@tikoci/rosetta 0.5.1 → 0.6.0

package/src/db.ts

@@ -21,6 +21,9 @@
  *   videos_fts       — FTS5 over title, description
  *   video_segments   — transcript segments (one per chapter, or full video if no chapters)
  *   video_segments_fts — FTS5 over chapter_title, transcript
+ *   dude_pages       — The Dude documentation pages (archived from wiki.mikrotik.com via Wayback Machine)
+ *   dude_pages_fts   — FTS5 over title, path, text, code
+ *   dude_images      — screenshot images from The Dude wiki pages
  */
 
 import sqlite from "bun:sqlite";
@@ -205,17 +208,83 @@ export function initDb() {
   db.run(`CREATE INDEX IF NOT EXISTS idx_commands_type ON commands(type);`);
 
   // -- Command version tracking --
+  //
+  // ros_versions is keyed on (version, arch) — restraml emits per-arch
+  // deep-inspect.<arch>.json (x86 vs arm64; arm64 carries ~1K extra nodes
+  // for wifi-qcom etc.). _meta fields capture provenance from the inspect
+  // file (generatedAt, crashPaths, completionStats). _attrs is a JSON
+  // catch-all for forward-compat: anything restraml later emits lands here
+  // first and gets promoted to a column once shape is stable.
+  //
+  // command_versions intentionally has no FK to ros_versions — the composite
+  // PK on the parent makes a single-column FK invalid, and command_versions
+  // is slated for replacement by schema_node_presence in the upcoming
+  // schema_nodes refactor (BACKLOG.md "Multi-arch schema import").
+
+  // Migration: legacy ros_versions had `version TEXT PRIMARY KEY` with no
+  // arch column. Detect via PRAGMA table_info and rebuild both tables in
+  // place — the FK on command_versions.ros_version requires the dance.
+  // Idempotent: only fires when arch column is missing.
+  {
+    const rvCols = db.prepare("PRAGMA table_info(ros_versions)").all() as Array<{ name: string }>;
+    if (rvCols.length > 0 && !rvCols.some((c) => c.name === "arch")) {
+      db.run("PRAGMA foreign_keys=OFF;");
+      db.run("BEGIN;");
+      try {
+        db.run(`CREATE TABLE ros_versions_new (
+          version              TEXT NOT NULL,
+          arch                 TEXT NOT NULL DEFAULT 'x86',
+          channel              TEXT,
+          extra_packages       INTEGER NOT NULL DEFAULT 0,
+          extracted_at         TEXT NOT NULL,
+          generated_at         TEXT,
+          crash_paths_tested   TEXT,
+          crash_paths_crashed  TEXT,
+          completion_stats     TEXT,
+          source_url           TEXT,
+          _attrs               TEXT,
+          PRIMARY KEY (version, arch)
+        );`);
+        db.run(`INSERT INTO ros_versions_new (version, arch, channel, extra_packages, extracted_at)
+                SELECT version, 'x86', channel, extra_packages, extracted_at FROM ros_versions;`);
+        db.run("DROP TABLE ros_versions;");
+        db.run("ALTER TABLE ros_versions_new RENAME TO ros_versions;");
+
+        db.run(`CREATE TABLE command_versions_new (
+          command_path TEXT NOT NULL,
+          ros_version  TEXT NOT NULL,
+          PRIMARY KEY (command_path, ros_version)
+        );`);
+        db.run("INSERT INTO command_versions_new SELECT command_path, ros_version FROM command_versions;");
+        db.run("DROP TABLE command_versions;");
+        db.run("ALTER TABLE command_versions_new RENAME TO command_versions;");
+        db.run("COMMIT;");
+      } catch (e) {
+        db.run("ROLLBACK;");
+        throw e;
+      }
+      db.run("PRAGMA foreign_keys=ON;");
+    }
+  }
 
   db.run(`CREATE TABLE IF NOT EXISTS ros_versions (
-    version        TEXT PRIMARY KEY,
-    channel        TEXT,
-    extra_packages INTEGER NOT NULL DEFAULT 0,
-    extracted_at   TEXT NOT NULL
+    version              TEXT NOT NULL,
+    arch                 TEXT NOT NULL DEFAULT 'x86',
+    channel              TEXT,
+    extra_packages       INTEGER NOT NULL DEFAULT 0,
+    extracted_at         TEXT NOT NULL,
+    generated_at         TEXT,
+    crash_paths_tested   TEXT,
+    crash_paths_crashed  TEXT,
+    completion_stats     TEXT,
+    source_url           TEXT,
+    _attrs               TEXT,
+    PRIMARY KEY (version, arch)
   );`);
 
   db.run(`CREATE TABLE IF NOT EXISTS command_versions (
     command_path TEXT NOT NULL,
-    ros_version  TEXT NOT NULL REFERENCES ros_versions(version),
+    ros_version  TEXT NOT NULL,
     PRIMARY KEY (command_path, ros_version)
   );`);
 
@@ -426,6 +495,58 @@ export function initDb() {
   END;`);
 
   db.run(`CREATE INDEX IF NOT EXISTS idx_video_segs_video ON video_segments(video_id);`);
+
+  // -- The Dude documentation (archived from MikroTik wiki via Wayback Machine) --
+
+  db.run(`CREATE TABLE IF NOT EXISTS dude_pages (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    slug        TEXT NOT NULL UNIQUE,
+    title       TEXT NOT NULL,
+    path        TEXT NOT NULL,
+    version     TEXT NOT NULL DEFAULT 'v6',
+    url         TEXT NOT NULL,
+    wayback_url TEXT NOT NULL,
+    text        TEXT NOT NULL,
+    code        TEXT,
+    last_edited TEXT,
+    word_count  INTEGER
+  );`);
+
+  db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS dude_pages_fts USING fts5(
+    title, path, text, code,
+    content=dude_pages,
+    content_rowid=id,
+    tokenize='porter unicode61'
+  );`);
+
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_ai AFTER INSERT ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(rowid, title, path, text, code)
+    VALUES (new.id, new.title, new.path, new.text, new.code);
+  END;`);
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_ad AFTER DELETE ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(dude_pages_fts, rowid, title, path, text, code)
+    VALUES('delete', old.id, old.title, old.path, old.text, old.code);
+  END;`);
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_au AFTER UPDATE ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(dude_pages_fts, rowid, title, path, text, code)
+    VALUES('delete', old.id, old.title, old.path, old.text, old.code);
+    INSERT INTO dude_pages_fts(rowid, title, path, text, code)
+    VALUES (new.id, new.title, new.path, new.text, new.code);
+  END;`);
+
+  db.run(`CREATE TABLE IF NOT EXISTS dude_images (
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    page_id      INTEGER NOT NULL REFERENCES dude_pages(id),
+    filename     TEXT NOT NULL,
+    alt_text     TEXT,
+    caption      TEXT,
+    local_path   TEXT NOT NULL,
+    original_url TEXT,
+    wayback_url  TEXT,
+    sort_order   INTEGER NOT NULL
+  );`);
+
+  db.run(`CREATE INDEX IF NOT EXISTS idx_dude_images_page ON dude_images(page_id);`);
 }
 
 /**
@@ -441,8 +562,24 @@ export function checkSchemaVersion(): { ok: boolean; actual: number; expected: n
 export function getDbStats() {
   const count = (sql: string) =>
     Number((db.prepare(sql).get() as { c: number }).c ?? 0);
+  const dbSizeBytes = (() => {
+    try {
+      return Bun.file(DB_PATH).size;
+    } catch {
+      return null;
+    }
+  })();
+  const schemaVersion = (() => {
+    try {
+      return (db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version;
+    } catch {
+      return null;
+    }
+  })();
   return {
     db_path: DB_PATH,
+    db_size_bytes: dbSizeBytes,
+    schema_version: schemaVersion,
     pages: count("SELECT COUNT(*) AS c FROM pages"),
     sections: count("SELECT COUNT(*) AS c FROM sections"),
     properties: count("SELECT COUNT(*) AS c FROM properties"),
@@ -454,12 +591,14 @@ export function getDbStats() {
     devices_with_tests: count("SELECT COUNT(DISTINCT device_id) AS c FROM device_test_results"),
     changelogs: count("SELECT COUNT(*) AS c FROM changelogs"),
     changelog_versions: count("SELECT COUNT(DISTINCT version) AS c FROM changelogs"),
-    ros_versions: count("SELECT COUNT(*) AS c FROM ros_versions"),
+    ros_versions: count("SELECT COUNT(DISTINCT version) AS c FROM ros_versions"),
     videos: count("SELECT COUNT(*) AS c FROM videos"),
     video_segments: count("SELECT COUNT(*) AS c FROM video_segments"),
+    dude_pages: count("SELECT COUNT(*) AS c FROM dude_pages"),
+    dude_images: count("SELECT COUNT(*) AS c FROM dude_images"),
     ...(() => {
       // Semantic version sort — SQL MIN/MAX is lexicographic ("7.10" < "7.9")
-      const versions = (db.prepare("SELECT version FROM ros_versions").all() as Array<{ version: string }>).map((r) => r.version);
+      const versions = (db.prepare("SELECT DISTINCT version FROM ros_versions").all() as Array<{ version: string }>).map((r) => r.version);
       if (versions.length === 0) return { ros_version_min: null, ros_version_max: null };
       const norm = (v: string) => {
         const clean = v.replace(/beta\d*/, "").replace(/rc\d*/, "");

package/src/extract-commands.ts

@@ -1,18 +1,24 @@
 #!/usr/bin/env bun
 /**
- * extract-commands.ts — Load RouterOS command tree from inspect.json into SQLite.
+ * extract-commands.ts — Load RouterOS command tree from inspect.json (or
+ * deep-inspect.<arch>.json) into SQLite.
  *
  * Walks the nested JSON tree and flattens it into a `commands` table with:
  *   path, name, type (dir|cmd|arg), parent_path, description
  *
- * Also populates command_versions for version tracking.
+ * Also populates command_versions for version tracking, and ros_versions
+ * with provenance (arch, _meta.generatedAt, crashPaths, completionStats)
+ * when the source file is a deep-inspect.json (i.e. has a `_meta` envelope).
  *
  * Usage:
- *   bun run src/extract-commands.ts [inspect.json-path-or-url] [--version=7.22] [--channel=stable] [--extra]
- *   bun run src/extract-commands.ts --accumulate [inspect.json-path] [--version=X]
+ *   bun run src/extract-commands.ts [path-or-url] [--version=7.22] [--channel=stable] [--extra] [--arch=x86|arm64]
+ *   bun run src/extract-commands.ts --accumulate [path] [--version=X]
  *
  * In default mode: replaces commands table and sets as primary version.
  * With --accumulate: only adds to command_versions, does not touch commands table.
+ *
+ * --arch defaults to x86 (back-compat for legacy single-arch inspect.json).
+ * deep-inspect.x86.json / deep-inspect.arm64.json filenames auto-derive arch.
  */
 
 import { db, initDb } from "./db.ts";
@@ -33,6 +39,13 @@ const flagArgs = Object.fromEntries(
 const DEFAULT_INSPECT_URL = `${RESTRAML_PAGES_URL}/7.22.1/extra/inspect.json`;
 const INSPECT_SOURCE = positional[0] || DEFAULT_INSPECT_URL;
 
+function deriveArch(filepath: string, flag: string | undefined): "x86" | "arm64" {
+  if (flag === "x86" || flag === "arm64") return flag;
+  if (/deep-inspect\.arm64\b/.test(filepath)) return "arm64";
+  if (/deep-inspect\.x86\b/.test(filepath)) return "x86";
+  return "x86";
+}
+
 // Derive version from path if not explicitly set
 function deriveVersion(filepath: string): string {
   const match = filepath.match(/\/(\d+\.\d+(?:\.\d+)?(?:beta\d+|rc\d+)?)\//);
@@ -48,11 +61,30 @@ function deriveChannel(version: string): string {
 
 const version = flagArgs.version || deriveVersion(INSPECT_SOURCE);
 const channel = flagArgs.channel || deriveChannel(version);
+const arch = deriveArch(INSPECT_SOURCE, flagArgs.arch);
 
 console.log(`Loading inspect.json from ${INSPECT_SOURCE}...`);
-console.log(`Version: ${version}, Channel: ${channel}, Extra: ${extraPackages}, Accumulate: ${accumulate}`);
+console.log(`Version: ${version}, Channel: ${channel}, Arch: ${arch}, Extra: ${extraPackages}, Accumulate: ${accumulate}`);
 const inspectData = await loadJson<Record<string, unknown>>(INSPECT_SOURCE);
 
+// deep-inspect.json files carry a `_meta` envelope; vanilla inspect.json does not.
+interface DeepInspectMeta {
+  version?: string;
+  generatedAt?: string;
+  crashPathsTested?: string[];
+  crashPathsCrashed?: string[];
+  completionStats?: Record<string, unknown>;
+}
+const meta = (inspectData._meta && typeof inspectData._meta === "object")
+  ? (inspectData._meta as DeepInspectMeta)
+  : null;
+if (meta) {
+  console.log(`  _meta.version=${meta.version} generatedAt=${meta.generatedAt}`);
+  if (meta.completionStats) {
+    console.log(`  completionStats=${JSON.stringify(meta.completionStats)}`);
+  }
+}
+
 interface CommandRow {
   path: string;
   name: string;
@@ -65,7 +97,7 @@ const rows: CommandRow[] = [];
 
 function walk(obj: Record<string, unknown>, parentPath: string) {
   for (const [key, value] of Object.entries(obj)) {
-    if (key === "_type" || key === "desc") continue;
+    if (key === "_type" || key === "desc" || key === "_meta") continue;
     if (typeof value !== "object" || value === null) continue;
 
     const node = value as Record<string, unknown>;
@@ -104,11 +136,24 @@ console.log(`  dirs: ${dirs}, cmds: ${cmds}, args: ${args}`);
 // Initialize DB and insert
 initDb();
 
-// Register this version
+// Register this version (per-arch row). _meta fields are populated when the
+// source is a deep-inspect.json; legacy inspect.json leaves them NULL.
 db.run(
-  `INSERT OR REPLACE INTO ros_versions (version, channel, extra_packages, extracted_at)
-   VALUES (?, ?, ?, datetime('now'))`,
-  [version, channel, extraPackages ? 1 : 0],
+  `INSERT OR REPLACE INTO ros_versions
+   (version, arch, channel, extra_packages, extracted_at,
+    generated_at, crash_paths_tested, crash_paths_crashed, completion_stats, source_url)
+   VALUES (?, ?, ?, ?, datetime('now'), ?, ?, ?, ?, ?)`,
+  [
+    version,
+    arch,
+    channel,
+    extraPackages ? 1 : 0,
+    meta?.generatedAt ?? null,
+    meta?.crashPathsTested ? JSON.stringify(meta.crashPathsTested) : null,
+    meta?.crashPathsCrashed ? JSON.stringify(meta.crashPathsCrashed) : null,
+    meta?.completionStats ? JSON.stringify(meta.completionStats) : null,
+    INSPECT_SOURCE,
+  ],
 );
 
 if (!accumulate) {

package/src/extract-devices.ts

@@ -120,7 +120,11 @@ if (lines.length < 2) {
 // Skip header row
 const dataLines = lines.slice(1);
 
-// Idempotent: clear existing data (FTS triggers handle cleanup)
+// Idempotent: clear existing data (FTS triggers handle cleanup).
+// device_test_results FKs into devices — wipe it first to avoid a FOREIGN KEY
+// constraint failure on re-run over a populated DB. extract-test-results runs
+// later in the pipeline and repopulates it.
+db.run("DELETE FROM device_test_results");
 db.run("DELETE FROM devices");
 
 const insert = db.prepare(`INSERT INTO devices (