@tikoci/rosetta 0.5.2 → 0.6.0

package/README.md

@@ -269,7 +269,7 @@ Ask your AI assistant questions like:
 
 ## MCP Tools
 
-The server exposes 14 tools designed to work together — agents start with `routeros_search` and drill into specific data as needed:
+The server exposes 16 tools designed to work together — agents start with `routeros_search` and drill into specific data as needed:
 
 | Tool | What it does |
 |------|-------------|
@@ -285,6 +285,8 @@ The server exposes 14 tools designed to work together — agents start with `rou
 | `routeros_device_lookup` | Hardware specs — filter by architecture, RAM, PoE, wireless, etc. |
 | `routeros_search_tests` | Cross-device ethernet and IPSec benchmarks |
 | `routeros_search_videos` | YouTube transcript search with chapter timestamps |
+| `routeros_dude_search` | FTS across archived Dude wiki docs (separate from RouterOS search) |
+| `routeros_dude_get_page` | Full Dude wiki page by ID or title, with screenshot metadata |
 | `routeros_stats` | Database health and coverage stats |
 | `routeros_current_versions` | Live-fetch current RouterOS versions from MikroTik |
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/browse.ts

@@ -19,6 +19,7 @@ import type {
   ChangelogResult,
   DeviceResult,
   DeviceTestRow,
+  DudeSearchResult,
   SearchResponse,
   SearchResult,
   SectionTocEntry,
@@ -29,6 +30,7 @@ import {
   checkCommandVersions,
   diffCommandVersions,
   fetchCurrentVersions,
+  getDudePage,
   getPage,
   getTestResultMeta,
   lookupProperty,
@@ -36,6 +38,7 @@ import {
   searchChangelogs,
   searchDevices,
   searchDeviceTests,
+  searchDude,
   searchPages,
   searchProperties,
   searchVideos,
@@ -196,6 +199,7 @@ type Context =
   | { type: "callouts"; query: string; results: CalloutResult[] }
   | { type: "changelogs"; results: ChangelogResult[] }
   | { type: "videos"; query: string; results: VideoSearchResult[] }
+  | { type: "dude"; query: string; results: DudeSearchResult[] }
   | { type: "properties"; query: string; pageId?: number; results: Array<{ name: string; page_id: number; page_title: string }> }
   | { type: "diff" }
   | { type: "vcheck"; path: string };
@@ -235,6 +239,7 @@ function contextLabel(c: Context): string {
     case "callouts": return c.query ? truncate(`cal: "${c.query}"`, 30) : "callouts";
     case "changelogs": return "changelogs";
     case "videos": return truncate(`vid: "${c.query}"`, 30);
+    case "dude": return truncate(`dude: "${c.query}"`, 30);
     case "diff": return "diff";
     case "vcheck": return truncate(`vc: ${c.path}`, 30);
   }
@@ -662,6 +667,65 @@ function renderVideos(results: VideoSearchResult[]): string {
   return out.join("\n");
 }
 
+function renderDudeResults(results: DudeSearchResult[]): string {
+  const out: string[] = [];
+  if (results.length === 0) {
+    out.push(`  ${dim("No Dude wiki results found.")}`);
+    return out.join("\n");
+  }
+
+  for (let i = 0; i < results.length; i++) {
+    const d = results[i];
+    const num = dim(`${String(i + 1).padStart(3)}  `);
+    const title = bold(truncate(d.title, 60));
+    const ver = dim(`[${d.version}]`);
+    const imgs = d.image_count > 0 ? dim(`📷 ${d.image_count}`) : "";
+    out.push(`${num}${title}  ${ver}  ${imgs}`);
+    out.push(`       ${dim(d.path)}`);
+    const excerpt = d.excerpt.replace(/\*\*/g, `${ESC}[1m`);
+    out.push(`       ${dim(truncate(excerpt, termWidth() - 8))}`);
+    out.push("");
+  }
+
+  out.push(`  ${cyan("[<n>]")} view page  ${cyan("[s <query>]")} search docs  ${cyan("[b]")} back`);
+  return out.join("\n");
+}
+
+function renderDudePage(page: import("./query.ts").DudePageResult): string {
+  const out: string[] = [];
+  out.push(`  ${bold(page.title)}  ${dim(`[${page.version}]`)}`);
+  out.push(`  ${dim(page.path)}`);
+  out.push(`  ${cyan(link(page.url))}`);
+  if (page.wayback_url) out.push(`  ${dim(`Archived: ${page.wayback_url}`)}`);
+  out.push("");
+
+  if (page.images.length > 0) {
+    out.push(`  ${bold("Screenshots:")} ${page.images.length}`);
+    for (const img of page.images) {
+      out.push(`    ${dim("•")} ${img.filename}${img.caption ? `  ${dim(img.caption)}` : ""}`);
+      out.push(`      ${dim(img.local_path)}`);
+    }
+    out.push("");
+  }
+
+  // Truncate long pages for TUI display
+  const maxChars = 8000;
+  const text = page.text.length > maxChars
+    ? `${page.text.slice(0, maxChars)}\n\n  ${dim(`... truncated (${page.text.length} chars total)`)}`
+    : page.text;
+  out.push(text);
+
+  if (page.code) {
+    out.push("");
+    out.push(`  ${bold("Code:")}`);
+    out.push(page.code.length > 2000 ? `${page.code.slice(0, 2000)}\n  ${dim("... truncated")}` : page.code);
+  }
+
+  out.push("");
+  out.push(`  ${cyan("[dude <query>]")} search dude  ${cyan("[s <query>]")} search docs  ${cyan("[b]")} back`);
+  return out.join("\n");
+}
+
 function renderDiff(result: ReturnType<typeof diffCommandVersions>): string {
   const out: string[] = [];
   out.push(`  ${bold("Command diff:")} ${result.from_version} → ${result.to_version}`);
@@ -775,6 +839,7 @@ function renderHelp(): string {
   cmd("callouts [query]", "cal", "Search callouts (type filter: cal warning)", "routeros_search_callouts");
   cmd("changelog [query]", "cl", "Search changelogs (cl 7.22, cl breaking)", "routeros_search_changelogs");
   cmd("videos <query>", "vid", "Search video transcripts", "routeros_search_videos");
+  cmd("dude <query>", "", "Search archived Dude wiki docs", "routeros_dude_search");
   cmd("diff <from> <to> [path]", "", "Command tree diff between versions", "routeros_command_diff");
   cmd("vcheck <path>", "vc", "Version range for a command path", "routeros_command_version_check");
   cmd("versions", "ver", "Live-fetch current RouterOS versions", "routeros_current_versions");
@@ -920,6 +985,12 @@ async function dispatch(input: string): Promise<void> {
       return;
     }
 
+    case "dude": {
+      if (!rest) { console.log(dim("  Usage: dude <query>")); return; }
+      await doSearchDude(rest);
+      return;
+    }
+
     case "diff": {
       const diffParts = rest.split(/\s+/);
       if (diffParts.length < 2) {
@@ -989,6 +1060,15 @@ async function handleNumberSelect(idx: number): Promise<void> {
     console.log(`  ${cyan(link(timeUrl))}\n`);
     return;
   }
+  if (ctx.type === "dude" && ctx.results[idx]) {
+    const d = ctx.results[idx];
+    const page = getDudePage(d.id);
+    if (page) {
+      await paged(renderDudePage(page));
+      pushCtx({ type: "dude", query: ctx.query, results: ctx.results });
+    }
+    return;
+  }
   if (ctx.type === "properties" && ctx.results[idx]) {
     const p = ctx.results[idx];
     await doPage(String(p.page_id));
@@ -1222,6 +1302,16 @@ async function doSearchVideos(query: string): Promise<void> {
   pushCtx({ type: "videos", query, results });
 }
 
+async function doSearchDude(query: string): Promise<void> {
+  const results = searchDude(query, 10);
+  if (results.length === 0) {
+    console.log(dim(`  No Dude wiki results for "${query}".`));
+    return;
+  }
+  await paged(`  ${bold(String(results.length))} Dude wiki results for ${cyan(`"${query}"`)}\n\n${renderDudeResults(results)}`);
+  pushCtx({ type: "dude", query, results });
+}
+
 async function doDiff(from: string, to: string, pathPrefix?: string): Promise<void> {
   const result = diffCommandVersions(from, to, pathPrefix);
   await paged(renderDiff(result));

package/src/db.ts

@@ -21,6 +21,9 @@
  *   videos_fts       — FTS5 over title, description
  *   video_segments   — transcript segments (one per chapter, or full video if no chapters)
  *   video_segments_fts — FTS5 over chapter_title, transcript
+ *   dude_pages       — The Dude documentation pages (archived from wiki.mikrotik.com via Wayback Machine)
+ *   dude_pages_fts   — FTS5 over title, path, text, code
+ *   dude_images      — screenshot images from The Dude wiki pages
  */
 
 import sqlite from "bun:sqlite";
@@ -205,17 +208,83 @@ export function initDb() {
   db.run(`CREATE INDEX IF NOT EXISTS idx_commands_type ON commands(type);`);
 
   // -- Command version tracking --
+  //
+  // ros_versions is keyed on (version, arch) — restraml emits per-arch
+  // deep-inspect.<arch>.json (x86 vs arm64; arm64 carries ~1K extra nodes
+  // for wifi-qcom etc.). _meta fields capture provenance from the inspect
+  // file (generatedAt, crashPaths, completionStats). _attrs is a JSON
+  // catch-all for forward-compat: anything restraml later emits lands here
+  // first and gets promoted to a column once shape is stable.
+  //
+  // command_versions intentionally has no FK to ros_versions — the composite
+  // PK on the parent makes a single-column FK invalid, and command_versions
+  // is slated for replacement by schema_node_presence in the upcoming
+  // schema_nodes refactor (BACKLOG.md "Multi-arch schema import").
+
+  // Migration: legacy ros_versions had `version TEXT PRIMARY KEY` with no
+  // arch column. Detect via PRAGMA table_info and rebuild both tables in
+  // place — the FK on command_versions.ros_version requires the dance.
+  // Idempotent: only fires when arch column is missing.
+  {
+    const rvCols = db.prepare("PRAGMA table_info(ros_versions)").all() as Array<{ name: string }>;
+    if (rvCols.length > 0 && !rvCols.some((c) => c.name === "arch")) {
+      db.run("PRAGMA foreign_keys=OFF;");
+      db.run("BEGIN;");
+      try {
+        db.run(`CREATE TABLE ros_versions_new (
+          version              TEXT NOT NULL,
+          arch                 TEXT NOT NULL DEFAULT 'x86',
+          channel              TEXT,
+          extra_packages       INTEGER NOT NULL DEFAULT 0,
+          extracted_at         TEXT NOT NULL,
+          generated_at         TEXT,
+          crash_paths_tested   TEXT,
+          crash_paths_crashed  TEXT,
+          completion_stats     TEXT,
+          source_url           TEXT,
+          _attrs               TEXT,
+          PRIMARY KEY (version, arch)
+        );`);
+        db.run(`INSERT INTO ros_versions_new (version, arch, channel, extra_packages, extracted_at)
+                SELECT version, 'x86', channel, extra_packages, extracted_at FROM ros_versions;`);
+        db.run("DROP TABLE ros_versions;");
+        db.run("ALTER TABLE ros_versions_new RENAME TO ros_versions;");
+
+        db.run(`CREATE TABLE command_versions_new (
+          command_path TEXT NOT NULL,
+          ros_version  TEXT NOT NULL,
+          PRIMARY KEY (command_path, ros_version)
+        );`);
+        db.run("INSERT INTO command_versions_new SELECT command_path, ros_version FROM command_versions;");
+        db.run("DROP TABLE command_versions;");
+        db.run("ALTER TABLE command_versions_new RENAME TO command_versions;");
+        db.run("COMMIT;");
+      } catch (e) {
+        db.run("ROLLBACK;");
+        throw e;
+      }
+      db.run("PRAGMA foreign_keys=ON;");
+    }
+  }
 
   db.run(`CREATE TABLE IF NOT EXISTS ros_versions (
-    version        TEXT PRIMARY KEY,
-    channel        TEXT,
-    extra_packages INTEGER NOT NULL DEFAULT 0,
-    extracted_at   TEXT NOT NULL
+    version              TEXT NOT NULL,
+    arch                 TEXT NOT NULL DEFAULT 'x86',
+    channel              TEXT,
+    extra_packages       INTEGER NOT NULL DEFAULT 0,
+    extracted_at         TEXT NOT NULL,
+    generated_at         TEXT,
+    crash_paths_tested   TEXT,
+    crash_paths_crashed  TEXT,
+    completion_stats     TEXT,
+    source_url           TEXT,
+    _attrs               TEXT,
+    PRIMARY KEY (version, arch)
   );`);
 
   db.run(`CREATE TABLE IF NOT EXISTS command_versions (
     command_path TEXT NOT NULL,
-    ros_version  TEXT NOT NULL REFERENCES ros_versions(version),
+    ros_version  TEXT NOT NULL,
     PRIMARY KEY (command_path, ros_version)
   );`);
 
@@ -426,6 +495,58 @@ export function initDb() {
   END;`);
 
   db.run(`CREATE INDEX IF NOT EXISTS idx_video_segs_video ON video_segments(video_id);`);
+
+  // -- The Dude documentation (archived from MikroTik wiki via Wayback Machine) --
+
+  db.run(`CREATE TABLE IF NOT EXISTS dude_pages (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    slug        TEXT NOT NULL UNIQUE,
+    title       TEXT NOT NULL,
+    path        TEXT NOT NULL,
+    version     TEXT NOT NULL DEFAULT 'v6',
+    url         TEXT NOT NULL,
+    wayback_url TEXT NOT NULL,
+    text        TEXT NOT NULL,
+    code        TEXT,
+    last_edited TEXT,
+    word_count  INTEGER
+  );`);
+
+  db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS dude_pages_fts USING fts5(
+    title, path, text, code,
+    content=dude_pages,
+    content_rowid=id,
+    tokenize='porter unicode61'
+  );`);
+
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_ai AFTER INSERT ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(rowid, title, path, text, code)
+    VALUES (new.id, new.title, new.path, new.text, new.code);
+  END;`);
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_ad AFTER DELETE ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(dude_pages_fts, rowid, title, path, text, code)
+    VALUES('delete', old.id, old.title, old.path, old.text, old.code);
+  END;`);
+  db.run(`CREATE TRIGGER IF NOT EXISTS dude_pages_au AFTER UPDATE ON dude_pages BEGIN
+    INSERT INTO dude_pages_fts(dude_pages_fts, rowid, title, path, text, code)
+    VALUES('delete', old.id, old.title, old.path, old.text, old.code);
+    INSERT INTO dude_pages_fts(rowid, title, path, text, code)
+    VALUES (new.id, new.title, new.path, new.text, new.code);
+  END;`);
+
+  db.run(`CREATE TABLE IF NOT EXISTS dude_images (
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    page_id      INTEGER NOT NULL REFERENCES dude_pages(id),
+    filename     TEXT NOT NULL,
+    alt_text     TEXT,
+    caption      TEXT,
+    local_path   TEXT NOT NULL,
+    original_url TEXT,
+    wayback_url  TEXT,
+    sort_order   INTEGER NOT NULL
+  );`);
+
+  db.run(`CREATE INDEX IF NOT EXISTS idx_dude_images_page ON dude_images(page_id);`);
 }
 
 /**
@@ -470,12 +591,14 @@ export function getDbStats() {
     devices_with_tests: count("SELECT COUNT(DISTINCT device_id) AS c FROM device_test_results"),
     changelogs: count("SELECT COUNT(*) AS c FROM changelogs"),
     changelog_versions: count("SELECT COUNT(DISTINCT version) AS c FROM changelogs"),
-    ros_versions: count("SELECT COUNT(*) AS c FROM ros_versions"),
+    ros_versions: count("SELECT COUNT(DISTINCT version) AS c FROM ros_versions"),
     videos: count("SELECT COUNT(*) AS c FROM videos"),
     video_segments: count("SELECT COUNT(*) AS c FROM video_segments"),
+    dude_pages: count("SELECT COUNT(*) AS c FROM dude_pages"),
+    dude_images: count("SELECT COUNT(*) AS c FROM dude_images"),
     ...(() => {
       // Semantic version sort — SQL MIN/MAX is lexicographic ("7.10" < "7.9")
-      const versions = (db.prepare("SELECT version FROM ros_versions").all() as Array<{ version: string }>).map((r) => r.version);
+      const versions = (db.prepare("SELECT DISTINCT version FROM ros_versions").all() as Array<{ version: string }>).map((r) => r.version);
       if (versions.length === 0) return { ros_version_min: null, ros_version_max: null };
       const norm = (v: string) => {
         const clean = v.replace(/beta\d*/, "").replace(/rc\d*/, "");

package/src/extract-commands.ts

@@ -1,18 +1,24 @@
 #!/usr/bin/env bun
 /**
- * extract-commands.ts — Load RouterOS command tree from inspect.json into SQLite.
+ * extract-commands.ts — Load RouterOS command tree from inspect.json (or
+ * deep-inspect.<arch>.json) into SQLite.
  *
  * Walks the nested JSON tree and flattens it into a `commands` table with:
  *   path, name, type (dir|cmd|arg), parent_path, description
  *
- * Also populates command_versions for version tracking.
+ * Also populates command_versions for version tracking, and ros_versions
+ * with provenance (arch, _meta.generatedAt, crashPaths, completionStats)
+ * when the source file is a deep-inspect.json (i.e. has a `_meta` envelope).
  *
  * Usage:
- *   bun run src/extract-commands.ts [inspect.json-path-or-url] [--version=7.22] [--channel=stable] [--extra]
- *   bun run src/extract-commands.ts --accumulate [inspect.json-path] [--version=X]
+ *   bun run src/extract-commands.ts [path-or-url] [--version=7.22] [--channel=stable] [--extra] [--arch=x86|arm64]
+ *   bun run src/extract-commands.ts --accumulate [path] [--version=X]
  *
  * In default mode: replaces commands table and sets as primary version.
  * With --accumulate: only adds to command_versions, does not touch commands table.
+ *
+ * --arch defaults to x86 (back-compat for legacy single-arch inspect.json).
+ * deep-inspect.x86.json / deep-inspect.arm64.json filenames auto-derive arch.
  */
 
 import { db, initDb } from "./db.ts";
@@ -33,6 +39,13 @@ const flagArgs = Object.fromEntries(
 const DEFAULT_INSPECT_URL = `${RESTRAML_PAGES_URL}/7.22.1/extra/inspect.json`;
 const INSPECT_SOURCE = positional[0] || DEFAULT_INSPECT_URL;
 
+function deriveArch(filepath: string, flag: string | undefined): "x86" | "arm64" {
+  if (flag === "x86" || flag === "arm64") return flag;
+  if (/deep-inspect\.arm64\b/.test(filepath)) return "arm64";
+  if (/deep-inspect\.x86\b/.test(filepath)) return "x86";
+  return "x86";
+}
+
 // Derive version from path if not explicitly set
 function deriveVersion(filepath: string): string {
   const match = filepath.match(/\/(\d+\.\d+(?:\.\d+)?(?:beta\d+|rc\d+)?)\//);
@@ -48,11 +61,30 @@ function deriveChannel(version: string): string {
 
 const version = flagArgs.version || deriveVersion(INSPECT_SOURCE);
 const channel = flagArgs.channel || deriveChannel(version);
+const arch = deriveArch(INSPECT_SOURCE, flagArgs.arch);
 
 console.log(`Loading inspect.json from ${INSPECT_SOURCE}...`);
-console.log(`Version: ${version}, Channel: ${channel}, Extra: ${extraPackages}, Accumulate: ${accumulate}`);
+console.log(`Version: ${version}, Channel: ${channel}, Arch: ${arch}, Extra: ${extraPackages}, Accumulate: ${accumulate}`);
 const inspectData = await loadJson<Record<string, unknown>>(INSPECT_SOURCE);
 
+// deep-inspect.json files carry a `_meta` envelope; vanilla inspect.json does not.
+interface DeepInspectMeta {
+  version?: string;
+  generatedAt?: string;
+  crashPathsTested?: string[];
+  crashPathsCrashed?: string[];
+  completionStats?: Record<string, unknown>;
+}
+const meta = (inspectData._meta && typeof inspectData._meta === "object")
+  ? (inspectData._meta as DeepInspectMeta)
+  : null;
+if (meta) {
+  console.log(`  _meta.version=${meta.version} generatedAt=${meta.generatedAt}`);
+  if (meta.completionStats) {
+    console.log(`  completionStats=${JSON.stringify(meta.completionStats)}`);
+  }
+}
+
 interface CommandRow {
   path: string;
   name: string;
@@ -65,7 +97,7 @@ const rows: CommandRow[] = [];
 
 function walk(obj: Record<string, unknown>, parentPath: string) {
   for (const [key, value] of Object.entries(obj)) {
-    if (key === "_type" || key === "desc") continue;
+    if (key === "_type" || key === "desc" || key === "_meta") continue;
     if (typeof value !== "object" || value === null) continue;
 
     const node = value as Record<string, unknown>;
@@ -104,11 +136,24 @@ console.log(`  dirs: ${dirs}, cmds: ${cmds}, args: ${args}`);
 // Initialize DB and insert
 initDb();
 
-// Register this version
+// Register this version (per-arch row). _meta fields are populated when the
+// source is a deep-inspect.json; legacy inspect.json leaves them NULL.
 db.run(
-  `INSERT OR REPLACE INTO ros_versions (version, channel, extra_packages, extracted_at)
-   VALUES (?, ?, ?, datetime('now'))`,
-  [version, channel, extraPackages ? 1 : 0],
+  `INSERT OR REPLACE INTO ros_versions
+   (version, arch, channel, extra_packages, extracted_at,
+    generated_at, crash_paths_tested, crash_paths_crashed, completion_stats, source_url)
+   VALUES (?, ?, ?, ?, datetime('now'), ?, ?, ?, ?, ?)`,
+  [
+    version,
+    arch,
+    channel,
+    extraPackages ? 1 : 0,
+    meta?.generatedAt ?? null,
+    meta?.crashPathsTested ? JSON.stringify(meta.crashPathsTested) : null,
+    meta?.crashPathsCrashed ? JSON.stringify(meta.crashPathsCrashed) : null,
+    meta?.completionStats ? JSON.stringify(meta.completionStats) : null,
+    INSPECT_SOURCE,
+  ],
 );
 
 if (!accumulate) {