@tikoci/rosetta 0.8.9 → 0.8.11

package/src/mcp.ts

@@ -53,62 +53,44 @@ function link(url: string, display?: string): string {
  * on fresh installs.
  *
  * Failure modes are explicit:
- *   - Missing or empty DB → download once. If download fails, return — server
- *     will start but tool calls will surface the underlying SQL error.
+ *   - Missing or empty DB → download once. If download fails, abort startup —
+ *     package-mode startup must not continue into db.ts and create a schema-only DB.
  *   - Schema mismatch → re-download once, then re-probe. If still mismatched,
  *     fail hard with an actionable message rather than silently using a DB
  *     that the running code can't query correctly.
  */
 async function ensureDbReady(log: (msg: string) => void): Promise<void> {
   const { resolveDbPath, SCHEMA_VERSION, resolveVersion } = await import("./paths.ts");
-  const { downloadDb } = await import("./setup.ts");
-  const { default: sqlite } = await import("bun:sqlite");
+  const { downloadDb, hasMinimumDbContent, probeDb, cleanupStaleTempArtifacts } = await import("./setup.ts");
 
   const dbPath = resolveDbPath(import.meta.dirname);
   const runningVersion = resolveVersion(import.meta.dirname);
 
-  /** Probe an existing DB. Returns null if the file is missing or unreadable.
-   *  Do NOT pass { readonly: true } — freshly written SQLite WAL-mode files fail
-   *  to open readonly on macOS until a read-write connection initialises the WAL
-   *  shared-memory file. Same gotcha as setup.ts::probeDb. */
-  function probe(): { pages: number; schemaVersion: number; releaseTag: string | null } | null {
-    try {
-      const check = new sqlite(dbPath);
-      const pages = (check.prepare("SELECT COUNT(*) AS c FROM pages").get() as { c: number }).c;
-      const ver = (check.prepare("PRAGMA user_version").get() as { user_version: number }).user_version;
-      let releaseTag: string | null = null;
-      try {
-        const meta = check.prepare("SELECT value FROM db_meta WHERE key = 'release_tag'").get() as { value: string } | null;
-        releaseTag = meta?.value ?? null;
-      } catch {
-        // db_meta missing on pre-v5 DBs — leave null
-      }
-      check.close();
-      return { pages, schemaVersion: ver, releaseTag };
-    } catch {
-      return null;
-    }
-  }
+  // Always clean stale .tmp.* artifacts from previous failed runs, regardless of
+  // whether a download is needed this time.
+  cleanupStaleTempArtifacts(dbPath);
 
-  let p = probe();
+  let p = probeDb(dbPath);
 
   // Case 1: DB missing or empty → first-time download.
-  if (!p || p.pages === 0) {
+  if (!hasMinimumDbContent(p)) {
     log(`No usable database at ${dbPath} — downloading...`);
     try {
-      await downloadDb(dbPath, log);
+      p = await downloadDb(dbPath, log);
       log("Database downloaded successfully.");
-      p = probe();
     } catch (e) {
       log(`Auto-download failed: ${e instanceof Error ? e.message : e}`);
-      log(`Run: bunx @tikoci/rosetta --refresh`);
-      return;
+      log(`Close other rosetta clients and run: bunx @tikoci/rosetta --refresh`);
+      throw new Error(`Unable to start rosetta without a usable database at ${dbPath}.`);
     }
   }
 
   if (!p) {
-    log(`Database probe failed after download.`);
-    return;
+    throw new Error(`Database probe failed after download: ${dbPath}`);
+  }
+
+  if (!hasMinimumDbContent(p)) {
+    throw new Error(`Database remained incomplete after recovery: ${dbPath}`);
   }
 
   // Case 2: Schema mismatch → re-download, then re-probe and fail hard if
@@ -119,27 +101,25 @@ async function ensureDbReady(log: (msg: string) => void): Promise<void> {
         `Re-downloading database...`,
     );
     try {
-      await downloadDb(dbPath, log);
+      p = await downloadDb(dbPath, log);
     } catch (e) {
       log(`✗ Auto-recovery download failed: ${e instanceof Error ? e.message : e}`);
       log(
         `  This rosetta build (v${runningVersion}) cannot use the existing DB. ` +
-          `Run \`bun pm cache rm\` to clear the bunx cache and relaunch.`,
+          `Close other rosetta clients and run: bunx @tikoci/rosetta@latest --refresh`,
       );
-      process.exit(1);
+      throw new Error(`Unable to recover an incompatible database at ${dbPath}.`);
     }
-    const p2 = probe();
-    if (!p2 || p2.schemaVersion !== SCHEMA_VERSION) {
+    if (!p || !hasMinimumDbContent(p) || p.schemaVersion !== SCHEMA_VERSION) {
       log(
-        `✗ Still incompatible after re-download (DB=${p2?.schemaVersion ?? "unreadable"}, expected=${SCHEMA_VERSION}).`,
+        `✗ Still incompatible after re-download (DB=${p?.schemaVersion ?? "unreadable"}, expected=${SCHEMA_VERSION}).`,
       );
       log(
         `  The published database does not match this rosetta build (v${runningVersion}). ` +
-          `Run \`bun pm cache rm && bunx @tikoci/rosetta --refresh\` to update both the package and the DB.`,
+          `Run: bunx @tikoci/rosetta@latest --refresh`,
       );
-      process.exit(1);
+      throw new Error(`Database remained incompatible after recovery: ${dbPath}`);
     }
-    p = p2;
   }
 
   // Quietly emit a one-line provenance banner so MCP-client logs show what's loaded.
@@ -226,6 +206,7 @@ const {
   browseCommandsAtVersion,
   checkCommandVersions,
   diffCommandVersions,
+  explainCommand,
   exportDevicesCsv,
   exportDeviceTestsCsv,
   fetchCurrentVersions,
@@ -575,8 +556,8 @@ and FTS in parallel, returning pages plus classifier-informed side queries in a
 single response. Consolidates what used to require 3–5 separate tool calls.
 
 Response shape:
-- classified: { version, topics, command_path, device, property } — what the
-  classifier detected from your input
+- classified: { version, topics, command_path, command_path_confidence, device, property } — what the
+  classifier detected from your input; command_path_confidence is high/medium/low
 - pages: top FTS matches (title, path, URL, excerpt, best_section)
 - related: callouts, properties, changelogs, videos, commands, devices, skills,
   glossary — empty sections are omitted. Cap scales with \`limit\`: small limit
@@ -696,8 +677,10 @@ server.registerTool(
   {
     description: `Look up a specific RouterOS configuration property by exact name.
 
-Returns type, default value, description, and documentation page.
+Returns type, default value, description, documentation page, and confidence.
 Optionally filter by command path to disambiguate (e.g., "disabled" appears everywhere).
+Confidence is high for exact matches on the page linked from command_path, medium for
+global exact matches without command_path, and low for global fallback with command_path.
 
 This requires the **exact property name**. If you don't know the name:
 → routeros_search: find the documentation page, then routeros_get_page to read properties in context
@@ -734,6 +717,57 @@ Examples:
   },
 );
 
+// ---- routeros_explain_command ----
+
+server.registerTool(
+  "routeros_explain_command",
+  {
+    description: `Explain a candidate RouterOS CLI command using rosetta's offline docs and command tree.
+
+This is a read-only tier-1 helper for write-shaped questions: it canonicalizes
+the command, annotates key=value arguments with documented properties, checks
+tracked RouterOS version presence, and returns compact docs/changelog context.
+It never connects to a router, validates against a live device, or executes anything.
+
+Returns:
+- command: original input
+- canonical: { path, verb, args, confidence } for the primary non-subshell command
+- confidence: high/medium/low/none from the CLI canonicalizer
+- args: parsed key=value args with first property match and lookup confidence when found
+- warnings: no-command, low-confidence, unknown-arg, command-not-in-version, or model-context-unused signals
+- pages: compact documentation search hits
+- changelog_hits: compact changelog hits
+- version_check: command version range when a canonical path is available
+
+Workflow:
+→ routeros_get_page: read full docs for a returned page
+→ routeros_lookup_property: inspect a specific argument/property in more detail
+→ routeros_command_tree: browse available commands/arguments under the canonical path
+→ routeros_command_version_check / routeros_command_diff: investigate version-specific availability
+
+Boundaries: Documentation covers RouterOS v7, aligned with long-term ~7.22; command data covers 7.9–7.23beta2. This tool is explanatory only — use a separate validator/runner before touching a router.`,
+    inputSchema: {
+      command: z
+        .string()
+        .describe("RouterOS CLI command to explain (e.g., '/ip/firewall/filter add chain=forward action=drop')"),
+      ros_version: z
+        .string()
+        .optional()
+        .describe("Optional RouterOS version to check against tracked command availability (e.g., '7.22')."),
+      model: z
+        .string()
+        .optional()
+        .describe("Optional device model context. Accepted for future use; device-specific validation is not implemented."),
+    },
+  },
+  async ({ command, ros_version, model }) => {
+    const result = explainCommand(command, ros_version, model);
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+    };
+  },
+);
+
 // ---- routeros_command_tree ----
 
 server.registerTool(
@@ -811,7 +845,9 @@ Knowledge boundaries:
 - No RouterOS v6 data available — v6 syntax and subsystems differ significantly from v7
 - For versions older than 7.9, no command tree data exists
 - Versions older than current long-term are unpatched by MikroTik
-- Absence of a peripheral in docs doesn't mean unsupported — most MBIM modems work`,
+- Absence of a peripheral in docs doesn't mean unsupported — most MBIM modems work
+
+→ routeros_search: probe the corpus these stats describe with any RouterOS question`,
     inputSchema: {},
   },
   async () => {
@@ -1402,7 +1438,9 @@ Key context for version reasoning:
 - Our command tree data covers 7.9–7.23beta2
 - If a user's version is older than the current long-term, recommend upgrading
 
-Requires network access to upgrade.mikrotik.com.`,
+Requires network access to upgrade.mikrotik.com.
+
+→ routeros_search_changelogs: read what changed *into* the current/target version (use to_version=<latest> and from_version=<user's version>)`,
     inputSchema: {},
   },
   async () => {
@@ -1558,4 +1596,8 @@ if (useHttp) {
   await server.connect(transport);
 }
 
-})();
+})().catch((err) => {
+  const message = err instanceof Error ? err.message : String(err);
+  process.stderr.write(`${message}\n`);
+  process.exit(1);
+});

package/src/query.test.ts

@@ -54,6 +54,7 @@ const {
   listGlossary,
   KNOWN_TOPICS,
   searchAll,
+  explainCommand,
 } = await import("./query.ts");
 const { parseChangelog } = await import("./extract-changelogs.ts");
 const { parseVtt, segmentTranscript } = await import("./extract-videos.ts");
@@ -122,6 +123,14 @@ beforeAll(() => {
     (id, page_id, name, type, default_val, description, section, sort_order)
     VALUES (2, 1, 'address-pool', 'string', '', 'Name of the address pool to use', NULL, 1)`);
 
+  db.run(`INSERT INTO properties
+    (id, page_id, name, type, default_val, description, section, sort_order)
+    VALUES (3, 2, 'chain', 'string', '', 'Firewall chain to match or create', NULL, 0)`);
+
+  db.run(`INSERT INTO properties
+    (id, page_id, name, type, default_val, description, section, sort_order)
+    VALUES (4, 2, 'action', 'string', 'accept', 'Action to take when a packet matches the rule', NULL, 1)`);
+
   db.run(`INSERT INTO ros_versions (version, arch, channel, extra_packages, extracted_at)
     VALUES ('7.22', 'x86', 'stable', 0, '2024-01-01T00:00:00Z')`);
   db.run(`INSERT INTO ros_versions (version, arch, channel, extra_packages, extracted_at)
@@ -137,6 +146,14 @@ beforeAll(() => {
     (id, path, name, type, parent_path, page_id, description, ros_version)
     VALUES (2, '/ip/dhcp-server', 'dhcp-server', 'dir', '/ip', 1, 'DHCP Server configuration', '7.22')`);
 
+  db.run(`INSERT INTO commands
+    (id, path, name, type, parent_path, page_id, description, ros_version)
+    VALUES (3, '/ip/firewall', 'firewall', 'dir', '/ip', 2, 'Firewall configuration', '7.22')`);
+
+  db.run(`INSERT INTO commands
+    (id, path, name, type, parent_path, page_id, description, ros_version)
+    VALUES (4, '/ip/firewall/filter', 'filter', 'dir', '/ip/firewall', 2, 'Firewall filter rules', '7.22')`);
+
   // schema_nodes for arch-filter tests: shared + x86-only child of /ip
   db.run(`INSERT INTO schema_nodes (path, name, type, parent_path, dir_role, _arch)
     VALUES ('/ip', 'ip', 'dir', NULL, 'namespace', NULL)`);
@@ -153,6 +170,8 @@ beforeAll(() => {
     VALUES ('/ip/dhcp-server', '7.22')`);
   db.run(`INSERT INTO command_versions (command_path, ros_version)
     VALUES ('/ip/dhcp-server', '7.9')`);
+  db.run(`INSERT INTO command_versions (command_path, ros_version)
+    VALUES ('/ip/firewall/filter', '7.22')`);
 
   // Extra command_versions entries to support diffCommandVersions tests
   // /ip/dhcp-server/lease only in 7.22 (added)
@@ -383,8 +402,12 @@ beforeAll(() => {
     VALUES ('7.22', '2026-Mar-09 10:38', 'bgp', 0, 'added BGP unnumbered support', 1)`);
   db.run(`INSERT INTO changelogs (version, released, category, is_breaking, description, sort_order)
     VALUES ('7.22', '2026-Mar-09 10:38', 'bridge', 0, 'added local and static MAC synchronization for MLAG', 2)`);
+  db.run(`INSERT INTO changelogs (version, released, category, is_breaking, description, sort_order)
+    VALUES ('7.22', '2026-Mar-09 10:38', 'firewall', 0, 'improved firewall filter rule handling', 3)`);
   db.run(`INSERT INTO changelogs (version, released, category, is_breaking, description, sort_order)
     VALUES ('7.22.1', '2026-Apr-01 09:00', 'wifi', 0, 'fixed channel switching for MediaTek access points', 0)`);
+  db.run(`INSERT INTO changelogs (version, released, category, is_breaking, description, sort_order)
+    VALUES ('7.23.1', '2026-Apr-15 09:00', 'routing', 0, 'fixed route cache after upgrade', 0)`);
 
   // Video and transcript fixtures for searchVideos tests
   db.run(`INSERT INTO videos
@@ -460,6 +483,10 @@ describe("extractTerms", () => {
     expect(extractTerms("how and the with without")).toEqual([]);
   });
 
+  test("drops switch as context for bridge VLAN filtering queries", () => {
+    expect(extractTerms("bridge vlan filtering on a switch")).toEqual(["bridge", "vlan", "filtering"]);
+  });
+
   test("filters terms shorter than 2 characters", () => {
     expect(extractTerms("a x y")).toEqual([]);
   });
@@ -777,11 +804,13 @@ describe("lookupProperty", () => {
     expect(rows.length).toBeGreaterThan(0);
     expect(rows[0].name).toBe("lease-time");
     expect(rows[0].page_title).toBe("DHCP Server");
+    expect(rows[0].confidence).toBe("medium");
   });
 
   test("case-insensitive name lookup", () => {
     const rows = lookupProperty("LEASE-TIME");
     expect(rows.length).toBeGreaterThan(0);
+    expect(rows.every((row) => row.confidence === "medium")).toBe(true);
   });
 
   test("returns empty for unknown property", () => {
@@ -792,12 +821,74 @@ describe("lookupProperty", () => {
     const rows = lookupProperty("lease-time", "/ip/dhcp-server");
     expect(rows.length).toBeGreaterThan(0);
     expect(rows[0].name).toBe("lease-time");
+    expect(rows[0].confidence).toBe("high");
   });
 
-  test("returns empty when command path has no linked page", () => {
+  test("marks global fallback low when command path has no linked page", () => {
     const rows = lookupProperty("lease-time", "/ip/unlinked");
     // /ip/unlinked has no page_id → falls through to global search
     expect(Array.isArray(rows)).toBe(true);
+    expect(rows.length).toBeGreaterThan(0);
+    expect(rows.every((row) => row.confidence === "low")).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// DB integration: explainCommand
+// ---------------------------------------------------------------------------
+
+describe("explainCommand", () => {
+  test("returns canonical command, known arg property, pages, changelog hits, and version check", () => {
+    const result = explainCommand("/ip firewall filter add chain=forward action=drop", "7.22");
+
+    expect(result.command).toBe("/ip firewall filter add chain=forward action=drop");
+    expect(result.canonical).toEqual({
+      path: "/ip/firewall/filter",
+      verb: "add",
+      args: ["chain=forward", "action=drop"],
+      confidence: "high",
+    });
+    expect(result.confidence).toBe("high");
+    expect(result.args.map((arg) => arg.name)).toEqual(["chain", "action"]);
+    expect(result.args[0].property?.name).toBe("chain");
+    expect(result.args[0].property?.confidence).toBe("high");
+    expect(result.warnings).toEqual([]);
+    expect(result.pages.some((page) => page.title === "Firewall Filter")).toBe(true);
+    expect(result.changelog_hits.some((hit) => hit.category === "firewall")).toBe(true);
+    expect(result.version_check?.versions).toContain("7.22");
+  });
+
+  test("warns for unknown args, absent target versions, and unused model context", () => {
+    const result = explainCommand("/ip firewall filter add frobnicate=yes", "7.9", "hAP ax3");
+
+    expect(result.canonical?.path).toBe("/ip/firewall/filter");
+    expect(result.confidence).toBe("high");
+    expect(result.args[0]).toMatchObject({ name: "frobnicate", value: "yes" });
+    expect(result.args[0].property).toBeUndefined();
+    expect(result.warnings.map((warning) => warning.kind)).toEqual(
+      expect.arrayContaining([
+        "unknown-arg",
+        "command-not-in-version",
+        "model-context-unused",
+      ]),
+    );
+  });
+
+  test("warns on low-confidence canonicalization", () => {
+    const result = explainCommand("print");
+
+    expect(result.canonical).toMatchObject({ path: "/", verb: "print", confidence: "low" });
+    expect(result.confidence).toBe("low");
+    expect(result.warnings.some((warning) => warning.kind === "low-confidence")).toBe(true);
+  });
+
+  test("warns when no primary command can be extracted", () => {
+    const result = explainCommand("not a routeros command");
+
+    expect(result.canonical).toBeNull();
+    expect(result.confidence).toBe("none");
+    expect(result.args).toEqual([]);
+    expect(result.warnings.some((warning) => warning.kind === "no-command")).toBe(true);
   });
 });
 
@@ -1563,12 +1654,33 @@ describe("searchChangelogs", () => {
     expect(results.some((r) => r.category === "bgp")).toBe(true);
   });
 
-  test("version filter returns only that version", () => {
+  test("exact patch version stays exact", () => {
+    const results = searchChangelogs("", { version: "7.22.1" });
+    expect(results.length).toBeGreaterThan(0);
+    for (const r of results) {
+      expect(r.version).toBe("7.22.1");
+    }
+  });
+
+  test("major.minor version filter preserves exact rows before patch fallback", () => {
     const results = searchChangelogs("", { version: "7.22" });
     expect(results.length).toBeGreaterThan(0);
     for (const r of results) {
       expect(r.version).toBe("7.22");
     }
+    expect(results.some((r) => r.version === "7.22.1")).toBe(false);
+  });
+
+  test("major.minor version filter falls back to patch rows when exact rows are absent", () => {
+    const results = searchChangelogs("", { version: "7.23" });
+    expect(results.length).toBeGreaterThan(0);
+    expect(results.every((r) => r.version === "7.23.1")).toBe(true);
+  });
+
+  test("generic major.minor version questions browse fallback patch rows", () => {
+    const results = searchChangelogs("what changed in 7.23", { version: "7.23" });
+    expect(results.length).toBeGreaterThan(0);
+    expect(results.every((r) => r.version === "7.23.1")).toBe(true);
   });
 
   test("version range filter works (from_version exclusive, to_version inclusive)", () => {
@@ -1885,7 +1997,19 @@ Today we will cover trunking. Let us begin.
     const vtt = `WEBVTT\n\n00:00:01.000 --> 00:00:03.000\n<b>Bold text</b> and <i>italic</i>.\n`;
     const cues = parseVtt(vtt);
     expect(cues[0].text).not.toContain("<b>");
-    expect(cues[0].text).toContain("Bold text");
+    expect(cues[0].text).toBe("Bold text and italic.");
+  });
+
+  test("drops malformed cue markup without leaking tag fragments", () => {
+    const vtt = `WEBVTT\n\n00:00:01.000 --> 00:00:03.000\n<scr<script>ipt>alert</scr<script>ipt> safe text.\n`;
+    const cues = parseVtt(vtt);
+    expect(cues[0].text).toBe("safe text.");
+  });
+
+  test("recovers plain text after malformed cue markup", () => {
+    const vtt = `WEBVTT\n\n00:00:01.000 --> 00:00:03.000\nClick here<a h<a href='x'>ref='evil'>safe text.\n`;
+    const cues = parseVtt(vtt);
+    expect(cues[0].text).toBe("Click heresafe text.");
   });
 });
 
@@ -2169,6 +2293,25 @@ describe("listGlossary", () => {
 // ---------------------------------------------------------------------------
 
 describe("searchAll related block", () => {
+  test("exposes command path confidence in classified output", () => {
+    const res = searchAll("/ip/dhcp-server");
+    expect(res.classified.command_path).toBe("/ip/dhcp-server");
+    expect(res.classified.command_path_confidence).toBe("medium");
+  });
+
+  test("includes property lookup confidence in related properties", () => {
+    const res = searchAll("lease-time");
+    expect(res.related.properties?.length).toBeGreaterThan(0);
+    expect(res.related.properties?.[0].confidence).toBe("medium");
+  });
+
+  test("includes changelogs for major.minor version questions backed only by patch rows", () => {
+    const res = searchAll("what changed in 7.23");
+    expect(res.classified.version).toBe("7.23");
+    expect(res.related.changelogs?.length).toBeGreaterThan(0);
+    expect(res.related.changelogs?.every((row) => row.version === "7.23.1")).toBe(true);
+  });
+
   test("includes glossary when input matches a glossary term", () => {
     const res = searchAll("chr");
     expect(res.related).toBeDefined();