@tikoci/rosetta 0.4.0 → 0.4.2

package/README.md

@@ -17,6 +17,7 @@ Instead of vector embeddings, rosetta uses **SQLite [FTS5](https://www.sqlite.or
 - **144 hardware products** — CPU, RAM, storage, ports, PoE, wireless, license level, pricing
 - **2,874 performance benchmarks** — ethernet and IPSec throughput test results for 125 devices (64/512/1518-byte packets, multiple routing/bridging modes), plus block diagrams for 110
 - **46 RouterOS versions tracked** (7.9 through 7.23beta2) for command history
+- **2 MCP CSV resources** for bulk reporting workflows: full benchmark dataset and full device catalog
 - Direct links to help.mikrotik.com for every page and section
 
 ---
@@ -212,6 +213,15 @@ powershell -c "irm bun.sh/install.ps1 | iex"
 
 > **Auto-update:** `bunx` checks the npm registry each session and uses the latest published version automatically. The database in `~/.rosetta/ros-help.db` persists across updates.
 
+### MCP Resources for Reporting
+
+If your MCP client supports resources, rosetta also exposes two read-only CSV datasets for bulk analysis and reporting:
+
+- `rosetta://datasets/device-test-results.csv`
+- `rosetta://datasets/devices.csv`
+
+In VS Code Copilot, attach them via **Add Context > MCP Resources** or **MCP: Browse Resources**. Use tools for normal search and drill-down; use resources when you explicitly want the whole dataset as CSV.
+
 ---
 
 ## Install from Binary
@@ -245,10 +255,11 @@ Ask your AI assistant questions like:
 - *"Show me warnings about hardware offloading"*
 - *"Which MikroTik routers have L3HW offload, and more than 8 ports of 48V PoE? Include cost."*
 - *"Compare the RB5009 and CCR2004 IPSec throughput at 1518-byte packets."*
+- *"My BGP routes stopped working after upgrading from 7.15 to 7.22 — what changed in the routing commands?"*
 
 ## MCP Tools
 
-The server provides 11 tools, designed to work together:
+The server provides 13 tools, designed to work together:
 
 | Tool | What it does |
 |------|-------------|
@@ -260,6 +271,7 @@ The server provides 11 tools, designed to work together:
 | `routeros_search_callouts` | Search warnings, notes, and tips across all pages |
 | `routeros_search_changelogs` | Search parsed changelog entries — filter by version range, category, breaking changes |
 | `routeros_command_version_check` | Check which RouterOS versions include a command |
+| `routeros_command_diff` | Diff two RouterOS versions — which command paths were added or removed between them |
 | `routeros_device_lookup` | Hardware specs for 144 MikroTik products — filter by architecture, RAM, storage, PoE, wireless, LTE. Includes ethernet/IPSec benchmarks and block diagrams for most devices |
 | `routeros_stats` | Database health: page/property/command counts, coverage stats |
 | `routeros_current_versions` | Fetch current RouterOS versions from MikroTik (live) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/db.ts

@@ -20,7 +20,9 @@
  */
 
 import sqlite from "bun:sqlite";
-import { resolveDbPath } from "./paths.ts";
+import { resolveDbPath, SCHEMA_VERSION } from "./paths.ts";
+
+export { SCHEMA_VERSION };
 
 export const DB_PATH = resolveDbPath(import.meta.dirname);
 
@@ -29,7 +31,11 @@ export const db = new sqlite(DB_PATH);
 export function initDb() {
   db.run("PRAGMA journal_mode=WAL;");
   db.run("PRAGMA foreign_keys=ON;");
-
+  // Stamp schema version unconditionally — initDb() is only called by extractors
+  // (which produce a current-schema DB) and by the MCP server after the version
+  // check in mcp.ts. If you ever need to open a DB read-only without touching
+  // user_version, call `db.run("PRAGMA foreign_keys=ON;")` directly and skip initDb().
+  db.run(`PRAGMA user_version = ${SCHEMA_VERSION};`);
   db.run(`CREATE TABLE IF NOT EXISTS schema_migrations (
     version TEXT PRIMARY KEY,
     applied_at TEXT NOT NULL
@@ -342,13 +348,19 @@ export function initDb() {
   END;`);
 }
 
+/**
+ * Verify the open DB was built with the expected schema version.
+ * Only meaningful after initDb() — initDb() itself stamps the version,
+ * so this is mainly a regression guard and for the test suite.
+ */
+export function checkSchemaVersion(): { ok: boolean; actual: number; expected: number } {
+  const row = db.prepare("PRAGMA user_version").get() as { user_version: number };
+  return { ok: row.user_version === SCHEMA_VERSION, actual: row.user_version, expected: SCHEMA_VERSION };
+}
+
 export function getDbStats() {
   const count = (sql: string) =>
     Number((db.prepare(sql).get() as { c: number }).c ?? 0);
-  const scalar = (sql: string) => {
-    const row = db.prepare(sql).get() as { v: string | null } | null;
-    return row?.v ?? null;
-  };
   return {
     db_path: DB_PATH,
     pages: count("SELECT COUNT(*) AS c FROM pages"),
@@ -363,8 +375,27 @@ export function getDbStats() {
     changelogs: count("SELECT COUNT(*) AS c FROM changelogs"),
     changelog_versions: count("SELECT COUNT(DISTINCT version) AS c FROM changelogs"),
     ros_versions: count("SELECT COUNT(*) AS c FROM ros_versions"),
-    ros_version_min: scalar("SELECT MIN(version) AS v FROM ros_versions"),
-    ros_version_max: scalar("SELECT MAX(version) AS v FROM ros_versions"),
+    ...(() => {
+      // Semantic version sort — SQL MIN/MAX is lexicographic ("7.10" < "7.9")
+      const versions = (db.prepare("SELECT version FROM ros_versions").all() as Array<{ version: string }>).map((r) => r.version);
+      if (versions.length === 0) return { ros_version_min: null, ros_version_max: null };
+      const norm = (v: string) => {
+        const clean = v.replace(/beta\d*/, "").replace(/rc\d*/, "");
+        const parts = clean.split(".").map(Number);
+        const suffix = v.includes("beta") ? 0 : v.includes("rc") ? 1 : 2;
+        return { parts, suffix };
+      };
+      const cmp = (a: string, b: string) => {
+        const na = norm(a), nb = norm(b);
+        for (let i = 0; i < Math.max(na.parts.length, nb.parts.length); i++) {
+          const d = (na.parts[i] ?? 0) - (nb.parts[i] ?? 0);
+          if (d !== 0) return d;
+        }
+        return na.suffix - nb.suffix;
+      };
+      versions.sort(cmp);
+      return { ros_version_min: versions[0], ros_version_max: versions[versions.length - 1] };
+    })(),
     doc_export: "2026-03-25 (Confluence HTML)",
   };
 }

package/src/extract-test-results.ts

@@ -12,8 +12,10 @@
  *
  * Usage: bun run src/extract-test-results.ts [--concurrency N] [--delay MS]
  *
- * Product page URL slug discovery: fetches the product matrix page to build
- * a name→slug mapping, then fetches each product page by slug.
+ * Product page URL slug discovery: fetches sitemap.xml once to build a validated
+ * slug set, then applies (1) a hardcoded override table for the 15 products with
+ * opaque/unpredictable slugs, (2) heuristic candidates validated against the sitemap,
+ * (3) raw heuristics as a fallback for new products not yet in the sitemap.
  */
 
 import { parseHTML } from "linkedom";
@@ -31,6 +33,28 @@ function getFlag(name: string, fallback: number): number {
 const CONCURRENCY = getFlag("concurrency", 4);
 const DELAY_MS = getFlag("delay", 500);
 const PRODUCT_BASE = "https://mikrotik.com/product/";
+const SITEMAP_URL = "https://mikrotik.com/sitemap.xml";
+
+// ── Slug overrides ──
+// Products whose URL slugs are undiscoverable by heuristics alone.
+// Derived from sitemap.xml research (2026-04-04 — all 15 missing devices resolved).
+const SLUG_OVERRIDES: Record<string, string> = {
+  "ATL LTE18 kit": "atl18",
+  "CRS418-8P-8G-2S+5axQ2axQ-RM": "crs418_8p_8g_2s_wifi",
+  "CRS518-16XS-2XQ-RM": "crs518_16xs_2xq",
+  "Chateau LTE18 ax": "chateaulte18_ax",
+  "FiberBox Plus": "fiberboxplus",
+  "KNOT Embedded LTE4 Global": "knot_emb_lte4_global",
+  "LHG LTE18 kit": "lhg_lte18",
+  "LHG XL 5 ax": "lhg_5_ax_xl",
+  "LHGG LTE7 kit": "lhgg_lte7",
+  "RB5009UPr+S+OUT": "rb5009_out",
+  "ROSE Data server (RDS)": "rds2216",
+  "SXTsq 5 ax": "sxtsq_5ax",
+  "cAP lite": "RBcAPL-2nD-307",
+  "hEX refresh": "hex_2024",
+  "wAP ax LTE7 kit": "wap_ax_lte7",
+};
 
 // ── Types ──
 
@@ -51,14 +75,6 @@ interface ProductPageData {
 
 // ── HTML Parsing ──
 
-/** Decode HTML entities like &#110;&#111;&#110;&#101; to text. */
-function decodeEntities(html: string): string {
-  const { document } = parseHTML("<div></div>");
-  const el = document.createElement("div");
-  el.innerHTML = html;
-  return el.textContent || "";
-}
-
 /** Parse a performance-table element into test result rows. */
 function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
   const rows: TestResultRow[] = [];
@@ -162,10 +178,10 @@ function generateSlugs(name: string, code: string | null): string[] {
     const cleanCode = norm(code);
 
     // 3. Product code as-is (original casing, + → plus, strip other specials)
-    add(cleanCode.replace(/\+/g, "plus").replace(/[^a-zA-Z0-9plus\-]+/g, "").replace(/^-|-$/g, ""));
+    add(cleanCode.replace(/\+/g, "plus").replace(/[^a-zA-Z0-9plus-]+/g, "").replace(/^-|-$/g, ""));
 
     // 4. Product code as-is (original casing)
-    add(cleanCode.replace(/[^a-zA-Z0-9\-]+/g, "").replace(/^-|-$/g, ""));
+    add(cleanCode.replace(/[^a-zA-Z0-9-]+/g, "").replace(/^-|-$/g, ""));
 
     // 5. Lowercased code: + → plus
     add(cleanCode.toLowerCase().replace(/\+/g, "plus").replace(/[^a-z0-9plus]+/g, "_").replace(/^_|_$/g, ""));
@@ -177,6 +193,59 @@ function generateSlugs(name: string, code: string | null): string[] {
   return slugs;
 }
 
+/** Fetch sitemap.xml and return the set of all valid product slugs.
+ *  Returns an empty set on error (callers fall back to heuristics). */
+async function fetchSitemap(): Promise<Set<string>> {
+  try {
+    const resp = await fetch(SITEMAP_URL);
+    if (!resp.ok) {
+      console.warn(`  [sitemap] HTTP ${resp.status} — falling back to heuristics`);
+      return new Set();
+    }
+    const xml = await resp.text();
+    const slugs = new Set<string>();
+    const re = /\/product\/([^<"]+)/g;
+    while (true) {
+      const m = re.exec(xml);
+      if (m === null) break;
+      slugs.add(m[1]);
+    }
+    console.log(`  [sitemap] ${slugs.size} product slugs loaded`);
+    return slugs;
+  } catch {
+    console.warn("  [sitemap] fetch failed — falling back to heuristics");
+    return new Set();
+  }
+}
+
+/** Build ordered slug candidates for a product.
+ *  Priority: (1) override table, (2) generated slugs that appear in sitemap,
+ *  (3) remaining generated slugs as fallback. */
+function buildSlugCandidates(name: string, code: string | null, sitemapSlugs: Set<string>): string[] {
+  const slugs: string[] = [];
+  const seen = new Set<string>();
+  const add = (s: string) => {
+    if (s && !seen.has(s)) { seen.add(s); slugs.push(s); }
+  };
+
+  // 1. Override table (highest priority — known-opaque slugs)
+  const override = SLUG_OVERRIDES[name];
+  if (override) add(override);
+
+  // 2. Heuristic candidates validated against sitemap
+  const generated = generateSlugs(name, code);
+  if (sitemapSlugs.size > 0) {
+    for (const s of generated) {
+      if (sitemapSlugs.has(s)) add(s);
+    }
+  }
+
+  // 3. All remaining heuristic candidates (fallback for new products not yet in sitemap)
+  for (const s of generated) add(s);
+
+  return slugs;
+}
+
 /** Fetch and parse a single product page, trying multiple slug candidates. */
 async function fetchProductPage(slugs: string[]): Promise<ProductPageData | null> {
   for (const slug of slugs) {
@@ -257,10 +326,14 @@ if (devices.length === 0) {
 
 console.log(`Found ${devices.length} devices in database`);
 
+// Fetch sitemap once to validate slugs and prioritize correct candidates
+console.log("Loading product sitemap...");
+const sitemapSlugs = await fetchSitemap();
+
 // Build device → candidate slugs mapping
 const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
 for (const dev of devices) {
-  const slugs = generateSlugs(dev.product_name, dev.product_code);
+  const slugs = buildSlugCandidates(dev.product_name, dev.product_code, sitemapSlugs);
   deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
 }
 

package/src/mcp-http.test.ts

@@ -13,7 +13,11 @@
  *
  * Each test gets an isolated server on a fresh port.
  */
+import sqlite from "bun:sqlite";
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import type { Subprocess } from "bun";
 
 // ── Helpers ──
@@ -31,42 +35,146 @@ interface ServerHandle {
   proc: Subprocess;
 }
 
+interface ProcessLogs {
+  stdout: string[];
+  stderr: string[];
+}
+
+/** Continuously consume a subprocess stream and append decoded chunks. */
+async function collectStream(stream: ReadableStream<Uint8Array>, sink: string[]): Promise<void> {
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      sink.push(decoder.decode(value));
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+/** Keep only the last N characters from aggregated process logs for error messages. */
+function tailLogs(logs: ProcessLogs, maxChars = 4000): string {
+  const merged = `--- stdout ---\n${logs.stdout.join("")}\n--- stderr ---\n${logs.stderr.join("")}`;
+  return merged.length > maxChars ? merged.slice(-maxChars) : merged;
+}
+
 /** Start the MCP server on a random port, wait for it to be ready. */
-async function startServer(): Promise<ServerHandle> {
+function createFixtureDb(dbPath: string): void {
+  const fixture = new sqlite(dbPath);
+  fixture.run(`CREATE TABLE pages (
+    id           INTEGER PRIMARY KEY,
+    slug         TEXT NOT NULL,
+    title        TEXT NOT NULL,
+    path         TEXT NOT NULL,
+    depth        INTEGER NOT NULL,
+    parent_id    INTEGER REFERENCES pages(id),
+    url          TEXT NOT NULL,
+    text         TEXT NOT NULL,
+    code         TEXT NOT NULL,
+    code_lang    TEXT,
+    author       TEXT,
+    last_updated TEXT,
+    word_count   INTEGER NOT NULL,
+    code_lines   INTEGER NOT NULL,
+    html_file    TEXT NOT NULL
+  );`);
+
+  fixture.run(`CREATE TABLE commands (
+    id          INTEGER PRIMARY KEY,
+    path        TEXT NOT NULL UNIQUE,
+    name        TEXT NOT NULL,
+    type        TEXT NOT NULL,
+    parent_path TEXT,
+    page_id     INTEGER REFERENCES pages(id),
+    description TEXT,
+    ros_version TEXT
+  );`);
+
+  fixture.run(`INSERT INTO pages (
+    id, slug, title, path, depth, parent_id, url, text, code, code_lang,
+    author, last_updated, word_count, code_lines, html_file
+  ) VALUES (
+    1, 'fixture', 'Fixture Page', 'RouterOS > Fixture', 1, NULL,
+    'https://help.mikrotik.com/docs/spaces/ROS/pages/1/Fixture',
+    'fixture text', '', NULL, 'test', NULL, 2, 0, 'fixture.html'
+  );`);
+
+  fixture.run(`INSERT INTO commands (
+    id, path, name, type, parent_path, page_id, description, ros_version
+  ) VALUES (
+    1, '/system', 'system', 'dir', NULL, 1, 'fixture command', '7.22'
+  );`);
+
+  fixture.close();
+}
+
+async function startServer(dbPath: string): Promise<ServerHandle> {
   const port = nextPort();
   const proc = Bun.spawn(["bun", "run", "src/mcp.ts", "--http", "--port", String(port)], {
     cwd: `${import.meta.dirname}/..`,
     stdout: "pipe",
     stderr: "pipe",
-    // Explicitly set DB_PATH so query.test.ts's process.env.DB_PATH=":memory:" override
-    // is not inherited by the server subprocess.
-    env: { ...process.env, HOST: "127.0.0.1", DB_PATH: `${import.meta.dirname}/../ros-help.db` },
+    // Use an isolated fixture DB and avoid network-dependent auto-downloads.
+    env: { ...process.env, HOST: "127.0.0.1", DB_PATH: dbPath },
+  });
+
+  const logs: ProcessLogs = { stdout: [], stderr: [] };
+  const stdoutCollector = collectStream(proc.stdout, logs.stdout);
+  const stderrCollector = collectStream(proc.stderr, logs.stderr);
+
+  let exitCode: number | null = null;
+  void proc.exited.then((code) => {
+    exitCode = code;
   });
 
-  // Wait for server to be ready (reads stderr for the startup message)
-  // 30s: 3 servers start in parallel during full test suite, contention extends startup time
-  const deadline = Date.now() + 30_000;
+  // Wait for server readiness by polling the actual HTTP endpoint.
+  // Use a generous timeout because first boot may auto-download the DB.
+  const deadline = Date.now() + 120_000;
   let ready = false;
-  const decoder = new TextDecoder();
-  const reader = proc.stderr.getReader();
 
   while (Date.now() < deadline) {
-    const { value, done } = await reader.read();
-    if (done) break;
-    const text = decoder.decode(value);
-    if (text.includes(`/mcp`)) {
-      ready = true;
+    if (exitCode !== null) {
       break;
     }
+
+    try {
+      const resp = await fetch(`http://127.0.0.1:${port}/mcp`, {
+        method: "GET",
+        headers: { Accept: "text/event-stream" },
+      });
+
+      // Endpoint is alive; missing session ID should return 400.
+      if (resp.status === 400) {
+        ready = true;
+        break;
+      }
+    } catch {
+      // Connection refused while server is still starting.
+    }
+
+    await Bun.sleep(200);
   }
-  // Release the reader lock so the stream can be consumed later or cleaned up
-  reader.releaseLock();
 
   if (!ready) {
     proc.kill();
-    throw new Error(`Server failed to start on port ${port} within 10s`);
+    await proc.exited.catch(() => undefined);
+    await Promise.all([stdoutCollector, stderrCollector]);
+
+    const reason = exitCode !== null
+      ? `exited early with code ${exitCode}`
+      : "did not become ready before timeout";
+    throw new Error(
+      `Server failed to start on port ${port}: ${reason}\n${tailLogs(logs)}`,
+    );
   }
 
+  // Keep collectors running in background; they naturally finish when process exits.
+  void stdoutCollector;
+  void stderrCollector;
+
   return { port, url: `http://127.0.0.1:${port}/mcp`, proc };
 }
 
@@ -178,13 +286,19 @@ async function mcpNotification(
 // so sharing a server does not affect test independence. Starting one server instead
 // of three avoids startup-time contention when all test files run in parallel.
 let server: ServerHandle;
+let fixtureDir: string;
+let fixtureDbPath: string;
 
 beforeAll(async () => {
-  server = await startServer();
-}, 30_000);
+  fixtureDir = mkdtempSync(join(tmpdir(), "rosetta-http-test-"));
+  fixtureDbPath = join(fixtureDir, "ros-help.db");
+  createFixtureDb(fixtureDbPath);
+  server = await startServer(fixtureDbPath);
+}, 130_000);
 
 afterAll(async () => {
   await killServer(server);
+  rmSync(fixtureDir, { recursive: true, force: true });
 }, 15_000);
 
 describe("HTTP transport: session lifecycle", () => {
@@ -212,7 +326,7 @@ describe("HTTP transport: session lifecycle", () => {
 
     const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
     const tools = result.tools as Array<{ name: string }>;
-    expect(tools.length).toBe(11);
+    expect(tools.length).toBe(13);
 
     const toolNames = tools.map((t) => t.name).sort();
     expect(toolNames).toContain("routeros_search");
@@ -228,6 +342,37 @@ describe("HTTP transport: session lifecycle", () => {
     expect(toolNames).toContain("routeros_current_versions");
   });
 
+  test("resources/list returns dataset resources after initialization", async () => {
+    const { sessionId } = await mcpInitialize(server.url);
+    await mcpNotification(server.url, sessionId, "notifications/initialized");
+
+    const messages = await mcpRequest(server.url, sessionId, "resources/list", 21);
+    expect(messages.length).toBe(1);
+
+    const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
+    const resources = result.resources as Array<{ uri: string; mimeType?: string }>;
+
+    expect(resources.some((resource) => resource.uri === "rosetta://datasets/device-test-results.csv")).toBe(true);
+    expect(resources.some((resource) => resource.uri === "rosetta://datasets/devices.csv")).toBe(true);
+  });
+
+  test("resources/read returns CSV content for device test dataset", async () => {
+    const { sessionId } = await mcpInitialize(server.url);
+    await mcpNotification(server.url, sessionId, "notifications/initialized");
+
+    const messages = await mcpRequest(server.url, sessionId, "resources/read", 22, {
+      uri: "rosetta://datasets/device-test-results.csv",
+    });
+    expect(messages.length).toBe(1);
+
+    const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
+    const contents = result.contents as Array<{ uri: string; mimeType?: string; text?: string }>;
+
+    expect(contents[0].uri).toBe("rosetta://datasets/device-test-results.csv");
+    expect(contents[0].mimeType).toBe("text/csv");
+    expect(contents[0].text).toContain("product_name,product_code,architecture,cpu,cpu_cores,cpu_frequency");
+  });
+
   test("tools/call works for routeros_stats", async () => {
     const { sessionId } = await mcpInitialize(server.url);
     await mcpNotification(server.url, sessionId, "notifications/initialized");
@@ -402,8 +547,8 @@ describe("HTTP transport: multi-session", () => {
     const tools1 = ((msgs1[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
     const tools2 = ((msgs2[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
 
-    expect(tools1.length).toBe(11);
-    expect(tools2.length).toBe(11);
+    expect(tools1.length).toBe(13);
+    expect(tools2.length).toBe(13);
   });
 
   test("deleting one session does not affect another", async () => {
@@ -425,7 +570,7 @@ describe("HTTP transport: multi-session", () => {
     // Client2 still works
     const msgs = await mcpRequest(server.url, client2.sessionId, "tools/list", 2);
     const tools = ((msgs[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
-    expect(tools.length).toBe(11);
+    expect(tools.length).toBe(13);
 
     // Client1 is gone
     const resp = await fetch(server.url, {