@tikoci/rosetta 0.5.0 → 0.5.2

package/src/extract-html.test.ts

@@ -0,0 +1,67 @@
+// Set BEFORE any import that transitively loads db.ts
+process.env.DB_PATH = ":memory:";
+
+import { describe, expect, it } from "bun:test";
+import { parseHTML } from "linkedom";
+
+// Dynamic import so the DB_PATH assignment above wins over module caching
+const { extractPlainText, sanitizeExtractedText } = await import("./extract-html.ts");
+
+describe("sanitizeExtractedText", () => {
+  it("removes Confluence TOC CDATA style blocks", () => {
+    const input = `
+Intro
+/*<![CDATA[*/
+div.rbtoc1774430868497 {padding: 0px;}
+div.rbtoc1774430868497 ul {margin-left: 0px;}
+div.rbtoc1774430868497 li {margin-left: 0px;padding-left: 0px;}
+/*]]>*/
+Basic Setup
+`;
+
+    const out = sanitizeExtractedText(input);
+    expect(out).toContain("Intro");
+    expect(out).toContain("Basic Setup");
+    expect(out).not.toContain("rbtoc1774430868497");
+    expect(out).not.toContain("/*<![CDATA[*/");
+    expect(out).not.toContain("/*]]>*/");
+  });
+
+  it("removes bare rbtoc css lines without wrappers", () => {
+    const input = `
+Top
+
+div.rbtoc1234 {padding: 0px;}
+div.rbtoc1234 ul {margin-left: 0px;}
+
+after
+`;
+
+    const out = sanitizeExtractedText(input);
+    expect(out).toContain("Top");
+    expect(out).toContain("after");
+    expect(out).not.toContain("div.rbtoc1234");
+  });
+
+  it("keeps normal RouterOS content", () => {
+    const input = "/interface vrrp add interface=ether1 vrid=49 priority=254";
+    const out = sanitizeExtractedText(input);
+    expect(out).toBe(input);
+  });
+});
+
+describe("extractPlainText", () => {
+  it("keeps heading and paragraph separated", () => {
+    const { document } = parseHTML("<div><h1>Basic Setup</h1><p>This is the basic VRRP configuration example.</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("Basic Setup\nThis is the basic VRRP configuration example.");
+    expect(out).not.toContain("Basic SetupThis");
+  });
+
+  it("keeps subsection heading boundaries", () => {
+    const { document } = parseHTML("<div><h2>Overview</h2><p>The Point-to-Point Protocol (PPP) provides...</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("Overview\nThe Point-to-Point Protocol (PPP) provides...");
+    expect(out).not.toContain("OverviewThe");
+  });
+});

package/src/extract-html.ts

@@ -64,6 +64,120 @@ interface SectionRow {
   sort_order: number;
 }
 
+const BLOCK_TAGS = new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tbody",
+  "td",
+  "th",
+  "thead",
+  "tr",
+  "ul",
+]);
+
+/**
+ * Remove Confluence TOC style leakage that may survive DOM cleanup when HTML is malformed.
+ * Keeps cleanup narrow to rbtoc CSS so normal content is not affected.
+ */
+export function sanitizeExtractedText(input: string): string {
+  if (!input) return "";
+
+  return input
+    // Full CDATA-wrapped TOC style block.
+    .replace(/\/\*<!\[CDATA\[\*\/[\s\S]*?\/\*\]\]>\*\//g, "")
+    // Bare TOC CSS lines that can leak without wrappers.
+    .replace(/^\s*div\.rbtoc\d+[^\n]*(?:\n\s*div\.rbtoc\d+[^\n]*)*/gm, "")
+    // Any orphaned CDATA markers.
+    .replace(/\/\*<!\[CDATA\[\*\//g, "")
+    .replace(/\/\*\]\]>\*\//g, "")
+    // Prevent giant gaps in rendered output.
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/**
+ * Convert a DOM subtree to readable plain text while preserving block boundaries.
+ * This avoids merged tokens like "OverviewThe" caused by raw textContent collapse.
+ */
+export function extractPlainText(root: Element | null): string {
+  if (!root) return "";
+
+  let out = "";
+
+  const addBreak = () => {
+    if (!out) return;
+    if (!out.endsWith("\n")) out += "\n";
+  };
+
+  const addText = (text: string) => {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    if (!normalized) return;
+    if (out && !out.endsWith("\n") && !out.endsWith(" ")) out += " ";
+    out += normalized;
+  };
+
+  const walk = (node: Node) => {
+    if (node.nodeType === 3) {
+      addText(node.textContent || "");
+      return;
+    }
+
+    if (node.nodeType !== 1) return;
+
+    const el = node as Element;
+    const tag = el.tagName.toLowerCase();
+
+    if (tag === "style" || tag === "script" || tag === "noscript") return;
+    if (tag === "br") {
+      addBreak();
+      return;
+    }
+
+    const isBlock = BLOCK_TAGS.has(tag);
+    if (isBlock) addBreak();
+
+    for (const child of el.childNodes) {
+      walk(child as unknown as Node);
+    }
+
+    if (isBlock) addBreak();
+  };
+
+  walk(root as unknown as Node);
+
+  return out
+    .replace(/[ \t]*\n[ \t]*/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
 /**
  * Split main content into sections by h1–h3 headings with id attributes.
  * Uses innerHTML + regex to locate heading boundaries, then parses each
@@ -105,7 +219,7 @@ function extractSections(mainContent: Element, pageId: number): SectionRow[] {
       codeChunks.push(ce.textContent?.trim() || "");
     }
 
-    const text = root?.textContent?.trim() || "";
+    const text = sanitizeExtractedText(extractPlainText(root));
     const code = codeChunks.join("\n\n");
 
     return {
@@ -179,8 +293,15 @@ function extractPage(file: string, html: string): (PageRow & { callouts: Callout
   const codeLang = codeLangs.size > 0 ? [...codeLangs].join(",") : null;
   const codeLines = code.split("\n").filter((l) => l.trim()).length;
 
+  // Remove <style> elements before extracting text — they contain Confluence
+  // table-of-contents CSS (CDATA blocks like div.rbtoc... {padding:0}) that
+  // otherwise pollute the plain-text index and appear in page views.
+  for (const styleEl of mainContent?.querySelectorAll("style") ?? []) {
+    styleEl.remove();
+  }
+
   // Plain text from main content (includes code block text too, which is fine for FTS)
-  const text = mainContent?.textContent?.trim() || "";
+  const text = sanitizeExtractedText(extractPlainText(mainContent));
   const wordCount = text.split(/\s+/).filter(Boolean).length;
 
   // Callouts: extract note/warning/info blocks
@@ -231,149 +352,155 @@ function extractPage(file: string, html: string): (PageRow & { callouts: Callout
 
 // ---- Main ----
 
-console.log("Initializing database...");
-initDb();
+function main() {
+  console.log("Initializing database...");
+  initDb();
 
 // Drop existing data for clean re-extraction (respect FK order)
-db.run("DELETE FROM sections;");
-db.run("DELETE FROM callouts;");
-db.run("INSERT INTO callouts_fts(callouts_fts) VALUES('rebuild');");
-db.run("DELETE FROM properties;");
-db.run("INSERT INTO properties_fts(properties_fts) VALUES('rebuild');");
-db.run("PRAGMA foreign_keys = OFF;");
-db.run("DELETE FROM pages;");
-db.run("PRAGMA foreign_keys = ON;");
-db.run("INSERT INTO pages_fts(pages_fts) VALUES('rebuild');");
-
-const htmlFiles = readdirSync(HTML_DIR)
-  .filter((f) => f.endsWith(".html") && f !== "index.html")
-  .sort();
-
-console.log(`Extracting ${htmlFiles.length} HTML files from ${HTML_DIR}`);
+  db.run("DELETE FROM sections;");
+  db.run("DELETE FROM callouts;");
+  db.run("INSERT INTO callouts_fts(callouts_fts) VALUES('rebuild');");
+  db.run("DELETE FROM properties;");
+  db.run("INSERT INTO properties_fts(properties_fts) VALUES('rebuild');");
+  db.run("PRAGMA foreign_keys = OFF;");
+  db.run("DELETE FROM pages;");
+  db.run("PRAGMA foreign_keys = ON;");
+  db.run("INSERT INTO pages_fts(pages_fts) VALUES('rebuild');");
+
+  const htmlFiles = readdirSync(HTML_DIR)
+    .filter((f) => f.endsWith(".html") && f !== "index.html")
+    .sort();
+
+  console.log(`Extracting ${htmlFiles.length} HTML files from ${HTML_DIR}`);
 
 // Two-pass insert: first without parent_id (avoids FK ordering issues),
 // then update parent relationships.
-const insertPage = db.prepare(`
+  const insertPage = db.prepare(`
   INSERT OR REPLACE INTO pages
     (id, slug, title, path, depth, parent_id, url, text, code, code_lang,
      author, last_updated, word_count, code_lines, html_file)
   VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 `);
-const updateParent = db.prepare("UPDATE pages SET parent_id = ? WHERE id = ?");
+  const updateParent = db.prepare("UPDATE pages SET parent_id = ? WHERE id = ?");
 
-let extracted = 0;
-let skipped = 0;
-let totalWords = 0;
-let totalCodeLines = 0;
-let totalCallouts = 0;
+  let extracted = 0;
+  let skipped = 0;
+  let totalWords = 0;
+  let totalCodeLines = 0;
+  let totalCallouts = 0;
 
-const allPages: (PageRow & { callouts: CalloutRow[]; sections: SectionRow[] })[] = [];
+  const allPages: (PageRow & { callouts: CalloutRow[]; sections: SectionRow[] })[] = [];
 
 // Pass 1: extract and insert all pages (parent_id = NULL)
-const insertAll = db.transaction(() => {
-  for (const file of htmlFiles) {
-    const html = readFileSync(resolve(HTML_DIR, file), "utf-8");
-    const page = extractPage(file, html);
-    if (!page) {
-      skipped++;
-      console.warn(`  skipped: ${file}`);
-      continue;
+  const insertAll = db.transaction(() => {
+    for (const file of htmlFiles) {
+      const html = readFileSync(resolve(HTML_DIR, file), "utf-8");
+      const page = extractPage(file, html);
+      if (!page) {
+        skipped++;
+        console.warn(`  skipped: ${file}`);
+        continue;
+      }
+      insertPage.run(
+        page.id,
+        page.slug,
+        page.title,
+        page.path,
+        page.depth,
+        page.url,
+        page.text,
+        page.code,
+        page.code_lang,
+        page.author,
+        page.last_updated,
+        page.word_count,
+        page.code_lines,
+        page.html_file,
+      );
+      allPages.push(page);
+      extracted++;
+      totalWords += page.word_count;
+      totalCodeLines += page.code_lines;
     }
-    insertPage.run(
-      page.id,
-      page.slug,
-      page.title,
-      page.path,
-      page.depth,
-      page.url,
-      page.text,
-      page.code,
-      page.code_lang,
-      page.author,
-      page.last_updated,
-      page.word_count,
-      page.code_lines,
-      page.html_file,
-    );
-    allPages.push(page);
-    extracted++;
-    totalWords += page.word_count;
-    totalCodeLines += page.code_lines;
-  }
-});
-insertAll();
+  });
+  insertAll();
 
 // Pass 2: set parent_id where the parent actually exists in the DB
-const pageIds = new Set(allPages.map((p) => p.id));
-const setParents = db.transaction(() => {
-  for (const page of allPages) {
-    if (page.parent_id && pageIds.has(page.parent_id)) {
-      updateParent.run(page.parent_id, page.id);
+  const pageIds = new Set(allPages.map((p) => p.id));
+  const setParents = db.transaction(() => {
+    for (const page of allPages) {
+      if (page.parent_id && pageIds.has(page.parent_id)) {
+        updateParent.run(page.parent_id, page.id);
+      }
     }
-  }
-});
-setParents();
+  });
+  setParents();
 
 // Pass 3: insert callouts
-const insertCallout = db.prepare(`
+  const insertCallout = db.prepare(`
   INSERT INTO callouts (page_id, type, content, sort_order)
   VALUES (?, ?, ?, ?)
 `);
-const insertCallouts = db.transaction(() => {
-  for (const page of allPages) {
-    for (const c of page.callouts) {
-      insertCallout.run(c.page_id, c.type, c.content, c.sort_order);
-      totalCallouts++;
+  const insertCallouts = db.transaction(() => {
+    for (const page of allPages) {
+      for (const c of page.callouts) {
+        insertCallout.run(c.page_id, c.type, c.content, c.sort_order);
+        totalCallouts++;
+      }
     }
-  }
-});
-insertCallouts();
+  });
+  insertCallouts();
 
 // Pass 4: insert sections
-let totalSections = 0;
-let pagesWithSections = 0;
-const insertSection = db.prepare(`
+  let totalSections = 0;
+  let pagesWithSections = 0;
+  const insertSection = db.prepare(`
   INSERT INTO sections (page_id, heading, level, anchor_id, text, code, word_count, sort_order)
   VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 `);
-const insertSections = db.transaction(() => {
-  for (const page of allPages) {
-    if (page.sections.length > 0) {
-      pagesWithSections++;
-      for (const s of page.sections) {
-        insertSection.run(s.page_id, s.heading, s.level, s.anchor_id, s.text, s.code, s.word_count, s.sort_order);
-        totalSections++;
+  const insertSections = db.transaction(() => {
+    for (const page of allPages) {
+      if (page.sections.length > 0) {
+        pagesWithSections++;
+        for (const s of page.sections) {
+          insertSection.run(s.page_id, s.heading, s.level, s.anchor_id, s.text, s.code, s.word_count, s.sort_order);
+          totalSections++;
+        }
       }
     }
-  }
-});
-insertSections();
+  });
+  insertSections();
 
-const ftsCount = (db.prepare("SELECT COUNT(*) as c FROM pages_fts").get() as { c: number }).c;
+  const ftsCount = (db.prepare("SELECT COUNT(*) as c FROM pages_fts").get() as { c: number }).c;
 
-console.log(`\nExtraction complete:`);
-console.log(`  Pages extracted: ${extracted}`);
-console.log(`  Pages skipped:   ${skipped}`);
-console.log(`  Total words:     ${totalWords.toLocaleString()}`);
-console.log(`  Total code lines: ${totalCodeLines.toLocaleString()}`);
-console.log(`  Total callouts:  ${totalCallouts}`);
-console.log(`  Total sections:  ${totalSections} (across ${pagesWithSections} pages)`);
-console.log(`  FTS index rows:  ${ftsCount}`);
+  console.log(`\nExtraction complete:`);
+  console.log(`  Pages extracted: ${extracted}`);
+  console.log(`  Pages skipped:   ${skipped}`);
+  console.log(`  Total words:     ${totalWords.toLocaleString()}`);
+  console.log(`  Total code lines: ${totalCodeLines.toLocaleString()}`);
+  console.log(`  Total callouts:  ${totalCallouts}`);
+  console.log(`  Total sections:  ${totalSections} (across ${pagesWithSections} pages)`);
+  console.log(`  FTS index rows:  ${ftsCount}`);
 
 // Quick search test
-const testResults = db
-  .prepare(
-    `SELECT s.id, s.title, s.path,
-            snippet(pages_fts, 2, '>>>', '<<<', '...', 20) as excerpt
-     FROM pages_fts fts
-     JOIN pages s ON s.id = fts.rowid
-     WHERE pages_fts MATCH 'firewall filter'
-     ORDER BY rank LIMIT 5`,
-  )
-  .all();
-
-console.log(`\nTest search for "firewall filter":`);
-for (const r of testResults as Array<{ id: number; title: string; path: string; excerpt: string }>) {
-  console.log(`  [${r.id}] ${r.path}`);
-  console.log(`    ${r.excerpt}`);
+  const testResults = db
+    .prepare(
+      `SELECT s.id, s.title, s.path,
+              snippet(pages_fts, 2, '>>>', '<<<', '...', 20) as excerpt
+       FROM pages_fts fts
+       JOIN pages s ON s.id = fts.rowid
+       WHERE pages_fts MATCH 'firewall filter'
+       ORDER BY rank LIMIT 5`,
+    )
+    .all();
+
+  console.log(`\nTest search for "firewall filter":`);
+  for (const r of testResults as Array<{ id: number; title: string; path: string; excerpt: string }>) {
+    console.log(`  [${r.id}] ${r.path}`);
+    console.log(`    ${r.excerpt}`);
+  }
+}
+
+if (import.meta.main) {
+  main();
 }

package/src/mcp.ts

@@ -978,6 +978,10 @@ Workflow:
 → routeros_device_lookup: get full specs (CPU, RAM, pricing) + block diagram for a specific device
 → routeros_search: find documentation about features relevant to the test type`,
     inputSchema: {
+      device: z
+        .string()
+        .optional()
+        .describe("Filter by device product name (substring match, e.g., 'RB5009', 'hAP', 'CCR2216')"),
       test_type: z
         .string()
         .optional()
@@ -1010,8 +1014,8 @@ Workflow:
         .describe("Max results (default 50, max 200)"),
     },
   },
-  async ({ test_type, mode, configuration, packet_size, sort_by, limit }) => {
-    const hasFilters = test_type || mode || configuration || packet_size;
+  async ({ device, test_type, mode, configuration, packet_size, sort_by, limit }) => {
+    const hasFilters = device || test_type || mode || configuration || packet_size;
 
     if (!hasFilters) {
       // Discovery mode: return available filter values
@@ -1029,7 +1033,7 @@ Workflow:
     }
 
     const result = searchDeviceTests(
-      { test_type, mode, configuration, packet_size, sort_by },
+      { device, test_type, mode, configuration, packet_size, sort_by },
       limit,
     );
 
@@ -1065,7 +1069,7 @@ server.registerTool(
   {
     description: `Fetch current RouterOS version numbers from MikroTik's upgrade server.
 
-Returns the latest version for each release channel: stable, long-term, testing, development.
+Returns the latest version for each release channel (stable, long-term, testing, development) plus the current WinBox 4 version.
 Useful for determining if a user's version is current, outdated, or unpatched.
 
 Key context for version reasoning:

package/src/query.test.ts

@@ -267,6 +267,10 @@ beforeAll(() => {
   db.run(`INSERT INTO device_test_results
     (device_id, test_type, mode, configuration, packet_size, throughput_kpps, throughput_mbps)
     VALUES (1, 'ipsec', 'Single tunnel', 'AES-128-CBC + SHA1', 1400, 120.9, 1354.1)`);
+  // RB5009UG+S+IN = id 9 (9th device inserted)
+  db.run(`INSERT INTO device_test_results
+    (device_id, test_type, mode, configuration, packet_size, throughput_kpps, throughput_mbps)
+    VALUES (9, 'ethernet', 'Routing', 'none (fast path)', 1518, 1613.0, 19577.3)`);
 
   // Page 3: a "large" page with sections for TOC testing
   // Text is ~200 chars to keep fixture small, but we'll use max_length=50 to trigger truncation
@@ -1154,14 +1158,14 @@ describe("searchDevices", () => {
 describe("searchDeviceTests", () => {
   test("returns all test results with no filters", () => {
     const res = searchDeviceTests({});
-    expect(res.results.length).toBe(3);
-    expect(res.total).toBe(3);
+    expect(res.results.length).toBe(4);
+    expect(res.total).toBe(4);
   });
 
   test("filters by test_type", () => {
     const res = searchDeviceTests({ test_type: "ethernet" });
     expect(res.results.every((r) => r.test_type === "ethernet")).toBe(true);
-    expect(res.results.length).toBe(2);
+    expect(res.results.length).toBe(3);
   });
 
   test("filters by test_type and mode", () => {
@@ -1181,6 +1185,32 @@ describe("searchDeviceTests", () => {
     expect(res.results.every((r) => r.packet_size === 512)).toBe(true);
   });
 
+  test("filters by device name (substring match)", () => {
+    const res = searchDeviceTests({ device: "hAP" });
+    expect(res.results).toHaveLength(3);
+    expect(res.results.every((r) => r.product_name.includes("hAP"))).toBe(true);
+  });
+
+  test("device filter combined with test_type", () => {
+    const res = searchDeviceTests({ device: "hAP", test_type: "ethernet" });
+    expect(res.results).toHaveLength(2);
+    expect(res.results.every((r) => r.test_type === "ethernet")).toBe(true);
+    expect(res.results.every((r) => r.product_name.includes("hAP"))).toBe(true);
+  });
+
+  test("device filter with no matches returns empty", () => {
+    const res = searchDeviceTests({ device: "Audience" });
+    expect(res.results).toHaveLength(0);
+    expect(res.total).toBe(0);
+  });
+
+  test("device filter for RB5009 returns its test results", () => {
+    const res = searchDeviceTests({ device: "RB5009" });
+    expect(res.results).toHaveLength(1);
+    expect(res.results[0].product_name).toBe("RB5009UG+S+IN");
+    expect(res.results[0].packet_size).toBe(1518);
+  });
+
   test("sorts by mbps descending by default", () => {
     const res = searchDeviceTests({ test_type: "ethernet" });
     if (res.results.length >= 2) {
@@ -1210,7 +1240,7 @@ describe("searchDeviceTests", () => {
   test("respects limit", () => {
     const res = searchDeviceTests({}, 1);
     expect(res.results).toHaveLength(1);
-    expect(res.total).toBe(3);
+    expect(res.total).toBe(4);
   });
 });
 
@@ -1220,9 +1250,10 @@ describe("dataset CSV exports", () => {
     const lines = csv.trim().split("\n");
 
     expect(lines[0]).toBe("product_name,product_code,architecture,cpu,cpu_cores,cpu_frequency,test_type,mode,configuration,packet_size,throughput_kpps,throughput_mbps,product_url");
-    expect(lines).toHaveLength(4);
+    expect(lines).toHaveLength(5);
     expect(csv).toContain("hAP ax3");
     expect(csv).toContain("IPQ-6010");
+    expect(csv).toContain("RB5009UG+S+IN");
     expect(csv).toContain("https://mikrotik.com/product/hap_ax3");
   });