@tikoci/rosetta 0.5.1 → 0.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/browse.ts

@@ -281,7 +281,7 @@ function renderSearchResults(resp: SearchResponse): string {
 
   // Navigation hints
   const hints = [
-    `${cyan("#N")} view page`,
+    `${cyan("[N]")} view page`,
     `${cyan("[s <query>]")} search`,
     `${cyan("[p <query>]")} properties`,
     `${cyan("[cmd <path>]")} commands`,
@@ -374,7 +374,7 @@ function renderPage(page: NonNullable<ReturnType<typeof getPage>>): string {
   // Navigation hints
   out.push("");
   const hints: string[] = [];
-  if (page.sections && page.sections.length > 0) hints.push(`${cyan("#N")} section`);
+  if (page.sections && page.sections.length > 0) hints.push(`${cyan("[N]")} section`);
   hints.push(`${cyan("[p]")} properties`);
   hints.push(`${cyan("[cmd]")} command tree`);
   hints.push(`${cyan("[cal]")} callouts`);
@@ -480,7 +480,7 @@ function renderDeviceResults(results: DeviceResult[], mode: string, total: numbe
     out.push("");
   }
 
-  out.push(`  ${cyan("#N")} view device  ${cyan("[tests]")} benchmarks  ${cyan("[b]")} back`);
+  out.push(`  ${cyan("[N]")} view device  ${cyan("[tests]")} benchmarks  ${cyan("[b]")} back`);
   return out.join("\n");
 }
 
@@ -529,24 +529,17 @@ function renderDeviceCard(d: DeviceResult): string {
     out.push(`  ${dim("Block diagram:")} ${cyan(link(d.block_diagram_url, "view"))}`);
   }
 
-  // Test results (attached for exact matches) — default to 512B for a useful baseline
+  // Test results (attached for exact matches)
   if (d.test_results && d.test_results.length > 0) {
-    const pkt512 = d.test_results.filter((t) => t.packet_size === 512);
-    const displayTests = pkt512.length > 0 ? pkt512 : d.test_results.slice(0, 20);
-    const pktNote = pkt512.length > 0 ? " (512B baseline)" : "";
     out.push("");
-    out.push(`  ${bold("Benchmarks:")}  ${dim(`(${displayTests.length} of ${d.test_results.length} tests${pktNote})`)}`);
-    for (const t of displayTests) {
+    out.push(`  ${bold("Benchmarks:")}  ${dim(`(${d.test_results.length} tests)`)}`);
+    for (const t of d.test_results.slice(0, 12)) {
       const mbps = t.throughput_mbps ? `${fmt(t.throughput_mbps)} Mbps` : "";
       const kpps = t.throughput_kpps ? `${fmt(t.throughput_kpps)} Kpps` : "";
       out.push(`    ${dim(pad(t.test_type, 9))} ${pad(t.mode, 16)} ${dim(pad(t.configuration, 28))} ${pad(`${t.packet_size}B`, 6)} ${bold(mbps)} ${dim(kpps)}`);
     }
-    if (d.test_results.length > displayTests.length) {
-      const otherSizes = [...new Set(
-        d.test_results.filter((t) => t.packet_size !== 512).map((t) => t.packet_size),
-      )].sort((a, b) => a - b);
-      const sizesStr = otherSizes.map((s) => `${s}B`).join(", ");
-      out.push(`    ${dim(`Also at ${sizesStr} — use`)} ${cyan("tests")} ${dim("<type> <size> for other packet sizes")}`);
+    if (d.test_results.length > 12) {
+      out.push(`    ${dim(`... and ${d.test_results.length - 12} more (use`)} ${cyan("tests")} ${dim("for full listing)")}`);
     }
   }
 
@@ -637,7 +630,7 @@ function renderChangelogs(results: ChangelogResult[]): string {
   }
 
   out.push("");
-  out.push(`  ${cyan("[cl 7.22 bgp]")} version + topic  ${cyan("[cl 7.21..7.22]")} range  ${cyan("[cl breaking]")} breaking only  ${cyan("[b]")} back`);
+  out.push(`  ${cyan("[cl breaking]")} breaking only  ${cyan("[cl <ver>]")} specific version  ${cyan("[b]")} back`);
   return out.join("\n");
 }
 
@@ -728,6 +721,13 @@ function renderStats(): string {
   const out: string[] = [];
   out.push(`  ${bold("Database Statistics")}`);
   out.push(`  ${dim("Path:")} ${stats.db_path}`);
+  if (stats.db_size_bytes != null) {
+    const mb = stats.db_size_bytes / (1024 * 1024);
+    out.push(`  ${dim("Size:")} ${mb.toFixed(1)} MB`);
+  }
+  if (stats.schema_version != null) {
+    out.push(`  ${dim("Schema:")} v${stats.schema_version}`);
+  }
   out.push(`  ${dim("Export:")} ${stats.doc_export}`);
   out.push("");
 
@@ -771,7 +771,7 @@ function renderHelp(): string {
   cmd("props <query>", "sp", "Search properties by FTS", "routeros_search_properties");
   cmd("cmd [path]", "tree", "Browse command tree", "routeros_command_tree");
   cmd("device <query>", "dev", "Look up device specs", "routeros_device_lookup");
-  cmd("tests [device] <type> [packet_size] [mode]", "", "Cross-device benchmarks (512B default)", "routeros_search_tests");
+  cmd("tests [device] [type]", "", "Cross-device benchmarks", "routeros_search_tests");
   cmd("callouts [query]", "cal", "Search callouts (type filter: cal warning)", "routeros_search_callouts");
   cmd("changelog [query]", "cl", "Search changelogs (cl 7.22, cl breaking)", "routeros_search_changelogs");
   cmd("videos <query>", "vid", "Search video transcripts", "routeros_search_videos");
@@ -791,69 +791,8 @@ function renderHelp(): string {
   return out.join("\n");
 }
 
-/** Fetch the full transcript text for a specific video segment. */
-function getVideoSegmentTranscript(youtubeVideoId: string, startS: number): string | null {
-  try {
-    const row = db
-      .prepare(
-        `SELECT vs.transcript FROM video_segments vs
-         JOIN videos v ON v.id = vs.video_id
-         WHERE v.video_id = ? AND vs.start_s = ? LIMIT 1`,
-      )
-      .get(youtubeVideoId, startS) as { transcript: string } | null;
-    return row?.transcript ?? null;
-  } catch {
-    return null;
-  }
-}
-
 // ── Command dispatcher ──
 
-/** Re-render the current context after going back — uses cached results where possible. */
-async function renderCurrentCtx(): Promise<void> {
-  switch (ctx.type) {
-    case "home":
-      console.log(renderWelcome());
-      return;
-    case "search":
-      await paged(renderSearchResults(ctx.response));
-      return;
-    case "commands": {
-      const children = browseCommands(ctx.path);
-      if (children.length > 0) await paged(renderCommandTree(ctx.path, children));
-      else console.log(dim(`  No children at "${ctx.path}".`));
-      return;
-    }
-    case "devices":
-      await paged(renderDeviceResults(ctx.results, "cached", ctx.results.length));
-      return;
-    case "device":
-      await paged(renderDeviceCard(ctx.device));
-      return;
-    case "callouts":
-      if (ctx.results.length > 0) await paged(renderCallouts(ctx.results));
-      return;
-    case "changelogs":
-      if (ctx.results.length > 0) await paged(renderChangelogs(ctx.results));
-      return;
-    case "videos":
-      if (ctx.results.length > 0) await paged(renderVideos(ctx.results));
-      return;
-    case "page": {
-      const page = getPage(ctx.pageId);
-      if (page) await paged(renderPage(page));
-      return;
-    }
-    case "sections": {
-      const page = getPage(ctx.pageId);
-      if (page) await paged(renderPage(page));
-      return;
-    }
-    default:
-      console.log(dim(`  In ${ctx.type} context. Type a command or help.`));
-  }
-}
-
 async function dispatch(input: string): Promise<void> {
   const trimmed = input.trim();
   if (!trimmed) return;
@@ -888,8 +827,7 @@ async function dispatch(input: string): Promise<void> {
       if (!popCtx()) {
         console.log(dim("  Already at top."));
       } else {
-        console.log(dim(`  ← ${ctx.type}`));
-        await renderCurrentCtx();
+        console.log(dim(`  ← back to ${ctx.type}`));
       }
       return;
 
@@ -904,20 +842,7 @@ async function dispatch(input: string): Promise<void> {
       return;
 
     case "page": {
-      if (!rest) {
-        // From commands context: navigate to the linked page
-        if (ctx.type === "commands") {
-          const row = db.prepare("SELECT page_id FROM commands WHERE path = ? LIMIT 1").get(ctx.path) as { page_id: number | null } | null;
-          if (row?.page_id) {
-            await doPage(String(row.page_id));
-            return;
-          }
-          console.log(dim(`  No linked page for ${ctx.path}.`));
-          return;
-        }
-        console.log(dim("  Usage: page <id|title>"));
-        return;
-      }
+      if (!rest) { console.log(dim("  Usage: page <id|title>")); return; }
       await doPage(rest);
       return;
     }
@@ -925,12 +850,13 @@ async function dispatch(input: string): Promise<void> {
     case "p":
     case "prop": {
       if (!rest) {
-        // Context-scoped: show properties for current page or sections context
-        const pageId = ctx.type === "page" ? ctx.pageId : ctx.type === "sections" ? ctx.pageId : null;
-        const title = ctx.type === "page" ? ctx.title : ctx.type === "sections" ? ctx.title : null;
-        if (pageId !== null && title !== null) {
-          await doPropsForPage(pageId, title);
-          return;
+        // Context-scoped: show properties for current page
+        if (ctx.type === "page") {
+          const page = getPage(ctx.pageId, 0); // just get metadata
+          if (page) {
+            await doPropsForPage(ctx.pageId, ctx.title);
+            return;
+          }
         }
         console.log(dim("  Usage: prop <name> — or navigate to a page first"));
         return;
@@ -948,15 +874,8 @@ async function dispatch(input: string): Promise<void> {
 
     case "cmd":
     case "tree": {
-      let cmdPath = rest;
-      if (!cmdPath && ctx.type === "commands") {
-        // Re-display current commands context
-        cmdPath = ctx.path;
-      } else if (cmdPath && !cmdPath.startsWith("/") && ctx.type === "commands") {
-        // Relative path: resolve against current commands path
-        cmdPath = `${ctx.path}/${cmdPath}`;
-      }
-      await doCommandTree(cmdPath);
+      const path = rest || (ctx.type === "commands" ? ctx.path : "");
+      await doCommandTree(path);
       return;
     }
 
@@ -974,24 +893,13 @@ async function dispatch(input: string): Promise<void> {
 
     case "cal":
     case "callouts": {
-      if (!rest) {
-        // Context-aware: show callouts for the current page directly
-        const pageId = ctx.type === "page" ? ctx.pageId
-          : ctx.type === "sections" ? ctx.pageId
-          : null;
-        if (pageId !== null) {
-          const pageCallouts = db.prepare(
-            `SELECT c.type, c.content, p.title as page_title, p.url as page_url,
-                    c.page_id, c.content as excerpt
-             FROM callouts c JOIN pages p ON p.id = c.page_id
-             WHERE c.page_id = ? ORDER BY c.sort_order`,
-          ).all(pageId) as CalloutResult[];
-          if (pageCallouts.length > 0) {
-            await paged(renderCallouts(pageCallouts));
-            pushCtx({ type: "callouts", query: "", results: pageCallouts });
-          } else {
-            console.log(dim("  No callouts for this page."));
-          }
+      if (!rest && ctx.type === "page") {
+        // Show callouts for current page
+        const results = searchCallouts("", undefined, 50);
+        const pageCallouts = results.filter((c) => c.page_id === (ctx as { pageId: number }).pageId);
+        if (pageCallouts.length > 0) {
+          await paged(renderCallouts(pageCallouts));
+          pushCtx({ type: "callouts", query: "", results: pageCallouts });
           return;
         }
       }
@@ -1006,7 +914,6 @@ async function dispatch(input: string): Promise<void> {
     }
 
     case "vid":
-    case "video":
     case "videos": {
       if (!rest) { console.log(dim("  Usage: videos <query>")); return; }
       await doSearchVideos(rest);
@@ -1077,18 +984,9 @@ async function handleNumberSelect(idx: number): Promise<void> {
   if (ctx.type === "videos" && ctx.results[idx]) {
     const v = ctx.results[idx];
     const timeUrl = v.start_s > 0 ? `${v.url}&t=${v.start_s}` : v.url;
-    const out: string[] = [];
-    out.push(`\n  ${bold(v.title)}`);
-    if (v.chapter_title) out.push(`  ${magenta(`§ ${v.chapter_title}`)}  ${dim(`@ ${formatTime(v.start_s)}`)}`);
-    out.push(`  ${cyan(link(timeUrl))}`);
-    // Fetch and show full transcript for this segment
-    const transcript = getVideoSegmentTranscript(v.video_id, v.start_s);
-    if (transcript) {
-      out.push("");
-      out.push(`  ${dim("── transcript ──")}`);
-      out.push(transcript.split("\n").map((l) => `  ${l}`).join("\n"));
-    }
-    await paged(out.join("\n"));
+    console.log(`\n  ${bold(v.title)}`);
+    if (v.chapter_title) console.log(`  ${magenta(`§ ${v.chapter_title}`)}  ${dim(`@ ${formatTime(v.start_s)}`)}`);
+    console.log(`  ${cyan(link(timeUrl))}\n`);
     return;
   }
   if (ctx.type === "properties" && ctx.results[idx]) {
@@ -1215,64 +1113,33 @@ async function doDeviceLookup(query: string): Promise<void> {
 }
 
 async function doTests(argsStr: string): Promise<void> {
-  const parts = argsStr.split(/\s+/).filter(Boolean);
-  const filters: Record<string, string | number> = {};
-
-  // Parse args if provided
-  if (argsStr) {
-    const knownTypes = ["ethernet", "ipsec"];
-    let offset = 0;
-    if (parts[0] && !knownTypes.includes(parts[0].toLowerCase())) {
-      filters.device = parts[0];
-      offset = 1;
-    }
-    if (parts[offset]) filters.test_type = parts[offset];
-    // packet_size is always numeric; mode can be multi-word — identify by type
-    const remaining = parts.slice(offset + 1);
-    const pktIdx = remaining.findIndex((p) => /^\d+$/.test(p));
-    if (pktIdx !== -1) {
-      filters.packet_size = Number.parseInt(remaining[pktIdx], 10);
-      const modeParts = [...remaining.slice(0, pktIdx), ...remaining.slice(pktIdx + 1)];
-      if (modeParts.length > 0) filters.mode = modeParts.join(" ");
-    } else if (remaining.length > 0) {
-      filters.mode = remaining.join(" ");
-    }
-  }
-
-  // Auto-inject device name from context when in a device view and none specified
-  if (!filters.device && ctx.type === "device") {
-    filters.device = ctx.device.product_name;
-  }
-
-  // Show help if no filters could be determined
-  if (!argsStr && !filters.device) {
+  if (!argsStr) {
+    // Show available filter values
     const meta = getTestResultMeta();
     console.log(`  ${bold("Test filters:")}`);
     console.log(`  ${dim("Types:")} ${meta.test_types.join(", ")}`);
     console.log(`  ${dim("Modes:")} ${meta.modes.join(", ")}`);
     console.log(`  ${dim("Packet sizes:")} ${meta.packet_sizes.join(", ")}`);
     console.log("");
-    console.log(`  ${dim("Usage: tests [device] <type> [packet_size] [mode]")}`);
+    console.log(`  ${dim("Usage: tests [device] <type> [mode] [packet_size]")}`);
     console.log(`  ${dim("Example: tests ethernet Routing 1518")}`);
     console.log(`  ${dim("Example: tests rb5009 ethernet 1518")}`);
     return;
   }
 
-  // Round packet_size to the nearest valid test packet size
-  if (filters.packet_size) {
-    const validSizes = [64, 512, 1400, 1518];
-    const size = Number(filters.packet_size);
-    const rounded = validSizes.reduce((prev, curr) =>
-      Math.abs(curr - size) < Math.abs(prev - size) ? curr : prev,
-    );
-    if (rounded !== size) {
-      console.log(dim(`  Rounding ${size}B → ${rounded}B (nearest test packet size)`));
-      filters.packet_size = rounded;
-    }
-  }
+  const parts = argsStr.split(/\s+/);
+  const filters: Record<string, string | number> = {};
 
-  // Default packet_size to 512 for a focused baseline comparison
-  if (!filters.packet_size && !filters.mode) filters.packet_size = 512;
+  // Known test types — if parts[0] is not a known type, treat it as a device filter
+  const knownTypes = ["ethernet", "ipsec"];
+  let offset = 0;
+  if (parts[0] && !knownTypes.includes(parts[0].toLowerCase())) {
+    filters.device = parts[0];
+    offset = 1;
+  }
+  if (parts[offset]) filters.test_type = parts[offset];
+  if (parts[offset + 1]) filters.mode = parts[offset + 1];
+  if (parts[offset + 2] && /^\d+$/.test(parts[offset + 2])) filters.packet_size = Number.parseInt(parts[offset + 2], 10);
 
   const result = searchDeviceTests(filters);
   if (result.results.length === 0) {
@@ -1379,6 +1246,7 @@ async function doCurrentVersions(): Promise<void> {
     const ch = pad(channel, 14);
     out.push(`  ${dim(ch)} ${bold(String(v))}`);
   }
+  out.push(`  ${dim(pad("winbox 4", 14))} ${bold(String(result.winbox ?? dim("unavailable")))}`);
   console.log(out.join("\n"));
 }
 
@@ -1429,9 +1297,6 @@ async function main() {
     } catch (err) {
       console.error(red(`  Error: ${err instanceof Error ? err.message : String(err)}`));
     }
-    // Clear any keys that leaked from the pager into readline's internal line buffer
-    // (the key pressed to exit paging — q/SPACE/ENTER — can appear in the next prompt)
-    rl.write(null as unknown as string, { ctrl: true, name: "u" });
     rl.setPrompt(buildPrompt());
     rl.prompt();
   });

package/src/db.ts

@@ -441,8 +441,24 @@ export function checkSchemaVersion(): { ok: boolean; actual: number; expected: n
 export function getDbStats() {
   const count = (sql: string) =>
     Number((db.prepare(sql).get() as { c: number }).c ?? 0);
+  const dbSizeBytes = (() => {
+    try {
+      return Bun.file(DB_PATH).size;
+    } catch {
+      return null;
+    }
+  })();
+  const schemaVersion = (() => {
+    try {
+      return (db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version;
+    } catch {
+      return null;
+    }
+  })();
   return {
     db_path: DB_PATH,
+    db_size_bytes: dbSizeBytes,
+    schema_version: schemaVersion,
     pages: count("SELECT COUNT(*) AS c FROM pages"),
     sections: count("SELECT COUNT(*) AS c FROM sections"),
     properties: count("SELECT COUNT(*) AS c FROM properties"),

package/src/extract-devices.ts

@@ -120,7 +120,11 @@ if (lines.length < 2) {
 // Skip header row
 const dataLines = lines.slice(1);
 
-// Idempotent: clear existing data (FTS triggers handle cleanup)
+// Idempotent: clear existing data (FTS triggers handle cleanup).
+// device_test_results FKs into devices — wipe it first to avoid a FOREIGN KEY
+// constraint failure on re-run over a populated DB. extract-test-results runs
+// later in the pipeline and repopulates it.
+db.run("DELETE FROM device_test_results");
 db.run("DELETE FROM devices");
 
 const insert = db.prepare(`INSERT INTO devices (

package/src/extract-html.test.ts

@@ -0,0 +1,67 @@
+// Set BEFORE any import that transitively loads db.ts
+process.env.DB_PATH = ":memory:";
+
+import { describe, expect, it } from "bun:test";
+import { parseHTML } from "linkedom";
+
+// Dynamic import so the DB_PATH assignment above wins over module caching
+const { extractPlainText, sanitizeExtractedText } = await import("./extract-html.ts");
+
+describe("sanitizeExtractedText", () => {
+  it("removes Confluence TOC CDATA style blocks", () => {
+    const input = `
+Intro
+/*<![CDATA[*/
+div.rbtoc1774430868497 {padding: 0px;}
+div.rbtoc1774430868497 ul {margin-left: 0px;}
+div.rbtoc1774430868497 li {margin-left: 0px;padding-left: 0px;}
+/*]]>*/
+Basic Setup
+`;
+
+    const out = sanitizeExtractedText(input);
+    expect(out).toContain("Intro");
+    expect(out).toContain("Basic Setup");
+    expect(out).not.toContain("rbtoc1774430868497");
+    expect(out).not.toContain("/*<![CDATA[*/");
+    expect(out).not.toContain("/*]]>*/");
+  });
+
+  it("removes bare rbtoc css lines without wrappers", () => {
+    const input = `
+Top
+
+div.rbtoc1234 {padding: 0px;}
+div.rbtoc1234 ul {margin-left: 0px;}
+
+after
+`;
+
+    const out = sanitizeExtractedText(input);
+    expect(out).toContain("Top");
+    expect(out).toContain("after");
+    expect(out).not.toContain("div.rbtoc1234");
+  });
+
+  it("keeps normal RouterOS content", () => {
+    const input = "/interface vrrp add interface=ether1 vrid=49 priority=254";
+    const out = sanitizeExtractedText(input);
+    expect(out).toBe(input);
+  });
+});
+
+describe("extractPlainText", () => {
+  it("keeps heading and paragraph separated", () => {
+    const { document } = parseHTML("<div><h1>Basic Setup</h1><p>This is the basic VRRP configuration example.</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("Basic Setup\nThis is the basic VRRP configuration example.");
+    expect(out).not.toContain("Basic SetupThis");
+  });
+
+  it("keeps subsection heading boundaries", () => {
+    const { document } = parseHTML("<div><h2>Overview</h2><p>The Point-to-Point Protocol (PPP) provides...</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("Overview\nThe Point-to-Point Protocol (PPP) provides...");
+    expect(out).not.toContain("OverviewThe");
+  });
+});

package/src/extract-html.ts

@@ -64,6 +64,120 @@ interface SectionRow {
   sort_order: number;
 }
 
+const BLOCK_TAGS = new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tbody",
+  "td",
+  "th",
+  "thead",
+  "tr",
+  "ul",
+]);
+
+/**
+ * Remove Confluence TOC style leakage that may survive DOM cleanup when HTML is malformed.
+ * Keeps cleanup narrow to rbtoc CSS so normal content is not affected.
+ */
+export function sanitizeExtractedText(input: string): string {
+  if (!input) return "";
+
+  return input
+    // Full CDATA-wrapped TOC style block.
+    .replace(/\/\*<!\[CDATA\[\*\/[\s\S]*?\/\*\]\]>\*\//g, "")
+    // Bare TOC CSS lines that can leak without wrappers.
+    .replace(/^\s*div\.rbtoc\d+[^\n]*(?:\n\s*div\.rbtoc\d+[^\n]*)*/gm, "")
+    // Any orphaned CDATA markers.
+    .replace(/\/\*<!\[CDATA\[\*\//g, "")
+    .replace(/\/\*\]\]>\*\//g, "")
+    // Prevent giant gaps in rendered output.
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/**
+ * Convert a DOM subtree to readable plain text while preserving block boundaries.
+ * This avoids merged tokens like "OverviewThe" caused by raw textContent collapse.
+ */
+export function extractPlainText(root: Element | null): string {
+  if (!root) return "";
+
+  let out = "";
+
+  const addBreak = () => {
+    if (!out) return;
+    if (!out.endsWith("\n")) out += "\n";
+  };
+
+  const addText = (text: string) => {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    if (!normalized) return;
+    if (out && !out.endsWith("\n") && !out.endsWith(" ")) out += " ";
+    out += normalized;
+  };
+
+  const walk = (node: Node) => {
+    if (node.nodeType === 3) {
+      addText(node.textContent || "");
+      return;
+    }
+
+    if (node.nodeType !== 1) return;
+
+    const el = node as Element;
+    const tag = el.tagName.toLowerCase();
+
+    if (tag === "style" || tag === "script" || tag === "noscript") return;
+    if (tag === "br") {
+      addBreak();
+      return;
+    }
+
+    const isBlock = BLOCK_TAGS.has(tag);
+    if (isBlock) addBreak();
+
+    for (const child of el.childNodes) {
+      walk(child as unknown as Node);
+    }
+
+    if (isBlock) addBreak();
+  };
+
+  walk(root as unknown as Node);
+
+  return out
+    .replace(/[ \t]*\n[ \t]*/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
 /**
  * Split main content into sections by h1–h3 headings with id attributes.
  * Uses innerHTML + regex to locate heading boundaries, then parses each
@@ -105,7 +219,7 @@ function extractSections(mainContent: Element, pageId: number): SectionRow[] {
       codeChunks.push(ce.textContent?.trim() || "");
     }
 
-    const text = root?.textContent?.trim() || "";
+    const text = sanitizeExtractedText(extractPlainText(root));
     const code = codeChunks.join("\n\n");
 
     return {
@@ -187,7 +301,7 @@ function extractPage(file: string, html: string): (PageRow & { callouts: Callout
   }
 
   // Plain text from main content (includes code block text too, which is fine for FTS)
-  const text = mainContent?.textContent?.trim() || "";
+  const text = sanitizeExtractedText(extractPlainText(mainContent));
   const wordCount = text.split(/\s+/).filter(Boolean).length;
 
   // Callouts: extract note/warning/info blocks
@@ -238,149 +352,155 @@ function extractPage(file: string, html: string): (PageRow & { callouts: Callout
 
 // ---- Main ----
 
-console.log("Initializing database...");
-initDb();
+function main() {
+  console.log("Initializing database...");
+  initDb();
 
 // Drop existing data for clean re-extraction (respect FK order)
-db.run("DELETE FROM sections;");
-db.run("DELETE FROM callouts;");
-db.run("INSERT INTO callouts_fts(callouts_fts) VALUES('rebuild');");
-db.run("DELETE FROM properties;");
-db.run("INSERT INTO properties_fts(properties_fts) VALUES('rebuild');");
-db.run("PRAGMA foreign_keys = OFF;");
-db.run("DELETE FROM pages;");
-db.run("PRAGMA foreign_keys = ON;");
-db.run("INSERT INTO pages_fts(pages_fts) VALUES('rebuild');");
-
-const htmlFiles = readdirSync(HTML_DIR)
-  .filter((f) => f.endsWith(".html") && f !== "index.html")
-  .sort();
-
-console.log(`Extracting ${htmlFiles.length} HTML files from ${HTML_DIR}`);
+  db.run("DELETE FROM sections;");
+  db.run("DELETE FROM callouts;");
+  db.run("INSERT INTO callouts_fts(callouts_fts) VALUES('rebuild');");
+  db.run("DELETE FROM properties;");
+  db.run("INSERT INTO properties_fts(properties_fts) VALUES('rebuild');");
+  db.run("PRAGMA foreign_keys = OFF;");
+  db.run("DELETE FROM pages;");
+  db.run("PRAGMA foreign_keys = ON;");
+  db.run("INSERT INTO pages_fts(pages_fts) VALUES('rebuild');");
+
+  const htmlFiles = readdirSync(HTML_DIR)
+    .filter((f) => f.endsWith(".html") && f !== "index.html")
+    .sort();
+
+  console.log(`Extracting ${htmlFiles.length} HTML files from ${HTML_DIR}`);
 
 // Two-pass insert: first without parent_id (avoids FK ordering issues),
 // then update parent relationships.
-const insertPage = db.prepare(`
+  const insertPage = db.prepare(`
   INSERT OR REPLACE INTO pages
     (id, slug, title, path, depth, parent_id, url, text, code, code_lang,
      author, last_updated, word_count, code_lines, html_file)
   VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 `);
-const updateParent = db.prepare("UPDATE pages SET parent_id = ? WHERE id = ?");
+  const updateParent = db.prepare("UPDATE pages SET parent_id = ? WHERE id = ?");
 
-let extracted = 0;
-let skipped = 0;
-let totalWords = 0;
-let totalCodeLines = 0;
-let totalCallouts = 0;
+  let extracted = 0;
+  let skipped = 0;
+  let totalWords = 0;
+  let totalCodeLines = 0;
+  let totalCallouts = 0;
 
-const allPages: (PageRow & { callouts: CalloutRow[]; sections: SectionRow[] })[] = [];
+  const allPages: (PageRow & { callouts: CalloutRow[]; sections: SectionRow[] })[] = [];
 
 // Pass 1: extract and insert all pages (parent_id = NULL)
-const insertAll = db.transaction(() => {
-  for (const file of htmlFiles) {
-    const html = readFileSync(resolve(HTML_DIR, file), "utf-8");
-    const page = extractPage(file, html);
-    if (!page) {
-      skipped++;
-      console.warn(`  skipped: ${file}`);
-      continue;
+  const insertAll = db.transaction(() => {
+    for (const file of htmlFiles) {
+      const html = readFileSync(resolve(HTML_DIR, file), "utf-8");
+      const page = extractPage(file, html);
+      if (!page) {
+        skipped++;
+        console.warn(`  skipped: ${file}`);
+        continue;
+      }
+      insertPage.run(
+        page.id,
+        page.slug,
+        page.title,
+        page.path,
+        page.depth,
+        page.url,
+        page.text,
+        page.code,
+        page.code_lang,
+        page.author,
+        page.last_updated,
+        page.word_count,
+        page.code_lines,
+        page.html_file,
+      );
+      allPages.push(page);
+      extracted++;
+      totalWords += page.word_count;
+      totalCodeLines += page.code_lines;
     }
-    insertPage.run(
-      page.id,
-      page.slug,
-      page.title,
-      page.path,
-      page.depth,
-      page.url,
-      page.text,
-      page.code,
-      page.code_lang,
-      page.author,
-      page.last_updated,
-      page.word_count,
-      page.code_lines,
-      page.html_file,
-    );
-    allPages.push(page);
-    extracted++;
-    totalWords += page.word_count;
-    totalCodeLines += page.code_lines;
-  }
-});
-insertAll();
+  });
+  insertAll();
 
 // Pass 2: set parent_id where the parent actually exists in the DB
-const pageIds = new Set(allPages.map((p) => p.id));
-const setParents = db.transaction(() => {
-  for (const page of allPages) {
-    if (page.parent_id && pageIds.has(page.parent_id)) {
-      updateParent.run(page.parent_id, page.id);
+  const pageIds = new Set(allPages.map((p) => p.id));
+  const setParents = db.transaction(() => {
+    for (const page of allPages) {
+      if (page.parent_id && pageIds.has(page.parent_id)) {
+        updateParent.run(page.parent_id, page.id);
+      }
     }
-  }
-});
-setParents();
+  });
+  setParents();
 
 // Pass 3: insert callouts
-const insertCallout = db.prepare(`
+  const insertCallout = db.prepare(`
   INSERT INTO callouts (page_id, type, content, sort_order)
   VALUES (?, ?, ?, ?)
 `);
-const insertCallouts = db.transaction(() => {
-  for (const page of allPages) {
-    for (const c of page.callouts) {
-      insertCallout.run(c.page_id, c.type, c.content, c.sort_order);
-      totalCallouts++;
+  const insertCallouts = db.transaction(() => {
+    for (const page of allPages) {
+      for (const c of page.callouts) {
+        insertCallout.run(c.page_id, c.type, c.content, c.sort_order);
+        totalCallouts++;
+      }
     }
-  }
-});
-insertCallouts();
+  });
+  insertCallouts();
 
 // Pass 4: insert sections
-let totalSections = 0;
-let pagesWithSections = 0;
-const insertSection = db.prepare(`
+  let totalSections = 0;
+  let pagesWithSections = 0;
+  const insertSection = db.prepare(`
   INSERT INTO sections (page_id, heading, level, anchor_id, text, code, word_count, sort_order)
   VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 `);
-const insertSections = db.transaction(() => {
-  for (const page of allPages) {
-    if (page.sections.length > 0) {
-      pagesWithSections++;
-      for (const s of page.sections) {
-        insertSection.run(s.page_id, s.heading, s.level, s.anchor_id, s.text, s.code, s.word_count, s.sort_order);
-        totalSections++;
+  const insertSections = db.transaction(() => {
+    for (const page of allPages) {
+      if (page.sections.length > 0) {
+        pagesWithSections++;
+        for (const s of page.sections) {
+          insertSection.run(s.page_id, s.heading, s.level, s.anchor_id, s.text, s.code, s.word_count, s.sort_order);
+          totalSections++;
+        }
       }
     }
-  }
-});
-insertSections();
+  });
+  insertSections();
 
-const ftsCount = (db.prepare("SELECT COUNT(*) as c FROM pages_fts").get() as { c: number }).c;
+  const ftsCount = (db.prepare("SELECT COUNT(*) as c FROM pages_fts").get() as { c: number }).c;
 
-console.log(`\nExtraction complete:`);
-console.log(`  Pages extracted: ${extracted}`);
-console.log(`  Pages skipped:   ${skipped}`);
-console.log(`  Total words:     ${totalWords.toLocaleString()}`);
-console.log(`  Total code lines: ${totalCodeLines.toLocaleString()}`);
-console.log(`  Total callouts:  ${totalCallouts}`);
-console.log(`  Total sections:  ${totalSections} (across ${pagesWithSections} pages)`);
-console.log(`  FTS index rows:  ${ftsCount}`);
+  console.log(`\nExtraction complete:`);
+  console.log(`  Pages extracted: ${extracted}`);
+  console.log(`  Pages skipped:   ${skipped}`);
+  console.log(`  Total words:     ${totalWords.toLocaleString()}`);
+  console.log(`  Total code lines: ${totalCodeLines.toLocaleString()}`);
+  console.log(`  Total callouts:  ${totalCallouts}`);
+  console.log(`  Total sections:  ${totalSections} (across ${pagesWithSections} pages)`);
+  console.log(`  FTS index rows:  ${ftsCount}`);
 
 // Quick search test
-const testResults = db
-  .prepare(
-    `SELECT s.id, s.title, s.path,
-            snippet(pages_fts, 2, '>>>', '<<<', '...', 20) as excerpt
-     FROM pages_fts fts
-     JOIN pages s ON s.id = fts.rowid
-     WHERE pages_fts MATCH 'firewall filter'
-     ORDER BY rank LIMIT 5`,
-  )
-  .all();
-
-console.log(`\nTest search for "firewall filter":`);
-for (const r of testResults as Array<{ id: number; title: string; path: string; excerpt: string }>) {
-  console.log(`  [${r.id}] ${r.path}`);
-  console.log(`    ${r.excerpt}`);
+  const testResults = db
+    .prepare(
+      `SELECT s.id, s.title, s.path,
+              snippet(pages_fts, 2, '>>>', '<<<', '...', 20) as excerpt
+       FROM pages_fts fts
+       JOIN pages s ON s.id = fts.rowid
+       WHERE pages_fts MATCH 'firewall filter'
+       ORDER BY rank LIMIT 5`,
+    )
+    .all();
+
+  console.log(`\nTest search for "firewall filter":`);
+  for (const r of testResults as Array<{ id: number; title: string; path: string; excerpt: string }>) {
+    console.log(`  [${r.id}] ${r.path}`);
+    console.log(`    ${r.excerpt}`);
+  }
+}
+
+if (import.meta.main) {
+  main();
 }

package/src/mcp.ts

@@ -1032,17 +1032,8 @@ Workflow:
       };
     }
 
-    // Round packet_size to the nearest valid test size (64, 512, 1400, 1518)
-    let effectivePacketSize = packet_size;
-    if (packet_size !== undefined) {
-      const validSizes = [64, 512, 1400, 1518];
-      effectivePacketSize = validSizes.reduce((prev, curr) =>
-        Math.abs(curr - packet_size) < Math.abs(prev - packet_size) ? curr : prev,
-      );
-    }
-
     const result = searchDeviceTests(
-      { device, test_type, mode, configuration, packet_size: effectivePacketSize, sort_by },
+      { device, test_type, mode, configuration, packet_size, sort_by },
       limit,
     );
 
@@ -1065,9 +1056,6 @@ Workflow:
         text: JSON.stringify({
           ...result,
           has_more: result.total > result.results.length,
-          ...(effectivePacketSize !== packet_size && packet_size !== undefined
-            ? { packet_size_rounded: `${packet_size} → ${effectivePacketSize} (nearest valid test size)` }
-            : {}),
         }, null, 2),
       }],
     };
@@ -1081,7 +1069,7 @@ server.registerTool(
   {
     description: `Fetch current RouterOS version numbers from MikroTik's upgrade server.
 
-Returns the latest version for each release channel: stable, long-term, testing, development.
+Returns the latest version for each release channel (stable, long-term, testing, development) plus the current WinBox 4 version.
 Useful for determining if a user's version is current, outdated, or unpatched.
 
 Key context for version reasoning:

package/src/query.ts

@@ -1493,27 +1493,31 @@ function runVideosFtsQuery(ftsQuery: string, limit: number): VideoSearchResult[]
 const VERSION_BASE_URL = "https://upgrade.mikrotik.com/routeros/NEWESTa7";
 
 /** Fetch current RouterOS versions from MikroTik's upgrade server. */
+const WINBOX_URL = "https://upgrade.mikrotik.com/routeros/winbox/LATEST.4";
+
 export async function fetchCurrentVersions(): Promise<{
   channels: Record<string, string | null>;
+  winbox: string | null;
   fetched_at: string;
 }> {
   const channels: Record<string, string | null> = {};
-  await Promise.all(
-    VERSION_CHANNELS.map(async (channel) => {
-      try {
-        const resp = await fetch(`${VERSION_BASE_URL}.${channel}`, {
-          signal: AbortSignal.timeout(10_000),
-        });
-        if (resp.ok) {
-          const text = await resp.text();
-          channels[channel] = text.trim().split(/\s+/)[0] || null;
-        } else {
-          channels[channel] = null;
-        }
-      } catch {
-        channels[channel] = null;
-      }
-    }),
-  );
-  return { channels, fetched_at: new Date().toISOString() };
+  const fetchOne = async (url: string): Promise<string | null> => {
+    try {
+      const resp = await fetch(url, { signal: AbortSignal.timeout(10_000) });
+      if (!resp.ok) return null;
+      const text = await resp.text();
+      return text.trim().split(/\s+/)[0] || null;
+    } catch {
+      return null;
+    }
+  };
+  const [, winbox] = await Promise.all([
+    Promise.all(
+      VERSION_CHANNELS.map(async (channel) => {
+        channels[channel] = await fetchOne(`${VERSION_BASE_URL}.${channel}`);
+      }),
+    ),
+    fetchOne(WINBOX_URL),
+  ]);
+  return { channels, winbox, fetched_at: new Date().toISOString() };
 }