@tikoci/rosetta 0.6.2 → 0.6.4

package/src/extract-html.test.ts

@@ -51,17 +51,41 @@ after
 });
 
 describe("extractPlainText", () => {
-  it("keeps heading and paragraph separated", () => {
+  it("emits heading markers", () => {
     const { document } = parseHTML("<div><h1>Basic Setup</h1><p>This is the basic VRRP configuration example.</p></div>");
     const out = extractPlainText(document.querySelector("div"));
-    expect(out).toContain("Basic Setup\nThis is the basic VRRP configuration example.");
-    expect(out).not.toContain("Basic SetupThis");
+    expect(out).toContain("# Basic Setup\nThis is the basic VRRP configuration example.");
   });
 
-  it("keeps subsection heading boundaries", () => {
+  it("emits h2 markers for subsections", () => {
     const { document } = parseHTML("<div><h2>Overview</h2><p>The Point-to-Point Protocol (PPP) provides...</p></div>");
     const out = extractPlainText(document.querySelector("div"));
-    expect(out).toContain("Overview\nThe Point-to-Point Protocol (PPP) provides...");
-    expect(out).not.toContain("OverviewThe");
+    expect(out).toContain("## Overview\nThe Point-to-Point Protocol (PPP) provides...");
+  });
+
+  it("emits list markers for ul", () => {
+    const { document } = parseHTML("<div><ul><li>First item</li><li>Second item</li></ul></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("- First item");
+    expect(out).toContain("- Second item");
+  });
+
+  it("emits numbered markers for ol", () => {
+    const { document } = parseHTML("<div><ol><li>Step one</li><li>Step two</li></ol></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("1. Step one");
+    expect(out).toContain("2. Step two");
+  });
+
+  it("wraps strong/b in **bold**", () => {
+    const { document } = parseHTML("<div><p>The <strong>disabled</strong> property controls this.</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("**disabled**");
+  });
+
+  it("wraps code in backticks", () => {
+    const { document } = parseHTML("<div><p>Use <code>/ip/address</code> to configure.</p></div>");
+    const out = extractPlainText(document.querySelector("div"));
+    expect(out).toContain("`/ip/address`");
   });
 });

package/src/extract-html.ts

@@ -123,13 +123,17 @@ export function sanitizeExtractedText(input: string): string {
 }
 
 /**
- * Convert a DOM subtree to readable plain text while preserving block boundaries.
- * This avoids merged tokens like "OverviewThe" caused by raw textContent collapse.
+ * Convert a DOM subtree to readable text with lightweight markdown signals.
+ * Preserves block boundaries (avoids "OverviewThe" token merge) and emits:
+ *   - Heading markers: ## Heading (matching h1–h6 level)
+ *   - List markers: - item (ul) or 1. item (ol)
+ *   - Emphasis: **bold** for <strong>/<b>, `code` for <code>
  */
 export function extractPlainText(root: Element | null): string {
   if (!root) return "";
 
   let out = "";
+  let olCounter = 0;
 
   const addBreak = () => {
     if (!out) return;
@@ -160,6 +164,78 @@ export function extractPlainText(root: Element | null): string {
       return;
     }
 
+    // Heading markers: ## Heading
+    const headingMatch = tag.match(/^h([1-6])$/);
+    if (headingMatch) {
+      addBreak();
+      const level = Number.parseInt(headingMatch[1], 10);
+      const prefix = "#".repeat(level);
+      const headingText = (el.textContent || "").replace(/\s+/g, " ").trim();
+      if (headingText) {
+        addText(`${prefix} ${headingText}`);
+      }
+      addBreak();
+      return; // don't recurse into heading children — already captured via textContent
+    }
+
+    // Ordered list: reset counter
+    if (tag === "ol") {
+      const savedCounter = olCounter;
+      olCounter = 0;
+      addBreak();
+      for (const child of el.childNodes) {
+        walk(child as unknown as Node);
+      }
+      addBreak();
+      olCounter = savedCounter;
+      return;
+    }
+
+    // Unordered list
+    if (tag === "ul") {
+      addBreak();
+      for (const child of el.childNodes) {
+        walk(child as unknown as Node);
+      }
+      addBreak();
+      return;
+    }
+
+    // List item: prefix with - or N.
+    if (tag === "li") {
+      addBreak();
+      const parentTag = (el.parentElement?.tagName || "").toLowerCase();
+      if (parentTag === "ol") {
+        olCounter++;
+        addText(`${olCounter}.`);
+      } else {
+        addText("-");
+      }
+      for (const child of el.childNodes) {
+        walk(child as unknown as Node);
+      }
+      addBreak();
+      return;
+    }
+
+    // Inline emphasis: **bold**
+    if (tag === "strong" || tag === "b") {
+      const inner = (el.textContent || "").replace(/\s+/g, " ").trim();
+      if (inner) {
+        addText(`**${inner}**`);
+      }
+      return; // don't recurse — already captured
+    }
+
+    // Inline code: `code`
+    if (tag === "code") {
+      const inner = (el.textContent || "").replace(/\s+/g, " ").trim();
+      if (inner) {
+        addText(`\`${inner}\``);
+      }
+      return; // don't recurse — already captured
+    }
+
     const isBlock = BLOCK_TAGS.has(tag);
     if (isBlock) addBreak();
 

package/src/mcp.ts

@@ -46,6 +46,62 @@ function link(url: string, display?: string): string {
   return `\x1b]8;;${url}\x07${display ?? url}\x1b]8;;\x07`;
 }
 
+/**
+ * Ensure the DB exists, has page data, and matches the current schema version.
+ * This must run before importing db.ts/query.ts to avoid creating an empty DB file
+ * on fresh installs.
+ */
+async function ensureDbReady(log: (msg: string) => void): Promise<void> {
+  const { resolveDbPath, SCHEMA_VERSION } = await import("./paths.ts");
+  const { downloadDb } = await import("./setup.ts");
+
+  const dbPath = resolveDbPath(import.meta.dirname);
+
+  const pageCount = (() => {
+    try {
+      const check = new (require("bun:sqlite").default)(dbPath, { readonly: true });
+      const row = check.prepare("SELECT COUNT(*) AS c FROM pages").get() as { c: number };
+      check.close();
+      return row.c;
+    } catch {
+      return 0;
+    }
+  })();
+
+  if (pageCount === 0) {
+    try {
+      await downloadDb(dbPath, log);
+      log("Database downloaded successfully.");
+    } catch (e) {
+      log(`Auto-download failed: ${e}`);
+      log(`Run: ${process.argv[0]} --setup`);
+      return;
+    }
+  }
+
+  const dbSchemaVersion = (() => {
+    try {
+      const check = new (require("bun:sqlite").default)(dbPath, { readonly: true });
+      const row = check.prepare("PRAGMA user_version").get() as { user_version: number };
+      check.close();
+      return row.user_version;
+    } catch {
+      return SCHEMA_VERSION;
+    }
+  })();
+
+  if (dbSchemaVersion !== SCHEMA_VERSION) {
+    log(`DB schema version mismatch (DB=${dbSchemaVersion}, expected=${SCHEMA_VERSION}) - re-downloading updated database...`);
+    try {
+      await downloadDb(dbPath, log);
+      log("Database updated successfully.");
+    } catch (e) {
+      log(`Auto-download failed: ${e}`);
+      log(`Run: ${process.argv[0]} --refresh`);
+    }
+  }
+}
+
 if (args.includes("--version") || args.includes("-v")) {
   console.log(`rosetta ${RESOLVED_VERSION}`);
   process.exit(0);
@@ -85,6 +141,7 @@ if (args.includes("--help") || args.includes("-h")) {
 (async () => {
 
 if (args[0] === "browse") {
+  await ensureDbReady((msg) => process.stderr.write(`${msg}\n`));
   // Strip "browse" from argv so browse.ts only sees flags/queries
   process.argv.splice(2, 1);
   await import("./browse.ts");
@@ -113,62 +170,7 @@ const { z } = await import("zod/v3");
 // Dynamic imports — db.ts eagerly opens the DB file on import,
 // so we must import after the --setup guard to avoid creating
 // an empty ros-help.db on fresh installs.
-//
-// Check if DB has data BEFORE importing db.ts. If empty/missing,
-// auto-download so db.ts opens the real database.
-const { resolveDbPath, SCHEMA_VERSION } = await import("./paths.ts");
-const _dbPath = resolveDbPath(import.meta.dirname);
-
-const _pageCount = (() => {
-  try {
-    const check = new (require("bun:sqlite").default)(_dbPath, { readonly: true });
-    const row = check.prepare("SELECT COUNT(*) AS c FROM pages").get() as { c: number };
-    check.close();
-    return row.c;
-  } catch {
-    return 0;
-  }
-})();
-
-if (_pageCount === 0) {
-  const { downloadDb } = await import("./setup.ts");
-  // Use stderr — stdout is the MCP stdio transport
-  const log = (msg: string) => process.stderr.write(`${msg}\n`);
-  try {
-    await downloadDb(_dbPath, log);
-    log("Database downloaded successfully.");
-  } catch (e) {
-    log(`Auto-download failed: ${e}`);
-    log(`Run: ${process.argv[0]} --setup`);
-  }
-}
-
-// Check schema version — a bunx auto-update may bring a new code version whose
-// schema is incompatible with the existing ~/.rosetta/ros-help.db.
-// MUST be checked before importing db.ts, because initDb() stamps user_version.
-const _dbSchemaVersion = (() => {
-  try {
-    const check = new (require("bun:sqlite").default)(_dbPath, { readonly: true });
-    const row = check.prepare("PRAGMA user_version").get() as { user_version: number };
-    check.close();
-    return row.user_version;
-  } catch {
-    return SCHEMA_VERSION; // unreadable — assume ok, initDb() will stamp it
-  }
-})();
-
-if (_dbSchemaVersion !== SCHEMA_VERSION) {
-  const { downloadDb } = await import("./setup.ts");
-  const log = (msg: string) => process.stderr.write(`${msg}\n`);
-  log(`DB schema version mismatch (DB=${_dbSchemaVersion}, expected=${SCHEMA_VERSION}) — re-downloading updated database...`);
-  try {
-    await downloadDb(_dbPath, log);
-    log("Database updated successfully.");
-  } catch (e) {
-    log(`Auto-download failed: ${e}`);
-    log(`Run: ${process.argv[0]} --refresh`);
-  }
-}
+await ensureDbReady((msg) => process.stderr.write(`${msg}\n`));
 
 // Now import db.ts (opens the DB) and query.ts
 const { db, getDbStats, initDb } = await import("./db.ts");
@@ -203,6 +205,8 @@ function createServer() {
 const server = new McpServer({
   name: "rosetta",
   version: RESOLVED_VERSION,
+}, {
+  instructions: "RouterOS documentation search. Start with routeros_search for any RouterOS question — it searches across pages, callouts, and properties with BM25 ranking. Drill into specific pages with routeros_get_page (by ID or title). For device hardware specs use routeros_device_lookup; for version-specific command changes use routeros_command_diff. Only v7 data exists (7.9+) — v6 is out of scope and answers will be unreliable.",
 });
 
 server.registerResource(

package/src/query.test.ts

@@ -505,6 +505,24 @@ describe("searchPages", () => {
     expect(res.ftsQuery).toBeTruthy();
     expect(res.query).toBe("dhcp server");
   });
+
+  test("returns best_section for pages with matching sections", () => {
+    // Page 3 (Bridging and Switching) has sections including "VLAN Setup" with VLAN content
+    const res = searchPages("bridge vlan");
+    const bridging = res.results.find((r) => r.id === 3);
+    expect(bridging).toBeDefined();
+    expect(bridging?.best_section).toBeDefined();
+    expect(bridging?.best_section?.heading).toBe("VLAN Setup");
+    expect(bridging?.best_section?.anchor_id).toBe("BridgingandSwitching-VLANSetup");
+    expect(bridging?.best_section?.url).toContain("#BridgingandSwitching-VLANSetup");
+  });
+
+  test("omits best_section for pages without sections", () => {
+    // Page 1 (DHCP Server) has no sections
+    const res = searchPages("dhcp lease");
+    expect(res.results.length).toBeGreaterThan(0);
+    expect(res.results[0].best_section).toBeUndefined();
+  });
 });
 
 // ---------------------------------------------------------------------------

package/src/query.ts

@@ -15,6 +15,7 @@ export type SearchResult = {
   word_count: number;
   code_lines: number;
   excerpt: string;
+  best_section?: { heading: string; anchor_id: string; url: string };
 };
 
 export type SearchResponse = {
@@ -157,6 +158,8 @@ export function searchPages(question: string, limit = DEFAULT_LIMIT): SearchResp
     fallbackMode = "or";
   }
 
+  attachBestSections(results, terms);
+
   return { query: question, ftsQuery, fallbackMode, results, total: results.length };
 }
 
@@ -179,6 +182,43 @@ function runFtsQuery(ftsQuery: string, limit: number): SearchResult[] {
   }
 }
 
+/** For each search result with sections, find the section whose text best matches the search terms. */
+function attachBestSections(results: SearchResult[], terms: string[]): void {
+  if (results.length === 0 || terms.length === 0) return;
+
+  const stmt = db.prepare(
+    `SELECT heading, anchor_id, text FROM sections WHERE page_id = ? ORDER BY sort_order`,
+  );
+
+  for (const result of results) {
+    const sections = stmt.all(result.id) as Array<{ heading: string; anchor_id: string; text: string }>;
+    if (sections.length === 0) continue;
+
+    let bestSection: (typeof sections)[0] | null = null;
+    let bestScore = 0;
+
+    for (const sec of sections) {
+      const haystack = `${sec.heading} ${sec.text}`.toLowerCase();
+      let score = 0;
+      for (const term of terms) {
+        if (haystack.includes(term)) score++;
+      }
+      if (score > bestScore) {
+        bestScore = score;
+        bestSection = sec;
+      }
+    }
+
+    if (bestSection && bestScore > 0) {
+      result.best_section = {
+        heading: bestSection.heading,
+        anchor_id: bestSection.anchor_id,
+        url: `${result.url}#${bestSection.anchor_id}`,
+      };
+    }
+  }
+}
+
 /** Section TOC entry returned when a large page would be truncated. */
 export type SectionTocEntry = {
   heading: string;

package/src/release.test.ts

@@ -324,6 +324,11 @@ describe("CLI flags", () => {
   test("supports --refresh flag", () => {
     expect(src).toContain("--refresh");
   });
+
+  test("browse mode bootstraps database before importing browse.ts", () => {
+    expect(src).toContain('if (args[0] === "browse")');
+    expect(src).toContain("await ensureDbReady");
+  });
 });
 
 // ---------------------------------------------------------------------------

package/src/setup.ts

@@ -81,7 +81,9 @@ export async function runSetup(force = false) {
   console.log();
   try {
     const { default: sqlite } = await import("bun:sqlite");
-    const db = new sqlite(dbPath, { readonly: true });
+    // Note: open in read-write mode (not readonly) to allow WAL checkpoint.
+    // WAL-mode databases can fail to initialize properly in readonly mode.
+    const db = new sqlite(dbPath);
     const row = db.prepare("SELECT COUNT(*) AS c FROM pages").get() as { c: number };
     const cmdRow = db.prepare("SELECT COUNT(*) AS c FROM commands WHERE type='cmd'").get() as { c: number };
     const versionRow = db.prepare("PRAGMA user_version").get() as { user_version: number };