@tikoci/rosetta 0.6.5 → 0.6.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.6.5",
+  "version": "0.6.6",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/browse.ts

@@ -265,6 +265,9 @@ function renderSearchResults(resp: SearchResponse): string {
   const out: string[] = [];
   const modeNote = resp.fallbackMode === "or" ? dim(" (OR fallback)") : "";
   out.push(`  ${bold(String(resp.results.length))} of ${resp.total} results for ${cyan(`"${resp.query}"`)}${modeNote}`);
+  if (resp.note) {
+    out.push(`  ${yellow("Hint:")} ${resp.note}`);
+  }
   out.push("");
 
   const w = termWidth();
@@ -1092,7 +1095,8 @@ async function handleNumberSelect(idx: number): Promise<void> {
 async function doSearch(query: string): Promise<void> {
   const resp = searchPages(query);
   if (resp.results.length === 0) {
-    console.log(`  ${dim("No results.")} Try: ${cyan("props")} ${query}, ${cyan("cal")} ${query}, ${cyan("vid")} ${query}`);
+    const extra = resp.note ? ` ${yellow("Hint:")} ${resp.note}` : "";
+    console.log(`  ${dim("No results.")} Try: ${cyan("props")} ${query}, ${cyan("cal")} ${query}, ${cyan("vid")} ${query}${extra}`);
     return;
   }
   await paged(renderSearchResults(resp));

package/src/extract-dude.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseDudePage } from "./extract-dude.ts";
+
+const probesHtml = readFileSync(join(import.meta.dirname, "..", "dude", "pages", "Probes.html"), "utf-8");
+
+describe("parseDudePage", () => {
+  test("uses the inner article body instead of wiki chrome", () => {
+    const parsed = parseDudePage(probesHtml, "https://web.archive.org/web/2024/https://wiki.mikrotik.com/wiki/Manual:The_Dude_v6/Probes");
+    expect(parsed.text).not.toContain("Jump to navigation");
+    expect(parsed.text).not.toContain("Jump to search");
+    expect(parsed.text).not.toContain("Retrieved from");
+    expect(parsed.text).not.toContain("FILE ARCHIVED ON");
+    expect(parsed.text).not.toContain("Contents");
+    expect(parsed.text).toContain("The Probes pane shows the available methods");
+  });
+
+  test("extracts useful inline and block code snippets", () => {
+    const parsed = parseDudePage(probesHtml, "https://web.archive.org/web/2024/https://wiki.mikrotik.com/wiki/Manual:The_Dude_v6/Probes");
+    expect(parsed.code).toContain("Custom_Voltage_Function()");
+    expect(parsed.code).toContain('if((oid("1.3.6.1.4.1.14988.1.1.3.8.0")>19),"1", "0")');
+  });
+
+  test("keeps content images but skips wiki UI icons", () => {
+    const parsed = parseDudePage(probesHtml, "https://web.archive.org/web/2024/https://wiki.mikrotik.com/wiki/Manual:The_Dude_v6/Probes");
+    expect(parsed.images.some((img) => img.filename === "Version.png")).toBeFalse();
+    expect(parsed.images.some((img) => img.filename === "Dude-probes-all.JPG")).toBeTrue();
+  });
+});

package/src/extract-dude.ts

@@ -44,7 +44,7 @@ interface PageDef {
 // ── Page list (from CDX API enumeration) ──
 
 const V6_PAGES: PageDef[] = [
-  { wikiPath: "Manual:The_Dude", slug: "The_Dude", path: "The Dude", version: "v6" },
+  // Note: Manual:The_Dude and Manual:The_Dude_v6 are wiki stubs (no content)
   { wikiPath: "Manual:The_Dude_v6/Installation", slug: "Installation", path: "The Dude > v6 > Installation", version: "v6" },
   { wikiPath: "Manual:The_Dude_v6/First_use", slug: "First_use", path: "The Dude > v6 > First Use", version: "v6" },
   { wikiPath: "Manual:The_Dude_v6/Interface", slug: "Interface", path: "The Dude > v6 > Interface", version: "v6" },
@@ -90,8 +90,7 @@ const V3_PAGES: PageDef[] = [
   { wikiPath: "Manual:The_Dude/Interface", slug: "v3_Interface", path: "The Dude > v3/v4 > Interface", version: "v3" },
   { wikiPath: "Manual:The_Dude/Device_settings", slug: "v3_Device_settings", path: "The Dude > v3/v4 > Device Settings", version: "v3" },
   { wikiPath: "Manual:The_Dude/Device_discovery", slug: "v3_Device_discovery", path: "The Dude > v3/v4 > Device Discovery", version: "v3" },
-  { wikiPath: "Manual:The_Dude/Device_map", slug: "v3_Device_map", path: "The Dude > v3/v4 > Device Map", version: "v3" },
-  { wikiPath: "Manual:The_Dude/Device_list", slug: "v3_Device_list", path: "The Dude > v3/v4 > Device List", version: "v3" },
+  // Note: Manual:The_Dude/Device_map and Manual:The_Dude/Device_list are wiki stubs (no content)
   { wikiPath: "Manual:The_Dude/Networks", slug: "v3_Networks", path: "The Dude > v3/v4 > Networks", version: "v3" },
   { wikiPath: "Manual:The_Dude/Links", slug: "v3_Links", path: "The Dude > v3/v4 > Links", version: "v3" },
   { wikiPath: "Manual:The_Dude/Agents", slug: "v3_Agents", path: "The Dude > v3/v4 > Agents", version: "v3" },
@@ -161,7 +160,44 @@ interface ParsedPage {
   images: Array<{ filename: string; altText: string | null; originalUrl: string; waybackImageUrl: string }>;
 }
 
-function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
+const DUDE_NOISE_SELECTORS = [
+  "#toc",
+  ".toc",
+  ".mw-editsection",
+  ".navbox",
+  "#catlinks",
+  ".printfooter",
+  "#wm-ipp-base",
+  "#wm-ipp",
+  "#donato",
+  ".wb-autocomplete-suggestions",
+  "#mw-navigation",
+  "#footer",
+  ".noprint",
+  "#jump-to-nav",
+  ".mw-jump-link",
+  "#siteSub",
+  "#contentSub",
+  "#contentSub2",
+  "script",
+  "style",
+  "noscript",
+] as const;
+
+function removeNoise(root: ParentNode): void {
+  for (const sel of DUDE_NOISE_SELECTORS) {
+    for (const el of root.querySelectorAll(sel)) el.remove();
+  }
+}
+
+function getPrimaryContent(document: Document): Element | null {
+  return document.querySelector("#mw-content-text .mw-parser-output")
+    ?? document.querySelector("#mw-content-text")
+    ?? document.querySelector("#bodyContent .mw-parser-output")
+    ?? document.querySelector("#bodyContent");
+}
+
+export function parseDudePage(html: string, _waybackBaseUrl: string): ParsedPage {
   const { document } = parseHTML(html);
 
   // Title from first h1 or page heading
@@ -172,21 +208,17 @@ function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
     .replace(/_/g, " ") ?? "Unknown";
 
   // Main content area
-  const content = document.querySelector("#mw-content-text, .mw-parser-output, #bodyContent");
+  const content = getPrimaryContent(document);
   if (!content) return { title, text: "", code: "", lastEdited: null, images: [] };
 
-  // Remove navigation, TOC, edit links, Wayback Machine toolbar
-  for (const sel of ["#toc", ".toc", ".mw-editsection", ".navbox", "#catlinks",
-    ".printfooter", "#wm-ipp-base", "#wm-ipp", "#donato", ".wb-autocomplete-suggestions",
-    "#mw-navigation", "#footer", ".noprint"]) {
-    for (const el of content.querySelectorAll(sel)) el.remove();
-  }
-
   // Extract code blocks
   const codeBlocks: string[] = [];
-  for (const pre of content.querySelectorAll("pre, code.ros")) {
+  const seenCodeBlocks = new Set<string>();
+  for (const pre of content.querySelectorAll("pre, code")) {
     const text = pre.textContent?.trim();
-    if (text && text.length > 10) codeBlocks.push(text);
+    if (!text || text.length < 6 || seenCodeBlocks.has(text)) continue;
+    seenCodeBlocks.add(text);
+    codeBlocks.push(text);
   }
 
   // Extract images
@@ -195,6 +227,8 @@ function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
   for (const img of content.querySelectorAll("img")) {
     const src = img.getAttribute("src") ?? "";
     const alt = img.getAttribute("alt") ?? null;
+    const parentA = img.closest("a");
+    const href = parentA?.getAttribute("href") ?? "";
 
     // Skip tiny icons, navigation icons, and Wayback Machine UI images
     const width = Number.parseInt(img.getAttribute("width") ?? "0", 10);
@@ -204,21 +238,17 @@ function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
     // Extract filename from MediaWiki image URLs
     // Patterns: /images/X/XX/Filename.JPG or /wiki/File:Filename.JPG
     let filename: string | null = null;
+    const fileMatch = href.match(/File:([^&"]+)/);
+    if (fileMatch) filename = decodeURIComponent(fileMatch[1]);
+
     const srcMatch = src.match(/\/images\/[0-9a-f]\/[0-9a-f]{2}\/([^/?]+)/i)
       ?? src.match(/\/([^/]+\.(?:jpg|jpeg|png|gif|svg))(?:\?|$)/i);
     if (srcMatch) {
-      filename = decodeURIComponent(srcMatch[1]);
-    }
-
-    // Also check parent <a> links for File: references
-    if (!filename) {
-      const parentA = img.closest("a");
-      const href = parentA?.getAttribute("href") ?? "";
-      const fileMatch = href.match(/File:([^&"]+)/);
-      if (fileMatch) filename = decodeURIComponent(fileMatch[1]);
+      filename ??= decodeURIComponent(srcMatch[1]);
     }
 
     if (!filename) continue;
+    filename = filename.replace(/^\d+px-/, "");
     if (seenFilenames.has(filename)) continue;
     // Skip common wiki icons
     if (/^Icon-\w+\.png$/i.test(filename)) continue;
@@ -242,9 +272,17 @@ function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
   }
 
   // Extract body text
-  const textContent = content.textContent ?? "";
+  const textRoot = content.cloneNode(true) as Element;
+  removeNoise(textRoot);
+  for (const img of textRoot.querySelectorAll("img")) img.remove();
+
+  const textContent = textRoot.textContent ?? "";
   // Clean up whitespace: collapse multiple blank lines, trim
-  const text = textContent.replace(/\n{3,}/g, "\n\n").trim();
+  const text = textContent
+    .replace(/Retrieved from\s+"https?:\/\/[^\n"]+"/gi, "")
+    .replace(/FILE ARCHIVED ON[\s\S]*$/i, "")
+    .replace(/\n[ \t]*\n[ \t]*\n+/g, "\n\n")
+    .trim();
 
   // Last edited date from footer
   const lastEditedEl = document.querySelector("#footer-info-lastmod, .lastmod");
@@ -326,7 +364,7 @@ async function main() {
     }
 
     // Parse HTML
-    const parsed = parsePage(html, wbUrl);
+    const parsed = parseDudePage(html, wbUrl);
     if (!parsed.text && !parsed.code) {
       console.log(`  WARN: empty content for ${pageDef.slug}`);
     }
@@ -403,7 +441,9 @@ async function main() {
   console.log(`DB: ${stats.c} dude_pages, ${imgStats.c} dude_images`);
 }
 
-main().catch((e) => {
-  console.error("Fatal:", e);
-  process.exit(1);
-});
+if (import.meta.main) {
+  main().catch((e) => {
+    console.error("Fatal:", e);
+    process.exit(1);
+  });
+}

package/src/mcp.ts

@@ -462,6 +462,7 @@ Workflow — what to do next:
 
 Tips:
 - Use specific technical terms: "DHCP relay agent" not "how to set up DHCP"
+- For retired Dude GUI or wiki topics, prefer routeros_dude_search; routeros_search covers current RouterOS v7 docs
 - Documentation: 317 pages from March 2026 Confluence export (~515K words)
 - Docs reflect the then-current long-term release (~7.22), not version-pinned
 - Command data: RouterOS 7.9–7.23beta2. No v6 data available.
@@ -990,14 +991,26 @@ server.registerTool(
 Returns the complete page text, code blocks, and a list of GUI screenshots with local file paths.
 Screenshots are downloaded images from the archived wiki — use a file viewer for multimodal analysis.
 
+max_length defaults to 16000. If the page text+code exceeds it, content is truncated and a
+truncated field shows the original lengths. Dude pages are generally small (< 12K chars)
+so truncation is uncommon.
+
 → routeros_dude_search: find pages by topic
 → routeros_command_tree: browse /dude commands in current RouterOS`,
     inputSchema: {
       id: z.union([z.number().int(), z.string()]).describe("Page ID (number) or title/slug (string)"),
+      max_length: z
+        .number()
+        .int()
+        .min(1000)
+        .max(200000)
+        .optional()
+        .default(16000)
+        .describe("Max combined text+code characters to return (default: 16000)."),
     },
   },
-  async ({ id }) => {
-    const page = getDudePage(typeof id === "string" && /^\d+$/.test(id) ? Number.parseInt(id, 10) : id);
+  async ({ id, max_length }) => {
+    const page = getDudePage(typeof id === "string" && /^\d+$/.test(id) ? Number.parseInt(id, 10) : id, max_length);
     if (!page) {
       return {
         content: [

package/src/query.test.ts

@@ -1770,6 +1770,11 @@ describe("searchDude", () => {
     const results = searchDude("dude", 1);
     expect(results.length).toBeLessThanOrEqual(1);
   });
+
+  test("searchPages adds a Dude-specific hint for explicit Dude queries", () => {
+    const results = searchPages("the dude probes");
+    expect(results.note).toContain("routeros_dude_search");
+  });
 });
 
 // ---------------------------------------------------------------------------
@@ -1780,22 +1785,22 @@ describe("getDudePage", () => {
   test("returns page by ID with images", () => {
     const page = getDudePage(1);
     expect(page).not.toBeNull();
-    expect(page!.title).toBe("Probes");
-    expect(page!.images.length).toBe(2);
-    expect(page!.images[0].filename).toBe("Dude-probes-all.JPG");
+    expect(page?.title).toBe("Probes");
+    expect(page?.images.length).toBe(2);
+    expect(page?.images[0]?.filename).toBe("Dude-probes-all.JPG");
   });
 
   test("returns page by title", () => {
     const page = getDudePage("Probes");
     expect(page).not.toBeNull();
-    expect(page!.id).toBe(1);
+    expect(page?.id).toBe(1);
   });
 
   test("returns page by slug", () => {
     const page = getDudePage("Device_discovery");
     expect(page).not.toBeNull();
-    expect(page!.title).toBe("Device Discovery");
-    expect(page!.images.length).toBe(1);
+    expect(page?.title).toBe("Device Discovery");
+    expect(page?.images.length).toBe(1);
   });
 
   test("returns null for non-existent page", () => {

package/src/query.ts

@@ -24,6 +24,7 @@ export type SearchResponse = {
   fallbackMode: "or" | null;
   results: SearchResult[];
   total: number;
+  note?: string;
 };
 
 type CsvScalar = string | number | null;
@@ -88,6 +89,14 @@ const COMPOUND_TERMS: [string, string][] = [
   ["address", "list"],
 ];
 
+function getSpecialSearchHint(question: string): string | undefined {
+  const normalized = question.toLowerCase();
+  if (/(?:^|\b)(?:the\s+)?dude(?:\b|$)/.test(normalized)) {
+    return 'This looks like a Dude query. For retired Dude GUI or wiki topics, use routeros_dude_search. For current RouterOS CLI and server paths, use routeros_command_tree with path "/dude".';
+  }
+  return undefined;
+}
+
 function escapeCsv(value: CsvScalar): string {
   if (value === null) return "";
   const text = String(value);
@@ -141,8 +150,9 @@ export function buildFtsQuery(terms: string[], mode: "AND" | "OR"): string {
 
 export function searchPages(question: string, limit = DEFAULT_LIMIT): SearchResponse {
   const terms = extractTerms(question);
+  const note = getSpecialSearchHint(question);
   if (terms.length === 0) {
-    return { query: question, ftsQuery: "", fallbackMode: null, results: [], total: 0 };
+    return { query: question, ftsQuery: "", fallbackMode: null, results: [], total: 0, ...(note ? { note } : {}) };
   }
 
   // Try AND first
@@ -160,7 +170,7 @@ export function searchPages(question: string, limit = DEFAULT_LIMIT): SearchResp
 
   attachBestSections(results, terms);
 
-  return { query: question, ftsQuery, fallbackMode, results, total: results.length };
+  return { query: question, ftsQuery, fallbackMode, results, total: results.length, ...(note ? { note } : {}) };
 }
 
 function runFtsQuery(ftsQuery: string, limit: number): SearchResult[] {
@@ -1566,6 +1576,7 @@ export type DudePageResult = {
   last_edited: string | null;
   word_count: number | null;
   images: DudeImageResult[];
+  truncated?: { text_total: number; code_total: number };
 };
 
 /** Search archived Dude wiki pages via FTS, with AND→OR fallback. */
@@ -1596,7 +1607,9 @@ function runDudeFtsQuery(ftsQuery: string, limit: number): DudeSearchResult[] {
          FROM dude_pages_fts fts
          JOIN dude_pages dp ON dp.id = fts.rowid
          WHERE dude_pages_fts MATCH ?
-         ORDER BY rank
+         ORDER BY bm25(dude_pages_fts, 3.0, 2.0, 1.0, 0.5),
+                  CASE dp.version WHEN 'v6' THEN 0 ELSE 1 END,
+                  dp.title
          LIMIT ?`,
       )
       .all(ftsQuery, limit) as DudeSearchResult[];
@@ -1605,8 +1618,9 @@ function runDudeFtsQuery(ftsQuery: string, limit: number): DudeSearchResult[] {
   }
 }
 
-/** Get a Dude wiki page by ID or title, with associated images. */
-export function getDudePage(idOrTitle: number | string): DudePageResult | null {
+/** Get a Dude wiki page by ID or title, with associated images.
+ *  maxLength: if set, truncates text+code combined and adds a truncated field. */
+export function getDudePage(idOrTitle: number | string, maxLength?: number): DudePageResult | null {
   const row =
     typeof idOrTitle === "number"
       ? db.prepare("SELECT * FROM dude_pages WHERE id = ?").get(idOrTitle)
@@ -1616,6 +1630,21 @@ export function getDudePage(idOrTitle: number | string): DudePageResult | null {
   page.images = db
     .prepare("SELECT id, filename, alt_text, caption, local_path, original_url, wayback_url FROM dude_images WHERE page_id = ? ORDER BY sort_order")
     .all(page.id) as DudeImageResult[];
+
+  if (maxLength) {
+    const textLen = page.text.length;
+    const codeLen = page.code?.length ?? 0;
+    if (textLen + codeLen > maxLength) {
+      const codeBudget = Math.min(codeLen, Math.floor(maxLength * 0.2));
+      const textBudget = maxLength - codeBudget;
+      page.truncated = { text_total: textLen, code_total: codeLen };
+      page.text = textLen > textBudget ? `${page.text.slice(0, textBudget)}\n\n[... truncated — ${textLen} chars total, showing first ${textBudget}]` : page.text;
+      if (page.code && codeLen > codeBudget) {
+        page.code = `${page.code.slice(0, codeBudget)}\n# [... truncated — ${codeLen} chars total]`;
+      }
+    }
+  }
+
   return page;
 }