@tikoci/rosetta 0.3.1 → 0.4.1

package/src/extract-test-results.ts

@@ -0,0 +1,359 @@
+/**
+ * extract-test-results.ts — Scrape MikroTik product pages for test results + block diagram URLs.
+ *
+ * Fetches each product page from mikrotik.com and extracts:
+ *   - Ethernet test results (bridging/routing throughput at various packet sizes)
+ *   - IPSec test results (tunnel throughput with various ciphers)
+ *   - Block diagram PNG URL
+ *   - Product page URL slug
+ *
+ * Idempotent: deletes all existing test results, updates device rows.
+ * Requires devices table to be populated first (via extract-devices.ts).
+ *
+ * Usage: bun run src/extract-test-results.ts [--concurrency N] [--delay MS]
+ *
+ * Product page URL slug discovery: fetches the product matrix page to build
+ * a name→slug mapping, then fetches each product page by slug.
+ */
+
+import { parseHTML } from "linkedom";
+import { db, initDb } from "./db.ts";
+
+// ── CLI flags ──
+
+const args = process.argv.slice(2);
+function getFlag(name: string, fallback: number): number {
+  const idx = args.indexOf(`--${name}`);
+  if (idx !== -1 && args[idx + 1]) return Number(args[idx + 1]);
+  return fallback;
+}
+
+const CONCURRENCY = getFlag("concurrency", 4);
+const DELAY_MS = getFlag("delay", 500);
+const PRODUCT_BASE = "https://mikrotik.com/product/";
+
+// ── Types ──
+
+interface TestResultRow {
+  mode: string;
+  configuration: string;
+  packet_size: number;
+  throughput_kpps: number | null;
+  throughput_mbps: number | null;
+}
+
+interface ProductPageData {
+  slug: string;
+  ethernet_results: TestResultRow[];
+  ipsec_results: TestResultRow[];
+  block_diagram_url: string | null;
+}
+
+// ── HTML Parsing ──
+
+/** Decode HTML entities like none to text. */
+function decodeEntities(html: string): string {
+  const { document } = parseHTML("<div></div>");
+  const el = document.createElement("div");
+  el.innerHTML = html;
+  return el.textContent || "";
+}
+
+/** Parse a performance-table element into test result rows. */
+function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
+  const rows: TestResultRow[] = [];
+
+  // Header row: first <tr> in <thead> has [product_code, test_description]
+  const thead = table.querySelector("thead");
+  if (!thead) return { testType: "unknown", rows };
+
+  const headerRows = thead.querySelectorAll("tr");
+  if (headerRows.length < 2) return { testType: "unknown", rows };
+
+  // Determine test type from header description
+  const headerCells = headerRows[0].querySelectorAll("td");
+  const testDesc = headerCells.length >= 2 ? (headerCells[1].textContent || "").trim().toLowerCase() : "";
+  const testType = testDesc.includes("ipsec") ? "ipsec" : "ethernet";
+
+  // Determine packet sizes from the second header row
+  // Structure: [Mode, Configuration, (1518|1400) byte, 512 byte, 64 byte]
+  // The colspan=2 means each size has kpps + Mbps columns
+  const sizeRow = headerRows[1];
+  const sizeCells = sizeRow.querySelectorAll("td");
+  const packetSizes: number[] = [];
+  for (const cell of sizeCells) {
+    const text = (cell.textContent || "").trim();
+    const match = text.match(/^(\d+)\s*byte/i);
+    if (match) packetSizes.push(Number.parseInt(match[1], 10));
+  }
+
+  // If we couldn't find sizes in the header, use defaults
+  if (packetSizes.length === 0) {
+    if (testType === "ipsec") {
+      packetSizes.push(1400, 512, 64);
+    } else {
+      packetSizes.push(1518, 512, 64);
+    }
+  }
+
+  // Parse data rows from <tbody>
+  const tbody = table.querySelector("tbody");
+  if (!tbody) return { testType, rows };
+
+  for (const tr of tbody.querySelectorAll("tr")) {
+    const cells = tr.querySelectorAll("td");
+    if (cells.length < 2) continue;
+
+    const mode = (cells[0].textContent || "").trim();
+    const config = (cells[1].textContent || "").trim();
+
+    // Each packet size has 2 columns: kpps, Mbps
+    for (let i = 0; i < packetSizes.length; i++) {
+      const kppsIdx = 2 + i * 2;
+      const mbpsIdx = 3 + i * 2;
+      if (kppsIdx >= cells.length) break;
+
+      const kpps = Number.parseFloat((cells[kppsIdx].textContent || "").trim());
+      const mbps = mbpsIdx < cells.length
+        ? Number.parseFloat((cells[mbpsIdx].textContent || "").trim())
+        : null;
+
+      rows.push({
+        mode,
+        configuration: config,
+        packet_size: packetSizes[i],
+        throughput_kpps: Number.isNaN(kpps) ? null : kpps,
+        throughput_mbps: mbps !== null && Number.isNaN(mbps) ? null : mbps,
+      });
+    }
+  }
+
+  return { testType, rows };
+}
+
+/** Generate candidate URL slugs for a product.
+ *  MikroTik slugs are wildly inconsistent — some use lowercased names with underscores,
+ *  some use product codes with original casing, and + is sometimes "plus", sometimes dropped.
+ *  Unicode superscripts (², ³) are transliterated to digits.
+ *  We try multiple variants and use the first that returns 200. */
+function generateSlugs(name: string, code: string | null): string[] {
+  const slugs: string[] = [];
+  const seen = new Set<string>();
+  const add = (s: string) => {
+    if (s && !seen.has(s)) {
+      seen.add(s);
+      slugs.push(s);
+    }
+  };
+
+  // Normalize Unicode superscripts to regular digits
+  const norm = (s: string) =>
+    s.replace(/²/g, "2").replace(/³/g, "3").replace(/¹/g, "1");
+
+  const cleanName = norm(name);
+
+  // 1. Lowercased name: + → plus, non-alphanum → _
+  add(cleanName.toLowerCase().replace(/\+/g, "plus").replace(/[^a-z0-9plus]+/g, "_").replace(/^_|_$/g, ""));
+
+  // 2. Lowercased name: drop + entirely
+  add(cleanName.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, ""));
+
+  if (code) {
+    const cleanCode = norm(code);
+
+    // 3. Product code as-is (original casing, + → plus, strip other specials)
+    add(cleanCode.replace(/\+/g, "plus").replace(/[^a-zA-Z0-9plus\-]+/g, "").replace(/^-|-$/g, ""));
+
+    // 4. Product code as-is (original casing)
+    add(cleanCode.replace(/[^a-zA-Z0-9\-]+/g, "").replace(/^-|-$/g, ""));
+
+    // 5. Lowercased code: + → plus
+    add(cleanCode.toLowerCase().replace(/\+/g, "plus").replace(/[^a-z0-9plus]+/g, "_").replace(/^_|_$/g, ""));
+
+    // 6. Lowercased code: drop +
+    add(cleanCode.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, ""));
+  }
+
+  return slugs;
+}
+
+/** Fetch and parse a single product page, trying multiple slug candidates. */
+async function fetchProductPage(slugs: string[]): Promise<ProductPageData | null> {
+  for (const slug of slugs) {
+    const url = `${PRODUCT_BASE}${slug}`;
+    try {
+      const resp = await fetch(url);
+      if (resp.ok) {
+        const html = await resp.text();
+        return parseProductHtml(html, slug);
+      }
+      // Don't warn for intermediary attempts — only the last slug matters
+    } catch {
+      // network error, try next slug
+    }
+  }
+  console.warn(`  [404] ${slugs[0]} (tried ${slugs.length} variants)`);
+  return null;
+}
+
+/** Parse product page HTML into structured data. */
+function parseProductHtml(html: string, slug: string): ProductPageData | null {
+
+  const { document } = parseHTML(html);
+
+  // Parse performance tables
+  const tables = document.querySelectorAll("table.performance-table");
+  const ethernet_results: TestResultRow[] = [];
+  const ipsec_results: TestResultRow[] = [];
+
+  for (const table of tables) {
+    const { testType, rows } = parsePerformanceTable(table);
+    if (testType === "ipsec") {
+      ipsec_results.push(...rows);
+    } else {
+      ethernet_results.push(...rows);
+    }
+  }
+
+  // Find block diagram URL
+  let block_diagram_url: string | null = null;
+  const links = document.querySelectorAll("a");
+  for (const a of links) {
+    const text = (a.textContent || "").trim();
+    if (text === "Block Diagram") {
+      const href = a.getAttribute("href");
+      if (href) {
+        block_diagram_url = href.startsWith("http")
+          ? href
+          : `https://cdn.mikrotik.com${href}`;
+      }
+      break;
+    }
+  }
+
+  return { slug, ethernet_results, ipsec_results, block_diagram_url };
+}
+
+/** Sleep helper for rate limiting. */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// ── Main ──
+
+initDb();
+
+// Get all devices from DB
+const devices = db.prepare("SELECT id, product_name, product_code FROM devices ORDER BY product_name").all() as Array<{
+  id: number;
+  product_name: string;
+  product_code: string | null;
+}>;
+
+if (devices.length === 0) {
+  console.error("No devices in database. Run extract-devices.ts first.");
+  process.exit(1);
+}
+
+console.log(`Found ${devices.length} devices in database`);
+
+// Build device → candidate slugs mapping
+const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
+for (const dev of devices) {
+  const slugs = generateSlugs(dev.product_name, dev.product_code);
+  deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
+}
+
+// Idempotent: clear existing test results
+db.run("DELETE FROM device_test_results");
+
+// Prepare statements
+const insertTest = db.prepare(`INSERT OR IGNORE INTO device_test_results (
+  device_id, test_type, mode, configuration, packet_size,
+  throughput_kpps, throughput_mbps
+) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+
+const updateDevice = db.prepare(`UPDATE devices
+  SET product_url = ?, block_diagram_url = ?
+  WHERE id = ?`);
+
+console.log(`Fetching ${deviceSlugs.length} product pages (concurrency=${CONCURRENCY}, delay=${DELAY_MS}ms)...`);
+
+let totalTests = 0;
+let devicesWithTests = 0;
+let devicesWithDiagrams = 0;
+let fetchErrors = 0;
+
+const insertAll = db.transaction(
+  (results: Array<{ deviceId: number; data: ProductPageData | null }>) => {
+    for (const { deviceId, data } of results) {
+      if (!data) {
+        fetchErrors++;
+        continue;
+      }
+
+      // Update device with URL and block diagram
+      updateDevice.run(
+        `https://mikrotik.com/product/${data.slug}`,
+        data.block_diagram_url,
+        deviceId,
+      );
+
+      if (data.block_diagram_url) devicesWithDiagrams++;
+
+      // Insert test results
+      const allResults = [
+        ...data.ethernet_results.map((r) => ({ ...r, test_type: "ethernet" as const })),
+        ...data.ipsec_results.map((r) => ({ ...r, test_type: "ipsec" as const })),
+      ];
+
+      if (allResults.length > 0) devicesWithTests++;
+
+      for (const r of allResults) {
+        insertTest.run(
+          deviceId,
+          r.test_type,
+          r.mode,
+          r.configuration,
+          r.packet_size,
+          r.throughput_kpps,
+          r.throughput_mbps,
+        );
+        totalTests++;
+      }
+    }
+  },
+);
+
+// Fetch all products with rate limiting
+const allResults: Array<{ deviceId: number; data: ProductPageData | null }> = [];
+let processed = 0;
+
+for (let i = 0; i < deviceSlugs.length; i += CONCURRENCY) {
+  const batch = deviceSlugs.slice(i, i + CONCURRENCY);
+  const batchResults = await Promise.all(
+    batch.map(async (dev) => {
+      const data = await fetchProductPage(dev.slugs);
+      return { deviceId: dev.id, data };
+    }),
+  );
+  allResults.push(...batchResults);
+  processed += batch.length;
+
+  const pct = Math.round((processed / deviceSlugs.length) * 100);
+  process.stdout.write(`\r  ${processed}/${deviceSlugs.length} (${pct}%)`);
+
+  if (i + CONCURRENCY < deviceSlugs.length) {
+    await sleep(DELAY_MS);
+  }
+}
+console.log(""); // newline after progress
+
+// Insert all results in one transaction
+insertAll(allResults);
+
+console.log(`Test results: ${totalTests} rows for ${devicesWithTests} devices`);
+console.log(`Block diagrams: ${devicesWithDiagrams} devices`);
+if (fetchErrors > 0) {
+  console.warn(`Fetch errors: ${fetchErrors} products`);
+}

package/src/mcp-http.test.ts

@@ -212,7 +212,7 @@ describe("HTTP transport: session lifecycle", () => {
 
     const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
     const tools = result.tools as Array<{ name: string }>;
-    expect(tools.length).toBe(11);
+    expect(tools.length).toBe(12);
 
     const toolNames = tools.map((t) => t.name).sort();
     expect(toolNames).toContain("routeros_search");
@@ -402,8 +402,8 @@ describe("HTTP transport: multi-session", () => {
     const tools1 = ((msgs1[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
     const tools2 = ((msgs2[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
 
-    expect(tools1.length).toBe(11);
-    expect(tools2.length).toBe(11);
+    expect(tools1.length).toBe(12);
+    expect(tools2.length).toBe(12);
   });
 
   test("deleting one session does not affect another", async () => {
@@ -425,7 +425,7 @@ describe("HTTP transport: multi-session", () => {
     // Client2 still works
     const msgs = await mcpRequest(server.url, client2.sessionId, "tools/list", 2);
     const tools = ((msgs[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
-    expect(tools.length).toBe(11);
+    expect(tools.length).toBe(12);
 
     // Client1 is gone
     const resp = await fetch(server.url, {

package/src/mcp.ts

@@ -38,6 +38,12 @@ function getArg(name: string): string | undefined {
   return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
 }
 
+/** Format a clickable terminal hyperlink using OSC 8 escape sequences.
+ *  Falls back to plain URL in terminals that don't support OSC 8. */
+function link(url: string, display?: string): string {
+  return `\x1b]8;;${url}\x07${display ?? url}\x1b]8;;\x07`;
+}
+
 if (args.includes("--version") || args.includes("-v")) {
   console.log(`rosetta ${RESOLVED_VERSION}`);
   process.exit(0);
@@ -64,8 +70,10 @@ if (args.includes("--help") || args.includes("-h")) {
   console.log("  DB_PATH  Absolute path to ros-help.db (optional)");
   console.log("  PORT     HTTP listen port (lower precedence than --port)");
   console.log("  HOST     HTTP bind address (lower precedence than --host)");
-  console.log("  TLS_CERT_PATH  TLS certificate path (lower precedence than --tls-cert)");
-  console.log("  TLS_KEY_PATH   TLS private key path (lower precedence than --tls-key)");
+  console.log();
+  console.log(`Quick start:  bunx @tikoci/rosetta --setup`);
+  console.log(`Project:      ${link("https://github.com/tikoci/rosetta")}`);
+  console.log(`Docs:         ${link("https://help.mikrotik.com/docs/spaces/ROS/overview", "help.mikrotik.com")}`);
   process.exit(0);
 }
 
@@ -130,6 +138,8 @@ const {
   searchCallouts,
   searchChangelogs,
   searchDevices,
+  searchDeviceTests,
+  getTestResultMeta,
   searchPages,
   searchProperties,
 } = await import("./query.ts");
@@ -630,16 +640,26 @@ Examples:
 server.registerTool(
   "routeros_device_lookup",
   {
-    description: `Look up MikroTik hardware specs or search for devices matching criteria.
+    description: `Look up MikroTik hardware specs, performance benchmarks, or search for devices matching criteria.
 
-144 products from mikrotik.com/products/matrix (March 2026). Returns hardware specs
-including CPU, RAM, storage, ports, PoE, wireless, license level, and price.
+144 products from mikrotik.com (March 2026). Returns hardware specs, official test results,
+block diagram URLs, and pricing.
 
 **How it works:**
-- If query matches a product name or code exactly → returns full specs for that device
-- Otherwise → FTS search + optional structured filters → returns matching devices
+- If query matches a product name or code exactly → returns full specs + test results + block diagram
+- Otherwise → FTS search + optional structured filters → returns matching devices (compact)
 - Filters can be used alone (no query) to find devices by capability
 
+**Test results** (from mikrotik.com per-product pages):
+- Ethernet: bridging/routing throughput at 64/512/1518 byte packets (kpps + Mbps)
+- IPSec: tunnel throughput with various AES/SHA configurations
+- Key metric: "Routing 25 ip filter rules @ 512 byte" is a common routing performance gauge
+- Devices with L3HW offload show additional hardware-accelerated routing rows
+- Included automatically for exact/single-device lookups — no extra call needed
+
+**Block diagram**: internal switch/CPU/PHY architecture diagram URL (PNG).
+Shows bus topology and per-port bandwidth limits — useful for understanding SoC bottlenecks.
+
 **License levels** determine feature availability:
 - L3: CPE/home (no routing protocols, limited queues)
 - L4: standard (OSPF, BGP, all firewall features)
@@ -654,6 +674,7 @@ including CPU, RAM, storage, ports, PoE, wireless, license level, and price.
 - SMIPS: lowest-end (hAP lite)
 
 Workflow — combine with other tools:
+→ routeros_search_tests: cross-device performance ranking (all 125 devices at once, e.g., 512B routing benchmark)
 → routeros_search: find documentation for features relevant to a device
 → routeros_command_tree: check commands available for a feature
 → routeros_current_versions: check latest firmware for the device
@@ -739,6 +760,115 @@ Data: 144 products, March 2026 snapshot. Not all MikroTik products ever made —
   },
 );
 
+// ---- routeros_search_tests ----
+
+server.registerTool(
+  "routeros_search_tests",
+  {
+    description: `Query device performance test results across all devices.
+
+Returns throughput benchmarks from mikrotik.com product pages — one call replaces
+what would otherwise require 125+ individual device lookups.
+
+**Data:** 2,874 test results across 125 devices (March 2026).
+- Ethernet: bridging/routing throughput at 64/512/1518 byte packets
+- IPSec: tunnel throughput with AES/SHA cipher configurations
+- Results include kpps (packets/sec) and Mbps
+
+**Common queries:**
+- Routing performance ranking: test_type="ethernet", mode="Routing", configuration="25 ip filter rules", packet_size=512
+- Bridge performance: test_type="ethernet", mode="Bridging", configuration="25 bridge filter"
+- IPSec throughput: test_type="ipsec", mode="Single tunnel", configuration="AES-128-CBC"
+
+**Configuration matching:** Uses LIKE (substring) — "25 ip filter" matches "25 ip filter rules".
+Note: some devices use slightly different names (e.g., "25 bridge filter" vs "25 bridge filter rules").
+
+**Tip:** Call with no filters first to see available test_types, modes, configurations, and packet_sizes via the metadata field.
+
+Workflow:
+→ routeros_device_lookup: get full specs + block diagram for a specific device from results
+→ routeros_search: find documentation about features relevant to the test type`,
+    inputSchema: {
+      test_type: z
+        .string()
+        .optional()
+        .describe("Filter: 'ethernet' or 'ipsec'"),
+      mode: z
+        .string()
+        .optional()
+        .describe("Filter: e.g., 'Routing', 'Bridging', 'Single tunnel', '256 tunnels'"),
+      configuration: z
+        .string()
+        .optional()
+        .describe("Filter (substring match): e.g., '25 ip filter rules', 'AES-128-CBC + SHA1', 'none (fast path)'"),
+      packet_size: z
+        .number()
+        .int()
+        .optional()
+        .describe("Filter: packet size in bytes (64, 512, 1400, 1518)"),
+      sort_by: z
+        .enum(["mbps", "kpps"])
+        .optional()
+        .default("mbps")
+        .describe("Sort results by throughput metric (default: mbps)"),
+      limit: z
+        .number()
+        .int()
+        .min(1)
+        .max(200)
+        .optional()
+        .default(50)
+        .describe("Max results (default 50, max 200)"),
+    },
+  },
+  async ({ test_type, mode, configuration, packet_size, sort_by, limit }) => {
+    const hasFilters = test_type || mode || configuration || packet_size;
+
+    if (!hasFilters) {
+      // Discovery mode: return available filter values
+      const meta = getTestResultMeta();
+      return {
+        content: [{
+          type: "text",
+          text: JSON.stringify({
+            message: "No filters provided. Here are the available values — use these to build your query:",
+            ...meta,
+            hint: "Common query: test_type='ethernet', mode='Routing', configuration='25 ip filter rules', packet_size=512",
+          }, null, 2),
+        }],
+      };
+    }
+
+    const result = searchDeviceTests(
+      { test_type, mode, configuration, packet_size, sort_by },
+      limit,
+    );
+
+    if (result.results.length === 0) {
+      const hints = [
+        "Call with no filters to see available test types, modes, and configurations",
+        configuration ? `Try a shorter configuration substring (e.g., "25 ip filter" instead of the full string)` : null,
+      ].filter(Boolean);
+      return {
+        content: [{
+          type: "text",
+          text: `No test results matched the filters.\n\nTry:\n${hints.map((h) => `- ${h}`).join("\n")}`,
+        }],
+      };
+    }
+
+    return {
+      content: [{
+        type: "text",
+        text: JSON.stringify({
+          ...result,
+          has_more: result.total > result.results.length,
+        }, null, 2),
+      }],
+    };
+  },
+);
+
 // ---- routeros_current_versions ----
 
 server.registerTool(