@tikoci/rosetta 0.2.0

package/src/extract-properties.ts

@@ -0,0 +1,234 @@
+#!/usr/bin/env bun
+
+/**
+ * extract-properties.ts — Parse confluenceTable property tables from HTML.
+ *
+ * For each page in the DB, re-reads the HTML file and extracts property tables:
+ *   - Tables with "Property" in the first header cell
+ *   - Each row: name (from <strong>), type (from <em>), default, description
+ *   - Section heading: nearest h1/h2/h3 above the table
+ *
+ * Usage: bun run src/extract-properties.ts [html-dir]
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parseHTML } from "linkedom";
+import { db, initDb } from "./db.ts";
+
+const HTML_DIR =
+  process.argv[2] || resolve(import.meta.dirname, "../box/latest/ROS");
+
+/**
+ * Parse the first cell of a property row to extract name, type, and default value.
+ *
+ * Patterns observed:
+ *   <strong>name</strong> (<em>type</em>; Default: <strong>value</strong>)
+ *   <strong>name</strong> (<em>type</em>; Default: value)
+ *   <strong>name</strong> (<em>type</em>)
+ *   <strong>name</strong>
+ */
+function parsePropertyCell(td: Element): { name: string; type: string | null; defaultVal: string | null } | null {
+  // Get the property name from the first <strong> element
+  const strongEl = td.querySelector("strong");
+  if (!strongEl) return null;
+
+  const name = strongEl.textContent?.trim() || "";
+  if (!name || name.length > 80) return null;
+
+  // Get the full cell text for type and default parsing
+  const cellText = td.textContent?.trim() || "";
+
+  // Extract type from <em> tag
+  let type: string | null = null;
+  const emEl = td.querySelector("em");
+  if (emEl) {
+    type = emEl.textContent?.trim() || null;
+    // Clean up trailing punctuation
+    if (type) type = type.replace(/[);,\s]+$/, "").trim();
+  }
+
+  // Extract default value — look for "Default:" pattern in cell text
+  let defaultVal: string | null = null;
+  const defaultMatch = cellText.match(/Default:\s*(.+?)(?:\)|$)/i);
+  if (defaultMatch) {
+    defaultVal = defaultMatch[1].trim();
+    // Remove trailing parenthesis or semicolons
+    defaultVal = defaultVal.replace(/[)\s]+$/, "").trim() || null;
+  }
+
+  return { name, type, defaultVal };
+}
+
+/** Get plain text from an element, stripping HTML. */
+function plainText(el: Element): string {
+  return (el.textContent || "").trim().replace(/\s+/g, " ");
+}
+
+interface PropertyRow {
+  pageId: number;
+  name: string;
+  type: string | null;
+  defaultVal: string | null;
+  description: string;
+  section: string | null;
+  sortOrder: number;
+}
+
+function extractProperties(pageId: number, htmlFile: string): PropertyRow[] {
+  const html = readFileSync(resolve(HTML_DIR, htmlFile), "utf-8");
+  const { document } = parseHTML(html);
+  const mainContent = document.querySelector("#main-content");
+  if (!mainContent) return [];
+
+  const properties: PropertyRow[] = [];
+  let sortOrder = 0;
+
+  // Build a map of element → nearest preceding heading for section context.
+  // Walk all children of main-content and track the current heading.
+  let currentHeading: string | null = null;
+  const headingMap = new Map<Element, string | null>();
+
+  // We need to find tables and their preceding headings.
+  // Walk through all elements in document order.
+  const allElements = mainContent.querySelectorAll("*");
+  for (const el of allElements) {
+    const tag = el.tagName?.toLowerCase();
+    if (tag === "h1" || tag === "h2" || tag === "h3") {
+      currentHeading = el.textContent?.trim() || null;
+    }
+    if (tag === "table") {
+      headingMap.set(el, currentHeading);
+    }
+  }
+
+  // Process each confluenceTable
+  const tables = mainContent.querySelectorAll("table.confluenceTable, table.wrapped");
+  for (const table of tables) {
+    // Check if this is a property table by looking at header cells
+    const headerCells = table.querySelectorAll("th.confluenceTh, thead th");
+    const hasPropertyHeader = Array.from(headerCells).some((th) => {
+      const text = th.textContent?.trim().toLowerCase() || "";
+      return text === "property" || text === "read-only property";
+    });
+    if (!hasPropertyHeader) continue;
+
+    const section = headingMap.get(table) || null;
+
+    // Process data rows (all <tr> in <tbody>, skip header row)
+    const rows = table.querySelectorAll("tbody tr");
+    let isFirstRow = true;
+    for (const row of rows) {
+      const cells = row.querySelectorAll("td.confluenceTd, td");
+      // Skip header rows (th cells, not td)
+      const thCells = row.querySelectorAll("th.confluenceTh, th");
+      if (thCells.length > 0 && cells.length === 0) {
+        isFirstRow = false;
+        continue;
+      }
+      // Also skip if first row has header-like content
+      if (isFirstRow && thCells.length > 0) {
+        isFirstRow = false;
+        continue;
+      }
+      isFirstRow = false;
+
+      if (cells.length < 2) continue;
+
+      const parsed = parsePropertyCell(cells[0]);
+      if (!parsed) continue;
+
+      const description = plainText(cells[1]);
+      if (!description && !parsed.type) continue;
+
+      properties.push({
+        pageId,
+        name: parsed.name,
+        type: parsed.type,
+        defaultVal: parsed.defaultVal,
+        description: description || "",
+        section,
+        sortOrder: sortOrder++,
+      });
+    }
+  }
+
+  return properties;
+}
+
+// ---- Main ----
+
+console.log("Initializing database...");
+initDb();
+
+// Clear existing properties for clean re-extraction
+db.run("DELETE FROM properties;");
+db.run("INSERT INTO properties_fts(properties_fts) VALUES('rebuild');");
+
+// Get all pages that have HTML files
+type PageRef = { id: number; html_file: string; title: string };
+const pages = db.prepare("SELECT id, html_file, title FROM pages").all() as PageRef[];
+
+const insert = db.prepare(`
+  INSERT OR IGNORE INTO properties
+    (page_id, name, type, default_val, description, section, sort_order)
+  VALUES (?, ?, ?, ?, ?, ?, ?)
+`);
+
+let totalProperties = 0;
+let pagesWithProps = 0;
+
+const insertAll = db.transaction(() => {
+  for (const page of pages) {
+    const props = extractProperties(page.id, page.html_file);
+    if (props.length > 0) {
+      pagesWithProps++;
+      for (const p of props) {
+        insert.run(p.pageId, p.name, p.type, p.defaultVal, p.description, p.section, p.sortOrder);
+        totalProperties++;
+      }
+    }
+  }
+});
+insertAll();
+
+const ftsCount = (db.prepare("SELECT COUNT(*) as c FROM properties_fts").get() as { c: number }).c;
+
+console.log(`\nProperty extraction complete:`);
+console.log(`  Properties extracted: ${totalProperties}`);
+console.log(`  Pages with properties: ${pagesWithProps}`);
+console.log(`  FTS index rows: ${ftsCount}`);
+
+// Sample output
+console.log(`\nSample properties from DHCP page:`);
+const dhcpProps = db
+  .prepare(
+    `SELECT name, type, default_val, section
+     FROM properties
+     WHERE page_id = (SELECT id FROM pages WHERE title = 'DHCP')
+     ORDER BY sort_order LIMIT 10`,
+  )
+  .all() as Array<{ name: string; type: string; default_val: string; section: string }>;
+
+for (const p of dhcpProps) {
+  console.log(`  ${p.name} (${p.type || "?"}) [default: ${p.default_val || "none"}] — ${p.section}`);
+}
+
+// Test FTS search
+console.log(`\nSearch for "gateway":`);
+const results = db
+  .prepare(
+    `SELECT p.name, p.type, p.default_val, pg.title as page_title, p.section,
+            snippet(properties_fts, 1, '>>>', '<<<', '...', 20) as excerpt
+     FROM properties_fts fts
+     JOIN properties p ON p.id = fts.rowid
+     JOIN pages pg ON pg.id = p.page_id
+     WHERE properties_fts MATCH 'gateway'
+     ORDER BY rank LIMIT 5`,
+  )
+  .all() as Array<{ name: string; type: string; default_val: string; page_title: string; section: string; excerpt: string }>;
+
+for (const r of results) {
+  console.log(`  ${r.name} [${r.page_title} / ${r.section}]`);
+  console.log(`    ${r.excerpt}`);
+}

package/src/link-commands.ts

@@ -0,0 +1,208 @@
+#!/usr/bin/env bun
+
+/**
+ * link-commands.ts — Link commands to documentation pages.
+ *
+ * Strategies:
+ * 1. Code block paths: extract RouterOS menu paths from code blocks in each page
+ * 2. Known mappings: hardcoded path prefixes to page slugs
+ *
+ * The linking is page_id on the `commands` table (nullable, many commands per page).
+ *
+ * Usage: bun run src/link-commands.ts
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parseHTML } from "linkedom";
+import { db, initDb } from "./db.ts";
+
+const HTML_DIR =
+  process.argv[2] || resolve(import.meta.dirname, "../box/latest/ROS");
+
+initDb();
+
+// Reset all links
+db.run("UPDATE commands SET page_id = NULL;");
+
+type PageRef = { id: number; title: string; html_file: string; code: string; path: string };
+const pages = db.prepare("SELECT id, title, html_file, code, path FROM pages").all() as PageRef[];
+
+type DirCmd = { id: number; path: string };
+const dirCommands = db
+  .prepare("SELECT id, path FROM commands WHERE type = 'dir'")
+  .all() as DirCmd[];
+
+// Build lookup: command path -> command row id
+const cmdPathToId = new Map<string, number>();
+for (const c of dirCommands) {
+  cmdPathToId.set(c.path, c.id);
+}
+
+// Strategy 1: Extract menu paths from first code block and page text
+// RouterOS paths in code: /ip/firewall/filter, /system/clock, etc.
+// Also handle old syntax: /ip firewall filter -> /ip/firewall/filter
+const menuPathRe = /\/[a-z][a-z0-9-]+(?:[/ ][a-z][a-z0-9-]+)+/g;
+
+function normalizeMenuPath(p: string): string {
+  return p.replace(/ /g, "/").toLowerCase();
+}
+
+// Set of known non-RouterOS paths to ignore
+const ignorePaths = new Set([
+  "/bin/bash", "/bin/sh", "/dev/null", "/usr/bin", "/usr/local",
+  "/etc/config", "/tmp/backup", "/var/log", "/proc/sys",
+]);
+
+function isRouterOsPath(p: string): boolean {
+  if (ignorePaths.has(p)) return false;
+  // Must start with a known top-level RouterOS dir
+  const firstSegment = p.split("/")[1];
+  return cmdPathToId.has(`/${firstSegment}`) || [
+    "ip", "ipv6", "interface", "system", "routing", "tool", "queue",
+    "ppp", "mpls", "certificate", "user", "snmp", "radius", "log",
+    "file", "disk", "container", "iot", "caps-man",
+  ].includes(firstSegment);
+}
+
+// Map: page_id -> set of command paths found in that page
+const pageToCommandPaths = new Map<number, Set<string>>();
+
+for (const page of pages) {
+  const paths = new Set<string>();
+
+  // Extract from code blocks
+  const codeMatches = page.code.matchAll(menuPathRe);
+  for (const m of codeMatches) {
+    const normalized = normalizeMenuPath(m[0]);
+    if (isRouterOsPath(normalized)) {
+      paths.add(normalized);
+    }
+  }
+
+  // Also look in the HTML for <strong>/path/syntax</strong> and code elements
+  try {
+    const html = readFileSync(resolve(HTML_DIR, page.html_file), "utf-8");
+    const { document } = parseHTML(html);
+    const mainContent = document.querySelector("#main-content");
+    if (mainContent) {
+      // Check <strong> tags containing paths (common pattern for "Sub-menu: /ip/firewall/filter")
+      for (const strong of mainContent.querySelectorAll("strong")) {
+        const text = strong.textContent?.trim() || "";
+        if (text.startsWith("/") && text.length > 3) {
+          const normalized = normalizeMenuPath(text);
+          if (isRouterOsPath(normalized)) {
+            paths.add(normalized);
+          }
+        }
+      }
+      // Check code elements too
+      for (const codeEl of mainContent.querySelectorAll("code")) {
+        const text = codeEl.textContent?.trim() || "";
+        if (text.startsWith("/") && text.length > 3) {
+          const normalized = normalizeMenuPath(text);
+          if (isRouterOsPath(normalized)) {
+            paths.add(normalized);
+          }
+        }
+      }
+    }
+  } catch {
+    // Skip if file can't be read
+  }
+
+  if (paths.size > 0) {
+    pageToCommandPaths.set(page.id, paths);
+  }
+}
+
+// Now link: for each page's paths, find the best matching dir command
+// A page "owns" a dir if the dir path matches one found in the page.
+// For each command dir, pick the most specific page (the one whose found path
+// is the longest match for the command path).
+
+const updateSingle = db.prepare("UPDATE commands SET page_id = ? WHERE path = ?");
+
+// Build reverse map: command_path -> candidate page_ids
+const cmdToCandidatePages = new Map<string, number[]>();
+
+for (const [pageId, paths] of pageToCommandPaths) {
+  for (const p of paths) {
+    // Direct match
+    if (cmdPathToId.has(p)) {
+      const existing = cmdToCandidatePages.get(p) || [];
+      existing.push(pageId);
+      cmdToCandidatePages.set(p, existing);
+    }
+    // Also try parent paths (e.g., /ip/dhcp-client/add -> /ip/dhcp-client)
+    const segments = p.split("/").filter(Boolean);
+    for (let i = segments.length - 1; i >= 1; i--) {
+      const parent = `/${segments.slice(0, i).join("/")}`;
+      if (cmdPathToId.has(parent)) {
+        const existing = cmdToCandidatePages.get(parent) || [];
+        if (!existing.includes(pageId)) {
+          existing.push(pageId);
+          cmdToCandidatePages.set(parent, existing);
+        }
+        break; // Only link to the most specific parent
+      }
+    }
+  }
+}
+
+// For each command dir, pick the page that seems most authoritative:
+// - Prefer the page whose breadcrumb path is closest to the command path
+// - If tied, prefer the page with more property tables
+const linkDir = db.transaction(() => {
+  for (const [cmdPath, candidatePageIds] of cmdToCandidatePages) {
+    const pageId = candidatePageIds[0];
+
+    // Link the dir itself
+    if (cmdPathToId.has(cmdPath)) {
+      updateSingle.run(pageId, cmdPath);
+    }
+
+    // Also link child commands
+    const children = db
+      .prepare("SELECT path FROM commands WHERE parent_path = ? AND page_id IS NULL")
+      .all(cmdPath) as Array<{ path: string }>;
+    for (const child of children) {
+      updateSingle.run(pageId, child.path);
+    }
+  }
+});
+linkDir();
+
+// Stats
+const totalDirs = (db.prepare("SELECT COUNT(*) as c FROM commands WHERE type='dir'").get() as { c: number }).c;
+const linkedDirs = (db.prepare("SELECT COUNT(*) as c FROM commands WHERE type='dir' AND page_id IS NOT NULL").get() as { c: number }).c;
+const totalCmds = (db.prepare("SELECT COUNT(*) as c FROM commands").get() as { c: number }).c;
+const linkedCmds = (db.prepare("SELECT COUNT(*) as c FROM commands WHERE page_id IS NOT NULL").get() as { c: number }).c;
+
+console.log("Linking complete:");
+console.log(`  Total commands: ${totalCmds}`);
+console.log(`  Linked commands: ${linkedCmds} (${((linkedCmds / totalCmds) * 100).toFixed(1)}%)`);
+console.log(`  Linked dirs: ${linkedDirs}/${totalDirs} (${((linkedDirs / totalDirs) * 100).toFixed(1)}%)`);
+
+// Sample linked commands
+console.log("\nSample linked dirs:");
+const samples = db
+  .prepare(
+    `SELECT c.path, p.title, p.url
+     FROM commands c JOIN pages p ON c.page_id = p.id
+     WHERE c.type = 'dir'
+     ORDER BY c.path LIMIT 20`,
+  )
+  .all() as Array<{ path: string; title: string; url: string }>;
+for (const s of samples) {
+  console.log(`  ${s.path} -> ${s.title}`);
+}
+
+// Unlinked dirs
+console.log("\nUnlinked dirs (sample):");
+const unlinked = db
+  .prepare("SELECT path FROM commands WHERE type='dir' AND page_id IS NULL ORDER BY path LIMIT 20")
+  .all() as Array<{ path: string }>;
+for (const u of unlinked) {
+  console.log(`  ${u.path}`);
+}