@tikoci/rosetta 0.2.0

package/src/extract-all-versions.ts

@@ -0,0 +1,147 @@
+#!/usr/bin/env bun
+/**
+ * extract-all-versions.ts — Extract command trees from all RouterOS versions.
+ *
+ * Discovers RouterOS versions from restraml and extracts each inspect.json
+ * (preferring extra/ variant) into command_versions.
+ *
+ * The latest stable version is loaded as the primary commands table.
+ *
+ * Usage:
+ *   bun run src/extract-all-versions.ts [restraml-base-url-or-local-docs-dir]
+ */
+
+import { existsSync, readdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { discoverRemoteVersions as discoverRemoteVersionList, isHttpUrl, RESTRAML_PAGES_URL } from "./restraml.ts";
+
+const SOURCE = process.argv[2];
+
+interface VersionInfo {
+  version: string;
+  channel: "stable" | "development";
+  inspectPath: string;
+  hasExtra: boolean;
+}
+
+function classifyChannel(version: string): "stable" | "development" {
+  if (version.includes("beta") || version.includes("rc")) return "development";
+  return "stable";
+}
+
+function parseVersionKey(version: string): number[] {
+  // "7.22beta1" → [7, 22, 0, -2, 1] (beta=-2, rc=-1, release=0)
+  const match = version.match(/^(\d+)\.(\d+)(?:\.(\d+))?(?:(beta|rc)(\d+))?$/);
+  if (!match) return [0];
+  const major = Number(match[1]);
+  const minor = Number(match[2]);
+  const patch = Number(match[3] ?? 0);
+  const preType = match[4] === "beta" ? -2 : match[4] === "rc" ? -1 : 0;
+  const preNum = Number(match[5] ?? 0);
+  return [major, minor, patch, preType, preNum];
+}
+
+function compareVersions(a: string, b: string): number {
+  const ka = parseVersionKey(a);
+  const kb = parseVersionKey(b);
+  for (let i = 0; i < Math.max(ka.length, kb.length); i++) {
+    const diff = (ka[i] ?? 0) - (kb[i] ?? 0);
+    if (diff !== 0) return diff;
+  }
+  return 0;
+}
+
+async function discoverRemoteVersions(): Promise<VersionInfo[]> {
+  const versionNames = await discoverRemoteVersionList();
+  const baseUrl = RESTRAML_PAGES_URL;
+
+  // restraml publishes extra/ for every version that has extra-packages,
+  // and the GitHub API listing already confirmed these dirs exist.
+  // Assume extra/inspect.json is available (restraml always generates it for CHR builds).
+  return versionNames
+    .filter((name) => /^\d+\.\d+/.test(name))
+    .map((name) => ({
+      version: name,
+      channel: classifyChannel(name),
+      inspectPath: `${baseUrl}/${name}/extra/inspect.json`,
+      hasExtra: true,
+    }))
+    .sort((a, b) => compareVersions(a.version, b.version));
+}
+
+function discoverLocalVersions(docsDir: string): VersionInfo[] {
+  const entries = readdirSync(docsDir).filter((name) => /^\d+\.\d+/.test(name));
+  return entries
+    .map((name) => {
+      const dir = resolve(docsDir, name);
+      const extraPath = resolve(dir, "extra/inspect.json");
+      const basePath = resolve(dir, "inspect.json");
+      const hasExtra = existsSync(extraPath);
+      const inspectPath = hasExtra ? extraPath : basePath;
+
+      if (!existsSync(inspectPath)) return null;
+
+      return {
+        version: name,
+        channel: classifyChannel(name),
+        inspectPath,
+        hasExtra,
+      };
+    })
+    .filter((v): v is VersionInfo => v !== null)
+    .sort((a, b) => compareVersions(a.version, b.version));
+}
+
+const localMode = SOURCE && !isHttpUrl(SOURCE);
+
+const versions = localMode
+  ? discoverLocalVersions(resolve(SOURCE))
+  : await discoverRemoteVersions();
+
+console.log(
+  `Found ${versions.length} RouterOS versions${localMode ? ` in ${resolve(SOURCE)}` : " from restraml GitHub"}`,
+);
+
+if (versions.length === 0) {
+  throw new Error(`No inspect.json files found${localMode ? ` in ${resolve(SOURCE)}` : " from restraml GitHub"}`);
+}
+
+// Determine the latest stable version for primary extraction
+const latestStable = [...versions].filter((v) => v.channel === "stable").pop();
+const latest = latestStable || versions[versions.length - 1];
+console.log(`Latest stable: ${latest?.version ?? "none"}`);
+
+// Run extraction for each version
+// First pass: accumulate all non-primary versions
+// Second pass: primary version (replaces commands table)
+
+const extractCmd = resolve(import.meta.dir, "extract-commands.ts");
+
+let extracted = 0;
+for (const v of versions) {
+  const isPrimary = v === latest;
+  const flags = [
+    `--version=${v.version}`,
+    `--channel=${v.channel}`,
+    ...(v.hasExtra ? ["--extra"] : []),
+    ...(isPrimary ? [] : ["--accumulate"]),
+  ];
+
+  console.log(`\n${"=".repeat(60)}`);
+  console.log(`${isPrimary ? "PRIMARY" : "accumulate"}: ${v.version} (${v.channel})`);
+
+  const proc = Bun.spawnSync(["bun", "run", extractCmd, v.inspectPath, ...flags], {
+    cwd: resolve(import.meta.dir, ".."),
+    stdio: ["inherit", "inherit", "inherit"],
+  });
+
+  if (proc.exitCode !== 0) {
+    console.error(`FAILED: ${v.version} (exit ${proc.exitCode})`);
+    continue;
+  }
+  extracted++;
+}
+
+console.log(`\n${"=".repeat(60)}`);
+console.log(`Done. Extracted ${extracted}/${versions.length} versions.`);
+console.log(`Primary version: ${latest?.version ?? "none"}`);

package/src/extract-changelogs.ts

@@ -0,0 +1,266 @@
+/**
+ * extract-changelogs.ts — Fetch and parse MikroTik changelogs into the changelogs table.
+ *
+ * Idempotent: deletes all existing changelog rows, then fetches and inserts.
+ * FTS5 index auto-populated via triggers defined in db.ts.
+ *
+ * Usage:
+ *   bun run src/extract-changelogs.ts                     # fetch for all ros_versions
+ *   bun run src/extract-changelogs.ts --versions=7.21,7.22,7.22.1  # explicit versions
+ *   bun run src/extract-changelogs.ts --probe-patches     # discover patch releases
+ */
+
+import { db, initDb } from "./db.ts";
+
+const CHANGELOG_BASE = "https://download.mikrotik.com/routeros";
+const FETCH_DELAY_MS = 200; // polite delay between requests
+
+// ── Types ──
+
+type ChangelogEntry = {
+  version: string;
+  released: string | null;
+  category: string;
+  is_breaking: number;
+  description: string;
+  sort_order: number;
+};
+
+// ── Parser ──
+
+const HEADER_RE = /^What's new in (\S+) \(([^)]+)\):/i;
+const ENTRY_RE = /^([*!])\)/;
+
+/** Parse changelog text into structured entries. */
+export function parseChangelog(text: string, expectedVersion?: string): ChangelogEntry[] {
+  const lines = text.split("\n");
+  const entries: ChangelogEntry[] = [];
+
+  let version: string | null = null;
+  let released: string | null = null;
+  let currentCategory: string | null = null;
+  let currentDesc = "";
+  let currentBreaking = 0;
+  let sortOrder = 0;
+
+  function flush() {
+    if (version && currentCategory !== null && currentDesc) {
+      entries.push({
+        version,
+        released,
+        category: currentCategory,
+        is_breaking: currentBreaking,
+        description: currentDesc.trim(),
+        sort_order: sortOrder++,
+      });
+    }
+    currentCategory = null;
+    currentDesc = "";
+    currentBreaking = 0;
+  }
+
+  for (const rawLine of lines) {
+    const line = rawLine.trimEnd();
+
+    // Check for header
+    const headerMatch = line.match(HEADER_RE);
+    if (headerMatch) {
+      flush();
+      version = headerMatch[1];
+      released = headerMatch[2];
+      sortOrder = 0;
+      continue;
+    }
+
+    // Check for entry start: *) or !)
+    const entryMatch = line.match(ENTRY_RE);
+    if (entryMatch) {
+      flush();
+      currentBreaking = entryMatch[1] === "!" ? 1 : 0;
+
+      // Rest of line after "*) " or "!) "
+      const rest = line.slice(2).trim();
+
+      // Extract category: text before first " - " within first 40 chars
+      const dashIdx = rest.indexOf(" - ");
+      if (dashIdx > 0 && dashIdx <= 40) {
+        currentCategory = rest.slice(0, dashIdx).trim().toLowerCase();
+        currentDesc = rest.slice(dashIdx + 3);
+      } else {
+        // No clear category separator — use "other"
+        currentCategory = "other";
+        currentDesc = rest;
+      }
+      continue;
+    }
+
+    // Continuation line — append to current entry
+    if (currentCategory !== null && line.trim()) {
+      currentDesc += ` ${line.trim()}`;
+    }
+  }
+
+  // Flush last entry
+  flush();
+
+  // If expectedVersion was given but header didn't match, override
+  // (some changelogs have slightly different header formats)
+  if (expectedVersion && entries.length > 0 && !entries.some((e) => e.version === expectedVersion)) {
+    for (const entry of entries) {
+      entry.version = expectedVersion;
+    }
+  }
+
+  return entries;
+}
+
+// ── Fetch ──
+
+async function fetchChangelog(version: string): Promise<string | null> {
+  const url = `${CHANGELOG_BASE}/${encodeURIComponent(version)}/CHANGELOG`;
+  try {
+    const resp = await fetch(url);
+    if (!resp.ok) {
+      if (resp.status === 404) return null;
+      console.warn(`  ${version}: HTTP ${resp.status}`);
+      return null;
+    }
+    return await resp.text();
+  } catch (err) {
+    console.warn(`  ${version}: fetch error — ${err}`);
+    return null;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// ── Version discovery ──
+
+function getKnownVersions(): string[] {
+  const rows = db
+    .prepare("SELECT version FROM ros_versions ORDER BY version")
+    .all() as Array<{ version: string }>;
+  return rows.map((r) => r.version);
+}
+
+/** Probe patch versions: for each minor (7.X), try 7.X.1, 7.X.2, ... up to first 404. */
+async function probePatchVersions(): Promise<string[]> {
+  const known = new Set(getKnownVersions());
+  const patches: string[] = [];
+
+  // Find all minor versions: extract unique 7.X prefixes
+  const minors = new Set<string>();
+  for (const v of known) {
+    const match = v.match(/^(\d+\.\d+)/);
+    if (match) minors.add(match[1]);
+  }
+
+  for (const minor of [...minors].sort()) {
+    // Probe .1 through .20 per minor — up to ~920 requests total at 200ms each (~3 min)
+    for (let p = 1; p <= 20; p++) {
+      const patchVersion = `${minor}.${p}`;
+      if (known.has(patchVersion)) {
+        patches.push(patchVersion);
+        continue;
+      }
+      // Probe by fetching
+      const text = await fetchChangelog(patchVersion);
+      if (text) {
+        patches.push(patchVersion);
+        console.log(`  Discovered patch: ${patchVersion}`);
+      } else {
+        break; // No more patches for this minor
+      }
+      await sleep(FETCH_DELAY_MS);
+    }
+  }
+
+  return patches;
+}
+
+// ── Main ──
+
+if (import.meta.main) {
+
+initDb();
+
+// Parse CLI args
+const args = process.argv.slice(2);
+const versionsArg = args.find((a) => a.startsWith("--versions="));
+const probePatches = args.includes("--probe-patches");
+
+let versions: string[];
+
+if (versionsArg) {
+  versions = versionsArg.slice("--versions=".length).split(",").map((v) => v.trim()).filter(Boolean);
+  console.log(`Changelog extraction: ${versions.length} explicit versions`);
+} else if (probePatches) {
+  console.log("Changelog extraction: probing patch versions...");
+  const known = getKnownVersions();
+  const patches = await probePatchVersions();
+  // Merge: known + discovered patches (deduplicated)
+  const all = new Set([...known, ...patches]);
+  versions = [...all];
+  console.log(`  ${versions.length} versions (${patches.length} from patch probing)`);
+} else {
+  versions = getKnownVersions();
+  console.log(`Changelog extraction: ${versions.length} versions from ros_versions`);
+}
+
+if (versions.length === 0) {
+  console.error("No versions to process. Run extract-commands first, or use --versions=");
+  process.exit(1);
+}
+
+// Fetch all changelogs into memory first, then delete+insert in one transaction.
+// This avoids leaving the table partially populated if a network failure occurs mid-run.
+const fetched: Array<{ version: string; entries: ChangelogEntry[] }> = [];
+let versionsFailed = 0;
+
+for (const version of versions) {
+  const text = await fetchChangelog(version);
+  if (!text) {
+    versionsFailed++;
+    continue;
+  }
+
+  const entries = parseChangelog(text, version);
+  if (entries.length === 0) {
+    console.warn(`  ${version}: no entries parsed`);
+    versionsFailed++;
+    continue;
+  }
+
+  fetched.push({ version, entries });
+  console.log(`  ${version}: ${entries.length} entries${entries.some((e) => e.is_breaking) ? ` (${entries.filter((e) => e.is_breaking).length} breaking)` : ""}`);
+
+  await sleep(FETCH_DELAY_MS);
+}
+
+// Atomic: delete all + insert all in one transaction
+const insert = db.prepare(`INSERT OR IGNORE INTO changelogs
+  (version, released, category, is_breaking, description, sort_order)
+  VALUES (?, ?, ?, ?, ?, ?)`);
+
+const upsertVersion = db.prepare(`INSERT OR IGNORE INTO ros_versions
+  (version, channel, extracted_at)
+  VALUES (?, ?, datetime('now'))`);
+
+const commitAll = db.transaction(() => {
+  db.run("DELETE FROM changelogs");
+  for (const { version, entries } of fetched) {
+    const channel = version.includes("beta") || version.includes("rc") ? "development" : "stable";
+    upsertVersion.run(version, channel);
+    for (const e of entries) {
+      insert.run(e.version, e.released, e.category, e.is_breaking, e.description, e.sort_order);
+    }
+  }
+});
+commitAll();
+
+const totalEntries = fetched.reduce((sum, f) => sum + f.entries.length, 0);
+console.log(`\nChangelogs: ${totalEntries} entries from ${fetched.length} versions (${versionsFailed} failed/skipped)`);
+
+} // end if (import.meta.main)

package/src/extract-commands.ts

@@ -0,0 +1,175 @@
+#!/usr/bin/env bun
+/**
+ * extract-commands.ts — Load RouterOS command tree from inspect.json into SQLite.
+ *
+ * Walks the nested JSON tree and flattens it into a `commands` table with:
+ *   path, name, type (dir|cmd|arg), parent_path, description
+ *
+ * Also populates command_versions for version tracking.
+ *
+ * Usage:
+ *   bun run src/extract-commands.ts [inspect.json-path-or-url] [--version=7.22] [--channel=stable] [--extra]
+ *   bun run src/extract-commands.ts --accumulate [inspect.json-path] [--version=X]
+ *
+ * In default mode: replaces commands table and sets as primary version.
+ * With --accumulate: only adds to command_versions, does not touch commands table.
+ */
+
+import { db, initDb } from "./db.ts";
+import { loadJson, RESTRAML_PAGES_URL } from "./restraml.ts";
+
+// Parse flags
+const cliArgs = process.argv.slice(2);
+const accumulate = cliArgs.includes("--accumulate");
+const extraPackages = cliArgs.includes("--extra");
+const positional = cliArgs.filter((a) => !a.startsWith("--"));
+const flagArgs = Object.fromEntries(
+  cliArgs.filter((a) => a.startsWith("--") && a.includes("=")).map((a) => {
+    const [k, ...v] = a.slice(2).split("=");
+    return [k, v.join("=")];
+  }),
+);
+
+const DEFAULT_INSPECT_URL = `${RESTRAML_PAGES_URL}/7.22.1/extra/inspect.json`;
+const INSPECT_SOURCE = positional[0] || DEFAULT_INSPECT_URL;
+
+// Derive version from path if not explicitly set
+function deriveVersion(filepath: string): string {
+  const match = filepath.match(/\/(\d+\.\d+(?:\.\d+)?(?:beta\d+|rc\d+)?)\//);
+  return match?.[1] ?? "unknown";
+}
+
+function deriveChannel(version: string): string {
+  if (version.includes("beta") || version.includes("rc")) return "development";
+  const parts = version.split(".");
+  if (parts.length === 3) return "stable";
+  return "stable";
+}
+
+const version = flagArgs.version || deriveVersion(INSPECT_SOURCE);
+const channel = flagArgs.channel || deriveChannel(version);
+
+console.log(`Loading inspect.json from ${INSPECT_SOURCE}...`);
+console.log(`Version: ${version}, Channel: ${channel}, Extra: ${extraPackages}, Accumulate: ${accumulate}`);
+const inspectData = await loadJson<Record<string, unknown>>(INSPECT_SOURCE);
+
+interface CommandRow {
+  path: string;
+  name: string;
+  type: string;
+  parentPath: string | null;
+  description: string | null;
+}
+
+const rows: CommandRow[] = [];
+
+function walk(obj: Record<string, unknown>, parentPath: string) {
+  for (const [key, value] of Object.entries(obj)) {
+    if (key === "_type" || key === "desc") continue;
+    if (typeof value !== "object" || value === null) continue;
+
+    const node = value as Record<string, unknown>;
+    const nodeType = node._type as string | undefined;
+    if (!nodeType) continue;
+
+    const currentPath = parentPath ? `${parentPath}/${key}` : `/${key}`;
+    const desc = typeof node.desc === "string" ? node.desc : null;
+
+    // Normalize "path" type to "dir" — inspect.json uses both
+    const normalizedType = nodeType === "path" ? "dir" : nodeType;
+
+    rows.push({
+      path: currentPath,
+      name: key,
+      type: normalizedType,
+      parentPath: parentPath || null,
+      description: desc,
+    });
+
+    // Recurse into children (dirs, paths, and cmds have children)
+    if (normalizedType === "dir" || normalizedType === "cmd") {
+      walk(node, currentPath);
+    }
+  }
+}
+
+walk(inspectData, "");
+
+console.log(`Parsed ${rows.length} command tree entries`);
+const dirs = rows.filter((r) => r.type === "dir").length;
+const cmds = rows.filter((r) => r.type === "cmd").length;
+const args = rows.filter((r) => r.type === "arg").length;
+console.log(`  dirs: ${dirs}, cmds: ${cmds}, args: ${args}`);
+
+// Initialize DB and insert
+initDb();
+
+// Register this version
+db.run(
+  `INSERT OR REPLACE INTO ros_versions (version, channel, extra_packages, extracted_at)
+   VALUES (?, ?, ?, datetime('now'))`,
+  [version, channel, extraPackages ? 1 : 0],
+);
+
+if (!accumulate) {
+  // Primary mode: replace commands table entirely, set ros_version
+  db.run("DELETE FROM commands;");
+
+  const insert = db.prepare(`
+    INSERT OR IGNORE INTO commands (path, name, type, parent_path, description, ros_version)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  const insertCommands = db.transaction(() => {
+    for (const r of rows) {
+      insert.run(r.path, r.name, r.type, r.parentPath, r.description, version);
+    }
+  });
+  insertCommands();
+
+  const total = (db.prepare("SELECT COUNT(*) as c FROM commands").get() as { c: number }).c;
+  console.log(`\nInserted ${total} commands into database (primary: ${version})`);
+}
+
+// Always populate command_versions for this version
+db.run("DELETE FROM command_versions WHERE ros_version = ?", [version]);
+
+const insertVersion = db.prepare(`
+  INSERT OR IGNORE INTO command_versions (command_path, ros_version)
+  VALUES (?, ?)
+`);
+
+const insertVersions = db.transaction(() => {
+  for (const r of rows) {
+    insertVersion.run(r.path, version);
+  }
+});
+insertVersions();
+
+const versionCount = (
+  db.prepare("SELECT COUNT(*) as c FROM command_versions WHERE ros_version = ?").get(version) as { c: number }
+).c;
+console.log(`Recorded ${versionCount} command_versions entries for ${version}`);
+
+const totalVersions = (db.prepare("SELECT COUNT(DISTINCT ros_version) as c FROM command_versions").get() as { c: number }).c;
+console.log(`Total versions tracked: ${totalVersions}`);
+
+if (!accumulate) {
+  // Sample: show the /ip/firewall subtree
+  console.log("\nSample: /ip/firewall children:");
+  const children = db
+    .prepare("SELECT path, type FROM commands WHERE parent_path = '/ip/firewall' ORDER BY path")
+    .all() as Array<{ path: string; type: string }>;
+  for (const c of children) {
+    console.log(`  ${c.type.padEnd(4)} ${c.path}`);
+  }
+
+  // Show top-level dirs
+  console.log("\nTop-level directories:");
+  const topLevel = db
+    .prepare("SELECT path FROM commands WHERE parent_path = '' AND type = 'dir' ORDER BY path")
+    .all() as Array<{ path: string }>;
+  for (const t of topLevel) {
+    console.log(`  ${t.path}`);
+  }
+}