@tikoci/rosetta 0.6.9 → 0.7.2

package/src/classify.ts

@@ -0,0 +1,216 @@
+/**
+ * classify.ts — Pre-search input classifier for `routeros_search` / TUI.
+ *
+ * Cheap regex-based detectors that run before any DB query. Each detector
+ * fires independently and non-exclusively: "bgp 7.22 route reflection" fires
+ * topic + version + general FTS together. `searchAll()` uses the output to
+ * route side queries in parallel (changelogs by version/category, device
+ * lookups, property resolution) without re-parsing the input.
+ *
+ * Pure module — no DB, no I/O. DB-backed resolution (property exists?
+ * device matches a real product?) happens in `searchAll()`.
+ *
+ * See DESIGN.md "North Star Architecture — Unified routeros_search".
+ */
+
+import { canonicalize } from "./canonicalize.ts";
+import { KNOWN_TOPICS } from "./query.ts";
+
+export type CommandFragment = {
+  /** `key=value` pairs extracted from the input (e.g. `chain=forward`). */
+  pairs: Array<{ key: string; value: string }>;
+  /** RouterOS verbs mentioned in the input (e.g. `add`, `set`, `print`). */
+  verbs: string[];
+};
+
+export type QueryClassification = {
+  /** Raw input (unchanged). */
+  input: string;
+  /** RouterOS version tag if a `7.X[.Y][betaN|rcN]` pattern was found. First match wins. */
+  version?: string;
+  /** Known-topic tokens — matched against `KNOWN_TOPICS`. Capped at 5 to keep side queries cheap. */
+  topics: string[];
+  /** Canonical command path (e.g. `/ip/firewall/filter`) if the input looks path-ish. */
+  command_path?: string;
+  /** `key=value` pairs / verbs parsed from fragment-style input (`add chain=forward`). */
+  command_fragment?: CommandFragment;
+  /** Device model candidate (e.g. `RB1100AHx4`, `hAP`, `CCR2216`). DB resolution happens in searchAll. */
+  device?: string;
+  /**
+   * Property-name candidate — single lowercase token that *might* be a property name.
+   * Only set when input is a single short token not matched by any other detector.
+   * searchAll must verify existence in `properties.name` before acting on it.
+   */
+  property?: string;
+};
+
+/** Version pattern: `7.22`, `7.22.1`, `7.23beta2`, `7.22rc1`. No leading \b so `v7.22` still matches. */
+const VERSION_RE = /7\.\d+(?:\.\d+)?(?:beta\d+|rc\d+)?/;
+
+/** key=value pattern — captures `chain=forward`, `action=accept`, `ssid="My Net"` (unquoted only here). */
+const FRAGMENT_RE = /([a-z][a-z0-9-]{0,30})=([^\s]+)/gi;
+
+/**
+ * Device model patterns. Order matters — specific prefixes before generic.
+ * Captures the full model token so searchAll can pass it to searchDevices.
+ */
+const DEVICE_PATTERNS: RegExp[] = [
+  /\bRB\d+[A-Za-z0-9+-]*\b/,           // RB450G, RB1100AHx4, RB5009UG+S+IN
+  /\bCCR\d+[A-Za-z0-9+-]*\b/,          // CCR2216-1G-12XS-2XQ
+  /\bCRS\d+[A-Za-z0-9+-]*\b/,          // CRS354-48G-4S+2Q+RM
+  /\bCSS\d+[A-Za-z0-9+-]*\b/,          // CSS610-8P-2S+IN
+  /\bSXT\d*[A-Za-z0-9+-]*\b/,          // SXTsq, SXT5
+  /\bLHG[A-Za-z0-9+-]*\b/i,            // LHG, LHGG-60ad
+  /\bLtAP[A-Za-z0-9+-]*\b/i,           // LtAP, LtAP mini
+  /\bh(?:EX|AP)[A-Za-z0-9+-]*\b/,      // hEX, hEX S, hAP, hAP ax²
+  /\b[cwm]AP[A-Za-z0-9+-]*\b/,         // cAP, wAP, mAP, cAP ac
+  /\bnetPower[A-Za-z0-9+-]*\b/i,       // netPower 15FR
+  /\bnetMetal[A-Za-z0-9+-]*\b/i,       // netMetal 5
+  /\bGroove[A-Za-z0-9+-]*\b/i,         // Groove A-52HPn
+];
+
+const VERB_TOKENS = new Set([
+  "add", "set", "print", "remove", "edit", "enable", "disable",
+  "export", "import", "find", "get", "reset", "move",
+  "monitor", "scan", "run", "start", "stop",
+]);
+
+/** Single-word tokens that look like a potential property name. */
+const PROPERTY_CANDIDATE_RE = /^[a-z][a-z0-9-]{2,30}$/;
+
+const MAX_TOPICS = 5;
+
+/** Tokenize input into lowercase alphanumeric words, preserving dashes. */
+function tokenize(input: string): string[] {
+  return input
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 0);
+}
+
+/** Detect RouterOS version tag. First match only — "compare 7.21 and 7.22" keeps `7.21`. */
+function detectVersion(input: string): string | undefined {
+  const m = VERSION_RE.exec(input);
+  return m ? m[0] : undefined;
+}
+
+/** Detect command path. Uses canonicalize.primaryPath for robust handling of `/ip address`, `ip/address`, etc. */
+function detectCommandPath(input: string): string | undefined {
+  // Heuristic: only run the canonicalizer if the input contains a forward slash
+  // OR starts with a known top-level command word. Otherwise every "routing"
+  // query would be classified as a command path.
+  const trimmed = input.trim();
+  const hasSlash = trimmed.includes("/");
+  const firstToken = trimmed.split(/\s+/)[0]?.toLowerCase();
+  const topLevels = new Set(["ip", "ipv6", "interface", "routing", "system", "bridge", "container", "port", "radius", "snmp", "tool", "user", "queue", "certificate", "file", "log", "disk", "caps-man", "mpls", "ppp", "special-login", "app"]);
+  if (!hasSlash && (!firstToken || !topLevels.has(firstToken))) return undefined;
+
+  try {
+    // Prefer the parsed command's dir path if a verb was recognized; otherwise
+    // fall back to finalPath (pure navigation like `/ip/firewall/filter`).
+    const { commands, finalPath } = canonicalize(trimmed, "/");
+    const primary = commands.find((c) => !c.subshell) ?? commands[0];
+    const path = primary?.path ?? finalPath;
+    if (!path || path === "/") return undefined;
+    return path;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Extract `key=value` pairs and RouterOS verbs from fragment-style input. */
+function detectCommandFragment(input: string): CommandFragment | undefined {
+  const pairs: Array<{ key: string; value: string }> = [];
+  FRAGMENT_RE.lastIndex = 0;
+  for (let m = FRAGMENT_RE.exec(input); m !== null; m = FRAGMENT_RE.exec(input)) {
+    pairs.push({ key: m[1].toLowerCase(), value: m[2] });
+  }
+
+  const verbs: string[] = [];
+  for (const tok of tokenize(input)) {
+    if (VERB_TOKENS.has(tok) && !verbs.includes(tok)) verbs.push(tok);
+  }
+
+  if (pairs.length === 0 && verbs.length === 0) return undefined;
+  return { pairs, verbs };
+}
+
+/** Detect device model mention. Returns first match; searchDevices handles disambiguation. */
+function detectDevice(input: string): string | undefined {
+  for (const pattern of DEVICE_PATTERNS) {
+    const m = pattern.exec(input);
+    if (m) return m[0];
+  }
+  return undefined;
+}
+
+/** Match tokens against KNOWN_TOPICS. Deduplicated, capped, in first-seen order. */
+function detectTopics(input: string): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const tok of tokenize(input)) {
+    if (KNOWN_TOPICS.has(tok) && !seen.has(tok)) {
+      seen.add(tok);
+      out.push(tok);
+      if (out.length >= MAX_TOPICS) break;
+    }
+  }
+  return out;
+}
+
+/**
+ * Flag a property-name candidate: single token that looks like an identifier
+ * and isn't already classified as anything else. searchAll will verify against
+ * `properties.name`.
+ */
+function detectProperty(
+  input: string,
+  opts: { hasPath: boolean; hasDevice: boolean; hasFragment: boolean; topics: string[] },
+): string | undefined {
+  const trimmed = input.trim();
+  // Only single-token inputs are property candidates — multi-word inputs go to FTS search.
+  if (!PROPERTY_CANDIDATE_RE.test(trimmed)) return undefined;
+  if (opts.hasPath || opts.hasDevice || opts.hasFragment) return undefined;
+  // If the token is itself a known topic, prefer that classification — topic routing beats property lookup.
+  if (opts.topics.includes(trimmed.toLowerCase())) return undefined;
+  return trimmed.toLowerCase();
+}
+
+/**
+ * Classify a user query. Detectors run independently; return object lists every
+ * signal found so `searchAll()` can route parallel side queries.
+ */
+export function classifyQuery(input: string): QueryClassification {
+  const normalized = input ?? "";
+  const result: QueryClassification = {
+    input: normalized,
+    topics: [],
+  };
+
+  if (!normalized.trim()) return result;
+
+  const version = detectVersion(normalized);
+  if (version) result.version = version;
+
+  const commandPath = detectCommandPath(normalized);
+  if (commandPath) result.command_path = commandPath;
+
+  const fragment = detectCommandFragment(normalized);
+  if (fragment) result.command_fragment = fragment;
+
+  const device = detectDevice(normalized);
+  if (device) result.device = device;
+
+  result.topics = detectTopics(normalized);
+
+  const property = detectProperty(normalized, {
+    hasPath: !!commandPath,
+    hasDevice: !!device,
+    hasFragment: !!fragment,
+    topics: result.topics,
+  });
+  if (property) result.property = property;
+
+  return result;
+}

package/src/db.ts

@@ -656,6 +656,105 @@ export function initDb() {
   );`);
 
   db.run(`CREATE INDEX IF NOT EXISTS idx_skill_refs_skill ON skill_references(skill_id);`);
+
+  // -- Glossary (domain jargon resolution for RouterOS terms) --
+
+  db.run(`CREATE TABLE IF NOT EXISTS glossary (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    term        TEXT NOT NULL UNIQUE,
+    definition  TEXT NOT NULL,
+    aliases     TEXT,
+    category    TEXT,
+    search_hint TEXT
+  );`);
+
+  db.run(`CREATE INDEX IF NOT EXISTS idx_glossary_term ON glossary(term);`);
+
+  seedGlossary();
+}
+
+/**
+ * Seed the glossary table with RouterOS domain jargon. Idempotent — uses
+ * INSERT OR IGNORE so existing rows are preserved across schema reinits.
+ *
+ * Fields:
+ *   term        — canonical lookup key (lowercase)
+ *   definition  — brief one-line explanation
+ *   aliases     — comma-separated alternate spellings/abbreviations
+ *   category    — grouping: 'product', 'protocol', 'subsystem', 'concept'
+ *   search_hint — query expansion term(s) for FTS — what to actually search for
+ */
+function seedGlossary() {
+  const entries: Array<[string, string, string | null, string, string]> = [
+    // Products & platforms
+    ["chr", "Cloud Hosted Router — RouterOS VM image for hypervisors", "cloud hosted router", "product", "CHR cloud hosted router"],
+    ["routerboard", "MikroTik hardware product line running RouterOS", "rb,routerboard", "product", "RouterBOARD hardware"],
+    ["swos", "SwOS — switch operating system for CRS1xx/2xx and CSS series", "switchos,switch os", "product", "SwOS switch operating system"],
+    ["winbox", "WinBox — native Windows/macOS GUI management tool for RouterOS", null, "product", "WinBox GUI management"],
+    ["webfig", "WebFig — web-based management interface for RouterOS", null, "product", "WebFig web interface"],
+    ["the dude", "The Dude — MikroTik network monitor (GUI retired, server remains in RouterOS)", "dude", "product", "Dude network monitor"],
+    ["netinstall", "Tool for reinstalling RouterOS on MikroTik hardware via network boot", null, "product", "netinstall network boot"],
+
+    // Wireless
+    ["capsman", "Controlled Access Point Manager — centralized wireless management", "caps man,caps,csman", "subsystem", "CAPsMAN wireless controller"],
+    ["wifiwave2", "WiFi Wave2 driver — successor to legacy wireless, for newer hardware", "wave2,wifi-qcom", "subsystem", "WiFi Wave2 wireless"],
+    ["w60g", "Wireless 60GHz — 802.11ad wireless bridge interface", "60ghz,wireless wire", "subsystem", "60GHz wireless bridge"],
+
+    // Firewall & filtering
+    ["mangle", "Firewall mangle table — packet marking and header modification", null, "subsystem", "firewall mangle packet marking"],
+    ["raw", "Firewall raw table — pre-connection-tracking filtering", null, "subsystem", "firewall raw pre-conntrack"],
+    ["conntrack", "Connection tracking — stateful firewall session table", "connection tracking", "subsystem", "conntrack connection tracking"],
+    ["fasttrack", "FastTrack — kernel-level connection fast path bypassing firewall rules", "fast track,fast-track", "concept", "fasttrack fast path"],
+
+    // Routing
+    ["ospf", "OSPF — Open Shortest Path First routing protocol", null, "protocol", "OSPF routing"],
+    ["bgp", "BGP — Border Gateway Protocol for inter-AS routing", null, "protocol", "BGP routing"],
+    ["rpki", "RPKI — Resource Public Key Infrastructure for BGP route origin validation", null, "protocol", "RPKI route origin validation"],
+    ["mpls", "MPLS — Multi-Protocol Label Switching", "ldp,vpls,rsvp-te", "protocol", "MPLS label switching"],
+    ["vrf", "VRF — Virtual Routing and Forwarding (multiple routing tables)", null, "concept", "VRF virtual routing"],
+
+    // VPN & tunnels
+    ["ipsec", "IPsec — IP Security protocol suite for encrypted tunnels", "ike,ike1,ike2", "protocol", "IPsec IKE tunnel encryption"],
+    ["wireguard", "WireGuard — modern VPN tunnel protocol", "wg", "protocol", "WireGuard VPN tunnel"],
+    ["sstp", "SSTP — Secure Socket Tunneling Protocol (Microsoft VPN)", null, "protocol", "SSTP tunnel"],
+    ["l2tp", "L2TP — Layer 2 Tunneling Protocol, often paired with IPsec", null, "protocol", "L2TP tunnel"],
+    ["ovpn", "OpenVPN — SSL/TLS VPN tunnel implementation in RouterOS", "openvpn", "protocol", "OpenVPN tunnel"],
+    ["vxlan", "VXLAN — Virtual Extensible LAN overlay network", null, "protocol", "VXLAN overlay"],
+    ["gre", "GRE — Generic Routing Encapsulation tunnel", null, "protocol", "GRE tunnel"],
+    ["eoip", "EoIP — Ethernet over IP tunnel (MikroTik proprietary)", null, "protocol", "EoIP Ethernet tunnel"],
+
+    // Bridging & switching
+    ["l3hw", "L3 Hardware Offloading — hardware-accelerated IP routing on supported switches", "l3 hardware,hw offload", "concept", "L3 hardware offloading"],
+    ["vlan", "VLAN — Virtual LAN (802.1Q tagging)", "802.1q", "concept", "VLAN tagging"],
+    ["dot1x", "802.1X — port-based network access control", "802.1x,port authentication", "protocol", "dot1x port authentication"],
+    ["mlag", "MLAG — Multi-Chassis Link Aggregation", null, "protocol", "MLAG multi-chassis"],
+    ["macsec", "MACsec — 802.1AE Media Access Control Security", "802.1ae", "protocol", "MACsec layer2 encryption"],
+
+    // Services
+    ["user-manager", "User Manager — RADIUS server and hotspot user management built into RouterOS", "userman", "subsystem", "User Manager RADIUS"],
+    ["hotspot", "Hotspot — captive portal with authentication and walled garden", "captive portal", "subsystem", "Hotspot captive portal"],
+    ["mqtt", "MQTT — IoT messaging protocol client in RouterOS", null, "protocol", "MQTT IoT messaging"],
+    ["tr069", "TR-069 — CPE WAN Management Protocol for remote device management", "cwmp,acs", "protocol", "TR-069 remote management"],
+    ["snmp", "SNMP — Simple Network Management Protocol for monitoring", null, "protocol", "SNMP monitoring"],
+    ["romon", "RoMON — Router Management Overlay Network for out-of-band management", null, "subsystem", "RoMON management overlay"],
+    ["zerotier", "ZeroTier — peer-to-peer VPN overlay (extra package)", null, "protocol", "ZeroTier peer VPN"],
+
+    // System concepts
+    ["npk", "NPK — RouterOS package file format (.npk)", "package", "concept", "NPK package file"],
+    ["supout", "Support output file — diagnostic bundle generated by /system/sup-output", "support output,sup-output", "concept", "supout support output diagnostics"],
+    ["defconf", "Default configuration — factory-reset configuration script", "default config", "concept", "default configuration"],
+    ["container", "OCI container support in RouterOS (extra package, 7.4+)", "docker", "subsystem", "container Docker OCI"],
+    ["lora", "LoRa — Long Range IoT radio support (extra package)", "lorawan", "subsystem", "LoRa IoT radio"],
+    ["poe", "PoE — Power over Ethernet (in/out)", "power over ethernet", "concept", "PoE power ethernet"],
+    ["etherboot", "Etherboot — network boot mode for RouterOS reinstallation via netinstall", "net boot", "concept", "etherboot network boot netinstall"],
+  ];
+
+  const stmt = db.prepare(
+    "INSERT OR IGNORE INTO glossary (term, definition, aliases, category, search_hint) VALUES (?, ?, ?, ?, ?)",
+  );
+  for (const [term, definition, aliases, category, search_hint] of entries) {
+    stmt.run(term, definition, aliases, category, search_hint);
+  }
 }
 
 /**
@@ -707,6 +806,7 @@ export function getDbStats() {
     dude_images: count("SELECT COUNT(*) AS c FROM dude_images"),
     skills: count("SELECT COUNT(*) AS c FROM skills"),
     skill_references: count("SELECT COUNT(*) AS c FROM skill_references"),
+    glossary: count("SELECT COUNT(*) AS c FROM glossary"),
     schema_nodes: count("SELECT COUNT(*) AS c FROM schema_nodes"),
     schema_node_presence: count("SELECT COUNT(*) AS c FROM schema_node_presence"),
     ...(() => {

package/src/mcp-http.test.ts

@@ -317,7 +317,7 @@ describe("HTTP transport: session lifecycle", () => {
     expect(serverInfo.name).toBe("rosetta");
   });
 
-  test("tools/list returns all 16 tools after initialization", async () => {
+  test("tools/list returns all 13 tools after initialization", async () => {
     const { sessionId } = await mcpInitialize(server.url);
 
     // Send initialized notification first (required by protocol)
@@ -328,20 +328,21 @@ describe("HTTP transport: session lifecycle", () => {
 
     const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
     const tools = result.tools as Array<{ name: string }>;
-    expect(tools.length).toBe(16);
+    expect(tools.length).toBe(13);
 
     const toolNames = tools.map((t) => t.name).sort();
     expect(toolNames).toContain("routeros_search");
     expect(toolNames).toContain("routeros_get_page");
     expect(toolNames).toContain("routeros_lookup_property");
-    expect(toolNames).toContain("routeros_search_properties");
     expect(toolNames).toContain("routeros_command_tree");
-    expect(toolNames).toContain("routeros_search_callouts");
     expect(toolNames).toContain("routeros_search_changelogs");
     expect(toolNames).toContain("routeros_command_version_check");
     expect(toolNames).toContain("routeros_device_lookup");
     expect(toolNames).toContain("routeros_stats");
     expect(toolNames).toContain("routeros_current_versions");
+    // Dropped: routeros_search_callouts, routeros_search_videos (folded into routeros_search.related)
+    expect(toolNames).not.toContain("routeros_search_callouts");
+    expect(toolNames).not.toContain("routeros_search_videos");
   });
 
   test("resources/list returns dataset resources after initialization", async () => {
@@ -549,8 +550,8 @@ describe("HTTP transport: multi-session", () => {
     const tools1 = ((msgs1[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
     const tools2 = ((msgs2[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
 
-    expect(tools1.length).toBe(16);
-    expect(tools2.length).toBe(16);
+    expect(tools1.length).toBe(13);
+    expect(tools2.length).toBe(13);
   });
 
   test("deleting one session does not affect another", async () => {
@@ -572,7 +573,7 @@ describe("HTTP transport: multi-session", () => {
     // Client2 still works
     const msgs = await mcpRequest(server.url, client2.sessionId, "tools/list", 2);
     const tools = ((msgs[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
-    expect(tools.length).toBe(16);
+    expect(tools.length).toBe(13);
 
     // Client1 is gone
     const resp = await fetch(server.url, {