@tikoci/rosetta 0.8.7 → 0.8.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.8.7",
+  "version": "0.8.8",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/eval/retrieval.ts

@@ -0,0 +1,469 @@
+/**
+ * Phase 0 — Golden-query retrieval evaluation runner.
+ *
+ * Loads fixtures/eval/queries.json, calls searchAll() for each query, and
+ * computes classical IR metrics (recall@k, MRR) plus classifier-detection
+ * accuracy. No LLM call anywhere — this is fully deterministic and runs
+ * against the committed ros-help.db in seconds.
+ *
+ * Modes:
+ *   bun run src/eval/retrieval.ts            → run + print report + compare to baseline
+ *   bun run src/eval/retrieval.ts --json     → run + emit JSON report on stdout
+ *   bun run src/eval/retrieval.ts --update-baseline
+ *                                           → run + overwrite fixtures/eval/baseline.json
+ *   bun run src/eval/retrieval.ts --filter <id-prefix>
+ *                                           → only run queries whose id starts with prefix
+ *
+ * Exit codes:
+ *   0  all metrics meet thresholds AND no regression vs baseline
+ *   1  threshold or baseline regression
+ *   2  runner error (bad fixture, DB missing, etc.)
+ *
+ * See BACKLOG.md "MCP Behavioral Testing — Phase 0" for design rationale.
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { searchAll } from "../query.ts";
+
+// ── Types ──────────────────────────────────────────────────────────────────
+
+type Shape =
+  | "nl-question"
+  | "command-path"
+  | "version-question"
+  | "device"
+  | "topic-multi"
+  | "ambiguous";
+
+type ExpectedClassified = {
+  command_path?: string;
+  version?: string;
+  device?: string;
+  property?: string;
+};
+
+type MatchMode = "any" | "all";
+
+type GoldenQuery = {
+  id: string;
+  query: string;
+  shape: Shape;
+  expected_pages?: number[];
+  /** "any" (default): top-k contains ≥1 expected page → recall=1. "all": classical subset recall. */
+  match_mode?: MatchMode;
+  expected_classified?: ExpectedClassified;
+  expected_topics_any?: string[];
+  expected_related?: string[];
+  /** Skip this entire query's checks unless DB has at least this many commands. Lets us keep
+   * command-tree-dependent checks in the golden set without false-failing on slim dev DBs. */
+  requires_commands_min?: number;
+  notes?: string;
+};
+
+type Thresholds = {
+  recall_at_5: number;
+  recall_at_3: number;
+  mrr: number;
+  classifier_accuracy: number;
+};
+
+type GoldenSet = {
+  _thresholds: Thresholds;
+  queries: GoldenQuery[];
+};
+
+type QueryResult = {
+  id: string;
+  query: string;
+  shape: Shape;
+  recall_at_5: number;
+  recall_at_3: number;
+  reciprocal_rank: number;
+  classifier_ok: boolean;
+  related_ok: boolean;
+  topics_ok: boolean;
+  skipped: boolean;
+  skip_reason?: string;
+  top_pages: { id: number; title: string }[];
+  classified_actual: Record<string, unknown>;
+  notes: string[];
+};
+
+type Report = {
+  generated_at: string;
+  total_queries: number;
+  metrics: {
+    recall_at_5: number;
+    recall_at_3: number;
+    mrr: number;
+    classifier_accuracy: number;
+    related_block_accuracy: number;
+    topics_accuracy: number;
+  };
+  per_shape: Record<string, { count: number; recall_at_5: number; mrr: number }>;
+  results: QueryResult[];
+};
+
+// ── Loaders ────────────────────────────────────────────────────────────────
+
+const FIXTURE_PATH = join(import.meta.dir, "../../fixtures/eval/queries.json");
+const BASELINE_PATH = join(import.meta.dir, "../../fixtures/eval/baseline.json");
+
+function loadGoldenSet(): GoldenSet {
+  if (!existsSync(FIXTURE_PATH)) {
+    console.error(`[eval] golden set not found at ${FIXTURE_PATH}`);
+    process.exit(2);
+  }
+  const raw = readFileSync(FIXTURE_PATH, "utf-8");
+  const parsed = JSON.parse(raw) as GoldenSet & { _doc?: string };
+  if (!parsed.queries || !Array.isArray(parsed.queries)) {
+    console.error("[eval] fixture missing 'queries' array");
+    process.exit(2);
+  }
+  return parsed;
+}
+
+// ── Per-query evaluation ───────────────────────────────────────────────────
+
+function evalQuery(q: GoldenQuery, commandsCount: number, k = 5): QueryResult {
+  const empty: QueryResult = {
+    id: q.id,
+    query: q.query,
+    shape: q.shape,
+    recall_at_5: 1,
+    recall_at_3: 1,
+    reciprocal_rank: 1,
+    classifier_ok: true,
+    related_ok: true,
+    topics_ok: true,
+    skipped: false,
+    top_pages: [],
+    classified_actual: {},
+    notes: [],
+  };
+
+  if (q.requires_commands_min && commandsCount < q.requires_commands_min) {
+    return {
+      ...empty,
+      skipped: true,
+      skip_reason: `requires commands ≥ ${q.requires_commands_min}, DB has ${commandsCount}`,
+    };
+  }
+
+  const resp = searchAll(q.query, k * 2);
+  const topIds = resp.pages.slice(0, k).map((p) => p.id);
+  const top3Ids = resp.pages.slice(0, 3).map((p) => p.id);
+
+  // Recall semantics — default "any" (≥1 expected page in top-k counts as full recall).
+  // For QA-style retrieval we usually only need ONE good answer; classical subset recall
+  // ("all" mode) is opt-in for cases where coverage actually matters.
+  const expected = q.expected_pages ?? [];
+  const mode: MatchMode = q.match_mode ?? "any";
+  const recallFor = (ids: number[]): number => {
+    if (expected.length === 0) return 1;
+    if (mode === "any") {
+      return expected.some((id) => ids.includes(id)) ? 1 : 0;
+    }
+    // "all" mode: classical subset recall
+    return expected.filter((id) => ids.includes(id)).length / expected.length;
+  };
+  const recall_at_5 = recallFor(topIds);
+  const recall_at_3 = recallFor(top3Ids);
+
+  // Reciprocal rank: 1/rank of first expected page in top-k. 0 if none found.
+  let rr = 0;
+  if (expected.length > 0) {
+    for (let i = 0; i < topIds.length; i++) {
+      if (expected.includes(topIds[i] as number)) {
+        rr = 1 / (i + 1);
+        break;
+      }
+    }
+  } else {
+    rr = 1; // N/A — don't penalize MRR
+  }
+
+  // Classifier check: every key in expected_classified must match exactly.
+  const notes: string[] = [];
+  let classifier_ok = true;
+  if (q.expected_classified) {
+    for (const [key, want] of Object.entries(q.expected_classified)) {
+      const got = (resp.classified as Record<string, unknown>)[key];
+      if (got !== want) {
+        classifier_ok = false;
+        notes.push(`classifier.${key}: want=${JSON.stringify(want)} got=${JSON.stringify(got)}`);
+      }
+    }
+  }
+
+  // Related-block check: each name in expected_related must appear in resp.related.
+  let related_ok = true;
+  if (q.expected_related && q.expected_related.length > 0) {
+    for (const key of q.expected_related) {
+      if (!(key in resp.related) || resp.related[key as keyof typeof resp.related] == null) {
+        related_ok = false;
+        notes.push(`related.${key}: missing`);
+      }
+    }
+  }
+
+  // Topics check: at least one expected topic must appear in classified.topics.
+  let topics_ok = true;
+  if (q.expected_topics_any && q.expected_topics_any.length > 0) {
+    const got = resp.classified.topics ?? [];
+    const hit = q.expected_topics_any.some((t) => got.includes(t));
+    if (!hit) {
+      topics_ok = false;
+      notes.push(`topics: want any of ${JSON.stringify(q.expected_topics_any)} got=${JSON.stringify(got)}`);
+    }
+  }
+
+  if (expected.length > 0 && recall_at_5 === 0) {
+    notes.push(`top-${k} pages: ${topIds.join(", ")} (none of expected ${expected.join(", ")})`);
+  }
+
+  return {
+    id: q.id,
+    query: q.query,
+    shape: q.shape,
+    recall_at_5,
+    recall_at_3,
+    reciprocal_rank: rr,
+    classifier_ok,
+    related_ok,
+    topics_ok,
+    skipped: false,
+    top_pages: resp.pages.slice(0, 5).map((p) => ({ id: p.id, title: p.title })),
+    classified_actual: { ...resp.classified },
+    notes,
+  };
+}
+
+// ── Aggregation ────────────────────────────────────────────────────────────
+
+function aggregate(results: QueryResult[]): Report["metrics"] {
+  const active = results.filter((r) => !r.skipped);
+  const n = active.length;
+  if (n === 0) {
+    return {
+      recall_at_5: 0,
+      recall_at_3: 0,
+      mrr: 0,
+      classifier_accuracy: 0,
+      related_block_accuracy: 0,
+      topics_accuracy: 0,
+    };
+  }
+  const sum = (f: (r: QueryResult) => number) => active.reduce((a, r) => a + f(r), 0);
+  return {
+    recall_at_5: sum((r) => r.recall_at_5) / n,
+    recall_at_3: sum((r) => r.recall_at_3) / n,
+    mrr: sum((r) => r.reciprocal_rank) / n,
+    classifier_accuracy: sum((r) => (r.classifier_ok ? 1 : 0)) / n,
+    related_block_accuracy: sum((r) => (r.related_ok ? 1 : 0)) / n,
+    topics_accuracy: sum((r) => (r.topics_ok ? 1 : 0)) / n,
+  };
+}
+
+function perShape(results: QueryResult[]): Report["per_shape"] {
+  const out: Report["per_shape"] = {};
+  for (const r of results) {
+    if (r.skipped) continue;
+    if (!out[r.shape]) {
+      out[r.shape] = { count: 0, recall_at_5: 0, mrr: 0 };
+    }
+    const bucket = out[r.shape];
+    bucket.count += 1;
+    bucket.recall_at_5 += r.recall_at_5;
+    bucket.mrr += r.reciprocal_rank;
+  }
+  for (const k of Object.keys(out)) {
+    const b = out[k];
+    if (!b) continue;
+    b.recall_at_5 = b.recall_at_5 / b.count;
+    b.mrr = b.mrr / b.count;
+  }
+  return out;
+}
+
+// ── Reporting ──────────────────────────────────────────────────────────────
+
+function fmtPct(x: number): string {
+  return `${(x * 100).toFixed(1)}%`;
+}
+
+function printReport(report: Report, thresholds: Thresholds, baseline: Report | null): void {
+  const m = report.metrics;
+  console.log(`\n📊 Rosetta retrieval eval — ${report.total_queries} queries`);
+  console.log(`   generated_at: ${report.generated_at}\n`);
+
+  const rows: { label: string; value: number; threshold?: number }[] = [
+    { label: "Recall@5", value: m.recall_at_5, threshold: thresholds.recall_at_5 },
+    { label: "Recall@3", value: m.recall_at_3, threshold: thresholds.recall_at_3 },
+    { label: "MRR", value: m.mrr, threshold: thresholds.mrr },
+    {
+      label: "Classifier accuracy",
+      value: m.classifier_accuracy,
+      threshold: thresholds.classifier_accuracy,
+    },
+    { label: "Related-block accuracy", value: m.related_block_accuracy },
+    { label: "Topics accuracy", value: m.topics_accuracy },
+  ];
+
+  for (const row of rows) {
+    const ok =
+      row.threshold === undefined ? "  " : row.value >= row.threshold ? "✅" : "❌";
+    const thresh = row.threshold === undefined ? "" : `  (≥ ${fmtPct(row.threshold)})`;
+    let delta = "";
+    if (baseline) {
+      const prev = (baseline.metrics as Record<string, number>)[
+        row.label.toLowerCase().replace(/[^a-z0-9]+/g, "_")
+      ];
+      // Map labels to baseline keys
+      const map: Record<string, number> = {
+        "Recall@5": baseline.metrics.recall_at_5,
+        "Recall@3": baseline.metrics.recall_at_3,
+        MRR: baseline.metrics.mrr,
+        "Classifier accuracy": baseline.metrics.classifier_accuracy,
+        "Related-block accuracy": baseline.metrics.related_block_accuracy,
+        "Topics accuracy": baseline.metrics.topics_accuracy,
+      };
+      const b = map[row.label] ?? prev;
+      if (typeof b === "number") {
+        const d = row.value - b;
+        if (Math.abs(d) >= 0.001) {
+          delta = `  Δ ${d > 0 ? "+" : ""}${(d * 100).toFixed(1)}pp`;
+        }
+      }
+    }
+    console.log(`  ${ok}  ${row.label.padEnd(24)} ${fmtPct(row.value).padStart(7)}${thresh}${delta}`);
+  }
+
+  console.log("\n  Per shape:");
+  for (const [shape, b] of Object.entries(report.per_shape)) {
+    console.log(
+      `    ${shape.padEnd(20)} n=${String(b.count).padStart(2)}  recall@5=${fmtPct(b.recall_at_5).padStart(7)}  mrr=${fmtPct(b.mrr).padStart(7)}`,
+    );
+  }
+
+  // Skipped + failure detail
+  const skipped = report.results.filter((r) => r.skipped);
+  if (skipped.length > 0) {
+    console.log(`\n  ⏭  ${skipped.length} queries skipped (env doesn't support):`);
+    for (const s of skipped) console.log(`    [${s.id}] ${s.skip_reason}`);
+  }
+
+  const failures = report.results.filter(
+    (r) =>
+      !r.skipped &&
+      (r.recall_at_5 < 1 || !r.classifier_ok || !r.related_ok || !r.topics_ok),
+  );
+  if (failures.length > 0) {
+    console.log(`\n  ⚠️  ${failures.length} queries with issues:`);
+    for (const f of failures) {
+      console.log(`    [${f.id}] "${f.query}"`);
+      for (const note of f.notes) console.log(`        ${note}`);
+    }
+  }
+}
+
+// ── Threshold + baseline gating ────────────────────────────────────────────
+
+function checkThresholds(metrics: Report["metrics"], t: Thresholds): string[] {
+  const fails: string[] = [];
+  if (metrics.recall_at_5 < t.recall_at_5)
+    fails.push(`recall@5 ${fmtPct(metrics.recall_at_5)} < ${fmtPct(t.recall_at_5)}`);
+  if (metrics.recall_at_3 < t.recall_at_3)
+    fails.push(`recall@3 ${fmtPct(metrics.recall_at_3)} < ${fmtPct(t.recall_at_3)}`);
+  if (metrics.mrr < t.mrr) fails.push(`mrr ${fmtPct(metrics.mrr)} < ${fmtPct(t.mrr)}`);
+  if (metrics.classifier_accuracy < t.classifier_accuracy)
+    fails.push(
+      `classifier ${fmtPct(metrics.classifier_accuracy)} < ${fmtPct(t.classifier_accuracy)}`,
+    );
+  return fails;
+}
+
+function checkRegression(curr: Report, base: Report, tolerance = 0.02): string[] {
+  // Tolerance = 2pp; FTS5 BM25 tweaks shouldn't trigger on noise.
+  const fails: string[] = [];
+  const keys: (keyof Report["metrics"])[] = [
+    "recall_at_5",
+    "recall_at_3",
+    "mrr",
+    "classifier_accuracy",
+  ];
+  for (const k of keys) {
+    const d = curr.metrics[k] - base.metrics[k];
+    if (d < -tolerance) {
+      fails.push(`${k} regressed ${(d * 100).toFixed(1)}pp (was ${fmtPct(base.metrics[k])}, now ${fmtPct(curr.metrics[k])})`);
+    }
+  }
+  return fails;
+}
+
+// ── Main ───────────────────────────────────────────────────────────────────
+
+export function runEval(filterPrefix?: string): Report {
+  // Lazy import db to avoid pulling it at module-load (so test isolation rules
+  // around DB_PATH still work — see extraction.instructions.md).
+  const { db } = require("../db.ts") as typeof import("../db.ts");
+  const commandsCount = (db.prepare("SELECT COUNT(*) as c FROM commands").get() as { c: number }).c;
+
+  const set = loadGoldenSet();
+  const queries = filterPrefix
+    ? set.queries.filter((q) => q.id.startsWith(filterPrefix))
+    : set.queries;
+
+  const results = queries.map((q) => evalQuery(q, commandsCount));
+  return {
+    generated_at: new Date().toISOString(),
+    total_queries: results.length,
+    metrics: aggregate(results),
+    per_shape: perShape(results),
+    results,
+  };
+}
+
+if (import.meta.main) {
+  const args = process.argv.slice(2);
+  const wantJson = args.includes("--json");
+  const wantUpdate = args.includes("--update-baseline");
+  const filterIdx = args.indexOf("--filter");
+  const filter = filterIdx >= 0 ? args[filterIdx + 1] : undefined;
+
+  const set = loadGoldenSet();
+  const report = runEval(filter);
+
+  if (wantJson) {
+    console.log(JSON.stringify(report, null, 2));
+    process.exit(0);
+  }
+
+  const baseline: Report | null = existsSync(BASELINE_PATH)
+    ? (JSON.parse(readFileSync(BASELINE_PATH, "utf-8")) as Report)
+    : null;
+
+  printReport(report, set._thresholds, baseline);
+
+  if (wantUpdate) {
+    writeFileSync(BASELINE_PATH, `${JSON.stringify(report, null, 2)}\n`);
+    console.log(`\n  💾 baseline updated → ${BASELINE_PATH}`);
+    process.exit(0);
+  }
+
+  // Gate: thresholds + regression
+  const thresholdFails = filter ? [] : checkThresholds(report.metrics, set._thresholds);
+  const regressionFails = baseline && !filter ? checkRegression(report, baseline) : [];
+
+  if (thresholdFails.length > 0 || regressionFails.length > 0) {
+    console.log("\n  ❌ FAIL");
+    for (const f of thresholdFails) console.log(`     threshold: ${f}`);
+    for (const f of regressionFails) console.log(`     regression: ${f}`);
+    console.log("\n  Run with --update-baseline if this is intentional.\n");
+    process.exit(1);
+  }
+
+  console.log("\n  ✅ all checks passed\n");
+  process.exit(0);
+}

package/src/eval/self-supervised.ts

@@ -0,0 +1,585 @@
+/**
+ * Phase 1 — Self-supervised retrieval evaluation runner.
+ *
+ * Auto-generates query→expected-page pairs from the live SQLite DB (section
+ * headings, property names, command paths, page titles), then scores searchAll()
+ * against them. No LLM cost, fully deterministic — a cheap "wide net" companion
+ * to Phase 0's hand-curated golden set.
+ *
+ * Modes:
+ *   bun run src/eval/self-supervised.ts
+ *                                          → run + print report + compare to baseline
+ *   bun run src/eval/self-supervised.ts --json
+ *                                          → run + emit JSON report on stdout
+ *   bun run src/eval/self-supervised.ts --update-baseline
+ *                                          → run + overwrite fixtures/eval/self-supervised-baseline.json
+ *   bun run src/eval/self-supervised.ts --filter <strategy>
+ *                                          → only run one strategy (section|property|cmd-path|title)
+ *   bun run src/eval/self-supervised.ts --limit <N>
+ *                                          → cap total queries (for fast iteration)
+ *   bun run src/eval/self-supervised.ts --top-misses <N>
+ *                                          → print top N highest-rank-but-still-missed queries
+ *
+ * Exit codes:
+ *   0  all metrics meet thresholds AND no regression vs baseline
+ *   1  threshold or baseline regression
+ *   2  runner error (DB missing, etc.)
+ *
+ * See BACKLOG.md "MCP Behavioral Testing — Phase 1" for design rationale.
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+// ── Types ──────────────────────────────────────────────────────────────────
+
+type Strategy = "section" | "property" | "cmd-path" | "title";
+
+type QueryCase = {
+  query: string;
+  expected_page_id: number;
+  source: Strategy;
+};
+
+type Thresholds = {
+  title_hit_at_5: number;
+  section_hit_at_10: number;
+  property_hit_at_10: number;
+  cmd_path_hit_at_5: number;
+  overall_mrr: number;
+};
+
+const THRESHOLDS: Thresholds = {
+  title_hit_at_5: 0.9,
+  section_hit_at_10: 0.65,
+  property_hit_at_10: 0.55,
+  cmd_path_hit_at_5: 0.7,
+  overall_mrr: 0.45,
+};
+
+type QueryResult = {
+  query: string;
+  expected_page_id: number;
+  source: Strategy;
+  hit_at_1: number;
+  hit_at_5: number;
+  hit_at_10: number;
+  mrr: number;
+  rank: number | null; // null if not found in top 10
+  top_3_pages: { id: number; title: string }[];
+};
+
+type StrategyMetrics = {
+  count: number;
+  hit_at_1: number;
+  hit_at_5: number;
+  hit_at_10: number;
+  mrr: number;
+};
+
+type Report = {
+  generated_at: string;
+  total_queries: number;
+  per_strategy: Record<Strategy, StrategyMetrics>;
+  overall: {
+    hit_at_1: number;
+    hit_at_5: number;
+    hit_at_10: number;
+    mrr: number;
+  };
+  results: QueryResult[];
+};
+
+// ── Seeded RNG for deterministic sampling ──────────────────────────────────
+
+function splitmix32(seed: number): () => number {
+  let a = seed;
+  return () => {
+    a |= 0;
+    a = (a + 0x9e3779b9) | 0;
+    let t = a ^ (a >>> 16);
+    t = Math.imul(t, 0x21f0aaad);
+    t = t ^ (t >>> 15);
+    t = Math.imul(t, 0x735a2d97);
+    t = t ^ (t >>> 15);
+    return (t >>> 0) / 4294967296;
+  };
+}
+
+function shuffle<T>(arr: T[], rng: () => number): T[] {
+  const result = [...arr];
+  for (let i = result.length - 1; i > 0; i--) {
+    const j = Math.floor(rng() * (i + 1));
+    [result[i], result[j]] = [result[j] as T, result[i] as T];
+  }
+  return result;
+}
+
+// ── Query generation ───────────────────────────────────────────────────────
+
+const GENERIC_HEADINGS = new Set([
+  "Overview",
+  "Example",
+  "Examples",
+  "Properties",
+  "Notes",
+  "Description",
+  "Configuration",
+  "Introduction",
+  "Summary",
+  "See also",
+  "See Also",
+  "Usage",
+  "General",
+  "Basic",
+  "Advanced",
+  "Settings",
+]);
+
+const GENERIC_PROPERTIES = new Set(["disabled", "comment", "name"]);
+
+async function generateQueries(
+  filter?: Strategy,
+  limit?: number,
+): Promise<QueryCase[]> {
+  // Dynamic import to respect DB_PATH env var overrides (see extraction.instructions.md)
+  const { db } = await import("../db.ts");
+
+  const rng = splitmix32(0xc0ffee);
+  const queries: QueryCase[] = [];
+
+  const strategies: Strategy[] = filter
+    ? [filter]
+    : ["section", "property", "cmd-path", "title"];
+
+  for (const strategy of strategies) {
+    let cases: QueryCase[] = [];
+
+    if (strategy === "section") {
+      // Target ~80 queries from section headings
+      const rows = db
+        .prepare(
+          `SELECT page_id, heading FROM sections
+           WHERE level IN (2, 3) AND length(heading) BETWEEN 6 AND 60
+           ORDER BY page_id`,
+        )
+        .all() as { page_id: number; heading: string }[];
+
+      const filtered = rows.filter((r) => {
+        const words = r.heading.trim().split(/\s+/);
+        if (words.length <= 1) return false;
+        if (GENERIC_HEADINGS.has(r.heading.trim())) return false;
+        return true;
+      });
+
+      cases = shuffle(filtered, rng)
+        .slice(0, limit ? Math.floor(limit / 4) : 80)
+        .map((r) => ({
+          query: r.heading,
+          expected_page_id: r.page_id,
+          source: "section" as Strategy,
+        }));
+    } else if (strategy === "property") {
+      // Target ~60 queries from property names
+      const rows = db
+        .prepare(
+          `SELECT DISTINCT page_id, name FROM properties
+           WHERE length(name) >= 4
+           ORDER BY name`,
+        )
+        .all() as { page_id: number; name: string }[];
+
+      const filtered = rows.filter((r) => !GENERIC_PROPERTIES.has(r.name));
+
+      cases = shuffle(filtered, rng)
+        .slice(0, limit ? Math.floor(limit / 4) : 60)
+        .map((r) => ({
+          query: r.name.includes("-") ? r.name : `${r.name} property`,
+          expected_page_id: r.page_id,
+          source: "property" as Strategy,
+        }));
+    } else if (strategy === "cmd-path") {
+      // Target ~30 queries from command paths (skip if table is small)
+      const commandsCount = (
+        db.prepare("SELECT COUNT(*) as c FROM commands").get() as { c: number }
+      ).c;
+
+      if (commandsCount >= 1000) {
+        const rows = db
+          .prepare(
+            `SELECT page_id, path FROM commands
+              WHERE page_id IS NOT NULL AND type = 'dir'
+              ORDER BY path`,
+          )
+          .all() as { page_id: number; path: string }[];
+
+        cases = shuffle(rows, rng)
+          .slice(0, limit ? Math.floor(limit / 4) : 30)
+          .map((r) => ({
+            query: r.path,
+            expected_page_id: r.page_id,
+            source: "cmd-path" as Strategy,
+          }));
+      }
+    } else if (strategy === "title") {
+      // Target ~30 queries from page titles (3-6 words)
+      const rows = db
+        .prepare(
+          `SELECT id, title FROM pages
+           ORDER BY id`,
+        )
+        .all() as { id: number; title: string }[];
+
+      const filtered = rows.filter((r) => {
+        const words = r.title.trim().split(/\s+/);
+        return words.length >= 3 && words.length <= 6;
+      });
+
+      cases = shuffle(filtered, rng)
+        .slice(0, limit ? Math.floor(limit / 4) : 30)
+        .map((r) => ({
+          query: r.title,
+          expected_page_id: r.id,
+          source: "title" as Strategy,
+        }));
+    }
+
+    queries.push(...cases);
+  }
+
+  // Apply overall limit if specified
+  if (limit && queries.length > limit) {
+    return shuffle(queries, rng).slice(0, limit);
+  }
+
+  return queries;
+}
+
+// ── Evaluation ─────────────────────────────────────────────────────────────
+
+async function evalQuery(q: QueryCase): Promise<QueryResult> {
+  const { searchAll } = await import("../query.ts");
+
+  const resp = searchAll(q.query, 10);
+  const pageIds = resp.pages.map((p) => p.id);
+
+  let rank: number | null = null;
+  for (let i = 0; i < pageIds.length; i++) {
+    if (pageIds[i] === q.expected_page_id) {
+      rank = i + 1;
+      break;
+    }
+  }
+
+  const hit_at_1 = rank === 1 ? 1 : 0;
+  const hit_at_5 = rank !== null && rank <= 5 ? 1 : 0;
+  const hit_at_10 = rank !== null && rank <= 10 ? 1 : 0;
+  const mrr = rank !== null ? 1 / rank : 0;
+
+  return {
+    query: q.query,
+    expected_page_id: q.expected_page_id,
+    source: q.source,
+    hit_at_1,
+    hit_at_5,
+    hit_at_10,
+    mrr,
+    rank,
+    top_3_pages: resp.pages.slice(0, 3).map((p) => ({ id: p.id, title: p.title })),
+  };
+}
+
+function aggregate(results: QueryResult[]): Report {
+  const perStrategy: Record<Strategy, StrategyMetrics> = {
+    section: { count: 0, hit_at_1: 0, hit_at_5: 0, hit_at_10: 0, mrr: 0 },
+    property: { count: 0, hit_at_1: 0, hit_at_5: 0, hit_at_10: 0, mrr: 0 },
+    "cmd-path": { count: 0, hit_at_1: 0, hit_at_5: 0, hit_at_10: 0, mrr: 0 },
+    title: { count: 0, hit_at_1: 0, hit_at_5: 0, hit_at_10: 0, mrr: 0 },
+  };
+
+  for (const r of results) {
+    const s = perStrategy[r.source];
+    s.count += 1;
+    s.hit_at_1 += r.hit_at_1;
+    s.hit_at_5 += r.hit_at_5;
+    s.hit_at_10 += r.hit_at_10;
+    s.mrr += r.mrr;
+  }
+
+  for (const s of Object.values(perStrategy)) {
+    if (s.count > 0) {
+      s.hit_at_1 /= s.count;
+      s.hit_at_5 /= s.count;
+      s.hit_at_10 /= s.count;
+      s.mrr /= s.count;
+    }
+  }
+
+  const total = results.length;
+  const overall = {
+    hit_at_1: total > 0 ? results.reduce((a, r) => a + r.hit_at_1, 0) / total : 0,
+    hit_at_5: total > 0 ? results.reduce((a, r) => a + r.hit_at_5, 0) / total : 0,
+    hit_at_10: total > 0 ? results.reduce((a, r) => a + r.hit_at_10, 0) / total : 0,
+    mrr: total > 0 ? results.reduce((a, r) => a + r.mrr, 0) / total : 0,
+  };
+
+  return {
+    generated_at: new Date().toISOString(),
+    total_queries: total,
+    per_strategy: perStrategy,
+    overall,
+    results,
+  };
+}
+
+// ── Reporting ──────────────────────────────────────────────────────────────
+
+function fmtPct(x: number): string {
+  return `${(x * 100).toFixed(1)}%`;
+}
+
+function printReport(
+  report: Report,
+  baseline: Report | null,
+  topMisses: number,
+): void {
+  console.log(
+    `\n📊 Rosetta self-supervised eval — ${report.total_queries} queries`,
+  );
+  console.log(`   generated_at: ${report.generated_at}\n`);
+
+  console.log("  Overall metrics:");
+  const overallRows: { label: string; value: number; threshold?: number }[] = [
+    { label: "Hit@1", value: report.overall.hit_at_1 },
+    { label: "Hit@5", value: report.overall.hit_at_5 },
+    { label: "Hit@10", value: report.overall.hit_at_10 },
+    { label: "MRR", value: report.overall.mrr, threshold: THRESHOLDS.overall_mrr },
+  ];
+
+  for (const row of overallRows) {
+    const ok =
+      row.threshold === undefined ? "  " : row.value >= row.threshold ? "✅" : "❌";
+    const thresh = row.threshold === undefined ? "" : `  (≥ ${fmtPct(row.threshold)})`;
+    let delta = "";
+    if (baseline) {
+      const map: Record<string, number> = {
+        "Hit@1": baseline.overall.hit_at_1,
+        "Hit@5": baseline.overall.hit_at_5,
+        "Hit@10": baseline.overall.hit_at_10,
+        MRR: baseline.overall.mrr,
+      };
+      const b = map[row.label];
+      if (typeof b === "number") {
+        const d = row.value - b;
+        if (Math.abs(d) >= 0.001) {
+          delta = `  Δ ${d > 0 ? "+" : ""}${(d * 100).toFixed(1)}pp`;
+        }
+      }
+    }
+    console.log(
+      `  ${ok}  ${row.label.padEnd(24)} ${fmtPct(row.value).padStart(7)}${thresh}${delta}`,
+    );
+  }
+
+  console.log("\n  Per strategy:");
+  const strategies: Strategy[] = ["title", "section", "property", "cmd-path"];
+  for (const strategy of strategies) {
+    const s = report.per_strategy[strategy];
+    if (s.count === 0) continue;
+
+    let thresh = "";
+    let ok = "  ";
+    if (strategy === "title") {
+      ok = s.hit_at_5 >= THRESHOLDS.title_hit_at_5 ? "✅" : "❌";
+      thresh = `  (hit@5 ≥ ${fmtPct(THRESHOLDS.title_hit_at_5)})`;
+    } else if (strategy === "section") {
+      ok = s.hit_at_10 >= THRESHOLDS.section_hit_at_10 ? "✅" : "❌";
+      thresh = `  (hit@10 ≥ ${fmtPct(THRESHOLDS.section_hit_at_10)})`;
+    } else if (strategy === "property") {
+      ok = s.hit_at_10 >= THRESHOLDS.property_hit_at_10 ? "✅" : "❌";
+      thresh = `  (hit@10 ≥ ${fmtPct(THRESHOLDS.property_hit_at_10)})`;
+    } else if (strategy === "cmd-path") {
+      ok = s.hit_at_5 >= THRESHOLDS.cmd_path_hit_at_5 ? "✅" : "❌";
+      thresh = `  (hit@5 ≥ ${fmtPct(THRESHOLDS.cmd_path_hit_at_5)})`;
+    }
+
+    console.log(`  ${ok}  ${strategy.padEnd(12)}  n=${String(s.count).padStart(3)}  hit@1=${fmtPct(s.hit_at_1).padStart(7)}  hit@5=${fmtPct(s.hit_at_5).padStart(7)}  hit@10=${fmtPct(s.hit_at_10).padStart(7)}  mrr=${fmtPct(s.mrr).padStart(7)}${thresh}`);
+  }
+
+  // Top misses — highest-rank-but-still-missed queries
+  if (topMisses > 0) {
+    const misses = report.results
+      .filter((r) => r.rank === null || r.rank > 10)
+      .slice(0, topMisses);
+
+    if (misses.length > 0) {
+      console.log(`\n  🔍 Top ${topMisses} misses (expected page not in top 10):`);
+      for (const m of misses) {
+        console.log(`    [${m.source}] "${m.query}" (expected page_id=${m.expected_page_id})`);
+        console.log(`        top-3: ${m.top_3_pages.map((p) => `#${p.id} ${p.title}`).join(" | ")}`);
+      }
+    }
+  }
+}
+
+// ── Threshold + baseline gating ────────────────────────────────────────────
+
+function checkThresholds(report: Report, filter?: Strategy): string[] {
+  const fails: string[] = [];
+
+  // Skip overall threshold checks when filtering by strategy
+  if (!filter) {
+    if (report.overall.mrr < THRESHOLDS.overall_mrr) {
+      fails.push(
+        `overall mrr ${fmtPct(report.overall.mrr)} < ${fmtPct(THRESHOLDS.overall_mrr)}`,
+      );
+    }
+  }
+
+  // Per-strategy thresholds
+  const title = report.per_strategy.title;
+  if (title.count > 0 && title.hit_at_5 < THRESHOLDS.title_hit_at_5) {
+    fails.push(
+      `title hit@5 ${fmtPct(title.hit_at_5)} < ${fmtPct(THRESHOLDS.title_hit_at_5)}`,
+    );
+  }
+
+  const section = report.per_strategy.section;
+  if (section.count > 0 && section.hit_at_10 < THRESHOLDS.section_hit_at_10) {
+    fails.push(
+      `section hit@10 ${fmtPct(section.hit_at_10)} < ${fmtPct(THRESHOLDS.section_hit_at_10)}`,
+    );
+  }
+
+  const property = report.per_strategy.property;
+  if (property.count > 0 && property.hit_at_10 < THRESHOLDS.property_hit_at_10) {
+    fails.push(
+      `property hit@10 ${fmtPct(property.hit_at_10)} < ${fmtPct(THRESHOLDS.property_hit_at_10)}`,
+    );
+  }
+
+  const cmdPath = report.per_strategy["cmd-path"];
+  if (cmdPath.count > 0 && cmdPath.hit_at_5 < THRESHOLDS.cmd_path_hit_at_5) {
+    fails.push(
+      `cmd-path hit@5 ${fmtPct(cmdPath.hit_at_5)} < ${fmtPct(THRESHOLDS.cmd_path_hit_at_5)}`,
+    );
+  }
+
+  return fails;
+}
+
+function checkRegression(
+  curr: Report,
+  base: Report,
+  filter?: Strategy,
+  tolerance = 0.05,
+): string[] {
+  // Tolerance = 5pp; auto-gen queries are noisier than golden set.
+  const fails: string[] = [];
+
+  // Overall metrics
+  if (!filter) {
+    const keys: (keyof Report["overall"])[] = ["hit_at_1", "hit_at_5", "hit_at_10", "mrr"];
+    for (const k of keys) {
+      const d = curr.overall[k] - base.overall[k];
+      if (d < -tolerance) {
+        fails.push(
+          `overall ${k} regressed ${(d * 100).toFixed(1)}pp (was ${fmtPct(base.overall[k])}, now ${fmtPct(curr.overall[k])})`,
+        );
+      }
+    }
+  }
+
+  // Per-strategy hit@10 (most lenient metric)
+  const strategies: Strategy[] = filter
+    ? [filter]
+    : ["title", "section", "property", "cmd-path"];
+  for (const strategy of strategies) {
+    const currS = curr.per_strategy[strategy];
+    const baseS = base.per_strategy[strategy];
+    if (currS.count > 0 && baseS.count > 0) {
+      const d = currS.hit_at_10 - baseS.hit_at_10;
+      if (d < -tolerance) {
+        fails.push(
+          `${strategy} hit@10 regressed ${(d * 100).toFixed(1)}pp (was ${fmtPct(baseS.hit_at_10)}, now ${fmtPct(currS.hit_at_10)})`,
+        );
+      }
+    }
+  }
+
+  return fails;
+}
+
+// ── Main ───────────────────────────────────────────────────────────────────
+
+const BASELINE_PATH = join(
+  import.meta.dir,
+  "../../fixtures/eval/self-supervised-baseline.json",
+);
+
+async function runEval(
+  filter?: Strategy,
+  limit?: number,
+): Promise<Report> {
+  const queries = await generateQueries(filter, limit);
+  const results: QueryResult[] = [];
+
+  for (const q of queries) {
+    results.push(await evalQuery(q));
+  }
+
+  return aggregate(results);
+}
+
+if (import.meta.main) {
+  const args = process.argv.slice(2);
+  const wantJson = args.includes("--json");
+  const wantUpdate = args.includes("--update-baseline");
+  const filterIdx = args.indexOf("--filter");
+  const filter = filterIdx >= 0 ? (args[filterIdx + 1] as Strategy) : undefined;
+  const limitIdx = args.indexOf("--limit");
+  const limit = limitIdx >= 0 ? Number.parseInt(args[limitIdx + 1] as string, 10) : undefined;
+  const topMissesIdx = args.indexOf("--top-misses");
+  const topMisses =
+    topMissesIdx >= 0 ? Number.parseInt(args[topMissesIdx + 1] as string, 10) : 10;
+
+  try {
+    const report = await runEval(filter, limit);
+
+    if (wantJson) {
+      console.log(JSON.stringify(report, null, 2));
+      process.exit(0);
+    }
+
+    const baseline: Report | null = existsSync(BASELINE_PATH)
+      ? (JSON.parse(readFileSync(BASELINE_PATH, "utf-8")) as Report)
+      : null;
+
+    printReport(report, baseline, topMisses);
+
+    if (wantUpdate) {
+      writeFileSync(BASELINE_PATH, `${JSON.stringify(report, null, 2)}\n`);
+      console.log(`\n  💾 baseline updated → ${BASELINE_PATH}`);
+      process.exit(0);
+    }
+
+    // Gate: thresholds + regression
+    const thresholdFails = checkThresholds(report, filter);
+    const regressionFails =
+      baseline ? checkRegression(report, baseline, filter) : [];
+
+    if (thresholdFails.length > 0 || regressionFails.length > 0) {
+      console.log("\n  ❌ FAIL");
+      for (const f of thresholdFails) console.log(`     threshold: ${f}`);
+      for (const f of regressionFails) console.log(`     regression: ${f}`);
+      console.log("\n  Run with --update-baseline if this is intentional.\n");
+      process.exit(1);
+    }
+
+    console.log("\n  ✅ all checks passed\n");
+    process.exit(0);
+  } catch (err) {
+    console.error(`[eval] runner error: ${err}`);
+    process.exit(2);
+  }
+}

package/src/mcp-contract.test.ts

@@ -0,0 +1,263 @@
+/**
+ * mcp-contract.test.ts — MCP tool surface contract tests (Phase 2).
+ *
+ * Guards against silent breaking changes to the MCP tool registry:
+ * - Block A: Frozen 13-tool list + workflow-arrow (→) convention in descriptions
+ * - Block B: Token-budget guardrails for 10 canonical queries (rough chars/4)
+ * - Block C: Shape snapshots — fingerprint the *contract* (keys, counts,
+ *           classifier output), NOT the corpus. Deliberately omits page IDs
+ *           and titles so DB refreshes don't churn snapshots.
+ *
+ * These are fast, deterministic, CI-runnable structural tests. No LLM calls,
+ * no network. Use the real local DB (ros-help.db) for stable FTS results.
+ *
+ * When adding/removing/renaming a tool: update EXPECTED_TOOLS below AND add
+ * a CHANGELOG entry under [Unreleased] → Added/Changed/Removed. The test is
+ * designed to force an explicit decision.
+ */
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+const ROOT = path.resolve(import.meta.dirname, "..");
+
+// Block A (static file parse) runs unconditionally. Blocks B and C need the
+// real populated DB singleton. When another test file (extract-videos,
+// query, etc.) has already pinned the db.ts singleton to :memory:, we skip
+// the DB-dependent blocks — running `bun test src/mcp-contract.test.ts`
+// alone exercises them, and CI's `bun test` still gets Block A coverage.
+const { searchAll } = await import("./query.ts");
+const { DB_PATH, getDbStats } = await import("./db.ts");
+
+// getDbStats() throws if tables don't exist (clean checkout before any DB build).
+// Guard defensively: any failure → treat as "DB not usable" and skip B/C.
+function dbPagesOrZero(): number {
+  try {
+    return getDbStats().pages;
+  } catch {
+    return 0;
+  }
+}
+
+const dbPages = dbPagesOrZero();
+const dbIsReal = DB_PATH !== ":memory:" && dbPages > 100;
+const skipReason = dbIsReal
+  ? ""
+  : `DB singleton is "${DB_PATH}" (pages=${dbPages}); run \`bun test src/mcp-contract.test.ts\` solo against a populated DB for Blocks B/C.`;
+
+// ---------------------------------------------------------------------------
+// Block A: Frozen tool registry
+// ---------------------------------------------------------------------------
+
+describe("Frozen tool registry", () => {
+  const EXPECTED_TOOLS = [
+    "routeros_search",
+    "routeros_get_page",
+    "routeros_lookup_property",
+    "routeros_command_tree",
+    "routeros_stats",
+    "routeros_search_changelogs",
+    "routeros_dude_search",
+    "routeros_dude_get_page",
+    "routeros_command_version_check",
+    "routeros_command_diff",
+    "routeros_device_lookup",
+    "routeros_search_tests",
+    "routeros_current_versions",
+  ];
+
+  test("exactly 13 tools registered", () => {
+    const mcpSrc = readFileSync(path.join(ROOT, "src/mcp.ts"), "utf-8");
+    // Extract tool names from server.registerTool("<name>", patterns
+    const toolMatches = mcpSrc.matchAll(/server\.registerTool\(\s*["']([^"']+)["']/g);
+    const foundTools = Array.from(toolMatches, (m) => m[1]);
+
+    expect(foundTools.length).toBe(13);
+    expect(foundTools.sort()).toEqual(EXPECTED_TOOLS.sort());
+  });
+
+  test("all tools have workflow arrow (→) in description", () => {
+    const mcpSrc = readFileSync(path.join(ROOT, "src/mcp.ts"), "utf-8");
+
+    // Terminal/informational tools that don't have natural follow-ups
+    // (identified during Phase 2 implementation as missing workflow arrows)
+    const KNOWN_EXCEPTIONS = ["routeros_stats", "routeros_current_versions"];
+
+    // Extract each complete registerTool block (tool name to closing paren before next registerTool)
+    // Split on registerTool calls, then extract name + description from each block
+    const toolBlocks = mcpSrc.split(/(?=server\.registerTool\()/);
+
+    const toolsWithoutArrow: string[] = [];
+
+    for (const block of toolBlocks) {
+      // Extract tool name
+      const nameMatch = block.match(/server\.registerTool\(\s*["']([^"']+)["']/);
+      if (!nameMatch) continue;
+
+      const toolName = nameMatch[1];
+
+      // Extract description (from `description:` to the closing `inputSchema:`)
+      // This captures the full description including embedded backticks
+      const descMatch = block.match(/description:\s*`([\s\S]*?)`\s*,\s*inputSchema:/);
+      if (!descMatch) {
+        // Some tools might not have inputSchema, try alternate pattern
+        const altMatch = block.match(/description:\s*`([\s\S]*?)`\s*,?\s*\}/);
+        if (!altMatch) continue;
+
+        const description = altMatch[1];
+        if (!description.includes("→") && !KNOWN_EXCEPTIONS.includes(toolName)) {
+          toolsWithoutArrow.push(toolName);
+        }
+        continue;
+      }
+
+      const description = descMatch[1];
+      if (!description.includes("→") && !KNOWN_EXCEPTIONS.includes(toolName)) {
+        toolsWithoutArrow.push(toolName);
+      }
+    }
+
+    if (toolsWithoutArrow.length > 0) {
+      throw new Error(
+        `Tools lacking workflow arrow (→) convention: ${toolsWithoutArrow.join(", ")}`,
+      );
+    }
+
+    // Ensure we found tool descriptions (sanity check for regex)
+    const totalFound = Array.from(
+      mcpSrc.matchAll(/server\.registerTool\(\s*["']([^"']+)["']/g),
+    ).length;
+    expect(totalFound).toBe(13);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Block B: Token-budget guardrails
+// ---------------------------------------------------------------------------
+
+describe.skipIf(!dbIsReal)(`Token-budget guardrails${dbIsReal ? "" : ` [skipped: ${skipReason}]`}`, () => {
+  /**
+   * Rough token estimator: 1 token ≈ 4 chars for JSON.
+   * This is a guardrail to catch 10x regressions, not precise billing.
+   */
+  function estimateTokens(obj: unknown): number {
+    return Math.ceil(JSON.stringify(obj).length / 4);
+  }
+
+  const QUERIES: Array<{ query: string; limit: number; budget: number }> = [
+    { query: "dhcp server", limit: 8, budget: 8000 },
+    { query: "/ip/firewall/filter", limit: 8, budget: 8000 },
+    { query: "bridge vlan", limit: 8, budget: 8000 },
+    { query: "hAP ax3", limit: 8, budget: 6000 },
+    { query: "what changed in 7.22.1", limit: 8, budget: 8000 },
+    { query: "BGP", limit: 8, budget: 8000 },
+    { query: "container setup", limit: 8, budget: 8000 },
+    { query: "disabled property", limit: 8, budget: 6000 },
+    { query: "CAPsMAN", limit: 20, budget: 16000 }, // hunger knob test
+    { query: "firewall filter chain", limit: 8, budget: 8000 },
+  ];
+
+  for (const { query, limit, budget } of QUERIES) {
+    test(`"${query}" (limit=${limit}) ≤ ${budget} tokens`, () => {
+      const result = searchAll(query, limit);
+      const tokens = estimateTokens(result);
+
+      if (tokens > budget) {
+        throw new Error(
+          `Token budget exceeded: "${query}" | actual=${tokens} tokens | budget=${budget}`,
+        );
+      }
+
+      // Log for the record (visible on test run)
+      console.log(`  ✓ "${query}" (limit=${limit}): ${tokens} tokens`);
+    });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// Block C: Response-shape invariants
+// ---------------------------------------------------------------------------
+//
+// This block asserts shape contracts that hold on any populated DB — no
+// file-based snapshots (they coupled to the local dev DB's extraction state
+// and drifted against the full CI-built DB's richer `related_buckets`). The
+// invariants here protect against silent breakage of the searchAll return
+// shape while staying portable across DBs of varying richness.
+//
+// Corpus-linked expectations ("this page must rank for this query") live in
+// fixtures/eval/queries.json (Phase 0) — that's the right surface for them.
+
+describe.skipIf(!dbIsReal)(`Response-shape invariants${dbIsReal ? "" : ` [skipped: ${skipReason}]`}`, () => {
+  type Invariant = {
+    query: string;
+    limit: number;
+    classifier_expected: Record<string, unknown>;
+  };
+
+  const INVARIANTS: Invariant[] = [
+    { query: "dhcp server", limit: 8, classifier_expected: { topics: ["dhcp"] } },
+    {
+      query: "bridge vlan",
+      limit: 8,
+      classifier_expected: { command_path: "/bridge/vlan", topics: ["bridge", "vlan"] },
+    },
+    { query: "hAP ax3", limit: 8, classifier_expected: { device: "hAP" } },
+    {
+      query: "what changed in 7.22.1",
+      limit: 8,
+      classifier_expected: { version: "7.22.1" },
+    },
+    {
+      query: "/ip/firewall/filter",
+      limit: 8,
+      classifier_expected: {
+        command_path: "/ip/firewall/filter",
+        topics: ["ip", "firewall", "filter"],
+      },
+    },
+  ];
+
+  for (const { query, limit, classifier_expected } of INVARIANTS) {
+    test(`shape: "${query}"`, () => {
+      const result = searchAll(query, limit);
+
+      // Top-level keys
+      expect(result).toHaveProperty("query", query);
+      expect(result).toHaveProperty("classified");
+      expect(result).toHaveProperty("pages");
+      expect(result).toHaveProperty("related");
+      expect(result).toHaveProperty("next_steps");
+      expect(result).toHaveProperty("total_pages");
+
+      // Classifier output is DB-independent (pure regex) — assert exact subset
+      for (const [k, v] of Object.entries(classifier_expected)) {
+        expect(result.classified).toHaveProperty(k, v);
+      }
+
+      // Pages: at least one hit on a populated DB, bounded by limit
+      expect(Array.isArray(result.pages)).toBe(true);
+      expect(result.pages.length).toBeGreaterThan(0);
+      expect(result.pages.length).toBeLessThanOrEqual(limit);
+      expect(result.total_pages).toBeGreaterThanOrEqual(result.pages.length);
+      for (const page of result.pages) {
+        expect(page).toHaveProperty("id");
+        expect(page).toHaveProperty("title");
+      }
+
+      // Related block: always an object; buckets that are present are arrays
+      expect(typeof result.related).toBe("object");
+      for (const [bucket, entries] of Object.entries(result.related)) {
+        if (Array.isArray(entries)) {
+          expect(entries.length).toBeGreaterThan(0);
+        } else {
+          // command_node is a single object when present
+          expect(entries).toBeTruthy();
+        }
+        expect(bucket.length).toBeGreaterThan(0);
+      }
+
+      // next_steps: array of hint strings
+      expect(Array.isArray(result.next_steps)).toBe(true);
+    });
+  }
+});

package/src/release.test.ts

@@ -390,9 +390,20 @@ describe("release.yml", () => {
     const src = readText(".github/workflows/release.yml");
     expect(src).toContain("bun run typecheck");
     expect(src).toContain("bun test");
+    expect(src).toContain("bun test src/mcp-contract.test.ts");
     expect(src).toContain("bun run lint");
   });
 
+  test("runs MCP contract tests against the real built DB before eval/release", () => {
+    const src = readText(".github/workflows/release.yml");
+    const contractIdx = src.indexOf("bun test src/mcp-contract.test.ts");
+    const evalIdx = src.indexOf("MCP retrieval eval (Phase 0, non-blocking)");
+    const buildIdx = src.indexOf("Build release artifacts");
+    expect(contractIdx).toBeGreaterThan(0);
+    expect(contractIdx).toBeLessThan(evalIdx);
+    expect(contractIdx).toBeLessThan(buildIdx);
+  });
+
   test("creates GitHub Release", () => {
     const src = readText(".github/workflows/release.yml");
     expect(src).toContain("gh release create");