mixdog 0.9.1 → 0.9.3

package/scripts/recall-bench.mjs

@@ -0,0 +1,207 @@
+#!/usr/bin/env node
+// recall-bench.mjs — case-based recall quality bench against the LIVE memory
+// DB. READ-ONLY: every case goes through handleToolCall('search_memories')
+// which only performs SELECTs (handleSearch path). No mutation actions
+// ('memory' tool / cycle1 / cycle2 / prune / purge / etc.) are ever invoked
+// here — do not add any.
+//
+//   node scripts/recall-bench.mjs [--cases scripts/recall-bench-cases.json] [--json]
+//
+// Loads src/runtime/memory/index.mjs in-process (same module the daemon
+// uses), calls init() once, runs each bench case through handleToolCall,
+// prints params/result-count/top-3/latency/PASS-WARN per case, then a
+// summary table, then calls stop().
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const __dir = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dir, '..');
+const DEFAULT_CASES_PATH = resolve(__dir, 'recall-bench-cases.json');
+
+function argValue(name, fallback = null) {
+  const idx = process.argv.indexOf(`--${name}`);
+  if (idx >= 0 && idx + 1 < process.argv.length) return process.argv[idx + 1];
+  const pref = `--${name}=`;
+  const hit = process.argv.find((a) => a.startsWith(pref));
+  return hit ? hit.slice(pref.length) : fallback;
+}
+function hasFlag(name) { return process.argv.includes(`--${name}`); }
+
+const WARN_LATENCY_MS = 3000;
+
+const DEFAULT_CASES = [
+  { id: 'kw-ko', label: 'keyword query (ko)', args: { query: '메모리 재현' }, expect: 'results' },
+  { id: 'kw-en', label: 'keyword query (en)', args: { query: 'memory recall pipeline' }, expect: 'results' },
+  { id: 'short-1tok', label: 'short 1-token query', args: { query: 'recall' }, expect: 'results' },
+  { id: 'short-2tok', label: 'short 2-token query', args: { query: 'cycle1 drain' }, expect: 'results' },
+  { id: 'browse-last', label: 'query-less recent browse (period=last)', args: { period: 'last', limit: 10 }, expect: 'browse' },
+  { id: 'period-24h', label: 'period window 24h', args: { period: '24h', limit: 10 }, expect: 'browse' },
+  { id: 'period-7d', label: 'period window 7d', args: { period: '7d', limit: 10 }, expect: 'browse' },
+  { id: 'category-filter', label: 'category filter (decision)', args: { period: '30d', category: 'decision', limit: 10 }, expect: 'browse' },
+  { id: 'id-lookup', label: 'id lookup', args: { id: 1 }, expect: 'idlookup' },
+  { id: 'scope-project', label: 'project-scoped query', args: { query: 'recall', cwd: ROOT, limit: 10 }, expect: 'results' },
+  { id: 'scope-all', label: 'all-scope query', args: { query: 'recall', projectScope: 'all', limit: 10 }, expect: 'results' },
+  { id: 'raw-on', label: 'includeRaw on', args: { query: 'recall', includeRaw: true, limit: 10 }, expect: 'results' },
+  { id: 'raw-off', label: 'includeRaw off', args: { query: 'recall', includeRaw: false, limit: 10 }, expect: 'results' },
+];
+
+function loadCases(path) {
+  if (path && existsSync(resolve(path))) {
+    try {
+      const parsed = JSON.parse(readFileSync(resolve(path), 'utf8'));
+      if (Array.isArray(parsed) && parsed.length) return parsed;
+    } catch (e) {
+      process.stderr.write(`[recall-bench] failed to parse cases file ${path}: ${e.message}; using built-in cases\n`);
+    }
+  }
+  return DEFAULT_CASES;
+}
+
+function textOfResult(result) {
+  if (result && typeof result === 'object' && Array.isArray(result.content)) {
+    return result.content.map((p) => (p?.type === 'text' ? p.text || '' : JSON.stringify(p))).join('\n');
+  }
+  if (result && typeof result === 'object' && typeof result.text === 'string') return result.text;
+  if (typeof result === 'string') return result;
+  return JSON.stringify(result ?? '');
+}
+
+function countEntryLines(text) {
+  const t = String(text || '').trim();
+  if (!t || t === '(no results)' || t === '(no valid ids)') return 0;
+  return t.split('\n').filter((line) => line.trim() && !line.startsWith('[recall truncated') && !line.startsWith('note:')).length;
+}
+
+function topN(text, n = 3, maxLen = 140) {
+  const t = String(text || '').trim();
+  if (!t || t === '(no results)') return [];
+  return t.split('\n')
+    .filter((line) => line.trim() && !line.startsWith('[recall truncated') && !line.startsWith('note:'))
+    .slice(0, n)
+    .map((line) => (line.length > maxLen ? `${line.slice(0, maxLen - 1)}…` : line));
+}
+
+function evaluateCase(kase, { count, ms, isError }) {
+  const warnings = [];
+  if (isError) warnings.push('error result');
+  if (ms > WARN_LATENCY_MS) warnings.push(`latency ${ms}ms > ${WARN_LATENCY_MS}ms`);
+  if ((kase.expect === 'browse' || kase.expect === 'idlookup') && count === 0) {
+    warnings.push('0 results for a browse/id-lookup case (expected data present)');
+  }
+  const status = warnings.length ? 'WARN' : 'PASS';
+  return { status, warnings };
+}
+
+async function runCase(memoryModule, kase) {
+  const started = Date.now();
+  let result;
+  let isError = false;
+  let errMsg = null;
+  try {
+    result = await memoryModule.handleToolCall('search_memories', kase.args || {});
+    isError = Boolean(result?.isError);
+  } catch (e) {
+    isError = true;
+    errMsg = e?.message || String(e);
+    result = { text: `(error: ${errMsg})` };
+  }
+  const ms = Date.now() - started;
+  const text = textOfResult(result);
+  const count = countEntryLines(text);
+  const evalResult = evaluateCase(kase, { count, ms, isError });
+  return {
+    id: kase.id,
+    label: kase.label,
+    args: kase.args,
+    count,
+    ms,
+    isError,
+    errMsg,
+    top3: topN(text, 3),
+    status: evalResult.status,
+    warnings: evalResult.warnings,
+  };
+}
+
+function printCase(row) {
+  process.stdout.write(`\n[${row.status}] ${row.id} — ${row.label}\n`);
+  process.stdout.write(`  params: ${JSON.stringify(row.args)}\n`);
+  process.stdout.write(`  results: ${row.count}  latency: ${row.ms}ms${row.isError ? `  ERROR: ${row.errMsg}` : ''}\n`);
+  if (row.top3.length) {
+    for (const line of row.top3) process.stdout.write(`    - ${line}\n`);
+  } else {
+    process.stdout.write('    (no results)\n');
+  }
+  for (const w of row.warnings) process.stdout.write(`  WARN: ${w}\n`);
+}
+
+function printSummary(rows) {
+  process.stdout.write('\n=== recall-bench summary ===\n');
+  const widths = { id: 18, results: 8, ms: 8, status: 6 };
+  process.stdout.write(
+    `${'case'.padEnd(widths.id)}${'results'.padEnd(widths.results)}${'ms'.padEnd(widths.ms)}${'status'.padEnd(widths.status)}notes\n`,
+  );
+  for (const r of rows) {
+    process.stdout.write(
+      `${String(r.id).padEnd(widths.id)}${String(r.count).padEnd(widths.results)}${String(r.ms).padEnd(widths.ms)}${String(r.status).padEnd(widths.status)}${r.warnings.join('; ')}\n`,
+    );
+  }
+  const warnCount = rows.filter((r) => r.status === 'WARN').length;
+  const totalMs = rows.reduce((s, r) => s + r.ms, 0);
+  process.stdout.write(`\ncases=${rows.length} pass=${rows.length - warnCount} warn=${warnCount} total_latency=${totalMs}ms\n`);
+}
+
+async function main() {
+  const casesPath = argValue('cases', DEFAULT_CASES_PATH);
+  const jsonMode = hasFlag('json');
+  const cases = loadCases(casesPath);
+
+  let memoryModule;
+  try {
+    memoryModule = await import(pathToFileURL(resolve(ROOT, 'src/runtime/memory/index.mjs')).href);
+  } catch (e) {
+    process.stderr.write(`[recall-bench] failed to load memory module: ${e?.stack || e?.message || e}\n`);
+    process.exitCode = 1;
+    return;
+  }
+
+  try {
+    await memoryModule.init();
+  } catch (e) {
+    process.stdout.write(`recall-bench: DB unreachable / init failed — reporting clearly, no fabricated results.\n`);
+    process.stdout.write(`error: ${e?.stack || e?.message || e}\n`);
+    process.exitCode = 1;
+    return;
+  }
+
+  const rows = [];
+  try {
+    for (const kase of cases) {
+      const row = await runCase(memoryModule, kase);
+      rows.push(row);
+      if (!jsonMode) printCase(row);
+    }
+  } finally {
+    try { await memoryModule.stop?.(); } catch {}
+  }
+
+  const allZero = rows.length > 0 && rows.every((r) => r.count === 0 && !r.isError);
+  if (allZero) {
+    process.stdout.write('\nNOTE: every case returned 0 results — DB is likely empty (or unreachable pool). Treat WARNs below as expected-empty, not a recall bug, until data is present.\n');
+  }
+
+  if (jsonMode) {
+    process.stdout.write(JSON.stringify({ cases: rows }, null, 2) + '\n');
+  } else {
+    printSummary(rows);
+  }
+
+  const hardErrors = rows.filter((r) => r.isError);
+  if (hardErrors.length) process.exitCode = 1;
+}
+
+main().catch((e) => {
+  process.stderr.write(`${e?.stack || e?.message || e}\n`);
+  process.exitCode = 1;
+});

package/scripts/routing-corpus.mjs

@@ -0,0 +1,281 @@
+#!/usr/bin/env node
+// routing-corpus.mjs — mine the local agent trace for REAL tool-routing
+// patterns and freeze them as a fixed corpus. The corpus (input) is captured
+// ONCE with --save; rule changes are then A/B-evaluated against that frozen
+// set by re-running --eval before/after (input constant, only rules change).
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { basename, resolve } from 'node:path';
+import { resolvePluginData } from '../src/runtime/shared/plugin-paths.mjs';
+
+function argValue(name, fallback = null) {
+  const idx = process.argv.indexOf(name);
+  if (idx >= 0 && idx + 1 < process.argv.length) return process.argv[idx + 1];
+  const pref = `${name}=`;
+  const hit = process.argv.find((a) => a.startsWith(pref));
+  return hit ? hit.slice(pref.length) : fallback;
+}
+function hasFlag(name) { return process.argv.includes(name); }
+
+function parseDuration(value) {
+  const raw = String(value || '').trim();
+  if (!raw) return null;
+  if (/^now$/i.test(raw)) return Date.now();
+  if (/^\d+$/.test(raw)) { const n = Number(raw); return n > 10_000_000_000 ? n : n * 1000; }
+  const rel = raw.match(/^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/i);
+  if (rel) {
+    const n = Number(rel[1]); const u = rel[2].toLowerCase();
+    const mult = u === 'ms' ? 1 : u === 's' ? 1000 : u === 'm' ? 60_000 : u === 'h' ? 3_600_000 : 86_400_000;
+    return Date.now() - n * mult;
+  }
+  const parsed = Date.parse(raw);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function defaultTracePath() {
+  const data = process.env.MIXDOG_DATA_DIR || resolvePluginData() || resolve(homedir(), '.mixdog', 'data');
+  return resolve(data, 'history', 'agent-trace.jsonl');
+}
+
+function readRows(path) {
+  if (!existsSync(path)) return [];
+  const out = [];
+  for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {
+    if (!line) continue;
+    try { out.push(JSON.parse(line)); } catch { /* best-effort */ }
+  }
+  return out;
+}
+function payload(row) { return row?.payload && typeof row.payload === 'object' ? row.payload : {}; }
+function field(row, name) { if (row && row[name] != null) return row[name]; const p = payload(row); return p[name] != null ? p[name] : null; }
+function sessionId(row) { return String(row?.session_id || row?.sessionId || field(row, 'session_id') || ''); }
+function shortId(id) { const s = String(id || ''); return s.length <= 18 ? s : `${s.slice(0, 10)}…${s.slice(-6)}`; }
+function shortModel(m) { const s = String(m || '-'); const i = s.lastIndexOf('/'); return i >= 0 ? s.slice(i + 1) : s; }
+function groupBy(rows, keyFn) {
+  const map = new Map();
+  for (const r of rows) { const k = keyFn(r); if (!map.has(k)) map.set(k, []); map.get(k).push(r); }
+  return map;
+}
+
+function argsSummary(tool, args) {
+  if (!args || typeof args !== 'object') return '';
+  const clip = (v, n = 60) => { const s = String(v ?? '').replace(/\s+/g, ' ').trim(); return s.length > n ? `${s.slice(0, n - 1)}…` : s; };
+  const arr = (v) => Array.isArray(v) ? `[${v.length}:${clip(v[0], 40)}]` : clip(v, 50);
+  switch (tool) {
+    case 'grep': return clip(Array.isArray(args.pattern) ? `[${args.pattern.length}]${args.pattern[0]}` : args.pattern);
+    case 'read': return clip(basename(String(Array.isArray(args.path) ? args.path[0] : args.path || '')));
+    case 'code_graph': return clip(`${args.mode || '?'}:${args.symbol || args.file || ''}`);
+    case 'explore': return arr(args.query);
+    case 'find': return clip(args.query);
+    case 'glob': return clip(Array.isArray(args.pattern) ? args.pattern[0] : args.pattern);
+    case 'list': return clip(basename(String(args.path || '')));
+    case 'apply_patch': return '(edit)';
+    case 'shell': return clip(args.command);
+    default: return clip(JSON.stringify(args), 40);
+  }
+}
+
+function normGrep(args) {
+  const p = args?.pattern;
+  return String(Array.isArray(p) ? p.join('|') : (p ?? '')).trim().toLowerCase();
+}
+
+// shell used to INSPECT the filesystem (read/list/search/exists) instead of a
+// dedicated tool — the anti-pattern the shell description targets. Concept, not
+// an exhaustive list: a command whose leading verb only reads/lists/searches
+// files and does not change state or run a program (git/node/npm/build/test).
+const SHELL_INSPECT_VERB = /(?:^|[\s;|&(]|\bforeach-object\s*\{)\s*(get-content|get-childitem|gci|select-string|cat|type|ls|dir|head|tail|find|findstr|grep|rg|wc|test-path|resolve-path|readlink|realpath|stat)\b/i;
+const SHELL_EXEC_VERB = /(?:^|[\s;|&])\s*(git|node|npm|npx|pnpm|yarn|python|cargo|go|make|docker|rm|remove-item|mkdir|new-item|set-content|out-file|move-item|copy-item|>>|>)\b/i;
+function isShellInspect(command) {
+  const c = String(command || '');
+  if (!c.trim()) return false;
+  // If the command also changes state or runs a program, it is legitimate.
+  if (SHELL_EXEC_VERB.test(c)) return false;
+  return SHELL_INSPECT_VERB.test(c);
+}
+
+function buildCase(sid, toolRows) {
+  const sorted = [...toolRows].sort((a, b) => Number(a.ts || 0) - Number(b.ts || 0));
+  const agent = field(sorted[0], 'agent') || field(sorted.find((r) => field(r, 'agent')), 'agent') || null;
+  const model = field(sorted[0], 'model') || null;
+  const iters = new Set();
+  const sequence = [];
+  for (const r of sorted) {
+    const it = field(r, 'iteration');
+    if (it != null) iters.add(it);
+    const tool = field(r, 'tool_name') || '?';
+    sequence.push({ it: it ?? null, tool, args: argsSummary(tool, field(r, 'tool_args')), rawArgs: field(r, 'tool_args') });
+  }
+  // per-iteration tool counts for serial detection
+  const perIt = new Map();
+  for (const s of sequence) { if (s.it == null) continue; perIt.set(s.it, (perIt.get(s.it) || 0) + 1); }
+  const itOrder = [...perIt.keys()].sort((a, b) => a - b);
+  let longestSingleRun = 0, run = 0;
+  for (const it of itOrder) { if (perIt.get(it) === 1) { run += 1; longestSingleRun = Math.max(longestSingleRun, run); } else run = 0; }
+
+  const names = sequence.map((s) => s.tool);
+  const hasAnchor = names.some((n) => n === 'grep' || n === 'code_graph' || n === 'find' || n === 'read');
+  const exploreCalls = sequence.filter((s) => s.tool === 'explore');
+  const flags = [];
+  if (exploreCalls.length && hasAnchor) flags.push('explore_overuse');
+  if (exploreCalls.some((s) => Array.isArray(s.rawArgs?.query))) flags.push('explore_multiquery');
+  if (names[0] === 'explore') flags.push('explore_first');
+  if (sequence.some((s) => s.tool === 'code_graph' && s.rawArgs?.mode === 'find_symbol' && !s.rawArgs?.file)) flags.push('find_symbol_noscope');
+  if (longestSingleRun >= 4) flags.push('serial_single_tool');
+  // read fragmentation
+  const readCounts = new Map();
+  for (const s of sequence) { if (s.tool !== 'read') continue; const k = String(s.rawArgs?.path ?? s.args); readCounts.set(k, (readCounts.get(k) || 0) + 1); }
+  if ([...readCounts.values()].some((c) => c >= 3)) flags.push('read_fragmentation');
+  // grep retry
+  const grepCounts = new Map();
+  for (const s of sequence) { if (s.tool !== 'grep') continue; const k = normGrep(s.rawArgs); if (!k) continue; grepCounts.set(k, (grepCounts.get(k) || 0) + 1); }
+  if ([...grepCounts.values()].some((c) => c >= 2)) flags.push('grep_retry');
+  // shell used for filesystem inspection instead of dedicated tools
+  if (sequence.some((s) => s.tool === 'shell' && isShellInspect(s.rawArgs?.command))) flags.push('shell_inspect');
+
+  return {
+    session_id: sid,
+    short_id: shortId(sid),
+    agent, model,
+    turns: iters.size || itOrder.length,
+    tools: sequence.length,
+    first_tool: names[0] || null,
+    longest_single_run: longestSingleRun,
+    flags,
+    max_ts: Math.max(...sorted.map((r) => Number(r.ts || 0))),
+    sequence: sequence.map((s) => ({ it: s.it, tool: s.tool, args: s.args })),
+  };
+}
+
+const FLAG_KEYS = ['explore_overuse', 'explore_multiquery', 'explore_first', 'find_symbol_noscope', 'serial_single_tool', 'read_fragmentation', 'grep_retry', 'shell_inspect'];
+
+function buildCorpus(rows, { limit, sinceTs, agentFilter }) {
+  let filtered = rows.filter((r) => r.kind === 'tool' && sessionId(r));
+  if (sinceTs != null) filtered = filtered.filter((r) => Number(r.ts || 0) >= sinceTs);
+  if (agentFilter) { const q = String(agentFilter).toLowerCase(); filtered = filtered.filter((r) => String(field(r, 'agent') || '').toLowerCase().includes(q)); }
+  const bySession = groupBy(filtered, sessionId);
+  const cases = [...bySession.entries()].map(([sid, tr]) => buildCase(sid, tr))
+    .sort((a, b) => Number(b.max_ts || 0) - Number(a.max_ts || 0));
+  const emitted = cases.slice(0, limit);
+  const summary = {};
+  for (const k of FLAG_KEYS) {
+    const count = emitted.filter((c) => c.flags.includes(k)).length;
+    summary[k] = { count, pct: emitted.length ? Math.round((count / emitted.length) * 100) : 0 };
+  }
+  return { scanned: bySession.size, emitted: emitted.length, summary, cases: emitted };
+}
+
+function scoreCorpus(cases) {
+  // A single健全 index: sum of flagged cases per KPI (lower is better).
+  const summary = {};
+  let flaggedCases = 0;
+  for (const k of FLAG_KEYS) summary[k] = cases.filter((c) => c.flags.includes(k)).length;
+  for (const c of cases) if (c.flags.length) flaggedCases += 1;
+  return { total: cases.length, flagged_cases: flaggedCases, clean_pct: cases.length ? Math.round(((cases.length - flaggedCases) / cases.length) * 100) : 0, by_flag: summary };
+}
+
+function renderText(corpus, showCases) {
+  const L = [];
+  L.push(`routing corpus: scanned ${corpus.scanned} sessions, emitted ${corpus.emitted} cases`);
+  L.push('anti-pattern summary:');
+  for (const k of FLAG_KEYS) L.push(`- ${k.padEnd(20)} ${String(corpus.summary[k].count).padStart(3)} (${corpus.summary[k].pct}%)`);
+  if (showCases) {
+    L.push('');
+    corpus.cases.forEach((c, i) => {
+      L.push(`#${i + 1} agent=${c.agent || '-'} model=${shortModel(c.model)} turns=${c.turns} tools=${c.tools} flags=[${c.flags.join(',')}] sess=${c.short_id}`);
+      const shown = c.sequence.slice(0, 8).map((s) => `${s.tool}(${s.args})`).join(' → ');
+      const more = c.sequence.length > 8 ? ` +${c.sequence.length - 8} more` : '';
+      L.push(`  seq: ${shown}${more}`);
+    });
+  }
+  return L.join('\n');
+}
+
+// Extract one case by session id (or short-id prefix) from the current trace.
+function extractCaseById(rows, wanted) {
+  const q = String(wanted || '');
+  const tr = rows.filter((r) => r.kind === 'tool' && sessionId(r) && (sessionId(r) === q || sessionId(r).startsWith(q) || shortId(sessionId(r)).startsWith(q)));
+  if (!tr.length) return null;
+  const sid = sessionId(tr[0]);
+  return buildCase(sid, tr.filter((r) => sessionId(r) === sid));
+}
+
+// tool-name -> count for a case (the tool mix).
+function toolMix(cas) {
+  const mix = {};
+  for (const s of cas.sequence) mix[s.tool] = (mix[s.tool] || 0) + 1;
+  return mix;
+}
+
+function renderVs(before, after) {
+  const L = [];
+  const fmtMix = (m) => Object.entries(m).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join(' ') || '(none)';
+  L.push(`routing A/B  before=${before.short_id} (${before.agent || '-'}/${shortModel(before.model)})  after=${after.short_id} (${after.agent || '-'}/${shortModel(after.model)})`);
+  L.push('');
+  const metric = (label, b, a, lowerBetter = true) => {
+    const d = a - b; const arrow = d === 0 ? '=' : (lowerBetter ? (d < 0 ? 'v' : '^') : (d > 0 ? 'v' : '^'));
+    L.push(`- ${label.padEnd(14)} ${String(b).padStart(5)} -> ${String(a).padStart(5)}  (${d > 0 ? '+' : ''}${d}) ${arrow}`);
+  };
+  metric('turns', before.turns, after.turns);
+  metric('tools', before.tools, after.tools);
+  metric('flags', before.flags.length, after.flags.length);
+  L.push(`  before tool-mix: ${fmtMix(toolMix(before))}`);
+  L.push(`  after  tool-mix: ${fmtMix(toolMix(after))}`);
+  const gained = after.flags.filter((f) => !before.flags.includes(f));
+  const cleared = before.flags.filter((f) => !after.flags.includes(f));
+  if (cleared.length) L.push(`  cleared flags: ${cleared.join(', ')}`);
+  if (gained.length) L.push(`  NEW flags: ${gained.join(', ')}`);
+  if (!cleared.length && !gained.length) L.push(`  flags unchanged: [${before.flags.join(', ') || 'none'}]`);
+  return L.join('\n');
+}
+
+// ---- main ----
+const tracePath = argValue('--trace') ? resolve(argValue('--trace')) : defaultTracePath();
+const sinceTs = parseDuration(argValue('--since', null));
+const limit = Number.parseInt(argValue('--limit', '50'), 10) || 50;
+const agentFilter = argValue('--agent', null);
+const jsonMode = hasFlag('--json');
+const savePath = argValue('--save', null);   // freeze the corpus (session id list + cases) to a file
+const evalPath = argValue('--eval', null);   // re-score a frozen corpus's sessions against CURRENT trace
+const vsIdx = process.argv.indexOf('--vs');  // --vs <before-sid> <after-sid>: routing diff
+
+if (vsIdx >= 0) {
+  const beforeId = process.argv[vsIdx + 1];
+  const afterId = process.argv[vsIdx + 2];
+  if (!beforeId || !afterId) { console.error('usage: --vs <before-session> <after-session>'); process.exit(1); }
+  const rows = readRows(tracePath);
+  const before = extractCaseById(rows, beforeId);
+  const after = extractCaseById(rows, afterId);
+  if (!before) { console.error(`before session not found: ${beforeId}`); process.exit(1); }
+  if (!after) { console.error(`after session not found: ${afterId}`); process.exit(1); }
+  if (jsonMode) console.log(JSON.stringify({ before, after }, null, 2));
+  else console.log(renderVs(before, after));
+  process.exit(0);
+}
+
+if (evalPath) {
+  // A/B: load frozen corpus, re-extract those exact sessions from the current
+  // trace, and score. (Sessions are immutable once finished, so this is stable;
+  // the value is comparing two frozen corpora captured before/after a rule change.)
+  const frozen = JSON.parse(readFileSync(resolve(evalPath), 'utf8'));
+  const wantIds = new Set((frozen.cases || []).map((c) => c.session_id));
+  const rows = readRows(tracePath).filter((r) => r.kind === 'tool' && wantIds.has(sessionId(r)));
+  const bySession = groupBy(rows, sessionId);
+  const cases = [...bySession.entries()].map(([sid, tr]) => buildCase(sid, tr));
+  const score = scoreCorpus(cases);
+  if (jsonMode) console.log(JSON.stringify({ eval: resolve(evalPath), ...score }, null, 2));
+  else {
+    console.log(`routing eval of frozen corpus (${resolve(evalPath)})`);
+    console.log(`cases=${score.total} flagged=${score.flagged_cases} clean=${score.clean_pct}%`);
+    for (const k of FLAG_KEYS) console.log(`- ${k.padEnd(20)} ${score.by_flag[k]}`);
+  }
+  process.exit(0);
+}
+
+const corpus = buildCorpus(readRows(tracePath), { limit, sinceTs, agentFilter });
+if (savePath) {
+  writeFileSync(resolve(savePath), JSON.stringify(corpus, null, 2));
+  console.error(`saved ${corpus.emitted} cases → ${resolve(savePath)}`);
+}
+if (jsonMode) console.log(JSON.stringify(corpus, null, 2));
+else console.log(renderText(corpus, true));