@claude-flow/cli 3.21.0 → 3.22.0

package/scripts/beir-bootstrap-significance.mjs

@@ -1,162 +0,0 @@
-#!/usr/bin/env node
-// beir-bootstrap-significance.mjs — paired bootstrap significance test for
-// BEIR retrieval results (ADR-086).
-//
-// Given a run JSON with per-query nDCG@10 and a baseline mean nDCG@10,
-// estimates whether our result is significantly above the baseline.
-//
-// Method: paired bootstrap — resample N=10000 with replacement from our
-// per-query nDCG scores, compute mean each time, take 2.5%-97.5% percentile
-// for 95% CI. Compare baseline to our CI: if baseline < CI lower bound,
-// the difference is significant at p < 0.05.
-//
-// For a paired test against ANOTHER per-query run (e.g. our pure-BM25 vs
-// our BGE-base), resample paired DIFFERENCES instead. Pass --paired
-// path/to/baseline-run.json.
-//
-// Usage:
-//   node scripts/beir-bootstrap-significance.mjs <run-json-path>
-//   node scripts/beir-bootstrap-significance.mjs <ours> --paired <baseline>
-//
-// Example:
-//   node scripts/beir-bootstrap-significance.mjs \
-//     docs/benchmarks/runs/beir-nfcorpus-bge-latest.json
-
-import { readFileSync } from 'node:fs';
-import { performance } from 'node:perf_hooks';
-
-const args = process.argv.slice(2);
-if (args.length === 0) {
-  console.error('Usage: node beir-bootstrap-significance.mjs <run-json> [--paired <baseline-run-json>]');
-  process.exit(1);
-}
-const RUN_PATH = args[0];
-const PAIRED_IDX = args.indexOf('--paired');
-const PAIRED_PATH = PAIRED_IDX > 0 ? args[PAIRED_IDX + 1] : null;
-const ITERATIONS = Number(process.env.BOOTSTRAP_ITERATIONS) || 10000;
-
-// Deterministic PRNG (Mulberry32) — so re-runs are reproducible.
-const SEED = Number(process.env.BOOTSTRAP_SEED) || 42;
-function mulberry32(a) {
-  return function () {
-    a |= 0; a = (a + 0x6D2B79F5) | 0;
-    let t = a;
-    t = Math.imul(t ^ (t >>> 15), t | 1);
-    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
-}
-
-function mean(arr) { let s = 0; for (const v of arr) s += v; return s / arr.length; }
-
-function percentile(sortedArr, p) {
-  const idx = Math.max(0, Math.min(sortedArr.length - 1, Math.floor(p * sortedArr.length)));
-  return sortedArr[idx];
-}
-
-function pointEstimate(scores) {
-  return {
-    mean: mean(scores),
-    n: scores.length,
-  };
-}
-
-function bootstrapCI(scores, iterations, rng, percentiles = [0.025, 0.975]) {
-  const means = new Float64Array(iterations);
-  const n = scores.length;
-  for (let i = 0; i < iterations; i++) {
-    let s = 0;
-    for (let j = 0; j < n; j++) s += scores[Math.floor(rng() * n)];
-    means[i] = s / n;
-  }
-  const sorted = Array.from(means).sort((a, b) => a - b);
-  return {
-    point: mean(scores),
-    lower: percentile(sorted, percentiles[0]),
-    upper: percentile(sorted, percentiles[1]),
-    p_value_vs_baseline_zero: null,
-  };
-}
-
-// Paired bootstrap: compare our per-query scores to a baseline's per-query
-// scores. Resample (idx) and average paired differences. The 95% CI on the
-// mean difference tells us if it's significantly non-zero.
-function pairedBootstrap(oursByQid, baselineByQid, iterations, rng) {
-  const sharedIds = [...oursByQid.keys()].filter((id) => baselineByQid.has(id));
-  const diffs = sharedIds.map((id) => oursByQid.get(id) - baselineByQid.get(id));
-  const n = diffs.length;
-  const means = new Float64Array(iterations);
-  for (let i = 0; i < iterations; i++) {
-    let s = 0;
-    for (let j = 0; j < n; j++) s += diffs[Math.floor(rng() * n)];
-    means[i] = s / n;
-  }
-  const sorted = Array.from(means).sort((a, b) => a - b);
-  // One-sided p ≈ fraction of resamples where mean diff ≤ 0
-  let negCount = 0;
-  for (const m of means) if (m <= 0) negCount++;
-  return {
-    n_paired: n,
-    point_diff: mean(diffs),
-    lower_95: percentile(sorted, 0.025),
-    upper_95: percentile(sorted, 0.975),
-    p_one_sided: negCount / iterations,
-  };
-}
-
-function loadRun(path) {
-  const data = JSON.parse(readFileSync(path, 'utf-8'));
-  if (!data.perQuery || !Array.isArray(data.perQuery)) {
-    console.error(`Run JSON at ${path} has no perQuery array — re-run the bench with the updated harness.`);
-    process.exit(2);
-  }
-  return data;
-}
-
-const ours = loadRun(RUN_PATH);
-console.log(`# BEIR significance test (ADR-086)`);
-console.log(`Run: ${RUN_PATH}`);
-console.log(`Dataset: ${ours.dataset ?? '?'}  Model: ${ours.model ?? '?'}  Queries: ${ours.perQuery.length}`);
-console.log(`Bootstrap: ${ITERATIONS} iterations, seed=${SEED}`);
-
-const rng = mulberry32(SEED);
-const ourScores = ours.perQuery.map((q) => q.ndcg10);
-
-const t0 = performance.now();
-const oursCI = bootstrapCI(ourScores, ITERATIONS, rng);
-console.log(`\n=== Our nDCG@10 (1-sample bootstrap CI) ===`);
-console.log(`  point:    ${oursCI.point.toFixed(4)}`);
-console.log(`  95% CI:   [${oursCI.lower.toFixed(4)}, ${oursCI.upper.toFixed(4)}]`);
-
-if (ours.baselines) {
-  console.log(`\n=== vs each published baseline (CI overlap) ===`);
-  for (const [name, score] of Object.entries(ours.baselines)) {
-    const ciDiff = oursCI.point - score;
-    const lowerDiff = oursCI.lower - score;
-    const upperDiff = oursCI.upper - score;
-    const direction = ciDiff > 0 ? '↑ above' : '↓ below';
-    const sig = (lowerDiff > 0 && upperDiff > 0) ? 'p<0.05 (95% CI excludes baseline)'
-              : (lowerDiff < 0 && upperDiff < 0) ? 'p<0.05 (95% CI below baseline)'
-              : 'n.s. (95% CI overlaps baseline)';
-    console.log(`  ${score.toFixed(3)}  ${name.padEnd(28)}  Δ=${ciDiff >= 0 ? '+' : ''}${ciDiff.toFixed(4)}  ${direction}  [${sig}]`);
-  }
-}
-
-if (PAIRED_PATH) {
-  const baseline = loadRun(PAIRED_PATH);
-  console.log(`\n=== Paired bootstrap vs ${PAIRED_PATH} ===`);
-  const oursByQid = new Map(ours.perQuery.map((q) => [q.qid, q.ndcg10]));
-  const baseByQid = new Map(baseline.perQuery.map((q) => [q.qid, q.ndcg10]));
-  const rng2 = mulberry32(SEED + 1);
-  const paired = pairedBootstrap(oursByQid, baseByQid, ITERATIONS, rng2);
-  console.log(`  paired queries: ${paired.n_paired}`);
-  console.log(`  Δ (ours - baseline): ${paired.point_diff >= 0 ? '+' : ''}${paired.point_diff.toFixed(4)}`);
-  console.log(`  95% CI: [${paired.lower_95.toFixed(4)}, ${paired.upper_95.toFixed(4)}]`);
-  console.log(`  one-sided p (Δ ≤ 0): ${paired.p_one_sided.toFixed(4)}`);
-  const sig = paired.lower_95 > 0 ? 'SIGNIFICANT improvement (p<0.025 one-sided)' :
-              paired.upper_95 < 0 ? 'SIGNIFICANT regression (p<0.025 one-sided)' :
-              'NOT SIGNIFICANT — CI overlaps zero';
-  console.log(`  verdict: ${sig}`);
-}
-
-console.log(`\nBootstrap took ${((performance.now() - t0) / 1000).toFixed(1)}s.`);

package/scripts/bench-rvagent.mjs

@@ -1,312 +0,0 @@
-#!/usr/bin/env node
-/**
- * ADR-129 rvagent benchmark suite.
- *
- * Measures the four performance wins introduced by ADR-129 Phases 1–4:
- *
- *   P1 — Provider routing latency (JsModelProvider wiring).
- *        Compares echo-stub round-trip (keyless baseline) against provider
- *        attachment overhead (measured with a fake key that will 401 — we
- *        capture the latency to the WASM call boundary, not the network hop).
- *
- *   P2 — wasm_agent_compose throughput with N=10/50/100 MCP tools.
- *        Shows cost of allowlist validation + descriptor building at scale.
- *
- *   P3 — Gallery CRUD throughput: add → list → import → export → remove.
- *        Validates that the 10 new gallery operations land in <1 ms each.
- *
- *   P4 — Plugin enumeration overhead: wasm_agent_compose with/without
- *        includePlugins. Measures manifest-lookup cost when the plugin
- *        directory does not exist (graceful no-op path).
- *
- * Output: docs/benchmarks/rvagent-baseline.json
- *
- * Usage:
- *   node v3/@claude-flow/cli/scripts/bench-rvagent.mjs [--tag=baseline] [--trials=5]
- *
- * Pattern: mirrors v3/@claude-flow/guidance/scripts/bench-phase-1.mjs.
- * Standalone — does NOT require the test runner, only Node ≥20 + the WASM
- * package (@ruvector/rvagent-wasm must be installed for realistic numbers;
- * the bench gracefully degrades to timing the WASM-unavailable error path).
- */
-
-import { performance } from 'node:perf_hooks';
-import { mkdirSync, writeFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { dirname, resolve } from 'node:path';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const REPO_ROOT = resolve(__dirname, '../../../..');
-const OUT_DIR = resolve(REPO_ROOT, 'docs', 'benchmarks');
-
-const args = Object.fromEntries(
-  process.argv.slice(2).map(a => {
-    const [k, v] = a.replace(/^--/, '').split('=');
-    return [k, v ?? true];
-  }),
-);
-
-const TAG = args.tag || 'baseline';
-const TRIALS = Math.max(3, parseInt(args.trials || '5', 10));
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Timing harness — 5-trial median, same as bench-phase-1.mjs
-// ─────────────────────────────────────────────────────────────────────────────
-
-/**
- * Time N async calls (or sync if fn returns non-Promise), return median ms.
- */
-async function bench(name, fn, reps = 20) {
-  // Warmup — trigger any lazy init so it doesn't inflate the first trial.
-  for (let i = 0; i < Math.min(3, reps); i++) {
-    try { await fn(); } catch { /* ignore warmup errors */ }
-  }
-
-  const latencies = [];
-  for (let t = 0; t < TRIALS; t++) {
-    const start = performance.now();
-    for (let i = 0; i < reps; i++) {
-      try { await fn(); } catch { /* measure call overhead, not success */ }
-    }
-    latencies.push((performance.now() - start) / reps);
-  }
-  latencies.sort((a, b) => a - b);
-  const median = latencies[Math.floor(TRIALS / 2)];
-  const min = latencies[0];
-  const max = latencies[TRIALS - 1];
-
-  return {
-    name,
-    trials: TRIALS,
-    reps,
-    medianMs: Math.round(median * 1000) / 1000,
-    minMs: Math.round(min * 1000) / 1000,
-    maxMs: Math.round(max * 1000) / 1000,
-    variance: Math.round(((max - min) / (median || 1)) * 1000) / 1000,
-  };
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Load the live agent-wasm module (dist preferred, src fallback)
-// ─────────────────────────────────────────────────────────────────────────────
-
-// dist layout: dist/src/**/*.js (tsc rootDir=.)
-const DIST_SRC = resolve(__dirname, '../dist/src');
-let wasmMod;
-try {
-  wasmMod = await import(resolve(DIST_SRC, 'ruvector/agent-wasm.js'));
-} catch {
-  console.warn('[bench] Could not load agent-wasm dist — using stub metrics only.');
-  wasmMod = null;
-}
-
-// Load the wasm tools module to bench wasm_agent_compose handler directly
-let composeHandler;
-let galleryHandlers = {};
-try {
-  const toolsMod = await import(resolve(DIST_SRC, 'mcp-tools/wasm-agent-tools.js'));
-  const tools = toolsMod.wasmAgentTools ?? toolsMod.default ?? [];
-  const findHandler = (name) => tools.find(t => t.name === name)?.handler;
-  composeHandler = findHandler('wasm_agent_compose');
-  galleryHandlers = {
-    add: findHandler('wasm_gallery_add_custom'),
-    list: findHandler('wasm_gallery_list'),
-    importFn: findHandler('wasm_gallery_import'),
-    exportFn: findHandler('wasm_gallery_export'),
-    remove: findHandler('wasm_gallery_remove_custom'),
-    categories: findHandler('wasm_gallery_categories'),
-    listByCategory: findHandler('wasm_gallery_list_by_category'),
-    loadRvf: findHandler('wasm_gallery_load_rvf'),
-    active: findHandler('wasm_gallery_active'),
-    config: findHandler('wasm_gallery_config'),
-  };
-} catch {
-  // Handlers unavailable — bench will time the missing-module error path.
-  composeHandler = null;
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// P1 — Provider routing latency
-//
-// Baseline: echo-stub path (no keys set) — measures raw WASM call overhead.
-// Provider: createWasmAgent with a fake ANTHROPIC_API_KEY. The key is 401'd
-// by the Anthropic API before any tokens are billed, but the routing
-// logic (attachJsModelProvider, JsModelProvider instantiation, provider
-// lookup) runs synchronously before the network call. We measure the
-// in-process portion only by timing createWasmAgent + promptWasmAgent
-// with a deliberately short input and catching the 401 network error.
-// ─────────────────────────────────────────────────────────────────────────────
-
-const originalKey = process.env.ANTHROPIC_API_KEY;
-
-console.log('\nADR-129 rvagent benchmark suite');
-console.log('================================');
-console.log(`tag=${TAG}  trials=${TRIALS}  node=${process.version}`);
-console.log('');
-
-// P1a — echo-stub baseline (no key)
-console.log('P1a: provider routing — echo-stub baseline (keyless)...');
-delete process.env.ANTHROPIC_API_KEY;
-delete process.env.OPENROUTER_API_KEY;
-delete process.env.OLLAMA_API_KEY;
-
-const r_p1_echo = await bench('P1-echo-stub: createWasmAgent (keyless)', async () => {
-  if (!wasmMod) return; // module unavailable — timing the absence
-  try {
-    const info = await wasmMod.createWasmAgent({ maxTurns: 1 });
-    wasmMod.terminateWasmAgent(info.id);
-  } catch { /* WASM unavailable is expected in keyless CI */ }
-}, 5);
-
-// P1b — provider-path: fake key triggers attachJsModelProvider logic
-console.log('P1b: provider routing — with fake key (measures attach overhead)...');
-process.env.ANTHROPIC_API_KEY = 'sk-ant-bench-fake-key-00000000000000000000000000';
-
-const r_p1_provider = await bench('P1-provider-path: createWasmAgent (fake key)', async () => {
-  if (!wasmMod) return;
-  try {
-    const info = await wasmMod.createWasmAgent({ maxTurns: 1 });
-    wasmMod.terminateWasmAgent(info.id);
-  } catch { /* 401 expected */ }
-}, 5);
-
-// Restore original key state
-if (originalKey !== undefined) {
-  process.env.ANTHROPIC_API_KEY = originalKey;
-} else {
-  delete process.env.ANTHROPIC_API_KEY;
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// P2 — wasm_agent_compose throughput with N=10/50/100 MCP tools
-// ─────────────────────────────────────────────────────────────────────────────
-
-console.log('P2: compose throughput — N=10/50/100 MCP tools...');
-
-function makeMcpToolList(n) {
-  return Array.from({ length: n }, (_, i) => `memory_search_${i % 30 === 0 ? i : 'search'}`);
-}
-
-async function timeCompose(toolCount) {
-  if (!composeHandler) return null;
-  const tools = makeMcpToolList(toolCount);
-  return bench(`P2-compose: ${toolCount} MCP tools`, async () => {
-    await composeHandler({ mcpTools: tools, skills: [], prompts: [], tools: [] });
-  }, 10);
-}
-
-const r_p2_10 = await timeCompose(10);
-const r_p2_50 = await timeCompose(50);
-const r_p2_100 = await timeCompose(100);
-
-// ─────────────────────────────────────────────────────────────────────────────
-// P3 — Gallery CRUD throughput: full add → list → import → export → remove cycle
-// ─────────────────────────────────────────────────────────────────────────────
-
-console.log('P3: gallery CRUD — add/list/import/export/remove cycle...');
-
-const CUSTOM_TEMPLATE = JSON.stringify({
-  id: 'bench-custom-1',
-  name: 'bench-template',
-  description: 'Benchmark custom template',
-  category: 'testing',
-  tags: ['bench'],
-  version: '0.1.0',
-  author: 'bench',
-  builtin: false,
-  tools: [],
-  prompts: [],
-  skills: [],
-  mcp_tools: [],
-  capabilities: [],
-});
-
-const r_p3_cycle = await bench('P3-gallery-crud: add→list→import→export→remove', async () => {
-  const fns = galleryHandlers;
-  if (fns.add) await fns.add({ template: JSON.parse(CUSTOM_TEMPLATE) }).catch(() => {});
-  if (fns.list) await fns.list({}).catch(() => {});
-  if (fns.importFn) await fns.importFn({ templatesJson: `[${CUSTOM_TEMPLATE}]` }).catch(() => {});
-  if (fns.exportFn) await fns.exportFn({}).catch(() => {});
-  if (fns.remove) await fns.remove({ id: 'bench-custom-1' }).catch(() => {});
-}, 5);
-
-const r_p3_categories = await bench('P3-gallery-categories: getCategories', async () => {
-  if (galleryHandlers.categories) await galleryHandlers.categories({}).catch(() => {});
-}, 10);
-
-// ─────────────────────────────────────────────────────────────────────────────
-// P4 — Plugin enumeration: compose with/without includePlugins
-//
-// Tests the manifest-lookup path for plugins that don't exist on disk
-// (graceful no-op / warning). This measures that skipping absent plugins
-// doesn't add significant overhead.
-// ─────────────────────────────────────────────────────────────────────────────
-
-console.log('P4: plugin enumeration — compose with/without includePlugins...');
-
-const r_p4_without = await bench('P4-plugin-enum: compose WITHOUT includePlugins', async () => {
-  if (!composeHandler) return;
-  await composeHandler({ mcpTools: ['memory_search'], skills: [], prompts: [], tools: [] });
-}, 10);
-
-const r_p4_with = await bench('P4-plugin-enum: compose WITH includePlugins (absent plugins)', async () => {
-  if (!composeHandler) return;
-  await composeHandler({
-    mcpTools: ['memory_search'],
-    includePlugins: ['nonexistent-plugin-a', 'nonexistent-plugin-b'],
-    skills: [],
-    prompts: [],
-    tools: [],
-  });
-}, 10);
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Emit results
-// ─────────────────────────────────────────────────────────────────────────────
-
-const results = [
-  r_p1_echo,
-  r_p1_provider,
-  r_p2_10,
-  r_p2_50,
-  r_p2_100,
-  r_p3_cycle,
-  r_p3_categories,
-  r_p4_without,
-  r_p4_with,
-].filter(Boolean);
-
-const out = {
-  tag: TAG,
-  trials: TRIALS,
-  node: process.version,
-  platform: `${process.platform}-${process.arch}`,
-  capturedAt: new Date().toISOString(),
-  adr: 'ADR-129',
-  phases: ['P1-provider-routing', 'P2-compose-throughput', 'P3-gallery-crud', 'P4-plugin-enum'],
-  results,
-};
-
-mkdirSync(OUT_DIR, { recursive: true });
-const outPath = resolve(OUT_DIR, `rvagent-${TAG}.json`);
-writeFileSync(outPath, JSON.stringify(out, null, 2));
-
-console.log(`\nWrote ${outPath}\n`);
-
-const COL_NAME = 52;
-const COL_MS = 10;
-const COL_VAR = 8;
-
-console.log(`| ${'Benchmark'.padEnd(COL_NAME)} | ${'Median ms'.padStart(COL_MS)} | ${'Variance'.padStart(COL_VAR)} |`);
-console.log(`|${'-'.repeat(COL_NAME + 2)}|${'-'.repeat(COL_MS + 2)}|${'-'.repeat(COL_VAR + 2)}|`);
-
-for (const r of results) {
-  const name = r.name.padEnd(COL_NAME).slice(0, COL_NAME);
-  const ms = String(r.medianMs).padStart(COL_MS);
-  const vari = String(r.variance).padStart(COL_VAR);
-  console.log(`| ${name} | ${ms} | ${vari} |`);
-}
-
-console.log('');
-console.log('Note: ms values measure in-process overhead including WASM unavailable error path.');
-console.log('When @ruvector/rvagent-wasm is not installed, timings reflect the import error cost only.');

package/scripts/benchmark-codemods.mjs

@@ -1,127 +0,0 @@
-#!/usr/bin/env node
-// benchmark-codemods.mjs — measured benchmark for deterministic Tier-1 codemods (ADR-143).
-//
-// Runs every case in bench/codemod-corpus.json through applyCodemod() and records:
-//   - correctness vs the golden `expected` output
-//   - latency (avg / p50 / p99 / max)
-//   - cost: $0 measured (no API call) + estimated savings vs an LLM edit
-//
-// Writes a run JSON (tagged summary.benchmark="codemod-tier1") into the
-// cost-tracker plugin's runs dir, which is exactly what `cost-trend` reads:
-//   plugins/ruflo-cost-tracker/docs/benchmarks/runs/codemod-<timestamp>.json
-// View the series with: BENCH_NAME=codemod-tier1 node scripts/trend.mjs
-//
-// Usage:
-//   node scripts/benchmark-codemods.mjs            # build dist first (npm run build)
-//   BENCH_JSON=1 node scripts/benchmark-codemods.mjs   # machine-readable JSON to stdout
-//   BENCH_NO_WRITE=1 node scripts/benchmark-codemods.mjs  # don't write a run file
-
-import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { dirname, join, resolve } from 'node:path';
-import { performance } from 'node:perf_hooks';
-
-const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
-const CLI_ROOT = resolve(SCRIPT_DIR, '..');
-const REPO_ROOT = resolve(SCRIPT_DIR, '../../../..');
-// Write to the cost-tracker plugin's runs dir — the exact path `cost-trend` reads.
-const RUNS_DIR = join(REPO_ROOT, 'plugins', 'ruflo-cost-tracker', 'docs', 'benchmarks', 'runs');
-
-// Estimated per-edit LLM cost (USD). Documented estimates, NOT a live call —
-// used only to express the savings of a $0 deterministic codemod.
-const LLM_EDIT_COST = { haiku: 0.0002, sonnet: 0.003, opus: 0.015 };
-
-function percentile(sorted, p) {
-  if (sorted.length === 0) return 0;
-  const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
-  return sorted[idx];
-}
-
-async function main() {
-  const { applyCodemod } = await import('../dist/src/ruvector/codemods/engine.js');
-  const corpus = JSON.parse(readFileSync(join(CLI_ROOT, 'bench', 'codemod-corpus.json'), 'utf-8'));
-  const cases = corpus.cases;
-
-  const results = [];
-  const latencies = [];
-  let correct = 0;
-
-  for (const c of cases) {
-    const t0 = performance.now();
-    const r = applyCodemod(c.intent, c.code, { language: c.language });
-    const latencyMs = performance.now() - t0;
-    latencies.push(latencyMs);
-    const ok = r.success && r.output === c.expected;
-    if (ok) correct++;
-    results.push({
-      id: c.id, intent: c.intent, correct: ok, changed: r.changed,
-      edits: r.edits, latencyMs: Number(latencyMs.toFixed(4)),
-      ...(ok ? {} : { gotPrefix: (r.output ?? '').slice(0, 80), reason: r.reason }),
-    });
-  }
-
-  const sorted = [...latencies].sort((a, b) => a - b);
-  const avg = latencies.reduce((s, x) => s + x, 0) / latencies.length;
-  const winRate = correct / cases.length;
-  const estSavings = {
-    vsHaiku: Number((LLM_EDIT_COST.haiku * cases.length).toFixed(6)),
-    vsSonnet: Number((LLM_EDIT_COST.sonnet * cases.length).toFixed(6)),
-    vsOpus: Number((LLM_EDIT_COST.opus * cases.length).toFixed(6)),
-  };
-
-  const summary = {
-    runAt: new Date().toISOString(),
-    benchmark: 'codemod-tier1',
-    corpusVersion: corpus.version,
-    corpusSize: cases.length,
-    winRate,
-    winRatePct: `${(winRate * 100).toFixed(1)}%`,
-    successCount: correct,
-    avgLatencyMs: Number(avg.toFixed(4)),
-    p50LatencyMs: Number(percentile(sorted, 50).toFixed(4)),
-    p99LatencyMs: Number(percentile(sorted, 99).toFixed(4)),
-    maxLatencyMs: Number(Math.max(...latencies).toFixed(4)),
-    structuralCostUsd: 0,
-    llmBaseline: 'estimated',
-    estimatedSavingsUsd: estSavings,
-  };
-
-  const run = { summary, results };
-
-  if (process.env.BENCH_JSON) {
-    console.log(JSON.stringify(run, null, 2));
-  } else {
-    console.log(`# Codemod Tier-1 benchmark (${cases.length} cases, corpus v${corpus.version})`);
-    console.log('');
-    console.log('| Metric | Value |');
-    console.log('|---|---:|');
-    console.log(`| Win rate (correct vs golden) | ${summary.winRatePct} (${correct}/${cases.length}) |`);
-    console.log(`| Avg latency | ${summary.avgLatencyMs} ms |`);
-    console.log(`| p50 / p99 / max latency | ${summary.p50LatencyMs} / ${summary.p99LatencyMs} / ${summary.maxLatencyMs} ms |`);
-    console.log(`| Measured cost | $0 (no API call) |`);
-    console.log(`| Est. savings vs Haiku/Sonnet/Opus | $${estSavings.vsHaiku} / $${estSavings.vsSonnet} / $${estSavings.vsOpus} |`);
-    console.log('');
-    const failed = results.filter((r) => !r.correct);
-    if (failed.length) {
-      console.log('## Failures');
-      for (const f of failed) console.log(`- \`${f.id}\` (${f.intent}): ${f.reason ?? `got "${f.gotPrefix}"`}`);
-      console.log('');
-    }
-  }
-
-  if (!process.env.BENCH_NO_WRITE) {
-    mkdirSync(RUNS_DIR, { recursive: true });
-    const stamp = summary.runAt.replace(/[:.]/g, '-');
-    writeFileSync(join(RUNS_DIR, `codemod-${stamp}.json`), JSON.stringify(run, null, 2));
-    writeFileSync(join(RUNS_DIR, 'codemod-latest.json'), JSON.stringify(run, null, 2));
-    if (!process.env.BENCH_JSON) console.log(`Wrote run to ${join(RUNS_DIR, `codemod-${stamp}.json`)}`);
-  }
-
-  // Non-zero exit if any case regressed — usable as a CI guardrail.
-  if (correct !== cases.length) process.exit(1);
-}
-
-main().catch((err) => {
-  console.error('benchmark failed:', err);
-  process.exit(1);
-});