@ngockhoale/ukit 3.4.46 → 3.4.48

package/scripts/bench/parallel-agents.mjs

@@ -1,224 +0,0 @@
-#!/usr/bin/env node
-/**
- * parallel-agents.mjs — TASK-004 harness.
- *
- * Measures the *machine-contention* component of N parallel agents by spawning N concurrent
- * child processes (default `yarn test:release-core`) per level and recording wall-clock.
- * This deliberately does NOT claim to benchmark LLM agent wall-clock (token counts, server
- * load and retries make that irreproducible) — see TASK-004 planner note #1.
- *
- * Usage:
- *   node scripts/bench/parallel-agents.mjs [--levels 1,3,5,10] [--cmd "<test command>"] [--out <path>]
- */
-
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import { spawn } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-
-const SLOWDOWN_LIMIT = 2.0;
-
-// BUG-C22-21: a level must never wait forever — one wedged `shell:true` child
-// used to leave `settled < n` permanently and hang the whole benchmark
-// mid-level. A per-level deadline SIGKILLs unsettled children (their whole
-// process group — detached:true puts each `sh -c` in its own group so
-// grandchildren die too) and resolves the level marked timedOut.
-// UKIT_BENCH_LEVEL_TIMEOUT_MS overrides the budget for tests.
-const LEVEL_TIMEOUT_MS = Number(process.env.UKIT_BENCH_LEVEL_TIMEOUT_MS) || 30 * 60 * 1000; // 30 min
-
-function parseArgs(argv) {
-  const opts = { levels: '1,3,5,10', cmd: 'yarn test:release-core', out: '.cache/bench/parallel-agents.json' };
-  for (let i = 0; i < argv.length; i += 1) {
-    const key = argv[i];
-    if (key !== '--levels' && key !== '--cmd' && key !== '--out') {
-      throw new Error(`unknown argument: ${key}`);
-    }
-    const val = argv[i + 1];
-    if (val === undefined || val.startsWith('--')) {
-      throw new Error(`missing value for ${key}`);
-    }
-    opts[key.slice(2)] = val;
-    i += 1;
-  }
-  return opts;
-}
-
-function parseLevels(raw) {
-  if (!/^\d+(,\d+)*$/.test(raw.trim())) {
-    throw new Error(`unparseable --levels value: "${raw}" (expected comma-separated positive integers, e.g. 1,3,5,10)`);
-  }
-  const levels = raw.split(',').map((s) => parseInt(s, 10));
-  if (levels.some((n) => n < 1)) throw new Error(`--levels values must be >= 1, got: ${raw}`);
-  return levels;
-}
-
-function runLevel(n, cmd) {
-  return new Promise((resolve) => {
-    const t0 = Date.now();
-    let live = 0;
-    let highWater = 0;
-    let failures = 0;
-    let settled = 0;
-    let timedOut = false;
-    let finished = false;
-    const children = [];
-    // BUG-C22-21: per-level deadline — kill stragglers and resolve marked-failed.
-    const deadline = setTimeout(() => {
-      timedOut = true;
-      for (const child of children) {
-        if (!child.ukitSettled) {
-          child.ukitSettled = true;
-          settled += 1;
-          failures += 1;
-          try {
-            process.kill(-child.pid, 'SIGKILL'); // kill the whole process tree
-          } catch {
-            try { child.kill('SIGKILL'); } catch { /* already gone */ }
-          }
-        }
-      }
-      maybeFinish();
-    }, LEVEL_TIMEOUT_MS);
-    deadline.unref();
-    for (let i = 0; i < n; i += 1) {
-      const child = spawn(cmd, { shell: true, stdio: 'ignore', detached: true });
-      live += 1;
-      highWater = Math.max(highWater, live);
-      children.push(child);
-      child.on('error', () => {
-        if (child.ukitSettled) return;
-        child.ukitSettled = true;
-        live -= 1;
-        settled += 1;
-        failures += 1;
-        maybeFinish();
-      });
-      child.on('exit', (code) => {
-        if (child.ukitSettled) return;
-        child.ukitSettled = true;
-        live -= 1;
-        settled += 1;
-        if (code !== 0) failures += 1;
-        maybeFinish();
-      });
-    }
-    function maybeFinish() {
-      if (finished) return;
-      if (settled === n) {
-        finished = true;
-        clearTimeout(deadline);
-        if (timedOut) {
-          console.error(`[bench] level n=${n}: deadline ${LEVEL_TIMEOUT_MS}ms exceeded — unsettled children killed and counted as failures`);
-        }
-        resolve({ n, wallClockMs: Date.now() - t0, failures, concurrencyHighWaterMark: highWater, timedOut });
-      }
-    }
-  });
-}
-
-function fail(msg) {
-  process.stderr.write(`${msg}\n`);
-  process.exit(1);
-}
-
-// BUG-C27-24: `wallClockMs === 0` (a sub-ms --cmd) made the base `base` 0 →
-// every slowdownFactor NaN → `eligible` empty → `eligible.reduce(..., eligible[0])`
-// dereffed undefined and threw a cryptic TypeError AFTER the whole benchmark ran.
-// `wallClockMs === 0` is below Date.now() resolution for a spawned child, so it is
-// marked unmeasurable and excluded; an empty `eligible` yields recommended=null so
-// main() fails with a clear message, and the reduce is seeded with rows[0] so it can
-// never deref undefined.
-export function computeRecommendation(rows) {
-  const base = rows[0].wallClockMs;
-  for (const row of rows) {
-    row.unmeasurable = row.wallClockMs === 0 || base === 0;
-    row.slowdownFactor = base === 0 ? NaN : row.wallClockMs / base;
-    row.perRunCost = row.wallClockMs / row.n;
-  }
-  const eligible = rows.filter((r) => !r.unmeasurable && r.slowdownFactor <= SLOWDOWN_LIMIT);
-  const recommended = eligible.length === 0
-    ? null
-    : eligible.reduce((best, r) => (r.perRunCost < best.perRunCost ? r : best), rows[0]).n;
-  return { recommended, eligible };
-}
-
-
-async function main() {
-  const opts = (() => {
-    try {
-      return parseArgs(process.argv.slice(2));
-    } catch (err) {
-      fail(err.message);
-    }
-  })();
-  let levels;
-  try {
-    levels = parseLevels(opts.levels);
-  } catch (err) {
-    fail(err.message);
-  }
-
-  const cpuCount = os.cpus().length;
-  const loadAvgStart = os.loadavg();
-  const startedAt = new Date().toISOString();
-
-  const rows = [];
-  for (const n of levels) {
-    // eslint-disable-next-line no-await-in-loop -- levels are measured sequentially on purpose
-    rows.push(await runLevel(n, opts.cmd));
-  }
-  const { recommended } = computeRecommendation(rows);
-  if (recommended === null) {
-    fail('no level met slowdown limit (base wall clock 0ms?) — benchmark is unmeasurable; rerun with a slower --cmd');
-  }
-
-  const loadAvgEnd = os.loadavg();
-  const finishedAt = new Date().toISOString();
-
-  const result = {
-    cpuCount,
-    levels: rows.map(({ n, wallClockMs, slowdownFactor, perRunCost, failures, concurrencyHighWaterMark, timedOut, unmeasurable }) => ({
-      n, wallClockMs, slowdownFactor, perRunCost, failures, concurrencyHighWaterMark, timedOut, unmeasurable,
-    })),
-    recommended,
-    measurementConditions: { loadAvgStart, loadAvgEnd, startedAt, finishedAt, cpuCount },
-  };
-
-  // Print the human table first so a failed --out write still shows the numbers.
-  const header = 'n'.padStart(4) + '  ' + 'wallClockMs'.padStart(12) + '  ' + 'slowdownFactor'.padStart(15) + '  ' + 'perRunCost'.padStart(11) + '  ' + 'failures'.padStart(8) + '  ' + 'concurrencyHighWaterMark'.padStart(23);
-  console.log(header);
-  for (const r of result.levels) {
-    console.log(
-      String(r.n).padStart(4)
-      + '  ' + String(r.wallClockMs).padStart(12)
-      + '  ' + (r.unmeasurable ? 'unmeasurable'.padStart(15) : r.slowdownFactor.toFixed(2).padStart(15))
-      + '  ' + r.perRunCost.toFixed(1).padStart(11)
-      + '  ' + String(r.failures).padStart(8)
-      + '  ' + String(r.concurrencyHighWaterMark).padStart(23),
-    );
-  }
-  console.log(`recommended: ${recommended} (best perRunCost with slowdownFactor <= ${SLOWDOWN_LIMIT}) on ${cpuCount} CPUs`);
-  console.log(`loadAvgStart: [${loadAvgStart.map((v) => v.toFixed(2)).join(', ')}]  loadAvgEnd: [${loadAvgEnd.map((v) => v.toFixed(2)).join(', ')}]`);
-
-  // Overwrite (never append), and never leave a partial file behind.
-  const outPath = path.resolve(opts.out);
-  try {
-    fs.mkdirSync(path.dirname(outPath), { recursive: true });
-    const body = JSON.stringify(result, null, 2) + '\n';
-    fs.writeFileSync(outPath, body);
-  } catch (err) {
-    fail(`cannot write --out ${outPath}: ${err.message}`);
-  }
-
-  const totalFailures = result.levels.reduce((s, r) => s + r.failures, 0);
-  if (totalFailures > 0) {
-    const perLevel = result.levels.filter((r) => r.failures > 0).map((r) => `n=${r.n}: ${r.failures}/${r.n} failed`).join('; ');
-    fail(`bench command failed: ${perLevel}`);
-  }
-}
-
-const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
-if (isMain) {
-  main().catch((err) => fail(err && err.message ? err.message : String(err)));
-}

package/scripts/bench/recorder-overhead.mjs

@@ -1,204 +0,0 @@
-#!/usr/bin/env node
-// recorder-overhead.mjs (TASK-001, SPEC §2 G2-FR03) — paired baseline vs
-// instrumented overhead measurement for the observer seam. REAL measurement
-// on this host; no fabricated targets.
-//
-// Workload: hook-telemetry-shaped events derived deterministically from the
-// seeded corpus (tests/fixtures/observability/corpus.json, seed=42) — one
-// event per corpus record, timings taken from the record's own duration so
-// the fixture stays the single source of truth.
-//
-//   baseline arm:     appendFileSync(JSON.stringify(row)) to a jsonl file —
-//                     the status-quo owner write path (hook-telemetry append).
-//   instrumented arm: adaptHookTelemetryEvent(row) → recorder.emit() → flush()
-//                     per event into a real temp segment store — the seam a
-//                     hook would run inside its deadline.
-//
-// Both arms repeat the same event corpus --reps times, interleaved per rep
-// (baseline pass, then instrumented pass) so drift hits both arms equally.
-//
-//   node scripts/bench/recorder-overhead.mjs [--json] [--reps N]
-
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import { execSync } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
-
-import { adaptHookTelemetryEvent } from '../../src/core/observability/adapters/hookTelemetryAdapter.js';
-import { createAdapterContext } from '../../src/core/observability/adapters/common.js';
-import { createRecorder } from '../../src/core/observability/emit/recorder.js';
-
-const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
-const corpusPath = path.join(repoRoot, 'tests/fixtures/observability/corpus.json');
-
-const args = process.argv.slice(2);
-const json = args.includes('--json');
-const repsFlag = args.indexOf('--reps');
-const REPS = repsFlag !== -1 ? Number.parseInt(args[repsFlag + 1], 10) : 30;
-if (!Number.isInteger(REPS) || REPS <= 0) {
-  console.error('--reps must be a positive integer');
-  process.exit(2);
-}
-
-function percentile(sorted, p) {
-  if (sorted.length === 0) return null;
-  const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
-  return sorted[Math.max(0, idx)];
-}
-
-function stats(samples) {
-  const sorted = [...samples].sort((a, b) => a - b);
-  const n = sorted.length;
-  const mean = n ? sorted.reduce((s, v) => s + v, 0) / n : 0;
-  const variance = n ? sorted.reduce((s, v) => s + (v - mean) ** 2, 0) / n : 0;
-  return {
-    n,
-    min: n ? sorted[0] : null,
-    max: n ? sorted[n - 1] : null,
-    mean: Number(mean.toFixed(4)),
-    stdev: Number(Math.sqrt(variance).toFixed(4)),
-    p50: percentile(sorted, 50),
-    p95: percentile(sorted, 95),
-    p99: percentile(sorted, 99),
-  };
-}
-
-function dirBytes(dir) {
-  let total = 0;
-  if (!fs.existsSync(dir)) return 0;
-  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-    const p = path.join(dir, entry.name);
-    if (entry.isDirectory()) total += dirBytes(p);
-    else total += fs.statSync(p).size;
-  }
-  return total;
-}
-
-function gitRevision() {
-  try {
-    return execSync('git rev-parse --short HEAD', { cwd: repoRoot, encoding: 'utf8', timeout: 10_000 }).trim();
-  } catch {
-    return 'unknown';
-  }
-}
-
-// --- workload: corpus records → hook-telemetry-shaped rows ------------------
-let corpus;
-let sourceRecords;
-try {
-  corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
-  sourceRecords = corpus.traces.flatMap((t) => t.records);
-} catch {
-  console.error(
-    `recorder-overhead: cannot load ${corpusPath} — run 'node scripts/bench/data-foundation.mjs --fixture' first`,
-  );
-  process.exit(1);
-}
-const events = sourceRecords.map((r, i) => ({
-  v: 1,
-  ts: Date.parse(r.wall_time_utc),
-  hookEvent: 'PostToolUse',
-  toolName: typeof r.payload?.tool_name === 'string' ? r.payload.tool_name : 'BenchTool',
-  toolUseId: `bench-${r.record_id ?? i}`,
-  hook: 'post-tool-use/bench-hook',
-  elapsedMs: Number.isFinite(r.payload?.duration_ms) ? r.payload.duration_ms : (i % 53),
-  outcome: 'ok',
-  ...(i % 3 === 0 ? { stageMs: i % 7 } : {}),
-}));
-
-const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ukit-bench-'));
-const baselineFile = path.join(tmpRoot, 'baseline.jsonl');
-const segmentRoot = path.join(tmpRoot, 'segments');
-
-const baselineMs = [];
-const emitMs = [];
-const flushMs = [];
-
-const heapBefore = process.memoryUsage().heapUsed;
-let recordsWritten = 0;
-let recordsEmitted = 0;
-let lastHealth = null;
-const wallStart = Date.now();
-
-const recorder = createRecorder({
-  root: segmentRoot,
-  config: { observability: { stage: 'default' } },
-});
-
-try {
-  for (let rep = 0; rep < REPS; rep += 1) {
-    // baseline arm — status-quo owner append path
-    for (const event of events) {
-      const line = JSON.stringify(event) + '\n';
-      const t0 = process.hrtime.bigint();
-      fs.appendFileSync(baselineFile, line);
-      baselineMs.push(Number(process.hrtime.bigint() - t0) / 1e6);
-    }
-    // instrumented arm — adapt → emit → flush per event (hook-deadline seam)
-    const ctx = createAdapterContext({ writerId: 'adapter-hook-bench' });
-    for (const event of events) {
-      const t0 = process.hrtime.bigint();
-      const record = adaptHookTelemetryEvent(event, ctx);
-      if (record) recorder.emit(record);
-      emitMs.push(Number(process.hrtime.bigint() - t0) / 1e6);
-
-      const f0 = process.hrtime.bigint();
-      await recorder.flush({ deadlineMs: 100 });
-      flushMs.push(Number(process.hrtime.bigint() - f0) / 1e6);
-    }
-    const h = recorder.health();
-    lastHealth = h;
-    recordsEmitted += events.length;
-  }
-} finally {
-  const heapAfter = process.memoryUsage().heapUsed;
-  const segmentBytes = dirBytes(segmentRoot);
-  recordsWritten = lastHealth ? lastHealth.written : 0;
-
-  const report = {
-    bench: 'recorder-overhead',
-    host: {
-      platform: os.platform(),
-      arch: os.arch(),
-      release: os.release(),
-      cpu: os.cpus()?.[0]?.model ?? 'unknown',
-      node: process.version,
-    },
-    revision: gitRevision(),
-    measured_at: new Date().toISOString(),
-    workload: {
-      corpus: path.relative(repoRoot, corpusPath),
-      seed: corpus.seed,
-      events_per_rep: events.length,
-      reps: REPS,
-    },
-    reps: REPS,
-    baseline: stats(baselineMs),
-    emit: stats(emitMs),
-    flush: stats(flushMs),
-    bytesPerRecord: recordsWritten > 0 ? Math.round(segmentBytes / recordsWritten) : null,
-    heapDeltaBytes: heapAfter - heapBefore,
-    wall_clock_ms: Date.now() - wallStart,
-    instrumented: {
-      events: recordsEmitted,
-      recordsWritten,
-      segmentBytes,
-      health: lastHealth,
-    },
-    baselineBytes: fs.existsSync(baselineFile) ? fs.statSync(baselineFile).size : 0,
-  };
-
-  if (json) {
-    process.stdout.write(JSON.stringify(report, null, 2) + '\n');
-  } else {
-    const f = (v) => (v === null ? 'null' : v.toFixed(3));
-    console.log(`reps=${report.reps} events/rep=${events.length} node=${report.host.node} rev=${report.revision}`);
-    console.log(`baseline append ms  p50=${f(report.baseline.p50)} p95=${f(report.baseline.p95)} p99=${f(report.baseline.p99)}`);
-    console.log(`instrumented emit ms p50=${f(report.emit.p50)} p95=${f(report.emit.p95)} p99=${f(report.emit.p99)} stdev=${report.emit.stdev}`);
-    console.log(`instrumented flush ms p50=${f(report.flush.p50)} p95=${f(report.flush.p95)} p99=${f(report.flush.p99)}`);
-    console.log(`bytes/record=${report.bytesPerRecord} heapDelta=${report.heapDeltaBytes}B written=${recordsWritten}`);
-  }
-
-  fs.rmSync(tmpRoot, { recursive: true, force: true });
-}

package/scripts/bench/runGold.mjs

@@ -1,238 +0,0 @@
-#!/usr/bin/env node
-// Gold-task benchmark harness skeleton (SPEC §10).
-// Loads a corpus of {id, prompt, expectedMode, expectedAnchors[], maxTokens}
-// cases, runs routeTask + compileContext per case, prints a scorecard with a
-// totals block, optionally writes an aggregated JSON scorecard (--json <path>),
-// and exits 0 only when every case passes. CI-3xx gates on the JSON summary.
-
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
-const DEFAULT_CORPUS = path.join(REPO_ROOT, 'scripts', 'bench', 'goldTasks.json');
-
-const USAGE = 'usage: runGold.mjs [--corpus <path>] [--project <dir>] [--json <path>]';
-
-function parseArgs(argv) {
-  let corpusPath = DEFAULT_CORPUS;
-  let projectRoot = REPO_ROOT;
-  let jsonPath = null;
-  const valueFor = (flag, i) => {
-    const next = argv[i + 1];
-    if (next === undefined || next.startsWith('--')) {
-      throw new Error(`missing value for ${flag}`);
-    }
-    return next;
-  };
-  for (let i = 0; i < argv.length; i += 1) {
-    if (argv[i] === '--corpus') {
-      corpusPath = path.resolve(valueFor('--corpus', i));
-      i += 1;
-    } else if (argv[i] === '--project') {
-      projectRoot = path.resolve(valueFor('--project', i));
-      i += 1;
-    } else if (argv[i] === '--json') {
-      jsonPath = path.resolve(valueFor('--json', i));
-      i += 1;
-    } else {
-      throw new Error(`unknown argument: ${argv[i]}`);
-    }
-  }
-  return { corpusPath, projectRoot, jsonPath };
-}
-
-async function loadCorpus(corpusPath) {
-  const raw = await fs.readFile(corpusPath, 'utf8');
-  const cases = JSON.parse(raw);
-  if (!Array.isArray(cases)) {
-    throw new Error(`bench corpus must be a JSON array: ${corpusPath}`);
-  }
-  return cases;
-}
-
-function anchorPathsOf(packet) {
-  const fromAnchors = (packet.anchors ?? []).map((a) => a.path).filter(Boolean);
-  const fromEvidence = (packet.evidence ?? []).map((e) => e.path).filter(Boolean);
-  return new Set([...fromAnchors, ...fromEvidence]);
-}
-
-function normalizeMode(mode) {
-  return typeof mode === 'string' && mode.trim() !== '' ? mode.trim() : null;
-}
-
-export async function runGoldCase(projectRoot, testCase, { compileContext, routeTask }) {
-  const failures = [];
-  const expectedMode = normalizeMode(testCase.expectedMode);
-  const expectedAnchors = Array.isArray(testCase.expectedAnchors) ? testCase.expectedAnchors : [];
-  const maxTokens = typeof testCase.maxTokens === 'number' ? testCase.maxTokens : Infinity;
-
-  const routed = routeTask({
-    prompt: testCase.prompt ?? '',
-    hasError: testCase.hasError === true,
-    filesHinted: Array.isArray(testCase.filesHinted) ? testCase.filesHinted : undefined,
-    flags: testCase.flags && typeof testCase.flags === 'object' ? testCase.flags : {},
-  });
-
-  const packet = await compileContext(projectRoot, {
-    prompt: testCase.prompt ?? '',
-    hasError: testCase.hasError === true,
-    filesHinted: Array.isArray(testCase.filesHinted) ? testCase.filesHinted : undefined,
-    flags: testCase.flags && typeof testCase.flags === 'object' ? testCase.flags : {},
-  });
-
-  if (expectedMode && packet.task_type !== expectedMode) {
-    failures.push(`mode: expected ${expectedMode}, got ${packet.task_type || '(empty)'}`);
-  }
-
-  const anchorPaths = anchorPathsOf(packet);
-  const anchorHits = [];
-  const anchorMisses = [];
-  for (const expected of expectedAnchors) {
-    if (anchorPaths.has(expected)) {
-      anchorHits.push(expected);
-    } else {
-      anchorMisses.push(expected);
-      failures.push(`anchor missing: ${expected}`);
-    }
-  }
-
-  if (packet.budget?.used > maxTokens) {
-    failures.push(`budget: used ${packet.budget.used} > maxTokens ${maxTokens}`);
-  }
-
-  return {
-    id: testCase.id ?? '(unnamed)',
-    mode: packet.task_type,
-    expectedMode,
-    anchors: [...anchorPaths],
-    expectedAnchors,
-    anchorHits,
-    anchorMisses,
-    budgetUsed: packet.budget?.used ?? 0,
-    maxTokens,
-    routedMode: routed.mode,
-    pass: failures.length === 0,
-    failures,
-  };
-}
-
-export async function runGold({ projectRoot, corpusPath } = {}) {
-  const rootDir = projectRoot ?? REPO_ROOT;
-  const { compileContext } = await import(path.join(REPO_ROOT, 'src/core/codeintel/compiler.js'));
-  const { routeTask } = await import(path.join(REPO_ROOT, 'src/core/codeintel/router.js'));
-  const { buildCodeIndex } = await import(path.join(REPO_ROOT, 'src/index/buildIndex.js'));
-  const { INDEX_ARTIFACTS, getArtifactPath } = await import(path.join(REPO_ROOT, 'src/index/paths.js'));
-
-  // The bench needs an index to score anchors; build once when missing.
-  try {
-    await fs.stat(getArtifactPath(rootDir, INDEX_ARTIFACTS.files));
-  } catch {
-    console.log('[bench] index missing — building code index first');
-    await buildCodeIndex({ rootDir });
-  }
-
-  const cases = await loadCorpus(corpusPath ?? DEFAULT_CORPUS);
-  const results = [];
-  for (const testCase of cases) {
-    results.push(await runGoldCase(rootDir, testCase, { compileContext, routeTask }));
-  }
-  return { results, pass: results.every((r) => r.pass) };
-}
-
-/**
- * Aggregate per-case results into a machine-readable summary for CI gating.
- * Pooled anchor precision = hits/found, recall = hits/expected; cases with
- * empty expectedAnchors contribute 0 to both denominators (excluded, not 1).
- * @param {Array<object>} results per-case runGoldCase outputs
- * @returns {{total:number, passed:number, passRate:number, modeAccuracy:number|null, anchorPrecision:number|null, anchorRecall:number|null, meanBudgetUsedPct:number|null}}
- */
-export function summarizeResults(results) {
-  const total = results.length;
-  const passed = results.filter((r) => r.pass).length;
-  const passRate = total === 0 ? 0 : passed / total;
-
-  const withMode = results.filter((r) => r.expectedMode);
-  const modeCorrect = withMode.filter((r) => r.mode === r.expectedMode).length;
-  const modeAccuracy = withMode.length === 0 ? null : modeCorrect / withMode.length;
-
-  let hits = 0;
-  let found = 0;
-  let expected = 0;
-  for (const r of results) {
-    hits += Array.isArray(r.anchorHits) ? r.anchorHits.length : 0;
-    found += Array.isArray(r.anchors) ? r.anchors.length : 0;
-    expected += Array.isArray(r.expectedAnchors) ? r.expectedAnchors.length : 0;
-  }
-  const anchorPrecision = found === 0 ? null : hits / found;
-  const anchorRecall = expected === 0 ? null : hits / expected;
-
-  const budgeted = results.filter((r) => typeof r.maxTokens === 'number' && r.maxTokens > 0);
-  const meanBudgetUsedPct = budgeted.length === 0
-    ? null
-    : budgeted.reduce((acc, r) => acc + (r.budgetUsed ?? 0) / r.maxTokens, 0) / budgeted.length;
-
-  return { total, passed, passRate, modeAccuracy, anchorPrecision, anchorRecall, meanBudgetUsedPct };
-}
-
-/**
- * Write { generatedAt, results, summary } scorecard JSON to `jsonPath` (pretty-printed).
- * @param {string} jsonPath output file path
- * @param {Array<object>} results per-case runGoldCase outputs
- */
-export async function writeScorecardJson(jsonPath, results) {
-  const payload = {
-    generatedAt: new Date().toISOString(),
-    results,
-    summary: summarizeResults(results),
-  };
-  await fs.writeFile(jsonPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
-}
-
-function fmtPct(value) {
-  return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
-}
-
-export function printScorecard(results) {
-  console.log('id | mode | expected | anchors | budget | verdict');
-  console.log('---|------|----------|---------|--------|--------');
-  for (const r of results) {
-    console.log(
-      `${r.id} | ${r.mode || '-'} | ${r.expectedMode || '-'} | ${r.anchors.length} | ${r.budgetUsed}/${r.maxTokens === Infinity ? '∞' : r.maxTokens} | ${r.pass ? 'PASS' : 'FAIL'}`,
-    );
-    for (const failure of r.failures) {
-      console.log(`  ! ${failure}`);
-    }
-  }
-  const s = summarizeResults(results);
-  console.log(`scorecard: ${s.passed}/${s.total} passed`);
-  console.log(
-    `totals: passRate: ${fmtPct(s.passRate)} | modeAccuracy: ${fmtPct(s.modeAccuracy)} | anchorPrecision: ${fmtPct(s.anchorPrecision)} | anchorRecall: ${fmtPct(s.anchorRecall)} | meanBudgetUsedPct: ${fmtPct(s.meanBudgetUsedPct)}`,
-  );
-}
-
-const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
-if (isMain) {
-  let parsed;
-  try {
-    parsed = parseArgs(process.argv.slice(2));
-  } catch (error) {
-    console.error(`${USAGE}\n${error?.message ?? error}`);
-    process.exit(2);
-  }
-  const { corpusPath, projectRoot, jsonPath } = parsed;
-  try {
-    const { results, pass } = await runGold({ projectRoot, corpusPath });
-    printScorecard(results);
-    if (jsonPath) {
-      await writeScorecardJson(jsonPath, results);
-      console.log(`[bench] scorecard JSON written: ${jsonPath}`);
-    }
-    if (!pass) {
-      process.exitCode = 1;
-    }
-  } catch (error) {
-    console.error(`[bench] ${error?.message ?? error}`);
-    process.exitCode = 1;
-  }
-}