akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/cli.js

@@ -1,1018 +0,0 @@
-#!/usr/bin/env bun
-/**
- * akm-bench CLI dispatcher.
- *
- * Subcommands:
- *   • `utility`    — paired noakm vs akm utility benchmark (Track A).
- *   • `compare`    — diff two report JSON files; refuses on hash/model mismatch.
- *   • `attribute`  — per-asset marginal contribution via leave-one-out masking.
- *   • `evolve`     — longitudinal evolution loop (Track B). Stub.
- *   • `kill`       — send SIGTERM to a running bench process (reads bench.pid).
- *
- * Implementation status and validity rules live in `tests/bench/BENCH.md`.
- *
- * NOTE: The bench binary is intentionally argv-light. citty is the project's
- * CLI framework but the bench is not part of the public CLI surface, so a
- * hand-rolled parser keeps the dependency graph tight.
- */
-import fs from "node:fs";
-import path from "node:path";
-import process from "node:process";
-import { getCacheDir } from "../../src/core/paths";
-import { listTasks } from "./corpus";
-import { runEvolve } from "./evolve";
-import { compareReports, rehydrateRunFromSerialized, runMaskedCorpus, } from "./metrics";
-import { BenchConfigError, loadOpencodeProviders } from "./opencode-config";
-import { renderAttributionTable, renderCompareMarkdown, renderEvolveReport, renderUtilityReport, } from "./report";
-import { loadBenchRunConfig } from "./run-config";
-import { runUtility } from "./runner";
-import { isPidRunning, readBenchPid, writeBenchPid } from "./tmp";
-const HELP = `akm-bench — agent-plus-akm evaluation framework
-
-Usage:
-  bun run tests/bench/cli.ts <config.json> [--json] [--seeds N] [--parallel N] [--tasks <list>]
-  bun run tests/bench/cli.ts <subcommand> [...flags]
-
-Config-file mode (recommended):
-  Pass a path to a tests/bench/configs/*.json file as the first argument.
-  See tests/bench/BENCH.md for the run-config schema and provider
-  discovery chain (BENCH_OPENCODE_CONFIG → providers/providersRef →
-  ~/.config/akm/bench-providers.json).
-
-Subcommands (legacy; still supported):
-  utility       Track A: paired noakm vs akm utility benchmark.
-  evolve        Track B: longitudinal feedback → distill → propose loop.
-  compare       Diff two report JSON files (refuses cross-model diffs).
-  attribute     Per-asset marginal pass-rate contribution.
-  kill          Send SIGTERM to a running bench process (reads bench.pid).
-  clean         Remove all bench tmp dirs under \${AKM_CACHE_DIR}/bench/.
-
-utility flags:
-  --tasks <slice>          train | eval | all  (default: all)
-  --seeds <N>              seeds per arm  (default: 5)
-  --budget-tokens <N>      per-run token cap (default: 30000)
-  --budget-wall-ms <N>     per-run wallclock cap in ms (default: 120000)
-  --parallel <N>           number of (task, arm, seed) triples to run concurrently
-                           (default: 1 — sequential). Clamped to [1, 8]. Values > 4
-                           print a warning unless --force-parallel is also set.
-  --force-parallel         suppress the high-parallelism warning when --parallel > 4.
-  --no-noakm               exclude the noakm baseline arm (default: included). The noakm arm is
-                           the control condition — pass_rate(akm) − pass_rate(noakm) is the
-                           primary utility metric. Omit only when you are measuring workflow
-                           compliance rather than marginal utility.
-  --include-synthetic      add a third 'synthetic' arm where the model writes/uses its own
-                           scratch notes (no AKM stash). Reports akm_over_synthetic_lift so
-                           operators can see whether AKM beats a self-notes baseline.
-  --opencode-config <path> path to a bench opencode providers JSON file. Auto-discovered
-                           from BENCH_OPENCODE_CONFIG env var or the fixture defaults when
-                           omitted. See BENCH.md for the discovery order.
-  --json                   suppress the markdown summary on stderr (machine-readable only).
-                           Without --json, JSON still goes to stdout and the markdown
-                           summary is also written to stderr for human-friendly reads.
-  -h, --help               show this message.
-
-evolve flags:
-  --tasks <domain>         domain id (e.g., docker-homelab) or 'all'. REQUIRED.
-  --seeds <N>              seeds per arm (default: 5)
-  --budget-tokens <N>      per-run token cap (default: 30000)
-  --budget-wall-ms <N>     per-run wallclock cap in ms (default: 120000)
-  --parallel <N>           same as utility --parallel (default: 1).
-  --force-parallel         suppress the high-parallelism warning when --parallel > 4.
-  --negative-threshold-count <N>  absolute negative-feedback count to evolve (default: 2)
-  --negative-threshold-ratio <R>  ratio of negatives to total feedback (default: 0.5)
-  --opencode-config <path> path to a bench opencode providers JSON file (same as utility).
-  --json                   suppress the markdown summary on stderr.
-
-compare flags:
-  --base <path>                 path to baseline UtilityRunReport JSON file. REQUIRED.
-  --current <path>              path to current  UtilityRunReport JSON file. REQUIRED.
-  --json                        emit the structured CompareResult JSON to stdout instead
-                                of a markdown diff.
-  --allow-corpus-mismatch       proceed even when the two reports disagree on the
-                                selected task IDs / taskCorpusHash. The diff is
-                                rendered with a warning instead of being refused.
-  --allow-fixture-mismatch      proceed even when the two reports disagree on the
-                                fixtureContentHash. Renders a warning instead of
-                                refusing.
-  Exit codes: 0 on successful diff, 1 on refusal (model/corpus/fixture-hash/schema/track mismatch),
-              2 on input errors (missing files, malformed JSON, unknown flags).
-
-attribute flags:
-  --base <path>            path to a §13.3 utility run JSON (required).
-  --top <N>                number of top-loaded assets to mask (default: 5; clamped).
-  --opencode-config <path> path to a bench opencode providers JSON file (same as utility).
-  --json                   suppress the markdown summary on stderr.
-
-kill flags:
-  (none)        Reads \${AKM_CACHE_DIR}/bench/bench.pid and sends SIGTERM.
-                Prints "no bench running" and exits 0 when the PID file is absent.
-                Run again (or press Ctrl-C) to escalate to SIGKILL.
-
-Environment:
-  BENCH_OPENCODE_MODEL      model id stamped into every RunResult. REQUIRED for utility/evolve
-                            unless the loaded providers file supplies a defaultModel.
-  BENCH_OPENCODE_CONFIG     path to the bench opencode providers JSON file. Overrides the
-                            default fixture; overridden by --opencode-config flag.
-
-Auto-discovery order for --opencode-config (first existing file wins):
-  1. --opencode-config <path>  flag value
-  2. BENCH_OPENCODE_CONFIG     env var
-  3. tests/fixtures/bench/opencode-providers.local.json  (gitignored operator overlay)
-  4. tests/fixtures/bench/opencode-providers.json        (committed fixture)
-  5. None found → empty isolated dir, opencode uses cloud-provider defaults.
-
-See tests/bench/BENCH.md for the operator guide.
-`;
-function parseArgs(argv) {
-    const flags = new Map();
-    const bool = new Set();
-    const positional = [];
-    const subcommand = argv[0] ?? "";
-    for (let i = 1; i < argv.length; i++) {
-        const arg = argv[i];
-        if (arg === "--help" || arg === "-h") {
-            bool.add("help");
-            continue;
-        }
-        if (arg === "--json") {
-            bool.add("json");
-            continue;
-        }
-        if (arg.startsWith("--")) {
-            const eq = arg.indexOf("=");
-            if (eq !== -1) {
-                flags.set(arg.slice(2, eq), arg.slice(eq + 1));
-            }
-            else {
-                const next = argv[i + 1];
-                if (next === undefined || next.startsWith("--")) {
-                    bool.add(arg.slice(2));
-                }
-                else {
-                    flags.set(arg.slice(2), next);
-                    i += 1;
-                }
-            }
-            continue;
-        }
-        positional.push(arg);
-    }
-    return { subcommand, flags, bool, positional };
-}
-/**
- * Absolute path to the committed default bench opencode providers fixture.
- * Used as the final fallback in the auto-discovery chain.
- */
-const DEFAULT_PROVIDERS_PATH = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
-/**
- * Absolute path to the gitignored operator overlay. Takes precedence over
- * the committed fixture when it exists.
- */
-const LOCAL_PROVIDERS_PATH = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
-/**
- * Auto-discover and load the bench opencode providers file.
- *
- * Discovery order (first existing file wins):
- *   1. `flagPath`  — `--opencode-config` flag value (already resolved by caller).
- *   2. `BENCH_OPENCODE_CONFIG` env var.
- *   3. `tests/fixtures/bench/opencode-providers.local.json` (gitignored overlay).
- *   4. `tests/fixtures/bench/opencode-providers.json` (committed fixture).
- *   5. None found → returns `undefined` (empty isolated dir, cloud-provider fallback).
- *
- * When a file is found but fails to load, the BenchConfigError is re-thrown
- * so the caller can map it to the correct exit code.
- */
-export function discoverOpencodeProviders(flagPath) {
-    const candidates = [
-        flagPath,
-        getEnv("BENCH_OPENCODE_CONFIG"),
-        LOCAL_PROVIDERS_PATH,
-        DEFAULT_PROVIDERS_PATH,
-    ];
-    for (const candidate of candidates) {
-        if (!candidate)
-            continue;
-        const absPath = path.isAbsolute(candidate) ? candidate : path.resolve(candidate);
-        if (fs.existsSync(absPath)) {
-            // Throws BenchConfigError on validation failure; callers propagate it.
-            return loadOpencodeProviders(absPath);
-        }
-        // When a flag or env-var path was given explicitly but doesn't exist,
-        // treat it as a usage error rather than silently skipping to the next
-        // candidate. Only the auto-discovered defaults (local + committed) are
-        // silently skipped when absent.
-        if (candidate === flagPath && flagPath) {
-            // This throws with isUsageError: true (file not found).
-            return loadOpencodeProviders(absPath);
-        }
-        if (candidate === getEnv("BENCH_OPENCODE_CONFIG") && candidate) {
-            return loadOpencodeProviders(absPath);
-        }
-    }
-    return undefined;
-}
-/**
- * `utility` subcommand. Walks the corpus, runs K seeds per arm per task,
- * and produces the §13.3 report.
- *
- * Returns rather than mutates process to keep this unit-testable. The
- * `main()` driver below maps the result onto the actual stdout/stderr/exit.
- */
-export async function runUtilityCli(options) {
-    const sliceFilter = options.slice === "all" ? undefined : options.slice;
-    const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
-    // noakm arm is included by default: it is the control condition for the
-    // primary utility metric (pass_rate(akm) − pass_rate(noakm), spec §4).
-    // Pass --no-noakm (options.includeNoakm === false) to exclude it.
-    const arms = options.includeNoakm === false ? ["akm"] : ["noakm", "akm"];
-    const report = await runUtility({
-        tasks,
-        arms,
-        model: options.model,
-        seedsPerArm: options.seedsPerArm,
-        budgetTokens: options.budgetTokens,
-        budgetWallMs: options.budgetWallMs,
-        slice: options.slice,
-        // #261: thread the synthetic-arm gate. Default off — the envelope shape
-        // is byte-identical to the pre-#261 output unless the operator opts in.
-        ...(options.includeSynthetic ? { includeSynthetic: true } : {}),
-        ...(options.parallel !== undefined ? { parallel: options.parallel } : {}),
-        ...(options.forceParallel ? { forceParallel: true } : {}),
-        ...(options.branch !== undefined ? { branch: options.branch } : {}),
-        ...(options.commit !== undefined ? { commit: options.commit } : {}),
-        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
-        ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
-    });
-    const { json, markdown } = renderUtilityReport(report);
-    const jsonText = `${JSON.stringify(json, null, 2)}\n`;
-    const markdownText = `${markdown}\n`;
-    // JSON ALWAYS goes to stdout. This is the bench's machine-readable
-    // contract (matches `tests/benchmark-suite.ts` and the future `bench
-    // compare`/`attribute` subcommands). The `--json` flag means
-    // "suppress the human-friendly markdown summary on stderr"; without it,
-    // both the JSON envelope (stdout) and the markdown summary (stderr) are
-    // emitted so an operator running it interactively gets both views.
-    const stdout = jsonText;
-    let stderr = options.json ? "" : markdownText;
-    stderr += `tasks discovered: ${tasks.length} (slice=${options.slice})\n`;
-    if (tasks.length === 0) {
-        stderr += "no tasks found — corpus is empty or the slice filter excluded all tasks\n";
-    }
-    return { exitCode: 0, stdout, stderr };
-}
-/**
- * `compare` subcommand. Reads two UtilityRunReport JSON files from disk,
- * dispatches to `compareReports`, and renders either markdown (default) or
- * the structured JSON envelope to stdout.
- *
- * Exit-code shape:
- *   • 0 on a successful diff (regardless of whether the diff shows wins).
- *   • 1 on a refusal (model/hash/schema/track mismatch).
- *   • 2 on input errors (file missing, malformed JSON).
- *
- * Returned `UtilityCliResult` keeps this unit-testable; the `main()` driver
- * splices the result onto the actual process.
- */
-export function runCompareCli(options) {
-    let baseRaw;
-    let currentRaw;
-    try {
-        baseRaw = fs.readFileSync(options.basePath, "utf8");
-    }
-    catch (err) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench compare: cannot read --base ${options.basePath}: ${err.message}\n`,
-        };
-    }
-    try {
-        currentRaw = fs.readFileSync(options.currentPath, "utf8");
-    }
-    catch (err) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench compare: cannot read --current ${options.currentPath}: ${err.message}\n`,
-        };
-    }
-    let base;
-    let current;
-    try {
-        base = JSON.parse(baseRaw);
-    }
-    catch (err) {
-        return { exitCode: 2, stdout: "", stderr: `bench compare: malformed JSON in --base: ${err.message}\n` };
-    }
-    try {
-        current = JSON.parse(currentRaw);
-    }
-    catch (err) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench compare: malformed JSON in --current: ${err.message}\n`,
-        };
-    }
-    const result = compareReports(base, current, {
-        ...(options.allowCorpusMismatch ? { allowCorpusMismatch: true } : {}),
-        ...(options.allowFixtureMismatch ? { allowFixtureMismatch: true } : {}),
-    });
-    const stdout = options.json ? `${JSON.stringify(result, null, 2)}\n` : `${renderCompareMarkdown(result)}\n`;
-    let stderr = "";
-    if (!result.ok) {
-        stderr = `bench compare: ${result.message}\n`;
-        return { exitCode: 1, stdout, stderr };
-    }
-    // One-line summary on stderr so an interactive operator sees it without
-    // having to scan the markdown body.
-    const agg = result.aggregate;
-    stderr = `bench compare: pass_rate Δ=${agg.passRateDelta.toFixed(2)} (${agg.passRateSign}); ${result.perTask.length} tasks compared.\n`;
-    for (const w of result.warnings)
-        stderr += `warning: ${w}\n`;
-    return { exitCode: 0, stdout, stderr };
-}
-/**
- * `attribute` subcommand. Loads a base utility report, picks the top-N
- * most-loaded assets, masks each in turn, re-runs the corpus, and emits a
- * marginal-contribution report.
- *
- * Cost: N × (tasks × arms × seedsPerArm) re-runs. Reported to stderr up
- * front so the operator can abort if the projection is too expensive.
- */
-export async function runAttributeCli(options) {
-    if (!fs.existsSync(options.basePath)) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench attribute: base report not found: ${options.basePath}\n`,
-        };
-    }
-    let baseEnvelope;
-    try {
-        baseEnvelope = JSON.parse(fs.readFileSync(options.basePath, "utf8"));
-    }
-    catch (err) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench attribute: failed to parse ${options.basePath}: ${err instanceof Error ? err.message : String(err)}\n`,
-        };
-    }
-    const corpus = (baseEnvelope.corpus ?? {});
-    const sliceRaw = (corpus.slice ?? "all");
-    const slice = sliceRaw === "train" || sliceRaw === "eval" ? sliceRaw : "all";
-    const sliceFilter = slice === "all" ? undefined : slice;
-    const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
-    const seedsPerArm = typeof corpus.seedsPerArm === "number" ? corpus.seedsPerArm : 5;
-    const agent = (baseEnvelope.agent ?? {});
-    const model = options.modelOverride ?? (typeof agent.model === "string" ? agent.model : "unknown");
-    // The stored envelope's perAsset is in snake_case. Convert it back to the
-    // PerAssetAttribution shape the metrics module expects so we can pass it
-    // through to runMaskedCorpus.
-    const perAssetSerialised = baseEnvelope.perAsset;
-    const perAsset = {
-        totalAkmRuns: perAssetSerialised?.total_akm_runs ?? 0,
-        rows: (perAssetSerialised?.rows ?? []).map((r) => ({
-            assetRef: String(r.asset_ref ?? ""),
-            loadCount: Number(r.load_count ?? 0),
-            loadCountPassing: Number(r.load_count_passing ?? 0),
-            loadCountFailing: Number(r.load_count_failing ?? 0),
-            loadPassRate: r.load_pass_rate === null ? null : Number(r.load_pass_rate),
-        })),
-    };
-    if (perAsset.rows.length === 0) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: "bench attribute: base report has no per-asset attribution rows (no assets loaded). Re-run `bench utility` first.\n",
-        };
-    }
-    const desired = Math.max(1, options.topN);
-    const clamped = Math.min(desired, perAsset.rows.length);
-    // Prefer the persisted `runs[]` array (#249). When the report carries
-    // serialised raw runs we hydrate them back into RunResult shape and feed
-    // them to `runMaskedCorpus` directly — that keeps attribution faithful to
-    // the original (task, arm, seed) bag instead of synthesising stubs from
-    // the per-asset aggregate. Falls back to the legacy aggregate path when
-    // the report pre-dates the `runs[]` field.
-    const persistedRuns = readPersistedRuns(baseEnvelope);
-    const akmRuns = persistedRuns !== null ? persistedRuns.filter((r) => r.arm === "akm") : synthesiseAkmRunsFromAttribution(perAsset);
-    const baseReport = {
-        timestamp: String(baseEnvelope.timestamp ?? ""),
-        branch: String(baseEnvelope.branch ?? ""),
-        commit: String(baseEnvelope.commit ?? ""),
-        model,
-        corpus: {
-            domains: typeof corpus.domains === "number" ? corpus.domains : 0,
-            tasks: typeof corpus.tasks === "number" ? corpus.tasks : tasks.length,
-            slice,
-            seedsPerArm,
-        },
-        aggregateNoakm: extractCorpusMetrics(baseEnvelope, "noakm"),
-        aggregateAkm: extractCorpusMetrics(baseEnvelope, "akm"),
-        aggregateDelta: extractCorpusMetrics(baseEnvelope, "delta"),
-        trajectoryAkm: { correctAssetLoaded: null, feedbackRecorded: 0 },
-        failureModes: { byLabel: {}, byTask: {} },
-        tasks: [],
-        warnings: [],
-        perAsset,
-        akmRuns,
-        taskMetadata: tasks,
-    };
-    const projection = clamped * tasks.length * 2 * seedsPerArm;
-    let stderr = `bench attribute: masking top ${clamped} of ${perAsset.rows.length} assets; ${clamped} × ${tasks.length} tasks × 2 arms × ${seedsPerArm} seeds = ${projection} re-runs.\n`;
-    const baseOptions = {
-        arms: ["noakm", "akm"],
-        model,
-        seedsPerArm,
-    };
-    const maskedRunner = options.runUtility ?? defaultMaskedRunner;
-    let maskedResult;
-    try {
-        maskedResult = await runMaskedCorpus({
-            baseReport,
-            topN: clamped,
-            runUtility: maskedRunner,
-            baseOptions,
-            ...(options.fixturesRoot ? { fixturesRoot: options.fixturesRoot } : {}),
-        });
-    }
-    catch (err) {
-        return {
-            exitCode: 1,
-            stdout: "",
-            stderr: `${stderr}bench attribute: masked-corpus run failed: ${err instanceof Error ? err.message : String(err)}\n`,
-        };
-    }
-    const json = {
-        schemaVersion: 1,
-        track: "attribute",
-        base: { path: options.basePath, model },
-        // Issue #251: surface the masking strategy + the exact masked refs in
-        // the JSON envelope so operators can audit the marginal-contribution
-        // numbers without re-running the masker. The `maskedRefs` order matches
-        // `attributions[]`. Strategy is currently always `"leave-one-out"`;
-        // future strategies extend the union in `MaskedCorpusResult`.
-        attribution: {
-            maskingStrategy: maskedResult.maskingStrategy,
-            maskedRefs: maskedResult.maskedRefs,
-        },
-        maskingStrategy: maskedResult.maskingStrategy,
-        runsPerformed: maskedResult.runsPerformed,
-        perAsset: {
-            total_akm_runs: perAsset.totalAkmRuns,
-            rows: perAsset.rows.map((r) => ({
-                asset_ref: r.assetRef,
-                load_count: r.loadCount,
-                load_count_passing: r.loadCountPassing,
-                load_count_failing: r.loadCountFailing,
-                load_pass_rate: r.loadPassRate,
-            })),
-        },
-        attributions: maskedResult.attributions.map((a) => ({
-            asset_ref: a.assetRef,
-            base_pass_rate: a.basePassRate,
-            masked_pass_rate: a.maskedPassRate,
-            marginal_contribution: a.marginalContribution,
-        })),
-    };
-    const stdout = `${JSON.stringify(json, null, 2)}\n`;
-    if (!options.json) {
-        stderr += `${renderAttributionTable(perAsset)}\n`;
-        stderr += `\n## Marginal contributions (leave-one-out)\n\n`;
-        stderr += `| asset_ref | base_pass_rate | masked_pass_rate | marginal_contribution |\n`;
-        stderr += `|-----------|----------------|------------------|-----------------------|\n`;
-        for (const a of maskedResult.attributions) {
-            stderr += `| \`${a.assetRef}\` | ${a.basePassRate.toFixed(2)} | ${a.maskedPassRate.toFixed(2)} | ${signed(a.marginalContribution.toFixed(2))} |\n`;
-        }
-    }
-    return { exitCode: 0, stdout, stderr };
-}
-/** Default real-runner wrapper for masked re-runs. */
-async function defaultMaskedRunner(options) {
-    const arms = options.arms;
-    return runUtility({
-        tasks: options.tasks,
-        arms,
-        model: options.model,
-        ...(options.seedsPerArm !== undefined ? { seedsPerArm: options.seedsPerArm } : {}),
-        ...(options.budgetTokens !== undefined ? { budgetTokens: options.budgetTokens } : {}),
-        ...(options.budgetWallMs !== undefined ? { budgetWallMs: options.budgetWallMs } : {}),
-        ...(options.slice !== undefined ? { slice: options.slice } : {}),
-        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
-        ...(options.branch !== undefined ? { branch: options.branch } : {}),
-        ...(options.commit !== undefined ? { commit: options.commit } : {}),
-        ...(options.spawn ? { spawn: options.spawn } : {}),
-        ...(options.materialiseStash !== undefined ? { materialiseStash: options.materialiseStash } : {}),
-    });
-}
-/**
- * Best-effort extractor for `aggregate.<arm>` corpus metrics from the
- * persisted §13.3 envelope. The envelope keys are snake-cased.
- */
-function extractCorpusMetrics(envelope, key) {
-    const aggregate = (envelope.aggregate ?? {});
-    const node = (aggregate[key] ?? {});
-    return {
-        passRate: typeof node.pass_rate === "number" ? node.pass_rate : 0,
-        tokensPerPass: node.tokens_per_pass === null ? null : typeof node.tokens_per_pass === "number" ? node.tokens_per_pass : null,
-        tokensPerRun: node.tokens_per_run === null ? null : typeof node.tokens_per_run === "number" ? node.tokens_per_run : null,
-        wallclockMs: typeof node.wallclock_ms === "number" ? node.wallclock_ms : 0,
-    };
-}
-/**
- * Read the persisted `runs[]` array (#249) from a §13.3 envelope and hydrate
- * each row back into the in-memory `RunResult` shape. Returns `null` when the
- * envelope pre-dates the field (legacy reports) so callers can fall back to
- * the aggregate-only path.
- *
- * Structurally validates each row: rows that don't carry the required keys
- * are skipped silently. We intentionally avoid throwing — older envelopes
- * with partial shapes still want to flow through the legacy path.
- */
-function readPersistedRuns(envelope) {
-    const raw = envelope.runs;
-    if (!Array.isArray(raw))
-        return null;
-    const out = [];
-    for (const r of raw) {
-        if (!r || typeof r !== "object")
-            continue;
-        const row = r;
-        if (typeof row.task_id !== "string")
-            continue;
-        if (typeof row.arm !== "string")
-            continue;
-        if (typeof row.seed !== "number")
-            continue;
-        if (typeof row.outcome !== "string")
-            continue;
-        // Trajectory shape: tolerate missing/partial sub-object so we don't
-        // reject otherwise-valid rows.
-        const traj = (row.trajectory ?? {});
-        const normalised = {
-            task_id: row.task_id,
-            arm: row.arm,
-            seed: row.seed,
-            model: typeof row.model === "string" ? row.model : "unknown",
-            outcome: row.outcome,
-            tokens: row.tokens ?? { input: 0, output: 0 },
-            wallclock_ms: typeof row.wallclock_ms === "number" ? row.wallclock_ms : 0,
-            verifier_exit_code: typeof row.verifier_exit_code === "number" ? row.verifier_exit_code : 0,
-            trajectory: {
-                correct_asset_loaded: traj.correct_asset_loaded ?? null,
-                feedback_recorded: traj.feedback_recorded ?? null,
-            },
-            assets_loaded: Array.isArray(row.assets_loaded) ? row.assets_loaded : [],
-            failure_mode: typeof row.failure_mode === "string" ? row.failure_mode : null,
-        };
-        out.push(rehydrateRunFromSerialized(normalised));
-    }
-    return out;
-}
-/**
- * Build a synthetic akm-arm RunResult bag from a previously-computed
- * attribution table. Used when we load a §13.3 envelope from disk: the
- * envelope doesn't carry raw RunResults, but the attribution table is
- * lossless w.r.t. (asset, pass/fail) counts — which is all
- * `computePerAssetAttribution` needs to reproduce the top-N ranking.
- *
- * Each synthetic run loads exactly one asset. This over-counts the run
- * total but keeps the per-asset counts faithful to the original table. The
- * synthesised runs are NOT consumed by `runMaskedCorpus` for the masked
- * runs themselves — those go through the injected runner — they only seed
- * `report.akmRuns` so that recomputing the attribution gives back the
- * same top-N ordering.
- */
-function synthesiseAkmRunsFromAttribution(perAsset) {
-    const out = [];
-    for (const row of perAsset.rows) {
-        for (let i = 0; i < row.loadCountPassing; i++) {
-            out.push(makeSyntheticRun("pass", row.assetRef));
-        }
-        for (let i = 0; i < row.loadCountFailing; i++) {
-            out.push(makeSyntheticRun("fail", row.assetRef));
-        }
-    }
-    return out;
-}
-function makeSyntheticRun(outcome, ref) {
-    return {
-        schemaVersion: 1,
-        taskId: "synthetic",
-        arm: "akm",
-        seed: 0,
-        model: "synthetic",
-        outcome,
-        tokens: { input: 0, output: 0 },
-        wallclockMs: 0,
-        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
-        events: [],
-        verifierStdout: "",
-        verifierExitCode: outcome === "pass" ? 0 : 1,
-        assetsLoaded: [ref],
-    };
-}
-function signed(text) {
-    if (text.startsWith("-"))
-        return text;
-    if (text === "0" || text === "0.00" || text === "0.0")
-        return text;
-    return `+${text}`;
-}
-function getEnv(name) {
-    const value = process.env[name];
-    return value && value.length > 0 ? value : undefined;
-}
-function parseInt32(text, fallback) {
-    if (text === undefined)
-        return fallback;
-    const n = Number.parseInt(text, 10);
-    if (!Number.isFinite(n) || n <= 0)
-        return fallback;
-    return n;
-}
-function parseFloatArg(text, fallback) {
-    if (text === undefined)
-        return fallback;
-    const n = Number.parseFloat(text);
-    if (!Number.isFinite(n) || n < 0)
-        return fallback;
-    return n;
-}
-/**
- * Set of obsolete-flag names that have already emitted their stderr
- * warning during this process. Used by `warnObsolete` to dedupe.
- *
- * Exported as a test seam so unit tests can clear the cache between
- * cases without restarting the process.
- */
-export const obsoleteFlagWarned = new Set();
-/**
- * Emit a one-line `[obsolete] ...` warning on stderr the FIRST time the
- * named flag is used in this process. Subsequent calls with the same
- * `name` are silent so a single invocation never emits the same warning
- * twice. Internal: wired into the existing utility/evolve dispatch
- * branches; the new config-file path never triggers these.
- */
-function warnObsolete(name, replacement) {
-    if (obsoleteFlagWarned.has(name))
-        return;
-    obsoleteFlagWarned.add(name);
-    process.stderr.write(`[obsolete] ${name} → ${replacement}\n`);
-}
-/**
- * Detect whether the user supplied any of the flags marked obsolete in
- * BENCH.md §G and emit one warning per used flag. The flags continue to
- * function — the warning is the only behavior change.
- */
-function warnIfObsoleteFlagsUsed(parsed) {
-    if (parsed.bool.has("no-noakm")) {
-        warnObsolete("--no-noakm", 'use `arms: ["akm"]` in a run config');
-    }
-    if (parsed.bool.has("include-synthetic")) {
-        warnObsolete("--include-synthetic", 'use `arms: [..., "synthetic"]` in a run config');
-    }
-    if (parsed.flags.has("opencode-config")) {
-        warnObsolete("--opencode-config", "use `providersRef` in a run config, BENCH_OPENCODE_CONFIG, or ~/.config/akm/bench-providers.json");
-    }
-    if (parsed.flags.has("budget-tokens")) {
-        warnObsolete("--budget-tokens", "use `budgetTokens` in a run config");
-    }
-    if (parsed.flags.has("budget-wall-ms")) {
-        warnObsolete("--budget-wall-ms", "use `budgetWallMs` in a run config");
-    }
-}
-/**
- * `<config.json>` dispatch — load a bench run config, resolve providers
- * and tasks, invoke `runUtility`, and emit the §13.3 envelope on stdout.
- *
- * Behaves identically to `runUtilityCli` from a stdout/stderr/exit-code
- * standpoint: JSON on stdout, optional markdown summary on stderr,
- * returns 0/2/78 mirroring the existing semantics.
- */
-export async function runConfigCli(options) {
-    let resolved;
-    try {
-        const overrides = {};
-        if (options.tasksList)
-            overrides.tasksList = options.tasksList;
-        if (options.seedsPerArm !== undefined)
-            overrides.seedsPerArm = options.seedsPerArm;
-        if (options.parallel !== undefined)
-            overrides.parallel = options.parallel;
-        resolved = loadBenchRunConfig(options.configPath, overrides);
-    }
-    catch (err) {
-        const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
-        return {
-            exitCode,
-            stdout: "",
-            stderr: `bench: ${err instanceof Error ? err.message : String(err)}\n`,
-        };
-    }
-    const report = await runUtility({
-        tasks: resolved.tasks,
-        arms: resolved.arms,
-        model: resolved.model,
-        slice: resolved.slice,
-        ...(resolved.seedsPerArm !== undefined ? { seedsPerArm: resolved.seedsPerArm } : {}),
-        ...(resolved.budgetTokens !== undefined ? { budgetTokens: resolved.budgetTokens } : {}),
-        ...(resolved.budgetWallMs !== undefined ? { budgetWallMs: resolved.budgetWallMs } : {}),
-        ...(resolved.parallel !== undefined ? { parallel: resolved.parallel } : {}),
-        ...(resolved.forceParallel ? { forceParallel: true } : {}),
-        ...(resolved.baselineByTaskId ? { baselineByTaskId: resolved.baselineByTaskId } : {}),
-        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
-        ...(options.branch !== undefined ? { branch: options.branch } : {}),
-        ...(options.commit !== undefined ? { commit: options.commit } : {}),
-        opencodeProviders: resolved.providers,
-        // Synthetic arm follows from `arms` containing "synthetic".
-        ...(resolved.arms.includes("synthetic") ? { includeSynthetic: true } : {}),
-    });
-    const { json, markdown } = renderUtilityReport(report);
-    const stdout = `${JSON.stringify(json, null, 2)}\n`;
-    let stderr = options.json ? "" : `${markdown}\n`;
-    stderr += `bench: config=${resolved.name} tasks=${resolved.tasks.length} arms=${resolved.arms.join(",")} model=${resolved.model}\n`;
-    return { exitCode: 0, stdout, stderr };
-}
-/**
- * `evolve` subcommand. Filters the corpus to one domain (or `all`), then
- * dispatches `runEvolve` and renders the §6.3+§6.4 envelope.
- */
-export async function runEvolveCli(options) {
-    // Discover all tasks then filter on domain.
-    const allTasks = listTasks();
-    const tasks = options.domain === "all" ? allTasks : allTasks.filter((t) => t.domain === options.domain);
-    if (tasks.length === 0) {
-        return {
-            exitCode: 2,
-            stdout: "",
-            stderr: `bench evolve: no tasks matched domain "${options.domain}".\n`,
-        };
-    }
-    const report = await runEvolve({
-        tasks,
-        model: options.model,
-        seedsPerArm: options.seedsPerArm,
-        budgetTokens: options.budgetTokens,
-        budgetWallMs: options.budgetWallMs,
-        negativeThreshold: options.negativeThreshold,
-        ...(options.branch !== undefined ? { branch: options.branch } : {}),
-        ...(options.commit !== undefined ? { commit: options.commit } : {}),
-        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
-    });
-    const { json, markdown } = renderEvolveReport(report);
-    const stdout = `${JSON.stringify(json, null, 2)}\n`;
-    let stderr = options.json ? "" : `${markdown}\n`;
-    stderr += `tasks discovered: ${tasks.length} (domain=${options.domain})\n`;
-    return { exitCode: 0, stdout, stderr };
-}
-async function main(argv) {
-    const parsed = parseArgs(argv);
-    if (parsed.bool.has("help") || parsed.subcommand === "" || parsed.subcommand === "help") {
-        process.stdout.write(HELP);
-        return parsed.subcommand === "" ? 2 : 0;
-    }
-    // Config-file dispatch: when argv[0] looks like a JSON path that exists,
-    // route to `runConfigCli`. This is the new common-case entry point —
-    // existing subcommand-style invocations are untouched.
-    if (parsed.subcommand.endsWith(".json") && fs.existsSync(parsed.subcommand)) {
-        const tasksRaw = parsed.flags.get("tasks");
-        const tasksList = tasksRaw
-            ? tasksRaw
-                .split(",")
-                .map((s) => s.trim())
-                .filter((s) => s.length > 0)
-            : undefined;
-        const seedsRaw = parsed.flags.get("seeds");
-        const parallelRaw = parsed.flags.get("parallel");
-        const removePid = writeBenchPid();
-        let result;
-        try {
-            result = await runConfigCli({
-                configPath: parsed.subcommand,
-                json: parsed.bool.has("json"),
-                ...(tasksList ? { tasksList } : {}),
-                ...(seedsRaw !== undefined ? { seedsPerArm: parseInt32(seedsRaw, 5) } : {}),
-                ...(parallelRaw !== undefined ? { parallel: parseInt32(parallelRaw, 1) } : {}),
-            });
-        }
-        finally {
-            removePid();
-        }
-        if (result.stdout)
-            process.stdout.write(result.stdout);
-        if (result.stderr)
-            process.stderr.write(result.stderr);
-        return result.exitCode;
-    }
-    // Emit obsolete-flag warnings for the legacy subcommand path. The new
-    // config-file path above never reaches this code so config-mode runs
-    // never spuriously warn.
-    warnIfObsoleteFlagsUsed(parsed);
-    switch (parsed.subcommand) {
-        case "utility": {
-            const sliceRaw = parsed.flags.get("tasks") ?? "all";
-            if (sliceRaw !== "train" && sliceRaw !== "eval" && sliceRaw !== "all") {
-                process.stderr.write(`bench utility: invalid --tasks value "${sliceRaw}"; expected one of: all, train, eval.\n`);
-                return 2;
-            }
-            const slice = sliceRaw;
-            // Load provider config at CLI startup (before any runs).
-            let opencodeProviders;
-            try {
-                opencodeProviders = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
-            }
-            catch (err) {
-                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
-                process.stderr.write(`bench utility: ${err instanceof Error ? err.message : String(err)}\n`);
-                return exitCode;
-            }
-            // BENCH_OPENCODE_MODEL: flag → env var → loaded.defaultModel → error exit 2.
-            const model = getEnv("BENCH_OPENCODE_MODEL") ?? opencodeProviders?.defaultModel;
-            if (!model) {
-                process.stderr.write("bench utility: BENCH_OPENCODE_MODEL environment variable is required.\n");
-                return 2;
-            }
-            // Write PID file so `bench kill` can terminate this run.
-            const removePid = writeBenchPid();
-            let result;
-            try {
-                result = await runUtilityCli({
-                    slice,
-                    json: parsed.bool.has("json"),
-                    seedsPerArm: parseInt32(parsed.flags.get("seeds"), 5),
-                    budgetTokens: parseInt32(parsed.flags.get("budget-tokens"), 30000),
-                    budgetWallMs: parseInt32(parsed.flags.get("budget-wall-ms"), 120000),
-                    model,
-                    parallel: parseInt32(parsed.flags.get("parallel"), 1),
-                    ...(parsed.bool.has("force-parallel") ? { forceParallel: true } : {}),
-                    ...(parsed.bool.has("include-synthetic") ? { includeSynthetic: true } : {}),
-                    ...(parsed.bool.has("no-noakm") ? { includeNoakm: false } : {}),
-                    ...(opencodeProviders ? { opencodeProviders } : {}),
-                });
-            }
-            finally {
-                removePid();
-            }
-            if (result.stdout)
-                process.stdout.write(result.stdout);
-            if (result.stderr)
-                process.stderr.write(result.stderr);
-            return result.exitCode;
-        }
-        case "evolve": {
-            const domain = parsed.flags.get("tasks");
-            if (!domain) {
-                process.stderr.write("bench evolve: --tasks <domain> is required (use --tasks all to run the full corpus).\n");
-                return 2;
-            }
-            // Load provider config at CLI startup (before any runs).
-            let opencodeProvidersEvolve;
-            try {
-                opencodeProvidersEvolve = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
-            }
-            catch (err) {
-                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
-                process.stderr.write(`bench evolve: ${err instanceof Error ? err.message : String(err)}\n`);
-                return exitCode;
-            }
-            const model = getEnv("BENCH_OPENCODE_MODEL") ?? opencodeProvidersEvolve?.defaultModel;
-            if (!model) {
-                process.stderr.write("bench evolve: BENCH_OPENCODE_MODEL environment variable is required.\n");
-                return 2;
-            }
-            // Write PID file so `bench kill` can terminate this run.
-            const removePidEvolve = writeBenchPid();
-            let resultEvolve;
-            try {
-                resultEvolve = await runEvolveCli({
-                    domain,
-                    json: parsed.bool.has("json"),
-                    seedsPerArm: parseInt32(parsed.flags.get("seeds"), 5),
-                    budgetTokens: parseInt32(parsed.flags.get("budget-tokens"), 30000),
-                    budgetWallMs: parseInt32(parsed.flags.get("budget-wall-ms"), 120000),
-                    model,
-                    parallel: parseInt32(parsed.flags.get("parallel"), 1),
-                    ...(parsed.bool.has("force-parallel") ? { forceParallel: true } : {}),
-                    negativeThreshold: {
-                        absoluteCount: parseInt32(parsed.flags.get("negative-threshold-count"), 2),
-                        ratio: parseFloatArg(parsed.flags.get("negative-threshold-ratio"), 0.5),
-                    },
-                    ...(opencodeProvidersEvolve ? { opencodeProviders: opencodeProvidersEvolve } : {}),
-                });
-            }
-            finally {
-                removePidEvolve();
-            }
-            if (resultEvolve.stdout)
-                process.stdout.write(resultEvolve.stdout);
-            if (resultEvolve.stderr)
-                process.stderr.write(resultEvolve.stderr);
-            return resultEvolve.exitCode;
-        }
-        case "compare": {
-            const basePath = parsed.flags.get("base");
-            const currentPath = parsed.flags.get("current");
-            if (!basePath || !currentPath) {
-                process.stderr.write("bench compare: --base and --current are both required.\n");
-                return 2;
-            }
-            const result = runCompareCli({
-                basePath,
-                currentPath,
-                json: parsed.bool.has("json"),
-                allowCorpusMismatch: parsed.bool.has("allow-corpus-mismatch"),
-                allowFixtureMismatch: parsed.bool.has("allow-fixture-mismatch"),
-            });
-            if (result.stdout)
-                process.stdout.write(result.stdout);
-            if (result.stderr)
-                process.stderr.write(result.stderr);
-            return result.exitCode;
-        }
-        case "attribute": {
-            const basePath = parsed.flags.get("base");
-            if (!basePath) {
-                process.stderr.write("bench attribute: --base <path> is required.\n");
-                return 2;
-            }
-            // Load provider config for parity (attribute re-runs use the same model + provider).
-            let opencodeProvidersAttr;
-            try {
-                opencodeProvidersAttr = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
-            }
-            catch (err) {
-                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
-                process.stderr.write(`bench attribute: ${err instanceof Error ? err.message : String(err)}\n`);
-                return exitCode;
-            }
-            // Suppress unused variable warning — opencodeProvidersAttr is exposed for
-            // future wiring when runMaskedCorpus accepts it. For now it only validates.
-            void opencodeProvidersAttr;
-            const topN = parseInt32(parsed.flags.get("top"), 5);
-            const result = await runAttributeCli({
-                basePath,
-                topN,
-                json: parsed.bool.has("json"),
-            });
-            if (result.stdout)
-                process.stdout.write(result.stdout);
-            if (result.stderr)
-                process.stderr.write(result.stderr);
-            return result.exitCode;
-        }
-        case "kill": {
-            const pid = readBenchPid();
-            if (pid === undefined) {
-                process.stdout.write("no bench running\n");
-                return 0;
-            }
-            if (!isPidRunning(pid)) {
-                process.stdout.write(`no bench running (stale PID file for PID ${pid} — cleaning up)\n`);
-                try {
-                    const { benchPidPath } = await import("./tmp");
-                    const { default: nodeFs } = await import("node:fs");
-                    nodeFs.rmSync(benchPidPath(), { force: true });
-                }
-                catch {
-                    /* best-effort */
-                }
-                return 0;
-            }
-            try {
-                process.kill(pid, "SIGTERM");
-                process.stdout.write(`sent SIGTERM to bench process ${pid}\n`);
-                process.stdout.write("run `bench kill` again to escalate to SIGKILL if the process does not exit.\n");
-            }
-            catch (err) {
-                process.stderr.write(`bench kill: failed to signal PID ${pid}: ${err instanceof Error ? err.message : String(err)}\n`);
-                return 1;
-            }
-            return 0;
-        }
-        case "clean": {
-            const benchRoot = path.join(getCacheDir(), "bench");
-            try {
-                fs.rmSync(benchRoot, { recursive: true, force: true });
-                process.stderr.write(`bench clean: removed ${benchRoot}\n`);
-            }
-            catch (err) {
-                process.stderr.write(`bench clean: failed to remove ${benchRoot}: ${err.message}\n`);
-                return 1;
-            }
-            return 0;
-        }
-        default:
-            process.stderr.write(`unknown subcommand: ${parsed.subcommand}\n`);
-            process.stderr.write(HELP);
-            return 2;
-    }
-}
-// Only execute when invoked directly. The exported `runUtilityCli` is callable
-// from tests without triggering process.exit.
-if (import.meta.main) {
-    const code = await main(process.argv.slice(2));
-    process.exit(code);
-}