akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/cli.js

@@ -0,0 +1,1018 @@
+#!/usr/bin/env bun
+/**
+ * akm-bench CLI dispatcher.
+ *
+ * Subcommands:
+ *   • `utility`    — paired noakm vs akm utility benchmark (Track A).
+ *   • `compare`    — diff two report JSON files; refuses on hash/model mismatch.
+ *   • `attribute`  — per-asset marginal contribution via leave-one-out masking.
+ *   • `evolve`     — longitudinal evolution loop (Track B). Stub.
+ *   • `kill`       — send SIGTERM to a running bench process (reads bench.pid).
+ *
+ * Implementation status and validity rules live in `tests/bench/BENCH.md`.
+ *
+ * NOTE: The bench binary is intentionally argv-light. citty is the project's
+ * CLI framework but the bench is not part of the public CLI surface, so a
+ * hand-rolled parser keeps the dependency graph tight.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { getCacheDir } from "../../src/core/paths";
+import { listTasks } from "./corpus";
+import { runEvolve } from "./evolve";
+import { compareReports, rehydrateRunFromSerialized, runMaskedCorpus, } from "./metrics";
+import { BenchConfigError, loadOpencodeProviders } from "./opencode-config";
+import { renderAttributionTable, renderCompareMarkdown, renderEvolveReport, renderUtilityReport, } from "./report";
+import { loadBenchRunConfig } from "./run-config";
+import { runUtility } from "./runner";
+import { isPidRunning, readBenchPid, writeBenchPid } from "./tmp";
+const HELP = `akm-bench — agent-plus-akm evaluation framework
+
+Usage:
+  bun run tests/bench/cli.ts <config.json> [--json] [--seeds N] [--parallel N] [--tasks <list>]
+  bun run tests/bench/cli.ts <subcommand> [...flags]
+
+Config-file mode (recommended):
+  Pass a path to a tests/bench/configs/*.json file as the first argument.
+  See tests/bench/BENCH.md for the run-config schema and provider
+  discovery chain (BENCH_OPENCODE_CONFIG → providers/providersRef →
+  ~/.config/akm/bench-providers.json).
+
+Subcommands (legacy; still supported):
+  utility       Track A: paired noakm vs akm utility benchmark.
+  evolve        Track B: longitudinal feedback → distill → propose loop.
+  compare       Diff two report JSON files (refuses cross-model diffs).
+  attribute     Per-asset marginal pass-rate contribution.
+  kill          Send SIGTERM to a running bench process (reads bench.pid).
+  clean         Remove all bench tmp dirs under \${AKM_CACHE_DIR}/bench/.
+
+utility flags:
+  --tasks <slice>          train | eval | all  (default: all)
+  --seeds <N>              seeds per arm  (default: 5)
+  --budget-tokens <N>      per-run token cap (default: 30000)
+  --budget-wall-ms <N>     per-run wallclock cap in ms (default: 120000)
+  --parallel <N>           number of (task, arm, seed) triples to run concurrently
+                           (default: 1 — sequential). Clamped to [1, 8]. Values > 4
+                           print a warning unless --force-parallel is also set.
+  --force-parallel         suppress the high-parallelism warning when --parallel > 4.
+  --no-noakm               exclude the noakm baseline arm (default: included). The noakm arm is
+                           the control condition — pass_rate(akm) − pass_rate(noakm) is the
+                           primary utility metric. Omit only when you are measuring workflow
+                           compliance rather than marginal utility.
+  --include-synthetic      add a third 'synthetic' arm where the model writes/uses its own
+                           scratch notes (no AKM stash). Reports akm_over_synthetic_lift so
+                           operators can see whether AKM beats a self-notes baseline.
+  --opencode-config <path> path to a bench opencode providers JSON file. Auto-discovered
+                           from BENCH_OPENCODE_CONFIG env var or the fixture defaults when
+                           omitted. See BENCH.md for the discovery order.
+  --json                   suppress the markdown summary on stderr (machine-readable only).
+                           Without --json, JSON still goes to stdout and the markdown
+                           summary is also written to stderr for human-friendly reads.
+  -h, --help               show this message.
+
+evolve flags:
+  --tasks <domain>         domain id (e.g., docker-homelab) or 'all'. REQUIRED.
+  --seeds <N>              seeds per arm (default: 5)
+  --budget-tokens <N>      per-run token cap (default: 30000)
+  --budget-wall-ms <N>     per-run wallclock cap in ms (default: 120000)
+  --parallel <N>           same as utility --parallel (default: 1).
+  --force-parallel         suppress the high-parallelism warning when --parallel > 4.
+  --negative-threshold-count <N>  absolute negative-feedback count to evolve (default: 2)
+  --negative-threshold-ratio <R>  ratio of negatives to total feedback (default: 0.5)
+  --opencode-config <path> path to a bench opencode providers JSON file (same as utility).
+  --json                   suppress the markdown summary on stderr.
+
+compare flags:
+  --base <path>                 path to baseline UtilityRunReport JSON file. REQUIRED.
+  --current <path>              path to current  UtilityRunReport JSON file. REQUIRED.
+  --json                        emit the structured CompareResult JSON to stdout instead
+                                of a markdown diff.
+  --allow-corpus-mismatch       proceed even when the two reports disagree on the
+                                selected task IDs / taskCorpusHash. The diff is
+                                rendered with a warning instead of being refused.
+  --allow-fixture-mismatch      proceed even when the two reports disagree on the
+                                fixtureContentHash. Renders a warning instead of
+                                refusing.
+  Exit codes: 0 on successful diff, 1 on refusal (model/corpus/fixture-hash/schema/track mismatch),
+              2 on input errors (missing files, malformed JSON, unknown flags).
+
+attribute flags:
+  --base <path>            path to a §13.3 utility run JSON (required).
+  --top <N>                number of top-loaded assets to mask (default: 5; clamped).
+  --opencode-config <path> path to a bench opencode providers JSON file (same as utility).
+  --json                   suppress the markdown summary on stderr.
+
+kill flags:
+  (none)        Reads \${AKM_CACHE_DIR}/bench/bench.pid and sends SIGTERM.
+                Prints "no bench running" and exits 0 when the PID file is absent.
+                Run again (or press Ctrl-C) to escalate to SIGKILL.
+
+Environment:
+  BENCH_OPENCODE_MODEL      model id stamped into every RunResult. REQUIRED for utility/evolve
+                            unless the loaded providers file supplies a defaultModel.
+  BENCH_OPENCODE_CONFIG     path to the bench opencode providers JSON file. Overrides the
+                            default fixture; overridden by --opencode-config flag.
+
+Auto-discovery order for --opencode-config (first existing file wins):
+  1. --opencode-config <path>  flag value
+  2. BENCH_OPENCODE_CONFIG     env var
+  3. tests/fixtures/bench/opencode-providers.local.json  (gitignored operator overlay)
+  4. tests/fixtures/bench/opencode-providers.json        (committed fixture)
+  5. None found → empty isolated dir, opencode uses cloud-provider defaults.
+
+See tests/bench/BENCH.md for the operator guide.
+`;
+function parseArgs(argv) {
+    const flags = new Map();
+    const bool = new Set();
+    const positional = [];
+    const subcommand = argv[0] ?? "";
+    for (let i = 1; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === "--help" || arg === "-h") {
+            bool.add("help");
+            continue;
+        }
+        if (arg === "--json") {
+            bool.add("json");
+            continue;
+        }
+        if (arg.startsWith("--")) {
+            const eq = arg.indexOf("=");
+            if (eq !== -1) {
+                flags.set(arg.slice(2, eq), arg.slice(eq + 1));
+            }
+            else {
+                const next = argv[i + 1];
+                if (next === undefined || next.startsWith("--")) {
+                    bool.add(arg.slice(2));
+                }
+                else {
+                    flags.set(arg.slice(2), next);
+                    i += 1;
+                }
+            }
+            continue;
+        }
+        positional.push(arg);
+    }
+    return { subcommand, flags, bool, positional };
+}
+/**
+ * Absolute path to the committed default bench opencode providers fixture.
+ * Used as the final fallback in the auto-discovery chain.
+ */
+const DEFAULT_PROVIDERS_PATH = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+/**
+ * Absolute path to the gitignored operator overlay. Takes precedence over
+ * the committed fixture when it exists.
+ */
+const LOCAL_PROVIDERS_PATH = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+/**
+ * Auto-discover and load the bench opencode providers file.
+ *
+ * Discovery order (first existing file wins):
+ *   1. `flagPath`  — `--opencode-config` flag value (already resolved by caller).
+ *   2. `BENCH_OPENCODE_CONFIG` env var.
+ *   3. `tests/fixtures/bench/opencode-providers.local.json` (gitignored overlay).
+ *   4. `tests/fixtures/bench/opencode-providers.json` (committed fixture).
+ *   5. None found → returns `undefined` (empty isolated dir, cloud-provider fallback).
+ *
+ * When a file is found but fails to load, the BenchConfigError is re-thrown
+ * so the caller can map it to the correct exit code.
+ */
+export function discoverOpencodeProviders(flagPath) {
+    const candidates = [
+        flagPath,
+        getEnv("BENCH_OPENCODE_CONFIG"),
+        LOCAL_PROVIDERS_PATH,
+        DEFAULT_PROVIDERS_PATH,
+    ];
+    for (const candidate of candidates) {
+        if (!candidate)
+            continue;
+        const absPath = path.isAbsolute(candidate) ? candidate : path.resolve(candidate);
+        if (fs.existsSync(absPath)) {
+            // Throws BenchConfigError on validation failure; callers propagate it.
+            return loadOpencodeProviders(absPath);
+        }
+        // When a flag or env-var path was given explicitly but doesn't exist,
+        // treat it as a usage error rather than silently skipping to the next
+        // candidate. Only the auto-discovered defaults (local + committed) are
+        // silently skipped when absent.
+        if (candidate === flagPath && flagPath) {
+            // This throws with isUsageError: true (file not found).
+            return loadOpencodeProviders(absPath);
+        }
+        if (candidate === getEnv("BENCH_OPENCODE_CONFIG") && candidate) {
+            return loadOpencodeProviders(absPath);
+        }
+    }
+    return undefined;
+}
+/**
+ * `utility` subcommand. Walks the corpus, runs K seeds per arm per task,
+ * and produces the §13.3 report.
+ *
+ * Returns rather than mutates process to keep this unit-testable. The
+ * `main()` driver below maps the result onto the actual stdout/stderr/exit.
+ */
+export async function runUtilityCli(options) {
+    const sliceFilter = options.slice === "all" ? undefined : options.slice;
+    const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
+    // noakm arm is included by default: it is the control condition for the
+    // primary utility metric (pass_rate(akm) − pass_rate(noakm), spec §4).
+    // Pass --no-noakm (options.includeNoakm === false) to exclude it.
+    const arms = options.includeNoakm === false ? ["akm"] : ["noakm", "akm"];
+    const report = await runUtility({
+        tasks,
+        arms,
+        model: options.model,
+        seedsPerArm: options.seedsPerArm,
+        budgetTokens: options.budgetTokens,
+        budgetWallMs: options.budgetWallMs,
+        slice: options.slice,
+        // #261: thread the synthetic-arm gate. Default off — the envelope shape
+        // is byte-identical to the pre-#261 output unless the operator opts in.
+        ...(options.includeSynthetic ? { includeSynthetic: true } : {}),
+        ...(options.parallel !== undefined ? { parallel: options.parallel } : {}),
+        ...(options.forceParallel ? { forceParallel: true } : {}),
+        ...(options.branch !== undefined ? { branch: options.branch } : {}),
+        ...(options.commit !== undefined ? { commit: options.commit } : {}),
+        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
+        ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+    });
+    const { json, markdown } = renderUtilityReport(report);
+    const jsonText = `${JSON.stringify(json, null, 2)}\n`;
+    const markdownText = `${markdown}\n`;
+    // JSON ALWAYS goes to stdout. This is the bench's machine-readable
+    // contract (matches `tests/benchmark-suite.ts` and the future `bench
+    // compare`/`attribute` subcommands). The `--json` flag means
+    // "suppress the human-friendly markdown summary on stderr"; without it,
+    // both the JSON envelope (stdout) and the markdown summary (stderr) are
+    // emitted so an operator running it interactively gets both views.
+    const stdout = jsonText;
+    let stderr = options.json ? "" : markdownText;
+    stderr += `tasks discovered: ${tasks.length} (slice=${options.slice})\n`;
+    if (tasks.length === 0) {
+        stderr += "no tasks found — corpus is empty or the slice filter excluded all tasks\n";
+    }
+    return { exitCode: 0, stdout, stderr };
+}
+/**
+ * `compare` subcommand. Reads two UtilityRunReport JSON files from disk,
+ * dispatches to `compareReports`, and renders either markdown (default) or
+ * the structured JSON envelope to stdout.
+ *
+ * Exit-code shape:
+ *   • 0 on a successful diff (regardless of whether the diff shows wins).
+ *   • 1 on a refusal (model/hash/schema/track mismatch).
+ *   • 2 on input errors (file missing, malformed JSON).
+ *
+ * Returned `UtilityCliResult` keeps this unit-testable; the `main()` driver
+ * splices the result onto the actual process.
+ */
+export function runCompareCli(options) {
+    let baseRaw;
+    let currentRaw;
+    try {
+        baseRaw = fs.readFileSync(options.basePath, "utf8");
+    }
+    catch (err) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench compare: cannot read --base ${options.basePath}: ${err.message}\n`,
+        };
+    }
+    try {
+        currentRaw = fs.readFileSync(options.currentPath, "utf8");
+    }
+    catch (err) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench compare: cannot read --current ${options.currentPath}: ${err.message}\n`,
+        };
+    }
+    let base;
+    let current;
+    try {
+        base = JSON.parse(baseRaw);
+    }
+    catch (err) {
+        return { exitCode: 2, stdout: "", stderr: `bench compare: malformed JSON in --base: ${err.message}\n` };
+    }
+    try {
+        current = JSON.parse(currentRaw);
+    }
+    catch (err) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench compare: malformed JSON in --current: ${err.message}\n`,
+        };
+    }
+    const result = compareReports(base, current, {
+        ...(options.allowCorpusMismatch ? { allowCorpusMismatch: true } : {}),
+        ...(options.allowFixtureMismatch ? { allowFixtureMismatch: true } : {}),
+    });
+    const stdout = options.json ? `${JSON.stringify(result, null, 2)}\n` : `${renderCompareMarkdown(result)}\n`;
+    let stderr = "";
+    if (!result.ok) {
+        stderr = `bench compare: ${result.message}\n`;
+        return { exitCode: 1, stdout, stderr };
+    }
+    // One-line summary on stderr so an interactive operator sees it without
+    // having to scan the markdown body.
+    const agg = result.aggregate;
+    stderr = `bench compare: pass_rate Δ=${agg.passRateDelta.toFixed(2)} (${agg.passRateSign}); ${result.perTask.length} tasks compared.\n`;
+    for (const w of result.warnings)
+        stderr += `warning: ${w}\n`;
+    return { exitCode: 0, stdout, stderr };
+}
+/**
+ * `attribute` subcommand. Loads a base utility report, picks the top-N
+ * most-loaded assets, masks each in turn, re-runs the corpus, and emits a
+ * marginal-contribution report.
+ *
+ * Cost: N × (tasks × arms × seedsPerArm) re-runs. Reported to stderr up
+ * front so the operator can abort if the projection is too expensive.
+ */
+export async function runAttributeCli(options) {
+    if (!fs.existsSync(options.basePath)) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench attribute: base report not found: ${options.basePath}\n`,
+        };
+    }
+    let baseEnvelope;
+    try {
+        baseEnvelope = JSON.parse(fs.readFileSync(options.basePath, "utf8"));
+    }
+    catch (err) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench attribute: failed to parse ${options.basePath}: ${err instanceof Error ? err.message : String(err)}\n`,
+        };
+    }
+    const corpus = (baseEnvelope.corpus ?? {});
+    const sliceRaw = (corpus.slice ?? "all");
+    const slice = sliceRaw === "train" || sliceRaw === "eval" ? sliceRaw : "all";
+    const sliceFilter = slice === "all" ? undefined : slice;
+    const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
+    const seedsPerArm = typeof corpus.seedsPerArm === "number" ? corpus.seedsPerArm : 5;
+    const agent = (baseEnvelope.agent ?? {});
+    const model = options.modelOverride ?? (typeof agent.model === "string" ? agent.model : "unknown");
+    // The stored envelope's perAsset is in snake_case. Convert it back to the
+    // PerAssetAttribution shape the metrics module expects so we can pass it
+    // through to runMaskedCorpus.
+    const perAssetSerialised = baseEnvelope.perAsset;
+    const perAsset = {
+        totalAkmRuns: perAssetSerialised?.total_akm_runs ?? 0,
+        rows: (perAssetSerialised?.rows ?? []).map((r) => ({
+            assetRef: String(r.asset_ref ?? ""),
+            loadCount: Number(r.load_count ?? 0),
+            loadCountPassing: Number(r.load_count_passing ?? 0),
+            loadCountFailing: Number(r.load_count_failing ?? 0),
+            loadPassRate: r.load_pass_rate === null ? null : Number(r.load_pass_rate),
+        })),
+    };
+    if (perAsset.rows.length === 0) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: "bench attribute: base report has no per-asset attribution rows (no assets loaded). Re-run `bench utility` first.\n",
+        };
+    }
+    const desired = Math.max(1, options.topN);
+    const clamped = Math.min(desired, perAsset.rows.length);
+    // Prefer the persisted `runs[]` array (#249). When the report carries
+    // serialised raw runs we hydrate them back into RunResult shape and feed
+    // them to `runMaskedCorpus` directly — that keeps attribution faithful to
+    // the original (task, arm, seed) bag instead of synthesising stubs from
+    // the per-asset aggregate. Falls back to the legacy aggregate path when
+    // the report pre-dates the `runs[]` field.
+    const persistedRuns = readPersistedRuns(baseEnvelope);
+    const akmRuns = persistedRuns !== null ? persistedRuns.filter((r) => r.arm === "akm") : synthesiseAkmRunsFromAttribution(perAsset);
+    const baseReport = {
+        timestamp: String(baseEnvelope.timestamp ?? ""),
+        branch: String(baseEnvelope.branch ?? ""),
+        commit: String(baseEnvelope.commit ?? ""),
+        model,
+        corpus: {
+            domains: typeof corpus.domains === "number" ? corpus.domains : 0,
+            tasks: typeof corpus.tasks === "number" ? corpus.tasks : tasks.length,
+            slice,
+            seedsPerArm,
+        },
+        aggregateNoakm: extractCorpusMetrics(baseEnvelope, "noakm"),
+        aggregateAkm: extractCorpusMetrics(baseEnvelope, "akm"),
+        aggregateDelta: extractCorpusMetrics(baseEnvelope, "delta"),
+        trajectoryAkm: { correctAssetLoaded: null, feedbackRecorded: 0 },
+        failureModes: { byLabel: {}, byTask: {} },
+        tasks: [],
+        warnings: [],
+        perAsset,
+        akmRuns,
+        taskMetadata: tasks,
+    };
+    const projection = clamped * tasks.length * 2 * seedsPerArm;
+    let stderr = `bench attribute: masking top ${clamped} of ${perAsset.rows.length} assets; ${clamped} × ${tasks.length} tasks × 2 arms × ${seedsPerArm} seeds = ${projection} re-runs.\n`;
+    const baseOptions = {
+        arms: ["noakm", "akm"],
+        model,
+        seedsPerArm,
+    };
+    const maskedRunner = options.runUtility ?? defaultMaskedRunner;
+    let maskedResult;
+    try {
+        maskedResult = await runMaskedCorpus({
+            baseReport,
+            topN: clamped,
+            runUtility: maskedRunner,
+            baseOptions,
+            ...(options.fixturesRoot ? { fixturesRoot: options.fixturesRoot } : {}),
+        });
+    }
+    catch (err) {
+        return {
+            exitCode: 1,
+            stdout: "",
+            stderr: `${stderr}bench attribute: masked-corpus run failed: ${err instanceof Error ? err.message : String(err)}\n`,
+        };
+    }
+    const json = {
+        schemaVersion: 1,
+        track: "attribute",
+        base: { path: options.basePath, model },
+        // Issue #251: surface the masking strategy + the exact masked refs in
+        // the JSON envelope so operators can audit the marginal-contribution
+        // numbers without re-running the masker. The `maskedRefs` order matches
+        // `attributions[]`. Strategy is currently always `"leave-one-out"`;
+        // future strategies extend the union in `MaskedCorpusResult`.
+        attribution: {
+            maskingStrategy: maskedResult.maskingStrategy,
+            maskedRefs: maskedResult.maskedRefs,
+        },
+        maskingStrategy: maskedResult.maskingStrategy,
+        runsPerformed: maskedResult.runsPerformed,
+        perAsset: {
+            total_akm_runs: perAsset.totalAkmRuns,
+            rows: perAsset.rows.map((r) => ({
+                asset_ref: r.assetRef,
+                load_count: r.loadCount,
+                load_count_passing: r.loadCountPassing,
+                load_count_failing: r.loadCountFailing,
+                load_pass_rate: r.loadPassRate,
+            })),
+        },
+        attributions: maskedResult.attributions.map((a) => ({
+            asset_ref: a.assetRef,
+            base_pass_rate: a.basePassRate,
+            masked_pass_rate: a.maskedPassRate,
+            marginal_contribution: a.marginalContribution,
+        })),
+    };
+    const stdout = `${JSON.stringify(json, null, 2)}\n`;
+    if (!options.json) {
+        stderr += `${renderAttributionTable(perAsset)}\n`;
+        stderr += `\n## Marginal contributions (leave-one-out)\n\n`;
+        stderr += `| asset_ref | base_pass_rate | masked_pass_rate | marginal_contribution |\n`;
+        stderr += `|-----------|----------------|------------------|-----------------------|\n`;
+        for (const a of maskedResult.attributions) {
+            stderr += `| \`${a.assetRef}\` | ${a.basePassRate.toFixed(2)} | ${a.maskedPassRate.toFixed(2)} | ${signed(a.marginalContribution.toFixed(2))} |\n`;
+        }
+    }
+    return { exitCode: 0, stdout, stderr };
+}
+/** Default real-runner wrapper for masked re-runs. */
+async function defaultMaskedRunner(options) {
+    const arms = options.arms;
+    return runUtility({
+        tasks: options.tasks,
+        arms,
+        model: options.model,
+        ...(options.seedsPerArm !== undefined ? { seedsPerArm: options.seedsPerArm } : {}),
+        ...(options.budgetTokens !== undefined ? { budgetTokens: options.budgetTokens } : {}),
+        ...(options.budgetWallMs !== undefined ? { budgetWallMs: options.budgetWallMs } : {}),
+        ...(options.slice !== undefined ? { slice: options.slice } : {}),
+        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
+        ...(options.branch !== undefined ? { branch: options.branch } : {}),
+        ...(options.commit !== undefined ? { commit: options.commit } : {}),
+        ...(options.spawn ? { spawn: options.spawn } : {}),
+        ...(options.materialiseStash !== undefined ? { materialiseStash: options.materialiseStash } : {}),
+    });
+}
+/**
+ * Best-effort extractor for `aggregate.<arm>` corpus metrics from the
+ * persisted §13.3 envelope. The envelope keys are snake-cased.
+ */
+function extractCorpusMetrics(envelope, key) {
+    const aggregate = (envelope.aggregate ?? {});
+    const node = (aggregate[key] ?? {});
+    return {
+        passRate: typeof node.pass_rate === "number" ? node.pass_rate : 0,
+        tokensPerPass: node.tokens_per_pass === null ? null : typeof node.tokens_per_pass === "number" ? node.tokens_per_pass : null,
+        tokensPerRun: node.tokens_per_run === null ? null : typeof node.tokens_per_run === "number" ? node.tokens_per_run : null,
+        wallclockMs: typeof node.wallclock_ms === "number" ? node.wallclock_ms : 0,
+    };
+}
+/**
+ * Read the persisted `runs[]` array (#249) from a §13.3 envelope and hydrate
+ * each row back into the in-memory `RunResult` shape. Returns `null` when the
+ * envelope pre-dates the field (legacy reports) so callers can fall back to
+ * the aggregate-only path.
+ *
+ * Structurally validates each row: rows that don't carry the required keys
+ * are skipped silently. We intentionally avoid throwing — older envelopes
+ * with partial shapes still want to flow through the legacy path.
+ */
+function readPersistedRuns(envelope) {
+    const raw = envelope.runs;
+    if (!Array.isArray(raw))
+        return null;
+    const out = [];
+    for (const r of raw) {
+        if (!r || typeof r !== "object")
+            continue;
+        const row = r;
+        if (typeof row.task_id !== "string")
+            continue;
+        if (typeof row.arm !== "string")
+            continue;
+        if (typeof row.seed !== "number")
+            continue;
+        if (typeof row.outcome !== "string")
+            continue;
+        // Trajectory shape: tolerate missing/partial sub-object so we don't
+        // reject otherwise-valid rows.
+        const traj = (row.trajectory ?? {});
+        const normalised = {
+            task_id: row.task_id,
+            arm: row.arm,
+            seed: row.seed,
+            model: typeof row.model === "string" ? row.model : "unknown",
+            outcome: row.outcome,
+            tokens: row.tokens ?? { input: 0, output: 0 },
+            wallclock_ms: typeof row.wallclock_ms === "number" ? row.wallclock_ms : 0,
+            verifier_exit_code: typeof row.verifier_exit_code === "number" ? row.verifier_exit_code : 0,
+            trajectory: {
+                correct_asset_loaded: traj.correct_asset_loaded ?? null,
+                feedback_recorded: traj.feedback_recorded ?? null,
+            },
+            assets_loaded: Array.isArray(row.assets_loaded) ? row.assets_loaded : [],
+            failure_mode: typeof row.failure_mode === "string" ? row.failure_mode : null,
+        };
+        out.push(rehydrateRunFromSerialized(normalised));
+    }
+    return out;
+}
+/**
+ * Build a synthetic akm-arm RunResult bag from a previously-computed
+ * attribution table. Used when we load a §13.3 envelope from disk: the
+ * envelope doesn't carry raw RunResults, but the attribution table is
+ * lossless w.r.t. (asset, pass/fail) counts — which is all
+ * `computePerAssetAttribution` needs to reproduce the top-N ranking.
+ *
+ * Each synthetic run loads exactly one asset. This over-counts the run
+ * total but keeps the per-asset counts faithful to the original table. The
+ * synthesised runs are NOT consumed by `runMaskedCorpus` for the masked
+ * runs themselves — those go through the injected runner — they only seed
+ * `report.akmRuns` so that recomputing the attribution gives back the
+ * same top-N ordering.
+ */
+function synthesiseAkmRunsFromAttribution(perAsset) {
+    const out = [];
+    for (const row of perAsset.rows) {
+        for (let i = 0; i < row.loadCountPassing; i++) {
+            out.push(makeSyntheticRun("pass", row.assetRef));
+        }
+        for (let i = 0; i < row.loadCountFailing; i++) {
+            out.push(makeSyntheticRun("fail", row.assetRef));
+        }
+    }
+    return out;
+}
+function makeSyntheticRun(outcome, ref) {
+    return {
+        schemaVersion: 1,
+        taskId: "synthetic",
+        arm: "akm",
+        seed: 0,
+        model: "synthetic",
+        outcome,
+        tokens: { input: 0, output: 0 },
+        wallclockMs: 0,
+        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
+        events: [],
+        verifierStdout: "",
+        verifierExitCode: outcome === "pass" ? 0 : 1,
+        assetsLoaded: [ref],
+    };
+}
+function signed(text) {
+    if (text.startsWith("-"))
+        return text;
+    if (text === "0" || text === "0.00" || text === "0.0")
+        return text;
+    return `+${text}`;
+}
+function getEnv(name) {
+    const value = process.env[name];
+    return value && value.length > 0 ? value : undefined;
+}
+function parseInt32(text, fallback) {
+    if (text === undefined)
+        return fallback;
+    const n = Number.parseInt(text, 10);
+    if (!Number.isFinite(n) || n <= 0)
+        return fallback;
+    return n;
+}
+function parseFloatArg(text, fallback) {
+    if (text === undefined)
+        return fallback;
+    const n = Number.parseFloat(text);
+    if (!Number.isFinite(n) || n < 0)
+        return fallback;
+    return n;
+}
+/**
+ * Set of obsolete-flag names that have already emitted their stderr
+ * warning during this process. Used by `warnObsolete` to dedupe.
+ *
+ * Exported as a test seam so unit tests can clear the cache between
+ * cases without restarting the process.
+ */
+export const obsoleteFlagWarned = new Set();
+/**
+ * Emit a one-line `[obsolete] ...` warning on stderr the FIRST time the
+ * named flag is used in this process. Subsequent calls with the same
+ * `name` are silent so a single invocation never emits the same warning
+ * twice. Internal: wired into the existing utility/evolve dispatch
+ * branches; the new config-file path never triggers these.
+ */
+function warnObsolete(name, replacement) {
+    if (obsoleteFlagWarned.has(name))
+        return;
+    obsoleteFlagWarned.add(name);
+    process.stderr.write(`[obsolete] ${name} → ${replacement}\n`);
+}
+/**
+ * Detect whether the user supplied any of the flags marked obsolete in
+ * BENCH.md §G and emit one warning per used flag. The flags continue to
+ * function — the warning is the only behavior change.
+ */
+function warnIfObsoleteFlagsUsed(parsed) {
+    if (parsed.bool.has("no-noakm")) {
+        warnObsolete("--no-noakm", 'use `arms: ["akm"]` in a run config');
+    }
+    if (parsed.bool.has("include-synthetic")) {
+        warnObsolete("--include-synthetic", 'use `arms: [..., "synthetic"]` in a run config');
+    }
+    if (parsed.flags.has("opencode-config")) {
+        warnObsolete("--opencode-config", "use `providersRef` in a run config, BENCH_OPENCODE_CONFIG, or ~/.config/akm/bench-providers.json");
+    }
+    if (parsed.flags.has("budget-tokens")) {
+        warnObsolete("--budget-tokens", "use `budgetTokens` in a run config");
+    }
+    if (parsed.flags.has("budget-wall-ms")) {
+        warnObsolete("--budget-wall-ms", "use `budgetWallMs` in a run config");
+    }
+}
+/**
+ * `<config.json>` dispatch — load a bench run config, resolve providers
+ * and tasks, invoke `runUtility`, and emit the §13.3 envelope on stdout.
+ *
+ * Behaves identically to `runUtilityCli` from a stdout/stderr/exit-code
+ * standpoint: JSON on stdout, optional markdown summary on stderr,
+ * returns 0/2/78 mirroring the existing semantics.
+ */
+export async function runConfigCli(options) {
+    let resolved;
+    try {
+        const overrides = {};
+        if (options.tasksList)
+            overrides.tasksList = options.tasksList;
+        if (options.seedsPerArm !== undefined)
+            overrides.seedsPerArm = options.seedsPerArm;
+        if (options.parallel !== undefined)
+            overrides.parallel = options.parallel;
+        resolved = loadBenchRunConfig(options.configPath, overrides);
+    }
+    catch (err) {
+        const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
+        return {
+            exitCode,
+            stdout: "",
+            stderr: `bench: ${err instanceof Error ? err.message : String(err)}\n`,
+        };
+    }
+    const report = await runUtility({
+        tasks: resolved.tasks,
+        arms: resolved.arms,
+        model: resolved.model,
+        slice: resolved.slice,
+        ...(resolved.seedsPerArm !== undefined ? { seedsPerArm: resolved.seedsPerArm } : {}),
+        ...(resolved.budgetTokens !== undefined ? { budgetTokens: resolved.budgetTokens } : {}),
+        ...(resolved.budgetWallMs !== undefined ? { budgetWallMs: resolved.budgetWallMs } : {}),
+        ...(resolved.parallel !== undefined ? { parallel: resolved.parallel } : {}),
+        ...(resolved.forceParallel ? { forceParallel: true } : {}),
+        ...(resolved.baselineByTaskId ? { baselineByTaskId: resolved.baselineByTaskId } : {}),
+        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
+        ...(options.branch !== undefined ? { branch: options.branch } : {}),
+        ...(options.commit !== undefined ? { commit: options.commit } : {}),
+        opencodeProviders: resolved.providers,
+        // Synthetic arm follows from `arms` containing "synthetic".
+        ...(resolved.arms.includes("synthetic") ? { includeSynthetic: true } : {}),
+    });
+    const { json, markdown } = renderUtilityReport(report);
+    const stdout = `${JSON.stringify(json, null, 2)}\n`;
+    let stderr = options.json ? "" : `${markdown}\n`;
+    stderr += `bench: config=${resolved.name} tasks=${resolved.tasks.length} arms=${resolved.arms.join(",")} model=${resolved.model}\n`;
+    return { exitCode: 0, stdout, stderr };
+}
+/**
+ * `evolve` subcommand. Filters the corpus to one domain (or `all`), then
+ * dispatches `runEvolve` and renders the §6.3+§6.4 envelope.
+ */
+export async function runEvolveCli(options) {
+    // Discover all tasks then filter on domain.
+    const allTasks = listTasks();
+    const tasks = options.domain === "all" ? allTasks : allTasks.filter((t) => t.domain === options.domain);
+    if (tasks.length === 0) {
+        return {
+            exitCode: 2,
+            stdout: "",
+            stderr: `bench evolve: no tasks matched domain "${options.domain}".\n`,
+        };
+    }
+    const report = await runEvolve({
+        tasks,
+        model: options.model,
+        seedsPerArm: options.seedsPerArm,
+        budgetTokens: options.budgetTokens,
+        budgetWallMs: options.budgetWallMs,
+        negativeThreshold: options.negativeThreshold,
+        ...(options.branch !== undefined ? { branch: options.branch } : {}),
+        ...(options.commit !== undefined ? { commit: options.commit } : {}),
+        ...(options.timestamp !== undefined ? { timestamp: options.timestamp } : {}),
+    });
+    const { json, markdown } = renderEvolveReport(report);
+    const stdout = `${JSON.stringify(json, null, 2)}\n`;
+    let stderr = options.json ? "" : `${markdown}\n`;
+    stderr += `tasks discovered: ${tasks.length} (domain=${options.domain})\n`;
+    return { exitCode: 0, stdout, stderr };
+}
+async function main(argv) {
+    const parsed = parseArgs(argv);
+    if (parsed.bool.has("help") || parsed.subcommand === "" || parsed.subcommand === "help") {
+        process.stdout.write(HELP);
+        return parsed.subcommand === "" ? 2 : 0;
+    }
+    // Config-file dispatch: when argv[0] looks like a JSON path that exists,
+    // route to `runConfigCli`. This is the new common-case entry point —
+    // existing subcommand-style invocations are untouched.
+    if (parsed.subcommand.endsWith(".json") && fs.existsSync(parsed.subcommand)) {
+        const tasksRaw = parsed.flags.get("tasks");
+        const tasksList = tasksRaw
+            ? tasksRaw
+                .split(",")
+                .map((s) => s.trim())
+                .filter((s) => s.length > 0)
+            : undefined;
+        const seedsRaw = parsed.flags.get("seeds");
+        const parallelRaw = parsed.flags.get("parallel");
+        const removePid = writeBenchPid();
+        let result;
+        try {
+            result = await runConfigCli({
+                configPath: parsed.subcommand,
+                json: parsed.bool.has("json"),
+                ...(tasksList ? { tasksList } : {}),
+                ...(seedsRaw !== undefined ? { seedsPerArm: parseInt32(seedsRaw, 5) } : {}),
+                ...(parallelRaw !== undefined ? { parallel: parseInt32(parallelRaw, 1) } : {}),
+            });
+        }
+        finally {
+            removePid();
+        }
+        if (result.stdout)
+            process.stdout.write(result.stdout);
+        if (result.stderr)
+            process.stderr.write(result.stderr);
+        return result.exitCode;
+    }
+    // Emit obsolete-flag warnings for the legacy subcommand path. The new
+    // config-file path above never reaches this code so config-mode runs
+    // never spuriously warn.
+    warnIfObsoleteFlagsUsed(parsed);
+    switch (parsed.subcommand) {
+        case "utility": {
+            const sliceRaw = parsed.flags.get("tasks") ?? "all";
+            if (sliceRaw !== "train" && sliceRaw !== "eval" && sliceRaw !== "all") {
+                process.stderr.write(`bench utility: invalid --tasks value "${sliceRaw}"; expected one of: all, train, eval.\n`);
+                return 2;
+            }
+            const slice = sliceRaw;
+            // Load provider config at CLI startup (before any runs).
+            let opencodeProviders;
+            try {
+                opencodeProviders = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
+            }
+            catch (err) {
+                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
+                process.stderr.write(`bench utility: ${err instanceof Error ? err.message : String(err)}\n`);
+                return exitCode;
+            }
+            // BENCH_OPENCODE_MODEL: flag → env var → loaded.defaultModel → error exit 2.
+            const model = getEnv("BENCH_OPENCODE_MODEL") ?? opencodeProviders?.defaultModel;
+            if (!model) {
+                process.stderr.write("bench utility: BENCH_OPENCODE_MODEL environment variable is required.\n");
+                return 2;
+            }
+            // Write PID file so `bench kill` can terminate this run.
+            const removePid = writeBenchPid();
+            let result;
+            try {
+                result = await runUtilityCli({
+                    slice,
+                    json: parsed.bool.has("json"),
+                    seedsPerArm: parseInt32(parsed.flags.get("seeds"), 5),
+                    budgetTokens: parseInt32(parsed.flags.get("budget-tokens"), 30000),
+                    budgetWallMs: parseInt32(parsed.flags.get("budget-wall-ms"), 120000),
+                    model,
+                    parallel: parseInt32(parsed.flags.get("parallel"), 1),
+                    ...(parsed.bool.has("force-parallel") ? { forceParallel: true } : {}),
+                    ...(parsed.bool.has("include-synthetic") ? { includeSynthetic: true } : {}),
+                    ...(parsed.bool.has("no-noakm") ? { includeNoakm: false } : {}),
+                    ...(opencodeProviders ? { opencodeProviders } : {}),
+                });
+            }
+            finally {
+                removePid();
+            }
+            if (result.stdout)
+                process.stdout.write(result.stdout);
+            if (result.stderr)
+                process.stderr.write(result.stderr);
+            return result.exitCode;
+        }
+        case "evolve": {
+            const domain = parsed.flags.get("tasks");
+            if (!domain) {
+                process.stderr.write("bench evolve: --tasks <domain> is required (use --tasks all to run the full corpus).\n");
+                return 2;
+            }
+            // Load provider config at CLI startup (before any runs).
+            let opencodeProvidersEvolve;
+            try {
+                opencodeProvidersEvolve = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
+            }
+            catch (err) {
+                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
+                process.stderr.write(`bench evolve: ${err instanceof Error ? err.message : String(err)}\n`);
+                return exitCode;
+            }
+            const model = getEnv("BENCH_OPENCODE_MODEL") ?? opencodeProvidersEvolve?.defaultModel;
+            if (!model) {
+                process.stderr.write("bench evolve: BENCH_OPENCODE_MODEL environment variable is required.\n");
+                return 2;
+            }
+            // Write PID file so `bench kill` can terminate this run.
+            const removePidEvolve = writeBenchPid();
+            let resultEvolve;
+            try {
+                resultEvolve = await runEvolveCli({
+                    domain,
+                    json: parsed.bool.has("json"),
+                    seedsPerArm: parseInt32(parsed.flags.get("seeds"), 5),
+                    budgetTokens: parseInt32(parsed.flags.get("budget-tokens"), 30000),
+                    budgetWallMs: parseInt32(parsed.flags.get("budget-wall-ms"), 120000),
+                    model,
+                    parallel: parseInt32(parsed.flags.get("parallel"), 1),
+                    ...(parsed.bool.has("force-parallel") ? { forceParallel: true } : {}),
+                    negativeThreshold: {
+                        absoluteCount: parseInt32(parsed.flags.get("negative-threshold-count"), 2),
+                        ratio: parseFloatArg(parsed.flags.get("negative-threshold-ratio"), 0.5),
+                    },
+                    ...(opencodeProvidersEvolve ? { opencodeProviders: opencodeProvidersEvolve } : {}),
+                });
+            }
+            finally {
+                removePidEvolve();
+            }
+            if (resultEvolve.stdout)
+                process.stdout.write(resultEvolve.stdout);
+            if (resultEvolve.stderr)
+                process.stderr.write(resultEvolve.stderr);
+            return resultEvolve.exitCode;
+        }
+        case "compare": {
+            const basePath = parsed.flags.get("base");
+            const currentPath = parsed.flags.get("current");
+            if (!basePath || !currentPath) {
+                process.stderr.write("bench compare: --base and --current are both required.\n");
+                return 2;
+            }
+            const result = runCompareCli({
+                basePath,
+                currentPath,
+                json: parsed.bool.has("json"),
+                allowCorpusMismatch: parsed.bool.has("allow-corpus-mismatch"),
+                allowFixtureMismatch: parsed.bool.has("allow-fixture-mismatch"),
+            });
+            if (result.stdout)
+                process.stdout.write(result.stdout);
+            if (result.stderr)
+                process.stderr.write(result.stderr);
+            return result.exitCode;
+        }
+        case "attribute": {
+            const basePath = parsed.flags.get("base");
+            if (!basePath) {
+                process.stderr.write("bench attribute: --base <path> is required.\n");
+                return 2;
+            }
+            // Load provider config for parity (attribute re-runs use the same model + provider).
+            let opencodeProvidersAttr;
+            try {
+                opencodeProvidersAttr = discoverOpencodeProviders(parsed.flags.get("opencode-config"));
+            }
+            catch (err) {
+                const exitCode = err instanceof BenchConfigError && err.isUsageError ? 2 : 78;
+                process.stderr.write(`bench attribute: ${err instanceof Error ? err.message : String(err)}\n`);
+                return exitCode;
+            }
+            // Suppress unused variable warning — opencodeProvidersAttr is exposed for
+            // future wiring when runMaskedCorpus accepts it. For now it only validates.
+            void opencodeProvidersAttr;
+            const topN = parseInt32(parsed.flags.get("top"), 5);
+            const result = await runAttributeCli({
+                basePath,
+                topN,
+                json: parsed.bool.has("json"),
+            });
+            if (result.stdout)
+                process.stdout.write(result.stdout);
+            if (result.stderr)
+                process.stderr.write(result.stderr);
+            return result.exitCode;
+        }
+        case "kill": {
+            const pid = readBenchPid();
+            if (pid === undefined) {
+                process.stdout.write("no bench running\n");
+                return 0;
+            }
+            if (!isPidRunning(pid)) {
+                process.stdout.write(`no bench running (stale PID file for PID ${pid} — cleaning up)\n`);
+                try {
+                    const { benchPidPath } = await import("./tmp");
+                    const { default: nodeFs } = await import("node:fs");
+                    nodeFs.rmSync(benchPidPath(), { force: true });
+                }
+                catch {
+                    /* best-effort */
+                }
+                return 0;
+            }
+            try {
+                process.kill(pid, "SIGTERM");
+                process.stdout.write(`sent SIGTERM to bench process ${pid}\n`);
+                process.stdout.write("run `bench kill` again to escalate to SIGKILL if the process does not exit.\n");
+            }
+            catch (err) {
+                process.stderr.write(`bench kill: failed to signal PID ${pid}: ${err instanceof Error ? err.message : String(err)}\n`);
+                return 1;
+            }
+            return 0;
+        }
+        case "clean": {
+            const benchRoot = path.join(getCacheDir(), "bench");
+            try {
+                fs.rmSync(benchRoot, { recursive: true, force: true });
+                process.stderr.write(`bench clean: removed ${benchRoot}\n`);
+            }
+            catch (err) {
+                process.stderr.write(`bench clean: failed to remove ${benchRoot}: ${err.message}\n`);
+                return 1;
+            }
+            return 0;
+        }
+        default:
+            process.stderr.write(`unknown subcommand: ${parsed.subcommand}\n`);
+            process.stderr.write(HELP);
+            return 2;
+    }
+}
+// Only execute when invoked directly. The exported `runUtilityCli` is callable
+// from tests without triggering process.exit.
+if (import.meta.main) {
+    const code = await main(process.argv.slice(2));
+    process.exit(code);
+}