akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/runner.js

@@ -0,0 +1,699 @@
+/**
+ * akm-bench K-seed runner (spec §5 + §6).
+ *
+ * `runUtility(options)` is the single entry point used by both the CLI
+ * dispatcher (`tests/bench/cli.ts utility`) and unit tests. It expands the
+ * caller's `(tasks × arms × seeds)` cartesian product, calls `runOne` for
+ * each triple, splices the trajectory record back in, and returns a
+ * `UtilityRunReport` that `renderUtilityReport` can stamp into JSON +
+ * markdown.
+ *
+ * Per-(arm, seed) isolation:
+ *   • Workspace: each (task, arm, seed) gets a fresh tmp dir seeded from the
+ *     task's `workspace/` template so runs cannot pollute each other.
+ *   • Stash: only the `akm` arm materialises a stash via `loadFixtureStash`.
+ *     We materialise once per task (the stash content is identical across
+ *     the K seeds) and reuse it.
+ *
+ * Cleanup: every tmp resource is wrapped in `try/finally`. We never leak
+ * tmp dirs even on harness exceptions.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { warn } from "../../src/core/warn";
+import { computeFixtureContentHash, loadFixtureStash } from "../fixtures/stashes/load";
+import { registerCleanup } from "./cleanup";
+import { computeTaskCorpusHash, readTaskBody } from "./corpus";
+import { runOne } from "./driver";
+import { validateFixtureCorpus } from "./environment";
+import { aggregateCorpus, aggregateFailureModes, aggregatePerTask, aggregateTrajectory, classifyFailureMode, computeCorpusDelta, computePerAssetAttribution, computePerTaskDelta, computeSearchBridge, extractAssetLoads, extractGoldRanks, } from "./metrics";
+import { resolveGitBranch, resolveGitCommit } from "./report";
+import { benchMkdtemp, benchTmpRoot } from "./tmp";
+import { computeTrajectory } from "./trajectory";
+import { evaluateRunAgainstAllSpecs, } from "./workflow-evaluator";
+import { loadAllWorkflowSpecs } from "./workflow-spec";
+import { normalizeRunToTrace } from "./workflow-trace";
+/** Checkpoint write interval: write a partial file every N completed runs. */
+const CHECKPOINT_INTERVAL = 5;
+/** Partial file max age before cleanup: 24 hours in milliseconds. */
+const PARTIAL_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+/**
+ * Emit a one-line progress update to stderr after each (task, arm, seed)
+ * completes. Goes to stderr even when --json is passed so operators always
+ * have a heartbeat signal during long runs.
+ *
+ * Format: `[<completed>/<total>] <taskId> <arm> <outcome> <wallclockSeconds>s`
+ */
+function emitProgress(completed, total, run) {
+    const secs = Math.round(run.wallclockMs / 1000);
+    process.stderr.write(`[${completed}/${total}] ${run.taskId} ${run.arm} ${run.outcome} ${secs}s\n`);
+}
+/**
+ * Write a partial checkpoint file under `${AKM_CACHE_DIR}/bench/`.
+ * The file contains the runs completed so far plus a `partial: true` marker
+ * and a `summary.total_runs_completed` counter. Old partial files (>24h)
+ * are not cleaned up here — that is done at startup via `cleanupOldPartials`.
+ */
+function writePartialCheckpoint(runs, timestamp) {
+    try {
+        const root = benchTmpRoot();
+        const filename = `bench-partial-${timestamp.replace(/[:.]/g, "-")}.json`;
+        const outPath = path.join(root, filename);
+        const envelope = {
+            partial: true,
+            summary: {
+                total_runs_completed: runs.length,
+            },
+            timestamp,
+            runs: runs.map((r) => ({
+                task_id: r.taskId,
+                arm: r.arm,
+                seed: r.seed,
+                model: r.model,
+                outcome: r.outcome,
+                wallclock_ms: r.wallclockMs,
+            })),
+        };
+        fs.writeFileSync(outPath, JSON.stringify(envelope, null, 2), "utf8");
+    }
+    catch {
+        // Checkpoint writes are best-effort — never abort a run for a write failure.
+    }
+}
+/**
+ * Remove partial checkpoint files older than 24 hours from the bench tmp root.
+ * Called once at the start of `runUtility` to reap orphans from prior crashed runs.
+ */
+function cleanupOldPartials() {
+    try {
+        const root = benchTmpRoot();
+        const now = Date.now();
+        const entries = fs.readdirSync(root);
+        for (const entry of entries) {
+            if (!entry.startsWith("bench-partial-"))
+                continue;
+            const fullPath = path.join(root, entry);
+            try {
+                const stat = fs.statSync(fullPath);
+                if (now - stat.mtimeMs > PARTIAL_MAX_AGE_MS) {
+                    fs.unlinkSync(fullPath);
+                }
+            }
+            catch {
+                /* swallow per-file errors */
+            }
+        }
+    }
+    catch {
+        /* swallow — cleanup is best-effort */
+    }
+}
+/**
+ * Default workflows directory. Can be overridden by callers (tests) via
+ * `RunUtilityOptions.workflowsDir`. Specs in this directory are loaded ONCE
+ * per `runUtility` call (not per run) — the evaluator filters via each spec's
+ * `applies_to` so we don't I/O in the hot loop.
+ */
+const DEFAULT_WORKFLOWS_DIR = path.resolve(__dirname, "..", "fixtures", "bench", "workflows");
+/**
+ * Run `items` in batches of `n` concurrently, calling `fn` for each item.
+ * Batches are executed sequentially; within each batch all items run with
+ * `Promise.all`. This gives bounded concurrency without a full work-queue.
+ */
+async function runInBatches(items, n, fn) {
+    for (let i = 0; i < items.length; i += n) {
+        await Promise.all(items.slice(i, i + n).map(fn));
+    }
+}
+/**
+ * Run K seeds × len(arms) × len(tasks) and return the §13.3 report.
+ *
+ * The function is robust to per-run failures — `runOne` already captures
+ * every failure path into a RunResult, so the runner only has to worry
+ * about its own infrastructure (stash materialisation, workspace copy).
+ * Those failures are recorded as `harness_error` runs.
+ *
+ * When `options.parallel > 1`, work items are batched and run concurrently
+ * via `runInBatches`. The shared `warnings`, `goldRankRecords`, and
+ * `workflowChecks` arrays are updated atomically at the end of each item so
+ * no JS-level races occur (Node/Bun is single-threaded).
+ */
+export async function runUtility(options) {
+    const seedsPerArm = options.seedsPerArm ?? 5;
+    const budgetTokens = options.budgetTokens ?? 30000;
+    const budgetWallMs = options.budgetWallMs ?? 120000;
+    const slice = options.slice ?? "all";
+    const materialiseStash = options.materialiseStash ?? true;
+    // Clamp parallel to [1, 8].
+    const parallel = Math.min(8, Math.max(1, options.parallel ?? 1));
+    if (parallel > 4 && !options.forceParallel) {
+        process.stderr.write(`bench: --parallel ${parallel} exceeds 4; high concurrency may overwhelm local providers. ` +
+            `Pass --force-parallel to suppress this warning.\n`);
+    }
+    // Clean up orphaned partial files from prior crashed runs (best-effort).
+    cleanupOldPartials();
+    const grouped = new Map();
+    const warnings = [];
+    // Validate all task stash references before starting any work. Missing
+    // fixtures produce harness_error at run time; better to surface them loudly
+    // at startup with the fixture name than to discover them per-seed mid-run.
+    if (materialiseStash && options.arms.includes("akm")) {
+        const { missing } = validateFixtureCorpus(options.tasks);
+        for (const [fixture, taskIds] of missing) {
+            const w = `fixture "${fixture}" missing MANIFEST.json — tasks will harness_error: ${taskIds.join(", ")}`;
+            process.stderr.write(`bench: WARNING: ${w}\n`);
+            warnings.push(w);
+        }
+    }
+    const goldRankRecords = [];
+    // Progress tracking: compute total run count upfront so progress lines show
+    // `[7/40]` rather than an unbounded counter.
+    const armsForProgress = options.includeSynthetic
+        ? [...new Set([...options.arms, "synthetic"])]
+        : options.arms;
+    const totalRuns = options.tasks.length * armsForProgress.length * seedsPerArm;
+    let completedRuns = 0;
+    // Partial checkpoint accumulator: collects all RunResults as they land so
+    // we can write a partial envelope periodically without keeping duplicates.
+    const allCompletedRuns = [];
+    const runTimestamp = options.timestamp ?? new Date().toISOString();
+    // #257: load workflow specs ONCE per runUtility call. Skipped when the
+    // caller passes an empty `workflowsDir` string (test escape hatch). Errors
+    // are surfaced as warnings — workflow evaluation is best-effort and a
+    // missing/malformed spec must not abort the whole bench run.
+    const workflowSpecs = [];
+    const workflowsDir = options.workflowsDir ?? DEFAULT_WORKFLOWS_DIR;
+    if (workflowsDir.length > 0) {
+        try {
+            const loaded = loadAllWorkflowSpecs(workflowsDir);
+            workflowSpecs.push(...loaded);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            warnings.push(`workflow specs: failed to load from "${workflowsDir}": ${msg}`);
+            warn(`[runUtility] workflow specs unavailable: ${msg}`);
+        }
+    }
+    const workflowChecks = [];
+    for (const task of options.tasks) {
+        const taskRuns = new Map();
+        grouped.set(task.id, taskRuns);
+        // Resolve a caller-supplied stash override before materialising. When
+        // `stashDirByFixture` provides a directory for this task's fixture, we
+        // skip `loadFixtureStash` entirely and forward the override.
+        const overrideStashDir = options.stashDirByFixture?.get(task.stash);
+        // Materialise the akm-arm stash once per task. We share it across the K
+        // seeds because the stash content is identical and re-running `akm
+        // index` for every seed is wasted work.
+        let stash;
+        let stashError;
+        if (options.arms.includes("akm") && materialiseStash && !overrideStashDir) {
+            try {
+                stash = loadFixtureStash(task.stash);
+            }
+            catch (err) {
+                stashError = err instanceof Error ? err.message : String(err);
+                warnings.push(`task ${task.id}: stash "${task.stash}" failed to load: ${stashError}`);
+            }
+        }
+        // SIGINT/SIGTERM trap (#267): register the per-task stash cleanup so an
+        // external signal mid-run reaps the tmp dir we just created.
+        const stashSnapshot = stash;
+        const deregisterStash = stashSnapshot
+            ? registerCleanup(() => {
+                try {
+                    stashSnapshot.cleanup();
+                }
+                catch {
+                    /* swallow */
+                }
+            })
+            : () => { };
+        // #261: when `includeSynthetic` is set, splice the synthetic arm into the
+        // per-task arm iteration alongside whatever the caller asked for. We
+        // dedupe so a caller that already passes `synthetic` in `arms` does not
+        // see it run twice. Pre-#261 callers (no flag, no `synthetic` in arms)
+        // see the old loop verbatim — that's the byte-identical default contract.
+        const armsForTask = (() => {
+            if (!options.includeSynthetic)
+                return options.arms;
+            if (options.arms.includes("synthetic"))
+                return options.arms;
+            return [...options.arms, "synthetic"];
+        })();
+        const workItems = [];
+        for (const arm of armsForTask) {
+            taskRuns.set(arm, []);
+            for (let seed = 0; seed < seedsPerArm; seed += 1) {
+                workItems.push({ arm, seed });
+            }
+        }
+        // Per-run worker: resolves stash/prompt, executes runOneIsolated, then
+        // splices the result into the shared accumulators. Because Bun/Node is
+        // single-threaded these splices are race-free even across concurrent
+        // awaits — only one microtask runs at a time between yield points.
+        const runItem = async ({ arm, seed }) => {
+            // Resolve the stashDir we'll forward to the agent. The akm arm
+            // always carries a stashDir so AKM_STASH_DIR is set in the child
+            // env — this is how downstream tooling (and the trajectory parser
+            // event-stream lookup) distinguishes arms. When the operator opted
+            // out of fixture materialisation (tests, dry-run), we still pass a
+            // stable placeholder so the env keys are wired correctly.
+            let stashDir;
+            if (arm === "akm") {
+                // Resolution order (must match the issue #251 acceptance criteria):
+                //   1. Per-task explicit override (used by `runMaskedCorpus` to
+                //      point at a tmp stash with one asset removed). Highest
+                //      priority because attribution correctness depends on this
+                //      branch never being shadowed by the `__no-stash__`
+                //      placeholder fallback.
+                //   2. Per-(task, arm)-call `stashDirByFixture` override (Phase
+                //      3 evolve persistence).
+                //   3. Per-task materialised fixture stash from `loadFixtureStash`.
+                //   4. `materialiseStash: false` placeholder so AKM_STASH_DIR is
+                //      still wired into the child env.
+                if (task.stashDirOverride)
+                    stashDir = task.stashDirOverride;
+                else if (overrideStashDir)
+                    stashDir = overrideStashDir;
+                else if (stash)
+                    stashDir = stash.stashDir;
+                else if (!materialiseStash)
+                    stashDir = path.join(task.taskDir, "__no-stash__");
+            }
+            // Build the prompt-override (#267). The builder is invoked once
+            // per (task, arm) — seeds share a prompt. `undefined` keeps the
+            // driver's default prompt in play.
+            //
+            // #261: the synthetic arm has a scratch-notes prompt contract —
+            // the model is told no AKM stash is available and instructed to
+            // write/use its own procedural notes. When the caller does not
+            // supply a `buildPrompt` override for the synthetic arm we fall
+            // back to a built-in scratch-notes prompt so the contract is
+            // honoured by every utility-track caller, not just `runEvolve`.
+            let promptOverride = options.buildPrompt?.(task, arm);
+            if (promptOverride === undefined && arm === "synthetic") {
+                promptOverride = buildUtilitySyntheticPrompt(task.id);
+            }
+            // Collect per-run warnings separately and merge after the run so
+            // concurrent runs don't interleave partial warning sequences.
+            const runWarnings = [];
+            const run = await runOneIsolated({
+                task,
+                arm,
+                seed,
+                model: options.model,
+                stashDir,
+                budgetTokens,
+                budgetWallMs,
+                spawn: options.spawn,
+                warnings: runWarnings,
+                ...(promptOverride !== undefined ? { prompt: promptOverride } : {}),
+                ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+                ...(stash?.indexCacheHome ? { indexCacheHome: stash.indexCacheHome } : {}),
+            });
+            // Merge per-run warnings into the shared array.
+            if (runWarnings.length > 0)
+                warnings.push(...runWarnings);
+            taskRuns.get(arm)?.push(run);
+            // Emit a compact progress line to stderr (unconditional — even under
+            // --json so operators have a heartbeat during long runs).
+            completedRuns += 1;
+            emitProgress(completedRuns, totalRuns, run);
+            // Accumulate for partial checkpointing.
+            allCompletedRuns.push(run);
+            if (completedRuns % CHECKPOINT_INTERVAL === 0) {
+                writePartialCheckpoint(allCompletedRuns, runTimestamp);
+            }
+            // §6.7 search-pipeline bridge: only the akm arm consults the stash,
+            // and we only attribute ranks for tasks with a gold ref. Both
+            // guards mean noakm and gold-less runs are silently excluded.
+            if (arm === "akm" && task.goldRef) {
+                const searches = extractGoldRanks(run, task.goldRef);
+                goldRankRecords.push({
+                    taskId: task.id,
+                    arm,
+                    seed,
+                    outcome: run.outcome,
+                    goldRef: task.goldRef,
+                    searches,
+                });
+            }
+            // #257: evaluate the akm-arm run against every workflow spec. The
+            // evaluator's `specApplies` filter handles applicability (arm,
+            // domain, gold ref, repeated-failures threshold), so we hand it the
+            // entire spec list and append whatever it returns. noakm/synthetic
+            // arms are not evaluated — workflow specs target the akm arm.
+            if (arm === "akm" && workflowSpecs.length > 0) {
+                const trace = normalizeRunToTrace(run, {
+                    warnings: runWarnings,
+                    harness: {
+                        agentStartedTs: run.startedAt,
+                        agentFinishedTs: run.finishedAt,
+                    },
+                });
+                const runCtx = {
+                    arm: run.arm,
+                    taskId: run.taskId,
+                    seed: run.seed,
+                    outcome: run.outcome,
+                };
+                const taskMetadata = buildWorkflowTaskMetadata(task, trace);
+                const checks = evaluateRunAgainstAllSpecs(trace, workflowSpecs, runCtx, taskMetadata);
+                workflowChecks.push(...checks);
+            }
+        };
+        try {
+            await runInBatches(workItems, parallel, runItem);
+        }
+        finally {
+            // Deregister BEFORE running cleanup so a SIGINT arriving during this
+            // block doesn't double-fire the cleanup (per cleanup.ts contract).
+            deregisterStash();
+            stash?.cleanup();
+        }
+    }
+    return buildReport({
+        grouped,
+        options,
+        seedsPerArm,
+        slice,
+        warnings,
+        goldRankRecords,
+        workflowChecks,
+    });
+}
+function buildWorkflowTaskMetadata(task, trace) {
+    const flags = {
+        search_has_relevant_result: searchResultIncludesGoldRef(trace, task.goldRef),
+        task_has_tests: taskHasTests(task),
+    };
+    return {
+        ...(task.goldRef !== undefined ? { goldRef: task.goldRef } : {}),
+        flags,
+    };
+}
+function searchResultIncludesGoldRef(trace, goldRef) {
+    if (!goldRef)
+        return false;
+    for (const event of trace.events) {
+        if (event.type !== "akm_search")
+            continue;
+        if (event.resultRefs?.includes(goldRef))
+            return true;
+    }
+    return false;
+}
+function taskHasTests(task) {
+    if (task.verifier === "pytest")
+        return true;
+    const testsDir = path.join(task.taskDir, "tests");
+    if (!fs.existsSync(testsDir))
+        return false;
+    try {
+        return fs.readdirSync(testsDir).some((name) => name.endsWith(".py") || name.endsWith(".sh"));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Set up a fresh workspace for one (task, arm, seed) triple, run `runOne`
+ * against it, splice in the trajectory record, then tear everything down.
+ */
+async function runOneIsolated(args) {
+    const workspace = benchMkdtemp(`akm-bench-ws-${args.task.domain}-`);
+    // SIGINT trap: register workspace cleanup so external signals don't leak
+    // tmp dirs. Deregistered in `finally` before we do the synchronous rm so
+    // the handler doesn't double-fire (per cleanup.ts contract).
+    const deregisterWorkspace = registerCleanup(() => {
+        try {
+            fs.rmSync(workspace, { recursive: true, force: true });
+        }
+        catch {
+            /* swallow */
+        }
+    });
+    try {
+        seedWorkspace(args.task.taskDir, workspace);
+        const runOptions = {
+            track: "utility",
+            arm: args.arm,
+            taskId: args.task.id,
+            taskTitle: args.task.title,
+            workspace,
+            model: args.model,
+            seed: args.seed,
+            budgetTokens: args.budgetTokens,
+            budgetWallMs: args.budgetWallMs,
+            verifier: args.task.verifier,
+            taskDir: args.task.taskDir,
+            ...(args.task.expectedMatch ? { expectedMatch: args.task.expectedMatch } : {}),
+            ...(args.task.akmKeywords ? { akmKeywords: args.task.akmKeywords } : {}),
+            ...(args.stashDir ? { stashDir: args.stashDir } : {}),
+            ...(args.spawn ? { spawn: args.spawn } : {}),
+            ...(args.prompt !== undefined ? { prompt: args.prompt } : {}),
+            warnings: args.warnings,
+            ...(args.opencodeProviders ? { opencodeProviders: args.opencodeProviders } : {}),
+            ...(args.indexCacheHome ? { indexCacheHome: args.indexCacheHome } : {}),
+        };
+        const result = await runOne(runOptions);
+        // Splice in the trajectory metric. The driver always returns
+        // `{ null, null }` — this is where the real values get filled.
+        const trajectory = computeTrajectory({ goldRef: args.task.goldRef }, result, {
+            warnings: args.warnings,
+        });
+        // Per-asset attribution is post-processing on the trace; it's free, so we
+        // run it on every (task, arm, seed) result. The driver emits an empty
+        // assetsLoaded[]; this is where the real refs get filled. Spec §6.5.
+        const assetsLoaded = extractAssetLoads(result);
+        // Splice in the failure-mode label. Only the akm arm carries one; the
+        // noakm baseline is the control and isn't part of the §6.6 to-do list.
+        // `classifyFailureMode` returns null for non-failed runs.
+        const failureMode = args.arm === "akm" ? classifyFailureMode(args.task, { ...result, trajectory, assetsLoaded }) : null;
+        return { ...result, trajectory, assetsLoaded, failureMode };
+    }
+    finally {
+        deregisterWorkspace();
+        fs.rmSync(workspace, { recursive: true, force: true });
+    }
+}
+/**
+ * Copy the task's `workspace/` template into the per-run tmp dir. If the
+ * task has no `workspace/` (loader-test fixtures), the run starts with an
+ * empty cwd — that is also valid for verifier-only tasks.
+ */
+function seedWorkspace(taskDir, dest) {
+    const src = path.join(taskDir, "workspace");
+    if (!fs.existsSync(src))
+        return;
+    copyDirRecursive(src, dest);
+}
+/**
+ * Default synthetic-arm prompt (#261). Used by Track A `runUtility` when the
+ * caller opts in via `includeSynthetic: true` and does not also supply a
+ * `buildPrompt` override for the synthetic arm.
+ *
+ * The prompt is a clear scratch-notes contract: the model is told no AKM
+ * stash is available and instructed to write/use its own procedural notes
+ * before solving the task. This mirrors the prompt shape used by Track B's
+ * `buildSyntheticPrompt(taskId)` but is intentionally duplicated here so
+ * Track A has no module-level dependency on `evolve.ts`.
+ *
+ * Exported for tests.
+ */
+export function buildUtilitySyntheticPrompt(taskId) {
+    return [
+        `Task: ${taskId}`,
+        "Arm: synthetic (Bring Your Own Skills)",
+        "No akm stash is available; AKM_STASH_DIR is intentionally absent. Before solving",
+        "the task, write a short scratchpad of the skills and steps you intend to use,",
+        "then proceed. Cite the scratchpad in your trace so the verifier can attribute",
+        "the approach to your own reasoning rather than retrieved guidance.",
+    ].join("\n");
+}
+function copyDirRecursive(src, dest) {
+    fs.mkdirSync(dest, { recursive: true });
+    const entries = fs.readdirSync(src, { withFileTypes: true });
+    for (const entry of entries) {
+        if (entry.name === ".gitkeep")
+            continue;
+        const s = path.join(src, entry.name);
+        const d = path.join(dest, entry.name);
+        if (entry.isDirectory())
+            copyDirRecursive(s, d);
+        else if (entry.isFile())
+            fs.copyFileSync(s, d);
+    }
+}
+function buildReport(args) {
+    const tasks = [];
+    const noakmPerTask = {};
+    const akmPerTask = {};
+    const synthPerTask = {};
+    const akmRunsAll = [];
+    const allRuns = [];
+    const includeSynth = args.options.includeSynthetic === true;
+    // #257: index workflow checks by taskId so we can attach a per-task
+    // mean compliance to each `UtilityReportTaskEntry`. Only `pass` and
+    // `partial` statuses contribute non-zero scores; `not_applicable` is
+    // skipped (the spec did not target this run); `harness_error` rolls in
+    // as a 0 so corrupt traces drag the per-task number down.
+    const checksByTask = new Map();
+    for (const c of args.workflowChecks) {
+        const arr = checksByTask.get(c.taskId);
+        if (arr)
+            arr.push(c);
+        else
+            checksByTask.set(c.taskId, [c]);
+    }
+    for (const task of args.options.tasks) {
+        const taskRuns = args.grouped.get(task.id);
+        const noakmRuns = taskRuns?.get("noakm") ?? [];
+        const akmRuns = taskRuns?.get("akm") ?? [];
+        // #261: synthetic-arm runs are only consulted when the caller opted in.
+        // A missing arm is NOT a zero-pass arm — we leave `synthPerTask[task.id]`
+        // unset rather than defaulting to a zeroed PerTaskMetrics so downstream
+        // consumers can distinguish "arm not run" from "arm ran with 0 passes".
+        const synthRuns = includeSynth ? (taskRuns?.get("synthetic") ?? []) : [];
+        const noakmMetrics = aggregatePerTask(noakmRuns);
+        const akmMetrics = aggregatePerTask(akmRuns);
+        const delta = computePerTaskDelta(noakmMetrics, akmMetrics);
+        noakmPerTask[task.id] = noakmMetrics;
+        akmPerTask[task.id] = akmMetrics;
+        if (includeSynth) {
+            synthPerTask[task.id] = aggregatePerTask(synthRuns);
+        }
+        akmRunsAll.push(...akmRuns);
+        // Preserve arm order (noakm, synthetic when enabled, then akm) so the
+        // persisted runs[] array is deterministic across reruns. #249. The
+        // synthetic block is omitted entirely when includeSynth is false so the
+        // pre-#261 envelope stays byte-identical.
+        if (includeSynth) {
+            allRuns.push(...noakmRuns, ...synthRuns, ...akmRuns);
+        }
+        else {
+            allRuns.push(...noakmRuns, ...akmRuns);
+        }
+        // #257: per-task workflow compliance, mean of `score` over applicable
+        // checks (excludes `not_applicable`). Undefined when this task has no
+        // applicable checks at all so downstream renderers can distinguish
+        // "not measured" from "measured at 0".
+        const taskChecks = checksByTask.get(task.id) ?? [];
+        const applicableTaskChecks = taskChecks.filter((c) => c.status !== "not_applicable");
+        let workflowCompliance;
+        if (applicableTaskChecks.length > 0) {
+            let sum = 0;
+            for (const c of applicableTaskChecks)
+                sum += c.score;
+            workflowCompliance = sum / applicableTaskChecks.length;
+        }
+        tasks.push({
+            id: task.id,
+            noakm: noakmMetrics,
+            akm: akmMetrics,
+            delta,
+            ...(includeSynth ? { synthetic: aggregatePerTask(synthRuns) } : {}),
+            ...(workflowCompliance !== undefined ? { workflowCompliance } : {}),
+        });
+    }
+    const aggregateNoakm = aggregateCorpus(noakmPerTask);
+    const aggregateAkm = aggregateCorpus(akmPerTask);
+    const aggregateDelta = computeCorpusDelta(aggregateNoakm, aggregateAkm);
+    // #261: synthetic-arm aggregate is built ONLY when the caller opted in.
+    // We compute it once here so the report renderer can stamp `arms.synthetic`
+    // and `akm_over_synthetic_lift` without recomputing.
+    const aggregateSynth = includeSynth ? aggregateCorpus(synthPerTask) : undefined;
+    const trajectoryAkm = aggregateTrajectory(akmRunsAll);
+    // Failure-mode aggregate (§6.6). Walks every akm-arm run; runs that are
+    // not "fail" carry `failureMode: null` and are skipped here.
+    const failureEntries = [];
+    for (const r of akmRunsAll) {
+        if (r.failureMode)
+            failureEntries.push({ taskId: r.taskId, mode: r.failureMode });
+    }
+    const failureModes = aggregateFailureModes(failureEntries);
+    const domains = new Set(args.options.tasks.map((t) => t.domain)).size;
+    const branch = args.options.branch ?? resolveGitBranch();
+    const commit = args.options.commit ?? resolveGitCommit();
+    const timestamp = args.options.timestamp ?? new Date().toISOString();
+    // §6.7 — compute the search-pipeline bridge once over the whole corpus.
+    // The function tolerates an empty record list (renders the N/A sentence
+    // downstream).
+    const searchBridge = computeSearchBridge({ goldRankRecords: args.goldRankRecords });
+    // #250 — stamp deterministic corpus + fixture identity into the report
+    // so `bench compare` can refuse cross-corpus / cross-fixture diffs unless
+    // the operator explicitly opts in via --allow-corpus-mismatch /
+    // --allow-fixture-mismatch.
+    const selectedTaskIds = [...args.options.tasks.map((t) => t.id)].sort();
+    const taskBodies = new Map();
+    for (const t of args.options.tasks)
+        taskBodies.set(t.id, readTaskBody(t.taskDir));
+    const taskCorpusHash = computeTaskCorpusHash(selectedTaskIds, taskBodies);
+    const fixtureNames = [...new Set(args.options.tasks.map((t) => t.stash))].sort();
+    const fixtures = {};
+    for (const name of fixtureNames) {
+        try {
+            fixtures[name] = computeFixtureContentHash(name);
+        }
+        catch (err) {
+            // Loader-test tasks point at fixtures that may not exist on disk; we
+            // still want to stamp identity for the present fixtures, so we record
+            // the failure as a warning and continue with the remaining set.
+            args.warnings.push(`corpus stamp: cannot hash fixture "${name}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // Combined fixture-content hash. Hash input is the same `<name>\0<hash>\0`
+    // pattern used elsewhere — order-stable because `fixtureNames` is sorted.
+    const combinedHash = createHash("sha256");
+    for (const name of fixtureNames) {
+        combinedHash.update(name);
+        combinedHash.update("\0");
+        combinedHash.update(fixtures[name] ?? "");
+        combinedHash.update("\0");
+    }
+    const fixtureContentHash = combinedHash.digest("hex");
+    const baseReport = {
+        timestamp,
+        branch,
+        commit,
+        model: args.options.model,
+        corpus: {
+            domains,
+            tasks: args.options.tasks.length,
+            slice: args.slice,
+            seedsPerArm: args.seedsPerArm,
+            selectedTaskIds,
+            taskCorpusHash,
+            fixtures,
+            fixtureContentHash,
+        },
+        aggregateNoakm,
+        aggregateAkm,
+        aggregateDelta,
+        ...(aggregateSynth ? { aggregateSynth } : {}),
+        trajectoryAkm,
+        failureModes,
+        tasks,
+        warnings: args.warnings,
+        akmRuns: akmRunsAll,
+        allRuns,
+        taskMetadata: args.options.tasks,
+        goldRankRecords: args.goldRankRecords,
+        searchBridge,
+        workflowChecks: args.workflowChecks,
+    };
+    // Compute per-asset attribution as post-processing on the akm-arm runs
+    // we just collected. This is the §6.5 "free" diagnostic — it runs on every
+    // utility invocation, no extra spawns.
+    baseReport.perAsset = computePerAssetAttribution(baseReport);
+    // Stamp the optional baseline pass-rate map onto the report so the
+    // renderer surfaces a `vs base` column in markdown and a
+    // `baseline_by_task_id` field in JSON. Additive — when the caller did
+    // not pass a baseline the report shape is byte-identical to before.
+    if (args.options.baselineByTaskId) {
+        baseReport.baselineByTaskId = { ...args.options.baselineByTaskId };
+    }
+    return baseReport;
+}