akm-cli 0.6.1 → 0.7.0-rc1

package/dist/tests/bench/metrics.js

@@ -0,0 +1,2319 @@
+/**
+ * akm-bench metrics (spec §6).
+ *
+ * Outcome metrics (§6.1) and trajectory metrics (§6.2). Both are pure
+ * functions over `RunResult[]` slices so the runner can compose them
+ * however it likes. The §6.3+ catalog (proposal-quality, longitudinal,
+ * attribution, failure-mode taxonomy) lands in #239/#240/#243.
+ *
+ * The failure-mode taxonomy classifier (§6.6) lives in this file
+ * (`classifyFailureMode`).
+ *
+ * Search-pipeline bridge metrics (§6.7) are below: they tie the synthetic
+ * MRR/Recall@K view in `tests/benchmark-suite.ts` to real-task pass rate
+ * by logging gold-rank-of-search per `akm search` invocation and slicing
+ * pass-rate by the rank of the agent's *chosen* search.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { safeRealpath } from "../../src/core/common";
+import { MEMORY_ABILITY_VALUES } from "./corpus";
+import { serializeRunForReport } from "./report";
+import { benchMkdtemp } from "./tmp";
+import { normalizeRunToTrace } from "./workflow-trace";
+/**
+ * Aggregate outcome metrics over a flat list of RunResults.
+ *
+ * Aggregations across multiple arms are the caller's responsibility — pass
+ * each arm's slice in separately. Backward-compatible v1 contract; the
+ * richer per-task / corpus shapes below subsume this.
+ */
+export function computeOutcomeAggregate(results) {
+    if (results.length === 0) {
+        return { passRate: 0, tokensPerPass: 0, wallclockMs: 0, budgetExceeded: 0, runsWithMeasuredTokens: 0 };
+    }
+    let passes = 0;
+    let budgetExceeded = 0;
+    let totalTokensInMeasuredPasses = 0;
+    let measuredPasses = 0;
+    let runsWithMeasuredTokens = 0;
+    let totalWallclock = 0;
+    for (const r of results) {
+        totalWallclock += r.wallclockMs;
+        if (isMeasured(r)) {
+            runsWithMeasuredTokens += 1;
+        }
+        if (r.outcome === "pass") {
+            passes += 1;
+            // Only fold tokens into the mean when we actually measured them
+            // (issue #252) — otherwise a `0` would silently understate cost.
+            if (isMeasured(r)) {
+                measuredPasses += 1;
+                totalTokensInMeasuredPasses += r.tokens.input + r.tokens.output;
+            }
+        }
+        else if (r.outcome === "budget_exceeded") {
+            budgetExceeded += 1;
+        }
+    }
+    return {
+        passRate: passes / results.length,
+        tokensPerPass: measuredPasses === 0 ? 0 : totalTokensInMeasuredPasses / measuredPasses,
+        wallclockMs: totalWallclock / results.length,
+        budgetExceeded,
+        runsWithMeasuredTokens,
+    };
+}
+/**
+ * Treat older artefacts without `tokenMeasurement` as `"parsed"` for backward
+ * compatibility — pre-#252 reports always returned numeric zero, and rejecting
+ * them entirely would break compare/attribute over historical runs.
+ */
+function isMeasured(r) {
+    return (r.tokenMeasurement ?? "parsed") === "parsed";
+}
+/**
+ * Aggregate K seed runs of one (task, arm) pair into PerTaskMetrics. Returns
+ * a zeroed envelope on empty input — callers decide whether to skip or render.
+ */
+export function aggregatePerTask(results) {
+    if (results.length === 0) {
+        return {
+            passRate: 0,
+            passAt1: 0,
+            tokensPerPass: null,
+            wallclockMs: 0,
+            passRateStdev: 0,
+            budgetExceededCount: 0,
+            harnessErrorCount: 0,
+            count: 0,
+            runsWithMeasuredTokens: 0,
+        };
+    }
+    let passes = 0;
+    let measuredPasses = 0;
+    let totalTokensInMeasuredPasses = 0;
+    let totalWallclock = 0;
+    let budgetExceeded = 0;
+    let harnessError = 0;
+    let runsWithMeasuredTokens = 0;
+    // For the standard deviation we need a fixed-iteration buffer of pass/fail.
+    const passSamples = [];
+    for (const r of results) {
+        totalWallclock += r.wallclockMs;
+        if (isMeasured(r)) {
+            runsWithMeasuredTokens += 1;
+        }
+        const isPass = r.outcome === "pass" ? 1 : 0;
+        passSamples.push(isPass);
+        if (isPass === 1) {
+            passes += 1;
+            // Only count tokens for measured passes (issue #252). A pass with
+            // missing measurement contributes to `passRate` but NOT to
+            // `tokensPerPass` — preserving "tokens per measured pass" semantics.
+            if (isMeasured(r)) {
+                measuredPasses += 1;
+                totalTokensInMeasuredPasses += r.tokens.input + r.tokens.output;
+            }
+        }
+        else if (r.outcome === "budget_exceeded") {
+            budgetExceeded += 1;
+        }
+        else if (r.outcome === "harness_error") {
+            harnessError += 1;
+        }
+    }
+    const seed0 = results.find((r) => r.seed === 0) ?? results[0];
+    const passAt1 = seed0 && seed0.outcome === "pass" ? 1 : 0;
+    return {
+        passRate: passes / results.length,
+        passAt1,
+        tokensPerPass: measuredPasses === 0 ? null : totalTokensInMeasuredPasses / measuredPasses,
+        wallclockMs: totalWallclock / results.length,
+        passRateStdev: stdev(passSamples),
+        budgetExceededCount: budgetExceeded,
+        harnessErrorCount: harnessError,
+        count: results.length,
+        runsWithMeasuredTokens,
+    };
+}
+/** Sample standard deviation. Returns 0 for length ≤ 1 (no spread to measure). */
+function stdev(values) {
+    if (values.length <= 1)
+        return 0;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    const sumSq = values.reduce((acc, v) => acc + (v - mean) * (v - mean), 0);
+    // Sample stdev (Bessel's correction) — n-1 denominator.
+    return Math.sqrt(sumSq / (values.length - 1));
+}
+/**
+ * Mean across per-task metrics. Each task contributes once, regardless of
+ * how many seeds it ran (K is already collapsed in `aggregatePerTask`).
+ *
+ * `tokensPerPass`: tasks where `tokensPerPass` is `null` (no passes) are
+ * dropped from that mean. The result is `null` if every task failed.
+ */
+export function aggregateCorpus(perTask) {
+    const tasks = Object.values(perTask);
+    if (tasks.length === 0) {
+        return { passRate: 0, tokensPerPass: null, wallclockMs: 0 };
+    }
+    const passRate = tasks.reduce((a, t) => a + t.passRate, 0) / tasks.length;
+    const wallclockMs = tasks.reduce((a, t) => a + t.wallclockMs, 0) / tasks.length;
+    const tppValues = tasks.map((t) => t.tokensPerPass).filter((v) => v !== null);
+    const tokensPerPass = tppValues.length === 0 ? null : tppValues.reduce((a, b) => a + b, 0) / tppValues.length;
+    return { passRate, tokensPerPass, wallclockMs };
+}
+/**
+ * Compute the akm − noakm delta. Negative `tokensPerPass`/`wallclockMs` mean
+ * akm was cheaper / faster; positive means it cost more. Pass-rate uses the
+ * opposite convention (positive = akm wins).
+ */
+export function computeCorpusDelta(noakm, akm) {
+    return {
+        passRate: akm.passRate - noakm.passRate,
+        tokensPerPass: akm.tokensPerPass === null || noakm.tokensPerPass === null ? null : akm.tokensPerPass - noakm.tokensPerPass,
+        wallclockMs: akm.wallclockMs - noakm.wallclockMs,
+    };
+}
+/** Per-task delta with the same null-safety as the corpus delta. */
+export function computePerTaskDelta(noakm, akm) {
+    return {
+        passRate: akm.passRate - noakm.passRate,
+        tokensPerPass: akm.tokensPerPass === null || noakm.tokensPerPass === null ? null : akm.tokensPerPass - noakm.tokensPerPass,
+        wallclockMs: akm.wallclockMs - noakm.wallclockMs,
+    };
+}
+/**
+ * Extract the domain prefix from a task ID. The corpus convention is
+ * `<domain>/<task-name>`; we split on the first `/`. Tasks lacking a slash
+ * fall back to the literal `unknown` bucket so they aggregate predictably
+ * rather than producing per-task domains-of-one.
+ */
+export function domainOfTaskId(taskId) {
+    const idx = taskId.indexOf("/");
+    if (idx <= 0)
+        return "unknown";
+    return taskId.slice(0, idx);
+}
+/**
+ * Compute the negative-transfer aggregate over a set of per-task entries
+ * (one entry per task; both arms already aggregated into PerTaskMetrics).
+ *
+ * A task is "regressed" when `akm.passRate < noakm.passRate`. Ties (equal
+ * pass rate, including 0=0) are NOT regressions. `topRegressedTasks` is
+ * sorted by `severity` descending then `taskId` ascending so output is
+ * deterministic.
+ */
+export function computeNegativeTransfer(tasks) {
+    const regressed = [];
+    for (const t of tasks) {
+        const delta = t.akm.passRate - t.noakm.passRate;
+        if (delta >= 0)
+            continue;
+        regressed.push({
+            taskId: t.id,
+            domain: domainOfTaskId(t.id),
+            noakmPassRate: t.noakm.passRate,
+            akmPassRate: t.akm.passRate,
+            delta,
+            severity: -delta,
+        });
+    }
+    regressed.sort((a, b) => {
+        if (b.severity !== a.severity)
+            return b.severity - a.severity;
+        return a.taskId.localeCompare(b.taskId);
+    });
+    const severity = regressed.reduce((acc, r) => acc + r.severity, 0);
+    return { count: regressed.length, severity, topRegressedTasks: regressed };
+}
+/**
+ * Compute per-domain aggregates over a set of per-task entries. Each task
+ * contributes once to its domain (K seeds already collapsed). Output rows
+ * are sorted by `domain` ascending so JSON / markdown are byte-stable.
+ *
+ * Domain extraction uses `domainOfTaskId` (split on first `/`).
+ */
+export function computeDomainAggregates(tasks) {
+    const buckets = new Map();
+    for (const t of tasks) {
+        const d = domainOfTaskId(t.id);
+        let arr = buckets.get(d);
+        if (!arr) {
+            arr = [];
+            buckets.set(d, arr);
+        }
+        arr.push(t);
+    }
+    const rows = [];
+    for (const [domain, group] of buckets) {
+        const n = group.length;
+        let noakmSum = 0;
+        let akmSum = 0;
+        let wallNoakm = 0;
+        let wallAkm = 0;
+        let regressionCount = 0;
+        const noakmTpp = [];
+        const akmTpp = [];
+        for (const t of group) {
+            noakmSum += t.noakm.passRate;
+            akmSum += t.akm.passRate;
+            wallNoakm += t.noakm.wallclockMs;
+            wallAkm += t.akm.wallclockMs;
+            if (t.akm.passRate < t.noakm.passRate)
+                regressionCount += 1;
+            if (t.noakm.tokensPerPass !== null)
+                noakmTpp.push(t.noakm.tokensPerPass);
+            if (t.akm.tokensPerPass !== null)
+                akmTpp.push(t.akm.tokensPerPass);
+        }
+        const passRateNoakm = noakmSum / n;
+        const passRateAkm = akmSum / n;
+        const meanNoakmTpp = noakmTpp.length === 0 ? null : noakmTpp.reduce((a, b) => a + b, 0) / noakmTpp.length;
+        const meanAkmTpp = akmTpp.length === 0 ? null : akmTpp.reduce((a, b) => a + b, 0) / akmTpp.length;
+        const tokensPerPassDelta = meanNoakmTpp === null || meanAkmTpp === null ? null : meanAkmTpp - meanNoakmTpp;
+        rows.push({
+            domain,
+            taskCount: n,
+            regressionCount,
+            passRateNoakm,
+            passRateAkm,
+            passRateDelta: passRateAkm - passRateNoakm,
+            tokensPerPassDelta,
+            wallclockMsDelta: wallAkm / n - wallNoakm / n,
+        });
+    }
+    rows.sort((a, b) => a.domain.localeCompare(b.domain));
+    return rows;
+}
+/**
+ * Compute asset-regression-candidate rows (#260). Walks the AKM-arm runs,
+ * keeps only those whose `taskId` is in `regressedTaskIds`, and tallies how
+ * often each loaded asset shows up. `regressedTaskCount` (distinct task IDs
+ * touched) is the primary sort key — assets that hurt many tasks are more
+ * actionable than assets that flooded one task across seeds.
+ *
+ * Sort: regressedTaskCount desc, totalLoadCount desc, assetRef asc.
+ */
+export function computeAssetRegressionCandidates(regressedTaskIds, akmRuns) {
+    const regressed = new Set(regressedTaskIds);
+    if (regressed.size === 0)
+        return [];
+    const taskIdsByAsset = new Map();
+    const totalLoadByAsset = new Map();
+    for (const run of akmRuns) {
+        if (!regressed.has(run.taskId))
+            continue;
+        const assets = run.assetsLoaded ?? [];
+        for (const ref of assets) {
+            let bucket = taskIdsByAsset.get(ref);
+            if (!bucket) {
+                bucket = new Set();
+                taskIdsByAsset.set(ref, bucket);
+            }
+            bucket.add(run.taskId);
+            totalLoadByAsset.set(ref, (totalLoadByAsset.get(ref) ?? 0) + 1);
+        }
+    }
+    const rows = [];
+    for (const [assetRef, taskIds] of taskIdsByAsset) {
+        rows.push({
+            assetRef,
+            regressedTaskCount: taskIds.size,
+            regressedTaskIds: [...taskIds].sort(),
+            totalLoadCount: totalLoadByAsset.get(assetRef) ?? 0,
+        });
+    }
+    rows.sort((a, b) => {
+        if (b.regressedTaskCount !== a.regressedTaskCount)
+            return b.regressedTaskCount - a.regressedTaskCount;
+        if (b.totalLoadCount !== a.totalLoadCount)
+            return b.totalLoadCount - a.totalLoadCount;
+        return a.assetRef.localeCompare(b.assetRef);
+    });
+    return rows;
+}
+// ── Per-asset attribution (§6.5) ───────────────────────────────────────────
+/**
+ * Extract the unique asset refs an agent loaded during a run by scanning
+ * `events[]` and `verifierStdout` for `akm show <ref>` invocations.
+ *
+ * Detection strategy (all heuristic, all conservative):
+ *   1. `event.eventType === "show"` with `event.ref` (forward-compat — akm
+ *      itself does not currently emit `show` events).
+ *   2. Substring match on `akm show <ref>` in stdout. The ref shape is
+ *      `[origin//]type:name` per the v1 contract; we accept word-boundary
+ *      terminators after the name.
+ *   3. Tool-call JSON `{"args":["show","<ref>"]}` — the form opencode logs
+ *      when the agent invokes the akm CLI as a tool. We extract refs that
+ *      look like asset refs from the args array entries adjacent to "show".
+ *
+ * Returns refs in first-seen order, deduplicated. Bounded scan: stdout is
+ * truncated at 16 MiB (the same cap the trajectory parser uses) to keep
+ * runaway agents from OOMing the bench.
+ */
+const ASSET_LOAD_STDOUT_SCAN_CAP = 16 * 1024 * 1024;
+// Asset ref grammar: optional `origin//` prefix, type:name, where type and
+// name are lowercase letters, digits, `_`, `-`. We deliberately do NOT match
+// `://` schemes (those are install locators, not asset refs). The character
+// class is intentionally tight so we don't mis-pickup arbitrary words after
+// `akm show`. The `name` segment is restricted to `[A-Za-z0-9_-]+` (no `/`,
+// no `.`) — the v1 grammar in src/core/asset-ref.ts permits `/` and `.` in
+// names (e.g. `script:db/migrate/run.sh`), but the masker treats names as
+// untrusted input and rejects any traversal-shaped value, so the bench-side
+// scanner does not need (or want) to extract such refs from agent stdout.
+// Limiting the regex here is defense-in-depth against a prompt-injected
+// agent emitting `akm show "skill:../../etc"` and us pulling that ref into
+// the masking flow.
+const ASSET_REF_PATTERN = /(?:[a-z0-9_-]+\/\/)?[a-z][a-z0-9_-]*:[A-Za-z0-9_-]+/g;
+export function extractAssetLoads(runResult) {
+    const seen = new Set();
+    const out = [];
+    const push = (ref) => {
+        if (!ref)
+            return;
+        if (seen.has(ref))
+            return;
+        seen.add(ref);
+        out.push(ref);
+    };
+    // 1. Events stream.
+    for (const event of runResult.events) {
+        if (event.eventType === "show" && typeof event.ref === "string") {
+            push(event.ref);
+        }
+        const meta = event.metadata;
+        if (meta && typeof meta === "object" && event.eventType === "show") {
+            const candidate = meta.ref;
+            if (typeof candidate === "string")
+                push(candidate);
+        }
+    }
+    // 2 & 3. Stdout scanning. Bound the scan so a runaway agent stdout cannot
+    // OOM the bench. Truncation is silent — the trajectory parser already
+    // surfaces a warning for the same data on its own scan.
+    let haystack = runResult.verifierStdout || "";
+    if (haystack.length > ASSET_LOAD_STDOUT_SCAN_CAP) {
+        haystack = haystack.slice(0, ASSET_LOAD_STDOUT_SCAN_CAP);
+    }
+    // `akm show <ref>` literal form. Accept optional quoting around the ref so
+    // shell traces like `akm show "skill:foo"` work too.
+    const literalRe = /akm\s+show\s+["']?((?:[a-z0-9_-]+\/\/)?[a-z][a-z0-9_-]*:[A-Za-z0-9_-]+)["']?/g;
+    for (const literalMatch of haystack.matchAll(literalRe)) {
+        push(literalMatch[1]);
+    }
+    // Tool-call JSON form. `"args":[..., "show", "<ref>", ...]`. We extract
+    // every refish token in the haystack that follows a "show" arg in JSON-y
+    // form. A second cheap pass keeps the pattern simple.
+    const toolCallRe = /"show"\s*,\s*"((?:[a-z0-9_-]+\/\/)?[a-z][a-z0-9_-]*:[A-Za-z0-9_-]+)"/g;
+    for (const toolCallMatch of haystack.matchAll(toolCallRe)) {
+        push(toolCallMatch[1]);
+    }
+    return out;
+}
+// Suppress the unused warning for `ASSET_REF_PATTERN` above. The constant is
+// retained as the documentation seam called out by the #251 review addenda,
+// even though `extractAssetLoads` uses inline regexes for its two scan forms.
+void ASSET_REF_PATTERN;
+/**
+ * Anchored variant of `ASSET_REF_PATTERN` for whole-string validation.
+ *
+ * Used by `materialiseMaskedStash` (#251) to gate every asset ref BEFORE we
+ * touch the filesystem. The base `ASSET_REF_PATTERN` is `/g`-flagged for
+ * scanning agent stdout; we re-anchor here so a hostile string like
+ * `skill:foo/../../etc` is rejected as a whole even though the regex would
+ * happily match a `skill:foo` substring under `/g`.
+ *
+ * Rejects `..`, absolute paths, drive letters, null bytes, `/`, `\`, and
+ * anything else outside the v1 ref grammar (mirrors src/core/asset-ref.ts).
+ */
+const ASSET_REF_ANCHORED = /^(?:[a-z0-9_-]+\/\/)?[a-z][a-z0-9_-]*:[A-Za-z0-9_-]+$/;
+/**
+ * Reject hostile asset refs before they reach any `fs.rmSync` call. The ref
+ * comes from agent stdout (untrusted; the agent could be prompt-injected) so
+ * we apply the anchored grammar pattern first, then the per-segment shape
+ * check after the colon-split. Defense in depth — each layer is sufficient
+ * on its own; the layered structure makes a future grammar relax safe.
+ */
+function isSafeAssetRef(ref) {
+    if (!ref)
+        return false;
+    if (ref.includes("\0"))
+        return false;
+    return ASSET_REF_ANCHORED.test(ref);
+}
+/**
+ * Aggregate per-asset load + pass counts across all akm-arm runs in a report.
+ *
+ * Sort order (stable, deterministic):
+ *   1. loadCount descending (most-used first)
+ *   2. loadPassRate descending (working assets above broken ones at the same load count)
+ *   3. assetRef ascending (alphabetical tiebreak)
+ *
+ * Only `arm === "akm"` runs contribute. The `noakm` arm has no stash and
+ * cannot load assets, so including it would zero-bias the rates.
+ */
+export function computePerAssetAttribution(report) {
+    const passing = new Map();
+    const failing = new Map();
+    let totalAkmRuns = 0;
+    // The §13.3 task entry doesn't carry RunResults — we read them from the
+    // shared akm-arm runs collection that the runner stamps onto `report.akmRuns`.
+    const akmRuns = collectAkmRuns(report);
+    for (const r of akmRuns) {
+        totalAkmRuns += 1;
+        const isPass = r.outcome === "pass";
+        for (const ref of r.assetsLoaded ?? []) {
+            const bucket = isPass ? passing : failing;
+            bucket.set(ref, (bucket.get(ref) ?? 0) + 1);
+        }
+    }
+    const refs = new Set([...passing.keys(), ...failing.keys()]);
+    const rows = [];
+    for (const ref of refs) {
+        const p = passing.get(ref) ?? 0;
+        const f = failing.get(ref) ?? 0;
+        const total = p + f;
+        rows.push({
+            assetRef: ref,
+            loadCountPassing: p,
+            loadCountFailing: f,
+            loadCount: total,
+            loadPassRate: total === 0 ? null : p / total,
+        });
+    }
+    rows.sort((a, b) => {
+        if (b.loadCount !== a.loadCount)
+            return b.loadCount - a.loadCount;
+        const ar = a.loadPassRate ?? -1;
+        const br = b.loadPassRate ?? -1;
+        if (br !== ar)
+            return br - ar;
+        return a.assetRef.localeCompare(b.assetRef);
+    });
+    return { rows, totalAkmRuns };
+}
+/**
+ * Pull the akm-arm RunResults out of a UtilityRunReport. The runner stamps
+ * them into the optional `akmRuns` field on the report so attribution can
+ * post-process them without re-running.
+ */
+function collectAkmRuns(report) {
+    if (Array.isArray(report.akmRuns))
+        return report.akmRuns;
+    return [];
+}
+// ── runs[] serialisation (#249) ────────────────────────────────────────────
+/**
+ * Project a list of RunResults onto the compact `runs[]` rows persisted
+ * inside the §13.3 JSON envelope (#249). One row per (task, arm, seed)
+ * triple; the renderer walks the input order verbatim, which the runner
+ * already builds deterministically (per-task block, noakm before akm,
+ * seeds in ascending order).
+ *
+ * Aggregate metrics (per-task, trajectory, failure-mode, search-bridge,
+ * attribution) MUST be recomputable from these rows + task metadata. This
+ * helper is the canonical projection — keep it in lockstep with the field
+ * list in the issue body.
+ */
+export function aggregateRunsForReport(runs) {
+    return runs.map(serializeRunForReport);
+}
+/**
+ * Hydrate a persisted `runs[]` row back into the `RunResult` shape that
+ * downstream metrics helpers (`computePerAssetAttribution`, `aggregateCorpus`,
+ * etc.) expect. Used by `bench attribute` / `bench compare` when they read a
+ * §13.3 envelope from disk: the persisted row carries a compact subset, but
+ * it carries everything those helpers need.
+ *
+ * Fields the row deliberately does NOT carry are filled with safe defaults:
+ *   • `events: []` — events.jsonl is not persisted; downstream attribution
+ *     only consults `assetsLoaded` and `verifierStdout`.
+ *   • `verifierStdout: ""` — full stdout is intentionally omitted from the
+ *     envelope (#249 acceptance criterion). `assetsLoaded` already carries
+ *     the post-hoc extraction the agent run produced.
+ *   • `schemaVersion: 1` — the report schema implies it.
+ *
+ * Tokens are passed through as-is so a future `measurement` field added by
+ * #252 lands on the rehydrated row automatically. TODO(#252): keep this
+ * spread.
+ */
+export function rehydrateRunFromSerialized(row) {
+    // The compact row uses a permissive Record shape for tokens (see
+    // RunRecordSerialized). Coerce defensively so older artefacts with only
+    // {input, output} hydrate cleanly.
+    const tok = row.tokens;
+    return {
+        schemaVersion: 1,
+        taskId: row.task_id,
+        arm: row.arm,
+        seed: row.seed,
+        model: row.model,
+        outcome: row.outcome,
+        tokens: {
+            ...tok,
+            input: typeof tok.input === "number" ? tok.input : 0,
+            output: typeof tok.output === "number" ? tok.output : 0,
+        },
+        wallclockMs: row.wallclock_ms,
+        trajectory: {
+            correctAssetLoaded: row.trajectory.correct_asset_loaded,
+            feedbackRecorded: row.trajectory.feedback_recorded,
+        },
+        events: [],
+        verifierStdout: "",
+        verifierExitCode: row.verifier_exit_code,
+        assetsLoaded: [...row.assets_loaded],
+        failureMode: (row.failure_mode ?? null),
+    };
+}
+/**
+ * Pick the top-N most-loaded assets from a base report and re-run the corpus
+ * with each one masked from its source stash. Returns a marginal-contribution
+ * row per masked asset.
+ *
+ * Cost: N * (tasks × arms × seedsPerArm) re-runs. Operators clamp N before
+ * calling — but we also clamp internally if `topN` exceeds the unique-asset
+ * count to avoid surprising no-op runs.
+ *
+ * Source-fixture safety: every masked re-run materialises a fresh tmp copy
+ * of the fixture stash, deletes the masked asset's files there, and points
+ * the re-run at the tmp dir. The shipped fixture in `tests/fixtures/stashes/`
+ * is NEVER mutated.
+ */
+export async function runMaskedCorpus(opts) {
+    const baseReport = opts.baseReport;
+    const fixturesRoot = opts.fixturesRoot ?? path.resolve(__dirname, "..", "fixtures", "stashes");
+    const attribution = computePerAssetAttribution(baseReport);
+    const desired = Math.max(1, opts.topN ?? 5);
+    const clamped = Math.min(desired, attribution.rows.length);
+    const baseAkmPassRate = baseReport.aggregateAkm.passRate;
+    const top = attribution.rows.slice(0, clamped);
+    const attributions = [];
+    const maskedRefs = [];
+    for (const row of top) {
+        const maskedTasks = [];
+        const tmpDirs = [];
+        try {
+            for (const baseTask of baseReport.taskMetadata ?? []) {
+                const maskedStashDir = materialiseMaskedStash(fixturesRoot, baseTask.stash, row.assetRef);
+                if (maskedStashDir)
+                    tmpDirs.push(maskedStashDir);
+                // Issue #251: forward the masked stashDir via the explicit
+                // `stashDirOverride` field on the cloned TaskMetadata. We MUST NOT
+                // mutate `baseTask.stash` (the fixture name) — the runner uses that
+                // to call `loadFixtureStash`, and overloading it breaks the
+                // `__no-stash__` resolution branch in runner.ts. The runner's AKM-arm
+                // branch checks `task.stashDirOverride` first.
+                //
+                // When `materialiseMaskedStash` returned `null` (asset not present in
+                // this fixture, or hostile ref shape rejected by the validator), we
+                // intentionally leave both fields untouched. The runner falls back to
+                // the normal materialisation flow against the unchanged source
+                // fixture — so the re-run still happens, but the result mirrors the
+                // base. This is a meaningful diagnostic (the ref didn't bind in this
+                // fixture) and is the same accounting `cost-accounting`-style tests
+                // assert against.
+                if (maskedStashDir) {
+                    maskedTasks.push({ ...baseTask, stashDirOverride: maskedStashDir });
+                }
+                else {
+                    maskedTasks.push({ ...baseTask });
+                }
+            }
+            const maskedReport = await opts.runUtility({
+                ...opts.baseOptions,
+                tasks: maskedTasks,
+                // The masked stash already has the correct content on disk, and the
+                // runner now resolves it via `task.stashDirOverride`. We still pass
+                // `materialiseStash: false` so the runner does not call
+                // `loadFixtureStash` against the (unmasked) named fixture — that
+                // would waste work and risk re-indexing the source dir.
+                materialiseStash: false,
+            });
+            const maskedPassRate = maskedReport.aggregateAkm.passRate;
+            attributions.push({
+                assetRef: row.assetRef,
+                basePassRate: baseAkmPassRate,
+                maskedPassRate,
+                marginalContribution: baseAkmPassRate - maskedPassRate,
+            });
+            maskedRefs.push(row.assetRef);
+        }
+        finally {
+            // Cleanup runs in BOTH success and failure paths (acceptance criterion).
+            // Best-effort: a tmpfs failure here is logged via the `try/catch` below
+            // and the host OS reaps the tmp dir on reboot.
+            for (const dir of tmpDirs) {
+                try {
+                    fs.rmSync(dir, { recursive: true, force: true });
+                }
+                catch {
+                    // Best-effort cleanup; tmpfs cleanup will handle leaks.
+                }
+            }
+        }
+    }
+    return {
+        baseReport,
+        attributions,
+        runsPerformed: clamped,
+        maskingStrategy: "leave-one-out",
+        maskedRefs,
+    };
+}
+/**
+ * Copy a fixture stash into a fresh tmp dir, delete every file matching the
+ * masked asset ref, and return the tmp dir path. Returns `null` if the named
+ * asset is not present in the fixture (we still re-run, but the result will
+ * mirror the base — which is itself a meaningful diagnostic).
+ *
+ * The masking heuristic:
+ *   1. Walk `<stash>/*<...>/.stash.json` files.
+ *   2. For each entry whose `name` + `type` matches the asset ref, drop the
+ *      entry and delete its `filename` if present.
+ *   3. Rewrite the `.stash.json` with the trimmed entries (or remove it if
+ *      it is now empty).
+ */
+export function materialiseMaskedStash(fixturesRoot, stashName, assetRef) {
+    // #271: validate stashName containment BEFORE touching the filesystem.
+    // `stashName` originates from a task YAML which, while authored, is part
+    // of the fixture corpus the bench loads; a fixture with `stash: "../../etc"`
+    // would otherwise resolve outside `fixturesRoot` and let masking edits or
+    // copies escape the bench sandbox. path.relative gives the cleanest
+    // containment check (handles `..` AND absolute path injection in one go).
+    const fixturesRootResolved = path.resolve(fixturesRoot);
+    const sourceDir = path.resolve(fixturesRootResolved, stashName);
+    const rel = path.relative(fixturesRootResolved, sourceDir);
+    if (rel.startsWith("..") || path.isAbsolute(rel))
+        return null;
+    if (!fs.existsSync(path.join(sourceDir, "MANIFEST.json")))
+        return null;
+    // Issue #251 review addendum: validate the WHOLE ref against the anchored
+    // grammar before we touch the filesystem. The downstream `isSafeAssetNameSegment`
+    // + `isPathContained` checks are still applied — this is defense in depth.
+    if (!isSafeAssetRef(assetRef))
+        return null;
+    const colonIdx = assetRef.indexOf(":");
+    if (colonIdx < 0) {
+        // Malformed ref: still produce a tmp copy with no edits so the caller's
+        // re-run sees the unmodified fixture.
+        const tmpRoot = benchMkdtemp(`akm-bench-masked-${stashName}-`);
+        copyDirRecursive(sourceDir, tmpRoot);
+        return tmpRoot;
+    }
+    const typeWithOrigin = assetRef.slice(0, colonIdx);
+    const name = assetRef.slice(colonIdx + 1);
+    const type = typeWithOrigin.includes("//") ? (typeWithOrigin.split("//")[1] ?? typeWithOrigin) : typeWithOrigin;
+    // SECURITY: the asset ref originates from agent stdout (untrusted; the
+    // agent could be prompt-injected). The masking heuristic below will
+    // `fs.rmSync` files under the tmp stash dir whose names are derived from
+    // `name`. A traversal-shaped name (`../etc`, `/abs/path`, `..\\..`) would
+    // escape the tmp root and delete arbitrary disk content. Reject those
+    // shapes BEFORE we materialise — and re-validate after path-resolving
+    // each candidate. Mirrors src/core/asset-ref.ts validateName().
+    if (!isSafeAssetNameSegment(name))
+        return null;
+    const tmpRoot = benchMkdtemp(`akm-bench-masked-${stashName}-`);
+    copyDirRecursive(sourceDir, tmpRoot);
+    // Walk every .stash.json under the tmp root and edit in place.
+    walkStashJsonFiles(tmpRoot, (jsonPath) => {
+        let raw;
+        try {
+            raw = fs.readFileSync(jsonPath, "utf8");
+        }
+        catch {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        const entries = parsed.entries ?? [];
+        const kept = [];
+        const jsonDir = path.dirname(jsonPath);
+        for (const entry of entries) {
+            if (entry.type === type && entry.name === name) {
+                // Remove the entry's content file(s). The on-disk `filename` is read
+                // from the fixture .stash.json (trusted) but the value still passes
+                // through path.relative containment so a malicious fixture can't use
+                // this path to escape either.
+                const filename = entry.filename;
+                if (typeof filename === "string" && isSafeAssetNameSegment(filename)) {
+                    const target = path.resolve(jsonDir, filename);
+                    if (isPathContained(tmpRoot, target)) {
+                        try {
+                            fs.rmSync(target, { force: true });
+                        }
+                        catch {
+                            // ignore
+                        }
+                    }
+                }
+                // Some fixtures keep a per-asset directory (e.g. skills/<name>/SKILL.md).
+                const dirCandidate = path.resolve(jsonDir, name);
+                if (isPathContained(tmpRoot, dirCandidate) &&
+                    fs.existsSync(dirCandidate) &&
+                    fs.statSync(dirCandidate).isDirectory()) {
+                    try {
+                        fs.rmSync(dirCandidate, { recursive: true, force: true });
+                    }
+                    catch {
+                        // ignore
+                    }
+                }
+                continue;
+            }
+            kept.push(entry);
+        }
+        if (kept.length === entries.length)
+            return; // nothing changed
+        if (kept.length === 0) {
+            try {
+                fs.rmSync(jsonPath, { force: true });
+            }
+            catch {
+                // ignore
+            }
+        }
+        else {
+            fs.writeFileSync(jsonPath, `${JSON.stringify({ ...parsed, entries: kept }, null, 2)}\n`);
+        }
+    });
+    return tmpRoot;
+}
+/**
+ * Reject any segment that could escape the tmp stash root when used as a
+ * relative path component:
+ *   - empty string
+ *   - any `/` or `\\` (path separators)
+ *   - a `..` segment in any form
+ *   - a leading `/` (POSIX absolute) or `C:` (Windows drive)
+ *   - any null byte
+ *
+ * Mirrors src/core/asset-ref.ts validateName(), but returns a boolean
+ * (callers map this to "skip" rather than "throw").
+ */
+function isSafeAssetNameSegment(value) {
+    if (!value)
+        return false;
+    if (value.includes("\0"))
+        return false;
+    if (value.includes("/") || value.includes("\\"))
+        return false;
+    if (value === ".." || value === ".")
+        return false;
+    if (/^[A-Za-z]:/.test(value))
+        return false;
+    return true;
+}
+/**
+ * After resolving a target path, confirm it lives under `root`. Defense in
+ * depth: even if a traversal-shaped name slipped past the segment check,
+ * this catches escapes via symlinks or odd `path.join` semantics.
+ *
+ * #271: aligned with `isWithin` in `src/core/common.ts` — both inputs go
+ * through `safeRealpath` so a symlink inside `root` that points outside
+ * cannot fool the `path.relative` containment check. The shared helper
+ * also handles not-yet-existing children (walks up to the closest existing
+ * ancestor and resolves symlinks there) so we keep the existing semantics
+ * for `target` paths the masking heuristic is about to create.
+ */
+export function isPathContained(root, target) {
+    const rootResolved = safeRealpath(root);
+    const targetResolved = safeRealpath(target);
+    const rel = path.relative(rootResolved, targetResolved);
+    if (rel === "")
+        return true;
+    if (rel.startsWith(".."))
+        return false;
+    if (path.isAbsolute(rel))
+        return false;
+    return true;
+}
+function walkStashJsonFiles(root, visit) {
+    const stack = [root];
+    while (stack.length > 0) {
+        const cur = stack.pop();
+        if (!cur)
+            continue;
+        let entries;
+        try {
+            entries = fs.readdirSync(cur, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries) {
+            const abs = path.join(cur, entry.name);
+            if (entry.isDirectory())
+                stack.push(abs);
+            else if (entry.isFile() && entry.name === ".stash.json")
+                visit(abs);
+        }
+    }
+}
+function copyDirRecursive(src, dest) {
+    fs.mkdirSync(dest, { recursive: true });
+    const entries = fs.readdirSync(src, { withFileTypes: true });
+    for (const entry of entries) {
+        const s = path.join(src, entry.name);
+        const d = path.join(dest, entry.name);
+        if (entry.isDirectory())
+            copyDirRecursive(s, d);
+        else if (entry.isFile())
+            fs.copyFileSync(s, d);
+    }
+}
+/** Aggregate trajectory booleans across a bag of runs. */
+export function aggregateTrajectory(results) {
+    if (results.length === 0) {
+        return { correctAssetLoaded: null, feedbackRecorded: 0 };
+    }
+    let knownAsset = 0;
+    let assetLoaded = 0;
+    let feedback = 0;
+    for (const r of results) {
+        if (r.trajectory.correctAssetLoaded !== null) {
+            knownAsset += 1;
+            if (r.trajectory.correctAssetLoaded)
+                assetLoaded += 1;
+        }
+        if (r.trajectory.feedbackRecorded === true)
+            feedback += 1;
+    }
+    return {
+        correctAssetLoaded: knownAsset === 0 ? null : assetLoaded / knownAsset,
+        feedbackRecorded: feedback / results.length,
+    };
+}
+/**
+ * Sign threshold below which a delta is rendered as `flat`. `pass_rate` is
+ * normalised to `[0, 1]`, so a 0.005 (0.5pp) tolerance keeps tiny K-seed
+ * sampling jitter from looking like a regression.
+ */
+const PASS_RATE_FLAT_TOLERANCE = 0.005;
+/** `tokens_per_pass` and `wallclock_ms` use raw counts; 0 is the only "flat". */
+const COUNT_FLAT_TOLERANCE = 0;
+function classifyPassRate(delta) {
+    if (delta === null)
+        return "flat";
+    if (Math.abs(delta) <= PASS_RATE_FLAT_TOLERANCE)
+        return "flat";
+    return delta > 0 ? "improve" : "regress";
+}
+function classifyCount(delta, lowerIsBetter) {
+    if (delta === null)
+        return "flat";
+    if (Math.abs(delta) <= COUNT_FLAT_TOLERANCE)
+        return "flat";
+    if (lowerIsBetter)
+        return delta < 0 ? "improve" : "regress";
+    return delta > 0 ? "improve" : "regress";
+}
+function readModel(r) {
+    return r.agent?.model ?? "<unknown>";
+}
+function readFixtureHash(r) {
+    const v = r.corpus?.fixtureContentHash;
+    return v === undefined || v === null ? null : v;
+}
+function readTaskCorpusHash(r) {
+    const v = r.corpus?.taskCorpusHash;
+    return v === undefined || v === null ? null : v;
+}
+function readSelectedTaskIds(r) {
+    const v = r.corpus?.selectedTaskIds;
+    return Array.isArray(v) ? v : null;
+}
+function arraysEqualIgnoringOrder(a, b) {
+    if (a.length !== b.length)
+        return false;
+    const sa = [...a].sort();
+    const sb = [...b].sort();
+    for (let i = 0; i < sa.length; i += 1)
+        if (sa[i] !== sb[i])
+            return false;
+    return true;
+}
+function akmAgg(r) {
+    const a = r.aggregate?.akm ?? {};
+    return {
+        pass_rate: a.pass_rate ?? 0,
+        tokens_per_pass: a.tokens_per_pass ?? null,
+        wallclock_ms: a.wallclock_ms ?? 0,
+    };
+}
+/**
+ * Diff two parsed UtilityRunReport JSONs.
+ *
+ * Refusal cases:
+ *   • Either side missing `schemaVersion: 1` or `track: "utility"` →
+ *     `schema_mismatch` / `track_mismatch`.
+ *   • `agent.model` differs → `model_mismatch`.
+ *   • Both sides report a `corpus.fixtureContentHash` and they differ →
+ *     `hash_mismatch`. Missing hash on either side proceeds with a warning
+ *     (Wave A may add it; older reports won't have it).
+ *
+ * On success the per-task table includes rows for every task in either side,
+ * plus aggregate deltas computed against the akm arm only (the noakm arm is
+ * the control — its delta is meaningless). `pass_rate` is in `[0, 1]`,
+ * higher is better; `tokens_per_pass` and `wallclock_ms` are counts, lower
+ * is better.
+ */
+export function compareReports(base, current, options = {}) {
+    // Schema-version gate.
+    if (base.schemaVersion !== 1 || current.schemaVersion !== 1) {
+        return {
+            ok: false,
+            reason: "schema_mismatch",
+            message: `compare requires schemaVersion=1 on both sides; got base=${String(base.schemaVersion)}, current=${String(current.schemaVersion)}`,
+        };
+    }
+    // Track gate. Cross-track diffs are nonsensical.
+    if (base.track !== "utility" || current.track !== "utility") {
+        return {
+            ok: false,
+            reason: "track_mismatch",
+            message: `compare only supports track="utility"; got base="${String(base.track)}", current="${String(current.track)}"`,
+        };
+    }
+    const baseModel = readModel(base);
+    const currentModel = readModel(current);
+    if (baseModel !== currentModel) {
+        return {
+            ok: false,
+            reason: "model_mismatch",
+            message: `cannot compare across different models: base="${baseModel}", current="${currentModel}". Rerun on the same model.`,
+            baseModel,
+            currentModel,
+        };
+    }
+    const baseHash = readFixtureHash(base);
+    const currentHash = readFixtureHash(current);
+    const warnings = [];
+    // #250 — task corpus hash + selected task IDs. Refused unless either side
+    // is legacy (missing the hash) or the operator passed
+    // `allowCorpusMismatch`. Legacy reports (no taskCorpusHash) degrade to a
+    // warning so older artefacts can still be diffed.
+    const baseTaskHash = readTaskCorpusHash(base);
+    const currentTaskHash = readTaskCorpusHash(current);
+    const baseIds = readSelectedTaskIds(base);
+    const currentIds = readSelectedTaskIds(current);
+    if (baseTaskHash !== null && currentTaskHash !== null && baseTaskHash !== currentTaskHash) {
+        if (!options.allowCorpusMismatch) {
+            return {
+                ok: false,
+                reason: "corpus_mismatch",
+                message: `cannot compare across different task corpora: base taskCorpusHash="${baseTaskHash}", current="${currentTaskHash}". Rerun against the same task selection or pass --allow-corpus-mismatch to override.`,
+                baseModel,
+                currentModel,
+                baseTaskCorpusHash: baseTaskHash,
+                currentTaskCorpusHash: currentTaskHash,
+                ...(baseIds ? { baseSelectedTaskIds: baseIds } : {}),
+                ...(currentIds ? { currentSelectedTaskIds: currentIds } : {}),
+            };
+        }
+        warnings.push(`task corpus hashes differ (base="${baseTaskHash}", current="${currentTaskHash}") — diff requested via --allow-corpus-mismatch`);
+    }
+    else if (baseTaskHash === null &&
+        currentTaskHash === null &&
+        baseIds !== null &&
+        currentIds !== null &&
+        !arraysEqualIgnoringOrder(baseIds, currentIds)) {
+        // Both sides legacy (no taskCorpusHash) but both expose selectedTaskIds
+        // and they differ. We can still detect a mismatched corpus from the ID
+        // list alone — refuse unless the operator opted in.
+        if (!options.allowCorpusMismatch) {
+            return {
+                ok: false,
+                reason: "corpus_mismatch",
+                message: `cannot compare across different selected task IDs. Rerun against the same task selection or pass --allow-corpus-mismatch to override.`,
+                baseModel,
+                currentModel,
+                baseSelectedTaskIds: baseIds,
+                currentSelectedTaskIds: currentIds,
+            };
+        }
+        warnings.push("selected task IDs differ — diff requested via --allow-corpus-mismatch");
+    }
+    if (baseTaskHash === null)
+        warnings.push("base report has no corpus.taskCorpusHash; proceeding without task-corpus-pin check");
+    if (currentTaskHash === null)
+        warnings.push("current report has no corpus.taskCorpusHash; proceeding without task-corpus-pin check");
+    if (baseHash !== null && currentHash !== null && baseHash !== currentHash) {
+        if (!options.allowFixtureMismatch) {
+            return {
+                ok: false,
+                reason: "hash_mismatch",
+                message: `cannot compare across different fixture-content hashes: base="${baseHash}", current="${currentHash}". Rerun against matching fixtures or pass --allow-fixture-mismatch to override.`,
+                baseModel,
+                currentModel,
+                baseFixtureContentHash: baseHash,
+                currentFixtureContentHash: currentHash,
+            };
+        }
+        warnings.push(`fixture-content hashes differ (base="${baseHash}", current="${currentHash}") — diff requested via --allow-fixture-mismatch`);
+    }
+    if (baseHash === null)
+        warnings.push("base report has no corpus.fixtureContentHash; proceeding without fixture-pin check");
+    if (currentHash === null)
+        warnings.push("current report has no corpus.fixtureContentHash; proceeding without fixture-pin check");
+    // Aggregate (akm arm is the one that matters — noakm is the control).
+    const ba = akmAgg(base);
+    const ca = akmAgg(current);
+    const passRateDelta = ca.pass_rate - ba.pass_rate;
+    const tokensPerPassDelta = ba.tokens_per_pass === null || ca.tokens_per_pass === null ? null : ca.tokens_per_pass - ba.tokens_per_pass;
+    const wallclockMsDelta = ca.wallclock_ms - ba.wallclock_ms;
+    const aggregate = {
+        passRateDelta,
+        passRateSign: classifyPassRate(passRateDelta),
+        tokensPerPassDelta,
+        tokensPerPassSign: classifyCount(tokensPerPassDelta, true),
+        wallclockMsDelta,
+        wallclockMsSign: classifyCount(wallclockMsDelta, true),
+    };
+    // Per-task rows. Outer-join on task id.
+    const baseTasks = new Map();
+    for (const t of base.tasks ?? [])
+        baseTasks.set(t.id, t);
+    const currentTasks = new Map();
+    for (const t of current.tasks ?? [])
+        currentTasks.set(t.id, t);
+    const allIds = new Set();
+    for (const id of baseTasks.keys())
+        allIds.add(id);
+    for (const id of currentTasks.keys())
+        allIds.add(id);
+    const perTask = [];
+    for (const id of [...allIds].sort()) {
+        const b = baseTasks.get(id);
+        const c = currentTasks.get(id);
+        const bM = b?.akm ?? null;
+        const cM = c?.akm ?? null;
+        const presence = b !== undefined && c !== undefined ? "both" : b !== undefined ? "base-only" : "current-only";
+        const passRateDelta_ = bM !== null && cM !== null ? cM.pass_rate - bM.pass_rate : null;
+        const tokensPerPassDelta_ = bM !== null && cM !== null && bM.tokens_per_pass !== null && cM.tokens_per_pass !== null
+            ? cM.tokens_per_pass - bM.tokens_per_pass
+            : null;
+        const wallclockMsDelta_ = bM !== null && cM !== null ? cM.wallclock_ms - bM.wallclock_ms : null;
+        perTask.push({
+            id,
+            presence,
+            baseMetrics: bM,
+            currentMetrics: cM,
+            delta: { passRate: passRateDelta_, tokensPerPass: tokensPerPassDelta_, wallclockMs: wallclockMsDelta_ },
+            signMarker: classifyPassRate(passRateDelta_),
+        });
+    }
+    return {
+        ok: true,
+        baseModel,
+        currentModel,
+        baseFixtureContentHash: baseHash,
+        currentFixtureContentHash: currentHash,
+        warnings,
+        aggregate,
+        perTask,
+    };
+}
+/** Maximum rank at which the gold ref still counts as "found"; > this is `search_low_rank`. */
+const SEARCH_RANK_CUTOFF = 5;
+/** Cap on the number of characters of `verifierStdout` we substring-scan. Mirrors trajectory.ts. */
+const FAILURE_MODE_STDOUT_SCAN_CAP = 16 * 1024 * 1024;
+/**
+ * Classify a single failed run into one of the seven §6.6 labels. Pure
+ * function — string-matches `runResult.events[]` and `runResult.verifierStdout`,
+ * never calls an LLM, never touches the filesystem.
+ *
+ * Decision tree (priority order — first match wins):
+ *   1. Run not failed (`pass`, `budget_exceeded`, `harness_error`) → `null`.
+ *   2. No `akm search` call in the trace → `no_search`.
+ *   3. Search ran; gold ref absent from search results → `search_no_gold`.
+ *   4. Gold ref present in search results at rank > 5 → `search_low_rank`.
+ *   5. `akm show` invoked on a non-gold ref AND gold ref never loaded → `loaded_wrong`.
+ *   6. Gold ref loaded; verifier output suggests the action contradicts the
+ *      asset's guidance (heuristic: verifier mentions the gold pattern was
+ *      explicitly NOT followed) → `loaded_ignored`.
+ *   7. Gold ref loaded and apparently followed → `followed_wrong`.
+ *   8. Default → `unrelated_bug`.
+ *
+ * Tasks without `goldRef`: rules that depend on the gold ref (3-7) are
+ * skipped; only `no_search` and `unrelated_bug` are reachable.
+ */
+export function classifyFailureMode(taskMeta, runResult) {
+    if (runResult.outcome !== "fail")
+        return null;
+    const trace = collectTrace(runResult);
+    const goldRef = taskMeta.goldRef;
+    // 1. no_search — no `akm search` invocation anywhere in the trace.
+    if (!hasAkmSearch(trace, runResult)) {
+        return "no_search";
+    }
+    // Without a gold ref the search-based and load-based checks are undefined.
+    // We can only distinguish "no_search" from everything else.
+    if (!goldRef) {
+        return "unrelated_bug";
+    }
+    const searchRank = findGoldSearchRank(trace, goldRef);
+    // 2. search_no_gold — search ran (precondition above) but gold ref absent.
+    if (searchRank === null) {
+        return "search_no_gold";
+    }
+    // 3. search_low_rank — present but below the cutoff.
+    if (searchRank > SEARCH_RANK_CUTOFF) {
+        return "search_low_rank";
+    }
+    const goldLoaded = hasAkmShow(trace, runResult, goldRef);
+    const otherRefLoaded = hasAkmShowOtherRef(trace, runResult, goldRef);
+    // 4. loaded_wrong — agent showed a non-gold ref AND never loaded the gold.
+    if (otherRefLoaded && !goldLoaded) {
+        return "loaded_wrong";
+    }
+    // The remaining branches all assume the gold was loaded.
+    if (!goldLoaded) {
+        // Gold ref was found in search at an acceptable rank, but the agent
+        // never loaded anything (gold or otherwise) before failing. The taxonomy
+        // table has no row for "found but never opened" — treat as unrelated_bug.
+        return "unrelated_bug";
+    }
+    // 5. loaded_ignored — verifier diagnostic indicates the action contradicts
+    //    the loaded asset. Conservative heuristic: look for explicit "ignored"
+    //    or "not applied" markers in the verifier stdout. Without an LLM we
+    //    cannot detect subtler contradictions, so this branch only fires when
+    //    the verifier itself flagged the contradiction.
+    if (verifierIndicatesIgnored(runResult.verifierStdout)) {
+        return "loaded_ignored";
+    }
+    // 6. followed_wrong — gold loaded, apparently followed, verifier still
+    //    failed. The §6.6 spec maps this to "the asset itself is wrong".
+    return "followed_wrong";
+}
+/** Build a `FailureModeAggregate` from a list of (taskId, label) pairs. */
+export function aggregateFailureModes(entries) {
+    const byLabel = {};
+    const byTask = {};
+    for (const { taskId, mode } of entries) {
+        byLabel[mode] = (byLabel[mode] ?? 0) + 1;
+        if (!byTask[taskId])
+            byTask[taskId] = {};
+        byTask[taskId][mode] = (byTask[taskId][mode] ?? 0) + 1;
+    }
+    return { byLabel, byTask };
+}
+// ── Failure-mode classifier helpers ────────────────────────────────────────
+/**
+ * Concatenated string used for substring scans. We pre-build this once per
+ * classify call so the helper functions can share it. Stdout is capped per
+ * the trajectory parser convention to keep runaway agents from OOMing the
+ * bench.
+ */
+function collectTrace(runResult) {
+    const stdout = runResult.verifierStdout ?? "";
+    const capped = stdout.length > FAILURE_MODE_STDOUT_SCAN_CAP ? stdout.slice(0, FAILURE_MODE_STDOUT_SCAN_CAP) : stdout;
+    return capped;
+}
+/** Does the trace contain any `akm search` invocation (CLI form OR event)? */
+function hasAkmSearch(trace, runResult) {
+    // Tool-call CLI form, e.g. `akm search "deploy homelab"`.
+    if (/\bakm\s+search\b/.test(trace))
+        return true;
+    // Tool-call JSON form, e.g. `"args":["search","..."]`.
+    if (trace.includes(`"search"`) && /["']search["']/.test(trace))
+        return true;
+    // Event-stream form (search verbs aren't currently emitted but the field
+    // is forward-compatible — see core/events.ts).
+    for (const event of runResult.events) {
+        if (event.eventType === "search" || event.eventType === "search_invoked")
+            return true;
+    }
+    return false;
+}
+/**
+ * Find the 1-based rank of `goldRef` in the search results captured in the
+ * trace, or `null` if not present. Best-effort heuristics:
+ *   1. Look for an `akm search` block followed by a numbered list (`1. skill:foo`).
+ *   2. Look for a JSON-ish results array containing the ref.
+ *   3. Fall back to substring presence — if the ref appears anywhere after
+ *      a search invocation, treat it as rank-unknown. We err on the side of
+ *      `1` (best case for the agent) so the classifier doesn't false-positive
+ *      on `search_low_rank`.
+ */
+function findGoldSearchRank(trace, goldRef) {
+    // Locate the first `akm search` invocation; restrict the rank search to
+    // text after it so we don't pick up `akm show` output.
+    const searchMatch = trace.match(/\bakm\s+search\b/);
+    if (!searchMatch || searchMatch.index === undefined) {
+        // Caller already verified search ran; if our regex disagrees, fall back
+        // to scanning the full trace.
+        return findRefRankInText(trace, goldRef);
+    }
+    const after = trace.slice(searchMatch.index);
+    return findRefRankInText(after, goldRef);
+}
+function findRefRankInText(text, goldRef) {
+    const escaped = goldRef.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // Numbered list: lines of the form `<rank>. <ref>` or `<rank>) <ref>`.
+    const numberedRe = /^\s*(\d{1,3})[.)]\s+([^\s]+)/gm;
+    let match;
+    while (true) {
+        match = numberedRe.exec(text);
+        if (match === null)
+            break;
+        const ref = match[2];
+        if (refsMatch(ref, goldRef)) {
+            return Number.parseInt(match[1], 10);
+        }
+    }
+    // JSON array form: `"results":["a","b","skill:foo"]`. Estimate rank by
+    // splitting on commas after the bracket. Best-effort.
+    const jsonRe = /"results"\s*:\s*\[([^\]]+)\]/;
+    const jsonMatch = text.match(jsonRe);
+    if (jsonMatch) {
+        const items = jsonMatch[1].split(",").map((s) => s.trim().replace(/^["']|["']$/g, ""));
+        const idx = items.findIndex((item) => refsMatch(item, goldRef));
+        if (idx >= 0)
+            return idx + 1;
+    }
+    // Substring presence — assume rank 1 (best case for the agent, conservative
+    // for the `search_low_rank` rule).
+    const refRe = new RegExp(`\\b${escaped}\\b`);
+    if (refRe.test(text))
+        return 1;
+    return null;
+}
+/** True when `candidate` is `goldRef` or a strict ref-extension thereof. */
+function refsMatch(candidate, goldRef) {
+    if (candidate === goldRef)
+        return true;
+    if (candidate.endsWith(`//${goldRef}`))
+        return true;
+    if (candidate.startsWith(`${goldRef}/`))
+        return true;
+    return false;
+}
+/** Did the agent invoke `akm show <goldRef>` at any point? */
+function hasAkmShow(trace, runResult, goldRef) {
+    const escaped = goldRef.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // CLI form, exact ref. Also matches origin-prefixed variants like
+    // `akm show team//skill:foo` because the `[\w/]*//` prefix is optional.
+    const cliRe = new RegExp(`\\bakm\\s+show\\s+["']?(?:[\\w-]+//)?${escaped}(?:\\b|\\W)`);
+    if (cliRe.test(trace))
+        return true;
+    // Tool-call JSON form: `"args":["show","skill:foo"]`.
+    if (trace.includes(`"show"`) && trace.includes(goldRef))
+        return true;
+    // Event-stream metadata.ref.
+    for (const event of runResult.events) {
+        if (typeof event.ref === "string" && refsMatch(event.ref, goldRef)) {
+            // Only count "show" or "load" eventTypes; a `feedback` event mentioning
+            // the ref doesn't mean the agent loaded it during this run.
+            if (event.eventType === "show" || event.eventType === "load" || event.eventType === "tool_call")
+                return true;
+        }
+        const meta = event.metadata;
+        if (meta && typeof meta === "object") {
+            const candidate = meta.ref;
+            if (typeof candidate === "string" && refsMatch(candidate, goldRef)) {
+                if (event.eventType === "show" || event.eventType === "load" || event.eventType === "tool_call")
+                    return true;
+            }
+        }
+    }
+    return false;
+}
+/** Did the agent invoke `akm show <ref>` for some ref OTHER than `goldRef`? */
+function hasAkmShowOtherRef(trace, runResult, goldRef) {
+    // CLI form: capture the ref argument and reject when it matches the gold.
+    const cliRe = /\bakm\s+show\s+["']?([^\s"'`]+)/g;
+    let match;
+    while (true) {
+        match = cliRe.exec(trace);
+        if (match === null)
+            break;
+        if (!refsMatch(match[1], goldRef))
+            return true;
+    }
+    // Tool-call JSON form: `"args":["show","..."]`. Best-effort scan.
+    const jsonRe = /\["show",\s*"([^"]+)"/g;
+    while (true) {
+        match = jsonRe.exec(trace);
+        if (match === null)
+            break;
+        if (!refsMatch(match[1], goldRef))
+            return true;
+    }
+    // Event-stream form.
+    for (const event of runResult.events) {
+        if (event.eventType !== "show" && event.eventType !== "load" && event.eventType !== "tool_call")
+            continue;
+        if (typeof event.ref === "string" && !refsMatch(event.ref, goldRef))
+            return true;
+        const meta = event.metadata;
+        if (meta && typeof meta === "object") {
+            const candidate = meta.ref;
+            if (typeof candidate === "string" && !refsMatch(candidate, goldRef))
+                return true;
+        }
+    }
+    return false;
+}
+/**
+ * Conservative heuristic for the `loaded_ignored` branch. Without an LLM we
+ * cannot reliably decide whether an arbitrary action contradicts arbitrary
+ * asset content; we only fire when the verifier's own diagnostic explicitly
+ * flags the gold-asset guidance as ignored.
+ *
+ * The verifier stdout strings are deterministic — they come from
+ * `runVerifier` and the per-task `verify.sh` scripts. Tasks that want to
+ * surface this label should emit one of the agreed-upon markers below.
+ */
+function verifierIndicatesIgnored(verifierStdout) {
+    if (!verifierStdout)
+        return false;
+    const lower = verifierStdout.toLowerCase();
+    return (lower.includes("ignored gold guidance") ||
+        lower.includes("guidance ignored") ||
+        lower.includes("did not follow loaded asset") ||
+        lower.includes("contradicts loaded asset"));
+}
+/** Cap on the number of result refs we extract per `akm search` invocation. */
+const TOP_K = 10;
+/**
+ * Extract the gold rank for every `akm search` invocation in a run.
+ *
+ * The parser scans `runResult.verifierStdout` (which carries the captured
+ * agent stdout including its tool-call trace) for `akm search` commands
+ * and the result lists that follow them. The first 10 hits are considered;
+ * if the gold ref appears, `rankOfGold` is its 1-based position, else
+ * `null`.
+ *
+ * Pure function: never reads from disk and never mutates inputs. When
+ * `goldRef` is undefined the function returns `[]` — we only attribute
+ * ranks for tasks that actually have a gold asset.
+ */
+export function extractGoldRanks(runResult, goldRef) {
+    if (!goldRef)
+        return [];
+    const haystack = runResult.verifierStdout;
+    if (!haystack)
+        return [];
+    const events = [];
+    // Walk the stdout linearly. A search invocation looks like
+    //   `akm search "<query>"` or `akm search <query>`
+    // and the subsequent block carries the result list. A new `akm` command
+    // (or end of stdout) terminates the previous search's result block.
+    const lines = haystack.split(/\r?\n/);
+    let active = null;
+    // Regex for an `akm search` invocation. Captures the rest of the line
+    // after `search ` so we can pick up the query whether it's quoted or not.
+    const searchInvocationRe = /\bakm\s+search\s+(.+?)(?:\s+--|$)/;
+    // A different `akm <verb>` (not `search`) terminates the active block.
+    const akmInvocationRe = /\bakm\s+(\w+)/;
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        const searchMatch = line.match(searchInvocationRe);
+        if (searchMatch) {
+            // Flush any active block before starting a new one.
+            if (active) {
+                active.rankOfGold = computeRank(active.results, goldRef);
+                events.push(active);
+            }
+            const query = stripQuotes(searchMatch[1].trim());
+            active = { query, results: [], rankOfGold: null };
+            // Some traces inline the JSON result on the same line — try to extract.
+            collectRefsFromLine(line, active.results);
+            continue;
+        }
+        if (!active)
+            continue;
+        // A non-search akm invocation closes the active search block.
+        const akmMatch = line.match(akmInvocationRe);
+        if (akmMatch && akmMatch[1] !== "search") {
+            active.rankOfGold = computeRank(active.results, goldRef);
+            events.push(active);
+            active = null;
+            continue;
+        }
+        collectRefsFromLine(line, active.results);
+    }
+    if (active) {
+        active.rankOfGold = computeRank(active.results, goldRef);
+        events.push(active);
+    }
+    return events;
+}
+/** Trim leading/trailing single or double quotes from a query string. */
+function stripQuotes(s) {
+    if (s.length >= 2) {
+        const first = s[0];
+        const last = s[s.length - 1];
+        if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+            return s.slice(1, -1);
+        }
+    }
+    return s;
+}
+/**
+ * Pull asset refs from a single line into `out`. Matches both plain
+ * `ref: <ref>` lines (text mode) and `"ref":"<ref>"` (JSON mode). We
+ * stop at TOP_K results to mirror the spec's top-10 cutoff.
+ */
+function collectRefsFromLine(line, out) {
+    if (out.length >= TOP_K)
+        return;
+    // JSON form: `"ref":"skill:foo"` or `"ref": "skill:foo"`. Multiple per line possible.
+    const jsonRe = /"ref"\s*:\s*"([^"]+)"/g;
+    let m;
+    m = jsonRe.exec(line);
+    while (m !== null) {
+        if (out.length >= TOP_K)
+            return;
+        out.push(m[1]);
+        m = jsonRe.exec(line);
+    }
+    // Plain text form: `  ref: skill:foo`. Only treat the line as a ref-bearing
+    // line if it starts with `ref:` (after whitespace). Avoids picking up
+    // every `:` in arbitrary stdout.
+    const textRe = /^ref:\s*([^\s,]+)/;
+    const tm = line.match(textRe);
+    if (tm && out.length < TOP_K) {
+        out.push(tm[1]);
+    }
+}
+/**
+ * 1-based rank of `goldRef` in `results`, or `null` if absent within the
+ * top 10. We use `matchesGold` for prefix-tolerant matching so
+ * `team//skill:foo` counts as `skill:foo` (mirrors trajectory parser).
+ */
+function computeRank(results, goldRef) {
+    const cap = Math.min(results.length, TOP_K);
+    for (let i = 0; i < cap; i += 1) {
+        if (matchesGold(results[i], goldRef))
+            return i + 1;
+    }
+    return null;
+}
+function matchesGold(candidate, gold) {
+    if (candidate === gold)
+        return true;
+    if (candidate.endsWith(`//${gold}`))
+        return true;
+    if (candidate.startsWith(`${gold}/`))
+        return true;
+    return false;
+}
+/**
+ * Aggregate gold-rank records across all akm-arm runs in the corpus.
+ *
+ * The function operates on `report.goldRankRecords`, which the runner
+ * populates per (task, arm, seed). When the corpus has no gold-ref tasks
+ * at all (every record list is empty), every metric collapses to a zero
+ * envelope and the `passRateByRank` table is empty — the renderer turns
+ * that into a single "(N/A)" sentence.
+ */
+export function computeSearchBridge(report) {
+    const records = report.goldRankRecords ?? [];
+    // Histogram + percentile inputs across every search.
+    const histogram = emptyHistogram();
+    const allRanks = [];
+    let totalSearches = 0;
+    for (const rec of records) {
+        for (const ev of rec.searches) {
+            totalSearches += 1;
+            allRanks.push(ev.rankOfGold);
+            const bucket = ev.rankOfGold === null ? "missing" : String(ev.rankOfGold);
+            histogram[bucket] = (histogram[bucket] ?? 0) + 1;
+        }
+    }
+    const goldAtRank1 = totalSearches === 0 ? 0 : (histogram["1"] ?? 0) / totalSearches;
+    const goldMissing = totalSearches === 0 ? 0 : (histogram.missing ?? 0) / totalSearches;
+    const goldRankP50 = totalSearches === 0 ? null : percentile(allRanks, 50);
+    const goldRankP90 = totalSearches === 0 ? null : percentile(allRanks, 90);
+    // pass_rate_by_rank — split runs by the rank in *the search the agent
+    // actually ran*. We use the last `akm search` of the run (or "missing"
+    // when no search at all happened, or "missing" when the agent searched
+    // but gold wasn't in the top 10 in that final search). Runs without any
+    // `akm search` invocation are dropped from this slice — `pass_rate_by_rank`
+    // only describes what happened given a search.
+    const passRateBuckets = new Map();
+    for (const rec of records) {
+        if (rec.searches.length === 0)
+            continue;
+        const chosen = rec.searches[rec.searches.length - 1];
+        const bucket = chosen.rankOfGold === null ? "missing" : String(chosen.rankOfGold);
+        const slot = passRateBuckets.get(bucket) ?? { passes: 0, total: 0 };
+        slot.total += 1;
+        if (rec.outcome === "pass")
+            slot.passes += 1;
+        passRateBuckets.set(bucket, slot);
+    }
+    const passRateByRank = [];
+    for (const rank of histogramKeys()) {
+        const slot = passRateBuckets.get(rank);
+        if (!slot)
+            continue;
+        passRateByRank.push({
+            rank,
+            passRate: slot.total === 0 ? 0 : slot.passes / slot.total,
+            runCount: slot.total,
+        });
+    }
+    return {
+        goldRankDistribution: histogram,
+        goldRankP50,
+        goldRankP90,
+        goldAtRank1,
+        goldMissing,
+        passRateByRank,
+        runsObserved: records.length,
+        searchesObserved: totalSearches,
+    };
+}
+/** Ordered keys used for both the histogram and the pass_rate_by_rank table. */
+export function histogramKeys() {
+    return ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "missing"];
+}
+function emptyHistogram() {
+    const out = {};
+    for (const k of histogramKeys())
+        out[k] = 0;
+    return out;
+}
+/**
+ * Linear-interpolated percentile over a list of ranks. `null` ranks are
+ * treated as `Infinity` so the missing bucket pushes percentiles up
+ * correctly. Returns `Infinity` when the percentile lands in the missing
+ * region; the renderer surfaces that as the literal `"missing"` token so
+ * downstream JSON consumers don't choke on `Infinity`.
+ */
+function percentile(ranks, p) {
+    if (ranks.length === 0)
+        return Number.NaN;
+    const sorted = ranks.map((r) => (r === null ? Number.POSITIVE_INFINITY : r)).sort((a, b) => a - b);
+    // Nearest-rank method (avoids interpolation between Infinity and a finite).
+    // index = ceil(p/100 * N) - 1, clamped to [0, N-1].
+    const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
+    return sorted[idx];
+}
+/**
+ * Aggregate proposal-quality metrics from the evolve runner's proposal log.
+ * Pure function — does not touch disk and does not invoke any subprocess.
+ */
+export function computeProposalQualityMetrics(proposalLog) {
+    const byRef = new Map();
+    let totalAccepted = 0;
+    let totalLintPass = 0;
+    for (const entry of proposalLog) {
+        let row = byRef.get(entry.assetRef);
+        if (!row) {
+            row = { assetRef: entry.assetRef, proposalCount: 0, lintPassCount: 0, acceptedCount: 0 };
+            byRef.set(entry.assetRef, row);
+        }
+        row.proposalCount += 1;
+        if (entry.lintPass) {
+            row.lintPassCount += 1;
+            totalLintPass += 1;
+        }
+        if (entry.decision === "accept") {
+            row.acceptedCount += 1;
+            totalAccepted += 1;
+        }
+    }
+    const rows = [...byRef.values()].sort((a, b) => a.assetRef.localeCompare(b.assetRef));
+    const totalProposals = proposalLog.length;
+    return {
+        rows,
+        totalProposals,
+        totalAccepted,
+        acceptanceRate: totalProposals === 0 ? 0 : totalAccepted / totalProposals,
+        lintPassRate: totalProposals === 0 ? 0 : totalLintPass / totalProposals,
+    };
+}
+/**
+ * Compute longitudinal metrics from three §13.3 utility-shaped reports. Each
+ * input report is expected to share the same eval-slice corpus, with one arm
+ * driving the akm side: `pre` = pre-evolve stash, `post` = evolved stash,
+ * `synthetic` = no-stash scratchpad arm.
+ *
+ * The "arm" we read off each report is `aggregateAkm.passRate` — the runners
+ * produce the akm arm for all three (synthetic is just the akm arm with a
+ * stripped stashDir; pre/post differ by stash content). `seedsPerArm` for
+ * the degradation threshold is taken from the post report's corpus envelope.
+ */
+export function computeLongitudinalMetrics(preReport, postReport, syntheticReport) {
+    const prePassRate = preReport.aggregateAkm.passRate;
+    const postPassRate = postReport.aggregateAkm.passRate;
+    const syntheticPassRate = syntheticReport.aggregateAkm.passRate;
+    const seedsPerArm = Math.max(1, postReport.corpus.seedsPerArm);
+    const oneSeedFraction = 1 / seedsPerArm;
+    // Per-task degradation: outer-join pre and post on task id.
+    const preTasks = new Map();
+    for (const t of preReport.tasks)
+        preTasks.set(t.id, t);
+    const postTasks = new Map();
+    for (const t of postReport.tasks)
+        postTasks.set(t.id, t);
+    // Index post failure-mode labels by task id (one mode per task — first
+    // failed run wins; matches the §6.6 by-task aggregate's natural ordering).
+    const postFailureByTask = {};
+    const postFailureByTaskMap = postReport.failureModes?.byTask ?? {};
+    for (const [taskId, byMode] of Object.entries(postFailureByTaskMap)) {
+        const labels = Object.keys(byMode);
+        if (labels.length > 0)
+            postFailureByTask[taskId] = labels[0];
+    }
+    const degradations = [];
+    const allIds = new Set();
+    for (const id of preTasks.keys())
+        allIds.add(id);
+    for (const id of postTasks.keys())
+        allIds.add(id);
+    for (const id of [...allIds].sort()) {
+        const pre = preTasks.get(id);
+        const post = postTasks.get(id);
+        if (!pre || !post)
+            continue;
+        const preRate = pre.akm.passRate;
+        const postRate = post.akm.passRate;
+        const dropped = preRate - postRate;
+        if (dropped > oneSeedFraction) {
+            degradations.push({
+                taskId: id,
+                prePassRate: preRate,
+                postPassRate: postRate,
+                delta: postRate - preRate,
+                failureMode: postFailureByTask[id] ?? null,
+            });
+        }
+    }
+    return {
+        improvementSlope: postPassRate - prePassRate,
+        overSyntheticLift: postPassRate - syntheticPassRate,
+        degradationCount: degradations.length,
+        degradations,
+        prePassRate,
+        postPassRate,
+        syntheticPassRate,
+    };
+}
+/** Threshold above `pass_rate[0]` that defines "improvement" for §6.4. */
+export const LEARNING_IMPROVEMENT_THRESHOLD = 0.05;
+export function computeLearningCurve(episodes) {
+    // Stable sort by episode_index — defensive against unordered inputs.
+    const sorted = [...episodes].sort((a, b) => a.episode_index - b.episode_index);
+    // Recompute per-episode deltas so the contract holds regardless of what
+    // the caller stamped on the input record.
+    const normalised = sorted.map((ep, i) => {
+        const prev = i === 0 ? null : sorted[i - 1];
+        const delta = prev === null ? 0 : ep.pass_rate - prev.pass_rate;
+        return { ...ep, delta_from_previous_episode: delta };
+    });
+    const passRateByEpisode = normalised.map((ep) => ep.pass_rate);
+    // Linear regression slope: sum((xi - x_mean) * (yi - y_mean)) /
+    // sum((xi - x_mean)^2). For a single episode the denominator is 0 — we
+    // return 0 (no observable trend) rather than NaN.
+    const n = normalised.length;
+    let learningSlope = 0;
+    if (n >= 2) {
+        const xs = normalised.map((ep) => ep.episode_index);
+        const xMean = xs.reduce((s, v) => s + v, 0) / n;
+        const yMean = passRateByEpisode.reduce((s, v) => s + v, 0) / n;
+        let num = 0;
+        let den = 0;
+        for (let i = 0; i < n; i += 1) {
+            const dx = xs[i] - xMean;
+            const dy = passRateByEpisode[i] - yMean;
+            num += dx * dy;
+            den += dx * dx;
+        }
+        learningSlope = den === 0 ? 0 : num / den;
+    }
+    // time_to_improvement: smallest episode_index strictly greater than
+    // `pass_rate[0] + threshold`. Episode 0 itself is excluded — improvement
+    // is only meaningful relative to baseline.
+    let timeToImprovement = null;
+    if (n >= 2) {
+        const baseline = passRateByEpisode[0];
+        for (let i = 1; i < n; i += 1) {
+            if (passRateByEpisode[i] > baseline + LEARNING_IMPROVEMENT_THRESHOLD) {
+                timeToImprovement = normalised[i].episode_index;
+                break;
+            }
+        }
+    }
+    return {
+        episodes: normalised,
+        pass_rate_by_episode: passRateByEpisode,
+        learning_slope: learningSlope,
+        time_to_improvement: timeToImprovement,
+    };
+}
+/**
+ * Compute the §6.8 feedback-signal integrity confusion matrix.
+ *
+ * Pure function — does not touch disk and does not invoke any subprocess.
+ * The join is by `(taskId, seed)` so that a feedback event is attributed
+ * to the run that produced it, NOT to a later run that happens to touch
+ * the same gold ref. This matters when the same gold ref appears across
+ * multiple Phase 1 runs (e.g. multiple seeds, or two tasks sharing a
+ * skill); the per-asset row aggregates across all runs that referenced it
+ * in feedback, but each individual feedback event's matrix cell is
+ * decided by its own run's outcome.
+ *
+ * NaN-safety: a per-asset row with zero feedback events (cannot happen via
+ * this function — every row is derived from at least one feedback entry —
+ * but defensive against future callers passing curated subsets) emits all
+ * three rates as `null`. `false_positive_rate` is `null` when `FP+TN===0`
+ * even if the row has `FN+TP>0`, and vice versa.
+ */
+export function computeFeedbackIntegrity(input) {
+    const akmRuns = input.phase1.akmRuns ?? [];
+    // Build a (taskId, seed) → outcome lookup so every feedback event
+    // resolves in O(1). When two runs share the same key (shouldn't happen
+    // — runner emits unique seeds per task — but defensive) the first
+    // wins.
+    const runOutcomeByKey = new Map();
+    for (const r of akmRuns) {
+        const key = `${r.taskId}::${r.seed}`;
+        if (!runOutcomeByKey.has(key))
+            runOutcomeByKey.set(key, r.outcome);
+    }
+    const perRef = new Map();
+    let aggTP = 0;
+    let aggFP = 0;
+    let aggTN = 0;
+    let aggFN = 0;
+    // Track which (taskId, seed) keys had any feedback dispatched (ok or
+    // not), for the coverage denominator. We count an attempted dispatch as
+    // covered — if `ok===false`, the operator wanted feedback but the CLI
+    // failed; that's still a covered run for the purpose of §6.8 (and is
+    // surfaced in the warnings list elsewhere).
+    const coveredKeys = new Set();
+    for (const fb of input.feedbackLog) {
+        const key = `${fb.taskId}::${fb.seed}`;
+        coveredKeys.add(key);
+        if (!fb.ok)
+            continue; // failed dispatches don't label a matrix cell.
+        const outcome = runOutcomeByKey.get(key);
+        if (outcome === undefined)
+            continue; // run not found — defensive, drop.
+        // harness_error runs are not labelled (the bench skips dispatching
+        // feedback for them; if a fake test injects one, we drop it from the
+        // matrix to avoid mislabelling).
+        if (outcome === "harness_error")
+            continue;
+        const passed = outcome === "pass";
+        let row = perRef.get(fb.goldRef);
+        if (!row) {
+            row = { truePositive: 0, falsePositive: 0, trueNegative: 0, falseNegative: 0 };
+            perRef.set(fb.goldRef, row);
+        }
+        if (fb.signal === "positive" && passed) {
+            row.truePositive += 1;
+            aggTP += 1;
+        }
+        else if (fb.signal === "positive" && !passed) {
+            row.falsePositive += 1;
+            aggFP += 1;
+        }
+        else if (fb.signal === "negative" && !passed) {
+            row.trueNegative += 1;
+            aggTN += 1;
+        }
+        else if (fb.signal === "negative" && passed) {
+            row.falseNegative += 1;
+            aggFN += 1;
+        }
+    }
+    const aggTotal = aggTP + aggFP + aggTN + aggFN;
+    const totalPhase1Runs = akmRuns.length;
+    const aggregate = {
+        truePositive: aggTP,
+        falsePositive: aggFP,
+        trueNegative: aggTN,
+        falseNegative: aggFN,
+        feedback_agreement: aggTotal === 0 ? 0 : (aggTP + aggTN) / aggTotal,
+        false_positive_rate: aggFP + aggTN === 0 ? 0 : aggFP / (aggFP + aggTN),
+        false_negative_rate: aggFN + aggTP === 0 ? 0 : aggFN / (aggFN + aggTP),
+        feedback_coverage: totalPhase1Runs === 0 ? 0 : coveredKeys.size / totalPhase1Runs,
+    };
+    const perAsset = [];
+    for (const [ref, row] of [...perRef.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+        const total = row.truePositive + row.falsePositive + row.trueNegative + row.falseNegative;
+        const fpDenom = row.falsePositive + row.trueNegative;
+        const fnDenom = row.falseNegative + row.truePositive;
+        perAsset.push({
+            ref,
+            truePositive: row.truePositive,
+            falsePositive: row.falsePositive,
+            trueNegative: row.trueNegative,
+            falseNegative: row.falseNegative,
+            feedback_agreement: total === 0 ? null : (row.truePositive + row.trueNegative) / total,
+            false_positive_rate: fpDenom === 0 ? null : row.falsePositive / fpDenom,
+            false_negative_rate: fnDenom === 0 ? null : row.falseNegative / fnDenom,
+        });
+    }
+    return { aggregate, perAsset };
+}
+function aggregateByKey(entries, pickKey) {
+    const buckets = new Map();
+    for (const entry of entries) {
+        const key = pickKey(entry);
+        if (!key)
+            continue;
+        let arr = buckets.get(key);
+        if (!arr) {
+            arr = [];
+            buckets.set(key, arr);
+        }
+        arr.push(entry);
+    }
+    const rows = [];
+    for (const [category, group] of buckets) {
+        const n = group.length;
+        let noakmSum = 0;
+        let akmSum = 0;
+        let regressionCount = 0;
+        let complianceSum = 0;
+        let complianceCount = 0;
+        for (const t of group) {
+            noakmSum += t.noakm.passRate;
+            akmSum += t.akm.passRate;
+            if (t.akm.passRate < t.noakm.passRate)
+                regressionCount += 1;
+            if (typeof t.workflowCompliance === "number" && Number.isFinite(t.workflowCompliance)) {
+                complianceSum += t.workflowCompliance;
+                complianceCount += 1;
+            }
+        }
+        rows.push({
+            category,
+            taskCount: n,
+            passRateNoakm: noakmSum / n,
+            passRateAkm: akmSum / n,
+            passRateDelta: akmSum / n - noakmSum / n,
+            negativeTransferCount: regressionCount,
+            workflowCompliance: complianceCount === 0 ? null : complianceSum / complianceCount,
+        });
+    }
+    rows.sort((a, b) => a.category.localeCompare(b.category));
+    return rows;
+}
+/**
+ * Aggregate per-task entries by `memoryAbility` (#262). Tasks lacking a tag
+ * are skipped so the report only surfaces categories with explicit
+ * coverage. Output rows are sorted by category for byte-stable JSON.
+ *
+ * The closed set of memory-ability values is exported as
+ * {@link MEMORY_ABILITY_VALUES} from `corpus.ts`.
+ */
+export function aggregateByMemoryAbility(entries) {
+    return aggregateByKey(entries, (e) => e.memoryAbility);
+}
+/**
+ * Aggregate per-task entries by `taskFamily` (#262). Tasks lacking a tag
+ * are skipped. `taskFamily` follows the `<domain>/<short-name>` grammar —
+ * tasks sharing a family are expected to transfer knowledge between each
+ * other. Output rows are sorted by category for byte-stable JSON.
+ */
+export function aggregateByTaskFamily(entries) {
+    return aggregateByKey(entries, (e) => e.taskFamily);
+}
+export function computeCorpusCoverage(tasks) {
+    const memoryAbilityCounts = {
+        untagged: 0,
+    };
+    for (const ability of MEMORY_ABILITY_VALUES) {
+        memoryAbilityCounts[ability] = 0;
+    }
+    const taskFamilyCounts = {};
+    let untaggedFamily = 0;
+    for (const task of tasks) {
+        if (task.memoryAbility) {
+            memoryAbilityCounts[task.memoryAbility] = (memoryAbilityCounts[task.memoryAbility] ?? 0) + 1;
+        }
+        else {
+            memoryAbilityCounts.untagged += 1;
+        }
+        if (task.taskFamily) {
+            taskFamilyCounts[task.taskFamily] = (taskFamilyCounts[task.taskFamily] ?? 0) + 1;
+        }
+        else {
+            untaggedFamily += 1;
+        }
+    }
+    if (untaggedFamily > 0)
+        taskFamilyCounts.untagged = untaggedFamily;
+    return {
+        totalTasks: tasks.length,
+        memoryAbilityCounts,
+        taskFamilyCounts,
+    };
+}
+/**
+ * Verb counts considered "AKM tool calls" for `totalToolCalls`. We
+ * deliberately keep this list small — each verb folded in MUST be a
+ * user-initiated CLI invocation, not a background bookkeeping event.
+ * Adding new verbs here is additive and changes only `totalToolCalls`.
+ */
+export const AKM_TOOL_CALL_TYPES = new Set([
+    "akm_search",
+    "akm_show",
+    "akm_feedback",
+]);
+/**
+ * Compute per-run AKM overhead records by replaying #254's normalised trace.
+ *
+ * Pure function: never mutates `runs` and never reads disk. The optional
+ * `taskMetadata` lookup is used only to label loads as relevant / irrelevant
+ * and to compute `timeToFirstCorrectAssetMs`.
+ *
+ * Returned array length matches `runs.length`; element order matches input
+ * order. Runs whose trace contains no AKM events still produce a record
+ * with all counts at zero and timings at `null`.
+ */
+export function computeAkmOverhead(runs, options = {}) {
+    const out = [];
+    for (const run of runs) {
+        out.push(perRun(run, options.taskMetadata));
+    }
+    return out;
+}
+function perRun(run, taskMetadata) {
+    const trace = normalizeRunToTrace(run);
+    const events = trace.events;
+    let searchCount = 0;
+    let showCount = 0;
+    let feedbackCount = 0;
+    const uniqueShowRefs = new Set();
+    for (const ev of events) {
+        if (ev.type === "akm_search")
+            searchCount += 1;
+        else if (ev.type === "akm_show") {
+            showCount += 1;
+            if (typeof ev.assetRef === "string" && ev.assetRef.length > 0) {
+                uniqueShowRefs.add(ev.assetRef);
+            }
+        }
+        else if (ev.type === "akm_feedback")
+            feedbackCount += 1;
+    }
+    const totalToolCalls = searchCount + showCount + feedbackCount;
+    // Run-start anchor: earliest parseable ts in the trace. We use the trace
+    // (not RunResult.events directly) so harness lifecycle markers, when
+    // supplied, can serve as the anchor for stdout-derived events that lack a
+    // native ts.
+    const runStartMs = earliestEventMs(events);
+    const timeToFirstSearchMs = computeFirstEventOffsetMs(events, runStartMs, (ev) => ev.type === "akm_search");
+    // Resolve task metadata once. Missing metadata means we can't judge
+    // relevance — emit null counts rather than zero.
+    const meta = taskMetadata?.get(run.taskId);
+    const goldRef = meta?.goldRef;
+    const transferFrom = meta?.expectedTransferFrom ?? [];
+    const knownRelevant = new Set();
+    if (typeof goldRef === "string" && goldRef.length > 0)
+        knownRelevant.add(goldRef);
+    for (const r of transferFrom) {
+        if (typeof r === "string" && r.length > 0)
+            knownRelevant.add(r);
+    }
+    let irrelevantAssetsLoadedCount;
+    if (!meta) {
+        // No metadata: cannot tell relevant from irrelevant. Surface null.
+        irrelevantAssetsLoadedCount = null;
+    }
+    else {
+        let count = 0;
+        for (const ref of uniqueShowRefs) {
+            if (!knownRelevant.has(ref))
+                count += 1;
+        }
+        irrelevantAssetsLoadedCount = count;
+    }
+    let timeToFirstCorrectAssetMs = null;
+    if (typeof goldRef === "string" && goldRef.length > 0) {
+        timeToFirstCorrectAssetMs = computeFirstEventOffsetMs(events, runStartMs, (ev) => ev.type === "akm_show" && ev.assetRef === goldRef);
+    }
+    return {
+        taskId: run.taskId,
+        arm: run.arm,
+        seed: run.seed,
+        outcome: run.outcome,
+        searchCount,
+        showCount,
+        feedbackCount,
+        totalToolCalls,
+        assetsLoadedCount: uniqueShowRefs.size,
+        irrelevantAssetsLoadedCount,
+        timeToFirstSearchMs,
+        timeToFirstCorrectAssetMs,
+        // Byte sizes are not yet wired through the trace (#254 does not capture
+        // payload sizes). Callers MUST treat null as "unavailable", not zero.
+        contextBytesLoaded: null,
+        assetBytesLoaded: null,
+    };
+}
+/**
+ * Aggregate per-run AKM overhead records into the corpus-wide block (#263).
+ *
+ * Pure: never mutates `perRun`. When `perRun` is empty, returns a zero/null
+ * envelope so callers can render a "no AKM activity" section without
+ * branching. `passingRuns === 0` always implies `toolCallsPerSuccess === null`
+ * and `costPerSuccess === null`.
+ */
+export function aggregateAkmOverhead(perRun, rawRuns = []) {
+    const n = perRun.length;
+    if (n === 0) {
+        return {
+            totalRuns: 0,
+            passingRuns: 0,
+            meanSearchCount: 0,
+            meanShowCount: 0,
+            meanFeedbackCount: 0,
+            meanToolCalls: 0,
+            meanAssetsLoaded: 0,
+            meanIrrelevantAssetsLoaded: null,
+            meanTimeToFirstSearchMs: null,
+            meanTimeToFirstCorrectAssetMs: null,
+            meanContextBytesLoaded: null,
+            meanAssetBytesLoaded: null,
+            totalToolCalls: 0,
+            toolCallsPerSuccess: null,
+            costPerSuccess: null,
+        };
+    }
+    let searchSum = 0;
+    let showSum = 0;
+    let feedbackSum = 0;
+    let toolCallsSum = 0;
+    let assetsSum = 0;
+    let irrelevantSum = 0;
+    let irrelevantCount = 0;
+    let firstSearchSum = 0;
+    let firstSearchCount = 0;
+    let firstCorrectSum = 0;
+    let firstCorrectCount = 0;
+    let contextBytesSum = 0;
+    let contextBytesCount = 0;
+    let assetBytesSum = 0;
+    let assetBytesCount = 0;
+    // Build a quick lookup for token measurement off `rawRuns` so the cost-
+    // per-success calc can honour the parsed/missing/unsupported distinction
+    // without forcing the caller to project tokens onto AkmOverheadPerRun.
+    const rawByKey = new Map();
+    for (const r of rawRuns) {
+        rawByKey.set(`${r.taskId}${r.arm}${r.seed}`, r);
+    }
+    let passingRuns = 0;
+    let parsedPassTokenSum = 0;
+    let parsedPassCount = 0;
+    let anyPassMissingMeasurement = false;
+    for (const row of perRun) {
+        searchSum += row.searchCount;
+        showSum += row.showCount;
+        feedbackSum += row.feedbackCount;
+        toolCallsSum += row.totalToolCalls;
+        assetsSum += row.assetsLoadedCount;
+        if (row.irrelevantAssetsLoadedCount !== null) {
+            irrelevantSum += row.irrelevantAssetsLoadedCount;
+            irrelevantCount += 1;
+        }
+        if (row.timeToFirstSearchMs !== null) {
+            firstSearchSum += row.timeToFirstSearchMs;
+            firstSearchCount += 1;
+        }
+        if (row.timeToFirstCorrectAssetMs !== null) {
+            firstCorrectSum += row.timeToFirstCorrectAssetMs;
+            firstCorrectCount += 1;
+        }
+        if (row.contextBytesLoaded !== null) {
+            contextBytesSum += row.contextBytesLoaded;
+            contextBytesCount += 1;
+        }
+        if (row.assetBytesLoaded !== null) {
+            assetBytesSum += row.assetBytesLoaded;
+            assetBytesCount += 1;
+        }
+        if (row.outcome === "pass") {
+            passingRuns += 1;
+            const raw = rawByKey.get(`${row.taskId}${row.arm}${row.seed}`);
+            // Treat absent tokenMeasurement as `parsed` for backward compat with
+            // older artefacts (mirrors `isMeasured` behaviour above).
+            const measurement = raw?.tokenMeasurement ?? "parsed";
+            if (raw && measurement === "parsed") {
+                parsedPassTokenSum += raw.tokens.input + raw.tokens.output;
+                parsedPassCount += 1;
+            }
+            else if (raw) {
+                anyPassMissingMeasurement = true;
+            }
+            else {
+                // No matching raw run supplied — cannot honour cost-per-success.
+                anyPassMissingMeasurement = true;
+            }
+        }
+    }
+    const toolCallsPerSuccess = passingRuns === 0 ? null : toolCallsSum / passingRuns;
+    // Cost-per-success: null unless EVERY passing run has parsed measurement.
+    // Mixed measurement statuses cannot be averaged honestly (issue #252).
+    const costPerSuccess = passingRuns === 0 || anyPassMissingMeasurement || parsedPassCount === 0
+        ? null
+        : parsedPassTokenSum / parsedPassCount;
+    return {
+        totalRuns: n,
+        passingRuns,
+        meanSearchCount: searchSum / n,
+        meanShowCount: showSum / n,
+        meanFeedbackCount: feedbackSum / n,
+        meanToolCalls: toolCallsSum / n,
+        meanAssetsLoaded: assetsSum / n,
+        meanIrrelevantAssetsLoaded: irrelevantCount === 0 ? null : irrelevantSum / irrelevantCount,
+        meanTimeToFirstSearchMs: firstSearchCount === 0 ? null : firstSearchSum / firstSearchCount,
+        meanTimeToFirstCorrectAssetMs: firstCorrectCount === 0 ? null : firstCorrectSum / firstCorrectCount,
+        meanContextBytesLoaded: contextBytesCount === 0 ? null : contextBytesSum / contextBytesCount,
+        meanAssetBytesLoaded: assetBytesCount === 0 ? null : assetBytesSum / assetBytesCount,
+        totalToolCalls: toolCallsSum,
+        toolCallsPerSuccess,
+        costPerSuccess,
+    };
+}
+/**
+ * Bucket a workflow check status onto pass / non-pass for reliability.
+ *
+ * Reliability is a strict pass-or-not metric (issue #258). Anything other
+ * than `pass` (including `partial`, `fail`, `harness_error`) counts as a
+ * non-pass. `not_applicable` returns `null` so the caller can skip the
+ * entire (task, seed) pair — it never contributes to either numerator or
+ * denominator.
+ */
+function bucketReliabilityStatus(status) {
+    if (status === "not_applicable")
+        return null;
+    if (status === "pass")
+        return "pass";
+    return "non_pass";
+}
+/**
+ * Compute workflow reliability metrics (`pass@k` and `pass^k`) per workflow
+ * and corpus-wide from a flat list of `WorkflowCheckResult`.
+ *
+ * Methodology (per #258 review addendum):
+ *   1. Filter out `not_applicable` checks entirely.
+ *   2. For each `(workflow_id, task_id)` group, collapse seeds to the set
+ *      of statuses observed.
+ *   3. `pass_at_k` per task = 1 if at least one seed is `pass`, else 0.
+ *   4. `pass_all_k` per task = 1 if every seed is `pass`, else 0.
+ *   5. Per-workflow row averages over its task set.
+ *   6. Corpus rollup averages over every (workflow, task) group equally.
+ *
+ * Pure: never mutates `checks`. Returns a stable shape for empty input.
+ */
+export function computeWorkflowReliability(checks) {
+    // Group by (workflow_id, task_id) → list of statuses across seeds.
+    // Use Map<string, Map<string, WorkflowCheckStatus[]>> so iteration order
+    // is insertion order (deterministic given deterministic input).
+    const grouped = new Map();
+    for (const c of checks) {
+        if (bucketReliabilityStatus(c.status) === null)
+            continue;
+        let perWorkflow = grouped.get(c.workflowId);
+        if (!perWorkflow) {
+            perWorkflow = new Map();
+            grouped.set(c.workflowId, perWorkflow);
+        }
+        const list = perWorkflow.get(c.taskId);
+        if (list)
+            list.push(c.status);
+        else
+            perWorkflow.set(c.taskId, [c.status]);
+    }
+    const byWorkflow = {};
+    let corpusPassAtKSum = 0;
+    let corpusPassAllKSum = 0;
+    let corpusGroupCount = 0;
+    const corpusTasks = new Set();
+    for (const [workflowId, perTask] of grouped) {
+        let passAtKSum = 0;
+        let passAllKSum = 0;
+        let kMax = 0;
+        for (const [taskId, statuses] of perTask) {
+            if (statuses.length > kMax)
+                kMax = statuses.length;
+            const allPass = statuses.every((s) => s === "pass");
+            const anyPass = statuses.some((s) => s === "pass");
+            if (anyPass)
+                passAtKSum += 1;
+            if (allPass)
+                passAllKSum += 1;
+            corpusPassAtKSum += anyPass ? 1 : 0;
+            corpusPassAllKSum += allPass ? 1 : 0;
+            corpusGroupCount += 1;
+            corpusTasks.add(taskId);
+        }
+        const taskCount = perTask.size;
+        byWorkflow[workflowId] = {
+            workflow_id: workflowId,
+            pass_at_k: taskCount === 0 ? 0 : passAtKSum / taskCount,
+            pass_all_k: taskCount === 0 ? 0 : passAllKSum / taskCount,
+            tasks: taskCount,
+            k: kMax,
+        };
+    }
+    const corpus = {
+        pass_at_k: corpusGroupCount === 0 ? 0 : corpusPassAtKSum / corpusGroupCount,
+        pass_all_k: corpusGroupCount === 0 ? 0 : corpusPassAllKSum / corpusGroupCount,
+        groups: corpusGroupCount,
+        tasks: corpusTasks.size,
+    };
+    return { byWorkflow, corpus };
+}
+/** Earliest parseable ts (ms epoch) among events; null when none. */
+function earliestEventMs(events) {
+    let earliest = null;
+    for (const ev of events) {
+        const ms = parseTsToMs(ev.ts);
+        if (ms === null)
+            continue;
+        if (earliest === null || ms < earliest)
+            earliest = ms;
+    }
+    return earliest;
+}
+/**
+ * Find the first event matching `predicate`, parse its ts, and return
+ * `(ts - runStartMs)`. Returns `null` if no matching event has a parseable
+ * ts, if `runStartMs` is null, or if the offset would be negative (a clock
+ * inversion we refuse to silently coerce to zero).
+ */
+function computeFirstEventOffsetMs(events, runStartMs, predicate) {
+    if (runStartMs === null)
+        return null;
+    for (const ev of events) {
+        if (!predicate(ev))
+            continue;
+        const ms = parseTsToMs(ev.ts);
+        if (ms === null)
+            continue;
+        const offset = ms - runStartMs;
+        if (offset < 0)
+            return null;
+        return offset;
+    }
+    return null;
+}
+/** Parse an ISO ts to ms-epoch; null when missing or unparseable. */
+function parseTsToMs(ts) {
+    if (typeof ts !== "string" || ts.length === 0)
+        return null;
+    const ms = Date.parse(ts);
+    if (Number.isNaN(ms))
+        return null;
+    return ms;
+}