akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/runner.test.js

@@ -1,958 +0,0 @@
-/**
- * Unit tests for the K-seed runner.
- *
- * The runner is exercised end-to-end with an injected fake spawn so no real
- * opencode binary is required. We assert:
- *   • Cardinality: tasks × arms × seeds RunResults are produced.
- *   • Workspace isolation: each (arm, seed) sees a fresh cwd.
- *   • Cleanup: tmp dirs are torn down on success and failure.
- *   • Trajectory splice: goldRef + tool-call output produces the right boolean.
- */
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import path from "node:path";
-import { runUtility } from "./runner";
-import { benchMkdtemp } from "./tmp";
-function asReadableStream(text) {
-    const bytes = new TextEncoder().encode(text);
-    return new ReadableStream({
-        start(controller) {
-            controller.enqueue(bytes);
-            controller.close();
-        },
-    });
-}
-function fakeSpawnFactory(agentStdoutByArm = {}, options = {}) {
-    const observed = { cmd: [], cwd: undefined, armSeen: [] };
-    let lastArmCacheHome;
-    const spawn = (cmd, opts) => {
-        observed.cmd = cmd;
-        observed.cwd = opts.cwd;
-        const isAgent = cmd[0] === "opencode";
-        let arm = "noakm";
-        if (isAgent) {
-            arm = opts.env?.AKM_STASH_DIR ? "akm" : "noakm";
-            observed.armSeen.push(arm);
-            lastArmCacheHome = opts.env?.XDG_CACHE_HOME;
-        }
-        const stdout = isAgent
-            ? ((arm === "akm" ? agentStdoutByArm.akm : agentStdoutByArm.noakm) ?? "")
-            : (options.verifierStdout ?? "");
-        const exitCode = isAgent ? (options.agentExitCode ?? 0) : (options.verifierExitCode ?? 0);
-        const proc = {
-            exitCode,
-            exited: Promise.resolve(exitCode),
-            stdout: asReadableStream(stdout),
-            stderr: asReadableStream(""),
-            stdin: null,
-            kill() { },
-        };
-        // For akm-arm runs, drop a synthetic events.jsonl into the cache home so
-        // the trajectory parser sees a feedback event when the test wants one.
-        if (isAgent && arm === "akm" && lastArmCacheHome && agentStdoutByArm.akm?.includes("FEEDBACK")) {
-            const akmDir = path.join(lastArmCacheHome, "akm");
-            fs.mkdirSync(akmDir, { recursive: true });
-            fs.writeFileSync(path.join(akmDir, "events.jsonl"), `${JSON.stringify({ schemaVersion: 1, ts: "2026-04-27T00:00:00Z", eventType: "feedback", ref: "skill:foo" })}\n`);
-        }
-        return proc;
-    };
-    return { spawn, observed };
-}
-function fakeTask(taskDir, overrides = {}) {
-    return {
-        id: "fake/task-a",
-        title: "Fake task",
-        domain: "fake",
-        difficulty: "easy",
-        stash: "minimal",
-        verifier: "regex",
-        expectedMatch: "ok",
-        budget: { tokens: 1000, wallMs: 5000 },
-        taskDir,
-        ...overrides,
-    };
-}
-describe("runUtility", () => {
-    let workspaceRoot;
-    let taskDir;
-    beforeAll(() => {
-        workspaceRoot = benchMkdtemp("bench-runner-test-");
-        taskDir = path.join(workspaceRoot, "task-a");
-        fs.mkdirSync(taskDir, { recursive: true });
-        // No workspace template — runs start with empty cwd, which is valid.
-    });
-    afterAll(() => {
-        fs.rmSync(workspaceRoot, { recursive: true, force: true });
-    });
-    test("stamps corpus identity (selectedTaskIds, taskCorpusHash, fixtures, fixtureContentHash) (#250)", async () => {
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir, { id: "domain-a/task-z" }), fakeTask(taskDir, { id: "domain-a/task-a" })],
-            arms: ["noakm"],
-            model: "test-model",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            branch: "test-branch",
-            commit: "abc123",
-            timestamp: "2026-04-27T00:00:00Z",
-        });
-        // selectedTaskIds is sorted alphabetically.
-        expect(report.corpus.selectedTaskIds).toEqual(["domain-a/task-a", "domain-a/task-z"]);
-        expect(report.corpus.taskCorpusHash).toMatch(/^[0-9a-f]{64}$/);
-        expect(report.corpus.fixtureContentHash).toMatch(/^[0-9a-f]{64}$/);
-        // The "minimal" fixture is referenced by both fakeTasks; one entry expected.
-        expect(report.corpus.fixtures).toBeDefined();
-        expect(Object.keys(report.corpus.fixtures ?? {})).toEqual(["minimal"]);
-        expect(report.corpus.fixtures?.minimal).toMatch(/^[0-9a-f]{64}$/);
-        // Re-running with the same inputs yields the same hashes (determinism).
-        const report2 = await runUtility({
-            tasks: [fakeTask(taskDir, { id: "domain-a/task-z" }), fakeTask(taskDir, { id: "domain-a/task-a" })],
-            arms: ["noakm"],
-            model: "test-model",
-            seedsPerArm: 1,
-            spawn: fakeSpawnFactory({ noakm: "ok", akm: "ok" }).spawn,
-            materialiseStash: false,
-            branch: "test-branch",
-            commit: "abc123",
-            timestamp: "2026-04-27T00:00:00Z",
-        });
-        expect(report2.corpus.taskCorpusHash).toBe(report.corpus.taskCorpusHash);
-        expect(report2.corpus.fixtureContentHash).toBe(report.corpus.fixtureContentHash);
-    });
-    test("produces tasks × arms × seeds run records (cardinality)", async () => {
-        const { spawn, observed } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test-model",
-            seedsPerArm: 3,
-            spawn,
-            materialiseStash: false,
-            branch: "test-branch",
-            commit: "abc123",
-            timestamp: "2026-04-27T00:00:00Z",
-        });
-        expect(report.tasks.length).toBe(1);
-        // 3 seeds × 2 arms = 6 agent invocations.
-        expect(observed.armSeen.length).toBe(6);
-        expect(observed.armSeen.filter((a) => a === "akm").length).toBe(3);
-        expect(observed.armSeen.filter((a) => a === "noakm").length).toBe(3);
-        // Per-task aggregates were filled.
-        const t = report.tasks[0];
-        expect(t).toBeDefined();
-        expect(t?.noakm.count).toBe(3);
-        expect(t?.akm.count).toBe(3);
-        expect(t?.noakm.passRate).toBe(1); // verifier exitCode 0 → pass
-        expect(t?.akm.passRate).toBe(1);
-    });
-    test("workspace isolation: each run gets a fresh cwd", async () => {
-        const cwds = new Set();
-        const spawn = (cmd, opts) => {
-            if (cmd[0] === "opencode" && opts.cwd)
-                cwds.add(opts.cwd);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(report.tasks.length).toBe(1);
-        // 2 seeds × 2 arms = 4 unique cwds.
-        expect(cwds.size).toBe(4);
-    });
-    test("cleanup: workspace tmp dirs are removed after each run", async () => {
-        const cwdsObserved = new Set();
-        const spawn = (cmd, opts) => {
-            if (cmd[0] === "opencode" && opts.cwd)
-                cwdsObserved.add(opts.cwd);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(cwdsObserved.size).toBe(2);
-        for (const cwd of cwdsObserved) {
-            expect(fs.existsSync(cwd)).toBe(false);
-        }
-    });
-    test("cleanup happens even when verifier reports failure (workspace still removed)", async () => {
-        const cwdsObserved = new Set();
-        const spawn = (cmd, opts) => {
-            if (cmd[0] === "opencode" && opts.cwd)
-                cwdsObserved.add(opts.cwd);
-            const isAgent = cmd[0] === "opencode";
-            const exitCode = isAgent ? 0 : 1;
-            return {
-                exitCode,
-                exited: Promise.resolve(exitCode),
-                stdout: asReadableStream(isAgent ? "nope" : ""),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(report.tasks[0]?.noakm.passRate).toBe(0);
-        for (const cwd of cwdsObserved) {
-            expect(fs.existsSync(cwd)).toBe(false);
-        }
-    });
-    test("trajectory splice: correctAssetLoaded + feedbackRecorded fill from akm-arm runs", async () => {
-        const akmStdout = "tool: akm show skill:foo\nFEEDBACK emitted\n";
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: akmStdout });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir, { goldRef: "skill:foo" })],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(report.trajectoryAkm.correctAssetLoaded).toBe(1);
-        expect(report.trajectoryAkm.feedbackRecorded).toBe(1);
-    });
-    test("workspace template files are copied into per-run cwd", async () => {
-        // Drop a sentinel file into the task's workspace/ template.
-        const wsTemplate = path.join(taskDir, "workspace");
-        fs.mkdirSync(wsTemplate, { recursive: true });
-        fs.writeFileSync(path.join(wsTemplate, "marker.txt"), "hello");
-        const seenContents = [];
-        const spawn = (cmd, opts) => {
-            if (cmd[0] === "opencode" && opts.cwd) {
-                const p = path.join(opts.cwd, "marker.txt");
-                if (fs.existsSync(p))
-                    seenContents.push(fs.readFileSync(p, "utf8"));
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        try {
-            await runUtility({
-                tasks: [fakeTask(taskDir)],
-                arms: ["noakm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-            });
-            expect(seenContents).toEqual(["hello"]);
-        }
-        finally {
-            fs.rmSync(wsTemplate, { recursive: true, force: true });
-        }
-    });
-    test("default seedsPerArm is 5", async () => {
-        const { spawn, observed } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm"],
-            model: "test",
-            spawn,
-            materialiseStash: false,
-        });
-        expect(observed.armSeen.length).toBe(5);
-    });
-    test("multi-task: each task lands in tasks[] in input order", async () => {
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const taskA = fakeTask(taskDir, { id: "alpha/x", domain: "alpha" });
-        const taskB = fakeTask(taskDir, { id: "beta/y", domain: "beta" });
-        const report = await runUtility({
-            tasks: [taskA, taskB],
-            arms: ["noakm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(report.tasks.map((t) => t.id)).toEqual(["alpha/x", "beta/y"]);
-        expect(report.corpus.domains).toBe(2);
-        expect(report.corpus.tasks).toBe(2);
-    });
-    // ── #267: per-arm prompt override ──────────────────────────────────────────
-    test("buildPrompt override forwards prompt to runOne when defined", async () => {
-        // Capture the agent command to assert the prompt forms its trailing arg
-        // (runAgent appends `prompt` as the last cmd token).
-        const capturedAgentCmds = [];
-        const spawn = (cmd, _opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent)
-                capturedAgentCmds.push([...cmd]);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            buildPrompt: (task, arm) => (arm === "akm" ? `BYOS scratchpad for ${task.id}` : undefined),
-        });
-        // 1 noakm + 1 akm = 2 agent invocations.
-        expect(capturedAgentCmds.length).toBe(2);
-        const akmInvocation = capturedAgentCmds.find((c) => c.some((t) => t.startsWith("BYOS scratchpad")));
-        expect(akmInvocation).toBeDefined();
-        const noakmInvocation = capturedAgentCmds.find((c) => !c.some((t) => t.startsWith("BYOS scratchpad")));
-        expect(noakmInvocation).toBeDefined();
-        // The noakm arm received the default prompt — assert the override didn't
-        // leak across arms.
-        expect(noakmInvocation?.some((t) => t.startsWith("BYOS"))).toBe(false);
-    });
-    // ── #251: TaskMetadata.stashDirOverride ───────────────────────────────────
-    test("stashDirOverride: AKM_STASH_DIR points at the override, never at __no-stash__", async () => {
-        // Issue #251 regression. Before the fix, `runMaskedCorpus` mutated
-        // `task.stash` and called the runner with `materialiseStash: false`,
-        // which ended up wiring `AKM_STASH_DIR` to `<taskDir>/__no-stash__` —
-        // so masked re-runs never saw the masked content. The fix is a
-        // dedicated `stashDirOverride` field that the runner consults FIRST.
-        const observedAkmStashDirs = [];
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent && opts.env?.AKM_STASH_DIR) {
-                observedAkmStashDirs.push(opts.env.AKM_STASH_DIR);
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const overrideDir = benchMkdtemp("akm-bench-251-override-");
-        try {
-            await runUtility({
-                tasks: [fakeTask(taskDir, { stashDirOverride: overrideDir })],
-                arms: ["akm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-            });
-            expect(observedAkmStashDirs.length).toBe(1);
-            expect(observedAkmStashDirs[0]).toBe(overrideDir);
-            // Critical: the runner MUST NOT have fallen back to the __no-stash__
-            // placeholder. This is the regression guard.
-            expect(observedAkmStashDirs[0]?.endsWith("__no-stash__")).toBe(false);
-        }
-        finally {
-            fs.rmSync(overrideDir, { recursive: true, force: true });
-        }
-    });
-    test("stashDirOverride: takes precedence over stashDirByFixture and materialised stash", async () => {
-        // Resolution order acceptance criterion: per-task override beats both
-        // the per-fixture map and the runner's own loadFixtureStash.
-        const observedAkmStashDirs = [];
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent && opts.env?.AKM_STASH_DIR) {
-                observedAkmStashDirs.push(opts.env.AKM_STASH_DIR);
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const overrideDir = benchMkdtemp("akm-bench-251-prec-");
-        const fixtureMapDir = benchMkdtemp("akm-bench-251-mapdir-");
-        try {
-            await runUtility({
-                tasks: [fakeTask(taskDir, { stashDirOverride: overrideDir, stash: "ignored-fixture" })],
-                arms: ["akm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-                stashDirByFixture: new Map([["ignored-fixture", fixtureMapDir]]),
-            });
-            expect(observedAkmStashDirs).toEqual([overrideDir]);
-        }
-        finally {
-            fs.rmSync(overrideDir, { recursive: true, force: true });
-            fs.rmSync(fixtureMapDir, { recursive: true, force: true });
-        }
-    });
-    test("buildPrompt returning undefined keeps the default prompt path", async () => {
-        const capturedAgentCmds = [];
-        const spawn = (cmd, _opts) => {
-            if (cmd[0] === "opencode")
-                capturedAgentCmds.push([...cmd]);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            buildPrompt: () => undefined,
-        });
-        // The driver's defaultPrompt() embeds the task id; assert that's what we
-        // got, not a custom prompt.
-        expect(capturedAgentCmds.length).toBe(1);
-        const trailing = capturedAgentCmds[0]?.[capturedAgentCmds[0].length - 1] ?? "";
-        expect(trailing).toContain("Task: fake/task-a");
-    });
-    // ── #261: optional synthetic arm ───────────────────────────────────────────
-    test("includeSynthetic: runs noakm + akm + synthetic over the same tasks/seeds", async () => {
-        // Spy on every agent invocation so we can assert which arm fired and
-        // that AKM_STASH_DIR is absent for the synthetic arm only.
-        const observedArms = [];
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent) {
-                const promptArg = cmd[cmd.length - 1] ?? "";
-                let arm;
-                if (promptArg.includes("Arm: synthetic"))
-                    arm = "synthetic";
-                else if (opts.env?.AKM_STASH_DIR)
-                    arm = "akm";
-                else
-                    arm = "noakm";
-                observedArms.push({ arm, akmStashDir: opts.env?.AKM_STASH_DIR });
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-            includeSynthetic: true,
-        });
-        // 2 seeds × 3 arms = 6 invocations.
-        expect(observedArms.length).toBe(6);
-        expect(observedArms.filter((o) => o.arm === "noakm").length).toBe(2);
-        expect(observedArms.filter((o) => o.arm === "akm").length).toBe(2);
-        expect(observedArms.filter((o) => o.arm === "synthetic").length).toBe(2);
-        // Synthetic-arm child env MUST NOT carry AKM_STASH_DIR.
-        for (const o of observedArms.filter((o) => o.arm === "synthetic")) {
-            expect(o.akmStashDir).toBeUndefined();
-        }
-        // Per-task synthetic metrics are spliced in; aggregateSynth populated.
-        expect(report.aggregateSynth).toBeDefined();
-        expect(report.aggregateSynth?.passRate).toBe(1);
-        expect(report.tasks[0]?.synthetic).toBeDefined();
-        expect(report.tasks[0]?.synthetic?.count).toBe(2);
-    });
-    test("default behaviour (no includeSynthetic) is byte-identical to two-arm output (no synthetic keys)", async () => {
-        // SNAPSHOT-STYLE TEST: run runUtility WITHOUT includeSynthetic, capture
-        // the JSON envelope keys, and assert no key contains 'synthetic'.
-        // Without this gate the byte-identical contract is unverifiable.
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            branch: "test-branch",
-            commit: "abc123",
-            timestamp: "2026-04-27T00:00:00Z",
-        });
-        // In-memory report shape: aggregateSynth absent.
-        expect(report.aggregateSynth).toBeUndefined();
-        expect(report.tasks[0]?.synthetic).toBeUndefined();
-        // Render the JSON envelope and walk every key recursively. None should
-        // contain 'synthetic'. We exercise the byte-identical envelope contract.
-        const { renderUtilityReport } = await import("./report");
-        const { json } = renderUtilityReport(report);
-        const allKeys = [];
-        const walk = (node) => {
-            if (Array.isArray(node)) {
-                for (const item of node)
-                    walk(item);
-            }
-            else if (node && typeof node === "object") {
-                for (const [k, v] of Object.entries(node)) {
-                    allKeys.push(k);
-                    walk(v);
-                }
-            }
-        };
-        walk(json);
-        const synthMentions = allKeys.filter((k) => k.toLowerCase().includes("synth"));
-        expect(synthMentions).toEqual([]);
-        // Markdown body must not mention 'synthetic' (case-insensitive).
-        const { markdown } = renderUtilityReport(report);
-        expect(markdown.toLowerCase().includes("synthetic")).toBe(false);
-    });
-    test("includeSynthetic: arm-keyed map defaults synthetic to 'arm-not-run' when arm is absent on a task", async () => {
-        // Edge case: build a grouped map manually that lacks the synthetic arm
-        // for a given task. The runner ALWAYS dispatches the arm when the flag
-        // is set, so this guard is enforced through the buildReport branch:
-        // when includeSynth is true but no synth runs landed, the per-task
-        // PerTaskMetrics carries `count: 0` (zero-result aggregate) — distinct
-        // from "arm not run at all" which omits the field. Verify the
-        // distinction: report tasks WITHOUT the flag have synthetic absent.
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const reportNoFlag = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(reportNoFlag.tasks[0]?.synthetic).toBeUndefined();
-    });
-    test("includeSynthetic: forwards a clear scratch-notes prompt to the synthetic arm", async () => {
-        // Acceptance criterion: synthetic arm has a clear scratch-notes prompt
-        // contract. We assert the default built-in prompt is forwarded when
-        // the caller does not supply a buildPrompt override.
-        const capturedAgentCmds = [];
-        const spawn = (cmd, _opts) => {
-            if (cmd[0] === "opencode")
-                capturedAgentCmds.push([...cmd]);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            includeSynthetic: true,
-        });
-        const synthCmd = capturedAgentCmds.find((c) => c[c.length - 1]?.includes("Arm: synthetic"));
-        expect(synthCmd).toBeDefined();
-        const synthPrompt = synthCmd?.[synthCmd.length - 1] ?? "";
-        expect(synthPrompt).toContain("Bring Your Own Skills");
-        expect(synthPrompt).toContain("scratchpad");
-        expect(synthPrompt).toContain("AKM_STASH_DIR is intentionally absent");
-    });
-    test("includeSynthetic: caller buildPrompt override wins for synthetic arm", async () => {
-        // When the caller threads a buildPrompt override that returns a string
-        // for the synthetic arm, the runner forwards that override verbatim
-        // instead of the built-in scratch-notes prompt. This preserves the
-        // Track B `runEvolve` integration where synthetic arm prompts are
-        // built by `buildSyntheticPrompt(task.id)`.
-        const capturedAgentCmds = [];
-        const spawn = (cmd, _opts) => {
-            if (cmd[0] === "opencode")
-                capturedAgentCmds.push([...cmd]);
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            includeSynthetic: true,
-            buildPrompt: (task, arm) => (arm === "synthetic" ? `OVERRIDE for ${task.id}` : undefined),
-        });
-        const synthCmd = capturedAgentCmds.find((c) => c[c.length - 1]?.startsWith("OVERRIDE for"));
-        expect(synthCmd).toBeDefined();
-        // The built-in scratch-notes contract MUST NOT have leaked through.
-        expect(synthCmd?.[synthCmd.length - 1]).not.toContain("Bring Your Own Skills");
-    });
-    test("includeSynthetic: report carries akm_over_synthetic_lift in aggregate", async () => {
-        // akm + noakm pass (regex verifier matches "ok"); synthetic fails (we
-        // emit "nope" on the synthetic agent stdout so the regex verifier
-        // rejects). The fake task uses regex/expectedMatch="ok" — see fakeTask().
-        const spawn = (cmd, _opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (!isAgent) {
-                return {
-                    exitCode: 0,
-                    exited: Promise.resolve(0),
-                    stdout: asReadableStream(""),
-                    stderr: asReadableStream(""),
-                    stdin: null,
-                    kill() { },
-                };
-            }
-            const promptArg = cmd[cmd.length - 1] ?? "";
-            const isSynth = promptArg.includes("Arm: synthetic");
-            const stdout = isSynth ? "nope" : "ok";
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream(stdout),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-            includeSynthetic: true,
-        });
-        expect(report.aggregateAkm.passRate).toBe(1);
-        expect(report.aggregateSynth?.passRate).toBe(0);
-        const { renderUtilityReport } = await import("./report");
-        const { json, markdown } = renderUtilityReport(report);
-        const aggregate = json.aggregate;
-        expect(aggregate.synthetic).toBeDefined();
-        expect(aggregate.akm_over_synthetic_lift).toBe(1);
-        // Markdown should mention the lift.
-        expect(markdown).toContain("akm_over_synthetic_lift");
-    });
-    test("includeSynthetic: markdown surfaces a warning when AKM fails to beat synthetic", async () => {
-        // Acceptance criterion: utility markdown summarizes when AKM fails to
-        // beat synthetic notes. Construct a scenario where akm_pass_rate <=
-        // synth_pass_rate — regex verifier expects "ok"; AKM agent emits "nope"
-        // (fail), synthetic + noakm emit "ok" (pass).
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (!isAgent) {
-                return {
-                    exitCode: 0,
-                    exited: Promise.resolve(0),
-                    stdout: asReadableStream(""),
-                    stderr: asReadableStream(""),
-                    stdin: null,
-                    kill() { },
-                };
-            }
-            const promptArg = cmd[cmd.length - 1] ?? "";
-            const isSynth = promptArg.includes("Arm: synthetic");
-            const isAkm = !isSynth && opts.env?.AKM_STASH_DIR !== undefined;
-            const stdout = isAkm ? "nope" : "ok";
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream(stdout),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["noakm", "akm"],
-            model: "test",
-            seedsPerArm: 2,
-            spawn,
-            materialiseStash: false,
-            includeSynthetic: true,
-        });
-        expect(report.aggregateAkm.passRate).toBe(0);
-        expect(report.aggregateSynth?.passRate).toBe(1);
-        const { renderUtilityReport } = await import("./report");
-        const { markdown } = renderUtilityReport(report);
-        expect(markdown).toContain(":warning:");
-        expect(markdown).toContain("AKM did not beat the synthetic-notes baseline");
-    });
-});
-// ── Workflow compliance plumbing (#257) ────────────────────────────────────
-describe("runUtility workflow compliance (#257)", () => {
-    let workspaceRoot;
-    let taskDir;
-    beforeAll(() => {
-        workspaceRoot = benchMkdtemp("bench-runner-wf-");
-        taskDir = path.join(workspaceRoot, "task");
-        fs.mkdirSync(taskDir, { recursive: true });
-    });
-    afterAll(() => {
-        fs.rmSync(workspaceRoot, { recursive: true, force: true });
-    });
-    test("loads workflow specs from the default directory and stamps workflowChecks on the report", async () => {
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [
-                // domain-name needs to match a task_domain in at least one shipping spec
-                // so the evaluator emits applicable checks (not_applicable rows are
-                // expected for the rest).
-                fakeTask(taskDir, { id: "docker-homelab/redis", domain: "docker-homelab" }),
-            ],
-            arms: ["akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-        });
-        expect(Array.isArray(report.workflowChecks)).toBe(true);
-        // At least one shipping spec applies to docker-homelab → some checks emitted.
-        expect((report.workflowChecks ?? []).length).toBeGreaterThan(0);
-    });
-    test("workflowsDir: '' disables workflow evaluation and yields no workflowChecks", async () => {
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir)],
-            arms: ["akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-            workflowsDir: "",
-        });
-        expect(report.workflowChecks).toEqual([]);
-    });
-    test("missing workflowsDir is recorded as a warning, not a crash", async () => {
-        const { setQuiet, resetQuiet } = await import("../../src/core/warn");
-        setQuiet(true);
-        try {
-            const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-            const report = await runUtility({
-                tasks: [fakeTask(taskDir)],
-                arms: ["akm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-                workflowsDir: "/path/that/does/not/exist",
-            });
-            expect(report.workflowChecks).toEqual([]);
-            const hasWarn = report.warnings.some((w) => w.includes("workflow specs"));
-            expect(hasWarn).toBe(true);
-        }
-        finally {
-            resetQuiet();
-        }
-    });
-    test("runner attaches taskOutcome to each workflowCheck so the cross-tab can attribute it", async () => {
-        const { spawn } = fakeSpawnFactory({ noakm: "ok", akm: "ok" });
-        const report = await runUtility({
-            tasks: [fakeTask(taskDir, { id: "docker-homelab/redis", domain: "docker-homelab" })],
-            arms: ["akm"],
-            model: "test",
-            seedsPerArm: 1,
-            spawn,
-            materialiseStash: false,
-        });
-        const c = (report.workflowChecks ?? [])[0];
-        expect(c).toBeDefined();
-        expect(typeof c?.taskOutcome).toBe("string");
-    });
-    test("passes search/result and test-presence flags into workflow evaluation", async () => {
-        const workflowsDir = benchMkdtemp("akm-bench-workflow-flags-");
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent && opts.env?.XDG_CACHE_HOME) {
-                const akmDir = path.join(opts.env.XDG_CACHE_HOME, "akm");
-                fs.mkdirSync(akmDir, { recursive: true });
-                fs.writeFileSync(path.join(akmDir, "events.jsonl"), [
-                    JSON.stringify({
-                        schemaVersion: 1,
-                        ts: "2026-04-27T00:00:00Z",
-                        eventType: "search",
-                        metadata: { query: "deploy", resultRefs: ["skill:foo"] },
-                    }),
-                    JSON.stringify({
-                        schemaVersion: 1,
-                        ts: "2026-04-27T00:00:01Z",
-                        eventType: "propose_invoked",
-                    }),
-                    JSON.stringify({
-                        schemaVersion: 1,
-                        ts: "2026-04-27T00:00:02Z",
-                        eventType: "promoted",
-                    }),
-                ].join("\n"));
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        try {
-            fs.writeFileSync(path.join(workflowsDir, "search.yaml"), `id: search-guard
-title: search guard
-applies_to:
-  arms: ["akm"]
-required_sequence:
-  - event: akm_search
-  - event: akm_show
-    required_if: search_has_relevant_result
-scoring:
-  required_steps_weight: 0.7
-  forbidden_steps_weight: 0.2
-  evidence_quality_weight: 0.1
-`, "utf8");
-            fs.writeFileSync(path.join(workflowsDir, "tests.yaml"), `id: tests-guard
-title: tests guard
-applies_to:
-  arms: ["akm"]
-required_sequence:
-  - event: akm_propose
-  - event: test_run
-    required_if: task_has_tests
-  - event: akm_proposal_accept
-scoring:
-  required_steps_weight: 0.7
-  forbidden_steps_weight: 0.2
-  evidence_quality_weight: 0.1
-`, "utf8");
-            const testsDir = path.join(taskDir, "tests");
-            fs.mkdirSync(testsDir, { recursive: true });
-            fs.writeFileSync(path.join(testsDir, "test_sample.py"), "def test_ok():\n    assert True\n");
-            const report = await runUtility({
-                tasks: [fakeTask(taskDir, { verifier: "pytest", goldRef: "skill:foo" })],
-                arms: ["akm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-                workflowsDir,
-            });
-            const byId = new Map((report.workflowChecks ?? []).map((check) => [check.workflowId, check]));
-            expect(byId.get("search-guard")?.violations.some((v) => v.code === "missing_required_event")).toBe(true);
-            expect(byId.get("tests-guard")?.violations.some((v) => v.code === "missing_required_event")).toBe(true);
-        }
-        finally {
-            fs.rmSync(workflowsDir, { recursive: true, force: true });
-            fs.rmSync(path.join(taskDir, "tests"), { recursive: true, force: true });
-        }
-    });
-    test("skips required_if workflow steps when runner-derived flags are false", async () => {
-        const workflowsDir = benchMkdtemp("akm-bench-workflow-flags-off-");
-        const spawn = (cmd, opts) => {
-            const isAgent = cmd[0] === "opencode";
-            if (isAgent && opts.env?.XDG_CACHE_HOME) {
-                const akmDir = path.join(opts.env.XDG_CACHE_HOME, "akm");
-                fs.mkdirSync(akmDir, { recursive: true });
-                fs.writeFileSync(path.join(akmDir, "events.jsonl"), `${JSON.stringify({
-                    schemaVersion: 1,
-                    ts: "2026-04-27T00:00:00Z",
-                    eventType: "search",
-                    metadata: { query: "deploy", resultRefs: ["skill:other"] },
-                })}\n`);
-            }
-            return {
-                exitCode: 0,
-                exited: Promise.resolve(0),
-                stdout: asReadableStream("ok"),
-                stderr: asReadableStream(""),
-                stdin: null,
-                kill() { },
-            };
-        };
-        try {
-            fs.writeFileSync(path.join(workflowsDir, "search.yaml"), `id: search-guard
-title: search guard
-applies_to:
-  arms: ["akm"]
-required_sequence:
-  - event: akm_search
-  - event: akm_show
-    required_if: search_has_relevant_result
-scoring:
-  required_steps_weight: 0.7
-  forbidden_steps_weight: 0.2
-  evidence_quality_weight: 0.1
-`, "utf8");
-            fs.writeFileSync(path.join(workflowsDir, "tests.yaml"), `id: tests-guard
-title: tests guard
-applies_to:
-  arms: ["akm"]
-required_sequence:
-  - event: akm_search
-  - event: test_run
-    required_if: task_has_tests
-scoring:
-  required_steps_weight: 0.7
-  forbidden_steps_weight: 0.2
-  evidence_quality_weight: 0.1
-`, "utf8");
-            const report = await runUtility({
-                tasks: [fakeTask(taskDir, { verifier: "regex", goldRef: "skill:foo" })],
-                arms: ["akm"],
-                model: "test",
-                seedsPerArm: 1,
-                spawn,
-                materialiseStash: false,
-                workflowsDir,
-            });
-            const byId = new Map((report.workflowChecks ?? []).map((check) => [check.workflowId, check]));
-            expect(byId.get("search-guard")?.status).toBe("pass");
-            expect(byId.get("tests-guard")?.status).toBe("pass");
-        }
-        finally {
-            fs.rmSync(workflowsDir, { recursive: true, force: true });
-        }
-    });
-});