akm-cli 0.6.0 → 0.7.0-rc1

package/dist/tests/bench/runner.js

@@ -0,0 +1,536 @@
+/**
+ * akm-bench K-seed runner (spec §5 + §6).
+ *
+ * `runUtility(options)` is the single entry point used by both the CLI
+ * dispatcher (`tests/bench/cli.ts utility`) and unit tests. It expands the
+ * caller's `(tasks × arms × seeds)` cartesian product, calls `runOne` for
+ * each triple, splices the trajectory record back in, and returns a
+ * `UtilityRunReport` that `renderUtilityReport` can stamp into JSON +
+ * markdown.
+ *
+ * Per-(arm, seed) isolation:
+ *   • Workspace: each (task, arm, seed) gets a fresh tmp dir seeded from the
+ *     task's `workspace/` template so runs cannot pollute each other.
+ *   • Stash: only the `akm` arm materialises a stash via `loadFixtureStash`.
+ *     We materialise once per task (the stash content is identical across
+ *     the K seeds) and reuse it.
+ *
+ * Cleanup: every tmp resource is wrapped in `try/finally`. We never leak
+ * tmp dirs even on harness exceptions.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { warn } from "../../src/core/warn";
+import { computeFixtureContentHash, loadFixtureStash } from "../fixtures/stashes/load";
+import { registerCleanup } from "./cleanup";
+import { computeTaskCorpusHash, readTaskBody } from "./corpus";
+import { runOne } from "./driver";
+import { aggregateCorpus, aggregateFailureModes, aggregatePerTask, aggregateTrajectory, classifyFailureMode, computeCorpusDelta, computePerAssetAttribution, computePerTaskDelta, computeSearchBridge, extractAssetLoads, extractGoldRanks, } from "./metrics";
+import { resolveGitBranch, resolveGitCommit } from "./report";
+import { benchMkdtemp } from "./tmp";
+import { computeTrajectory } from "./trajectory";
+import { evaluateRunAgainstAllSpecs, } from "./workflow-evaluator";
+import { loadAllWorkflowSpecs } from "./workflow-spec";
+import { normalizeRunToTrace } from "./workflow-trace";
+/**
+ * Default workflows directory. Can be overridden by callers (tests) via
+ * `RunUtilityOptions.workflowsDir`. Specs in this directory are loaded ONCE
+ * per `runUtility` call (not per run) — the evaluator filters via each spec's
+ * `applies_to` so we don't I/O in the hot loop.
+ */
+const DEFAULT_WORKFLOWS_DIR = path.resolve(__dirname, "..", "fixtures", "bench", "workflows");
+/**
+ * Run K seeds × len(arms) × len(tasks) and return the §13.3 report.
+ *
+ * The function is robust to per-run failures — `runOne` already captures
+ * every failure path into a RunResult, so the runner only has to worry
+ * about its own infrastructure (stash materialisation, workspace copy).
+ * Those failures are recorded as `harness_error` runs.
+ */
+export async function runUtility(options) {
+    const seedsPerArm = options.seedsPerArm ?? 5;
+    const budgetTokens = options.budgetTokens ?? 30000;
+    const budgetWallMs = options.budgetWallMs ?? 120000;
+    const slice = options.slice ?? "all";
+    const materialiseStash = options.materialiseStash ?? true;
+    const grouped = new Map();
+    const warnings = [];
+    const goldRankRecords = [];
+    // #257: load workflow specs ONCE per runUtility call. Skipped when the
+    // caller passes an empty `workflowsDir` string (test escape hatch). Errors
+    // are surfaced as warnings — workflow evaluation is best-effort and a
+    // missing/malformed spec must not abort the whole bench run.
+    const workflowSpecs = [];
+    const workflowsDir = options.workflowsDir ?? DEFAULT_WORKFLOWS_DIR;
+    if (workflowsDir.length > 0) {
+        try {
+            const loaded = loadAllWorkflowSpecs(workflowsDir);
+            workflowSpecs.push(...loaded);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            warnings.push(`workflow specs: failed to load from "${workflowsDir}": ${msg}`);
+            warn(`[runUtility] workflow specs unavailable: ${msg}`);
+        }
+    }
+    const workflowChecks = [];
+    for (const task of options.tasks) {
+        const taskRuns = new Map();
+        grouped.set(task.id, taskRuns);
+        // Resolve a caller-supplied stash override before materialising. When
+        // `stashDirByFixture` provides a directory for this task's fixture, we
+        // skip `loadFixtureStash` entirely and forward the override.
+        const overrideStashDir = options.stashDirByFixture?.get(task.stash);
+        // Materialise the akm-arm stash once per task. We share it across the K
+        // seeds because the stash content is identical and re-running `akm
+        // index` for every seed is wasted work.
+        let stash;
+        let stashError;
+        if (options.arms.includes("akm") && materialiseStash && !overrideStashDir) {
+            try {
+                stash = loadFixtureStash(task.stash, { skipIndex: true });
+            }
+            catch (err) {
+                stashError = err instanceof Error ? err.message : String(err);
+                warnings.push(`task ${task.id}: stash "${task.stash}" failed to load: ${stashError}`);
+            }
+        }
+        // SIGINT/SIGTERM trap (#267): register the per-task stash cleanup so an
+        // external signal mid-run reaps the tmp dir we just created.
+        const stashSnapshot = stash;
+        const deregisterStash = stashSnapshot
+            ? registerCleanup(() => {
+                try {
+                    stashSnapshot.cleanup();
+                }
+                catch {
+                    /* swallow */
+                }
+            })
+            : () => { };
+        // #261: when `includeSynthetic` is set, splice the synthetic arm into the
+        // per-task arm iteration alongside whatever the caller asked for. We
+        // dedupe so a caller that already passes `synthetic` in `arms` does not
+        // see it run twice. Pre-#261 callers (no flag, no `synthetic` in arms)
+        // see the old loop verbatim — that's the byte-identical default contract.
+        const armsForTask = (() => {
+            if (!options.includeSynthetic)
+                return options.arms;
+            if (options.arms.includes("synthetic"))
+                return options.arms;
+            return [...options.arms, "synthetic"];
+        })();
+        try {
+            for (const arm of armsForTask) {
+                const armRuns = [];
+                taskRuns.set(arm, armRuns);
+                for (let seed = 0; seed < seedsPerArm; seed += 1) {
+                    // Resolve the stashDir we'll forward to the agent. The akm arm
+                    // always carries a stashDir so AKM_STASH_DIR is set in the child
+                    // env — this is how downstream tooling (and the trajectory parser
+                    // event-stream lookup) distinguishes arms. When the operator opted
+                    // out of fixture materialisation (tests, dry-run), we still pass a
+                    // stable placeholder so the env keys are wired correctly.
+                    let stashDir;
+                    if (arm === "akm") {
+                        // Resolution order (must match the issue #251 acceptance criteria):
+                        //   1. Per-task explicit override (used by `runMaskedCorpus` to
+                        //      point at a tmp stash with one asset removed). Highest
+                        //      priority because attribution correctness depends on this
+                        //      branch never being shadowed by the `__no-stash__`
+                        //      placeholder fallback.
+                        //   2. Per-(task, arm)-call `stashDirByFixture` override (Phase
+                        //      3 evolve persistence).
+                        //   3. Per-task materialised fixture stash from `loadFixtureStash`.
+                        //   4. `materialiseStash: false` placeholder so AKM_STASH_DIR is
+                        //      still wired into the child env.
+                        if (task.stashDirOverride)
+                            stashDir = task.stashDirOverride;
+                        else if (overrideStashDir)
+                            stashDir = overrideStashDir;
+                        else if (stash)
+                            stashDir = stash.stashDir;
+                        else if (!materialiseStash)
+                            stashDir = path.join(task.taskDir, "__no-stash__");
+                    }
+                    // Build the prompt-override (#267). The builder is invoked once
+                    // per (task, arm) — seeds share a prompt. `undefined` keeps the
+                    // driver's default prompt in play.
+                    //
+                    // #261: the synthetic arm has a scratch-notes prompt contract —
+                    // the model is told no AKM stash is available and instructed to
+                    // write/use its own procedural notes. When the caller does not
+                    // supply a `buildPrompt` override for the synthetic arm we fall
+                    // back to a built-in scratch-notes prompt so the contract is
+                    // honoured by every utility-track caller, not just `runEvolve`.
+                    let promptOverride = options.buildPrompt?.(task, arm);
+                    if (promptOverride === undefined && arm === "synthetic") {
+                        promptOverride = buildUtilitySyntheticPrompt(task.id);
+                    }
+                    const run = await runOneIsolated({
+                        task,
+                        arm,
+                        seed,
+                        model: options.model,
+                        stashDir,
+                        budgetTokens,
+                        budgetWallMs,
+                        spawn: options.spawn,
+                        warnings,
+                        ...(promptOverride !== undefined ? { prompt: promptOverride } : {}),
+                    });
+                    armRuns.push(run);
+                    // §6.7 search-pipeline bridge: only the akm arm consults the stash,
+                    // and we only attribute ranks for tasks with a gold ref. Both
+                    // guards mean noakm and gold-less runs are silently excluded.
+                    if (arm === "akm" && task.goldRef) {
+                        const searches = extractGoldRanks(run, task.goldRef);
+                        goldRankRecords.push({
+                            taskId: task.id,
+                            arm,
+                            seed,
+                            outcome: run.outcome,
+                            goldRef: task.goldRef,
+                            searches,
+                        });
+                    }
+                    // #257: evaluate the akm-arm run against every workflow spec. The
+                    // evaluator's `specApplies` filter handles applicability (arm,
+                    // domain, gold ref, repeated-failures threshold), so we hand it the
+                    // entire spec list and append whatever it returns. noakm/synthetic
+                    // arms are not evaluated — workflow specs target the akm arm.
+                    if (arm === "akm" && workflowSpecs.length > 0) {
+                        const trace = normalizeRunToTrace(run, { warnings });
+                        const runCtx = {
+                            arm: run.arm,
+                            taskId: run.taskId,
+                            seed: run.seed,
+                            outcome: run.outcome,
+                        };
+                        const taskMetadata = buildWorkflowTaskMetadata(task, trace);
+                        const checks = evaluateRunAgainstAllSpecs(trace, workflowSpecs, runCtx, taskMetadata);
+                        workflowChecks.push(...checks);
+                    }
+                }
+            }
+        }
+        finally {
+            // Deregister BEFORE running cleanup so a SIGINT arriving during this
+            // block doesn't double-fire the cleanup (per cleanup.ts contract).
+            deregisterStash();
+            stash?.cleanup();
+        }
+    }
+    return buildReport({
+        grouped,
+        options,
+        seedsPerArm,
+        slice,
+        warnings,
+        goldRankRecords,
+        workflowChecks,
+    });
+}
+function buildWorkflowTaskMetadata(task, trace) {
+    const flags = {
+        search_has_relevant_result: searchResultIncludesGoldRef(trace, task.goldRef),
+        task_has_tests: taskHasTests(task),
+    };
+    return {
+        ...(task.goldRef !== undefined ? { goldRef: task.goldRef } : {}),
+        flags,
+    };
+}
+function searchResultIncludesGoldRef(trace, goldRef) {
+    if (!goldRef)
+        return false;
+    for (const event of trace.events) {
+        if (event.type !== "akm_search")
+            continue;
+        if (event.resultRefs?.includes(goldRef))
+            return true;
+    }
+    return false;
+}
+function taskHasTests(task) {
+    if (task.verifier === "pytest")
+        return true;
+    const testsDir = path.join(task.taskDir, "tests");
+    if (!fs.existsSync(testsDir))
+        return false;
+    try {
+        return fs.readdirSync(testsDir).some((name) => name.endsWith(".py") || name.endsWith(".sh"));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Set up a fresh workspace for one (task, arm, seed) triple, run `runOne`
+ * against it, splice in the trajectory record, then tear everything down.
+ */
+async function runOneIsolated(args) {
+    const workspace = benchMkdtemp(`akm-bench-ws-${args.task.domain}-`);
+    // SIGINT trap: register workspace cleanup so external signals don't leak
+    // tmp dirs. Deregistered in `finally` before we do the synchronous rm so
+    // the handler doesn't double-fire (per cleanup.ts contract).
+    const deregisterWorkspace = registerCleanup(() => {
+        try {
+            fs.rmSync(workspace, { recursive: true, force: true });
+        }
+        catch {
+            /* swallow */
+        }
+    });
+    try {
+        seedWorkspace(args.task.taskDir, workspace);
+        const runOptions = {
+            track: "utility",
+            arm: args.arm,
+            taskId: args.task.id,
+            workspace,
+            model: args.model,
+            seed: args.seed,
+            budgetTokens: args.budgetTokens,
+            budgetWallMs: args.budgetWallMs,
+            verifier: args.task.verifier,
+            taskDir: args.task.taskDir,
+            ...(args.task.expectedMatch ? { expectedMatch: args.task.expectedMatch } : {}),
+            ...(args.stashDir ? { stashDir: args.stashDir } : {}),
+            ...(args.spawn ? { spawn: args.spawn } : {}),
+            ...(args.prompt !== undefined ? { prompt: args.prompt } : {}),
+            warnings: args.warnings,
+        };
+        const result = await runOne(runOptions);
+        // Splice in the trajectory metric. The driver always returns
+        // `{ null, null }` — this is where the real values get filled.
+        const trajectory = computeTrajectory({ goldRef: args.task.goldRef }, result, {
+            warnings: args.warnings,
+        });
+        // Per-asset attribution is post-processing on the trace; it's free, so we
+        // run it on every (task, arm, seed) result. The driver emits an empty
+        // assetsLoaded[]; this is where the real refs get filled. Spec §6.5.
+        const assetsLoaded = extractAssetLoads(result);
+        // Splice in the failure-mode label. Only the akm arm carries one; the
+        // noakm baseline is the control and isn't part of the §6.6 to-do list.
+        // `classifyFailureMode` returns null for non-failed runs.
+        const failureMode = args.arm === "akm" ? classifyFailureMode(args.task, { ...result, trajectory, assetsLoaded }) : null;
+        return { ...result, trajectory, assetsLoaded, failureMode };
+    }
+    finally {
+        deregisterWorkspace();
+        fs.rmSync(workspace, { recursive: true, force: true });
+    }
+}
+/**
+ * Copy the task's `workspace/` template into the per-run tmp dir. If the
+ * task has no `workspace/` (loader-test fixtures), the run starts with an
+ * empty cwd — that is also valid for verifier-only tasks.
+ */
+function seedWorkspace(taskDir, dest) {
+    const src = path.join(taskDir, "workspace");
+    if (!fs.existsSync(src))
+        return;
+    copyDirRecursive(src, dest);
+}
+/**
+ * Default synthetic-arm prompt (#261). Used by Track A `runUtility` when the
+ * caller opts in via `includeSynthetic: true` and does not also supply a
+ * `buildPrompt` override for the synthetic arm.
+ *
+ * The prompt is a clear scratch-notes contract: the model is told no AKM
+ * stash is available and instructed to write/use its own procedural notes
+ * before solving the task. This mirrors the prompt shape used by Track B's
+ * `buildSyntheticPrompt(taskId)` but is intentionally duplicated here so
+ * Track A has no module-level dependency on `evolve.ts`.
+ *
+ * Exported for tests.
+ */
+export function buildUtilitySyntheticPrompt(taskId) {
+    return [
+        `Task: ${taskId}`,
+        "Arm: synthetic (Bring Your Own Skills)",
+        "No akm stash is available; AKM_STASH_DIR is intentionally absent. Before solving",
+        "the task, write a short scratchpad of the skills and steps you intend to use,",
+        "then proceed. Cite the scratchpad in your trace so the verifier can attribute",
+        "the approach to your own reasoning rather than retrieved guidance.",
+    ].join("\n");
+}
+function copyDirRecursive(src, dest) {
+    fs.mkdirSync(dest, { recursive: true });
+    const entries = fs.readdirSync(src, { withFileTypes: true });
+    for (const entry of entries) {
+        if (entry.name === ".gitkeep")
+            continue;
+        const s = path.join(src, entry.name);
+        const d = path.join(dest, entry.name);
+        if (entry.isDirectory())
+            copyDirRecursive(s, d);
+        else if (entry.isFile())
+            fs.copyFileSync(s, d);
+    }
+}
+function buildReport(args) {
+    const tasks = [];
+    const noakmPerTask = {};
+    const akmPerTask = {};
+    const synthPerTask = {};
+    const akmRunsAll = [];
+    const allRuns = [];
+    const includeSynth = args.options.includeSynthetic === true;
+    // #257: index workflow checks by taskId so we can attach a per-task
+    // mean compliance to each `UtilityReportTaskEntry`. Only `pass` and
+    // `partial` statuses contribute non-zero scores; `not_applicable` is
+    // skipped (the spec did not target this run); `harness_error` rolls in
+    // as a 0 so corrupt traces drag the per-task number down.
+    const checksByTask = new Map();
+    for (const c of args.workflowChecks) {
+        const arr = checksByTask.get(c.taskId);
+        if (arr)
+            arr.push(c);
+        else
+            checksByTask.set(c.taskId, [c]);
+    }
+    for (const task of args.options.tasks) {
+        const taskRuns = args.grouped.get(task.id);
+        const noakmRuns = taskRuns?.get("noakm") ?? [];
+        const akmRuns = taskRuns?.get("akm") ?? [];
+        // #261: synthetic-arm runs are only consulted when the caller opted in.
+        // A missing arm is NOT a zero-pass arm — we leave `synthPerTask[task.id]`
+        // unset rather than defaulting to a zeroed PerTaskMetrics so downstream
+        // consumers can distinguish "arm not run" from "arm ran with 0 passes".
+        const synthRuns = includeSynth ? (taskRuns?.get("synthetic") ?? []) : [];
+        const noakmMetrics = aggregatePerTask(noakmRuns);
+        const akmMetrics = aggregatePerTask(akmRuns);
+        const delta = computePerTaskDelta(noakmMetrics, akmMetrics);
+        noakmPerTask[task.id] = noakmMetrics;
+        akmPerTask[task.id] = akmMetrics;
+        if (includeSynth) {
+            synthPerTask[task.id] = aggregatePerTask(synthRuns);
+        }
+        akmRunsAll.push(...akmRuns);
+        // Preserve arm order (noakm, synthetic when enabled, then akm) so the
+        // persisted runs[] array is deterministic across reruns. #249. The
+        // synthetic block is omitted entirely when includeSynth is false so the
+        // pre-#261 envelope stays byte-identical.
+        if (includeSynth) {
+            allRuns.push(...noakmRuns, ...synthRuns, ...akmRuns);
+        }
+        else {
+            allRuns.push(...noakmRuns, ...akmRuns);
+        }
+        // #257: per-task workflow compliance, mean of `score` over applicable
+        // checks (excludes `not_applicable`). Undefined when this task has no
+        // applicable checks at all so downstream renderers can distinguish
+        // "not measured" from "measured at 0".
+        const taskChecks = checksByTask.get(task.id) ?? [];
+        const applicableTaskChecks = taskChecks.filter((c) => c.status !== "not_applicable");
+        let workflowCompliance;
+        if (applicableTaskChecks.length > 0) {
+            let sum = 0;
+            for (const c of applicableTaskChecks)
+                sum += c.score;
+            workflowCompliance = sum / applicableTaskChecks.length;
+        }
+        tasks.push({
+            id: task.id,
+            noakm: noakmMetrics,
+            akm: akmMetrics,
+            delta,
+            ...(includeSynth ? { synthetic: aggregatePerTask(synthRuns) } : {}),
+            ...(workflowCompliance !== undefined ? { workflowCompliance } : {}),
+        });
+    }
+    const aggregateNoakm = aggregateCorpus(noakmPerTask);
+    const aggregateAkm = aggregateCorpus(akmPerTask);
+    const aggregateDelta = computeCorpusDelta(aggregateNoakm, aggregateAkm);
+    // #261: synthetic-arm aggregate is built ONLY when the caller opted in.
+    // We compute it once here so the report renderer can stamp `arms.synthetic`
+    // and `akm_over_synthetic_lift` without recomputing.
+    const aggregateSynth = includeSynth ? aggregateCorpus(synthPerTask) : undefined;
+    const trajectoryAkm = aggregateTrajectory(akmRunsAll);
+    // Failure-mode aggregate (§6.6). Walks every akm-arm run; runs that are
+    // not "fail" carry `failureMode: null` and are skipped here.
+    const failureEntries = [];
+    for (const r of akmRunsAll) {
+        if (r.failureMode)
+            failureEntries.push({ taskId: r.taskId, mode: r.failureMode });
+    }
+    const failureModes = aggregateFailureModes(failureEntries);
+    const domains = new Set(args.options.tasks.map((t) => t.domain)).size;
+    const branch = args.options.branch ?? resolveGitBranch();
+    const commit = args.options.commit ?? resolveGitCommit();
+    const timestamp = args.options.timestamp ?? new Date().toISOString();
+    // §6.7 — compute the search-pipeline bridge once over the whole corpus.
+    // The function tolerates an empty record list (renders the N/A sentence
+    // downstream).
+    const searchBridge = computeSearchBridge({ goldRankRecords: args.goldRankRecords });
+    // #250 — stamp deterministic corpus + fixture identity into the report
+    // so `bench compare` can refuse cross-corpus / cross-fixture diffs unless
+    // the operator explicitly opts in via --allow-corpus-mismatch /
+    // --allow-fixture-mismatch.
+    const selectedTaskIds = [...args.options.tasks.map((t) => t.id)].sort();
+    const taskBodies = new Map();
+    for (const t of args.options.tasks)
+        taskBodies.set(t.id, readTaskBody(t.taskDir));
+    const taskCorpusHash = computeTaskCorpusHash(selectedTaskIds, taskBodies);
+    const fixtureNames = [...new Set(args.options.tasks.map((t) => t.stash))].sort();
+    const fixtures = {};
+    for (const name of fixtureNames) {
+        try {
+            fixtures[name] = computeFixtureContentHash(name);
+        }
+        catch (err) {
+            // Loader-test tasks point at fixtures that may not exist on disk; we
+            // still want to stamp identity for the present fixtures, so we record
+            // the failure as a warning and continue with the remaining set.
+            args.warnings.push(`corpus stamp: cannot hash fixture "${name}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // Combined fixture-content hash. Hash input is the same `<name>\0<hash>\0`
+    // pattern used elsewhere — order-stable because `fixtureNames` is sorted.
+    const combinedHash = createHash("sha256");
+    for (const name of fixtureNames) {
+        combinedHash.update(name);
+        combinedHash.update("\0");
+        combinedHash.update(fixtures[name] ?? "");
+        combinedHash.update("\0");
+    }
+    const fixtureContentHash = combinedHash.digest("hex");
+    const baseReport = {
+        timestamp,
+        branch,
+        commit,
+        model: args.options.model,
+        corpus: {
+            domains,
+            tasks: args.options.tasks.length,
+            slice: args.slice,
+            seedsPerArm: args.seedsPerArm,
+            selectedTaskIds,
+            taskCorpusHash,
+            fixtures,
+            fixtureContentHash,
+        },
+        aggregateNoakm,
+        aggregateAkm,
+        aggregateDelta,
+        ...(aggregateSynth ? { aggregateSynth } : {}),
+        trajectoryAkm,
+        failureModes,
+        tasks,
+        warnings: args.warnings,
+        akmRuns: akmRunsAll,
+        allRuns,
+        taskMetadata: args.options.tasks,
+        goldRankRecords: args.goldRankRecords,
+        searchBridge,
+        workflowChecks: args.workflowChecks,
+    };
+    // Compute per-asset attribution as post-processing on the akm-arm runs
+    // we just collected. This is the §6.5 "free" diagnostic — it runs on every
+    // utility invocation, no extra spawns.
+    baseReport.perAsset = computePerAssetAttribution(baseReport);
+    return baseReport;
+}