akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/evolve.js

@@ -0,0 +1,647 @@
+/**
+ * akm-bench `evolve` — Track B longitudinal three-phase runner (spec §4 + §6.4).
+ *
+ * `runEvolve()` orchestrates three phases against a single eval-domain corpus:
+ *
+ *   • Phase 1 (signal accumulation): run K seeds × tasks (train slice only)
+ *     under the akm arm, then record `akm feedback <gold_ref> --positive` /
+ *     `--negative` events per outcome.
+ *   • Phase 2 (evolve): for every asset whose negative feedback crosses the
+ *     threshold, invoke `akm distill` and `akm reflect`, validate every
+ *     resulting proposal via `akm proposal show --json`, then accept or
+ *     reject per lint outcome. After processing, rebuild the index.
+ *   • Phase 3 (re-evaluate): run the eval slice under THREE arms — `pre` (the
+ *     original un-evolved fixture), `post` (the evolved fixture), `synthetic`
+ *     (no stash, scratchpad-only "Bring Your Own Skills" prompt).
+ *
+ * Leakage prevention (spec §7.4): before invoking distill we compute the set
+ * of eval-slice gold refs and pass it to `akm distill` via
+ * `--exclude-feedback-from <csv>` (#267). `akmDistill` filters those
+ * feedback events out of its LLM input before constructing the prompt.
+ * Refs in the exclusion list still see distillation run — but distillation
+ * runs from asset content alone, with no feedback signal that could have
+ * leaked from the eval slice. The proposal log + Phase 1 feedback stream
+ * are also filtered before computeProposalQualityMetrics ever sees them.
+ *
+ * Test seams: every external interaction is funnelled through one of three
+ * injectable functions:
+ *   - `spawn` — forwarded to `runOne` (drives the agent harness).
+ *   - `akmCli(args, cwd, env)` — invoked for every `akm <verb>` subprocess.
+ *   - `materialiseStash` — when false, `runUtility` doesn't touch
+ *     fixtures/stashes/.
+ * Tests inject fakes; production wires the real `Bun.spawnSync` and the
+ * real `loadFixtureStash`.
+ */
+import path from "node:path";
+import { loadFixtureStash } from "../fixtures/stashes/load";
+import { registerCleanup } from "./cleanup";
+import { computeLessonMetrics } from "./evolve-metrics";
+import { computeFeedbackIntegrity, computeLongitudinalMetrics, computeProposalQualityMetrics, } from "./metrics";
+import { runUtility } from "./runner";
+import { benchMkdtemp } from "./tmp";
+/**
+ * Drive the three-phase Track B runner.
+ *
+ * Pre: `tasks` is already filtered to one domain (or `all`). The runner
+ * partitions internally on `task.slice`.
+ *
+ * Sandboxing: at the start of every real run the runner materialises one
+ * dedicated tmp stash per fixture (the `evolveStash`) plus a fresh sibling
+ * snapshot per fixture (the `preStash`). Phase 1 + Phase 2 pin
+ * `AKM_STASH_DIR` to the appropriate `evolveStash` for every spawned `akm`
+ * invocation; Phase 3's pre arm uses `preStash`, the post arm uses
+ * `evolveStash`, and the synthetic arm uses no stash. The operator's real
+ * `process.env.AKM_STASH_DIR` is never read or written by `runEvolve`. All
+ * stashes are cleaned up in a top-level try/finally.
+ */
+export async function runEvolve(options) {
+    const seedsPerArm = options.seedsPerArm ?? 5;
+    const budgetTokens = options.budgetTokens ?? 30000;
+    const budgetWallMs = options.budgetWallMs ?? 120000;
+    const negativeThreshold = options.negativeThreshold ?? { absoluteCount: 2, ratio: 0.5 };
+    const materialiseStash = options.materialiseStash ?? true;
+    const akmCli = options.akmCli ?? defaultAkmCli;
+    const warnings = [];
+    const trainTasks = options.tasks.filter((t) => effectiveSlice(t) === "train");
+    const evalTasks = options.tasks.filter((t) => effectiveSlice(t) === "eval");
+    // Use the first task's domain (or "all") as the corpus label. The CLI
+    // already filtered to one domain; this is just for the report header.
+    const domain = uniqueDomain(options.tasks);
+    // ── Sandbox setup: per-fixture evolveStash + preStash. ───────────────────
+    // We materialise one tmp stash per unique `task.stash` so Phase 1
+    // accumulates feedback into the same on-disk stash that Phase 2 mutates,
+    // and that Phase 3's post arm reads back. The operator's real
+    // AKM_STASH_DIR is never touched. The pre arm gets a fresh snapshot of
+    // the same starting fixture (no Phase 2 mutations applied).
+    const fixtureNames = new Set();
+    for (const t of options.tasks)
+        fixtureNames.add(t.stash);
+    const evolveStashes = new Map();
+    const preStashes = new Map();
+    const evolveDirByFixture = new Map();
+    const preDirByFixture = new Map();
+    /** Per-fixture XDG_CACHE_HOME dirs allocated for evolve-stash indexing. */
+    const evolveCacheDirByFixture = new Map();
+    // SIGINT trap (#267): every per-fixture stash registers its cleanup with
+    // the shared registry so an external Ctrl-C reaps the tmp dirs even when
+    // the top-level try/finally never runs. We deregister in the matching
+    // finally block before invoking the synchronous cleanup so the handler
+    // doesn't double-fire.
+    const stashDeregistrations = [];
+    if (materialiseStash) {
+        for (const name of fixtureNames) {
+            try {
+                const evolved = loadFixtureStash(name, { skipIndex: false });
+                evolveStashes.set(name, evolved);
+                evolveDirByFixture.set(name, evolved.stashDir);
+                // Allocate a per-fixture cache dir for the evolve-stash re-index.
+                // `loadFixtureStash` used its own isolated XDG_CACHE_HOME; subsequent
+                // `akmCli` calls (feedback, distill, reflect) must look in the same
+                // cache. We allocate a fresh bench cache dir and pass it through
+                // `indexEvolveStash` + `envForRef` so the FTS5 DB is in a known place.
+                evolveCacheDirByFixture.set(name, benchMkdtemp(`akm-evolve-cache-${name}-`));
+                stashDeregistrations.push(registerCleanup(() => {
+                    try {
+                        evolved.cleanup();
+                    }
+                    catch {
+                        /* swallow */
+                    }
+                }));
+            }
+            catch (err) {
+                warnings.push(`evolve: failed to materialise evolve stash for fixture "${name}": ${err.message}`);
+            }
+            try {
+                const pre = loadFixtureStash(name, { skipIndex: false });
+                preStashes.set(name, pre);
+                preDirByFixture.set(name, pre.stashDir);
+                stashDeregistrations.push(registerCleanup(() => {
+                    try {
+                        pre.cleanup();
+                    }
+                    catch {
+                        /* swallow */
+                    }
+                }));
+            }
+            catch (err) {
+                warnings.push(`evolve: failed to materialise pre stash for fixture "${name}": ${err.message}`);
+            }
+        }
+    }
+    // Resolve the evolveStash dir for a given asset ref. We map ref → fixture
+    // by looking up which task's gold ref it matches; if no task owns it (or
+    // multiple do, which is unusual), we fall back to the first available
+    // evolveStash. The simple — and most common — case is a single fixture
+    // per `--tasks <domain>` invocation.
+    const refToFixture = new Map();
+    for (const t of options.tasks) {
+        if (t.goldRef)
+            refToFixture.set(t.goldRef, t.stash);
+    }
+    const fallbackEvolveDir = [...evolveDirByFixture.values()][0];
+    const fallbackEvolveCacheDir = [...evolveCacheDirByFixture.values()][0];
+    function envForRef(ref) {
+        const baseEnv = { ...process.env };
+        if (!materialiseStash) {
+            // Tests opt out of fixture materialisation entirely; we still strip
+            // the operator's AKM_STASH_DIR so the fake CLI sees a known sentinel.
+            delete baseEnv.AKM_STASH_DIR;
+            return baseEnv;
+        }
+        const fixture = ref ? refToFixture.get(ref) : undefined;
+        const dir = (fixture && evolveDirByFixture.get(fixture)) ?? fallbackEvolveDir;
+        const cacheDir = (fixture && evolveCacheDirByFixture.get(fixture)) ?? fallbackEvolveCacheDir;
+        if (dir)
+            baseEnv.AKM_STASH_DIR = dir;
+        else
+            delete baseEnv.AKM_STASH_DIR;
+        if (cacheDir)
+            baseEnv.XDG_CACHE_HOME = cacheDir;
+        return baseEnv;
+    }
+    // ── Phase 1 pre-flight: index each evolve stash in its dedicated cache. ───
+    // `loadFixtureStash` already ran `akm index` but used an isolated
+    // XDG_CACHE_HOME that subsequent `akmCli` calls (feedback, distill, reflect)
+    // cannot see. Re-running `akm index` here via `akmCli` with the same
+    // AKM_STASH_DIR + XDG_CACHE_HOME that `envForRef` will produce ensures the
+    // FTS5 database is populated where Phase 1 feedback will look.
+    // Non-zero exit adds a warning but does not abort — Phase 1 can still run
+    // with degraded feedback if the index step fails.
+    if (materialiseStash) {
+        const phase1Cwd = options.tasks[0]?.taskDir ?? process.cwd();
+        for (const [fixtureName, stashDir] of evolveDirByFixture) {
+            const cacheDir = evolveCacheDirByFixture.get(fixtureName);
+            if (!cacheDir)
+                continue;
+            try {
+                const result = await indexEvolveStash(stashDir, cacheDir, akmCli, phase1Cwd);
+                if (!result.ok) {
+                    warnings.push(`evolve: pre-flight akm index failed for stash ${stashDir}: ${result.stderr.trim()}`);
+                }
+            }
+            catch (err) {
+                warnings.push(`evolve: pre-flight akm index threw for stash ${stashDir}: ${err.message}`);
+            }
+        }
+    }
+    let preReport;
+    let postReport;
+    let syntheticReport;
+    let phase1Report;
+    const feedbackLog = [];
+    const proposalLog = [];
+    try {
+        // ── Phase 1: accumulate signal on the train slice (akm arm only). ─────
+        phase1Report = await runUtility({
+            tasks: trainTasks,
+            arms: ["akm"],
+            model: options.model,
+            seedsPerArm,
+            budgetTokens,
+            budgetWallMs,
+            slice: "train",
+            ...(options.spawn ? { spawn: options.spawn } : {}),
+            // We pre-materialised the per-fixture evolve stash above; tell the
+            // runner to forward those dirs and skip its own per-task materialise.
+            materialiseStash,
+            ...(materialiseStash ? { stashDirByFixture: evolveDirByFixture } : {}),
+            ...(options.timestamp ? { timestamp: options.timestamp } : {}),
+            ...(options.branch ? { branch: options.branch } : {}),
+            ...(options.commit ? { commit: options.commit } : {}),
+            ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+        });
+        // Issue feedback events per (task, seed) outcome on the akm arm.
+        const feedbackByRef = new Map();
+        const phase1Cwd = options.tasks[0]?.taskDir ?? process.cwd();
+        for (const run of phase1Report.akmRuns ?? []) {
+            const taskMeta = options.tasks.find((t) => t.id === run.taskId);
+            const goldRef = taskMeta?.goldRef;
+            if (!goldRef)
+                continue;
+            if (run.outcome === "harness_error")
+                continue;
+            const signal = run.outcome === "pass" ? "positive" : "negative";
+            const args = ["feedback", goldRef, signal === "positive" ? "--positive" : "--negative"];
+            // Wrap in try/catch so a single throwing akmCli (e.g. subprocess
+            // crash) cannot leave `feedbackByRef` partially populated and let
+            // Phase 2 proceed on corrupt state.
+            try {
+                const cliResult = await akmCli(args, phase1Cwd, envForRef(goldRef));
+                feedbackLog.push({ taskId: run.taskId, seed: run.seed, goldRef, signal, ok: cliResult.exitCode === 0 });
+                if (cliResult.exitCode !== 0) {
+                    warnings.push(`phase1: akm feedback for ${goldRef} (${signal}) failed: ${cliResult.stderr.trim()}`);
+                }
+            }
+            catch (err) {
+                feedbackLog.push({ taskId: run.taskId, seed: run.seed, goldRef, signal, ok: false });
+                warnings.push(`phase1.feedback_dispatch_failed: ${goldRef} ${err.message}`);
+            }
+            const counts = feedbackByRef.get(goldRef) ?? { positive: 0, negative: 0 };
+            if (signal === "positive")
+                counts.positive += 1;
+            else
+                counts.negative += 1;
+            feedbackByRef.set(goldRef, counts);
+        }
+        // ── Phase 2: evolve. ────────────────────────────────────────────────────
+        const evalGoldRefs = new Set();
+        for (const t of evalTasks) {
+            if (t.goldRef)
+                evalGoldRefs.add(t.goldRef);
+        }
+        const refsToEvolve = [];
+        for (const [ref, counts] of feedbackByRef.entries()) {
+            if (crossesNegativeThreshold(counts, negativeThreshold))
+                refsToEvolve.push(ref);
+        }
+        refsToEvolve.sort();
+        // §7.4 leakage prevention (#267): instead of hard-skipping refs that
+        // overlap eval-slice gold refs, we now pass the gold-ref set through
+        // `--exclude-feedback-from` (and the matching env var) so `akm distill`
+        // filters those events out of its LLM input. The behaviour collapses
+        // back to "no useful feedback shown" for refs that ARE the gold ref —
+        // distill then runs from asset content only, which is what we want.
+        const evalGoldRefList = [...evalGoldRefs].sort();
+        const excludeFeedbackCsv = evalGoldRefList.join(",");
+        for (const ref of refsToEvolve) {
+            // The env var fallback is the contract `akm distill` honours; it lets
+            // the bench keep working even if a hypothetical caller invokes
+            // distill via a wrapper that mangles flags.
+            const evolveEnv = {
+                ...envForRef(ref),
+                AKM_BENCH_EXCLUDE_GOLD_REFS: excludeFeedbackCsv,
+                ...(excludeFeedbackCsv ? { AKM_DISTILL_EXCLUDE_FEEDBACK_FROM: excludeFeedbackCsv } : {}),
+            };
+            // Pass the eval-gold list explicitly via the CLI flag so the contract
+            // is observable in test logs (the env var is a fallback for harnesses
+            // that strip flags). Reflect doesn't accept this flag — it's a distill
+            // concern only.
+            const distillArgs = ["distill", ref];
+            if (excludeFeedbackCsv) {
+                distillArgs.push("--exclude-feedback-from", excludeFeedbackCsv);
+            }
+            const distillResult = await akmCli(distillArgs, phase1Cwd, evolveEnv);
+            if (distillResult.exitCode !== 0) {
+                warnings.push(`phase2: akm distill ${ref} failed: ${distillResult.stderr.trim()}`);
+            }
+            else if (evalGoldRefs.has(ref) && excludeFeedbackCsv) {
+                // Per-ref leakage info — replaces the previous "skipped" message.
+                // Operator can audit which refs ran through the filter and confirm
+                // distillation didn't see leaked feedback.
+                warnings.push(`phase2: filtered eval-slice gold-ref feedback from distill input for ${ref} (--exclude-feedback-from ${excludeFeedbackCsv}).`);
+            }
+            const reflectResult = await akmCli(["reflect", ref], phase1Cwd, evolveEnv);
+            if (reflectResult.exitCode !== 0) {
+                // `reflect` requires `agent.default` to be configured — a missing
+                // config is non-fatal for the bench; we record and continue.
+                warnings.push(`phase2: akm reflect ${ref} skipped/failed: ${reflectResult.stderr.trim()}`);
+            }
+        }
+        // Walk the proposal queue per fixture (each evolveStash has its own
+        // proposal log on disk). When we materialised stashes we iterate every
+        // fixture that produced proposals; in the common single-fixture case
+        // this is one pass.
+        const proposalFixtures = materialiseStash ? [...evolveDirByFixture.keys()] : [undefined];
+        for (const fixtureName of proposalFixtures) {
+            const proposalEnv = { ...process.env };
+            if (materialiseStash && fixtureName) {
+                const dir = evolveDirByFixture.get(fixtureName);
+                if (dir)
+                    proposalEnv.AKM_STASH_DIR = dir;
+                const cacheDir = evolveCacheDirByFixture.get(fixtureName);
+                if (cacheDir)
+                    proposalEnv.XDG_CACHE_HOME = cacheDir;
+            }
+            else if (!materialiseStash) {
+                delete proposalEnv.AKM_STASH_DIR;
+            }
+            const listResult = await akmCli(["proposal", "list", "--json"], phase1Cwd, proposalEnv);
+            const proposals = parseProposalList(listResult.stdout);
+            for (const p of proposals) {
+                const showResult = await akmCli(["proposal", "show", p.id, "--json"], phase1Cwd, proposalEnv);
+                const lintInfo = parseProposalShow(showResult.stdout);
+                const lintPass = lintInfo.lintPass;
+                if (lintPass) {
+                    const acceptResult = await akmCli(["proposal", "accept", p.id], phase1Cwd, proposalEnv);
+                    proposalLog.push({
+                        proposalId: p.id,
+                        assetRef: p.assetRef,
+                        kind: p.kind,
+                        lintPass: true,
+                        decision: acceptResult.exitCode === 0 ? "accept" : "reject",
+                        ...(acceptResult.exitCode === 0 ? {} : { rejectReason: `accept failed: ${acceptResult.stderr.trim()}` }),
+                    });
+                }
+                else {
+                    const reason = lintInfo.lintMessage ?? "lint failed";
+                    const rejectResult = await akmCli(["proposal", "reject", p.id, "--reason", `lint failed: ${reason}`], phase1Cwd, proposalEnv);
+                    proposalLog.push({
+                        proposalId: p.id,
+                        assetRef: p.assetRef,
+                        kind: p.kind,
+                        lintPass: false,
+                        decision: "reject",
+                        rejectReason: reason,
+                    });
+                    if (rejectResult.exitCode !== 0) {
+                        warnings.push(`phase2: akm proposal reject ${p.id} failed: ${rejectResult.stderr.trim()}`);
+                    }
+                }
+            }
+            // Rebuild the index so accepted lessons surface in Phase 3.
+            const indexResult = await akmCli(["index"], phase1Cwd, proposalEnv);
+            if (indexResult.exitCode !== 0) {
+                warnings.push(`phase2: akm index rebuild failed: ${indexResult.stderr.trim()}`);
+            }
+        }
+        // ── Phase 3: re-evaluate (eval slice). ─────────────────────────────────
+        // pre arm: fresh snapshot of the starting fixture (no Phase 2 mutations
+        // applied). post arm: the mutated evolveStash so accepted lessons reach
+        // the eval slice. synthetic arm: no stash.
+        preReport = await runUtility({
+            tasks: evalTasks,
+            arms: ["akm"],
+            model: options.model,
+            seedsPerArm,
+            budgetTokens,
+            budgetWallMs,
+            slice: "eval",
+            ...(options.spawn ? { spawn: options.spawn } : {}),
+            materialiseStash,
+            ...(materialiseStash ? { stashDirByFixture: preDirByFixture } : {}),
+            ...(options.timestamp ? { timestamp: options.timestamp } : {}),
+            ...(options.branch ? { branch: options.branch } : {}),
+            ...(options.commit ? { commit: options.commit } : {}),
+            ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+        });
+        postReport = await runUtility({
+            tasks: evalTasks,
+            arms: ["akm"],
+            model: options.model,
+            seedsPerArm,
+            budgetTokens,
+            budgetWallMs,
+            slice: "eval",
+            // Stamp arm metadata so spawn fakes can distinguish pre-vs-post via
+            // an env probe. We thread it via a fresh `spawn` wrapper when one
+            // was supplied.
+            materialiseStash,
+            ...(materialiseStash ? { stashDirByFixture: evolveDirByFixture } : {}),
+            ...(options.timestamp ? { timestamp: options.timestamp } : {}),
+            ...(options.branch ? { branch: options.branch } : {}),
+            ...(options.commit ? { commit: options.commit } : {}),
+            ...(options.spawn ? { spawn: wrapSpawnWithArm(options.spawn, "post") } : {}),
+            ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+        });
+        // synthetic: no stash. We pass a spawn wrapper that strips
+        // AKM_STASH_DIR and injects the "Bring Your Own Skills" tag so test
+        // fakes (and a future real harness) can branch. #267 — also forward a
+        // per-task scratchpad prompt via the runner's `buildPrompt` seam so the
+        // synthetic arm actually exercises the BYOS prompt path rather than
+        // relying on the noakm default.
+        syntheticReport = await runUtility({
+            tasks: evalTasks,
+            arms: ["akm"],
+            model: options.model,
+            seedsPerArm,
+            budgetTokens,
+            budgetWallMs,
+            slice: "eval",
+            materialiseStash: false,
+            buildPrompt: (task, _arm) => buildSyntheticPrompt(task.id),
+            ...(options.timestamp ? { timestamp: options.timestamp } : {}),
+            ...(options.branch ? { branch: options.branch } : {}),
+            ...(options.commit ? { commit: options.commit } : {}),
+            ...(options.spawn ? { spawn: wrapSpawnWithArm(options.spawn, "synthetic", undefined, true) } : {}),
+            ...(options.opencodeProviders ? { opencodeProviders: options.opencodeProviders } : {}),
+        });
+    }
+    finally {
+        // Deregister BEFORE running cleanup so a SIGINT during teardown
+        // doesn't double-fire the cleanup fns (per cleanup.ts contract).
+        for (const deregister of stashDeregistrations)
+            deregister();
+        for (const s of evolveStashes.values()) {
+            try {
+                s.cleanup();
+            }
+            catch {
+                /* swallow — best-effort tmp cleanup */
+            }
+        }
+        for (const s of preStashes.values()) {
+            try {
+                s.cleanup();
+            }
+            catch {
+                /* swallow — best-effort tmp cleanup */
+            }
+        }
+    }
+    // ── Compute aggregates. ──────────────────────────────────────────────────
+    const proposalsMetrics = computeProposalQualityMetrics(proposalLog);
+    const longitudinal = computeLongitudinalMetrics(preReport, postReport, syntheticReport);
+    const feedbackIntegrity = computeFeedbackIntegrity({ phase1: phase1Report, feedbackLog });
+    // #264 — lesson quality + reuse metrics. The runner doesn't (yet) read
+    // accepted lesson bodies off disk or load verifier source text; we pass
+    // empty maps so the leakage check defaults to "low" until the read seam
+    // lands. Reuse + negative-transfer attribution work today off the
+    // pre/post arm `assetsLoaded` stream.
+    const lessons = computeLessonMetrics({
+        proposalLog,
+        feedbackLog,
+        preRuns: preReport.akmRuns ?? [],
+        postRuns: postReport.akmRuns ?? [],
+    });
+    return {
+        timestamp: options.timestamp ?? new Date().toISOString(),
+        branch: options.branch ?? preReport.branch,
+        commit: options.commit ?? preReport.commit,
+        model: options.model,
+        domain,
+        seedsPerArm,
+        feedbackLog,
+        proposalLog,
+        proposals: proposalsMetrics,
+        lessons,
+        longitudinal,
+        feedbackIntegrity,
+        phase1: phase1Report,
+        arms: { pre: preReport, post: postReport, synthetic: syntheticReport },
+        warnings: [
+            ...warnings,
+            ...phase1Report.warnings,
+            ...preReport.warnings,
+            ...postReport.warnings,
+            ...syntheticReport.warnings,
+        ],
+    };
+}
+/**
+ * Default subprocess invoker — runs `bun run src/cli.ts <args>` in `cwd`
+ * with the supplied env. Real runs use this; tests inject a fake.
+ */
+async function defaultAkmCli(args, cwd, env) {
+    const cli = path.resolve(__dirname, "..", "..", "src", "cli.ts");
+    const proc = Bun.spawnSync({
+        cmd: ["bun", "run", cli, ...args],
+        cwd,
+        env: { ...process.env, ...env },
+        stdout: "pipe",
+        stderr: "pipe",
+    });
+    const stdout = proc.stdout ? new TextDecoder().decode(proc.stdout) : "";
+    const stderr = proc.stderr ? new TextDecoder().decode(proc.stderr) : "";
+    return { exitCode: proc.exitCode ?? -1, stdout, stderr };
+}
+/**
+ * Threshold check: an asset crosses the negative threshold if either the
+ * absolute negative count meets `absoluteCount` OR the negative *ratio* among
+ * total feedback exceeds `ratio`. Either branch is sufficient — both are
+ * spec-mandated defaults.
+ */
+function crossesNegativeThreshold(counts, threshold) {
+    if (counts.negative >= threshold.absoluteCount)
+        return true;
+    const total = counts.positive + counts.negative;
+    if (total === 0)
+        return false;
+    return counts.negative / total > threshold.ratio;
+}
+/** Best-effort partition. Honours explicit `slice:` and falls back to id-hash. */
+function effectiveSlice(task) {
+    if (task.slice)
+        return task.slice;
+    // Mirror corpus.effectiveSlice — SHA-1 first byte parity.
+    // We avoid the import cycle by inlining the trivial fallback.
+    let h = 0;
+    for (let i = 0; i < task.id.length; i += 1)
+        h = (h * 31 + task.id.charCodeAt(i)) | 0;
+    return Math.abs(h) % 2 === 0 ? "train" : "eval";
+}
+function uniqueDomain(tasks) {
+    const set = new Set(tasks.map((t) => t.domain));
+    if (set.size === 1)
+        return [...set][0] ?? "all";
+    return "all";
+}
+/**
+ * Wrap a spawn fake so every child sees `BENCH_EVOLVE_ARM=<arm>` (and
+ * `BENCH_EVOLVE_SCRATCHPAD=1` for the synthetic arm). Used by Phase 3 so
+ * test fakes can distinguish the three arms without us having to expose a
+ * `prompt` override on `runUtility`. Real production runs receive the same
+ * env keys; the real `runAgent` harness ignores them.
+ */
+function wrapSpawnWithArm(inner, arm, stashDir, scratchpad = false) {
+    return (cmd, opts) => {
+        const env = { ...(opts.env ?? {}) };
+        env.BENCH_EVOLVE_ARM = arm;
+        if (scratchpad)
+            env.BENCH_EVOLVE_SCRATCHPAD = "1";
+        if (stashDir)
+            env.AKM_STASH_DIR = stashDir;
+        if (arm === "synthetic")
+            delete env.AKM_STASH_DIR;
+        return inner(cmd, { ...opts, env });
+    };
+}
+/** Tolerant parser for `akm proposal list --json` stdout. */
+function parseProposalList(stdout) {
+    if (!stdout.trim())
+        return [];
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch {
+        return [];
+    }
+    const arr = Array.isArray(parsed)
+        ? parsed
+        : Array.isArray(parsed.proposals)
+            ? parsed.proposals
+            : [];
+    const out = [];
+    for (const item of arr) {
+        if (!item || typeof item !== "object")
+            continue;
+        const rec = item;
+        const id = typeof rec.id === "string" ? rec.id : null;
+        const assetRef = typeof rec.target_ref === "string"
+            ? rec.target_ref
+            : typeof rec.targetRef === "string"
+                ? rec.targetRef
+                : typeof rec.ref === "string"
+                    ? rec.ref
+                    : null;
+        const kindRaw = typeof rec.kind === "string" ? rec.kind : typeof rec.source === "string" ? rec.source : "unknown";
+        const kind = kindRaw === "lesson" || kindRaw === "distill"
+            ? "lesson"
+            : kindRaw === "revision" || kindRaw === "reflect"
+                ? "revision"
+                : "unknown";
+        if (!id || !assetRef)
+            continue;
+        out.push({ id, assetRef, kind });
+    }
+    return out;
+}
+function parseProposalShow(stdout) {
+    if (!stdout.trim())
+        return { lintPass: false, lintMessage: "empty proposal show output" };
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch (err) {
+        return { lintPass: false, lintMessage: `proposal show: parse error (${err.message})` };
+    }
+    const lintPass = parsed.lint_pass === true ||
+        parsed.lintPass === true ||
+        (typeof parsed.lint === "object" && parsed.lint !== null && parsed.lint.pass === true);
+    const lintRaw = parsed.lint;
+    let lintMessage;
+    if (lintRaw && typeof lintRaw === "object") {
+        const issues = lintRaw.issues;
+        if (Array.isArray(issues) && issues.length > 0) {
+            lintMessage = issues
+                .map((i) => (typeof i === "string" ? i : (i?.message ?? JSON.stringify(i))))
+                .join("; ");
+        }
+    }
+    return { lintPass, ...(lintMessage ? { lintMessage } : {}) };
+}
+/**
+ * Run `akm index` on the evolve stash to populate the FTS5 database in the
+ * cache directory that Phase 1 `akmCli` calls will use.
+ *
+ * `loadFixtureStash` already indexed the stash into an isolated XDG_CACHE_HOME
+ * that is invisible to subsequent `akmCli` calls. Calling this helper with the
+ * same `stashDir` + `cacheDir` that `envForRef` will forward ensures `akm
+ * feedback` (and later `akm distill` / `akm reflect`) can look up refs in the
+ * FTS5 index.
+ *
+ * Returns `{ ok: true }` on exit code 0, `{ ok: false, stderr }` otherwise.
+ * Exported for tests.
+ */
+export async function indexEvolveStash(stashDir, cacheDir, akmCli, cwd) {
+    const env = {
+        ...process.env,
+        AKM_STASH_DIR: stashDir,
+        XDG_CACHE_HOME: cacheDir,
+    };
+    const result = await akmCli(["index"], cwd, env);
+    return { ok: result.exitCode === 0, stderr: result.stderr };
+}
+/** Exposed for tests so the synthetic-arm prompt construction can be asserted. */
+export function buildSyntheticPrompt(taskId) {
+    return [
+        `Task: ${taskId}`,
+        "Arm: synthetic (Bring Your Own Skills)",
+        "No akm stash is available. Before solving the task, write a short scratchpad of the skills",
+        "and steps you intend to use, then proceed. Cite the scratchpad in your trace so the verifier",
+        "can attribute the approach to your own reasoning rather than retrieved guidance.",
+    ].join("\n");
+}