akm-cli 0.6.1 → 0.7.0-rc1

package/dist/tests/bench/driver.test.js

@@ -0,0 +1,443 @@
+/**
+ * Unit tests for the bench driver — exercises every RunResult outcome
+ * (`pass`, `fail`, `budget_exceeded`, `harness_error`) via an injected fake
+ * spawn. Real opencode is never invoked.
+ */
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { _ISOLATED_ENV_NAMES, _SCRUBBED_OPERATOR_ENV_NAMES, buildIsolatedEnv, buildSanitizedEnvSource, createIsolationDirs, EVENTS_READ_CAP_BYTES, parseTokenUsage, readRunEvents, runOne, stripAkmStashDir, } from "./driver";
+import { benchMkdtemp } from "./tmp";
+function asReadableStream(text) {
+    const bytes = new TextEncoder().encode(text);
+    return new ReadableStream({
+        start(controller) {
+            controller.enqueue(bytes);
+            controller.close();
+        },
+    });
+}
+/**
+ * Build a spawn fn that scripts the agent run first, then any subsequent
+ * verifier run. Distinguishes by command: opencode is the configured `bin`
+ * for the built-in opencode profile (i.e. cmd[0] === "opencode"); anything
+ * else is a verifier.
+ */
+function scriptedSpawn(agent, verifier) {
+    const invocations = [];
+    const spawn = (cmd, options) => {
+        invocations.push({ cmd, env: options.env });
+        const isAgent = cmd[0] === "opencode";
+        const config = isAgent ? agent : (verifier ?? { exitCode: 0, stdout: "" });
+        if (isAgent && agent.throwSync)
+            throw agent.throwSync;
+        let resolveExit = () => { };
+        const exited = new Promise((resolve) => {
+            resolveExit = resolve;
+            if (!(isAgent && agent.hangsUntilKilled))
+                resolve(config.exitCode);
+        });
+        const proc = {
+            exitCode: isAgent && agent.hangsUntilKilled ? null : config.exitCode,
+            exited,
+            stdout: asReadableStream(config.stdout ?? ""),
+            stderr: asReadableStream(config.stderr ?? ""),
+            stdin: null,
+            kill() {
+                // Honour kill so timeout path resolves cleanly.
+                resolveExit(143);
+            },
+        };
+        return proc;
+    };
+    return { spawn, invocations };
+}
+const baseOptions = {
+    track: "utility",
+    arm: "noakm",
+    taskId: "_example/example-task",
+    workspace: "",
+    model: "anthropic/claude-opus-4-7",
+    seed: 0,
+    budgetTokens: 100000,
+    budgetWallMs: 60_000,
+    verifier: "regex",
+    taskDir: "",
+    expectedMatch: "ok",
+};
+describe("runOne", () => {
+    let workspace;
+    beforeAll(() => {
+        workspace = benchMkdtemp("bench-driver-test-");
+    });
+    afterAll(() => {
+        fs.rmSync(workspace, { recursive: true, force: true });
+    });
+    test("pass: agent exits 0, verifier exits 0", async () => {
+        const { spawn, invocations } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+        const result = await runOne({ ...baseOptions, workspace, spawn });
+        expect(result.outcome).toBe("pass");
+        expect(result.verifierExitCode).toBe(0);
+        expect(result.taskId).toBe("_example/example-task");
+        expect(result.model).toBe("anthropic/claude-opus-4-7");
+        expect(result.seed).toBe(0);
+        expect(result.schemaVersion).toBe(1);
+        expect(invocations[0]?.cmd[0]).toBe("opencode");
+    });
+    test("fail: agent exits 0 but verifier rejects output", async () => {
+        const { spawn } = scriptedSpawn({ exitCode: 0, stdout: "nope" });
+        const result = await runOne({ ...baseOptions, workspace, spawn });
+        expect(result.outcome).toBe("fail");
+        expect(result.verifierExitCode).toBe(1);
+    });
+    test("budget_exceeded: agent times out (runAgent reason: timeout)", async () => {
+        const { spawn } = scriptedSpawn({ exitCode: 0, hangsUntilKilled: true });
+        const result = await runOne({
+            ...baseOptions,
+            workspace,
+            spawn,
+            // Tiny budget so the timer fires before the fake agent ever exits.
+            budgetWallMs: 50,
+        });
+        expect(result.outcome).toBe("budget_exceeded");
+    });
+    test("harness_error: agent spawn throws synchronously", async () => {
+        const { spawn } = scriptedSpawn({ exitCode: 0, throwSync: new Error("ENOENT") });
+        const result = await runOne({ ...baseOptions, workspace, spawn });
+        expect(result.outcome).toBe("harness_error");
+    });
+    test("budget_exceeded: parsed token usage exceeds budgetTokens", async () => {
+        // Agent reports 70k input + 50k output = 120k tokens, budget is 100k.
+        // Verifier should NOT run; outcome must be budget_exceeded.
+        const { spawn } = scriptedSpawn({
+            exitCode: 0,
+            stdout: "input_tokens: 70000 output_tokens: 50000",
+        });
+        const result = await runOne({
+            ...baseOptions,
+            workspace,
+            spawn,
+            budgetTokens: 100_000,
+        });
+        expect(result.outcome).toBe("budget_exceeded");
+        expect(result.tokens.input + result.tokens.output).toBeGreaterThan(100_000);
+        expect(result.tokens.input).toBe(70_000);
+        expect(result.tokens.output).toBe(50_000);
+        expect(result.tokenMeasurement).toBe("parsed");
+    });
+    test("tokenMeasurement: parsed when stdout reports tokens", async () => {
+        const { spawn } = scriptedSpawn({
+            exitCode: 0,
+            stdout: "ok\ninput_tokens: 10 output_tokens: 5",
+        });
+        const result = await runOne({ ...baseOptions, workspace, spawn });
+        expect(result.outcome).toBe("pass");
+        expect(result.tokenMeasurement).toBe("parsed");
+        expect(result.tokens.input).toBe(10);
+        expect(result.tokens.output).toBe(5);
+    });
+    test("tokenMeasurement: missing when stdout has no token line — and budget is NOT enforced", async () => {
+        // Agent never reports tokens. budgetTokens is 1, but the harness must not
+        // mark this as budget_exceeded (issue #252) — measurement is missing.
+        const { spawn } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+        const result = await runOne({ ...baseOptions, workspace, spawn, budgetTokens: 1 });
+        expect(result.tokenMeasurement).toBe("missing");
+        expect(result.tokens).toEqual({ input: 0, output: 0 });
+        expect(result.outcome).not.toBe("budget_exceeded");
+    });
+    test("tokenMeasurement: harness_error path leaves measurement as 'missing'", async () => {
+        const { spawn } = scriptedSpawn({ exitCode: 0, throwSync: new Error("ENOENT") });
+        const result = await runOne({ ...baseOptions, workspace, spawn });
+        expect(result.outcome).toBe("harness_error");
+        // No agent stdout was ever observed → measurement stays at the default.
+        expect(result.tokenMeasurement).toBe("missing");
+    });
+    test("isolation: child env carries pinned XDG/OPENCODE/AKM dirs and not operator values", async () => {
+        const sentinel = "/tmp/operator-config-must-not-leak";
+        const priors = {};
+        for (const name of _ISOLATED_ENV_NAMES) {
+            priors[name] = process.env[name];
+            process.env[name] = sentinel;
+        }
+        try {
+            const { spawn, invocations } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+            await runOne({
+                ...baseOptions,
+                workspace,
+                stashDir: "/tmp/some-stash",
+                arm: "akm",
+                spawn,
+            });
+            const childEnv = invocations[0]?.env ?? {};
+            // Each isolated key MUST be present and MUST NOT equal the operator sentinel.
+            for (const name of _ISOLATED_ENV_NAMES) {
+                expect(childEnv[name]).toBeDefined();
+                expect(childEnv[name]).not.toBe(sentinel);
+            }
+            expect(childEnv.AKM_STASH_DIR).toBe("/tmp/some-stash");
+            expect(childEnv.BENCH_OPENCODE_MODEL).toBe("anthropic/claude-opus-4-7");
+        }
+        finally {
+            for (const name of _ISOLATED_ENV_NAMES) {
+                if (priors[name] === undefined)
+                    delete process.env[name];
+                else
+                    process.env[name] = priors[name];
+            }
+        }
+    });
+    // ── #271: operator-env isolation (OPENCODE_API_KEY/ANTHROPIC_API_KEY/AKM_CONFIG_DIR)
+    test("operator env isolation: bench child never inherits OPENCODE_API_KEY/ANTHROPIC_API_KEY/AKM_CONFIG_DIR (#271)", async () => {
+        // Even though `OPENCODE_API_KEY` is in the opencode profile's
+        // `envPassthrough` list, the bench driver MUST scrub these operator-env
+        // names before profile.envPassthrough copies them into the child. This
+        // is the regression guard the #271 review identified — without it,
+        // operator credentials and the operator's `AKM_CONFIG_DIR` would leak
+        // into every (task × arm × seed) child.
+        const sentinels = {
+            OPENCODE_API_KEY: "sentinel-A-must-not-leak",
+            ANTHROPIC_API_KEY: "sentinel-B-must-not-leak",
+            AKM_CONFIG_DIR: "sentinel-C-must-not-leak",
+        };
+        const priors = {};
+        for (const [name, value] of Object.entries(sentinels)) {
+            priors[name] = process.env[name];
+            process.env[name] = value;
+        }
+        try {
+            const { spawn, invocations } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+            await runOne({
+                ...baseOptions,
+                workspace,
+                arm: "akm",
+                stashDir: "/tmp/some-stash",
+                spawn,
+            });
+            const childEnv = invocations[0]?.env ?? {};
+            // None of the operator sentinels reach the child env that runAgent
+            // hands to spawn — neither as a key:value pair nor as a substring
+            // match (paranoid: confirm the literal sentinel strings are absent
+            // even from values like `OPENCODE_CONFIG`).
+            for (const name of _SCRUBBED_OPERATOR_ENV_NAMES) {
+                expect(childEnv[name]).toBeUndefined();
+            }
+            for (const sentinel of Object.values(sentinels)) {
+                for (const value of Object.values(childEnv)) {
+                    expect(value).not.toContain(sentinel);
+                }
+            }
+            // The explicit bench keys ARE present and pinned to the per-run
+            // tmpdirs (sanity: the scrubbing didn't accidentally drop them).
+            expect(childEnv.XDG_CACHE_HOME).toBeDefined();
+            expect(childEnv.XDG_CONFIG_HOME).toBeDefined();
+            expect(childEnv.OPENCODE_CONFIG).toBeDefined();
+            expect(childEnv.AKM_STASH_DIR).toBe("/tmp/some-stash");
+            expect(childEnv.BENCH_OPENCODE_MODEL).toBe("anthropic/claude-opus-4-7");
+        }
+        finally {
+            for (const [name, prior] of Object.entries(priors)) {
+                if (prior === undefined)
+                    delete process.env[name];
+                else
+                    process.env[name] = prior;
+            }
+        }
+    });
+    // ── #261: synthetic-arm AKM_STASH_DIR isolation ─────────────────────────────
+    test("synthetic arm: child env never carries AKM_STASH_DIR (recurrence guard for #243 fixup)", async () => {
+        // CRITICAL: synthetic-arm runs MUST NOT carry AKM_STASH_DIR. Without
+        // this guard the operator's real AKM_STASH_DIR leaks in via parent-env
+        // inheritance — exactly the failure mode the #243 fixup chased. We
+        // exercise both the explicit-stashDir case (bad caller passes one
+        // anyway) and the no-stashDir case.
+        const operatorStash = "/tmp/operator-stash-must-never-leak-into-synthetic";
+        const prior = process.env.AKM_STASH_DIR;
+        process.env.AKM_STASH_DIR = operatorStash;
+        try {
+            // 1) Synthetic arm with NO stashDir option: AKM_STASH_DIR must be
+            //    absent in the child env.
+            const { spawn, invocations } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+            await runOne({
+                ...baseOptions,
+                workspace,
+                arm: "synthetic",
+                spawn,
+            });
+            const childEnv1 = invocations[0]?.env ?? {};
+            expect(childEnv1.AKM_STASH_DIR).toBeUndefined();
+            expect(childEnv1.AKM_STASH_DIR).not.toBe(operatorStash);
+            // 2) Even when a buggy caller forwards a stashDir to the synthetic
+            //    arm, the driver MUST refuse to wire it into the child env.
+            const { spawn: spawn2, invocations: invocations2 } = scriptedSpawn({ exitCode: 0, stdout: "ok" });
+            await runOne({
+                ...baseOptions,
+                workspace,
+                arm: "synthetic",
+                stashDir: "/tmp/buggy-caller-stash",
+                spawn: spawn2,
+            });
+            const childEnv2 = invocations2[0]?.env ?? {};
+            expect(childEnv2.AKM_STASH_DIR).toBeUndefined();
+        }
+        finally {
+            if (prior === undefined)
+                delete process.env.AKM_STASH_DIR;
+            else
+                process.env.AKM_STASH_DIR = prior;
+        }
+    });
+});
+describe("driver helpers", () => {
+    test("createIsolationDirs creates four dirs under a single root", () => {
+        const dirs = createIsolationDirs();
+        try {
+            expect(fs.existsSync(dirs.cacheHome)).toBe(true);
+            expect(fs.existsSync(dirs.configHome)).toBe(true);
+            expect(fs.existsSync(dirs.opencodeConfig)).toBe(true);
+            expect(dirs.cacheHome.startsWith(dirs.root)).toBe(true);
+        }
+        finally {
+            fs.rmSync(dirs.root, { recursive: true, force: true });
+        }
+    });
+    test("stripAkmStashDir deletes AKM_STASH_DIR in place (#261 synthetic-arm guard)", () => {
+        const env = {
+            AKM_STASH_DIR: "/tmp/operator-stash",
+            XDG_CACHE_HOME: "/tmp/cache",
+        };
+        const result = stripAkmStashDir(env);
+        expect(result).toBe(env); // mutates in place + returns same ref
+        expect(env.AKM_STASH_DIR).toBeUndefined();
+        expect(env.XDG_CACHE_HOME).toBe("/tmp/cache"); // siblings untouched
+        // No-op on env without AKM_STASH_DIR.
+        const env2 = { XDG_CACHE_HOME: "/tmp/cache" };
+        stripAkmStashDir(env2);
+        expect(env2).toEqual({ XDG_CACHE_HOME: "/tmp/cache" });
+    });
+    test("buildSanitizedEnvSource strips OPENCODE_API_KEY/ANTHROPIC_API_KEY/AKM_CONFIG_DIR (#271)", () => {
+        const source = {
+            OPENCODE_API_KEY: "leak-A",
+            ANTHROPIC_API_KEY: "leak-B",
+            AKM_CONFIG_DIR: "/operator/akm",
+            PATH: "/usr/bin",
+            HOME: "/home/op",
+            OPENCODE_CONFIG: "/operator/opencode",
+            UNRELATED: "kept",
+        };
+        const result = buildSanitizedEnvSource(source);
+        // Operator names removed.
+        expect(result.OPENCODE_API_KEY).toBeUndefined();
+        expect(result.ANTHROPIC_API_KEY).toBeUndefined();
+        expect(result.AKM_CONFIG_DIR).toBeUndefined();
+        // Everything else preserved verbatim.
+        expect(result.PATH).toBe("/usr/bin");
+        expect(result.HOME).toBe("/home/op");
+        expect(result.OPENCODE_CONFIG).toBe("/operator/opencode");
+        expect(result.UNRELATED).toBe("kept");
+        // Result is a copy, not the same reference (caller can mutate freely).
+        expect(result).not.toBe(source);
+        // Source object is untouched.
+        expect(source.OPENCODE_API_KEY).toBe("leak-A");
+    });
+    test("buildSanitizedEnvSource defaults to process.env when no source given", () => {
+        const prior = process.env.OPENCODE_API_KEY;
+        process.env.OPENCODE_API_KEY = "default-source-leak";
+        try {
+            const result = buildSanitizedEnvSource();
+            expect(result.OPENCODE_API_KEY).toBeUndefined();
+        }
+        finally {
+            if (prior === undefined)
+                delete process.env.OPENCODE_API_KEY;
+            else
+                process.env.OPENCODE_API_KEY = prior;
+        }
+    });
+    test("buildIsolatedEnv pins the four isolation keys plus model", () => {
+        const dirs = createIsolationDirs("/tmp/stash");
+        try {
+            const env = buildIsolatedEnv(dirs, "model-x");
+            expect(env.XDG_CACHE_HOME).toBe(dirs.cacheHome);
+            expect(env.XDG_CONFIG_HOME).toBe(dirs.configHome);
+            expect(env.OPENCODE_CONFIG).toBe(dirs.opencodeConfig);
+            expect(env.AKM_STASH_DIR).toBe("/tmp/stash");
+            expect(env.BENCH_OPENCODE_MODEL).toBe("model-x");
+        }
+        finally {
+            fs.rmSync(dirs.root, { recursive: true, force: true });
+        }
+    });
+    test("parseTokenUsage extracts numbers when present, missing otherwise", () => {
+        // No matchable token line at all → measurement is "missing", not a real zero (issue #252).
+        expect(parseTokenUsage("")).toEqual({ input: 0, output: 0, measurement: "missing" });
+        expect(parseTokenUsage("noise")).toEqual({ input: 0, output: 0, measurement: "missing" });
+        // Both keys present → "parsed" with the actual numbers.
+        expect(parseTokenUsage("input_tokens: 123 output_tokens: 456")).toEqual({
+            input: 123,
+            output: 456,
+            measurement: "parsed",
+        });
+        // Only one key present → still "parsed", missing key defaults to 0.
+        expect(parseTokenUsage("input_tokens: 99")).toEqual({ input: 99, output: 0, measurement: "parsed" });
+        expect(parseTokenUsage("output_tokens: 55")).toEqual({ input: 0, output: 55, measurement: "parsed" });
+    });
+    test("readRunEvents returns [] when events.jsonl is missing and parses lines when present", () => {
+        const tmp = benchMkdtemp("bench-events-");
+        try {
+            expect(readRunEvents(tmp)).toEqual([]);
+            const akm = path.join(tmp, "akm");
+            fs.mkdirSync(akm, { recursive: true });
+            fs.writeFileSync(path.join(akm, "events.jsonl"), `${JSON.stringify({ schemaVersion: 1, ts: "2026-04-27T00:00:00Z", eventType: "feedback" })}\n`);
+            const events = readRunEvents(tmp);
+            expect(events.length).toBe(1);
+            expect(events[0]?.eventType).toBe("feedback");
+        }
+        finally {
+            fs.rmSync(tmp, { recursive: true, force: true });
+        }
+    });
+    test("readRunEvents caps reads at EVENTS_READ_CAP_BYTES and records a warning when exceeded", () => {
+        const tmp = benchMkdtemp("bench-events-cap-");
+        try {
+            const akm = path.join(tmp, "akm");
+            fs.mkdirSync(akm, { recursive: true });
+            const eventsPath = path.join(akm, "events.jsonl");
+            // Write a leading parseable record, then a giant filler line that
+            // pushes total size past the cap.
+            const firstLine = `${JSON.stringify({ schemaVersion: 1, ts: "2026-04-27T00:00:00Z", eventType: "feedback" })}\n`;
+            const fd = fs.openSync(eventsPath, "w");
+            try {
+                fs.writeSync(fd, firstLine);
+                // Filler line: a single very long line that — combined with the
+                // first — exceeds the cap. We cap at 16MiB so write 17MiB of 'x'.
+                const fillerSize = EVENTS_READ_CAP_BYTES + 1024 * 1024;
+                const chunk = Buffer.alloc(64 * 1024, "x".charCodeAt(0));
+                let written = 0;
+                while (written < fillerSize) {
+                    const remaining = fillerSize - written;
+                    const toWrite = remaining < chunk.length ? chunk.subarray(0, remaining) : chunk;
+                    fs.writeSync(fd, toWrite);
+                    written += toWrite.length;
+                }
+                fs.writeSync(fd, "\n");
+            }
+            finally {
+                fs.closeSync(fd);
+            }
+            const totalSize = fs.statSync(eventsPath).size;
+            expect(totalSize).toBeGreaterThan(EVENTS_READ_CAP_BYTES);
+            const warnings = [];
+            const events = readRunEvents(tmp, { warnings });
+            // The first parseable record should still be returned from the prefix.
+            expect(events.length).toBe(1);
+            expect(events[0]?.eventType).toBe("feedback");
+            // A warning was appended that mentions the cap and the actual size.
+            expect(warnings.length).toBe(1);
+            expect(warnings[0]).toContain("events.jsonl truncated");
+            expect(warnings[0]).toContain(String(EVENTS_READ_CAP_BYTES));
+        }
+        finally {
+            fs.rmSync(tmp, { recursive: true, force: true });
+        }
+    });
+});

package/dist/tests/bench/evolve-metrics.js

@@ -0,0 +1,179 @@
+/**
+ * Track B lesson quality + reuse metrics (issue #264, spec §6.3 follow-up).
+ *
+ * `computeLessonMetrics` walks the evolve runner's proposal log and the
+ * Phase 3 pre/post arm `RunResult[]`s and emits one `LessonRecord` per
+ * lesson-kind proposal. The record captures:
+ *
+ *   - `source_failures` — eval/train tasks whose negative feedback events
+ *     targeted this asset ref (joined via the supplied `feedbackLog`).
+ *   - `lint_pass` / `accepted` — verbatim from the proposal log entry.
+ *   - `first_reused_on` / `reuse_count` / `reuse_pass_rate` — how often the
+ *     accepted lesson's ref appeared in post-arm runs' `assetsLoaded`, and
+ *     the pass-rate among those reuses.
+ *   - `negative_transfer_count` — count of (taskId, seed) pairs where the
+ *     same task PASSED in pre but FAILED in post AND the post run loaded
+ *     this lesson's ref. Spec §6.4 negative-transfer attribution.
+ *   - `leakage_risk` — `"high"` when any verbatim 4-token-or-longer phrase
+ *     in the supplied verifier source(s) appears verbatim in the lesson
+ *     body; `"medium"` for 3-token leakage; `"low"` otherwise. Mirrors the
+ *     Wave 3 `leakage.test.ts` philosophy: structural fragments are red
+ *     flags, lone tokens are not.
+ *
+ * The function is pure: no disk I/O, no subprocess. Callers (the evolve
+ * runner) thread lesson bodies + verifier sources through optional maps so
+ * the leakage check is fully deterministic and testable with mock inputs.
+ */
+/**
+ * Compute lesson-quality + reuse metrics from the evolve runner's outputs.
+ * Pure function — does not touch disk and does not invoke any subprocess.
+ *
+ * Only `proposalLog` entries with `kind === "lesson"` are surfaced as
+ * `LessonRecord`s. Revision-kind proposals are tracked elsewhere (the
+ * §6.3 `proposals` block already covers them) and would skew the lesson
+ * reuse rate if mixed in.
+ */
+export function computeLessonMetrics(input) {
+    const lessons = input.proposalLog.filter((p) => p.kind === "lesson");
+    const feedbackLog = input.feedbackLog ?? [];
+    const preRuns = input.preRuns ?? [];
+    const postRuns = input.postRuns ?? [];
+    const lessonBodies = input.lessonBodies ?? {};
+    const verifierSources = input.verifierSources ?? {};
+    // Pre-index pre-arm task → seed → outcome so negative-transfer attribution
+    // is a constant-time lookup per post run.
+    const preOutcomes = new Map();
+    for (const r of preRuns) {
+        let inner = preOutcomes.get(r.taskId);
+        if (!inner) {
+            inner = new Map();
+            preOutcomes.set(r.taskId, inner);
+        }
+        inner.set(r.seed, r.outcome);
+    }
+    // Pre-index negative feedback by ref so source_failures is O(events).
+    const negativeFeedbackByRef = new Map();
+    for (const ev of feedbackLog) {
+        if (ev.signal !== "negative")
+            continue;
+        let set = negativeFeedbackByRef.get(ev.goldRef);
+        if (!set) {
+            set = new Set();
+            negativeFeedbackByRef.set(ev.goldRef, set);
+        }
+        set.add(ev.taskId);
+    }
+    const records = lessons.map((p) => {
+        const ref = p.assetRef;
+        const sourceFailures = [...(negativeFeedbackByRef.get(ref) ?? [])].sort();
+        // Reuse: post-arm runs that loaded this ref.
+        let firstReusedOn = null;
+        let reuseCount = 0;
+        let reusePassCount = 0;
+        // Negative transfer: post-FAIL where pre-PASS for the same (task, seed)
+        // AND this lesson was loaded in the post run. Dedupe by taskId so a
+        // task that regresses across multiple seeds counts once.
+        const negativeTransferTasks = new Set();
+        if (p.decision === "accept") {
+            for (const r of postRuns) {
+                if (!r.assetsLoaded?.includes(ref))
+                    continue;
+                if (firstReusedOn === null)
+                    firstReusedOn = r.taskId;
+                reuseCount += 1;
+                if (r.outcome === "pass")
+                    reusePassCount += 1;
+                if (r.outcome === "fail" || r.outcome === "budget_exceeded") {
+                    const prePerSeed = preOutcomes.get(r.taskId);
+                    if (prePerSeed && prePerSeed.get(r.seed) === "pass") {
+                        negativeTransferTasks.add(r.taskId);
+                    }
+                }
+            }
+        }
+        const reusePassRate = reuseCount === 0 ? 0 : reusePassCount / reuseCount;
+        const leakageRisk = classifyLeakageRisk(lessonBodies[ref], verifierSources[ref]);
+        return {
+            ref,
+            source_failures: sourceFailures,
+            lint_pass: p.lintPass,
+            accepted: p.decision === "accept",
+            first_reused_on: firstReusedOn,
+            reuse_count: reuseCount,
+            reuse_pass_rate: reusePassRate,
+            negative_transfer_count: negativeTransferTasks.size,
+            leakage_risk: leakageRisk,
+        };
+    });
+    records.sort((a, b) => a.ref.localeCompare(b.ref));
+    const total = records.length;
+    const accepted = records.filter((r) => r.accepted);
+    const lintPassed = records.filter((r) => r.lint_pass).length;
+    const reusedAccepted = accepted.filter((r) => r.reuse_count > 0);
+    const reusePassRateSum = reusedAccepted.reduce((sum, r) => sum + r.reuse_pass_rate, 0);
+    const negativeTransferTotal = records.reduce((sum, r) => sum + r.negative_transfer_count, 0);
+    return {
+        lessons: records,
+        lessons_created_count: total,
+        lessons_accepted_count: accepted.length,
+        proposal_lint_pass_rate: total === 0 ? 0 : lintPassed / total,
+        proposal_acceptance_rate: total === 0 ? 0 : accepted.length / total,
+        lesson_reuse_rate: accepted.length === 0 ? 0 : reusedAccepted.length / accepted.length,
+        lesson_reuse_success_rate: reusedAccepted.length === 0 ? 0 : reusePassRateSum / reusedAccepted.length,
+        lesson_negative_transfer_count: negativeTransferTotal,
+    };
+}
+/**
+ * Classify lesson-body leakage against verifier source text. Returns
+ * `"high"` when a 4+-word verbatim phrase from any verifier-source entry
+ * appears in the body; `"medium"` for 3-word overlap; `"low"` otherwise.
+ *
+ * The check is intentionally simple — Wave 3's `leakage.test.ts` uses
+ * structural assertion extraction (regex literals, dotted paths, jq/grep
+ * patterns); here we just slide an N-gram window over the verifier text
+ * and ask "does the body contain this exact run of words?". Tokens are
+ * normalised to lowercase and split on non-word boundaries so trivial
+ * whitespace differences don't hide leakage.
+ */
+export function classifyLeakageRisk(body, verifierSources) {
+    if (!body || !verifierSources || verifierSources.length === 0)
+        return "low";
+    const bodyTokens = tokenize(body);
+    if (bodyTokens.length === 0)
+        return "low";
+    const bodyJoined = ` ${bodyTokens.join(" ")} `;
+    let mediumHit = false;
+    for (const source of verifierSources) {
+        const sourceTokens = tokenize(source);
+        if (sourceTokens.length < 3)
+            continue;
+        if (containsNGram(bodyJoined, sourceTokens, 4))
+            return "high";
+        if (!mediumHit && containsNGram(bodyJoined, sourceTokens, 3))
+            mediumHit = true;
+    }
+    return mediumHit ? "medium" : "low";
+}
+/**
+ * Slide an N-gram window of size `n` across `tokens` and return true if any
+ * window appears as a contiguous substring inside `bodyJoined` (which is
+ * pre-padded with spaces so word boundaries match cleanly). Skips windows
+ * shorter than `n`; returns false on empty input.
+ */
+function containsNGram(bodyJoined, tokens, n) {
+    if (tokens.length < n)
+        return false;
+    for (let i = 0; i + n <= tokens.length; i += 1) {
+        const phrase = ` ${tokens.slice(i, i + n).join(" ")} `;
+        if (bodyJoined.includes(phrase))
+            return true;
+    }
+    return false;
+}
+/** Lowercase tokens split on non-word characters. Empty strings dropped. */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/)
+        .filter((t) => t.length > 0);
+}