akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/environment.js

@@ -0,0 +1,233 @@
+/**
+ * environment.ts — unified bench environment setup.
+ *
+ * `setupBenchEnvironment` is the single function that owns all per-run
+ * isolation: isolation dirs, opencode.json, akm config, FTS5 index. Both
+ * `runOne` (driver.ts) and the doctor's live-run check call this function,
+ * guaranteeing they produce identical environments.
+ *
+ * Key design decisions:
+ * - `BENCH_OPENCODE_INVARIANTS` (plugin:[], permission block) are always
+ *   written — they are bench isolation invariants, not conditional on the
+ *   provider path. No silent stub fallbacks.
+ * - `dryRun: true` skips the akm config and index writes. Unit tests set
+ *   this so the setup path is exercised without spawning a real agent.
+ * - `validateFixtureCorpus` is called at bench startup to catch missing
+ *   fixtures before any work items start, not per-task mid-run.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { buildIsolatedEnv, buildSanitizedEnvSource, createIsolationDirs } from "./driver";
+import { BenchConfigError, selectProviderForModel } from "./opencode-config";
+import { benchMkdtemp } from "./tmp";
+// ── Bench isolation invariants ───────────────────────────────────────────────
+/**
+ * Top-level keys written unconditionally into every bench-generated
+ * opencode.json. These are isolation invariants — never conditional on
+ * provider resolution or model type.
+ *
+ * - `plugin: []` — prevents operator plugins (akm-opencode, etc.) from
+ *   running lifecycle hooks that override AKM_STASH_DIR, warm indexes
+ *   against the wrong stash, or prompt akm setup wizards.
+ * - `permission` — opencode in non-interactive (`opencode run`) mode
+ *   silently skips tool calls without explicit permission grants.
+ */
+export const BENCH_OPENCODE_INVARIANTS = {
+    plugin: [],
+    permission: {
+        bash: "allow",
+        edit: "allow",
+        write: "allow",
+        read: "allow",
+        webfetch: "allow",
+    },
+};
+// ── Built-in cloud prefixes ──────────────────────────────────────────────────
+/**
+ * opencode provider prefixes that resolve via its built-in cloud-provider
+ * registry. Models with one of these prefixes do not need a custom provider
+ * entry in the bench providers JSON. Models with any other prefix require
+ * `opencodeProviders` — the harness refuses to run without it to prevent
+ * silent cloud-model fallback and unexpected API charges.
+ */
+export const BUILTIN_CLOUD_PREFIXES = new Set([
+    "anthropic",
+    "openai",
+    "openrouter",
+    "opencode",
+    "google",
+    "amazon",
+    "azure",
+    "vertex",
+    "bedrock",
+    "mistral",
+    "groq",
+    "together",
+    "fireworks",
+]);
+/**
+ * Write an `opencode.json` into `opencodeConfigDir`.
+ *
+ * Always includes `BENCH_OPENCODE_INVARIANTS` (plugin:[], permission block).
+ * When `providers` is supplied and the model prefix resolves, the `provider`
+ * block is added. When the prefix is not found in the providers map (built-in
+ * cloud model), the file is written without a provider block and a warning is
+ * returned — this is not an error because built-in cloud models resolve via
+ * opencode's own registry.
+ *
+ * Returns a `WriteOpencodeJsonResult` — never throws for expected cases.
+ * Throws for unexpected FS errors.
+ */
+export function writeOpencodeJson(opencodeConfigDir, model, providers) {
+    const warnings = [];
+    let providerKey;
+    let providerBlock;
+    if (providers) {
+        try {
+            const selected = selectProviderForModel(providers, model);
+            providerKey = selected.providerKey;
+            providerBlock = { [selected.providerKey]: selected.entry };
+        }
+        catch (err) {
+            if (err instanceof BenchConfigError) {
+                // Check if this is a local-provider model that MUST have a provider block.
+                const modelPrefix = model.split("/")[0];
+                if (modelPrefix && !BUILTIN_CLOUD_PREFIXES.has(modelPrefix)) {
+                    // Local-prefix model not in providers map — this is a hard error, not a
+                    // fallback. Writing opencode.json without a provider block would cause
+                    // opencode to use cloud resolution, skewing results and incurring costs.
+                    throw new BenchConfigError(`model "${model}" uses local prefix "${modelPrefix}" but was not found in the providers config. ` +
+                        `Add it to the providers file or use a built-in cloud model prefix.`, true);
+                }
+                warnings.push(`model "${model}" not found in providers config; writing stub (expected for built-in cloud models)`);
+            }
+            else {
+                throw err;
+            }
+        }
+    }
+    const config = {
+        $schema: "https://opencode.ai/config.json",
+        model,
+        ...BENCH_OPENCODE_INVARIANTS,
+        ...(providerBlock ? { provider: providerBlock } : {}),
+    };
+    fs.writeFileSync(path.join(opencodeConfigDir, "opencode.json"), JSON.stringify(config, null, 2), { mode: 0o600 });
+    return { providerKey, warnings };
+}
+/**
+ * Set up a complete bench run environment.
+ *
+ * 1. Creates isolation dirs (XDG_CACHE_HOME, XDG_CONFIG_HOME, OPENCODE_CONFIG).
+ * 2. Writes opencode.json with BENCH_OPENCODE_INVARIANTS + optional provider.
+ * 3. Writes $XDG_CONFIG_HOME/akm/config.json so the akm CLI and any plugin
+ *    find the correct stash via `akm config get stashDir`.
+ * 4. Copies the pre-built FTS5 index into XDG_CACHE_HOME, or re-indexes as
+ *    fallback if no pre-built cache is available.
+ *
+ * Throws `BenchConfigError` for model prefix / provider mismatches.
+ */
+export function setupBenchEnvironment(params) {
+    const { model, arm, stashDir: rawStashDir, indexCacheHome, providers, dryRun = false, warnings = [] } = params;
+    // Synthetic arm must never carry a stash.
+    const stashDir = arm === "synthetic" ? undefined : rawStashDir;
+    // Safety: refuse to run local-provider models without a providers config.
+    const modelParts = model.split("/");
+    if (modelParts.length >= 2 && !BUILTIN_CLOUD_PREFIXES.has(modelParts[0]) && !providers) {
+        throw new BenchConfigError(`model "${model}" uses custom provider prefix "${modelParts[0]}" — supply opencodeProviders to avoid silent fallback to a cloud model`, false);
+    }
+    const dirs = createIsolationDirs(stashDir);
+    const env = buildIsolatedEnv(dirs, model);
+    // Synthetic arm must not carry AKM_STASH_DIR even if createIsolationDirs
+    // somehow set it (recurrence guard for the #243 fixup pattern).
+    if (arm === "synthetic") {
+        delete env.AKM_STASH_DIR;
+    }
+    // Write opencode.json with invariants + optional provider block.
+    const result = writeOpencodeJson(dirs.opencodeConfig, model, providers);
+    for (const w of result.warnings)
+        warnings.push(w);
+    // Wire akm config and index only when a real stash is on disk.
+    const stashOnDisk = stashDir ? fs.existsSync(stashDir) : false;
+    if (stashDir && stashOnDisk && !dryRun) {
+        // akm config: so `akm config get stashDir` returns the fixture path
+        // and the akm-opencode plugin (if somehow re-enabled) injects the right
+        // AKM_STASH_DIR into the bash-tool env via its shell.env hook.
+        const akmConfigDir = path.join(dirs.configHome, "akm");
+        fs.mkdirSync(akmConfigDir, { recursive: true });
+        fs.writeFileSync(path.join(akmConfigDir, "config.json"), JSON.stringify({ stashDir }), { mode: 0o600 });
+        // FTS5 index: fast-path copy from pre-built cache; slow-path re-index.
+        const destAkmDir = path.join(dirs.cacheHome, "akm");
+        fs.mkdirSync(destAkmDir, { recursive: true });
+        if (indexCacheHome) {
+            const srcAkmDir = path.join(indexCacheHome, "akm");
+            try {
+                for (const entry of fs.readdirSync(srcAkmDir)) {
+                    fs.copyFileSync(path.join(srcAkmDir, entry), path.join(destAkmDir, entry));
+                }
+            }
+            catch (err) {
+                warnings.push(`index copy failed, falling back to re-index: ${err.message}`);
+                _runAkmIndex(stashDir, env);
+            }
+        }
+        else {
+            _runAkmIndex(stashDir, env);
+        }
+    }
+    return {
+        dirs,
+        env,
+        teardown() {
+            try {
+                fs.rmSync(dirs.root, { recursive: true, force: true });
+            }
+            catch {
+                /* swallow */
+            }
+        },
+    };
+}
+function _runAkmIndex(stashDir, env) {
+    const cliEntry = path.resolve(__dirname, "..", "..", "src", "cli.ts");
+    Bun.spawnSync({
+        cmd: ["bun", "run", cliEntry, "index", "--full"],
+        cwd: stashDir,
+        env: { ...buildSanitizedEnvSource(), ...env },
+        stdout: "pipe",
+        stderr: "pipe",
+    });
+}
+// ── validateFixtureCorpus ────────────────────────────────────────────────────
+const FIXTURES_ROOT = path.resolve(__dirname, "..", "fixtures", "stashes");
+/**
+ * Validate that all task stash references name fixtures that exist on disk
+ * (i.e. have a MANIFEST.json). Returns the set of missing fixture names.
+ *
+ * Call at bench startup before creating any work items. A non-empty `missing`
+ * set means those tasks will produce `harness_error` at run time — better to
+ * surface that now with named failures than to discover it per-seed.
+ */
+export function validateFixtureCorpus(tasks) {
+    const byFixture = new Map();
+    for (const t of tasks) {
+        if (!byFixture.has(t.stash))
+            byFixture.set(t.stash, []);
+        byFixture.get(t.stash)?.push(t.id);
+    }
+    const valid = new Set();
+    const missing = new Map();
+    for (const [fixture, taskIds] of byFixture) {
+        const manifestPath = path.join(FIXTURES_ROOT, fixture, "MANIFEST.json");
+        if (fs.existsSync(manifestPath)) {
+            valid.add(fixture);
+        }
+        else {
+            missing.set(fixture, taskIds);
+        }
+    }
+    return { valid, missing };
+}
+// Re-export from driver for consumers that previously imported from there.
+export { buildIsolatedEnv, buildSanitizedEnvSource, createIsolationDirs } from "./driver";
+export { benchMkdtemp };

package/dist/tests/bench/environment.test.js

@@ -0,0 +1,199 @@
+/**
+ * Tests for environment.ts — writeOpencodeJson, validateFixtureCorpus,
+ * BENCH_OPENCODE_INVARIANTS, and setupBenchEnvironment (dryRun mode).
+ */
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { BENCH_OPENCODE_INVARIANTS, BUILTIN_CLOUD_PREFIXES, setupBenchEnvironment, validateFixtureCorpus, writeOpencodeJson, } from "./environment";
+import { benchMkdtemp } from "./tmp";
+// ── writeOpencodeJson ────────────────────────────────────────────────────────
+describe("writeOpencodeJson", () => {
+    let tmp;
+    beforeAll(() => {
+        tmp = benchMkdtemp("bench-env-test-");
+    });
+    afterAll(() => {
+        fs.rmSync(tmp, { recursive: true, force: true });
+    });
+    test("always writes plugin:[] and permission block (isolation invariants)", () => {
+        const dir = path.join(tmp, "invariants");
+        fs.mkdirSync(dir, { recursive: true });
+        writeOpencodeJson(dir, "anthropic/claude-opus-4-7");
+        const config = JSON.parse(fs.readFileSync(path.join(dir, "opencode.json"), "utf8"));
+        expect(config.plugin).toEqual([]);
+        expect(config.permission?.bash).toBe("allow");
+        expect(config.permission?.edit).toBe("allow");
+        expect(config.permission?.write).toBe("allow");
+    });
+    test("writes provider block when model resolves in providers map", () => {
+        const dir = path.join(tmp, "with-provider");
+        fs.mkdirSync(dir, { recursive: true });
+        const providers = {
+            source: "/fake/providers.json",
+            providers: { myprov: { npm: "@ai-sdk/openai-compatible", name: "My Provider" } },
+        };
+        const result = writeOpencodeJson(dir, "myprov/my-model", providers);
+        expect(result.providerKey).toBe("myprov");
+        expect(result.warnings).toHaveLength(0);
+        const config = JSON.parse(fs.readFileSync(path.join(dir, "opencode.json"), "utf8"));
+        expect(config.provider?.myprov).toBeDefined();
+        expect(config.model).toBe("myprov/my-model");
+    });
+    test("writes stub (no provider block) and returns warning for built-in cloud model not in providers map", () => {
+        const dir = path.join(tmp, "cloud-stub");
+        fs.mkdirSync(dir, { recursive: true });
+        const providers = {
+            source: "/fake/providers.json",
+            providers: { otherprov: {} },
+        };
+        const result = writeOpencodeJson(dir, "opencode/big-pickle", providers);
+        expect(result.providerKey).toBeUndefined();
+        expect(result.warnings.length).toBeGreaterThan(0);
+        const config = JSON.parse(fs.readFileSync(path.join(dir, "opencode.json"), "utf8"));
+        expect(config.provider).toBeUndefined();
+        // Invariants still present.
+        expect(config.plugin).toEqual([]);
+    });
+    test("throws BenchConfigError for local-prefix model not found in providers map", () => {
+        const dir = path.join(tmp, "local-prefix-missing");
+        fs.mkdirSync(dir, { recursive: true });
+        const providers = {
+            source: "/fake/providers.json",
+            providers: { otherprov: {} },
+        };
+        // "shredder" is not in BUILTIN_CLOUD_PREFIXES and not in the providers map.
+        expect(() => writeOpencodeJson(dir, "shredder/qwen3.5-9b", providers)).toThrow(/local prefix/);
+        // The opencode.json must NOT have been written (or if partially written, provider block is absent).
+        // We check that the function threw rather than silently wrote a cloud-fallback stub.
+    });
+    test("writes provider block for local-prefix model that IS found in providers map", () => {
+        const dir = path.join(tmp, "local-prefix-found");
+        fs.mkdirSync(dir, { recursive: true });
+        const providers = {
+            source: "/fake/providers.json",
+            providers: { shredder: { npm: "@ai-sdk/openai-compatible", name: "Shredder" } },
+        };
+        const result = writeOpencodeJson(dir, "shredder/qwen3.5-9b", providers);
+        expect(result.providerKey).toBe("shredder");
+        expect(result.warnings).toHaveLength(0);
+        const config = JSON.parse(fs.readFileSync(path.join(dir, "opencode.json"), "utf8"));
+        expect(config.provider?.shredder).toBeDefined();
+        expect(config.model).toBe("shredder/qwen3.5-9b");
+    });
+    test("mode 0o600 (not world-readable)", () => {
+        const dir = path.join(tmp, "mode-check");
+        fs.mkdirSync(dir, { recursive: true });
+        writeOpencodeJson(dir, "anthropic/claude-opus-4-7");
+        const stat = fs.statSync(path.join(dir, "opencode.json"));
+        expect(stat.mode & 0o777).toBe(0o600);
+    });
+});
+// ── BENCH_OPENCODE_INVARIANTS ────────────────────────────────────────────────
+describe("BENCH_OPENCODE_INVARIANTS", () => {
+    test("plugin is an empty readonly array", () => {
+        expect(BENCH_OPENCODE_INVARIANTS.plugin).toEqual([]);
+        expect(Array.isArray(BENCH_OPENCODE_INVARIANTS.plugin)).toBe(true);
+    });
+    test("permission.bash is 'allow'", () => {
+        expect(BENCH_OPENCODE_INVARIANTS.permission.bash).toBe("allow");
+    });
+});
+// ── BUILTIN_CLOUD_PREFIXES ───────────────────────────────────────────────────
+describe("BUILTIN_CLOUD_PREFIXES", () => {
+    test("includes anthropic, openai, opencode", () => {
+        expect(BUILTIN_CLOUD_PREFIXES.has("anthropic")).toBe(true);
+        expect(BUILTIN_CLOUD_PREFIXES.has("openai")).toBe(true);
+        expect(BUILTIN_CLOUD_PREFIXES.has("opencode")).toBe(true);
+    });
+    test("does not include custom provider prefixes like 'shredder' or 'don'", () => {
+        expect(BUILTIN_CLOUD_PREFIXES.has("shredder")).toBe(false);
+        expect(BUILTIN_CLOUD_PREFIXES.has("don")).toBe(false);
+    });
+});
+// ── validateFixtureCorpus ────────────────────────────────────────────────────
+describe("validateFixtureCorpus", () => {
+    test("returns known fixtures as valid", () => {
+        const tasks = [{ id: "az-cli/foo", stash: "az-cli" }];
+        const { valid, missing } = validateFixtureCorpus(tasks);
+        expect(valid.has("az-cli")).toBe(true);
+        expect(missing.size).toBe(0);
+    });
+    test("returns nonexistent fixture as missing with its task IDs", () => {
+        const tasks = [
+            { id: "ghost/task-1", stash: "ghost-fixture" },
+            { id: "ghost/task-2", stash: "ghost-fixture" },
+        ];
+        const { valid, missing } = validateFixtureCorpus(tasks);
+        expect(valid.has("ghost-fixture")).toBe(false);
+        expect(missing.has("ghost-fixture")).toBe(true);
+        expect(missing.get("ghost-fixture")).toEqual(["ghost/task-1", "ghost/task-2"]);
+    });
+    test("handles empty task list", () => {
+        const { valid, missing } = validateFixtureCorpus([]);
+        expect(valid.size).toBe(0);
+        expect(missing.size).toBe(0);
+    });
+    test("deduplicates fixture names across tasks", () => {
+        const tasks = [
+            { id: "az-cli/a", stash: "az-cli" },
+            { id: "az-cli/b", stash: "az-cli" },
+            { id: "az-cli/c", stash: "az-cli" },
+        ];
+        const { valid } = validateFixtureCorpus(tasks);
+        expect(valid.size).toBe(1);
+    });
+});
+// ── setupBenchEnvironment (dryRun) ───────────────────────────────────────────
+describe("setupBenchEnvironment dryRun", () => {
+    test("creates isolation dirs and writes opencode.json with invariants", () => {
+        const env = setupBenchEnvironment({
+            model: "anthropic/claude-opus-4-7",
+            arm: "akm",
+            dryRun: true,
+        });
+        try {
+            expect(fs.existsSync(env.dirs.cacheHome)).toBe(true);
+            expect(fs.existsSync(env.dirs.configHome)).toBe(true);
+            expect(fs.existsSync(env.dirs.opencodeConfig)).toBe(true);
+            const config = JSON.parse(fs.readFileSync(path.join(env.dirs.opencodeConfig, "opencode.json"), "utf8"));
+            expect(config.plugin).toEqual([]);
+            expect(config.permission?.bash).toBe("allow");
+        }
+        finally {
+            env.teardown();
+        }
+    });
+    test("throws for custom provider prefix without providers config", () => {
+        expect(() => setupBenchEnvironment({
+            model: "shredder/qwen/qwen3.5-9b",
+            arm: "akm",
+            dryRun: true,
+        })).toThrow(/custom provider prefix/);
+    });
+    test("synthetic arm never sets AKM_STASH_DIR", () => {
+        const env = setupBenchEnvironment({
+            model: "anthropic/claude-opus-4-7",
+            arm: "synthetic",
+            stashDir: "/some/stash",
+            dryRun: true,
+        });
+        try {
+            expect(env.env.AKM_STASH_DIR).toBeUndefined();
+        }
+        finally {
+            env.teardown();
+        }
+    });
+    test("teardown removes the isolation dirs", () => {
+        const env = setupBenchEnvironment({
+            model: "anthropic/claude-opus-4-7",
+            arm: "akm",
+            dryRun: true,
+        });
+        const { root } = env.dirs;
+        expect(fs.existsSync(root)).toBe(true);
+        env.teardown();
+        expect(fs.existsSync(root)).toBe(false);
+    });
+});

package/dist/tests/bench/evolve-metrics.js

@@ -0,0 +1,179 @@
+/**
+ * Track B lesson quality + reuse metrics (issue #264, spec §6.3 follow-up).
+ *
+ * `computeLessonMetrics` walks the evolve runner's proposal log and the
+ * Phase 3 pre/post arm `RunResult[]`s and emits one `LessonRecord` per
+ * lesson-kind proposal. The record captures:
+ *
+ *   - `source_failures` — eval/train tasks whose negative feedback events
+ *     targeted this asset ref (joined via the supplied `feedbackLog`).
+ *   - `lint_pass` / `accepted` — verbatim from the proposal log entry.
+ *   - `first_reused_on` / `reuse_count` / `reuse_pass_rate` — how often the
+ *     accepted lesson's ref appeared in post-arm runs' `assetsLoaded`, and
+ *     the pass-rate among those reuses.
+ *   - `negative_transfer_count` — count of (taskId, seed) pairs where the
+ *     same task PASSED in pre but FAILED in post AND the post run loaded
+ *     this lesson's ref. Spec §6.4 negative-transfer attribution.
+ *   - `leakage_risk` — `"high"` when any verbatim 4-token-or-longer phrase
+ *     in the supplied verifier source(s) appears verbatim in the lesson
+ *     body; `"medium"` for 3-token leakage; `"low"` otherwise. Mirrors the
+ *     Wave 3 `leakage.test.ts` philosophy: structural fragments are red
+ *     flags, lone tokens are not.
+ *
+ * The function is pure: no disk I/O, no subprocess. Callers (the evolve
+ * runner) thread lesson bodies + verifier sources through optional maps so
+ * the leakage check is fully deterministic and testable with mock inputs.
+ */
+/**
+ * Compute lesson-quality + reuse metrics from the evolve runner's outputs.
+ * Pure function — does not touch disk and does not invoke any subprocess.
+ *
+ * Only `proposalLog` entries with `kind === "lesson"` are surfaced as
+ * `LessonRecord`s. Revision-kind proposals are tracked elsewhere (the
+ * §6.3 `proposals` block already covers them) and would skew the lesson
+ * reuse rate if mixed in.
+ */
+export function computeLessonMetrics(input) {
+    const lessons = input.proposalLog.filter((p) => p.kind === "lesson");
+    const feedbackLog = input.feedbackLog ?? [];
+    const preRuns = input.preRuns ?? [];
+    const postRuns = input.postRuns ?? [];
+    const lessonBodies = input.lessonBodies ?? {};
+    const verifierSources = input.verifierSources ?? {};
+    // Pre-index pre-arm task → seed → outcome so negative-transfer attribution
+    // is a constant-time lookup per post run.
+    const preOutcomes = new Map();
+    for (const r of preRuns) {
+        let inner = preOutcomes.get(r.taskId);
+        if (!inner) {
+            inner = new Map();
+            preOutcomes.set(r.taskId, inner);
+        }
+        inner.set(r.seed, r.outcome);
+    }
+    // Pre-index negative feedback by ref so source_failures is O(events).
+    const negativeFeedbackByRef = new Map();
+    for (const ev of feedbackLog) {
+        if (ev.signal !== "negative")
+            continue;
+        let set = negativeFeedbackByRef.get(ev.goldRef);
+        if (!set) {
+            set = new Set();
+            negativeFeedbackByRef.set(ev.goldRef, set);
+        }
+        set.add(ev.taskId);
+    }
+    const records = lessons.map((p) => {
+        const ref = p.assetRef;
+        const sourceFailures = [...(negativeFeedbackByRef.get(ref) ?? [])].sort();
+        // Reuse: post-arm runs that loaded this ref.
+        let firstReusedOn = null;
+        let reuseCount = 0;
+        let reusePassCount = 0;
+        // Negative transfer: post-FAIL where pre-PASS for the same (task, seed)
+        // AND this lesson was loaded in the post run. Dedupe by taskId so a
+        // task that regresses across multiple seeds counts once.
+        const negativeTransferTasks = new Set();
+        if (p.decision === "accept") {
+            for (const r of postRuns) {
+                if (!r.assetsLoaded?.includes(ref))
+                    continue;
+                if (firstReusedOn === null)
+                    firstReusedOn = r.taskId;
+                reuseCount += 1;
+                if (r.outcome === "pass")
+                    reusePassCount += 1;
+                if (r.outcome === "fail" || r.outcome === "budget_exceeded") {
+                    const prePerSeed = preOutcomes.get(r.taskId);
+                    if (prePerSeed && prePerSeed.get(r.seed) === "pass") {
+                        negativeTransferTasks.add(r.taskId);
+                    }
+                }
+            }
+        }
+        const reusePassRate = reuseCount === 0 ? 0 : reusePassCount / reuseCount;
+        const leakageRisk = classifyLeakageRisk(lessonBodies[ref], verifierSources[ref]);
+        return {
+            ref,
+            source_failures: sourceFailures,
+            lint_pass: p.lintPass,
+            accepted: p.decision === "accept",
+            first_reused_on: firstReusedOn,
+            reuse_count: reuseCount,
+            reuse_pass_rate: reusePassRate,
+            negative_transfer_count: negativeTransferTasks.size,
+            leakage_risk: leakageRisk,
+        };
+    });
+    records.sort((a, b) => a.ref.localeCompare(b.ref));
+    const total = records.length;
+    const accepted = records.filter((r) => r.accepted);
+    const lintPassed = records.filter((r) => r.lint_pass).length;
+    const reusedAccepted = accepted.filter((r) => r.reuse_count > 0);
+    const reusePassRateSum = reusedAccepted.reduce((sum, r) => sum + r.reuse_pass_rate, 0);
+    const negativeTransferTotal = records.reduce((sum, r) => sum + r.negative_transfer_count, 0);
+    return {
+        lessons: records,
+        lessons_created_count: total,
+        lessons_accepted_count: accepted.length,
+        proposal_lint_pass_rate: total === 0 ? 0 : lintPassed / total,
+        proposal_acceptance_rate: total === 0 ? 0 : accepted.length / total,
+        lesson_reuse_rate: accepted.length === 0 ? 0 : reusedAccepted.length / accepted.length,
+        lesson_reuse_success_rate: reusedAccepted.length === 0 ? 0 : reusePassRateSum / reusedAccepted.length,
+        lesson_negative_transfer_count: negativeTransferTotal,
+    };
+}
+/**
+ * Classify lesson-body leakage against verifier source text. Returns
+ * `"high"` when a 4+-word verbatim phrase from any verifier-source entry
+ * appears in the body; `"medium"` for 3-word overlap; `"low"` otherwise.
+ *
+ * The check is intentionally simple — Wave 3's `leakage.test.ts` uses
+ * structural assertion extraction (regex literals, dotted paths, jq/grep
+ * patterns); here we just slide an N-gram window over the verifier text
+ * and ask "does the body contain this exact run of words?". Tokens are
+ * normalised to lowercase and split on non-word boundaries so trivial
+ * whitespace differences don't hide leakage.
+ */
+export function classifyLeakageRisk(body, verifierSources) {
+    if (!body || !verifierSources || verifierSources.length === 0)
+        return "low";
+    const bodyTokens = tokenize(body);
+    if (bodyTokens.length === 0)
+        return "low";
+    const bodyJoined = ` ${bodyTokens.join(" ")} `;
+    let mediumHit = false;
+    for (const source of verifierSources) {
+        const sourceTokens = tokenize(source);
+        if (sourceTokens.length < 3)
+            continue;
+        if (containsNGram(bodyJoined, sourceTokens, 4))
+            return "high";
+        if (!mediumHit && containsNGram(bodyJoined, sourceTokens, 3))
+            mediumHit = true;
+    }
+    return mediumHit ? "medium" : "low";
+}
+/**
+ * Slide an N-gram window of size `n` across `tokens` and return true if any
+ * window appears as a contiguous substring inside `bodyJoined` (which is
+ * pre-padded with spaces so word boundaries match cleanly). Skips windows
+ * shorter than `n`; returns false on empty input.
+ */
+function containsNGram(bodyJoined, tokens, n) {
+    if (tokens.length < n)
+        return false;
+    for (let i = 0; i + n <= tokens.length; i += 1) {
+        const phrase = ` ${tokens.slice(i, i + n).join(" ")} `;
+        if (bodyJoined.includes(phrase))
+            return true;
+    }
+    return false;
+}
+/** Lowercase tokens split on non-word characters. Empty strings dropped. */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/)
+        .filter((t) => t.length > 0);
+}