akm-cli 0.6.0 → 0.7.0-rc1

package/dist/tests/bench/feedback-integrity.test.js

@@ -0,0 +1,456 @@
+/**
+ * Unit tests for §6.8 feedback-signal integrity (#244).
+ *
+ * Coverage:
+ *   • All four 2×2 quadrants (TP, FP, TN, FN).
+ *   • Per-asset breakdown when an asset has mixed signals across runs.
+ *   • `feedback_agreement < 0.80` triggers the warning marker (markdown +
+ *     structured `warnings[]` JSON entry).
+ *   • `feedback_coverage` correctly counts runs with feedback dispatched
+ *     vs total Phase 1 runs.
+ *   • NaN-safety: zero-feedback asset emits all rates as `null`, never
+ *     `0` or `NaN`.
+ *   • Attribution rule (§6.8): a feedback event is attributed to the run
+ *     that produced it, not to a later run touching the same asset.
+ *
+ * The metric is a pure function over RunResult[] + feedbackLog[]; no spawn
+ * fakes are needed. We build small synthetic streams directly.
+ */
+import { describe, expect, test } from "bun:test";
+import { computeFeedbackIntegrity } from "./metrics";
+import { FEEDBACK_AGREEMENT_WARNING_THRESHOLD, renderEvolveReport, renderFeedbackIntegrityTable } from "./report";
+function fakeRun(overrides) {
+    return {
+        schemaVersion: 1,
+        taskId: "t",
+        arm: "akm",
+        seed: 0,
+        model: "m",
+        outcome: "pass",
+        tokens: { input: 0, output: 0 },
+        wallclockMs: 0,
+        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
+        events: [],
+        verifierStdout: "",
+        verifierExitCode: 0,
+        assetsLoaded: [],
+        ...overrides,
+    };
+}
+function fb(overrides) {
+    return {
+        taskId: "t",
+        seed: 0,
+        goldRef: "skill:s",
+        signal: "positive",
+        ok: true,
+        ...overrides,
+    };
+}
+describe("computeFeedbackIntegrity — 2x2 quadrants", () => {
+    test("TP: feedback + on a passed run", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "pass" })] };
+        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "positive" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(1);
+        expect(m.aggregate.falsePositive).toBe(0);
+        expect(m.aggregate.trueNegative).toBe(0);
+        expect(m.aggregate.falseNegative).toBe(0);
+        expect(m.aggregate.feedback_agreement).toBeCloseTo(1);
+        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
+        expect(m.perAsset).toHaveLength(1);
+        expect(m.perAsset[0].ref).toBe("skill:a");
+        expect(m.perAsset[0].truePositive).toBe(1);
+        expect(m.perAsset[0].feedback_agreement).toBeCloseTo(1);
+    });
+    test("FP: feedback + on a failed run", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "fail" })] };
+        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "positive" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(0);
+        expect(m.aggregate.falsePositive).toBe(1);
+        expect(m.aggregate.trueNegative).toBe(0);
+        expect(m.aggregate.falseNegative).toBe(0);
+        expect(m.aggregate.feedback_agreement).toBeCloseTo(0);
+        expect(m.aggregate.false_positive_rate).toBeCloseTo(1);
+        expect(m.perAsset[0].falsePositive).toBe(1);
+    });
+    test("TN: feedback - on a failed run", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "fail" })] };
+        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "negative" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.trueNegative).toBe(1);
+        expect(m.aggregate.feedback_agreement).toBeCloseTo(1);
+        expect(m.aggregate.false_positive_rate).toBeCloseTo(0);
+        expect(m.perAsset[0].trueNegative).toBe(1);
+    });
+    test("FN: feedback - on a passed run", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "pass" })] };
+        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "negative" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.falseNegative).toBe(1);
+        expect(m.aggregate.feedback_agreement).toBeCloseTo(0);
+        expect(m.aggregate.false_negative_rate).toBeCloseTo(1);
+        expect(m.perAsset[0].falseNegative).toBe(1);
+    });
+});
+describe("computeFeedbackIntegrity — aggregate over mixed quadrants", () => {
+    test("computes feedback_agreement and rates correctly across mixed runs", () => {
+        // 4 runs covering all four quadrants — exactly one of each.
+        const phase1 = {
+            akmRuns: [
+                fakeRun({ taskId: "tp", seed: 0, outcome: "pass" }),
+                fakeRun({ taskId: "fp", seed: 0, outcome: "fail" }),
+                fakeRun({ taskId: "tn", seed: 0, outcome: "fail" }),
+                fakeRun({ taskId: "fn", seed: 0, outcome: "pass" }),
+            ],
+        };
+        const feedbackLog = [
+            fb({ taskId: "tp", seed: 0, goldRef: "skill:tp", signal: "positive" }),
+            fb({ taskId: "fp", seed: 0, goldRef: "skill:fp", signal: "positive" }),
+            fb({ taskId: "tn", seed: 0, goldRef: "skill:tn", signal: "negative" }),
+            fb({ taskId: "fn", seed: 0, goldRef: "skill:fn", signal: "negative" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(1);
+        expect(m.aggregate.falsePositive).toBe(1);
+        expect(m.aggregate.trueNegative).toBe(1);
+        expect(m.aggregate.falseNegative).toBe(1);
+        expect(m.aggregate.feedback_agreement).toBeCloseTo(0.5); // 2/4
+        expect(m.aggregate.false_positive_rate).toBeCloseTo(0.5); // 1 / (1+1)
+        expect(m.aggregate.false_negative_rate).toBeCloseTo(0.5); // 1 / (1+1)
+        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
+        expect(m.perAsset).toHaveLength(4);
+        // Per-asset rows should be sorted by ref
+        expect(m.perAsset.map((r) => r.ref)).toEqual(["skill:fn", "skill:fp", "skill:tn", "skill:tp"]);
+    });
+});
+describe("computeFeedbackIntegrity — per-asset mixed signals", () => {
+    test("aggregates correctly when one asset appears across multiple Phase 1 runs", () => {
+        // skill:shared has 2 TP, 1 FP, 1 TN, 1 FN across 5 runs.
+        const phase1 = {
+            akmRuns: [
+                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 1, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 2, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 3, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 4, outcome: "pass" }),
+            ],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:shared", signal: "positive" }), // TP
+            fb({ taskId: "t", seed: 1, goldRef: "skill:shared", signal: "positive" }), // TP
+            fb({ taskId: "t", seed: 2, goldRef: "skill:shared", signal: "positive" }), // FP
+            fb({ taskId: "t", seed: 3, goldRef: "skill:shared", signal: "negative" }), // TN
+            fb({ taskId: "t", seed: 4, goldRef: "skill:shared", signal: "negative" }), // FN
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.perAsset).toHaveLength(1);
+        const row = m.perAsset[0];
+        expect(row.ref).toBe("skill:shared");
+        expect(row.truePositive).toBe(2);
+        expect(row.falsePositive).toBe(1);
+        expect(row.trueNegative).toBe(1);
+        expect(row.falseNegative).toBe(1);
+        expect(row.feedback_agreement).toBeCloseTo(3 / 5);
+        expect(row.false_positive_rate).toBeCloseTo(1 / 2); // FP / (FP+TN) = 1/2
+        expect(row.false_negative_rate).toBeCloseTo(1 / 3); // FN / (FN+TP) = 1/3
+    });
+});
+describe("computeFeedbackIntegrity — attribution rule", () => {
+    test("attributes feedback to the run that produced it, not a later run touching the same asset", () => {
+        // skill:contested appears across two Phase 1 runs:
+        //   run #0: passed, feedback +  → TP
+        //   run #1: failed, feedback +  → FP
+        // The naive (wrong) implementation would conflate both events with
+        // run #1's outcome and label both as FP. The correct implementation
+        // joins each event to its own (taskId, seed) → gets one TP, one FP.
+        const phase1 = {
+            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:contested", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:contested", signal: "positive" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(1);
+        expect(m.aggregate.falsePositive).toBe(1);
+        expect(m.aggregate.trueNegative).toBe(0);
+        expect(m.aggregate.falseNegative).toBe(0);
+        expect(m.perAsset[0].truePositive).toBe(1);
+        expect(m.perAsset[0].falsePositive).toBe(1);
+    });
+});
+describe("computeFeedbackIntegrity — feedback_coverage", () => {
+    test("counts runs with feedback dispatched vs total Phase 1 runs", () => {
+        // 4 phase-1 runs, only 2 had feedback dispatched.
+        const phase1 = {
+            akmRuns: [
+                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 1, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 2, outcome: "harness_error" }),
+                fakeRun({ taskId: "t", seed: 3, outcome: "budget_exceeded" }),
+            ],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.feedback_coverage).toBeCloseTo(0.5); // 2 of 4
+    });
+    test("zero coverage when no feedback dispatched", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] };
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog: [] });
+        expect(m.aggregate.feedback_coverage).toBe(0);
+        expect(m.aggregate.feedback_agreement).toBe(0);
+        expect(m.perAsset).toEqual([]);
+    });
+    test("zero coverage and zero runs returns 0 (not NaN)", () => {
+        const m = computeFeedbackIntegrity({ phase1: { akmRuns: [] }, feedbackLog: [] });
+        expect(m.aggregate.feedback_coverage).toBe(0);
+        expect(m.aggregate.feedback_agreement).toBe(0);
+        expect(m.aggregate.false_positive_rate).toBe(0);
+        expect(m.aggregate.false_negative_rate).toBe(0);
+        expect(Number.isFinite(m.aggregate.feedback_coverage)).toBe(true);
+        expect(Number.isFinite(m.aggregate.feedback_agreement)).toBe(true);
+    });
+});
+describe("computeFeedbackIntegrity — NaN safety", () => {
+    test("per-asset row with FP+TN === 0 emits null false_positive_rate (only positive feedback on passes)", () => {
+        const phase1 = {
+            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "pass" })],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:only-tp", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:only-tp", signal: "positive" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        const row = m.perAsset[0];
+        expect(row.feedback_agreement).toBeCloseTo(1);
+        expect(row.false_positive_rate).toBeNull(); // FP+TN === 0
+        expect(row.false_negative_rate).toBeCloseTo(0); // FN/(FN+TP) = 0/2 = 0
+    });
+    test("per-asset row with FN+TP === 0 emits null false_negative_rate (only negative feedback on fails)", () => {
+        const phase1 = {
+            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "fail" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:only-tn", signal: "negative" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:only-tn", signal: "negative" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        const row = m.perAsset[0];
+        expect(row.feedback_agreement).toBeCloseTo(1);
+        expect(row.false_negative_rate).toBeNull(); // FN+TP === 0
+        expect(row.false_positive_rate).toBeCloseTo(0); // FP/(FP+TN) = 0/2 = 0
+    });
+    test("ok=false feedback events are excluded from the matrix but still count toward coverage", () => {
+        const phase1 = {
+            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive", ok: true }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative", ok: false }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        // Only the ok=true entry contributes to the matrix (TP=1).
+        expect(m.aggregate.truePositive).toBe(1);
+        expect(m.aggregate.trueNegative).toBe(0);
+        // But coverage counts both attempts.
+        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
+    });
+    test("harness_error runs are excluded from the matrix even with a stamped feedback event", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "harness_error" })] };
+        const feedbackLog = [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(0);
+        expect(m.aggregate.falsePositive).toBe(0);
+        expect(m.perAsset).toEqual([]);
+    });
+    test("feedback for a run not present in akmRuns is silently dropped", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "real", seed: 0, outcome: "pass" })] };
+        const feedbackLog = [fb({ taskId: "ghost", seed: 99, goldRef: "skill:a", signal: "positive" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(m.aggregate.truePositive).toBe(0);
+        expect(m.perAsset).toEqual([]);
+        // Coverage still records the dispatch attempt — operator wanted feedback.
+        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
+    });
+});
+// ── Render-side coverage ───────────────────────────────────────────────────
+function emptyUtilityReport() {
+    // Build a minimal §13.3-shaped utility report. The renderer reads
+    // many subfields; we stub them to safe zeros.
+    return {
+        timestamp: "2026-04-27T00:00:00Z",
+        branch: "test",
+        commit: "deadbee",
+        model: "m",
+        corpus: { domains: 0, tasks: 0, slice: "all", seedsPerArm: 1 },
+        aggregateNoakm: { passRate: 0, tokensPerPass: 0, wallclockMs: 0 },
+        aggregateAkm: { passRate: 0, tokensPerPass: 0, wallclockMs: 0 },
+        aggregateDelta: {
+            passRate: 0,
+            tokensPerPass: 0,
+            wallclockMs: 0,
+        },
+        trajectoryAkm: {
+            correctAssetLoaded: null,
+            feedbackRecorded: 0,
+        },
+        failureModes: { byLabel: {}, byTask: {} },
+        tasks: [],
+        warnings: [],
+        akmRuns: [],
+        taskMetadata: [],
+        goldRankRecords: [],
+    };
+}
+function evolveInputWith(metrics) {
+    return {
+        timestamp: "2026-04-27T00:00:00Z",
+        branch: "test",
+        commit: "deadbee",
+        model: "m",
+        domain: "test",
+        seedsPerArm: 1,
+        proposals: { rows: [], totalProposals: 0, totalAccepted: 0, acceptanceRate: 0, lintPassRate: 0 },
+        longitudinal: {
+            improvementSlope: 0.1,
+            overSyntheticLift: 0.05,
+            degradationCount: 0,
+            degradations: [],
+            prePassRate: 0.5,
+            postPassRate: 0.6,
+            syntheticPassRate: 0.55,
+        },
+        arms: { pre: emptyUtilityReport(), post: emptyUtilityReport(), synthetic: emptyUtilityReport() },
+        warnings: [],
+        ...(metrics ? { feedbackIntegrity: metrics } : {}),
+    };
+}
+describe("renderFeedbackIntegrityTable", () => {
+    test("emits aggregate matrix + per-asset rows", () => {
+        const phase1 = {
+            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative" }),
+        ];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        const md = renderFeedbackIntegrityTable(m);
+        expect(md).toContain("Feedback-signal integrity");
+        expect(md).toContain("feedback_agreement | 1.00");
+        expect(md).toContain("feedback_coverage | 1.00");
+        expect(md).toContain("`skill:a`");
+    });
+    test("renders n/a when a per-asset rate is null", () => {
+        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] };
+        const feedbackLog = [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })];
+        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
+        const md = renderFeedbackIntegrityTable(m);
+        // Only TP — false_positive_rate denom is 0 → null → "n/a".
+        expect(md).toContain("n/a");
+    });
+    test("renders 'No feedback events recorded' when perAsset is empty", () => {
+        const m = {
+            aggregate: {
+                truePositive: 0,
+                falsePositive: 0,
+                trueNegative: 0,
+                falseNegative: 0,
+                feedback_agreement: 0,
+                false_positive_rate: 0,
+                false_negative_rate: 0,
+                feedback_coverage: 0,
+            },
+            perAsset: [],
+        };
+        expect(renderFeedbackIntegrityTable(m)).toContain("No feedback events recorded");
+    });
+});
+describe("renderEvolveReport — feedback_agreement headline + warning marker", () => {
+    test("places real feedback_agreement after improvement_slope when metrics provided", () => {
+        const metrics = computeFeedbackIntegrity({
+            phase1: { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] },
+            feedbackLog: [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })],
+        });
+        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
+        // feedback_agreement is on a line directly after improvement_slope.
+        const slopeIdx = markdown.indexOf("improvement_slope:");
+        const agreementIdx = markdown.indexOf("feedback_agreement:");
+        expect(slopeIdx).toBeGreaterThanOrEqual(0);
+        expect(agreementIdx).toBeGreaterThan(slopeIdx);
+        expect(markdown).toContain("feedback_agreement: 1.00");
+        expect(markdown).not.toContain("pending (#244)");
+        // JSON envelope carries `feedback_integrity` as a top-level key.
+        const parsed = json;
+        expect(parsed.feedback_integrity).toBeDefined();
+        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(false);
+    });
+    test("placeholder remains when metrics omitted (legacy path)", () => {
+        const { markdown, json } = renderEvolveReport(evolveInputWith(undefined));
+        expect(markdown).toContain("_feedback_agreement: pending (#244)_");
+        const parsed = json;
+        expect(parsed.feedback_integrity).toBeUndefined();
+    });
+    test("agreement < 0.80 prepends warning marker to markdown and structured warnings[]", () => {
+        // 1 TP + 4 FP → agreement = 1/5 = 0.20.
+        const phase1 = {
+            akmRuns: [
+                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 1, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 2, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 3, outcome: "fail" }),
+                fakeRun({ taskId: "t", seed: 4, outcome: "fail" }),
+            ],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 2, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 3, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 4, goldRef: "skill:a", signal: "positive" }),
+        ];
+        const metrics = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(metrics.aggregate.feedback_agreement).toBeCloseTo(0.2);
+        expect(metrics.aggregate.feedback_agreement).toBeLessThan(FEEDBACK_AGREEMENT_WARNING_THRESHOLD);
+        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
+        // Marker appears above the headline, not after it.
+        const warnIdx = markdown.indexOf("feedback_agreement = 0.20");
+        const slopeIdx = markdown.indexOf("**improvement_slope:");
+        expect(warnIdx).toBeGreaterThanOrEqual(0);
+        expect(warnIdx).toBeLessThan(slopeIdx);
+        expect(markdown).toContain("Track B headline numbers");
+        // Structured warning surfaces in the JSON envelope.
+        const parsed = json;
+        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(true);
+    });
+    test("agreement at exactly 0.80 does NOT trigger the warning marker", () => {
+        // 4 TP + 1 FP → agreement = 4/5 = 0.80 exactly.
+        const phase1 = {
+            akmRuns: [
+                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 1, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 2, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 3, outcome: "pass" }),
+                fakeRun({ taskId: "t", seed: 4, outcome: "fail" }),
+            ],
+        };
+        const feedbackLog = [
+            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 2, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 3, goldRef: "skill:a", signal: "positive" }),
+            fb({ taskId: "t", seed: 4, goldRef: "skill:a", signal: "positive" }),
+        ];
+        const metrics = computeFeedbackIntegrity({ phase1, feedbackLog });
+        expect(metrics.aggregate.feedback_agreement).toBeCloseTo(0.8);
+        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
+        expect(markdown).not.toContain("Track B headline numbers");
+        const parsed = json;
+        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(false);
+    });
+});

package/dist/tests/bench/leakage.test.js

@@ -0,0 +1,125 @@
+/**
+ * Leakage smoke test for the seeded bench corpus (spec §7.4).
+ *
+ * For every task that declares a `gold_ref` of the form `skill:<name>`,
+ * locate the SKILL.md inside the named fixture stash and assert that the
+ * verifier's *structural assertions* do not appear verbatim in the gold-ref
+ * content. The gold ref is allowed (and expected) to discuss the topic in
+ * general terms — what it must NOT do is hand the agent a copy-pasteable
+ * fragment that satisfies the verifier directly.
+ *
+ * The check extracts:
+ *   • for `regex` verifiers — the literal segments of `expected_match`
+ *     between regex meta-characters (these are the substrings the agent
+ *     must produce);
+ *   • for `pytest` verifiers — Python-style structural assertion paths and
+ *     dictionary lookups (e.g., `services.redis.healthcheck.test`,
+ *     `redis["healthcheck"]["test"]`);
+ *   • for `script` (shell) verifiers — single-quoted `grep` patterns and
+ *     `jq -e` expressions, which encode the exact assertion shape.
+ *
+ * Each fragment is checked individually. Lone tokens that legitimately
+ * appear in any reasonable description of the topic (e.g., `redis-cli`,
+ * `akm`, `bridge`, `feedback`) are filtered out by a minimum-length and
+ * minimum-token-count rule.
+ */
+import { describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { getTasksRoot, listTasks } from "./corpus";
+const STASHES_ROOT = path.resolve(getTasksRoot(), "..", "..", "stashes");
+/** Resolve `skill:<name>` against the named stash; returns SKILL.md path or `undefined`. */
+function resolveGoldRefPath(stashName, goldRef) {
+    const match = /^skill:([a-z0-9][a-z0-9-]*)$/.exec(goldRef);
+    if (!match)
+        return undefined;
+    const skillDir = path.join(STASHES_ROOT, stashName, "skills", match[1]);
+    const skillFile = path.join(skillDir, "SKILL.md");
+    return fs.existsSync(skillFile) ? skillFile : undefined;
+}
+/**
+ * Pull the literal segments out of a regex pattern. Splits on regex
+ * meta-characters and discards short fragments. The remaining strings are
+ * what the agent's stdout must contain — and therefore what the gold ref
+ * must NOT spell out verbatim.
+ */
+function regexLiterals(pattern) {
+    return pattern
+        .split(/[.*+?^${}()|[\]\\]/)
+        .map((s) => s.trim())
+        .filter((s) => s.length >= 6 && s.includes(" "));
+}
+/** Pull structural assertion fragments out of a pytest verifier file. */
+function pytestStructuralFragments(text) {
+    const out = new Set();
+    // Subscript chains like compose["services"]["redis"]["healthcheck"]["test"].
+    const subscriptRe = /(?:\["[a-z0-9_]+"\]){2,}/g;
+    for (const m of text.matchAll(subscriptRe))
+        out.add(m[0]);
+    // Dotted attribute paths used in error messages, e.g. services.redis.healthcheck.test.
+    const dottedRe = /[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}/g;
+    for (const m of text.matchAll(dottedRe))
+        out.add(m[0]);
+    return [...out];
+}
+/** Pull shell-verifier assertions: single-quoted greps and jq -e expressions. */
+function shellAssertionFragments(text) {
+    const out = new Set();
+    // grep -q '<pattern>' or grep -qi '<pattern>'.
+    const grepRe = /grep\s+-[a-zA-Z]+\s+'([^']{4,})'/g;
+    for (const m of text.matchAll(grepRe))
+        out.add(m[1]);
+    // jq -e '<expr>'.
+    const jqRe = /jq\s+-e\s+'([^']{4,})'/g;
+    for (const m of text.matchAll(jqRe))
+        out.add(m[1]);
+    return [...out];
+}
+function readVerifierFiles(task) {
+    let combined = "";
+    if (task.verifier === "pytest") {
+        const testsDir = path.join(task.taskDir, "tests");
+        if (fs.existsSync(testsDir)) {
+            for (const entry of fs.readdirSync(testsDir)) {
+                if (entry.endsWith(".py"))
+                    combined += `${fs.readFileSync(path.join(testsDir, entry), "utf8")}\n`;
+            }
+        }
+    }
+    else if (task.verifier === "script") {
+        const verify = path.join(task.taskDir, "verify.sh");
+        if (fs.existsSync(verify))
+            combined += fs.readFileSync(verify, "utf8");
+    }
+    return combined;
+}
+describe("gold-ref leakage check", () => {
+    const tasks = listTasks().filter((t) => t.goldRef);
+    test("at least one task ships with a gold_ref", () => {
+        expect(tasks.length).toBeGreaterThan(0);
+    });
+    for (const task of tasks) {
+        test(`${task.id}: verifier text does not appear in gold-ref content`, () => {
+            const goldRef = task.goldRef;
+            const goldPath = resolveGoldRefPath(task.stash, goldRef);
+            // A declared gold_ref MUST resolve to an existing fixture asset. Silent
+            // skipping here previously masked typos and stash-name drift; we now
+            // fail loudly so the corpus author is forced to fix the reference.
+            if (!goldPath) {
+                throw new Error(`${task.id}: gold_ref "${goldRef}" against stash "${task.stash}" did not resolve to a SKILL.md under tests/fixtures/stashes/. Fix the gold_ref, fix the stash name, or remove the gold_ref.`);
+            }
+            const goldContent = fs.readFileSync(goldPath, "utf8");
+            const fragments = [];
+            if (task.verifier === "regex" && task.expectedMatch) {
+                fragments.push(...regexLiterals(task.expectedMatch));
+            }
+            else {
+                const verifierText = readVerifierFiles(task);
+                fragments.push(...pytestStructuralFragments(verifierText));
+                fragments.push(...shellAssertionFragments(verifierText));
+            }
+            const leaked = fragments.filter((frag) => goldContent.includes(frag));
+            expect(leaked).toEqual([]);
+        });
+    }
+});