akm-cli 0.7.0-rc1 → 0.7.1

package/dist/tests/bench/feedback-integrity.test.js

@@ -1,456 +0,0 @@
-/**
- * Unit tests for §6.8 feedback-signal integrity (#244).
- *
- * Coverage:
- *   • All four 2×2 quadrants (TP, FP, TN, FN).
- *   • Per-asset breakdown when an asset has mixed signals across runs.
- *   • `feedback_agreement < 0.80` triggers the warning marker (markdown +
- *     structured `warnings[]` JSON entry).
- *   • `feedback_coverage` correctly counts runs with feedback dispatched
- *     vs total Phase 1 runs.
- *   • NaN-safety: zero-feedback asset emits all rates as `null`, never
- *     `0` or `NaN`.
- *   • Attribution rule (§6.8): a feedback event is attributed to the run
- *     that produced it, not to a later run touching the same asset.
- *
- * The metric is a pure function over RunResult[] + feedbackLog[]; no spawn
- * fakes are needed. We build small synthetic streams directly.
- */
-import { describe, expect, test } from "bun:test";
-import { computeFeedbackIntegrity } from "./metrics";
-import { FEEDBACK_AGREEMENT_WARNING_THRESHOLD, renderEvolveReport, renderFeedbackIntegrityTable } from "./report";
-function fakeRun(overrides) {
-    return {
-        schemaVersion: 1,
-        taskId: "t",
-        arm: "akm",
-        seed: 0,
-        model: "m",
-        outcome: "pass",
-        tokens: { input: 0, output: 0 },
-        wallclockMs: 0,
-        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
-        events: [],
-        verifierStdout: "",
-        verifierExitCode: 0,
-        assetsLoaded: [],
-        ...overrides,
-    };
-}
-function fb(overrides) {
-    return {
-        taskId: "t",
-        seed: 0,
-        goldRef: "skill:s",
-        signal: "positive",
-        ok: true,
-        ...overrides,
-    };
-}
-describe("computeFeedbackIntegrity — 2x2 quadrants", () => {
-    test("TP: feedback + on a passed run", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "pass" })] };
-        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "positive" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(1);
-        expect(m.aggregate.falsePositive).toBe(0);
-        expect(m.aggregate.trueNegative).toBe(0);
-        expect(m.aggregate.falseNegative).toBe(0);
-        expect(m.aggregate.feedback_agreement).toBeCloseTo(1);
-        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
-        expect(m.perAsset).toHaveLength(1);
-        expect(m.perAsset[0].ref).toBe("skill:a");
-        expect(m.perAsset[0].truePositive).toBe(1);
-        expect(m.perAsset[0].feedback_agreement).toBeCloseTo(1);
-    });
-    test("FP: feedback + on a failed run", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "fail" })] };
-        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "positive" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(0);
-        expect(m.aggregate.falsePositive).toBe(1);
-        expect(m.aggregate.trueNegative).toBe(0);
-        expect(m.aggregate.falseNegative).toBe(0);
-        expect(m.aggregate.feedback_agreement).toBeCloseTo(0);
-        expect(m.aggregate.false_positive_rate).toBeCloseTo(1);
-        expect(m.perAsset[0].falsePositive).toBe(1);
-    });
-    test("TN: feedback - on a failed run", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "fail" })] };
-        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "negative" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.trueNegative).toBe(1);
-        expect(m.aggregate.feedback_agreement).toBeCloseTo(1);
-        expect(m.aggregate.false_positive_rate).toBeCloseTo(0);
-        expect(m.perAsset[0].trueNegative).toBe(1);
-    });
-    test("FN: feedback - on a passed run", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t1", seed: 0, outcome: "pass" })] };
-        const feedbackLog = [fb({ taskId: "t1", seed: 0, goldRef: "skill:a", signal: "negative" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.falseNegative).toBe(1);
-        expect(m.aggregate.feedback_agreement).toBeCloseTo(0);
-        expect(m.aggregate.false_negative_rate).toBeCloseTo(1);
-        expect(m.perAsset[0].falseNegative).toBe(1);
-    });
-});
-describe("computeFeedbackIntegrity — aggregate over mixed quadrants", () => {
-    test("computes feedback_agreement and rates correctly across mixed runs", () => {
-        // 4 runs covering all four quadrants — exactly one of each.
-        const phase1 = {
-            akmRuns: [
-                fakeRun({ taskId: "tp", seed: 0, outcome: "pass" }),
-                fakeRun({ taskId: "fp", seed: 0, outcome: "fail" }),
-                fakeRun({ taskId: "tn", seed: 0, outcome: "fail" }),
-                fakeRun({ taskId: "fn", seed: 0, outcome: "pass" }),
-            ],
-        };
-        const feedbackLog = [
-            fb({ taskId: "tp", seed: 0, goldRef: "skill:tp", signal: "positive" }),
-            fb({ taskId: "fp", seed: 0, goldRef: "skill:fp", signal: "positive" }),
-            fb({ taskId: "tn", seed: 0, goldRef: "skill:tn", signal: "negative" }),
-            fb({ taskId: "fn", seed: 0, goldRef: "skill:fn", signal: "negative" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(1);
-        expect(m.aggregate.falsePositive).toBe(1);
-        expect(m.aggregate.trueNegative).toBe(1);
-        expect(m.aggregate.falseNegative).toBe(1);
-        expect(m.aggregate.feedback_agreement).toBeCloseTo(0.5); // 2/4
-        expect(m.aggregate.false_positive_rate).toBeCloseTo(0.5); // 1 / (1+1)
-        expect(m.aggregate.false_negative_rate).toBeCloseTo(0.5); // 1 / (1+1)
-        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
-        expect(m.perAsset).toHaveLength(4);
-        // Per-asset rows should be sorted by ref
-        expect(m.perAsset.map((r) => r.ref)).toEqual(["skill:fn", "skill:fp", "skill:tn", "skill:tp"]);
-    });
-});
-describe("computeFeedbackIntegrity — per-asset mixed signals", () => {
-    test("aggregates correctly when one asset appears across multiple Phase 1 runs", () => {
-        // skill:shared has 2 TP, 1 FP, 1 TN, 1 FN across 5 runs.
-        const phase1 = {
-            akmRuns: [
-                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 1, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 2, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 3, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 4, outcome: "pass" }),
-            ],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:shared", signal: "positive" }), // TP
-            fb({ taskId: "t", seed: 1, goldRef: "skill:shared", signal: "positive" }), // TP
-            fb({ taskId: "t", seed: 2, goldRef: "skill:shared", signal: "positive" }), // FP
-            fb({ taskId: "t", seed: 3, goldRef: "skill:shared", signal: "negative" }), // TN
-            fb({ taskId: "t", seed: 4, goldRef: "skill:shared", signal: "negative" }), // FN
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.perAsset).toHaveLength(1);
-        const row = m.perAsset[0];
-        expect(row.ref).toBe("skill:shared");
-        expect(row.truePositive).toBe(2);
-        expect(row.falsePositive).toBe(1);
-        expect(row.trueNegative).toBe(1);
-        expect(row.falseNegative).toBe(1);
-        expect(row.feedback_agreement).toBeCloseTo(3 / 5);
-        expect(row.false_positive_rate).toBeCloseTo(1 / 2); // FP / (FP+TN) = 1/2
-        expect(row.false_negative_rate).toBeCloseTo(1 / 3); // FN / (FN+TP) = 1/3
-    });
-});
-describe("computeFeedbackIntegrity — attribution rule", () => {
-    test("attributes feedback to the run that produced it, not a later run touching the same asset", () => {
-        // skill:contested appears across two Phase 1 runs:
-        //   run #0: passed, feedback +  → TP
-        //   run #1: failed, feedback +  → FP
-        // The naive (wrong) implementation would conflate both events with
-        // run #1's outcome and label both as FP. The correct implementation
-        // joins each event to its own (taskId, seed) → gets one TP, one FP.
-        const phase1 = {
-            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:contested", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:contested", signal: "positive" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(1);
-        expect(m.aggregate.falsePositive).toBe(1);
-        expect(m.aggregate.trueNegative).toBe(0);
-        expect(m.aggregate.falseNegative).toBe(0);
-        expect(m.perAsset[0].truePositive).toBe(1);
-        expect(m.perAsset[0].falsePositive).toBe(1);
-    });
-});
-describe("computeFeedbackIntegrity — feedback_coverage", () => {
-    test("counts runs with feedback dispatched vs total Phase 1 runs", () => {
-        // 4 phase-1 runs, only 2 had feedback dispatched.
-        const phase1 = {
-            akmRuns: [
-                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 1, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 2, outcome: "harness_error" }),
-                fakeRun({ taskId: "t", seed: 3, outcome: "budget_exceeded" }),
-            ],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.feedback_coverage).toBeCloseTo(0.5); // 2 of 4
-    });
-    test("zero coverage when no feedback dispatched", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] };
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog: [] });
-        expect(m.aggregate.feedback_coverage).toBe(0);
-        expect(m.aggregate.feedback_agreement).toBe(0);
-        expect(m.perAsset).toEqual([]);
-    });
-    test("zero coverage and zero runs returns 0 (not NaN)", () => {
-        const m = computeFeedbackIntegrity({ phase1: { akmRuns: [] }, feedbackLog: [] });
-        expect(m.aggregate.feedback_coverage).toBe(0);
-        expect(m.aggregate.feedback_agreement).toBe(0);
-        expect(m.aggregate.false_positive_rate).toBe(0);
-        expect(m.aggregate.false_negative_rate).toBe(0);
-        expect(Number.isFinite(m.aggregate.feedback_coverage)).toBe(true);
-        expect(Number.isFinite(m.aggregate.feedback_agreement)).toBe(true);
-    });
-});
-describe("computeFeedbackIntegrity — NaN safety", () => {
-    test("per-asset row with FP+TN === 0 emits null false_positive_rate (only positive feedback on passes)", () => {
-        const phase1 = {
-            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "pass" })],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:only-tp", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:only-tp", signal: "positive" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        const row = m.perAsset[0];
-        expect(row.feedback_agreement).toBeCloseTo(1);
-        expect(row.false_positive_rate).toBeNull(); // FP+TN === 0
-        expect(row.false_negative_rate).toBeCloseTo(0); // FN/(FN+TP) = 0/2 = 0
-    });
-    test("per-asset row with FN+TP === 0 emits null false_negative_rate (only negative feedback on fails)", () => {
-        const phase1 = {
-            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "fail" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:only-tn", signal: "negative" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:only-tn", signal: "negative" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        const row = m.perAsset[0];
-        expect(row.feedback_agreement).toBeCloseTo(1);
-        expect(row.false_negative_rate).toBeNull(); // FN+TP === 0
-        expect(row.false_positive_rate).toBeCloseTo(0); // FP/(FP+TN) = 0/2 = 0
-    });
-    test("ok=false feedback events are excluded from the matrix but still count toward coverage", () => {
-        const phase1 = {
-            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive", ok: true }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative", ok: false }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        // Only the ok=true entry contributes to the matrix (TP=1).
-        expect(m.aggregate.truePositive).toBe(1);
-        expect(m.aggregate.trueNegative).toBe(0);
-        // But coverage counts both attempts.
-        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
-    });
-    test("harness_error runs are excluded from the matrix even with a stamped feedback event", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "harness_error" })] };
-        const feedbackLog = [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(0);
-        expect(m.aggregate.falsePositive).toBe(0);
-        expect(m.perAsset).toEqual([]);
-    });
-    test("feedback for a run not present in akmRuns is silently dropped", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "real", seed: 0, outcome: "pass" })] };
-        const feedbackLog = [fb({ taskId: "ghost", seed: 99, goldRef: "skill:a", signal: "positive" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(m.aggregate.truePositive).toBe(0);
-        expect(m.perAsset).toEqual([]);
-        // Coverage still records the dispatch attempt — operator wanted feedback.
-        expect(m.aggregate.feedback_coverage).toBeCloseTo(1);
-    });
-});
-// ── Render-side coverage ───────────────────────────────────────────────────
-function emptyUtilityReport() {
-    // Build a minimal §13.3-shaped utility report. The renderer reads
-    // many subfields; we stub them to safe zeros.
-    return {
-        timestamp: "2026-04-27T00:00:00Z",
-        branch: "test",
-        commit: "deadbee",
-        model: "m",
-        corpus: { domains: 0, tasks: 0, slice: "all", seedsPerArm: 1 },
-        aggregateNoakm: { passRate: 0, tokensPerPass: 0, wallclockMs: 0 },
-        aggregateAkm: { passRate: 0, tokensPerPass: 0, wallclockMs: 0 },
-        aggregateDelta: {
-            passRate: 0,
-            tokensPerPass: 0,
-            wallclockMs: 0,
-        },
-        trajectoryAkm: {
-            correctAssetLoaded: null,
-            feedbackRecorded: 0,
-        },
-        failureModes: { byLabel: {}, byTask: {} },
-        tasks: [],
-        warnings: [],
-        akmRuns: [],
-        taskMetadata: [],
-        goldRankRecords: [],
-    };
-}
-function evolveInputWith(metrics) {
-    return {
-        timestamp: "2026-04-27T00:00:00Z",
-        branch: "test",
-        commit: "deadbee",
-        model: "m",
-        domain: "test",
-        seedsPerArm: 1,
-        proposals: { rows: [], totalProposals: 0, totalAccepted: 0, acceptanceRate: 0, lintPassRate: 0 },
-        longitudinal: {
-            improvementSlope: 0.1,
-            overSyntheticLift: 0.05,
-            degradationCount: 0,
-            degradations: [],
-            prePassRate: 0.5,
-            postPassRate: 0.6,
-            syntheticPassRate: 0.55,
-        },
-        arms: { pre: emptyUtilityReport(), post: emptyUtilityReport(), synthetic: emptyUtilityReport() },
-        warnings: [],
-        ...(metrics ? { feedbackIntegrity: metrics } : {}),
-    };
-}
-describe("renderFeedbackIntegrityTable", () => {
-    test("emits aggregate matrix + per-asset rows", () => {
-        const phase1 = {
-            akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" }), fakeRun({ taskId: "t", seed: 1, outcome: "fail" })],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "negative" }),
-        ];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        const md = renderFeedbackIntegrityTable(m);
-        expect(md).toContain("Feedback-signal integrity");
-        expect(md).toContain("feedback_agreement | 1.00");
-        expect(md).toContain("feedback_coverage | 1.00");
-        expect(md).toContain("`skill:a`");
-    });
-    test("renders n/a when a per-asset rate is null", () => {
-        const phase1 = { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] };
-        const feedbackLog = [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })];
-        const m = computeFeedbackIntegrity({ phase1, feedbackLog });
-        const md = renderFeedbackIntegrityTable(m);
-        // Only TP — false_positive_rate denom is 0 → null → "n/a".
-        expect(md).toContain("n/a");
-    });
-    test("renders 'No feedback events recorded' when perAsset is empty", () => {
-        const m = {
-            aggregate: {
-                truePositive: 0,
-                falsePositive: 0,
-                trueNegative: 0,
-                falseNegative: 0,
-                feedback_agreement: 0,
-                false_positive_rate: 0,
-                false_negative_rate: 0,
-                feedback_coverage: 0,
-            },
-            perAsset: [],
-        };
-        expect(renderFeedbackIntegrityTable(m)).toContain("No feedback events recorded");
-    });
-});
-describe("renderEvolveReport — feedback_agreement headline + warning marker", () => {
-    test("places real feedback_agreement after improvement_slope when metrics provided", () => {
-        const metrics = computeFeedbackIntegrity({
-            phase1: { akmRuns: [fakeRun({ taskId: "t", seed: 0, outcome: "pass" })] },
-            feedbackLog: [fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" })],
-        });
-        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
-        // feedback_agreement is on a line directly after improvement_slope.
-        const slopeIdx = markdown.indexOf("improvement_slope:");
-        const agreementIdx = markdown.indexOf("feedback_agreement:");
-        expect(slopeIdx).toBeGreaterThanOrEqual(0);
-        expect(agreementIdx).toBeGreaterThan(slopeIdx);
-        expect(markdown).toContain("feedback_agreement: 1.00");
-        expect(markdown).not.toContain("pending (#244)");
-        // JSON envelope carries `feedback_integrity` as a top-level key.
-        const parsed = json;
-        expect(parsed.feedback_integrity).toBeDefined();
-        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(false);
-    });
-    test("placeholder remains when metrics omitted (legacy path)", () => {
-        const { markdown, json } = renderEvolveReport(evolveInputWith(undefined));
-        expect(markdown).toContain("_feedback_agreement: pending (#244)_");
-        const parsed = json;
-        expect(parsed.feedback_integrity).toBeUndefined();
-    });
-    test("agreement < 0.80 prepends warning marker to markdown and structured warnings[]", () => {
-        // 1 TP + 4 FP → agreement = 1/5 = 0.20.
-        const phase1 = {
-            akmRuns: [
-                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 1, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 2, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 3, outcome: "fail" }),
-                fakeRun({ taskId: "t", seed: 4, outcome: "fail" }),
-            ],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 2, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 3, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 4, goldRef: "skill:a", signal: "positive" }),
-        ];
-        const metrics = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(metrics.aggregate.feedback_agreement).toBeCloseTo(0.2);
-        expect(metrics.aggregate.feedback_agreement).toBeLessThan(FEEDBACK_AGREEMENT_WARNING_THRESHOLD);
-        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
-        // Marker appears above the headline, not after it.
-        const warnIdx = markdown.indexOf("feedback_agreement = 0.20");
-        const slopeIdx = markdown.indexOf("**improvement_slope:");
-        expect(warnIdx).toBeGreaterThanOrEqual(0);
-        expect(warnIdx).toBeLessThan(slopeIdx);
-        expect(markdown).toContain("Track B headline numbers");
-        // Structured warning surfaces in the JSON envelope.
-        const parsed = json;
-        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(true);
-    });
-    test("agreement at exactly 0.80 does NOT trigger the warning marker", () => {
-        // 4 TP + 1 FP → agreement = 4/5 = 0.80 exactly.
-        const phase1 = {
-            akmRuns: [
-                fakeRun({ taskId: "t", seed: 0, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 1, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 2, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 3, outcome: "pass" }),
-                fakeRun({ taskId: "t", seed: 4, outcome: "fail" }),
-            ],
-        };
-        const feedbackLog = [
-            fb({ taskId: "t", seed: 0, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 1, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 2, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 3, goldRef: "skill:a", signal: "positive" }),
-            fb({ taskId: "t", seed: 4, goldRef: "skill:a", signal: "positive" }),
-        ];
-        const metrics = computeFeedbackIntegrity({ phase1, feedbackLog });
-        expect(metrics.aggregate.feedback_agreement).toBeCloseTo(0.8);
-        const { markdown, json } = renderEvolveReport(evolveInputWith(metrics));
-        expect(markdown).not.toContain("Track B headline numbers");
-        const parsed = json;
-        expect(parsed.warnings.some((w) => w.startsWith("feedback_agreement_below_threshold"))).toBe(false);
-    });
-});

package/dist/tests/bench/leakage.test.js

@@ -1,125 +0,0 @@
-/**
- * Leakage smoke test for the seeded bench corpus (spec §7.4).
- *
- * For every task that declares a `gold_ref` of the form `skill:<name>`,
- * locate the SKILL.md inside the named fixture stash and assert that the
- * verifier's *structural assertions* do not appear verbatim in the gold-ref
- * content. The gold ref is allowed (and expected) to discuss the topic in
- * general terms — what it must NOT do is hand the agent a copy-pasteable
- * fragment that satisfies the verifier directly.
- *
- * The check extracts:
- *   • for `regex` verifiers — the literal segments of `expected_match`
- *     between regex meta-characters (these are the substrings the agent
- *     must produce);
- *   • for `pytest` verifiers — Python-style structural assertion paths and
- *     dictionary lookups (e.g., `services.redis.healthcheck.test`,
- *     `redis["healthcheck"]["test"]`);
- *   • for `script` (shell) verifiers — single-quoted `grep` patterns and
- *     `jq -e` expressions, which encode the exact assertion shape.
- *
- * Each fragment is checked individually. Lone tokens that legitimately
- * appear in any reasonable description of the topic (e.g., `redis-cli`,
- * `akm`, `bridge`, `feedback`) are filtered out by a minimum-length and
- * minimum-token-count rule.
- */
-import { describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import path from "node:path";
-import { getTasksRoot, listTasks } from "./corpus";
-const STASHES_ROOT = path.resolve(getTasksRoot(), "..", "..", "stashes");
-/** Resolve `skill:<name>` against the named stash; returns SKILL.md path or `undefined`. */
-function resolveGoldRefPath(stashName, goldRef) {
-    const match = /^skill:([a-z0-9][a-z0-9-]*)$/.exec(goldRef);
-    if (!match)
-        return undefined;
-    const skillDir = path.join(STASHES_ROOT, stashName, "skills", match[1]);
-    const skillFile = path.join(skillDir, "SKILL.md");
-    return fs.existsSync(skillFile) ? skillFile : undefined;
-}
-/**
- * Pull the literal segments out of a regex pattern. Splits on regex
- * meta-characters and discards short fragments. The remaining strings are
- * what the agent's stdout must contain — and therefore what the gold ref
- * must NOT spell out verbatim.
- */
-function regexLiterals(pattern) {
-    return pattern
-        .split(/[.*+?^${}()|[\]\\]/)
-        .map((s) => s.trim())
-        .filter((s) => s.length >= 6 && s.includes(" "));
-}
-/** Pull structural assertion fragments out of a pytest verifier file. */
-function pytestStructuralFragments(text) {
-    const out = new Set();
-    // Subscript chains like compose["services"]["redis"]["healthcheck"]["test"].
-    const subscriptRe = /(?:\["[a-z0-9_]+"\]){2,}/g;
-    for (const m of text.matchAll(subscriptRe))
-        out.add(m[0]);
-    // Dotted attribute paths used in error messages, e.g. services.redis.healthcheck.test.
-    const dottedRe = /[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}/g;
-    for (const m of text.matchAll(dottedRe))
-        out.add(m[0]);
-    return [...out];
-}
-/** Pull shell-verifier assertions: single-quoted greps and jq -e expressions. */
-function shellAssertionFragments(text) {
-    const out = new Set();
-    // grep -q '<pattern>' or grep -qi '<pattern>'.
-    const grepRe = /grep\s+-[a-zA-Z]+\s+'([^']{4,})'/g;
-    for (const m of text.matchAll(grepRe))
-        out.add(m[1]);
-    // jq -e '<expr>'.
-    const jqRe = /jq\s+-e\s+'([^']{4,})'/g;
-    for (const m of text.matchAll(jqRe))
-        out.add(m[1]);
-    return [...out];
-}
-function readVerifierFiles(task) {
-    let combined = "";
-    if (task.verifier === "pytest") {
-        const testsDir = path.join(task.taskDir, "tests");
-        if (fs.existsSync(testsDir)) {
-            for (const entry of fs.readdirSync(testsDir)) {
-                if (entry.endsWith(".py"))
-                    combined += `${fs.readFileSync(path.join(testsDir, entry), "utf8")}\n`;
-            }
-        }
-    }
-    else if (task.verifier === "script") {
-        const verify = path.join(task.taskDir, "verify.sh");
-        if (fs.existsSync(verify))
-            combined += fs.readFileSync(verify, "utf8");
-    }
-    return combined;
-}
-describe("gold-ref leakage check", () => {
-    const tasks = listTasks().filter((t) => t.goldRef);
-    test("at least one task ships with a gold_ref", () => {
-        expect(tasks.length).toBeGreaterThan(0);
-    });
-    for (const task of tasks) {
-        test(`${task.id}: verifier text does not appear in gold-ref content`, () => {
-            const goldRef = task.goldRef;
-            const goldPath = resolveGoldRefPath(task.stash, goldRef);
-            // A declared gold_ref MUST resolve to an existing fixture asset. Silent
-            // skipping here previously masked typos and stash-name drift; we now
-            // fail loudly so the corpus author is forced to fix the reference.
-            if (!goldPath) {
-                throw new Error(`${task.id}: gold_ref "${goldRef}" against stash "${task.stash}" did not resolve to a SKILL.md under tests/fixtures/stashes/. Fix the gold_ref, fix the stash name, or remove the gold_ref.`);
-            }
-            const goldContent = fs.readFileSync(goldPath, "utf8");
-            const fragments = [];
-            if (task.verifier === "regex" && task.expectedMatch) {
-                fragments.push(...regexLiterals(task.expectedMatch));
-            }
-            else {
-                const verifierText = readVerifierFiles(task);
-                fragments.push(...pytestStructuralFragments(verifierText));
-                fragments.push(...shellAssertionFragments(verifierText));
-            }
-            const leaked = fragments.filter((frag) => goldContent.includes(frag));
-            expect(leaked).toEqual([]);
-        });
-    }
-});