akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/compare.test.js

@@ -0,0 +1,556 @@
+/**
+ * Unit tests for the `bench compare` subcommand (#239).
+ *
+ * Covers:
+ *   • happy-path comparison: deltas + sign markers correct.
+ *   • model-mismatch refusal: both models named in the message.
+ *   • missing fixture-content hash on either side: proceeds with a warning.
+ *   • markdown output is byte-stable across two calls with identical input.
+ *   • CLI driver: invalid input file (missing path / malformed JSON) → exit 2.
+ *   • CLI driver: refusal → exit 1; success → exit 0.
+ */
+import { describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { runCompareCli } from "./cli";
+import { compareReports } from "./metrics";
+import { renderCompareMarkdown } from "./report";
+import { benchMkdtemp } from "./tmp";
+const MODEL = "anthropic/claude-opus-4-7";
+function makeReport(overrides = {}) {
+    return {
+        schemaVersion: 1,
+        track: "utility",
+        branch: "release/0.7.0",
+        commit: "deadbee",
+        timestamp: "2026-04-27T12:00:00Z",
+        agent: { harness: "opencode", model: MODEL },
+        corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5 },
+        aggregate: {
+            noakm: { pass_rate: 0.4, tokens_per_pass: 18000, wallclock_ms: 41000 },
+            akm: { pass_rate: 0.6, tokens_per_pass: 14000, wallclock_ms: 36000 },
+            delta: { pass_rate: 0.2, tokens_per_pass: -4000, wallclock_ms: -5000 },
+        },
+        tasks: [
+            {
+                id: "domain-a/task-1",
+                akm: {
+                    pass_rate: 0.6,
+                    pass_at_1: 1,
+                    tokens_per_pass: 13000,
+                    wallclock_ms: 35000,
+                    pass_rate_stdev: 0.1,
+                    budget_exceeded_count: 0,
+                    harness_error_count: 0,
+                    count: 5,
+                },
+            },
+            {
+                id: "domain-b/task-2",
+                akm: {
+                    pass_rate: 0.6,
+                    pass_at_1: 1,
+                    tokens_per_pass: 15000,
+                    wallclock_ms: 37000,
+                    pass_rate_stdev: 0.2,
+                    budget_exceeded_count: 0,
+                    harness_error_count: 0,
+                    count: 5,
+                },
+            },
+        ],
+        warnings: [],
+        ...overrides,
+    };
+}
+describe("compareReports — happy path", () => {
+    test("computes aggregate delta with correct sign markers", () => {
+        const base = makeReport();
+        // Current improves pass_rate by +0.2, reduces tokens by 1000, slower by 1000ms.
+        const current = makeReport({
+            aggregate: {
+                noakm: { pass_rate: 0.4, tokens_per_pass: 18000, wallclock_ms: 41000 },
+                akm: { pass_rate: 0.8, tokens_per_pass: 13000, wallclock_ms: 37000 },
+                delta: { pass_rate: 0.4, tokens_per_pass: -5000, wallclock_ms: -4000 },
+            },
+        });
+        const result = compareReports(base, current);
+        if (!result.ok)
+            throw new Error("expected ok=true");
+        expect(result.aggregate.passRateDelta).toBeCloseTo(0.2);
+        expect(result.aggregate.passRateSign).toBe("improve");
+        expect(result.aggregate.tokensPerPassDelta).toBeCloseTo(-1000);
+        expect(result.aggregate.tokensPerPassSign).toBe("improve"); // lower tokens = better
+        expect(result.aggregate.wallclockMsDelta).toBeCloseTo(1000);
+        expect(result.aggregate.wallclockMsSign).toBe("regress"); // higher wallclock = worse
+        expect(result.perTask.length).toBe(2);
+    });
+    test("flat sign for tiny pass-rate jitter", () => {
+        const base = makeReport();
+        const current = makeReport({
+            aggregate: {
+                noakm: { pass_rate: 0.4, tokens_per_pass: 18000, wallclock_ms: 41000 },
+                akm: { pass_rate: 0.602, tokens_per_pass: 14000, wallclock_ms: 36000 },
+                delta: { pass_rate: 0.202, tokens_per_pass: -4000, wallclock_ms: -5000 },
+            },
+        });
+        const result = compareReports(base, current);
+        if (!result.ok)
+            throw new Error("expected ok=true");
+        // 0.602 − 0.6 = 0.002 < 0.005 tolerance
+        expect(result.aggregate.passRateSign).toBe("flat");
+    });
+    test("per-task row carries baseMetrics + currentMetrics + signMarker", () => {
+        const base = makeReport();
+        const current = makeReport({
+            tasks: [
+                {
+                    id: "domain-a/task-1",
+                    akm: {
+                        pass_rate: 0.8,
+                        pass_at_1: 1,
+                        tokens_per_pass: 12000,
+                        wallclock_ms: 33000,
+                        pass_rate_stdev: 0.05,
+                        budget_exceeded_count: 0,
+                        harness_error_count: 0,
+                        count: 5,
+                    },
+                },
+                {
+                    id: "domain-b/task-2",
+                    akm: {
+                        pass_rate: 0.4,
+                        pass_at_1: 0,
+                        tokens_per_pass: 16000,
+                        wallclock_ms: 38000,
+                        pass_rate_stdev: 0.3,
+                        budget_exceeded_count: 1,
+                        harness_error_count: 0,
+                        count: 5,
+                    },
+                },
+            ],
+        });
+        const result = compareReports(base, current);
+        if (!result.ok)
+            throw new Error("expected ok=true");
+        const row1 = result.perTask.find((r) => r.id === "domain-a/task-1");
+        const row2 = result.perTask.find((r) => r.id === "domain-b/task-2");
+        expect(row1?.signMarker).toBe("improve");
+        expect(row1?.delta.passRate).toBeCloseTo(0.2);
+        expect(row1?.baseMetrics?.pass_rate_stdev).toBeCloseTo(0.1);
+        expect(row1?.currentMetrics?.pass_rate_stdev).toBeCloseTo(0.05);
+        expect(row2?.signMarker).toBe("regress");
+        expect(row2?.delta.passRate).toBeCloseTo(-0.2);
+    });
+});
+describe("compareReports — refusal cases", () => {
+    test("model mismatch: ok=false with both models named", () => {
+        const base = makeReport();
+        const current = makeReport({ agent: { harness: "opencode", model: "anthropic/claude-sonnet-4-5" } });
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("model_mismatch");
+        expect(result.baseModel).toBe(MODEL);
+        expect(result.currentModel).toBe("anthropic/claude-sonnet-4-5");
+        expect(result.message).toContain(MODEL);
+        expect(result.message).toContain("anthropic/claude-sonnet-4-5");
+    });
+    test("schema mismatch: refuses non-v1 envelopes", () => {
+        const base = makeReport({ schemaVersion: 2 });
+        const current = makeReport();
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("schema_mismatch");
+    });
+    test("track mismatch: refuses non-utility tracks", () => {
+        const base = makeReport({ track: "evolve" });
+        const current = makeReport();
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("track_mismatch");
+    });
+    test("hash mismatch: refuses with both hashes named", () => {
+        const base = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "abc123" },
+        });
+        const current = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "def456" },
+        });
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("hash_mismatch");
+        expect(result.message).toContain("abc123");
+        expect(result.message).toContain("def456");
+    });
+});
+describe("compareReports — fixture-hash warnings", () => {
+    test("missing hash on base: proceeds with warning", () => {
+        const base = makeReport(); // no fixtureContentHash
+        const current = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "abc123" },
+        });
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.some((w) => w.includes("base") && w.includes("fixtureContentHash"))).toBe(true);
+    });
+    test("missing hash on current: proceeds with warning", () => {
+        const base = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "abc123" },
+        });
+        const current = makeReport(); // no fixtureContentHash
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.some((w) => w.includes("current") && w.includes("fixtureContentHash"))).toBe(true);
+    });
+    test("missing on both: two fixture warnings (#250 also adds two corpus warnings)", () => {
+        const base = makeReport();
+        const current = makeReport();
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.filter((w) => w.includes("fixtureContentHash")).length).toBe(2);
+    });
+    test("matching fixture hash: no fixture warnings", () => {
+        const base = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "abc123" },
+        });
+        const current = makeReport({
+            corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "abc123" },
+        });
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.filter((w) => w.includes("fixtureContentHash")).length).toBe(0);
+    });
+});
+describe("compareReports — corpus identity (#250)", () => {
+    function withCorpusIdentity(taskCorpusHash, selectedTaskIds, fixtureContentHash) {
+        return makeReport({
+            corpus: {
+                domains: 2,
+                tasks: selectedTaskIds.length,
+                slice: "all",
+                seedsPerArm: 5,
+                taskCorpusHash,
+                selectedTaskIds,
+                ...(fixtureContentHash ? { fixtureContentHash } : {}),
+            },
+        });
+    }
+    test("matching corpus + fixture hashes: ok=true, no warnings", () => {
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-a");
+        const current = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-a");
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.length).toBe(0);
+    });
+    test("taskCorpusHash mismatch: refuses by default", () => {
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"]);
+        const current = withCorpusIdentity("tc-b", ["a/one", "b/two"]);
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("corpus_mismatch");
+        expect(result.message).toContain("tc-a");
+        expect(result.message).toContain("tc-b");
+        expect(result.baseTaskCorpusHash).toBe("tc-a");
+        expect(result.currentTaskCorpusHash).toBe("tc-b");
+    });
+    test("selectedTaskIds differ but hashes both present and matching: still ok", () => {
+        // Defensive: in practice two reports with identical taskCorpusHash should
+        // also share IDs; but if a producer ever forgets to align them, the hash
+        // dominates so we don't false-positive.
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"]);
+        const current = withCorpusIdentity("tc-a", ["a/one", "b/two", "c/three"]);
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+    });
+    test("allowCorpusMismatch converts refusal to warning", () => {
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"]);
+        const current = withCorpusIdentity("tc-b", ["a/one", "b/two"]);
+        const result = compareReports(base, current, { allowCorpusMismatch: true });
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.some((w) => w.includes("--allow-corpus-mismatch"))).toBe(true);
+    });
+    test("legacy report (missing taskCorpusHash) gets a warning, not refusal", () => {
+        const base = makeReport(); // no taskCorpusHash
+        const current = withCorpusIdentity("tc-a", ["a/one", "b/two"]);
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.some((w) => w.includes("base") && w.includes("taskCorpusHash"))).toBe(true);
+    });
+    test("legacy on both sides: two warnings, still ok", () => {
+        const base = makeReport();
+        const current = makeReport();
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        // 2 missing taskCorpusHash + 2 missing fixtureContentHash warnings.
+        expect(result.warnings.filter((w) => w.includes("taskCorpusHash")).length).toBe(2);
+        expect(result.warnings.filter((w) => w.includes("fixtureContentHash")).length).toBe(2);
+    });
+    test("fixture-content hash mismatch: refuses by default (existing behaviour)", () => {
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-a");
+        const current = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-b");
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("hash_mismatch");
+    });
+    test("allowFixtureMismatch converts fixture-hash refusal to warning", () => {
+        const base = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-a");
+        const current = withCorpusIdentity("tc-a", ["a/one", "b/two"], "fh-b");
+        const result = compareReports(base, current, { allowFixtureMismatch: true });
+        expect(result.ok).toBe(true);
+        if (!result.ok)
+            return;
+        expect(result.warnings.some((w) => w.includes("--allow-fixture-mismatch"))).toBe(true);
+    });
+    test("corpus mismatch is checked before fixture mismatch (refusal precedence)", () => {
+        // When both differ and neither flag is set, the corpus refusal wins.
+        const base = withCorpusIdentity("tc-a", ["a/one"], "fh-a");
+        const current = withCorpusIdentity("tc-b", ["a/one"], "fh-b");
+        const result = compareReports(base, current);
+        expect(result.ok).toBe(false);
+        if (result.ok)
+            return;
+        expect(result.reason).toBe("corpus_mismatch");
+    });
+});
+describe("renderCompareMarkdown determinism", () => {
+    test("byte-stable across two calls with identical input", () => {
+        const base = makeReport();
+        const current = makeReport();
+        const r1 = compareReports(base, current);
+        const r2 = compareReports(base, current);
+        expect(renderCompareMarkdown(r1)).toBe(renderCompareMarkdown(r2));
+    });
+    test("contains aggregate header and per-task table", () => {
+        const base = makeReport();
+        const current = makeReport({
+            aggregate: {
+                noakm: { pass_rate: 0.4, tokens_per_pass: 18000, wallclock_ms: 41000 },
+                akm: { pass_rate: 0.8, tokens_per_pass: 13000, wallclock_ms: 36000 },
+                delta: { pass_rate: 0.4, tokens_per_pass: -5000, wallclock_ms: -5000 },
+            },
+        });
+        const md = renderCompareMarkdown(compareReports(base, current));
+        expect(md).toContain("# akm-bench compare");
+        expect(md).toContain("## Aggregate");
+        expect(md).toContain("## Per-task");
+        expect(md).toContain("pass_rate");
+        expect(md).toContain("+0.20"); // pass_rate delta
+        expect(md).toContain("▲"); // improve glyph
+    });
+    test("refusal renders as a single error block, not a diff table", () => {
+        const base = makeReport();
+        const current = makeReport({ agent: { harness: "opencode", model: "anthropic/claude-sonnet-4-5" } });
+        const md = renderCompareMarkdown(compareReports(base, current));
+        expect(md).toContain("refused");
+        expect(md).toContain("model_mismatch");
+        expect(md).toContain(MODEL);
+        expect(md).toContain("anthropic/claude-sonnet-4-5");
+        expect(md).not.toContain("## Aggregate"); // no diff body
+    });
+});
+// ── CLI driver ────────────────────────────────────────────────────────────
+describe("runCompareCli", () => {
+    function withTmpFiles(cb, base, current) {
+        const tmp = benchMkdtemp("bench-compare-");
+        try {
+            const basePath = path.join(tmp, "base.json");
+            const currentPath = path.join(tmp, "current.json");
+            fs.writeFileSync(basePath, JSON.stringify(base ?? makeReport()));
+            fs.writeFileSync(currentPath, JSON.stringify(current ?? makeReport()));
+            cb({ basePath, currentPath, tmp });
+        }
+        finally {
+            fs.rmSync(tmp, { recursive: true, force: true });
+        }
+    }
+    test("happy path: exit 0, markdown to stdout", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: false });
+            expect(result.exitCode).toBe(0);
+            expect(result.stdout).toContain("# akm-bench compare");
+            expect(result.stderr).toContain("pass_rate");
+        });
+    });
+    test("happy path with --json: exit 0, structured JSON to stdout", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: true });
+            expect(result.exitCode).toBe(0);
+            const parsed = JSON.parse(result.stdout);
+            expect(parsed.ok).toBe(true);
+        });
+    });
+    test("model mismatch: exit 1 + clear stderr", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: false });
+            expect(result.exitCode).toBe(1);
+            expect(result.stderr).toContain("different models");
+        }, makeReport(), makeReport({ agent: { harness: "opencode", model: "anthropic/claude-sonnet-4-5" } }));
+    });
+    test("hash mismatch: exit 1", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: false });
+            expect(result.exitCode).toBe(1);
+            expect(result.stderr).toContain("fixture-content");
+        }, makeReport({ corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "h1" } }), makeReport({ corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "h2" } }));
+    });
+    test("malformed JSON in --base: exit 2", () => {
+        const tmp = benchMkdtemp("bench-compare-bad-");
+        try {
+            const basePath = path.join(tmp, "base.json");
+            const currentPath = path.join(tmp, "current.json");
+            fs.writeFileSync(basePath, "{ not valid json");
+            fs.writeFileSync(currentPath, JSON.stringify(makeReport()));
+            const result = runCompareCli({ basePath, currentPath, json: false });
+            expect(result.exitCode).toBe(2);
+            expect(result.stderr).toContain("malformed JSON");
+        }
+        finally {
+            fs.rmSync(tmp, { recursive: true, force: true });
+        }
+    });
+    test("round-trip: reports with persisted runs[] (#249) compare cleanly", () => {
+        // The runs[] field is additive — compare ignores it but must NOT reject
+        // reports that carry it. Confirms the new key is forward-compatible with
+        // the existing aggregate-based diff path.
+        const baseWithRuns = {
+            ...makeReport(),
+            runs: [
+                {
+                    task_id: "domain-a/task-1",
+                    arm: "akm",
+                    seed: 0,
+                    model: MODEL,
+                    outcome: "pass",
+                    tokens: { input: 1, output: 2 },
+                    wallclock_ms: 100,
+                    verifier_exit_code: 0,
+                    trajectory: { correct_asset_loaded: true, feedback_recorded: false },
+                    assets_loaded: ["skill:foo"],
+                    failure_mode: null,
+                },
+            ],
+        };
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: true });
+            expect(result.exitCode).toBe(0);
+            const parsed = JSON.parse(result.stdout);
+            expect(parsed.ok).toBe(true);
+        }, baseWithRuns, baseWithRuns);
+    });
+    test("corpus mismatch: exit 1 (#250)", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({ basePath, currentPath, json: false });
+            expect(result.exitCode).toBe(1);
+            expect(result.stderr).toContain("task corpora");
+        }, makeReport({
+            corpus: {
+                domains: 2,
+                tasks: 2,
+                slice: "all",
+                seedsPerArm: 5,
+                taskCorpusHash: "tc1",
+                selectedTaskIds: ["a/one", "b/two"],
+            },
+        }), makeReport({
+            corpus: {
+                domains: 2,
+                tasks: 2,
+                slice: "all",
+                seedsPerArm: 5,
+                taskCorpusHash: "tc2",
+                selectedTaskIds: ["a/one", "b/two"],
+            },
+        }));
+    });
+    test("corpus mismatch with --allow-corpus-mismatch: exit 0 + warning (#250)", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({
+                basePath,
+                currentPath,
+                json: false,
+                allowCorpusMismatch: true,
+            });
+            expect(result.exitCode).toBe(0);
+            expect(result.stderr).toContain("warning");
+            expect(result.stderr).toContain("task corpus");
+        }, makeReport({
+            corpus: {
+                domains: 2,
+                tasks: 2,
+                slice: "all",
+                seedsPerArm: 5,
+                taskCorpusHash: "tc1",
+                selectedTaskIds: ["a/one", "b/two"],
+            },
+        }), makeReport({
+            corpus: {
+                domains: 2,
+                tasks: 2,
+                slice: "all",
+                seedsPerArm: 5,
+                taskCorpusHash: "tc2",
+                selectedTaskIds: ["a/one", "b/two"],
+            },
+        }));
+    });
+    test("fixture mismatch with --allow-fixture-mismatch: exit 0 + warning (#250)", () => {
+        withTmpFiles(({ basePath, currentPath }) => {
+            const result = runCompareCli({
+                basePath,
+                currentPath,
+                json: false,
+                allowFixtureMismatch: true,
+            });
+            expect(result.exitCode).toBe(0);
+            expect(result.stderr).toContain("warning");
+            expect(result.stderr).toContain("fixture-content");
+        }, makeReport({ corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "h1" } }), makeReport({ corpus: { domains: 2, tasks: 2, slice: "all", seedsPerArm: 5, fixtureContentHash: "h2" } }));
+    });
+    test("missing --base file: exit 2", () => {
+        const tmp = benchMkdtemp("bench-compare-missing-");
+        try {
+            const currentPath = path.join(tmp, "current.json");
+            fs.writeFileSync(currentPath, JSON.stringify(makeReport()));
+            const result = runCompareCli({
+                basePath: path.join(tmp, "nope.json"),
+                currentPath,
+                json: false,
+            });
+            expect(result.exitCode).toBe(2);
+            expect(result.stderr).toContain("cannot read --base");
+        }
+        finally {
+            fs.rmSync(tmp, { recursive: true, force: true });
+        }
+    });
+});