akm-cli 0.7.0 → 0.7.1

package/dist/tests/bench/failure-modes.test.js

@@ -1,349 +0,0 @@
-/**
- * Unit tests for the §6.6 failure-mode taxonomy classifier.
- *
- * The classifier is a pure function over (TaskMetadata, RunResult). We
- * exercise each of the seven labels with a synthetic event stream / verifier
- * stdout, verify priority/tie-breaking, and assert the report renderer
- * orders bins by descending count.
- */
-import { describe, expect, test } from "bun:test";
-import { aggregateFailureModes, classifyFailureMode } from "./metrics";
-import { renderFailureModeBreakdown } from "./report";
-function fakeTask(overrides = {}) {
-    return {
-        id: "domain-a/task-1",
-        title: "fake",
-        domain: "domain-a",
-        difficulty: "easy",
-        stash: "fake-stash",
-        verifier: "regex",
-        budget: { tokens: 1000, wallMs: 60000 },
-        taskDir: "/tmp/fake",
-        goldRef: "skill:docker-homelab",
-        ...overrides,
-    };
-}
-function fakeRun(overrides = {}) {
-    return {
-        schemaVersion: 1,
-        taskId: "domain-a/task-1",
-        arm: "akm",
-        seed: 0,
-        model: "m",
-        outcome: "fail",
-        tokens: { input: 0, output: 0 },
-        wallclockMs: 0,
-        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
-        events: [],
-        verifierStdout: "",
-        verifierExitCode: 1,
-        assetsLoaded: [],
-        ...overrides,
-    };
-}
-describe("classifyFailureMode — non-failed runs", () => {
-    test("returns null when outcome is 'pass'", () => {
-        const out = classifyFailureMode(fakeTask(), fakeRun({ outcome: "pass" }));
-        expect(out).toBeNull();
-    });
-    test("returns null when outcome is 'budget_exceeded'", () => {
-        const out = classifyFailureMode(fakeTask(), fakeRun({ outcome: "budget_exceeded" }));
-        expect(out).toBeNull();
-    });
-    test("returns null when outcome is 'harness_error'", () => {
-        const out = classifyFailureMode(fakeTask(), fakeRun({ outcome: "harness_error" }));
-        expect(out).toBeNull();
-    });
-});
-describe("classifyFailureMode — seven labels", () => {
-    test("no_search: empty trace returns no_search", () => {
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: "" }));
-        expect(out).toBe("no_search");
-    });
-    test("no_search: trace mentions show but never search", () => {
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: "akm show skill:docker-homelab\nresult: ok" }));
-        expect(out).toBe("no_search");
-    });
-    test("search_no_gold: search ran, gold ref absent from results", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:foo",
-            "2. skill:bar",
-            "3. skill:baz",
-            "verifier: missing required output",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("search_no_gold");
-    });
-    test("search_low_rank: gold ref appears at rank 7 in numbered list", () => {
-        const lines = ["$ akm search homelab"];
-        for (let i = 1; i <= 6; i += 1)
-            lines.push(`${i}. skill:filler-${i}`);
-        lines.push("7. skill:docker-homelab");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: lines.join("\n") }));
-        expect(out).toBe("search_low_rank");
-    });
-    test("loaded_wrong: agent showed a non-gold ref and never loaded gold", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:docker-homelab",
-            "2. skill:az-cli",
-            "$ akm show skill:az-cli",
-            "(content of az-cli)",
-            "verifier: action wrong",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("loaded_wrong");
-    });
-    test("loaded_ignored: gold loaded but verifier flags ignored guidance", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:docker-homelab",
-            "$ akm show skill:docker-homelab",
-            "(content of docker-homelab)",
-            "verifier: agent did not follow loaded asset",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("loaded_ignored");
-    });
-    test("followed_wrong: gold loaded, no ignored marker, verifier still failed", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:docker-homelab",
-            "$ akm show skill:docker-homelab",
-            "(content of docker-homelab)",
-            "verifier: pattern mismatch — expected 'X' got 'Y'",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("followed_wrong");
-    });
-    test("unrelated_bug: gold ref in search results, agent didn't load anything", () => {
-        // Search ran (so not no_search), gold present at rank 1 (so not search_no_gold,
-        // not search_low_rank), no `akm show` calls at all (so not loaded_wrong,
-        // not loaded_ignored, not followed_wrong) → unrelated_bug.
-        const trace = ["$ akm search homelab", "1. skill:docker-homelab", "verifier: missing config"].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("unrelated_bug");
-    });
-    test("unrelated_bug: task has no goldRef and search ran", () => {
-        const trace = "$ akm search foo\nresults: nothing relevant";
-        const out = classifyFailureMode(fakeTask({ goldRef: undefined }), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("unrelated_bug");
-    });
-    test("no_events: task has no goldRef and no search in trace", () => {
-        // When there is no goldRef and no search evidence, trajectory.correctAssetLoaded
-        // is always null (metric undefined). We cannot tell whether the agent searched
-        // or whether events data was absent. Surfaces as `no_events`.
-        const out = classifyFailureMode(fakeTask({ goldRef: undefined }), fakeRun({ verifierStdout: "" }));
-        expect(out).toBe("no_events");
-    });
-});
-describe("classifyFailureMode — trajectory-aware classification (REC-07 / REC-13)", () => {
-    test("loaded_ignored: correctAssetLoaded=true + fail → loaded_ignored (short-circuit)", () => {
-        // The agent loaded the correct asset (confirmed by trajectory data) but still
-        // produced wrong output. This is the dominant failure pattern in the
-        // 2026-05-03 baseline: 24/25 `search_no_gold` labels were wrong because the
-        // classifier didn't consult trajectory.correctAssetLoaded.
-        const out = classifyFailureMode(fakeTask(), fakeRun({
-            trajectory: { correctAssetLoaded: true, feedbackRecorded: null },
-            verifierStdout: "verifier: field values wrong",
-        }));
-        expect(out).toBe("loaded_ignored");
-    });
-    test("loaded_ignored: correctAssetLoaded=true overrides stdout-scan — fires even with no search in trace", () => {
-        // Trajectory data is authoritative. Even if verifierStdout shows no `akm
-        // search`, the trajectory says the gold was loaded → loaded_ignored, not
-        // no_search.
-        const out = classifyFailureMode(fakeTask(), fakeRun({
-            trajectory: { correctAssetLoaded: true, feedbackRecorded: null },
-            verifierStdout: "",
-        }));
-        expect(out).toBe("loaded_ignored");
-    });
-    test("search_no_gold: correctAssetLoaded=false + search ran + gold absent → search_no_gold", () => {
-        // When trajectory says gold was NOT loaded and search ran but gold ref absent
-        // from results, this is a genuine search failure.
-        const trace = ["$ akm search homelab", "1. skill:foo", "2. skill:bar"].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({
-            trajectory: { correctAssetLoaded: false, feedbackRecorded: null },
-            verifierStdout: trace,
-        }));
-        expect(out).toBe("search_no_gold");
-    });
-    test("no_search: correctAssetLoaded=false + no search in trace → no_search", () => {
-        // When trajectory says gold was NOT loaded and there is no search evidence,
-        // the agent genuinely didn't search.
-        const out = classifyFailureMode(fakeTask(), fakeRun({
-            trajectory: { correctAssetLoaded: false, feedbackRecorded: null },
-            verifierStdout: "verifier: missing output",
-        }));
-        expect(out).toBe("no_search");
-    });
-});
-describe("classifyFailureMode — tie-breaking and priority", () => {
-    test("no_search beats search_no_gold when both could apply (no search call)", () => {
-        // No `akm search` text, but gold is also missing. The first rule wins.
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: "verifier: nope" }));
-        expect(out).toBe("no_search");
-    });
-    test("search_no_gold beats search_low_rank when gold absent", () => {
-        const trace = ["$ akm search homelab", "1. skill:foo", "2. skill:bar"].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("search_no_gold");
-    });
-    test("search_low_rank beats loaded_wrong when gold is present at rank 7 even after wrong show", () => {
-        const lines = ["$ akm search homelab"];
-        for (let i = 1; i <= 6; i += 1)
-            lines.push(`${i}. skill:filler-${i}`);
-        lines.push("7. skill:docker-homelab");
-        lines.push("$ akm show skill:az-cli");
-        lines.push("verifier: failed");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: lines.join("\n") }));
-        expect(out).toBe("search_low_rank");
-    });
-    test("loaded_wrong beats followed_wrong when gold never loaded but other ref shown", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:docker-homelab",
-            "$ akm show skill:az-cli",
-            "verifier: agent did not follow loaded asset",
-        ].join("\n");
-        // Note `did not follow loaded asset` would otherwise trip loaded_ignored,
-        // but loaded_wrong's "gold never loaded" precondition wins first.
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("loaded_wrong");
-    });
-    test("loaded_ignored beats followed_wrong when verifier flags ignored", () => {
-        const trace = [
-            "$ akm search homelab",
-            "1. skill:docker-homelab",
-            "$ akm show skill:docker-homelab",
-            "verifier: contradicts loaded asset; agent ignored it",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("loaded_ignored");
-    });
-});
-describe("classifyFailureMode — input variants", () => {
-    test("event-stream-only search invocation counts as a search call", () => {
-        const event = {
-            schemaVersion: 1,
-            id: 0,
-            ts: "2026-04-27T00:00:00Z",
-            eventType: "search_invoked",
-        };
-        // No CLI marker in stdout, but the event makes hasAkmSearch return true.
-        // Gold ref is absent from any result list → search_no_gold.
-        const out = classifyFailureMode(fakeTask(), fakeRun({ events: [event], verifierStdout: "verifier: nope" }));
-        expect(out).toBe("search_no_gold");
-    });
-    test("tool-call JSON form for show counts as loading the gold ref", () => {
-        const trace = [
-            "$ akm search homelab",
-            '{"results":["skill:docker-homelab"]}',
-            '{"command":"akm","args":["show","skill:docker-homelab"]}',
-            "verifier: pattern mismatch",
-        ].join("\n");
-        const out = classifyFailureMode(fakeTask(), fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("followed_wrong");
-    });
-    test("origin-prefixed gold ref also matches", () => {
-        const task = fakeTask({ goldRef: "skill:docker-homelab" });
-        const trace = [
-            "$ akm search homelab",
-            "1. team//skill:docker-homelab",
-            "$ akm show team//skill:docker-homelab",
-            "verifier: pattern mismatch",
-        ].join("\n");
-        const out = classifyFailureMode(task, fakeRun({ verifierStdout: trace }));
-        expect(out).toBe("followed_wrong");
-    });
-});
-describe("classifyFailureMode — purity", () => {
-    test("same input twice yields the same label", () => {
-        const task = fakeTask();
-        const run = fakeRun({
-            verifierStdout: ["$ akm search homelab", "1. skill:docker-homelab", "verifier: missing config"].join("\n"),
-        });
-        const a = classifyFailureMode(task, run);
-        const b = classifyFailureMode(task, run);
-        expect(a).toBe(b);
-    });
-    test("classifier does not mutate its inputs", () => {
-        const task = fakeTask();
-        const run = fakeRun({ verifierStdout: "$ akm search foo\n1. skill:docker-homelab" });
-        const taskJson = JSON.stringify(task);
-        const runJson = JSON.stringify(run);
-        classifyFailureMode(task, run);
-        expect(JSON.stringify(task)).toBe(taskJson);
-        expect(JSON.stringify(run)).toBe(runJson);
-    });
-});
-describe("aggregateFailureModes", () => {
-    test("counts per label and per task", () => {
-        const agg = aggregateFailureModes([
-            { taskId: "t1", mode: "no_search" },
-            { taskId: "t1", mode: "no_search" },
-            { taskId: "t1", mode: "followed_wrong" },
-            { taskId: "t2", mode: "no_search" },
-        ]);
-        expect(agg.byLabel.no_search).toBe(3);
-        expect(agg.byLabel.followed_wrong).toBe(1);
-        expect(agg.byTask.t1?.no_search).toBe(2);
-        expect(agg.byTask.t1?.followed_wrong).toBe(1);
-        expect(agg.byTask.t2?.no_search).toBe(1);
-    });
-    test("empty input produces empty aggregate", () => {
-        const agg = aggregateFailureModes([]);
-        expect(agg.byLabel).toEqual({});
-        expect(agg.byTask).toEqual({});
-    });
-});
-describe("renderFailureModeBreakdown", () => {
-    function makeReport(byLabel) {
-        return {
-            timestamp: "2026-04-27T00:00:00Z",
-            branch: "x",
-            commit: "y",
-            model: "m",
-            corpus: { domains: 1, tasks: 1, slice: "all", seedsPerArm: 5 },
-            aggregateNoakm: { passRate: 0, tokensPerPass: null, tokensPerRun: null, wallclockMs: 0 },
-            aggregateAkm: { passRate: 0, tokensPerPass: null, tokensPerRun: null, wallclockMs: 0 },
-            aggregateDelta: { passRate: 0, tokensPerPass: null, tokensPerRun: null, wallclockMs: 0 },
-            trajectoryAkm: { correctAssetLoaded: null, feedbackRecorded: 0 },
-            failureModes: { byLabel, byTask: {} },
-            tasks: [],
-            warnings: [],
-        };
-    }
-    test("orders bins by descending count", () => {
-        const md = renderFailureModeBreakdown(makeReport({ no_search: 3, followed_wrong: 5, search_no_gold: 1 }));
-        const lines = md.split("\n").filter((l) => l.startsWith("- "));
-        expect(lines[0]).toContain("followed_wrong");
-        expect(lines[0]).toContain("5");
-        expect(lines[1]).toContain("no_search");
-        expect(lines[1]).toContain("3");
-        expect(lines[2]).toContain("search_no_gold");
-        expect(lines[2]).toContain("1");
-    });
-    test("ties broken alphabetically by label", () => {
-        const md = renderFailureModeBreakdown(makeReport({ followed_wrong: 2, no_search: 2 }));
-        const lines = md.split("\n").filter((l) => l.startsWith("- "));
-        expect(lines[0]).toContain("followed_wrong");
-        expect(lines[1]).toContain("no_search");
-    });
-    test("includes percent of failed runs", () => {
-        const md = renderFailureModeBreakdown(makeReport({ no_search: 1, followed_wrong: 3 }));
-        // 3/4 = 75.0%, 1/4 = 25.0%
-        expect(md).toContain("75.0%");
-        expect(md).toContain("25.0%");
-    });
-    test("returns empty string when no failures", () => {
-        const md = renderFailureModeBreakdown(makeReport({}));
-        expect(md).toBe("");
-    });
-    test("section header present when bins exist", () => {
-        const md = renderFailureModeBreakdown(makeReport({ no_search: 1 }));
-        expect(md).toContain("## Failure modes");
-    });
-});