akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/search-bridge.test.js

@@ -1,331 +0,0 @@
-/**
- * Unit tests for the §6.7 search-pipeline bridge.
- *
- * Covers:
- *   • `extractGoldRanks` — pure-function rank extraction from synthetic
- *     verifier-stdout traces, including JSON tool-call form, plain-text
- *     `ref:` lines, multiple searches per run, and gold-not-in-top-10
- *     (the "missing" bucket).
- *   • `computeSearchBridge` — histogram, p50/p90, gold_at_rank_1,
- *     gold_missing, and the keystone `pass_rate_by_rank` slice.
- *   • Empty-corpus path — no records → renderer emits the N/A sentence.
- *
- * No real opencode is invoked; every fixture is a hand-crafted `RunResult`.
- */
-import { describe, expect, test } from "bun:test";
-import { computeSearchBridge, extractGoldRanks } from "./metrics";
-import { renderSearchBridgeTable } from "./report";
-function fakeResult(stdout, overrides = {}) {
-    return {
-        schemaVersion: 1,
-        taskId: "t",
-        arm: "akm",
-        seed: 0,
-        model: "m",
-        outcome: "pass",
-        tokens: { input: 0, output: 0 },
-        wallclockMs: 0,
-        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
-        events: [],
-        verifierStdout: stdout,
-        verifierExitCode: 0,
-        assetsLoaded: [],
-        ...overrides,
-    };
-}
-describe("extractGoldRanks", () => {
-    test("returns [] when goldRef is undefined", () => {
-        const r = fakeResult('akm search "foo"\nref: skill:foo');
-        expect(extractGoldRanks(r, undefined)).toEqual([]);
-    });
-    test("returns [] when verifierStdout is empty", () => {
-        const r = fakeResult("");
-        expect(extractGoldRanks(r, "skill:foo")).toEqual([]);
-    });
-    test("extracts a single search with text-mode ref output, gold at rank 1", () => {
-        const stdout = [
-            `> akm search "redis healthcheck"`,
-            `skill: docker-homelab`,
-            `  ref: skill:docker-homelab`,
-            `  score: 0.92`,
-            `skill: nginx-tls`,
-            `  ref: skill:nginx-tls`,
-            `  score: 0.81`,
-        ].join("\n");
-        const events = extractGoldRanks(fakeResult(stdout), "skill:docker-homelab");
-        expect(events).toHaveLength(1);
-        expect(events[0].query).toBe("redis healthcheck");
-        expect(events[0].results).toEqual(["skill:docker-homelab", "skill:nginx-tls"]);
-        expect(events[0].rankOfGold).toBe(1);
-    });
-    test("extracts JSON tool-call form, gold at rank 3", () => {
-        const stdout = [
-            'tool: akm search "kubernetes pod restart" --output json',
-            '{"hits":[{"ref":"skill:k8s-debug"},{"ref":"skill:k8s-monitoring"},{"ref":"skill:k8s-restart"},{"ref":"skill:k8s-deploy"}]}',
-        ].join("\n");
-        const events = extractGoldRanks(fakeResult(stdout), "skill:k8s-restart");
-        expect(events).toHaveLength(1);
-        expect(events[0].results.slice(0, 4)).toEqual([
-            "skill:k8s-debug",
-            "skill:k8s-monitoring",
-            "skill:k8s-restart",
-            "skill:k8s-deploy",
-        ]);
-        expect(events[0].rankOfGold).toBe(3);
-    });
-    test("returns null rank when gold is missing from top 10", () => {
-        const refs = Array.from({ length: 12 }, (_, i) => `  ref: skill:other-${i}`).join("\n");
-        const stdout = `akm search "missing-target"\n${refs}`;
-        const events = extractGoldRanks(fakeResult(stdout), "skill:gold");
-        expect(events).toHaveLength(1);
-        expect(events[0].rankOfGold).toBeNull();
-        // Top-10 cap: only 10 results retained.
-        expect(events[0].results.length).toBeLessThanOrEqual(10);
-    });
-    test("multiple searches per run are each emitted in order", () => {
-        const stdout = [
-            'akm search "first query"',
-            "  ref: skill:a",
-            "  ref: skill:b",
-            'akm search "second query"',
-            "  ref: skill:gold",
-            "  ref: skill:c",
-        ].join("\n");
-        const events = extractGoldRanks(fakeResult(stdout), "skill:gold");
-        expect(events).toHaveLength(2);
-        expect(events[0].query).toBe("first query");
-        expect(events[0].rankOfGold).toBeNull();
-        expect(events[1].query).toBe("second query");
-        expect(events[1].rankOfGold).toBe(1);
-    });
-    test("non-search akm invocation closes the active search block", () => {
-        const stdout = [
-            'akm search "q"',
-            "  ref: skill:a",
-            "  ref: skill:gold",
-            "akm show skill:gold",
-            "  ref: skill:gold (this should NOT extend the previous search)",
-            "  ref: skill:other",
-        ].join("\n");
-        const events = extractGoldRanks(fakeResult(stdout), "skill:gold");
-        // Only the search block contributes to results; the show block is closed.
-        expect(events).toHaveLength(1);
-        expect(events[0].results).toEqual(["skill:a", "skill:gold"]);
-        expect(events[0].rankOfGold).toBe(2);
-    });
-    test("origin-prefixed ref counts as gold (team//skill:foo matches skill:foo)", () => {
-        const stdout = ['akm search "q"', "  ref: team//skill:foo", "  ref: skill:bar"].join("\n");
-        const events = extractGoldRanks(fakeResult(stdout), "skill:foo");
-        expect(events[0].rankOfGold).toBe(1);
-    });
-});
-describe("computeSearchBridge — histogram + percentiles", () => {
-    function fakeRecord(seed, outcome, rankOrNullPerSearch) {
-        return {
-            taskId: `t${seed}`,
-            arm: "akm",
-            seed,
-            outcome,
-            goldRef: "skill:gold",
-            searches: rankOrNullPerSearch.map((rank, i) => ({
-                query: `q${i}`,
-                // Reconstruct a plausible result list: gold at the requested rank,
-                // others as fillers. The aggregator only looks at rankOfGold.
-                results: rank === null ? Array.from({ length: 10 }, (_, j) => `skill:other-${j}`) : [],
-                rankOfGold: rank,
-            })),
-        };
-    }
-    test("empty corpus produces zero envelope", () => {
-        const m = computeSearchBridge({ goldRankRecords: [] });
-        expect(m.runsObserved).toBe(0);
-        expect(m.searchesObserved).toBe(0);
-        expect(m.goldRankP50).toBeNull();
-        expect(m.goldRankP90).toBeNull();
-        expect(m.goldAtRank1).toBe(0);
-        expect(m.goldMissing).toBe(0);
-        expect(m.passRateByRank).toEqual([]);
-        // Histogram is fully zeroed for every key.
-        for (const k of ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "missing"]) {
-            expect(m.goldRankDistribution[k]).toBe(0);
-        }
-    });
-    test("histogram counts ranks across all searches", () => {
-        const records = [
-            fakeRecord(0, "pass", [1, 2]),
-            fakeRecord(1, "fail", [1, null]),
-            fakeRecord(2, "pass", [3]),
-        ];
-        const m = computeSearchBridge({ goldRankRecords: records });
-        expect(m.searchesObserved).toBe(5);
-        expect(m.runsObserved).toBe(3);
-        expect(m.goldRankDistribution["1"]).toBe(2);
-        expect(m.goldRankDistribution["2"]).toBe(1);
-        expect(m.goldRankDistribution["3"]).toBe(1);
-        expect(m.goldRankDistribution.missing).toBe(1);
-        expect(m.goldAtRank1).toBeCloseTo(2 / 5);
-        expect(m.goldMissing).toBeCloseTo(1 / 5);
-    });
-    test("p50/p90 use nearest-rank with missing treated as Infinity", () => {
-        // Ranks: [1,1,2,3,5,5,7,9,null,null] across one record.
-        const records = [fakeRecord(0, "pass", [1, 1, 2, 3, 5, 5, 7, 9, null, null])];
-        const m = computeSearchBridge({ goldRankRecords: records });
-        // Sorted: [1,1,2,3,5,5,7,9,Inf,Inf]
-        // p50 = idx ceil(0.5*10)-1 = 4 → 5
-        expect(m.goldRankP50).toBe(5);
-        // p90 = idx ceil(0.9*10)-1 = 8 → Infinity
-        expect(m.goldRankP90).toBe(Number.POSITIVE_INFINITY);
-    });
-});
-describe("computeSearchBridge — pass_rate_by_rank uses the agent's chosen search", () => {
-    test("attributes pass/fail to the rank in the LAST search, not the highest-ranked", () => {
-        // Run A passed; first search had gold at rank 1 (great rank!), but the
-        // *chosen* (last) search had gold at rank 5. The bridge must attribute
-        // run A to rank 5, not rank 1, otherwise it overstates the value of
-        // having gold at rank 1.
-        const records = [
-            {
-                taskId: "ta",
-                arm: "akm",
-                seed: 0,
-                outcome: "pass",
-                goldRef: "skill:gold",
-                searches: [
-                    { query: "first", results: [], rankOfGold: 1 },
-                    { query: "last", results: [], rankOfGold: 5 },
-                ],
-            },
-            {
-                taskId: "tb",
-                arm: "akm",
-                seed: 0,
-                outcome: "fail",
-                goldRef: "skill:gold",
-                searches: [{ query: "only", results: [], rankOfGold: 5 }],
-            },
-            {
-                taskId: "tc",
-                arm: "akm",
-                seed: 0,
-                outcome: "pass",
-                goldRef: "skill:gold",
-                searches: [{ query: "only", results: [], rankOfGold: 1 }],
-            },
-        ];
-        const m = computeSearchBridge({ goldRankRecords: records });
-        // Buckets: rank 1 → {1 pass / 1 total}, rank 5 → {1 pass / 2 total}.
-        const rank1 = m.passRateByRank.find((e) => e.rank === "1");
-        const rank5 = m.passRateByRank.find((e) => e.rank === "5");
-        expect(rank1).toBeDefined();
-        expect(rank1?.passRate).toBe(1);
-        expect(rank1?.runCount).toBe(1);
-        expect(rank5).toBeDefined();
-        expect(rank5?.passRate).toBe(0.5);
-        expect(rank5?.runCount).toBe(2);
-    });
-    test("missing bucket gets its own pass-rate row instead of being dropped", () => {
-        const records = [
-            {
-                taskId: "tm1",
-                arm: "akm",
-                seed: 0,
-                outcome: "pass",
-                goldRef: "skill:gold",
-                searches: [{ query: "q", results: [], rankOfGold: null }],
-            },
-            {
-                taskId: "tm2",
-                arm: "akm",
-                seed: 0,
-                outcome: "fail",
-                goldRef: "skill:gold",
-                searches: [{ query: "q", results: [], rankOfGold: null }],
-            },
-        ];
-        const m = computeSearchBridge({ goldRankRecords: records });
-        const missing = m.passRateByRank.find((e) => e.rank === "missing");
-        expect(missing).toBeDefined();
-        expect(missing?.runCount).toBe(2);
-        expect(missing?.passRate).toBe(0.5);
-    });
-    test("runs without any akm search invocation are excluded from pass_rate_by_rank", () => {
-        const records = [
-            {
-                taskId: "no-search",
-                arm: "akm",
-                seed: 0,
-                outcome: "fail",
-                goldRef: "skill:gold",
-                searches: [],
-            },
-        ];
-        const m = computeSearchBridge({ goldRankRecords: records });
-        expect(m.runsObserved).toBe(1);
-        expect(m.searchesObserved).toBe(0);
-        expect(m.passRateByRank).toEqual([]);
-    });
-});
-describe("renderSearchBridgeTable", () => {
-    test("empty corpus renders the N/A sentence", () => {
-        const md = renderSearchBridgeTable({
-            goldRankDistribution: {
-                "1": 0,
-                "2": 0,
-                "3": 0,
-                "4": 0,
-                "5": 0,
-                "6": 0,
-                "7": 0,
-                "8": 0,
-                "9": 0,
-                "10": 0,
-                missing: 0,
-            },
-            goldRankP50: null,
-            goldRankP90: null,
-            goldAtRank1: 0,
-            goldMissing: 0,
-            passRateByRank: [],
-            runsObserved: 0,
-            searchesObserved: 0,
-        });
-        expect(md).toContain("Search → outcome bridge");
-        expect(md).toContain("(no gold-ref tasks in corpus; bridge metrics N/A)");
-    });
-    test("populated corpus surfaces histogram, p50/p90, and pass-rate-by-rank table", () => {
-        const md = renderSearchBridgeTable({
-            goldRankDistribution: {
-                "1": 3,
-                "2": 1,
-                "3": 0,
-                "4": 0,
-                "5": 1,
-                "6": 0,
-                "7": 0,
-                "8": 0,
-                "9": 0,
-                "10": 0,
-                missing: 1,
-            },
-            goldRankP50: 1,
-            goldRankP90: 5,
-            goldAtRank1: 0.5,
-            goldMissing: 1 / 6,
-            passRateByRank: [
-                { rank: "1", passRate: 0.67, runCount: 3 },
-                { rank: "5", passRate: 0, runCount: 1 },
-                { rank: "missing", passRate: 0, runCount: 1 },
-            ],
-            runsObserved: 5,
-            searchesObserved: 6,
-        });
-        expect(md).toContain("| 1 | 3 |");
-        expect(md).toContain("| missing | 1 |");
-        expect(md).toContain("p50=1.0");
-        expect(md).toContain("p90=5.0");
-        expect(md).toContain("gold_at_rank_1=50.0%");
-        expect(md).toContain("| rank | pass_rate | run_count |");
-        expect(md).toContain("| 1 | 0.67 | 3 |");
-        expect(md).toContain("| missing | 0.00 | 1 |");
-    });
-});

package/dist/tests/bench/tmp.js

@@ -1,131 +0,0 @@
-/**
- * Bench tmp-root redirection (#276).
- *
- * Every bench tmp directory — per-(task, arm, seed) workspace, per-task
- * fixture stash, per-fixture evolveStash + preStash, plus the scratch dirs
- * spun up inside unit tests — lives under `${AKM_CACHE_DIR}/bench/`, NOT
- * `os.tmpdir()`.
- *
- * Why: during long bench/workflow runs, orphan tmp dirs from crashed agents
- * accumulate. When they pile up under `/tmp` the OS-level partition fills,
- * which breaks shells, browsers, npm caches, and the rest of the system.
- * Pinning bench tmp to the akm cache dir means a single
- * `rm -rf "$(akm config get cache.dir)/bench"` purges all bench scratch
- * without disturbing anything else.
- *
- * The bench cleanup machinery (`tests/bench/cleanup.ts`) also reaps
- * `${AKM_CACHE_DIR}/bench/*` entries older than 6 hours on the first
- * `registerCleanup` call to catch orphans from prior crashed runs.
- *
- * NOTE: this helper deliberately does NOT import `os.tmpdir()`. The
- * invariant test (`tests/bench/no-os-tmpdir-invariant.test.ts`) asserts
- * zero `os.tmpdir` references across `tests/bench/*.ts`.
- */
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { getCacheDir } from "../../src/core/paths";
-/** Bench-tmp root: `${AKM_CACHE_DIR}/bench/`. Created lazily. */
-export function benchTmpRoot() {
-    const root = path.join(getCacheDir(), "bench");
-    fs.mkdirSync(root, { recursive: true });
-    return root;
-}
-/**
- * Create a fresh tmp directory under `benchTmpRoot()`.
- *
- * Drop-in replacement for `fs.mkdtempSync(path.join(os.tmpdir(), prefix))`.
- * The returned absolute path is unique per call.
- */
-export function benchMkdtemp(prefix) {
-    return fs.mkdtempSync(path.join(benchTmpRoot(), prefix));
-}
-// ── PID file ────────────────────────────────────────────────────────────────
-/** Absolute path to the bench PID file: `${AKM_CACHE_DIR}/bench/bench.pid`. */
-export function benchPidPath() {
-    return path.join(benchTmpRoot(), "bench.pid");
-}
-/**
- * Write `process.pid` to `bench.pid`.
- *
- * If a stale PID file exists and the referenced process is no longer running,
- * it is removed with a warning before writing the new one.
- *
- * Returns a cleanup function that removes the PID file. Call it in a
- * `finally` block so the file is removed on both clean exit and exceptions.
- */
-export function writeBenchPid() {
-    const pidPath = benchPidPath();
-    // Check for an existing PID file and warn if stale.
-    if (fs.existsSync(pidPath)) {
-        let existingPid;
-        try {
-            const raw = fs.readFileSync(pidPath, "utf8").trim();
-            existingPid = Number.parseInt(raw, 10);
-        }
-        catch {
-            // Unreadable — treat as stale.
-        }
-        if (existingPid !== undefined && Number.isFinite(existingPid) && !isPidRunning(existingPid)) {
-            // Stale PID — warn and remove.
-            process.stderr.write(`bench: removing stale PID file for PID ${existingPid} (process not running)\n`);
-            try {
-                fs.rmSync(pidPath, { force: true });
-            }
-            catch {
-                /* best-effort */
-            }
-        }
-    }
-    try {
-        fs.writeFileSync(pidPath, String(process.pid), "utf8");
-    }
-    catch {
-        /* best-effort — PID file is diagnostic, not critical */
-    }
-    return () => {
-        try {
-            // Only remove if it still contains our PID (guard against races).
-            const current = fs.readFileSync(pidPath, "utf8").trim();
-            if (current === String(process.pid)) {
-                fs.rmSync(pidPath, { force: true });
-            }
-        }
-        catch {
-            /* best-effort */
-        }
-    };
-}
-/**
- * Read the PID from `bench.pid`. Returns `undefined` when the file does not
- * exist or cannot be parsed.
- */
-export function readBenchPid() {
-    const pidPath = benchPidPath();
-    if (!fs.existsSync(pidPath))
-        return undefined;
-    try {
-        const raw = fs.readFileSync(pidPath, "utf8").trim();
-        const n = Number.parseInt(raw, 10);
-        return Number.isFinite(n) && n > 0 ? n : undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
-/**
- * Return `true` when the process with the given PID is running on this host.
- * Uses `process.kill(pid, 0)` — signal 0 is a no-op probe that throws ESRCH
- * when the process does not exist and EPERM when it exists but is owned by
- * another user (in which case it IS running).
- */
-export function isPidRunning(pid) {
-    try {
-        process.kill(pid, 0);
-        return true;
-    }
-    catch (err) {
-        const code = err.code;
-        // EPERM means the process exists but we don't have permission to signal it.
-        return code === "EPERM";
-    }
-}

package/dist/tests/bench/trajectory.js

@@ -1,116 +0,0 @@
-/**
- * akm-bench trajectory parser (spec §6.2).
- *
- * Trajectory metrics describe the *path* the agent took through the run, not
- * just the terminal outcome. For #238 we score two booleans per run:
- *
- *   • `correctAssetLoaded` — did the agent invoke `akm show <goldRef>` (or
- *     a sufficient prefix thereof) at any point during the run? `null` when
- *     the task carries no `goldRef` (and so the metric is undefined).
- *   • `feedbackRecorded` — did the agent emit any `feedback` event into
- *     `events.jsonl` during the run? Always `false` for the `noakm` arm
- *     because that arm runs without a stash.
- *
- * The driver hands us a `RunResult` after the run has finished. We never
- * mutate it; we return a fresh `TrajectoryRecord` and let the runner splice
- * it back in. This keeps `runOne`'s signature stable and lets `#239`/`#240`
- * extend the trajectory shape without touching the driver.
- */
-/**
- * Cap on the number of characters of `verifierStdout` we substring-scan for
- * the `akm show <ref>` heuristic. A runaway agent could emit GBs of stdout;
- * scanning all of it would OOM the bench. The first 16 MiB is plenty to
- * decide whether the agent invoked `akm show` for the gold ref.
- */
-export const VERIFIER_STDOUT_SCAN_CAP = 16 * 1024 * 1024;
-/**
- * Compute the trajectory record for a single run.
- *
- * The `correctAssetLoaded` heuristic looks for the `akm show <ref>` invocation
- * in two places:
- *   1. The `events.jsonl` events array (if `akm show` ever emits an event —
- *      currently it does not, but we future-proof).
- *   2. The agent's stdout/verifier stdout (`runResult.verifierStdout`). When
- *      opencode logs its tool calls, the literal string `akm show <ref>`
- *      appears verbatim in the trace.
- *
- * We accept a "sufficient prefix": `skill:docker-homelab` matches both the
- * exact ref and `skill:docker-homelab/anything`. The match is conservative
- * — case-sensitive, exact substring on `akm show <ref>` (whitespace-flexible).
- */
-export function computeTrajectory(task, runResult, opts) {
-    const correctAssetLoaded = computeCorrectAssetLoaded(task, runResult, opts);
-    const feedbackRecorded = computeFeedbackRecorded(runResult);
-    return { correctAssetLoaded, feedbackRecorded };
-}
-function computeCorrectAssetLoaded(task, runResult, opts) {
-    if (!task.goldRef)
-        return null;
-    const ref = task.goldRef;
-    // Search the events stream for any tool-call event that carries the ref.
-    // akm show emits an event to events.jsonl, so this path is the primary
-    // detection route when the structured event stream is available.
-    for (const event of runResult.events) {
-        const refField = event.ref;
-        if (typeof refField === "string" && matchesRef(refField, ref))
-            return true;
-        const meta = event.metadata;
-        if (meta && typeof meta === "object") {
-            const candidate = meta.ref;
-            if (typeof candidate === "string" && matchesRef(candidate, ref))
-                return true;
-        }
-    }
-    // Substring scan on the captured agent/verifier stdout. We look for either
-    //   - `akm show <ref>` (the canonical form opencode logs when the agent
-    //     invokes the akm CLI as a tool), or
-    //   - the bare ref appearing on a line that mentions `show` (covers tool-
-    //     call JSON like `{"command":"akm","args":["show","skill:foo"]}`).
-    // Cap the scan at VERIFIER_STDOUT_SCAN_CAP so a runaway agent's GBs of
-    // stdout cannot OOM the bench. When we truncate, push a warning so the
-    // top-level report aggregates it under `warnings[]`.
-    const haystackFull = runResult.verifierStdout;
-    let haystack = haystackFull;
-    if (haystack && haystack.length > VERIFIER_STDOUT_SCAN_CAP) {
-        haystack = haystack.slice(0, VERIFIER_STDOUT_SCAN_CAP);
-        if (opts?.warnings) {
-            opts.warnings.push(`verifierStdout truncated for trajectory scan: ${haystackFull.length} chars exceeds ${VERIFIER_STDOUT_SCAN_CAP}-char cap; correct_asset_loaded computed from the prefix.`);
-        }
-    }
-    if (haystack && containsAkmShow(haystack, ref))
-        return true;
-    return false;
-}
-function matchesRef(candidate, gold) {
-    if (candidate === gold)
-        return true;
-    // Allow goldRef to be a prefix of a more-specific ref (e.g. team//skill:foo
-    // when the task says skill:foo). Keep the check anchored to ref segments.
-    if (candidate.endsWith(`//${gold}`))
-        return true;
-    if (candidate.startsWith(`${gold}/`))
-        return true;
-    return false;
-}
-function containsAkmShow(text, ref) {
-    // Whitespace-flexible match for `akm show <ref>`. We escape regex metas in
-    // the ref because asset refs may contain `:` (always) and `/` (origin form).
-    const escaped = ref.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-    const pattern = new RegExp(`akm\\s+show\\s+(?:["'])?${escaped}(?:\\b|\\W)`);
-    if (pattern.test(text))
-        return true;
-    // Tool-call JSON form: `"args":["show","<ref>"]` or similar. Cheap heuristic.
-    if (text.includes(`"show"`) && text.includes(ref))
-        return true;
-    return false;
-}
-function computeFeedbackRecorded(runResult) {
-    // The `noakm` arm runs without an akm stash, so events.jsonl will be empty
-    // by construction. Still honour the same scan — the assertion is an
-    // invariant of the events stream, not arm-specific behaviour.
-    for (const event of runResult.events) {
-        if (event.eventType === "feedback")
-            return true;
-    }
-    return false;
-}