akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/trajectory.test.js

@@ -1,127 +0,0 @@
-/**
- * Unit tests for the trajectory parser.
- */
-import { describe, expect, test } from "bun:test";
-import { computeTrajectory, VERIFIER_STDOUT_SCAN_CAP } from "./trajectory";
-function fakeRun(overrides = {}) {
-    return {
-        schemaVersion: 1,
-        taskId: "x",
-        arm: "akm",
-        seed: 0,
-        model: "m",
-        outcome: "pass",
-        tokens: { input: 0, output: 0 },
-        wallclockMs: 0,
-        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
-        events: [],
-        verifierStdout: "",
-        verifierExitCode: 0,
-        assetsLoaded: [],
-        ...overrides,
-    };
-}
-function feedbackEvent() {
-    return {
-        schemaVersion: 1,
-        id: 0,
-        ts: "2026-04-27T00:00:00.000Z",
-        eventType: "feedback",
-        ref: "skill:foo",
-    };
-}
-describe("computeTrajectory.correctAssetLoaded", () => {
-    test("null when goldRef is missing on the task", () => {
-        const traj = computeTrajectory({}, fakeRun({ verifierStdout: "akm show skill:irrelevant" }));
-        expect(traj.correctAssetLoaded).toBeNull();
-    });
-    test("true when verifierStdout contains `akm show <goldRef>`", () => {
-        const traj = computeTrajectory({ goldRef: "skill:docker-homelab" }, fakeRun({
-            verifierStdout: "tool: akm show skill:docker-homelab\nresult: ok\n",
-        }));
-        expect(traj.correctAssetLoaded).toBe(true);
-    });
-    test("true when tool-call JSON form contains the ref", () => {
-        const traj = computeTrajectory({ goldRef: "skill:docker-homelab" }, fakeRun({
-            verifierStdout: '{"command":"akm","args":["show","skill:docker-homelab"]}',
-        }));
-        expect(traj.correctAssetLoaded).toBe(true);
-    });
-    test("false when verifierStdout shows a different ref", () => {
-        const traj = computeTrajectory({ goldRef: "skill:docker-homelab" }, fakeRun({ verifierStdout: "akm show skill:az-cli\n" }));
-        expect(traj.correctAssetLoaded).toBe(false);
-    });
-    test("false on empty trace", () => {
-        const traj = computeTrajectory({ goldRef: "skill:docker-homelab" }, fakeRun({ verifierStdout: "" }));
-        expect(traj.correctAssetLoaded).toBe(false);
-    });
-    test("true when an event metadata.ref carries the goldRef", () => {
-        const event = {
-            schemaVersion: 1,
-            id: 1,
-            ts: "2026-04-27T00:00:00.000Z",
-            eventType: "tool_call",
-            metadata: { ref: "skill:docker-homelab" },
-        };
-        const traj = computeTrajectory({ goldRef: "skill:docker-homelab" }, fakeRun({ events: [event] }));
-        expect(traj.correctAssetLoaded).toBe(true);
-    });
-});
-describe("computeTrajectory.feedbackRecorded", () => {
-    test("true when events stream contains a `feedback` event", () => {
-        const traj = computeTrajectory({ goldRef: "skill:foo" }, fakeRun({ events: [feedbackEvent()] }));
-        expect(traj.feedbackRecorded).toBe(true);
-    });
-    test("false when events stream is empty", () => {
-        const traj = computeTrajectory({ goldRef: "skill:foo" }, fakeRun({ events: [] }));
-        expect(traj.feedbackRecorded).toBe(false);
-    });
-    test("false when events contain other types but no `feedback`", () => {
-        const event = {
-            schemaVersion: 1,
-            id: 0,
-            ts: "2026-04-27T00:00:00.000Z",
-            eventType: "remember",
-            ref: "memory:alpha",
-        };
-        const traj = computeTrajectory({ goldRef: "skill:foo" }, fakeRun({ events: [event] }));
-        expect(traj.feedbackRecorded).toBe(false);
-    });
-});
-describe("computeTrajectory verifierStdout cap", () => {
-    test("trajectory still computes from the prefix when stdout exceeds the cap, and a warning is recorded", () => {
-        // Construct a stdout: prefix has the canonical `akm show` invocation;
-        // the rest is GBs-of-junk simulated as a long filler past the cap.
-        const ref = "skill:docker-homelab";
-        const prefix = `tool: akm show ${ref}\n`;
-        const fillerSize = VERIFIER_STDOUT_SCAN_CAP + 1024;
-        // Use repeated 'a' so total length comfortably exceeds the cap.
-        const filler = "a".repeat(fillerSize);
-        const verifierStdout = prefix + filler;
-        expect(verifierStdout.length).toBeGreaterThan(VERIFIER_STDOUT_SCAN_CAP);
-        const warnings = [];
-        const traj = computeTrajectory({ goldRef: ref }, fakeRun({ verifierStdout }), { warnings });
-        expect(traj.correctAssetLoaded).toBe(true);
-        expect(warnings.length).toBe(1);
-        expect(warnings[0]).toContain("verifierStdout truncated");
-        expect(warnings[0]).toContain(String(VERIFIER_STDOUT_SCAN_CAP));
-    });
-    test("no warning when stdout is within the cap", () => {
-        const warnings = [];
-        computeTrajectory({ goldRef: "skill:foo" }, fakeRun({ verifierStdout: "akm show skill:foo\n" }), { warnings });
-        expect(warnings).toEqual([]);
-    });
-    test("match found in the prefix even though tail mentions the ref past the cap", () => {
-        // Prefix has only filler; the gold ref appears only AFTER the cap.
-        // The scan should miss it (correctly — the agent's effective behaviour
-        // within the budgeted prefix did not include the show call).
-        const ref = "skill:never-loaded";
-        const filler = "x".repeat(VERIFIER_STDOUT_SCAN_CAP);
-        const verifierStdout = `${filler}akm show ${ref}\n`;
-        const warnings = [];
-        const traj = computeTrajectory({ goldRef: ref }, fakeRun({ verifierStdout }), { warnings });
-        expect(traj.correctAssetLoaded).toBe(false);
-        expect(warnings.length).toBe(1);
-        expect(warnings[0]).toContain("verifierStdout truncated");
-    });
-});

package/dist/tests/bench/verifier.js

@@ -1,114 +0,0 @@
-/**
- * akm-bench verifier dispatcher (spec §5.3).
- *
- *   • `script` — spawn `<taskDir>/verify.sh` with cwd = workspace.
- *   • `pytest` — spawn `pytest -q --tb=line` with cwd = workspace.
- *   • `regex`  — match `expected_match` against `agentStdout`.
- *
- * No LLM-as-judge anywhere. Static dispatch only.
- *
- * Missing runtime (e.g. `pytest` not on PATH) returns exit code 127 with a
- * clear stdout message. The driver maps that to `outcome: "harness_error"`,
- * NOT `fail` — a missing tool is not an agent failure.
- */
-import fs from "node:fs";
-import path from "node:path";
-function resolveSpawn(config) {
-    if (config?.spawn)
-        return config.spawn;
-    const bun = globalThis.Bun;
-    if (!bun?.spawn)
-        throw new Error("Bun.spawn unavailable; pass config.spawn");
-    return bun.spawn.bind(bun);
-}
-async function readStream(stream) {
-    if (!stream)
-        return "";
-    try {
-        return await new Response(stream).text();
-    }
-    catch {
-        return "";
-    }
-}
-async function runProcess(cmd, cwd, spawn) {
-    let proc;
-    try {
-        proc = spawn(cmd, {
-            cwd,
-            stdout: "pipe",
-            stderr: "pipe",
-        });
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        // ENOENT (binary missing) maps to 127 — the conventional "command not found".
-        return {
-            exitCode: 127,
-            stdout: `verifier failed to spawn: ${message}`,
-        };
-    }
-    const stdoutPromise = readStream(proc.stdout ?? null);
-    const stderrPromise = readStream(proc.stderr ?? null);
-    let exitCode;
-    try {
-        exitCode = await proc.exited;
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return {
-            exitCode: 127,
-            stdout: `verifier exited with error: ${message}`,
-        };
-    }
-    const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]);
-    // Combine stdout+stderr so the operator sees the full verifier output.
-    const combined = stderr ? `${stdout}\n--- stderr ---\n${stderr}` : stdout;
-    return { exitCode, stdout: combined };
-}
-/**
- * Dispatch a verifier run. Each branch maps a `task.yaml` `verifier:` field
- * onto a deterministic check.
- */
-export async function runVerifier(taskDir, workspace, kind, config) {
-    if (kind === "script") {
-        const script = path.join(taskDir, "verify.sh");
-        if (!fs.existsSync(script)) {
-            return { exitCode: 127, stdout: `verify.sh not found at ${script}` };
-        }
-        return runProcess(["bash", script], workspace, resolveSpawn(config));
-    }
-    if (kind === "pytest") {
-        // Test files live at <taskDir>/tests/, not inside the workspace copy.
-        // Pass the absolute path so pytest discovers them while running with
-        // cwd=workspace (which lets relative paths like pathlib.Path("file.yml") work).
-        const testsDir = path.join(taskDir, "tests");
-        const testArgs = fs.existsSync(testsDir) ? [testsDir] : [];
-        return runProcess(["pytest", "-q", "--tb=line", ...testArgs], workspace, resolveSpawn(config));
-    }
-    if (kind === "regex") {
-        const pattern = config?.expectedMatch;
-        const input = config?.agentStdout ?? "";
-        if (!pattern) {
-            return {
-                exitCode: 127,
-                stdout: 'regex verifier requires "expected_match" in task.yaml',
-            };
-        }
-        let regex;
-        try {
-            regex = new RegExp(pattern);
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            return { exitCode: 127, stdout: `invalid regex: ${message}` };
-        }
-        const matched = regex.test(input);
-        return {
-            exitCode: matched ? 0 : 1,
-            stdout: matched ? `regex match: ${pattern}` : `regex did not match: ${pattern}`,
-        };
-    }
-    // Compiler should refuse to land an unknown kind; runtime guard is belt-and-braces.
-    return { exitCode: 127, stdout: `unknown verifier kind: ${String(kind)}` };
-}

package/dist/tests/bench/verifier.test.js

@@ -1,118 +0,0 @@
-/**
- * Unit tests for the verifier dispatcher. Covers each of the three
- * verifier kinds plus the missing-pytest graceful-127 path.
- */
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import path from "node:path";
-import { benchMkdtemp } from "./tmp";
-import { runVerifier } from "./verifier";
-let scratch;
-beforeAll(() => {
-    scratch = benchMkdtemp("bench-verifier-test-");
-});
-afterAll(() => {
-    fs.rmSync(scratch, { recursive: true, force: true });
-});
-function fakeSpawn(exitCode, stdout = "", stderr = "", throwSync) {
-    return (_cmd, _options) => {
-        if (throwSync)
-            throw throwSync;
-        const proc = {
-            exitCode,
-            exited: Promise.resolve(exitCode),
-            stdout: stdout
-                ? new ReadableStream({
-                    start(controller) {
-                        controller.enqueue(new TextEncoder().encode(stdout));
-                        controller.close();
-                    },
-                })
-                : null,
-            stderr: stderr
-                ? new ReadableStream({
-                    start(controller) {
-                        controller.enqueue(new TextEncoder().encode(stderr));
-                        controller.close();
-                    },
-                })
-                : null,
-            stdin: null,
-            kill() {
-                /* noop */
-            },
-        };
-        return proc;
-    };
-}
-describe("runVerifier — script", () => {
-    test("returns exit 0 when verify.sh succeeds", async () => {
-        const taskDir = path.join(scratch, "script-pass");
-        fs.mkdirSync(taskDir);
-        fs.writeFileSync(path.join(taskDir, "verify.sh"), "");
-        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
-        const result = await runVerifier(taskDir, workspace, "script", {
-            spawn: fakeSpawn(0, "ok"),
-        });
-        expect(result.exitCode).toBe(0);
-        expect(result.stdout).toContain("ok");
-    });
-    test("returns 127 when verify.sh is missing", async () => {
-        const taskDir = path.join(scratch, "script-missing");
-        fs.mkdirSync(taskDir);
-        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
-        const result = await runVerifier(taskDir, workspace, "script", {
-            spawn: fakeSpawn(0),
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("verify.sh not found");
-    });
-});
-describe("runVerifier — regex", () => {
-    test("returns 0 when expected_match matches agent stdout", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "the agent printed: hello world",
-            expectedMatch: "hello",
-        });
-        expect(result.exitCode).toBe(0);
-    });
-    test("returns 1 when expected_match does not match", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "different output",
-            expectedMatch: "hello",
-        });
-        expect(result.exitCode).toBe(1);
-    });
-    test("returns 127 when expected_match missing", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "anything",
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("expected_match");
-    });
-    test("returns 127 on invalid regex pattern", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "x",
-            expectedMatch: "(",
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("invalid regex");
-    });
-});
-describe("runVerifier — pytest", () => {
-    test("returns 127 with a clear message when pytest is missing", async () => {
-        const result = await runVerifier(scratch, scratch, "pytest", {
-            // Simulate ENOENT: spawn throws when bin not on PATH.
-            spawn: fakeSpawn(0, "", "", new Error("ENOENT: pytest not found")),
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("ENOENT");
-    });
-    test("returns the pytest exit code when present", async () => {
-        const result = await runVerifier(scratch, scratch, "pytest", {
-            spawn: fakeSpawn(0, "1 passed in 0.05s"),
-        });
-        expect(result.exitCode).toBe(0);
-        expect(result.stdout).toContain("passed");
-    });
-});