akm-cli 0.6.1 → 0.7.0-rc1

package/dist/tests/bench/verifier.js

@@ -0,0 +1,109 @@
+/**
+ * akm-bench verifier dispatcher (spec §5.3).
+ *
+ *   • `script` — spawn `<taskDir>/verify.sh` with cwd = workspace.
+ *   • `pytest` — spawn `pytest -q --tb=line` with cwd = workspace.
+ *   • `regex`  — match `expected_match` against `agentStdout`.
+ *
+ * No LLM-as-judge anywhere. Static dispatch only.
+ *
+ * Missing runtime (e.g. `pytest` not on PATH) returns exit code 127 with a
+ * clear stdout message. The driver maps that to `outcome: "harness_error"`,
+ * NOT `fail` — a missing tool is not an agent failure.
+ */
+import fs from "node:fs";
+import path from "node:path";
+function resolveSpawn(config) {
+    if (config?.spawn)
+        return config.spawn;
+    const bun = globalThis.Bun;
+    if (!bun?.spawn)
+        throw new Error("Bun.spawn unavailable; pass config.spawn");
+    return bun.spawn.bind(bun);
+}
+async function readStream(stream) {
+    if (!stream)
+        return "";
+    try {
+        return await new Response(stream).text();
+    }
+    catch {
+        return "";
+    }
+}
+async function runProcess(cmd, cwd, spawn) {
+    let proc;
+    try {
+        proc = spawn(cmd, {
+            cwd,
+            stdout: "pipe",
+            stderr: "pipe",
+        });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        // ENOENT (binary missing) maps to 127 — the conventional "command not found".
+        return {
+            exitCode: 127,
+            stdout: `verifier failed to spawn: ${message}`,
+        };
+    }
+    const stdoutPromise = readStream(proc.stdout ?? null);
+    const stderrPromise = readStream(proc.stderr ?? null);
+    let exitCode;
+    try {
+        exitCode = await proc.exited;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            exitCode: 127,
+            stdout: `verifier exited with error: ${message}`,
+        };
+    }
+    const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]);
+    // Combine stdout+stderr so the operator sees the full verifier output.
+    const combined = stderr ? `${stdout}\n--- stderr ---\n${stderr}` : stdout;
+    return { exitCode, stdout: combined };
+}
+/**
+ * Dispatch a verifier run. Each branch maps a `task.yaml` `verifier:` field
+ * onto a deterministic check.
+ */
+export async function runVerifier(taskDir, workspace, kind, config) {
+    if (kind === "script") {
+        const script = path.join(taskDir, "verify.sh");
+        if (!fs.existsSync(script)) {
+            return { exitCode: 127, stdout: `verify.sh not found at ${script}` };
+        }
+        return runProcess(["bash", script], workspace, resolveSpawn(config));
+    }
+    if (kind === "pytest") {
+        return runProcess(["pytest", "-q", "--tb=line"], workspace, resolveSpawn(config));
+    }
+    if (kind === "regex") {
+        const pattern = config?.expectedMatch;
+        const input = config?.agentStdout ?? "";
+        if (!pattern) {
+            return {
+                exitCode: 127,
+                stdout: 'regex verifier requires "expected_match" in task.yaml',
+            };
+        }
+        let regex;
+        try {
+            regex = new RegExp(pattern);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            return { exitCode: 127, stdout: `invalid regex: ${message}` };
+        }
+        const matched = regex.test(input);
+        return {
+            exitCode: matched ? 0 : 1,
+            stdout: matched ? `regex match: ${pattern}` : `regex did not match: ${pattern}`,
+        };
+    }
+    // Compiler should refuse to land an unknown kind; runtime guard is belt-and-braces.
+    return { exitCode: 127, stdout: `unknown verifier kind: ${String(kind)}` };
+}

package/dist/tests/bench/verifier.test.js

@@ -0,0 +1,118 @@
+/**
+ * Unit tests for the verifier dispatcher. Covers each of the three
+ * verifier kinds plus the missing-pytest graceful-127 path.
+ */
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { benchMkdtemp } from "./tmp";
+import { runVerifier } from "./verifier";
+let scratch;
+beforeAll(() => {
+    scratch = benchMkdtemp("bench-verifier-test-");
+});
+afterAll(() => {
+    fs.rmSync(scratch, { recursive: true, force: true });
+});
+function fakeSpawn(exitCode, stdout = "", stderr = "", throwSync) {
+    return (_cmd, _options) => {
+        if (throwSync)
+            throw throwSync;
+        const proc = {
+            exitCode,
+            exited: Promise.resolve(exitCode),
+            stdout: stdout
+                ? new ReadableStream({
+                    start(controller) {
+                        controller.enqueue(new TextEncoder().encode(stdout));
+                        controller.close();
+                    },
+                })
+                : null,
+            stderr: stderr
+                ? new ReadableStream({
+                    start(controller) {
+                        controller.enqueue(new TextEncoder().encode(stderr));
+                        controller.close();
+                    },
+                })
+                : null,
+            stdin: null,
+            kill() {
+                /* noop */
+            },
+        };
+        return proc;
+    };
+}
+describe("runVerifier — script", () => {
+    test("returns exit 0 when verify.sh succeeds", async () => {
+        const taskDir = path.join(scratch, "script-pass");
+        fs.mkdirSync(taskDir);
+        fs.writeFileSync(path.join(taskDir, "verify.sh"), "");
+        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
+        const result = await runVerifier(taskDir, workspace, "script", {
+            spawn: fakeSpawn(0, "ok"),
+        });
+        expect(result.exitCode).toBe(0);
+        expect(result.stdout).toContain("ok");
+    });
+    test("returns 127 when verify.sh is missing", async () => {
+        const taskDir = path.join(scratch, "script-missing");
+        fs.mkdirSync(taskDir);
+        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
+        const result = await runVerifier(taskDir, workspace, "script", {
+            spawn: fakeSpawn(0),
+        });
+        expect(result.exitCode).toBe(127);
+        expect(result.stdout).toContain("verify.sh not found");
+    });
+});
+describe("runVerifier — regex", () => {
+    test("returns 0 when expected_match matches agent stdout", async () => {
+        const result = await runVerifier(scratch, scratch, "regex", {
+            agentStdout: "the agent printed: hello world",
+            expectedMatch: "hello",
+        });
+        expect(result.exitCode).toBe(0);
+    });
+    test("returns 1 when expected_match does not match", async () => {
+        const result = await runVerifier(scratch, scratch, "regex", {
+            agentStdout: "different output",
+            expectedMatch: "hello",
+        });
+        expect(result.exitCode).toBe(1);
+    });
+    test("returns 127 when expected_match missing", async () => {
+        const result = await runVerifier(scratch, scratch, "regex", {
+            agentStdout: "anything",
+        });
+        expect(result.exitCode).toBe(127);
+        expect(result.stdout).toContain("expected_match");
+    });
+    test("returns 127 on invalid regex pattern", async () => {
+        const result = await runVerifier(scratch, scratch, "regex", {
+            agentStdout: "x",
+            expectedMatch: "(",
+        });
+        expect(result.exitCode).toBe(127);
+        expect(result.stdout).toContain("invalid regex");
+    });
+});
+describe("runVerifier — pytest", () => {
+    test("returns 127 with a clear message when pytest is missing", async () => {
+        const result = await runVerifier(scratch, scratch, "pytest", {
+            // Simulate ENOENT: spawn throws when bin not on PATH.
+            spawn: fakeSpawn(0, "", "", new Error("ENOENT: pytest not found")),
+        });
+        expect(result.exitCode).toBe(127);
+        expect(result.stdout).toContain("ENOENT");
+    });
+    test("returns the pytest exit code when present", async () => {
+        const result = await runVerifier(scratch, scratch, "pytest", {
+            spawn: fakeSpawn(0, "1 passed in 0.05s"),
+        });
+        expect(result.exitCode).toBe(0);
+        expect(result.stdout).toContain("passed");
+    });
+});