akm-cli 0.7.0-rc1 → 0.7.1

package/dist/tests/bench/verifier.js

@@ -1,109 +0,0 @@
-/**
- * akm-bench verifier dispatcher (spec §5.3).
- *
- *   • `script` — spawn `<taskDir>/verify.sh` with cwd = workspace.
- *   • `pytest` — spawn `pytest -q --tb=line` with cwd = workspace.
- *   • `regex`  — match `expected_match` against `agentStdout`.
- *
- * No LLM-as-judge anywhere. Static dispatch only.
- *
- * Missing runtime (e.g. `pytest` not on PATH) returns exit code 127 with a
- * clear stdout message. The driver maps that to `outcome: "harness_error"`,
- * NOT `fail` — a missing tool is not an agent failure.
- */
-import fs from "node:fs";
-import path from "node:path";
-function resolveSpawn(config) {
-    if (config?.spawn)
-        return config.spawn;
-    const bun = globalThis.Bun;
-    if (!bun?.spawn)
-        throw new Error("Bun.spawn unavailable; pass config.spawn");
-    return bun.spawn.bind(bun);
-}
-async function readStream(stream) {
-    if (!stream)
-        return "";
-    try {
-        return await new Response(stream).text();
-    }
-    catch {
-        return "";
-    }
-}
-async function runProcess(cmd, cwd, spawn) {
-    let proc;
-    try {
-        proc = spawn(cmd, {
-            cwd,
-            stdout: "pipe",
-            stderr: "pipe",
-        });
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        // ENOENT (binary missing) maps to 127 — the conventional "command not found".
-        return {
-            exitCode: 127,
-            stdout: `verifier failed to spawn: ${message}`,
-        };
-    }
-    const stdoutPromise = readStream(proc.stdout ?? null);
-    const stderrPromise = readStream(proc.stderr ?? null);
-    let exitCode;
-    try {
-        exitCode = await proc.exited;
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return {
-            exitCode: 127,
-            stdout: `verifier exited with error: ${message}`,
-        };
-    }
-    const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]);
-    // Combine stdout+stderr so the operator sees the full verifier output.
-    const combined = stderr ? `${stdout}\n--- stderr ---\n${stderr}` : stdout;
-    return { exitCode, stdout: combined };
-}
-/**
- * Dispatch a verifier run. Each branch maps a `task.yaml` `verifier:` field
- * onto a deterministic check.
- */
-export async function runVerifier(taskDir, workspace, kind, config) {
-    if (kind === "script") {
-        const script = path.join(taskDir, "verify.sh");
-        if (!fs.existsSync(script)) {
-            return { exitCode: 127, stdout: `verify.sh not found at ${script}` };
-        }
-        return runProcess(["bash", script], workspace, resolveSpawn(config));
-    }
-    if (kind === "pytest") {
-        return runProcess(["pytest", "-q", "--tb=line"], workspace, resolveSpawn(config));
-    }
-    if (kind === "regex") {
-        const pattern = config?.expectedMatch;
-        const input = config?.agentStdout ?? "";
-        if (!pattern) {
-            return {
-                exitCode: 127,
-                stdout: 'regex verifier requires "expected_match" in task.yaml',
-            };
-        }
-        let regex;
-        try {
-            regex = new RegExp(pattern);
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            return { exitCode: 127, stdout: `invalid regex: ${message}` };
-        }
-        const matched = regex.test(input);
-        return {
-            exitCode: matched ? 0 : 1,
-            stdout: matched ? `regex match: ${pattern}` : `regex did not match: ${pattern}`,
-        };
-    }
-    // Compiler should refuse to land an unknown kind; runtime guard is belt-and-braces.
-    return { exitCode: 127, stdout: `unknown verifier kind: ${String(kind)}` };
-}

package/dist/tests/bench/verifier.test.js

@@ -1,118 +0,0 @@
-/**
- * Unit tests for the verifier dispatcher. Covers each of the three
- * verifier kinds plus the missing-pytest graceful-127 path.
- */
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import path from "node:path";
-import { benchMkdtemp } from "./tmp";
-import { runVerifier } from "./verifier";
-let scratch;
-beforeAll(() => {
-    scratch = benchMkdtemp("bench-verifier-test-");
-});
-afterAll(() => {
-    fs.rmSync(scratch, { recursive: true, force: true });
-});
-function fakeSpawn(exitCode, stdout = "", stderr = "", throwSync) {
-    return (_cmd, _options) => {
-        if (throwSync)
-            throw throwSync;
-        const proc = {
-            exitCode,
-            exited: Promise.resolve(exitCode),
-            stdout: stdout
-                ? new ReadableStream({
-                    start(controller) {
-                        controller.enqueue(new TextEncoder().encode(stdout));
-                        controller.close();
-                    },
-                })
-                : null,
-            stderr: stderr
-                ? new ReadableStream({
-                    start(controller) {
-                        controller.enqueue(new TextEncoder().encode(stderr));
-                        controller.close();
-                    },
-                })
-                : null,
-            stdin: null,
-            kill() {
-                /* noop */
-            },
-        };
-        return proc;
-    };
-}
-describe("runVerifier — script", () => {
-    test("returns exit 0 when verify.sh succeeds", async () => {
-        const taskDir = path.join(scratch, "script-pass");
-        fs.mkdirSync(taskDir);
-        fs.writeFileSync(path.join(taskDir, "verify.sh"), "");
-        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
-        const result = await runVerifier(taskDir, workspace, "script", {
-            spawn: fakeSpawn(0, "ok"),
-        });
-        expect(result.exitCode).toBe(0);
-        expect(result.stdout).toContain("ok");
-    });
-    test("returns 127 when verify.sh is missing", async () => {
-        const taskDir = path.join(scratch, "script-missing");
-        fs.mkdirSync(taskDir);
-        const workspace = fs.mkdtempSync(path.join(scratch, "ws-"));
-        const result = await runVerifier(taskDir, workspace, "script", {
-            spawn: fakeSpawn(0),
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("verify.sh not found");
-    });
-});
-describe("runVerifier — regex", () => {
-    test("returns 0 when expected_match matches agent stdout", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "the agent printed: hello world",
-            expectedMatch: "hello",
-        });
-        expect(result.exitCode).toBe(0);
-    });
-    test("returns 1 when expected_match does not match", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "different output",
-            expectedMatch: "hello",
-        });
-        expect(result.exitCode).toBe(1);
-    });
-    test("returns 127 when expected_match missing", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "anything",
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("expected_match");
-    });
-    test("returns 127 on invalid regex pattern", async () => {
-        const result = await runVerifier(scratch, scratch, "regex", {
-            agentStdout: "x",
-            expectedMatch: "(",
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("invalid regex");
-    });
-});
-describe("runVerifier — pytest", () => {
-    test("returns 127 with a clear message when pytest is missing", async () => {
-        const result = await runVerifier(scratch, scratch, "pytest", {
-            // Simulate ENOENT: spawn throws when bin not on PATH.
-            spawn: fakeSpawn(0, "", "", new Error("ENOENT: pytest not found")),
-        });
-        expect(result.exitCode).toBe(127);
-        expect(result.stdout).toContain("ENOENT");
-    });
-    test("returns the pytest exit code when present", async () => {
-        const result = await runVerifier(scratch, scratch, "pytest", {
-            spawn: fakeSpawn(0, "1 passed in 0.05s"),
-        });
-        expect(result.exitCode).toBe(0);
-        expect(result.stdout).toContain("passed");
-    });
-});