akm-cli 0.6.0 → 0.7.0-rc1

package/dist/tests/bench/driver.js

@@ -0,0 +1,346 @@
+/**
+ * akm-bench driver — `runOne(options)` executes a single (task, arm, seed)
+ * triple end-to-end and returns a v1 RunResult envelope.
+ *
+ * See `docs/technical/benchmark.md` §5.2 for the locked schema and §7.1/§7.2
+ * for the isolation/budget rules. The shapes here are the v1 contract that
+ * #238/#239/#240/#243 will extend without breaking.
+ *
+ * Design notes:
+ *   • The driver invokes opencode through `runAgent` with the built-in
+ *     `opencode` profile. No new harness abstraction.
+ *   • Per-run isolation: every run gets fresh tmpdirs for `XDG_CACHE_HOME`,
+ *     `XDG_CONFIG_HOME`, `OPENCODE_CONFIG`, and (when `stashDir` is provided)
+ *     `AKM_STASH_DIR`. The operator's personal opencode/akm config is NEVER
+ *     read or written.
+ *   • Hard budgets: `budgetWallMs` is enforced via `runAgent`'s timeout. A
+ *     timeout produces `outcome: "budget_exceeded"`, which is a distinct
+ *     state from `fail` so cost regressions stay visible.
+ *   • This issue (#236) does not need a real opencode call to work end-to-end.
+ *     The harness shape, isolation, and result envelope must be correct and
+ *     unit-testable with an injected fake spawn.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { BUILTIN_AGENT_PROFILE_NAMES, getBuiltinAgentProfile } from "../../src/integrations/agent/profiles";
+import { runAgent } from "../../src/integrations/agent/spawn";
+import { benchMkdtemp } from "./tmp";
+import { runVerifier } from "./verifier";
+/** Operator-config env names that MUST NOT leak into per-run children. */
+const ISOLATED_ENV_NAMES = ["OPENCODE_CONFIG", "AKM_STASH_DIR", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"];
+/**
+ * Operator-env names that MUST be stripped from `envSource` before the bench
+ * driver hands it to `runAgent`. These are credentials and config-dir hints
+ * that belong to the operator's *interactive* environment and have no
+ * business inside a bench-arm child:
+ *
+ *   • `OPENCODE_API_KEY` / `ANTHROPIC_API_KEY` — real-money credentials. The
+ *     opencode profile lists `OPENCODE_API_KEY` in `envPassthrough`, so
+ *     without explicit scrubbing the bench would forward the operator's key
+ *     into every (task × arm × seed) child. Bench is hermetic by design;
+ *     credentials must be supplied through the bench's own config surface,
+ *     not inherited.
+ *   • `AKM_CONFIG_DIR` — points akm at the operator's stash config. Letting
+ *     this leak defeats the per-run isolation tmpdirs `createIsolationDirs`
+ *     materialises (XDG_CACHE_HOME / XDG_CONFIG_HOME) and would cause
+ *     bench runs to read the operator's writable config.
+ *
+ * Recurrence guard for #271 (mirrors the #243/#251 fixup pattern of
+ * pinning isolation behaviour with regression tests).
+ */
+const SCRUBBED_OPERATOR_ENV_NAMES = ["OPENCODE_API_KEY", "ANTHROPIC_API_KEY", "AKM_CONFIG_DIR"];
+/**
+ * Build the `envSource` passed to `runAgent`. Returns a copy of `source`
+ * (default: `process.env`) with `SCRUBBED_OPERATOR_ENV_NAMES` removed so
+ * profile-level passthrough (`profile.envPassthrough`) cannot drag operator
+ * credentials/config-dir hints into the bench-arm child.
+ *
+ * The returned object is a shallow copy — callers may mutate it without
+ * touching the real `process.env`.
+ */
+export function buildSanitizedEnvSource(source) {
+    const src = source ?? process.env;
+    const out = { ...src };
+    for (const name of SCRUBBED_OPERATOR_ENV_NAMES) {
+        delete out[name];
+    }
+    return out;
+}
+export function createIsolationDirs(stashDir) {
+    const root = benchMkdtemp("akm-bench-run-");
+    const cacheHome = path.join(root, "cache");
+    const configHome = path.join(root, "config");
+    const opencodeConfig = path.join(root, "opencode-config");
+    fs.mkdirSync(cacheHome, { recursive: true });
+    fs.mkdirSync(configHome, { recursive: true });
+    fs.mkdirSync(opencodeConfig, { recursive: true });
+    return {
+        root,
+        cacheHome,
+        configHome,
+        opencodeConfig,
+        akmStashDir: stashDir,
+    };
+}
+/** Build the env passed to `runAgent`. The XDG/AKM/OPENCODE keys are pinned. */
+export function buildIsolatedEnv(dirs, model) {
+    const env = {
+        XDG_CACHE_HOME: dirs.cacheHome,
+        XDG_CONFIG_HOME: dirs.configHome,
+        OPENCODE_CONFIG: dirs.opencodeConfig,
+        BENCH_OPENCODE_MODEL: model,
+    };
+    if (dirs.akmStashDir)
+        env.AKM_STASH_DIR = dirs.akmStashDir;
+    return env;
+}
+/**
+ * Strip `AKM_STASH_DIR` from a child env object. Used by the synthetic-arm
+ * spawn path (#261) so the operator's real `AKM_STASH_DIR` cannot leak in
+ * via the parent process even when the harness has copied a wider env via
+ * `{ ...process.env, ...env }`. This is the recurrence guard for the #243
+ * fixup pattern — a synthetic-arm child must NEVER inherit a stash.
+ *
+ * Mutates `env` in place and returns it for ergonomic chaining.
+ */
+export function stripAkmStashDir(env) {
+    delete env.AKM_STASH_DIR;
+    return env;
+}
+/**
+ * Best-effort token-usage parser for opencode stdout. Returns numeric token
+ * counts AND a measurement status so callers can distinguish a real zero
+ * (`"parsed"`, both fields legitimately 0) from an unparseable / absent
+ * report (`"missing"`, both fields default to 0 but downstream aggregation
+ * MUST skip the run rather than treat that 0 as measured).
+ *
+ * The harness never emits `"unsupported"` from this parser — that label is
+ * stamped on results from arms that don't run a token-reporting agent
+ * (e.g. the synthetic arm), and is set by the caller, not here.
+ */
+export function parseTokenUsage(stdout) {
+    // opencode prints lines like `tokens: input=1234 output=5678` in some
+    // configurations. We look for the keys defensively; absent values mean we
+    // could not measure (`measurement: "missing"`).
+    const inputMatch = stdout.match(/(?:input[_\s-]?tokens?|tokens?[_\s-]?input)[\s:=]+(\d+)/i);
+    const outputMatch = stdout.match(/(?:output[_\s-]?tokens?|tokens?[_\s-]?output)[\s:=]+(\d+)/i);
+    if (!inputMatch && !outputMatch) {
+        return { input: 0, output: 0, measurement: "missing" };
+    }
+    return {
+        input: inputMatch ? Number.parseInt(inputMatch[1], 10) : 0,
+        output: outputMatch ? Number.parseInt(outputMatch[1], 10) : 0,
+        measurement: "parsed",
+    };
+}
+/**
+ * Maximum bytes read from events.jsonl per run. A runaway agent producing
+ * GBs of structured-log output would otherwise OOM the bench. Trajectory
+ * parsing operates on the prefix; a warning is appended when the cap is
+ * hit so the report surfaces the truncation.
+ */
+export const EVENTS_READ_CAP_BYTES = 16 * 1024 * 1024;
+/**
+ * Read the events.jsonl file produced by this run, if any. The path is
+ * `<XDG_CACHE_HOME>/akm/events.jsonl` per `src/core/events.ts`.
+ *
+ * Caps the number of bytes read at `EVENTS_READ_CAP_BYTES` (16 MiB). When the
+ * file is larger, the prefix is parsed and a warning is appended to
+ * `opts.warnings` (when supplied). The trailing partial line after a
+ * truncation is dropped, since `JSON.parse` would reject it anyway.
+ */
+export function readRunEvents(cacheHome, opts) {
+    const eventsPath = path.join(cacheHome, "akm", "events.jsonl");
+    if (!fs.existsSync(eventsPath))
+        return [];
+    // Read up to the cap. We open the file rather than `readFileSync` so we
+    // don't allocate an arbitrarily large buffer just to throw most of it away.
+    let totalSize = 0;
+    try {
+        totalSize = fs.statSync(eventsPath).size;
+    }
+    catch {
+        return [];
+    }
+    const cap = EVENTS_READ_CAP_BYTES;
+    const truncated = totalSize > cap;
+    let text;
+    if (truncated) {
+        const buf = Buffer.alloc(cap);
+        const fd = fs.openSync(eventsPath, "r");
+        try {
+            fs.readSync(fd, buf, 0, cap, 0);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+        text = buf.toString("utf8");
+        // Drop the partial trailing line so we don't try to parse half a record.
+        const lastNl = text.lastIndexOf("\n");
+        if (lastNl !== -1)
+            text = text.slice(0, lastNl);
+        if (opts?.warnings) {
+            opts.warnings.push(`events.jsonl truncated: ${totalSize} bytes exceeds ${cap}-byte cap; trajectory computed from the prefix.`);
+        }
+    }
+    else {
+        text = fs.readFileSync(eventsPath, "utf8");
+    }
+    const out = [];
+    let id = 0;
+    for (const line of text.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            const parsed = JSON.parse(trimmed);
+            out.push({ ...parsed, id: parsed.id ?? id });
+            id += 1;
+        }
+        catch {
+            // Skip malformed lines — events stream is best-effort upstream.
+        }
+    }
+    return out;
+}
+/** Default prompt forwarded to opencode when caller omits one. */
+function defaultPrompt(options) {
+    return [
+        `Task: ${options.taskId}`,
+        `Arm: ${options.arm}`,
+        `Workspace: ${options.workspace}`,
+        options.arm === "akm"
+            ? "An akm stash is configured via AKM_STASH_DIR. Use `akm search` and `akm show` to find relevant assets before acting."
+            : "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+}
+/**
+ * Run a single (task, arm, seed) and return the v1 RunResult envelope.
+ *
+ * The function never throws on infrastructure failures — every error path
+ * is captured into the returned RunResult with a stable outcome value.
+ */
+export async function runOne(options) {
+    // Stamp a baseline result; we mutate fields below as the run progresses.
+    const result = {
+        schemaVersion: 1,
+        taskId: options.taskId,
+        arm: options.arm,
+        seed: options.seed,
+        model: options.model,
+        outcome: "harness_error",
+        tokens: { input: 0, output: 0 },
+        tokenMeasurement: "missing",
+        wallclockMs: 0,
+        trajectory: { correctAssetLoaded: null, feedbackRecorded: null },
+        events: [],
+        verifierStdout: "",
+        verifierExitCode: -1,
+        assetsLoaded: [],
+    };
+    // Look up the built-in opencode profile defensively. The lookup is a pure
+    // map read today, but wrapping it preserves the doc-comment guarantee that
+    // runOne never throws on infrastructure failures even if the registry
+    // shape changes. A missing/throwing profile becomes harness_error.
+    let profile;
+    try {
+        profile = getBuiltinAgentProfile("opencode");
+    }
+    catch (err) {
+        result.verifierStdout = `harness: getBuiltinAgentProfile("opencode") threw: ${err instanceof Error ? err.message : String(err)}`;
+        return result;
+    }
+    if (!profile) {
+        result.verifierStdout = `harness: built-in agent profile "opencode" missing; available: ${BUILTIN_AGENT_PROFILE_NAMES.join(", ")}`;
+        return result;
+    }
+    // #261: synthetic-arm runs MUST NOT carry AKM_STASH_DIR. We refuse to
+    // forward a stashDir for the synthetic arm even when the caller mistakenly
+    // supplies one, and we explicitly delete the key from the built env so the
+    // operator's real AKM_STASH_DIR can never leak in through any parent-env
+    // inheritance the harness happens to do downstream. Recurrence guard for
+    // the #243 fixup pattern.
+    const stashDir = options.arm === "synthetic" ? undefined : options.stashDir;
+    const dirs = createIsolationDirs(stashDir);
+    const env = buildIsolatedEnv(dirs, options.model);
+    if (options.arm === "synthetic") {
+        stripAkmStashDir(env);
+    }
+    try {
+        const agentResult = await runAgent(profile, options.prompt ?? defaultPrompt(options), {
+            env,
+            // #271: scrub operator credentials + config-dir hints from the env
+            // source BEFORE profile.envPassthrough copies them into the child.
+            // Without this, OPENCODE_API_KEY (in opencode's passthrough list) and
+            // AKM_CONFIG_DIR (read by akm at startup) would leak the operator's
+            // interactive environment into every bench child.
+            envSource: buildSanitizedEnvSource(),
+            cwd: options.workspace,
+            timeoutMs: options.budgetWallMs,
+            stdio: "captured",
+            ...(options.spawn ? { spawn: options.spawn } : {}),
+        });
+        result.wallclockMs = agentResult.durationMs;
+        const parsed = parseTokenUsage(agentResult.stdout);
+        result.tokens = { input: parsed.input, output: parsed.output };
+        result.tokenMeasurement = parsed.measurement;
+        result.events = readRunEvents(dirs.cacheHome, { warnings: options.warnings });
+        if (!agentResult.ok) {
+            if (agentResult.reason === "timeout") {
+                result.outcome = "budget_exceeded";
+                return result;
+            }
+            // spawn_failed / non_zero_exit / parse_error all mean the harness
+            // itself broke; the verifier never saw the workspace.
+            if (agentResult.reason === "spawn_failed" || agentResult.reason === "parse_error") {
+                result.outcome = "harness_error";
+                return result;
+            }
+            // non_zero_exit from the agent: intentionally falls through to the
+            // verifier path. Per spec §5.3 ("deterministic verifiers, never LLM"),
+            // the agent is the system under test, not the judge — its exit code
+            // does not gate verification. The verifier always runs against
+            // whatever workspace state the agent left behind, even on a crash.
+        }
+        // Token-budget enforcement is best-effort: only mark `budget_exceeded`
+        // if measurement was actually parsed (issue #252) AND the total exceeds
+        // the cap. A `"missing"` / `"unsupported"` measurement MUST NOT silently
+        // mask a budget overrun as a pass — it leaves the verifier to decide.
+        if (result.tokenMeasurement === "parsed") {
+            const totalTokens = result.tokens.input + result.tokens.output;
+            if (totalTokens > options.budgetTokens) {
+                result.outcome = "budget_exceeded";
+                return result;
+            }
+        }
+        const verifierResult = await runVerifier(options.taskDir, options.workspace, options.verifier, {
+            agentStdout: agentResult.stdout,
+            expectedMatch: options.expectedMatch,
+            ...(options.spawn ? { spawn: options.spawn } : {}),
+        });
+        result.verifierStdout = verifierResult.stdout;
+        result.verifierExitCode = verifierResult.exitCode;
+        if (verifierResult.exitCode === 127) {
+            // Missing runtime (e.g. pytest not on PATH) — not the agent's fault.
+            result.outcome = "harness_error";
+        }
+        else {
+            result.outcome = verifierResult.exitCode === 0 ? "pass" : "fail";
+        }
+        return result;
+    }
+    finally {
+        // Always tear down the isolation tmpdir. We copy events out before
+        // deletion (see readRunEvents above), so this is safe.
+        fs.rmSync(dirs.root, { recursive: true, force: true });
+    }
+}
+/** Exposed for the unit test that asserts operator env never leaks. */
+export const _ISOLATED_ENV_NAMES = ISOLATED_ENV_NAMES;
+/**
+ * Exposed for the #271 regression test that asserts operator credentials +
+ * `AKM_CONFIG_DIR` never reach a bench-arm child via profile.envPassthrough.
+ */
+export const _SCRUBBED_OPERATOR_ENV_NAMES = SCRUBBED_OPERATOR_ENV_NAMES;