@alignfirst/openclaw-test 0.20.0

package/dist/exec-rpc.js

@@ -0,0 +1,46 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+export const IPC_DIR = "/var/run/openclaw-test-ipc";
+// Wait this much longer than the requested timeout before declaring the
+// host-side poll dead. Gives the watcher time to kill the child on its own
+// timeout (exitCode 124) and write the truncated response file.
+const WATCHER_DEADLINE_HEADROOM_MS = 5_000;
+const EXEC_POLL_INTERVAL_MS = 100;
+const WATCHER_TIMEOUT_EXIT_CODE = 124;
+export async function execInGateway(argv, opts = {}) {
+    const id = randomUUID();
+    const timeoutMs = opts.timeoutMs ?? 30_000;
+    const payload = { id, argv, timeoutMs };
+    if (opts.cwd !== undefined)
+        payload.cwd = opts.cwd;
+    if (opts.env !== undefined)
+        payload.env = opts.env;
+    if (opts.stdin !== undefined)
+        payload.stdin = opts.stdin;
+    const reqPath = `${IPC_DIR}/${id}.req.json`;
+    const reqTmp = `${reqPath}.tmp`;
+    const resPath = `${IPC_DIR}/${id}.res.json`;
+    writeFileSync(reqTmp, JSON.stringify(payload));
+    renameSync(reqTmp, reqPath);
+    const deadline = Date.now() + timeoutMs + WATCHER_DEADLINE_HEADROOM_MS;
+    while (Date.now() < deadline) {
+        if (existsSync(resPath)) {
+            const raw = readFileSync(resPath, "utf8");
+            rmSync(resPath, { force: true });
+            rmSync(reqPath, { force: true });
+            const parsed = JSON.parse(raw);
+            if (parsed.exitCode === WATCHER_TIMEOUT_EXIT_CODE) {
+                console.warn(`execInGateway: watcher killed child after ${timeoutMs}ms (id ${id}, argv ${JSON.stringify(argv)})`);
+            }
+            return parsed;
+        }
+        await new Promise((r) => setTimeout(r, EXEC_POLL_INTERVAL_MS));
+    }
+    rmSync(reqPath, { force: true });
+    rmSync(`${reqPath}.processing`, { force: true });
+    // The watcher can still write `<id>.res.json` after this deadline (its own
+    // timeout kill plus the write can trail it). Nobody will read it, and the
+    // IPC volume is persistent — sweep it once it had time to land.
+    setTimeout(() => rmSync(resPath, { force: true }), WATCHER_DEADLINE_HEADROOM_MS * 2).unref();
+    throw new Error(`execInGateway timed out waiting for response (request id ${id})`);
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export type { BusMessage, ChannelId, MockCliRegisterMode, MockCliRegisterOptions, ScenarioContext, SendInboundResult, WaitForOutboundOptions, WaitForOutboundResult, } from "./context.js";
+export type { JudgeUsage, JudgeVerdict, JudgeVerdictJson, JudgeVerdictRaw, } from "./judge.js";
+export type { ActionEntry, ActionEntryBase, AgentToolCall, AgentToolCallEntry, AssertionRecord, CliMockCall, CliMockEntry, CliMockHandler, CliMockHandlerArgs, CostBreakdown, EmitSink, ErrorScenarioResult, FailedEntryScenarioResult, FailScenarioResult, InboundSentEntry, OutboundReceivedEntry, PassScenarioResult, ReportEntry, ReportEntryBase, ScenarioFailure, ScenarioLogEntry, ScenarioLogNote, ScenarioReport, ScenarioResult, } from "./report.js";

package/dist/index.js

@@ -0,0 +1 @@
+export {};

package/dist/judge.d.ts

@@ -0,0 +1,47 @@
+export interface JudgeUsage {
+    model: string;
+    inputTokens: number;
+    outputTokens: number;
+}
+export interface JudgeVerdict {
+    verdict: "pass" | "fail";
+    reasoning: string;
+    raw: string;
+    usage: JudgeUsage;
+}
+export interface JudgeVerdictJson<T> {
+    parsed: T;
+    raw: string;
+    usage: JudgeUsage;
+}
+export interface JudgeVerdictRaw {
+    raw: string;
+    usage: JudgeUsage;
+}
+export declare function judgeLLM(params: {
+    message: string;
+    rubric: string;
+    maxTokens?: number;
+}): Promise<JudgeVerdict>;
+export declare function judgeLLMJson<T>(params: {
+    message: string;
+    prompt: string;
+    returnType: string;
+    maxTokens?: number;
+}): Promise<JudgeVerdictJson<T>>;
+export declare function judgeLLMRaw(prompt: string, opts?: {
+    maxTokens?: number;
+}): Promise<JudgeVerdictRaw>;
+export declare function buildVerdictPrompt(params: {
+    message: string;
+    rubric: string;
+}): string;
+export declare function buildJsonPrompt(params: {
+    message: string;
+    prompt: string;
+    returnType: string;
+}): string;
+export declare function parseJudgeModelRef(ref: string): {
+    provider: "anthropic" | "openrouter";
+    bareModel: string;
+};

package/dist/judge.js

@@ -0,0 +1,183 @@
+import Anthropic from "@anthropic-ai/sdk";
+import OpenAI from "openai";
+import { extractTaggedBlock } from "./parse-tagged-json.js";
+const DEFAULT_JUDGE_MODEL = "anthropic/claude-haiku-4-5";
+const DEFAULT_MAX_TOKENS = 1024;
+const RESULT_TAG = "result-json";
+let cached;
+export async function judgeLLM(params) {
+    const prompt = buildVerdictPrompt({ message: params.message, rubric: params.rubric });
+    const { raw, usage } = await callJudge(prompt, params.maxTokens ?? DEFAULT_MAX_TOKENS);
+    const body = extractTaggedBlock(raw, RESULT_TAG);
+    const parsed = parseJson(body, raw);
+    const obj = parsed;
+    if (obj.verdict !== "pass" && obj.verdict !== "fail") {
+        throw new Error(`judge verdict invalid: ${JSON.stringify(obj.verdict)}. raw=${JSON.stringify(raw)}`);
+    }
+    if (typeof obj.reasoning !== "string") {
+        throw new Error(`judge reasoning not a string: ${JSON.stringify(obj.reasoning)}. raw=${JSON.stringify(raw)}`);
+    }
+    return { verdict: obj.verdict, reasoning: obj.reasoning, raw, usage };
+}
+export async function judgeLLMJson(params) {
+    const prompt = buildJsonPrompt({
+        message: params.message,
+        prompt: params.prompt,
+        returnType: params.returnType,
+    });
+    const { raw, usage } = await callJudge(prompt, params.maxTokens ?? DEFAULT_MAX_TOKENS);
+    const body = extractTaggedBlock(raw, RESULT_TAG);
+    const parsed = parseJson(body, raw);
+    return { parsed, raw, usage };
+}
+export async function judgeLLMRaw(prompt, opts) {
+    return await callJudge(prompt, opts?.maxTokens ?? DEFAULT_MAX_TOKENS);
+}
+export function buildVerdictPrompt(params) {
+    return `You are a strict QA judge. Evaluate the message below against the rubric. Return "pass" only when the message satisfies the rubric without violations; otherwise return "fail".
+
+<rubric>
+${params.rubric}
+</rubric>
+
+<message-to-evaluate>
+${params.message}
+</message-to-evaluate>
+
+Here is the return type we expect:
+
+<return-type>
+interface JudgeResult {
+  reasoning?: string; // concise, <=20 words
+  verdict: "pass" | "fail";
+}
+</return-type>
+
+You may write free-form prose first if useful. Then, emit the result JSON value in a \`<${RESULT_TAG}>\` tag.
+`;
+}
+export function buildJsonPrompt(params) {
+    return `${params.prompt}
+
+Message under evaluation:
+
+<message-to-evaluate>
+${params.message}
+</message-to-evaluate>
+
+Here is the return type we expect:
+
+<return-type>
+${params.returnType};
+</return-type>
+
+You may write free-form prose first if useful. Then, emit the result JSON value in a \`<${RESULT_TAG}>\` tag.
+`;
+}
+// The SDK's own retries (see `maxRetries` below) cap out around 30s of backoff;
+// observed container DNS/network outages last minutes and have killed most of a
+// batch at once. Between them, these outer delays hold a cell alive ~2 minutes.
+const CONNECTION_RETRY_DELAYS_MS = [15_000, 30_000, 60_000];
+async function callJudge(prompt, maxTokens) {
+    for (let attempt = 0;; ++attempt) {
+        try {
+            return await callJudgeOnce(prompt, maxTokens);
+        }
+        catch (err) {
+            const delayMs = CONNECTION_RETRY_DELAYS_MS[attempt];
+            if (delayMs === undefined || !isConnectionError(err))
+                throw err;
+            console.warn(`judge connection error, retrying in ${delayMs / 1000}s: ${String(err)}`);
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+        }
+    }
+}
+async function callJudgeOnce(prompt, maxTokens) {
+    const judge = getJudge();
+    if (judge.provider === "openrouter") {
+        return callOpenRouterOnce(judge, prompt, maxTokens);
+    }
+    const resp = await judge.client.messages.create({
+        model: judge.bareModel,
+        max_tokens: maxTokens,
+        temperature: 0,
+        messages: [{ role: "user", content: prompt }],
+    });
+    const raw = resp.content.map((b) => (b.type === "text" ? b.text : "")).join("");
+    if (!resp.usage)
+        throw new Error("judge response missing usage");
+    const usage = {
+        model: judge.model,
+        inputTokens: resp.usage.input_tokens,
+        outputTokens: resp.usage.output_tokens,
+    };
+    return { raw, usage };
+}
+async function callOpenRouterOnce(judge, prompt, maxTokens) {
+    const response = await judge.client.chat.completions.create({
+        model: judge.bareModel,
+        max_tokens: maxTokens,
+        temperature: 0,
+        messages: [{ role: "user", content: prompt }],
+    });
+    const raw = response.choices.map((choice) => choice.message.content ?? "").join("");
+    if (!response.usage)
+        throw new Error("judge response missing usage");
+    const usage = {
+        model: judge.model,
+        inputTokens: response.usage.prompt_tokens,
+        outputTokens: response.usage.completion_tokens,
+    };
+    return { raw, usage };
+}
+function parseJson(body, raw) {
+    try {
+        return JSON.parse(body);
+    }
+    catch (err) {
+        throw new Error(`judge JSON parse failed: ${err.message}. body=${JSON.stringify(body)} raw=${JSON.stringify(raw)}`);
+    }
+}
+function getJudge() {
+    if (cached)
+        return cached;
+    const model = process.env.OPENCLAW_TEST_JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL;
+    const { provider, bareModel } = parseJudgeModelRef(model);
+    // Container DNS blips (EAI_AGAIN) outlast the SDK's default 2 retries and have
+    // killed whole cells at once; 5 retries ride out a multi-second outage.
+    if (provider === "openrouter") {
+        const apiKey = process.env.OPENROUTER_API_KEY;
+        if (!apiKey) {
+            throw new Error("OPENROUTER_API_KEY is not set — fill in your project's .env.local");
+        }
+        cached = {
+            provider,
+            client: new OpenAI({ apiKey, baseURL: "https://openrouter.ai/api/v1", maxRetries: 5 }),
+            model,
+            bareModel,
+        };
+        return cached;
+    }
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey)
+        throw new Error("ANTHROPIC_API_KEY is not set — fill in your project's .env.local");
+    cached = { provider, client: new Anthropic({ apiKey, maxRetries: 5 }), model, bareModel };
+    return cached;
+}
+export function parseJudgeModelRef(ref) {
+    const slash = ref.indexOf("/");
+    if (slash < 0) {
+        throw new Error(`OPENCLAW_TEST_JUDGE_MODEL must be a LiteLLM-style "provider/model" reference (for example, "anthropic/claude-haiku-4-5" or "openrouter/anthropic/claude-haiku-4.5"); got ${JSON.stringify(ref)}`);
+    }
+    const provider = ref.slice(0, slash);
+    const bareModel = ref.slice(slash + 1);
+    if (provider !== "anthropic" && provider !== "openrouter") {
+        throw new Error(`OPENCLAW_TEST_JUDGE_MODEL provider ${JSON.stringify(provider)} is not supported; use "anthropic/" or "openrouter/"`);
+    }
+    if (!bareModel)
+        throw new Error(`OPENCLAW_TEST_JUDGE_MODEL model name is empty after the "${provider}/" prefix`);
+    return { provider, bareModel };
+}
+function isConnectionError(error) {
+    return (error instanceof Anthropic.APIConnectionError || error instanceof OpenAI.APIConnectionError);
+}

package/dist/loop.d.ts

@@ -0,0 +1,62 @@
+import { type ChildProcess } from "node:child_process";
+import { type CellResult } from "./cell-result.js";
+import type { SelectedModel } from "./models.js";
+export declare function expandChannelSelection(raw: string, openclawConfigPath: string): string[];
+export declare function discoverScenarios(scenariosDir: string): string[];
+export interface MatrixOptions {
+    scenarios: string[];
+    channels: string[];
+    models: SelectedModel[];
+    /**
+     * Renders the chosen model's ref into a run-scoped config and returns its path
+     * (memoized per model). Worker loops hand it to every spawn via per-spawn env.
+     */
+    renderConfigPath: (model: SelectedModel) => string;
+    iterations: number;
+    maxFailures: number;
+    /** Stop dispatching at the first failing cell; in-flight cells drain. */
+    stopOnFail: boolean;
+    reuseStack: boolean;
+    /** Pool size; a serial run is a pool of 1 (inherited stdio, no status lines). */
+    parallel: number;
+    workers: WorkerContext[];
+    /** Refresh the worker's private workspace copy; called before every cell. */
+    refreshWorkspace: (worker: WorkerContext) => void;
+    artifactsDir: string;
+    /** Host-side path to the results dir; the host reads `*.json` from here. */
+    resultsDir: string;
+    /** Container-side path passed to the runner via `--results-dir`. */
+    runnerResultsDir: string;
+    baseStamp: string;
+    spawnRunner?: SpawnFn;
+    spawnRecreate?: SpawnFn;
+    readResult?: (path: string) => CellResult | undefined;
+}
+export interface WorkerContext {
+    /** 1-based. */
+    index: number;
+    /** Compose base args + `["-p", "<base>-w<i>"]`. */
+    composeArgs: string[];
+    /** Host-side per-worker gateway logs dir, bind-mounted into the worker's gateway. */
+    gatewayLogsDir: string;
+    /** Host-side per-worker workspace copy, bind-mounted into the worker's gateway. */
+    workspaceDir: string;
+    /** True when the worker's bus+gateway were already running before this command. */
+    wasRunningBefore: boolean;
+}
+export type SpawnFn = (args: SpawnRequest) => Promise<number>;
+export interface SpawnRequest {
+    command: string;
+    argv: string[];
+    /** Extra env merged over `process.env` for this spawn only. */
+    env?: Record<string, string>;
+    /** When set, stdout+stderr are appended to this file instead of inherited. */
+    logFile?: string;
+    onChild?: (child: ChildProcess) => void;
+}
+/**
+ * Per-worker env for a compose spawn. Compose re-interpolates bind mounts on
+ * every invocation, so recreate, `run runner` and `up` must all carry it.
+ */
+export declare function workerSpawnEnv(worker: WorkerContext, configPath: string): Record<string, string>;
+export declare function runMatrix(opts: MatrixOptions): Promise<number>;

package/dist/loop.js

@@ -0,0 +1,316 @@
+import { spawn } from "node:child_process";
+import { closeSync, openSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { cellLeafName, readCellResult } from "./cell-result.js";
+import { judgeCostUsd } from "./cost.js";
+import { printSummary, printTotalCost } from "./summary.js";
+export function expandChannelSelection(raw, openclawConfigPath) {
+    if (!raw)
+        throw new Error("run: --channel expects a non-empty value");
+    const cfg = JSON.parse(readFileSync(openclawConfigPath, "utf8"));
+    const allowed = Object.keys(cfg.channels ?? {});
+    if (allowed.length === 0)
+        throw new Error(`no channels declared in ${openclawConfigPath}`);
+    if (raw === "all")
+        return [...allowed].sort();
+    const ids = raw
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+    if (ids.length === 0) {
+        throw new Error(`run: --channel expects a non-empty list, got ${JSON.stringify(raw)}`);
+    }
+    const unknown = ids.filter((c) => !allowed.includes(c));
+    if (unknown.length > 0) {
+        throw new Error(`unknown channel(s): ${unknown.join(", ")} — allowed: ${allowed.join(", ")}`);
+    }
+    return [...new Set(ids)];
+}
+export function discoverScenarios(scenariosDir) {
+    return readdirSync(scenariosDir)
+        .filter((f) => f.endsWith(".ts"))
+        .map((f) => f.slice(0, -".ts".length))
+        .sort();
+}
+/**
+ * Per-worker env for a compose spawn. Compose re-interpolates bind mounts on
+ * every invocation, so recreate, `run runner` and `up` must all carry it.
+ */
+export function workerSpawnEnv(worker, configPath) {
+    return {
+        OPENCLAW_CONFIG_PATH: configPath,
+        OPENCLAW_TEST_GATEWAY_LOGS_DIR: worker.gatewayLogsDir,
+        OPENCLAW_WORKSPACE_DIR: worker.workspaceDir,
+    };
+}
+export async function runMatrix(opts) {
+    const state = {
+        pending: expandCells(opts),
+        runningPerModel: new Map(),
+        pairFailures: new Map(),
+        bailedPairs: new Set(),
+        stopped: false,
+        signalled: false,
+        liveChildren: new Set(),
+        records: [],
+        exitCode: 0,
+    };
+    const ctx = {
+        opts,
+        state,
+        spawnRunner: opts.spawnRunner ?? defaultSpawn,
+        spawnRecreate: opts.spawnRecreate ?? defaultSpawn,
+        readResult: opts.readResult ?? readCellResult,
+        iterationWidth: opts.iterations > 1 ? String(opts.iterations).length : 0,
+    };
+    const onSignal = (sig) => {
+        state.stopped = true;
+        state.signalled = true;
+        for (const child of state.liveChildren)
+            child.kill(sig);
+    };
+    process.on("SIGINT", onSignal);
+    process.on("SIGTERM", onSignal);
+    try {
+        await Promise.all(opts.workers.map((worker) => workerLoop(ctx, worker)));
+    }
+    finally {
+        process.off("SIGINT", onSignal);
+        process.off("SIGTERM", onSignal);
+    }
+    const results = [...state.records]
+        .sort((a, b) => a.cell.index - b.cell.index)
+        .map((r) => r.result);
+    printSummary(results, opts.baseStamp);
+    printTotalCost(results);
+    if (state.exitCode !== 0)
+        return state.exitCode;
+    return results.some((r) => r.verdict === "fail") ? 1 : 0;
+}
+/** Full expansion in `model × scenario × channel × iteration` nesting order. */
+function expandCells(opts) {
+    const cells = [];
+    for (const model of opts.models) {
+        for (const scenario of opts.scenarios) {
+            for (const channel of opts.channels) {
+                for (let iter = 1; iter <= opts.iterations; ++iter) {
+                    cells.push({
+                        index: cells.length,
+                        model,
+                        scenario,
+                        channel,
+                        iteration: iter,
+                        pairKey: `${model.id}|${scenario}|${channel}`,
+                    });
+                }
+            }
+        }
+    }
+    return cells;
+}
+async function workerLoop(ctx, worker) {
+    const { opts, state } = ctx;
+    // The model the worker's gateway booted on. Workers are created lazily: the
+    // first cell's `up -d --force-recreate` also creates a stack that isn't running.
+    let loadedModelId;
+    for (;;) {
+        const cell = takeNextCell(state, opts.reuseStack ? loadedModelId : undefined);
+        if (!cell)
+            return;
+        changeRunningCount(state, cell.model.id, 1);
+        try {
+            const recreated = await runCell(ctx, worker, cell, loadedModelId);
+            if (recreated)
+                loadedModelId = cell.model.id;
+        }
+        finally {
+            changeRunningCount(state, cell.model.id, -1);
+        }
+    }
+}
+/**
+ * Skip bailed pairs. Under `--reuse-stack` (`loadedModelId` given), first look for
+ * a pending cell matching the loaded model to avoid the recreate; otherwise pick
+ * the pending cell whose model has the fewest running cells (spreads providers,
+ * softens rate-limit bursts; tie → expansion order).
+ */
+function takeNextCell(state, loadedModelId) {
+    if (state.stopped)
+        return;
+    state.pending = state.pending.filter((c) => !state.bailedPairs.has(c.pairKey));
+    if (state.pending.length === 0)
+        return;
+    const affinity = loadedModelId !== undefined
+        ? state.pending.find((c) => c.model.id === loadedModelId)
+        : undefined;
+    const cell = affinity ?? leastLoadedModelCell(state);
+    state.pending.splice(state.pending.indexOf(cell), 1);
+    return cell;
+}
+// Linear scan is fine: `pending` is small and shrinks per take.
+function leastLoadedModelCell(state) {
+    let best = state.pending[0];
+    let bestCount = state.runningPerModel.get(best.model.id) ?? 0;
+    for (const cell of state.pending) {
+        const count = state.runningPerModel.get(cell.model.id) ?? 0;
+        if (count < bestCount) {
+            best = cell;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+function changeRunningCount(state, modelId, delta) {
+    state.runningPerModel.set(modelId, (state.runningPerModel.get(modelId) ?? 0) + delta);
+}
+/** Returns true when the worker's stack was (re)created for this cell. */
+async function runCell(ctx, worker, cell, loadedModelId) {
+    const { opts, state } = ctx;
+    const leaf = cellLeafName({
+        scenarioId: cell.scenario,
+        modelId: cell.model.id,
+        channel: cell.channel,
+        iterationIndex: cell.iteration,
+        iterationWidth: ctx.iterationWidth,
+    });
+    const env = workerSpawnEnv(worker, opts.renderConfigPath(cell.model));
+    const logFile = opts.parallel > 1 ? join(opts.resultsDir, `${leaf}.log`) : undefined;
+    if (opts.parallel > 1)
+        console.log(`[w${worker.index}] ${leaf} started`);
+    // Refresh even on the reuse-stack no-recreate path: the live gateway reads the
+    // workspace lazily, so this is intentional despite the reuse-stack state-leak caveat.
+    opts.refreshWorkspace(worker);
+    const needRecreate = !opts.reuseStack || loadedModelId !== cell.model.id;
+    if (needRecreate) {
+        const code = await spawnTracked(ctx.spawnRecreate, state, {
+            command: "docker",
+            argv: [...worker.composeArgs, "up", "-d", "--force-recreate", "--wait", "bus", "gateway"],
+            env,
+            logFile,
+        });
+        if (state.signalled)
+            return false;
+        if (code !== 0) {
+            state.stopped = true;
+            state.exitCode = code;
+            if (opts.parallel > 1) {
+                console.error(`[w${worker.index}] ${leaf} bus+gateway recreate failed (exit ${code})`);
+            }
+            return false;
+        }
+    }
+    const resultsPath = join(opts.resultsDir, `${leaf}.json`);
+    const childExit = await spawnTracked(ctx.spawnRunner, state, {
+        command: "docker",
+        argv: [
+            ...worker.composeArgs,
+            "run",
+            "--rm",
+            "--use-aliases",
+            "runner",
+            "--scenario",
+            cell.scenario,
+            "--channel",
+            cell.channel,
+            "--model-id",
+            cell.model.id,
+            "--model-ref",
+            cell.model.ref,
+            "--iteration-index",
+            String(cell.iteration),
+            "--iteration-width",
+            String(ctx.iterationWidth),
+            "--base-stamp",
+            opts.baseStamp,
+            "--results-dir",
+            opts.runnerResultsDir,
+        ],
+        env,
+        logFile,
+    });
+    const result = ctx.readResult(resultsPath) ??
+        synthesizeFailedResult({
+            scenario: cell.scenario,
+            channel: cell.channel,
+            model: cell.model.id,
+            iter: cell.iteration,
+            resultsPath,
+            childExit,
+        });
+    state.records.push({ cell, result });
+    if (opts.parallel > 1)
+        console.log(formatCellDone(worker.index, leaf, result));
+    if (result.verdict === "fail") {
+        if (opts.stopOnFail)
+            state.stopped = true;
+        const failures = (state.pairFailures.get(cell.pairKey) ?? 0) + 1;
+        state.pairFailures.set(cell.pairKey, failures);
+        if (failures > opts.maxFailures)
+            state.bailedPairs.add(cell.pairKey);
+    }
+    return needRecreate;
+}
+async function spawnTracked(fn, state, req) {
+    return fn({
+        ...req,
+        onChild: (child) => {
+            state.liveChildren.add(child);
+            child.on("exit", () => state.liveChildren.delete(child));
+        },
+    });
+}
+function formatCellDone(workerIndex, leaf, r) {
+    const verdict = r.verdict === "pass" ? "PASS" : "FAIL";
+    const judgeCost = r.judgeUsages.reduce((sum, u) => sum + judgeCostUsd(u), 0);
+    const cost = r.agentCostUsd + judgeCost;
+    return `[w${workerIndex}] ${leaf} ${verdict} (${Math.round(r.durationMs / 1000)}s, $${cost.toFixed(4)})`;
+}
+function defaultSpawn(req) {
+    return new Promise((resolve) => {
+        const env = req.env ? { ...process.env, ...req.env } : process.env;
+        const logFd = req.logFile === undefined ? undefined : openSync(req.logFile, "a");
+        const child = spawn(req.command, req.argv, {
+            env,
+            stdio: logFd === undefined ? "inherit" : ["ignore", logFd, logFd],
+        });
+        // The child holds dups of the fd; safe to close ours right away.
+        if (logFd !== undefined)
+            closeSync(logFd);
+        req.onChild?.(child);
+        child.on("exit", (code, signal) => {
+            if (signal)
+                resolve(128 + signalNumber(signal));
+            else
+                resolve(code ?? 1);
+        });
+        child.on("error", (err) => {
+            console.error(err.message);
+            resolve(1);
+        });
+    });
+}
+function signalNumber(signal) {
+    if (signal === "SIGINT")
+        return 2;
+    if (signal === "SIGTERM")
+        return 15;
+    return 1;
+}
+function synthesizeFailedResult(params) {
+    console.warn(`run: missing or invalid cell record at ${params.resultsPath} (runner exit ${params.childExit}) — counting as fail`);
+    return {
+        schemaVersion: 3,
+        scenarioId: params.scenario,
+        channel: params.channel,
+        model: params.model,
+        iterationIndex: params.iter,
+        verdict: "fail",
+        durationMs: 0,
+        conversationId: "",
+        artifactDirName: "",
+        agentCostUsd: 0,
+        agentTurns: 0,
+        judgeUsd: 0,
+        judgeUsages: [],
+    };
+}