@crewhaus/eval-runner 0.1.3 → 0.1.5

package/dist/wire-once.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Build the agent stack ONCE per eval run, then share across samples.
+ *
+ * This factors the runRun logic from apps/cli/src/index.ts so the CLI's
+ * `crewhaus run` and the eval runner's `crewhaus eval` use the same
+ * tool/hook/skill/MCP/sub-agent wiring. Single source of truth for the
+ * "what is the full agent stack from an IR" question.
+ *
+ * The MCP host (if any) is shared across all eval samples — re-spinning
+ * stdio MCP servers per sample for 200 samples would burn ~30s in process
+ * startup and exceed the T7 SLO. The trade-off is documented: eval-runner
+ * assumes the agent's MCP usage is read-mostly. An `isolateMcpPerSample`
+ * escape hatch is reserved for a future where this matters.
+ */
+import type { SubAgentDefinition } from "@crewhaus/agent-context-isolation";
+import { type HookDef } from "@crewhaus/hooks-engine";
+import type { IrV0 } from "@crewhaus/ir";
+import { McpHost } from "@crewhaus/mcp-host";
+import { type RuleSet } from "@crewhaus/permission-engine";
+import { type SkillRef } from "@crewhaus/skills-registry";
+import { type SlashCommand } from "@crewhaus/slash-commands";
+import { spawnSubAgent } from "@crewhaus/sub-agent-spawner";
+import { type RegisteredTool } from "@crewhaus/tool-catalog";
+type SpawnSubAgentFn = typeof spawnSubAgent;
+export type SharedAgentDeps = {
+    readonly tools: ReadonlyArray<RegisteredTool>;
+    readonly hooks: ReadonlyArray<HookDef>;
+    readonly skills: ReadonlyArray<SkillRef>;
+    readonly slashCommands: ReadonlyMap<string, SlashCommand>;
+    readonly subAgents?: ReadonlyMap<string, SubAgentDefinition>;
+    readonly spawnSubAgent?: SpawnSubAgentFn;
+    readonly permissionRules: RuleSet;
+    readonly mcpHost?: McpHost;
+    readonly model: string;
+    readonly instructions: string;
+    readonly sessionName: string;
+    readonly sessionTarget: string;
+};
+export declare function wireRunOnce(ir: IrV0, opts?: {
+    cwd?: string;
+}): Promise<SharedAgentDeps>;
+export {};

package/dist/wire-once.js

@@ -0,0 +1,150 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { loadHooks } from "@crewhaus/hooks-engine";
+import { createLogger } from "@crewhaus/logging";
+import { McpHost } from "@crewhaus/mcp-host";
+import { BUILTIN_DEFAULT_RULES, PermissionConfigError, parsePermissionsConfig, tagRules, } from "@crewhaus/permission-engine";
+import { createSkillTool, discoverSkills } from "@crewhaus/skills-registry";
+import { loadCommands } from "@crewhaus/slash-commands";
+import { spawnSubAgent } from "@crewhaus/sub-agent-spawner";
+import { ToolCatalog } from "@crewhaus/tool-catalog";
+import { registerMcpServer } from "@crewhaus/tool-mcp";
+import { createTaskTool } from "@crewhaus/tool-task";
+import { RunnerError } from "./errors";
+const logger = createLogger({ bindings: { module: "eval-runner.wire" } });
+export async function wireRunOnce(ir, opts = {}) {
+    const cwd = opts.cwd ?? process.cwd();
+    // Tools.
+    let tools = [];
+    if (ir.tools.length > 0) {
+        await applyToolConfigs(ir.tools, ir.toolConfigs);
+        const toolMap = await loadToolMap();
+        tools = ir.tools.map((name) => {
+            const tool = toolMap[name];
+            if (!tool) {
+                const known = Object.keys(toolMap).sort().join(", ");
+                throw new RunnerError(`unknown tool "${name}" — known tools: ${known}`);
+            }
+            return tool;
+        });
+    }
+    // MCP servers (shared across samples).
+    let mcpHost;
+    if (Object.keys(ir.mcp_servers).length > 0) {
+        const host = new McpHost({ logger });
+        mcpHost = host;
+        for (const [name, cfg] of Object.entries(ir.mcp_servers))
+            host.addServer(name, cfg);
+        const tempCatalog = new ToolCatalog();
+        for (const t of tools)
+            tempCatalog.register(t);
+        await Promise.all(Object.keys(ir.mcp_servers).map((name) => registerMcpServer(host, name, tempCatalog)));
+        tools = tempCatalog.list().slice();
+    }
+    // Permission rules.
+    const permissionRules = buildRuleSet(ir.permissions.rules, cwd);
+    // Hooks / skills / slash-commands.
+    const [hooks, skills, slashCommands] = await Promise.all([
+        loadHooks({ cwd }),
+        discoverSkills({ cwd }),
+        loadCommands({ cwd }),
+    ]);
+    if (skills.length > 0)
+        tools.push(createSkillTool(skills));
+    // Sub-agents.
+    let subAgents;
+    if (ir.subAgents.length > 0) {
+        subAgents = new Map(ir.subAgents.map((d) => [
+            d.name,
+            {
+                name: d.name,
+                description: d.description,
+                instructions: d.instructions,
+                tools: d.tools,
+                ...(d.model !== undefined ? { model: d.model } : {}),
+                permissions: d.permissions,
+                inherit_bypass: d.inheritBypass,
+            },
+        ]));
+        tools.push(createTaskTool({ subAgents }));
+    }
+    return {
+        tools,
+        hooks,
+        skills,
+        slashCommands,
+        permissionRules,
+        model: ir.agent.model,
+        instructions: ir.agent.instructions,
+        sessionName: ir.name,
+        sessionTarget: ir.target,
+        ...(subAgents !== undefined ? { subAgents, spawnSubAgent } : {}),
+        ...(mcpHost !== undefined ? { mcpHost } : {}),
+    };
+}
+async function loadToolMap() {
+    const [fs, bash, todo, web, image, fetchPkg] = await Promise.all([
+        import("@crewhaus/tool-fs"),
+        import("@crewhaus/tool-bash"),
+        import("@crewhaus/tool-todo"),
+        import("@crewhaus/tool-web"),
+        import("@crewhaus/tool-image"),
+        import("@crewhaus/tool-fetch"),
+    ]);
+    return {
+        read: fs.read,
+        write: fs.write,
+        edit: fs.edit,
+        glob: fs.glob,
+        grep: fs.grep,
+        bash: bash.bash,
+        todoWrite: todo.todoWrite,
+        webFetch: web.webFetch,
+        webSearch: web.webSearch,
+        readImage: image.readImage,
+        fetch: fetchPkg.fetch,
+    };
+}
+async function applyToolConfigs(toolNames, toolConfigs) {
+    const used = new Set(toolNames);
+    if (used.has("fetch") && toolConfigs["fetch"] !== undefined) {
+        const { registerFetchConfig } = await import("@crewhaus/tool-fetch");
+        registerFetchConfig(toolConfigs["fetch"]);
+    }
+    if (used.has("webFetch") && toolConfigs["webFetch"] !== undefined) {
+        const { registerWebFetchConfig } = await import("@crewhaus/tool-web");
+        registerWebFetchConfig(toolConfigs["webFetch"]);
+    }
+}
+function buildRuleSet(yamlRules, cwd) {
+    let settings = [];
+    const settingsPath = join(cwd, ".crewhaus", "settings.json");
+    if (existsSync(settingsPath)) {
+        let raw;
+        try {
+            raw = JSON.parse(readFileSync(settingsPath, "utf-8"));
+        }
+        catch (err) {
+            throw new RunnerError(`failed to parse ${settingsPath}: ${err.message}`, err);
+        }
+        const root = raw.permissions;
+        if (root !== undefined) {
+            try {
+                const parsed = parsePermissionsConfig(root, "settings");
+                settings = tagRules(parsed.rules, "settings");
+            }
+            catch (err) {
+                if (err instanceof PermissionConfigError)
+                    throw new RunnerError(err.message, err);
+                throw err;
+            }
+        }
+    }
+    return {
+        flag: [],
+        settings,
+        yaml: tagRules(yamlRules, "yaml"),
+        hooks: [],
+        builtin: BUILTIN_DEFAULT_RULES,
+    };
+}

package/package.json

@@ -1,46 +1,49 @@
 {
   "name": "@crewhaus/eval-runner",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "type": "module",
   "description": "Eval runner — per-sample isolation, concurrency, grading, persistence",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "scripts": {
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/agent-context-isolation": "0.1.3",
-    "@crewhaus/compiler": "0.1.3",
-    "@crewhaus/errors": "0.1.3",
-    "@crewhaus/eval-dataset": "0.1.3",
-    "@crewhaus/eval-grader": "0.1.3",
-    "@crewhaus/eval-judge": "0.1.3",
-    "@crewhaus/event-log": "0.1.3",
-    "@crewhaus/hooks-engine": "0.1.3",
-    "@crewhaus/ir": "0.1.3",
-    "@crewhaus/logging": "0.1.3",
-    "@crewhaus/mcp-host": "0.1.3",
-    "@crewhaus/permission-engine": "0.1.3",
-    "@crewhaus/run-context": "0.1.3",
-    "@crewhaus/runtime-core": "0.1.3",
-    "@crewhaus/skills-registry": "0.1.3",
-    "@crewhaus/slash-commands": "0.1.3",
-    "@crewhaus/spec": "0.1.3",
-    "@crewhaus/sub-agent-spawner": "0.1.3",
-    "@crewhaus/tenancy": "0.1.3",
-    "@crewhaus/tool-bash": "0.1.3",
-    "@crewhaus/tool-catalog": "0.1.3",
-    "@crewhaus/tool-fetch": "0.1.3",
-    "@crewhaus/tool-fs": "0.1.3",
-    "@crewhaus/tool-image": "0.1.3",
-    "@crewhaus/tool-mcp": "0.1.3",
-    "@crewhaus/tool-task": "0.1.3",
-    "@crewhaus/tool-todo": "0.1.3",
-    "@crewhaus/tool-web": "0.1.3",
-    "@crewhaus/trace-event-bus": "0.1.3",
+    "@crewhaus/agent-context-isolation": "0.1.5",
+    "@crewhaus/compiler": "0.1.5",
+    "@crewhaus/errors": "0.1.5",
+    "@crewhaus/eval-dataset": "0.1.5",
+    "@crewhaus/eval-grader": "0.1.5",
+    "@crewhaus/eval-judge": "0.1.5",
+    "@crewhaus/event-log": "0.1.5",
+    "@crewhaus/hooks-engine": "0.1.5",
+    "@crewhaus/ir": "0.1.5",
+    "@crewhaus/logging": "0.1.5",
+    "@crewhaus/mcp-host": "0.1.5",
+    "@crewhaus/permission-engine": "0.1.5",
+    "@crewhaus/run-context": "0.1.5",
+    "@crewhaus/runtime-core": "0.1.5",
+    "@crewhaus/skills-registry": "0.1.5",
+    "@crewhaus/slash-commands": "0.1.5",
+    "@crewhaus/spec": "0.1.5",
+    "@crewhaus/sub-agent-spawner": "0.1.5",
+    "@crewhaus/tenancy": "0.1.5",
+    "@crewhaus/tool-bash": "0.1.5",
+    "@crewhaus/tool-catalog": "0.1.5",
+    "@crewhaus/tool-fetch": "0.1.5",
+    "@crewhaus/tool-fs": "0.1.5",
+    "@crewhaus/tool-image": "0.1.5",
+    "@crewhaus/tool-mcp": "0.1.5",
+    "@crewhaus/tool-task": "0.1.5",
+    "@crewhaus/tool-todo": "0.1.5",
+    "@crewhaus/tool-web": "0.1.5",
+    "@crewhaus/trace-event-bus": "0.1.5",
     "zod": "^3.23.8"
   },
   "license": "Apache-2.0",
@@ -61,5 +64,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "files": ["src", "README.md", "LICENSE", "NOTICE"]
+  "files": ["dist", "README.md", "LICENSE", "NOTICE"]
 }

package/src/aggregate.ts

@@ -1,48 +0,0 @@
-import type { SampleResult } from "./types";
-
-export function quantile(sorted: ReadonlyArray<number>, q: number): number {
-  if (sorted.length === 0) return 0;
-  if (sorted.length === 1) return sorted[0] ?? 0;
-  const idx = q * (sorted.length - 1);
-  const lo = Math.floor(idx);
-  const hi = Math.ceil(idx);
-  if (lo === hi) return sorted[lo] ?? 0;
-  const fract = idx - lo;
-  return (sorted[lo] ?? 0) * (1 - fract) + (sorted[hi] ?? 0) * fract;
-}
-
-export function aggregate(samples: ReadonlyArray<SampleResult>): {
-  passRate: number;
-  meanScore: number;
-  p50Turns: number;
-  p95Turns: number;
-  p50LatencyMs: number;
-  p95LatencyMs: number;
-  totalTokens: { input: number; output: number };
-  errorCount: number;
-} {
-  const ok = samples.filter((s) => s.error === undefined);
-  const total = samples.length;
-  const passed = ok.filter((s) => s.grades.overall.passed).length;
-  const meanScore =
-    ok.length === 0 ? 0 : ok.reduce((sum, s) => sum + s.grades.overall.score, 0) / ok.length;
-  const turnsSorted = ok.map((s) => s.turns).sort((a, b) => a - b);
-  const latSorted = ok.map((s) => s.latencyMs).sort((a, b) => a - b);
-  const totalTokens = ok.reduce(
-    (acc, s) => ({
-      input: acc.input + s.tokens.input,
-      output: acc.output + s.tokens.output,
-    }),
-    { input: 0, output: 0 },
-  );
-  return {
-    passRate: total === 0 ? 0 : passed / total,
-    meanScore,
-    p50Turns: quantile(turnsSorted, 0.5),
-    p95Turns: quantile(turnsSorted, 0.95),
-    p50LatencyMs: quantile(latSorted, 0.5),
-    p95LatencyMs: quantile(latSorted, 0.95),
-    totalTokens,
-    errorCount: samples.length - ok.length,
-  };
-}

package/src/errors.ts

@@ -1,7 +0,0 @@
-import { RuntimeError } from "@crewhaus/errors";
-
-export class RunnerError extends RuntimeError {
-  constructor(message: string, cause?: unknown) {
-    super(`eval-runner: ${message}`, cause);
-  }
-}

package/src/index.default-invoker.test.ts

@@ -1,172 +0,0 @@
-/**
- * Isolated test for `runEval`'s built-in `defaultInvoker` — the production
- * path taken when the caller supplies no `opts.invoker`. It wires the agent
- * stack once via `wireRunOnce`, then drives `runChatLoop` per sample.
- *
- * We stub both `./wire-once` and `@crewhaus/runtime-core` so nothing real is
- * spun up (no tools, no MCP, no model call). The stub `runChatLoop` records
- * the options it was handed so we can assert the per-sample session name, the
- * forced `permissionMode: "auto"`, `singleTurn`, and the seeded user message.
- * `mock.module` is process-global, so this lives in its own file.
- */
-import { afterAll, describe, expect, mock, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { lower } from "@crewhaus/compiler";
-import type { Sample } from "@crewhaus/eval-dataset";
-import { parseGradersConfig } from "@crewhaus/eval-grader";
-import type { IrNode, IrV0 } from "@crewhaus/ir";
-import { parseSpec } from "@crewhaus/spec";
-
-type ChatLoopCall = Record<string, unknown>;
-const chatLoopCalls: ChatLoopCall[] = [];
-const wireCalls: Array<{ cwd?: string }> = [];
-
-// Capture real modules so `afterAll` can restore them — `mock.module` is
-// process-global and does not auto-restore across test files. Each capture is
-// a plain-object SNAPSHOT (`{ ...ns }`): an ESM namespace is a live view that
-// resolves to the stubs once mock.module patches the module, so restoring
-// from the namespace itself would silently reinstall the stubs.
-const realWireOnce = { ...(await import("./wire-once")) };
-const realRuntimeCore = { ...(await import("@crewhaus/runtime-core")) };
-
-// Toggle whether the wired deps include sub-agents (drives the optional
-// subAgents/spawnSubAgent spread in defaultInvoker).
-let includeSubAgents = false;
-
-function baseDeps() {
-  return {
-    tools: [],
-    hooks: [],
-    skills: [],
-    slashCommands: new Map(),
-    permissionRules: { flag: [], settings: [], yaml: [], hooks: [], builtin: [] },
-    model: "claude-wired",
-    instructions: "wired instructions",
-    sessionName: "wired-session",
-    sessionTarget: "cli",
-  };
-}
-
-mock.module("./wire-once", () => ({
-  wireRunOnce: async (_ir: unknown, opts: { cwd?: string } = {}) => {
-    wireCalls.push(opts);
-    if (includeSubAgents) {
-      return {
-        ...baseDeps(),
-        subAgents: new Map([["helper", { name: "helper" }]]),
-        spawnSubAgent: () => undefined,
-      };
-    }
-    return baseDeps();
-  },
-}));
-
-mock.module("@crewhaus/runtime-core", () => ({
-  ...realRuntimeCore,
-  runChatLoop: async (opts: ChatLoopCall) => {
-    chatLoopCalls.push(opts);
-    return `answer for ${(opts["seedMessages"] as Array<{ content: string }>)[0]?.content}`;
-  },
-}));
-
-const { runEval } = await import("./index");
-
-function narrowToAgent(ir: IrNode): IrV0 {
-  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
-  return ir;
-}
-
-const SPEC = `name: default-invoker-test
-target: cli
-agent:
-  model: claude-opus-4-7
-  instructions: spec instructions
-`;
-
-async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
-  for (const s of samples) yield s;
-}
-
-const TMP_ROOTS: string[] = [];
-function newTempRoot(): string {
-  const dir = mkdtempSync(join(tmpdir(), "crewhaus-definvoker-"));
-  TMP_ROOTS.push(dir);
-  return dir;
-}
-afterAll(() => {
-  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
-  mock.module("./wire-once", () => realWireOnce);
-  mock.module("@crewhaus/runtime-core", () => realRuntimeCore);
-});
-
-describe("runEval — defaultInvoker (no caller invoker)", () => {
-  test("wires once, drives runChatLoop per sample (no sub-agents)", async () => {
-    const outDir = newTempRoot();
-    chatLoopCalls.length = 0;
-    wireCalls.length = 0;
-    includeSubAgents = false;
-
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = [
-      { id: "a", input: "first", expected_output: "answer for first" },
-      { id: "b", input: "second", expected_output: "answer for second" },
-    ];
-    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
-
-    const summary = await runEval({
-      ir,
-      dataset: { name: "def", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      // No invoker → defaultInvoker path. cwd threads into wireRunOnce.
-      opts: { outDir, cwd: "/tmp/some-cwd" },
-    });
-
-    // wireRunOnce called exactly once, with the cwd.
-    expect(wireCalls).toHaveLength(1);
-    expect(wireCalls[0]).toEqual({ cwd: "/tmp/some-cwd" });
-
-    // runChatLoop invoked per sample with the wired stack + forced options.
-    expect(chatLoopCalls).toHaveLength(2);
-    const call = chatLoopCalls.find(
-      (c) => (c["seedMessages"] as Array<{ content: string }>)[0]?.content === "first",
-    );
-    expect(call?.["permissionMode"]).toBe("auto");
-    expect(call?.["singleTurn"]).toBe(true);
-    expect(call?.["model"]).toBe("claude-wired");
-    expect(call?.["instructions"]).toBe("wired instructions");
-    expect(call?.["sessionName"]).toBe("wired-session_a");
-    expect(call?.["subAgents"]).toBeUndefined();
-    expect(call?.["spawnSubAgent"]).toBeUndefined();
-
-    // The agentOutput flowed back through to grading (exact_match passes).
-    expect(summary.aggregates.passRate).toBe(1);
-    expect(summary.samples.find((s) => s.sampleId === "a")?.agentOutput).toBe("answer for first");
-  });
-
-  test("threads sub-agents through to runChatLoop when wired", async () => {
-    const outDir = newTempRoot();
-    chatLoopCalls.length = 0;
-    wireCalls.length = 0;
-    includeSubAgents = true;
-
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = [{ id: "a", input: "q", expected_output: "answer for q" }];
-    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
-
-    await runEval({
-      ir,
-      dataset: { name: "def-sub", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      // No cwd → wireRunOnce receives `{}` (the cwd-absent spread branch).
-      opts: { outDir },
-    });
-
-    expect(wireCalls[0]).toEqual({});
-    expect(chatLoopCalls).toHaveLength(1);
-    const call = chatLoopCalls[0];
-    expect(call?.["subAgents"]).toBeInstanceOf(Map);
-    expect(typeof call?.["spawnSubAgent"]).toBe("function");
-  });
-});

package/src/index.interrupt.test.ts

@@ -1,99 +0,0 @@
-/**
- * Isolated test for the SIGINT mid-run interrupt path in `runEval`.
- *
- * Lives in its own file because it emits a process-level `SIGINT`: keeping it
- * apart from the rest of the suite ensures the emitted signal only interacts
- * with the handler `runEval` itself registers (and removes) for this run.
- *
- * The interrupt check sits *after* `sem.acquire()`: every sample callback's
- * synchronous prefix runs during `samples.map(...)`, before any SIGINT can
- * fire, so a pre-acquire check would never observe a mid-run interrupt. With
- * `concurrency: 1`, sample s0 holds the only slot while the rest queue; firing
- * SIGINT inside s0's invoker means each later sample is skipped as its turn
- * comes — exercising both the interrupt throw and the rejected→SampleResult
- * mapping.
- */
-import { afterAll, describe, expect, test } from "bun:test";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { lower } from "@crewhaus/compiler";
-import type { Sample } from "@crewhaus/eval-dataset";
-import { parseGradersConfig } from "@crewhaus/eval-grader";
-import type { IrNode, IrV0 } from "@crewhaus/ir";
-import { parseSpec } from "@crewhaus/spec";
-import { type AgentInvoker, runEval } from "./index";
-
-function narrowToAgent(ir: IrNode): IrV0 {
-  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
-  return ir;
-}
-
-const SPEC = `name: interrupt-test
-target: cli
-agent:
-  model: claude-opus-4-7
-  instructions: hi
-`;
-
-async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
-  for (const s of samples) yield s;
-}
-
-const TMP_ROOTS: string[] = [];
-function newTempRoot(): string {
-  const dir = mkdtempSync(join(tmpdir(), "crewhaus-interrupt-"));
-  TMP_ROOTS.push(dir);
-  return dir;
-}
-afterAll(() => {
-  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
-});
-
-describe("runEval — SIGINT mid-run interrupt", () => {
-  test("interrupts still-queued samples and maps them to failed results", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = Array.from({ length: 4 }).map((_, i) => ({
-      id: `s${i}`,
-      input: "x",
-      expected_output: "y",
-    }));
-
-    let count = 0;
-    const invoker: AgentInvoker = async ({ sample }) => {
-      count += 1;
-      // Fire SIGINT while the first sample holds the single concurrency slot;
-      // the rest are queued and will observe `interrupted` as they dequeue.
-      if (count === 1) process.emit("SIGINT" as NodeJS.Signals);
-      return { agentOutput: sample.expected_output ?? "", events: [] };
-    };
-
-    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
-    const summary = await runEval({
-      ir,
-      dataset: { name: "interrupt", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      opts: { invoker, outDir, concurrency: 1 },
-    });
-
-    // s0 completed; s1..s3 were interrupted before running.
-    expect(summary.samples).toHaveLength(4);
-    expect(summary.samples.find((s) => s.sampleId === "s0")?.grades.overall.passed).toBe(true);
-    const interrupted = summary.samples.filter((s) => s.error?.includes("interrupted"));
-    expect(interrupted.length).toBe(3);
-    expect(summary.aggregates.errorCount).toBe(3);
-
-    // The mapped failure carries the canonical placeholder fields.
-    const s3 = summary.samples.find((s) => s.sampleId === "s3");
-    expect(s3?.sessionId).toBe("(unset)");
-    expect(s3?.latencyMs).toBe(0);
-    expect(s3?.turns).toBe(0);
-    expect(s3?.tokens).toEqual({ input: 0, output: 0 });
-    expect(s3?.grades.overall.rationale).toBe("sample failed entirely");
-
-    // results.json was still persisted despite the interrupt.
-    const results = JSON.parse(readFileSync(join(outDir, "results.json"), "utf-8"));
-    expect(results.aggregates.errorCount).toBe(3);
-  });
-});