@crewhaus/eval-runner 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/eval-runner",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "description": "Eval runner — per-sample isolation, concurrency, grading, persistence",
   "main": "src/index.ts",
@@ -12,41 +12,42 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/agent-context-isolation": "0.0.0",
-    "@crewhaus/compiler": "0.0.0",
-    "@crewhaus/errors": "0.0.0",
-    "@crewhaus/eval-dataset": "0.0.0",
-    "@crewhaus/eval-grader": "0.0.0",
-    "@crewhaus/eval-judge": "0.0.0",
-    "@crewhaus/event-log": "0.0.0",
-    "@crewhaus/hooks-engine": "0.0.0",
-    "@crewhaus/ir": "0.0.0",
-    "@crewhaus/logging": "0.0.0",
-    "@crewhaus/mcp-host": "0.0.0",
-    "@crewhaus/permission-engine": "0.0.0",
-    "@crewhaus/run-context": "0.0.0",
-    "@crewhaus/runtime-core": "0.0.0",
-    "@crewhaus/skills-registry": "0.0.0",
-    "@crewhaus/slash-commands": "0.0.0",
-    "@crewhaus/spec": "0.0.0",
-    "@crewhaus/sub-agent-spawner": "0.0.0",
-    "@crewhaus/tool-bash": "0.0.0",
-    "@crewhaus/tool-catalog": "0.0.0",
-    "@crewhaus/tool-fetch": "0.0.0",
-    "@crewhaus/tool-fs": "0.0.0",
-    "@crewhaus/tool-image": "0.0.0",
-    "@crewhaus/tool-mcp": "0.0.0",
-    "@crewhaus/tool-task": "0.0.0",
-    "@crewhaus/tool-todo": "0.0.0",
-    "@crewhaus/tool-web": "0.0.0",
-    "@crewhaus/trace-event-bus": "0.0.0",
+    "@crewhaus/agent-context-isolation": "0.1.2",
+    "@crewhaus/compiler": "0.1.2",
+    "@crewhaus/errors": "0.1.2",
+    "@crewhaus/eval-dataset": "0.1.2",
+    "@crewhaus/eval-grader": "0.1.2",
+    "@crewhaus/eval-judge": "0.1.2",
+    "@crewhaus/event-log": "0.1.2",
+    "@crewhaus/hooks-engine": "0.1.2",
+    "@crewhaus/ir": "0.1.2",
+    "@crewhaus/logging": "0.1.2",
+    "@crewhaus/mcp-host": "0.1.2",
+    "@crewhaus/permission-engine": "0.1.2",
+    "@crewhaus/run-context": "0.1.2",
+    "@crewhaus/runtime-core": "0.1.2",
+    "@crewhaus/skills-registry": "0.1.2",
+    "@crewhaus/slash-commands": "0.1.2",
+    "@crewhaus/spec": "0.1.2",
+    "@crewhaus/sub-agent-spawner": "0.1.2",
+    "@crewhaus/tenancy": "0.1.2",
+    "@crewhaus/tool-bash": "0.1.2",
+    "@crewhaus/tool-catalog": "0.1.2",
+    "@crewhaus/tool-fetch": "0.1.2",
+    "@crewhaus/tool-fs": "0.1.2",
+    "@crewhaus/tool-image": "0.1.2",
+    "@crewhaus/tool-mcp": "0.1.2",
+    "@crewhaus/tool-task": "0.1.2",
+    "@crewhaus/tool-todo": "0.1.2",
+    "@crewhaus/tool-web": "0.1.2",
+    "@crewhaus/trace-event-bus": "0.1.2",
     "zod": "^3.23.8"
   },
   "license": "Apache-2.0",
   "author": {
     "name": "Max Meier",
-    "email": "max@studiomax.io",
-    "url": "https://studiomax.io"
+    "email": "max@crewhaus.ai",
+    "url": "https://crewhaus.ai"
   },
   "repository": {
     "type": "git",
@@ -58,12 +59,7 @@
     "url": "https://github.com/crewhaus/factory/issues"
   },
   "publishConfig": {
-    "access": "restricted"
+    "access": "public"
   },
-  "files": [
-    "src",
-    "README.md",
-    "LICENSE",
-    "NOTICE"
-  ]
+  "files": ["src", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.default-invoker.test.ts

@@ -0,0 +1,172 @@
+/**
+ * Isolated test for `runEval`'s built-in `defaultInvoker` — the production
+ * path taken when the caller supplies no `opts.invoker`. It wires the agent
+ * stack once via `wireRunOnce`, then drives `runChatLoop` per sample.
+ *
+ * We stub both `./wire-once` and `@crewhaus/runtime-core` so nothing real is
+ * spun up (no tools, no MCP, no model call). The stub `runChatLoop` records
+ * the options it was handed so we can assert the per-sample session name, the
+ * forced `permissionMode: "auto"`, `singleTurn`, and the seeded user message.
+ * `mock.module` is process-global, so this lives in its own file.
+ */
+import { afterAll, describe, expect, mock, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { lower } from "@crewhaus/compiler";
+import type { Sample } from "@crewhaus/eval-dataset";
+import { parseGradersConfig } from "@crewhaus/eval-grader";
+import type { IrNode, IrV0 } from "@crewhaus/ir";
+import { parseSpec } from "@crewhaus/spec";
+
+type ChatLoopCall = Record<string, unknown>;
+const chatLoopCalls: ChatLoopCall[] = [];
+const wireCalls: Array<{ cwd?: string }> = [];
+
+// Capture real modules so `afterAll` can restore them — `mock.module` is
+// process-global and does not auto-restore across test files. Each capture is
+// a plain-object SNAPSHOT (`{ ...ns }`): an ESM namespace is a live view that
+// resolves to the stubs once mock.module patches the module, so restoring
+// from the namespace itself would silently reinstall the stubs.
+const realWireOnce = { ...(await import("./wire-once")) };
+const realRuntimeCore = { ...(await import("@crewhaus/runtime-core")) };
+
+// Toggle whether the wired deps include sub-agents (drives the optional
+// subAgents/spawnSubAgent spread in defaultInvoker).
+let includeSubAgents = false;
+
+function baseDeps() {
+  return {
+    tools: [],
+    hooks: [],
+    skills: [],
+    slashCommands: new Map(),
+    permissionRules: { flag: [], settings: [], yaml: [], hooks: [], builtin: [] },
+    model: "claude-wired",
+    instructions: "wired instructions",
+    sessionName: "wired-session",
+    sessionTarget: "cli",
+  };
+}
+
+mock.module("./wire-once", () => ({
+  wireRunOnce: async (_ir: unknown, opts: { cwd?: string } = {}) => {
+    wireCalls.push(opts);
+    if (includeSubAgents) {
+      return {
+        ...baseDeps(),
+        subAgents: new Map([["helper", { name: "helper" }]]),
+        spawnSubAgent: () => undefined,
+      };
+    }
+    return baseDeps();
+  },
+}));
+
+mock.module("@crewhaus/runtime-core", () => ({
+  ...realRuntimeCore,
+  runChatLoop: async (opts: ChatLoopCall) => {
+    chatLoopCalls.push(opts);
+    return `answer for ${(opts["seedMessages"] as Array<{ content: string }>)[0]?.content}`;
+  },
+}));
+
+const { runEval } = await import("./index");
+
+function narrowToAgent(ir: IrNode): IrV0 {
+  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
+  return ir;
+}
+
+const SPEC = `name: default-invoker-test
+target: cli
+agent:
+  model: claude-opus-4-7
+  instructions: spec instructions
+`;
+
+async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
+  for (const s of samples) yield s;
+}
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-definvoker-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+  mock.module("./wire-once", () => realWireOnce);
+  mock.module("@crewhaus/runtime-core", () => realRuntimeCore);
+});
+
+describe("runEval — defaultInvoker (no caller invoker)", () => {
+  test("wires once, drives runChatLoop per sample (no sub-agents)", async () => {
+    const outDir = newTempRoot();
+    chatLoopCalls.length = 0;
+    wireCalls.length = 0;
+    includeSubAgents = false;
+
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = [
+      { id: "a", input: "first", expected_output: "answer for first" },
+      { id: "b", input: "second", expected_output: "answer for second" },
+    ];
+    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
+
+    const summary = await runEval({
+      ir,
+      dataset: { name: "def", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      // No invoker → defaultInvoker path. cwd threads into wireRunOnce.
+      opts: { outDir, cwd: "/tmp/some-cwd" },
+    });
+
+    // wireRunOnce called exactly once, with the cwd.
+    expect(wireCalls).toHaveLength(1);
+    expect(wireCalls[0]).toEqual({ cwd: "/tmp/some-cwd" });
+
+    // runChatLoop invoked per sample with the wired stack + forced options.
+    expect(chatLoopCalls).toHaveLength(2);
+    const call = chatLoopCalls.find(
+      (c) => (c["seedMessages"] as Array<{ content: string }>)[0]?.content === "first",
+    );
+    expect(call?.["permissionMode"]).toBe("auto");
+    expect(call?.["singleTurn"]).toBe(true);
+    expect(call?.["model"]).toBe("claude-wired");
+    expect(call?.["instructions"]).toBe("wired instructions");
+    expect(call?.["sessionName"]).toBe("wired-session_a");
+    expect(call?.["subAgents"]).toBeUndefined();
+    expect(call?.["spawnSubAgent"]).toBeUndefined();
+
+    // The agentOutput flowed back through to grading (exact_match passes).
+    expect(summary.aggregates.passRate).toBe(1);
+    expect(summary.samples.find((s) => s.sampleId === "a")?.agentOutput).toBe("answer for first");
+  });
+
+  test("threads sub-agents through to runChatLoop when wired", async () => {
+    const outDir = newTempRoot();
+    chatLoopCalls.length = 0;
+    wireCalls.length = 0;
+    includeSubAgents = true;
+
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = [{ id: "a", input: "q", expected_output: "answer for q" }];
+    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
+
+    await runEval({
+      ir,
+      dataset: { name: "def-sub", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      // No cwd → wireRunOnce receives `{}` (the cwd-absent spread branch).
+      opts: { outDir },
+    });
+
+    expect(wireCalls[0]).toEqual({});
+    expect(chatLoopCalls).toHaveLength(1);
+    const call = chatLoopCalls[0];
+    expect(call?.["subAgents"]).toBeInstanceOf(Map);
+    expect(typeof call?.["spawnSubAgent"]).toBe("function");
+  });
+});

package/src/index.interrupt.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Isolated test for the SIGINT mid-run interrupt path in `runEval`.
+ *
+ * Lives in its own file because it emits a process-level `SIGINT`: keeping it
+ * apart from the rest of the suite ensures the emitted signal only interacts
+ * with the handler `runEval` itself registers (and removes) for this run.
+ *
+ * The interrupt check sits *after* `sem.acquire()`: every sample callback's
+ * synchronous prefix runs during `samples.map(...)`, before any SIGINT can
+ * fire, so a pre-acquire check would never observe a mid-run interrupt. With
+ * `concurrency: 1`, sample s0 holds the only slot while the rest queue; firing
+ * SIGINT inside s0's invoker means each later sample is skipped as its turn
+ * comes — exercising both the interrupt throw and the rejected→SampleResult
+ * mapping.
+ */
+import { afterAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { lower } from "@crewhaus/compiler";
+import type { Sample } from "@crewhaus/eval-dataset";
+import { parseGradersConfig } from "@crewhaus/eval-grader";
+import type { IrNode, IrV0 } from "@crewhaus/ir";
+import { parseSpec } from "@crewhaus/spec";
+import { type AgentInvoker, runEval } from "./index";
+
+function narrowToAgent(ir: IrNode): IrV0 {
+  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
+  return ir;
+}
+
+const SPEC = `name: interrupt-test
+target: cli
+agent:
+  model: claude-opus-4-7
+  instructions: hi
+`;
+
+async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
+  for (const s of samples) yield s;
+}
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-interrupt-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+});
+
+describe("runEval — SIGINT mid-run interrupt", () => {
+  test("interrupts still-queued samples and maps them to failed results", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = Array.from({ length: 4 }).map((_, i) => ({
+      id: `s${i}`,
+      input: "x",
+      expected_output: "y",
+    }));
+
+    let count = 0;
+    const invoker: AgentInvoker = async ({ sample }) => {
+      count += 1;
+      // Fire SIGINT while the first sample holds the single concurrency slot;
+      // the rest are queued and will observe `interrupted` as they dequeue.
+      if (count === 1) process.emit("SIGINT" as NodeJS.Signals);
+      return { agentOutput: sample.expected_output ?? "", events: [] };
+    };
+
+    const { compiled } = parseGradersConfig("graders:\n  - name: m\n    type: exact_match\n");
+    const summary = await runEval({
+      ir,
+      dataset: { name: "interrupt", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      opts: { invoker, outDir, concurrency: 1 },
+    });
+
+    // s0 completed; s1..s3 were interrupted before running.
+    expect(summary.samples).toHaveLength(4);
+    expect(summary.samples.find((s) => s.sampleId === "s0")?.grades.overall.passed).toBe(true);
+    const interrupted = summary.samples.filter((s) => s.error?.includes("interrupted"));
+    expect(interrupted.length).toBe(3);
+    expect(summary.aggregates.errorCount).toBe(3);
+
+    // The mapped failure carries the canonical placeholder fields.
+    const s3 = summary.samples.find((s) => s.sampleId === "s3");
+    expect(s3?.sessionId).toBe("(unset)");
+    expect(s3?.latencyMs).toBe(0);
+    expect(s3?.turns).toBe(0);
+    expect(s3?.tokens).toEqual({ input: 0, output: 0 });
+    expect(s3?.grades.overall.rationale).toBe("sample failed entirely");
+
+    // results.json was still persisted despite the interrupt.
+    const results = JSON.parse(readFileSync(join(outDir, "results.json"), "utf-8"));
+    expect(results.aggregates.errorCount).toBe(3);
+  });
+});

package/src/index.judge.test.ts

@@ -0,0 +1,167 @@
+/**
+ * Isolated test for the `llm_judge` grader-resolution branch in `runEval`.
+ *
+ * The runner replaces any compiled grader carrying a `judgeSpec` with a real
+ * judge grader bound to the run's `judgeModel` (or the per-grader override).
+ * We stub `@crewhaus/eval-judge` so no LLM/network is touched: `loadRubric`
+ * echoes its input and `createJudgeGrader` returns a deterministic grader that
+ * records the model it was bound to. `mock.module` is process-global, so this
+ * lives in its own file (Bun gives each test file a fresh module graph).
+ */
+import { afterAll, describe, expect, mock, test } from "bun:test";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { lower } from "@crewhaus/compiler";
+import type { Sample } from "@crewhaus/eval-dataset";
+import type { CompiledGrader, GradeResult } from "@crewhaus/eval-grader";
+import type { IrNode, IrV0 } from "@crewhaus/ir";
+import { parseSpec } from "@crewhaus/spec";
+
+const boundModels: Array<string | undefined> = [];
+const loadedRubrics: unknown[] = [];
+
+// Capture the real module so `afterAll` can restore it — `mock.module` is
+// process-global and does not auto-restore across test files. The capture is
+// a plain-object SNAPSHOT (`{ ...ns }`): an ESM namespace is a live view that
+// resolves to the stubs once mock.module patches the module, so restoring
+// from the namespace itself would silently reinstall the stubs.
+const realEvalJudge = { ...(await import("@crewhaus/eval-judge")) };
+
+mock.module("@crewhaus/eval-judge", () => ({
+  ...realEvalJudge,
+  loadRubric: (input: unknown) => {
+    loadedRubrics.push(input);
+    return { criteria: [{ name: "quality", weight: 1, description: "is it good" }] };
+  },
+  createJudgeGrader: (_rubric: unknown, opts: { model?: string } = {}) => {
+    boundModels.push(opts.model);
+    // Deterministic grader: always passes, no network.
+    return async (): Promise<GradeResult> => ({
+      passed: true,
+      score: 1,
+      rationale: `judged with ${opts.model ?? "(default)"}`,
+    });
+  },
+}));
+
+const { runEval } = await import("./index");
+
+function narrowToAgent(ir: IrNode): IrV0 {
+  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
+  return ir;
+}
+
+const SPEC = `name: judge-test
+target: cli
+agent:
+  model: claude-opus-4-7
+  instructions: hi
+`;
+
+async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
+  for (const s of samples) yield s;
+}
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-judge-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+  mock.module("@crewhaus/eval-judge", () => realEvalJudge);
+});
+
+// A compiled grader with a judgeSpec but no per-grader model → uses opts.judgeModel.
+function judgeGraderNoModel(name: string): CompiledGrader {
+  return {
+    name,
+    grader: async () => {
+      throw new Error("placeholder must be replaced");
+    },
+    weight: 1,
+    judgeSpec: { rubric: { criteria: [{ name: "q", weight: 1, description: "d" }] } } as never,
+  };
+}
+
+// A compiled grader whose judgeSpec carries its own model override.
+function judgeGraderWithModel(name: string, model: string): CompiledGrader {
+  return {
+    name,
+    grader: async () => {
+      throw new Error("placeholder must be replaced");
+    },
+    weight: 1,
+    judgeSpec: {
+      rubric: { criteria: [{ name: "q", weight: 1, description: "d" }] },
+      model,
+    } as never,
+  };
+}
+
+describe("runEval — llm_judge resolution", () => {
+  test("binds judge grader to opts.judgeModel when no per-grader override", async () => {
+    const outDir = newTempRoot();
+    boundModels.length = 0;
+    loadedRubrics.length = 0;
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
+    const invoker = async () => ({ agentOutput: "anything", events: [] });
+
+    const summary = await runEval({
+      ir,
+      dataset: { name: "judged", samples: yieldSamples(samples) },
+      compiledGraders: [judgeGraderNoModel("rubricA")],
+      opts: { invoker, outDir, judgeModel: "claude-judge-x" },
+    });
+
+    expect(loadedRubrics).toHaveLength(1);
+    expect(boundModels).toEqual(["claude-judge-x"]);
+    expect(summary.config.graderNames).toEqual(["rubricA"]);
+    expect(summary.config.judgeModel).toBe("claude-judge-x");
+    expect(summary.aggregates.passRate).toBe(1);
+
+    // judgeModel surfaced in the persisted run.json snapshot.
+    const runJson = JSON.parse(readFileSync(join(outDir, "run.json"), "utf-8"));
+    expect(runJson.judgeModel).toBe("claude-judge-x");
+    expect(runJson.graderNames).toEqual(["rubricA"]);
+  });
+
+  test("per-grader judgeSpec.model overrides the run judgeModel", async () => {
+    const outDir = newTempRoot();
+    boundModels.length = 0;
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
+    const invoker = async () => ({ agentOutput: "anything", events: [] });
+
+    await runEval({
+      ir,
+      dataset: { name: "judged2", samples: yieldSamples(samples) },
+      compiledGraders: [judgeGraderWithModel("rubricB", "claude-override")],
+      // No opts.judgeModel → the per-grader override must win.
+      opts: { invoker, outDir },
+    });
+
+    expect(boundModels).toEqual(["claude-override"]);
+  });
+
+  test("a judge grader with neither override nor judgeModel binds the default", async () => {
+    const outDir = newTempRoot();
+    boundModels.length = 0;
+    const ir = narrowToAgent(lower(parseSpec(SPEC)));
+    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
+    const invoker = async () => ({ agentOutput: "anything", events: [] });
+
+    await runEval({
+      ir,
+      dataset: { name: "judged3", samples: yieldSamples(samples) },
+      compiledGraders: [judgeGraderNoModel("rubricC")],
+      opts: { invoker, outDir },
+    });
+
+    // model is undefined → createJudgeGrader called with `{}` (no model key).
+    expect(boundModels).toEqual([undefined]);
+  });
+});

package/src/index.ts

@@ -28,6 +28,7 @@ import type { CompiledGrader, Grader } from "@crewhaus/eval-grader";
 import { createJudgeGrader, loadRubric } from "@crewhaus/eval-judge";
 import type { IrV0 } from "@crewhaus/ir";
 import { runChatLoop } from "@crewhaus/runtime-core";
+import { currentTenantContext } from "@crewhaus/tenancy";
 import { aggregate } from "./aggregate";
 import { RunnerError } from "./errors";
 import { runSample } from "./run-sample";
@@ -59,6 +60,20 @@ export type RunEvalArgs = {
   readonly opts?: RunEvalOptions;
 };
 
+/**
+ * Resolve the eval output directory. When an eval runs inside a tenant scope
+ * (e.g. a cloud/managed eval), the tenant's rebased `evalRoot` is used so one
+ * tenant's eval artifacts never share a directory with another's; the global
+ * default is only used outside any tenant scope (#150). An explicit
+ * `optsOutDir` always wins for trusted callers.
+ */
+export function resolveEvalOutDir(runId: string, optsOutDir?: string): string {
+  if (optsOutDir !== undefined) return optsOutDir;
+  const tenant = currentTenantContext()?.tenant;
+  if (tenant !== undefined) return join(tenant.evalRoot, runId);
+  return join(".crewhaus", "evals", runId);
+}
+
 /**
  * Execute an evaluation. Returns a summary; per-sample artifacts and
  * the summary itself are also persisted under `outDir`.
@@ -68,7 +83,7 @@ export async function runEval(args: RunEvalArgs): Promise<EvalRunSummary> {
   const opts = args.opts ?? {};
 
   const runId = opts.runId ?? generateRunId();
-  const outDir = opts.outDir ?? join(".crewhaus", "evals", runId);
+  const outDir = resolveEvalOutDir(runId, opts.outDir);
   mkdirSync(outDir, { recursive: true });
 
   // Resolve graders. Replace any `llm_judge` placeholder with a real judge
@@ -129,10 +144,15 @@ export async function runEval(args: RunEvalArgs): Promise<EvalRunSummary> {
 
   const settled = await Promise.allSettled(
     samples.map(async (sample) => {
+      const release = await sem.acquire();
+      // Check *after* acquiring the slot: every callback's synchronous prefix
+      // runs during `.map()` (before any SIGINT can fire), so a pre-acquire
+      // check would never observe a mid-run interrupt. Samples still queued on
+      // the semaphore when SIGINT arrives are skipped here as their turn comes.
       if (interrupted) {
+        release();
         throw new RunnerError(`run interrupted before sample "${sample.id}"`);
       }
-      const release = await sem.acquire();
       try {
         return await runSample({
           sample,