@crewhaus/eval-runner 0.1.0

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@crewhaus/eval-runner",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Eval runner — per-sample isolation, concurrency, grading, persistence",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/agent-context-isolation": "0.0.0",
+    "@crewhaus/compiler": "0.0.0",
+    "@crewhaus/errors": "0.0.0",
+    "@crewhaus/eval-dataset": "0.0.0",
+    "@crewhaus/eval-grader": "0.0.0",
+    "@crewhaus/eval-judge": "0.0.0",
+    "@crewhaus/event-log": "0.0.0",
+    "@crewhaus/hooks-engine": "0.0.0",
+    "@crewhaus/ir": "0.0.0",
+    "@crewhaus/logging": "0.0.0",
+    "@crewhaus/mcp-host": "0.0.0",
+    "@crewhaus/permission-engine": "0.0.0",
+    "@crewhaus/run-context": "0.0.0",
+    "@crewhaus/runtime-core": "0.0.0",
+    "@crewhaus/skills-registry": "0.0.0",
+    "@crewhaus/slash-commands": "0.0.0",
+    "@crewhaus/spec": "0.0.0",
+    "@crewhaus/sub-agent-spawner": "0.0.0",
+    "@crewhaus/tool-bash": "0.0.0",
+    "@crewhaus/tool-catalog": "0.0.0",
+    "@crewhaus/tool-fetch": "0.0.0",
+    "@crewhaus/tool-fs": "0.0.0",
+    "@crewhaus/tool-image": "0.0.0",
+    "@crewhaus/tool-mcp": "0.0.0",
+    "@crewhaus/tool-task": "0.0.0",
+    "@crewhaus/tool-todo": "0.0.0",
+    "@crewhaus/tool-web": "0.0.0",
+    "@crewhaus/trace-event-bus": "0.0.0",
+    "zod": "^3.23.8"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/eval-runner"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/eval-runner#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/aggregate.ts

@@ -0,0 +1,48 @@
+import type { SampleResult } from "./types";
+
+export function quantile(sorted: ReadonlyArray<number>, q: number): number {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0] ?? 0;
+  const idx = q * (sorted.length - 1);
+  const lo = Math.floor(idx);
+  const hi = Math.ceil(idx);
+  if (lo === hi) return sorted[lo] ?? 0;
+  const fract = idx - lo;
+  return (sorted[lo] ?? 0) * (1 - fract) + (sorted[hi] ?? 0) * fract;
+}
+
+export function aggregate(samples: ReadonlyArray<SampleResult>): {
+  passRate: number;
+  meanScore: number;
+  p50Turns: number;
+  p95Turns: number;
+  p50LatencyMs: number;
+  p95LatencyMs: number;
+  totalTokens: { input: number; output: number };
+  errorCount: number;
+} {
+  const ok = samples.filter((s) => s.error === undefined);
+  const total = samples.length;
+  const passed = ok.filter((s) => s.grades.overall.passed).length;
+  const meanScore =
+    ok.length === 0 ? 0 : ok.reduce((sum, s) => sum + s.grades.overall.score, 0) / ok.length;
+  const turnsSorted = ok.map((s) => s.turns).sort((a, b) => a - b);
+  const latSorted = ok.map((s) => s.latencyMs).sort((a, b) => a - b);
+  const totalTokens = ok.reduce(
+    (acc, s) => ({
+      input: acc.input + s.tokens.input,
+      output: acc.output + s.tokens.output,
+    }),
+    { input: 0, output: 0 },
+  );
+  return {
+    passRate: total === 0 ? 0 : passed / total,
+    meanScore,
+    p50Turns: quantile(turnsSorted, 0.5),
+    p95Turns: quantile(turnsSorted, 0.95),
+    p50LatencyMs: quantile(latSorted, 0.5),
+    p95LatencyMs: quantile(latSorted, 0.95),
+    totalTokens,
+    errorCount: samples.length - ok.length,
+  };
+}

package/src/errors.ts

@@ -0,0 +1,7 @@
+import { RuntimeError } from "@crewhaus/errors";
+
+export class RunnerError extends RuntimeError {
+  constructor(message: string, cause?: unknown) {
+    super(`eval-runner: ${message}`, cause);
+  }
+}

package/src/index.test.ts

@@ -0,0 +1,254 @@
+import { afterAll, describe, expect, test } from "bun:test";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { lower } from "@crewhaus/compiler";
+import type { Sample } from "@crewhaus/eval-dataset";
+import { parseGradersConfig } from "@crewhaus/eval-grader";
+import type { IrNode, IrV0 } from "@crewhaus/ir";
+import { parseSpec } from "@crewhaus/spec";
+
+function narrowToAgent(ir: IrNode): IrV0 {
+  if (ir.target !== "cli") throw new Error(`test fixture must be target:cli, got ${ir.target}`);
+  return ir;
+}
+import { aggregate, quantile } from "./aggregate";
+import { type AgentInvoker, runEval } from "./index";
+import { Semaphore } from "./semaphore";
+
+const TMP_ROOTS: string[] = [];
+function newTempRoot(): string {
+  const dir = mkdtempSync(join(tmpdir(), "crewhaus-eval-runner-"));
+  TMP_ROOTS.push(dir);
+  return dir;
+}
+afterAll(() => {
+  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
+});
+
+const HELLO_SPEC = `name: hello-test
+target: cli
+agent:
+  model: claude-opus-4-7
+  instructions: |
+    You are a helpful, concise assistant.
+`;
+
+async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
+  for (const s of samples) yield s;
+}
+
+const FIXED_GRADERS = `
+graders:
+  - name: math
+    type: exact_match
+`;
+
+describe("Semaphore", () => {
+  test("respects capacity", async () => {
+    const sem = new Semaphore(2);
+    let active = 0;
+    let maxActive = 0;
+    const tasks = Array.from({ length: 10 }).map(async () => {
+      const release = await sem.acquire();
+      active += 1;
+      maxActive = Math.max(maxActive, active);
+      await new Promise((r) => setTimeout(r, 5));
+      active -= 1;
+      release();
+    });
+    await Promise.all(tasks);
+    expect(maxActive).toBeLessThanOrEqual(2);
+  });
+
+  test("rejects capacity < 1", () => {
+    expect(() => new Semaphore(0)).toThrow();
+  });
+});
+
+describe("aggregate", () => {
+  test("quantile handles edge cases", () => {
+    expect(quantile([], 0.5)).toBe(0);
+    expect(quantile([42], 0.5)).toBe(42);
+    expect(quantile([1, 2, 3, 4, 5], 0.5)).toBe(3);
+    expect(quantile([1, 2, 3, 4, 5], 0.95)).toBeCloseTo(4.8);
+  });
+
+  test("computes pass rate, mean score, percentiles", () => {
+    const samples = [
+      mockSample("s1", true, 1, 100, 1),
+      mockSample("s2", true, 0.5, 200, 2),
+      mockSample("s3", false, 0, 300, 3),
+    ];
+    const agg = aggregate(samples);
+    expect(agg.passRate).toBeCloseTo(2 / 3);
+    expect(agg.meanScore).toBeCloseTo(0.5);
+    expect(agg.p50LatencyMs).toBe(200);
+    expect(agg.errorCount).toBe(0);
+  });
+});
+
+describe("runEval — T3 5-sample fixture", () => {
+  test("runs 5 samples, persists artifacts, computes aggregates", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
+    const samples: Sample[] = [
+      { id: "q1", input: "What is 2+2?", expected_output: "4" },
+      { id: "q2", input: "What is 5-3?", expected_output: "2" },
+      { id: "q3", input: "What is 6/2?", expected_output: "3" },
+      { id: "q4", input: "What is 3*3?", expected_output: "9" },
+      { id: "q5", input: "What is 10-7?", expected_output: "3" },
+    ];
+
+    // Stub invoker: returns the expected_output verbatim. exact_match grader passes.
+    const invoker: AgentInvoker = async ({ sample }) => ({
+      agentOutput: sample.expected_output ?? "",
+      transcript: [],
+      events: [],
+    });
+
+    const { compiled } = parseGradersConfig(FIXED_GRADERS);
+    const summary = await runEval({
+      ir,
+      dataset: { name: "fixture5", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      opts: { invoker, outDir, concurrency: 2 },
+    });
+
+    expect(summary.samples).toHaveLength(5);
+    expect(summary.aggregates.passRate).toBe(1);
+    expect(summary.aggregates.meanScore).toBe(1);
+
+    // Per-sample artifacts
+    for (const id of ["q1", "q2", "q3", "q4", "q5"]) {
+      expect(existsSync(join(outDir, id, "transcript.jsonl"))).toBe(true);
+      expect(existsSync(join(outDir, id, "events.jsonl"))).toBe(true);
+      expect(existsSync(join(outDir, id, "grades.json"))).toBe(true);
+      expect(existsSync(join(outDir, id, "meta.json"))).toBe(true);
+    }
+    // Run-level artifacts
+    expect(existsSync(join(outDir, "run.json"))).toBe(true);
+    expect(existsSync(join(outDir, "results.json"))).toBe(true);
+    const results = JSON.parse(readFileSync(join(outDir, "results.json"), "utf-8"));
+    expect(results.samples).toHaveLength(5);
+    expect(results.aggregates.passRate).toBe(1);
+  });
+
+  test("captures grader failures correctly", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
+    const samples: Sample[] = [
+      { id: "ok", input: "x", expected_output: "y" },
+      { id: "wrong", input: "x", expected_output: "y" },
+    ];
+    const invoker: AgentInvoker = async ({ sample }) => ({
+      agentOutput: sample.id === "ok" ? "y" : "z",
+      transcript: [],
+      events: [],
+    });
+    const { compiled } = parseGradersConfig(FIXED_GRADERS);
+    const summary = await runEval({
+      ir,
+      dataset: { name: "mixed", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      opts: { invoker, outDir },
+    });
+    expect(summary.aggregates.passRate).toBe(0.5);
+    expect(summary.samples.find((s) => s.sampleId === "wrong")?.grades.overall.passed).toBe(false);
+  });
+
+  test("invoker errors do not kill the rest of the run", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
+    const samples: Sample[] = [
+      { id: "ok", input: "x", expected_output: "y" },
+      { id: "boom", input: "x", expected_output: "y" },
+    ];
+    const invoker: AgentInvoker = async ({ sample }) => {
+      if (sample.id === "boom") throw new Error("invoker exploded");
+      return { agentOutput: sample.expected_output ?? "", transcript: [], events: [] };
+    };
+    const { compiled } = parseGradersConfig(FIXED_GRADERS);
+    const summary = await runEval({
+      ir,
+      dataset: { name: "withErr", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      opts: { invoker, outDir },
+    });
+    expect(summary.samples).toHaveLength(2);
+    expect(summary.aggregates.errorCount).toBe(1);
+    const boom = summary.samples.find((s) => s.sampleId === "boom");
+    expect(boom?.error).toBe("invoker exploded");
+    expect(boom?.grades.overall.passed).toBe(false);
+    // The good sample still completed and was scored.
+    expect(summary.samples.find((s) => s.sampleId === "ok")?.grades.overall.passed).toBe(true);
+  });
+
+  test("rejects empty dataset", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
+    const invoker: AgentInvoker = async () => ({
+      agentOutput: "",
+      transcript: [],
+      events: [],
+    });
+    const { compiled } = parseGradersConfig(FIXED_GRADERS);
+    await expect(
+      runEval({
+        ir,
+        dataset: { name: "empty", samples: yieldSamples([]) },
+        compiledGraders: compiled,
+        opts: { invoker, outDir },
+      }),
+    ).rejects.toThrow(/yielded zero samples/);
+  });
+});
+
+describe("runEval — T7 200-sample concurrency-8 SLO", () => {
+  test("completes 200 samples at concurrency 8 well under 60s", async () => {
+    const outDir = newTempRoot();
+    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
+    const samples: Sample[] = Array.from({ length: 200 }).map((_, i) => ({
+      id: `s${i.toString().padStart(3, "0")}`,
+      input: `q${i}`,
+      expected_output: `a${i}`,
+    }));
+    const invoker: AgentInvoker = async ({ sample }) => {
+      // Tiny delay to mimic real-world async work without burning CPU.
+      await new Promise((r) => setTimeout(r, 1));
+      return { agentOutput: sample.expected_output ?? "", transcript: [], events: [] };
+    };
+    const { compiled } = parseGradersConfig(FIXED_GRADERS);
+
+    const t0 = performance.now();
+    const summary = await runEval({
+      ir,
+      dataset: { name: "load200", samples: yieldSamples(samples) },
+      compiledGraders: compiled,
+      opts: { invoker, outDir, concurrency: 8 },
+    });
+    const elapsedMs = performance.now() - t0;
+
+    expect(summary.samples).toHaveLength(200);
+    expect(summary.aggregates.passRate).toBe(1);
+    expect(elapsedMs).toBeLessThan(60_000);
+  }, 90_000);
+});
+
+function mockSample(id: string, passed: boolean, score: number, latencyMs: number, turns: number) {
+  return {
+    sampleId: id,
+    sessionId: `sess_${id.padEnd(16, "0")}`,
+    startedAt: "2026-01-01T00:00:00Z",
+    endedAt: "2026-01-01T00:00:01Z",
+    latencyMs,
+    turns,
+    tokens: { input: 10, output: 20 },
+    model: "claude-opus-4-7",
+    agentOutput: "x",
+    grades: {
+      overall: { passed, score, rationale: "" },
+      perGrader: [],
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,244 @@
+/**
+ * Catalog R-eval `eval-runner` — run an agent against a dataset.
+ *
+ * Per-sample isolation: fresh `RunContext` (and therefore fresh
+ * `TraceEventBus` + `sessionId`) so concurrent samples don't see each
+ * other's events. Concurrency is enforced by a tiny in-package semaphore
+ * (default 4). Per-sample artifacts persist to
+ * `.crewhaus/evals/<runId>/<sampleId>/{transcript.jsonl, events.jsonl,
+ * grades.json, meta.json}`. Run-level: `run.json` (config snapshot)
+ * + `results.json` (aggregates).
+ *
+ * `--seed` is honored only for providers that surface temperature
+ * reproducibility (Anthropic does not). Document divergence; do not
+ * promise byte-identical reruns.
+ *
+ * `permissionMode` is forced to `"auto"` so `alwaysAsk` rules auto-deny
+ * rather than blocking on stdin in a non-interactive eval run.
+ *
+ * MCP servers are shared across samples (read-mostly assumption). An
+ * `isolateMcpPerSample` escape hatch is reserved for future use.
+ *
+ * Reference: build-roadmap.md §16.
+ */
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import type { Sample } from "@crewhaus/eval-dataset";
+import type { CompiledGrader, Grader } from "@crewhaus/eval-grader";
+import { createJudgeGrader, loadRubric } from "@crewhaus/eval-judge";
+import type { IrV0 } from "@crewhaus/ir";
+import { runChatLoop } from "@crewhaus/runtime-core";
+import { aggregate } from "./aggregate";
+import { RunnerError } from "./errors";
+import { runSample } from "./run-sample";
+import { Semaphore } from "./semaphore";
+import type {
+  AgentInvoker,
+  EvalRunSummary,
+  GraderEntry,
+  RunEvalOptions,
+  SampleResult,
+} from "./types";
+import { type SharedAgentDeps, wireRunOnce } from "./wire-once";
+
+export type { AgentInvoker, EvalRunSummary, GraderEntry, RunEvalOptions, SampleResult };
+export type { SharedAgentDeps };
+export { wireRunOnce };
+export { Semaphore };
+export { aggregate };
+export { RunnerError };
+export { runSample };
+
+const DEFAULT_CONCURRENCY = 4;
+
+export type RunEvalArgs = {
+  /** Lowered agent IR (target: cli). Caller is responsible for narrowing. */
+  readonly ir: IrV0;
+  readonly dataset: { name: string; samples: AsyncIterable<Sample> };
+  readonly compiledGraders: ReadonlyArray<CompiledGrader>;
+  readonly opts?: RunEvalOptions;
+};
+
+/**
+ * Execute an evaluation. Returns a summary; per-sample artifacts and
+ * the summary itself are also persisted under `outDir`.
+ */
+export async function runEval(args: RunEvalArgs): Promise<EvalRunSummary> {
+  const { ir, dataset, compiledGraders } = args;
+  const opts = args.opts ?? {};
+
+  const runId = opts.runId ?? generateRunId();
+  const outDir = opts.outDir ?? join(".crewhaus", "evals", runId);
+  mkdirSync(outDir, { recursive: true });
+
+  // Resolve graders. Replace any `llm_judge` placeholder with a real judge
+  // grader bound to the runner's judgeModel (or the per-grader override).
+  const graders: GraderEntry[] = compiledGraders.map((g) => {
+    if (g.judgeSpec) {
+      const rubric = loadRubric(g.judgeSpec.rubric);
+      const model = g.judgeSpec.model ?? opts.judgeModel;
+      const grader = createJudgeGrader(rubric, model !== undefined ? { model } : {});
+      return { name: g.name, grader };
+    }
+    return { name: g.name, grader: g.grader };
+  });
+
+  // The default invoker calls runChatLoop with the per-sample fresh runContext.
+  const invoker = opts.invoker ?? (await defaultInvoker(ir, opts));
+
+  const startedAt = new Date().toISOString();
+  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
+  const sem = new Semaphore(concurrency);
+
+  // Persist run-level config snapshot up front so SIGINT mid-run still leaves
+  // a usable directory.
+  const specHash = await hashSpec(ir);
+  await Bun.write(
+    join(outDir, "run.json"),
+    JSON.stringify(
+      {
+        runId,
+        startedAt,
+        specHash,
+        datasetName: dataset.name,
+        graderNames: graders.map((g) => g.name),
+        model: ir.agent.model,
+        ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
+        concurrency,
+        ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+      },
+      null,
+      2,
+    ),
+  );
+
+  // Materialize the dataset into a list so we can run samples concurrently.
+  // For very large datasets the streaming loaders still avoid loading the
+  // whole file into memory — but eventually all sample objects sit in RAM.
+  const samples: Sample[] = [];
+  for await (const s of dataset.samples) samples.push(s);
+  if (samples.length === 0) {
+    throw new RunnerError(`dataset "${dataset.name}" yielded zero samples`);
+  }
+
+  let interrupted = false;
+  const abortHandler = () => {
+    interrupted = true;
+  };
+  process.once("SIGINT", abortHandler);
+
+  const settled = await Promise.allSettled(
+    samples.map(async (sample) => {
+      if (interrupted) {
+        throw new RunnerError(`run interrupted before sample "${sample.id}"`);
+      }
+      const release = await sem.acquire();
+      try {
+        return await runSample({
+          sample,
+          invoker,
+          graders,
+          outDir,
+          model: ir.agent.model,
+          ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+        });
+      } finally {
+        release();
+      }
+    }),
+  );
+
+  process.removeListener("SIGINT", abortHandler);
+
+  const results: SampleResult[] = settled.map((r, i) => {
+    if (r.status === "fulfilled") return r.value;
+    const sample = samples[i];
+    return {
+      sampleId: sample?.id ?? `unknown-${i}`,
+      sessionId: "(unset)",
+      startedAt,
+      endedAt: new Date().toISOString(),
+      latencyMs: 0,
+      turns: 0,
+      tokens: { input: 0, output: 0 },
+      model: ir.agent.model,
+      agentOutput: "",
+      grades: {
+        overall: { passed: false, score: 0, rationale: "sample failed entirely" },
+        perGrader: [],
+      },
+      error: r.reason instanceof Error ? r.reason.message : String(r.reason),
+    };
+  });
+
+  const endedAt = new Date().toISOString();
+  const aggregates = aggregate(results);
+
+  const summary: EvalRunSummary = {
+    runId,
+    startedAt,
+    endedAt,
+    samples: results,
+    aggregates,
+    config: {
+      specHash,
+      datasetName: dataset.name,
+      graderNames: graders.map((g) => g.name),
+      model: ir.agent.model,
+      ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
+      concurrency,
+      ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+    },
+    outDir,
+  };
+
+  await Bun.write(join(outDir, "results.json"), JSON.stringify(summary, null, 2));
+
+  return summary;
+}
+
+async function defaultInvoker(ir: IrV0, opts: RunEvalOptions): Promise<AgentInvoker> {
+  const wired: SharedAgentDeps = await wireRunOnce(
+    ir,
+    opts.cwd !== undefined ? { cwd: opts.cwd } : {},
+  );
+  return async (req) => {
+    const agentOutput = await runChatLoop({
+      model: wired.model,
+      instructions: wired.instructions,
+      tools: [...wired.tools],
+      hooks: wired.hooks,
+      skills: wired.skills,
+      slashCommands: wired.slashCommands,
+      ...(wired.subAgents !== undefined && wired.spawnSubAgent !== undefined
+        ? { subAgents: wired.subAgents, spawnSubAgent: wired.spawnSubAgent }
+        : {}),
+      permissionRules: wired.permissionRules,
+      permissionMode: "auto",
+      sessionName: `${wired.sessionName}_${req.sample.id}`,
+      sessionTarget: wired.sessionTarget,
+      runContext: req.runContext,
+      sessionRootDir: req.sessionRootDir,
+      singleTurn: true,
+      seedMessages: [{ role: "user", content: req.sample.input }],
+    });
+    return { agentOutput };
+  };
+}
+
+function generateRunId(): string {
+  const bytes = new Uint8Array(8);
+  crypto.getRandomValues(bytes);
+  return `run_${Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")}`;
+}
+
+async function hashSpec(ir: IrV0): Promise<string> {
+  const text = JSON.stringify({
+    name: ir.name,
+    target: ir.target,
+    model: ir.agent.model,
+    instructions: ir.agent.instructions,
+    tools: ir.tools,
+  });
+  return Bun.hash(text).toString(16);
+}

package/src/run-sample.ts

@@ -0,0 +1,232 @@
+import { mkdirSync, renameSync } from "node:fs";
+import { join } from "node:path";
+import type { Sample } from "@crewhaus/eval-dataset";
+import {
+  type CompiledGrader,
+  type GradeResult,
+  type Grader,
+  type RunResult,
+  type ToolCall,
+  combineCompiledGraders,
+} from "@crewhaus/eval-grader";
+import { type Event as TranscriptEvent, openEventLog } from "@crewhaus/event-log";
+import { createRunContext } from "@crewhaus/run-context";
+import type { ModelResponseEvent, TraceEvent } from "@crewhaus/trace-event-bus";
+import { RunnerError } from "./errors";
+import type { AgentInvoker, GraderEntry, SampleResult } from "./types";
+
+/**
+ * Per-sample logic. Mints a fresh runContext (and therefore a fresh
+ * TraceEventBus + sessionId), subscribes to the bus, calls the invoker,
+ * and persists `transcript.jsonl`, `events.jsonl`, `grades.json`,
+ * `meta.json` to the per-sample directory.
+ */
+export async function runSample(args: {
+  sample: Sample;
+  invoker: AgentInvoker;
+  graders: ReadonlyArray<GraderEntry>;
+  outDir: string; // .crewhaus/evals/<runId>
+  model: string;
+  seed?: number;
+}): Promise<SampleResult> {
+  const { sample, invoker, graders, outDir, model } = args;
+  const sampleDir = join(outDir, sanitize(sample.id));
+  mkdirSync(sampleDir, { recursive: true });
+
+  const runContext = createRunContext();
+  const events: TraceEvent[] = [];
+  const unsubscribe = runContext.eventBus.subscribe((e) => {
+    events.push(e);
+  });
+
+  const startedAt = new Date().toISOString();
+  const t0 = performance.now();
+
+  let agentOutput = "";
+  let transcript: TranscriptEvent[] = [];
+  let invokerEvents: ReadonlyArray<TraceEvent> | undefined;
+  let error: string | undefined;
+  try {
+    const result = await invoker({
+      sample,
+      runContext,
+      sessionRootDir: sampleDir,
+      ...(args.seed !== undefined ? { seed: args.seed } : {}),
+    });
+    agentOutput = result.agentOutput;
+    transcript = result.transcript ? [...result.transcript] : [];
+    invokerEvents = result.events;
+  } catch (err) {
+    error = err instanceof Error ? err.message : String(err);
+  } finally {
+    unsubscribe();
+  }
+
+  const endedAt = new Date().toISOString();
+  const latencyMs = Math.round(performance.now() - t0);
+
+  // If the invoker didn't supply a transcript, attempt to read it from disk
+  // (this is the path the production runChatLoop wrapper takes).
+  if (transcript.length === 0 && error === undefined) {
+    transcript = await readTranscript(sampleDir, runContext.sessionId);
+  }
+
+  const finalEvents = invokerEvents ? [...invokerEvents] : events;
+
+  // Rename auto-named event-log file → transcript.jsonl for stable artifact
+  // names. (Skip if the invoker didn't actually write one, e.g. test stubs.)
+  const autoPath = join(sampleDir, `${runContext.sessionId}.jsonl`);
+  const transcriptPath = join(sampleDir, "transcript.jsonl");
+  if (await Bun.file(autoPath).exists()) {
+    renameSync(autoPath, transcriptPath);
+  } else if (transcript.length > 0) {
+    // Stub-supplied transcript — write it ourselves.
+    await Bun.write(
+      transcriptPath,
+      transcript.map((e) => JSON.stringify(e)).join("\n") + (transcript.length > 0 ? "\n" : ""),
+    );
+  } else {
+    await Bun.write(transcriptPath, "");
+  }
+
+  // Persist captured trace events.
+  await Bun.write(
+    join(sampleDir, "events.jsonl"),
+    finalEvents.map((e) => JSON.stringify(e)).join("\n") + (finalEvents.length > 0 ? "\n" : ""),
+  );
+
+  const turns = transcript.filter((e) => e.kind === "assistant_message").length;
+  const toolCalls = extractToolCalls(finalEvents);
+  const tokens = sumTokens(finalEvents);
+
+  // Apply graders.
+  const runResult: RunResult = {
+    agentOutput,
+    events: finalEvents,
+    transcript,
+    toolCalls,
+    turns,
+    latencyMs,
+  };
+
+  const perGrader: Array<{ name: string } & GradeResult> = [];
+  for (const g of graders) {
+    let result: GradeResult;
+    try {
+      result = await g.grader(sample, runResult);
+    } catch (err) {
+      result = {
+        passed: false,
+        score: 0,
+        rationale: `grader threw: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+    perGrader.push({ name: g.name, ...result });
+  }
+
+  // Overall = AND of all graders, score = mean.
+  const overall: GradeResult = {
+    passed: error === undefined && perGrader.every((g) => g.passed),
+    score:
+      perGrader.length === 0 ? 0 : perGrader.reduce((s, g) => s + g.score, 0) / perGrader.length,
+    rationale:
+      error !== undefined
+        ? `agent invocation error: ${error}`
+        : perGrader.map((g) => `[${g.name}: ${g.passed ? "✓" : "✗"}] ${g.rationale}`).join(" & "),
+  };
+
+  const sampleResult: SampleResult = {
+    sampleId: sample.id,
+    sessionId: runContext.sessionId,
+    startedAt,
+    endedAt,
+    latencyMs,
+    turns,
+    tokens,
+    model,
+    agentOutput,
+    grades: { overall, perGrader },
+    ...(error !== undefined ? { error } : {}),
+  };
+
+  await Bun.write(join(sampleDir, "grades.json"), JSON.stringify(sampleResult.grades, null, 2));
+  await Bun.write(
+    join(sampleDir, "meta.json"),
+    JSON.stringify(
+      {
+        sampleId: sample.id,
+        sessionId: runContext.sessionId,
+        startedAt,
+        endedAt,
+        latencyMs,
+        turns,
+        tokens,
+        model,
+        ...(error !== undefined ? { error } : {}),
+        ...(args.seed !== undefined ? { seed: args.seed } : {}),
+      },
+      null,
+      2,
+    ),
+  );
+
+  return sampleResult;
+}
+
+/** Helper: produce one combined grader from an array of CompiledGrader. */
+export function combineGraderEntries(graders: ReadonlyArray<CompiledGrader>): Grader {
+  return combineCompiledGraders(graders);
+}
+
+/** Strip path separators from sample IDs to keep the on-disk layout flat. */
+function sanitize(id: string): string {
+  return id.replace(/[^A-Za-z0-9_.-]/g, "_");
+}
+
+async function readTranscript(dir: string, sessionId: string): Promise<TranscriptEvent[]> {
+  const out: TranscriptEvent[] = [];
+  try {
+    const log = await openEventLog(sessionId, { rootDir: dir });
+    for await (const e of log.read()) out.push(e);
+    await log.close();
+  } catch (err) {
+    // Tolerate missing log (test stubs may skip persistence).
+    if (!(err instanceof Error && /invalid sessionId/.test(err.message))) {
+      throw new RunnerError(`failed to read transcript for ${sessionId}`, err);
+    }
+  }
+  return out;
+}
+
+function extractToolCalls(events: ReadonlyArray<TraceEvent>): ToolCall[] {
+  const calls: Array<ToolCall & { ts: number }> = [];
+  const startByUseId = new Map<string, { toolName: string; ts: number }>();
+  for (const ev of events) {
+    if (ev.kind === "tool_call_start") {
+      startByUseId.set(ev.toolUseId, { toolName: ev.toolName, ts: Date.parse(ev.timestamp) });
+    } else if (ev.kind === "tool_call_end") {
+      const start = startByUseId.get(ev.toolUseId);
+      calls.push({
+        toolName: ev.toolName,
+        toolUseId: ev.toolUseId,
+        isError: ev.isError,
+        ts: start?.ts ?? Date.parse(ev.timestamp),
+      });
+    }
+  }
+  calls.sort((a, b) => a.ts - b.ts);
+  return calls.map(({ ts: _ts, ...rest }) => rest);
+}
+
+function sumTokens(events: ReadonlyArray<TraceEvent>): { input: number; output: number } {
+  let input = 0;
+  let output = 0;
+  for (const ev of events) {
+    if (ev.kind === "model_response") {
+      const e = ev as ModelResponseEvent;
+      input += e.usage.input;
+      output += e.usage.output;
+    }
+  }
+  return { input, output };
+}

package/src/semaphore.ts

@@ -0,0 +1,35 @@
+/**
+ * Trivial async semaphore — no `p-limit` dep. Acquire returns a release fn;
+ * callers MUST call it (use try/finally). When the semaphore would otherwise
+ * be over-capacity, acquirers queue FIFO.
+ */
+export class Semaphore {
+  private inflight = 0;
+  private readonly waitQueue: Array<() => void> = [];
+  constructor(private readonly capacity: number) {
+    if (capacity < 1) throw new Error(`Semaphore capacity must be ≥ 1 (got ${capacity})`);
+  }
+  async acquire(): Promise<() => void> {
+    if (this.inflight < this.capacity) {
+      this.inflight += 1;
+      return () => this.release();
+    }
+    return new Promise<() => void>((resolve) => {
+      this.waitQueue.push(() => {
+        this.inflight += 1;
+        resolve(() => this.release());
+      });
+    });
+  }
+  private release(): void {
+    this.inflight -= 1;
+    const next = this.waitQueue.shift();
+    if (next) next();
+  }
+  get pending(): number {
+    return this.waitQueue.length;
+  }
+  get active(): number {
+    return this.inflight;
+  }
+}

package/src/types.ts

@@ -0,0 +1,93 @@
+import type { Sample } from "@crewhaus/eval-dataset";
+import type { GradeResult, Grader } from "@crewhaus/eval-grader";
+import type { Event as TranscriptEvent } from "@crewhaus/event-log";
+import type { RunContext } from "@crewhaus/run-context";
+import type { TraceEvent } from "@crewhaus/trace-event-bus";
+
+/**
+ * The agent-invocation contract. The runner provides a per-sample
+ * `RunContext` (fresh `TraceEventBus`, fresh `sessionId`) and the
+ * destination directory for the event log. The invoker is responsible
+ * for actually running the chat loop and returning the assistant text.
+ *
+ * In production this is a thin wrapper around `runChatLoop`. In tests it's
+ * a deterministic stub that returns a canned answer per sample.
+ */
+export type AgentInvoker = (req: AgentInvokeRequest) => Promise<AgentInvokeResult>;
+
+export type AgentInvokeRequest = {
+  readonly sample: Sample;
+  readonly runContext: RunContext;
+  /** Where the runtime should write the per-sample event-log JSONL. */
+  readonly sessionRootDir: string;
+  readonly seed?: number;
+};
+
+export type AgentInvokeResult = {
+  /** Final assistant text returned by the chat loop. */
+  readonly agentOutput: string;
+  /**
+   * If the invoker has out-of-band knowledge (e.g. a stub), it can supply
+   * the captured event-log entries here. The default invoker reads them
+   * from disk after `runChatLoop` returns.
+   */
+  readonly transcript?: ReadonlyArray<TranscriptEvent>;
+  /** Same as transcript — invoker can short-circuit the bus subscription. */
+  readonly events?: ReadonlyArray<TraceEvent>;
+};
+
+export type SampleResult = {
+  readonly sampleId: string;
+  readonly sessionId: string;
+  readonly startedAt: string;
+  readonly endedAt: string;
+  readonly latencyMs: number;
+  readonly turns: number;
+  readonly tokens: { input: number; output: number };
+  readonly model: string;
+  readonly agentOutput: string;
+  readonly grades: { overall: GradeResult; perGrader: Array<{ name: string } & GradeResult> };
+  readonly error?: string;
+};
+
+export type EvalRunSummary = {
+  readonly runId: string;
+  readonly startedAt: string;
+  readonly endedAt: string;
+  readonly samples: ReadonlyArray<SampleResult>;
+  readonly aggregates: {
+    readonly passRate: number;
+    readonly meanScore: number;
+    readonly p50Turns: number;
+    readonly p95Turns: number;
+    readonly p50LatencyMs: number;
+    readonly p95LatencyMs: number;
+    readonly totalTokens: { input: number; output: number };
+    readonly errorCount: number;
+  };
+  readonly config: {
+    readonly specHash: string;
+    readonly datasetName: string;
+    readonly graderNames: ReadonlyArray<string>;
+    readonly model: string;
+    readonly judgeModel?: string;
+    readonly concurrency: number;
+    readonly seed?: number;
+  };
+  readonly outDir: string;
+};
+
+export type RunEvalOptions = {
+  readonly runId?: string;
+  readonly concurrency?: number;
+  readonly seed?: number;
+  readonly outDir?: string;
+  readonly judgeModel?: string;
+  readonly invoker?: AgentInvoker;
+  readonly cwd?: string;
+};
+
+export type GraderEntry = {
+  readonly name: string;
+  readonly grader: Grader;
+};

package/src/wire-once.ts

@@ -0,0 +1,204 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * Build the agent stack ONCE per eval run, then share across samples.
+ *
+ * This factors the runRun logic from apps/cli/src/index.ts so the CLI's
+ * `crewhaus run` and the eval runner's `crewhaus eval` use the same
+ * tool/hook/skill/MCP/sub-agent wiring. Single source of truth for the
+ * "what is the full agent stack from an IR" question.
+ *
+ * The MCP host (if any) is shared across all eval samples — re-spinning
+ * stdio MCP servers per sample for 200 samples would burn ~30s in process
+ * startup and exceed the T7 SLO. The trade-off is documented: eval-runner
+ * assumes the agent's MCP usage is read-mostly. An `isolateMcpPerSample`
+ * escape hatch is reserved for a future where this matters.
+ */
+import type { SubAgentDefinition } from "@crewhaus/agent-context-isolation";
+import { type HookDef, loadHooks } from "@crewhaus/hooks-engine";
+import type { IrV0 } from "@crewhaus/ir";
+import { createLogger } from "@crewhaus/logging";
+import { McpHost } from "@crewhaus/mcp-host";
+import {
+  BUILTIN_DEFAULT_RULES,
+  PermissionConfigError,
+  type RuleSet,
+  parsePermissionsConfig,
+  tagRules,
+} from "@crewhaus/permission-engine";
+import { type SkillRef, createSkillTool, discoverSkills } from "@crewhaus/skills-registry";
+import { type SlashCommand, loadCommands } from "@crewhaus/slash-commands";
+import { spawnSubAgent } from "@crewhaus/sub-agent-spawner";
+import { type RegisteredTool, ToolCatalog } from "@crewhaus/tool-catalog";
+import { registerMcpServer } from "@crewhaus/tool-mcp";
+import { createTaskTool } from "@crewhaus/tool-task";
+import { RunnerError } from "./errors";
+
+type SpawnSubAgentFn = typeof spawnSubAgent;
+
+export type SharedAgentDeps = {
+  readonly tools: ReadonlyArray<RegisteredTool>;
+  readonly hooks: ReadonlyArray<HookDef>;
+  readonly skills: ReadonlyArray<SkillRef>;
+  readonly slashCommands: ReadonlyMap<string, SlashCommand>;
+  readonly subAgents?: ReadonlyMap<string, SubAgentDefinition>;
+  readonly spawnSubAgent?: SpawnSubAgentFn;
+  readonly permissionRules: RuleSet;
+  readonly mcpHost?: McpHost;
+  readonly model: string;
+  readonly instructions: string;
+  readonly sessionName: string;
+  readonly sessionTarget: string;
+};
+
+const logger = createLogger({ bindings: { module: "eval-runner.wire" } });
+
+export async function wireRunOnce(ir: IrV0, opts: { cwd?: string } = {}): Promise<SharedAgentDeps> {
+  const cwd = opts.cwd ?? process.cwd();
+
+  // Tools.
+  let tools: RegisteredTool[] = [];
+  if (ir.tools.length > 0) {
+    await applyToolConfigs(ir.tools, ir.toolConfigs);
+    const toolMap = await loadToolMap();
+    tools = ir.tools.map((name) => {
+      const tool = toolMap[name];
+      if (!tool) {
+        const known = Object.keys(toolMap).sort().join(", ");
+        throw new RunnerError(`unknown tool "${name}" — known tools: ${known}`);
+      }
+      return tool;
+    });
+  }
+
+  // MCP servers (shared across samples).
+  let mcpHost: McpHost | undefined;
+  if (Object.keys(ir.mcp_servers).length > 0) {
+    const host = new McpHost({ logger });
+    mcpHost = host;
+    for (const [name, cfg] of Object.entries(ir.mcp_servers)) host.addServer(name, cfg);
+    const tempCatalog = new ToolCatalog();
+    for (const t of tools) tempCatalog.register(t);
+    await Promise.all(
+      Object.keys(ir.mcp_servers).map((name) => registerMcpServer(host, name, tempCatalog)),
+    );
+    tools = tempCatalog.list().slice();
+  }
+
+  // Permission rules.
+  const permissionRules = buildRuleSet(ir.permissions.rules, cwd);
+
+  // Hooks / skills / slash-commands.
+  const [hooks, skills, slashCommands] = await Promise.all([
+    loadHooks({ cwd }),
+    discoverSkills({ cwd }),
+    loadCommands({ cwd }),
+  ]);
+  if (skills.length > 0) tools.push(createSkillTool(skills));
+
+  // Sub-agents.
+  let subAgents: ReadonlyMap<string, SubAgentDefinition> | undefined;
+  if (ir.subAgents.length > 0) {
+    subAgents = new Map(
+      ir.subAgents.map((d) => [
+        d.name,
+        {
+          name: d.name,
+          description: d.description,
+          instructions: d.instructions,
+          tools: d.tools,
+          ...(d.model !== undefined ? { model: d.model } : {}),
+          permissions: d.permissions,
+          inherit_bypass: d.inheritBypass,
+        } satisfies SubAgentDefinition,
+      ]),
+    );
+    tools.push(createTaskTool({ subAgents }));
+  }
+
+  return {
+    tools,
+    hooks,
+    skills,
+    slashCommands,
+    permissionRules,
+    model: ir.agent.model,
+    instructions: ir.agent.instructions,
+    sessionName: ir.name,
+    sessionTarget: ir.target,
+    ...(subAgents !== undefined ? { subAgents, spawnSubAgent } : {}),
+    ...(mcpHost !== undefined ? { mcpHost } : {}),
+  };
+}
+
+async function loadToolMap(): Promise<Record<string, RegisteredTool>> {
+  const [fs, bash, todo, web, image, fetchPkg] = await Promise.all([
+    import("@crewhaus/tool-fs"),
+    import("@crewhaus/tool-bash"),
+    import("@crewhaus/tool-todo"),
+    import("@crewhaus/tool-web"),
+    import("@crewhaus/tool-image"),
+    import("@crewhaus/tool-fetch"),
+  ]);
+  return {
+    read: fs.read,
+    write: fs.write,
+    edit: fs.edit,
+    glob: fs.glob,
+    grep: fs.grep,
+    bash: bash.bash,
+    todoWrite: todo.todoWrite,
+    webFetch: web.webFetch,
+    webSearch: web.webSearch,
+    readImage: image.readImage,
+    fetch: fetchPkg.fetch,
+  };
+}
+
+async function applyToolConfigs(
+  toolNames: readonly string[],
+  toolConfigs: Readonly<Record<string, unknown>>,
+): Promise<void> {
+  const used = new Set(toolNames);
+  if (used.has("fetch") && toolConfigs["fetch"] !== undefined) {
+    const { registerFetchConfig } = await import("@crewhaus/tool-fetch");
+    registerFetchConfig(toolConfigs["fetch"] as Parameters<typeof registerFetchConfig>[0]);
+  }
+  if (used.has("webFetch") && toolConfigs["webFetch"] !== undefined) {
+    const { registerWebFetchConfig } = await import("@crewhaus/tool-web");
+    registerWebFetchConfig(toolConfigs["webFetch"] as Parameters<typeof registerWebFetchConfig>[0]);
+  }
+}
+
+function buildRuleSet(
+  yamlRules: ReadonlyArray<{ type: "alwaysAllow" | "alwaysDeny" | "alwaysAsk"; pattern: string }>,
+  cwd: string,
+): RuleSet {
+  let settings: RuleSet["settings"] = [];
+  const settingsPath = join(cwd, ".crewhaus", "settings.json");
+  if (existsSync(settingsPath)) {
+    let raw: unknown;
+    try {
+      raw = JSON.parse(readFileSync(settingsPath, "utf-8"));
+    } catch (err) {
+      throw new RunnerError(`failed to parse ${settingsPath}: ${(err as Error).message}`, err);
+    }
+    const root = (raw as { permissions?: unknown }).permissions;
+    if (root !== undefined) {
+      try {
+        const parsed = parsePermissionsConfig(root, "settings");
+        settings = tagRules(parsed.rules, "settings");
+      } catch (err) {
+        if (err instanceof PermissionConfigError) throw new RunnerError(err.message, err);
+        throw err;
+      }
+    }
+  }
+  return {
+    flag: [],
+    settings,
+    yaml: tagRules(yamlRules, "yaml"),
+    hooks: [],
+    builtin: BUILTIN_DEFAULT_RULES,
+  };
+}