@crewhaus/eval-runner 0.1.4 → 0.1.5

package/src/index.judge.test.ts

@@ -1,167 +0,0 @@
-/**
- * Isolated test for the `llm_judge` grader-resolution branch in `runEval`.
- *
- * The runner replaces any compiled grader carrying a `judgeSpec` with a real
- * judge grader bound to the run's `judgeModel` (or the per-grader override).
- * We stub `@crewhaus/eval-judge` so no LLM/network is touched: `loadRubric`
- * echoes its input and `createJudgeGrader` returns a deterministic grader that
- * records the model it was bound to. `mock.module` is process-global, so this
- * lives in its own file (Bun gives each test file a fresh module graph).
- */
-import { afterAll, describe, expect, mock, test } from "bun:test";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { lower } from "@crewhaus/compiler";
-import type { Sample } from "@crewhaus/eval-dataset";
-import type { CompiledGrader, GradeResult } from "@crewhaus/eval-grader";
-import type { IrNode, IrV0 } from "@crewhaus/ir";
-import { parseSpec } from "@crewhaus/spec";
-
-const boundModels: Array<string | undefined> = [];
-const loadedRubrics: unknown[] = [];
-
-// Capture the real module so `afterAll` can restore it — `mock.module` is
-// process-global and does not auto-restore across test files. The capture is
-// a plain-object SNAPSHOT (`{ ...ns }`): an ESM namespace is a live view that
-// resolves to the stubs once mock.module patches the module, so restoring
-// from the namespace itself would silently reinstall the stubs.
-const realEvalJudge = { ...(await import("@crewhaus/eval-judge")) };
-
-mock.module("@crewhaus/eval-judge", () => ({
-  ...realEvalJudge,
-  loadRubric: (input: unknown) => {
-    loadedRubrics.push(input);
-    return { criteria: [{ name: "quality", weight: 1, description: "is it good" }] };
-  },
-  createJudgeGrader: (_rubric: unknown, opts: { model?: string } = {}) => {
-    boundModels.push(opts.model);
-    // Deterministic grader: always passes, no network.
-    return async (): Promise<GradeResult> => ({
-      passed: true,
-      score: 1,
-      rationale: `judged with ${opts.model ?? "(default)"}`,
-    });
-  },
-}));
-
-const { runEval } = await import("./index");
-
-function narrowToAgent(ir: IrNode): IrV0 {
-  if (ir.target !== "cli") throw new Error(`expected target:cli, got ${ir.target}`);
-  return ir;
-}
-
-const SPEC = `name: judge-test
-target: cli
-agent:
-  model: claude-opus-4-7
-  instructions: hi
-`;
-
-async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
-  for (const s of samples) yield s;
-}
-
-const TMP_ROOTS: string[] = [];
-function newTempRoot(): string {
-  const dir = mkdtempSync(join(tmpdir(), "crewhaus-judge-"));
-  TMP_ROOTS.push(dir);
-  return dir;
-}
-afterAll(() => {
-  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
-  mock.module("@crewhaus/eval-judge", () => realEvalJudge);
-});
-
-// A compiled grader with a judgeSpec but no per-grader model → uses opts.judgeModel.
-function judgeGraderNoModel(name: string): CompiledGrader {
-  return {
-    name,
-    grader: async () => {
-      throw new Error("placeholder must be replaced");
-    },
-    weight: 1,
-    judgeSpec: { rubric: { criteria: [{ name: "q", weight: 1, description: "d" }] } } as never,
-  };
-}
-
-// A compiled grader whose judgeSpec carries its own model override.
-function judgeGraderWithModel(name: string, model: string): CompiledGrader {
-  return {
-    name,
-    grader: async () => {
-      throw new Error("placeholder must be replaced");
-    },
-    weight: 1,
-    judgeSpec: {
-      rubric: { criteria: [{ name: "q", weight: 1, description: "d" }] },
-      model,
-    } as never,
-  };
-}
-
-describe("runEval — llm_judge resolution", () => {
-  test("binds judge grader to opts.judgeModel when no per-grader override", async () => {
-    const outDir = newTempRoot();
-    boundModels.length = 0;
-    loadedRubrics.length = 0;
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
-    const invoker = async () => ({ agentOutput: "anything", events: [] });
-
-    const summary = await runEval({
-      ir,
-      dataset: { name: "judged", samples: yieldSamples(samples) },
-      compiledGraders: [judgeGraderNoModel("rubricA")],
-      opts: { invoker, outDir, judgeModel: "claude-judge-x" },
-    });
-
-    expect(loadedRubrics).toHaveLength(1);
-    expect(boundModels).toEqual(["claude-judge-x"]);
-    expect(summary.config.graderNames).toEqual(["rubricA"]);
-    expect(summary.config.judgeModel).toBe("claude-judge-x");
-    expect(summary.aggregates.passRate).toBe(1);
-
-    // judgeModel surfaced in the persisted run.json snapshot.
-    const runJson = JSON.parse(readFileSync(join(outDir, "run.json"), "utf-8"));
-    expect(runJson.judgeModel).toBe("claude-judge-x");
-    expect(runJson.graderNames).toEqual(["rubricA"]);
-  });
-
-  test("per-grader judgeSpec.model overrides the run judgeModel", async () => {
-    const outDir = newTempRoot();
-    boundModels.length = 0;
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
-    const invoker = async () => ({ agentOutput: "anything", events: [] });
-
-    await runEval({
-      ir,
-      dataset: { name: "judged2", samples: yieldSamples(samples) },
-      compiledGraders: [judgeGraderWithModel("rubricB", "claude-override")],
-      // No opts.judgeModel → the per-grader override must win.
-      opts: { invoker, outDir },
-    });
-
-    expect(boundModels).toEqual(["claude-override"]);
-  });
-
-  test("a judge grader with neither override nor judgeModel binds the default", async () => {
-    const outDir = newTempRoot();
-    boundModels.length = 0;
-    const ir = narrowToAgent(lower(parseSpec(SPEC)));
-    const samples: Sample[] = [{ id: "q1", input: "hi", expected_output: "y" }];
-    const invoker = async () => ({ agentOutput: "anything", events: [] });
-
-    await runEval({
-      ir,
-      dataset: { name: "judged3", samples: yieldSamples(samples) },
-      compiledGraders: [judgeGraderNoModel("rubricC")],
-      opts: { invoker, outDir },
-    });
-
-    // model is undefined → createJudgeGrader called with `{}` (no model key).
-    expect(boundModels).toEqual([undefined]);
-  });
-});

package/src/index.test.ts

@@ -1,254 +0,0 @@
-import { afterAll, describe, expect, test } from "bun:test";
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { lower } from "@crewhaus/compiler";
-import type { Sample } from "@crewhaus/eval-dataset";
-import { parseGradersConfig } from "@crewhaus/eval-grader";
-import type { IrNode, IrV0 } from "@crewhaus/ir";
-import { parseSpec } from "@crewhaus/spec";
-
-function narrowToAgent(ir: IrNode): IrV0 {
-  if (ir.target !== "cli") throw new Error(`test fixture must be target:cli, got ${ir.target}`);
-  return ir;
-}
-import { aggregate, quantile } from "./aggregate";
-import { type AgentInvoker, runEval } from "./index";
-import { Semaphore } from "./semaphore";
-
-const TMP_ROOTS: string[] = [];
-function newTempRoot(): string {
-  const dir = mkdtempSync(join(tmpdir(), "crewhaus-eval-runner-"));
-  TMP_ROOTS.push(dir);
-  return dir;
-}
-afterAll(() => {
-  for (const dir of TMP_ROOTS) rmSync(dir, { recursive: true, force: true });
-});
-
-const HELLO_SPEC = `name: hello-test
-target: cli
-agent:
-  model: claude-opus-4-7
-  instructions: |
-    You are a helpful, concise assistant.
-`;
-
-async function* yieldSamples(samples: Sample[]): AsyncIterable<Sample> {
-  for (const s of samples) yield s;
-}
-
-const FIXED_GRADERS = `
-graders:
-  - name: math
-    type: exact_match
-`;
-
-describe("Semaphore", () => {
-  test("respects capacity", async () => {
-    const sem = new Semaphore(2);
-    let active = 0;
-    let maxActive = 0;
-    const tasks = Array.from({ length: 10 }).map(async () => {
-      const release = await sem.acquire();
-      active += 1;
-      maxActive = Math.max(maxActive, active);
-      await new Promise((r) => setTimeout(r, 5));
-      active -= 1;
-      release();
-    });
-    await Promise.all(tasks);
-    expect(maxActive).toBeLessThanOrEqual(2);
-  });
-
-  test("rejects capacity < 1", () => {
-    expect(() => new Semaphore(0)).toThrow();
-  });
-});
-
-describe("aggregate", () => {
-  test("quantile handles edge cases", () => {
-    expect(quantile([], 0.5)).toBe(0);
-    expect(quantile([42], 0.5)).toBe(42);
-    expect(quantile([1, 2, 3, 4, 5], 0.5)).toBe(3);
-    expect(quantile([1, 2, 3, 4, 5], 0.95)).toBeCloseTo(4.8);
-  });
-
-  test("computes pass rate, mean score, percentiles", () => {
-    const samples = [
-      mockSample("s1", true, 1, 100, 1),
-      mockSample("s2", true, 0.5, 200, 2),
-      mockSample("s3", false, 0, 300, 3),
-    ];
-    const agg = aggregate(samples);
-    expect(agg.passRate).toBeCloseTo(2 / 3);
-    expect(agg.meanScore).toBeCloseTo(0.5);
-    expect(agg.p50LatencyMs).toBe(200);
-    expect(agg.errorCount).toBe(0);
-  });
-});
-
-describe("runEval — T3 5-sample fixture", () => {
-  test("runs 5 samples, persists artifacts, computes aggregates", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
-    const samples: Sample[] = [
-      { id: "q1", input: "What is 2+2?", expected_output: "4" },
-      { id: "q2", input: "What is 5-3?", expected_output: "2" },
-      { id: "q3", input: "What is 6/2?", expected_output: "3" },
-      { id: "q4", input: "What is 3*3?", expected_output: "9" },
-      { id: "q5", input: "What is 10-7?", expected_output: "3" },
-    ];
-
-    // Stub invoker: returns the expected_output verbatim. exact_match grader passes.
-    const invoker: AgentInvoker = async ({ sample }) => ({
-      agentOutput: sample.expected_output ?? "",
-      transcript: [],
-      events: [],
-    });
-
-    const { compiled } = parseGradersConfig(FIXED_GRADERS);
-    const summary = await runEval({
-      ir,
-      dataset: { name: "fixture5", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      opts: { invoker, outDir, concurrency: 2 },
-    });
-
-    expect(summary.samples).toHaveLength(5);
-    expect(summary.aggregates.passRate).toBe(1);
-    expect(summary.aggregates.meanScore).toBe(1);
-
-    // Per-sample artifacts
-    for (const id of ["q1", "q2", "q3", "q4", "q5"]) {
-      expect(existsSync(join(outDir, id, "transcript.jsonl"))).toBe(true);
-      expect(existsSync(join(outDir, id, "events.jsonl"))).toBe(true);
-      expect(existsSync(join(outDir, id, "grades.json"))).toBe(true);
-      expect(existsSync(join(outDir, id, "meta.json"))).toBe(true);
-    }
-    // Run-level artifacts
-    expect(existsSync(join(outDir, "run.json"))).toBe(true);
-    expect(existsSync(join(outDir, "results.json"))).toBe(true);
-    const results = JSON.parse(readFileSync(join(outDir, "results.json"), "utf-8"));
-    expect(results.samples).toHaveLength(5);
-    expect(results.aggregates.passRate).toBe(1);
-  });
-
-  test("captures grader failures correctly", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
-    const samples: Sample[] = [
-      { id: "ok", input: "x", expected_output: "y" },
-      { id: "wrong", input: "x", expected_output: "y" },
-    ];
-    const invoker: AgentInvoker = async ({ sample }) => ({
-      agentOutput: sample.id === "ok" ? "y" : "z",
-      transcript: [],
-      events: [],
-    });
-    const { compiled } = parseGradersConfig(FIXED_GRADERS);
-    const summary = await runEval({
-      ir,
-      dataset: { name: "mixed", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      opts: { invoker, outDir },
-    });
-    expect(summary.aggregates.passRate).toBe(0.5);
-    expect(summary.samples.find((s) => s.sampleId === "wrong")?.grades.overall.passed).toBe(false);
-  });
-
-  test("invoker errors do not kill the rest of the run", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
-    const samples: Sample[] = [
-      { id: "ok", input: "x", expected_output: "y" },
-      { id: "boom", input: "x", expected_output: "y" },
-    ];
-    const invoker: AgentInvoker = async ({ sample }) => {
-      if (sample.id === "boom") throw new Error("invoker exploded");
-      return { agentOutput: sample.expected_output ?? "", transcript: [], events: [] };
-    };
-    const { compiled } = parseGradersConfig(FIXED_GRADERS);
-    const summary = await runEval({
-      ir,
-      dataset: { name: "withErr", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      opts: { invoker, outDir },
-    });
-    expect(summary.samples).toHaveLength(2);
-    expect(summary.aggregates.errorCount).toBe(1);
-    const boom = summary.samples.find((s) => s.sampleId === "boom");
-    expect(boom?.error).toBe("invoker exploded");
-    expect(boom?.grades.overall.passed).toBe(false);
-    // The good sample still completed and was scored.
-    expect(summary.samples.find((s) => s.sampleId === "ok")?.grades.overall.passed).toBe(true);
-  });
-
-  test("rejects empty dataset", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
-    const invoker: AgentInvoker = async () => ({
-      agentOutput: "",
-      transcript: [],
-      events: [],
-    });
-    const { compiled } = parseGradersConfig(FIXED_GRADERS);
-    await expect(
-      runEval({
-        ir,
-        dataset: { name: "empty", samples: yieldSamples([]) },
-        compiledGraders: compiled,
-        opts: { invoker, outDir },
-      }),
-    ).rejects.toThrow(/yielded zero samples/);
-  });
-});
-
-describe("runEval — T7 200-sample concurrency-8 SLO", () => {
-  test("completes 200 samples at concurrency 8 well under 60s", async () => {
-    const outDir = newTempRoot();
-    const ir = narrowToAgent(lower(parseSpec(HELLO_SPEC)));
-    const samples: Sample[] = Array.from({ length: 200 }).map((_, i) => ({
-      id: `s${i.toString().padStart(3, "0")}`,
-      input: `q${i}`,
-      expected_output: `a${i}`,
-    }));
-    const invoker: AgentInvoker = async ({ sample }) => {
-      // Tiny delay to mimic real-world async work without burning CPU.
-      await new Promise((r) => setTimeout(r, 1));
-      return { agentOutput: sample.expected_output ?? "", transcript: [], events: [] };
-    };
-    const { compiled } = parseGradersConfig(FIXED_GRADERS);
-
-    const t0 = performance.now();
-    const summary = await runEval({
-      ir,
-      dataset: { name: "load200", samples: yieldSamples(samples) },
-      compiledGraders: compiled,
-      opts: { invoker, outDir, concurrency: 8 },
-    });
-    const elapsedMs = performance.now() - t0;
-
-    expect(summary.samples).toHaveLength(200);
-    expect(summary.aggregates.passRate).toBe(1);
-    expect(elapsedMs).toBeLessThan(60_000);
-  }, 90_000);
-});
-
-function mockSample(id: string, passed: boolean, score: number, latencyMs: number, turns: number) {
-  return {
-    sampleId: id,
-    sessionId: `sess_${id.padEnd(16, "0")}`,
-    startedAt: "2026-01-01T00:00:00Z",
-    endedAt: "2026-01-01T00:00:01Z",
-    latencyMs,
-    turns,
-    tokens: { input: 10, output: 20 },
-    model: "claude-opus-4-7",
-    agentOutput: "x",
-    grades: {
-      overall: { passed, score, rationale: "" },
-      perGrader: [],
-    },
-  };
-}

package/src/index.ts

@@ -1,264 +0,0 @@
-/**
- * Catalog R-eval `eval-runner` — run an agent against a dataset.
- *
- * Per-sample isolation: fresh `RunContext` (and therefore fresh
- * `TraceEventBus` + `sessionId`) so concurrent samples don't see each
- * other's events. Concurrency is enforced by a tiny in-package semaphore
- * (default 4). Per-sample artifacts persist to
- * `.crewhaus/evals/<runId>/<sampleId>/{transcript.jsonl, events.jsonl,
- * grades.json, meta.json}`. Run-level: `run.json` (config snapshot)
- * + `results.json` (aggregates).
- *
- * `--seed` is honored only for providers that surface temperature
- * reproducibility (Anthropic does not). Document divergence; do not
- * promise byte-identical reruns.
- *
- * `permissionMode` is forced to `"auto"` so `alwaysAsk` rules auto-deny
- * rather than blocking on stdin in a non-interactive eval run.
- *
- * MCP servers are shared across samples (read-mostly assumption). An
- * `isolateMcpPerSample` escape hatch is reserved for future use.
- *
- * Reference: build-roadmap.md §16.
- */
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import type { Sample } from "@crewhaus/eval-dataset";
-import type { CompiledGrader, Grader } from "@crewhaus/eval-grader";
-import { createJudgeGrader, loadRubric } from "@crewhaus/eval-judge";
-import type { IrV0 } from "@crewhaus/ir";
-import { runChatLoop } from "@crewhaus/runtime-core";
-import { currentTenantContext } from "@crewhaus/tenancy";
-import { aggregate } from "./aggregate";
-import { RunnerError } from "./errors";
-import { runSample } from "./run-sample";
-import { Semaphore } from "./semaphore";
-import type {
-  AgentInvoker,
-  EvalRunSummary,
-  GraderEntry,
-  RunEvalOptions,
-  SampleResult,
-} from "./types";
-import { type SharedAgentDeps, wireRunOnce } from "./wire-once";
-
-export type { AgentInvoker, EvalRunSummary, GraderEntry, RunEvalOptions, SampleResult };
-export type { SharedAgentDeps };
-export { wireRunOnce };
-export { Semaphore };
-export { aggregate };
-export { RunnerError };
-export { runSample };
-
-const DEFAULT_CONCURRENCY = 4;
-
-export type RunEvalArgs = {
-  /** Lowered agent IR (target: cli). Caller is responsible for narrowing. */
-  readonly ir: IrV0;
-  readonly dataset: { name: string; samples: AsyncIterable<Sample> };
-  readonly compiledGraders: ReadonlyArray<CompiledGrader>;
-  readonly opts?: RunEvalOptions;
-};
-
-/**
- * Resolve the eval output directory. When an eval runs inside a tenant scope
- * (e.g. a cloud/managed eval), the tenant's rebased `evalRoot` is used so one
- * tenant's eval artifacts never share a directory with another's; the global
- * default is only used outside any tenant scope (#150). An explicit
- * `optsOutDir` always wins for trusted callers.
- */
-export function resolveEvalOutDir(runId: string, optsOutDir?: string): string {
-  if (optsOutDir !== undefined) return optsOutDir;
-  const tenant = currentTenantContext()?.tenant;
-  if (tenant !== undefined) return join(tenant.evalRoot, runId);
-  return join(".crewhaus", "evals", runId);
-}
-
-/**
- * Execute an evaluation. Returns a summary; per-sample artifacts and
- * the summary itself are also persisted under `outDir`.
- */
-export async function runEval(args: RunEvalArgs): Promise<EvalRunSummary> {
-  const { ir, dataset, compiledGraders } = args;
-  const opts = args.opts ?? {};
-
-  const runId = opts.runId ?? generateRunId();
-  const outDir = resolveEvalOutDir(runId, opts.outDir);
-  mkdirSync(outDir, { recursive: true });
-
-  // Resolve graders. Replace any `llm_judge` placeholder with a real judge
-  // grader bound to the runner's judgeModel (or the per-grader override).
-  const graders: GraderEntry[] = compiledGraders.map((g) => {
-    if (g.judgeSpec) {
-      const rubric = loadRubric(g.judgeSpec.rubric);
-      const model = g.judgeSpec.model ?? opts.judgeModel;
-      const grader = createJudgeGrader(rubric, model !== undefined ? { model } : {});
-      return { name: g.name, grader };
-    }
-    return { name: g.name, grader: g.grader };
-  });
-
-  // The default invoker calls runChatLoop with the per-sample fresh runContext.
-  const invoker = opts.invoker ?? (await defaultInvoker(ir, opts));
-
-  const startedAt = new Date().toISOString();
-  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
-  const sem = new Semaphore(concurrency);
-
-  // Persist run-level config snapshot up front so SIGINT mid-run still leaves
-  // a usable directory.
-  const specHash = await hashSpec(ir);
-  await Bun.write(
-    join(outDir, "run.json"),
-    JSON.stringify(
-      {
-        runId,
-        startedAt,
-        specHash,
-        datasetName: dataset.name,
-        graderNames: graders.map((g) => g.name),
-        model: ir.agent.model,
-        ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
-        concurrency,
-        ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
-      },
-      null,
-      2,
-    ),
-  );
-
-  // Materialize the dataset into a list so we can run samples concurrently.
-  // For very large datasets the streaming loaders still avoid loading the
-  // whole file into memory — but eventually all sample objects sit in RAM.
-  const samples: Sample[] = [];
-  for await (const s of dataset.samples) samples.push(s);
-  if (samples.length === 0) {
-    throw new RunnerError(`dataset "${dataset.name}" yielded zero samples`);
-  }
-
-  let interrupted = false;
-  const abortHandler = () => {
-    interrupted = true;
-  };
-  process.once("SIGINT", abortHandler);
-
-  const settled = await Promise.allSettled(
-    samples.map(async (sample) => {
-      const release = await sem.acquire();
-      // Check *after* acquiring the slot: every callback's synchronous prefix
-      // runs during `.map()` (before any SIGINT can fire), so a pre-acquire
-      // check would never observe a mid-run interrupt. Samples still queued on
-      // the semaphore when SIGINT arrives are skipped here as their turn comes.
-      if (interrupted) {
-        release();
-        throw new RunnerError(`run interrupted before sample "${sample.id}"`);
-      }
-      try {
-        return await runSample({
-          sample,
-          invoker,
-          graders,
-          outDir,
-          model: ir.agent.model,
-          ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
-        });
-      } finally {
-        release();
-      }
-    }),
-  );
-
-  process.removeListener("SIGINT", abortHandler);
-
-  const results: SampleResult[] = settled.map((r, i) => {
-    if (r.status === "fulfilled") return r.value;
-    const sample = samples[i];
-    return {
-      sampleId: sample?.id ?? `unknown-${i}`,
-      sessionId: "(unset)",
-      startedAt,
-      endedAt: new Date().toISOString(),
-      latencyMs: 0,
-      turns: 0,
-      tokens: { input: 0, output: 0 },
-      model: ir.agent.model,
-      agentOutput: "",
-      grades: {
-        overall: { passed: false, score: 0, rationale: "sample failed entirely" },
-        perGrader: [],
-      },
-      error: r.reason instanceof Error ? r.reason.message : String(r.reason),
-    };
-  });
-
-  const endedAt = new Date().toISOString();
-  const aggregates = aggregate(results);
-
-  const summary: EvalRunSummary = {
-    runId,
-    startedAt,
-    endedAt,
-    samples: results,
-    aggregates,
-    config: {
-      specHash,
-      datasetName: dataset.name,
-      graderNames: graders.map((g) => g.name),
-      model: ir.agent.model,
-      ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
-      concurrency,
-      ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
-    },
-    outDir,
-  };
-
-  await Bun.write(join(outDir, "results.json"), JSON.stringify(summary, null, 2));
-
-  return summary;
-}
-
-async function defaultInvoker(ir: IrV0, opts: RunEvalOptions): Promise<AgentInvoker> {
-  const wired: SharedAgentDeps = await wireRunOnce(
-    ir,
-    opts.cwd !== undefined ? { cwd: opts.cwd } : {},
-  );
-  return async (req) => {
-    const agentOutput = await runChatLoop({
-      model: wired.model,
-      instructions: wired.instructions,
-      tools: [...wired.tools],
-      hooks: wired.hooks,
-      skills: wired.skills,
-      slashCommands: wired.slashCommands,
-      ...(wired.subAgents !== undefined && wired.spawnSubAgent !== undefined
-        ? { subAgents: wired.subAgents, spawnSubAgent: wired.spawnSubAgent }
-        : {}),
-      permissionRules: wired.permissionRules,
-      permissionMode: "auto",
-      sessionName: `${wired.sessionName}_${req.sample.id}`,
-      sessionTarget: wired.sessionTarget,
-      runContext: req.runContext,
-      sessionRootDir: req.sessionRootDir,
-      singleTurn: true,
-      seedMessages: [{ role: "user", content: req.sample.input }],
-    });
-    return { agentOutput };
-  };
-}
-
-function generateRunId(): string {
-  const bytes = new Uint8Array(8);
-  crypto.getRandomValues(bytes);
-  return `run_${Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")}`;
-}
-
-async function hashSpec(ir: IrV0): Promise<string> {
-  const text = JSON.stringify({
-    name: ir.name,
-    target: ir.target,
-    model: ir.agent.model,
-    instructions: ir.agent.instructions,
-    tools: ir.tools,
-  });
-  return Bun.hash(text).toString(16);
-}