@crewhaus/eval-runner 0.1.3 → 0.1.5

package/dist/aggregate.d.ts

@@ -0,0 +1,15 @@
+import type { SampleResult } from "./types";
+export declare function quantile(sorted: ReadonlyArray<number>, q: number): number;
+export declare function aggregate(samples: ReadonlyArray<SampleResult>): {
+    passRate: number;
+    meanScore: number;
+    p50Turns: number;
+    p95Turns: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    totalTokens: {
+        input: number;
+        output: number;
+    };
+    errorCount: number;
+};

package/dist/aggregate.js

@@ -0,0 +1,35 @@
+export function quantile(sorted, q) {
+    if (sorted.length === 0)
+        return 0;
+    if (sorted.length === 1)
+        return sorted[0] ?? 0;
+    const idx = q * (sorted.length - 1);
+    const lo = Math.floor(idx);
+    const hi = Math.ceil(idx);
+    if (lo === hi)
+        return sorted[lo] ?? 0;
+    const fract = idx - lo;
+    return (sorted[lo] ?? 0) * (1 - fract) + (sorted[hi] ?? 0) * fract;
+}
+export function aggregate(samples) {
+    const ok = samples.filter((s) => s.error === undefined);
+    const total = samples.length;
+    const passed = ok.filter((s) => s.grades.overall.passed).length;
+    const meanScore = ok.length === 0 ? 0 : ok.reduce((sum, s) => sum + s.grades.overall.score, 0) / ok.length;
+    const turnsSorted = ok.map((s) => s.turns).sort((a, b) => a - b);
+    const latSorted = ok.map((s) => s.latencyMs).sort((a, b) => a - b);
+    const totalTokens = ok.reduce((acc, s) => ({
+        input: acc.input + s.tokens.input,
+        output: acc.output + s.tokens.output,
+    }), { input: 0, output: 0 });
+    return {
+        passRate: total === 0 ? 0 : passed / total,
+        meanScore,
+        p50Turns: quantile(turnsSorted, 0.5),
+        p95Turns: quantile(turnsSorted, 0.95),
+        p50LatencyMs: quantile(latSorted, 0.5),
+        p95LatencyMs: quantile(latSorted, 0.95),
+        totalTokens,
+        errorCount: samples.length - ok.length,
+    };
+}

package/dist/errors.d.ts

@@ -0,0 +1,4 @@
+import { RuntimeError } from "@crewhaus/errors";
+export declare class RunnerError extends RuntimeError {
+    constructor(message: string, cause?: unknown);
+}

package/dist/errors.js

@@ -0,0 +1,6 @@
+import { RuntimeError } from "@crewhaus/errors";
+export class RunnerError extends RuntimeError {
+    constructor(message, cause) {
+        super(`eval-runner: ${message}`, cause);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,39 @@
+import type { Sample } from "@crewhaus/eval-dataset";
+import type { CompiledGrader } from "@crewhaus/eval-grader";
+import type { IrV0 } from "@crewhaus/ir";
+import { aggregate } from "./aggregate";
+import { RunnerError } from "./errors";
+import { runSample } from "./run-sample";
+import { Semaphore } from "./semaphore";
+import type { AgentInvoker, EvalRunSummary, GraderEntry, RunEvalOptions, SampleResult } from "./types";
+import { type SharedAgentDeps, wireRunOnce } from "./wire-once";
+export type { AgentInvoker, EvalRunSummary, GraderEntry, RunEvalOptions, SampleResult };
+export type { SharedAgentDeps };
+export { wireRunOnce };
+export { Semaphore };
+export { aggregate };
+export { RunnerError };
+export { runSample };
+export type RunEvalArgs = {
+    /** Lowered agent IR (target: cli). Caller is responsible for narrowing. */
+    readonly ir: IrV0;
+    readonly dataset: {
+        name: string;
+        samples: AsyncIterable<Sample>;
+    };
+    readonly compiledGraders: ReadonlyArray<CompiledGrader>;
+    readonly opts?: RunEvalOptions;
+};
+/**
+ * Resolve the eval output directory. When an eval runs inside a tenant scope
+ * (e.g. a cloud/managed eval), the tenant's rebased `evalRoot` is used so one
+ * tenant's eval artifacts never share a directory with another's; the global
+ * default is only used outside any tenant scope (#150). An explicit
+ * `optsOutDir` always wins for trusted callers.
+ */
+export declare function resolveEvalOutDir(runId: string, optsOutDir?: string): string;
+/**
+ * Execute an evaluation. Returns a summary; per-sample artifacts and
+ * the summary itself are also persisted under `outDir`.
+ */
+export declare function runEval(args: RunEvalArgs): Promise<EvalRunSummary>;

package/dist/index.js

@@ -0,0 +1,216 @@
+/**
+ * Catalog R-eval `eval-runner` — run an agent against a dataset.
+ *
+ * Per-sample isolation: fresh `RunContext` (and therefore fresh
+ * `TraceEventBus` + `sessionId`) so concurrent samples don't see each
+ * other's events. Concurrency is enforced by a tiny in-package semaphore
+ * (default 4). Per-sample artifacts persist to
+ * `.crewhaus/evals/<runId>/<sampleId>/{transcript.jsonl, events.jsonl,
+ * grades.json, meta.json}`. Run-level: `run.json` (config snapshot)
+ * + `results.json` (aggregates).
+ *
+ * `--seed` is honored only for providers that surface temperature
+ * reproducibility (Anthropic does not). Document divergence; do not
+ * promise byte-identical reruns.
+ *
+ * `permissionMode` is forced to `"auto"` so `alwaysAsk` rules auto-deny
+ * rather than blocking on stdin in a non-interactive eval run.
+ *
+ * MCP servers are shared across samples (read-mostly assumption). An
+ * `isolateMcpPerSample` escape hatch is reserved for future use.
+ *
+ * Reference: build-roadmap.md §16.
+ */
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { createJudgeGrader, loadRubric } from "@crewhaus/eval-judge";
+import { runChatLoop } from "@crewhaus/runtime-core";
+import { currentTenantContext } from "@crewhaus/tenancy";
+import { aggregate } from "./aggregate";
+import { RunnerError } from "./errors";
+import { runSample } from "./run-sample";
+import { Semaphore } from "./semaphore";
+import { wireRunOnce } from "./wire-once";
+export { wireRunOnce };
+export { Semaphore };
+export { aggregate };
+export { RunnerError };
+export { runSample };
+const DEFAULT_CONCURRENCY = 4;
+/**
+ * Resolve the eval output directory. When an eval runs inside a tenant scope
+ * (e.g. a cloud/managed eval), the tenant's rebased `evalRoot` is used so one
+ * tenant's eval artifacts never share a directory with another's; the global
+ * default is only used outside any tenant scope (#150). An explicit
+ * `optsOutDir` always wins for trusted callers.
+ */
+export function resolveEvalOutDir(runId, optsOutDir) {
+    if (optsOutDir !== undefined)
+        return optsOutDir;
+    const tenant = currentTenantContext()?.tenant;
+    if (tenant !== undefined)
+        return join(tenant.evalRoot, runId);
+    return join(".crewhaus", "evals", runId);
+}
+/**
+ * Execute an evaluation. Returns a summary; per-sample artifacts and
+ * the summary itself are also persisted under `outDir`.
+ */
+export async function runEval(args) {
+    const { ir, dataset, compiledGraders } = args;
+    const opts = args.opts ?? {};
+    const runId = opts.runId ?? generateRunId();
+    const outDir = resolveEvalOutDir(runId, opts.outDir);
+    mkdirSync(outDir, { recursive: true });
+    // Resolve graders. Replace any `llm_judge` placeholder with a real judge
+    // grader bound to the runner's judgeModel (or the per-grader override).
+    const graders = compiledGraders.map((g) => {
+        if (g.judgeSpec) {
+            const rubric = loadRubric(g.judgeSpec.rubric);
+            const model = g.judgeSpec.model ?? opts.judgeModel;
+            const grader = createJudgeGrader(rubric, model !== undefined ? { model } : {});
+            return { name: g.name, grader };
+        }
+        return { name: g.name, grader: g.grader };
+    });
+    // The default invoker calls runChatLoop with the per-sample fresh runContext.
+    const invoker = opts.invoker ?? (await defaultInvoker(ir, opts));
+    const startedAt = new Date().toISOString();
+    const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
+    const sem = new Semaphore(concurrency);
+    // Persist run-level config snapshot up front so SIGINT mid-run still leaves
+    // a usable directory.
+    const specHash = await hashSpec(ir);
+    await Bun.write(join(outDir, "run.json"), JSON.stringify({
+        runId,
+        startedAt,
+        specHash,
+        datasetName: dataset.name,
+        graderNames: graders.map((g) => g.name),
+        model: ir.agent.model,
+        ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
+        concurrency,
+        ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+    }, null, 2));
+    // Materialize the dataset into a list so we can run samples concurrently.
+    // For very large datasets the streaming loaders still avoid loading the
+    // whole file into memory — but eventually all sample objects sit in RAM.
+    const samples = [];
+    for await (const s of dataset.samples)
+        samples.push(s);
+    if (samples.length === 0) {
+        throw new RunnerError(`dataset "${dataset.name}" yielded zero samples`);
+    }
+    let interrupted = false;
+    const abortHandler = () => {
+        interrupted = true;
+    };
+    process.once("SIGINT", abortHandler);
+    const settled = await Promise.allSettled(samples.map(async (sample) => {
+        const release = await sem.acquire();
+        // Check *after* acquiring the slot: every callback's synchronous prefix
+        // runs during `.map()` (before any SIGINT can fire), so a pre-acquire
+        // check would never observe a mid-run interrupt. Samples still queued on
+        // the semaphore when SIGINT arrives are skipped here as their turn comes.
+        if (interrupted) {
+            release();
+            throw new RunnerError(`run interrupted before sample "${sample.id}"`);
+        }
+        try {
+            return await runSample({
+                sample,
+                invoker,
+                graders,
+                outDir,
+                model: ir.agent.model,
+                ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+            });
+        }
+        finally {
+            release();
+        }
+    }));
+    process.removeListener("SIGINT", abortHandler);
+    const results = settled.map((r, i) => {
+        if (r.status === "fulfilled")
+            return r.value;
+        const sample = samples[i];
+        return {
+            sampleId: sample?.id ?? `unknown-${i}`,
+            sessionId: "(unset)",
+            startedAt,
+            endedAt: new Date().toISOString(),
+            latencyMs: 0,
+            turns: 0,
+            tokens: { input: 0, output: 0 },
+            model: ir.agent.model,
+            agentOutput: "",
+            grades: {
+                overall: { passed: false, score: 0, rationale: "sample failed entirely" },
+                perGrader: [],
+            },
+            error: r.reason instanceof Error ? r.reason.message : String(r.reason),
+        };
+    });
+    const endedAt = new Date().toISOString();
+    const aggregates = aggregate(results);
+    const summary = {
+        runId,
+        startedAt,
+        endedAt,
+        samples: results,
+        aggregates,
+        config: {
+            specHash,
+            datasetName: dataset.name,
+            graderNames: graders.map((g) => g.name),
+            model: ir.agent.model,
+            ...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
+            concurrency,
+            ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+        },
+        outDir,
+    };
+    await Bun.write(join(outDir, "results.json"), JSON.stringify(summary, null, 2));
+    return summary;
+}
+async function defaultInvoker(ir, opts) {
+    const wired = await wireRunOnce(ir, opts.cwd !== undefined ? { cwd: opts.cwd } : {});
+    return async (req) => {
+        const agentOutput = await runChatLoop({
+            model: wired.model,
+            instructions: wired.instructions,
+            tools: [...wired.tools],
+            hooks: wired.hooks,
+            skills: wired.skills,
+            slashCommands: wired.slashCommands,
+            ...(wired.subAgents !== undefined && wired.spawnSubAgent !== undefined
+                ? { subAgents: wired.subAgents, spawnSubAgent: wired.spawnSubAgent }
+                : {}),
+            permissionRules: wired.permissionRules,
+            permissionMode: "auto",
+            sessionName: `${wired.sessionName}_${req.sample.id}`,
+            sessionTarget: wired.sessionTarget,
+            runContext: req.runContext,
+            sessionRootDir: req.sessionRootDir,
+            singleTurn: true,
+            seedMessages: [{ role: "user", content: req.sample.input }],
+        });
+        return { agentOutput };
+    };
+}
+function generateRunId() {
+    const bytes = new Uint8Array(8);
+    crypto.getRandomValues(bytes);
+    return `run_${Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")}`;
+}
+async function hashSpec(ir) {
+    const text = JSON.stringify({
+        name: ir.name,
+        target: ir.target,
+        model: ir.agent.model,
+        instructions: ir.agent.instructions,
+        tools: ir.tools,
+    });
+    return Bun.hash(text).toString(16);
+}

package/dist/run-sample.d.ts

@@ -0,0 +1,19 @@
+import type { Sample } from "@crewhaus/eval-dataset";
+import { type CompiledGrader, type Grader } from "@crewhaus/eval-grader";
+import type { AgentInvoker, GraderEntry, SampleResult } from "./types";
+/**
+ * Per-sample logic. Mints a fresh runContext (and therefore a fresh
+ * TraceEventBus + sessionId), subscribes to the bus, calls the invoker,
+ * and persists `transcript.jsonl`, `events.jsonl`, `grades.json`,
+ * `meta.json` to the per-sample directory.
+ */
+export declare function runSample(args: {
+    sample: Sample;
+    invoker: AgentInvoker;
+    graders: ReadonlyArray<GraderEntry>;
+    outDir: string;
+    model: string;
+    seed?: number;
+}): Promise<SampleResult>;
+/** Helper: produce one combined grader from an array of CompiledGrader. */
+export declare function combineGraderEntries(graders: ReadonlyArray<CompiledGrader>): Grader;

package/dist/run-sample.js

@@ -0,0 +1,187 @@
+import { mkdirSync, renameSync } from "node:fs";
+import { join } from "node:path";
+import { combineCompiledGraders, } from "@crewhaus/eval-grader";
+import { openEventLog } from "@crewhaus/event-log";
+import { createRunContext } from "@crewhaus/run-context";
+import { RunnerError } from "./errors";
+/**
+ * Per-sample logic. Mints a fresh runContext (and therefore a fresh
+ * TraceEventBus + sessionId), subscribes to the bus, calls the invoker,
+ * and persists `transcript.jsonl`, `events.jsonl`, `grades.json`,
+ * `meta.json` to the per-sample directory.
+ */
+export async function runSample(args) {
+    const { sample, invoker, graders, outDir, model } = args;
+    const sampleDir = join(outDir, sanitize(sample.id));
+    mkdirSync(sampleDir, { recursive: true });
+    const runContext = createRunContext();
+    const events = [];
+    const unsubscribe = runContext.eventBus.subscribe((e) => {
+        events.push(e);
+    });
+    const startedAt = new Date().toISOString();
+    const t0 = performance.now();
+    let agentOutput = "";
+    let transcript = [];
+    let invokerEvents;
+    let error;
+    try {
+        const result = await invoker({
+            sample,
+            runContext,
+            sessionRootDir: sampleDir,
+            ...(args.seed !== undefined ? { seed: args.seed } : {}),
+        });
+        agentOutput = result.agentOutput;
+        transcript = result.transcript ? [...result.transcript] : [];
+        invokerEvents = result.events;
+    }
+    catch (err) {
+        error = err instanceof Error ? err.message : String(err);
+    }
+    finally {
+        unsubscribe();
+    }
+    const endedAt = new Date().toISOString();
+    const latencyMs = Math.round(performance.now() - t0);
+    // If the invoker didn't supply a transcript, attempt to read it from disk
+    // (this is the path the production runChatLoop wrapper takes).
+    if (transcript.length === 0 && error === undefined) {
+        transcript = await readTranscript(sampleDir, runContext.sessionId);
+    }
+    const finalEvents = invokerEvents ? [...invokerEvents] : events;
+    // Rename auto-named event-log file → transcript.jsonl for stable artifact
+    // names. (Skip if the invoker didn't actually write one, e.g. test stubs.)
+    const autoPath = join(sampleDir, `${runContext.sessionId}.jsonl`);
+    const transcriptPath = join(sampleDir, "transcript.jsonl");
+    if (await Bun.file(autoPath).exists()) {
+        renameSync(autoPath, transcriptPath);
+    }
+    else if (transcript.length > 0) {
+        // Stub-supplied transcript — write it ourselves.
+        await Bun.write(transcriptPath, transcript.map((e) => JSON.stringify(e)).join("\n") + (transcript.length > 0 ? "\n" : ""));
+    }
+    else {
+        await Bun.write(transcriptPath, "");
+    }
+    // Persist captured trace events.
+    await Bun.write(join(sampleDir, "events.jsonl"), finalEvents.map((e) => JSON.stringify(e)).join("\n") + (finalEvents.length > 0 ? "\n" : ""));
+    const turns = transcript.filter((e) => e.kind === "assistant_message").length;
+    const toolCalls = extractToolCalls(finalEvents);
+    const tokens = sumTokens(finalEvents);
+    // Apply graders.
+    const runResult = {
+        agentOutput,
+        events: finalEvents,
+        transcript,
+        toolCalls,
+        turns,
+        latencyMs,
+    };
+    const perGrader = [];
+    for (const g of graders) {
+        let result;
+        try {
+            result = await g.grader(sample, runResult);
+        }
+        catch (err) {
+            result = {
+                passed: false,
+                score: 0,
+                rationale: `grader threw: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+        perGrader.push({ name: g.name, ...result });
+    }
+    // Overall = AND of all graders, score = mean.
+    const overall = {
+        passed: error === undefined && perGrader.every((g) => g.passed),
+        score: perGrader.length === 0 ? 0 : perGrader.reduce((s, g) => s + g.score, 0) / perGrader.length,
+        rationale: error !== undefined
+            ? `agent invocation error: ${error}`
+            : perGrader.map((g) => `[${g.name}: ${g.passed ? "✓" : "✗"}] ${g.rationale}`).join(" & "),
+    };
+    const sampleResult = {
+        sampleId: sample.id,
+        sessionId: runContext.sessionId,
+        startedAt,
+        endedAt,
+        latencyMs,
+        turns,
+        tokens,
+        model,
+        agentOutput,
+        grades: { overall, perGrader },
+        ...(error !== undefined ? { error } : {}),
+    };
+    await Bun.write(join(sampleDir, "grades.json"), JSON.stringify(sampleResult.grades, null, 2));
+    await Bun.write(join(sampleDir, "meta.json"), JSON.stringify({
+        sampleId: sample.id,
+        sessionId: runContext.sessionId,
+        startedAt,
+        endedAt,
+        latencyMs,
+        turns,
+        tokens,
+        model,
+        ...(error !== undefined ? { error } : {}),
+        ...(args.seed !== undefined ? { seed: args.seed } : {}),
+    }, null, 2));
+    return sampleResult;
+}
+/** Helper: produce one combined grader from an array of CompiledGrader. */
+export function combineGraderEntries(graders) {
+    return combineCompiledGraders(graders);
+}
+/** Strip path separators from sample IDs to keep the on-disk layout flat. */
+function sanitize(id) {
+    return id.replace(/[^A-Za-z0-9_.-]/g, "_");
+}
+async function readTranscript(dir, sessionId) {
+    const out = [];
+    try {
+        const log = await openEventLog(sessionId, { rootDir: dir });
+        for await (const e of log.read())
+            out.push(e);
+        await log.close();
+    }
+    catch (err) {
+        // Tolerate missing log (test stubs may skip persistence).
+        if (!(err instanceof Error && /invalid sessionId/.test(err.message))) {
+            throw new RunnerError(`failed to read transcript for ${sessionId}`, err);
+        }
+    }
+    return out;
+}
+function extractToolCalls(events) {
+    const calls = [];
+    const startByUseId = new Map();
+    for (const ev of events) {
+        if (ev.kind === "tool_call_start") {
+            startByUseId.set(ev.toolUseId, { toolName: ev.toolName, ts: Date.parse(ev.timestamp) });
+        }
+        else if (ev.kind === "tool_call_end") {
+            const start = startByUseId.get(ev.toolUseId);
+            calls.push({
+                toolName: ev.toolName,
+                toolUseId: ev.toolUseId,
+                isError: ev.isError,
+                ts: start?.ts ?? Date.parse(ev.timestamp),
+            });
+        }
+    }
+    calls.sort((a, b) => a.ts - b.ts);
+    return calls.map(({ ts: _ts, ...rest }) => rest);
+}
+function sumTokens(events) {
+    let input = 0;
+    let output = 0;
+    for (const ev of events) {
+        if (ev.kind === "model_response") {
+            const e = ev;
+            input += e.usage.input;
+            output += e.usage.output;
+        }
+    }
+    return { input, output };
+}

package/dist/semaphore.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Trivial async semaphore — no `p-limit` dep. Acquire returns a release fn;
+ * callers MUST call it (use try/finally). When the semaphore would otherwise
+ * be over-capacity, acquirers queue FIFO.
+ */
+export declare class Semaphore {
+    private readonly capacity;
+    private inflight;
+    private readonly waitQueue;
+    constructor(capacity: number);
+    acquire(): Promise<() => void>;
+    private release;
+    get pending(): number;
+    get active(): number;
+}

package/dist/semaphore.js

@@ -0,0 +1,39 @@
+/**
+ * Trivial async semaphore — no `p-limit` dep. Acquire returns a release fn;
+ * callers MUST call it (use try/finally). When the semaphore would otherwise
+ * be over-capacity, acquirers queue FIFO.
+ */
+export class Semaphore {
+    capacity;
+    inflight = 0;
+    waitQueue = [];
+    constructor(capacity) {
+        this.capacity = capacity;
+        if (capacity < 1)
+            throw new Error(`Semaphore capacity must be ≥ 1 (got ${capacity})`);
+    }
+    async acquire() {
+        if (this.inflight < this.capacity) {
+            this.inflight += 1;
+            return () => this.release();
+        }
+        return new Promise((resolve) => {
+            this.waitQueue.push(() => {
+                this.inflight += 1;
+                resolve(() => this.release());
+            });
+        });
+    }
+    release() {
+        this.inflight -= 1;
+        const next = this.waitQueue.shift();
+        if (next)
+            next();
+    }
+    get pending() {
+        return this.waitQueue.length;
+    }
+    get active() {
+        return this.inflight;
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,97 @@
+import type { Sample } from "@crewhaus/eval-dataset";
+import type { GradeResult, Grader } from "@crewhaus/eval-grader";
+import type { Event as TranscriptEvent } from "@crewhaus/event-log";
+import type { RunContext } from "@crewhaus/run-context";
+import type { TraceEvent } from "@crewhaus/trace-event-bus";
+/**
+ * The agent-invocation contract. The runner provides a per-sample
+ * `RunContext` (fresh `TraceEventBus`, fresh `sessionId`) and the
+ * destination directory for the event log. The invoker is responsible
+ * for actually running the chat loop and returning the assistant text.
+ *
+ * In production this is a thin wrapper around `runChatLoop`. In tests it's
+ * a deterministic stub that returns a canned answer per sample.
+ */
+export type AgentInvoker = (req: AgentInvokeRequest) => Promise<AgentInvokeResult>;
+export type AgentInvokeRequest = {
+    readonly sample: Sample;
+    readonly runContext: RunContext;
+    /** Where the runtime should write the per-sample event-log JSONL. */
+    readonly sessionRootDir: string;
+    readonly seed?: number;
+};
+export type AgentInvokeResult = {
+    /** Final assistant text returned by the chat loop. */
+    readonly agentOutput: string;
+    /**
+     * If the invoker has out-of-band knowledge (e.g. a stub), it can supply
+     * the captured event-log entries here. The default invoker reads them
+     * from disk after `runChatLoop` returns.
+     */
+    readonly transcript?: ReadonlyArray<TranscriptEvent>;
+    /** Same as transcript — invoker can short-circuit the bus subscription. */
+    readonly events?: ReadonlyArray<TraceEvent>;
+};
+export type SampleResult = {
+    readonly sampleId: string;
+    readonly sessionId: string;
+    readonly startedAt: string;
+    readonly endedAt: string;
+    readonly latencyMs: number;
+    readonly turns: number;
+    readonly tokens: {
+        input: number;
+        output: number;
+    };
+    readonly model: string;
+    readonly agentOutput: string;
+    readonly grades: {
+        overall: GradeResult;
+        perGrader: Array<{
+            name: string;
+        } & GradeResult>;
+    };
+    readonly error?: string;
+};
+export type EvalRunSummary = {
+    readonly runId: string;
+    readonly startedAt: string;
+    readonly endedAt: string;
+    readonly samples: ReadonlyArray<SampleResult>;
+    readonly aggregates: {
+        readonly passRate: number;
+        readonly meanScore: number;
+        readonly p50Turns: number;
+        readonly p95Turns: number;
+        readonly p50LatencyMs: number;
+        readonly p95LatencyMs: number;
+        readonly totalTokens: {
+            input: number;
+            output: number;
+        };
+        readonly errorCount: number;
+    };
+    readonly config: {
+        readonly specHash: string;
+        readonly datasetName: string;
+        readonly graderNames: ReadonlyArray<string>;
+        readonly model: string;
+        readonly judgeModel?: string;
+        readonly concurrency: number;
+        readonly seed?: number;
+    };
+    readonly outDir: string;
+};
+export type RunEvalOptions = {
+    readonly runId?: string;
+    readonly concurrency?: number;
+    readonly seed?: number;
+    readonly outDir?: string;
+    readonly judgeModel?: string;
+    readonly invoker?: AgentInvoker;
+    readonly cwd?: string;
+};
+export type GraderEntry = {
+    readonly name: string;
+    readonly grader: Grader;
+};

package/dist/types.js

@@ -0,0 +1 @@
+export {};