@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/evals/run.js

@@ -0,0 +1,452 @@
+/**
+ * `runSuite`: every case, `repeat` times, graded from the trace. Live mode
+ * runs the profile now through `runTurn` with the host's provider and grades
+ * what the memory sink caught; recorded mode grades records a host already
+ * has. Both build the same trials, so the same graders give the same results.
+ *
+ * Every trial's results become a `theorem.eval.trial` span in the judged
+ * trace; the run becomes one `theorem.eval.run` record. Both go to the sink
+ * the host names, or nowhere.
+ *
+ * @module
+ */
+import { errorKind, TheoremError } from '../guardrails/error.js';
+import { getProfile, runTurn } from '../kernel/default-scope.js';
+import { isModelProfile } from '../kernel/registry/resolve.js';
+import { isRecord } from '../kernel/util/record.js';
+import { resolveObservabilityPolicy } from '../observability/resolve-policy.js';
+import { memorySink, writeTrace } from '../observability/trace.js';
+import { attachmentData } from './attachments.js';
+import { modelCalls, spanDurationMs } from './graders/shared.js';
+import { buildRunRecord, startTrialRecord } from './record.js';
+import { buildTrial, groupByTrace, hasTurn } from './trial.js';
+import { caseVerdict, trialOutcome } from './verdict.js';
+/** A judge turn's stamp carries `judge`; the turn it judged does not. */
+function isJudgeStamp(record) {
+    const stamp = record.metadata?.eval;
+    return isRecord(stamp) && stamp.judge !== undefined;
+}
+function stampOf(record) {
+    const stamp = record?.metadata?.eval;
+    if (!isRecord(stamp) || stamp.judge !== undefined)
+        return undefined;
+    const { suite, case: caseId, trial } = stamp;
+    if (typeof suite !== 'string' || typeof caseId !== 'string' || typeof trial !== 'number') {
+        return undefined;
+    }
+    return { suite, case: caseId, trial };
+}
+/**
+ * Grade one trial with every grader that applies, all at once; a grader that
+ * throws yields `grader_error`. Judge records reach `context.traced` in grader
+ * order once every grader is done, so a run writes the same records in the
+ * same order whichever judge answered first.
+ */
+async function gradeTrial(graders, trial, context) {
+    const applicable = trial.case ? graders : graders.filter((grader) => !grader.needsExpect);
+    const graded = await Promise.all(applicable.map(async (grader) => {
+        const records = [];
+        const own = {
+            ...context,
+            traced: (traced) => {
+                records.push(...traced);
+            },
+        };
+        try {
+            return { records, result: await grader.grade(trial, own), graderIdentity: grader.identity };
+        }
+        catch (thrown) {
+            return {
+                records,
+                result: {
+                    name: grader.name,
+                    source: grader.source,
+                    errorType: 'grader_error',
+                    explanation: thrown instanceof Error ? thrown.message : String(thrown),
+                },
+                graderIdentity: grader.identity,
+            };
+        }
+    }));
+    for (const { records } of graded)
+        await context.traced(records);
+    return graded.map(({ result, graderIdentity }) => ({ result, graderIdentity }));
+}
+/** Every applicable grader's result as the error that kept the turn from producing a trace. */
+function erroredResults(graders, evalCase, error) {
+    const applicable = evalCase ? graders : graders.filter((grader) => !grader.needsExpect);
+    return applicable.map((grader) => ({
+        name: grader.name,
+        source: grader.source,
+        errorType: error,
+    }));
+}
+/** The record's root: the span no other span in it parents (a judge call's root has the trial span above it). */
+function recordRoot(record) {
+    const ids = new Set(record.spans.map((span) => span.spanId));
+    return record.spans.find((span) => span.parentSpanId === undefined || !ids.has(span.parentSpanId));
+}
+/** A record's root cost, and whether it is whole: absent or partial, some calls went unpriced. */
+function recordCost(record) {
+    const attributes = recordRoot(record)?.attributes;
+    const cost = attributes?.['theorem.usage.cost_usd'];
+    if (typeof cost !== 'number')
+        return { whole: false };
+    return { usd: cost, whole: attributes?.['theorem.usage.cost_partial'] !== true };
+}
+/** The profile a record ran on (`gen_ai.agent.name` on its root). */
+function recordAgent(record) {
+    const agent = recordRoot(record)?.attributes['gen_ai.agent.name'];
+    return typeof agent === 'string' ? agent : undefined;
+}
+/** Build, grade and write one trial from the records of one trace. */
+async function gradeRecords(grading, evalCase, index, records) {
+    const trial = buildTrial({ suite: grading.suite.suite.id, case: evalCase, index, records });
+    const trialRecord = startTrialRecord({
+        trial,
+        policy: grading.policy,
+        ...(grading.clock ? { clock: grading.clock } : {}),
+    });
+    const usage = trial.usage();
+    const judgeRecords = [];
+    let judgeCostUsd;
+    let unpriced = usage.costUsd === undefined || usage.tokens.cost?.partial ? 1 : 0;
+    let priced = usage.costUsd === undefined ? 0 : 1;
+    const { judging } = grading;
+    const context = {
+        ...(judging.suiteJudge ? { judge: judging.suiteJudge } : {}),
+        ...(judging.provider ? { judgeProvider: judging.provider } : {}),
+        ...(judging.decision ? { judgeDecision: judging.decision } : {}),
+        ...(judging.media ? { media: judging.media } : {}),
+        traceparent: trialRecord.traceparent,
+        traced: async (traced) => {
+            for (const record of traced) {
+                judgeRecords.push(record);
+                const cost = recordCost(record);
+                if (cost.usd !== undefined) {
+                    judgeCostUsd = (judgeCostUsd ?? 0) + cost.usd;
+                    priced += 1;
+                }
+                if (!cost.whole)
+                    unpriced += 1;
+                const policy = judging.policies.get(recordAgent(record) ?? '');
+                if (grading.sink && policy)
+                    await writeTrace(grading.sink, Promise.resolve(record), policy);
+            }
+        },
+        ...(grading.signal ? { signal: grading.signal } : {}),
+    };
+    const graded = await gradeTrial(grading.suite.suite.graders, trial, context);
+    const results = graded.map((entry) => entry.result);
+    const built = await trialRecord.finish(graded);
+    grading.trialSpans.push(built.span);
+    if (grading.sink)
+        await writeTrace(grading.sink, Promise.resolve(built.record), grading.policy);
+    const { costUsd } = usage;
+    return {
+        ...(evalCase ? { case: evalCase } : {}),
+        index,
+        outcome: trialOutcome(results),
+        results,
+        traceId: trial.root.traceId,
+        turn: {
+            durationMs: spanDurationMs(trial.top),
+            modelCalls: modelCalls(trial).length,
+            toolCalls: trial.spans('execute_tool').length,
+            ...(typeof trial.root.attributes['theorem.stop.kind'] === 'string'
+                ? { stop: trial.root.attributes['theorem.stop.kind'] }
+                : {}),
+        },
+        records,
+        ...(costUsd === undefined ? {} : { costUsd }),
+        ...(judgeCostUsd === undefined ? {} : { judgeCostUsd }),
+        unpriced,
+        priced,
+        judgeRecords,
+        trialRecord: built.record,
+    };
+}
+async function turnRequest(suite, evalCase, index) {
+    if ('session' in evalCase.input) {
+        throw new TheoremError('config', `case ${evalCase.id} is a session script; suite ${suite.suite.id} runs turns`);
+    }
+    const stamp = { suite: suite.suite.id, case: evalCase.id, trial: index };
+    return {
+        profile: suite.suite.profile,
+        input: {
+            ...(evalCase.input.text === undefined ? {} : { text: evalCase.input.text }),
+            ...(evalCase.input.attachments
+                ? {
+                    attachments: await Promise.all(evalCase.input.attachments.map(async (attachment) => ({
+                        mimeType: attachment.mimeType,
+                        data: await attachmentData(attachment),
+                        ...(attachment.name === undefined ? {} : { name: attachment.name }),
+                    }))),
+                }
+                : {}),
+        },
+        metadata: { eval: stamp },
+    };
+}
+/** Run one turn live; its records are what the memory sink caught. */
+async function runLiveTrial(grading, provider, evalCase, index, signal) {
+    const records = [];
+    let thrown;
+    try {
+        const request = {
+            ...(await turnRequest(grading.suite, evalCase, index)),
+            ...(signal ? { signal } : {}),
+        };
+        for await (const _event of runTurn(request, provider, memorySink(records))) {
+            // The trace is the record of the turn; events are not graded.
+        }
+    }
+    catch (error) {
+        thrown = error;
+    }
+    if (records.length > 0) {
+        // The turn's own records go to the host's sink too: a label points at a transcript someone can read.
+        if (grading.sink) {
+            for (const record of records) {
+                await writeTrace(grading.sink, Promise.resolve(record), grading.policy);
+            }
+        }
+        return gradeRecords(grading, evalCase, index, records);
+    }
+    const error = thrown === undefined ? 'internal' : errorKind(thrown);
+    const results = erroredResults(grading.suite.suite.graders, evalCase, error);
+    return {
+        case: evalCase,
+        index,
+        outcome: trialOutcome(results),
+        results,
+        records,
+        unpriced: 0,
+        priced: 0,
+        judgeRecords: [],
+        error,
+    };
+}
+/**
+ * What the suite's model graders judge with. Each grader names its judge
+ * profile (its own, else the suite's); a text judge needs a provider (the
+ * option, else the suite's `judgeProvider` export, else the agent's), a
+ * decision judge a key (the option, else the suite's `judgeDecision` export).
+ */
+function judgingOf(loaded, options) {
+    const { suite } = loaded;
+    const provider = options.judgeProvider ?? loaded.judgeProvider ?? options.provider;
+    const decision = options.judgeDecision ?? loaded.judgeDecision;
+    const media = options.media ?? loaded.media;
+    const policies = new Map();
+    for (const grader of suite.graders) {
+        for (const id of grader.judgeProfiles?.(suite.judge?.profile) ?? []) {
+            const profile = getProfile(id);
+            if (profile.type === 'decision' && !decision) {
+                throw new TheoremError('config', `suite ${suite.id}: grader ${grader.name} needs a key for decision judge profile ${id}; export judgeDecision`);
+            }
+            if (profile.type !== 'decision' && !provider) {
+                throw new TheoremError('config', `suite ${suite.id}: grader ${grader.name} needs a provider for judge profile ${id}`);
+            }
+            policies.set(id, resolveObservabilityPolicy(profile.observability));
+        }
+    }
+    return {
+        ...(suite.judge ? { suiteJudge: suite.judge.profile } : {}),
+        ...(provider ? { provider } : {}),
+        ...(decision ? { decision } : {}),
+        ...(media ? { media } : {}),
+        policies,
+    };
+}
+function checkSuite(loaded, options) {
+    const { suite } = loaded;
+    const profile = getProfile(suite.profile);
+    if (!isModelProfile(profile)) {
+        throw new TheoremError('config', `suite ${suite.id}: profile ${suite.profile} runs no model turn`); // lexicon-exempt: developer contract error
+    }
+    if (suite.mode === 'session') {
+        throw new TheoremError('config', `suite ${suite.id}: session suites are not run yet`); // lexicon-exempt: developer contract error
+    }
+    if (profile.type === 'live') {
+        throw new TheoremError('config', `suite ${suite.id}: a live profile needs mode 'session'`); // lexicon-exempt: developer contract error
+    }
+    if (!options.recorded && !options.provider) {
+        throw new TheoremError('config', `suite ${suite.id}: live mode needs a provider`); // lexicon-exempt: developer contract error
+    }
+    if (suite.graders.length === 0) {
+        throw new TheoremError('config', `suite ${suite.id}: no graders`); // lexicon-exempt: developer contract error
+    }
+    const warnings = [];
+    const repeat = options.repeat ?? suite.trials.repeat;
+    if (repeat === 1) {
+        warnings.push('one trial per case cannot tell noise from change; set trials.repeat to 3 or more');
+    }
+    return warnings;
+}
+/** A trial's whole cost: the agent's turn and its judge turns. */
+function reportCost(report) {
+    return (report.costUsd ?? 0) + (report.judgeCostUsd ?? 0);
+}
+function summedCost(reports) {
+    return reports.reduce((sum, report) => sum + reportCost(report), 0);
+}
+function summedCalls(reports, key) {
+    return reports.reduce((sum, report) => sum + report[key], 0);
+}
+/** Recorded mode: each trace's records become one trial, matched to a case by its eval stamp. */
+async function gradeRecorded(grading, records) {
+    const byCase = new Map();
+    const caseless = [];
+    const cases = new Map(grading.suite.cases.map((evalCase) => [evalCase.id, evalCase]));
+    const skipped = { judge: 0, other: 0 };
+    for (const [, group] of groupByTrace(records)) {
+        // A directory of eval traces holds judge calls (under their trial) and run records beside the turns; only turns are trials.
+        const traceRecords = group.filter((record) => !isJudgeStamp(record));
+        skipped.judge += group.length - traceRecords.length;
+        if (traceRecords.length === 0)
+            continue;
+        if (!hasTurn(traceRecords)) {
+            skipped.other += 1;
+            continue;
+        }
+        const stamp = traceRecords
+            .map(stampOf)
+            .find((found) => found?.suite === grading.suite.suite.id);
+        const evalCase = stamp ? cases.get(stamp.case) : undefined;
+        if (!evalCase) {
+            caseless.push(await gradeRecords(grading, undefined, caseless.length, traceRecords));
+            continue;
+        }
+        const index = stamp?.trial ?? 0;
+        const report = await gradeRecords(grading, evalCase, index, traceRecords);
+        byCase.set(evalCase.id, [...(byCase.get(evalCase.id) ?? []), report]);
+    }
+    const reports = [...byCase.values(), caseless].flat();
+    return {
+        byCase,
+        caseless,
+        costUsd: summedCost(reports),
+        unpriced: summedCalls(reports, 'unpriced'),
+        priced: summedCalls(reports, 'priced'),
+        skipped,
+    };
+}
+/**
+ * Live mode: every case, `repeat` times, started in suite order with up to
+ * `concurrency` in flight, until the cost ceiling. The ceiling is checked
+ * before each start, so a stop lets what is in flight finish and count.
+ */
+async function runLive(grading, provider, repeat, options) {
+    const jobs = grading.suite.cases.flatMap((evalCase) => Array.from({ length: repeat }, (_, index) => ({ evalCase, index })));
+    const reports = new Map(grading.suite.cases.map((evalCase) => [evalCase.id, []]));
+    let next = 0;
+    let costUsd = 0;
+    let stopped;
+    const worker = async () => {
+        while (next < jobs.length) {
+            if (options.maxCostUsd !== undefined && costUsd > options.maxCostUsd) {
+                stopped = 'budget';
+                return;
+            }
+            const job = jobs[next++];
+            if (!job)
+                return;
+            const report = await runLiveTrial(grading, provider, job.evalCase, job.index, options.signal);
+            costUsd += reportCost(report);
+            reports.get(job.evalCase.id)?.push(report);
+            options.onTrial?.(report);
+        }
+    };
+    const workers = Math.max(1, Math.min(options.concurrency ?? 1, jobs.length));
+    await Promise.all(Array.from({ length: workers }, worker));
+    const byCase = new Map();
+    for (const [id, list] of reports) {
+        byCase.set(id, [...list].sort((a, b) => a.index - b.index));
+    }
+    const all = [...byCase.values()].flat();
+    return {
+        byCase,
+        caseless: [],
+        ...(stopped ? { stopped } : {}),
+        costUsd,
+        unpriced: summedCalls(all, 'unpriced'),
+        priced: summedCalls(all, 'priced'),
+    };
+}
+async function ranOf(grading, repeat, options) {
+    if (options.recorded !== undefined) {
+        const ran = await gradeRecorded(grading, options.recorded);
+        for (const report of [...ran.byCase.values(), ran.caseless].flat())
+            options.onTrial?.(report);
+        return ran;
+    }
+    if (!options.provider) {
+        throw new TheoremError('config', `suite ${grading.suite.suite.id}: live mode needs a provider`); // lexicon-exempt: developer contract error
+    }
+    return runLive(grading, options.provider, repeat, options);
+}
+/** Run a loaded suite live or over records, write its trial and run records, and report every verdict. */
+async function runSuite(loaded, options = {}) {
+    const warnings = checkSuite(loaded, options);
+    const { suite } = loaded;
+    const repeat = options.repeat ?? suite.trials.repeat;
+    const profile = getProfile(suite.profile);
+    const policy = resolveObservabilityPolicy(isModelProfile(profile) ? profile.observability : undefined);
+    const grading = {
+        suite: loaded,
+        policy,
+        judging: judgingOf(loaded, options),
+        ...(options.sink ? { sink: options.sink } : {}),
+        ...(options.clock ? { clock: options.clock } : {}),
+        trialSpans: [],
+        ...(options.signal ? { signal: options.signal } : {}),
+    };
+    const ran = await ranOf(grading, repeat, options);
+    if (ran.skipped && ran.skipped.judge + ran.skipped.other > 0) {
+        warnings.push(`skipped ${ran.skipped.judge} judge call record(s) and ${ran.skipped.other} record(s) with no turn; they are not trials`);
+    }
+    if (options.maxCostUsd !== undefined && ran.unpriced > 0) {
+        warnings.push(`maxCostUsd counts only the costs providers report; ${ran.unpriced} call(s) reported none or only part, so the ceiling could not hold them`);
+    }
+    const rule = suite.trials.pass ?? 'all';
+    const verdicts = loaded.cases.map((evalCase) => caseVerdict(evalCase, (ran.byCase.get(evalCase.id) ?? []).map((report) => report.results), rule));
+    const decided = verdicts.filter((verdict) => verdict.decided);
+    const undecided = verdicts.length - decided.length;
+    if (undecided > 0) {
+        warnings.push(`${undecided} case(s) undecided: too few of their trials escaped error to apply the pass rule, so no pass rate counts them`);
+    }
+    const run = await buildRunRecord({
+        suite: {
+            id: suite.id,
+            trials: { repeat, ...(suite.trials.pass ? { pass: suite.trials.pass } : {}) },
+        },
+        verdicts,
+        trialSpans: grading.trialSpans,
+        policy,
+        ...(options.clock ? { clock: options.clock } : {}),
+        ...(ran.caseless.length > 0 ? { caseless: true } : {}),
+        ...(ran.stopped ? { stopped: ran.stopped } : {}),
+        ...(options.revision ? { revision: options.revision } : {}),
+    });
+    if (options.sink)
+        await writeTrace(options.sink, Promise.resolve(run), policy);
+    const trials = [...ran.byCase.values()].flat();
+    return {
+        suite: suite.id,
+        mode: options.recorded !== undefined ? 'recorded' : 'live',
+        repeat,
+        passRule: rule,
+        verdicts,
+        trials,
+        caseless: ran.caseless,
+        passed: ran.stopped === undefined && decided.length > 0 && decided.every((verdict) => verdict.passed),
+        ...(ran.stopped ? { stopped: ran.stopped } : {}),
+        costUsd: ran.costUsd,
+        unpriced: ran.unpriced,
+        priced: ran.priced,
+        warnings,
+        run,
+    };
+}
+export { runSuite };

package/esm/src/evals/suite.d.ts

@@ -0,0 +1,19 @@
+import type { z } from 'zod';
+import type { RunDecisionOptions } from '../kernel/engine/decision.js';
+import type { ModelProvider } from '../kernel/types.js';
+import { type TraceRecord } from '../observability/trace-schema.js';
+import { type EvalCase, type EvalMediaResolver, type EvalSuite } from './types.js';
+interface LoadedSuite {
+    suite: EvalSuite;
+    cases: EvalCase[];
+    provider?: ModelProvider;
+    judgeProvider?: ModelProvider;
+    judgeDecision?: Omit<RunDecisionOptions, 'sink'>;
+    media?: EvalMediaResolver;
+    path: string;
+}
+declare function readJsonl<T>(path: string, schema: z.ZodType<T>, what: string): Promise<T[]>;
+declare function readTraceRecords(path: string): Promise<TraceRecord[]>;
+declare function loadSuite(modulePath: string): Promise<LoadedSuite>;
+export type { LoadedSuite };
+export { loadSuite, readJsonl, readTraceRecords };

package/esm/src/evals/suite.js

@@ -0,0 +1,92 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { cwd } from 'node:process';
+import { TheoremError } from '../guardrails/error.js';
+import { isRecord } from '../kernel/util/record.js';
+import { traceRecordSchema } from '../observability/trace-schema.js';
+import { pinCaseFiles } from './attachments.js';
+import { evalCaseSchema, evalSuiteSchema, } from './types.js';
+function isProvider(value) {
+    return isRecord(value) && typeof value.complete === 'function';
+}
+function isDecisionKey(value) {
+    return isRecord(value) && isRecord(value.vault);
+}
+function absolute(path) {
+    return path.startsWith('/') ? path : `${cwd()}/${path}`;
+}
+function siblingOf(modulePath, relative) {
+    const dir = modulePath.slice(0, modulePath.lastIndexOf('/'));
+    return relative.startsWith('/') ? relative : `${dir}/${relative}`;
+}
+async function readJsonl(path, schema, what) {
+    const text = await readFile(path, 'utf8');
+    const out = [];
+    const lines = text.split('\n');
+    for (const [index, line] of lines.entries()) {
+        if (line.trim() === '')
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            throw new TheoremError('config', `${path}:${index + 1}: ${what} is not JSON`); // lexicon-exempt: developer contract error
+        }
+        const result = schema.safeParse(parsed);
+        if (!result.success) {
+            const issue = result.error.issues[0];
+            const at = issue?.path.join('.') || '(root)';
+            throw new TheoremError('config', `${path}:${index + 1}: ${what} ${at}: ${issue?.message ?? 'invalid'}`);
+        }
+        out.push(result.data);
+    }
+    return out;
+}
+async function readTraceRecords(path) {
+    const target = absolute(path);
+    const info = await stat(target);
+    if (!info.isDirectory())
+        return readJsonl(target, traceRecordSchema, 'trace record');
+    const files = [];
+    for (const entry of await readdir(target, { withFileTypes: true })) {
+        if (entry.isFile() && entry.name.endsWith('.jsonl'))
+            files.push(`${target}/${entry.name}`);
+    }
+    files.sort();
+    const records = [];
+    for (const file of files)
+        records.push(...(await readJsonl(file, traceRecordSchema, 'trace record')));
+    return records;
+}
+async function loadSuite(modulePath) {
+    const path = absolute(modulePath);
+    const module = await import(new URL(`file://${path}`).href);
+    if (!isRecord(module)) {
+        throw new TheoremError('config', `${modulePath}: not a module`); // lexicon-exempt: developer contract error
+    }
+    const parsed = evalSuiteSchema.safeParse(module.default);
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        throw new TheoremError('config', `${modulePath}: default export is not an EvalSuite (${issue?.path.join('.') || 'root'}: ${issue?.message ?? 'invalid'})`);
+    }
+    const suite = parsed.data;
+    const casesPath = siblingOf(path, suite.cases);
+    const cases = await readJsonl(casesPath, evalCaseSchema, 'case');
+    const ids = new Set();
+    for (const evalCase of cases) {
+        if (ids.has(evalCase.id)) {
+            throw new TheoremError('config', `${suite.cases}: case id ${evalCase.id} appears twice`); // lexicon-exempt: developer contract error
+        }
+        ids.add(evalCase.id);
+    }
+    return {
+        suite,
+        cases: await pinCaseFiles(cases, casesPath),
+        path,
+        ...(isProvider(module.provider) ? { provider: module.provider } : {}),
+        ...(isProvider(module.judgeProvider) ? { judgeProvider: module.judgeProvider } : {}),
+        ...(isDecisionKey(module.judgeDecision) ? { judgeDecision: module.judgeDecision } : {}),
+        ...(typeof module.media === 'function' ? { media: module.media } : {}),
+    };
+}
+export { loadSuite, readJsonl, readTraceRecords };

package/esm/src/evals/summary.d.ts

@@ -0,0 +1,32 @@
+import { type GroupSummary } from './breakdown.js';
+import type { SuiteRun, TurnShape } from './run.js';
+import type { EvalPassRule, EvalResult } from './types.js';
+import type { CaseVerdict, TrialOutcome } from './verdict.js';
+interface TrialSummary {
+    case?: string;
+    index: number;
+    outcome: TrialOutcome;
+    traceId?: string;
+    turn?: TurnShape;
+    error?: string;
+    results: EvalResult[];
+}
+interface RunSummary {
+    suite: string;
+    mode: 'live' | 'recorded';
+    repeat: number;
+    passRule: EvalPassRule;
+    passed: boolean;
+    stopped?: 'budget';
+    costUsd: number;
+    unpriced: number;
+    priced: number;
+    verdicts: CaseVerdict[];
+    groups: GroupSummary[];
+    trials: TrialSummary[];
+    runTraceId?: string;
+    warnings: string[];
+}
+declare function summarizeRun(run: SuiteRun): RunSummary;
+export type { RunSummary, TrialSummary };
+export { summarizeRun };

package/esm/src/evals/summary.js

@@ -0,0 +1,30 @@
+import { groupSummaries } from './breakdown.js';
+function summarizeRun(run) {
+    const trials = [...run.trials, ...run.caseless].map((report) => ({
+        ...(report.case ? { case: report.case.id } : {}),
+        index: report.index,
+        outcome: report.outcome,
+        ...(report.traceId ? { traceId: report.traceId } : {}),
+        ...(report.turn ? { turn: report.turn } : {}),
+        ...(report.error ? { error: report.error } : {}),
+        results: report.results,
+    }));
+    const runTraceId = run.run.spans[0]?.traceId;
+    return {
+        suite: run.suite,
+        mode: run.mode,
+        repeat: run.repeat,
+        passRule: run.passRule,
+        passed: run.passed,
+        ...(run.stopped ? { stopped: run.stopped } : {}),
+        costUsd: run.costUsd,
+        unpriced: run.unpriced,
+        priced: run.priced,
+        verdicts: run.verdicts,
+        groups: groupSummaries(run),
+        trials,
+        ...(runTraceId ? { runTraceId } : {}),
+        warnings: run.warnings,
+    };
+}
+export { summarizeRun };

package/esm/src/evals/trial.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Records → `Trial`. A live run hands the runner what its memory sink caught;
+ * a recorded run hands it what a file held. Both build the same trial, so the
+ * same graders give the same results.
+ *
+ * The root is the `invoke_agent` span with no agent above it in the set; a host
+ * span above it (the host's request, when the host nests the turn under its own
+ * span) does not hide it, and the topmost span above it is the trial's `top`. It
+ * is never assumed to be the first record: a compaction turn's record can be
+ * written before its parent's (P9).
+ *
+ * @module
+ */
+import { type TraceRecord } from '../observability/trace-record.js';
+import type { EvalCase, Trial } from './types.js';
+/** Build the trial every grader receives from the records that share one trace. */
+declare function buildTrial(args: {
+    suite: string;
+    case?: EvalCase;
+    index: number;
+    records: TraceRecord[];
+}): Trial;
+/** Group records by trace id, in first-seen order. */
+declare function groupByTrace(records: TraceRecord[]): Map<string, TraceRecord[]>;
+/** Whether these records hold a turn at all: an `invoke_agent` span. An eval run record does not. */
+declare function hasTurn(records: readonly TraceRecord[]): boolean;
+export { buildTrial, groupByTrace, hasTurn };