@theoremjs/agents 0.3.0 → 0.3.1

package/esm/src/evals/record.d.ts

@@ -3,15 +3,18 @@ import { type TraceClock, type TraceSpan } from '../observability/trace-span.js'
 import type { ResolvedObservabilityPolicy } from '../observability/types.js';
 import type { EvalResult, EvalSuite, Trial } from './types.js';
 import { type CaseVerdict } from './verdict.js';
+/** A grader's result with the identity of the grader that gave it. */
 interface GradedResult {
     result: EvalResult;
     graderIdentity: string;
 }
+/** What a trial's record is built from. */
 interface TrialRecordInput {
     trial: Trial;
     policy: ResolvedObservabilityPolicy;
     clock?: TraceClock;
 }
+/** A trial record being built: its `traceparent` and a `finish` that writes it with the results. */
 interface OpenTrialRecord {
     traceparent: string;
     finish: (results: readonly GradedResult[]) => Promise<{
@@ -19,7 +22,9 @@ interface OpenTrialRecord {
         span: TraceSpan;
     }>;
 }
+/** Opens the record of a trial. */
 declare function startTrialRecord(input: TrialRecordInput): OpenTrialRecord;
+/** What a run's record is built from: the suite, the verdicts, and the trial spans. */
 interface RunRecordInput {
     suite: Pick<EvalSuite, 'id' | 'trials'>;
     verdicts: readonly CaseVerdict[];
@@ -30,6 +35,7 @@ interface RunRecordInput {
     stopped?: 'budget';
     revision?: string;
 }
+/** Builds the record of a whole run. */
 declare function buildRunRecord(input: RunRecordInput): Promise<TraceRecord>;
 export type { GradedResult, OpenTrialRecord, RunRecordInput, TrialRecordInput };
 export { buildRunRecord, startTrialRecord };

package/esm/src/evals/record.js

@@ -34,6 +34,7 @@ async function resultAttributes(result, graderIdentity, responseId) {
         ...optional('theorem.evaluation.passed', result.passed),
     };
 }
+/** Opens the record of a trial. */
 function startTrialRecord(input) {
     const { trial, policy } = input;
     const tree = startTrace(TRIAL_SPAN, {
@@ -60,7 +61,7 @@ function startTrialRecord(input) {
         });
         const [span] = record.spans;
         if (!span)
-            throw new TheoremError('internal', 'trial record has no span'); // lexicon-exempt: invariant
+            throw new TheoremError('internal', 'trial record has no span'); // labels-exempt: invariant
         return { record, span };
     };
     return { traceparent: tree.root.traceparent(), finish };
@@ -77,6 +78,7 @@ function verdictAttributes(verdict) {
         trials_ungraded: verdict.trialsUngraded,
     };
 }
+/** Builds the record of a whole run. */
 function buildRunRecord(input) {
     const rule = input.suite.trials.pass ?? 'all';
     const tree = startTrace(RUN_SPAN, {
@@ -98,7 +100,7 @@ function buildRunRecord(input) {
     for (const verdict of input.verdicts) {
         tree.root.event(VERDICT_EVENT, verdictAttributes(verdict));
     }
-    // A run that reached its end is OK whatever the verdicts say; one stopped early is not finished.
+    // why: A run that reached its end is OK whatever the verdicts say; one stopped early is not finished.
     tree.root.end(input.stopped ? { code: 'UNSET' } : { code: 'OK' });
     return buildRecord({ spans: tree.collect(), policy: input.policy });
 }

package/esm/src/evals/rubrics/mod.d.ts

@@ -4,7 +4,7 @@
  * Apache License 2.0), keyed in camelCase (`tool_selection` is
  * `rubrics.toolSelection`), and `rubric()` for a host's own.
  *
- * Each keeps Phoenix's name, labels and prompt. The labels that pass follow
+ * Each keeps Phoenix's name, labels, and prompt. The labels that pass follow
  * Phoenix's direction: the top-scoring labels when higher is better, the
  * bottom-scoring when lower is (`toxicity` passes `non-toxic`), none when
  * Phoenix names no direction (`refusal` informs, never decides). A decision

package/esm/src/evals/rubrics/mod.js

@@ -4,7 +4,7 @@
  * Apache License 2.0), keyed in camelCase (`tool_selection` is
  * `rubrics.toolSelection`), and `rubric()` for a host's own.
  *
- * Each keeps Phoenix's name, labels and prompt. The labels that pass follow
+ * Each keeps Phoenix's name, labels, and prompt. The labels that pass follow
  * Phoenix's direction: the top-scoring labels when higher is better, the
  * bottom-scoring when lower is (`toxicity` passes `non-toxic`), none when
  * Phoenix names no direction (`refusal` informs, never decides). A decision

package/esm/src/evals/rubrics/mustache.js

@@ -13,7 +13,7 @@
 import { TheoremError } from '../../guardrails/error.js';
 const TAG = /\{\{(\{?)([^{}]*)\}?\}\}/g;
 function templateError(message) {
-    return new TheoremError('config', `rubric template: ${message}`); // lexicon-exempt: developer contract error
+    return new TheoremError('config', `rubric template: ${message}`); // labels-exempt: developer contract error
 }
 /** A tag whose line holds nothing else: the whole line goes, newline included. */
 function standalone(template, start, end) {

package/esm/src/evals/rubrics/types.js

@@ -10,7 +10,7 @@
 import { TheoremError } from '../../guardrails/error.js';
 import { renderTemplate, templateVariables } from './mustache.js';
 function configError(rubric, message) {
-    return new TheoremError('config', `rubric ${rubric}: ${message}`); // lexicon-exempt: developer contract error
+    return new TheoremError('config', `rubric ${rubric}: ${message}`); // labels-exempt: developer contract error
 }
 /** Render the prompt over the view; a rubric without one, or a variable the view lacks, is a config error. */
 function fillRubric(rubric, view) {

package/esm/src/evals/run.d.ts

@@ -1,118 +1,75 @@
-/**
- * `runSuite`: every case, `repeat` times, graded from the trace. Live mode
- * runs the profile now through `runTurn` with the host's provider and grades
- * what the memory sink caught; recorded mode grades records a host already
- * has. Both build the same trials, so the same graders give the same results.
- *
- * Every trial's results become a `theorem.eval.trial` span in the judged
- * trace; the run becomes one `theorem.eval.run` record. Both go to the sink
- * the host names, or nowhere.
- *
- * @module
- */
 import type { RunDecisionOptions } from '../kernel/engine/decision.js';
-import type { ModelProvider } from '../kernel/types.js';
+import type { ProviderHostOptions } from '../kernel/protocols/runtime.js';
 import type { TraceRecord } from '../observability/trace-record.js';
 import type { TraceSink } from '../observability/trace-sink.js';
 import type { TraceClock } from '../observability/trace-span.js';
 import type { LoadedSuite } from './suite.js';
 import type { EvalCase, EvalMediaResolver, EvalPassRule, EvalResult } from './types.js';
 import { type CaseVerdict, type TrialOutcome } from './verdict.js';
-/** How the eval stamps a live turn so recorded mode can match its record to the case. */
+/** Names a trial by suite, case, and trial number. */
 interface EvalStamp {
     suite: string;
     case: string;
     trial: number;
 }
-/** A turn's whole length, root span start to end, its loop (model calls and tool calls), and why it stopped. */
+/** The size of a turn: its duration and the model and tool calls it made. */
 interface TurnShape {
     durationMs: number;
     modelCalls: number;
     toolCalls: number;
-    /** `theorem.stop.kind` on the root; absent when the turn recorded none. */
     stop?: string;
 }
-/** One trial as the run reports it, before and after it is written. */
+/** One trial's outcome, results, and records. */
 interface TrialReport {
     case?: EvalCase;
     index: number;
     outcome: TrialOutcome;
     results: EvalResult[];
-    /** The judged trace, when the turn produced one. */
     traceId?: string;
     records: TraceRecord[];
-    /** How long the turn took and how many times it looped, when it produced a trace. */
     turn?: TurnShape;
-    /** The kind of the error that stopped the turn before it had a trace, if any. */
     error?: string;
-    /** The turn's cost from its root, when it recorded one. */
     costUsd?: number;
-    /** The judge calls' summed cost, when a model grader ran and any judge recorded one. */
     judgeCostUsd?: number;
-    /** The turn and judge calls whose cost is unknown: none reported, or only part (`theorem.usage.cost_partial`). */
     unpriced: number;
-    /** The turn and judge calls that reported a cost, whole or part; a reported zero counts. */
     priced: number;
-    /** Every judge call's records, one trace per judge call. */
     judgeRecords: TraceRecord[];
-    /** The written `theorem.eval.trial` record, when there was a trace to hold it. */
     trialRecord?: TraceRecord;
 }
+/** Options for `runSuite`: the providers, the judge, and how often to repeat. */
 interface RunSuiteOptions {
-    /** Live mode: the host's provider for the profile under test. */
-    provider?: ModelProvider;
-    /** The provider for text judge profiles; default the suite's `judgeProvider` export, else `provider`. */
-    judgeProvider?: ModelProvider;
-    /** The key for decision judge profiles (Jev); default the suite's `judgeDecision` export. */
+    provider?: ProviderHostOptions;
+    judgeProvider?: ProviderHostOptions;
     judgeDecision?: Omit<RunDecisionOptions, 'sink'>;
-    /** Where a judge finds media the trace names only by hash; default the suite's `media` export. */
     media?: EvalMediaResolver;
-    /** Recorded mode: the records to grade; live mode when absent. */
     recorded?: TraceRecord[];
-    /** Where trial and run records go. Absent: they are returned only. */
     sink?: TraceSink;
-    /**
-     * Stop starting trials once the summed cost (agent and judge) crosses this;
-     * the run record says `stopped: budget`. Calls with no reported cost add
-     * nothing, so the run warns when any ran.
-     */
     maxCostUsd?: number;
-    /** `repeat` override from the command line. */
     repeat?: number;
-    /** Live mode: trials in flight at once (default 1). Order of start and report is the suite's either way. */
     concurrency?: number;
-    /** `vcs.ref.head.revision` on the run record. */
     revision?: string;
     clock?: TraceClock;
-    /** Called after each trial is graded, for progress output. */
     onTrial?: (report: TrialReport) => void;
     signal?: AbortSignal;
 }
-/** What a run produced. */
+/** A finished suite run: its mode and its cases' trials and verdicts. */
 interface SuiteRun {
     suite: string;
     mode: 'live' | 'recorded';
     repeat: number;
-    /** What a case's trials must do for it to pass. */
     passRule: EvalPassRule;
     verdicts: CaseVerdict[];
     trials: TrialReport[];
-    /** Recorded mode: trials whose records carried no matching case. */
     caseless: TrialReport[];
-    /** Some case was decided, every decided case passed, and nothing stopped the run. */
     passed: boolean;
     stopped?: 'budget';
-    /** The summed cost of agent turns and judge calls, over every one that recorded a cost. */
     costUsd: number;
-    /** Agent turns and judge calls whose cost went unreported, in whole or part, so `costUsd` leaves it out. */
     unpriced: number;
-    /** Agent turns and judge calls that reported a cost, so `costUsd` is theirs even when it is zero. */
     priced: number;
-    /** Plain-language warnings for the host to print. */
     warnings: string[];
     run: TraceRecord;
 }
-/** Run a loaded suite live or over records, write its trial and run records, and report every verdict. */
+/** Runs every case of a suite and returns the graded run. */
 declare function runSuite(loaded: LoadedSuite, options?: RunSuiteOptions): Promise<SuiteRun>;
 export type { EvalStamp, RunSuiteOptions, SuiteRun, TrialReport, TurnShape };
 export { runSuite };

package/esm/src/evals/run.js

@@ -1,15 +1,3 @@
-/**
- * `runSuite`: every case, `repeat` times, graded from the trace. Live mode
- * runs the profile now through `runTurn` with the host's provider and grades
- * what the memory sink caught; recorded mode grades records a host already
- * has. Both build the same trials, so the same graders give the same results.
- *
- * Every trial's results become a `theorem.eval.trial` span in the judged
- * trace; the run becomes one `theorem.eval.run` record. Both go to the sink
- * the host names, or nowhere.
- *
- * @module
- */
 import { errorKind, TheoremError } from '../guardrails/error.js';
 import { getProfile, runTurn } from '../kernel/default-scope.js';
 import { isModelProfile } from '../kernel/registry/resolve.js';
@@ -36,12 +24,7 @@ function stampOf(record) {
     }
     return { suite, case: caseId, trial };
 }
-/**
- * Grade one trial with every grader that applies, all at once; a grader that
- * throws yields `grader_error`. Judge records reach `context.traced` in grader
- * order once every grader is done, so a run writes the same records in the
- * same order whichever judge answered first.
- */
+/** Judge records reach `context.traced` in grader order once every grader is done, so a run writes the same records in the same order whichever judge answered first. */
 async function gradeTrial(graders, trial, context) {
     const applicable = trial.case ? graders : graders.filter((grader) => !grader.needsExpect);
     const graded = await Promise.all(applicable.map(async (grader) => {
@@ -72,7 +55,6 @@ async function gradeTrial(graders, trial, context) {
         await context.traced(records);
     return graded.map(({ result, graderIdentity }) => ({ result, graderIdentity }));
 }
-/** Every applicable grader's result as the error that kept the turn from producing a trace. */
 function erroredResults(graders, evalCase, error) {
     const applicable = evalCase ? graders : graders.filter((grader) => !grader.needsExpect);
     return applicable.map((grader) => ({
@@ -86,7 +68,6 @@ function recordRoot(record) {
     const ids = new Set(record.spans.map((span) => span.spanId));
     return record.spans.find((span) => span.parentSpanId === undefined || !ids.has(span.parentSpanId));
 }
-/** A record's root cost, and whether it is whole: absent or partial, some calls went unpriced. */
 function recordCost(record) {
     const attributes = recordRoot(record)?.attributes;
     const cost = attributes?.['theorem.usage.cost_usd'];
@@ -94,12 +75,10 @@ function recordCost(record) {
         return { whole: false };
     return { usd: cost, whole: attributes?.['theorem.usage.cost_partial'] !== true };
 }
-/** The profile a record ran on (`gen_ai.agent.name` on its root). */
 function recordAgent(record) {
     const agent = recordRoot(record)?.attributes['gen_ai.agent.name'];
     return typeof agent === 'string' ? agent : undefined;
 }
-/** Build, grade and write one trial from the records of one trace. */
 async function gradeRecords(grading, evalCase, index, records) {
     const trial = buildTrial({ suite: grading.suite.suite.id, case: evalCase, index, records });
     const trialRecord = startTrialRecord({
@@ -186,9 +165,10 @@ async function turnRequest(suite, evalCase, index) {
                 : {}),
         },
         metadata: { eval: stamp },
+        ...(suite.onStage ? { onStage: suite.onStage } : {}),
+        ...(suite.host === undefined ? {} : { host: suite.host }),
     };
 }
-/** Run one turn live; its records are what the memory sink caught. */
 async function runLiveTrial(grading, provider, evalCase, index, signal) {
     const records = [];
     let thrown;
@@ -198,14 +178,13 @@ async function runLiveTrial(grading, provider, evalCase, index, signal) {
             ...(signal ? { signal } : {}),
         };
         for await (const _event of runTurn(request, provider, memorySink(records))) {
-            // The trace is the record of the turn; events are not graded.
+            // why: The trace is the record of the turn; events are not graded.
         }
     }
     catch (error) {
         thrown = error;
     }
     if (records.length > 0) {
-        // The turn's own records go to the host's sink too: a label points at a transcript someone can read.
         if (grading.sink) {
             for (const record of records) {
                 await writeTrace(grading.sink, Promise.resolve(record), grading.policy);
@@ -227,12 +206,6 @@ async function runLiveTrial(grading, provider, evalCase, index, signal) {
         error,
     };
 }
-/**
- * What the suite's model graders judge with. Each grader names its judge
- * profile (its own, else the suite's); a text judge needs a provider (the
- * option, else the suite's `judgeProvider` export, else the agent's), a
- * decision judge a key (the option, else the suite's `judgeDecision` export).
- */
 function judgingOf(loaded, options) {
     const { suite } = loaded;
     const provider = options.judgeProvider ?? loaded.judgeProvider ?? options.provider;
@@ -263,19 +236,19 @@ function checkSuite(loaded, options) {
     const { suite } = loaded;
     const profile = getProfile(suite.profile);
     if (!isModelProfile(profile)) {
-        throw new TheoremError('config', `suite ${suite.id}: profile ${suite.profile} runs no model turn`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${suite.id}: profile ${suite.profile} runs no model turn`); // labels-exempt: developer contract error
     }
     if (suite.mode === 'session') {
-        throw new TheoremError('config', `suite ${suite.id}: session suites are not run yet`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${suite.id}: session suites are not run yet`); // labels-exempt: developer contract error
     }
     if (profile.type === 'live') {
-        throw new TheoremError('config', `suite ${suite.id}: a live profile needs mode 'session'`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${suite.id}: a live profile needs mode 'session'`); // labels-exempt: developer contract error
     }
     if (!options.recorded && !options.provider) {
-        throw new TheoremError('config', `suite ${suite.id}: live mode needs a provider`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${suite.id}: live mode needs a provider`); // labels-exempt: developer contract error
     }
     if (suite.graders.length === 0) {
-        throw new TheoremError('config', `suite ${suite.id}: no graders`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${suite.id}: no graders`); // labels-exempt: developer contract error
     }
     const warnings = [];
     const repeat = options.repeat ?? suite.trials.repeat;
@@ -284,7 +257,6 @@ function checkSuite(loaded, options) {
     }
     return warnings;
 }
-/** A trial's whole cost: the agent's turn and its judge turns. */
 function reportCost(report) {
     return (report.costUsd ?? 0) + (report.judgeCostUsd ?? 0);
 }
@@ -294,14 +266,12 @@ function summedCost(reports) {
 function summedCalls(reports, key) {
     return reports.reduce((sum, report) => sum + report[key], 0);
 }
-/** Recorded mode: each trace's records become one trial, matched to a case by its eval stamp. */
 async function gradeRecorded(grading, records) {
     const byCase = new Map();
     const caseless = [];
     const cases = new Map(grading.suite.cases.map((evalCase) => [evalCase.id, evalCase]));
     const skipped = { judge: 0, other: 0 };
     for (const [, group] of groupByTrace(records)) {
-        // A directory of eval traces holds judge calls (under their trial) and run records beside the turns; only turns are trials.
         const traceRecords = group.filter((record) => !isJudgeStamp(record));
         skipped.judge += group.length - traceRecords.length;
         if (traceRecords.length === 0)
@@ -332,11 +302,7 @@ async function gradeRecorded(grading, records) {
         skipped,
     };
 }
-/**
- * Live mode: every case, `repeat` times, started in suite order with up to
- * `concurrency` in flight, until the cost ceiling. The ceiling is checked
- * before each start, so a stop lets what is in flight finish and count.
- */
+/** The cost ceiling is checked before each start, so a stop lets what is in flight finish and count. */
 async function runLive(grading, provider, repeat, options) {
     const jobs = grading.suite.cases.flatMap((evalCase) => Array.from({ length: repeat }, (_, index) => ({ evalCase, index })));
     const reports = new Map(grading.suite.cases.map((evalCase) => [evalCase.id, []]));
@@ -382,11 +348,11 @@ async function ranOf(grading, repeat, options) {
         return ran;
     }
     if (!options.provider) {
-        throw new TheoremError('config', `suite ${grading.suite.suite.id}: live mode needs a provider`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `suite ${grading.suite.suite.id}: live mode needs a provider`); // labels-exempt: developer contract error
     }
     return runLive(grading, options.provider, repeat, options);
 }
-/** Run a loaded suite live or over records, write its trial and run records, and report every verdict. */
+/** Runs every case of a suite and returns the graded run. */
 async function runSuite(loaded, options = {}) {
     const warnings = checkSuite(loaded, options);
     const { suite } = loaded;

package/esm/src/evals/suite.d.ts

@@ -1,19 +1,28 @@
 import type { z } from 'zod';
 import type { RunDecisionOptions } from '../kernel/engine/decision.js';
-import type { ModelProvider } from '../kernel/types.js';
+import type { ProviderHostOptions } from '../kernel/protocols/runtime.js';
+import type { StageHandler } from '../kernel/stages.js';
 import { type TraceRecord } from '../observability/trace-schema.js';
 import { type EvalCase, type EvalMediaResolver, type EvalSuite } from './types.js';
+/** A suite with its cases read, the providers, media resolver, and stage handler its module exports, and the module's path. */
 interface LoadedSuite {
     suite: EvalSuite;
     cases: EvalCase[];
-    provider?: ModelProvider;
-    judgeProvider?: ModelProvider;
+    provider?: ProviderHostOptions;
+    judgeProvider?: ProviderHostOptions;
     judgeDecision?: Omit<RunDecisionOptions, 'sink'>;
     media?: EvalMediaResolver;
+    /** The stage handler every live trial's turn runs with, as the host's own requests do. */
+    onStage?: StageHandler;
+    /** The application context every live trial's turn carries as `TurnRequest.host`; its tools read it as `ctx.host`. */
+    host?: unknown;
     path: string;
 }
+/** Reads a JSON Lines file, parsing each line against the schema; throws with the file and line number on a line that is not JSON or does not match. */
 declare function readJsonl<T>(path: string, schema: z.ZodType<T>, what: string): Promise<T[]>;
+/** Reads trace records from a `.jsonl` file, or from every `.jsonl` file in a directory, in name order. */
 declare function readTraceRecords(path: string): Promise<TraceRecord[]>;
+/** Imports a suite module, checks its default export against `EvalSuite` and reads its cases; throws when a case id repeats. */
 declare function loadSuite(modulePath: string): Promise<LoadedSuite>;
 export type { LoadedSuite };
 export { loadSuite, readJsonl, readTraceRecords };

package/esm/src/evals/suite.js

@@ -6,7 +6,7 @@ import { traceRecordSchema } from '../observability/trace-schema.js';
 import { pinCaseFiles } from './attachments.js';
 import { evalCaseSchema, evalSuiteSchema, } from './types.js';
 function isProvider(value) {
-    return isRecord(value) && typeof value.complete === 'function';
+    return isRecord(value) && !('complete' in value);
 }
 function isDecisionKey(value) {
     return isRecord(value) && isRecord(value.vault);
@@ -18,6 +18,7 @@ function siblingOf(modulePath, relative) {
     const dir = modulePath.slice(0, modulePath.lastIndexOf('/'));
     return relative.startsWith('/') ? relative : `${dir}/${relative}`;
 }
+/** Reads a JSON Lines file, parsing each line against the schema; throws with the file and line number on a line that is not JSON or does not match. */
 async function readJsonl(path, schema, what) {
     const text = await readFile(path, 'utf8');
     const out = [];
@@ -30,7 +31,7 @@ async function readJsonl(path, schema, what) {
             parsed = JSON.parse(line);
         }
         catch {
-            throw new TheoremError('config', `${path}:${index + 1}: ${what} is not JSON`); // lexicon-exempt: developer contract error
+            throw new TheoremError('config', `${path}:${index + 1}: ${what} is not JSON`); // labels-exempt: developer contract error
         }
         const result = schema.safeParse(parsed);
         if (!result.success) {
@@ -42,6 +43,7 @@ async function readJsonl(path, schema, what) {
     }
     return out;
 }
+/** Reads trace records from a `.jsonl` file, or from every `.jsonl` file in a directory, in name order. */
 async function readTraceRecords(path) {
     const target = absolute(path);
     const info = await stat(target);
@@ -58,11 +60,12 @@ async function readTraceRecords(path) {
         records.push(...(await readJsonl(file, traceRecordSchema, 'trace record')));
     return records;
 }
+/** Imports a suite module, checks its default export against `EvalSuite` and reads its cases; throws when a case id repeats. */
 async function loadSuite(modulePath) {
     const path = absolute(modulePath);
     const module = await import(new URL(`file://${path}`).href);
     if (!isRecord(module)) {
-        throw new TheoremError('config', `${modulePath}: not a module`); // lexicon-exempt: developer contract error
+        throw new TheoremError('config', `${modulePath}: not a module`); // labels-exempt: developer contract error
     }
     const parsed = evalSuiteSchema.safeParse(module.default);
     if (!parsed.success) {
@@ -75,7 +78,7 @@ async function loadSuite(modulePath) {
     const ids = new Set();
     for (const evalCase of cases) {
         if (ids.has(evalCase.id)) {
-            throw new TheoremError('config', `${suite.cases}: case id ${evalCase.id} appears twice`); // lexicon-exempt: developer contract error
+            throw new TheoremError('config', `${suite.cases}: case id ${evalCase.id} appears twice`); // labels-exempt: developer contract error
         }
         ids.add(evalCase.id);
     }
@@ -87,6 +90,8 @@ async function loadSuite(modulePath) {
         ...(isProvider(module.judgeProvider) ? { judgeProvider: module.judgeProvider } : {}),
         ...(isDecisionKey(module.judgeDecision) ? { judgeDecision: module.judgeDecision } : {}),
         ...(typeof module.media === 'function' ? { media: module.media } : {}),
+        ...(typeof module.onStage === 'function' ? { onStage: module.onStage } : {}),
+        ...(module.host === undefined ? {} : { host: module.host }),
     };
 }
 export { loadSuite, readJsonl, readTraceRecords };

package/esm/src/evals/summary.d.ts

@@ -2,6 +2,7 @@ import { type GroupSummary } from './breakdown.js';
 import type { SuiteRun, TurnShape } from './run.js';
 import type { EvalPassRule, EvalResult } from './types.js';
 import type { CaseVerdict, TrialOutcome } from './verdict.js';
+/** One trial in a run summary: its case, index, outcome, trace id, turn shape, error, and grader results. */
 interface TrialSummary {
     case?: string;
     index: number;
@@ -11,6 +12,7 @@ interface TrialSummary {
     error?: string;
     results: EvalResult[];
 }
+/** A suite run reduced to what a report shows: the pass rule and verdict, cost, priced and unpriced counts, per-case verdicts, group summaries, and trials. */
 interface RunSummary {
     suite: string;
     mode: 'live' | 'recorded';
@@ -27,6 +29,7 @@ interface RunSummary {
     runTraceId?: string;
     warnings: string[];
 }
+/** Reduces a suite run to a `RunSummary`. */
 declare function summarizeRun(run: SuiteRun): RunSummary;
 export type { RunSummary, TrialSummary };
 export { summarizeRun };

package/esm/src/evals/summary.js

@@ -1,4 +1,5 @@
 import { groupSummaries } from './breakdown.js';
+/** Reduces a suite run to a `RunSummary`. */
 function summarizeRun(run) {
     const trials = [...run.trials, ...run.caseless].map((report) => ({
         ...(report.case ? { case: report.case.id } : {}),

package/esm/src/evals/trial.d.ts

@@ -1,27 +1,14 @@
-/**
- * Records → `Trial`. A live run hands the runner what its memory sink caught;
- * a recorded run hands it what a file held. Both build the same trial, so the
- * same graders give the same results.
- *
- * The root is the `invoke_agent` span with no agent above it in the set; a host
- * span above it (the host's request, when the host nests the turn under its own
- * span) does not hide it, and the topmost span above it is the trial's `top`. It
- * is never assumed to be the first record: a compaction turn's record can be
- * written before its parent's (P9).
- *
- * @module
- */
 import { type TraceRecord } from '../observability/trace-record.js';
 import type { EvalCase, Trial } from './types.js';
-/** Build the trial every grader receives from the records that share one trace. */
+/** Builds the `Trial` a grader reads from one trial's trace records: its root and top spans and accessors for spans, children, content, delivered output, and usage. */
 declare function buildTrial(args: {
     suite: string;
     case?: EvalCase;
     index: number;
     records: TraceRecord[];
 }): Trial;
-/** Group records by trace id, in first-seen order. */
+/** Groups trace records by the trace id of their first span; records with no spans are dropped. */
 declare function groupByTrace(records: TraceRecord[]): Map<string, TraceRecord[]>;
-/** Whether these records hold a turn at all: an `invoke_agent` span. An eval run record does not. */
+/** True when any record holds an `invoke_agent` span, meaning an agent turn ran. */
 declare function hasTurn(records: readonly TraceRecord[]): boolean;
 export { buildTrial, groupByTrace, hasTurn };