@theoremjs/agents 0.3.0 → 0.3.2

package/esm/src/cli/index.js

@@ -8,7 +8,7 @@
 import "../../_dnt.polyfills.js";
 import { argv, exit } from 'node:process';
 import { benchCommand } from './commands/bench.js';
-import { egressCompileCommand } from './commands/egress-compile.js';
+import { detectCompileCommand } from './commands/detect-compile.js';
 import { evalCommand } from './commands/eval.js';
 import { fuzzCanaryCommand } from './commands/fuzz-canary.js';
 import { fuzzGuardrailsCommand } from './commands/fuzz-guardrails.js';
@@ -28,8 +28,8 @@ COMMANDS:
     --iterations <n>   Measurement iterations (default: 50)
     --warmup <n>       Warmup iterations (default: 5)
 
-  egress-compile <module>  Compile a module's egress rules for egressPolicy
-    --export <name>    The export holding the rules (default: rules)
+  detect-compile <module>  Compile the patterns in a module's detect setting
+    --export <name>    The export holding the setting (default: detect)
     --out <path>       Where to write the compiled module
 
   eval <suite>         Run an eval suite live or over recorded traces
@@ -204,19 +204,19 @@ function handleProfile(flags) {
     }
     showProfileCommand(id);
 }
-async function handleEgressCompile(flags) {
+async function handleDetectCompile(flags) {
     const module = flags._[1];
     if (!module || typeof flags.out !== 'string') {
-        console.error('Error: usage `agents egress-compile <module> --out <path> [--export <name>]`');
+        console.error('Error: usage `agents detect-compile <module> --out <path> [--export <name>]`');
         exit(1);
     }
     try {
-        const count = await egressCompileCommand({
+        const detectors = await detectCompileCommand({
             module,
             exportName: typeof flags.export === 'string' ? flags.export : undefined,
             out: flags.out,
         });
-        console.log(`Compiled ${count} egress rule${count === 1 ? '' : 's'} to ${flags.out}`);
+        console.log(`Compiled the patterns of ${detectors.join(', ') || 'no detector'} to ${flags.out}`);
     }
     catch (err) {
         console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
@@ -239,8 +239,8 @@ export async function main(cliArgs = argv.slice(2)) {
             exit(1);
         }
     }
-    else if (command === 'egress-compile') {
-        await handleEgressCompile(flags);
+    else if (command === 'detect-compile') {
+        await handleDetectCompile(flags);
     }
     else if (command === 'bench') {
         await benchCommand({

package/esm/src/cli/matrix/fixtures.js

@@ -1,39 +1,34 @@
-// 1x1 pixel PNG (base64)
 export const FIXTURE_PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
 export const FIXTURE_PDF_BASE64 = btoa('%PDF-1.4\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n2 0 obj<</Type/Pages/Count 1/Kids[3 0 R]>>endobj\n3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<<>>>>endobj\nxref\n0 4\n0000000000 65535 f \n0000000009 00000 n \n0000000056 00000 n \n0000000111 00000 n \ntrailer<</Size 4/Root 1 0 R>>\nstartxref\n188\n%%EOF\n');
-// 16kHz 16-bit mono PCM WAV
 export function createSyntheticWavBase64(durationSec = 0.1, sampleRate = 16000) {
     const numSamples = Math.floor(sampleRate * durationSec);
-    const dataSize = numSamples * 2; // 16-bit = 2 bytes per sample
+    const dataSize = numSamples * 2;
     const buffer = new ArrayBuffer(44 + dataSize);
     const view = new DataView(buffer);
-    // RIFF identifier
-    view.setUint8(0, 0x52); // R
-    view.setUint8(1, 0x49); // I
-    view.setUint8(2, 0x46); // F
-    view.setUint8(3, 0x46); // F
-    view.setUint32(4, 36 + dataSize, true); // file size - 8
-    view.setUint8(8, 0x57); // W
-    view.setUint8(9, 0x41); // A
-    view.setUint8(10, 0x56); // V
-    view.setUint8(11, 0x45); // E
-    // fmt subchunk
-    view.setUint8(12, 0x66); // f
-    view.setUint8(13, 0x6d); // m
-    view.setUint8(14, 0x74); // t
-    view.setUint8(15, 0x20); // ' '
-    view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM)
-    view.setUint16(20, 1, true); // AudioFormat (1 = PCM)
-    view.setUint16(22, 1, true); // NumChannels (1 = Mono)
-    view.setUint32(24, sampleRate, true); // SampleRate
-    view.setUint32(28, sampleRate * 2, true); // ByteRate (SampleRate * NumChannels * BitsPerSample/8)
-    view.setUint16(32, 2, true); // BlockAlign (NumChannels * BitsPerSample/8)
-    view.setUint16(34, 16, true); // BitsPerSample (16 bits)
-    // data subchunk
-    view.setUint8(36, 0x64); // d
-    view.setUint8(37, 0x61); // a
-    view.setUint8(38, 0x74); // t
-    view.setUint8(39, 0x61); // a
+    view.setUint8(0, 0x52);
+    view.setUint8(1, 0x49);
+    view.setUint8(2, 0x46);
+    view.setUint8(3, 0x46);
+    view.setUint32(4, 36 + dataSize, true);
+    view.setUint8(8, 0x57);
+    view.setUint8(9, 0x41);
+    view.setUint8(10, 0x56);
+    view.setUint8(11, 0x45);
+    view.setUint8(12, 0x66);
+    view.setUint8(13, 0x6d);
+    view.setUint8(14, 0x74);
+    view.setUint8(15, 0x20);
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, sampleRate, true);
+    view.setUint32(28, sampleRate * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    view.setUint8(36, 0x64);
+    view.setUint8(37, 0x61);
+    view.setUint8(38, 0x74);
+    view.setUint8(39, 0x61);
     view.setUint32(40, dataSize, true);
     const freq = 440;
     for (let i = 0; i < numSamples; i++) {

package/esm/src/evals/attachments.d.ts

@@ -1,4 +1,5 @@
 import type { EvalAttachment, EvalCase } from './types.js';
+/** An attachment's content as base64, read from the file when the attachment is a path. */
 declare function attachmentData(attachment: EvalAttachment): Promise<string>;
 declare function pinCaseFiles(cases: EvalCase[], casesPath: string): Promise<EvalCase[]>;
 export { attachmentData, pinCaseFiles };

package/esm/src/evals/attachments.js

@@ -1,8 +1,9 @@
-// A file attachment is read when its trial runs, not when the suite loads, so thousands of photos are never held at once.
+// why: A file attachment is read when its trial runs, not when the suite loads, so thousands of photos are never held at once.
 import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { TheoremError } from '../guardrails/error.js';
+import { bytesToBase64 } from '../kernel/util/base64.js';
 async function hashedFile(path) {
     let bytes;
     try {
@@ -20,8 +21,9 @@ async function fileBytes(attachment) {
     }
     return bytes;
 }
+/** An attachment's content as base64, read from the file when the attachment is a path. */
 async function attachmentData(attachment) {
-    return 'data' in attachment ? attachment.data : (await fileBytes(attachment)).toString('base64');
+    return 'data' in attachment ? attachment.data : bytesToBase64(await fileBytes(attachment));
 }
 async function pinCaseFiles(cases, casesPath) {
     const pinned = [];

package/esm/src/evals/breakdown.d.ts

@@ -4,6 +4,7 @@ interface Spread {
     median: number;
     p90: number;
 }
+/** The results of one group of cases: all cases, or the cases with one tag. It counts cases and trials by outcome and, where they apply, the answers by grade, the stop kinds, and the spread of duration, model calls, and tool calls. */
 interface GroupSummary {
     group: string;
     cases: number;
@@ -23,6 +24,7 @@ interface GroupSummary {
     modelCalls?: Spread;
     toolCalls?: Spread;
 }
+/** Summarizes a run as one group for all cases and one for each tag its cases carry. */
 declare function groupSummaries(run: Pick<SuiteRun, 'trials' | 'verdicts'>): GroupSummary[];
 export type { GroupSummary, Spread };
 export { groupSummaries };

package/esm/src/evals/breakdown.js

@@ -49,6 +49,7 @@ function groupOf(group, verdicts, trials, gradesAnswers) {
         ...(toolCalls ? { toolCalls } : {}),
     };
 }
+/** Summarizes a run as one group for all cases and one for each tag its cases carry. */
 function groupSummaries(run) {
     const gradesAnswers = run.trials.some((report) => report.results.some((result) => result.name === 'answer'));
     const tags = [...new Set(run.trials.flatMap((report) => report.case?.tags ?? []))].sort();

package/esm/src/evals/graders/answer.d.ts

@@ -1,11 +1,14 @@
 import type { EvalGrader } from '../types.js';
+/** Where `answer` reads the agent's answer: a key of the reply's JSON, or an argument of a tool call. */
 type AnswerSource = {
     json: string;
 } | {
     tool: string;
     arg: string;
 };
+/** How an answer was graded: `accepted` passes, `partial` and `wrong` fail. */
 type AnswerLabel = 'accepted' | 'partial' | 'wrong';
+/** A code grader that checks the agent's answer against the case's `expect.answer` lists of accepted, partial, and rejected names, normalizing both sides; with `from` it compares the named JSON key or tool argument exactly, otherwise it looks for the names in the reply text. A provider error before an answer errors the trial instead. */
 declare function answer(options?: {
     from?: AnswerSource;
 }): EvalGrader;

package/esm/src/evals/graders/answer.js

@@ -82,6 +82,7 @@ function unanswered(trial, why) {
         explanation: `${why}; the turn stopped provider_error${kind} before it answered`,
     };
 }
+/** A code grader that checks the agent's answer against the case's `expect.answer` lists of accepted, partial, and rejected names, normalizing both sides; with `from` it compares the named JSON key or tool argument exactly, otherwise it looks for the names in the reply text. A provider error before an answer errors the trial instead. */
 function answer(options = {}) {
     const { from } = options;
     const identity = `answer:${from ? JSON.stringify(from) : 'reply'}`;

package/esm/src/evals/graders/code.d.ts

@@ -1,50 +1,29 @@
-/**
- * Code graders: deterministic checks over a trial's trace. Each returns one
- * pass/fail result with the reason in plain words. Outcome first: what was
- * delivered and what it cost matter more than the path taken.
- *
- * @module
- */
 import type { EvalGrader, EvalResult, Trial } from '../types.js';
-/** Graders over what the host received: the model's own text and JSON. */
+/** Graders over the text the turn delivered to the user. */
 interface DeliveredGraders {
-    /** The delivered text contains `text`. */
     includes: (text: string) => EvalGrader;
-    /** The delivered text matches `pattern` (a JavaScript regular expression source). */
     regex: (pattern: string, flags?: string) => EvalGrader;
-    /** The delivered text equals `text` exactly. */
     equals: (text: string) => EvalGrader;
-    /**
-     * The delivered JSON validates against a JSON Schema object (required
-     * fields present, recursively). The structured part is read when the
-     * profile has one; otherwise the text is parsed.
-     */
     jsonSchema: (schema: Record<string, unknown>) => EvalGrader;
-    /** Every field of the case's `expect.json` is present in the delivered JSON with the same value. */
     json: () => EvalGrader;
 }
+/** Graders over the text the turn delivered to the user. */
 declare const delivered: DeliveredGraders;
-/**
- * How the called tools are compared with the expected list:
- * - `exact`: the same tools in the same order, nothing more;
- * - `in_order`: the expected tools appear in that order, others may interleave;
- * - `any_order`: the same tools, order ignored;
- * - `subset`: every tool called is one that was expected (none unexpected; some may be missing).
- */
+/** `in_order` lets other tools interleave; `subset` means none unexpected, though some expected may be missing. */
 type TrajectoryMode = 'exact' | 'in_order' | 'any_order' | 'subset';
-/** The tools the agent called, in start order, against `expect` or the case's `expect.tools`. */
+/** Grades the tools a turn called against the expected ones, by order or by set. */
 declare function toolTrajectory(options: {
     expect?: string[];
     mode: TrajectoryMode;
 }): EvalGrader;
-/** The turn stopped for one of these reasons (`theorem.stop.kind`). */
+/** Grades that the turn stopped with one of the given kinds. */
 declare function stopKind(kind: string | string[]): EvalGrader;
 /** A guardrail did (`fired: true`) or did not (`fired: false`) act on any span; `allow` does not count as acting. */
 declare function guardrail(options: {
     fired: boolean;
     action?: 'redact' | 'flag' | 'block';
 }): EvalGrader;
-/** Ceilings the trial must stay under; every one that is crossed is named. */
+/** Limits a trial must stay under: cost, tokens, steps, and time. */
 interface BudgetOptions {
     maxCostUsd?: number;
     maxTokens?: number;
@@ -55,10 +34,7 @@ interface BudgetOptions {
 }
 /** The trial stayed within every ceiling given; a ceiling with no reading in the trace fails, since it cannot be shown. */
 declare function budget(options: BudgetOptions): EvalGrader;
-/**
- * The host's own check of the end state (a booking exists, a row changed):
- * "host decides". Return a boolean, or a full result to score and explain.
- */
+/** A grader that passes when `check` accepts the trial. */
 declare function outcome(name: string, check: (trial: Trial) => boolean | EvalResult | Promise<boolean | EvalResult>): EvalGrader;
 export type { BudgetOptions, DeliveredGraders, TrajectoryMode };
 export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory };

package/esm/src/evals/graders/code.js

@@ -1,10 +1,3 @@
-/**
- * Code graders: deterministic checks over a trial's trace. Each returns one
- * pass/fail result with the reason in plain words. Outcome first: what was
- * delivered and what it cost matter more than the path taken.
- *
- * @module
- */
 import { collectValidationFailures } from '../../kernel/engine/runner/schema-validation.js';
 import { isRecord } from '../../kernel/util/record.js';
 import { codeGrader, deliveredJson, deliveredText, listOf, modelCalls, passFail, spanDurationMs, } from './shared.js';
@@ -17,15 +10,14 @@ function numberAttribute(span, key) {
     const value = span.attributes[key];
     return typeof value === 'number' ? value : undefined;
 }
-/** Deep structural equality over JSON values. */
 function sameJson(left, right) {
     return JSON.stringify(left) === JSON.stringify(right);
 }
 function quote(text) {
     return JSON.stringify(text);
 }
+/** Graders over the text the turn delivered to the user. */
 const delivered = {
-    /** The delivered text contains `text`. */
     includes(text) {
         return codeGrader('delivered_includes', `delivered.includes:${text}`, false, (trial) => {
             const actual = deliveredText(trial, undefined);
@@ -33,7 +25,6 @@ const delivered = {
             return passFail('delivered_includes', passed, passed ? `delivered text includes ${quote(text)}` : `delivered text lacks ${quote(text)}`);
         });
     },
-    /** The delivered text matches `pattern` (a JavaScript regular expression source). */
     regex(pattern, flags = '') {
         const re = new RegExp(pattern, flags);
         return codeGrader('delivered_regex', `delivered.regex:/${pattern}/${flags}`, false, (trial) => {
@@ -43,7 +34,6 @@ const delivered = {
                 : `delivered text does not match /${pattern}/${flags}`);
         });
     },
-    /** The delivered text equals `text` exactly. */
     equals(text) {
         return codeGrader('delivered_equals', `delivered.equals:${text}`, false, (trial) => {
             const actual = deliveredText(trial, undefined);
@@ -51,11 +41,6 @@ const delivered = {
             return passFail('delivered_equals', passed, passed ? 'delivered text is exactly as expected' : `delivered text was ${quote(actual)}`);
         });
     },
-    /**
-     * The delivered JSON validates against a JSON Schema object (required
-     * fields present, recursively). The structured part is read when the
-     * profile has one; otherwise the text is parsed.
-     */
     jsonSchema(schema) {
         return codeGrader('delivered_json_schema', `delivered.jsonSchema:${JSON.stringify(schema)}`, false, async (trial) => {
             const value = deliveredJson(trial);
@@ -67,7 +52,6 @@ const delivered = {
             return passFail('delivered_json_schema', passed, passed ? 'delivered JSON fits the schema' : failures.map((f) => f.error).join('; '));
         });
     },
-    /** Every field of the case's `expect.json` is present in the delivered JSON with the same value. */
     json() {
         return codeGrader('delivered_json', 'delivered.json', true, (trial) => {
             const expected = trial.case?.expect?.json;
@@ -111,7 +95,7 @@ function trajectoryMatches(mode, expected, actual) {
             return false;
     }
 }
-/** The tools the agent called, in start order, against `expect` or the case's `expect.tools`. */
+/** Grades the tools a turn called against the expected ones, by order or by set. */
 function toolTrajectory(options) {
     const identity = `toolTrajectory:${options.mode}:${options.expect ? JSON.stringify(options.expect) : 'case'}`;
     return codeGrader('tool_trajectory', identity, options.expect === undefined, (trial) => {
@@ -126,8 +110,7 @@ function toolTrajectory(options) {
         return passFail('tool_trajectory', passed, `called ${listOf(actual)}; expected ${listOf(expected)} (${options.mode})`);
     });
 }
-// ── stop, guardrail, budget ─────────────────────────
-/** The turn stopped for one of these reasons (`theorem.stop.kind`). */
+/** Grades that the turn stopped with one of the given kinds. */
 function stopKind(kind) {
     const kinds = Array.isArray(kind) ? kind : [kind];
     return codeGrader('stop_kind', `stopKind:${kinds.join('|')}`, false, (trial) => {
@@ -188,11 +171,7 @@ function budget(options) {
         return passFail('budget', passed, passed ? 'within budget' : over.join('; '));
     });
 }
-// ── outcome ─────────────────────────────────────────
-/**
- * The host's own check of the end state (a booking exists, a row changed):
- * "host decides". Return a boolean, or a full result to score and explain.
- */
+/** A grader that passes when `check` accepts the trial. */
 function outcome(name, check) {
     return codeGrader(name, `outcome:${name}`, false, async (trial) => {
         const verdict = await check(trial);

package/esm/src/evals/graders/judge.d.ts

@@ -1,10 +1,13 @@
 import { type EvalRubric } from '../rubrics/types.js';
 import type { EvalGrader, Trial } from '../types.js';
+/** The id of the structured schema a text judge answers with, registered when this module loads. */
 declare const EVAL_JUDGMENT = "evalJudgment";
+/** A judge's answer: one of the rubric's labels, or `unknown`, and why. */
 interface Judgment {
     label: string;
     explanation: string;
 }
+/** Options for `judge`: the rubric and, optionally, a name, the labels that pass, a judge profile, the rubric's variables, the cost of a wrong pass and an escalation profile. */
 interface JudgeOptions {
     rubric: EvalRubric;
     name?: string;
@@ -15,6 +18,7 @@ interface JudgeOptions {
     wrongPassCost?: number;
     escalate?: string;
 }
+/** A grader in which a model judges the trial against a rubric: a text profile answers directly, a decision profile answers the rubric's question, and an `escalate` text profile takes trials with media a decision judge cannot see. A turn that failed or delivered nothing is not judged; a misconfiguration throws. */
 declare function judge(options: JudgeOptions): EvalGrader;
 export type { JudgeOptions, Judgment };
 export { EVAL_JUDGMENT, judge };

package/esm/src/evals/graders/judge.js

@@ -9,6 +9,7 @@ import { buildTrial } from '../trial.js';
 import { resolveMedia, trialMedia } from './media.js';
 import { deliveredJson, deliveredText } from './shared.js';
 import { rubricView, TRIAL_VARIABLES } from './transcript.js';
+/** The id of the structured schema a text judge answers with, registered when this module loads. */
 const EVAL_JUDGMENT = 'evalJudgment';
 const UNKNOWN = 'unknown';
 const UNKNOWN_CRITERION = 'The record does not show enough to decide.';
@@ -44,10 +45,10 @@ const JUDGMENT_SCHEMA = {
         required: ['label', 'explanation'],
     },
 };
-// Importing this module registers the schema, so a host's judge profile can name it at registration.
+// why: Importing this module registers the schema, so a host's judge profile can name it at registration.
 registerStructured(EVAL_JUDGMENT, JUDGMENT_SCHEMA);
 function configError(grader, message) {
-    return new TheoremError('config', `judge ${grader}: ${message}`); // lexicon-exempt: developer contract error
+    return new TheoremError('config', `judge ${grader}: ${message}`); // labels-exempt: developer contract error
 }
 function judgeKind(grader, rubric, profileId) {
     const profile = getProfile(profileId);
@@ -116,7 +117,7 @@ async function askText(args) {
     };
     try {
         for await (const _event of runTurn(request, context.judgeProvider, memorySink(records))) {
-            // The judge's trace is its record; its events are not read.
+            // why: The judge's trace is its record; its events are not read.
         }
     }
     catch (thrown) {
@@ -209,13 +210,13 @@ function stateOf(rubric, view) {
     for (const variable of rubric.variables) {
         const value = view[variable];
         if (value === undefined) {
-            throw new TheoremError('config', `rubric ${rubric.name}: no value for ${variable}`); // lexicon-exempt: developer contract error
+            throw new TheoremError('config', `rubric ${rubric.name}: no value for ${variable}`); // labels-exempt: developer contract error
         }
         state[variable] = value;
     }
     return state;
 }
-// Models echo the prompt's headings (`CORRECT -`), so a text judge's label is read case-insensitively.
+// why: Models echo the prompt's headings (`CORRECT -`), so a text judge's label is read case-insensitively.
 function declaredLabel(label, allowed) {
     const wanted = label.trim().toLowerCase();
     return [...allowed].find((declared) => declared.toLowerCase() === wanted);
@@ -250,6 +251,7 @@ function resultOf(run, rubric, name, pass) {
         ...(pass ? { passed: pass.includes(label) } : {}),
     };
 }
+/** A grader in which a model judges the trial against a rubric: a text profile answers directly, a decision profile answers the rubric's question, and an `escalate` text profile takes trials with media a decision judge cannot see. A turn that failed or delivered nothing is not judged; a misconfiguration throws. */
 function judge(options) {
     const { rubric } = options;
     const name = options.name ?? rubric.name;

package/esm/src/evals/graders/latency.d.ts

@@ -1,4 +1,5 @@
 import type { EvalGrader } from '../types.js';
+/** A grader that passes when each reply began within `maxMs`, measured from the start of the response or, for a live reply, from the end of the user's speech; a reply with no reading fails it. */
 declare function turnLatency(options: {
     maxMs: number;
 }): EvalGrader;

package/esm/src/evals/graders/latency.js

@@ -9,7 +9,7 @@ function numberAttribute(span, key) {
     const value = span.attributes[key];
     return typeof value === 'number' ? value : undefined;
 }
-// The successful (last) HTTP try's start plus its time to first chunk, or the call's own start when the call streamed over a socket (Live).
+// why: The successful (last) HTTP try's start plus its time to first chunk, or the call's own start when the call streamed over a socket (Live).
 function firstChunkAt(trial, call) {
     const seconds = numberAttribute(call, 'gen_ai.response.time_to_first_chunk');
     if (seconds === undefined)
@@ -18,7 +18,7 @@ function firstChunkAt(trial, call) {
     const from = tries.at(-1) ?? call;
     return BigInt(from.startTimeUnixNano) + BigInt(Math.round(seconds * MS_PER_S)) * NANOS_PER_MS;
 }
-// Speech that prompted an earlier reply never anchors a later one.
+// invariant: Speech that prompted an earlier reply never anchors a later one.
 function speechEndedAt(root, afterUnixNano, beforeUnixNano) {
     let latest;
     for (const event of root.events) {
@@ -59,6 +59,7 @@ function latencyReadings(trial) {
         reading(trial.root, BigInt(trial.root.startTimeUnixNano), firstChunkAt(trial, firstCall)),
     ];
 }
+/** A grader that passes when each reply began within `maxMs`, measured from the start of the response or, for a live reply, from the end of the user's speech; a reply with no reading fails it. */
 function turnLatency(options) {
     return codeGrader('turn_latency', `turnLatency:${options.maxMs}`, false, (trial) => {
         const readings = latencyReadings(trial);

package/esm/src/evals/graders/live.d.ts

@@ -1,9 +1,12 @@
 import type { EvalGrader } from '../types.js';
+/** Graders for the transcript of a live model's spoken output. */
 interface TranscriptionGraders {
     includes: (text?: string) => EvalGrader;
     regex: (pattern?: string, flags?: string) => EvalGrader;
 }
+/** Graders for a live reply's output transcription: `includes` (a substring) and `regex`, each taking its pattern as an argument or from the case's `expect.transcription`. */
 declare const transcription: TranscriptionGraders;
+/** A grader that passes when the model was interrupted at most `max` times in the trial. */
 declare function interruptions(options: {
     max: number;
 }): EvalGrader;

package/esm/src/evals/graders/live.js

@@ -1,5 +1,6 @@
 import { codeGrader, deliveredText, passFail } from './shared.js';
 const OUTPUT_TRANSCRIPTION = 'output_transcription';
+/** Graders for a live reply's output transcription: `includes` (a substring) and `regex`, each taking its pattern as an argument or from the case's `expect.transcription`. */
 const transcription = {
     includes(text) {
         const identity = `transcription.includes:${text ?? 'case'}`;
@@ -28,6 +29,7 @@ const transcription = {
         });
     },
 };
+/** A grader that passes when the model was interrupted at most `max` times in the trial. */
 function interruptions(options) {
     return codeGrader('interruptions', `interruptions:${options.max}`, false, (trial) => {
         const count = trial

package/esm/src/evals/graders/media.js

@@ -45,7 +45,6 @@ function mediaLabeler(trial) {
     const byKey = new Map(trialMedia(trial).map((media) => [media.sha256 ?? media.uri ?? '', media.label]));
     return (part) => byKey.get(mediaKey(part) ?? '') ?? `[${String(part.type)}]`;
 }
-// A file's hash is the one its case pins.
 async function caseAttachments(trial) {
     const byHash = new Map();
     const input = trial.case?.input;
@@ -61,7 +60,7 @@ async function caseAttachments(trial) {
     }
     return byHash;
 }
-// The host's bytes for a hash, only when they are the bytes the trace hashed.
+// why: The host's bytes for a hash, only when they are the bytes the trace hashed.
 async function fromHost(sha256, mimeType, context) {
     const data = await context.media?.({ sha256, mimeType });
     if (data === undefined)

package/esm/src/evals/graders/shared.js

@@ -25,7 +25,7 @@ function deliveredParts(trial) {
         return Array.isArray(parts) ? parts.filter(isObject) : [];
     });
 }
-// Interim transcripts are skipped; the final one holds the whole.
+// why: Interim transcripts are skipped; the final one holds the whole.
 function deliveredText(trial, source) {
     return deliveredParts(trial)
         .filter((part) => part.type === 'text' &&

package/esm/src/evals/graders/transcript.d.ts

@@ -2,6 +2,7 @@ import type { DecisionJson } from '../../kernel/types.js';
 import type { TraceSpan } from '../../observability/trace-span.js';
 import type { EvalRubric } from '../rubrics/types.js';
 import type { Trial } from '../types.js';
+/** The variables a judge rubric may use, each filled from the trial by `trialVariables`. */
 declare const TRIAL_VARIABLES: readonly ["input", "output", "context", "conversation", "user_message", "tool_call", "tool_result", "text"];
 interface ToolStep {
     span: TraceSpan;
@@ -11,6 +12,7 @@ interface ToolStep {
     outcome: string;
 }
 declare function toolSteps(trial: Trial): ToolStep[];
+/** Fills each rubric variable from a trial: the last user message, the reply, the tool activity, and the conversation as text. */
 declare function trialVariables(trial: Trial): Record<(typeof TRIAL_VARIABLES)[number], string>;
 declare function rubricView(rubric: EvalRubric, trial: Trial, host: Readonly<Record<string, string>>): Record<string, DecisionJson>;
 export type { ToolStep };

package/esm/src/evals/graders/transcript.js

@@ -2,6 +2,7 @@ import { isRecord } from '../../kernel/util/record.js';
 import { templatePaths } from '../rubrics/mustache.js';
 import { mediaLabeler, trialMedia } from './media.js';
 import { deliveredJson, deliveredText, modelCalls, startOf } from './shared.js';
+/** The variables a judge rubric may use, each filled from the trial by `trialVariables`. */
 const TRIAL_VARIABLES = [
     'input',
     'output',
@@ -54,7 +55,7 @@ function toolCallText(part) {
             args = JSON.parse(args);
         }
         catch {
-            // Not JSON: the arguments stand as the model sent them.
+            // why: Not JSON: the arguments stand as the model sent them.
         }
     }
     return JSON.stringify({ name: part.name, arguments: args });
@@ -96,7 +97,7 @@ function availableTools(trial) {
                 return parsed.map((tool) => JSON.stringify(tool));
         }
         catch {
-            // Not JSON: the definitions stand as one entry, as sent.
+            // why: Not JSON: the definitions stand as one entry, as sent.
         }
         return [text];
     }
@@ -147,6 +148,7 @@ function transcript(messages, steps, output) {
         `[assistant]\n${output}`,
     ].join('\n\n');
 }
+/** Fills each rubric variable from a trial: the last user message, the reply, the tool activity, and the conversation as text. */
 function trialVariables(trial) {
     const messages = inputMessages(trial);
     const input = messages.findLast((message) => message.role === 'user')?.text ?? '';
@@ -170,7 +172,7 @@ function trialVariables(trial) {
         text: output,
     };
 }
-// A rubric that reads `user_message` gets `conversation` as what came before that message, as Phoenix's user-friction prompt asks.
+// why: A rubric that reads `user_message` gets `conversation` as what came before that message, as Phoenix's user-friction prompt asks.
 function rubricView(rubric, trial, host) {
     const values = { ...trialVariables(trial) };
     const paths = rubric.template === undefined ? [] : templatePaths(rubric.template);

package/esm/src/evals/mod.d.ts

@@ -1,13 +1,5 @@
 /**
- * Evals over traces. A host says, for any profile: here are the cases, here is
- * what "good" means, run it k times and tell me, from the trace alone, whether
- * it passed. Graders read the v3 `TraceRecord` and nothing else; results are
- * trace records too, written through the same `TraceSink`.
- *
- * Hosts own the cases, the judge (a text model, Jev, or both) and the pass
- * rule. THEOREM owns the runner, the code graders and the result record;
- * viewing, labelling and comparing runs belong to the trace tool the host
- * already uses.
+ * Evals over traces: graders read the v3 `TraceRecord` and nothing else; results are trace records too, written through the same `TraceSink`.
  *
  * @module
  */

package/esm/src/evals/mod.js

@@ -1,13 +1,5 @@
 /**
- * Evals over traces. A host says, for any profile: here are the cases, here is
- * what "good" means, run it k times and tell me, from the trace alone, whether
- * it passed. Graders read the v3 `TraceRecord` and nothing else; results are
- * trace records too, written through the same `TraceSink`.
- *
- * Hosts own the cases, the judge (a text model, Jev, or both) and the pass
- * rule. THEOREM owns the runner, the code graders and the result record;
- * viewing, labelling and comparing runs belong to the trace tool the host
- * already uses.
+ * Evals over traces: graders read the v3 `TraceRecord` and nothing else; results are trace records too, written through the same `TraceSink`.
  *
  * @module
  */