@theoremjs/agents 0.0.0-stage → 0.3.1

package/esm/src/cli/matrix/synthesizer.js

@@ -0,0 +1,103 @@
+import { profileInputs } from '../../kernel/registry/catalog.js';
+import { FIXTURE_PNG_BASE64, FIXTURE_WAV_BASE64, getFixtureForMime } from './fixtures.js';
+export function synthesizeLiteCombo(profile) {
+    const model = profile.allowModelSelect && profile.models.fast ? 'fast' : undefined;
+    return {
+        profile: profile.id,
+        model,
+        input: {
+            text: `Ping test for profile ${profile.id}. Respond concisely with confirmation.`,
+        },
+    };
+}
+function resolveStressModel(profile) {
+    if (!profile.allowModelSelect) {
+        return undefined;
+    }
+    if (profile.models.smart) {
+        return 'smart';
+    }
+    const ids = Object.keys(profile.models);
+    return ids.length > 1 ? ids[ids.length - 1] : undefined;
+}
+function resolveStressAttachments(profile) {
+    const inputs = profileInputs(profile);
+    if (!inputs) {
+        return [];
+    }
+    const attachments = [];
+    const accept = inputs.attachments?.accept;
+    if (accept && accept.length > 0) {
+        const preferredMimes = ['image/png', 'application/pdf', 'text/plain'];
+        const chosenMime = preferredMimes.find((m) => accept.includes(m)) ?? accept[0];
+        const fixture = getFixtureForMime(chosenMime);
+        if (fixture) {
+            attachments.push(fixture);
+        }
+        else {
+            attachments.push({ mimeType: 'image/png', data: FIXTURE_PNG_BASE64 });
+        }
+    }
+    return attachments;
+}
+function resolveStressVoice(profile) {
+    const inputs = profileInputs(profile);
+    if (!inputs) {
+        return [];
+    }
+    const voice = [];
+    if (inputs.voice?.accept && inputs.voice.accept.length > 0) {
+        voice.push({ mimeType: 'audio/wav', data: FIXTURE_WAV_BASE64 });
+    }
+    return voice;
+}
+export function synthesizeStressCombo(profile) {
+    const model = resolveStressModel(profile);
+    const attachments = resolveStressAttachments(profile);
+    const voice = resolveStressVoice(profile);
+    const promptText = `Execute comprehensive test turn for profile ${profile.id}. Validate all instructions and produce required outputs.`;
+    return {
+        profile: profile.id,
+        model,
+        input: {
+            text: promptText,
+            attachments: attachments.length > 0 ? attachments : undefined,
+            voice: voice.length > 0 ? voice : undefined,
+        },
+    };
+}
+export function synthesizeMatrixCombos(profile) {
+    return [
+        {
+            name: 'Lite (connectivity)',
+            req: synthesizeLiteCombo(profile),
+        },
+        {
+            name: 'Stress (all modalities + primary tools)',
+            req: synthesizeStressCombo(profile),
+        },
+    ];
+}
+function assertGroundingFlagsOnModel(profile, options) {
+    const modelId = options.mode && profile.models[options.mode] ? options.mode : profile.defaultModel;
+    const builtins = new Set(profile.models[modelId]?.builtInTools ?? []);
+    if (options.search === true && !builtins.has('googleSearch')) {
+        throw new Error(`--search requires googleSearch on models.${modelId}.builtInTools`);
+    }
+    if (options.map === true && !builtins.has('googleMaps')) {
+        throw new Error(`--map requires googleMaps on models.${modelId}.builtInTools`);
+    }
+}
+export function buildCustomTurnRequest(profile, options) {
+    if (options.lite) {
+        return synthesizeLiteCombo(profile);
+    }
+    const base = synthesizeStressCombo(profile);
+    if (options.mode) {
+        base.model = options.mode;
+    }
+    if (options.search !== undefined || options.map !== undefined) {
+        assertGroundingFlagsOnModel(profile, options);
+    }
+    return base;
+}

package/esm/src/evals/attachments.d.ts

@@ -0,0 +1,5 @@
+import type { EvalAttachment, EvalCase } from './types.js';
+/** An attachment's content as base64, read from the file when the attachment is a path. */
+declare function attachmentData(attachment: EvalAttachment): Promise<string>;
+declare function pinCaseFiles(cases: EvalCase[], casesPath: string): Promise<EvalCase[]>;
+export { attachmentData, pinCaseFiles };

package/esm/src/evals/attachments.js

@@ -0,0 +1,55 @@
+// why: A file attachment is read when its trial runs, not when the suite loads, so thousands of photos are never held at once.
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { TheoremError } from '../guardrails/error.js';
+import { bytesToBase64 } from '../kernel/util/base64.js';
+async function hashedFile(path) {
+    let bytes;
+    try {
+        bytes = await readFile(path);
+    }
+    catch (error) {
+        throw new TheoremError('config', `attachment ${path} cannot be read: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return { bytes, sha256: createHash('sha256').update(bytes).digest('hex') };
+}
+async function fileBytes(attachment) {
+    const { bytes, sha256 } = await hashedFile(attachment.path);
+    if (sha256 !== attachment.sha256) {
+        throw new TheoremError('config', `attachment ${attachment.path} hashes to ${sha256}, not the ${attachment.sha256} its case pins`);
+    }
+    return bytes;
+}
+/** An attachment's content as base64, read from the file when the attachment is a path. */
+async function attachmentData(attachment) {
+    return 'data' in attachment ? attachment.data : bytesToBase64(await fileBytes(attachment));
+}
+async function pinCaseFiles(cases, casesPath) {
+    const pinned = [];
+    for (const evalCase of cases) {
+        const { input } = evalCase;
+        if (!('attachments' in input) || input.attachments === undefined) {
+            pinned.push(evalCase);
+            continue;
+        }
+        const attachments = [];
+        for (const attachment of input.attachments) {
+            if ('data' in attachment) {
+                attachments.push(attachment);
+                continue;
+            }
+            const path = resolve(dirname(casesPath), attachment.path);
+            try {
+                await fileBytes({ ...attachment, path });
+            }
+            catch (error) {
+                throw new TheoremError('config', `case ${evalCase.id}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            attachments.push({ ...attachment, path });
+        }
+        pinned.push({ ...evalCase, input: { ...input, attachments } });
+    }
+    return pinned;
+}
+export { attachmentData, pinCaseFiles };

package/esm/src/evals/breakdown.d.ts

@@ -0,0 +1,30 @@
+import type { SuiteRun } from './run.js';
+/** Nearest-rank percentiles, so each is a value some trial had. */
+interface Spread {
+    median: number;
+    p90: number;
+}
+/** The results of one group of cases: all cases, or the cases with one tag. It counts cases and trials by outcome and, where they apply, the answers by grade, the stop kinds, and the spread of duration, model calls, and tool calls. */
+interface GroupSummary {
+    group: string;
+    cases: number;
+    casesPassed: number;
+    casesUndecided: number;
+    trials: number;
+    trialsPassed: number;
+    trialsErrored: number;
+    answers?: {
+        accepted: number;
+        partial: number;
+        wrong: number;
+        none: number;
+    };
+    stops: Record<string, number>;
+    durationMs?: Spread;
+    modelCalls?: Spread;
+    toolCalls?: Spread;
+}
+/** Summarizes a run as one group for all cases and one for each tag its cases carry. */
+declare function groupSummaries(run: Pick<SuiteRun, 'trials' | 'verdicts'>): GroupSummary[];
+export type { GroupSummary, Spread };
+export { groupSummaries };

package/esm/src/evals/breakdown.js

@@ -0,0 +1,69 @@
+const MEDIAN = 0.5;
+const P90 = 0.9;
+function rank(sorted, quantile) {
+    return sorted[Math.max(0, Math.ceil(quantile * sorted.length) - 1)] ?? 0;
+}
+function spread(values) {
+    if (values.length === 0)
+        return undefined;
+    const sorted = values.toSorted((a, b) => a - b);
+    return { median: rank(sorted, MEDIAN), p90: rank(sorted, P90) };
+}
+function answerLabel(report) {
+    return report.results.find((result) => result.name === 'answer')?.score?.label;
+}
+function groupOf(group, verdicts, trials, gradesAnswers) {
+    const turns = trials.flatMap((report) => report.turn ?? []);
+    const of = (key) => spread(turns.map((turn) => turn[key]));
+    const labels = trials.map(answerLabel);
+    const count = (label) => labels.filter((each) => each === label).length;
+    const stops = {};
+    for (const turn of turns) {
+        const stop = turn.stop ?? 'unrecorded';
+        stops[stop] = (stops[stop] ?? 0) + 1;
+    }
+    const durationMs = of('durationMs');
+    const modelCalls = of('modelCalls');
+    const toolCalls = of('toolCalls');
+    return {
+        group,
+        cases: verdicts.length,
+        casesPassed: verdicts.filter((verdict) => verdict.passed).length,
+        casesUndecided: verdicts.filter((verdict) => !verdict.decided).length,
+        trials: trials.length,
+        trialsPassed: trials.filter((report) => report.outcome === 'passed').length,
+        trialsErrored: trials.filter((report) => report.outcome === 'errored').length,
+        ...(gradesAnswers
+            ? {
+                answers: {
+                    accepted: count('accepted'),
+                    partial: count('partial'),
+                    wrong: count('wrong'),
+                    none: count(undefined),
+                },
+            }
+            : {}),
+        stops,
+        ...(durationMs ? { durationMs } : {}),
+        ...(modelCalls ? { modelCalls } : {}),
+        ...(toolCalls ? { toolCalls } : {}),
+    };
+}
+/** Summarizes a run as one group for all cases and one for each tag its cases carry. */
+function groupSummaries(run) {
+    const gradesAnswers = run.trials.some((report) => report.results.some((result) => result.name === 'answer'));
+    const tags = [...new Set(run.trials.flatMap((report) => report.case?.tags ?? []))].sort();
+    const tagged = (tag) => run.trials.filter((report) => report.case?.tags?.includes(tag) === true);
+    const verdictsOf = (trials) => {
+        const ids = new Set(trials.map((report) => report.case?.id));
+        return run.verdicts.filter((verdict) => ids.has(verdict.case));
+    };
+    return [
+        groupOf('all', run.verdicts, run.trials, gradesAnswers),
+        ...tags.map((tag) => {
+            const trials = tagged(tag);
+            return groupOf(tag, verdictsOf(trials), trials, gradesAnswers);
+        }),
+    ];
+}
+export { groupSummaries };

package/esm/src/evals/graders/answer.d.ts

@@ -0,0 +1,16 @@
+import type { EvalGrader } from '../types.js';
+/** Where `answer` reads the agent's answer: a key of the reply's JSON, or an argument of a tool call. */
+type AnswerSource = {
+    json: string;
+} | {
+    tool: string;
+    arg: string;
+};
+/** How an answer was graded: `accepted` passes, `partial` and `wrong` fail. */
+type AnswerLabel = 'accepted' | 'partial' | 'wrong';
+/** A code grader that checks the agent's answer against the case's `expect.answer` lists of accepted, partial, and rejected names, normalizing both sides; with `from` it compares the named JSON key or tool argument exactly, otherwise it looks for the names in the reply text. A provider error before an answer errors the trial instead. */
+declare function answer(options?: {
+    from?: AnswerSource;
+}): EvalGrader;
+export type { AnswerLabel, AnswerSource };
+export { answer };

package/esm/src/evals/graders/answer.js

@@ -0,0 +1,134 @@
+import { isRecord } from '../../kernel/util/record.js';
+import { codeGrader, deliveredJson, deliveredText, listOf } from './shared.js';
+import { toolSteps } from './transcript.js';
+function normalized(text) {
+    return text
+        .normalize('NFKD')
+        .replace(/\p{M}/gu, '')
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}]+/gu, ' ')
+        .trim();
+}
+/** A name said only inside a longer one of them is not said: `Calathea` in `Calathea orbifolia` is the species. */
+function namedIn(text, names) {
+    const words = normalized(text).split(' ');
+    const spans = names.flatMap((name) => {
+        const key = normalized(name);
+        if (!key)
+            return [];
+        const parts = key.split(' ');
+        return words.flatMap((_, start) => parts.every((part, i) => words[start + i] === part)
+            ? [{ name, start, end: start + parts.length }]
+            : []);
+    });
+    const said = new Set(spans
+        .filter((span) => !spans.some((other) => other.start <= span.start &&
+        other.end >= span.end &&
+        other.end - other.start > span.end - span.start))
+        .map((span) => span.name));
+    return (among) => among.find((name) => said.has(name));
+}
+function at(value, path) {
+    let current = value;
+    for (const key of path.split('.')) {
+        if (!isRecord(current))
+            return undefined;
+        current = current[key];
+    }
+    return current;
+}
+function parsedArguments(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readAnswer(trial, source) {
+    if ('json' in source) {
+        const where = `JSON key ${source.json}`;
+        const value = at(deliveredJson(trial), source.json);
+        return typeof value === 'string' ? { value, where } : { missing: `no string at ${where}` };
+    }
+    const where = `${source.tool} argument ${source.arg}`;
+    const call = toolSteps(trial)
+        .filter((step) => step.name === source.tool)
+        .at(-1);
+    if (!call)
+        return { missing: `${source.tool} was not called` };
+    const value = at(parsedArguments(call.arguments), source.arg);
+    return typeof value === 'string' ? { value, where } : { missing: `no string at ${where}` };
+}
+function result(label, explanation) {
+    const passed = label === 'accepted';
+    return {
+        name: 'answer',
+        source: 'code',
+        score: { value: passed ? 1 : 0, label },
+        explanation,
+        passed,
+    };
+}
+/** A turn the provider failed before it answered: the agent never had its say, so the trial errors instead of counting as wrong. */
+function unanswered(trial, why) {
+    if (trial.root.attributes['theorem.stop.kind'] !== 'provider_error')
+        return undefined;
+    const kind = trial.root.status.message ? ` (${trial.root.status.message})` : '';
+    return {
+        name: 'answer',
+        source: 'code',
+        errorType: 'provider_error',
+        explanation: `${why}; the turn stopped provider_error${kind} before it answered`,
+    };
+}
+/** A code grader that checks the agent's answer against the case's `expect.answer` lists of accepted, partial, and rejected names, normalizing both sides; with `from` it compares the named JSON key or tool argument exactly, otherwise it looks for the names in the reply text. A provider error before an answer errors the trial instead. */
+function answer(options = {}) {
+    const { from } = options;
+    const identity = `answer:${from ? JSON.stringify(from) : 'reply'}`;
+    return codeGrader('answer', identity, true, (trial) => {
+        const expected = trial.case?.expect?.answer;
+        if (!expected)
+            return result('wrong', 'case has no expect.answer');
+        const accepted = expected.accepted;
+        const partial = expected.partial ?? [];
+        const rejected = expected.rejected ?? [];
+        if (from) {
+            const read = readAnswer(trial, from);
+            if ('missing' in read) {
+                return (unanswered(trial, read.missing) ??
+                    result('wrong', `${read.missing}; accepted ${listOf(accepted)}`));
+            }
+            const said = normalized(read.value);
+            const quoted = JSON.stringify(read.value);
+            if (accepted.some((name) => normalized(name) === said)) {
+                return result('accepted', `answered ${quoted} (${read.where})`);
+            }
+            if (rejected.some((name) => normalized(name) === said)) {
+                return result('wrong', `answered ${quoted} (${read.where}), which the case rejects; accepted ${listOf(accepted)}`);
+            }
+            if (partial.some((name) => normalized(name) === said)) {
+                return result('partial', `answered ${quoted} (${read.where}); accepted ${listOf(accepted)}`);
+            }
+            return result('wrong', `answered ${quoted} (${read.where}); accepted ${listOf(accepted)}`);
+        }
+        const found = namedIn(deliveredText(trial, undefined), [...accepted, ...partial, ...rejected]);
+        const hit = found(accepted);
+        const miss = found(rejected);
+        if (hit && miss) {
+            return result('partial', `reply names ${JSON.stringify(hit)} and ${JSON.stringify(miss)}, which the case rejects`);
+        }
+        if (hit)
+            return result('accepted', `reply names ${JSON.stringify(hit)}`);
+        if (miss) {
+            return result('wrong', `reply names ${JSON.stringify(miss)}, which the case rejects; accepted ${listOf(accepted)}`);
+        }
+        const near = found(partial);
+        if (near) {
+            return result('partial', `reply names ${JSON.stringify(near)}; accepted ${listOf(accepted)}`);
+        }
+        return (unanswered(trial, 'the reply names no answer') ??
+            result('wrong', `reply names none of ${listOf(accepted)}`));
+    });
+}
+export { answer };

package/esm/src/evals/graders/code.d.ts

@@ -0,0 +1,40 @@
+import type { EvalGrader, EvalResult, Trial } from '../types.js';
+/** Graders over the text the turn delivered to the user. */
+interface DeliveredGraders {
+    includes: (text: string) => EvalGrader;
+    regex: (pattern: string, flags?: string) => EvalGrader;
+    equals: (text: string) => EvalGrader;
+    jsonSchema: (schema: Record<string, unknown>) => EvalGrader;
+    json: () => EvalGrader;
+}
+/** Graders over the text the turn delivered to the user. */
+declare const delivered: DeliveredGraders;
+/** `in_order` lets other tools interleave; `subset` means none unexpected, though some expected may be missing. */
+type TrajectoryMode = 'exact' | 'in_order' | 'any_order' | 'subset';
+/** Grades the tools a turn called against the expected ones, by order or by set. */
+declare function toolTrajectory(options: {
+    expect?: string[];
+    mode: TrajectoryMode;
+}): EvalGrader;
+/** Grades that the turn stopped with one of the given kinds. */
+declare function stopKind(kind: string | string[]): EvalGrader;
+/** A guardrail did (`fired: true`) or did not (`fired: false`) act on any span; `allow` does not count as acting. */
+declare function guardrail(options: {
+    fired: boolean;
+    action?: 'redact' | 'flag' | 'block';
+}): EvalGrader;
+/** Limits a trial must stay under: cost, tokens, steps, and time. */
+interface BudgetOptions {
+    maxCostUsd?: number;
+    maxTokens?: number;
+    maxSteps?: number;
+    maxDurationMs?: number;
+    /** The slowest model call's time to first chunk. */
+    maxTimeToFirstChunkMs?: number;
+}
+/** The trial stayed within every ceiling given; a ceiling with no reading in the trace fails, since it cannot be shown. */
+declare function budget(options: BudgetOptions): EvalGrader;
+/** A grader that passes when `check` accepts the trial. */
+declare function outcome(name: string, check: (trial: Trial) => boolean | EvalResult | Promise<boolean | EvalResult>): EvalGrader;
+export type { BudgetOptions, DeliveredGraders, TrajectoryMode };
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory };

package/esm/src/evals/graders/code.js

@@ -0,0 +1,184 @@
+import { collectValidationFailures } from '../../kernel/engine/runner/schema-validation.js';
+import { isRecord } from '../../kernel/util/record.js';
+import { codeGrader, deliveredJson, deliveredText, listOf, modelCalls, passFail, spanDurationMs, } from './shared.js';
+const MS_PER_S = 1000;
+function stringAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'string' ? value : undefined;
+}
+function numberAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'number' ? value : undefined;
+}
+function sameJson(left, right) {
+    return JSON.stringify(left) === JSON.stringify(right);
+}
+function quote(text) {
+    return JSON.stringify(text);
+}
+/** Graders over the text the turn delivered to the user. */
+const delivered = {
+    includes(text) {
+        return codeGrader('delivered_includes', `delivered.includes:${text}`, false, (trial) => {
+            const actual = deliveredText(trial, undefined);
+            const passed = actual.includes(text);
+            return passFail('delivered_includes', passed, passed ? `delivered text includes ${quote(text)}` : `delivered text lacks ${quote(text)}`);
+        });
+    },
+    regex(pattern, flags = '') {
+        const re = new RegExp(pattern, flags);
+        return codeGrader('delivered_regex', `delivered.regex:/${pattern}/${flags}`, false, (trial) => {
+            const passed = re.test(deliveredText(trial, undefined));
+            return passFail('delivered_regex', passed, passed
+                ? `delivered text matches /${pattern}/${flags}`
+                : `delivered text does not match /${pattern}/${flags}`);
+        });
+    },
+    equals(text) {
+        return codeGrader('delivered_equals', `delivered.equals:${text}`, false, (trial) => {
+            const actual = deliveredText(trial, undefined);
+            const passed = actual === text;
+            return passFail('delivered_equals', passed, passed ? 'delivered text is exactly as expected' : `delivered text was ${quote(actual)}`);
+        });
+    },
+    jsonSchema(schema) {
+        return codeGrader('delivered_json_schema', `delivered.jsonSchema:${JSON.stringify(schema)}`, false, async (trial) => {
+            const value = deliveredJson(trial);
+            if (value === undefined) {
+                return passFail('delivered_json_schema', false, 'delivered output is not JSON');
+            }
+            const failures = await collectValidationFailures(schema, value, undefined);
+            const passed = failures.length === 0;
+            return passFail('delivered_json_schema', passed, passed ? 'delivered JSON fits the schema' : failures.map((f) => f.error).join('; '));
+        });
+    },
+    json() {
+        return codeGrader('delivered_json', 'delivered.json', true, (trial) => {
+            const expected = trial.case?.expect?.json;
+            if (!expected) {
+                return passFail('delivered_json', false, 'case has no expect.json');
+            }
+            const actual = deliveredJson(trial);
+            if (!isRecord(actual)) {
+                return passFail('delivered_json', false, 'delivered output is not a JSON object');
+            }
+            const wrong = Object.entries(expected).filter(([key, want]) => !sameJson(actual[key], want));
+            const passed = wrong.length === 0;
+            return passFail('delivered_json', passed, passed
+                ? 'delivered JSON carries every expected field'
+                : `fields differ: ${wrong.map(([key, want]) => `${key} expected ${JSON.stringify(want)} got ${JSON.stringify(actual[key])}`).join('; ')}`);
+        });
+    },
+};
+function inOrder(expected, actual) {
+    let at = 0;
+    for (const name of actual) {
+        if (name === expected[at])
+            at += 1;
+    }
+    return at === expected.length;
+}
+function anyOrder(expected, actual) {
+    return sameJson([...expected].sort(), [...actual].sort());
+}
+function trajectoryMatches(mode, expected, actual) {
+    switch (mode) {
+        case 'exact':
+            return sameJson(expected, actual);
+        case 'in_order':
+            return inOrder(expected, actual);
+        case 'any_order':
+            return anyOrder(expected, actual);
+        case 'subset':
+            return actual.every((name) => expected.includes(name));
+        default:
+            return false;
+    }
+}
+/** Grades the tools a turn called against the expected ones, by order or by set. */
+function toolTrajectory(options) {
+    const identity = `toolTrajectory:${options.mode}:${options.expect ? JSON.stringify(options.expect) : 'case'}`;
+    return codeGrader('tool_trajectory', identity, options.expect === undefined, (trial) => {
+        const expected = options.expect ?? trial.case?.expect?.tools;
+        if (!expected) {
+            return passFail('tool_trajectory', false, 'case has no expect.tools');
+        }
+        const actual = trial
+            .spans('execute_tool')
+            .flatMap((span) => stringAttribute(span, 'gen_ai.tool.name') ?? []);
+        const passed = trajectoryMatches(options.mode, expected, actual);
+        return passFail('tool_trajectory', passed, `called ${listOf(actual)}; expected ${listOf(expected)} (${options.mode})`);
+    });
+}
+/** Grades that the turn stopped with one of the given kinds. */
+function stopKind(kind) {
+    const kinds = Array.isArray(kind) ? kind : [kind];
+    return codeGrader('stop_kind', `stopKind:${kinds.join('|')}`, false, (trial) => {
+        const actual = stringAttribute(trial.root, 'theorem.stop.kind');
+        const passed = actual !== undefined && kinds.includes(actual);
+        return passFail('stop_kind', passed, `stopped ${actual ?? 'without a stop'}; expected ${listOf(kinds)}`);
+    });
+}
+/** A guardrail did (`fired: true`) or did not (`fired: false`) act on any span; `allow` does not count as acting. */
+function guardrail(options) {
+    const identity = `guardrail:${options.fired}:${options.action ?? 'any'}`;
+    return codeGrader('guardrail', identity, false, (trial) => {
+        const actions = trial.records
+            .flatMap((record) => record.spans)
+            .flatMap((span) => span.events)
+            .filter((event) => event.name === 'theorem.guardrail')
+            .flatMap((event) => typeof event.attributes.action === 'string' ? [event.attributes.action] : [])
+            .filter((action) => (options.action ? action === options.action : action !== 'allow'));
+        const fired = actions.length > 0;
+        const passed = fired === options.fired;
+        return passFail('guardrail', passed, fired ? `guardrail acted: ${listOf(actions)}` : 'no guardrail acted');
+    });
+}
+const BUDGET_KEYS = [
+    'maxCostUsd',
+    'maxTokens',
+    'maxSteps',
+    'maxDurationMs',
+    'maxTimeToFirstChunkMs',
+];
+function budgetReadings(trial) {
+    const usage = trial.usage();
+    const firstChunks = modelCalls(trial).flatMap((span) => numberAttribute(span, 'gen_ai.response.time_to_first_chunk') ?? []);
+    return {
+        maxCostUsd: usage.costUsd,
+        maxTokens: usage.tokens.total,
+        maxSteps: numberAttribute(trial.root, 'theorem.steps'),
+        maxDurationMs: spanDurationMs(trial.top),
+        maxTimeToFirstChunkMs: firstChunks.length > 0 ? Math.max(...firstChunks) * MS_PER_S : undefined,
+    };
+}
+/** The trial stayed within every ceiling given; a ceiling with no reading in the trace fails, since it cannot be shown. */
+function budget(options) {
+    return codeGrader('budget', `budget:${JSON.stringify(options)}`, false, (trial) => {
+        const readings = budgetReadings(trial);
+        const over = [];
+        for (const key of BUDGET_KEYS) {
+            const limit = options[key];
+            if (limit === undefined)
+                continue;
+            const reading = readings[key];
+            if (reading === undefined)
+                over.push(`${key}: not recorded`);
+            else if (reading > limit)
+                over.push(`${key}: ${reading} over ${limit}`);
+        }
+        const passed = over.length === 0;
+        return passFail('budget', passed, passed ? 'within budget' : over.join('; '));
+    });
+}
+/** A grader that passes when `check` accepts the trial. */
+function outcome(name, check) {
+    return codeGrader(name, `outcome:${name}`, false, async (trial) => {
+        const verdict = await check(trial);
+        if (typeof verdict === 'boolean') {
+            return passFail(name, verdict, verdict ? 'outcome check passed' : 'outcome check failed');
+        }
+        return verdict;
+    });
+}
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory };

package/esm/src/evals/graders/judge.d.ts

@@ -0,0 +1,24 @@
+import { type EvalRubric } from '../rubrics/types.js';
+import type { EvalGrader, Trial } from '../types.js';
+/** The id of the structured schema a text judge answers with, registered when this module loads. */
+declare const EVAL_JUDGMENT = "evalJudgment";
+/** A judge's answer: one of the rubric's labels, or `unknown`, and why. */
+interface Judgment {
+    label: string;
+    explanation: string;
+}
+/** Options for `judge`: the rubric and, optionally, a name, the labels that pass, a judge profile, the rubric's variables, the cost of a wrong pass and an escalation profile. */
+interface JudgeOptions {
+    rubric: EvalRubric;
+    name?: string;
+    pass?: readonly string[];
+    profile?: string;
+    variables?: (trial: Trial) => Record<string, string>;
+    /** Decision judges only: how many times worse a wrong pass is than a wrong fail; default 1. */
+    wrongPassCost?: number;
+    escalate?: string;
+}
+/** A grader in which a model judges the trial against a rubric: a text profile answers directly, a decision profile answers the rubric's question, and an `escalate` text profile takes trials with media a decision judge cannot see. A turn that failed or delivered nothing is not judged; a misconfiguration throws. */
+declare function judge(options: JudgeOptions): EvalGrader;
+export type { JudgeOptions, Judgment };
+export { EVAL_JUDGMENT, judge };