@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/cli/matrix/synthesizer.js

@@ -0,0 +1,103 @@
+import { profileInputs } from '../../kernel/registry/catalog.js';
+import { FIXTURE_PNG_BASE64, FIXTURE_WAV_BASE64, getFixtureForMime } from './fixtures.js';
+export function synthesizeLiteCombo(profile) {
+    const model = profile.allowModelSelect && profile.models.fast ? 'fast' : undefined;
+    return {
+        profile: profile.id,
+        model,
+        input: {
+            text: `Ping test for profile ${profile.id}. Respond concisely with confirmation.`,
+        },
+    };
+}
+function resolveStressModel(profile) {
+    if (!profile.allowModelSelect) {
+        return undefined;
+    }
+    if (profile.models.smart) {
+        return 'smart';
+    }
+    const ids = Object.keys(profile.models);
+    return ids.length > 1 ? ids[ids.length - 1] : undefined;
+}
+function resolveStressAttachments(profile) {
+    const inputs = profileInputs(profile);
+    if (!inputs) {
+        return [];
+    }
+    const attachments = [];
+    const accept = inputs.attachments?.accept;
+    if (accept && accept.length > 0) {
+        const preferredMimes = ['image/png', 'application/pdf', 'text/plain'];
+        const chosenMime = preferredMimes.find((m) => accept.includes(m)) ?? accept[0];
+        const fixture = getFixtureForMime(chosenMime);
+        if (fixture) {
+            attachments.push(fixture);
+        }
+        else {
+            attachments.push({ mimeType: 'image/png', data: FIXTURE_PNG_BASE64 });
+        }
+    }
+    return attachments;
+}
+function resolveStressVoice(profile) {
+    const inputs = profileInputs(profile);
+    if (!inputs) {
+        return [];
+    }
+    const voice = [];
+    if (inputs.voice?.accept && inputs.voice.accept.length > 0) {
+        voice.push({ mimeType: 'audio/wav', data: FIXTURE_WAV_BASE64 });
+    }
+    return voice;
+}
+export function synthesizeStressCombo(profile) {
+    const model = resolveStressModel(profile);
+    const attachments = resolveStressAttachments(profile);
+    const voice = resolveStressVoice(profile);
+    const promptText = `Execute comprehensive test turn for profile ${profile.id}. Validate all instructions and produce required outputs.`;
+    return {
+        profile: profile.id,
+        model,
+        input: {
+            text: promptText,
+            attachments: attachments.length > 0 ? attachments : undefined,
+            voice: voice.length > 0 ? voice : undefined,
+        },
+    };
+}
+export function synthesizeMatrixCombos(profile) {
+    return [
+        {
+            name: 'Lite (connectivity)',
+            req: synthesizeLiteCombo(profile),
+        },
+        {
+            name: 'Stress (all modalities + primary tools)',
+            req: synthesizeStressCombo(profile),
+        },
+    ];
+}
+function assertGroundingFlagsOnModel(profile, options) {
+    const modelId = options.mode && profile.models[options.mode] ? options.mode : profile.defaultModel;
+    const builtins = new Set(profile.models[modelId]?.builtInTools ?? []);
+    if (options.search === true && !builtins.has('googleSearch')) {
+        throw new Error(`--search requires googleSearch on models.${modelId}.builtInTools`);
+    }
+    if (options.map === true && !builtins.has('googleMaps')) {
+        throw new Error(`--map requires googleMaps on models.${modelId}.builtInTools`);
+    }
+}
+export function buildCustomTurnRequest(profile, options) {
+    if (options.lite) {
+        return synthesizeLiteCombo(profile);
+    }
+    const base = synthesizeStressCombo(profile);
+    if (options.mode) {
+        base.model = options.mode;
+    }
+    if (options.search !== undefined || options.map !== undefined) {
+        assertGroundingFlagsOnModel(profile, options);
+    }
+    return base;
+}

package/esm/src/evals/attachments.d.ts

@@ -0,0 +1,4 @@
+import type { EvalAttachment, EvalCase } from './types.js';
+declare function attachmentData(attachment: EvalAttachment): Promise<string>;
+declare function pinCaseFiles(cases: EvalCase[], casesPath: string): Promise<EvalCase[]>;
+export { attachmentData, pinCaseFiles };

package/esm/src/evals/attachments.js

@@ -0,0 +1,53 @@
+// A file attachment is read when its trial runs, not when the suite loads, so thousands of photos are never held at once.
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { TheoremError } from '../guardrails/error.js';
+async function hashedFile(path) {
+    let bytes;
+    try {
+        bytes = await readFile(path);
+    }
+    catch (error) {
+        throw new TheoremError('config', `attachment ${path} cannot be read: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return { bytes, sha256: createHash('sha256').update(bytes).digest('hex') };
+}
+async function fileBytes(attachment) {
+    const { bytes, sha256 } = await hashedFile(attachment.path);
+    if (sha256 !== attachment.sha256) {
+        throw new TheoremError('config', `attachment ${attachment.path} hashes to ${sha256}, not the ${attachment.sha256} its case pins`);
+    }
+    return bytes;
+}
+async function attachmentData(attachment) {
+    return 'data' in attachment ? attachment.data : (await fileBytes(attachment)).toString('base64');
+}
+async function pinCaseFiles(cases, casesPath) {
+    const pinned = [];
+    for (const evalCase of cases) {
+        const { input } = evalCase;
+        if (!('attachments' in input) || input.attachments === undefined) {
+            pinned.push(evalCase);
+            continue;
+        }
+        const attachments = [];
+        for (const attachment of input.attachments) {
+            if ('data' in attachment) {
+                attachments.push(attachment);
+                continue;
+            }
+            const path = resolve(dirname(casesPath), attachment.path);
+            try {
+                await fileBytes({ ...attachment, path });
+            }
+            catch (error) {
+                throw new TheoremError('config', `case ${evalCase.id}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            attachments.push({ ...attachment, path });
+        }
+        pinned.push({ ...evalCase, input: { ...input, attachments } });
+    }
+    return pinned;
+}
+export { attachmentData, pinCaseFiles };

package/esm/src/evals/breakdown.d.ts

@@ -0,0 +1,28 @@
+import type { SuiteRun } from './run.js';
+/** Nearest-rank percentiles, so each is a value some trial had. */
+interface Spread {
+    median: number;
+    p90: number;
+}
+interface GroupSummary {
+    group: string;
+    cases: number;
+    casesPassed: number;
+    casesUndecided: number;
+    trials: number;
+    trialsPassed: number;
+    trialsErrored: number;
+    answers?: {
+        accepted: number;
+        partial: number;
+        wrong: number;
+        none: number;
+    };
+    stops: Record<string, number>;
+    durationMs?: Spread;
+    modelCalls?: Spread;
+    toolCalls?: Spread;
+}
+declare function groupSummaries(run: Pick<SuiteRun, 'trials' | 'verdicts'>): GroupSummary[];
+export type { GroupSummary, Spread };
+export { groupSummaries };

package/esm/src/evals/breakdown.js

@@ -0,0 +1,68 @@
+const MEDIAN = 0.5;
+const P90 = 0.9;
+function rank(sorted, quantile) {
+    return sorted[Math.max(0, Math.ceil(quantile * sorted.length) - 1)] ?? 0;
+}
+function spread(values) {
+    if (values.length === 0)
+        return undefined;
+    const sorted = values.toSorted((a, b) => a - b);
+    return { median: rank(sorted, MEDIAN), p90: rank(sorted, P90) };
+}
+function answerLabel(report) {
+    return report.results.find((result) => result.name === 'answer')?.score?.label;
+}
+function groupOf(group, verdicts, trials, gradesAnswers) {
+    const turns = trials.flatMap((report) => report.turn ?? []);
+    const of = (key) => spread(turns.map((turn) => turn[key]));
+    const labels = trials.map(answerLabel);
+    const count = (label) => labels.filter((each) => each === label).length;
+    const stops = {};
+    for (const turn of turns) {
+        const stop = turn.stop ?? 'unrecorded';
+        stops[stop] = (stops[stop] ?? 0) + 1;
+    }
+    const durationMs = of('durationMs');
+    const modelCalls = of('modelCalls');
+    const toolCalls = of('toolCalls');
+    return {
+        group,
+        cases: verdicts.length,
+        casesPassed: verdicts.filter((verdict) => verdict.passed).length,
+        casesUndecided: verdicts.filter((verdict) => !verdict.decided).length,
+        trials: trials.length,
+        trialsPassed: trials.filter((report) => report.outcome === 'passed').length,
+        trialsErrored: trials.filter((report) => report.outcome === 'errored').length,
+        ...(gradesAnswers
+            ? {
+                answers: {
+                    accepted: count('accepted'),
+                    partial: count('partial'),
+                    wrong: count('wrong'),
+                    none: count(undefined),
+                },
+            }
+            : {}),
+        stops,
+        ...(durationMs ? { durationMs } : {}),
+        ...(modelCalls ? { modelCalls } : {}),
+        ...(toolCalls ? { toolCalls } : {}),
+    };
+}
+function groupSummaries(run) {
+    const gradesAnswers = run.trials.some((report) => report.results.some((result) => result.name === 'answer'));
+    const tags = [...new Set(run.trials.flatMap((report) => report.case?.tags ?? []))].sort();
+    const tagged = (tag) => run.trials.filter((report) => report.case?.tags?.includes(tag) === true);
+    const verdictsOf = (trials) => {
+        const ids = new Set(trials.map((report) => report.case?.id));
+        return run.verdicts.filter((verdict) => ids.has(verdict.case));
+    };
+    return [
+        groupOf('all', run.verdicts, run.trials, gradesAnswers),
+        ...tags.map((tag) => {
+            const trials = tagged(tag);
+            return groupOf(tag, verdictsOf(trials), trials, gradesAnswers);
+        }),
+    ];
+}
+export { groupSummaries };

package/esm/src/evals/graders/answer.d.ts

@@ -0,0 +1,13 @@
+import type { EvalGrader } from '../types.js';
+type AnswerSource = {
+    json: string;
+} | {
+    tool: string;
+    arg: string;
+};
+type AnswerLabel = 'accepted' | 'partial' | 'wrong';
+declare function answer(options?: {
+    from?: AnswerSource;
+}): EvalGrader;
+export type { AnswerLabel, AnswerSource };
+export { answer };

package/esm/src/evals/graders/answer.js

@@ -0,0 +1,133 @@
+import { isRecord } from '../../kernel/util/record.js';
+import { codeGrader, deliveredJson, deliveredText, listOf } from './shared.js';
+import { toolSteps } from './transcript.js';
+function normalized(text) {
+    return text
+        .normalize('NFKD')
+        .replace(/\p{M}/gu, '')
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}]+/gu, ' ')
+        .trim();
+}
+/** A name said only inside a longer one of them is not said: `Calathea` in `Calathea orbifolia` is the species. */
+function namedIn(text, names) {
+    const words = normalized(text).split(' ');
+    const spans = names.flatMap((name) => {
+        const key = normalized(name);
+        if (!key)
+            return [];
+        const parts = key.split(' ');
+        return words.flatMap((_, start) => parts.every((part, i) => words[start + i] === part)
+            ? [{ name, start, end: start + parts.length }]
+            : []);
+    });
+    const said = new Set(spans
+        .filter((span) => !spans.some((other) => other.start <= span.start &&
+        other.end >= span.end &&
+        other.end - other.start > span.end - span.start))
+        .map((span) => span.name));
+    return (among) => among.find((name) => said.has(name));
+}
+function at(value, path) {
+    let current = value;
+    for (const key of path.split('.')) {
+        if (!isRecord(current))
+            return undefined;
+        current = current[key];
+    }
+    return current;
+}
+function parsedArguments(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readAnswer(trial, source) {
+    if ('json' in source) {
+        const where = `JSON key ${source.json}`;
+        const value = at(deliveredJson(trial), source.json);
+        return typeof value === 'string' ? { value, where } : { missing: `no string at ${where}` };
+    }
+    const where = `${source.tool} argument ${source.arg}`;
+    const call = toolSteps(trial)
+        .filter((step) => step.name === source.tool)
+        .at(-1);
+    if (!call)
+        return { missing: `${source.tool} was not called` };
+    const value = at(parsedArguments(call.arguments), source.arg);
+    return typeof value === 'string' ? { value, where } : { missing: `no string at ${where}` };
+}
+function result(label, explanation) {
+    const passed = label === 'accepted';
+    return {
+        name: 'answer',
+        source: 'code',
+        score: { value: passed ? 1 : 0, label },
+        explanation,
+        passed,
+    };
+}
+/** A turn the provider failed before it answered: the agent never had its say, so the trial errors instead of counting as wrong. */
+function unanswered(trial, why) {
+    if (trial.root.attributes['theorem.stop.kind'] !== 'provider_error')
+        return undefined;
+    const kind = trial.root.status.message ? ` (${trial.root.status.message})` : '';
+    return {
+        name: 'answer',
+        source: 'code',
+        errorType: 'provider_error',
+        explanation: `${why}; the turn stopped provider_error${kind} before it answered`,
+    };
+}
+function answer(options = {}) {
+    const { from } = options;
+    const identity = `answer:${from ? JSON.stringify(from) : 'reply'}`;
+    return codeGrader('answer', identity, true, (trial) => {
+        const expected = trial.case?.expect?.answer;
+        if (!expected)
+            return result('wrong', 'case has no expect.answer');
+        const accepted = expected.accepted;
+        const partial = expected.partial ?? [];
+        const rejected = expected.rejected ?? [];
+        if (from) {
+            const read = readAnswer(trial, from);
+            if ('missing' in read) {
+                return (unanswered(trial, read.missing) ??
+                    result('wrong', `${read.missing}; accepted ${listOf(accepted)}`));
+            }
+            const said = normalized(read.value);
+            const quoted = JSON.stringify(read.value);
+            if (accepted.some((name) => normalized(name) === said)) {
+                return result('accepted', `answered ${quoted} (${read.where})`);
+            }
+            if (rejected.some((name) => normalized(name) === said)) {
+                return result('wrong', `answered ${quoted} (${read.where}), which the case rejects; accepted ${listOf(accepted)}`);
+            }
+            if (partial.some((name) => normalized(name) === said)) {
+                return result('partial', `answered ${quoted} (${read.where}); accepted ${listOf(accepted)}`);
+            }
+            return result('wrong', `answered ${quoted} (${read.where}); accepted ${listOf(accepted)}`);
+        }
+        const found = namedIn(deliveredText(trial, undefined), [...accepted, ...partial, ...rejected]);
+        const hit = found(accepted);
+        const miss = found(rejected);
+        if (hit && miss) {
+            return result('partial', `reply names ${JSON.stringify(hit)} and ${JSON.stringify(miss)}, which the case rejects`);
+        }
+        if (hit)
+            return result('accepted', `reply names ${JSON.stringify(hit)}`);
+        if (miss) {
+            return result('wrong', `reply names ${JSON.stringify(miss)}, which the case rejects; accepted ${listOf(accepted)}`);
+        }
+        const near = found(partial);
+        if (near) {
+            return result('partial', `reply names ${JSON.stringify(near)}; accepted ${listOf(accepted)}`);
+        }
+        return (unanswered(trial, 'the reply names no answer') ??
+            result('wrong', `reply names none of ${listOf(accepted)}`));
+    });
+}
+export { answer };

package/esm/src/evals/graders/code.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Code graders: deterministic checks over a trial's trace. Each returns one
+ * pass/fail result with the reason in plain words. Outcome first: what was
+ * delivered and what it cost matter more than the path taken.
+ *
+ * @module
+ */
+import type { EvalGrader, EvalResult, Trial } from '../types.js';
+/** Graders over what the host received: the model's own text and JSON. */
+interface DeliveredGraders {
+    /** The delivered text contains `text`. */
+    includes: (text: string) => EvalGrader;
+    /** The delivered text matches `pattern` (a JavaScript regular expression source). */
+    regex: (pattern: string, flags?: string) => EvalGrader;
+    /** The delivered text equals `text` exactly. */
+    equals: (text: string) => EvalGrader;
+    /**
+     * The delivered JSON validates against a JSON Schema object (required
+     * fields present, recursively). The structured part is read when the
+     * profile has one; otherwise the text is parsed.
+     */
+    jsonSchema: (schema: Record<string, unknown>) => EvalGrader;
+    /** Every field of the case's `expect.json` is present in the delivered JSON with the same value. */
+    json: () => EvalGrader;
+}
+declare const delivered: DeliveredGraders;
+/**
+ * How the called tools are compared with the expected list:
+ * - `exact`: the same tools in the same order, nothing more;
+ * - `in_order`: the expected tools appear in that order, others may interleave;
+ * - `any_order`: the same tools, order ignored;
+ * - `subset`: every tool called is one that was expected (none unexpected; some may be missing).
+ */
+type TrajectoryMode = 'exact' | 'in_order' | 'any_order' | 'subset';
+/** The tools the agent called, in start order, against `expect` or the case's `expect.tools`. */
+declare function toolTrajectory(options: {
+    expect?: string[];
+    mode: TrajectoryMode;
+}): EvalGrader;
+/** The turn stopped for one of these reasons (`theorem.stop.kind`). */
+declare function stopKind(kind: string | string[]): EvalGrader;
+/** A guardrail did (`fired: true`) or did not (`fired: false`) act on any span; `allow` does not count as acting. */
+declare function guardrail(options: {
+    fired: boolean;
+    action?: 'redact' | 'flag' | 'block';
+}): EvalGrader;
+/** Ceilings the trial must stay under; every one that is crossed is named. */
+interface BudgetOptions {
+    maxCostUsd?: number;
+    maxTokens?: number;
+    maxSteps?: number;
+    maxDurationMs?: number;
+    /** The slowest model call's time to first chunk. */
+    maxTimeToFirstChunkMs?: number;
+}
+/** The trial stayed within every ceiling given; a ceiling with no reading in the trace fails, since it cannot be shown. */
+declare function budget(options: BudgetOptions): EvalGrader;
+/**
+ * The host's own check of the end state (a booking exists, a row changed):
+ * "host decides". Return a boolean, or a full result to score and explain.
+ */
+declare function outcome(name: string, check: (trial: Trial) => boolean | EvalResult | Promise<boolean | EvalResult>): EvalGrader;
+export type { BudgetOptions, DeliveredGraders, TrajectoryMode };
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory };

package/esm/src/evals/graders/code.js

@@ -0,0 +1,205 @@
+/**
+ * Code graders: deterministic checks over a trial's trace. Each returns one
+ * pass/fail result with the reason in plain words. Outcome first: what was
+ * delivered and what it cost matter more than the path taken.
+ *
+ * @module
+ */
+import { collectValidationFailures } from '../../kernel/engine/runner/schema-validation.js';
+import { isRecord } from '../../kernel/util/record.js';
+import { codeGrader, deliveredJson, deliveredText, listOf, modelCalls, passFail, spanDurationMs, } from './shared.js';
+const MS_PER_S = 1000;
+function stringAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'string' ? value : undefined;
+}
+function numberAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'number' ? value : undefined;
+}
+/** Deep structural equality over JSON values. */
+function sameJson(left, right) {
+    return JSON.stringify(left) === JSON.stringify(right);
+}
+function quote(text) {
+    return JSON.stringify(text);
+}
+const delivered = {
+    /** The delivered text contains `text`. */
+    includes(text) {
+        return codeGrader('delivered_includes', `delivered.includes:${text}`, false, (trial) => {
+            const actual = deliveredText(trial, undefined);
+            const passed = actual.includes(text);
+            return passFail('delivered_includes', passed, passed ? `delivered text includes ${quote(text)}` : `delivered text lacks ${quote(text)}`);
+        });
+    },
+    /** The delivered text matches `pattern` (a JavaScript regular expression source). */
+    regex(pattern, flags = '') {
+        const re = new RegExp(pattern, flags);
+        return codeGrader('delivered_regex', `delivered.regex:/${pattern}/${flags}`, false, (trial) => {
+            const passed = re.test(deliveredText(trial, undefined));
+            return passFail('delivered_regex', passed, passed
+                ? `delivered text matches /${pattern}/${flags}`
+                : `delivered text does not match /${pattern}/${flags}`);
+        });
+    },
+    /** The delivered text equals `text` exactly. */
+    equals(text) {
+        return codeGrader('delivered_equals', `delivered.equals:${text}`, false, (trial) => {
+            const actual = deliveredText(trial, undefined);
+            const passed = actual === text;
+            return passFail('delivered_equals', passed, passed ? 'delivered text is exactly as expected' : `delivered text was ${quote(actual)}`);
+        });
+    },
+    /**
+     * The delivered JSON validates against a JSON Schema object (required
+     * fields present, recursively). The structured part is read when the
+     * profile has one; otherwise the text is parsed.
+     */
+    jsonSchema(schema) {
+        return codeGrader('delivered_json_schema', `delivered.jsonSchema:${JSON.stringify(schema)}`, false, async (trial) => {
+            const value = deliveredJson(trial);
+            if (value === undefined) {
+                return passFail('delivered_json_schema', false, 'delivered output is not JSON');
+            }
+            const failures = await collectValidationFailures(schema, value, undefined);
+            const passed = failures.length === 0;
+            return passFail('delivered_json_schema', passed, passed ? 'delivered JSON fits the schema' : failures.map((f) => f.error).join('; '));
+        });
+    },
+    /** Every field of the case's `expect.json` is present in the delivered JSON with the same value. */
+    json() {
+        return codeGrader('delivered_json', 'delivered.json', true, (trial) => {
+            const expected = trial.case?.expect?.json;
+            if (!expected) {
+                return passFail('delivered_json', false, 'case has no expect.json');
+            }
+            const actual = deliveredJson(trial);
+            if (!isRecord(actual)) {
+                return passFail('delivered_json', false, 'delivered output is not a JSON object');
+            }
+            const wrong = Object.entries(expected).filter(([key, want]) => !sameJson(actual[key], want));
+            const passed = wrong.length === 0;
+            return passFail('delivered_json', passed, passed
+                ? 'delivered JSON carries every expected field'
+                : `fields differ: ${wrong.map(([key, want]) => `${key} expected ${JSON.stringify(want)} got ${JSON.stringify(actual[key])}`).join('; ')}`);
+        });
+    },
+};
+function inOrder(expected, actual) {
+    let at = 0;
+    for (const name of actual) {
+        if (name === expected[at])
+            at += 1;
+    }
+    return at === expected.length;
+}
+function anyOrder(expected, actual) {
+    return sameJson([...expected].sort(), [...actual].sort());
+}
+function trajectoryMatches(mode, expected, actual) {
+    switch (mode) {
+        case 'exact':
+            return sameJson(expected, actual);
+        case 'in_order':
+            return inOrder(expected, actual);
+        case 'any_order':
+            return anyOrder(expected, actual);
+        case 'subset':
+            return actual.every((name) => expected.includes(name));
+        default:
+            return false;
+    }
+}
+/** The tools the agent called, in start order, against `expect` or the case's `expect.tools`. */
+function toolTrajectory(options) {
+    const identity = `toolTrajectory:${options.mode}:${options.expect ? JSON.stringify(options.expect) : 'case'}`;
+    return codeGrader('tool_trajectory', identity, options.expect === undefined, (trial) => {
+        const expected = options.expect ?? trial.case?.expect?.tools;
+        if (!expected) {
+            return passFail('tool_trajectory', false, 'case has no expect.tools');
+        }
+        const actual = trial
+            .spans('execute_tool')
+            .flatMap((span) => stringAttribute(span, 'gen_ai.tool.name') ?? []);
+        const passed = trajectoryMatches(options.mode, expected, actual);
+        return passFail('tool_trajectory', passed, `called ${listOf(actual)}; expected ${listOf(expected)} (${options.mode})`);
+    });
+}
+// ── stop, guardrail, budget ─────────────────────────
+/** The turn stopped for one of these reasons (`theorem.stop.kind`). */
+function stopKind(kind) {
+    const kinds = Array.isArray(kind) ? kind : [kind];
+    return codeGrader('stop_kind', `stopKind:${kinds.join('|')}`, false, (trial) => {
+        const actual = stringAttribute(trial.root, 'theorem.stop.kind');
+        const passed = actual !== undefined && kinds.includes(actual);
+        return passFail('stop_kind', passed, `stopped ${actual ?? 'without a stop'}; expected ${listOf(kinds)}`);
+    });
+}
+/** A guardrail did (`fired: true`) or did not (`fired: false`) act on any span; `allow` does not count as acting. */
+function guardrail(options) {
+    const identity = `guardrail:${options.fired}:${options.action ?? 'any'}`;
+    return codeGrader('guardrail', identity, false, (trial) => {
+        const actions = trial.records
+            .flatMap((record) => record.spans)
+            .flatMap((span) => span.events)
+            .filter((event) => event.name === 'theorem.guardrail')
+            .flatMap((event) => typeof event.attributes.action === 'string' ? [event.attributes.action] : [])
+            .filter((action) => (options.action ? action === options.action : action !== 'allow'));
+        const fired = actions.length > 0;
+        const passed = fired === options.fired;
+        return passFail('guardrail', passed, fired ? `guardrail acted: ${listOf(actions)}` : 'no guardrail acted');
+    });
+}
+const BUDGET_KEYS = [
+    'maxCostUsd',
+    'maxTokens',
+    'maxSteps',
+    'maxDurationMs',
+    'maxTimeToFirstChunkMs',
+];
+function budgetReadings(trial) {
+    const usage = trial.usage();
+    const firstChunks = modelCalls(trial).flatMap((span) => numberAttribute(span, 'gen_ai.response.time_to_first_chunk') ?? []);
+    return {
+        maxCostUsd: usage.costUsd,
+        maxTokens: usage.tokens.total,
+        maxSteps: numberAttribute(trial.root, 'theorem.steps'),
+        maxDurationMs: spanDurationMs(trial.top),
+        maxTimeToFirstChunkMs: firstChunks.length > 0 ? Math.max(...firstChunks) * MS_PER_S : undefined,
+    };
+}
+/** The trial stayed within every ceiling given; a ceiling with no reading in the trace fails, since it cannot be shown. */
+function budget(options) {
+    return codeGrader('budget', `budget:${JSON.stringify(options)}`, false, (trial) => {
+        const readings = budgetReadings(trial);
+        const over = [];
+        for (const key of BUDGET_KEYS) {
+            const limit = options[key];
+            if (limit === undefined)
+                continue;
+            const reading = readings[key];
+            if (reading === undefined)
+                over.push(`${key}: not recorded`);
+            else if (reading > limit)
+                over.push(`${key}: ${reading} over ${limit}`);
+        }
+        const passed = over.length === 0;
+        return passFail('budget', passed, passed ? 'within budget' : over.join('; '));
+    });
+}
+// ── outcome ─────────────────────────────────────────
+/**
+ * The host's own check of the end state (a booking exists, a row changed):
+ * "host decides". Return a boolean, or a full result to score and explain.
+ */
+function outcome(name, check) {
+    return codeGrader(name, `outcome:${name}`, false, async (trial) => {
+        const verdict = await check(trial);
+        if (typeof verdict === 'boolean') {
+            return passFail(name, verdict, verdict ? 'outcome check passed' : 'outcome check failed');
+        }
+        return verdict;
+    });
+}
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory };