@theoremjs/agents 0.0.0-stage → 0.3.1

package/esm/src/evals/graders/judge.js

@@ -0,0 +1,367 @@
+import { errorKind, TheoremError } from '../../guardrails/error.js';
+import { getProfile, registerStructured, runDecision, runTurn, } from '../../kernel/default-scope.js';
+import { mimeAllowed, profileAccept } from '../../kernel/registry/catalog.js';
+import { isRecord } from '../../kernel/util/record.js';
+import { memorySink } from '../../observability/trace.js';
+import { formatTraceparent } from '../../observability/trace-span.js';
+import { fillRubric } from '../rubrics/types.js';
+import { buildTrial } from '../trial.js';
+import { resolveMedia, trialMedia } from './media.js';
+import { deliveredJson, deliveredText } from './shared.js';
+import { rubricView, TRIAL_VARIABLES } from './transcript.js';
+/** The id of the structured schema a text judge answers with, registered when this module loads. */
+const EVAL_JUDGMENT = 'evalJudgment';
+const UNKNOWN = 'unknown';
+const UNKNOWN_CRITERION = 'The record does not show enough to decide.';
+const VERDICT = 'verdict';
+const FAILED_STOPS = new Set([
+    'provider_error',
+    'cancelled',
+    'stream_incomplete',
+    'interrupted',
+]);
+function turnFailure(trial) {
+    const stop = trial.root.attributes['theorem.stop.kind'];
+    if (typeof stop === 'string' && FAILED_STOPS.has(stop))
+        return `the turn stopped with ${stop}`;
+    if (deliveredText(trial, undefined) === '' && deliveredJson(trial) === undefined) {
+        return `the turn delivered nothing${typeof stop === 'string' ? ` (stopped ${stop})` : ''}`;
+    }
+    return undefined;
+}
+const JUDGMENT_SCHEMA = {
+    jsonSchema: {
+        type: 'object',
+        properties: {
+            label: {
+                type: 'string',
+                description: `One of the labels the rubric names, exactly as written, or "${UNKNOWN}" when the record does not let you decide.`,
+            },
+            explanation: {
+                type: 'string',
+                description: 'Why, in a few sentences, citing what the record shows.',
+            },
+        },
+        required: ['label', 'explanation'],
+    },
+};
+// why: Importing this module registers the schema, so a host's judge profile can name it at registration.
+registerStructured(EVAL_JUDGMENT, JUDGMENT_SCHEMA);
+function configError(grader, message) {
+    return new TheoremError('config', `judge ${grader}: ${message}`); // labels-exempt: developer contract error
+}
+function judgeKind(grader, rubric, profileId) {
+    const profile = getProfile(profileId);
+    if (profile.type === 'decision') {
+        if (!rubric.question) {
+            throw configError(grader, `rubric ${rubric.name} has no question for decision profile ${profileId}`);
+        }
+        return 'decision';
+    }
+    if (profile.type !== 'text') {
+        throw configError(grader, `judge profile ${profileId} is neither a text nor a decision profile`);
+    }
+    if (profile.outputs?.structured !== EVAL_JUDGMENT) {
+        throw configError(grader, `judge profile ${profileId} must set outputs.structured to '${EVAL_JUDGMENT}'`);
+    }
+    if (rubric.template === undefined) {
+        throw configError(grader, `rubric ${rubric.name} has no prompt for text profile ${profileId}`);
+    }
+    return 'text';
+}
+function traceparentsOf(records) {
+    const spans = records[0]?.spans ?? [];
+    const ids = new Set(spans.map((span) => span.spanId));
+    const root = spans.find((span) => span.parentSpanId === undefined || !ids.has(span.parentSpanId));
+    return root ? [formatTraceparent(root.traceId, root.spanId)] : [];
+}
+function judgeStamp(trial, grader) {
+    return {
+        eval: {
+            suite: trial.suite,
+            ...(trial.case ? { case: trial.case.id } : {}),
+            trial: trial.index,
+            judge: { grader },
+        },
+    };
+}
+function judgeCall(trial, grader, context) {
+    return {
+        metadata: judgeStamp(trial, grader),
+        ...(context.traceparent ? { traceparent: context.traceparent } : {}),
+        ...(context.signal ? { signal: context.signal } : {}),
+    };
+}
+function parseJudgment(value) {
+    if (!isRecord(value))
+        return undefined;
+    const { label, explanation } = value;
+    if (typeof label !== 'string' || typeof explanation !== 'string')
+        return undefined;
+    return { label, explanation };
+}
+async function askText(args) {
+    const { context } = args;
+    if (!context.judgeProvider) {
+        throw configError(args.grader, `no provider for text judge profile ${args.profile}`);
+    }
+    const records = [];
+    let error;
+    const request = {
+        profile: args.profile,
+        input: {
+            text: args.prompt,
+            ...(args.attachments.length > 0 ? { attachments: args.attachments } : {}),
+        },
+        ...judgeCall(args.trial, args.grader, context),
+    };
+    try {
+        for await (const _event of runTurn(request, context.judgeProvider, memorySink(records))) {
+            // why: The judge's trace is its record; its events are not read.
+        }
+    }
+    catch (thrown) {
+        error = errorKind(thrown);
+    }
+    await context.traced(records);
+    if (records.length === 0)
+        return { traceparents: [], error: error ?? 'internal' };
+    const judged = buildTrial({ suite: args.trial.suite, index: 0, records });
+    const traceparents = [formatTraceparent(judged.root.traceId, judged.root.spanId)];
+    const failure = judged.root.attributes['error.type'];
+    if (error === undefined && typeof failure === 'string')
+        error = failure;
+    if (error !== undefined)
+        return { traceparents, error };
+    const judgment = parseJudgment(deliveredJson(judged));
+    return judgment ? { traceparents, judgment } : { traceparents, error: 'bad_response' };
+}
+function percent(value) {
+    return `${Math.round(value * 100)}%`;
+}
+function choiceJudgment(answer, rule) {
+    if (answer?.type !== 'choice')
+        return undefined;
+    const p = answer.probabilities;
+    const odds = Object.entries(p)
+        .sort((a, b) => b[1] - a[1])
+        .map(([label, value]) => `${label} ${percent(value)}`)
+        .join(', ');
+    const { pass } = rule;
+    if (!pass)
+        return { label: answer.choice, explanation: `Jev chose ${answer.choice} (${odds}).` };
+    const mass = (labels) => labels.reduce((sum, l) => sum + (p[l] ?? 0), 0);
+    const likeliest = (labels) => labels.reduce((best, l) => ((p[l] ?? 0) > (p[best] ?? 0) ? l : best));
+    const failing = rule.labels.filter((label) => !pass.includes(label));
+    const needs = `passing needs ${pass.join(' or ')} above ${percent(rule.line)}`;
+    if (mass(pass) > rule.line) {
+        return { label: likeliest(pass), explanation: `Jev: ${odds}; ${needs}.` };
+    }
+    if (failing.length > 0 && mass(failing) > 0.5) {
+        return { label: likeliest(failing), explanation: `Jev: ${odds}; ${needs}.` };
+    }
+    return { label: UNKNOWN, explanation: `Jev was unsure: ${odds}; ${needs}.` };
+}
+async function askDecision(args) {
+    const { context, rubric } = args;
+    if (!context.judgeDecision) {
+        throw configError(args.grader, `no key for decision judge profile ${args.profile}`);
+    }
+    if (!rubric.question) {
+        throw configError(args.grader, `rubric ${rubric.name} has no question`);
+    }
+    const question = {
+        type: 'choice',
+        instructions: rubric.question.instructions,
+        criteria: { ...rubric.question.criteria, [UNKNOWN]: UNKNOWN_CRITERION },
+    };
+    const records = [];
+    try {
+        const result = await runDecision({
+            profile: args.profile,
+            state: args.state,
+            questions: { [VERDICT]: question },
+            ...judgeCall(args.trial, args.grader, context),
+        }, { ...context.judgeDecision, sink: memorySink(records) });
+        await context.traced(records);
+        const traceparents = traceparentsOf(records);
+        const judgment = choiceJudgment(result.answers[VERDICT], args.rule);
+        return judgment ? { traceparents, judgment } : { traceparents, error: 'bad_response' };
+    }
+    catch (thrown) {
+        await context.traced(records);
+        return { traceparents: traceparentsOf(records), error: errorKind(thrown) };
+    }
+}
+function escalated(decision, text) {
+    const traceparents = [...decision.traceparents, ...text.traceparents];
+    if (!text.judgment)
+        return { traceparents, error: text.error ?? 'internal' };
+    return {
+        traceparents,
+        judgment: {
+            label: text.judgment.label,
+            explanation: `${decision.judgment?.explanation ?? ''} The text judge decided: ${text.judgment.explanation}`,
+        },
+    };
+}
+function stateOf(rubric, view) {
+    const state = {};
+    for (const variable of rubric.variables) {
+        const value = view[variable];
+        if (value === undefined) {
+            throw new TheoremError('config', `rubric ${rubric.name}: no value for ${variable}`); // labels-exempt: developer contract error
+        }
+        state[variable] = value;
+    }
+    return state;
+}
+// why: Models echo the prompt's headings (`CORRECT -`), so a text judge's label is read case-insensitively.
+function declaredLabel(label, allowed) {
+    const wanted = label.trim().toLowerCase();
+    return [...allowed].find((declared) => declared.toLowerCase() === wanted);
+}
+function resultOf(run, rubric, name, pass) {
+    const base = {
+        name,
+        source: 'model',
+        ...(run.traceparents.length > 0 ? { judgeTraceparents: run.traceparents } : {}),
+    };
+    if (!run.judgment) {
+        return {
+            ...base,
+            errorType: run.error ?? 'internal',
+            explanation: `the judge gave no judgment (${run.error ?? 'internal'})`,
+        };
+    }
+    const allowed = new Set([...Object.keys(rubric.labels), UNKNOWN]);
+    const label = declaredLabel(run.judgment.label, allowed);
+    if (label === undefined) {
+        return {
+            ...base,
+            errorType: 'bad_response',
+            explanation: `the judge answered ${JSON.stringify(run.judgment.label)}, not one of ${[...allowed].join(', ')}`,
+        };
+    }
+    const value = rubric.labels[label];
+    return {
+        ...base,
+        score: { ...(value === undefined ? {} : { value }), label },
+        explanation: run.judgment.explanation,
+        ...(pass ? { passed: pass.includes(label) } : {}),
+    };
+}
+/** A grader in which a model judges the trial against a rubric: a text profile answers directly, a decision profile answers the rubric's question, and an `escalate` text profile takes trials with media a decision judge cannot see. A turn that failed or delivered nothing is not judged; a misconfiguration throws. */
+function judge(options) {
+    const { rubric } = options;
+    const name = options.name ?? rubric.name;
+    const pass = options.pass ?? rubric.pass;
+    const unknownPass = (pass ?? []).filter((label) => !(label in rubric.labels));
+    if (unknownPass.length > 0) {
+        throw configError(name, `pass names labels the rubric lacks: ${unknownPass.join(', ')}`);
+    }
+    const unfillable = rubric.variables.filter((variable) => !TRIAL_VARIABLES.includes(variable));
+    if (unfillable.length > 0 && !options.variables) {
+        throw configError(name, `the rubric needs ${unfillable.join(', ')}, which the trace cannot fill; pass variables`);
+    }
+    const wrongPassCost = options.wrongPassCost ?? 1;
+    if (!(Number.isFinite(wrongPassCost) && wrongPassCost > 0)) {
+        throw configError(name, `wrongPassCost must be a positive number, not ${wrongPassCost}`);
+    }
+    if (options.wrongPassCost !== undefined && !pass) {
+        throw configError(name, `wrongPassCost needs labels that pass, and rubric ${rubric.name} names none`);
+    }
+    const rule = {
+        pass,
+        labels: Object.keys(rubric.labels),
+        line: wrongPassCost / (1 + wrongPassCost),
+    };
+    const judgeProfile = (suiteJudge) => {
+        const profile = options.profile ?? suiteJudge;
+        if (!profile)
+            throw configError(name, 'names no judge profile, and neither does the suite');
+        const kind = judgeKind(name, rubric, profile);
+        const decisionOnly = [
+            ...(options.wrongPassCost === undefined ? [] : ['wrongPassCost']),
+            ...(options.escalate === undefined ? [] : ['escalate']),
+        ];
+        if (kind === 'text' && decisionOnly.length > 0) {
+            throw configError(name, `${decisionOnly.join(' and ')} ${decisionOnly.length > 1 ? 'need' : 'needs'} a decision judge, and ${profile} is a text profile`);
+        }
+        if (options.escalate !== undefined && judgeKind(name, rubric, options.escalate) !== 'text') {
+            throw configError(name, `escalate names ${options.escalate}, which is not a text profile`);
+        }
+        return profile;
+    };
+    const identity = [
+        `judge:${name}`,
+        `labels=${Object.entries(rubric.labels)
+            .map(([label, value]) => `${label}:${value}`)
+            .join(',')}`,
+        `pass=${pass ? [...pass].join(',') : 'none'}`,
+        ...(options.profile ? [`profile=${options.profile}`] : []),
+        ...(options.wrongPassCost === undefined ? [] : [`wrongPassCost=${options.wrongPassCost}`]),
+        ...(options.escalate ? [`escalate=${options.escalate}`] : []),
+        `template=${rubric.template ?? ''}`,
+        `question=${rubric.question ? JSON.stringify(rubric.question) : ''}`,
+    ].join('\n');
+    const grade = async (trial, context) => {
+        const notJudged = (why) => ({
+            name,
+            source: 'model',
+            explanation: `${why}; not judged`,
+        });
+        const failure = turnFailure(trial);
+        if (failure)
+            return notJudged(failure);
+        const profile = judgeProfile(context.judge);
+        const view = rubricView(rubric, trial, options.variables?.(trial) ?? {});
+        const read = JSON.stringify(view);
+        const media = trialMedia(trial).filter((item) => read.includes(item.label));
+        const resolved = await resolveMedia(trial, media, context);
+        if ('missing' in resolved) {
+            return notJudged(`the judge could not be shown ${resolved.missing.join(', ')}`);
+        }
+        const { attachments } = resolved;
+        const text = (textProfile) => {
+            const refused = attachments.filter((item) => !mimeAllowed(profileAccept(getProfile(textProfile), 'attachments') ?? [], item.mimeType));
+            if (refused.length > 0) {
+                throw configError(name, `judge profile ${textProfile} must accept ${[...new Set(refused.map((item) => item.mimeType))].join(', ')} to see the media it judges`);
+            }
+            return askText({
+                trial,
+                prompt: fillRubric(rubric, view),
+                profile: textProfile,
+                grader: name,
+                context,
+                attachments,
+            });
+        };
+        if (judgeKind(name, rubric, profile) === 'text')
+            return resultOf(await text(profile), rubric, name, pass);
+        if (attachments.length > 0) {
+            if (!options.escalate) {
+                return notJudged(`decision judge ${profile} cannot see ${media.map((item) => item.label).join(', ')} and no escalate judge can`);
+            }
+            return resultOf(await text(options.escalate), rubric, name, pass);
+        }
+        const run = await askDecision({
+            trial,
+            rubric,
+            rule,
+            state: stateOf(rubric, view),
+            profile,
+            grader: name,
+            context,
+        });
+        if (!options.escalate || run.judgment?.label !== UNKNOWN)
+            return resultOf(run, rubric, name, pass);
+        return resultOf(escalated(run, await text(options.escalate)), rubric, name, pass);
+    };
+    const judgeProfiles = (suiteJudge) => [
+        judgeProfile(suiteJudge),
+        ...(options.escalate === undefined ? [] : [options.escalate]),
+    ];
+    return { name, identity, source: 'model', needsExpect: false, judgeProfiles, grade };
+}
+export { EVAL_JUDGMENT, judge };

package/esm/src/evals/graders/latency.d.ts

@@ -0,0 +1,6 @@
+import type { EvalGrader } from '../types.js';
+/** A grader that passes when each reply began within `maxMs`, measured from the start of the response or, for a live reply, from the end of the user's speech; a reply with no reading fails it. */
+declare function turnLatency(options: {
+    maxMs: number;
+}): EvalGrader;
+export { turnLatency };

package/esm/src/evals/graders/latency.js

@@ -0,0 +1,82 @@
+import { codeGrader, passFail } from './shared.js';
+const MS_PER_S = 1000;
+const NANOS_PER_MS = 1000000n;
+const ACTIVITY_END = 'ACTIVITY_END';
+function msBetween(fromUnixNano, toUnixNano) {
+    return Number((toUnixNano - fromUnixNano) / NANOS_PER_MS);
+}
+function numberAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'number' ? value : undefined;
+}
+// why: The successful (last) HTTP try's start plus its time to first chunk, or the call's own start when the call streamed over a socket (Live).
+function firstChunkAt(trial, call) {
+    const seconds = numberAttribute(call, 'gen_ai.response.time_to_first_chunk');
+    if (seconds === undefined)
+        return undefined;
+    const tries = trial.children(call).filter((span) => 'http.request.method' in span.attributes);
+    const from = tries.at(-1) ?? call;
+    return BigInt(from.startTimeUnixNano) + BigInt(Math.round(seconds * MS_PER_S)) * NANOS_PER_MS;
+}
+// invariant: Speech that prompted an earlier reply never anchors a later one.
+function speechEndedAt(root, afterUnixNano, beforeUnixNano) {
+    let latest;
+    for (const event of root.events) {
+        if (event.name !== 'theorem.session')
+            continue;
+        if (event.attributes.kind !== 'voice_activity' || event.attributes.activity !== ACTIVITY_END) {
+            continue;
+        }
+        const at = BigInt(event.timeUnixNano);
+        if (afterUnixNano !== undefined && at <= afterUnixNano)
+            continue;
+        if (at <= beforeUnixNano && (latest === undefined || at > latest))
+            latest = at;
+    }
+    return latest;
+}
+function reading(span, from, chunk) {
+    if (chunk === undefined)
+        return { span, missing: 'first chunk not recorded' };
+    return { span, ms: msBetween(from, chunk) };
+}
+function latencyReadings(trial) {
+    const responses = trial.spans('generate_content');
+    if (responses.length > 0) {
+        return responses.map((response, index) => {
+            const chunk = firstChunkAt(trial, response);
+            const start = BigInt(response.startTimeUnixNano);
+            const previous = responses[index - 1];
+            const after = previous ? BigInt(previous.startTimeUnixNano) : undefined;
+            const from = chunk === undefined ? start : (speechEndedAt(trial.root, after, chunk) ?? start);
+            return reading(response, from, chunk);
+        });
+    }
+    const [firstCall] = trial.spans('chat');
+    if (!firstCall)
+        return [];
+    return [
+        reading(trial.root, BigInt(trial.root.startTimeUnixNano), firstChunkAt(trial, firstCall)),
+    ];
+}
+/** A grader that passes when each reply began within `maxMs`, measured from the start of the response or, for a live reply, from the end of the user's speech; a reply with no reading fails it. */
+function turnLatency(options) {
+    return codeGrader('turn_latency', `turnLatency:${options.maxMs}`, false, (trial) => {
+        const readings = latencyReadings(trial);
+        if (readings.length === 0)
+            return passFail('turn_latency', false, 'no model call recorded');
+        const over = readings.flatMap((entry, index) => {
+            const name = readings.length === 1 ? 'reply' : `reply ${index + 1}`;
+            if (entry.ms === undefined)
+                return [`${name}: ${entry.missing}`];
+            return entry.ms > options.maxMs ? [`${name}: ${entry.ms} ms over ${options.maxMs}`] : [];
+        });
+        const measured = readings.flatMap((entry) => entry.ms ?? []);
+        const summary = measured.length === 1
+            ? `reply began after ${measured[0]} ms`
+            : `${measured.length} replies began within ${Math.max(...measured)} ms`;
+        const passed = over.length === 0;
+        return passFail('turn_latency', passed, passed ? summary : over.join('; '));
+    });
+}
+export { turnLatency };

package/esm/src/evals/graders/live.d.ts

@@ -0,0 +1,14 @@
+import type { EvalGrader } from '../types.js';
+/** Graders for the transcript of a live model's spoken output. */
+interface TranscriptionGraders {
+    includes: (text?: string) => EvalGrader;
+    regex: (pattern?: string, flags?: string) => EvalGrader;
+}
+/** Graders for a live reply's output transcription: `includes` (a substring) and `regex`, each taking its pattern as an argument or from the case's `expect.transcription`. */
+declare const transcription: TranscriptionGraders;
+/** A grader that passes when the model was interrupted at most `max` times in the trial. */
+declare function interruptions(options: {
+    max: number;
+}): EvalGrader;
+export type { TranscriptionGraders };
+export { interruptions, transcription };

package/esm/src/evals/graders/live.js

@@ -0,0 +1,42 @@
+import { codeGrader, deliveredText, passFail } from './shared.js';
+const OUTPUT_TRANSCRIPTION = 'output_transcription';
+/** Graders for a live reply's output transcription: `includes` (a substring) and `regex`, each taking its pattern as an argument or from the case's `expect.transcription`. */
+const transcription = {
+    includes(text) {
+        const identity = `transcription.includes:${text ?? 'case'}`;
+        return codeGrader('transcription_includes', identity, text === undefined, (trial) => {
+            const want = text ?? trial.case?.expect?.transcription?.includes;
+            if (want === undefined) {
+                return passFail('transcription_includes', false, 'case has no expect.transcription.includes');
+            }
+            const passed = deliveredText(trial, OUTPUT_TRANSCRIPTION).includes(want);
+            return passFail('transcription_includes', passed, passed
+                ? `transcript includes ${JSON.stringify(want)}`
+                : `transcript lacks ${JSON.stringify(want)}`);
+        });
+    },
+    regex(pattern, flags = '') {
+        const identity = `transcription.regex:${pattern === undefined ? 'case' : `/${pattern}/${flags}`}`;
+        return codeGrader('transcription_regex', identity, pattern === undefined, (trial) => {
+            const source = pattern ?? trial.case?.expect?.transcription?.regex;
+            if (source === undefined) {
+                return passFail('transcription_regex', false, 'case has no expect.transcription.regex');
+            }
+            const passed = new RegExp(source, flags).test(deliveredText(trial, OUTPUT_TRANSCRIPTION));
+            return passFail('transcription_regex', passed, passed
+                ? `transcript matches /${source}/${flags}`
+                : `transcript does not match /${source}/${flags}`);
+        });
+    },
+};
+/** A grader that passes when the model was interrupted at most `max` times in the trial. */
+function interruptions(options) {
+    return codeGrader('interruptions', `interruptions:${options.max}`, false, (trial) => {
+        const count = trial
+            .spans('generate_content')
+            .filter((span) => span.attributes['theorem.stop.kind'] === 'interrupted').length;
+        const passed = count <= options.max;
+        return passFail('interruptions', passed, `${count} interruption(s); at most ${options.max} allowed`);
+    });
+}
+export { interruptions, transcription };

package/esm/src/evals/graders/media.d.ts

@@ -0,0 +1,18 @@
+import type { TurnBlob, TurnMediaRef } from '../../kernel/types.js';
+import type { EvalGradeContext, Trial } from '../types.js';
+interface TrialMedia {
+    label: string;
+    mimeType: string;
+    from: 'input' | 'output';
+    sha256?: string;
+    uri?: string;
+}
+declare function trialMedia(trial: Trial): TrialMedia[];
+declare function mediaLabeler(trial: Trial): (part: Record<string, unknown>) => string;
+declare function resolveMedia(trial: Trial, media: readonly TrialMedia[], context: EvalGradeContext): Promise<{
+    attachments: Array<TurnBlob | TurnMediaRef>;
+} | {
+    missing: string[];
+}>;
+export type { TrialMedia };
+export { mediaLabeler, resolveMedia, trialMedia };

package/esm/src/evals/graders/media.js

@@ -0,0 +1,91 @@
+import { sha256Base64 } from '../../kernel/engine/hash.js';
+import { isRecord } from '../../kernel/util/record.js';
+import { attachmentData } from '../attachments.js';
+import { modelCalls } from './shared.js';
+function mediaKey(part) {
+    if (part.type === 'blob' && typeof part.content_sha256 === 'string')
+        return part.content_sha256;
+    if (part.type === 'uri' && typeof part.uri === 'string')
+        return part.uri;
+    return undefined;
+}
+function messageParts(messages) {
+    if (!Array.isArray(messages))
+        return [];
+    return messages.flatMap((message) => isRecord(message) && Array.isArray(message.parts) ? message.parts.filter(isRecord) : []);
+}
+function trialMedia(trial) {
+    const parts = [
+        ...messageParts(trial.content(trial.root.attributes['gen_ai.input.messages'])).map((part) => ({
+            part,
+            from: 'input',
+        })),
+        ...modelCalls(trial).flatMap((span) => messageParts(trial.content(span.attributes['gen_ai.output.messages'])).map((part) => ({
+            part,
+            from: 'output',
+        }))),
+    ];
+    const media = new Map();
+    for (const { part, from } of parts) {
+        const key = mediaKey(part);
+        if (key === undefined || media.has(key))
+            continue;
+        const mimeType = typeof part.mime_type === 'string' ? part.mime_type : 'application/octet-stream';
+        const kind = typeof part.modality === 'string' ? part.modality : 'file';
+        media.set(key, {
+            label: `[${kind} ${media.size + 1}: ${mimeType}]`,
+            mimeType,
+            from,
+            ...(part.type === 'blob' ? { sha256: key } : { uri: key }),
+        });
+    }
+    return [...media.values()];
+}
+function mediaLabeler(trial) {
+    const byKey = new Map(trialMedia(trial).map((media) => [media.sha256 ?? media.uri ?? '', media.label]));
+    return (part) => byKey.get(mediaKey(part) ?? '') ?? `[${String(part.type)}]`;
+}
+async function caseAttachments(trial) {
+    const byHash = new Map();
+    const input = trial.case?.input;
+    const attachments = input && 'attachments' in input ? (input.attachments ?? []) : [];
+    for (const attachment of attachments) {
+        if (!('data' in attachment)) {
+            byHash.set(attachment.sha256, attachment);
+            continue;
+        }
+        const digest = await sha256Base64(attachment.data);
+        if (digest)
+            byHash.set(digest.hash, attachment);
+    }
+    return byHash;
+}
+// why: The host's bytes for a hash, only when they are the bytes the trace hashed.
+async function fromHost(sha256, mimeType, context) {
+    const data = await context.media?.({ sha256, mimeType });
+    if (data === undefined)
+        return undefined;
+    return (await sha256Base64(data))?.hash === sha256 ? data : undefined;
+}
+async function resolveMedia(trial, media, context) {
+    const fromCase = await caseAttachments(trial);
+    const attachments = [];
+    const missing = [];
+    for (const item of media) {
+        if (item.uri !== undefined) {
+            attachments.push({ mimeType: item.mimeType, uri: item.uri });
+            continue;
+        }
+        const sha256 = item.sha256 ?? '';
+        const attachment = fromCase.get(sha256);
+        const data = attachment
+            ? await attachmentData(attachment)
+            : await fromHost(sha256, item.mimeType, context);
+        if (data === undefined)
+            missing.push(item.label);
+        else
+            attachments.push({ mimeType: item.mimeType, data });
+    }
+    return missing.length > 0 ? { missing } : { attachments };
+}
+export { mediaLabeler, resolveMedia, trialMedia };

package/esm/src/evals/graders/shared.d.ts

@@ -0,0 +1,11 @@
+import type { TraceSpan } from '../../observability/trace-span.js';
+import type { EvalGrader, EvalResult, Trial } from '../types.js';
+declare function passFail(name: string, passed: boolean, explanation: string): EvalResult;
+declare function codeGrader(name: string, identity: string, needsExpect: boolean, grade: (trial: Trial) => EvalResult | Promise<EvalResult>): EvalGrader;
+declare function deliveredText(trial: Trial, source: string | undefined): string;
+declare function deliveredJson(trial: Trial): unknown;
+declare function startOf(span: TraceSpan): bigint;
+declare function modelCalls(trial: Trial): TraceSpan[];
+declare function spanDurationMs(span: TraceSpan): number;
+declare function listOf(items: readonly string[]): string;
+export { codeGrader, deliveredJson, deliveredText, listOf, modelCalls, passFail, spanDurationMs, startOf, };