@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/evals/graders/shared.js

@@ -0,0 +1,62 @@
+function passFail(name, passed, explanation) {
+    return {
+        name,
+        source: 'code',
+        score: { value: passed ? 1 : 0, label: passed ? 'pass' : 'fail' },
+        explanation,
+        passed,
+    };
+}
+function codeGrader(name, identity, needsExpect, grade) {
+    return {
+        name,
+        identity,
+        source: 'code',
+        needsExpect,
+        grade,
+    };
+}
+function isObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function deliveredParts(trial) {
+    return trial.delivered().flatMap((message) => {
+        const parts = message.parts;
+        return Array.isArray(parts) ? parts.filter(isObject) : [];
+    });
+}
+// Interim transcripts are skipped; the final one holds the whole.
+function deliveredText(trial, source) {
+    return deliveredParts(trial)
+        .filter((part) => part.type === 'text' &&
+        part['theorem.source'] === source &&
+        part['theorem.interim'] !== true)
+        .map((part) => trial.text(part) ?? '')
+        .join('');
+}
+function deliveredJson(trial) {
+    const structured = deliveredParts(trial).find((part) => part.type === 'structured');
+    const text = structured ? trial.text(structured.content) : deliveredText(trial, undefined);
+    if (text === undefined || text === '')
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+function startOf(span) {
+    return BigInt(span.startTimeUnixNano ?? '0');
+}
+function modelCalls(trial) {
+    return [...trial.spans('chat'), ...trial.spans('generate_content')].toSorted((a, b) => Number(startOf(a) - startOf(b)));
+}
+const NANOS_PER_MS = 1_000_000;
+function spanDurationMs(span) {
+    return Number((BigInt(span.endTimeUnixNano) - BigInt(span.startTimeUnixNano)) / BigInt(NANOS_PER_MS));
+}
+function listOf(items) {
+    return items.length === 0 ? 'none' : items.join(', ');
+}
+export { codeGrader, deliveredJson, deliveredText, listOf, modelCalls, passFail, spanDurationMs, startOf, };

package/esm/src/evals/graders/transcript.d.ts

@@ -0,0 +1,17 @@
+import type { DecisionJson } from '../../kernel/types.js';
+import type { TraceSpan } from '../../observability/trace-span.js';
+import type { EvalRubric } from '../rubrics/types.js';
+import type { Trial } from '../types.js';
+declare const TRIAL_VARIABLES: readonly ["input", "output", "context", "conversation", "user_message", "tool_call", "tool_result", "text"];
+interface ToolStep {
+    span: TraceSpan;
+    name: string;
+    arguments: string;
+    result: string;
+    outcome: string;
+}
+declare function toolSteps(trial: Trial): ToolStep[];
+declare function trialVariables(trial: Trial): Record<(typeof TRIAL_VARIABLES)[number], string>;
+declare function rubricView(rubric: EvalRubric, trial: Trial, host: Readonly<Record<string, string>>): Record<string, DecisionJson>;
+export type { ToolStep };
+export { rubricView, TRIAL_VARIABLES, toolSteps, trialVariables };

package/esm/src/evals/graders/transcript.js

@@ -0,0 +1,200 @@
+import { isRecord } from '../../kernel/util/record.js';
+import { templatePaths } from '../rubrics/mustache.js';
+import { mediaLabeler, trialMedia } from './media.js';
+import { deliveredJson, deliveredText, modelCalls, startOf } from './shared.js';
+const TRIAL_VARIABLES = [
+    'input',
+    'output',
+    'context',
+    'conversation',
+    'user_message',
+    'tool_call',
+    'tool_result',
+    'text',
+];
+function sameAsText(content, texts) {
+    const json = JSON.stringify(content);
+    return texts.some((text) => {
+        try {
+            return JSON.stringify(JSON.parse(text)) === json;
+        }
+        catch {
+            return false;
+        }
+    });
+}
+function partsText(parts, media) {
+    if (!Array.isArray(parts))
+        return '';
+    const texts = parts.flatMap((part) => isRecord(part) && part.type === 'text' && typeof part.content === 'string'
+        ? [part.content]
+        : []);
+    return parts
+        .map((part) => {
+        if (!isRecord(part) || part.type === 'tool_call')
+            return '';
+        if (typeof part.content === 'string')
+            return part.content;
+        if (part.type === 'structured') {
+            return sameAsText(part.content, texts) ? '' : JSON.stringify(part.content);
+        }
+        if (part.type === 'blob' || part.type === 'uri')
+            return media(part);
+        if (typeof part.type === 'string')
+            return `[${part.type}]`;
+        return '';
+    })
+        .filter((text) => text !== '')
+        .join('\n');
+}
+function toolCallText(part) {
+    let args = part.arguments;
+    if (typeof args === 'string') {
+        try {
+            args = JSON.parse(args);
+        }
+        catch {
+            // Not JSON: the arguments stand as the model sent them.
+        }
+    }
+    return JSON.stringify({ name: part.name, arguments: args });
+}
+function inputMessages(trial) {
+    const messages = trial.content(trial.root.attributes['gen_ai.input.messages']);
+    if (!Array.isArray(messages))
+        return [];
+    const media = mediaLabeler(trial);
+    return messages.flatMap((message) => {
+        if (!isRecord(message) || typeof message.role !== 'string')
+            return [];
+        return [{ role: message.role, text: partsText(message.parts, media) }];
+    });
+}
+function stringOf(trial, value) {
+    if (typeof value === 'string')
+        return value;
+    return trial.text(value) ?? '';
+}
+function toolSteps(trial) {
+    return trial.spans('execute_tool').map((span) => ({
+        span,
+        name: stringOf(trial, span.attributes['gen_ai.tool.name']),
+        arguments: stringOf(trial, span.attributes['gen_ai.tool.call.arguments']),
+        result: stringOf(trial, span.attributes['gen_ai.tool.call.result']),
+        outcome: stringOf(trial, span.attributes['theorem.tool.outcome']),
+    }));
+}
+function availableTools(trial) {
+    const calls = modelCalls(trial);
+    for (let i = calls.length - 1; i >= 0; i -= 1) {
+        const text = stringOf(trial, calls[i]?.attributes['gen_ai.tool.definitions']);
+        if (text === '')
+            continue;
+        try {
+            const parsed = JSON.parse(text);
+            if (Array.isArray(parsed))
+                return parsed.map((tool) => JSON.stringify(tool));
+        }
+        catch {
+            // Not JSON: the definitions stand as one entry, as sent.
+        }
+        return [text];
+    }
+    return [];
+}
+function outputMessages(trial, steps) {
+    const media = mediaLabeler(trial);
+    const fromCalls = modelCalls(trial).flatMap((span) => {
+        const messages = trial.content(span.attributes['gen_ai.output.messages']);
+        if (!Array.isArray(messages))
+            return [];
+        return messages.filter(isRecord).map((message) => ({
+            at: startOf(span),
+            message: {
+                role: typeof message.role === 'string' ? message.role : 'assistant',
+                content: partsText(message.parts, media),
+                tool_calls: (Array.isArray(message.parts) ? message.parts : [])
+                    .filter((part) => isRecord(part) && part.type === 'tool_call')
+                    .map(toolCallText),
+            },
+        }));
+    });
+    const fromTools = steps.map((step) => ({
+        at: startOf(step.span),
+        message: { role: 'tool', content: `${step.name}: ${step.result}`, tool_calls: [] },
+    }));
+    return [...fromCalls, ...fromTools]
+        .toSorted((a, b) => Number(a.at - b.at))
+        .map((entry) => entry.message);
+}
+function outputText(trial) {
+    const text = deliveredText(trial, undefined);
+    if (text !== '')
+        return text;
+    const json = deliveredJson(trial);
+    return json === undefined ? '' : JSON.stringify(json);
+}
+function stepText(step) {
+    return [
+        `[tool call ${step.name}]\n${step.arguments}`,
+        `[tool result ${step.name}]${step.outcome ? ` (${step.outcome})` : ''}\n${step.result}`,
+    ];
+}
+function transcript(messages, steps, output) {
+    return [
+        ...messages.map((message) => `[${message.role}]\n${message.text}`),
+        ...steps.flatMap(stepText),
+        `[assistant]\n${output}`,
+    ].join('\n\n');
+}
+function trialVariables(trial) {
+    const messages = inputMessages(trial);
+    const input = messages.findLast((message) => message.role === 'user')?.text ?? '';
+    const output = outputText(trial);
+    const steps = toolSteps(trial);
+    return {
+        input,
+        output,
+        context: [
+            ...trialMedia(trial)
+                .filter((media) => media.from === 'input')
+                .map((media) => `[attached]\n${media.label}`),
+            ...steps.flatMap(stepText),
+        ].join('\n\n'),
+        conversation: transcript(messages, steps, output),
+        user_message: input,
+        tool_call: steps
+            .map((step) => `${step.name}(${step.arguments})${step.outcome ? ` → ${step.outcome}` : ''}`)
+            .join('\n'),
+        tool_result: steps.map((step) => `${step.name}: ${step.result}`).join('\n\n'),
+        text: output,
+    };
+}
+// A rubric that reads `user_message` gets `conversation` as what came before that message, as Phoenix's user-friction prompt asks.
+function rubricView(rubric, trial, host) {
+    const values = { ...trialVariables(trial) };
+    const paths = rubric.template === undefined ? [] : templatePaths(rubric.template);
+    const view = {};
+    for (const variable of rubric.variables) {
+        if (host[variable] !== undefined)
+            view[variable] = host[variable];
+        else if (variable === 'output' && paths.some((path) => path.startsWith('output.'))) {
+            view.output = {
+                messages: outputMessages(trial, toolSteps(trial)),
+                available_tools: availableTools(trial),
+            };
+        }
+        else if (variable === 'conversation' && rubric.variables.includes('user_message')) {
+            const messages = inputMessages(trial);
+            const last = messages.findLastIndex((message) => message.role === 'user');
+            view.conversation = messages
+                .slice(0, Math.max(last, 0))
+                .map((message) => `[${message.role}]\n${message.text}`)
+                .join('\n\n');
+        }
+        else if (values[variable] !== undefined)
+            view[variable] = values[variable];
+    }
+    return view;
+}
+export { rubricView, TRIAL_VARIABLES, toolSteps, trialVariables };

package/esm/src/evals/mod.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Evals over traces. A host says, for any profile: here are the cases, here is
+ * what "good" means, run it k times and tell me, from the trace alone, whether
+ * it passed. Graders read the v3 `TraceRecord` and nothing else; results are
+ * trace records too, written through the same `TraceSink`.
+ *
+ * Hosts own the cases, the judge (a text model, Jev, or both) and the pass
+ * rule. THEOREM owns the runner, the code graders and the result record;
+ * viewing, labelling and comparing runs belong to the trace tool the host
+ * already uses.
+ *
+ * @module
+ */
+import "../../_dnt.polyfills.js";
+export { attachmentData } from './attachments.js';
+export type { GroupSummary, Spread } from './breakdown.js';
+export { groupSummaries } from './breakdown.js';
+export type { AnswerLabel, AnswerSource } from './graders/answer.js';
+export { answer } from './graders/answer.js';
+export type { BudgetOptions, DeliveredGraders, TrajectoryMode } from './graders/code.js';
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory } from './graders/code.js';
+export type { JudgeOptions, Judgment } from './graders/judge.js';
+export { EVAL_JUDGMENT, judge } from './graders/judge.js';
+export { turnLatency } from './graders/latency.js';
+export type { TranscriptionGraders } from './graders/live.js';
+export { interruptions, transcription } from './graders/live.js';
+export { TRIAL_VARIABLES, trialVariables } from './graders/transcript.js';
+export type { GradedResult, OpenTrialRecord, RunRecordInput, TrialRecordInput, } from './record.js';
+export { buildRunRecord, startTrialRecord } from './record.js';
+export type { EvalRubric, EvalRubricQuestion } from './rubrics/mod.js';
+export { fillRubric, rubric, rubrics } from './rubrics/mod.js';
+export type { EvalStamp, RunSuiteOptions, SuiteRun, TrialReport, TurnShape } from './run.js';
+export { runSuite } from './run.js';
+export type { LoadedSuite } from './suite.js';
+export { loadSuite, readJsonl, readTraceRecords } from './suite.js';
+export type { RunSummary, TrialSummary } from './summary.js';
+export { summarizeRun } from './summary.js';
+export { buildTrial, groupByTrace, hasTurn } from './trial.js';
+export type { EvalAnswer, EvalAttachment, EvalCase, EvalCaseInput, EvalCaseKind, EvalDifficulty, EvalExpect, EvalFileAttachment, EvalGradeContext, EvalGrader, EvalInlineAttachment, EvalMediaRef, EvalMediaResolver, EvalPassRule, EvalResult, EvalResultSource, EvalSessionInput, EvalSessionStep, EvalSuite, EvalTrials, EvalTurnInput, TraceOperation, Trial, TrialMessage, TrialUsage, } from './types.js';
+export { evalCaseSchema, evalResultSchema, evalSuiteSchema } from './types.js';
+export type { CaseVerdict, TrialOutcome } from './verdict.js';
+export { caseVerdict, passRuleName, trialOutcome } from './verdict.js';

package/esm/src/evals/mod.js

@@ -0,0 +1,30 @@
+/**
+ * Evals over traces. A host says, for any profile: here are the cases, here is
+ * what "good" means, run it k times and tell me, from the trace alone, whether
+ * it passed. Graders read the v3 `TraceRecord` and nothing else; results are
+ * trace records too, written through the same `TraceSink`.
+ *
+ * Hosts own the cases, the judge (a text model, Jev, or both) and the pass
+ * rule. THEOREM owns the runner, the code graders and the result record;
+ * viewing, labelling and comparing runs belong to the trace tool the host
+ * already uses.
+ *
+ * @module
+ */
+import "../../_dnt.polyfills.js";
+export { attachmentData } from './attachments.js';
+export { groupSummaries } from './breakdown.js';
+export { answer } from './graders/answer.js';
+export { budget, delivered, guardrail, outcome, stopKind, toolTrajectory } from './graders/code.js';
+export { EVAL_JUDGMENT, judge } from './graders/judge.js';
+export { turnLatency } from './graders/latency.js';
+export { interruptions, transcription } from './graders/live.js';
+export { TRIAL_VARIABLES, trialVariables } from './graders/transcript.js';
+export { buildRunRecord, startTrialRecord } from './record.js';
+export { fillRubric, rubric, rubrics } from './rubrics/mod.js';
+export { runSuite } from './run.js';
+export { loadSuite, readJsonl, readTraceRecords } from './suite.js';
+export { summarizeRun } from './summary.js';
+export { buildTrial, groupByTrace, hasTurn } from './trial.js';
+export { evalCaseSchema, evalResultSchema, evalSuiteSchema } from './types.js';
+export { caseVerdict, passRuleName, trialOutcome } from './verdict.js';

package/esm/src/evals/record.d.ts

@@ -0,0 +1,35 @@
+import { type TraceRecord } from '../observability/trace-record.js';
+import { type TraceClock, type TraceSpan } from '../observability/trace-span.js';
+import type { ResolvedObservabilityPolicy } from '../observability/types.js';
+import type { EvalResult, EvalSuite, Trial } from './types.js';
+import { type CaseVerdict } from './verdict.js';
+interface GradedResult {
+    result: EvalResult;
+    graderIdentity: string;
+}
+interface TrialRecordInput {
+    trial: Trial;
+    policy: ResolvedObservabilityPolicy;
+    clock?: TraceClock;
+}
+interface OpenTrialRecord {
+    traceparent: string;
+    finish: (results: readonly GradedResult[]) => Promise<{
+        record: TraceRecord;
+        span: TraceSpan;
+    }>;
+}
+declare function startTrialRecord(input: TrialRecordInput): OpenTrialRecord;
+interface RunRecordInput {
+    suite: Pick<EvalSuite, 'id' | 'trials'>;
+    verdicts: readonly CaseVerdict[];
+    trialSpans: readonly Pick<TraceSpan, 'traceId' | 'spanId'>[];
+    policy: ResolvedObservabilityPolicy;
+    clock?: TraceClock;
+    caseless?: boolean;
+    stopped?: 'budget';
+    revision?: string;
+}
+declare function buildRunRecord(input: RunRecordInput): Promise<TraceRecord>;
+export type { GradedResult, OpenTrialRecord, RunRecordInput, TrialRecordInput };
+export { buildRunRecord, startTrialRecord };

package/esm/src/evals/record.js

@@ -0,0 +1,105 @@
+import { TheoremError } from '../guardrails/error.js';
+import { sha256 } from '../kernel/engine/hash.js';
+import { buildRecord } from '../observability/trace-record.js';
+import { formatTraceparent, startTrace, traceContent, } from '../observability/trace-span.js';
+import { passRuleName } from './verdict.js';
+const TRIAL_SPAN = 'theorem.eval.trial';
+const RUN_SPAN = 'theorem.eval.run';
+const RESULT_EVENT = 'gen_ai.evaluation.result';
+const VERDICT_EVENT = 'theorem.eval.verdict';
+function optional(key, value) {
+    return value === undefined ? {} : { [key]: value };
+}
+function responseIdOf(trial) {
+    const calls = [...trial.spans('chat'), ...trial.spans('generate_content')];
+    for (let i = calls.length - 1; i >= 0; i -= 1) {
+        const id = calls[i]?.attributes['gen_ai.response.id'];
+        if (typeof id === 'string')
+            return id;
+    }
+    return undefined;
+}
+async function resultAttributes(result, graderIdentity, responseId) {
+    return {
+        'gen_ai.evaluation.name': result.name,
+        ...optional('gen_ai.evaluation.score.value', result.score?.value),
+        ...optional('gen_ai.evaluation.score.label', result.score?.label),
+        ...(result.explanation === undefined
+            ? {}
+            : { 'gen_ai.evaluation.explanation': traceContent(result.explanation) }),
+        ...optional('gen_ai.response.id', responseId),
+        ...optional('error.type', result.errorType),
+        'theorem.evaluation.source': result.source,
+        'theorem.evaluation.grader.version': await sha256(graderIdentity),
+        ...optional('theorem.evaluation.passed', result.passed),
+    };
+}
+function startTrialRecord(input) {
+    const { trial, policy } = input;
+    const tree = startTrace(TRIAL_SPAN, {
+        traceparent: formatTraceparent(trial.root.traceId, trial.root.spanId),
+        ...(input.clock ? { clock: input.clock } : {}),
+        attributes: {
+            'theorem.evaluation.suite': trial.suite,
+            ...optional('theorem.evaluation.case', trial.case?.id),
+            'theorem.evaluation.trial': trial.index,
+        },
+    });
+    const finish = async (results) => {
+        const responseId = responseIdOf(trial);
+        for (const { result, graderIdentity } of results) {
+            tree.root.event(RESULT_EVENT, await resultAttributes(result, graderIdentity, responseId));
+        }
+        const errorType = results.find(({ result }) => result.errorType !== undefined)?.result
+            .errorType;
+        tree.root.end(errorType ? { code: 'ERROR', message: errorType } : { code: 'OK' });
+        const record = await buildRecord({
+            spans: tree.collect(),
+            policy,
+            ...(trial.records[0]?.metadata ? { metadata: trial.records[0].metadata } : {}),
+        });
+        const [span] = record.spans;
+        if (!span)
+            throw new TheoremError('internal', 'trial record has no span'); // lexicon-exempt: invariant
+        return { record, span };
+    };
+    return { traceparent: tree.root.traceparent(), finish };
+}
+function verdictAttributes(verdict) {
+    return {
+        case: verdict.case,
+        kind: verdict.kind,
+        ...optional('difficulty', verdict.difficulty),
+        passed: verdict.passed,
+        trials: verdict.trials,
+        trials_passed: verdict.trialsPassed,
+        trials_errored: verdict.trialsErrored,
+        trials_ungraded: verdict.trialsUngraded,
+    };
+}
+function buildRunRecord(input) {
+    const rule = input.suite.trials.pass ?? 'all';
+    const tree = startTrace(RUN_SPAN, {
+        ...(input.clock ? { clock: input.clock } : {}),
+        attributes: {
+            'theorem.evaluation.suite': input.suite.id,
+            'theorem.eval.repeat': input.suite.trials.repeat,
+            'theorem.eval.pass_rule': passRuleName(rule),
+            ...(typeof rule === 'object' ? { 'theorem.eval.pass_at_least': rule.atLeast } : {}),
+            ...(input.caseless ? { 'theorem.eval.caseless': true } : {}),
+            ...optional('theorem.eval.stopped', input.stopped),
+            ...optional('vcs.ref.head.revision', input.revision),
+        },
+        links: input.trialSpans.map((span) => ({
+            traceparent: formatTraceparent(span.traceId, span.spanId),
+            attributes: { 'theorem.link.kind': 'trial' },
+        })),
+    });
+    for (const verdict of input.verdicts) {
+        tree.root.event(VERDICT_EVENT, verdictAttributes(verdict));
+    }
+    // A run that reached its end is OK whatever the verdicts say; one stopped early is not finished.
+    tree.root.end(input.stopped ? { code: 'UNSET' } : { code: 'OK' });
+    return buildRecord({ spans: tree.collect(), policy: input.policy });
+}
+export { buildRunRecord, startTrialRecord };