@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/evals/trial.js

@@ -0,0 +1,158 @@
+/**
+ * Records → `Trial`. A live run hands the runner what its memory sink caught;
+ * a recorded run hands it what a file held. Both build the same trial, so the
+ * same graders give the same results.
+ *
+ * The root is the `invoke_agent` span with no agent above it in the set; a host
+ * span above it (the host's request, when the host nests the turn under its own
+ * span) does not hide it, and the topmost span above it is the trial's `top`. It
+ * is never assumed to be the first record: a compaction turn's record can be
+ * written before its parent's (P9).
+ *
+ * @module
+ */
+import { TheoremError } from '../guardrails/error.js';
+import { contentOf, inlineContent } from '../observability/trace-record.js';
+function byStart(a, b) {
+    const left = BigInt(a.startTimeUnixNano);
+    const right = BigInt(b.startTimeUnixNano);
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function operationOf(span) {
+    const value = span.attributes['gen_ai.operation.name'];
+    return typeof value === 'string' ? value : undefined;
+}
+/** Every span of every record, in start order. */
+function allSpans(records) {
+    return records.flatMap((record) => record.spans).sort(byStart);
+}
+/** The spans above `span` in the set, nearest first; a parent outside the set ends the walk. */
+function ancestorsOf(span, byId) {
+    const above = [];
+    const seen = new Set([span.spanId]);
+    let parent = span.parentSpanId === undefined ? undefined : byId.get(span.parentSpanId);
+    while (parent && !seen.has(parent.spanId)) {
+        above.push(parent);
+        seen.add(parent.spanId);
+        parent = parent.parentSpanId === undefined ? undefined : byId.get(parent.parentSpanId);
+    }
+    return above;
+}
+/**
+ * The `invoke_agent` span with no `invoke_agent` above it in the set; the
+ * earliest when several qualify (a session root before its responses).
+ */
+function rootOf(spans, byId) {
+    const roots = spans.filter((span) => operationOf(span) === 'invoke_agent' &&
+        !ancestorsOf(span, byId).some((above) => operationOf(above) === 'invoke_agent'));
+    const [root] = roots;
+    if (!root) {
+        throw new TheoremError('config', 'trial records hold no invoke_agent root span'); // lexicon-exempt: developer contract error
+    }
+    return root;
+}
+/** One record whose `content` is every record's, so a reference from any span resolves. */
+function mergedRecord(records) {
+    const [first] = records;
+    if (!first) {
+        throw new TheoremError('config', 'a trial needs at least one trace record'); // lexicon-exempt: developer contract error
+    }
+    return { ...first, content: Object.assign({}, ...records.map((record) => record.content)) };
+}
+function numberAttribute(span, key) {
+    const value = span.attributes[key];
+    return typeof value === 'number' ? value : undefined;
+}
+function isObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isMessageList(value) {
+    return Array.isArray(value) && value.every(isObject);
+}
+/** The root's usage attributes as `TurnTokens`; zero when the root recorded none. */
+function usageOf(root) {
+    const input = numberAttribute(root, 'gen_ai.usage.input_tokens') ?? 0;
+    const output = numberAttribute(root, 'gen_ai.usage.output_tokens') ?? 0;
+    const thinking = numberAttribute(root, 'gen_ai.usage.reasoning.output_tokens');
+    const cached = numberAttribute(root, 'gen_ai.usage.cache_read.input_tokens');
+    const cacheWrite = numberAttribute(root, 'gen_ai.usage.cache_write.input_tokens');
+    const toolUse = numberAttribute(root, 'theorem.usage.tool_use.input_tokens');
+    const costUsd = numberAttribute(root, 'theorem.usage.cost_usd');
+    const partial = root.attributes['theorem.usage.cost_partial'] === true;
+    const estimated = root.attributes['theorem.usage.estimated'];
+    const tokens = {
+        input,
+        output,
+        total: input + output,
+        ...(thinking === undefined ? {} : { thinking }),
+        ...(cached === undefined ? {} : { cached }),
+        ...(cacheWrite === undefined ? {} : { cacheWrite }),
+        ...(toolUse === undefined ? {} : { toolUse }),
+        ...(costUsd === undefined
+            ? {}
+            : { cost: { usd: costUsd, ...(partial ? { partial: true } : {}) } }),
+        ...(Array.isArray(estimated)
+            ? {
+                estimated: estimated.flatMap((side) => side === 'input' || side === 'output' ? [side] : []),
+            }
+            : {}),
+    };
+    return costUsd === undefined ? { tokens } : { tokens, costUsd };
+}
+/**
+ * What the host received. A turn's root carries `gen_ai.output.messages`; a
+ * Live session's root carries none, and each response span carries its own
+ * `theorem.output.delivered`, taken in start order.
+ */
+function deliveredOf(root, spans) {
+    const own = root.attributes['gen_ai.output.messages'];
+    if (isMessageList(own))
+        return own;
+    return spans
+        .filter((span) => 'theorem.request.live' in span.attributes)
+        .flatMap((span) => {
+        const delivered = span.attributes['theorem.output.delivered'];
+        return isMessageList(delivered) ? delivered : [];
+    });
+}
+/** Build the trial every grader receives from the records that share one trace. */
+function buildTrial(args) {
+    const merged = mergedRecord(args.records);
+    const spans = allSpans(args.records);
+    const byId = new Map(spans.map((span) => [span.spanId, span]));
+    const root = rootOf(spans, byId);
+    return {
+        suite: args.suite,
+        ...(args.case ? { case: args.case } : {}),
+        index: args.index,
+        records: args.records,
+        root,
+        top: ancestorsOf(root, byId).at(-1) ?? root,
+        spans: (operation) => spans.filter((span) => operationOf(span) === operation),
+        children: (parent) => spans.filter((span) => span.parentSpanId === parent.spanId),
+        content: (value) => inlineContent(merged, value),
+        text: (value) => (isObject(value) ? contentOf(merged, value) : undefined),
+        delivered: () => deliveredOf(root, spans),
+        usage: () => usageOf(root),
+    };
+}
+/** Group records by trace id, in first-seen order. */
+function groupByTrace(records) {
+    const groups = new Map();
+    for (const record of records) {
+        const traceId = record.spans[0]?.traceId;
+        if (traceId === undefined)
+            continue;
+        const group = groups.get(traceId);
+        if (group)
+            group.push(record);
+        else
+            groups.set(traceId, [record]);
+    }
+    return groups;
+}
+/** Whether these records hold a turn at all: an `invoke_agent` span. An eval run record does not. */
+function hasTurn(records) {
+    return records.some((record) => record.spans.some((span) => operationOf(span) === 'invoke_agent'));
+}
+export { buildTrial, groupByTrace, hasTurn };

package/esm/src/evals/types.d.ts

@@ -0,0 +1,260 @@
+/**
+ * Eval shapes: the suite a host writes, the cases it keeps in JSONL,
+ * and the result every grader returns. Each is a documented type and a zod
+ * schema checked against it (`Equals`): the type is what graders read; the
+ * schema is what the runner runs on a file.
+ *
+ * Vocabulary follows Anthropic's evals guide so a host can read both: a
+ * **suite** of **cases**, each run as k **trials**; a **grader** reads a
+ * trial's transcript (its trace) and returns a **result**; the suite's pass
+ * rule turns k results into one **verdict**.
+ *
+ * @module
+ */
+import { z } from 'zod';
+import type { RunDecisionOptions } from '../kernel/engine/decision.js';
+import type { TurnTokens } from '../kernel/turn-events.js';
+import type { ModelProvider } from '../kernel/types.js';
+import type { TraceRecord } from '../observability/trace-record.js';
+import type { TraceAttributeValue, TraceSpan } from '../observability/trace-span.js';
+/** Anthropic's split: does the agent *do* the thing (capability), or *still* do it (regression)? */
+export type EvalCaseKind = 'capability' | 'regression';
+/** A file the case attaches to its turn, inline, as `TurnBlob` takes it. */
+export interface EvalInlineAttachment {
+    mimeType: string;
+    /** Base64 bytes. */
+    data: string;
+    name?: string;
+}
+/**
+ * A file the case attaches to its turn, kept beside the cases file and pinned
+ * by the hex SHA-256 of its bytes; a file whose bytes hash otherwise is refused.
+ */
+export interface EvalFileAttachment {
+    mimeType: string;
+    /** Relative to the cases file, or absolute; the loaded case holds it absolute. */
+    path: string;
+    sha256: string;
+    name?: string;
+}
+/** A file the case attaches to its turn: its bytes inline, or a file beside the cases. */
+export type EvalAttachment = EvalInlineAttachment | EvalFileAttachment;
+/** One turn's input (`mode: 'turn'`). */
+export interface EvalTurnInput {
+    text?: string;
+    attachments?: EvalAttachment[];
+}
+/** One scripted step of a Live session: send text, wait for the boundary it names. */
+export interface EvalSessionStep {
+    text: string;
+    until: 'turn_complete';
+}
+/** A scripted Live session (`mode: 'session'`): text steps in order. Audio steps are not in this slice. */
+export interface EvalSessionInput {
+    session: {
+        steps: EvalSessionStep[];
+    };
+}
+/** What the case sends: a turn's input or a session script. */
+export type EvalCaseInput = EvalTurnInput | EvalSessionInput;
+/**
+ * A dataset case's right answer. Only an `accepted` name passes; a `partial`
+ * one is close but not enough (the genus, the broader condition); a `rejected`
+ * one is a wrong answer the dataset records for the case (a confuser, an
+ * identification it superseded).
+ */
+export interface EvalAnswer {
+    /** Every name that counts: scientific name, synonyms, common names. */
+    accepted: string[];
+    partial?: string[];
+    /** A reply that names one beside an accepted name hedged: `partial`, not a pass. */
+    rejected?: string[];
+    /** Grounding notes for the person reading a miss; no grader reads them. */
+    reference?: string;
+}
+/** What the transcript should contain; graders that need it are skipped when it is absent. */
+export interface EvalExpect {
+    /** Tools the agent should call, in order (`toolTrajectory` reads it). Empty = no tool. */
+    tools?: string[];
+    /** Fields the delivered JSON should carry, each compared whole (`delivered.json` reads it). */
+    json?: Record<string, unknown>;
+    /** What the Live reply's transcript should say (`transcription.*` read it). */
+    transcription?: {
+        includes?: string;
+        regex?: string;
+    };
+    /** The right answer to a dataset question (`answer` reads it). */
+    answer?: EvalAnswer;
+    /** For the human reading the case; no grader reads it. */
+    notes?: string;
+}
+/** Perplexity's 1–5 difficulty; results are reported per bucket when set. */
+export type EvalDifficulty = 1 | 2 | 3 | 4 | 5;
+/** One line of `cases.jsonl`. */
+export interface EvalCase {
+    id: string;
+    kind: EvalCaseKind;
+    difficulty?: EvalDifficulty;
+    input: EvalCaseInput;
+    expect?: EvalExpect;
+    tags?: string[];
+}
+/** What the runner runs on each line of a cases file. */
+export declare const evalCaseSchema: z.ZodType<EvalCase>;
+/** The `gen_ai.operation.name` values a trial's spans carry. */
+export type TraceOperation = 'invoke_agent' | 'chat' | 'generate_content' | 'execute_tool';
+/** A message as the trace stores it: `{ role, parts: [...] }`, parts by hash; resolve them with `text` / `content`. */
+export type TrialMessage = Record<string, TraceAttributeValue>;
+/** What the root span reports the trial cost, as the kernel's `TurnTokens`. */
+export interface TrialUsage {
+    tokens: TurnTokens;
+    costUsd?: number;
+}
+/**
+ * One run of one case, as every grader receives it: the trace records that
+ * share the trial's trace id and nothing else. If a grader needs a fact the
+ * trace does not hold, the trace is wrong, not the grader.
+ */
+export interface Trial {
+    suite: string;
+    /** Absent in caseless recorded mode (a production trace with no `metadata.eval`). */
+    case?: EvalCase;
+    /** 0..k-1. */
+    index: number;
+    records: TraceRecord[];
+    /** The `invoke_agent` span with no agent above it in the set; never assumed to be first. */
+    root: TraceSpan;
+    /**
+     * The topmost span above `root` in the set: the host's own span when the host
+     * nested the turn under it, else `root`. Its length is the trial's time.
+     */
+    top: TraceSpan;
+    /** Spans by operation, in start order, across every record. */
+    spans: (operation: TraceOperation) => TraceSpan[];
+    /** The spans one span parents, in start order (a model call's HTTP tries, a response's tool calls). */
+    children: (span: TraceSpan) => TraceSpan[];
+    /** A value with every `{ content_sha256 }` / `{ json_sha256 }` reference replaced by its stored content. */
+    content: (value: unknown) => unknown;
+    /** The stored text one reference names; `undefined` for any other value. */
+    text: (value: unknown) => string | undefined;
+    /** What the host received: the root's output messages, or each Live response's, as stored. */
+    delivered: () => TrialMessage[];
+    /** Tokens and cost from the root's usage attributes. */
+    usage: () => TrialUsage;
+}
+/** Who produced a result: a code grader, or a judge model. */
+export type EvalResultSource = 'code' | 'model';
+/**
+ * One grader's reading of one trial: the semconv `gen_ai.evaluation.result`
+ * event, before it is written. `passed` is the grader's yes/no when it has one;
+ * absent means the result informs but does not decide the verdict.
+ */
+export interface EvalResult {
+    /** The grader's name, e.g. `tool_trajectory`, `faithfulness`. */
+    name: string;
+    source: EvalResultSource;
+    /** `value` is 0–1 unless the grader says otherwise; `label` is one of the grader's labels. */
+    score?: {
+        value?: number;
+        label?: string;
+    };
+    /** Why, in the grader's words. */
+    explanation?: string;
+    /** Whether this result counts as a pass. Absent: informational only. */
+    passed?: boolean;
+    /** Why no score could be produced; the result is then a failed one. */
+    errorType?: string;
+    /** Judge turns this result was drawn from (`traceparent` each); model graders only. */
+    judgeTraceparents?: string[];
+}
+/** A result as a reader validates it, e.g. one rebuilt from a written event. */
+export declare const evalResultSchema: z.ZodType<EvalResult>;
+/** How k trials become one verdict: every trial (pass^k), any trial (pass@k), or at least n. */
+export type EvalPassRule = 'all' | 'any' | {
+    atLeast: number;
+};
+/** How many times each case runs, and what counts as passing. */
+export interface EvalTrials {
+    /** Trials per case. No default: it is a cost decision, so the suite states it. `1` cannot tell noise from change. */
+    repeat: number;
+    /** Default `all`. */
+    pass?: EvalPassRule;
+}
+/**
+ * What the runner hands every grader beside the trial. Code graders ignore it;
+ * a model grader runs its judge through it and reports each judge call's
+ * records through `traced`, so the runner can count their cost and write them
+ * to the sink.
+ */
+export interface EvalGradeContext {
+    /** The suite's judge profile, when it names one; a grader may name its own. */
+    judge?: string;
+    /** The host's provider for text judge profiles. */
+    judgeProvider?: ModelProvider;
+    /** The host's vault for decision judge profiles. */
+    judgeDecision?: Omit<RunDecisionOptions, 'sink'>;
+    /**
+     * The trial span as a W3C `traceparent`: a judge call runs under it, so it
+     * lands in the judged trace beneath the trial. Absent, each judge call
+     * starts a trace of its own.
+     */
+    traceparent?: string;
+    /**
+     * The host's store for media a trace names only by hash. A judge that must
+     * see media the case did not attach asks here; absent, only the case's own
+     * attachments can be shown.
+     */
+    media?: EvalMediaResolver;
+    /** Every record a grader's own judge call produced. */
+    traced: (records: TraceRecord[]) => void | Promise<void>;
+    signal?: AbortSignal;
+}
+/** Media a trace names by the sha256 (hex) of its raw bytes. */
+export interface EvalMediaRef {
+    sha256: string;
+    mimeType: string;
+}
+/** The media's bytes as base64, or `undefined` when the host does not have them. */
+export type EvalMediaResolver = (ref: EvalMediaRef) => string | undefined | Promise<string | undefined>;
+/** Code or model logic that reads a trial and returns a result. */
+export interface EvalGrader {
+    /** The result's `gen_ai.evaluation.name`. */
+    name: string;
+    /**
+     * Stable text naming exactly what this grader checks (its options, its
+     * rubric). Its sha256 is written as `theorem.evaluation.grader.version`, so
+     * a later reader can tell which rubric produced which score.
+     */
+    identity: string;
+    source: EvalResultSource;
+    /** Reads the case's `expect`; skipped when a trial has no case (caseless recorded mode). */
+    needsExpect: boolean;
+    /**
+     * Model graders: every judge profile this grader may run, given the suite's
+     * `judge.profile`: its judge first, then any it escalates to. Throws when
+     * there is none, or a profile cannot run this grader, so a suite fails
+     * before its first trial.
+     */
+    judgeProfiles?: (suiteJudge: string | undefined) => string[];
+    grade: (trial: Trial, context: EvalGradeContext) => EvalResult | Promise<EvalResult>;
+}
+/** A named set of cases against one profile, with graders and a pass rule. */
+export interface EvalSuite {
+    id: string;
+    /** The profile under test; any type but `host`. */
+    profile: string;
+    mode: 'turn' | 'session';
+    /** Path to `cases.jsonl`, relative to the suite module. */
+    cases: string;
+    trials: EvalTrials;
+    graders: EvalGrader[];
+    /**
+     * The judge profile model graders run unless they name their own: a text
+     * profile that answers through `evalJudgment`, or a decision profile (Jev).
+     */
+    judge?: {
+        profile: string;
+    };
+}
+/** What the runner runs on a suite module's default export. */
+export declare const evalSuiteSchema: z.ZodType<EvalSuite>;

package/esm/src/evals/types.js

@@ -0,0 +1,126 @@
+/**
+ * Eval shapes: the suite a host writes, the cases it keeps in JSONL,
+ * and the result every grader returns. Each is a documented type and a zod
+ * schema checked against it (`Equals`): the type is what graders read; the
+ * schema is what the runner runs on a file.
+ *
+ * Vocabulary follows Anthropic's evals guide so a host can read both: a
+ * **suite** of **cases**, each run as k **trials**; a **grader** reads a
+ * trial's transcript (its trace) and returns a **result**; the suite's pass
+ * rule turns k results into one **verdict**.
+ *
+ * @module
+ */
+import { z } from 'zod';
+import { isRecord } from '../kernel/util/record.js';
+const evalCaseKind = z.enum(['capability', 'regression']);
+true;
+const evalInlineAttachment = z.object({
+    mimeType: z.string(),
+    data: z.string(),
+    name: z.string().optional(),
+});
+true;
+const evalFileAttachment = z.object({
+    mimeType: z.string(),
+    path: z.string().min(1),
+    sha256: z.string().regex(/^[0-9a-f]{64}$/, 'must be a lowercase hex SHA-256'),
+    name: z.string().optional(),
+});
+true;
+const evalAttachment = z.union([evalInlineAttachment, evalFileAttachment]);
+true;
+const evalTurnInput = z.object({
+    text: z.string().optional(),
+    attachments: z.array(evalAttachment).optional(),
+});
+true;
+const evalSessionStep = z.object({ text: z.string(), until: z.literal('turn_complete') });
+true;
+const evalSessionInput = z.object({ session: z.object({ steps: z.array(evalSessionStep) }) });
+true;
+const evalCaseInput = z.union([evalSessionInput, evalTurnInput]);
+true;
+const evalAnswer = z.object({
+    accepted: z.array(z.string().min(1)).min(1),
+    partial: z.array(z.string().min(1)).optional(),
+    rejected: z.array(z.string().min(1)).optional(),
+    reference: z.string().optional(),
+});
+true;
+const evalExpect = z.object({
+    tools: z.array(z.string()).optional(),
+    json: z.record(z.string(), z.unknown()).optional(),
+    transcription: z
+        .object({ includes: z.string().optional(), regex: z.string().optional() })
+        .optional(),
+    answer: evalAnswer.optional(),
+    notes: z.string().optional(),
+});
+true;
+const evalDifficulty = z.union([
+    z.literal(1),
+    z.literal(2),
+    z.literal(3),
+    z.literal(4),
+    z.literal(5),
+]);
+true;
+const evalCase = z.object({
+    id: z.string().min(1),
+    kind: evalCaseKind,
+    difficulty: evalDifficulty.optional(),
+    input: evalCaseInput,
+    expect: evalExpect.optional(),
+    tags: z.array(z.string()).optional(),
+});
+true;
+/** What the runner runs on each line of a cases file. */
+export const evalCaseSchema = evalCase;
+const evalResultSource = z.enum(['code', 'model']);
+true;
+const evalResult = z.object({
+    name: z.string().min(1),
+    source: evalResultSource,
+    score: z.object({ value: z.number().optional(), label: z.string().optional() }).optional(),
+    explanation: z.string().optional(),
+    passed: z.boolean().optional(),
+    errorType: z.string().optional(),
+    judgeTraceparents: z.array(z.string()).optional(),
+});
+true;
+/** A result as a reader validates it, e.g. one rebuilt from a written event. */
+export const evalResultSchema = evalResult;
+const evalPassRule = z.union([
+    z.literal('all'),
+    z.literal('any'),
+    z.object({ atLeast: z.number().int().min(1) }),
+]);
+true;
+const evalTrials = z.object({
+    repeat: z.number().int().min(1),
+    pass: evalPassRule.optional(),
+});
+true;
+function isGrader(value) {
+    return (isRecord(value) &&
+        typeof value.name === 'string' &&
+        typeof value.identity === 'string' &&
+        (value.source === 'code' || value.source === 'model') &&
+        typeof value.needsExpect === 'boolean' &&
+        (value.judgeProfiles === undefined || typeof value.judgeProfiles === 'function') &&
+        typeof value.grade === 'function');
+}
+const evalGrader = z.custom(isGrader);
+const evalSuite = z.object({
+    id: z.string().min(1),
+    profile: z.string().min(1),
+    mode: z.enum(['turn', 'session']),
+    cases: z.string().min(1),
+    trials: evalTrials,
+    graders: z.array(evalGrader),
+    judge: z.object({ profile: z.string().min(1) }).optional(),
+});
+true;
+/** What the runner runs on a suite module's default export. */
+export const evalSuiteSchema = evalSuite;

package/esm/src/evals/verdict.d.ts

@@ -0,0 +1,18 @@
+import type { EvalCase, EvalPassRule, EvalResult } from './types.js';
+type TrialOutcome = 'passed' | 'failed' | 'errored' | 'ungraded';
+declare function trialOutcome(results: readonly EvalResult[]): TrialOutcome;
+interface CaseVerdict {
+    case: string;
+    kind: EvalCase['kind'];
+    difficulty?: EvalCase['difficulty'];
+    passed: boolean;
+    decided: boolean;
+    trials: number;
+    trialsPassed: number;
+    trialsErrored: number;
+    trialsUngraded: number;
+}
+declare function passRuleName(rule: EvalPassRule): 'all' | 'any' | 'at_least';
+declare function caseVerdict(evalCase: EvalCase, trials: readonly (readonly EvalResult[])[], rule?: EvalPassRule): CaseVerdict;
+export type { CaseVerdict, TrialOutcome };
+export { caseVerdict, passRuleName, trialOutcome };

package/esm/src/evals/verdict.js

@@ -0,0 +1,45 @@
+function trialOutcome(results) {
+    if (results.some((result) => result.errorType !== undefined))
+        return 'errored';
+    const decided = results.filter((result) => result.passed !== undefined);
+    if (decided.length === 0)
+        return 'ungraded';
+    return decided.every((result) => result.passed) ? 'passed' : 'failed';
+}
+function ruleMet(rule, passed, trials) {
+    if (trials === 0)
+        return false;
+    if (rule === 'all')
+        return passed === trials;
+    if (rule === 'any')
+        return passed >= 1;
+    return passed >= rule.atLeast;
+}
+function trialsNeeded(rule) {
+    return typeof rule === 'object' ? rule.atLeast : 1;
+}
+function passRuleName(rule) {
+    return typeof rule === 'string' ? rule : 'at_least';
+}
+// An errored trial is one that could not be graded (a provider error, a judge that failed), so the rule reads it as a trial that never ran: a rate limit never counts against the agent.
+function caseVerdict(evalCase, trials, rule = 'all') {
+    const outcomes = trials.map(trialOutcome);
+    const count = (outcome) => outcomes.filter((o) => o === outcome).length;
+    const trialsPassed = count('passed');
+    const trialsErrored = count('errored');
+    const trialsUngraded = count('ungraded');
+    const ran = outcomes.length - trialsErrored;
+    const passed = trialsUngraded < ran && ruleMet(rule, trialsPassed, ran);
+    return {
+        case: evalCase.id,
+        kind: evalCase.kind,
+        ...(evalCase.difficulty === undefined ? {} : { difficulty: evalCase.difficulty }),
+        passed,
+        decided: passed || trialsErrored === 0 || ran >= trialsNeeded(rule),
+        trials: outcomes.length,
+        trialsPassed,
+        trialsErrored,
+        trialsUngraded,
+    };
+}
+export { caseVerdict, passRuleName, trialOutcome };

package/esm/src/guardrails/canary-gate.d.ts

@@ -0,0 +1,21 @@
+import type { TurnEvent } from '../kernel/types.js';
+import { type CanaryStreamGate } from './canary.js';
+export interface CanaryGateSession {
+    canary: string;
+    /** The system prompt as sent, when replies echoing it are leaks too. */
+    privateSystem?: readonly string[];
+    gate: CanaryStreamGate;
+}
+/** Pass the system prompt as sent to catch replies that echo it, as `runTurn` and Live do. */
+declare function createCanaryGateSession(canary: string, privateSystem?: readonly string[]): CanaryGateSession;
+/**
+ * Canary-only batch filter for hosts that need stream lookback without full egress. Stops at the
+ * first leak, so no unsafe event is returned; thoughts are unguarded (`isGuardedOutput`).
+ */
+declare function filterCanaryGatedEvents(session: CanaryGateSession, events: TurnEvent[]): {
+    leaked: true;
+} | {
+    leaked: false;
+    events: TurnEvent[];
+};
+export { createCanaryGateSession, filterCanaryGatedEvents };

package/esm/src/guardrails/canary-gate.js

@@ -0,0 +1,40 @@
+import { createCanaryStreamGate, eventHasCanary, guardedEventTexts, isStreamedCanaryEvent, } from './canary.js';
+import { scanTextForPromptEcho } from './prompt-echo.js';
+/** Pass the system prompt as sent to catch replies that echo it, as `runTurn` and Live do. */
+function createCanaryGateSession(canary, privateSystem) {
+    return {
+        canary,
+        ...(privateSystem ? { privateSystem } : {}),
+        gate: createCanaryStreamGate(canary, privateSystem),
+    };
+}
+function echoesPrompt(event, canary, privateSystem) {
+    return (privateSystem !== undefined &&
+        guardedEventTexts(event).some((text) => scanTextForPromptEcho(text, privateSystem, canary)));
+}
+/**
+ * Canary-only batch filter for hosts that need stream lookback without full egress. Stops at the
+ * first leak, so no unsafe event is returned; thoughts are unguarded (`isGuardedOutput`).
+ */
+function filterCanaryGatedEvents(session, events) {
+    const out = [];
+    for (const event of events) {
+        if (isStreamedCanaryEvent(event)) {
+            const result = session.gate.process(event.text ?? '');
+            if (result.leak) {
+                return { leaked: true };
+            }
+            if (result.emit) {
+                out.push({ ...event, text: result.emit });
+            }
+            continue;
+        }
+        if (eventHasCanary(event, session.canary) ||
+            echoesPrompt(event, session.canary, session.privateSystem)) {
+            return { leaked: true };
+        }
+        out.push(event);
+    }
+    return { leaked: false, events: out };
+}
+export { createCanaryGateSession, filterCanaryGatedEvents };