@archmax-ai/harness 0.0.0

package/dist/testing/driver.d.ts

@@ -0,0 +1,78 @@
+/**
+ * The host-side session engine cases drive the agent through. One engine per
+ * case; it owns the case's session on the agent under test, its declared start
+ * conditions (trigger, variables, seeded workspace files), and its latest turn
+ * view. A thin caller of `agent.workflow.send`: where a message lands — a turn,
+ * a reply to a session a person holds — is the session's business, decided by
+ * the runtime exactly as it is for a host.
+ */
+import type { GovernedAgent } from "../agent.js";
+import { type SessionView } from "../workflow/session-view.js";
+import type { ToolMockSpec } from "../core/tool-mocks.js";
+/** The session name steps without a `session:` key drive. */
+export declare const DEFAULT_SESSION_ID = "default";
+export interface DriveSession {
+    /** The session's name as the case declares it (`session:` on a step). */
+    name: string;
+    /**
+     * The id the runtime addresses this session by — namespaced per case, since one
+     * agent serves every case. One id: the runtime has no separate session beneath it.
+     */
+    sessionId: string;
+    /** The validated trigger every driven turn of this session starts from. */
+    trigger?: {
+        id: string;
+    };
+    /** Variables seeded into every driven turn of this session, locked by the runtime. */
+    variables?: Record<string, unknown>;
+    /** The session's latest turn view (empty until it first drives). */
+    lastView: SessionView;
+}
+export interface SessionStartConditions {
+    trigger?: {
+        id: string;
+    };
+    /**
+     * Variables seeded for this session, supplied per turn rather than at
+     * assembly: one agent serves every case of a suite, so a case's seeds cannot
+     * ride the assembly-time `variables` seeds.
+     */
+    variables?: Record<string, unknown>;
+    /** Workspace files seeded into the session before any turn. */
+    files?: Record<string, unknown>;
+}
+/**
+ * The refusal for a sub-workflow mock against a target that cannot delegate.
+ * Separate from the interception refusal because the fix is different: this one
+ * is not about middleware but about the target being workflow-governed at all.
+ */
+export declare function unDelegatableMocksMessage(): string;
+/** The refusal for mocks a target cannot intercept (single definition). */
+export declare function unInterceptableMocksMessage(mockCount: number): string;
+export interface SessionEngine {
+    /** Open a session, validating and applying its declared start conditions. */
+    open(sessionId: string, conditions?: SessionStartConditions): Promise<DriveSession>;
+    /** Drive one user turn on a session under its declared trigger. */
+    send(sessionId: string, message: string, toolMocks?: ToolMockSpec[]): Promise<SessionView>;
+    /** Resume a session parked at a human state with the person's chosen edge. */
+    decide(sessionId: string, to: string, comment?: string): Promise<SessionView>;
+    /**
+     * Resume a session parked awaiting an event with a delivered firing; the
+     * session continues in the state it parked in. Fails the case the way
+     * production fails the host — the session was not parked — rather than
+     * silently starting a new turn.
+     */
+    deliver(sessionId: string, trigger: string, variables?: Record<string, unknown>): Promise<SessionView>;
+}
+/**
+ * Create the per-case session engine. The default session runs on
+ * `defaultSessionId` (caller-nameable via `sessionIdForCase`); every other
+ * session's id derives from it, so naming the default names the whole case and
+ * no session is a timestamped orphan.
+ */
+export declare function createSessionEngine(opts: {
+    agent: GovernedAgent;
+    defaultSessionId: string;
+    /** Aborts every run this engine drives — the case's wall-clock budget. */
+    signal?: AbortSignal;
+}): SessionEngine;

package/dist/testing/driver.js

@@ -0,0 +1,117 @@
+import { messagesToSessionView } from "../workflow/session-view.js";
+/** The session name steps without a `session:` key drive. */
+export const DEFAULT_SESSION_ID = "default";
+/**
+ * Validate-then-seed-then-store for a session's declared start conditions.
+ * Fails closed before any turn: an unknown trigger id surfaces
+ * `UnknownTriggerError` with the declared triggers, and declared files against
+ * a target without a `sessions.seed` handle are refused rather than skipped.
+ */
+async function applyStartConditions(agent, session, input) {
+    if (input.trigger) {
+        agent.workflow.resolveTrigger(input.trigger);
+        session.trigger = input.trigger;
+    }
+    if (input.variables && Object.keys(input.variables).length > 0) {
+        session.variables = { ...input.variables };
+    }
+    const files = input.files ?? {};
+    if (Object.keys(files).length > 0) {
+        if (typeof agent.sessions?.seed !== "function") {
+            throw new Error(`this case declares ${Object.keys(files).length} workspace file(s), but the target ` +
+                `runtime's session handles expose no 'seed' operation, so nothing can be placed in the ` +
+                `session workspace before the agent runs. Supply a target whose sessions handle can seed ` +
+                `(createAgent provides one).`);
+        }
+        await agent.sessions.seed(session.sessionId, files);
+    }
+}
+/**
+ * The refusal for a sub-workflow mock against a target that cannot delegate.
+ * Separate from the interception refusal because the fix is different: this one
+ * is not about middleware but about the target being workflow-governed at all.
+ */
+export function unDelegatableMocksMessage() {
+    return (`this case mocks a sub-workflow, but the target agent is not workflow-governed ` +
+        `(agent.workflow is undefined) — it cannot dispatch sub-workflows, so the mock would ` +
+        `never be consulted. Run the case against a workflow-governed target.`);
+}
+/** The refusal for mocks a target cannot intercept (single definition). */
+export function unInterceptableMocksMessage(mockCount) {
+    return (`this case declares ${mockCount} tool mock(s), but the target agent ` +
+        `cannot intercept agent-initiated tool calls (interception would be partial: ` +
+        `scripts' PTC calls mocked, the agent's own calls real). Wire ` +
+        `createToolMockMiddleware() into the target's middleware, or set ` +
+        `toolMocks on a target that intercepts by other means.`);
+}
+/** The view of one settled {@link Outcome}, as assertions read it. */
+function viewOf(session, outcome) {
+    const view = messagesToSessionView(outcome.messages, session.name, outcome.auditTrail, outcome.variables);
+    if (outcome.kind === "parked") {
+        view.parked = true;
+        if (outcome.parkedChannel)
+            view.parkedChannel = outcome.parkedChannel;
+        if (outcome.state)
+            view.node = outcome.state;
+    }
+    return view;
+}
+/**
+ * Create the per-case session engine. The default session runs on
+ * `defaultSessionId` (caller-nameable via `sessionIdForCase`); every other
+ * session's id derives from it, so naming the default names the whole case and
+ * no session is a timestamped orphan.
+ */
+export function createSessionEngine(opts) {
+    const { agent, defaultSessionId } = opts;
+    const runConfig = opts.signal ? { signal: opts.signal } : {};
+    const sessions = new Map();
+    function get(sessionId) {
+        const session = sessions.get(sessionId);
+        if (!session) {
+            throw new Error(`unknown session '${sessionId}' (opened: ${[...sessions.keys()].join(", ") || "none"})`);
+        }
+        return session;
+    }
+    async function settle(session, outcome) {
+        const view = viewOf(session, await outcome);
+        session.lastView = view;
+        return view;
+    }
+    return {
+        async open(name, conditions = {}) {
+            if (sessions.has(name))
+                throw new Error(`session '${name}' is already open`);
+            const session = {
+                name,
+                sessionId: name === DEFAULT_SESSION_ID ? defaultSessionId : `${defaultSessionId}-${name}`,
+                lastView: messagesToSessionView([], name),
+            };
+            await applyStartConditions(agent, session, conditions);
+            sessions.set(name, session);
+            return session;
+        },
+        async send(sessionId, message, toolMocks = []) {
+            const session = get(sessionId);
+            // Every turn of a session runs under the session's declared trigger and
+            // seeds — consistency by construction: a send step carries neither, so a
+            // follow-up message cannot restart the session at a different start state.
+            // Mocks ride the config, the one per-invocation channel; the runtime
+            // decides whether this is a turn or a reply to a session a person holds.
+            return settle(session, agent.workflow.send(session.sessionId, {
+                message,
+                ...(session.trigger ? { trigger: session.trigger } : {}),
+                ...(session.variables ? { variables: session.variables } : {}),
+            }, { ...runConfig, configurable: { __toolMocks: toolMocks } }));
+        },
+        async deliver(sessionId, trigger, variables) {
+            const session = get(sessionId);
+            return settle(session, agent.workflow.send(session.sessionId, { delivery: { trigger: { id: trigger }, ...(variables ? { variables } : {}) } }, runConfig));
+        },
+        async decide(sessionId, to, comment) {
+            const session = get(sessionId);
+            return settle(session, agent.workflow.send(session.sessionId, { decision: { target: to, ...(comment ? { comment } : {}) } }, runConfig));
+        },
+    };
+}
+//# sourceMappingURL=driver.js.map

package/dist/testing/driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.js","sourceRoot":"","sources":["../../src/testing/driver.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,qBAAqB,EAAoB,MAAM,6BAA6B,CAAC;AAItF,6DAA6D;AAC7D,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC;AA8B5C;;;;;GAKG;AACH,KAAK,UAAU,oBAAoB,CACjC,KAAoB,EACpB,OAAqB,EACrB,KAA6B;IAE7B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7C,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAClC,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/D,OAAO,CAAC,SAAS,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,IAAI,OAAO,KAAK,CAAC,QAAQ,EAAE,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,sBAAsB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,qCAAqC;gBAClF,wFAAwF;gBACxF,0FAA0F;gBAC1F,6BAA6B,CAChC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB;IACvC,OAAO,CACL,gFAAgF;QAChF,sFAAsF;QACtF,sEAAsE,CACvE,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,2BAA2B,CAAC,SAAiB;IAC3D,OAAO,CACL,sBAAsB,SAAS,sCAAsC;QACrE,8EAA8E;QAC9E,+DAA+D;QAC/D,kEAAkE;QAClE,uDAAuD,CACxD,CAAC;AACJ,CAAC;AAkBD,sEAAsE;AACtE,SAAS,MAAM,CAAC,OAAqB,EAAE,OAAgB;IACrD,MAAM,IAAI,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1G,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,OAAO,CAAC,aAAa;YAAE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QACtE,IAAI,OAAO,CAAC,KAAK;YAAE,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAKnC;IACC,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEjD,SAAS,GAAG,CAAC,SAAiB;QAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,oBAAoB,SAAS,cAAc,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG,CACxF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,UAAU,MAAM,CAAC,OAAqB,EAAE,OAAyB;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC;QAC5C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,GAAG,EAAE;YAC9B,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,mBAAmB,CAAC,CAAC;YAC7E,MAAM,OAAO,GAAiB;gBAC5B,IAAI;gBACJ,SAAS,EAAE,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,gBAAgB,IAAI,IAAI,EAAE;gBACzF,QAAQ,EAAE,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC;aAC1C,CAAC;YACF,MAAM,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC5B,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,GAAG,EAAE;YAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,wEAAwE;YACxE,yEAAyE;YACzE,2EAA2E;YAC3E,qEAAqE;YACrE,yEAAyE;YACzE,OAAO,MAAM,CACX,OAAO,EACP,KAAK,CAAC,QAAQ,CAAC,IAAI,CACjB,OAAO,CAAC,SAAS,EACjB;gBACE,OAAO;gBACP,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/D,EACD,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,CAC3D,CACF,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS;YACzC,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,MAAM,CACX,OAAO,EACP,KAAK,CAAC,QAAQ,CAAC,IAAI,CACjB,OAAO,CAAC,SAAS,EACjB,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAC/E,SAAS,CACV,CACF,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,EAAE,OAAO;YACjC,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,MAAM,CACX,OAAO,EACP,KAAK,CAAC,QAAQ,CAAC,IAAI,CACjB,OAAO,CAAC,SAAS,EACjB,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAC7D,SAAS,CACV,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/testing/grade.d.ts

@@ -0,0 +1,116 @@
+/**
+ * The grader behind a `grade:` assertion.
+ *
+ * Two halves of one job: the **evidence** a graded turn is shown — what the
+ * session said, separated from what it did, bounded with visible elision — and
+ * the **prompt** that grades it against the case's stated expectation. They are
+ * one module because the evidence exists only to be graded; nothing else reads it.
+ *
+ * A grade is a soft threshold, not a gate: it scores, and the case's declared bar
+ * decides pass or fail.
+ */
+import type { SessionView, ToolCallFact } from "../workflow/session-view.js";
+import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
+import { type ModelFactory } from "../env.js";
+import type { WorkflowTestsConfig } from "../machine/types.js";
+/**
+ * The evidence an offline-test `judge:` assertion hands the grading model:
+ * the turn's final reply *and* a chronological record of how the run reached
+ * it. Grading only the last assistant message grades prose — a criterion about
+ * what the agent *did* ("looked up the order before answering", "never fetched
+ * another customer's record") has nothing to read.
+ *
+ * Building and rendering are separate on purpose: composition (ordering,
+ * call/output pairing, bounding) is asserted against a plain object, and only
+ * the final string formatting is prompt-shaped.
+ */
+export type JudgeEvidenceEntry = {
+    kind: "message";
+    text: string;
+}
+/** Something the runtime wrote into the transcript: a decision, a delivered event, an error route. */
+ | {
+    kind: "runtime";
+    note: string;
+    text: string;
+} | {
+    kind: "tool";
+    name: string;
+    input: unknown;
+    output?: unknown;
+    status: ToolCallFact["status"];
+};
+export interface JudgeEvidence {
+    /** The turn's final assistant reply — what the user was told. */
+    reply: string;
+    /**
+     * The turn in order: assistant messages and tool calls interleaved as they
+     * happened. The final reply is not repeated here when it closed the turn —
+     * it is rendered in its own section.
+     */
+    record: JudgeEvidenceEntry[];
+}
+/**
+ * Per-value character cap for a rendered tool input, tool output, or record
+ * message. The reply is deliberately exempt (see {@link renderJudgeEvidence}).
+ */
+export declare const MAX_VALUE_CHARS = 800;
+/** Total character budget for the rendered record section. */
+export declare const MAX_RECORD_CHARS = 12000;
+/**
+ * Derive judge evidence from the view the step's other assertions evaluate
+ * against, so a `judge` and a `calledTool` on the same step can never disagree
+ * about what the run did.
+ *
+ * Order comes from `events` (chronological, but with no call→output pairing)
+ * and pairing from `toolCalls` (paired by tool-call id, but flat). Both are
+ * built in the same pass of `messagesToSessionView`, so the nth `tool.called` event
+ * is `toolCalls[n]` — reading them together is exact, where re-pairing the
+ * events by name here would re-implement that matching less accurately.
+ */
+export declare function judgeEvidenceFromView(view: SessionView): JudgeEvidence;
+/**
+ * Render evidence for the grading prompt. Every reduction is visible: a capped
+ * value names the characters it dropped and a record trimmed to its budget
+ * opens with the number of entries omitted. A judge that silently received
+ * less than the run produced would grade a run nobody can reconstruct.
+ */
+export declare function renderJudgeEvidence(evidence: JudgeEvidence): string;
+/**
+ * What the judge model said: a score in `[0, 1]` and its reasoning. There is
+ * deliberately no pass/fail here — the case's declared `atLeast` decides that,
+ * in `assertions.ts`, so a record can never contradict the verdict beside it.
+ */
+export interface JudgeResult {
+    score: number;
+    reason?: string;
+}
+/**
+ * Resolve the judge model through the model factory's `judge` role (default:
+ * the env-configured model). `tests.judge.model` and `tests.judge.modelOptions`
+ * (`temperature`, `maxTokens`) are applied over the environment the factory is
+ * handed, so the default factory builds exactly the grader the spec names; a
+ * factory that brings its own model ignores the env, and these with it.
+ *
+ * The environment is loaded lazily (same shape as the harness): a factory that
+ * carries its own credentials never reads it, so a host with no `ARCHMAX_*`
+ * variables can still grade a `judge:` expectation.
+ */
+export declare function createJudgeModel(config: WorkflowTestsConfig, modelFactory?: ModelFactory): BaseChatModel;
+/**
+ * Grade one closed-QA criterion against a turn's run evidence (the offline-test
+ * judge). The evidence carries what the assistant *did* as well as what it
+ * said, so the two are weighed differently in the prompt: a criterion about
+ * what the user was told is met only by the final reply, while a criterion
+ * about behaviour may be met by the tool record. Without that split, adding the
+ * record would let "tells the customer the refund is approved" pass on the
+ * strength of an `approve_refund` call the customer never heard about.
+ */
+export declare function gradeClosedQA(model: BaseChatModel, criteria: string, evidence: JudgeEvidence): Promise<JudgeResult>;
+/**
+ * Read the verdict out of a grader's reply. The reply is scanned for balanced
+ * `{...}` blocks — so prose around the JSON, a markdown fence, or a brace
+ * inside the prose (`the reply {mentions} …`) is tolerated — and the first
+ * block that parses as an object wins. `null` when nothing parses.
+ */
+export declare function parseVerdict(raw: string): JudgeResult | null;

package/dist/testing/grade.js

@@ -0,0 +1,261 @@
+import { HumanMessage, SystemMessage } from "@langchain/core/messages";
+import { defaultModelFactory, loadEnv } from "../env.js";
+/**
+ * Per-value character cap for a rendered tool input, tool output, or record
+ * message. The reply is deliberately exempt (see {@link renderJudgeEvidence}).
+ */
+export const MAX_VALUE_CHARS = 800;
+/** Total character budget for the rendered record section. */
+export const MAX_RECORD_CHARS = 12_000;
+/**
+ * Derive judge evidence from the view the step's other assertions evaluate
+ * against, so a `judge` and a `calledTool` on the same step can never disagree
+ * about what the run did.
+ *
+ * Order comes from `events` (chronological, but with no call→output pairing)
+ * and pairing from `toolCalls` (paired by tool-call id, but flat). Both are
+ * built in the same pass of `messagesToSessionView`, so the nth `tool.called` event
+ * is `toolCalls[n]` — reading them together is exact, where re-pairing the
+ * events by name here would re-implement that matching less accurately.
+ */
+export function judgeEvidenceFromView(view) {
+    const record = [];
+    let nthCall = 0;
+    for (const event of view.events) {
+        if (event.type === "message.completed") {
+            const text = String(event.data?.message ?? "");
+            if (text)
+                record.push({ kind: "message", text });
+            continue;
+        }
+        if (event.type === "runtime.note") {
+            const text = String(event.data?.text ?? "");
+            if (text)
+                record.push({ kind: "runtime", note: String(event.data?.kind ?? ""), text });
+            continue;
+        }
+        if (event.type !== "tool.called")
+            continue;
+        // `tool.completed` carries no id, so its output is read from the paired
+        // fact instead — the event only marks position, which is already known.
+        const fact = view.toolCalls[nthCall++];
+        record.push({
+            kind: "tool",
+            name: fact?.name ?? String(event.data?.name ?? ""),
+            input: fact ? fact.input : event.data?.input,
+            ...(fact?.output !== undefined ? { output: fact.output } : {}),
+            status: fact?.status ?? "pending",
+        });
+    }
+    const reply = view.reply ?? "";
+    // The closing message is the graded reply; it gets its own section, so
+    // carrying it in the record too would spend budget saying it twice. Only
+    // when it *closed* the turn — a reply followed by tool calls stays in place,
+    // because "said this, then did that" is the ordering the judge needs.
+    const last = record[record.length - 1];
+    if (reply && last?.kind === "message" && last.text === reply)
+        record.pop();
+    return { reply, record };
+}
+/**
+ * Render evidence for the grading prompt. Every reduction is visible: a capped
+ * value names the characters it dropped and a record trimmed to its budget
+ * opens with the number of entries omitted. A judge that silently received
+ * less than the run produced would grade a run nobody can reconstruct.
+ */
+export function renderJudgeEvidence(evidence) {
+    return [
+        "What the assistant did, in order:",
+        renderRecord(evidence.record),
+        "",
+        "Final reply to the user:",
+        evidence.reply || "(empty)",
+    ].join("\n");
+}
+function renderRecord(record) {
+    if (record.length === 0)
+        return "(no tool calls or intermediate messages recorded)";
+    // Numbered by original position, so the numbering stays coherent with the
+    // omission marker when the oldest entries are dropped.
+    const lines = record.map((entry, i) => `${i + 1}. ${renderEntry(entry)}`);
+    // Oldest-first dropping: the graded reply is the end of the turn, so the
+    // work nearest it is what a criterion most often concerns.
+    let kept = 0;
+    let total = 0;
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const next = total + lines[i].length + 1;
+        if (kept > 0 && next > MAX_RECORD_CHARS)
+            break;
+        total = next;
+        kept += 1;
+    }
+    const dropped = lines.length - kept;
+    const body = lines.slice(dropped);
+    return dropped > 0
+        ? [
+            `… ${dropped} earlier ${dropped === 1 ? "entry" : "entries"} omitted (evidence budget)`,
+            ...body,
+        ].join("\n")
+        : body.join("\n");
+}
+function renderEntry(entry) {
+    if (entry.kind === "message")
+        return `[message] ${cap(entry.text)}`;
+    if (entry.kind === "runtime")
+        return `[runtime:${entry.note}] ${cap(entry.text)}`;
+    const output = entry.output === undefined ? "(no output recorded)" : `output=${cap(stringify(entry.output))}`;
+    return `[tool] ${entry.name} input=${cap(stringify(entry.input))} → ${entry.status} ${output}`;
+}
+function stringify(value) {
+    if (typeof value === "string")
+        return value;
+    try {
+        return JSON.stringify(value) ?? String(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function cap(text) {
+    if (text.length <= MAX_VALUE_CHARS)
+        return text;
+    return `${text.slice(0, MAX_VALUE_CHARS)} …(+${text.length - MAX_VALUE_CHARS} chars)`;
+}
+/**
+ * Resolve the judge model through the model factory's `judge` role (default:
+ * the env-configured model). `tests.judge.model` and `tests.judge.modelOptions`
+ * (`temperature`, `maxTokens`) are applied over the environment the factory is
+ * handed, so the default factory builds exactly the grader the spec names; a
+ * factory that brings its own model ignores the env, and these with it.
+ *
+ * The environment is loaded lazily (same shape as the harness): a factory that
+ * carries its own credentials never reads it, so a host with no `ARCHMAX_*`
+ * variables can still grade a `judge:` expectation.
+ */
+export function createJudgeModel(config, modelFactory = defaultModelFactory) {
+    const { model, modelOptions = {} } = config.judge ?? {};
+    let cachedEnv;
+    const env = () => (cachedEnv ??= {
+        ...loadEnv(),
+        ...(model ? { model } : {}),
+        ...(modelOptions.temperature !== undefined ? { temperature: modelOptions.temperature } : {}),
+        ...(modelOptions.maxTokens !== undefined ? { maxTokens: modelOptions.maxTokens } : {}),
+    });
+    return modelFactory("judge", env);
+}
+/**
+ * Grade one closed-QA criterion against a turn's run evidence (the offline-test
+ * judge). The evidence carries what the assistant *did* as well as what it
+ * said, so the two are weighed differently in the prompt: a criterion about
+ * what the user was told is met only by the final reply, while a criterion
+ * about behaviour may be met by the tool record. Without that split, adding the
+ * record would let "tells the customer the refund is approved" pass on the
+ * strength of an `approve_refund` call the customer never heard about.
+ */
+export async function gradeClosedQA(model, criteria, evidence) {
+    const messages = [
+        new SystemMessage([
+            "You grade whether an assistant's turn satisfies a yes/no criterion.",
+            "You are given a chronological record of the turn — the assistant's messages, the tool calls it made with their inputs, outputs and status, and the runtime's own notes (marked [runtime:<kind>]: a person's decision that routed the run, an event delivered to it, an error route) — followed by the assistant's final reply to the user.",
+            "Weigh them differently: a criterion about what the user was told is met only by the final reply, while a criterion about what the assistant did, or what happened to the run, may be met by the record.",
+            'Reply with JSON only: {"score": number between 0 and 1, "pass": boolean, "reason": string}',
+            "score 1 means the criterion is fully met; 0 means it is not met at all.",
+            // The reason is printed beside a verdict line, so it has to fit there.
+            // A bound in the prompt, not a truncation: cutting a model's
+            // explanation mid-clause reads worse than asking for a short one.
+            "Keep reason to 1-3 sentences: state the judgment and the evidence for it, nothing more.",
+        ].join(" ")),
+        new HumanMessage(`Criterion: ${criteria}\n\n${renderJudgeEvidence(evidence)}`),
+    ];
+    const first = await model.invoke(messages);
+    const verdict = parseVerdict(responseText(first));
+    if (verdict)
+        return verdict;
+    // A grader that wrapped or mangled its JSON gets exactly one reminder before
+    // its turn scores 0: a real model run failed a case this way, and re-asking
+    // is cheaper than a false failure. The retry carries the first reply, so the
+    // model corrects its own answer rather than grading afresh.
+    const retry = await model.invoke([
+        ...messages,
+        first,
+        new HumanMessage("Your previous reply could not be read as JSON. Reply with the JSON object only — " +
+            'no prose, no code fences: {"score": number, "pass": boolean, "reason": string}'),
+    ]);
+    return (parseVerdict(responseText(retry)) ?? {
+        score: 0,
+        reason: "grader returned no readable JSON verdict",
+    });
+}
+/** The textual content of a model reply, whatever shape the provider used. */
+function responseText(response) {
+    const { content } = response;
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((p) => (typeof p === "string" ? p : (p.text ?? "")))
+            .join("");
+    }
+    return String(content ?? "");
+}
+/**
+ * Read the verdict out of a grader's reply. The reply is scanned for balanced
+ * `{...}` blocks — so prose around the JSON, a markdown fence, or a brace
+ * inside the prose (`the reply {mentions} …`) is tolerated — and the first
+ * block that parses as an object wins. `null` when nothing parses.
+ */
+export function parseVerdict(raw) {
+    for (const block of balancedObjects(raw)) {
+        try {
+            const parsed = JSON.parse(block);
+            if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
+                continue;
+            // The prompt asks the model for its own `pass` boolean (unchanged
+            // wording, so grading stays calibrated); it is not read — the declared
+            // `atLeast` is the only bar.
+            const { score, reason } = parsed;
+            return {
+                score: Math.max(0, Math.min(1, Number(score ?? 0))),
+                reason: typeof reason === "string" ? reason : undefined,
+            };
+        }
+        catch {
+            // Not JSON: try the next candidate.
+        }
+    }
+    return null;
+}
+/** Every balanced `{...}` block in `raw`, in order of its opening brace. */
+function* balancedObjects(raw) {
+    let start = raw.indexOf("{");
+    while (start !== -1) {
+        let depth = 0;
+        let inString = false;
+        let escaped = false;
+        let end = -1;
+        for (let i = start; i < raw.length; i++) {
+            const ch = raw[i];
+            if (inString) {
+                if (escaped)
+                    escaped = false;
+                else if (ch === "\\")
+                    escaped = true;
+                else if (ch === '"')
+                    inString = false;
+                continue;
+            }
+            if (ch === '"')
+                inString = true;
+            else if (ch === "{")
+                depth += 1;
+            else if (ch === "}" && --depth === 0) {
+                end = i;
+                break;
+            }
+        }
+        if (end !== -1)
+            yield raw.slice(start, end + 1);
+        start = raw.indexOf("{", start + 1);
+    }
+}
+//# sourceMappingURL=grade.js.map

package/dist/testing/grade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../src/testing/grade.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAiB,mBAAmB,EAAE,OAAO,EAAqB,MAAM,WAAW,CAAC;AAsC3F;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AAEnC,8DAA8D;AAC9D,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEvC;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAiB;IACrD,MAAM,MAAM,GAAyB,EAAE,CAAC;IACxC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;YAC/C,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;YACvF,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;YAAE,SAAS;QAC3C,wEAAwE;QACxE,wEAAwE;QACxE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YAClD,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK;YAC5C,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,SAAS;SAClC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IAC/B,uEAAuE;IACvE,yEAAyE;IACzE,6EAA6E;IAC7E,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,IAAI,KAAK,IAAI,IAAI,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;QAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IAE3E,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAuB;IACzD,OAAO;QACL,mCAAmC;QACnC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC7B,EAAE;QACF,0BAA0B;QAC1B,QAAQ,CAAC,KAAK,IAAI,SAAS;KAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CAAC,MAAqC;IACzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,mDAAmD,CAAC;IAEpF,0EAA0E;IAC1E,uDAAuD;IACvD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAE1E,yEAAyE;IACzE,2DAA2D;IAC3D,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACzC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,gBAAgB;YAAE,MAAM;QAC/C,KAAK,GAAG,IAAI,CAAC;QACb,IAAI,IAAI,CAAC,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,OAAO,GAAG,CAAC;QAChB,CAAC,CAAC;YACE,KAAK,OAAO,YAAY,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,4BAA4B;YACvF,GAAG,IAAI;SACR,CAAC,IAAI,CAAC,IAAI,CAAC;QACd,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,aAAa,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;IACpE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,YAAY,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;IAClF,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;IACjG,OAAO,UAAU,KAAK,CAAC,IAAI,UAAU,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;AACjG,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,GAAG,CAAC,IAAY;IACvB,IAAI,IAAI,CAAC,MAAM,IAAI,eAAe;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,OAAO,IAAI,CAAC,MAAM,GAAG,eAAe,SAAS,CAAC;AACxF,CAAC;AAYD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAA2B,EAC3B,eAA6B,mBAAmB;IAEhD,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IACxD,IAAI,SAA+B,CAAC;IACpC,MAAM,GAAG,GAAG,GAAa,EAAE,CACzB,CAAC,SAAS,KAAK;QACb,GAAG,OAAO,EAAE;QACZ,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,YAAY,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,YAAY,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvF,CAAC,CAAC;IACL,OAAO,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAoB,EACpB,QAAgB,EAChB,QAAuB;IAEvB,MAAM,QAAQ,GAAG;QACf,IAAI,aAAa,CACf;YACE,qEAAqE;YACrE,4UAA4U;YAC5U,yMAAyM;YACzM,4FAA4F;YAC5F,yEAAyE;YACzE,uEAAuE;YACvE,6DAA6D;YAC7D,kEAAkE;YAClE,yFAAyF;SAC1F,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ;QACD,IAAI,YAAY,CAAC,cAAc,QAAQ,OAAO,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;KAC/E,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IAClD,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAE5B,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,4DAA4D;IAC5D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;QAC/B,GAAG,QAAQ;QACX,KAAK;QACL,IAAI,YAAY,CACd,mFAAmF;YACjF,gFAAgF,CACnF;KACF,CAAC,CAAC;IACH,OAAO,CACL,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI;QACnC,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,0CAA0C;KACnD,CACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,SAAS,YAAY,CAAC,QAA8B;IAClD,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAuB,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;aAC/E,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAC;YAC5C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrF,kEAAkE;YAClE,uEAAuE;YACvE,6BAA6B;YAC7B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAA+C,CAAC;YAC1E,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBACnD,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;aACxD,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4EAA4E;AAC5E,QAAQ,CAAC,CAAC,eAAe,CAAC,GAAW;IACnC,IAAI,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAClB,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,OAAO;oBAAE,OAAO,GAAG,KAAK,CAAC;qBACxB,IAAI,EAAE,KAAK,IAAI;oBAAE,OAAO,GAAG,IAAI,CAAC;qBAChC,IAAI,EAAE,KAAK,GAAG;oBAAE,QAAQ,GAAG,KAAK,CAAC;gBACtC,SAAS;YACX,CAAC;YACD,IAAI,EAAE,KAAK,GAAG;gBAAE,QAAQ,GAAG,IAAI,CAAC;iBAC3B,IAAI,EAAE,KAAK,GAAG;gBAAE,KAAK,IAAI,CAAC,CAAC;iBAC3B,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;gBACrC,GAAG,GAAG,CAAC,CAAC;gBACR,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QAChD,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;AACH,CAAC"}

package/dist/testing/interpreter.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Host-side interpretation of one parsed case document: open the case's
+ * session with its start conditions, then walk the flat step list — actions
+ * (`send`, `decide`, `deliver`) drive the agent through `workflow.send`, and each
+ * assertion evaluates against the view of the nearest action above it. No sandbox is involved —
+ * the QuickJS sandbox serves only lifecycle hooks and the agent's own sandbox
+ * tools inside the driven agent.
+ *
+ * A case that dies partway is still reported: termination is returned as an
+ * `error` on the outcome, next to the records the case had already produced,
+ * so the steps that passed before it are never thrown away.
+ */
+import type { GovernedAgent } from "../agent.js";
+import { type GradeFn } from "./assertions.js";
+import type { CaseDocument } from "./case-schema.js";
+import type { AssertionRecord } from "./runner.js";
+export interface ExecuteCaseOptions {
+    /** Aborts the runs the case drives (the case's wall-clock budget). */
+    signal?: AbortSignal;
+    doc: CaseDocument;
+    /** Seed files, `from:` references already materialized by the runner. */
+    seeds: Record<string, unknown>;
+    agent: GovernedAgent;
+    /** The runtime session id for this case (caller-nameable via `sessionIdForCase`). */
+    sessionId: string;
+    grade?: GradeFn;
+    /**
+     * Invoked with each assertion record as it is produced — observe-only; the
+     * record objects are the same ones returned in the outcome. A caller that
+     * may stop waiting on this promise (the runner abandons it at the case's
+     * wall-clock deadline) uses this to hold the records produced before it let
+     * go, which the return value can no longer deliver.
+     */
+    onRecord?: (record: AssertionRecord) => void;
+}
+export interface ExecuteCaseOutcome {
+    records: AssertionRecord[];
+    /**
+     * Set when the case terminated before its steps were exhausted: a failed
+     * `send`/`decide`, a refused start (bad trigger, failed seed,
+     * un-interceptable mocks). The records produced before that point are still
+     * returned alongside it — a case's failure is reported as data, not by
+     * throwing past what it already established.
+     */
+    error?: string;
+}
+/**
+ * Describe every assertion step a case did not account for, as `not-executed`
+ * records. The one place un-run steps are built, so the three ways a case can
+ * stop early — a structural halt, a thrown action, and the runner's wall-clock
+ * timeout — cannot drift apart in what they report.
+ */
+export declare function notExecutedRecords(doc: CaseDocument, alreadyAccountedFor: (step: number) => boolean): AssertionRecord[];
+/** Execute one case document against the agent under test. */
+export declare function executeCase(opts: ExecuteCaseOptions): Promise<ExecuteCaseOutcome>;