eve 0.27.13 → 0.29.0

package/dist/src/context/dynamic-model-lifecycle.d.ts

@@ -2,7 +2,7 @@ import type { ModelMessage } from "ai";
 import type { AlsContext } from "#context/container.js";
 import type { ContextKey } from "#context/key.js";
 import { type LiveDynamicModelSelection } from "#context/keys.js";
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
 import type { RuntimeDynamicModelReference, RuntimeModelReference } from "#runtime/agent/bootstrap.js";
 import { type RuntimeModelResolutionScope } from "#runtime/agent/resolve-model.js";
 export type ActiveDynamicModelSelection = LiveDynamicModelSelection;
@@ -12,7 +12,7 @@ export declare function getActiveDynamicModelSelection(ctx: {
 export declare function dispatchDynamicModelEvent(input: {
     readonly ctx: AlsContext;
     readonly dynamicModel: RuntimeDynamicModelReference | undefined;
-    readonly event: HandleMessageStreamEvent;
+    readonly event: UnstampedMessageStreamEvent;
     readonly fallback: RuntimeModelReference;
     readonly messages: readonly ModelMessage[];
     readonly scope: RuntimeModelResolutionScope;

package/dist/src/context/dynamic-skill-lifecycle.d.ts

@@ -1,5 +1,5 @@
 import type { ModelMessage } from "ai";
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
 import type { ResolvedDynamicSkillResolver } from "#runtime/types.js";
 import type { ContextContainer } from "#context/container.js";
 import { ContextKey } from "#context/key.js";
@@ -19,6 +19,6 @@ export declare const PendingSkillAnnouncementKey: ContextKey<string>;
 export declare function dispatchDynamicSkillEvent(input: {
     readonly ctx: ContextContainer;
     readonly resolvers: readonly ResolvedDynamicSkillResolver[];
-    readonly event: HandleMessageStreamEvent;
+    readonly event: UnstampedMessageStreamEvent;
     readonly messages: readonly ModelMessage[];
 }): Promise<void>;

package/dist/src/context/dynamic-tool-lifecycle.d.ts

@@ -1,6 +1,6 @@
 import type { ModelMessage } from "ai";
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
-import type { HandleMessageStreamEvent, SessionStartedStreamEvent } from "#protocol/message.js";
+import type { UnstampedMessageStreamEvent, SessionStartedStreamEvent } from "#protocol/message.js";
 import type { ResolvedDynamicToolResolver } from "#runtime/types.js";
 import type { ContextContainer } from "#context/container.js";
 import type { DurableDynamicToolMetadata } from "#context/keys.js";
@@ -20,7 +20,7 @@ export declare function replayDynamicSessionTools(metadata: readonly DurableDyna
 export declare function dispatchDynamicToolEvent(input: {
     readonly ctx: ContextContainer;
     readonly resolvers: readonly ResolvedDynamicToolResolver[];
-    readonly event: HandleMessageStreamEvent;
+    readonly event: UnstampedMessageStreamEvent;
     readonly messages: readonly ModelMessage[];
 }): Promise<void>;
 /**

package/dist/src/context/hook-lifecycle.d.ts

@@ -1,4 +1,4 @@
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent } from "#protocol/message.js";
 import type { RuntimeHookRegistry } from "#runtime/hooks/registry.js";
 import type { ContextContainer } from "./container.js";
 /**
@@ -10,5 +10,5 @@ import type { ContextContainer } from "./container.js";
 export declare function dispatchStreamEventHooks(input: {
     readonly ctx: ContextContainer;
     readonly registry: RuntimeHookRegistry;
-    readonly event: HandleMessageStreamEvent;
+    readonly event: MessageStreamEvent;
 }): Promise<void>;

package/dist/src/evals/assertions/collector.d.ts

@@ -1,14 +1,6 @@
-import type { AssertionHandle, AssertionResult, AssertionSeverity, EveEvalTaskResult } from "#evals/types.js";
+import type { AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalTaskResult } from "#evals/types.js";
 import type { EveEvalAssertionSubject } from "#evals/assertions/run.js";
-/**
- * Outcome of evaluating one assertion: a 0–1 score (boolean assertions use
- * exactly 0 or 1) with optional human-readable detail and metadata.
- */
-export interface AssertionOutcome {
-    readonly score: number;
-    readonly message?: string;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
+export type AssertionOutcome = AssertionEvaluation;
 /**
  * A scoped assertion evaluated lazily after `test(t)` returns. The selected
  * subject may be the aggregate run, one session, or one immutable turn.

package/dist/src/evals/assertions/collector.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";var AssertionCollector=class{#e=[];#t=[];get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){let r={name:e.name,severity:n,threshold:void 0,kind:`deferred`,spec:e,selectSubject:t,score:0,failed:!1};return this.#e.push(r),makeHandle(r)}recordValue(e){let t={name:e.name,severity:e.severity,threshold:e.threshold,kind:`resolved`,score:0,failed:!1};this.#e.push(t);let n=settleEntry(t,e.score);return this.#t.push(n),makeHandle(t)}recordOutcome(e){let t={name:e.name,severity:`gate`,threshold:void 0,kind:`resolved`,score:e.outcome.score,message:e.outcome.message,metadata:e.outcome.metadata,failed:!1};return this.#e.push(t),computePassed(t.severity,t.threshold,t.score,t.failed)}async recordRequirement(e){let t={name:e.name,severity:`gate`,threshold:e.threshold,kind:`resolved`,score:0,failed:!1};return this.#e.push(t),await settleEntry(t,e.score),computePassed(t.severity,t.threshold,t.score,t.failed)}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push({name:n.name,score:n.score,severity:n.severity,threshold:n.threshold,passed:computePassed(n.severity,n.threshold,n.score,n.failed),message:n.message,metadata:n.metadata})}return t}};function computePassed(e,t,n,r){if(r)return!1;let i=t??(e===`gate`?1:void 0);return i===void 0||n>=i}async function settleEntry(t,n){try{let e=await n();t.score=e.score,t.message=e.message,t.metadata=e.metadata}catch(n){t.score=0,t.severity=`gate`,t.threshold=void 0,t.message=toErrorMessage(n),t.failed=!0}}function makeHandle(e){let t={gate(n){return e.severity=`gate`,e.threshold=n,t},soft(n){return e.severity=`soft`,e.threshold=n,t},atLeast(n){return e.severity=`soft`,e.threshold=n,t}};return t}export{AssertionCollector};
+import{toErrorMessage}from"#shared/errors.js";import{formatAssertionName}from"#evals/diagnostics.js";var AssertionCollector=class{#e=[];#t=[];get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){let r={baseName:e.name,name:e.name,severity:n,threshold:void 0,kind:`deferred`,spec:e,selectSubject:t,score:0,failed:!1};return this.#e.push(r),makeHandle(r)}recordValue(e){let t={baseName:e.name,name:e.name,severity:e.severity,threshold:e.threshold,kind:`resolved`,score:0,failed:!1};this.#e.push(t);let n=settleEntry(t,e.score);return this.#t.push(n),makeHandle(t)}recordOutcome(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:void 0,kind:`resolved`,score:e.outcome.score,message:e.outcome.message,metadata:e.outcome.metadata,failed:!1};return this.#e.push(t),computePassed(t.severity,t.threshold,t.score,t.failed)}async recordRequirement(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:e.threshold,kind:`resolved`,score:0,failed:!1};return this.#e.push(t),await settleEntry(t,e.score),computePassed(t.severity,t.threshold,t.score,t.failed)}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push({name:n.name,score:n.score,severity:n.severity,threshold:n.threshold,passed:computePassed(n.severity,n.threshold,n.score,n.failed),message:n.message,metadata:n.metadata})}return t}};function computePassed(e,t,n,r){if(r)return!1;let i=t??(e===`gate`?1:void 0);return i===void 0||n>=i}async function settleEntry(t,n){try{let e=await n();t.score=e.score,t.message=e.message,t.metadata=e.metadata}catch(n){t.score=0,t.severity=`gate`,t.threshold=void 0,t.message=toErrorMessage(n),t.failed=!0}}function makeHandle(e){let n={gate(t){return e.severity=`gate`,e.threshold=t,n},soft(t){return e.severity=`soft`,e.threshold=t,n},atLeast(t){return e.severity=`soft`,e.threshold=t,n},label(r){return e.name=formatAssertionName(e.baseName,r),n}};return n}export{AssertionCollector};

package/dist/src/evals/assertions/run.d.ts

@@ -1,12 +1,12 @@
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent } from "#protocol/message.js";
 import { type EveEvalSkillLoadMatchOptions, type EveEvalSubagentCallMatchOptions, type EveEvalToolCallMatchOptions } from "#evals/match.js";
 import type { EveEvalEventMatch } from "#evals/match.js";
 import type { RunAssertion } from "#evals/assertions/collector.js";
 /** Minimal captured scope consumed by deterministic eval assertions. */
 export interface EveEvalAssertionSubject {
     readonly derived: import("#evals/types.js").EveEvalDerivedFacts;
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     readonly output: unknown;
     readonly status: "completed" | "failed" | "waiting";
 }
@@ -68,7 +68,7 @@ export declare function calledSubagent(name: string, options?: EveEvalSubagentCa
  * Escape hatch: asserts an arbitrary predicate over the full typed event
  * stream. `label` names the assertion in reports.
  */
-export declare function eventsSatisfy(label: string, predicate: (events: readonly HandleMessageStreamEvent[]) => boolean): RunAssertion;
+export declare function eventsSatisfy(label: string, predicate: (events: readonly MessageStreamEvent[]) => boolean): RunAssertion;
 /** Asserts a typed stream event exists, optionally constraining its count. */
 export declare function typedEvent(matcher: EveEvalEventMatch): RunAssertion;
 /** Asserts no typed stream event matches. */

package/dist/src/evals/cli/eval.d.ts

@@ -1,4 +1,5 @@
-interface EvalCliOptions {
+/** Parsed Commander options accepted by {@link runEvalCommand}. */
+export interface EvalCliOptions {
     url?: string;
     timeout?: string;
     maxConcurrency?: string;
@@ -8,6 +9,7 @@ interface EvalCliOptions {
     strict?: boolean;
     list?: boolean;
     tag?: string[];
+    excludeTag?: string[];
     verbose?: boolean;
 }
 type EvalCliLogger = {
@@ -19,7 +21,9 @@ type EvalCliLogger = {
  *
  * Exit codes: `0` when every executed eval passed its gate assertions (and
  * soft thresholds under `--strict`), `1` when any eval failed, `2` for runner
- * or configuration errors (no evals discovered, no evals matching filters).
+ * or configuration errors (no evals discovered, no evals matching `--tag`).
+ * When `--exclude-tag` removes every matching eval the run succeeds with
+ * nothing executed.
  */
 export declare function runEvalCommand(evalIds: readonly string[], options: EvalCliOptions, logger: EvalCliLogger): Promise<void>;
 export {};

package/dist/src/evals/cli/eval.js

@@ -1,4 +1,4 @@
-import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{createDevelopmentServer}from"#internal/nitro/host.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let s=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(s);let c=e.length>0?e:void 0,l=await discoverAndImportEvals(s,c);if(l.length===0){let e=await findMisplacedEvalDirs(s);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
+import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{createDevelopmentServer}from"#internal/nitro/host.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let s=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(s);let c=e.length>0?e:void 0,l=await discoverAndImportEvals(s,c);if(l.length===0){let e=await findMisplacedEvalDirs(s);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
 `+e.map(e=>`  - ${e}`).join(`
 `)+`
-eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):c?n.error(`No evals found matching: ${c.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let u=filterEvalsByTag(l,t.tag??[]);if(u.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let d,f;try{d=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),f=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(u,t.json===!0,n);return}let p;try{p=await discoverEvalConfig(s)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let m,h,g;try{if(t.url)g=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:s}),h=await resolveEvalTargetHandle({client:g,expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:t.url});else{m=createDevelopmentServer(s,{host:`127.0.0.1`,port:0});let e=await m.start();g=await createEvalClient({kind:`local`,url:e.url}),h=await resolveEvalTargetHandle({client:g,expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:e.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:u,config:p,target:h,client:g,appRoot:s,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:d,timeoutMs:f,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{m&&(await m.close(),await shutdownActiveSandboxHandles({log:e=>n.error(e)}))}let _=typeof process.exitCode==`number`?process.exitCode:0;process.exit(_)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};
+eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):c?n.error(`No evals found matching: ${c.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let u=t.tag??[],d=t.excludeTag??[],f=filterEvalsByTags({evaluations:l,includeTags:u,excludeTags:[]});if(f.length===0){n.error(`No evals matched the provided tags (${u.join(`, `)}).`),process.exitCode=2;return}let p=filterEvalsByTags({evaluations:f,includeTags:[],excludeTags:d});if(t.list===!0){printEvalList(p,t.json===!0,n);return}if(p.length===0){n.log(`All ${f.length} matching evals are excluded by tags (${d.join(`, `)}); nothing to run.`);return}let m,h;try{m=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),h=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let g;try{g=await discoverEvalConfig(s)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let _,v,y;try{if(t.url)y=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:s}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:t.url});else{_=createDevelopmentServer(s,{host:`127.0.0.1`,port:0});let e=await _.start();y=await createEvalClient({kind:`local`,url:e.url}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:e.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:p,config:g,target:v,client:y,appRoot:s,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:m,timeoutMs:h,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{_&&(await _.close(),await shutdownActiveSandboxHandles({log:e=>n.error(e)}))}let b=typeof process.exitCode==`number`?process.exitCode:0;process.exit(b)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};

package/dist/src/evals/cli/filter.d.ts

@@ -0,0 +1,14 @@
+import type { EveEval } from "#evals/types.js";
+/**
+ * Applies `--tag` and `--exclude-tag` filtering for the `eve eval` command.
+ *
+ * An eval runs when it carries at least one `includeTags` entry (or no
+ * include filter is given) and carries none of the `excludeTags` entries.
+ * Exclusion wins over inclusion so suite-level runs (e.g. a mock-model world
+ * suite excluding `real-model` evals) can never pull in an excluded eval.
+ */
+export declare function filterEvalsByTags(input: {
+    readonly evaluations: readonly EveEval[];
+    readonly includeTags: readonly string[];
+    readonly excludeTags: readonly string[];
+}): EveEval[];

package/dist/src/evals/cli/filter.js

@@ -0,0 +1 @@
+function filterEvalsByTags(e){return e.evaluations.filter(t=>{let n=t.tags??[];return e.includeTags.length>0&&!n.some(t=>e.includeTags.includes(t))?!1:!n.some(t=>e.excludeTags.includes(t))})}export{filterEvalsByTags};

package/dist/src/evals/context.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import"#evals/session.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(n){let i=n.collector,a=``,primary=()=>n.manager.primary,replyMessage=()=>n.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:n.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},cancel:()=>primary().cancel(),requireInputRequest:e=>primary().requireInputRequest(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),start:e=>(a=promptText(e),primary().start(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:n.signal,target:n.target,get reply(){return replyMessage()},log:n.log,sleep:e=>sleep(e,n.signal),newSession:()=>n.manager.newSession(),...createScopedAssertions(i,{timing:`final`,select:e=>e}),check:(e,t)=>recordCheck(i,e,t),require:(e,t)=>requireCheck(i,e,t),skip:e=>{throw e.trim().length===0?Error(`skip() requires a non-empty reason.`):i.hasEntries||n.manager.hasActivity()?Error(`skip() must be called before sending messages or recording assertions.`):new EvalSkipped(e)},judge:o},collector:i}}async function requireCheck(e,t,r){let i=r.gate(r.threshold);if(!await e.recordRequirement({name:i.name,threshold:i.threshold,score:async()=>({score:await i.score(t)})}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:async()=>({score:await n.score(t)})})}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};
+import"#evals/assertions/collector.js";import"#evals/session.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(n){let i=n.collector,a=``,primary=()=>n.manager.primary,replyMessage=()=>n.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:n.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},cancel:()=>primary().cancel(),requireInputRequest:e=>primary().requireInputRequest(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),start:e=>(a=promptText(e),primary().start(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:n.signal,target:n.target,get reply(){return replyMessage()},log:n.log,sleep:e=>sleep(e,n.signal),newSession:()=>n.manager.newSession(),...createScopedAssertions(i,{timing:`final`,select:e=>e}),check:(e,t)=>recordCheck(i,e,t),require:(e,t)=>requireCheck(i,e,t),skip:e=>{throw e.trim().length===0?Error(`skip() requires a non-empty reason.`):i.hasEntries||n.manager.hasActivity()?Error(`skip() must be called before sending messages or recording assertions.`):new EvalSkipped(e)},judge:o},collector:i}}async function requireCheck(e,t,r){let i=r.gate(r.threshold);if(!await e.recordRequirement({name:i.name,threshold:i.threshold,score:()=>evaluateAssertion(i,t)}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:()=>evaluateAssertion(n,t)})}async function evaluateAssertion(e,t){return e.evaluate===void 0?{score:await e.score(t)}:await e.evaluate(t)}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};

package/dist/src/evals/diagnostics.d.ts

@@ -0,0 +1,8 @@
+/** Adds an optional user label without replacing the assertion's stable family name. */
+export declare function formatAssertionName(name: string, label: string): string;
+/** Formats an assertion value for a compact human-readable diagnostic. */
+export declare function formatDiagnosticValue(value: unknown, maxLength?: number): string;
+/** Converts arbitrary assertion evidence into an artifact-safe JSON value. */
+export declare function toDiagnosticMetadataValue(value: unknown): unknown;
+/** Bounds diagnostic text while leaving full structured evidence in metadata. */
+export declare function truncateDiagnostic(text: string, maxLength: number): string;

package/dist/src/evals/diagnostics.js

@@ -0,0 +1 @@
+function formatAssertionName(e,t){let n=t.trim();if(n.length===0)throw Error(`Assertion labels must be non-empty.`);return`${e} [${n}]`}function formatDiagnosticValue(e,t=160){let n;try{n=JSON.stringify(e)??String(e)}catch{n=String(e)}return truncateDiagnostic(n,t)}function toDiagnosticMetadataValue(e){let t=[];try{let n=JSON.stringify(e,function(e,n){if(typeof n==`bigint`)return`${String(n)}n`;if(typeof n==`function`||typeof n==`symbol`)return String(n);if(n instanceof Error)return{name:n.name,message:n.message,stack:n.stack};if(typeof n==`object`&&n){for(;t.length>0&&t.at(-1)!==this;)t.pop();if(t.includes(n))return`[Circular]`;t.push(n)}return n});return n===void 0?String(e):JSON.parse(n)}catch{return String(e)}}function truncateDiagnostic(e,t){return e.length<=t?e:`${e.slice(0,t)}…`}export{formatAssertionName,formatDiagnosticValue,toDiagnosticMetadataValue,truncateDiagnostic};

package/dist/src/evals/expect/index.d.ts

@@ -1,6 +1,6 @@
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
 import type { Assertion } from "#evals/types.js";
-export type { Assertion, AssertionHandle, AssertionSeverity } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionSeverity, } from "#evals/types.js";
 /**
  * Passes when the value (coerced to a string) contains a substring or matches
  * a RegExp. A hard gate by default.

package/dist/src/evals/expect/index.js

@@ -1 +1 @@
-import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.1_bufferutil@4.1.0_/node_modules/autoevals/jsdist/index.js";import{deepEquals,testRegExp}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>{let r=String(t??``);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?1:0}})}function satisfies(e,t){if(t.trim().length===0)throw Error(`satisfies() requires a non-empty label.`);return makeAssertion({name:`satisfies(${t})`,severity:`gate`,score:t=>+!!e(t)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,satisfies,similarity};
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.1_bufferutil@4.1.0_/node_modules/autoevals/jsdist/index.js";import{formatDiagnosticValue,toDiagnosticMetadataValue}from"#evals/diagnostics.js";import{deepEquals,testRegExp}from"#evals/match.js";function makeAssertion(e){let evaluate=t=>{let n=e.evaluate(t);return n instanceof Promise?n.then(normalizeEvaluation):normalizeEvaluation(n)};return{name:e.name,severity:e.severity,threshold:e.threshold,evaluate,score(e){let t=evaluate(e);return t instanceof Promise?t.then(e=>e.score):t.score},gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function normalizeEvaluation(e){return typeof e==`number`?{score:e}:e}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,evaluate:n=>{let r=String(n??``);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?1:{score:0,message:`expected ${formatDiagnosticValue(r)} to include ${formatDiagnosticValue(String(e))}`,metadata:{actual:r,expected:String(e)}}}})}function satisfies(e,r){if(r.trim().length===0)throw Error(`satisfies() requires a non-empty label.`);return makeAssertion({name:`satisfies(${r})`,severity:`gate`,evaluate:i=>e(i)?1:{score:0,message:`predicate did not hold for ${formatDiagnosticValue(i)}`,metadata:{actual:toDiagnosticMetadataValue(i),predicate:r}}})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,evaluate:i=>deepEquals(i,e)?1:{score:0,message:`expected ${formatDiagnosticValue(e)}; received ${formatDiagnosticValue(i)}`,metadata:{actual:toDiagnosticMetadataValue(i),expected:toDiagnosticMetadataValue(e)}}})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,evaluate:async t=>{let r=await e[`~standard`].validate(t);return!(`issues`in r)||r.issues===void 0?1:{score:0,message:`schema validation failed: ${r.issues.map(formatSchemaIssue).join(`; `)}`,metadata:{actual:toDiagnosticMetadataValue(t),issues:toDiagnosticMetadataValue(r.issues)}}}})}function similarity(n){return makeAssertion({name:`similarity`,severity:`soft`,evaluate:async r=>{let i=String(r??``);return{score:(await Levenshtein({output:i,expected:n})).score??0,message:`expected similarity to ${formatDiagnosticValue(n)}; received ${formatDiagnosticValue(i)}`,metadata:{actual:i,expected:n}}}})}function formatSchemaIssue(e){return`${e.path===void 0?``:`${e.path.map(e=>String(typeof e==`object`?e.key:e)).join(`.`)}: `}${e.message}`}export{equals,includes,matches,satisfies,similarity};

package/dist/src/evals/index.d.ts

@@ -6,5 +6,5 @@ export type { RuntimeIdentity } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";

package/dist/src/evals/judge.js

@@ -1 +1,2 @@
-import{ClosedQA,Factuality,Sql,Summary}from"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.1_bufferutil@4.1.0_/node_modules/autoevals/jsdist/index.js";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import{createAutoevalsClient}from"#evals/autoevals-client.js";function buildJudgeContext(a){function grade(e,t,n){let r=t?.on??a.getReply(),o=String(r??``),s=a.getInput(),c=t?.model??a.judge?.model,l=t?.modelOptions??a.judge?.modelOptions;return a.collector.recordValue({name:e,severity:`soft`,score:async()=>{if(c===void 0)throw Error(`${e} needs a judge model. Set \`judge\` on the eval or in evals.config.ts, or pass { model } to the call.`);return{score:(await n({input:s,output:o,model:c,modelOptions:l})).score??0,metadata:{judge:formatLanguageModelGatewayId(c)}}}})}return{autoevals:{factuality:(e,n)=>grade(`judge.autoevals.factuality`,n,({input:n,output:r,model:i,modelOptions:a})=>Factuality({input:n,output:r,expected:e,...client(i,a)})),summarizes:(e,t)=>grade(`judge.autoevals.summarizes`,t,({input:t,output:n,model:i,modelOptions:a})=>Summary({input:t,output:n,expected:e,...client(i,a)})),closedQA:(t,n)=>grade(`judge.autoevals.closedQA`,n,({input:n,output:r,model:i,modelOptions:a})=>ClosedQA({input:n,output:r,criteria:t,...client(i,a)})),sql:(e,t)=>grade(`judge.autoevals.sql`,t,({input:t,output:r,model:i,modelOptions:a})=>Sql({input:t,output:r,expected:e,...client(i,a)}))}}}function client(e,t){return{model:formatLanguageModelGatewayId(e),client:createAutoevalsClient({languageModel:e,providerOptions:t?.providerOptions})}}export{buildJudgeContext};
+import{ClosedQA,Factuality,Sql,Summary}from"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.1_bufferutil@4.1.0_/node_modules/autoevals/jsdist/index.js";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import{formatDiagnosticValue}from"#evals/diagnostics.js";import{createAutoevalsClient}from"#evals/autoevals-client.js";function buildJudgeContext(a){function grade(e,t,n,r){let o=t?.on??a.getReply(),s=String(o??``),c=a.getInput(),l=t?.model??a.judge?.model,u=t?.modelOptions??a.judge?.modelOptions;return a.collector.recordValue({name:e,severity:`soft`,score:async()=>{if(l===void 0)throw Error(`${e} needs a judge model. Set \`judge\` on the eval or in evals.config.ts, or pass { model } to the call.`);let t=await r({input:c,output:s,model:l,modelOptions:u}),a=t.error===void 0?{}:{autoevalsError:t.error},o={...t.metadata,...a,judge:formatLanguageModelGatewayId(l),input:c,output:s,[n.label]:n.value,autoevalsName:t.name};return{score:t.score??0,message:formatJudgeDetail(c,s,n,t),metadata:o}}})}return{autoevals:{factuality:(e,n)=>grade(`judge.autoevals.factuality`,n,{label:`expected`,value:e},({input:n,output:r,model:i,modelOptions:a})=>Factuality({input:n,output:r,expected:e,...client(i,a)})),summarizes:(e,t)=>grade(`judge.autoevals.summarizes`,t,{label:`expected`,value:e},({input:t,output:n,model:i,modelOptions:a})=>Summary({input:t,output:n,expected:e,...client(i,a)})),closedQA:(t,n)=>grade(`judge.autoevals.closedQA`,n,{label:`criteria`,value:t},({input:n,output:r,model:i,modelOptions:a})=>ClosedQA({input:n,output:r,criteria:t,...client(i,a)})),sql:(e,t)=>grade(`judge.autoevals.sql`,t,{label:`expected`,value:e},({input:t,output:r,model:i,modelOptions:a})=>Sql({input:t,output:r,expected:e,...client(i,a)}))}}}function formatJudgeDetail(e,t,n,r){let i=[[`prompt`,e],[n.label,n.value],[`response`,t]];return r.metadata?.rationale!==void 0&&i.push([`rationale`,r.metadata.rationale]),r.metadata?.choice!==void 0&&i.push([`choice`,r.metadata.choice]),r.error!==void 0&&i.push([`autoevals error`,r.error]),i.map(([e,t])=>`${e}: ${formatDiagnosticValue(t)}`).join(`
+`)}function client(e,t){return{model:formatLanguageModelGatewayId(e),client:createAutoevalsClient({languageModel:e,providerOptions:t?.providerOptions})}}export{buildJudgeContext};

package/dist/src/evals/match.d.ts

@@ -1,4 +1,4 @@
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent } from "#protocol/message.js";
 import type { InputRequest } from "#runtime/input/types.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
@@ -74,11 +74,11 @@ export interface EveEvalInputRequestMatchOptions {
     readonly toolName?: string;
 }
 /** One typed stream-event matcher used by scoped event assertions. */
-export type EveEvalEventMatch<TType extends HandleMessageStreamEvent["type"] = HandleMessageStreamEvent["type"]> = TType extends HandleMessageStreamEvent["type"] ? {
+export type EveEvalEventMatch<TType extends MessageStreamEvent["type"] = MessageStreamEvent["type"]> = TType extends MessageStreamEvent["type"] ? {
     /** Stream-event type to match. */
     readonly type: TType;
     /** Partial-deep matcher over the event data. */
-    readonly data?: EveEvalDeepMatcher<Extract<HandleMessageStreamEvent, {
+    readonly data?: EveEvalDeepMatcher<Extract<MessageStreamEvent, {
         type: TType;
     }> extends {
         data: infer TData;
@@ -104,7 +104,7 @@ export declare function subagentCallMatches(call: EveEvalSubagentCall, options:
 /** Returns true when one HITL request satisfies every supplied constraint. */
 export declare function inputRequestMatches(request: InputRequest, options: EveEvalInputRequestMatchOptions): boolean;
 /** Returns true when one stream event satisfies a typed event matcher. */
-export declare function eventMatches(event: HandleMessageStreamEvent, matcher: EveEvalEventMatch): boolean;
+export declare function eventMatches(event: MessageStreamEvent, matcher: EveEvalEventMatch): boolean;
 /**
  * Strict structural equality used by scoped `outputEquals`: unlike matcher
  * comparison, objects must carry exactly the same keys on both sides.

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{dirname,join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.d.ts

@@ -1,4 +1,4 @@
-import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent } from "#protocol/message.js";
 import type { EveEvalDerivedFacts } from "#evals/types.js";
 /**
  * Options for {@link deriveRunFacts}.
@@ -15,7 +15,7 @@ export interface DeriveRunFactsOptions {
  * `subagent.started` with `subagent.completed` the same way. These facts
  * power checks, scorers, and reporters.
  */
-export declare function deriveRunFacts(events: readonly HandleMessageStreamEvent[], options?: DeriveRunFactsOptions): EveEvalDerivedFacts;
+export declare function deriveRunFacts(events: readonly MessageStreamEvent[], options?: DeriveRunFactsOptions): EveEvalDerivedFacts;
 /**
  * Returns empty derived facts, used when a case produced no events
  * (execution errors, transport failures).

package/dist/src/evals/runner/reporters/assertion-diagnostics.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionResult } from "#evals/types.js";
+export declare function formatAssertionFailureHeadline(assertion: AssertionResult): string;
+export declare function formatAssertionFailureDetailLines(assertion: AssertionResult): readonly string[];

package/dist/src/evals/runner/reporters/assertion-diagnostics.js

@@ -0,0 +1,3 @@
+import{truncateDiagnostic}from"#evals/diagnostics.js";function formatAssertionFailureHeadline(t){let n=t.threshold??(t.severity===`gate`?1:void 0),r=n===void 0?``:` (${formatPercent(t.score)} < ${formatPercent(n)})`,[i]=t.message?.split(`
+`)??[],a=i===void 0?``:`: ${truncateDiagnostic(i,240)}`;return`${t.name}${r}${a}`}function formatAssertionFailureDetailLines(t){let[,...n]=t.message?.split(`
+`)??[],r=n.slice(0,4).map(t=>truncateDiagnostic(t,240));return n.length>4?[...r,`…`]:r}function formatPercent(e){return`${Math.round(e*100)}%`}export{formatAssertionFailureDetailLines,formatAssertionFailureHeadline};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import picocolors from"#compiled/picocolors/index.js";function Console(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.map(e=>this.#i(e.name,e.score)).join(`  `),u=[s,this.#t.dim(e.id),c,l].filter(Boolean).join(`  `);this.#e(u);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{Console};
+import picocolors from"#compiled/picocolors/index.js";import{formatAssertionFailureDetailLines,formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function Console(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:r,verdict:i,error:a,skipReason:o}=e,s=r.filter(e=>e.severity===`gate`),c=r.filter(e=>e.severity===`soft`),l=this.#n(i),u=s.length>0?this.#r(s.filter(e=>e.passed).length,s.length):``,d=c.map(e=>this.#i(e.name,e.score)).join(`  `),f=[l,this.#t.dim(e.id),u,d].filter(Boolean).join(`  `);this.#e(f);for(let e of r)if(!e.passed){this.#e(`  ${this.#t.red(`✗ ${formatAssertionFailureHeadline(e)}`)}`);for(let n of formatAssertionFailureDetailLines(e))this.#e(`    ${this.#t.red(n)}`)}a&&this.#e(`  ${this.#t.red(a)}`),o&&this.#e(`  ${this.#t.dim(o)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{Console};

package/dist/src/evals/runner/reporters/junit.js

@@ -1,4 +1,4 @@
-import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";import{truncateDiagnostic}from"#evals/diagnostics.js";import{formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
 `)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
 `);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
-`)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.find(e=>!e.passed);return t===void 0?e.verdict===`scored`?`score below threshold`:e.verdict:t.message===void 0?t.name:`${t.name}: ${t.message}`}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};
+`)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.filter(e=>!e.passed);return t.length>0?truncateDiagnostic(t.map(formatAssertionFailureHeadline).join(`; `),1e3):e.verdict===`scored`?`score below threshold`:e.verdict}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/session.d.ts

@@ -1,7 +1,7 @@
 import type { Client } from "#client/client.js";
 import type { ClientSession } from "#client/session.js";
 import type { CancelSessionResult, SendTurnInput, SessionState } from "#client/types.js";
-import type { HandleMessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
 import type { EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
@@ -10,7 +10,7 @@ import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
  */
 export declare class EveEvalTurnFailedError extends Error {
-    readonly event: TurnFailureStreamEvent | undefined;
+    readonly event: (TurnFailureStreamEvent & MessageStreamEvent) | undefined;
     readonly turn: EveEvalTurn;
     constructor(turn: EveEvalTurn);
 }
@@ -23,7 +23,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
         readonly session: ClientSession;
         readonly signal?: AbortSignal;
     });
-    get events(): readonly HandleMessageStreamEvent[];
+    get events(): readonly MessageStreamEvent[];
     get lastTurn(): EveEvalTurn | undefined;
     get pendingInputRequests(): readonly InputRequest[];
     get sessionId(): string | undefined;

package/dist/src/evals/types.d.ts

@@ -1,6 +1,6 @@
 import type { LanguageModel } from "ai";
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
-import type { HandleMessageStreamEvent, RuntimeIdentity } from "#protocol/message.js";
+import type { RuntimeIdentity, MessageStreamEvent } from "#protocol/message.js";
 import type { CancelSessionResult, SendTurnInput, SessionState } from "#client/types.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
@@ -70,7 +70,7 @@ export interface EveEvalDerivedFacts {
  */
 export interface EveEvalSessionResult {
     readonly derived: EveEvalDerivedFacts;
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     readonly primary: boolean;
     readonly sessionId?: string;
     readonly state: SessionState;
@@ -94,7 +94,7 @@ export interface EveEvalTaskResult {
      */
     readonly status: "completed" | "failed" | "waiting";
     /** The captured stream events from the run. */
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     /** Lines written through `t.log` while the eval ran. */
     readonly logs?: readonly string[];
     /** Facts extracted from the stream (tool calls, message counts, etc.). */
@@ -114,14 +114,25 @@ export interface EveEvalTaskResult {
  * carries a threshold).
  */
 export type AssertionSeverity = "gate" | "soft";
+/**
+ * A rich value-assertion evaluation. Built-in assertions use this to retain a
+ * concise failure reason and structured evidence alongside the numeric score.
+ */
+export interface AssertionEvaluation {
+    /** Normalized 0–1 score. */
+    readonly score: number;
+    /** Concise human-readable reason, shown when the assertion fails. */
+    readonly message?: string;
+    /** Structured diagnostic evidence retained by artifacts and reporters. */
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
 /**
  * A value-level assertion produced by the builders in `eve/evals/expect`
  * (e.g. `includes`, `equals`, `similarity`) and applied to an explicit value
  * via `t.check(value, assertion)`. Boolean assertions score exactly 0 or 1.
  *
  * The chainable `gate`/`soft`/`atLeast` return a new assertion with the
- * severity or threshold overridden, so the threshold rides on the assertion
- * itself rather than a detached map.
+ * severity or threshold overridden.
  */
 export interface Assertion {
     readonly name: string;
@@ -129,20 +140,25 @@ export interface Assertion {
     /** Minimum passing score. `undefined` on a soft assertion = tracked only. */
     readonly threshold?: number;
     score(value: unknown): number | Promise<number>;
+    /** Rich evaluation used by `t.check` and `t.require`; custom assertions may omit it. */
+    evaluate?(value: unknown): AssertionEvaluation | Promise<AssertionEvaluation>;
     gate(threshold?: number): Assertion;
     soft(threshold?: number): Assertion;
     atLeast(threshold: number): Assertion;
 }
 /**
  * Handle to a recorded assertion, returned by every `t` assertion method.
- * Chain `gate`/`soft`/`atLeast` to override the recorded severity or
- * threshold. Recorded assertions are finalized by the runner; use `t.require`
- * or a `require*` lookup when later control flow depends on a passing result.
+ * Chain `gate`/`soft`/`atLeast` to override the recorded severity or threshold,
+ * and `label` to distinguish repeated assertion families. Recorded assertions
+ * are finalized by the runner; use `t.require` or a `require*` lookup when
+ * later control flow depends on a passing result.
  */
 export interface AssertionHandle {
     gate(threshold?: number): this;
     soft(threshold?: number): this;
     atLeast(threshold: number): this;
+    /** Adds a stable human-readable label to this recorded assertion. */
+    label(label: string): this;
 }
 /**
  * The recorded outcome of one assertion, consumed by the verdict, reporters,
@@ -173,14 +189,14 @@ export interface EveEvalAssertions {
     maxToolCalls(max: number): AssertionHandle;
     calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
     noFailedActions(): AssertionHandle;
-    event<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+    event<TType extends MessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
         type: TType;
     }>, "type">): AssertionHandle;
-    notEvent<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+    notEvent<TType extends MessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
         type: TType;
     }>, "type" | "count">): AssertionHandle;
     eventOrder(matchers: readonly EveEvalEventMatch[]): AssertionHandle;
-    eventsSatisfy(label: string, predicate: (events: readonly HandleMessageStreamEvent[]) => boolean): AssertionHandle;
+    eventsSatisfy(label: string, predicate: (events: readonly MessageStreamEvent[]) => boolean): AssertionHandle;
 }
 /** Assertions over the final output captured by one session or immutable turn. */
 export interface EveEvalOutputAssertions {
@@ -188,11 +204,11 @@ export interface EveEvalOutputAssertions {
     outputMatches(schema: StandardSchemaV1): AssertionHandle;
 }
 /** Typed stream event returned by {@link EveEvalLiveTurn.waitForEvent}. */
-export type EveEvalStreamEvent<TType extends HandleMessageStreamEvent["type"] = HandleMessageStreamEvent["type"]> = Extract<HandleMessageStreamEvent, {
+export type EveEvalStreamEvent<TType extends MessageStreamEvent["type"] = MessageStreamEvent["type"]> = Extract<MessageStreamEvent, {
     type: TType;
 }>;
 /** Matcher options for waiting until one live turn emits a specific event. */
-export type EveEvalWaitForEventOptions<TType extends HandleMessageStreamEvent["type"]> = Omit<Extract<EveEvalEventMatch, {
+export type EveEvalWaitForEventOptions<TType extends MessageStreamEvent["type"]> = Omit<Extract<EveEvalEventMatch, {
     type: TType;
 }>, "count" | "type">;
 /**
@@ -203,7 +219,7 @@ export type EveEvalWaitForEventOptions<TType extends HandleMessageStreamEvent["t
  */
 export interface EveEvalLiveTurn {
     /** Events observed on this turn so far. */
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     /** Session driver that started or owns this turn. */
     readonly session: EveEvalSession;
     /** Durable session id available as soon as the turn is accepted or attached. */
@@ -213,12 +229,12 @@ export interface EveEvalLiveTurn {
     /** Wait for the turn boundary and return the recorded immutable result. */
     result(): Promise<EveEvalTurn>;
     /** Wait until the live stream emits one typed event matching `options`. */
-    waitForEvent<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: EveEvalWaitForEventOptions<TType>): Promise<EveEvalStreamEvent<TType>>;
+    waitForEvent<TType extends MessageStreamEvent["type"]>(type: TType, options?: EveEvalWaitForEventOptions<TType>): Promise<EveEvalStreamEvent<TType>>;
 }
 /** Operations and state shared by the primary eval context and independent sessions. */
 export interface EveEvalSessionDriver {
     /** All events observed on this session so far. */
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     /** Input requests left pending by the last parked turn. */
     readonly pendingInputRequests: readonly InputRequest[];
     /** Serializable cursor for resuming this session. */
@@ -248,7 +264,7 @@ export interface EveEvalSession extends EveEvalSessionDriver, EveEvalAssertions,
  */
 export interface EveEvalTurn extends EveEvalAssertions, EveEvalOutputAssertions {
     readonly data: unknown;
-    readonly events: readonly HandleMessageStreamEvent[];
+    readonly events: readonly MessageStreamEvent[];
     readonly inputRequests: readonly InputRequest[];
     readonly message: string | undefined;
     readonly sessionId: string;

package/dist/src/execution/cancel-descendant-turns-step.js

@@ -1 +1 @@
-import{createLogger,logError}from"#internal/logging.js";import{BundleKey}from"#runtime/sessions/runtime-context-keys.js";import{deserializeContext}from"#context/serialize.js";import{readDurableSession}from"#execution/durable-session-store.js";import{cancelRemoteAgentTurn,isRetryableRemoteAgentCancelError,resolveRemoteAgentForAction}from"#execution/remote-agent-dispatch.js";import{requestWorkflowTurnCancellation}from"#execution/workflow-runtime.js";import{getPendingRuntimeActionBatch}from"#harness/runtime-actions.js";const log=createLogger(`execution.cancel-descendant-turns`);async function cancelDescendantTurnsStep(e){"use step";let r;try{r=getPendingRuntimeActionBatch((await readDurableSession(e.sessionState)).state)}catch(n){logError(log,`failed to read pending descendants during cancellation`,n,{sessionId:e.sessionState.sessionId});return}if(r===void 0)return;let i=r.childSessionIds;if(i===void 0)return;let a,getRemoteRegistry=()=>a??=deserializeContext(e.serializedContext).then(e=>e.require(BundleKey).subagentRegistry.subagentsByNodeId),o=r.actions.flatMap(e=>{let t=i[e.callId];return t===void 0?[]:e.kind===`subagent-call`?[cancelLocalDescendant({action:e,childSessionId:t})]:e.kind===`remote-agent-call`?[cancelRemoteDescendant({action:e,childSessionId:t,remoteRegistry:getRemoteRegistry()})]:[]});await Promise.all(o)}async function cancelLocalDescendant(e){try{await requestCancellationWithRetry({request:()=>requestWorkflowTurnCancellation({sessionId:e.childSessionId}),shouldRetryError:()=>!1})}catch(n){logError(log,`failed to cancel local descendant turn`,n,{callId:e.action.callId,childSessionId:e.childSessionId,subagentName:e.action.subagentName})}}async function cancelRemoteDescendant(e){try{let t=await e.remoteRegistry,n=resolveRemoteAgentForAction({nodeId:e.action.nodeId,remoteAgentName:e.action.remoteAgentName,registry:t});await requestCancellationWithRetry({request:()=>cancelRemoteAgentTurn({remote:n,sessionId:e.childSessionId}),shouldRetryError:isRetryableRemoteAgentCancelError})}catch(n){logError(log,`failed to cancel remote descendant turn`,n,{callId:e.action.callId,childSessionId:e.childSessionId,remoteAgentName:e.action.remoteAgentName})}}async function requestCancellationWithRetry(e){for(let t=1;t<=12;t+=1){try{if((await e.request()).status===`accepted`||t===12)return}catch(n){if(!e.shouldRetryError(n)||t===12)throw n}await new Promise(e=>setTimeout(e,250))}}export{cancelDescendantTurnsStep};
+import{createLogger,logError}from"#internal/logging.js";import{BundleKey}from"#runtime/sessions/runtime-context-keys.js";import{deserializeContext}from"#context/serialize.js";import{readDurableSession}from"#execution/durable-session-store.js";import{cancelRemoteAgentTurn,isRetryableRemoteAgentCancelError,resolveRemoteAgentForAction}from"#execution/remote-agent-dispatch.js";import{requestWorkflowTurnCancellation}from"#execution/workflow-runtime.js";import{getPendingRuntimeActionBatch}from"#harness/runtime-actions.js";const log=createLogger(`execution.cancel-descendant-turns`);async function cancelDescendantTurnsStep(e){"use step";let i;try{i=getPendingRuntimeActionBatch((await readDurableSession(e.sessionState)).state)}catch(n){logError(log,`failed to read pending descendants during cancellation`,n,{sessionId:e.sessionState.sessionId});return}if(i===void 0){log.warn(`no pending batch found while cancelling descendants; nothing to cancel`,{sessionId:e.sessionState.sessionId});return}let a=i.childSessionIds;if(a===void 0){log.warn(`pending batch carries no child session ids; descendants cannot be cancelled`,{actionCount:i.actions.length,sessionId:e.sessionState.sessionId});return}let o,getRemoteRegistry=()=>o??=deserializeContext(e.serializedContext).then(e=>e.require(BundleKey).subagentRegistry.subagentsByNodeId),s=i.actions.flatMap(e=>{let t=a[e.callId];return t===void 0?[]:e.kind===`subagent-call`?[cancelLocalDescendant({action:e,childSessionId:t})]:e.kind===`remote-agent-call`?[cancelRemoteDescendant({action:e,childSessionId:t,remoteRegistry:getRemoteRegistry()})]:[]});await Promise.all(s)}async function cancelLocalDescendant(e){try{let t=await requestCancellationWithRetry({request:()=>requestWorkflowTurnCancellation({sessionId:e.childSessionId}),shouldRetryError:()=>!1});t.status!==`accepted`&&log.warn(`descendant cancel was never accepted; the child may run to completion`,{callId:e.action.callId,childSessionId:e.childSessionId,finalStatus:t.status,reason:t.reason,subagentName:e.action.subagentName})}catch(n){logError(log,`failed to cancel local descendant turn`,n,{callId:e.action.callId,childSessionId:e.childSessionId,subagentName:e.action.subagentName})}}async function cancelRemoteDescendant(e){try{let t=await e.remoteRegistry,n=resolveRemoteAgentForAction({nodeId:e.action.nodeId,remoteAgentName:e.action.remoteAgentName,registry:t}),r=await requestCancellationWithRetry({request:()=>cancelRemoteAgentTurn({remote:n,sessionId:e.childSessionId}),shouldRetryError:isRetryableRemoteAgentCancelError});r.status!==`accepted`&&log.warn(`remote descendant cancel was never accepted; the child may run to completion`,{callId:e.action.callId,childSessionId:e.childSessionId,finalStatus:r.status,reason:r.reason,remoteAgentName:e.action.remoteAgentName})}catch(n){logError(log,`failed to cancel remote descendant turn`,n,{callId:e.action.callId,childSessionId:e.childSessionId,remoteAgentName:e.action.remoteAgentName})}}async function requestCancellationWithRetry(e){let t=250,n={status:`no_active_turn`};for(let r=1;r<=8;r+=1){try{if(n=await e.request(),n.status===`accepted`||r===8)return n}catch(t){if(!e.shouldRetryError(t)||r===8)throw t}await new Promise(e=>setTimeout(e,t)),t=Math.min(t*2,1500)}return n}export{cancelDescendantTurnsStep};