eve 0.14.0 → 0.15.0

package/dist/src/evals/define-eval.js

@@ -1 +1 @@
-function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.completed()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),rejectLegacyKey(e,`requires`,`Point real-model evals at credentialed targets directly; dev-only routes are enforced from the live target.`),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}export{defineEval};
+function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.succeeded()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),rejectLegacyKey(e,`requires`,`Point real-model evals at credentialed targets directly; dev-only routes are enforced from the live target.`),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}export{defineEval};

package/dist/src/evals/expect/index.d.ts

@@ -2,10 +2,12 @@ import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js"
 import type { Assertion } from "#evals/types.js";
 export type { Assertion, AssertionHandle, AssertionSeverity } from "#evals/types.js";
 /**
- * Passes when the value (coerced to a string) contains `substring`. A hard
- * gate by default. Apply with `t.check(value, includes("..."))`.
+ * Passes when the value (coerced to a string) contains a substring or matches
+ * a RegExp. A hard gate by default.
  */
-export declare function includes(substring: string): Assertion;
+export declare function includes(substring: string | RegExp): Assertion;
+/** Passes when `predicate` returns true for the supplied value. */
+export declare function satisfies<T = unknown>(predicate: (value: T) => boolean, label: string): Assertion;
 /**
  * Passes when the value deep-equals `expected` (exact structural equality).
  * A hard gate by default.

package/dist/src/evals/expect/index.js

@@ -1 +1 @@
-import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>+!!String(t??``).includes(e)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,similarity};
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals,testRegExp}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>{let r=String(t??``);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?1:0}})}function satisfies(e,t){if(t.trim().length===0)throw Error(`satisfies() requires a non-empty label.`);return makeAssertion({name:`satisfies(${t})`,severity:`gate`,score:t=>+!!e(t)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,satisfies,similarity};

package/dist/src/evals/index.d.ts

@@ -3,5 +3,5 @@ export { defineEvalConfig } from "#evals/define-eval-config.js";
 export { EveEvalTurnFailedError } from "#evals/session.js";
 export type { RuntimeIdentity } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
-export type { EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
+export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";

package/dist/src/evals/match.d.ts

@@ -1,3 +1,6 @@
+import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { InputRequest } from "#runtime/input/types.js";
+import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
 /**
  * One matcher accepted by the assertion options (`t.calledTool`,
@@ -9,26 +12,25 @@ import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
  *   compare with `Object.is`
  * - a **RegExp** tests string values directly and the JSON serialization of
  *   anything else
- * - a **function** receives the observed value and either returns a boolean
- *   verdict, or returns a resolved expected value that is then compared like a
- *   literal — e.g. `(o) => o === process.env.EVE_WEATHER_AGENT_HOST`. To assert
- *   a literal boolean field, use the literal directly; boolean returns are
- *   always treated as verdicts.
+ * - a **function** receives the observed value and returns a boolean verdict
  */
-export type EveEvalValueMatcher<T = unknown> = T | RegExp | ((value: T) => unknown);
+export type EveEvalValueMatcher<T = JsonValue | undefined> = EveEvalDeepMatcher<T>;
+type EveEvalDeepMatcher<T> = RegExp | ((value: T) => boolean) | (T extends readonly (infer TEntry)[] ? readonly EveEvalDeepMatcher<TEntry>[] : T extends object ? {
+    readonly [K in keyof T]?: EveEvalDeepMatcher<T[K]>;
+} : T);
 /**
  * Constraints applied to tool calls by `t.calledTool`. All provided
  * constraints must hold for a call to match.
  */
 export interface EveEvalToolCallMatchOptions {
     /** Partial-deep matcher over the call input (see {@link EveEvalValueMatcher}). */
-    readonly input?: EveEvalValueMatcher;
+    readonly input?: EveEvalValueMatcher<JsonObject>;
     /** Matcher over the call output. */
     readonly output?: EveEvalValueMatcher;
-    /** Required error state of matching calls. */
-    readonly isError?: boolean;
+    /** Required lifecycle outcome. Defaults to `"completed"`. */
+    readonly status?: EveEvalToolCall["status"];
     /** Exact number of matching calls required. Defaults to "at least one". */
-    readonly times?: number;
+    readonly count?: number;
 }
 /**
  * Constraints applied to a `load_skill` call by `t.loadedSkill`. Identical to
@@ -41,10 +43,40 @@ export type EveEvalSkillLoadMatchOptions = Omit<EveEvalToolCallMatchOptions, "in
  */
 export interface EveEvalSubagentCallMatchOptions {
     /** Matcher over the `subagent.called` remote URL. */
-    readonly remoteUrl?: EveEvalValueMatcher;
+    readonly remoteUrl?: EveEvalValueMatcher<string | undefined>;
     /** Matcher over the `subagent.completed` output. */
     readonly output?: EveEvalValueMatcher;
+    /** Required lifecycle outcome. Defaults to `"completed"`. */
+    readonly status?: EveEvalSubagentCall["status"];
+    /** Exact number of matching delegations required. Defaults to "at least one". */
+    readonly count?: number;
 }
+/** Constraints accepted by `requireInputRequest`. */
+export interface EveEvalInputRequestMatchOptions {
+    /** Matcher over the request's display hint. */
+    readonly display?: EveEvalValueMatcher<InputRequest["display"]>;
+    /** Partial-deep matcher over a tool-call action's input. */
+    readonly input?: EveEvalValueMatcher<JsonObject>;
+    /** Matcher over the complete option-id list in request order. */
+    readonly optionIds?: EveEvalValueMatcher<readonly string[]>;
+    /** Matcher over the request prompt. */
+    readonly prompt?: EveEvalValueMatcher<string>;
+    /** Required tool name for a tool-call action. */
+    readonly toolName?: string;
+}
+/** One typed stream-event matcher used by scoped event assertions. */
+export type EveEvalEventMatch<TType extends HandleMessageStreamEvent["type"] = HandleMessageStreamEvent["type"]> = TType extends HandleMessageStreamEvent["type"] ? {
+    /** Stream-event type to match. */
+    readonly type: TType;
+    /** Partial-deep matcher over the event data. */
+    readonly data?: EveEvalDeepMatcher<Extract<HandleMessageStreamEvent, {
+        type: TType;
+    }> extends {
+        data: infer TData;
+    } ? TData : never>;
+    /** Exact number of matching events required. Defaults to "at least one". */
+    readonly count?: number;
+} : never;
 /**
  * Returns true when the observed value satisfies a matcher (literal, RegExp,
  * or function — see {@link EveEvalValueMatcher}).
@@ -52,7 +84,7 @@ export interface EveEvalSubagentCallMatchOptions {
 export declare function matchesValue(matcher: unknown, value: unknown): boolean;
 /**
  * Returns true when one derived tool call satisfies the `input`/`output`/
- * `isError` constraints (the `times` count is the caller's concern).
+ * lifecycle constraints (the `count` option is the caller's concern).
  */
 export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
 /**
@@ -60,8 +92,12 @@ export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalT
  * `output` constraints.
  */
 export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions): boolean;
+/** Returns true when one HITL request satisfies every supplied constraint. */
+export declare function inputRequestMatches(request: InputRequest, options: EveEvalInputRequestMatchOptions): boolean;
+/** Returns true when one stream event satisfies a typed event matcher. */
+export declare function eventMatches(event: HandleMessageStreamEvent, matcher: EveEvalEventMatch): boolean;
 /**
- * Strict structural equality used by `t.outputEquals`: unlike matcher
+ * Strict structural equality used by scoped `outputEquals`: unlike matcher
  * comparison, objects must carry exactly the same keys on both sides.
  */
 export declare function deepEquals(a: unknown, b: unknown): boolean;
@@ -71,3 +107,4 @@ export declare function deepEquals(a: unknown, b: unknown): boolean;
  * a `g`/`y`-flagged pattern would otherwise return order-dependent results.
  */
 export declare function testRegExp(pattern: RegExp, text: string): boolean;
+export {};

package/dist/src/evals/match.js

@@ -1 +1 @@
-function matchesValue(e,t){if(e instanceof RegExp)return testRegExpAgainst(e,t);if(typeof e==`function`){let n=e(t);return typeof n==`boolean`?n:typeof n==`function`?Object.is(n,t):matchesValue(n,t)}return Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||t.isError!==void 0&&e.isError!==t.isError)}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};
+function matchesValue(e,t){return e instanceof RegExp?testRegExpAgainst(e,t):typeof e==`function`?e(t):Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function inputRequestMatches(e,t){if(t.display!==void 0&&!matchesValue(t.display,e.display)||t.prompt!==void 0&&!matchesValue(t.prompt,e.prompt))return!1;if(t.optionIds!==void 0){let n=(e.options??[]).map(e=>e.id);if(!matchesValue(t.optionIds,n))return!1}return!(t.toolName!==void 0&&(e.action.kind!==`tool-call`||e.action.toolName!==t.toolName)||t.input!==void 0&&(e.action.kind!==`tool-call`||!matchesValue(t.input,e.action.input)))}function eventMatches(e,t){return e.type===t.type&&(t.data===void 0||matchesValue(t.data,`data`in e?e.data:void 0))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,eventMatches,inputRequestMatches,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{dirname,join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={name:t,isError:!1,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions){if(e.kind!==`tool-call`)continue;let t={name:e.toolName,input:e.input,output:void 0,isError:!1,turnIndex:Math.max(c,0),sessionId:n};r.push(t),i.set(e.callId,t)}break;case`action.result`:{let{result:e,status:n}=t.data,r=n===`failed`||e.isError===!0;if(e.kind===`tool-result`){let t=i.get(e.callId);t!==void 0&&(t.output=e.output,t.isError=r)}else if(e.kind===`subagent-result`){let t=o.get(e.callId);t!==void 0&&(t.output=t.output??e.output,t.isError=r)}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output;break}case`input.requested`:s.push(...t.data.requests);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return r.push(s),i.set(e,s),s},ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={name:t,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`&&ensureToolCall(e.callId,e.toolName,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}else if(e.kind===`subagent-result`){let t=ensureSubagentCall(e.callId,e.subagentName);t.output=t.output??e.output,t.status=n}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output,e.status===`pending`&&(e.status=`completed`);break}case`input.requested`:s.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let u=computeEvalVerdict({error:l,assertions:c});return{id:r.id,result:s,assertions:c,verdict:u,error:l,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -20,6 +20,7 @@ export interface ExecuteTaskResult {
     readonly result: EveEvalTaskResult;
     readonly assertions: readonly AssertionResult[];
     readonly error?: string;
+    readonly skipReason?: string;
 }
 /**
  * Executes one eval's `test(t)` against an eve agent target: drives the

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(r){let{client:a,evaluation:o,target:s,timeoutMs:c}=r,l=c===void 0?neverAbortSignal():AbortSignal.timeout(c),u=new EvalSessionManager({client:a,signal:l}),d=scopeEvalTargetHandle(s,{sessions:u}),f=[],{context:p,collector:m}=createEvalContext({manager:u,target:d,signal:l,judge:o.judge,log:e=>{f.push(e),r.onLog?.(e)}}),h;try{await o.test(p)}catch(t){h=toErrorMessage(t)}let g=buildTaskResult({logs:f,sessions:u.snapshots(),turn:u.lastTurnSession()?.lastTurn});return{result:g,assertions:await m.finalize(g),error:h}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:n,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:a}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await s.test(h)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r}=e,i=t.filter(e=>e.severity===`gate`),a=t.filter(e=>e.severity===`soft`),o=this.#n(n),s=i.length>0?this.#r(i.filter(e=>e.passed).length,i.length):``,c=a.map(e=>this.#i(e.name,e.score)).join(`  `),l=[o,this.#t.dim(e.id),s,c].filter(Boolean).join(`  `);this.#e(l);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,results:i}=e,a=i.length,o=[];t>0&&o.push(this.#t.green(`${t} passed`)),n>0&&o.push(this.#t.red(`${n} failed`)),r>0&&o.push(this.#t.yellow(`${r} scored`)),o.length===0&&o.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${o.join(`, `)} ${this.#t.dim(`(${a} total)`)}`);let s=this.#a(i);if(s.total>0){let e=this.#t.green(`${s.passed} passed`),t=s.failed>0?`, ${this.#t.red(`${s.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let c=this.#o(i);if(c.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of c){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let l=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(l)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.map(e=>this.#i(e.name,e.score)).join(`  `),u=[s,this.#t.dim(e.id),c,l].filter(Boolean).join(`  `);this.#e(u);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/reporters/junit.js

@@ -1,3 +1,4 @@
-import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="0" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
-`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
+import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
+`);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
 `)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.find(e=>!e.passed);return t===void 0?e.verdict===`scored`?`score below threshold`:e.verdict:t.message===void 0?t.name:`${t.name}: ${t.message}`}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/runner/verdict.d.ts

@@ -10,4 +10,5 @@ import type { AssertionResult, EveEvalVerdict } from "#evals/types.js";
 export declare function computeEvalVerdict(input: {
     readonly error?: string;
     readonly assertions: readonly AssertionResult[];
+    readonly skipReason?: string;
 }): EveEvalVerdict;

package/dist/src/evals/runner/verdict.js

@@ -1 +1 @@
-function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return t?`scored`:`passed`}export{computeEvalVerdict};
+function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return e.skipReason===void 0?t?`scored`:`passed`:`skipped`}export{computeEvalVerdict};

package/dist/src/evals/session.d.ts

@@ -3,7 +3,9 @@ import type { ClientSession } from "#client/session.js";
 import type { SendTurnInput, SessionState } from "#client/types.js";
 import type { HandleMessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
-import type { EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalAssertions, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
  */
@@ -12,9 +14,12 @@ export declare class EveEvalTurnFailedError extends Error {
     readonly turn: EveEvalTurn;
     constructor(turn: EveEvalTurn);
 }
+export interface EvalSessionDriver extends EveEvalAssertions, EveEvalOutputAssertions {
+}
 export declare class EvalSessionDriver implements EveEvalSession {
     #private;
     constructor(input: {
+        readonly collector: AssertionCollector;
         readonly session: ClientSession;
         readonly signal?: AbortSignal;
     });
@@ -23,10 +28,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
     get pendingInputRequests(): readonly InputRequest[];
     get sessionId(): string | undefined;
     get state(): SessionState;
-    expectInputRequests(filter?: {
-        readonly display?: InputRequest["display"];
-        readonly toolName?: string;
-    }): readonly InputRequest[];
+    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(input: SendTurnInput): Promise<EveEvalTurn>;
@@ -40,6 +42,7 @@ export declare class EvalSessionManager {
     #private;
     constructor(input: {
         readonly client: Client;
+        readonly collector?: AssertionCollector;
         readonly signal?: AbortSignal;
     });
     get primary(): EvalSessionDriver;
@@ -49,4 +52,5 @@ export declare class EvalSessionManager {
     }): Promise<EvalSessionDriver>;
     snapshots(): readonly EveEvalSessionResult[];
     lastTurnSession(): EvalSessionDriver | undefined;
+    hasActivity(): boolean;
 }

package/dist/src/evals/session.js

@@ -1 +1 @@
-import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=[];#r;#i=[];constructor(e){this.#e=e.session,this.#t=e.signal}get events(){return this.#n}get lastTurn(){return this.#r}get pendingInputRequests(){return this.#i}get sessionId(){return this.#e.state.sessionId??this.#r?.sessionId}get state(){return this.#e.state}expectInputRequests(e){if(this.#i.length===0)throw Error(`Expected pending input requests, but the last turn did not park.`);let t=this.#i.filter(t=>inputRequestMatches(t,e));if(t.length===0)throw Error(`No pending input requests matched ${formatInputRequestFilter(e)}.`);return t}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.expectInputRequests();for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#a({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#a({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:t,status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#n,{sessionId:t}),events:[...this.#n],primary:e,sessionId:t,state:this.#e.state}}#a(e){this.#n.push(...e.events),this.#i=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({data:e.data,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId??this.sessionId??``,status:e.status,toolCalls:t.toolCalls});return this.#r=n,n}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}},EvalSessionManager=class{#e;#t;#n=[];#r;constructor(e){this.#e=e.client,this.#t=e.signal}get primary(){return this.#r??=this.#i(),this.#r}newSession(){return this.#i()}async attachSession(e,t){let n=new EvalSessionDriver({session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#n.push(n),await n.readTurn(t),n}snapshots(){return this.#n.map(e=>e.snapshot(e===this.#r))}lastTurnSession(){return this.#r?.lastTurn===void 0?this.#n.find(e=>e.lastTurn!==void 0):this.#r}#i(){let e=new EvalSessionDriver({session:this.#e.session(),signal:this.#t});return this.#n.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function inputRequestMatches(e,t){return t===void 0?!0:t.display!==void 0&&e.display!==t.display?!1:t.toolName===void 0?!0:e.action.kind===`tool-call`&&e.action.toolName===t.toolName}function formatInputRequestFilter(e){return e===void 0?`{}`:JSON.stringify(e)}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};
+import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r=[];#i;#a=[];constructor(e){this.#n=e.collector,this.#e=e.session,this.#t=e.signal,Object.assign(this,createScopedAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}),createOutputAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}))}get events(){return this.#r}get lastTurn(){return this.#i}get pendingInputRequests(){return this.#a}get sessionId(){return this.#e.state.sessionId??this.#i?.sessionId}get state(){return this.#e.state}requireInputRequest(e={}){this.#a.length===0&&this.#c(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#a.filter(t=>inputRequestMatches(t,e));return(this.#a.length!==1||t.length!==1)&&this.#c(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#a.length} pending request(s)`),this.#n.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.#a;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#o({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#o({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:requireSessionId(t),status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:t}),events:[...this.#r],primary:e,sessionId:t,state:this.#e.state}}#o(e){this.#r.push(...e.events),this.#a=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#n,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#i=n,n}#s(){let e=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:e}),events:[...this.#r],output:outputOf(this.#i),status:this.#i?.status??`completed`}}#c(e,t){throw this.#n.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r=[];#i;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#t=e.signal}get primary(){return this.#i??=this.#a(),this.#i}newSession(){return this.#a()}async attachSession(e,t){let n=new EvalSessionDriver({collector:this.#n,session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#r.push(n),await n.readTurn(t),n}snapshots(){return this.#r.map(e=>e.snapshot(e===this.#i))}lastTurnSession(){return this.#i?.lastTurn===void 0?this.#r.find(e=>e.lastTurn!==void 0):this.#i}hasActivity(){return this.#r.length>0}#a(){let e=new EvalSessionDriver({collector:this.#n,session:this.#e.session(),signal:this.#t});return this.#r.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};

package/dist/src/evals/types.d.ts

@@ -3,10 +3,12 @@ import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js"
 import type { HandleMessageStreamEvent, RuntimeIdentity } from "#protocol/message.js";
 import type { SendTurnInput, SessionState } from "#client/types.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
-import type { JsonObject } from "#shared/json.js";
+import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
-import type { EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
+import type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
+/** Lifecycle outcome of an eval-observed tool or subagent action. */
+export type EveEvalActionStatus = "pending" | "completed" | "failed" | "rejected";
 /**
  * One tool call extracted from the captured stream, pairing the
  * `actions.requested` request with its matching `action.result`.
@@ -17,9 +19,9 @@ export interface EveEvalToolCall {
     /** Tool input as requested by the model. */
     readonly input: JsonObject;
     /** Tool output from the matching `action.result`; `undefined` when the call never resolved. */
-    readonly output: unknown;
-    /** True when the matching `action.result` reported a failure. */
-    readonly isError: boolean;
+    readonly output: JsonValue | undefined;
+    /** Whether the request is unresolved, completed, failed, or user-rejected. */
+    readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the call happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -35,9 +37,9 @@ export interface EveEvalSubagentCall {
     /** Remote agent URL for remote delegations (`subagent.called` remote metadata). */
     readonly remoteUrl?: string;
     /** Output from the matching `subagent.completed` event; `undefined` when the call never completed. */
-    readonly output?: unknown;
-    /** True when the matching subagent action result reported a failure. */
-    readonly isError: boolean;
+    readonly output?: JsonValue;
+    /** Whether the delegation is unresolved, completed, failed, or rejected. */
+    readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the delegation happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -74,9 +76,8 @@ export interface EveEvalSessionResult {
  */
 export interface EveEvalTaskResult {
     /**
-     * The agent's last assistant message (same as {@link finalMessage}), retained
-     * for reporters and artifacts that log a single "output" value. Mutable
-     * because the runner assigns it after the run completes.
+     * The final turn's structured data when present, otherwise its last assistant
+     * message. Retained for reporters and artifacts that log one output value.
      */
     output: unknown;
     /** The agent's last assistant message, or null when none was produced. */
@@ -131,10 +132,10 @@ export interface Assertion {
 /**
  * Handle to a recorded assertion, returned by every `t` assertion method.
  * Chain `gate`/`soft`/`atLeast` to override the recorded severity or
- * threshold, and `await` it to surface model-backed (judge) errors and ensure
- * the assertion has resolved before the run continues.
+ * threshold. Recorded assertions are finalized by the runner; use `t.require`
+ * or a `require*` lookup when later control flow depends on a passing result.
  */
-export interface AssertionHandle extends PromiseLike<void> {
+export interface AssertionHandle {
     gate(threshold?: number): this;
     soft(threshold?: number): this;
     atLeast(threshold: number): this;
@@ -153,10 +154,37 @@ export interface AssertionResult {
     readonly message?: string;
     readonly metadata?: Readonly<Record<string, unknown>>;
 }
-/**
- * Driver for one session, exposed on the eval context and by `t.newSession()`.
- */
-export interface EveEvalSession {
+/** Recorded assertions shared by aggregate, session, and turn scopes. */
+export interface EveEvalAssertions {
+    succeeded(): AssertionHandle;
+    parked(): AssertionHandle;
+    messageIncludes(token: string | RegExp): AssertionHandle;
+    calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
+    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
+    loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
+    notCalledTool(name: string): AssertionHandle;
+    /** Asserts that tool requests appeared in order, allowing unrelated requests between them. */
+    toolOrder(names: readonly string[]): AssertionHandle;
+    usedNoTools(): AssertionHandle;
+    maxToolCalls(max: number): AssertionHandle;
+    calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
+    noFailedActions(): AssertionHandle;
+    event<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+        type: TType;
+    }>, "type">): AssertionHandle;
+    notEvent<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+        type: TType;
+    }>, "type" | "count">): AssertionHandle;
+    eventOrder(matchers: readonly EveEvalEventMatch[]): AssertionHandle;
+    eventsSatisfy(label: string, predicate: (events: readonly HandleMessageStreamEvent[]) => boolean): AssertionHandle;
+}
+/** Assertions over the final output captured by one session or immutable turn. */
+export interface EveEvalOutputAssertions {
+    outputEquals(value: unknown): AssertionHandle;
+    outputMatches(schema: StandardSchemaV1): AssertionHandle;
+}
+/** Operations and state shared by the primary eval context and independent sessions. */
+export interface EveEvalSessionDriver {
     /** All events observed on this session so far. */
     readonly events: readonly HandleMessageStreamEvent[];
     /** Input requests left pending by the last parked turn. */
@@ -165,11 +193,8 @@ export interface EveEvalSession {
     readonly state: SessionState;
     /** eve session id after the first successful send. */
     readonly sessionId: string | undefined;
-    /** Assert the last turn parked on HITL input and return matching requests. */
-    expectInputRequests(filter?: {
-        readonly display?: InputRequest["display"];
-        readonly toolName?: string;
-    }): readonly InputRequest[];
+    /** Require exactly one pending input request matching `filter`, or abort dependent control flow. */
+    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     /** Resolve specific pending requests and run the resumed turn. */
     respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
     /** Resolve every pending request with the same option id. */
@@ -179,16 +204,22 @@ export interface EveEvalSession {
     /** Send one text turn with a local file attached as a data URL. */
     sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
 }
+/** Driver for one independent session, exposed by `t.newSession()` and target attachment helpers. */
+export interface EveEvalSession extends EveEvalSessionDriver, EveEvalAssertions, EveEvalOutputAssertions {
+}
 /**
  * One completed eval-driver turn.
  */
-export interface EveEvalTurn {
+export interface EveEvalTurn extends EveEvalAssertions, EveEvalOutputAssertions {
     readonly data: unknown;
     readonly events: readonly HandleMessageStreamEvent[];
     readonly inputRequests: readonly InputRequest[];
     readonly message: string | undefined;
+    readonly sessionId: string;
     readonly status: "completed" | "failed" | "waiting";
     readonly toolCalls: readonly EveEvalToolCall[];
+    /** Require exactly one matching tool call, record a gate, and return it for dependent checks. */
+    requireToolCall(name: string, options?: Omit<EveEvalToolCallMatchOptions, "count">): EveEvalToolCall;
     expectOk(): this;
 }
 /**
@@ -232,15 +263,13 @@ export interface JudgeContext {
 }
 /**
  * The single context passed to an eval's `test(t)` function. It drives the
- * primary session (it extends {@link EveEvalSession}), carries the run-level
+ * primary session, carries the run-level
  * and value-level assertion vocabulary, and exposes `judge` for LLM-as-judge.
  *
- * Run-level assertions (`completed`, `calledTool`, …) record an entry
- * evaluated against the final run and never throw; `check` and `judge`
- * evaluate the supplied value immediately. Use plain `throw` /
- * `turn.expectOk()` for bespoke preconditions that should abort the run.
+ * Scoped assertions (`succeeded`, `calledTool`, …) record an entry evaluated
+ * after the test body; `check`, `require`, and `judge` evaluate explicit values.
  */
-export interface EveEvalContext extends EveEvalSession {
+export interface EveEvalContext extends EveEvalSessionDriver, EveEvalAssertions {
     /** Eval timeout signal. */
     readonly signal: AbortSignal;
     /** Current target under test. */
@@ -253,24 +282,12 @@ export interface EveEvalContext extends EveEvalSession {
     sleep(ms?: number): Promise<void>;
     /** Create an additional independent session against the same target. */
     newSession(): EveEvalSession;
-    completed(): AssertionHandle;
-    didNotFail(): AssertionHandle;
-    waiting(): AssertionHandle;
-    messageIncludes(token: string | RegExp): AssertionHandle;
-    calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
-    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
-    loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
-    notCalledTool(name: string): AssertionHandle;
-    toolOrder(names: readonly string[]): AssertionHandle;
-    usedNoTools(): AssertionHandle;
-    maxToolCalls(max: number): AssertionHandle;
-    calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
-    noFailedActions(): AssertionHandle;
-    event(predicate: (events: readonly HandleMessageStreamEvent[]) => boolean, label: string): AssertionHandle;
-    outputEquals(value: unknown): AssertionHandle;
-    outputMatches(schema: StandardSchemaV1): AssertionHandle;
     /** Apply a value-level assertion (from `eve/evals/expect`) to a value. */
     check(value: unknown, assertion: Assertion): AssertionHandle;
+    /** Record an immediate gate and abort dependent control flow when it fails. */
+    require<T>(value: T, assertion: Assertion): Promise<T>;
+    /** Mark this eval as intentionally skipped and stop executing its test body. */
+    skip(reason: string): never;
     /** LLM-as-judge assertions, bound to the resolved judge model. */
     readonly judge: JudgeContext;
 }
@@ -372,8 +389,9 @@ export type EveEval = EveEvalDefinition & {
  * - `"passed"`  — no execution error, every gate held, every soft threshold met
  * - `"failed"`  — a gate assertion failed or execution errored (timeout, transport, thrown task)
  * - `"scored"`  — every gate held but a soft assertion fell below its threshold
+ * - `"skipped"` — the test body intentionally called `t.skip(reason)`
  */
-export type EveEvalVerdict = "passed" | "failed" | "scored";
+export type EveEvalVerdict = "passed" | "failed" | "scored" | "skipped";
 /**
  * Result of executing and asserting one eval.
  *
@@ -388,6 +406,8 @@ export interface EveEvalResult {
     /** Per-eval verdict; see {@link EveEvalVerdict}. */
     readonly verdict: EveEvalVerdict;
     readonly error?: string;
+    /** Why the eval intentionally skipped, present only for a `"skipped"` verdict. */
+    readonly skipReason?: string;
     readonly startedAt: string;
     readonly completedAt: string;
 }
@@ -405,6 +425,8 @@ export interface EveEvalRunSummary {
     readonly failed: number;
     /** Evals with verdict `"scored"` (below-threshold soft assertions only). */
     readonly scored: number;
+    /** Evals intentionally skipped by their test body. */
+    readonly skipped: number;
     /** The execution-error subset of `failed` (timeouts, connection failures, exceptions). */
     readonly errored: number;
 }