eve 0.14.0 → 0.15.1

package/dist/src/evals/assertions/run.d.ts

@@ -1,37 +1,38 @@
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
 import type { HandleMessageStreamEvent } from "#protocol/message.js";
 import { type EveEvalSkillLoadMatchOptions, type EveEvalSubagentCallMatchOptions, type EveEvalToolCallMatchOptions } from "#evals/match.js";
+import type { EveEvalEventMatch } from "#evals/match.js";
 import type { RunAssertion } from "#evals/assertions/collector.js";
+/** Minimal captured scope consumed by deterministic eval assertions. */
+export interface EveEvalAssertionSubject {
+    readonly derived: import("#evals/types.js").EveEvalDerivedFacts;
+    readonly events: readonly HandleMessageStreamEvent[];
+    readonly output: unknown;
+    readonly status: "completed" | "failed" | "waiting";
+}
 /**
- * Asserts the run ran to completion: it did not fail and did not park on an
+ * Asserts the run finished successfully: it did not fail and did not park on an
  * unanswered HITL input request.
  */
-export declare function completed(): RunAssertion;
+export declare function succeeded(): RunAssertion;
 /**
- * Asserts the run ended parked on HITL input. For approval gates and
- * ask-question flows.
+ * Asserts the run ended cleanly parked on HITL input.
  */
-export declare function waiting(): RunAssertion;
-/**
- * Asserts the run did not fail: terminal status is not `"failed"` and no
- * `turn.failed` / `step.failed` events were emitted. Parked runs pass; use
- * {@link completed} to also reject parking.
- */
-export declare function didNotFail(): RunAssertion;
+export declare function parked(): RunAssertion;
 /**
  * Asserts the joined assistant message text contains `token` (substring for
  * strings, `test` for RegExps).
  */
 export declare function messageIncludes(token: string | RegExp): RunAssertion;
 /**
- * Asserts a tool call with `name` happened. Options constrain the call
- * further: `input` partial-deep-matches, `output` matches the result,
- * `isError` constrains error state, and `times` requires an exact count.
+ * Asserts a completed tool call with `name` happened. Options constrain the
+ * call further: `input` partial-deep-matches, `output` matches the result,
+ * `status` overrides the lifecycle state, and `count` requires an exact count.
  */
 export declare function calledTool(name: string, options?: EveEvalToolCallMatchOptions): RunAssertion;
 /**
  * Sugar over {@link calledTool} for the framework `load_skill` tool: asserts a
- * skill with id `skill` was loaded. `output`/`isError`/`times` constrain the
+ * skill with id `skill` was loaded. `output`/`status`/`count` constrain the
  * matching call exactly as for `calledTool`.
  */
 export declare function loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): RunAssertion;
@@ -40,7 +41,7 @@ export declare function loadedSkill(skill: string, options?: EveEvalSkillLoadMat
  */
 export declare function notCalledTool(name: string): RunAssertion;
 /**
- * Asserts the named tools were called in the given order (subsequence match:
+ * Asserts the named tools were requested in the given order (subsequence match:
  * other calls may interleave).
  */
 export declare function toolOrder(names: readonly string[]): RunAssertion;
@@ -66,7 +67,13 @@ export declare function calledSubagent(name: string, options?: EveEvalSubagentCa
  * Escape hatch: asserts an arbitrary predicate over the full typed event
  * stream. `label` names the assertion in reports.
  */
-export declare function event(predicate: (events: readonly HandleMessageStreamEvent[]) => boolean, label: string): RunAssertion;
+export declare function eventsSatisfy(label: string, predicate: (events: readonly HandleMessageStreamEvent[]) => boolean): RunAssertion;
+/** Asserts a typed stream event exists, optionally with an exact count. */
+export declare function typedEvent(matcher: EveEvalEventMatch): RunAssertion;
+/** Asserts no typed stream event matches. */
+export declare function notEvent(matcher: EveEvalEventMatch): RunAssertion;
+/** Asserts typed event groups occur in order, ignoring unrelated events. */
+export declare function eventOrder(matchers: readonly EveEvalEventMatch[]): RunAssertion;
 /**
  * Asserts `result.output` (the final assistant message) deep-equals `value`.
  */

package/dist/src/evals/assertions/run.js

@@ -1,2 +1,2 @@
-import{deepEquals,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function completed(){return{name:`completed`,evaluate(e){return e.status===`failed`?fail(failureDetail(`run failed`,e.derived.failureCode)):e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS}}}function waiting(){return{name:`waiting`,evaluate(e){return e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`)}}}function didNotFail(){return{name:`didNotFail`,evaluate(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?PASS:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let r=joinCompletedMessages(t.events);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(r)}`)}}}function calledTool(e,t={}){return{name:`calledTool(${e})`,evaluate(n){let i=n.derived.toolCalls.filter(t=>t.name===e),a=i.filter(e=>toolCallMatches(e,t));if(t.times===void 0?a.length>0:a.length===t.times)return{score:1,metadata:{matchingCalls:a.length}};let o=i.length>0?`observed ${e} calls: ${i.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`;return fail(`${t.times===void 0?`expected a matching call to "${e}"`:`expected exactly ${t.times} matching call(s), found ${a.length}`}; ${o}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=t.derived.toolCalls.map(e=>e.name),r=0;for(let t of n)if(t===e[r]&&(r+=1),r===e.length)break;return r===e.length?PASS:fail(`missing "${e[r]}" after [${e.slice(0,r).join(`, `)}]; observed order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(e=>e.data.result.kind===`tool-result`?e.data.result.toolName:e.data.result.kind);return fail(`${t.length} failed action(s): ${n.join(`, `)}`)}}}function calledSubagent(e,n={}){return{name:`calledSubagent(${e})`,evaluate(r){let i=r.derived.subagentCalls.filter(t=>t.name===e);return i.filter(e=>subagentCallMatches(e,n)).length>0?PASS:i.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but no call matched the constraints`,{observedSubagentCalls:i})}}}function event(e,t){return{name:`event(${t})`,evaluate(n){return e(n.events)?PASS:fail(`event predicate "${t}" did not hold`)}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
-`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}export{calledSubagent,calledTool,completed,didNotFail,event,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,outputEquals,outputMatches,toolOrder,usedNoTools,waiting};
+import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),a=r.filter(e=>toolCallMatches(e,t));if(t.count===void 0?a.length>0:a.length===t.count)return{score:1,metadata:{matchingCalls:a.length}};let o=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`;return fail(`${t.count===void 0?`expected a matching call to "${e}"`:`expected exactly ${t.count} matching call(s), found ${a.length}`}; ${o}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(e=>e.data.result.kind===`tool-result`?e.data.result.toolName:e.data.result.kind);return fail(`${t.length} failed action(s): ${n.join(`, `)}`)}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(r){let i=r.derived.subagentCalls.filter(t=>t.name===e),a=i.filter(e=>subagentCallMatches(e,t));return(t.count===void 0?a.length>0:a.length===t.count)?{score:1,metadata:{matchingCalls:a.length}}:i.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but ${a.length} call(s) matched the constraints`,{observedSubagentCalls:i})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(n){let r=n.events.filter(n=>eventMatches(n,e));return(e.count===void 0?r.length>0:r.length===e.count)?{score:1,metadata:{matchingEvents:r.length}}:fail(`expected ${e.count===void 0?`at least one`:`exactly ${e.count}`} matching ${e.type} event(s), found ${r.length}; observed: [${n.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(n){let r=n.events.filter(n=>eventMatches(n,e));return r.length===0?PASS:fail(`expected no matching ${e.type} events, found ${r.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(n){let r=-1;for(let i of e){let e=n.events.flatMap((e,n)=>eventMatches(e,i)?[n]:[]);if(!(i.count===void 0?e.length>0:e.length===i.count))return fail(`expected ${i.count===void 0?`at least one`:`exactly ${i.count}`} matching ${i.type} event(s), found ${e.length}`);let a=e[0];if(a===void 0||a<=r)return fail(`event group ${i.type} did not occur after the previous group`);r=e[e.length-1]??r}return PASS}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function validateCount(e){if(e!==void 0&&(!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer; received ${e}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};

package/dist/src/evals/assertions/scoped.d.ts

@@ -0,0 +1,17 @@
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalAssertionSubject } from "#evals/assertions/run.js";
+import type { EveEvalAssertions, EveEvalOutputAssertions, EveEvalTaskResult } from "#evals/types.js";
+type AssertionScope = {
+    readonly timing: "final";
+    select(result: EveEvalTaskResult): EveEvalAssertionSubject;
+} | {
+    readonly timing: "snapshot";
+    select(): EveEvalAssertionSubject;
+};
+/** Binds the shared assertion vocabulary to one aggregate, session, or turn scope. */
+export declare function createScopedAssertions(collector: AssertionCollector, scope: AssertionScope): EveEvalAssertions;
+/** Binds output-only assertions to a session or immutable turn snapshot. */
+export declare function createOutputAssertions(collector: AssertionCollector, scope: Extract<AssertionScope, {
+    timing: "snapshot";
+}>): EveEvalOutputAssertions;
+export {};

package/dist/src/evals/assertions/scoped.js

@@ -0,0 +1 @@
+import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";function createScopedAssertions(t,n){let r=createRecorder(t,n);return{succeeded:()=>r(RunAssertions.succeeded()),parked:()=>r(RunAssertions.parked()),messageIncludes:t=>r(RunAssertions.messageIncludes(t)),calledTool:(t,n)=>r(RunAssertions.calledTool(t,n)),loadedSkill:(t,n)=>r(RunAssertions.loadedSkill(t,n)),notCalledTool:t=>r(RunAssertions.notCalledTool(t)),toolOrder:t=>r(RunAssertions.toolOrder(t)),usedNoTools:()=>r(RunAssertions.usedNoTools()),maxToolCalls:t=>r(RunAssertions.maxToolCalls(t)),calledSubagent:(t,n)=>r(RunAssertions.calledSubagent(t,n)),noFailedActions:()=>r(RunAssertions.noFailedActions()),event:(t,n)=>r(RunAssertions.typedEvent({...n,type:t})),notEvent:(t,n)=>r(RunAssertions.notEvent({...n,type:t})),eventOrder:t=>r(RunAssertions.eventOrder(t)),eventsSatisfy:(t,n)=>r(RunAssertions.eventsSatisfy(t,n))}}function createOutputAssertions(t,n){let r=createRecorder(t,n);return{outputEquals:t=>r(RunAssertions.outputEquals(t)),outputMatches:t=>r(RunAssertions.outputMatches(t))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};

package/dist/src/evals/cli/eval.js

@@ -1 +1 @@
-import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let o=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(o);let s=e.length>0?e:void 0,c=await discoverAndImportEvals(o,s);if(c.length===0){s?n.error(`No evals found matching: ${s.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let l=filterEvalsByTag(c,t.tag??[]);if(l.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(l,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(o)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h;try{t.url?(h=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:o}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`remote`,url:t.url})):(p=await startDevelopmentServer(o,{host:`127.0.0.1`,port:0}),h=await createEvalClient({kind:`local`,url:p.url}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`local`,url:p.url}));let e=t.json===!0?[]:[new ConsoleReporter];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let r=await runEvals({evaluations:l,config:f,target:m,client:h,appRoot:o,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(r,null,2));let i=r.failed>0,a=t.strict===!0&&r.scored>0;(i||a)&&(process.exitCode=1)}finally{p&&await p.close()}let g=typeof process.exitCode==`number`?process.exitCode:0;process.exit(g)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};
+import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let o=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(o);let s=e.length>0?e:void 0,c=await discoverAndImportEvals(o,s);if(c.length===0){s?n.error(`No evals found matching: ${s.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let l=filterEvalsByTag(c,t.tag??[]);if(l.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(l,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(o)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h;try{if(t.url)h=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:o}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`remote`,url:t.url});else{p=createDevelopmentServer(o,{host:`127.0.0.1`,port:0});let e=await p.start();h=await createEvalClient({kind:`local`,url:e.url}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`local`,url:e.url})}let e=t.json===!0?[]:[new ConsoleReporter];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let r=await runEvals({evaluations:l,config:f,target:m,client:h,appRoot:o,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(r,null,2));let i=r.failed>0,a=t.strict===!0&&r.scored>0;(i||a)&&(process.exitCode=1)}finally{p&&await p.close()}let g=typeof process.exitCode==`number`?process.exitCode:0;process.exit(g)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};

package/dist/src/evals/context.d.ts

@@ -9,6 +9,7 @@ import type { EveEvalContext, EveEvalJudgeConfig, EveEvalTargetHandle } from "#e
  */
 export declare function createEvalContext(deps: {
     readonly manager: EvalSessionManager;
+    readonly collector: AssertionCollector;
     readonly target: EveEvalTargetHandle;
     readonly signal: AbortSignal;
     readonly judge: EveEvalJudgeConfig | undefined;

package/dist/src/evals/context.js

@@ -1 +1 @@
-import"#evals/session.js";import{AssertionCollector}from"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{buildJudgeContext}from"#evals/judge.js";function createEvalContext(r){let i=new AssertionCollector,a=``,primary=()=>r.manager.primary,replyMessage=()=>r.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:r.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},expectInputRequests:e=>primary().expectInputRequests(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:r.signal,target:r.target,get reply(){return replyMessage()},log:r.log,sleep:e=>sleep(e,r.signal),newSession:()=>r.manager.newSession(),completed:()=>i.recordRun(RunAssertions.completed()),didNotFail:()=>i.recordRun(RunAssertions.didNotFail()),waiting:()=>i.recordRun(RunAssertions.waiting()),messageIncludes:e=>i.recordRun(RunAssertions.messageIncludes(e)),calledTool:(e,n)=>i.recordRun(RunAssertions.calledTool(e,n)),loadedSkill:(e,n)=>i.recordRun(RunAssertions.loadedSkill(e,n)),notCalledTool:e=>i.recordRun(RunAssertions.notCalledTool(e)),toolOrder:e=>i.recordRun(RunAssertions.toolOrder(e)),usedNoTools:()=>i.recordRun(RunAssertions.usedNoTools()),maxToolCalls:e=>i.recordRun(RunAssertions.maxToolCalls(e)),calledSubagent:(e,n)=>i.recordRun(RunAssertions.calledSubagent(e,n)),noFailedActions:()=>i.recordRun(RunAssertions.noFailedActions()),event:(e,n)=>i.recordRun(RunAssertions.event(e,n)),outputEquals:e=>i.recordRun(RunAssertions.outputEquals(e)),outputMatches:e=>i.recordRun(RunAssertions.outputMatches(e)),check:(e,t)=>recordCheck(i,e,t),judge:o},collector:i}}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:async()=>({score:await n.score(t)})})}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};
+import"#evals/assertions/collector.js";import"#evals/session.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(n){let i=n.collector,a=``,primary=()=>n.manager.primary,replyMessage=()=>n.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:n.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},requireInputRequest:e=>primary().requireInputRequest(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:n.signal,target:n.target,get reply(){return replyMessage()},log:n.log,sleep:e=>sleep(e,n.signal),newSession:()=>n.manager.newSession(),...createScopedAssertions(i,{timing:`final`,select:e=>e}),check:(e,t)=>recordCheck(i,e,t),require:(e,t)=>requireCheck(i,e,t),skip:e=>{throw e.trim().length===0?Error(`skip() requires a non-empty reason.`):i.hasEntries||n.manager.hasActivity()?Error(`skip() must be called before sending messages or recording assertions.`):new EvalSkipped(e)},judge:o},collector:i}}async function requireCheck(e,t,r){let i=r.gate(r.threshold);if(!await e.recordRequirement({name:i.name,threshold:i.threshold,score:async()=>({score:await i.score(t)})}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:async()=>({score:await n.score(t)})})}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};

package/dist/src/evals/control-flow.d.ts

@@ -0,0 +1,9 @@
+/** Internal signal used to stop an eval after a recorded required assertion fails. */
+export declare class EvalRequirementFailed extends Error {
+    constructor();
+}
+/** Internal signal used by `t.skip()` to produce a skipped verdict. */
+export declare class EvalSkipped extends Error {
+    readonly reason: string;
+    constructor(reason: string);
+}

package/dist/src/evals/control-flow.js

@@ -0,0 +1 @@
+var EvalRequirementFailed=class extends Error{constructor(){super(`A required eval assertion failed.`),this.name=`EvalRequirementFailed`}},EvalSkipped=class extends Error{reason;constructor(e){super(e),this.name=`EvalSkipped`,this.reason=e}};export{EvalRequirementFailed,EvalSkipped};

package/dist/src/evals/define-eval.d.ts

@@ -2,7 +2,7 @@ import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";
 /**
  * Defines one eve eval. Each eval file is exactly one case: an imperative
  * `test(t)` function that drives the agent (`t.send`, `t.respond`, …) and
- * asserts on what it produced (`t.completed()`, `t.check(...)`,
+ * asserts on what it produced (`t.succeeded()`, `t.check(...)`,
  * `t.judge.autoevals.*`). Organize related evals with directory nesting under
  * `evals/`, or default-export an array of evals to fan one file out over a
  * dataset.

package/dist/src/evals/define-eval.js

@@ -1 +1 @@
-function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.completed()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),rejectLegacyKey(e,`requires`,`Point real-model evals at credentialed targets directly; dev-only routes are enforced from the live target.`),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}export{defineEval};
+function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.succeeded()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),rejectLegacyKey(e,`requires`,`Point real-model evals at credentialed targets directly; dev-only routes are enforced from the live target.`),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}export{defineEval};

package/dist/src/evals/expect/index.d.ts

@@ -2,10 +2,12 @@ import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js"
 import type { Assertion } from "#evals/types.js";
 export type { Assertion, AssertionHandle, AssertionSeverity } from "#evals/types.js";
 /**
- * Passes when the value (coerced to a string) contains `substring`. A hard
- * gate by default. Apply with `t.check(value, includes("..."))`.
+ * Passes when the value (coerced to a string) contains a substring or matches
+ * a RegExp. A hard gate by default.
  */
-export declare function includes(substring: string): Assertion;
+export declare function includes(substring: string | RegExp): Assertion;
+/** Passes when `predicate` returns true for the supplied value. */
+export declare function satisfies<T = unknown>(predicate: (value: T) => boolean, label: string): Assertion;
 /**
  * Passes when the value deep-equals `expected` (exact structural equality).
  * A hard gate by default.

package/dist/src/evals/expect/index.js

@@ -1 +1 @@
-import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>+!!String(t??``).includes(e)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,similarity};
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals,testRegExp}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>{let r=String(t??``);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?1:0}})}function satisfies(e,t){if(t.trim().length===0)throw Error(`satisfies() requires a non-empty label.`);return makeAssertion({name:`satisfies(${t})`,severity:`gate`,score:t=>+!!e(t)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,satisfies,similarity};

package/dist/src/evals/index.d.ts

@@ -3,5 +3,5 @@ export { defineEvalConfig } from "#evals/define-eval-config.js";
 export { EveEvalTurnFailedError } from "#evals/session.js";
 export type { RuntimeIdentity } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
-export type { EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
+export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";

package/dist/src/evals/match.d.ts

@@ -1,3 +1,6 @@
+import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import type { InputRequest } from "#runtime/input/types.js";
+import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
 /**
  * One matcher accepted by the assertion options (`t.calledTool`,
@@ -9,26 +12,25 @@ import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
  *   compare with `Object.is`
  * - a **RegExp** tests string values directly and the JSON serialization of
  *   anything else
- * - a **function** receives the observed value and either returns a boolean
- *   verdict, or returns a resolved expected value that is then compared like a
- *   literal — e.g. `(o) => o === process.env.EVE_WEATHER_AGENT_HOST`. To assert
- *   a literal boolean field, use the literal directly; boolean returns are
- *   always treated as verdicts.
+ * - a **function** receives the observed value and returns a boolean verdict
  */
-export type EveEvalValueMatcher<T = unknown> = T | RegExp | ((value: T) => unknown);
+export type EveEvalValueMatcher<T = JsonValue | undefined> = EveEvalDeepMatcher<T>;
+type EveEvalDeepMatcher<T> = RegExp | ((value: T) => boolean) | (T extends readonly (infer TEntry)[] ? readonly EveEvalDeepMatcher<TEntry>[] : T extends object ? {
+    readonly [K in keyof T]?: EveEvalDeepMatcher<T[K]>;
+} : T);
 /**
  * Constraints applied to tool calls by `t.calledTool`. All provided
  * constraints must hold for a call to match.
  */
 export interface EveEvalToolCallMatchOptions {
     /** Partial-deep matcher over the call input (see {@link EveEvalValueMatcher}). */
-    readonly input?: EveEvalValueMatcher;
+    readonly input?: EveEvalValueMatcher<JsonObject>;
     /** Matcher over the call output. */
     readonly output?: EveEvalValueMatcher;
-    /** Required error state of matching calls. */
-    readonly isError?: boolean;
+    /** Required lifecycle outcome. Defaults to `"completed"`. */
+    readonly status?: EveEvalToolCall["status"];
     /** Exact number of matching calls required. Defaults to "at least one". */
-    readonly times?: number;
+    readonly count?: number;
 }
 /**
  * Constraints applied to a `load_skill` call by `t.loadedSkill`. Identical to
@@ -41,10 +43,40 @@ export type EveEvalSkillLoadMatchOptions = Omit<EveEvalToolCallMatchOptions, "in
  */
 export interface EveEvalSubagentCallMatchOptions {
     /** Matcher over the `subagent.called` remote URL. */
-    readonly remoteUrl?: EveEvalValueMatcher;
+    readonly remoteUrl?: EveEvalValueMatcher<string | undefined>;
     /** Matcher over the `subagent.completed` output. */
     readonly output?: EveEvalValueMatcher;
+    /** Required lifecycle outcome. Defaults to `"completed"`. */
+    readonly status?: EveEvalSubagentCall["status"];
+    /** Exact number of matching delegations required. Defaults to "at least one". */
+    readonly count?: number;
 }
+/** Constraints accepted by `requireInputRequest`. */
+export interface EveEvalInputRequestMatchOptions {
+    /** Matcher over the request's display hint. */
+    readonly display?: EveEvalValueMatcher<InputRequest["display"]>;
+    /** Partial-deep matcher over a tool-call action's input. */
+    readonly input?: EveEvalValueMatcher<JsonObject>;
+    /** Matcher over the complete option-id list in request order. */
+    readonly optionIds?: EveEvalValueMatcher<readonly string[]>;
+    /** Matcher over the request prompt. */
+    readonly prompt?: EveEvalValueMatcher<string>;
+    /** Required tool name for a tool-call action. */
+    readonly toolName?: string;
+}
+/** One typed stream-event matcher used by scoped event assertions. */
+export type EveEvalEventMatch<TType extends HandleMessageStreamEvent["type"] = HandleMessageStreamEvent["type"]> = TType extends HandleMessageStreamEvent["type"] ? {
+    /** Stream-event type to match. */
+    readonly type: TType;
+    /** Partial-deep matcher over the event data. */
+    readonly data?: EveEvalDeepMatcher<Extract<HandleMessageStreamEvent, {
+        type: TType;
+    }> extends {
+        data: infer TData;
+    } ? TData : never>;
+    /** Exact number of matching events required. Defaults to "at least one". */
+    readonly count?: number;
+} : never;
 /**
  * Returns true when the observed value satisfies a matcher (literal, RegExp,
  * or function — see {@link EveEvalValueMatcher}).
@@ -52,7 +84,7 @@ export interface EveEvalSubagentCallMatchOptions {
 export declare function matchesValue(matcher: unknown, value: unknown): boolean;
 /**
  * Returns true when one derived tool call satisfies the `input`/`output`/
- * `isError` constraints (the `times` count is the caller's concern).
+ * lifecycle constraints (the `count` option is the caller's concern).
  */
 export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
 /**
@@ -60,8 +92,12 @@ export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalT
  * `output` constraints.
  */
 export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions): boolean;
+/** Returns true when one HITL request satisfies every supplied constraint. */
+export declare function inputRequestMatches(request: InputRequest, options: EveEvalInputRequestMatchOptions): boolean;
+/** Returns true when one stream event satisfies a typed event matcher. */
+export declare function eventMatches(event: HandleMessageStreamEvent, matcher: EveEvalEventMatch): boolean;
 /**
- * Strict structural equality used by `t.outputEquals`: unlike matcher
+ * Strict structural equality used by scoped `outputEquals`: unlike matcher
  * comparison, objects must carry exactly the same keys on both sides.
  */
 export declare function deepEquals(a: unknown, b: unknown): boolean;
@@ -71,3 +107,4 @@ export declare function deepEquals(a: unknown, b: unknown): boolean;
  * a `g`/`y`-flagged pattern would otherwise return order-dependent results.
  */
 export declare function testRegExp(pattern: RegExp, text: string): boolean;
+export {};

package/dist/src/evals/match.js

@@ -1 +1 @@
-function matchesValue(e,t){if(e instanceof RegExp)return testRegExpAgainst(e,t);if(typeof e==`function`){let n=e(t);return typeof n==`boolean`?n:typeof n==`function`?Object.is(n,t):matchesValue(n,t)}return Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||t.isError!==void 0&&e.isError!==t.isError)}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};
+function matchesValue(e,t){return e instanceof RegExp?testRegExpAgainst(e,t):typeof e==`function`?e(t):Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function inputRequestMatches(e,t){if(t.display!==void 0&&!matchesValue(t.display,e.display)||t.prompt!==void 0&&!matchesValue(t.prompt,e.prompt))return!1;if(t.optionIds!==void 0){let n=(e.options??[]).map(e=>e.id);if(!matchesValue(t.optionIds,n))return!1}return!(t.toolName!==void 0&&(e.action.kind!==`tool-call`||e.action.toolName!==t.toolName)||t.input!==void 0&&(e.action.kind!==`tool-call`||!matchesValue(t.input,e.action.input)))}function eventMatches(e,t){return e.type===t.type&&(t.data===void 0||matchesValue(t.data,`data`in e?e.data:void 0))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,eventMatches,inputRequestMatches,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{dirname,join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={name:t,isError:!1,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions){if(e.kind!==`tool-call`)continue;let t={name:e.toolName,input:e.input,output:void 0,isError:!1,turnIndex:Math.max(c,0),sessionId:n};r.push(t),i.set(e.callId,t)}break;case`action.result`:{let{result:e,status:n}=t.data,r=n===`failed`||e.isError===!0;if(e.kind===`tool-result`){let t=i.get(e.callId);t!==void 0&&(t.output=e.output,t.isError=r)}else if(e.kind===`subagent-result`){let t=o.get(e.callId);t!==void 0&&(t.output=t.output??e.output,t.isError=r)}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output;break}case`input.requested`:s.push(...t.data.requests);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return r.push(s),i.set(e,s),s},ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={name:t,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`&&ensureToolCall(e.callId,e.toolName,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}else if(e.kind===`subagent-result`){let t=ensureSubagentCall(e.callId,e.subagentName);t.output=t.output??e.output,t.status=n}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output,e.status===`pending`&&(e.status=`completed`);break}case`input.requested`:s.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let u=computeEvalVerdict({error:l,assertions:c});return{id:r.id,result:s,assertions:c,verdict:u,error:l,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -20,6 +20,7 @@ export interface ExecuteTaskResult {
     readonly result: EveEvalTaskResult;
     readonly assertions: readonly AssertionResult[];
     readonly error?: string;
+    readonly skipReason?: string;
 }
 /**
  * Executes one eval's `test(t)` against an eve agent target: drives the

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(r){let{client:a,evaluation:o,target:s,timeoutMs:c}=r,l=c===void 0?neverAbortSignal():AbortSignal.timeout(c),u=new EvalSessionManager({client:a,signal:l}),d=scopeEvalTargetHandle(s,{sessions:u}),f=[],{context:p,collector:m}=createEvalContext({manager:u,target:d,signal:l,judge:o.judge,log:e=>{f.push(e),r.onLog?.(e)}}),h;try{await o.test(p)}catch(t){h=toErrorMessage(t)}let g=buildTaskResult({logs:f,sessions:u.snapshots(),turn:u.lastTurnSession()?.lastTurn});return{result:g,assertions:await m.finalize(g),error:h}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:n,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:a}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await s.test(h)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r}=e,i=t.filter(e=>e.severity===`gate`),a=t.filter(e=>e.severity===`soft`),o=this.#n(n),s=i.length>0?this.#r(i.filter(e=>e.passed).length,i.length):``,c=a.map(e=>this.#i(e.name,e.score)).join(`  `),l=[o,this.#t.dim(e.id),s,c].filter(Boolean).join(`  `);this.#e(l);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,results:i}=e,a=i.length,o=[];t>0&&o.push(this.#t.green(`${t} passed`)),n>0&&o.push(this.#t.red(`${n} failed`)),r>0&&o.push(this.#t.yellow(`${r} scored`)),o.length===0&&o.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${o.join(`, `)} ${this.#t.dim(`(${a} total)`)}`);let s=this.#a(i);if(s.total>0){let e=this.#t.green(`${s.passed} passed`),t=s.failed>0?`, ${this.#t.red(`${s.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let c=this.#o(i);if(c.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of c){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let l=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(l)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.map(e=>this.#i(e.name,e.score)).join(`  `),u=[s,this.#t.dim(e.id),c,l].filter(Boolean).join(`  `);this.#e(u);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/reporters/junit.js

@@ -1,3 +1,4 @@
-import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="0" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
-`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
+import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
+`);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
 `)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.find(e=>!e.passed);return t===void 0?e.verdict===`scored`?`score below threshold`:e.verdict:t.message===void 0?t.name:`${t.name}: ${t.message}`}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/runner/verdict.d.ts

@@ -10,4 +10,5 @@ import type { AssertionResult, EveEvalVerdict } from "#evals/types.js";
 export declare function computeEvalVerdict(input: {
     readonly error?: string;
     readonly assertions: readonly AssertionResult[];
+    readonly skipReason?: string;
 }): EveEvalVerdict;

package/dist/src/evals/runner/verdict.js

@@ -1 +1 @@
-function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return t?`scored`:`passed`}export{computeEvalVerdict};
+function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return e.skipReason===void 0?t?`scored`:`passed`:`skipped`}export{computeEvalVerdict};

package/dist/src/evals/session.d.ts

@@ -3,7 +3,9 @@ import type { ClientSession } from "#client/session.js";
 import type { SendTurnInput, SessionState } from "#client/types.js";
 import type { HandleMessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
-import type { EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalAssertions, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
  */
@@ -12,9 +14,12 @@ export declare class EveEvalTurnFailedError extends Error {
     readonly turn: EveEvalTurn;
     constructor(turn: EveEvalTurn);
 }
+export interface EvalSessionDriver extends EveEvalAssertions, EveEvalOutputAssertions {
+}
 export declare class EvalSessionDriver implements EveEvalSession {
     #private;
     constructor(input: {
+        readonly collector: AssertionCollector;
         readonly session: ClientSession;
         readonly signal?: AbortSignal;
     });
@@ -23,10 +28,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
     get pendingInputRequests(): readonly InputRequest[];
     get sessionId(): string | undefined;
     get state(): SessionState;
-    expectInputRequests(filter?: {
-        readonly display?: InputRequest["display"];
-        readonly toolName?: string;
-    }): readonly InputRequest[];
+    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(input: SendTurnInput): Promise<EveEvalTurn>;
@@ -40,6 +42,7 @@ export declare class EvalSessionManager {
     #private;
     constructor(input: {
         readonly client: Client;
+        readonly collector?: AssertionCollector;
         readonly signal?: AbortSignal;
     });
     get primary(): EvalSessionDriver;
@@ -49,4 +52,5 @@ export declare class EvalSessionManager {
     }): Promise<EvalSessionDriver>;
     snapshots(): readonly EveEvalSessionResult[];
     lastTurnSession(): EvalSessionDriver | undefined;
+    hasActivity(): boolean;
 }

package/dist/src/evals/session.js

@@ -1 +1 @@
-import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=[];#r;#i=[];constructor(e){this.#e=e.session,this.#t=e.signal}get events(){return this.#n}get lastTurn(){return this.#r}get pendingInputRequests(){return this.#i}get sessionId(){return this.#e.state.sessionId??this.#r?.sessionId}get state(){return this.#e.state}expectInputRequests(e){if(this.#i.length===0)throw Error(`Expected pending input requests, but the last turn did not park.`);let t=this.#i.filter(t=>inputRequestMatches(t,e));if(t.length===0)throw Error(`No pending input requests matched ${formatInputRequestFilter(e)}.`);return t}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.expectInputRequests();for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#a({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#a({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:t,status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#n,{sessionId:t}),events:[...this.#n],primary:e,sessionId:t,state:this.#e.state}}#a(e){this.#n.push(...e.events),this.#i=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({data:e.data,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId??this.sessionId??``,status:e.status,toolCalls:t.toolCalls});return this.#r=n,n}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}},EvalSessionManager=class{#e;#t;#n=[];#r;constructor(e){this.#e=e.client,this.#t=e.signal}get primary(){return this.#r??=this.#i(),this.#r}newSession(){return this.#i()}async attachSession(e,t){let n=new EvalSessionDriver({session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#n.push(n),await n.readTurn(t),n}snapshots(){return this.#n.map(e=>e.snapshot(e===this.#r))}lastTurnSession(){return this.#r?.lastTurn===void 0?this.#n.find(e=>e.lastTurn!==void 0):this.#r}#i(){let e=new EvalSessionDriver({session:this.#e.session(),signal:this.#t});return this.#n.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function inputRequestMatches(e,t){return t===void 0?!0:t.display!==void 0&&e.display!==t.display?!1:t.toolName===void 0?!0:e.action.kind===`tool-call`&&e.action.toolName===t.toolName}function formatInputRequestFilter(e){return e===void 0?`{}`:JSON.stringify(e)}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};
+import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r=[];#i;#a=[];constructor(e){this.#n=e.collector,this.#e=e.session,this.#t=e.signal,Object.assign(this,createScopedAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}),createOutputAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}))}get events(){return this.#r}get lastTurn(){return this.#i}get pendingInputRequests(){return this.#a}get sessionId(){return this.#e.state.sessionId??this.#i?.sessionId}get state(){return this.#e.state}requireInputRequest(e={}){this.#a.length===0&&this.#c(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#a.filter(t=>inputRequestMatches(t,e));return(this.#a.length!==1||t.length!==1)&&this.#c(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#a.length} pending request(s)`),this.#n.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.#a;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#o({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#o({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:requireSessionId(t),status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:t}),events:[...this.#r],primary:e,sessionId:t,state:this.#e.state}}#o(e){this.#r.push(...e.events),this.#a=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#n,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#i=n,n}#s(){let e=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:e}),events:[...this.#r],output:outputOf(this.#i),status:this.#i?.status??`completed`}}#c(e,t){throw this.#n.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r=[];#i;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#t=e.signal}get primary(){return this.#i??=this.#a(),this.#i}newSession(){return this.#a()}async attachSession(e,t){let n=new EvalSessionDriver({collector:this.#n,session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#r.push(n),await n.readTurn(t),n}snapshots(){return this.#r.map(e=>e.snapshot(e===this.#i))}lastTurnSession(){return this.#i?.lastTurn===void 0?this.#r.find(e=>e.lastTurn!==void 0):this.#i}hasActivity(){return this.#r.length>0}#a(){let e=new EvalSessionDriver({collector:this.#n,session:this.#e.session(),signal:this.#t});return this.#r.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};