eve 0.13.8 → 0.15.0

package/dist/src/evals/runner/execute-task.d.ts

@@ -20,6 +20,7 @@ export interface ExecuteTaskResult {
     readonly result: EveEvalTaskResult;
     readonly assertions: readonly AssertionResult[];
     readonly error?: string;
+    readonly skipReason?: string;
 }
 /**
  * Executes one eval's `test(t)` against an eve agent target: drives the

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(r){let{client:a,evaluation:o,target:s,timeoutMs:c}=r,l=c===void 0?neverAbortSignal():AbortSignal.timeout(c),u=new EvalSessionManager({client:a,signal:l}),d=scopeEvalTargetHandle(s,{sessions:u}),f=[],{context:p,collector:m}=createEvalContext({manager:u,target:d,signal:l,judge:o.judge,log:e=>{f.push(e),r.onLog?.(e)}}),h;try{await o.test(p)}catch(t){h=toErrorMessage(t)}let g=buildTaskResult({logs:f,sessions:u.snapshots(),turn:u.lastTurnSession()?.lastTurn});return{result:g,assertions:await m.finalize(g),error:h}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:n,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:a}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await s.test(h)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r}=e,i=t.filter(e=>e.severity===`gate`),a=t.filter(e=>e.severity===`soft`),o=this.#n(n),s=i.length>0?this.#r(i.filter(e=>e.passed).length,i.length):``,c=a.map(e=>this.#i(e.name,e.score)).join(`  `),l=[o,this.#t.dim(e.id),s,c].filter(Boolean).join(`  `);this.#e(l);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,results:i}=e,a=i.length,o=[];t>0&&o.push(this.#t.green(`${t} passed`)),n>0&&o.push(this.#t.red(`${n} failed`)),r>0&&o.push(this.#t.yellow(`${r} scored`)),o.length===0&&o.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${o.join(`, `)} ${this.#t.dim(`(${a} total)`)}`);let s=this.#a(i);if(s.total>0){let e=this.#t.green(`${s.passed} passed`),t=s.failed>0?`, ${this.#t.red(`${s.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let c=this.#o(i);if(c.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of c){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let l=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(l)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.map(e=>this.#i(e.name,e.score)).join(`  `),u=[s,this.#t.dim(e.id),c,l].filter(Boolean).join(`  `);this.#e(u);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/reporters/junit.js

@@ -1,3 +1,4 @@
-import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="0" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
-`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
+import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
+`);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
 `)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.find(e=>!e.passed);return t===void 0?e.verdict===`scored`?`score below threshold`:e.verdict:t.message===void 0?t.name:`${t.name}: ${t.message}`}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/runner/verdict.d.ts

@@ -10,4 +10,5 @@ import type { AssertionResult, EveEvalVerdict } from "#evals/types.js";
 export declare function computeEvalVerdict(input: {
     readonly error?: string;
     readonly assertions: readonly AssertionResult[];
+    readonly skipReason?: string;
 }): EveEvalVerdict;

package/dist/src/evals/runner/verdict.js

@@ -1 +1 @@
-function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return t?`scored`:`passed`}export{computeEvalVerdict};
+function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return e.skipReason===void 0?t?`scored`:`passed`:`skipped`}export{computeEvalVerdict};

package/dist/src/evals/session.d.ts

@@ -3,7 +3,9 @@ import type { ClientSession } from "#client/session.js";
 import type { SendTurnInput, SessionState } from "#client/types.js";
 import type { HandleMessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
-import type { EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalAssertions, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
  */
@@ -12,9 +14,12 @@ export declare class EveEvalTurnFailedError extends Error {
     readonly turn: EveEvalTurn;
     constructor(turn: EveEvalTurn);
 }
+export interface EvalSessionDriver extends EveEvalAssertions, EveEvalOutputAssertions {
+}
 export declare class EvalSessionDriver implements EveEvalSession {
     #private;
     constructor(input: {
+        readonly collector: AssertionCollector;
         readonly session: ClientSession;
         readonly signal?: AbortSignal;
     });
@@ -23,10 +28,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
     get pendingInputRequests(): readonly InputRequest[];
     get sessionId(): string | undefined;
     get state(): SessionState;
-    expectInputRequests(filter?: {
-        readonly display?: InputRequest["display"];
-        readonly toolName?: string;
-    }): readonly InputRequest[];
+    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(input: SendTurnInput): Promise<EveEvalTurn>;
@@ -40,6 +42,7 @@ export declare class EvalSessionManager {
     #private;
     constructor(input: {
         readonly client: Client;
+        readonly collector?: AssertionCollector;
         readonly signal?: AbortSignal;
     });
     get primary(): EvalSessionDriver;
@@ -49,4 +52,5 @@ export declare class EvalSessionManager {
     }): Promise<EvalSessionDriver>;
     snapshots(): readonly EveEvalSessionResult[];
     lastTurnSession(): EvalSessionDriver | undefined;
+    hasActivity(): boolean;
 }

package/dist/src/evals/session.js

@@ -1 +1 @@
-import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=[];#r;#i=[];constructor(e){this.#e=e.session,this.#t=e.signal}get events(){return this.#n}get lastTurn(){return this.#r}get pendingInputRequests(){return this.#i}get sessionId(){return this.#e.state.sessionId??this.#r?.sessionId}get state(){return this.#e.state}expectInputRequests(e){if(this.#i.length===0)throw Error(`Expected pending input requests, but the last turn did not park.`);let t=this.#i.filter(t=>inputRequestMatches(t,e));if(t.length===0)throw Error(`No pending input requests matched ${formatInputRequestFilter(e)}.`);return t}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.expectInputRequests();for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#a({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#a({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:t,status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#n,{sessionId:t}),events:[...this.#n],primary:e,sessionId:t,state:this.#e.state}}#a(e){this.#n.push(...e.events),this.#i=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({data:e.data,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId??this.sessionId??``,status:e.status,toolCalls:t.toolCalls});return this.#r=n,n}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}},EvalSessionManager=class{#e;#t;#n=[];#r;constructor(e){this.#e=e.client,this.#t=e.signal}get primary(){return this.#r??=this.#i(),this.#r}newSession(){return this.#i()}async attachSession(e,t){let n=new EvalSessionDriver({session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#n.push(n),await n.readTurn(t),n}snapshots(){return this.#n.map(e=>e.snapshot(e===this.#r))}lastTurnSession(){return this.#r?.lastTurn===void 0?this.#n.find(e=>e.lastTurn!==void 0):this.#r}#i(){let e=new EvalSessionDriver({session:this.#e.session(),signal:this.#t});return this.#n.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function inputRequestMatches(e,t){return t===void 0?!0:t.display!==void 0&&e.display!==t.display?!1:t.toolName===void 0?!0:e.action.kind===`tool-call`&&e.action.toolName===t.toolName}function formatInputRequestFilter(e){return e===void 0?`{}`:JSON.stringify(e)}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};
+import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r=[];#i;#a=[];constructor(e){this.#n=e.collector,this.#e=e.session,this.#t=e.signal,Object.assign(this,createScopedAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}),createOutputAssertions(this.#n,{timing:`snapshot`,select:()=>this.#s()}))}get events(){return this.#r}get lastTurn(){return this.#i}get pendingInputRequests(){return this.#a}get sessionId(){return this.#e.state.sessionId??this.#i?.sessionId}get state(){return this.#e.state}requireInputRequest(e={}){this.#a.length===0&&this.#c(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#a.filter(t=>inputRequestMatches(t,e));return(this.#a.length!==1||t.length!==1)&&this.#c(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#a.length} pending request(s)`),this.#n.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.#a;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#o({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#o({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:requireSessionId(t),status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:t}),events:[...this.#r],primary:e,sessionId:t,state:this.#e.state}}#o(e){this.#r.push(...e.events),this.#a=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#n,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#i=n,n}#s(){let e=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:e}),events:[...this.#r],output:outputOf(this.#i),status:this.#i?.status??`completed`}}#c(e,t){throw this.#n.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r=[];#i;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#t=e.signal}get primary(){return this.#i??=this.#a(),this.#i}newSession(){return this.#a()}async attachSession(e,t){let n=new EvalSessionDriver({collector:this.#n,session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#r.push(n),await n.readTurn(t),n}snapshots(){return this.#r.map(e=>e.snapshot(e===this.#i))}lastTurnSession(){return this.#i?.lastTurn===void 0?this.#r.find(e=>e.lastTurn!==void 0):this.#i}hasActivity(){return this.#r.length>0}#a(){let e=new EvalSessionDriver({collector:this.#n,session:this.#e.session(),signal:this.#t});return this.#r.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};

package/dist/src/evals/types.d.ts

@@ -3,10 +3,12 @@ import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js"
 import type { HandleMessageStreamEvent, RuntimeIdentity } from "#protocol/message.js";
 import type { SendTurnInput, SessionState } from "#client/types.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
-import type { JsonObject } from "#shared/json.js";
+import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
-import type { EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
+import type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
+/** Lifecycle outcome of an eval-observed tool or subagent action. */
+export type EveEvalActionStatus = "pending" | "completed" | "failed" | "rejected";
 /**
  * One tool call extracted from the captured stream, pairing the
  * `actions.requested` request with its matching `action.result`.
@@ -17,9 +19,9 @@ export interface EveEvalToolCall {
     /** Tool input as requested by the model. */
     readonly input: JsonObject;
     /** Tool output from the matching `action.result`; `undefined` when the call never resolved. */
-    readonly output: unknown;
-    /** True when the matching `action.result` reported a failure. */
-    readonly isError: boolean;
+    readonly output: JsonValue | undefined;
+    /** Whether the request is unresolved, completed, failed, or user-rejected. */
+    readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the call happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -35,9 +37,9 @@ export interface EveEvalSubagentCall {
     /** Remote agent URL for remote delegations (`subagent.called` remote metadata). */
     readonly remoteUrl?: string;
     /** Output from the matching `subagent.completed` event; `undefined` when the call never completed. */
-    readonly output?: unknown;
-    /** True when the matching subagent action result reported a failure. */
-    readonly isError: boolean;
+    readonly output?: JsonValue;
+    /** Whether the delegation is unresolved, completed, failed, or rejected. */
+    readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the delegation happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -74,9 +76,8 @@ export interface EveEvalSessionResult {
  */
 export interface EveEvalTaskResult {
     /**
-     * The agent's last assistant message (same as {@link finalMessage}), retained
-     * for reporters and artifacts that log a single "output" value. Mutable
-     * because the runner assigns it after the run completes.
+     * The final turn's structured data when present, otherwise its last assistant
+     * message. Retained for reporters and artifacts that log one output value.
      */
     output: unknown;
     /** The agent's last assistant message, or null when none was produced. */
@@ -131,10 +132,10 @@ export interface Assertion {
 /**
  * Handle to a recorded assertion, returned by every `t` assertion method.
  * Chain `gate`/`soft`/`atLeast` to override the recorded severity or
- * threshold, and `await` it to surface model-backed (judge) errors and ensure
- * the assertion has resolved before the run continues.
+ * threshold. Recorded assertions are finalized by the runner; use `t.require`
+ * or a `require*` lookup when later control flow depends on a passing result.
  */
-export interface AssertionHandle extends PromiseLike<void> {
+export interface AssertionHandle {
     gate(threshold?: number): this;
     soft(threshold?: number): this;
     atLeast(threshold: number): this;
@@ -153,10 +154,37 @@ export interface AssertionResult {
     readonly message?: string;
     readonly metadata?: Readonly<Record<string, unknown>>;
 }
-/**
- * Driver for one session, exposed on the eval context and by `t.newSession()`.
- */
-export interface EveEvalSession {
+/** Recorded assertions shared by aggregate, session, and turn scopes. */
+export interface EveEvalAssertions {
+    succeeded(): AssertionHandle;
+    parked(): AssertionHandle;
+    messageIncludes(token: string | RegExp): AssertionHandle;
+    calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
+    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
+    loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
+    notCalledTool(name: string): AssertionHandle;
+    /** Asserts that tool requests appeared in order, allowing unrelated requests between them. */
+    toolOrder(names: readonly string[]): AssertionHandle;
+    usedNoTools(): AssertionHandle;
+    maxToolCalls(max: number): AssertionHandle;
+    calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
+    noFailedActions(): AssertionHandle;
+    event<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+        type: TType;
+    }>, "type">): AssertionHandle;
+    notEvent<TType extends HandleMessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+        type: TType;
+    }>, "type" | "count">): AssertionHandle;
+    eventOrder(matchers: readonly EveEvalEventMatch[]): AssertionHandle;
+    eventsSatisfy(label: string, predicate: (events: readonly HandleMessageStreamEvent[]) => boolean): AssertionHandle;
+}
+/** Assertions over the final output captured by one session or immutable turn. */
+export interface EveEvalOutputAssertions {
+    outputEquals(value: unknown): AssertionHandle;
+    outputMatches(schema: StandardSchemaV1): AssertionHandle;
+}
+/** Operations and state shared by the primary eval context and independent sessions. */
+export interface EveEvalSessionDriver {
     /** All events observed on this session so far. */
     readonly events: readonly HandleMessageStreamEvent[];
     /** Input requests left pending by the last parked turn. */
@@ -165,11 +193,8 @@ export interface EveEvalSession {
     readonly state: SessionState;
     /** eve session id after the first successful send. */
     readonly sessionId: string | undefined;
-    /** Assert the last turn parked on HITL input and return matching requests. */
-    expectInputRequests(filter?: {
-        readonly display?: InputRequest["display"];
-        readonly toolName?: string;
-    }): readonly InputRequest[];
+    /** Require exactly one pending input request matching `filter`, or abort dependent control flow. */
+    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     /** Resolve specific pending requests and run the resumed turn. */
     respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
     /** Resolve every pending request with the same option id. */
@@ -179,16 +204,22 @@ export interface EveEvalSession {
     /** Send one text turn with a local file attached as a data URL. */
     sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
 }
+/** Driver for one independent session, exposed by `t.newSession()` and target attachment helpers. */
+export interface EveEvalSession extends EveEvalSessionDriver, EveEvalAssertions, EveEvalOutputAssertions {
+}
 /**
  * One completed eval-driver turn.
  */
-export interface EveEvalTurn {
+export interface EveEvalTurn extends EveEvalAssertions, EveEvalOutputAssertions {
     readonly data: unknown;
     readonly events: readonly HandleMessageStreamEvent[];
     readonly inputRequests: readonly InputRequest[];
     readonly message: string | undefined;
+    readonly sessionId: string;
     readonly status: "completed" | "failed" | "waiting";
     readonly toolCalls: readonly EveEvalToolCall[];
+    /** Require exactly one matching tool call, record a gate, and return it for dependent checks. */
+    requireToolCall(name: string, options?: Omit<EveEvalToolCallMatchOptions, "count">): EveEvalToolCall;
     expectOk(): this;
 }
 /**
@@ -232,15 +263,13 @@ export interface JudgeContext {
 }
 /**
  * The single context passed to an eval's `test(t)` function. It drives the
- * primary session (it extends {@link EveEvalSession}), carries the run-level
+ * primary session, carries the run-level
  * and value-level assertion vocabulary, and exposes `judge` for LLM-as-judge.
  *
- * Run-level assertions (`completed`, `calledTool`, …) record an entry
- * evaluated against the final run and never throw; `check` and `judge`
- * evaluate the supplied value immediately. Use plain `throw` /
- * `turn.expectOk()` for bespoke preconditions that should abort the run.
+ * Scoped assertions (`succeeded`, `calledTool`, …) record an entry evaluated
+ * after the test body; `check`, `require`, and `judge` evaluate explicit values.
  */
-export interface EveEvalContext extends EveEvalSession {
+export interface EveEvalContext extends EveEvalSessionDriver, EveEvalAssertions {
     /** Eval timeout signal. */
     readonly signal: AbortSignal;
     /** Current target under test. */
@@ -253,24 +282,12 @@ export interface EveEvalContext extends EveEvalSession {
     sleep(ms?: number): Promise<void>;
     /** Create an additional independent session against the same target. */
     newSession(): EveEvalSession;
-    completed(): AssertionHandle;
-    didNotFail(): AssertionHandle;
-    waiting(): AssertionHandle;
-    messageIncludes(token: string | RegExp): AssertionHandle;
-    calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
-    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
-    loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
-    notCalledTool(name: string): AssertionHandle;
-    toolOrder(names: readonly string[]): AssertionHandle;
-    usedNoTools(): AssertionHandle;
-    maxToolCalls(max: number): AssertionHandle;
-    calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
-    noFailedActions(): AssertionHandle;
-    event(predicate: (events: readonly HandleMessageStreamEvent[]) => boolean, label: string): AssertionHandle;
-    outputEquals(value: unknown): AssertionHandle;
-    outputMatches(schema: StandardSchemaV1): AssertionHandle;
     /** Apply a value-level assertion (from `eve/evals/expect`) to a value. */
     check(value: unknown, assertion: Assertion): AssertionHandle;
+    /** Record an immediate gate and abort dependent control flow when it fails. */
+    require<T>(value: T, assertion: Assertion): Promise<T>;
+    /** Mark this eval as intentionally skipped and stop executing its test body. */
+    skip(reason: string): never;
     /** LLM-as-judge assertions, bound to the resolved judge model. */
     readonly judge: JudgeContext;
 }
@@ -372,8 +389,9 @@ export type EveEval = EveEvalDefinition & {
  * - `"passed"`  — no execution error, every gate held, every soft threshold met
  * - `"failed"`  — a gate assertion failed or execution errored (timeout, transport, thrown task)
  * - `"scored"`  — every gate held but a soft assertion fell below its threshold
+ * - `"skipped"` — the test body intentionally called `t.skip(reason)`
  */
-export type EveEvalVerdict = "passed" | "failed" | "scored";
+export type EveEvalVerdict = "passed" | "failed" | "scored" | "skipped";
 /**
  * Result of executing and asserting one eval.
  *
@@ -388,6 +406,8 @@ export interface EveEvalResult {
     /** Per-eval verdict; see {@link EveEvalVerdict}. */
     readonly verdict: EveEvalVerdict;
     readonly error?: string;
+    /** Why the eval intentionally skipped, present only for a `"skipped"` verdict. */
+    readonly skipReason?: string;
     readonly startedAt: string;
     readonly completedAt: string;
 }
@@ -405,6 +425,8 @@ export interface EveEvalRunSummary {
     readonly failed: number;
     /** Evals with verdict `"scored"` (below-threshold soft assertions only). */
     readonly scored: number;
+    /** Evals intentionally skipped by their test body. */
+    readonly skipped: number;
     /** The execution-error subset of `failed` (timeouts, connection failures, exceptions). */
     readonly errored: number;
 }

package/dist/src/execution/dispatch-runtime-actions-step.js

@@ -1 +1 @@
-import{createLogger,logError}from"#internal/logging.js";import{callAdapterEventHandler}from"#channel/adapter.js";import{AuthKey,CapabilitiesKey,ChannelInstrumentationKey,InitiatorAuthKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{createSubagentCalledEvent,encodeMessageStreamEvent,timestampHandleMessageStreamEvent}from"#protocol/message.js";import{BundleKey,ChannelKey}from"#runtime/sessions/runtime-context-keys.js";import{createDurableSessionState,readDurableSession}from"#execution/durable-session-store.js";import{hydrateDurableSession}from"#execution/session.js";import{deserializeContext}from"#context/serialize.js";import{getPendingRuntimeActionBatch,recordPendingSubagentChildToken}from"#harness/runtime-actions.js";import{buildAdapterContext}from"#channel/adapter-context.js";import{resolveRemoteAgentForAction,startRemoteAgentSession}from"#execution/remote-agent-dispatch.js";import{buildSubagentRunInput}from"#execution/subagent-tool.js";import{createWorkflowRuntime,workflowEntryReference}from"#execution/workflow-runtime.js";const log=createLogger(`execution.dispatch-runtime-actions`);async function dispatchRuntimeActionsStep(e){"use step";let s=await readDurableSession(e.sessionState),l=getPendingRuntimeActionBatch(s.state);if(l===void 0||l.actions.length===0)return{results:[],sessionState:e.sessionState};let u=await deserializeContext(e.serializedContext),d=u.require(BundleKey),f=hydrateDurableSession({compactionOverrides:{thresholdPercent:d.resolvedAgent.config.compaction?.thresholdPercent},durable:s,turnAgent:d.turnAgent}),p=u.require(ChannelKey),m=u.get(AuthKey)??null,h=u.get(CapabilitiesKey),g=u.get(ChannelInstrumentationKey),_=u.get(InitiatorAuthKey)??null,v=e.parentWritable.getWriter(),y=buildAdapterContext(p,u),b=f,x=[];try{for(let r of l.actions){let i,a,o,s;switch(r.kind){case`subagent-call`:{let e=createWorkflowRuntime({compiledArtifactsSource:d.compiledArtifactsSource,nodeId:r.nodeId}),{childContinuationToken:t,runInput:n}=buildSubagentRunInput({action:r,auth:m,batchEvent:l.event,capabilities:h,channelMetadata:g,initiatorAuth:_,session:f}),o=await e.run(n);b=recordPendingSubagentChildToken({callId:r.callId,childContinuationToken:t,session:b}),i=o.sessionId,a=r.name,s=r.subagentName;break}case`remote-agent-call`:{let n;try{n=resolveRemoteAgentForAction({nodeId:r.nodeId,remoteAgentName:r.remoteAgentName,registry:d.subagentRegistry.subagentsByNodeId}),i=await startRemoteAgentSession({action:r,callbackBaseUrl:e.callbackBaseUrl,remote:n,session:f})}catch(e){logError(log,`remote agent start failed`,e,{remoteAgentName:r.remoteAgentName,nodeId:r.nodeId,callId:r.callId}),x.push(createRemoteAgentStartFailureResult({action:r,error:e}));continue}a=r.name,o={url:n.url},s=r.remoteAgentName;break}default:throw Error(`Unsupported runtime action kind "${r.kind}" in workflow runtime.`)}let c=await callAdapterEventHandler(p,createSubagentCalledEvent({callId:r.callId,childSessionId:i,name:a,remote:o,sequence:l.event.sequence,sessionId:f.sessionId,toolName:s,turnId:l.event.turnId,workflowId:workflowEntryReference.workflowId}),y);await v.write(encodeMessageStreamEvent(timestampHandleMessageStreamEvent(c)))}}finally{v.releaseLock()}return{results:x,sessionState:b===f?e.sessionState:createDurableSessionState({session:b})}}function createRemoteAgentStartFailureResult(e){return{callId:e.action.callId,isError:!0,kind:`subagent-result`,output:{code:`REMOTE_AGENT_START_FAILED`,message:toErrorMessage(e.error)},subagentName:e.action.remoteAgentName}}export{dispatchRuntimeActionsStep};
+import{createLogger,logError}from"#internal/logging.js";import{callAdapterEventHandler}from"#channel/adapter.js";import{AuthKey,CapabilitiesKey,ChannelInstrumentationKey,InitiatorAuthKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{createSubagentCalledEvent,encodeMessageStreamEvent,timestampHandleMessageStreamEvent}from"#protocol/message.js";import{BundleKey,ChannelKey}from"#runtime/sessions/runtime-context-keys.js";import{createDurableSessionState,readDurableSession}from"#execution/durable-session-store.js";import{hydrateDurableSession}from"#execution/session.js";import{deserializeContext}from"#context/serialize.js";import{buildAdapterContext}from"#channel/adapter-context.js";import{getPendingRuntimeActionBatch,recordPendingSubagentChildToken}from"#harness/runtime-actions.js";import{resolveRemoteAgentForAction,startRemoteAgentSession}from"#execution/remote-agent-dispatch.js";import{buildSubagentRunInput}from"#execution/subagent-tool.js";import{createWorkflowRuntime,workflowEntryReference}from"#execution/workflow-runtime.js";const log=createLogger(`execution.dispatch-runtime-actions`);async function dispatchRuntimeActionsStep(e){"use step";let s=await readDurableSession(e.sessionState),l=getPendingRuntimeActionBatch(s.state);if(l===void 0||l.actions.length===0)return{results:[],sessionState:e.sessionState};let u=await deserializeContext(e.serializedContext),d=u.require(BundleKey),f=hydrateDurableSession({compactionOverrides:{thresholdPercent:d.resolvedAgent.config.compaction?.thresholdPercent},durable:s,turnAgent:d.turnAgent}),p=u.require(ChannelKey),m=u.get(AuthKey)??null,h=u.get(CapabilitiesKey),g=u.get(ChannelInstrumentationKey),_=u.get(InitiatorAuthKey)??null,v=e.parentWritable.getWriter(),y=buildAdapterContext(p,u),b=f,x=[];try{for(let r of l.actions){let i,a,o,s;switch(r.kind){case`subagent-call`:{let e=createWorkflowRuntime({compiledArtifactsSource:d.compiledArtifactsSource,nodeId:r.nodeId}),{childContinuationToken:t,runInput:n}=buildSubagentRunInput({action:r,auth:m,batchEvent:l.event,capabilities:h,channelMetadata:g,initiatorAuth:_,session:f}),o=await e.run(n);b=recordPendingSubagentChildToken({callId:r.callId,childContinuationToken:t,session:b}),i=o.sessionId,a=r.name,s=r.subagentName;break}case`remote-agent-call`:{let n;try{n=resolveRemoteAgentForAction({nodeId:r.nodeId,remoteAgentName:r.remoteAgentName,registry:d.subagentRegistry.subagentsByNodeId}),i=await startRemoteAgentSession({action:r,callbackBaseUrl:e.callbackBaseUrl,remote:n,session:f})}catch(e){logError(log,`remote agent start failed`,e,{remoteAgentName:r.remoteAgentName,nodeId:r.nodeId,callId:r.callId}),x.push(createRemoteAgentStartFailureResult({action:r,error:e}));continue}a=r.name,o={url:n.url},s=r.remoteAgentName;break}default:throw Error(`Unsupported runtime action kind "${r.kind}" in workflow runtime.`)}let c=await callAdapterEventHandler(p,createSubagentCalledEvent({callId:r.callId,childSessionId:i,name:a,remote:o,sequence:l.event.sequence,sessionId:f.sessionId,toolName:s,turnId:l.event.turnId,workflowId:workflowEntryReference.workflowId}),y);await v.write(encodeMessageStreamEvent(timestampHandleMessageStreamEvent(c)))}}finally{v.releaseLock()}return{results:x,sessionState:b===f?e.sessionState:createDurableSessionState({session:b})}}function createRemoteAgentStartFailureResult(e){return{callId:e.action.callId,isError:!0,kind:`subagent-result`,output:{code:`REMOTE_AGENT_START_FAILED`,message:toErrorMessage(e.error)},subagentName:e.action.remoteAgentName}}export{dispatchRuntimeActionsStep};

package/dist/src/execution/dispatch-workflow-runtime-actions-step.d.ts

@@ -0,0 +1,12 @@
+import { type DurableSessionState } from "#execution/durable-session-store.js";
+import type { RuntimeSubagentResultActionResult } from "#runtime/actions/types.js";
+/** Dispatches the child-agent action currently blocking a dynamic workflow. */
+export declare function dispatchWorkflowRuntimeActionsStep(input: {
+    readonly callbackBaseUrl?: string;
+    readonly parentWritable: WritableStream<Uint8Array>;
+    readonly serializedContext: Record<string, unknown>;
+    readonly sessionState: DurableSessionState;
+}): Promise<{
+    readonly results: readonly RuntimeSubagentResultActionResult[];
+    readonly sessionState: DurableSessionState;
+}>;

package/dist/src/execution/dispatch-workflow-runtime-actions-step.js

@@ -0,0 +1 @@
+import{BundleKey}from"#runtime/sessions/runtime-context-keys.js";import{createDurableSessionState,readDurableSession}from"#execution/durable-session-store.js";import{hydrateDurableSession}from"#execution/session.js";import{deserializeContext}from"#context/serialize.js";import{setPendingRuntimeActionBatch}from"#harness/runtime-actions.js";import{dispatchRuntimeActionsStep}from"#execution/dispatch-runtime-actions-step.js";import{getPendingWorkflowInterrupt}from"#harness/workflow-interrupt-state.js";import{buildRuntimeActionsFromWorkflowInterrupt}from"#harness/workflow-runtime-action-state.js";async function dispatchWorkflowRuntimeActionsStep(e){"use step";let t=await readDurableSession(e.sessionState),n=getPendingWorkflowInterrupt(t.state);if(n===void 0)return{results:[],sessionState:e.sessionState};let r=buildRuntimeActionsFromWorkflowInterrupt(n.interrupt);if(r.length===0)return{results:[],sessionState:e.sessionState};let i=(await deserializeContext(e.serializedContext)).require(BundleKey),a=setPendingRuntimeActionBatch({actions:r,event:{sequence:0,stepIndex:0,turnId:`workflow-dispatch`},responseMessages:[],session:hydrateDurableSession({compactionOverrides:{thresholdPercent:i.resolvedAgent.config.compaction?.thresholdPercent},durable:t,turnAgent:i.turnAgent})});return dispatchRuntimeActionsStep({callbackBaseUrl:e.callbackBaseUrl,parentWritable:e.parentWritable,serializedContext:e.serializedContext,sessionState:createDurableSessionState({session:a})})}export{dispatchWorkflowRuntimeActionsStep};

package/dist/src/execution/next-driver-action.d.ts

@@ -30,7 +30,7 @@ export type NextDriverAction = {
     readonly sessionState: DurableSessionState;
     readonly serializedContext: Record<string, unknown>;
 } | {
-    readonly kind: "dispatch-code-mode-runtime-actions";
+    readonly kind: "dispatch-workflow-runtime-actions";
     readonly pendingActionKeys: readonly string[];
     readonly sessionState: DurableSessionState;
     readonly serializedContext: Record<string, unknown>;

package/dist/src/execution/node-step.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{jsonSchema}from"ai";import{createToolLoopHarness}from"#harness/tool-loop.js";import{resolveCodeModeEnabled}from"#shared/code-mode.js";import{resolveRuntimeModelReference}from"#runtime/agent/resolve-model.js";import{findRegisteredRuntimeTool}from"#runtime/tools/registry.js";import{SUBAGENT_TOOL_INPUT_SCHEMA}from"#runtime/subagents/registry.js";import{preserveFrameworkStateOnCompaction}from"#execution/compaction.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";const log=createLogger(`execution.node-step`);function createExecutionNodeStep(e){let t=createRuntimeModelResolver(e.modelResolutionScope),n=createNodeHarnessTools({node:e.node});return createToolLoopHarness({capabilities:e.capabilities,codeMode:resolveCodeModeEnabled(e.node.agent.config?.experimental?.codeMode),workflow:e.node.agent.workflowEnabled===!0,handleEvent:e.handleEvent,mode:e.mode,onCompaction:preserveFrameworkStateOnCompaction,resolveModel:t,runtimeIdentity:buildRuntimeIdentity(e.node),tools:n})}function buildRuntimeIdentity(e){let n=resolveInstalledPackageInfo(),r={agentId:e.turnAgent.id,agentName:e.agent.config?.name,eveVersion:n.version,modelId:e.turnAgent.model.id},i=process.env.VERCEL_GIT_COMMIT_SHA?.trim(),a=process.env.VERCEL_GIT_COMMIT_REF?.trim(),o=process.env.VERCEL_DEPLOYMENT_CREATED_AT?.trim();return i||a||o?{...r,build:{deployedAt:o||void 0,gitBranch:a||void 0,gitSha:i||void 0}}:r}function createRuntimeModelResolver(e){return t=>resolveRuntimeModelReference(t,e)}function createNodeHarnessTools(e){let t=new Map;for(let n of e.node.turnAgent.tools){let r=resolveHarnessToolDefinition({node:e.node,tool:n});r!==null&&t.set(n.name,r)}return t.has(`agent`)||t.set(`agent`,{description:`Launch a new agent to handle a complex, multi-step subtask.`,inputSchema:jsonSchema(SUBAGENT_TOOL_INPUT_SCHEMA),name:`agent`,runtimeAction:{kind:`subagent-call`,nodeId:e.node.nodeId,subagentName:`agent`}}),t}function resolveHarnessToolDefinition(e){if(e.tool.kind===`subagent`)return{description:e.tool.description??``,inputSchema:jsonSchema(e.tool.inputSchema??{}),name:e.tool.name,outputSchema:e.tool.outputSchema===void 0?void 0:jsonSchema(e.tool.outputSchema),runtimeAction:{kind:`subagent-call`,nodeId:e.tool.nodeId,subagentName:e.tool.name}};if(e.tool.kind===`remote`)return{description:e.tool.description??``,inputSchema:jsonSchema(e.tool.inputSchema??{}),name:e.tool.name,outputSchema:e.tool.outputSchema===void 0?void 0:jsonSchema(e.tool.outputSchema),runtimeAction:{kind:`remote-agent-call`,nodeId:e.tool.nodeId,remoteAgentName:e.tool.name,subagentName:e.tool.name}};let t=findRegisteredRuntimeTool(e.node.toolRegistry,e.tool.name);if(t===null)return log.warn(`declared tool is not registered — omitting from toolset`,{toolName:e.tool.name,nodeId:e.node.nodeId}),null;let r=t.definition,i=r.sourceId.startsWith(`eve:`),a=r.execute;return{approvalKey:r.approvalKey,description:r.description,execute:resolveAuthoredExecute({isFrameworkTool:i,rawExecute:a,scope:r.name}),inputSchema:r.inputStandardSchema??jsonSchema(r.inputSchema??{}),name:r.name,needsApproval:r.needsApproval,outputSchema:r.outputStandardSchema??maybeJsonSchema(r.outputSchema),toModelOutput:r.toModelOutput}}function resolveAuthoredExecute(e){let{isFrameworkTool:t,rawExecute:n,scope:r}=e;return n===void 0?void 0:t?n:createToolExecuteWithAuth({execute:n,scope:r})}function maybeJsonSchema(e){return e===void 0?void 0:jsonSchema(e)}export{createExecutionNodeStep,createNodeHarnessTools};
+import{createLogger}from"#internal/logging.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{jsonSchema}from"ai";import{createToolLoopHarness}from"#harness/tool-loop.js";import{resolveRuntimeModelReference}from"#runtime/agent/resolve-model.js";import{findRegisteredRuntimeTool}from"#runtime/tools/registry.js";import{SUBAGENT_TOOL_INPUT_SCHEMA}from"#runtime/subagents/registry.js";import{preserveFrameworkStateOnCompaction}from"#execution/compaction.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";const log=createLogger(`execution.node-step`),BUILT_IN_AGENT_TOOL_DESCRIPTION=[`Delegate a focused subtask to a fresh copy of yourself.`,`Use it to isolate complex work or split a large task into independent pieces.`,"Issue multiple `agent` calls in one response to run a small fixed set in parallel.","Each child has fresh history and state but shares your tools and sandbox, so include essential context in `message` and give parallel writers non-overlapping scopes."].join(` `);function createExecutionNodeStep(e){let t=createRuntimeModelResolver(e.modelResolutionScope),n=createNodeHarnessTools({node:e.node});return createToolLoopHarness({capabilities:e.capabilities,workflow:e.node.agent.workflowEnabled===!0,handleEvent:e.handleEvent,mode:e.mode,onCompaction:preserveFrameworkStateOnCompaction,resolveModel:t,runtimeIdentity:buildRuntimeIdentity(e.node),tools:n})}function buildRuntimeIdentity(e){let n=resolveInstalledPackageInfo(),r={agentId:e.turnAgent.id,agentName:e.agent.config?.name,eveVersion:n.version,modelId:e.turnAgent.model.id},i=process.env.VERCEL_GIT_COMMIT_SHA?.trim(),a=process.env.VERCEL_GIT_COMMIT_REF?.trim(),o=process.env.VERCEL_DEPLOYMENT_CREATED_AT?.trim();return i||a||o?{...r,build:{deployedAt:o||void 0,gitBranch:a||void 0,gitSha:i||void 0}}:r}function createRuntimeModelResolver(e){return t=>resolveRuntimeModelReference(t,e)}function createNodeHarnessTools(e){let t=new Map;for(let n of e.node.turnAgent.tools){let r=resolveHarnessToolDefinition({node:e.node,tool:n});r!==null&&t.set(n.name,r)}return t.has(`agent`)||t.set(`agent`,{description:BUILT_IN_AGENT_TOOL_DESCRIPTION,inputSchema:jsonSchema(SUBAGENT_TOOL_INPUT_SCHEMA),name:`agent`,runtimeAction:{kind:`subagent-call`,nodeId:e.node.nodeId,subagentName:`agent`}}),t}function resolveHarnessToolDefinition(e){if(e.tool.kind===`subagent`)return{description:e.tool.description??``,inputSchema:jsonSchema(e.tool.inputSchema??{}),name:e.tool.name,outputSchema:e.tool.outputSchema===void 0?void 0:jsonSchema(e.tool.outputSchema),runtimeAction:{kind:`subagent-call`,nodeId:e.tool.nodeId,subagentName:e.tool.name}};if(e.tool.kind===`remote`)return{description:e.tool.description??``,inputSchema:jsonSchema(e.tool.inputSchema??{}),name:e.tool.name,outputSchema:e.tool.outputSchema===void 0?void 0:jsonSchema(e.tool.outputSchema),runtimeAction:{kind:`remote-agent-call`,nodeId:e.tool.nodeId,remoteAgentName:e.tool.name,subagentName:e.tool.name}};let t=findRegisteredRuntimeTool(e.node.toolRegistry,e.tool.name);if(t===null)return log.warn(`declared tool is not registered — omitting from toolset`,{toolName:e.tool.name,nodeId:e.node.nodeId}),null;let r=t.definition,i=r.sourceId.startsWith(`eve:`),o=r.execute;return{approvalKey:r.approvalKey,description:r.description,execute:resolveAuthoredExecute({isFrameworkTool:i,rawExecute:o,scope:r.name}),inputSchema:r.inputStandardSchema??jsonSchema(r.inputSchema??{}),name:r.name,approval:r.approval,outputSchema:r.outputStandardSchema??maybeJsonSchema(r.outputSchema),toModelOutput:r.toModelOutput}}function resolveAuthoredExecute(e){let{isFrameworkTool:t,rawExecute:n,scope:r}=e;return n===void 0?void 0:t?n:createToolExecuteWithAuth({execute:n,scope:r})}function maybeJsonSchema(e){return e===void 0?void 0:jsonSchema(e)}export{createExecutionNodeStep,createNodeHarnessTools};

package/dist/src/execution/session.js

@@ -1,5 +1,5 @@
-function createCompactionConfig(e={}){let t=e.thresholdPercent??.9,n={recentWindowSize:10,threshold:e.contextWindowTokens===void 0?1e5:Math.max(1,Math.floor(e.contextWindowTokens*t))};return e.lastKnownInputTokens===void 0?n:{...n,lastKnownInputTokens:e.lastKnownInputTokens,lastKnownPromptMessageCount:e.lastKnownPromptMessageCount}}function createSession(e){let{turnAgent:t}=e,n=createSessionToolDefinitions(t),r={agent:{compactionModelReference:t.compactionModel,modelReference:t.model,system:t.instructions.join(`
+function createCompactionConfig(e={}){let t=e.thresholdPercent??.9,n={recentWindowSize:10,threshold:e.contextWindowTokens===void 0?1e5:Math.max(1,Math.floor(e.contextWindowTokens*t))};return e.lastKnownInputTokens===void 0?n:{...n,lastKnownInputTokens:e.lastKnownInputTokens,lastKnownPromptMessageCount:e.lastKnownPromptMessageCount}}function createSession(e){let{turnAgent:t}=e,n=createSessionToolDefinitions(t),r={agent:{compactionModelReference:t.compactionModel,modelReference:t.model,reasoning:t.reasoning,system:t.instructions.join(`
 
-`),tools:n},compaction:createCompactionConfig({contextWindowTokens:t.model.contextWindowTokens,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:e.continuationToken,history:[],sessionId:e.sessionId};return e.rootSessionId!==void 0&&(r.rootSessionId=e.rootSessionId),e.outputSchema!==void 0&&(r.outputSchema=e.outputSchema),r}function refreshSessionFromTurnAgent(e){return{...e.session,agent:{compactionModelReference:e.turnAgent.compactionModel,modelReference:e.turnAgent.model,system:e.turnAgent.instructions.join(`
+`),tools:n},compaction:createCompactionConfig({contextWindowTokens:t.model.contextWindowTokens,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:e.continuationToken,history:[],sessionId:e.sessionId};return e.rootSessionId!==void 0&&(r.rootSessionId=e.rootSessionId),e.outputSchema!==void 0&&(r.outputSchema=e.outputSchema),r}function refreshSessionFromTurnAgent(e){return{...e.session,agent:{compactionModelReference:e.turnAgent.compactionModel,modelReference:e.turnAgent.model,reasoning:e.turnAgent.reasoning,system:e.turnAgent.instructions.join(`
 
-`),tools:createSessionToolDefinitions(e.turnAgent)},compaction:createCompactionConfig({contextWindowTokens:e.turnAgent.model.contextWindowTokens,lastKnownInputTokens:e.session.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.session.compaction.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent})}}function mintSubagentContinuationToken(e){return`subagent:${e??crypto.randomUUID()}`}function projectToDurableSession(e){let t={agent:{system:e.agent.system},continuationToken:e.continuationToken,history:e.history,sessionId:e.sessionId};return(e.compaction.lastKnownInputTokens!==void 0||e.compaction.lastKnownPromptMessageCount!==void 0)&&(t.compaction={lastKnownInputTokens:e.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.compaction.lastKnownPromptMessageCount}),e.rootSessionId!==void 0&&(t.rootSessionId=e.rootSessionId),e.outputSchema!==void 0&&(t.outputSchema=e.outputSchema),e.sandboxState!==void 0&&(t.sandboxState=e.sandboxState),e.state!==void 0&&(t.state=e.state),t}function hydrateDurableSession(e){let{durable:t,turnAgent:n}=e,r=createSessionToolDefinitions(n),i={agent:{compactionModelReference:n.compactionModel,modelReference:n.model,system:t.agent.system,tools:r},compaction:createCompactionConfig({contextWindowTokens:n.model.contextWindowTokens,lastKnownInputTokens:t.compaction?.lastKnownInputTokens,lastKnownPromptMessageCount:t.compaction?.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:t.continuationToken,history:t.history,sessionId:t.sessionId};return t.rootSessionId!==void 0&&(i.rootSessionId=t.rootSessionId),t.outputSchema!==void 0&&(i.outputSchema=t.outputSchema),t.sandboxState!==void 0&&(i.sandboxState=t.sandboxState),t.state!==void 0&&(i.state=t.state),i}function createSessionToolDefinitions(e){return e.tools.map(e=>({description:e.description??``,inputSchema:e.inputSchema,name:e.name,outputSchema:e.outputSchema}))}export{createCompactionConfig,createSession,hydrateDurableSession,mintSubagentContinuationToken,projectToDurableSession,refreshSessionFromTurnAgent};
+`),tools:createSessionToolDefinitions(e.turnAgent)},compaction:createCompactionConfig({contextWindowTokens:e.turnAgent.model.contextWindowTokens,lastKnownInputTokens:e.session.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.session.compaction.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent})}}function mintSubagentContinuationToken(e){return`subagent:${e??crypto.randomUUID()}`}function projectToDurableSession(e){let t={agent:{system:e.agent.system},continuationToken:e.continuationToken,history:e.history,sessionId:e.sessionId};return(e.compaction.lastKnownInputTokens!==void 0||e.compaction.lastKnownPromptMessageCount!==void 0)&&(t.compaction={lastKnownInputTokens:e.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.compaction.lastKnownPromptMessageCount}),e.rootSessionId!==void 0&&(t.rootSessionId=e.rootSessionId),e.outputSchema!==void 0&&(t.outputSchema=e.outputSchema),e.sandboxState!==void 0&&(t.sandboxState=e.sandboxState),e.state!==void 0&&(t.state=e.state),t}function hydrateDurableSession(e){let{durable:t,turnAgent:n}=e,r=createSessionToolDefinitions(n),i={agent:{compactionModelReference:n.compactionModel,modelReference:n.model,reasoning:n.reasoning,system:t.agent.system,tools:r},compaction:createCompactionConfig({contextWindowTokens:n.model.contextWindowTokens,lastKnownInputTokens:t.compaction?.lastKnownInputTokens,lastKnownPromptMessageCount:t.compaction?.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:t.continuationToken,history:t.history,sessionId:t.sessionId};return t.rootSessionId!==void 0&&(i.rootSessionId=t.rootSessionId),t.outputSchema!==void 0&&(i.outputSchema=t.outputSchema),t.sandboxState!==void 0&&(i.sandboxState=t.sandboxState),t.state!==void 0&&(i.state=t.state),i}function createSessionToolDefinitions(e){return e.tools.map(e=>({description:e.description??``,inputSchema:e.inputSchema,name:e.name,outputSchema:e.outputSchema}))}export{createCompactionConfig,createSession,hydrateDurableSession,mintSubagentContinuationToken,projectToDurableSession,refreshSessionFromTurnAgent};

package/dist/src/execution/turn-workflow.js

@@ -1 +1 @@
-import{resumeHook}from"#internal/workflow/runtime.js";import{normalizeSerializableError}from"#execution/workflow-errors.js";import{migrateTurnWorkflowInput}from"#execution/durable-session-migrations/turn-workflow.js";import{turnStep}from"#execution/workflow-steps.js";async function turnWorkflow(e){"use workflow";let r=migrateTurnWorkflowInput(e),i=r.stepInput;try{for(;;){let e=await turnStep(i);if(e.action===`done`){await notifyDriverStep({completionToken:r.completionToken,payload:{action:{kind:`done`,output:e.output??``,isError:e.isError,serializedContext:e.serializedContext,sessionState:e.sessionState},kind:`turn-result`}});return}if(e.action===`dispatch-code-mode-runtime-actions`){await notifyDriverStep({completionToken:r.completionToken,payload:{action:{kind:`dispatch-code-mode-runtime-actions`,pendingActionKeys:e.pendingRuntimeActionKeys,serializedContext:e.serializedContext,sessionState:e.sessionState},kind:`turn-result`}});return}if(e.action===`park`){let t=e.pendingRuntimeActionKeys;if(!(t!==void 0||e.hasPendingAuthorization||e.hasPendingInputBatch&&r.capabilities?.requestInput===!0||r.mode===`conversation`))throw Error("Task mode cannot wait for follow-up input (`next: null`).");let n=t===void 0?{kind:`park`,serializedContext:e.serializedContext,sessionState:e.sessionState,authorizationNames:e.authorizationNames}:{kind:`dispatch-runtime-actions`,pendingActionKeys:t,serializedContext:e.serializedContext,sessionState:e.sessionState};await notifyDriverStep({completionToken:r.completionToken,payload:{action:n,kind:`turn-result`}});return}i={input:void 0,parentWritable:i.parentWritable,serializedContext:e.serializedContext,sessionState:e.sessionState}}}catch(e){throw await notifyDriverStep({completionToken:r.completionToken,payload:{error:normalizeSerializableError(e),kind:`turn-error`}}),e}}async function notifyDriverStep(t){"use step";await resumeHook(t.completionToken,t.payload)}export{notifyDriverStep,turnWorkflow};
+import{resumeHook}from"#internal/workflow/runtime.js";import{normalizeSerializableError}from"#execution/workflow-errors.js";import{migrateTurnWorkflowInput}from"#execution/durable-session-migrations/turn-workflow.js";import{turnStep}from"#execution/workflow-steps.js";async function turnWorkflow(e){"use workflow";let r=migrateTurnWorkflowInput(e),i=r.stepInput;try{for(;;){let e=await turnStep(i);if(e.action===`done`){await notifyDriverStep({completionToken:r.completionToken,payload:{action:{kind:`done`,output:e.output??``,isError:e.isError,serializedContext:e.serializedContext,sessionState:e.sessionState},kind:`turn-result`}});return}if(e.action===`dispatch-workflow-runtime-actions`){await notifyDriverStep({completionToken:r.completionToken,payload:{action:{kind:`dispatch-workflow-runtime-actions`,pendingActionKeys:e.pendingRuntimeActionKeys,serializedContext:e.serializedContext,sessionState:e.sessionState},kind:`turn-result`}});return}if(e.action===`park`){let t=e.pendingRuntimeActionKeys;if(!(t!==void 0||e.hasPendingAuthorization||e.hasPendingInputBatch&&r.capabilities?.requestInput===!0||r.mode===`conversation`))throw Error("Task mode cannot wait for follow-up input (`next: null`).");let n=t===void 0?{kind:`park`,serializedContext:e.serializedContext,sessionState:e.sessionState,authorizationNames:e.authorizationNames}:{kind:`dispatch-runtime-actions`,pendingActionKeys:t,serializedContext:e.serializedContext,sessionState:e.sessionState};await notifyDriverStep({completionToken:r.completionToken,payload:{action:n,kind:`turn-result`}});return}i={input:void 0,parentWritable:i.parentWritable,serializedContext:e.serializedContext,sessionState:e.sessionState}}}catch(e){throw await notifyDriverStep({completionToken:r.completionToken,payload:{error:normalizeSerializableError(e),kind:`turn-error`}}),e}}async function notifyDriverStep(t){"use step";await resumeHook(t.completionToken,t.payload)}export{notifyDriverStep,turnWorkflow};

package/dist/src/execution/workflow-entry.js

@@ -1 +1 @@
-import{accumulateRuntimeActionResults}from"#harness/runtime-actions.js";import{dispatchRuntimeActionsStep}from"#execution/dispatch-runtime-actions-step.js";import{resolveVercelProductionCallbackBaseUrl}from"#execution/workflow-callback-url.js";import{normalizeSerializableError,rebuildSerializableError}from"#execution/workflow-errors.js";import{dispatchTurnStep,emitTerminalSessionFailureStep,routeProxiedDeliverStep,runProxyInputRequestStep}from"#execution/workflow-steps.js";import{createHook,getWorkflowMetadata,getWritable}from"#compiled/@workflow/core/index.js";import{coalesceDeliveries}from"#harness/messages.js";import{readChannelRequestId,readRootSessionId}from"#execution/eve-workflow-attributes.js";import{notifyDelegatedParentStep}from"#execution/delegated-parent-notification.js";import{createDelegatedSubagentErrorResult,createDelegatedSubagentSuccessResult}from"#execution/delegated-parent-result.js";import{createSessionStep}from"#execution/create-session-step.js";import{dispatchCodeModeRuntimeActionsStep}from"#execution/dispatch-code-mode-runtime-actions-step.js";import{fireSessionCallbackStep}from"#execution/session-callback-step.js";import{claimHookOwnership,closeHookIterator,disposeHook}from"#execution/hook-ownership.js";async function workflowEntry(e){"use workflow";let{workflowRunId:t}=getWorkflowMetadata(),n=e.serializedContext[`eve.continuationToken`]||``,i=e.serializedContext[`eve.mode`],a=e.serializedContext[`eve.capabilities`],s=e.serializedContext[`eve.bundle`];e.serializedContext[`eve.sessionId`]=t;let c=getWritable();try{let r=readRootSessionId(e.serializedContext),{state:o}=await createSessionStep({compiledArtifactsSource:s.source,continuationToken:n,nodeId:s.nodeId,outputSchema:e.input.outputSchema,rootSessionId:r,sessionId:t});return await runDriverLoop({capabilities:a,driverWritable:c,initialInput:{kind:`deliver`,payloads:[{message:e.input.message,context:e.input.context,outputSchema:e.input.outputSchema}],requestId:readChannelRequestId(e.serializedContext)},mode:i,serializedContext:e.serializedContext,sessionState:o})}catch(t){throw await emitTerminalSessionFailureStep({error:normalizeSerializableError(t),parentWritable:c,serializedContext:e.serializedContext}),await fireSessionCallbackStep({error:normalizeSerializableError(t),serializedContext:e.serializedContext,status:`failed`}),await notifyDelegatedParentStep({result:createDelegatedSubagentErrorResult(e.serializedContext,t),serializedContext:e.serializedContext}),t}}async function runDriverLoop(e){let r=createHook({token:`${e.sessionState.sessionId}:auth`}),i=r[Symbol.asyncIterator](),a=0,nextTurnCompletionToken=()=>`${e.sessionState.sessionId}:turn-completion:${String(a++)}`,o=``,s,c,l=null,u=[],getNextPromise=()=>{if(c===void 0)throw Error(`Cannot wait for deliveries before a continuation token is available.`);return l??=c.next(),l},consumeNext=()=>{l=null},closeParkHook=async()=>{let e=c,t=s;if(s=void 0,c=void 0,l=null,e!==void 0)try{await closeHookIterator(e)}catch(e){if(t!==void 0)try{await disposeHook(t)}catch{}throw e}t!==void 0&&await disposeHook(t)},rekeyHook=async e=>{if(!e||s!==void 0&&e===o)return;let t=createHook({token:e});await claimHookOwnership(t);try{await closeParkHook()}catch(e){try{await disposeHook(t)}catch{}throw e}o=e,s=t,c=t[Symbol.asyncIterator](),l=null};try{e.sessionState.continuationToken&&await rekeyHook(e.sessionState.continuationToken);let r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:e.initialInput,mode:e.mode,parentWritable:e.driverWritable,serializedContext:e.serializedContext,sessionState:e.sessionState});if(r.kind===`done`)return await finalizeDone({action:r,driverWritable:e.driverWritable});if(!r.sessionState.continuationToken)throw Error("Cannot park: no continuation token available. The channel must post the first message during the initial turn (anchoring the session) or `send()` must be called with an explicit continuationToken.");for(await rekeyHook(r.sessionState.continuationToken);;)switch(r.kind){case`done`:return await finalizeDone({action:r,driverWritable:e.driverWritable});case`dispatch-code-mode-runtime-actions`:case`dispatch-runtime-actions`:{let i=await(r.kind===`dispatch-code-mode-runtime-actions`?dispatchCodeModeRuntimeActionsStep:dispatchRuntimeActionsStep)({callbackBaseUrl:resolveVercelProductionCallbackBaseUrl()??getWorkflowMetadata().url,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),a=await waitForPendingRuntimeActionResults({bufferedDeliveries:u,consumeNext,getNextPromise,initialResults:i.results,parentWritable:e.driverWritable,pendingActionKeys:r.pendingActionKeys,rekeyHook,serializedContext:r.serializedContext,sessionState:i.sessionState});if(a===null)return{output:``};r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{kind:`runtime-action-result`,results:a.results},mode:e.mode,parentWritable:e.driverWritable,serializedContext:a.serializedContext,sessionState:a.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}case`park`:{if(r.authorizationNames&&r.authorizationNames.length>0){let t=r.authorizationNames.length,n=[];for(;n.length<t;){let e=await i.next();if(e.done)break;e.value.kind===`deliver`&&n.push(...e.value.payloads)}r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{kind:`deliver`,payloads:n},mode:e.mode,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}let t=await waitForNextDeliver({bufferedDeliveries:u,consumeNext,getNextPromise});if(t===null)return{output:``};let n=await routeDeliverForChildren({auth:t.auth,parentWritable:e.driverWritable,payloads:t.payloads,sessionState:r.sessionState});if(n===void 0)continue;r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{auth:t.auth,kind:`deliver`,payloads:[n],requestId:t.requestId},mode:e.mode,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}}}finally{await closeParkHook(),await closeHookIterator(i),await disposeHook(r)}}async function finalizeDone(e){let{output:t,serializedContext:n}=e.action,r=e.action.isError===!0;return await fireSessionCallbackStep({error:r?t:void 0,output:r?void 0:t,serializedContext:n,status:r?`failed`:`completed`}),await notifyDelegatedParentStep({result:r?createDelegatedSubagentErrorResult(n,t):createDelegatedSubagentSuccessResult(n,t),serializedContext:n}),{output:t}}async function dispatchAndAwaitTurn(e){let t=createHook({token:e.completionToken}),n=t.token;try{await dispatchTurnStep({capabilities:e.capabilities,completionToken:n,delivery:e.delivery,mode:e.mode,parentWritable:e.parentWritable,serializedContext:e.serializedContext,sessionState:e.sessionState});let r=await awaitHookPayload(t);if(r.kind===`turn-error`)throw rebuildSerializableError(r.error);return r.action}finally{await disposeHook(t)}}async function awaitHookPayload(e){for await(let t of e)return t;throw Error(`Turn completion hook closed before delivering a result.`)}async function waitForPendingRuntimeActionResults(t){let n=t.sessionState,r=t.serializedContext,i=await accumulateRuntimeActionResults({bufferedDeliveries:t.bufferedDeliveries,async getNext(){for(;;){let e=await t.getNextPromise();if(t.consumeNext(),e.done)return null;let i=e.value;if(i.kind===`deliver`){let e=await routeDeliverForChildren({auth:i.auth,parentWritable:t.parentWritable,payloads:i.payloads,sessionState:n});if(e===void 0)continue;return{kind:`deliver`,value:{...i,payloads:[e]}}}if(i.kind===`runtime-action-result`)return{kind:`runtime-action-result`,results:i.results};let a=await runProxyInputRequestStep({hookPayload:i,parentWritable:t.parentWritable,serializedContext:r,sessionState:n});n=a.sessionState,r=a.serializedContext,await t.rekeyHook(n.continuationToken)}},initialResults:t.initialResults,pendingActionKeys:t.pendingActionKeys});return i===null?null:{results:i,serializedContext:r,sessionState:n}}async function routeDeliverForChildren(e){let t=coalescePayloads(e.payloads);return e.sessionState.hasProxyInputRequests?(await routeProxiedDeliverStep({auth:e.auth,parentWritable:e.parentWritable,payload:t,sessionState:e.sessionState})).remainder:t}async function waitForNextDeliver(e){if(e.bufferedDeliveries.length>0)return coalesceDeliveries(e.bufferedDeliveries.splice(0));for(;;){let t=await e.getNextPromise();if(e.consumeNext(),t.done)return null;if(t.value.kind!==`deliver`)continue;let n=t.value;for(;;){let t=await takeReadyPayload(e.getNextPromise());if(t===NO_READY_MESSAGE||(e.consumeNext(),t.done))break;t.value.kind===`deliver`&&(n=coalesceDeliveries([n,t.value]))}return n}}function coalescePayloads(e){if(e.length===0)return{};if(e.length===1)return e[0]??{};let t={},n=[];for(let r of e){for(let[e,n]of Object.entries(r))e!==`inputResponses`&&n!==void 0&&(t[e]=n);r.inputResponses!==void 0&&n.push(...r.inputResponses)}return n.length>0&&(t.inputResponses=n),t}const NO_READY_MESSAGE=Symbol(`no-ready-message`);async function takeReadyPayload(e){return await Promise.resolve(),await Promise.race([e,Promise.resolve(NO_READY_MESSAGE)])}export{workflowEntry};
+import{accumulateRuntimeActionResults}from"#harness/runtime-actions.js";import{dispatchRuntimeActionsStep}from"#execution/dispatch-runtime-actions-step.js";import{resolveVercelProductionCallbackBaseUrl}from"#execution/workflow-callback-url.js";import{normalizeSerializableError,rebuildSerializableError}from"#execution/workflow-errors.js";import{dispatchTurnStep,emitTerminalSessionFailureStep,routeProxiedDeliverStep,runProxyInputRequestStep}from"#execution/workflow-steps.js";import{createHook,getWorkflowMetadata,getWritable}from"#compiled/@workflow/core/index.js";import{coalesceDeliveries}from"#harness/messages.js";import{readChannelRequestId,readRootSessionId}from"#execution/eve-workflow-attributes.js";import{notifyDelegatedParentStep}from"#execution/delegated-parent-notification.js";import{createDelegatedSubagentErrorResult,createDelegatedSubagentSuccessResult}from"#execution/delegated-parent-result.js";import{createSessionStep}from"#execution/create-session-step.js";import{dispatchWorkflowRuntimeActionsStep}from"#execution/dispatch-workflow-runtime-actions-step.js";import{fireSessionCallbackStep}from"#execution/session-callback-step.js";import{claimHookOwnership,closeHookIterator,disposeHook}from"#execution/hook-ownership.js";async function workflowEntry(e){"use workflow";let{workflowRunId:t}=getWorkflowMetadata(),n=e.serializedContext[`eve.continuationToken`]||``,i=e.serializedContext[`eve.mode`],a=e.serializedContext[`eve.capabilities`],s=e.serializedContext[`eve.bundle`];e.serializedContext[`eve.sessionId`]=t;let c=getWritable();try{let r=readRootSessionId(e.serializedContext),{state:o}=await createSessionStep({compiledArtifactsSource:s.source,continuationToken:n,nodeId:s.nodeId,outputSchema:e.input.outputSchema,rootSessionId:r,sessionId:t});return await runDriverLoop({capabilities:a,driverWritable:c,initialInput:{kind:`deliver`,payloads:[{message:e.input.message,context:e.input.context,outputSchema:e.input.outputSchema}],requestId:readChannelRequestId(e.serializedContext)},mode:i,serializedContext:e.serializedContext,sessionState:o})}catch(t){throw await emitTerminalSessionFailureStep({error:normalizeSerializableError(t),parentWritable:c,serializedContext:e.serializedContext}),await fireSessionCallbackStep({error:normalizeSerializableError(t),serializedContext:e.serializedContext,status:`failed`}),await notifyDelegatedParentStep({result:createDelegatedSubagentErrorResult(e.serializedContext,t),serializedContext:e.serializedContext}),t}}async function runDriverLoop(e){let r=createHook({token:`${e.sessionState.sessionId}:auth`}),i=r[Symbol.asyncIterator](),a=0,nextTurnCompletionToken=()=>`${e.sessionState.sessionId}:turn-completion:${String(a++)}`,o=``,s,c,l=null,u=[],getNextPromise=()=>{if(c===void 0)throw Error(`Cannot wait for deliveries before a continuation token is available.`);return l??=c.next(),l},consumeNext=()=>{l=null},closeParkHook=async()=>{let e=c,t=s;if(s=void 0,c=void 0,l=null,e!==void 0)try{await closeHookIterator(e)}catch(e){if(t!==void 0)try{await disposeHook(t)}catch{}throw e}t!==void 0&&await disposeHook(t)},rekeyHook=async e=>{if(!e||s!==void 0&&e===o)return;let t=createHook({token:e});await claimHookOwnership(t);try{await closeParkHook()}catch(e){try{await disposeHook(t)}catch{}throw e}o=e,s=t,c=t[Symbol.asyncIterator](),l=null};try{e.sessionState.continuationToken&&await rekeyHook(e.sessionState.continuationToken);let r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:e.initialInput,mode:e.mode,parentWritable:e.driverWritable,serializedContext:e.serializedContext,sessionState:e.sessionState});if(r.kind===`done`)return await finalizeDone({action:r,driverWritable:e.driverWritable});if(!r.sessionState.continuationToken)throw Error("Cannot park: no continuation token available. The channel must post the first message during the initial turn (anchoring the session) or `send()` must be called with an explicit continuationToken.");for(await rekeyHook(r.sessionState.continuationToken);;)switch(r.kind){case`done`:return await finalizeDone({action:r,driverWritable:e.driverWritable});case`dispatch-workflow-runtime-actions`:case`dispatch-runtime-actions`:{let i=await(r.kind===`dispatch-workflow-runtime-actions`?dispatchWorkflowRuntimeActionsStep:dispatchRuntimeActionsStep)({callbackBaseUrl:resolveVercelProductionCallbackBaseUrl()??getWorkflowMetadata().url,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),a=await waitForPendingRuntimeActionResults({bufferedDeliveries:u,consumeNext,getNextPromise,initialResults:i.results,parentWritable:e.driverWritable,pendingActionKeys:r.pendingActionKeys,rekeyHook,serializedContext:r.serializedContext,sessionState:i.sessionState});if(a===null)return{output:``};r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{kind:`runtime-action-result`,results:a.results},mode:e.mode,parentWritable:e.driverWritable,serializedContext:a.serializedContext,sessionState:a.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}case`park`:{if(r.authorizationNames&&r.authorizationNames.length>0){let t=r.authorizationNames.length,n=[];for(;n.length<t;){let e=await i.next();if(e.done)break;e.value.kind===`deliver`&&n.push(...e.value.payloads)}r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{kind:`deliver`,payloads:n},mode:e.mode,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}let t=await waitForNextDeliver({bufferedDeliveries:u,consumeNext,getNextPromise});if(t===null)return{output:``};let n=await routeDeliverForChildren({auth:t.auth,parentWritable:e.driverWritable,payloads:t.payloads,sessionState:r.sessionState});if(n===void 0)continue;r=await dispatchAndAwaitTurn({capabilities:e.capabilities,completionToken:nextTurnCompletionToken(),delivery:{auth:t.auth,kind:`deliver`,payloads:[n],requestId:t.requestId},mode:e.mode,parentWritable:e.driverWritable,serializedContext:r.serializedContext,sessionState:r.sessionState}),await rekeyHook(r.sessionState.continuationToken);break}}}finally{await closeParkHook(),await closeHookIterator(i),await disposeHook(r)}}async function finalizeDone(e){let{output:t,serializedContext:n}=e.action,r=e.action.isError===!0;return await fireSessionCallbackStep({error:r?t:void 0,output:r?void 0:t,serializedContext:n,status:r?`failed`:`completed`}),await notifyDelegatedParentStep({result:r?createDelegatedSubagentErrorResult(n,t):createDelegatedSubagentSuccessResult(n,t),serializedContext:n}),{output:t}}async function dispatchAndAwaitTurn(e){let t=createHook({token:e.completionToken}),n=t.token;try{await dispatchTurnStep({capabilities:e.capabilities,completionToken:n,delivery:e.delivery,mode:e.mode,parentWritable:e.parentWritable,serializedContext:e.serializedContext,sessionState:e.sessionState});let r=await awaitHookPayload(t);if(r.kind===`turn-error`)throw rebuildSerializableError(r.error);return r.action}finally{await disposeHook(t)}}async function awaitHookPayload(e){for await(let t of e)return t;throw Error(`Turn completion hook closed before delivering a result.`)}async function waitForPendingRuntimeActionResults(t){let n=t.sessionState,r=t.serializedContext,i=await accumulateRuntimeActionResults({bufferedDeliveries:t.bufferedDeliveries,async getNext(){for(;;){let e=await t.getNextPromise();if(t.consumeNext(),e.done)return null;let i=e.value;if(i.kind===`deliver`){let e=await routeDeliverForChildren({auth:i.auth,parentWritable:t.parentWritable,payloads:i.payloads,sessionState:n});if(e===void 0)continue;return{kind:`deliver`,value:{...i,payloads:[e]}}}if(i.kind===`runtime-action-result`)return{kind:`runtime-action-result`,results:i.results};let a=await runProxyInputRequestStep({hookPayload:i,parentWritable:t.parentWritable,serializedContext:r,sessionState:n});n=a.sessionState,r=a.serializedContext,await t.rekeyHook(n.continuationToken)}},initialResults:t.initialResults,pendingActionKeys:t.pendingActionKeys});return i===null?null:{results:i,serializedContext:r,sessionState:n}}async function routeDeliverForChildren(e){let t=coalescePayloads(e.payloads);return e.sessionState.hasProxyInputRequests?(await routeProxiedDeliverStep({auth:e.auth,parentWritable:e.parentWritable,payload:t,sessionState:e.sessionState})).remainder:t}async function waitForNextDeliver(e){if(e.bufferedDeliveries.length>0)return coalesceDeliveries(e.bufferedDeliveries.splice(0));for(;;){let t=await e.getNextPromise();if(e.consumeNext(),t.done)return null;if(t.value.kind!==`deliver`)continue;let n=t.value;for(;;){let t=await takeReadyPayload(e.getNextPromise());if(t===NO_READY_MESSAGE||(e.consumeNext(),t.done))break;t.value.kind===`deliver`&&(n=coalesceDeliveries([n,t.value]))}return n}}function coalescePayloads(e){if(e.length===0)return{};if(e.length===1)return e[0]??{};let t={},n=[];for(let r of e){for(let[e,n]of Object.entries(r))e!==`inputResponses`&&n!==void 0&&(t[e]=n);r.inputResponses!==void 0&&n.push(...r.inputResponses)}return n.length>0&&(t.inputResponses=n),t}const NO_READY_MESSAGE=Symbol(`no-ready-message`);async function takeReadyPayload(e){return await Promise.resolve(),await Promise.race([e,Promise.resolve(NO_READY_MESSAGE)])}export{workflowEntry};

package/dist/src/execution/workflow-runtime.d.ts

@@ -1,14 +1,6 @@
 import type { Runtime } from "#channel/types.js";
-import { type Run } from "#internal/workflow/runtime.js";
+import { type Run, type StartOptionsWithoutDeploymentId, type WorkflowFunction, type WorkflowMetadata } from "#internal/workflow/runtime.js";
 import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";
-type WorkflowFunction<TArgs extends unknown[], TResult> = (...args: TArgs) => Promise<TResult>;
-interface WorkflowMetadata {
-    readonly workflowId: string;
-}
-interface WorkflowStartOptions {
-    readonly allowReservedAttributes?: boolean;
-    readonly attributes?: Record<string, string>;
-}
 export declare const LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE = "deploymentId 'latest' requires a World that implements resolveLatestDeploymentId()";
 /**
  * Workflow function names whose bundled id is stable across deployments
@@ -52,5 +44,4 @@ export declare function createWorkflowRuntime(config: {
  * Starts a workflow on the latest deployment when latest routing applies,
  * while preserving local/dev worlds that do not implement latest routing.
  */
-export declare function startWorkflowPreferLatest<TArgs extends unknown[], TResult>(workflow: WorkflowFunction<TArgs, TResult> | WorkflowMetadata, args: TArgs, options?: WorkflowStartOptions): Promise<Run<unknown> | Run<TResult>>;
-export {};
+export declare function startWorkflowPreferLatest<TArgs extends unknown[], TResult>(workflow: WorkflowFunction<TArgs, TResult> | WorkflowMetadata, args: TArgs, options?: StartOptionsWithoutDeploymentId): Promise<Run<unknown> | Run<TResult>>;

package/dist/src/execution/workflow-runtime.js

@@ -1 +1 @@
-import{createLogger,logError}from"#internal/logging.js";import{RuntimeNoActiveSessionError}from"#execution/runtime-errors.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{getCompiledRuntimeAgentBundle}from"#runtime/sessions/compiled-agent-cache.js";import{serializeContext}from"#context/serialize.js";import{getRun,resumeHook,start}from"#internal/workflow/runtime.js";import{ROOT_RUNTIME_AGENT_NODE_ID}from"#runtime/graph.js";import{buildSessionAttributes,buildSubagentRootAttributes,readParentLineage}from"#execution/eve-workflow-attributes.js";import{HookNotFoundError}from"#compiled/@workflow/errors/index.js";import{normalizeEveAttributes}from"#runtime/attributes/normalize.js";import{buildRunContext}from"#execution/runtime-context.js";import{parseNdjsonStream}from"#execution/ndjson-stream.js";const WORKFLOW_ENTRY_NAME=`workflowEntry`,TURN_WORKFLOW_NAME=`turnWorkflow`,EVE_PACKAGE_INFO=resolveInstalledPackageInfo(),LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE=`deploymentId 'latest' requires a World that implements resolveLatestDeploymentId()`,STABLE_WORKFLOW_NAMES=new Set([WORKFLOW_ENTRY_NAME,TURN_WORKFLOW_NAME]),STABLE_ID_BASE=EVE_PACKAGE_INFO.name,log=createLogger(`execution.workflow-runtime`),workflowEntryReference={workflowId:`workflow//${STABLE_ID_BASE}//${WORKFLOW_ENTRY_NAME}`},turnWorkflowReference={workflowId:`workflow//${STABLE_ID_BASE}//${TURN_WORKFLOW_NAME}`};function createWorkflowRuntime(e){return{async run(n){let r=await getCompiledRuntimeAgentBundle({compiledArtifactsSource:e.compiledArtifactsSource,nodeId:e.nodeId}),i=serializeContext(buildRunContext({bundle:r,run:n})),a=readParentLineage(i),o=a.sessionId===void 0?buildSessionAttributes({inputMessage:n.input.message,serializedContext:i}):buildSubagentRootAttributes({identity:{nodeId:r.nodeId??ROOT_RUNTIME_AGENT_NODE_ID},parentCallId:a.callId,parentSessionId:a.sessionId,parentTurnId:a.turnId,rootSessionId:a.rootSessionId??a.sessionId,serializedContext:i}),s;try{s=await startWorkflowPreferLatest(workflowEntryReference,[{input:n.input,serializedContext:i}],{allowReservedAttributes:!0,attributes:normalizeEveAttributes(o)})}catch(e){throw logError(log,`failed to start workflow run`,e,{continuationToken:n.continuationToken}),e}let c,getEvents=()=>(c??=parseNdjsonStream(()=>getRun(s.runId).getReadable()),c);return{continuationToken:n.continuationToken??s.runId,get events(){return getEvents()},sessionId:s.runId}},async deliver(e){let r={auth:e.auth,kind:`deliver`,payloads:[e.payload],requestId:e.requestId};try{return{sessionId:normalizeWorkflowHook(await resumeHook(e.continuationToken,r)).runId}}catch(r){throw HookNotFoundError.is(r)?new RuntimeNoActiveSessionError(e.continuationToken):(logError(log,`failed to deliver to active session`,r,{continuationToken:e.continuationToken}),r)}},async getEventStream(e,t){return parseNdjsonStream(()=>getRun(e).getReadable({startIndex:t?.startIndex}))}}}async function startWorkflowPreferLatest(e,t,n){if(!shouldRouteToLatestDeployment())return n===void 0?await start(e,t):await start(e,t,n);try{return await start(e,t,{...n,deploymentId:`latest`})}catch(r){if(!isLatestDeploymentUnsupportedError(r))throw r;return n===void 0?await start(e,t):await start(e,t,n)}}function shouldRouteToLatestDeployment(){return process.env.VERCEL_ENV===`production`}function isLatestDeploymentUnsupportedError(e){return e instanceof Error&&e.message.includes(`deploymentId 'latest' requires a World that implements resolveLatestDeploymentId()`)}function normalizeWorkflowHook(e){if(typeof e!=`object`||!e||!(`runId`in e))throw Error(`Workflow hook did not include a run id.`);let t=e.runId;if(typeof t!=`string`||t.length===0)throw Error(`Workflow hook did not include a run id.`);return{runId:t}}export{LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE,STABLE_WORKFLOW_NAMES,createWorkflowRuntime,startWorkflowPreferLatest,turnWorkflowReference,workflowEntryReference};
+import{createLogger,logError}from"#internal/logging.js";import{RuntimeNoActiveSessionError}from"#execution/runtime-errors.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{getCompiledRuntimeAgentBundle}from"#runtime/sessions/compiled-agent-cache.js";import{serializeContext}from"#context/serialize.js";import{getRun,resumeHook,start}from"#internal/workflow/runtime.js";import{ROOT_RUNTIME_AGENT_NODE_ID}from"#runtime/graph.js";import{buildSessionAttributes,buildSubagentRootAttributes,readParentLineage}from"#execution/eve-workflow-attributes.js";import{HookNotFoundError}from"#compiled/@workflow/errors/index.js";import{normalizeEveAttributes}from"#runtime/attributes/normalize.js";import{buildRunContext}from"#execution/runtime-context.js";import{parseNdjsonStream}from"#execution/ndjson-stream.js";const WORKFLOW_ENTRY_NAME=`workflowEntry`,TURN_WORKFLOW_NAME=`turnWorkflow`,EVE_PACKAGE_INFO=resolveInstalledPackageInfo(),LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE=`deploymentId 'latest' requires a World that implements resolveLatestDeploymentId()`,STABLE_WORKFLOW_NAMES=new Set([WORKFLOW_ENTRY_NAME,TURN_WORKFLOW_NAME]),STABLE_ID_BASE=EVE_PACKAGE_INFO.name,log=createLogger(`execution.workflow-runtime`),workflowEntryReference={workflowId:`workflow//${STABLE_ID_BASE}//${WORKFLOW_ENTRY_NAME}`},turnWorkflowReference={workflowId:`workflow//${STABLE_ID_BASE}//${TURN_WORKFLOW_NAME}`};function createWorkflowRuntime(e){return{async run(n){let r=await getCompiledRuntimeAgentBundle({compiledArtifactsSource:e.compiledArtifactsSource,nodeId:e.nodeId}),i=serializeContext(buildRunContext({bundle:r,run:n})),a=readParentLineage(i),o=a.sessionId===void 0?buildSessionAttributes({inputMessage:n.title??n.input.message,serializedContext:i}):buildSubagentRootAttributes({identity:{nodeId:r.nodeId??ROOT_RUNTIME_AGENT_NODE_ID},parentCallId:a.callId,parentSessionId:a.sessionId,parentTurnId:a.turnId,rootSessionId:a.rootSessionId??a.sessionId,serializedContext:i}),s;try{s=await startWorkflowPreferLatest(workflowEntryReference,[{input:n.input,serializedContext:i}],{allowReservedAttributes:!0,attributes:normalizeEveAttributes(o)})}catch(e){throw logError(log,`failed to start workflow run`,e,{continuationToken:n.continuationToken}),e}let c,getEvents=()=>(c??=parseNdjsonStream(()=>getRun(s.runId).getReadable()),c);return{continuationToken:n.continuationToken??s.runId,get events(){return getEvents()},sessionId:s.runId}},async deliver(e){let r={auth:e.auth,kind:`deliver`,payloads:[e.payload],requestId:e.requestId};try{return{sessionId:normalizeWorkflowHook(await resumeHook(e.continuationToken,r)).runId}}catch(r){throw HookNotFoundError.is(r)?new RuntimeNoActiveSessionError(e.continuationToken):(logError(log,`failed to deliver to active session`,r,{continuationToken:e.continuationToken}),r)}},async getEventStream(e,t){return parseNdjsonStream(()=>getRun(e).getReadable({startIndex:t?.startIndex}))}}}async function startWorkflowPreferLatest(e,t,n){if(!shouldRouteToLatestDeployment())return n===void 0?await start(e,t):await start(e,t,n);try{return await start(e,t,{...n,deploymentId:`latest`})}catch(r){if(!isLatestDeploymentUnsupportedError(r))throw r;return n===void 0?await start(e,t):await start(e,t,n)}}function shouldRouteToLatestDeployment(){return process.env.VERCEL_ENV===`production`}function isLatestDeploymentUnsupportedError(e){return e instanceof Error&&e.message.includes(`deploymentId 'latest' requires a World that implements resolveLatestDeploymentId()`)}function normalizeWorkflowHook(e){if(typeof e!=`object`||!e||!(`runId`in e))throw Error(`Workflow hook did not include a run id.`);let t=e.runId;if(typeof t!=`string`||t.length===0)throw Error(`Workflow hook did not include a run id.`);return{runId:t}}export{LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE,STABLE_WORKFLOW_NAMES,createWorkflowRuntime,startWorkflowPreferLatest,turnWorkflowReference,workflowEntryReference};