eve 0.83.2 → 0.84.0

package/dist/src/evals/session.js

@@ -1,2 +1,2 @@
-import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{getMessageResponseReceipt}from"#client/message-response.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts,openInputRequests}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o=[];#s=[];#c;#l;#u=[];#d=new Set;#f;#p=!1;#m=[];constructor(e){this.#i=e.collector,this.#l=e.onSessionStart,this.#c=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#y()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#y()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#f}get pendingInputRequests(){return this.#m}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#m.length===0&&this.#b(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#m.filter(t=>inputRequestMatches(t,e));return(this.#m.length!==1||t.length!==1)&&this.#b(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#m.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#h({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#h({...t,inputResponses:e})}async respondAll(e){let t=this.#m;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#h({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#h({turnPolicy:`queue`,...t,message:e})}async#h(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.recordSend(t===void 0?`message`:`respond`,getMessageResponseReceipt(i)),this.consume(i,n)}recordSend(e,t){t!==void 0&&this.#s.push({kind:e,...t})}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#g(n,e),record:(e,r)=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
-`)),this.#v(n,e,r)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#c,sessionId:e,state:this.#e.state,traceContexts:[...this.#u],clock:{receivedAt:[...this.#o],sends:[...this.#s]}}}#g(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#d.has(r)||(this.#d.add(r),this.#u.push(n),!this.#p&&(this.#p=!0,this.#l?.({primary:this.#c,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#_(e){this.#a.push(...e.events),this.#o.push(...e.receivedAt),this.#m=e.status===`waiting`?openInputRequests(this.#a):[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n={...t,parked:t.parked||this.#m.length>0},r=new EvalTurn({collector:this.#i,data:e.data,derived:n,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:n.toolCalls});return this.#f=r,this.#t(this),r}#v(e,t,n){let r=summarizeTurnEvents(t);return this.#_({data:extractCompletedResult(t),events:t,inputRequests:r.inputRequests,message:r.message,receivedAt:n,sessionId:e,status:r.status})}#y(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#f),status:this.#f?.status??`completed`}}#b(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=[];#r=new Set;#i;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#a(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#i!==void 0)throw this.#i;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#r.add(n)})}async#a(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n),this.#n.push(Date.now());let e=i.observe(n);if(t(n),this.#o(n),isTurnFailureEvent(n)&&this.#s(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#s(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t,this.#n)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#s(t),e}}#o(e){for(let t of this.#r)t.matches(e)&&(this.#r.delete(t),t.resolve(e))}#s(e){if(this.#i===void 0){this.#i=e;for(let t of this.#r)t.reject(e);this.#r.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{ResponseSegment,failureOf,summarizeTurnEvents}from"#client/session-utils.js";import{getDeliveryReceipt}from"#client/delivery.js";import{deliveryEvents}from"#client/delivery-events.js";import{basename}from"node:path";import{emptySessionView,foldEvents,openInteractions}from"#protocol/session-projection/public-view.js";import{followClientSession}from"#client/session.js";import{createTextWithFileContent}from"#client/file-parts.js";import{outputOf}from"#evals/output.js";import{interactionMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(e=>failureOf(e)!==void 0),n=e.error,r=t===void 0||n===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${n.code} ${n.message}`.trim();super(`Eval turn failed: ${r}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s=[];#c=[];#l;#u;#d=[];#f=new Set;#p;#m=!1;#h=[];constructor(e){this.#o=e.history??[],this.#i=e.collector,this.#u=e.onSessionStart,this.#l=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#b()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#b()}))}get events(){return this.#a}get view(){return foldEvents(emptySessionView(),[...this.#o,...this.#a])}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#p}get pendingInteractions(){return this.#h}get id(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){let e=this.#e.state.streamIndex,t=await this.#e.compact({signal:this.#r});return this.consume(this.#e.stream({startIndex:e,signal:this.#r}),void 0,{deliveryId:t.deliveryId}).result()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,id:t.id,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInteraction(e={}){this.#h.length===0&&this.#x(`requireInteraction`,`expected one pending input request, but the last turn did not park`);let t=this.#h.filter(t=>interactionMatches(t,e));return(this.#h.length!==1||t.length!==1)&&this.#x(`requireInteraction`,`expected exactly one pending input request matching ${formatInteractionFilter(e)}, found ${t.length} match(es) across ${this.#h.length} pending request(s)`),this.#i.recordOutcome({name:`requireInteraction`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one answer.`);return await(await this.#g({...t,answers:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#g({...t,answers:e})}async respondAll(e){let t=this.#h;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({interactionId:t.interactionId,optionId:e})))}async send(e,t={}){return await(await this.#g({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#g({turnPolicy:`queue`,...t,message:e})}async#g(e){let{answers:t,message:n,...r}=attachSignal(e,this.#r),i=this.#e.state.streamIndex,a=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);this.recordSend(t===void 0?`message`:`respond`,getDeliveryReceipt(a));let o=a.deliveryId;return this.consume(followClientSession(this.#e,{startIndex:i,signal:r.signal,headers:r.headers,streamReconnectPolicy:r.streamReconnectPolicy}),n,{deliveryId:o})}recordSend(e,t){t!==void 0&&this.#c.push({kind:e,...t})}consume(e,t,n={}){let r=this.id;return new EvalLiveTurn({...n,events:e,observe:e=>this.#_(r,e),initialView:this.view,record:(e,i,a)=>(this.#a.push(...i),this.#s.push(...a.earlier),t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
+`)),this.#y(r,e,a.events,n.deliveryId)),session:this,sessionId:r})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async nextTurn(e){return await this.watch(e).result()}watch(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}),void 0,{endsWith:e?.until})}snapshot(){let e=this.id;return{view:this.view,derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#l,sessionId:e,state:this.#e.state,traceContexts:[...this.#d],clock:{receivedAt:[...this.#s],sends:[...this.#c]}}}#_(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#f.has(r)||(this.#f.add(r),this.#d.push(n),!this.#m&&(this.#m=!0,this.#u?.({primary:this.#l,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#v(e){this.#a.push(...e.events),this.#s.push(...e.receivedAt);let t=this.view,n=openInteractions(t);this.#h=e.status===`awaiting-input`?n:[];let r=deriveRunFacts(e.events,{sessionId:e.sessionId,usageEvents:this.#o.length===0?this.#a:[...this.#o,...this.#a]}),i=new EvalTurn({collector:this.#i,data:e.data,outcome:e.outcome,error:e.error,view:t,sessionEnded:e.sessionEnded,derived:r,events:e.events,interactions:n,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:r.toolCalls});return this.#p=i,this.#t(this),i}#y(e,t,n,r){let i=summarizeTurnEvents(t,r);return this.#v({data:i.result,outcome:i.outcome,error:i.failure,events:t,message:i.message,receivedAt:n,sessionEnded:i.sessionEnded,sessionId:e,status:i.status})}#b(){let e=this.id;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#p),status:this.#p?.status??`completed`}}#x(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t;#n=[];#r=[];#i=[];#a=[];#o=new Set;#s;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#t=e.initialView,this.#e=this.#l(e.events,e.observe,e.record,e),this.#e.catch(()=>{})}get events(){return this.#r}get view(){return foldEvents(this.#t,[...this.#n,...this.#r])}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0))&&(t?.scope===void 0||matchesValue(t.scope,n.scope??{}));return await this.#c(e=>matches(e)?e:void 0,this.#r.find(matches))}async waitForToolCall(e,t={}){let find=()=>deriveRunFacts(this.#r,{sessionId:this.sessionId}).toolCalls.find(n=>n.name===e&&n.status!==`pending`&&toolCallMatches(n,{...t,status:t.status??n.status}));return await this.#c(e=>e.type===`call.settled`?find():void 0,find())}async#c(e,t){if(t!==void 0)return t;if(this.#s!==void 0)throw this.#s;return await new Promise((t,n)=>{let r={reject:n,resolve:e=>t(e),take:e};this.#o.add(r)})}async#l(e,t,n,r){let{deliveryId:i,endsWith:a}=r,o=!1;try{let r=!1,s=new ResponseSegment,c=i===void 0?void 0:new ResponseSegment({deliveryId:i}),l=this.#n,u=i===void 0?e:deliveryEvents(e,i,void 0,e=>{l.push(e),this.#i.push(Date.now()),t(e)});for await(let e of u){this.#r.push(e),this.#a.push(Date.now());let n=s.observe(e);a?.(e)===!0&&(o=!0);let i=a===void 0?c===void 0?n:c.observe(e):o&&e.meta.endOfLine!==!1;if(t(e),this.#u(e),failureOf(e)!==void 0&&this.#d(Error(`Session ${this.sessionId} failed before the expected event (${e.type}).`)),i){r=!0,this.#d(Error(`Session ${this.sessionId} reached ${e.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#r,l,{earlier:this.#i,events:this.#a})}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#d(t),e}}#u(e){for(let t of this.#o){let n=t.take(e);n!==void 0&&(this.#o.delete(t),t.resolve(n))}}#d(e){if(this.#s===void 0){this.#s=e;for(let t of this.#o)t.reject(e);this.#o.clear()}}},EvalTurn=class{data;outcome;error;view;sessionEnded;events;interactions;reply;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.outcome=e.outcome,this.error=e.error,this.view=e.view,this.sessionEnded=e.sessionEnded,this.events=e.events,this.interactions=e.interactions,this.reply=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInteractionFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function assertRequestHasOption(e,t){if(e.request.options===void 0||e.request.options.length===0)throw Error(`Input request "${e.interactionId}" has no selectable options.`);if(!e.request.options.some(e=>e.id===t))throw Error(`Input request "${e.interactionId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};

package/dist/src/evals/target.js

@@ -1 +1 @@
-import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);let n=createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url});return toolsByTarget.set(n,{dynamic:t.tools.dynamic.length>0,static:t.tools.static.map(e=>e.name)}),n}const toolsByTarget=new WeakMap;function targetTools(e){return toolsByTarget.get(e)}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,async attachSession(r,i){if(e.sessions!==void 0)return await e.sessions.attachSession(r,i);if(t!==void 0)return await t.attachSession(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:n}).attachSession(r,i)},watchTurn(r,i){if(e.sessions!==void 0)return e.sessions.watchTurn(r,i);if(t!==void 0)return t.watchTurn(r,i);if(n===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:n}).watchTurn(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle,targetTools};
+import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);let n=createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url});return toolsByTarget.set(n,{dynamic:t.tools.dynamic.length>0,static:t.tools.static.map(e=>e.name)}),n}const toolsByTarget=new WeakMap;function targetTools(e){return toolsByTarget.get(e)}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,attach(r,i){if(e.sessions!==void 0)return e.sessions.attach(r,i);if(t!==void 0)return t.attach(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return new EvalSessionManager({client:n}).attach(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle,targetTools};

package/dist/src/evals/types.d.ts

@@ -1,28 +1,37 @@
+import type { Delivery } from "#shared/delivery.js";
+import type { DeliveryOutcome } from "#protocol/session-events/families/delivery.js";
+import type { ErrorInfo } from "#protocol/session-events/envelope.js";
+import type { SessionView } from "#protocol/session-projection/session-view.js";
+import type { RequestStatus } from "#protocol/session-projection/request-status.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
 import type { TokenUsage } from "#shared/token-usage.js";
-import type { Experimental_DecisionModel as DecisionModel } from "ai";
+import type { DecisionModel } from "#ai/decision-types.js";
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
 import type { ClientAgentSession } from "#client/agent-session.js";
-import type { RuntimeIdentity, RuntimeTraceContext, MessageStreamEvent, AgentStartedStreamEvent } from "#protocol/message.js";
-import type { CancelSessionResult, ClientSessionState, CreateSessionOptions, SendTurnInput, SendTurnOptions } from "#client/types.js";
-import type { InputRequest, InputResponse } from "#shared/input.js";
+import type { RuntimeIdentity, TraceContext } from "#protocol/session-events/families/session.js";
+import type { ClientSessionState, CreateSessionOptions, SendTurnInput, SendTurnOptions, RespondTurnOptions } from "#client/types.js";
+import type { Answer } from "#shared/input.js";
+import type { Interaction } from "#protocol/session-projection/reader.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
-import type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
+import type { EveEvalEventMatch, EveEvalInteractionMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
 /** Lifecycle outcome of an eval-observed tool action. */
-export type EveEvalActionStatus = "pending" | "completed" | "failed" | "rejected";
+export type EveEvalActionStatus = "pending" | "completed" | "failed" | "rejected" | "interrupted" | "abandoned";
 /**
  * One tool call extracted from the captured stream, pairing the
- * `actions.requested` request with its matching `action.result`.
+ * `call.requested` request with its matching `call.settled`.
  */
 export interface EveEvalToolCall {
+    /** The call's id, as its `call.*` facts name it. */
+    readonly callId: string;
     /** Authored tool name (e.g. `"get_weather"`). */
     readonly name: string;
     /** Tool input as requested by the model. */
     readonly input: JsonObject;
-    /** Tool output from the matching `action.result`; `undefined` when the call never resolved. */
+    /** Inline output from `call.settled`; absent when unresolved or only a reference was recorded. */
     readonly output: JsonValue | undefined;
-    /** Whether the request is unresolved, completed, failed, or user-rejected. */
+    /** Pending until settled, then the call's explicit outcome. */
     readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the call happened in. */
     readonly turnIndex: number;
@@ -30,8 +39,8 @@ export interface EveEvalToolCall {
     readonly sessionId?: string;
 }
 /**
- * One skill load extracted from the captured stream, pairing the `load-skill`
- * request with its matching `load-skill-result`.
+ * One skill call extracted from the captured stream, pairing its `call.requested`
+ * capability with its matching `call.settled`.
  */
 export interface EveEvalSkillLoad {
     /** The loaded skill's name. */
@@ -45,23 +54,20 @@ export interface EveEvalSkillLoad {
     readonly sessionId?: string;
 }
 /**
- * One call to an agent task extracted from the captured stream: its
- * `task.started`, joined with its `task.settled` and the task's
- * `agent.started`.
+ * One call served by an agent task: its `call.started {taskId}`, joined with
+ * `call.settled`, the task's single `task.started`, and its child-session link.
  */
 export interface EveEvalSubagentCall {
-    /** The agent tool call's id, as on its task events. */
+    /** The agent tool call's id. */
     readonly callId?: string;
     /** The agent's session id, shared by every call to one task; absent if it never opened. */
     readonly childSessionId?: string;
     /** Subagent name. */
     readonly name: string;
-    /** Remote agent URL for remote delegations (`agent.started` remote metadata). */
-    readonly remoteUrl?: string;
-    /** Output from the call's `task.settled`; `undefined` until the call completes. */
+    /** Output from `call.settled`, resolving shared replies; absent until it completes. */
     readonly output?: JsonValue;
-    /** The call's task status: `working` until its `task.settled` arrives. */
-    readonly status: "working" | "completed" | "failed" | "cancelled";
+    /** Pending until this call settles, then the call's explicit outcome. */
+    readonly status: EveEvalActionStatus;
     /** Zero-based index of the turn the delegation happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -76,26 +82,26 @@ export interface EveEvalDerivedFacts {
     readonly skillLoads: readonly EveEvalSkillLoad[];
     readonly subagentCalls: readonly EveEvalSubagentCall[];
     readonly subagentCallCount: number;
-    /** Every HITL input request raised during the run (`input.requested`). */
-    readonly inputRequests: readonly InputRequest[];
+    /** Every HITL input request raised during the run (`interaction.opened`). */
+    readonly interactions: readonly Interaction[];
     /** True when the run ended parked on unanswered HITL input requests. */
     readonly parked: boolean;
     readonly messageCount: number;
     readonly reasoningBlockCount: number;
     /**
-     * Distinct ids of the models the steps started with (`step.started`): in first-use order for one
-     * session, and in session order for an eval. Covers only the sessions the eval created or
-     * attached: a delegated subagent runs in its own session, so its models appear only when the eval
-     * attaches that session. Compaction and `auto` routing calls are not included.
+     * Distinct model ids from `model.started`: in first-use order for one session, and in session
+     * order for an eval. Includes compaction summaries. Covers only captured sessions; a delegated
+     * agent's models appear only when the eval captures its session. Provider-internal routing
+     * calls without an eve model lifecycle are not included.
      */
     readonly models: readonly string[];
     /**
-     * Token usage from the latest `session.waiting`, `turn.waiting`, `session.failed`, or
-     * `session.completed`: the
-     * session's own model calls plus what the agents it delegated to spent, so on a turn it is the
-     * session's total so far. For an eval, each captured session counts once, by its latest usage in
-     * time across captures, except sessions another captured session opened. Absent when a counted
-     * session reported none; `costUsd` is absent when a counted session reported no cost.
+     * Sum of the captured session's `usage.recorded` facts through this turn: its own model calls,
+     * compaction summaries, and delegated spend recorded on calls. A turn reports the session's
+     * total so far, not only that turn's spend. For an eval, each captured session counts once, by
+     * the capture whose latest `usage.recorded` is latest in time, except sessions another captured
+     * session opened, whose spend is attributed to that parent's calls. Absent when a counted session
+     * reported no usage; `costUsd` is absent when a counted session reported no cost.
      */
     readonly usage?: TokenUsage;
     readonly failureCode?: string;
@@ -104,13 +110,14 @@ export interface EveEvalDerivedFacts {
  * Captured event stream and facts for one session involved in an eval.
  */
 export interface EveEvalSessionResult {
+    readonly view: SessionView;
     readonly derived: EveEvalDerivedFacts;
-    readonly events: readonly MessageStreamEvent[];
+    readonly events: readonly SessionStreamEvent[];
     readonly primary: boolean;
     readonly sessionId?: string;
     readonly state: ClientSessionState | undefined;
     /** Distinct trace contexts observed for this session, in stream order. */
-    readonly traceContexts: readonly RuntimeTraceContext[];
+    readonly traceContexts: readonly TraceContext[];
     /**
      * When the eval client sent to this session and received its events, for
      * measuring eve's own overhead against the server's `meta.at` times.
@@ -141,7 +148,7 @@ export interface EveEvalSendTiming {
     readonly acceptedAt: number;
 }
 /** Trace context attributed to one session involved in an eval. */
-export interface EveEvalTraceContext extends RuntimeTraceContext {
+export interface EveEvalTraceContext extends TraceContext {
     readonly primary: boolean;
     readonly sessionId: string;
 }
@@ -149,22 +156,25 @@ export interface EveEvalTraceContext extends RuntimeTraceContext {
  * Full result of executing one eval against an eve agent.
  */
 export interface EveEvalTaskResult {
-    /**
-     * The final turn's structured data when present, otherwise its last assistant
-     * message. Retained for reporters and artifacts that log one output value.
-     */
-    output: unknown;
-    /** The agent's last assistant message, or null when none was produced. */
-    readonly finalMessage: string | null;
+    /** The last observed turn's session view; absent when no turn was read. */
+    readonly view?: SessionView;
+    /** Whether the session owning the last observed turn ended. */
+    readonly sessionEnded: boolean;
+    /** The last observed turn's structured output, when present. */
+    readonly data: unknown;
+    /** The last observed turn's reply text, when present. */
+    readonly reply: string | undefined;
+    /** Present when the read observed its accepted delivery's settlement. */
+    readonly outcome?: DeliveryOutcome;
+    readonly error?: ErrorInfo;
+    /** The primary captured session ID; other sessions are listed in sessions. */
     readonly sessionId?: string;
     /**
-     * How the run's final turn ended: `"completed"` (session finished),
-     * `"failed"` (terminal failure), or `"waiting"` (parked for the next
-     * user message).
+     * The last observed request's status, independent of whether its session ended.
      */
-    readonly status: "completed" | "failed" | "waiting";
+    readonly status: RequestStatus | "working";
     /** The captured stream events from the run. */
-    readonly events: readonly MessageStreamEvent[];
+    readonly events: readonly SessionStreamEvent[];
     /** Lines written through `t.log` while the eval ran. */
     readonly logs?: readonly string[];
     /** Facts extracted from the stream (tool calls, message counts, etc.). */
@@ -261,26 +271,26 @@ export interface AssertionResult {
 /** Recorded assertions shared by aggregate, session, and turn scopes. */
 export interface EveEvalAssertions {
     succeeded(): AssertionHandle;
-    parked(): AssertionHandle;
+    awaitingInput(): AssertionHandle;
     messageIncludes(token: string | RegExp): AssertionHandle;
     calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
     /** Asserts a completed load of `skill`, constrained like `calledTool`. */
-    loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
+    calledSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
     notCalledTool(name: string): AssertionHandle;
     /** Asserts that tool requests appeared in order, allowing unrelated requests between them. */
     toolOrder(names: readonly string[]): AssertionHandle;
     usedNoTools(): AssertionHandle;
     maxToolCalls(max: number): AssertionHandle;
-    calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
-    noFailedActions(): AssertionHandle;
-    event<TType extends MessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+    calledAgent(name: string, options?: EveEvalSubagentCallMatchOptions): AssertionHandle;
+    noFailedCalls(): AssertionHandle;
+    event<TType extends SessionStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
         type: TType;
     }>, "type">): AssertionHandle;
-    notEvent<TType extends MessageStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
+    notEvent<TType extends SessionStreamEvent["type"]>(type: TType, options?: Omit<Extract<EveEvalEventMatch, {
         type: TType;
     }>, "type" | "count">): AssertionHandle;
     eventOrder(matchers: readonly EveEvalEventMatch[]): AssertionHandle;
-    eventsSatisfy(label: string, predicate: (events: readonly MessageStreamEvent[]) => boolean): AssertionHandle;
+    eventsSatisfy(label: string, predicate: (events: readonly SessionStreamEvent[]) => boolean): AssertionHandle;
 }
 /** Assertions over the final output captured by one session or immutable turn. */
 export interface EveEvalOutputAssertions {
@@ -288,11 +298,11 @@ export interface EveEvalOutputAssertions {
     outputMatches(schema: StandardSchemaV1): AssertionHandle;
 }
 /** Typed stream event returned by {@link EveEvalLiveTurn.waitForEvent}. */
-export type EveEvalStreamEvent<TType extends MessageStreamEvent["type"] = MessageStreamEvent["type"]> = Extract<MessageStreamEvent, {
+export type EveEvalStreamEvent<TType extends SessionStreamEvent["type"] = SessionStreamEvent["type"]> = Extract<SessionStreamEvent, {
     type: TType;
 }>;
 /** Matcher options for waiting until one live turn emits a specific event. */
-export type EveEvalWaitForEventOptions<TType extends MessageStreamEvent["type"]> = Omit<Extract<EveEvalEventMatch, {
+export type EveEvalWaitForEventOptions<TType extends SessionStreamEvent["type"]> = Omit<Extract<EveEvalEventMatch, {
     type: TType;
 }>, "count" | "type">;
 /**
@@ -302,48 +312,58 @@ export type EveEvalWaitForEventOptions<TType extends MessageStreamEvent["type"]>
  * {@link result} settles and records that same stream exactly once.
  */
 export interface EveEvalLiveTurn {
+    readonly view: SessionView;
     /** Events observed on this turn so far. */
-    readonly events: readonly MessageStreamEvent[];
+    readonly events: readonly SessionStreamEvent[];
     /** Session driver that started or owns this turn. */
     readonly session: EveEvalSession;
     /** Durable session id available as soon as the turn is accepted or attached. */
     readonly sessionId: string;
     /** Request cooperative cancellation of this turn's session. */
-    cancel(): Promise<CancelSessionResult>;
+    cancel(): Promise<Delivery>;
     /** Wait for the turn boundary and return the recorded immutable result. */
     result(): Promise<EveEvalTurn>;
     /** Wait until the live stream emits one typed event matching `options`. */
-    waitForEvent<TType extends MessageStreamEvent["type"]>(type: TType, options?: EveEvalWaitForEventOptions<TType>): Promise<EveEvalStreamEvent<TType>>;
+    waitForEvent<TType extends SessionStreamEvent["type"]>(type: TType, options?: EveEvalWaitForEventOptions<TType>): Promise<EveEvalStreamEvent<TType>>;
+    /**
+     * Wait until a call to the tool named `name` settles, matching `options`. Without a `status`,
+     * any settled outcome matches.
+     */
+    waitForToolCall(name: string, options?: Omit<EveEvalToolCallMatchOptions, "count">): Promise<EveEvalToolCall>;
 }
 /** Operations and state belonging to one accepted session. */
 interface EveEvalSessionDriver {
     /** All events observed on this session so far. */
-    readonly events: readonly MessageStreamEvent[];
+    readonly events: readonly SessionStreamEvent[];
     /**
      * User and assistant messages observed on this session in turn order. Pass
      * this to a judge's `on` option to grade the complete conversation.
      */
     readonly transcript: string;
     /** Input requests still open when the last turn parked, including ones raised before a partial `respond()`. */
-    readonly pendingInputRequests: readonly InputRequest[];
+    readonly pendingInteractions: readonly Interaction[];
     /** Serializable cursor for resuming this session. */
     readonly state: ClientSessionState;
-    /** Durable session id, available when the session is returned. */
-    readonly sessionId: string;
+    /** Fixed session ID, available without reading the stream. */
+    readonly id: string;
+    /** Read and record the next turn from this session's current cursor. */
+    nextTurn(options?: EveEvalWatchOptions): Promise<EveEvalTurn>;
+    /** Start consuming from this session's current cursor and return its live-turn handle. */
+    watch(options?: EveEvalWatchOptions): EveEvalLiveTurn;
     /** Request cooperative cancellation of this session's active turn. */
-    cancel(): Promise<CancelSessionResult>;
+    cancel(): Promise<Delivery>;
     /**
      * Compact this session's history between turns and wait for it to finish.
-     * Returns the compaction events through `session.waiting`; throws when the
+     * Returns the compaction's events through its `context.settled`; throws when the
      * session is no longer active.
      */
     compact(): Promise<EveEvalTurn>;
     /** Require exactly one pending input request matching `filter`, or abort dependent control flow. */
-    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
+    requireInteraction(filter?: EveEvalInteractionMatchOptions): Interaction;
     /** Resolve specific pending requests and run the resumed turn. */
-    respond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalTurn>;
+    respond(responses: readonly Answer[], options?: RespondTurnOptions): Promise<EveEvalTurn>;
     /** Start a response turn without waiting for its boundary. */
-    startRespond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
+    startRespond(responses: readonly Answer[], options?: RespondTurnOptions): Promise<EveEvalLiveTurn>;
     /** Resolve every pending request with the same option id. */
     respondAll(optionId: string): Promise<EveEvalTurn>;
     /** Send one turn through this session. */
@@ -354,28 +374,36 @@ interface EveEvalSessionDriver {
     sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
     /**
      * The session an agent run opened, as this session's stream announced it with
-     * `agent.started`. Its `stream()` follows the child through this parent session
+     * `child.opened`. Its `stream()` follows the child through this parent session
      * with the eval client's credentials, and stops with the eval unless given a `signal`.
      */
-    agent(started: AgentStartedStreamEvent): EveEvalAgentSession;
+    agent(opened: Extract<SessionStreamEvent, {
+        type: "child.opened";
+    }>): EveEvalAgentSession;
 }
 /** A session an agent run opened, reached through the parent eval session. */
-export type EveEvalAgentSession = Pick<ClientAgentSession, "name" | "sessionId" | "stream" | "taskId">;
+export type EveEvalAgentSession = Pick<ClientAgentSession, "name" | "id" | "stream" | "taskId">;
 /** One accepted session, exposed by `t.session()`, turns, and target attachment helpers. */
 export interface EveEvalSession extends EveEvalSessionDriver, EveEvalAssertions, EveEvalOutputAssertions {
+    readonly view: SessionView;
 }
 /**
  * One completed eval-driver turn.
  */
 export interface EveEvalTurn extends EveEvalAssertions, EveEvalOutputAssertions {
+    readonly view: SessionView;
+    readonly sessionEnded: boolean;
     readonly data: unknown;
-    readonly events: readonly MessageStreamEvent[];
-    readonly inputRequests: readonly InputRequest[];
-    readonly message: string | undefined;
+    readonly events: readonly SessionStreamEvent[];
+    readonly interactions: readonly Interaction[];
+    readonly reply: string | undefined;
+    /** A watch may stop before a delivery settles, so its outcome can be absent. */
+    readonly outcome?: DeliveryOutcome;
+    readonly error?: ErrorInfo;
     /** Session that owns this turn; use it for follow-up messages. */
     readonly session: EveEvalSession;
     readonly sessionId: string;
-    readonly status: "completed" | "failed" | "waiting";
+    readonly status: RequestStatus | "working";
     readonly toolCalls: readonly EveEvalToolCall[];
     /** Require exactly one matching tool call, record a gate, and return it for dependent checks. */
     requireToolCall(name: string, options?: Omit<EveEvalToolCallMatchOptions, "count">): EveEvalToolCall;
@@ -446,7 +474,7 @@ export interface EveEvalContext<TContext = unknown> extends EveEvalAssertions {
     /** Run-wide setup context, shared by reference across evals and teardown. */
     readonly context: TContext;
     /** Eval timeout signal. */
-    readonly signal: AbortSignal;
+    readonly abortSignal: AbortSignal;
     /** Current target under test. */
     readonly target: EveEvalTargetHandle;
     /** Structured eval log hook. */
@@ -504,24 +532,17 @@ export interface EveEvalTargetHandle extends EveEvalTarget {
     dispatchSchedule(scheduleId: string): Promise<EveEvalScheduleDispatchResult>;
     /** Authenticated fetch against the target base URL. */
     fetch(path: string, init?: RequestInit): Promise<Response>;
-    /**
-     * Attach to a pre-existing session and consume one turn boundary.
-     *
-     * When that boundary is `session.waiting`, the attached session recovers
-     * the exact session ID, so `session.send(...)` and `session.respond(...)`
-     * continue the same durable session.
-     */
-    attachSession(sessionId: string, opts?: {
+    /** Attach to an exact session ID without I/O. Read it with nextTurn() or watch(). */
+    attach(sessionId: string, opts?: {
         readonly startIndex?: number;
-    }): Promise<EveEvalSession>;
-    /**
-     * Observe one in-progress turn from a session created outside the eval.
-     * The returned live-turn handle starts consuming immediately and owns the
-     * stream through its next turn boundary.
-     */
-    watchTurn(sessionId: string, opts?: {
-        readonly startIndex?: number;
-    }): EveEvalLiveTurn;
+    }): EveEvalSession;
+}
+/** Where a watched read starts, and what ends it when no turn boundary does. */
+export interface EveEvalWatchOptions {
+    /** The stream line to start from. */
+    readonly startIndex?: number;
+    /** Ends the read with the line holding the first matching event, instead of a turn boundary. */
+    readonly until?: (event: SessionStreamEvent) => boolean;
 }
 /**
  * Shared fields between the user-facing input and the validated eval.
@@ -657,7 +678,7 @@ export interface EveEvalConfigInput<TContext = unknown> {
     readonly judge?: EveEvalJudgeConfig;
     /**
      * Reporters that observe every eval in the run (e.g. a shared
-     * `Braintrust()` experiment). Suppressed by `eve eval --skip-report`.
+     * `braintrustReporter()` experiment). Suppressed by `eve eval --skip-report`.
      */
     readonly reporters?: readonly EvalReporter[];
     /**

package/dist/src/eve-channel/create-request.d.ts

@@ -1,6 +1,5 @@
-import type { UserContent } from "ai";
+import type { MessageContent } from "#shared/message-content.js";
 import type { SessionCallback, SessionCapabilities } from "#channel/types.js";
-import type { LegacyRemoteAgentCaller } from "#execution/legacy-remote-agent/protocol.js";
 import type { ClientContextValue } from "#eve-channel/types.js";
 import type { JsonObject } from "#shared/json.js";
 import type { ToolStub } from "#tool-stubs/types.js";
@@ -8,19 +7,18 @@ export interface ParsedCreateBody {
     stubs?: readonly ToolStub[];
     callback?: SessionCallback;
     capabilities?: SessionCapabilities;
-    message?: string | UserContent;
+    message?: MessageContent;
     context?: ClientContextValue;
     operationId?: string;
     outputSchema?: JsonObject;
     /** Remote agent protocol of a delegating caller; set only with {@link callback}. */
     protocolVersion?: number;
-    legacyRemoteAgentCaller?: LegacyRemoteAgentCaller;
 }
 /** Enforces the fields that only make sense when creation also starts a turn. */
 export declare function validateMessageFreeCreate(input: {
     readonly callback: SessionCallback | undefined;
     readonly hasClientContext: boolean;
     readonly hasMessageField: boolean;
-    readonly message: string | UserContent | undefined;
+    readonly message: MessageContent | undefined;
     readonly outputSchema: JsonObject | undefined;
 }): Response | undefined;

package/dist/src/eve-channel/descendant-stream.d.ts

@@ -0,0 +1,9 @@
+import { type Session } from "#channel/session.js";
+import { type RemoteAgentBinding } from "#eve-channel/support.js";
+/** Every hop resolves recorded authority on its owning deployment, never an event-supplied URL. */
+export declare function descendantStreamResponse(input: {
+    readonly request: Request;
+    readonly root: Session;
+    readonly attachSession: (id: string) => Session;
+    readonly resolveHeaders?: (binding: RemoteAgentBinding) => Promise<Record<string, string>>;
+}): Promise<Response>;

package/dist/src/eve-channel/descendant-stream.js

@@ -0,0 +1 @@
+import{internalSession}from"#channel/session.js";import{isObject}from"#shared/guards.js";import{createEveDescendantStreamRoutePath,createEveSessionStreamRoutePath}from"#protocol/routes.js";import{DESCENDANT_MAX_DEPTH,EVE_OBSERVATION_KEY_HEADER,localObservationKey}from"#protocol/descendant-observation.js";import{waitWithSignal}from"#shared/wait-with-signal.js";import{createSessionStreamResponse}from"#eve-channel/request.js";import{findRemoteAgentBinding}from"#eve-channel/support.js";import{proxyStreamResponse}from"#eve-channel/stream-headers.js";import{isExpiredWorkflowRunError}from"#internal/workflow/is-inactive-workflow-run-error.js";async function descendantStreamResponse(e){let t=new URL(e.request.url),n=t.searchParams.get(`children`),r;try{if(n===null||n.length>4096)throw Error();let e=parseDescendantPath(JSON.parse(n));if(e===void 0)throw Error();r=e}catch{return Response.json({error:`Invalid descendant path.`,ok:!1},{status:400})}let i=new AbortController,a=setTimeout(()=>i.abort(new DOMException(`Reader resolution timed out.`,`AbortError`)),1e4),o=AbortSignal.any([e.request.signal,i.signal]),s=e.root,c=new Set([s.id]);try{for(let[n,i]of r.entries()){let a=i.sessionId,l=await recordedChild(s,i,o);if(l===void 0)return unavailable();let u=r.slice(n+1);if(l.data.stream===createEveSessionStreamRoutePath(a)){if(c.has(a))return unavailable(`cycle`,422);if(c.add(a),s=e.attachSession(a),u.length!==0)continue;let t=await resolvedResponse(createSessionStreamResponse(e.request,s),o);return t.headers.set(EVE_OBSERVATION_KEY_HEADER,localObservationKey(a)),t}let d=await waitWithSignal(findRemoteAgentBinding({parent:internalSession(s),callId:`callId`in l.data.owner?l.data.owner.callId:``,childSessionId:a,childStreamPath:l.data.stream}),o);if(d===void 0||d.earlierProtocol!==void 0||e.resolveHeaders===void 0)return unavailable(`unsupported`,422);let f=await waitWithSignal(e.resolveHeaders(d),o),p=u.length===0?createEveSessionStreamRoutePath(a):createEveDescendantStreamRoutePath(a),m=new URL(p.replace(/^\/+/,``),`${d.url.replace(/\/+$/,``)}/`);for(let e of[`startIndex`,`includeTailIndex`,`follow`]){let n=t.searchParams.get(e);n!==null&&m.searchParams.set(e,n)}u.length!==0&&m.searchParams.set(`children`,JSON.stringify(u));let h=await resolvedResponse(fetch(m,{cache:`no-store`,headers:f,redirect:`manual`,signal:o}),o);if(h.status>=300&&h.status<400)return h.body?.cancel().catch(()=>{}),unavailable(`unsupported`,422);let g=proxyStreamResponse(h);if(g.ok){let e=u.length===0?localObservationKey(a):h.headers.get(EVE_OBSERVATION_KEY_HEADER);if(e===null||e.length>8192)return h.body?.cancel().catch(()=>{}),unavailable(`unsupported`,422);g.headers.set(EVE_OBSERVATION_KEY_HEADER,JSON.stringify([`remote`,s.id,a,e]))}return g}}catch(e){return isExpiredWorkflowRunError(e)?unavailable(`session_expired`,410):unavailable()}finally{clearTimeout(a)}return unavailable()}function parseDescendantPath(e){if(!Array.isArray(e)||e.length===0||e.length>DESCENDANT_MAX_DEPTH)return;let t=e,n=[];for(let e of t){if(!isObject(e)||typeof e.sessionId!=`string`||e.sessionId.length===0||e.sessionId.length>256||typeof e.position!=`number`||!Number.isSafeInteger(e.position)||e.position<0)return;n.push({sessionId:e.sessionId,position:e.position})}return n}async function resolvedResponse(e,t){return e.then(e=>{t.aborted&&e.body?.cancel().catch(()=>{})},()=>{}),await waitWithSignal(e,t)}function unavailable(e=`descendant_unavailable`,t=404){return Response.json({ok:!1,code:e,error:`Descendant history is unavailable.`},{status:t})}async function recordedChild(e,t,n){n.throwIfAborted();let r=new AbortController,i=e.stream({follow:!1,startIndex:t.position,signal:AbortSignal.any([n,r.signal])})[Symbol.asyncIterator]();try{for(;;){let e=await waitWithSignal(i.next(),n);if(e.done||e.value.meta.position.line>t.position)return;let r=e.value;if(r.meta.position.line===t.position&&r.type===`child.opened`&&r.data.sessionId===t.sessionId)return r}}finally{r.abort(),i.return?.().catch(()=>{})}}export{descendantStreamResponse};

package/dist/src/eve-channel/index.d.ts

@@ -1,3 +1,3 @@
-import type { EveChannel, EveChannelInput } from "#eve-channel/types.js";
+import type { EveChannel, EveChannelConfig } from "#eve-channel/types.js";
 export * from "#eve-channel/types.js";
-export declare function eveChannel(input: EveChannelInput): EveChannel;
+export declare function eveChannel(input: EveChannelConfig): EveChannel;