eve 0.68.0 → 0.70.0

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-import{isJsonObjectValue}from"#shared/json.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return r.push(s),i.set(e,s),s},ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={callId:e,name:t,status:`working`,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&ensureToolCall(e.callId,LOAD_SKILL_TOOL_NAME,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n;let r=e.output;n===`completed`&&isJsonObjectValue(r)&&r.status===`working`&&typeof r.taskId==`string`&&typeof r.agentId==`string`&&ensureSubagentCall(e.callId,e.toolName)}else if(e.kind===`subagent-result`){let t=ensureSubagentCall(e.callId,e.subagentName);if(e.origin===`child`&&e.backgroundTask!==void 0)break;t.output=t.output??e.output,t.status=e.origin===`child`&&e.outcome.result.kind===`cancelled`?`cancelled`:n===`rejected`?`failed`:n}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);e.childSessionId=t.data.childSessionId,t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);if(t.data.backgroundTask!==void 0||e.status!==`working`)break;e.output=t.data.output,e.status=`completed`;break}case`input.requested`:s.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||TURN_EPILOGUE_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=[],l=-1,u=0,d=0,f=new Set,p,m,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(l,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:l+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&ensureToolCall(e.callId,LOAD_SKILL_TOOL_NAME,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;a.push({callId:e,name:r,taskId:i,turnIndex:Math.max(l,0)});break}case`task.settled`:o.set(t.data.callId,t.data);break;case`agent.started`:s.push(t.data);break;case`input.requested`:c.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(u+=1);break;case`reasoning.completed`:d+=1;break;case`step.started`:f.add(t.data.modelId);break;case`session.waiting`:case`turn.waiting`:p=t.data.usage;break;case`session.completed`:p=t.data?.usage;break;case`session.failed`:p=t.data.usage,m=t.data.code}let h=deriveSubagentCalls({agentCalls:a,agentSessions:s,sessionId:n,settledTaskCalls:o});return{toolCalls:r,toolCallCount:r.length,subagentCalls:h,subagentCallCount:h.length,inputRequests:c,parked:endedParkedOnInput(e),messageCount:u,reasoningBlockCount:d,models:[...f],usage:p,failureCode:m}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||PARKING_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector,s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector,s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:i}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=[...new Map(e.map(e=>[e.sessionId,e.derived.usage]))].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n]);if(n.every(e=>e!==void 0))return n.reduce(addTokenUsage)}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/assertion-scores.d.ts

@@ -0,0 +1,19 @@
+import type { AssertionResult } from "#evals/types.js";
+/** Assertions reported under one name, scored together. */
+export interface AssertionScoreGroup {
+    readonly score: number;
+    readonly assertions: readonly AssertionResult[];
+}
+/**
+ * Groups an eval's assertions by reported name and scores each group by its lowest member.
+ *
+ * Repeated assertions, such as several unlabeled judges, share a name. The minimum keeps one
+ * comparable score per name, and a group passes only when every member does.
+ */
+export declare function groupAssertionScores(assertions: readonly AssertionResult[], getName: (assertion: AssertionResult) => string): Map<string, AssertionScoreGroup>;
+/** Per-assertion scores retained in row metadata once repeated assertions share one score. */
+export declare function composeAssertionScoreMetadata(assertions: readonly AssertionResult[]): {
+    name: string;
+    severity: import("#evals/types.js").AssertionSeverity;
+    score: number;
+}[];

package/dist/src/evals/runner/reporters/assertion-scores.js

@@ -0,0 +1 @@
+function groupAssertionScores(e,t){let n=new Map;for(let r of e){let e=t(r),i=n.get(e);n.set(e,{score:Math.min(i?.score??r.score,r.score),assertions:[...i?.assertions??[],r]})}return n}function composeAssertionScoreMetadata(e){return e.map(({name:e,severity:t,score:n})=>({name:e,severity:t,score:n}))}export{composeAssertionScoreMetadata,groupAssertionScores};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,e=>e.severity===`gate`?`gate:${e.name}`:e.name);for(let[e,t]of r)n[e]=t.score;let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/datadog-client.d.ts

@@ -0,0 +1,123 @@
+export type DatadogJsonValue = string | number | boolean | null | DatadogJsonValue[] | {
+    [key: string]: DatadogJsonValue;
+};
+export interface DatadogDatasetRecordInput {
+    inputData: DatadogJsonValue;
+    expectedOutput?: DatadogJsonValue;
+    metadata?: Record<string, DatadogJsonValue>;
+    tags?: string[];
+}
+export interface DatadogDatasetRecord {
+    readonly id: string | null;
+    readonly input: DatadogJsonValue;
+    readonly expectedOutput: DatadogJsonValue;
+    readonly metadata: Record<string, DatadogJsonValue>;
+}
+export interface DatadogDatasetRecordFields {
+    input: DatadogJsonValue;
+    expectedOutput: DatadogJsonValue;
+    metadata: Record<string, DatadogJsonValue>;
+}
+export interface DatadogDataset {
+    id(): string | null;
+    name(): string;
+    version(): number | null;
+    latestVersion(): number | null;
+    records(): readonly DatadogDatasetRecord[];
+    addRecords(records: DatadogDatasetRecordInput[]): DatadogDataset;
+    update(index: number, fields: Partial<DatadogDatasetRecordFields>): DatadogDataset;
+    url(): string | null;
+    push(): Promise<{
+        pushedCount: number;
+        totalCount: number;
+    }>;
+}
+export interface DatadogExperimentsClient {
+    createDataset?(name: string, options?: {
+        projectName?: string;
+        description?: string;
+    }): DatadogDataset;
+    pullDataset?(name: string, options?: {
+        projectName?: string;
+        maxWaitMs?: number;
+    }): Promise<DatadogDataset>;
+    startExperiment(options: DatadogStartExperimentOptions): Promise<DatadogExternalExperiment>;
+}
+export interface DatadogStartExperimentOptions {
+    name: string;
+    projectName?: string;
+    description?: string;
+    tags?: Record<string, string>;
+    metadata?: Record<string, DatadogJsonValue>;
+    config?: Record<string, DatadogJsonValue>;
+    dataset?: {
+        id?: string;
+        version?: number;
+        name?: string;
+        description?: string;
+    };
+}
+export interface DatadogExternalExperiment {
+    experimentId(): string;
+    url(): string | null;
+    submitSpan(row: DatadogExternalExperimentSpanInput): Promise<DatadogExternalExperimentSpan>;
+    submitEvaluationMetrics(span: Pick<DatadogExternalExperimentSpan, "experimentId" | "spanId" | "traceId">, metrics: DatadogEvaluationMetricInput[]): Promise<void>;
+    close(options?: {
+        status?: string;
+        error?: string | Error;
+    }): Promise<void>;
+}
+export interface DatadogExternalExperimentSpanInput {
+    name?: string;
+    input?: DatadogJsonValue;
+    output?: DatadogJsonValue;
+    expectedOutput?: DatadogJsonValue;
+    metadata?: Record<string, DatadogJsonValue>;
+    tags?: Record<string, string>;
+    startedAt?: Date | string | number;
+    completedAt?: Date | string | number;
+    durationMs?: number;
+    error?: string | Error | {
+        type?: string;
+        name?: string;
+        message?: string;
+        stack?: string;
+    };
+    datasetRecordId?: string;
+    runId?: string;
+    runIteration?: number;
+}
+interface DatadogExternalExperimentSpan {
+    experimentId: string;
+    spanId: string;
+    traceId: string;
+    url: string | null;
+}
+export interface DatadogEvaluationMetricInput {
+    label: string;
+    value?: DatadogJsonValue;
+    error?: string | Error;
+    timestamp?: Date | string | number;
+    tags?: Record<string, string>;
+    source?: string;
+}
+/** Options used when eve initializes `dd-trace` itself. */
+export interface DatadogClientConfig {
+    readonly client?: DatadogExperimentsClient;
+    readonly service?: string;
+    readonly env?: string;
+    readonly site?: string;
+    readonly mlApp?: string;
+}
+export declare const MISSING_DATASET_API_MESSAGE: string;
+/** Pulls the named dataset, creating it when Datadog has no dataset with that name. */
+export declare function getDatadogDataset(client: DatadogExperimentsClient, name: string, options: {
+    projectName?: string;
+    description: string;
+}): Promise<DatadogDataset>;
+/** Returns `config.client`, or initializes `dd-trace` and returns its experiments API. */
+export declare function resolveDatadogClient(config: DatadogClientConfig, projectName: string): Promise<DatadogExperimentsClient>;
+export declare function toOptionalDatadogJsonValue(value: unknown): DatadogJsonValue | undefined;
+export declare function toDatadogJsonRecord(value: Readonly<Record<string, unknown>> | undefined): Record<string, DatadogJsonValue> | undefined;
+export declare function toDatadogJsonValue(value: unknown): DatadogJsonValue;
+export {};

package/dist/src/evals/runner/reporters/datadog-client.js

@@ -0,0 +1,4 @@
+import{createRequire}from"node:module";import{parseJsonValue}from"#shared/json.js";const DD_TRACE_PACKAGE=`dd-trace`,MISSING_DATASET_API_MESSAGE=[`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.createDataset() and pullDataset().`,`Update to a release compatible with dd-trace@6.13.0.`].join(`
+`);async function getDatadogDataset(e,t,n){if(!e.pullDataset||!e.createDataset)throw Error(MISSING_DATASET_API_MESSAGE);try{return await e.pullDataset(t,{projectName:n.projectName,maxWaitMs:5e3})}catch(e){if(!(e instanceof Error)||!e.message.startsWith(`Dataset '${t}' not found`))throw e}return e.createDataset(t,n)}async function resolveDatadogClient(e,t){if(e.client)return e.client;let n=(await loadDatadogSdk()).init({service:e.service??process.env.DD_SERVICE??t,env:e.env??process.env.DD_ENV,site:e.site??process.env.DD_SITE,llmobs:{projectName:t,mlApp:e.mlApp??t,agentlessEnabled:!0}}).llmobs?.experiments;if(!n?.startExperiment)throw Error([`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().`,`Update to a release compatible with dd-trace@6.13.0.`].join(`
+`));return n}async function loadDatadogSdk(){try{return createRequire(`${process.cwd()}/package.json`)(DD_TRACE_PACKAGE)}catch{try{let e=await import(DD_TRACE_PACKAGE);return e.default??e}catch{throw Error([`The 'dd-trace' package is required for Datadog reporting but was not found.`,``,`Install the tested release with:`,`  npm install dd-trace@6.13.0`].join(`
+`))}}}function toOptionalDatadogJsonValue(e){return e===void 0?void 0:toDatadogJsonValue(e)}function toDatadogJsonRecord(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))r!==void 0&&(t[n]=toDatadogJsonValue(r));return t}function toDatadogJsonValue(e){return cloneDatadogJsonValue(parseJsonValue(e))}function cloneDatadogJsonValue(e){if(Array.isArray(e))return e.map(e=>cloneDatadogJsonValue(e));if(typeof e==`object`&&e){let t={};for(let[n,r]of Object.entries(e))t[n]=cloneDatadogJsonValue(r);return t}return e}export{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue};

package/dist/src/evals/runner/reporters/datadog.d.ts

@@ -1,9 +1,10 @@
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
+import { type DatadogExperimentsClient } from "#evals/runner/reporters/datadog-client.js";
 /** Configuration for the Datadog reporter. */
 export interface DatadogReporterConfig {
     /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_PROJECT_NAME`, the configured ml_app, `DD_SERVICE`, or the first eval id. */
     readonly projectName?: string;
-    /** Name for the dataset used by the experiment. Defaults to `<experimentName> dataset`. */
+    /** Dataset reused across runs. Defaults to `<projectName> evals`, or `<experimentName> dataset` without `recordInputs`. */
     readonly datasetName?: string;
     /** Name for the created experiment. Defaults to a timestamped eve eval run name. */
     readonly experimentName?: string;
@@ -23,115 +24,25 @@ export interface DatadogReporterConfig {
     readonly metadata?: Readonly<Record<string, unknown>>;
     /** JSON-serializable Datadog Experiment config. */
     readonly config?: Readonly<Record<string, unknown>>;
-    /** Include the first sent message, falling back to the eval description, in the experiment row and a linked dataset record. Defaults to false. */
+    /** Include the first sent message, falling back to the eval description, in the experiment row and a linked dataset record. Defaults to true. */
     readonly recordInputs?: boolean;
-    /** Include eval outputs in the synthetic experiment row output. Defaults to false. */
+    /** Include eval outputs in the synthetic experiment row output. Defaults to true. */
     readonly recordOutputs?: boolean;
-    /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to false. */
+    /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to true. */
     readonly recordExpectedOutputs?: boolean;
-    /** Include raw assertion names as metric tags and failure messages in row metadata. Defaults to false. */
+    /** Include each assertion's name and score, and failure messages, in row metadata. Defaults to true. */
     readonly recordAssertionDetails?: boolean;
-    /** Include execution error messages, which may contain application data. Defaults to false. */
+    /** Include execution error messages, which may contain application data. Defaults to true. */
     readonly recordErrors?: boolean;
     /** Console hook used for tests. */
     readonly log?: (line: string) => void;
     /** eve-owned client seam for tests and custom Datadog SDK wiring. */
     readonly client?: DatadogExperimentsClient;
 }
-type DatadogJsonValue = string | number | boolean | null | DatadogJsonValue[] | {
-    [key: string]: DatadogJsonValue;
-};
-interface DatadogDatasetRecordInput {
-    inputData: DatadogJsonValue;
-    expectedOutput?: DatadogJsonValue;
-    metadata?: Record<string, DatadogJsonValue>;
-    tags?: string[];
-}
-interface DatadogDatasetRecord {
-    readonly id: string | null;
-}
-interface DatadogDataset {
-    id(): string | null;
-    name(): string;
-    version(): number | null;
-    records(): readonly DatadogDatasetRecord[];
-    url(): string | null;
-    push(): Promise<{
-        pushedCount: number;
-        totalCount: number;
-    }>;
-}
-interface DatadogExperimentsClient {
-    createDataset?(name: string, options?: {
-        projectName?: string;
-        description?: string;
-        records?: DatadogDatasetRecordInput[];
-    }): DatadogDataset;
-    startExperiment(options: DatadogStartExperimentOptions): Promise<DatadogExternalExperiment>;
-}
-interface DatadogStartExperimentOptions {
-    name: string;
-    projectName?: string;
-    description?: string;
-    tags?: Record<string, string>;
-    metadata?: Record<string, DatadogJsonValue>;
-    config?: Record<string, DatadogJsonValue>;
-    dataset?: {
-        id?: string;
-        version?: number;
-        name?: string;
-        description?: string;
-    };
-}
-interface DatadogExternalExperiment {
-    experimentId(): string;
-    url(): string | null;
-    submitSpan(row: DatadogExternalExperimentSpanInput): Promise<DatadogExternalExperimentSpan>;
-    submitEvaluationMetrics(span: Pick<DatadogExternalExperimentSpan, "experimentId" | "spanId" | "traceId">, metrics: DatadogEvaluationMetricInput[]): Promise<void>;
-    close(options?: {
-        status?: string;
-        error?: string | Error;
-    }): Promise<void>;
-}
-interface DatadogExternalExperimentSpanInput {
-    name?: string;
-    input?: DatadogJsonValue;
-    output?: DatadogJsonValue;
-    expectedOutput?: DatadogJsonValue;
-    metadata?: Record<string, DatadogJsonValue>;
-    tags?: Record<string, string>;
-    startedAt?: Date | string | number;
-    completedAt?: Date | string | number;
-    durationMs?: number;
-    error?: string | Error | {
-        type?: string;
-        name?: string;
-        message?: string;
-        stack?: string;
-    };
-    datasetRecordId?: string;
-    runId?: string;
-    runIteration?: number;
-}
-interface DatadogExternalExperimentSpan {
-    experimentId: string;
-    spanId: string;
-    traceId: string;
-    url: string | null;
-}
-interface DatadogEvaluationMetricInput {
-    label: string;
-    value?: DatadogJsonValue;
-    error?: string | Error;
-    timestamp?: Date | string | number;
-    tags?: Record<string, string>;
-    source?: string;
-}
 /**
- * Creates an {@link EvalReporter} that uploads eval assertion scores to a
- * Datadog LLM Observability Experiment. Requires the optional `dd-trace`
+ * Creates an {@link EvalReporter} that uploads eval results to a Datadog LLM
+ * Observability Experiment. Requires the optional `dd-trace`
  * package and `DD_API_KEY`/`DD_APP_KEY` credentials unless `config.client` is
  * provided.
  */
 export declare function Datadog(config?: DatadogReporterConfig): EvalReporter;
-export {};

package/dist/src/evals/runner/reporters/datadog.js

@@ -1,3 +1,2 @@
-import{createRequire}from"node:module";import{parseJsonValue}from"#shared/json.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t=new Map;#n;#r;#i;#a;constructor(e){this.#e=e}async onRunStart(e,t){this.#n=void 0,this.#r=void 0,this.#i=void 0,this.#a=void 0,this.#t.clear();for(let t of e)this.#t.set(t.id,t);let n=await resolveDatadogClient(this.#e,e),r=resolveLocalGitMetadata(process.cwd()),i=this.#e.experimentName??defaultExperimentName(),a=resolveExperimentMetadata(e,t);r.sha&&(a.eveGitCommit=r.sha,a.eveGitBranch=r.branch),Object.assign(a,this.#e.metadata);let o={name:i,projectName:resolveProjectName(this.#e,e),description:this.#e.description,dataset:{name:this.#e.datasetName??`${i} dataset`},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(a),config:toDatadogJsonRecord(this.#e.config)};if(this.#e.recordInputs){if(!n.createDataset)throw Error(`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.createDataset().`);this.#n=n,this.#r=o;return}this.#i=await n.startExperiment(o)}async onEvalComplete(e){!this.#e.recordInputs&&this.#i&&await this.#s(e)}async onRunComplete(e){try{if(this.#e.recordInputs&&await this.#o(e.results),!this.#i)return;let t=e.failed>0||e.errored>0;await this.#i.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#a&&n(`Datadog dataset URL: ${this.#a}\n`);let r=this.#i.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#n=void 0,this.#r=void 0,this.#i=void 0,this.#a=void 0}}async#o(e){let t=this.#n,n=this.#r;if(!t?.createDataset||!n)return;let r=n.dataset?.name??`${n.name} dataset`,i=e.map(e=>{let t=this.#t.get(e.id),n={inputData:toDatadogJsonValue(resolveInput(e,t)),metadata:{eveEvalId:e.id}};if(this.#e.recordExpectedOutputs){let e=toOptionalDatadogJsonValue(resolveExpectedOutput(t));e!==void 0&&(n.expectedOutput=e)}return n}),a=t.createDataset(r,{projectName:n.projectName,description:this.#e.description??`Eve eval inputs for experiment '${n.name}'.`,records:i});await a.push();let o=a.id();if(!o)throw Error(`Datadog dataset '${r}' has no id after push().`);let s=a.records();if(s.length!==e.length)throw Error(`Datadog dataset '${r}' has ${s.length} records for ${e.length} eval results.`);let c={id:o,name:a.name()},l=a.version();l!==null&&(c.version=l),this.#a=a.url()??void 0,this.#i=await t.startExperiment({...n,dataset:c});for(let[t,n]of e.entries()){let e=s[t]?.id;if(!e)throw Error(`Datadog dataset record ${t+1} has no id after push().`);await this.#s(n,e)}}async#s(e,t){if(!this.#i)return;let n=this.#t.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#e.recordAssertionDetails===!0)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#e.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#e.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#e.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#e.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#i.submitSpan(r);await this.#i.submitEvaluationMetrics(i,resolveEvaluationMetrics(e,this.#e.recordAssertionDetails===!0))}};const DD_TRACE_PACKAGE=`dd-trace`,EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`];async function resolveDatadogClient(e,t){if(e.client)return e.client;let n=await loadDatadogSdk(),r=resolveProjectName(e,t),i=n.init({service:e.service??process.env.DD_SERVICE??r,env:e.env??process.env.DD_ENV,site:e.site??process.env.DD_SITE,llmobs:{projectName:r,mlApp:e.mlApp??r,agentlessEnabled:!0}}).llmobs?.experiments;if(!i?.startExperiment)throw Error([`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().`,`Update to a release compatible with dd-trace@6.13.0.`].join(`
-`));return i}async function loadDatadogSdk(){try{return createRequire(`${process.cwd()}/package.json`)(DD_TRACE_PACKAGE)}catch{try{let e=await import(DD_TRACE_PACKAGE);return e.default??e}catch{throw Error([`The 'dd-trace' package is required for Datadog reporting but was not found.`,``,`Install the tested release with:`,`  npm install dd-trace@6.13.0`].join(`
-`))}}}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);if(Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked}),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e,t){let n=[],r=new Set(BUILT_IN_METRIC_LABELS);for(let[i,a]of e.assertions.entries()){let e=a.severity===`gate`?`gate_${a.name}`:a.name,o={assertion_index:String(i+1),assertion_severity:a.severity,assertion_passed:String(a.passed)};t&&(o.assertion_name=a.name,o.assertion_label=e),n.push({label:reserveDatadogMetricLabel(toDatadogMetricLabel(e),r),value:a.score,tags:o})}return n.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),n}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(0,r-n)}function toOptionalDatadogJsonValue(e){return e===void 0?void 0:toDatadogJsonValue(e)}function toDatadogJsonRecord(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))r!==void 0&&(t[n]=toDatadogJsonValue(r));return t}function toDatadogJsonValue(e){return cloneDatadogJsonValue(parseJsonValue(e))}function cloneDatadogJsonValue(e){if(Array.isArray(e))return e.map(e=>cloneDatadogJsonValue(e));if(typeof e==`object`&&e){let t={};for(let[n,r]of Object.entries(e))t[n]=cloneDatadogJsonValue(r);return t}return e}export{Datadog};
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
+`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);if(Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked}),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(e.severity===`gate`?`gate_${e.name}`:e.name));for(let[e,i]of r)t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,assertion_passed:String(i.assertions.every(e=>e.passed)),assertion_count:String(i.assertions.length)}});return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(0,r-n)}export{Datadog};

package/dist/src/evals/session-content.js

@@ -1,3 +1,3 @@
-import{extname}from"node:path";function formatEvalTranscript(e){let t=[];for(let n of e)n.type===`message.received`?t.push(`User:\n${n.data.message}`):n.type===`message.completed`&&n.data.message!==null&&t.push(`Assistant:\n${n.data.message}`);return t.join(`
+import{extname}from"node:path";function formatEvalTranscript(e){let t=[];for(let n of e)n.type===`message.received`?t.push(`User:\n${n.data.message}`):n.type===`message.completed`&&t.push(`Assistant:\n${n.data.message}`);return t.join(`
 
 `)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{formatEvalTranscript,inferMediaType};

package/dist/src/evals/session.d.ts

@@ -1,9 +1,9 @@
 import type { ClientSession } from "#client/session.js";
-import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions, StreamOptions } from "#client/types.js";
-import type { MessageStreamEvent, RuntimeTraceContext, SubagentCalledStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
+import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions } from "#client/types.js";
+import type { MessageStreamEvent, RuntimeTraceContext, AgentStartedStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#shared/input.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
-import type { EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import type { EveEvalAgentSession, EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
 import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
@@ -39,7 +39,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
     get sessionId(): string;
     get state(): ClientSessionState;
     cancel(): Promise<CancelSessionResult>;
-    streamSubagent(called: SubagentCalledStreamEvent, options?: StreamOptions): AsyncIterable<MessageStreamEvent>;
+    agent(started: AgentStartedStreamEvent): EveEvalAgentSession;
     /** @internal */
     cleanup(signal: AbortSignal): Promise<void>;
     requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;

package/dist/src/evals/session.js

@@ -1,2 +1,2 @@
-import{readFile}from"node:fs/promises";import{basename}from"node:path";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{summarizeTurnEvents}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}streamSubagent(e,t={}){return this.#e.streamSubagent(e,{...t,signal:t.signal??this.#r})}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
-`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1;for await(let n of e)if(this.#t.push(n),t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),isCurrentTurnBoundaryEvent(n)){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
+`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};

package/dist/src/evals/types.d.ts

@@ -1,10 +1,11 @@
+import type { TokenUsage } from "#shared/token-usage.js";
 import type { Experimental_EvaluationModel as EvaluationModel } from "ai";
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
-import type { RuntimeIdentity, RuntimeTraceContext, MessageStreamEvent, SubagentCalledStreamEvent } from "#protocol/message.js";
-import type { CancelSessionResult, ClientSessionState, CreateSessionOptions, SendTurnInput, SendTurnOptions, StreamOptions } from "#client/types.js";
+import type { ClientAgentSession } from "#client/agent-session.js";
+import type { RuntimeIdentity, RuntimeTraceContext, MessageStreamEvent, AgentStartedStreamEvent } from "#protocol/message.js";
+import type { CancelSessionResult, ClientSessionState, CreateSessionOptions, SendTurnInput, SendTurnOptions } from "#client/types.js";
 import type { InputRequest, InputResponse } from "#shared/input.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
-import type { TaskStatus } from "#tasks/types.js";
 import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
 import type { EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, EveEvalToolCallMatchOptions } from "#evals/match.js";
@@ -29,22 +30,23 @@ export interface EveEvalToolCall {
     readonly sessionId?: string;
 }
 /**
- * One subagent delegation extracted from the captured stream
- * (`subagent.called` / `subagent.started`, joined with `subagent.completed`).
+ * One call to an agent task extracted from the captured stream: its
+ * `task.started`, joined with its `task.settled` and the task's
+ * `agent.started`.
  */
 export interface EveEvalSubagentCall {
-    /** Runtime-action call id joining this delegation's lifecycle events, when observed. */
+    /** The agent tool call's id, as on its task events. */
     readonly callId?: string;
-    /** Durable child session id for local and remote workflow delegations. */
+    /** The agent's session id, shared by every call to one task; absent if it never opened. */
     readonly childSessionId?: string;
     /** Subagent name. */
     readonly name: string;
-    /** Remote agent URL for remote delegations (`subagent.called` remote metadata). */
+    /** Remote agent URL for remote delegations (`agent.started` remote metadata). */
     readonly remoteUrl?: string;
-    /** Output from the matching `subagent.completed` event; `undefined` when the call never completed. */
+    /** Output from the call's `task.settled`; `undefined` until the call completes. */
     readonly output?: JsonValue;
-    /** Task lifecycle status inferred from the captured delegation events. */
-    readonly status: TaskStatus;
+    /** The call's task status: `working` until its `task.settled` arrives. */
+    readonly status: "working" | "completed" | "failed" | "cancelled";
     /** Zero-based index of the turn the delegation happened in. */
     readonly turnIndex: number;
     /** Owning session id, when the runner knows it. */
@@ -64,6 +66,21 @@ export interface EveEvalDerivedFacts {
     readonly parked: boolean;
     readonly messageCount: number;
     readonly reasoningBlockCount: number;
+    /**
+     * Distinct ids of the models the steps started with (`step.started`): in first-use order for one
+     * session, and in session order for an eval. Covers only the sessions the eval created or
+     * attached: a delegated subagent runs in its own session, so its models appear only when the eval
+     * attaches that session. Compaction and `auto` routing calls are not included.
+     */
+    readonly models: readonly string[];
+    /**
+     * Token usage from the latest `session.waiting`, `turn.waiting`, `session.failed`, or
+     * `session.completed`: the
+     * session's own model calls plus what the agents it delegated to spent, so on a turn it is the
+     * session's total so far. For an eval, each captured session counts once, by its latest usage,
+     * except sessions another captured session opened. Absent when a counted session reported none.
+     */
+    readonly usage?: TokenUsage;
     readonly failureCode?: string;
 }
 /**
@@ -275,12 +292,14 @@ interface EveEvalSessionDriver {
     /** Send one text turn with a local file attached as a data URL. */
     sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
     /**
-     * Follow one delegated child's stream through this parent session, with the
-     * eval client's credentials. Local children use their own stream route; remote
-     * children use the parent-origin proxy.
+     * The session an agent run opened, as this session's stream announced it with
+     * `agent.started`. Its `stream()` follows the child through this parent session
+     * with the eval client's credentials, and stops with the eval unless given a `signal`.
      */
-    streamSubagent(called: SubagentCalledStreamEvent, options?: StreamOptions): AsyncIterable<MessageStreamEvent>;
+    agent(started: AgentStartedStreamEvent): EveEvalAgentSession;
 }
+/** A session an agent run opened, reached through the parent eval session. */
+export type EveEvalAgentSession = Pick<ClientAgentSession, "name" | "sessionId" | "stream" | "taskId">;
 /** One accepted session, exposed by `t.session()`, turns, and target attachment helpers. */
 export interface EveEvalSession extends EveEvalSessionDriver, EveEvalAssertions, EveEvalOutputAssertions {
 }

package/dist/src/eve-channel/create-request.d.ts

@@ -1,19 +1,20 @@
 import type { UserContent } from "ai";
-import type { ActivityObserverConfig, SessionCallback, SessionCapabilities, TaskDeliveryPolicy } from "#channel/types.js";
+import type { SessionCallback, SessionCapabilities } from "#channel/types.js";
+import type { LegacyRemoteAgentCaller } from "#execution/legacy-remote-agent/protocol.js";
 import type { JsonObject } from "#shared/json.js";
 export interface ParsedCreateBody {
-    taskDeliveryPolicy?: TaskDeliveryPolicy;
-    activityObserver?: ActivityObserverConfig;
     callback?: SessionCallback;
     capabilities?: SessionCapabilities;
     message?: string | UserContent;
     context?: readonly string[];
     operationId?: string;
     outputSchema?: JsonObject;
+    /** Remote agent protocol of a delegating caller; set only with {@link callback}. */
+    protocolVersion?: number;
+    legacyRemoteAgentCaller?: LegacyRemoteAgentCaller;
 }
 /** Enforces the fields that only make sense when creation also starts a turn. */
 export declare function validateMessageFreeCreate(input: {
-    readonly activityObserver: ActivityObserverConfig | undefined;
     readonly callback: SessionCallback | undefined;
     readonly hasClientContext: boolean;
     readonly hasMessageField: boolean;

package/dist/src/eve-channel/create-request.js

@@ -1 +1 @@
-function validateMessageFreeCreate(e){if(e.hasMessageField&&e.message===void 0)return Response.json({error:`Expected 'message' to be non-empty when provided.`,ok:!1},{status:400});if(e.message===void 0&&(e.hasClientContext||e.callback!==void 0||e.activityObserver!==void 0||e.outputSchema!==void 0))return Response.json({error:`Creating a session without a message does not accept 'clientContext', 'callback', 'activityObserver', or 'outputSchema'.`,ok:!1},{status:400})}export{validateMessageFreeCreate};
+function validateMessageFreeCreate(e){if(e.hasMessageField&&e.message===void 0)return Response.json({error:`Expected 'message' to be non-empty when provided.`,ok:!1},{status:400});if(e.message===void 0&&(e.hasClientContext||e.callback!==void 0||e.outputSchema!==void 0))return Response.json({error:`Creating a session without a message does not accept 'clientContext', 'callback', or 'outputSchema'.`,ok:!1},{status:400})}export{validateMessageFreeCreate};

package/dist/src/eve-channel/index.d.ts

@@ -1,12 +1,3 @@
 import type { EveChannel, EveChannelInput } from "#eve-channel/types.js";
 export * from "#eve-channel/types.js";
-/**
- * Builds the default eve HTTP channel: a {@link defineChannel} instance serving the
- * built-in `/eve/v1` routes (GET inspects the agent, POST creates a session,
- * ID-addressed POST routes deliver follow-ups and controls, and GET streams a
- * session's NDJSON event feed). Every route
- * runs {@link EveChannelInput.auth} via {@link routeAuth} before dispatching.
- * Default-export the result as your `agent/channels/eve.ts` channel; reach for
- * {@link defineChannel} directly only for a custom transport.
- */
 export declare function eveChannel(input: EveChannelInput): EveChannel;