eve 0.83.2 → 0.84.0

package/dist/src/evals/mock-model.d.ts

@@ -47,7 +47,11 @@ export interface MockModelToolCall {
     readonly name: string;
     /** JSON-serializable tool input. Defaults to an empty object. */
     readonly input?: unknown;
-    /** Stable call id. eve derives one when omitted. */
+    /**
+     * The call's id. eve derives a new one when omitted. A session's call ids never repeat, so an id
+     * the session's model already emitted, as when a responder calls a tool again after a sign-in,
+     * takes `<id>#<n>`.
+     */
     readonly id?: string;
 }
 /** Optional token counts reported by a mock response. */

package/dist/src/evals/mock-model.js

@@ -1 +1 @@
-import{markMockModel}from"#internal/mock-model-identity.js";import{MockLanguageModelV4}from"ai/test";import{TASKS_NOTE_LABEL,TASK_RESULT_TAG}from"#execution/tasks/render.js";import{isPendingApprovalsSnippet}from"#harness/hitl/approval-prompt.js";const RESPONSE_TIMESTAMP=Date.parse(`2026-01-01T00:00:00.000Z`);function mockModel(e={}){let t=normalizeOptions(e),n=normalizeResponder(t.respond),r=t.modelId??`model`;return markMockModel(new MockLanguageModelV4({modelId:r,provider:t.provider??`eve-mock`,doGenerate:async e=>createGenerateResult(await n(createRequest(e)),e,r),doStream:async e=>createStreamResult(createGenerateResult(await n(createRequest(e)),e,r))}))}function normalizeOptions(e){return typeof e==`string`||typeof e==`function`?{respond:e}:e}function normalizeResponder(e){if(typeof e==`function`)return e;let t=e??`Mock response`;return()=>t}function createRequest(e){let t=e.prompt.map(e=>({role:e.role,text:extractMessageText(e)})),n=t.filter(e=>e.role===`user`).map(e=>e.text).filter(e=>!isFrameworkScaffolding(e));return{lastUserMessage:n.at(-1)??null,messages:t,toolResults:extractToolResults(e),tools:extractTools(e),userMessageCount:n.length,userMessages:n}}function isFrameworkScaffolding(e){let t=e.trim();return t.startsWith(TASKS_NOTE_LABEL)||t.startsWith(TASK_RESULT_TAG)||isPendingApprovalsSnippet(t)}function extractMessageText(e){return typeof e.content==`string`?e.content:e.content.flatMap(e=>{switch(e.type){case`reasoning`:case`text`:return[e.text];case`tool-result`:return[formatValue(normalizeToolOutput(e.output))];default:return[]}}).join(``)}function extractTools(e){return(e.tools??[]).map(e=>e.type===`function`?{description:e.description,inputSchema:e.inputSchema,name:e.name}:{name:e.name})}function extractToolResults(e){return e.prompt.flatMap(e=>typeof e.content==`string`?[]:e.content.flatMap(e=>e.type===`tool-result`?[{id:e.toolCallId,isError:e.output.type===`error-json`||e.output.type===`error-text`||e.output.type===`execution-denied`,name:e.toolName,output:normalizeToolOutput(e.output)}]:[]))}function normalizeToolOutput(e){switch(e.type){case`error-json`:case`error-text`:case`json`:case`text`:return e.value;case`execution-denied`:return e.reason??`Tool execution denied`;case`content`:return e.value}}function createGenerateResult(e,t,n){let r=typeof e==`string`?{text:e}:e,i=r.toolCalls??[];if(!(`text`in r)&&i.length===0)throw Error(`mockModel responders must return text or at least one item in "toolCalls".`);let a=[];r.text!==void 0&&a.push({text:r.text,type:`text`});for(let[e,n]of i.entries())a.push({input:JSON.stringify(n.input??{}),toolCallId:n.id??`mock-tool-call-${countUserMessages(t)}-${countToolResults(t)}-${e+1}`,toolName:n.name,type:`tool-call`});let o=t.prompt.map(e=>extractMessageText(e)).join(` `),s=[r.text??``,...i.map(e=>formatValue(e.input??{}))].join(` `),c=r.usage?.inputTokens??estimateTokens(o),l=r.usage?.outputTokens??estimateTokens(s);return{content:a,finishReason:{raw:void 0,unified:i.length>0?`tool-calls`:`stop`},response:{id:`mock-response-${countUserMessages(t)}-${countToolResults(t)}`,modelId:n,timestamp:new Date(RESPONSE_TIMESTAMP)},usage:{inputTokens:{cacheRead:0,cacheWrite:0,noCache:c,total:c},outputTokens:{reasoning:0,text:l,total:l}},warnings:[]}}function createStreamResult(e){let t=[{type:`stream-start`,warnings:e.warnings}];e.response!==void 0&&t.push({...e.response,type:`response-metadata`});let n=0;for(let r of e.content){if(r.type===`text`){let e=`mock-text-${n}`;n+=1,t.push({id:e,type:`text-start`}),r.text.length>0&&t.push({delta:r.text,id:e,type:`text-delta`}),t.push({id:e,type:`text-end`});continue}r.type===`tool-call`&&t.push(r)}return t.push({finishReason:e.finishReason,type:`finish`,usage:e.usage}),{stream:new ReadableStream({start(e){for(let n of t)e.enqueue(n);e.close()}})}}function countUserMessages(e){return e.prompt.filter(e=>e.role===`user`).length}function countToolResults(e){return extractToolResults(e).length}function estimateTokens(e){return Math.max(1,Math.ceil(e.trim().length/4))}function formatValue(e){if(typeof e==`string`)return e;try{return JSON.stringify(e)??String(e)}catch{return String(e)}}export{mockModel};
+import{contextStorage}from"#context/container.js";import{SessionIdKey}from"#context/keys.js";import{markMockModel}from"#internal/mock-model-identity.js";import{MockLanguageModelV4}from"ai/test";import{TASKS_NOTE_LABEL,TASK_RESULT_TAG}from"#execution/tasks/render.js";import{isPendingApprovalsSnippet}from"#harness/hitl/approval-prompt.js";const RESPONSE_TIMESTAMP=Date.parse(`2026-01-01T00:00:00.000Z`);let derivedCallIds=0;const emittedCallIds=new Map;function sessionCallId(e){let t=contextStorage.getStore()?.get(SessionIdKey);if(t===void 0)return e;let n=emittedCallIds.get(t);n===void 0&&(n=new Map,emittedCallIds.set(t,n));let r=(n.get(e)??0)+1;return n.set(e,r),r===1?e:`${e}#${String(r)}`}function mockModel(e={}){let t=normalizeOptions(e),n=normalizeResponder(t.respond),r=t.modelId??`model`;return markMockModel(new MockLanguageModelV4({modelId:r,provider:t.provider??`eve-mock`,doGenerate:async e=>createGenerateResult(await n(createRequest(e)),e,r),doStream:async e=>createStreamResult(createGenerateResult(await n(createRequest(e)),e,r))}))}function normalizeOptions(e){return typeof e==`string`||typeof e==`function`?{respond:e}:e}function normalizeResponder(e){if(typeof e==`function`)return e;let t=e??`Mock response`;return()=>t}function createRequest(e){let t=e.prompt.map(e=>({role:e.role,text:extractMessageText(e)})),n=t.filter(e=>e.role===`user`).map(e=>e.text).filter(e=>!isFrameworkScaffolding(e));return{lastUserMessage:n.at(-1)??null,messages:t,toolResults:extractToolResults(e),tools:extractTools(e),userMessageCount:n.length,userMessages:n}}function isFrameworkScaffolding(e){let t=e.trim();return t.startsWith(TASKS_NOTE_LABEL)||t.startsWith(TASK_RESULT_TAG)||isPendingApprovalsSnippet(t)}function extractMessageText(e){return typeof e.content==`string`?e.content:e.content.flatMap(e=>{switch(e.type){case`reasoning`:case`text`:return[e.text];case`tool-result`:return[formatValue(normalizeToolOutput(e.output))];default:return[]}}).join(``)}function extractTools(e){return(e.tools??[]).map(e=>e.type===`function`?{description:e.description,inputSchema:e.inputSchema,name:e.name}:{name:e.name})}function extractToolResults(e){return e.prompt.flatMap(e=>typeof e.content==`string`?[]:e.content.flatMap(e=>e.type===`tool-result`?[{id:e.toolCallId,isError:e.output.type===`error-json`||e.output.type===`error-text`||e.output.type===`execution-denied`,name:e.toolName,output:normalizeToolOutput(e.output)}]:[]))}function normalizeToolOutput(e){switch(e.type){case`error-json`:case`error-text`:case`json`:case`text`:return e.value;case`execution-denied`:return e.reason??`Tool execution denied`;case`content`:return e.value}}function createGenerateResult(e,t,n){let r=typeof e==`string`?{text:e}:e,i=r.toolCalls??[];if(!(`text`in r)&&i.length===0)throw Error(`mockModel responders must return text or at least one item in "toolCalls".`);let a=[];r.text!==void 0&&a.push({text:r.text,type:`text`});for(let[e,n]of i.entries())a.push({input:JSON.stringify(n.input??{}),toolCallId:(n.id===void 0?void 0:sessionCallId(n.id))??`mock-tool-call-${countUserMessages(t)}-${countToolResults(t)}-${e+1}-${++derivedCallIds}`,toolName:n.name,type:`tool-call`});let o=t.prompt.map(e=>extractMessageText(e)).join(` `),s=[r.text??``,...i.map(e=>formatValue(e.input??{}))].join(` `),c=r.usage?.inputTokens??estimateTokens(o),l=r.usage?.outputTokens??estimateTokens(s);return{content:a,finishReason:{raw:void 0,unified:i.length>0?`tool-calls`:`stop`},response:{id:`mock-response-${countUserMessages(t)}-${countToolResults(t)}`,modelId:n,timestamp:new Date(RESPONSE_TIMESTAMP)},usage:{inputTokens:{cacheRead:0,cacheWrite:0,noCache:c,total:c},outputTokens:{reasoning:0,text:l,total:l}},warnings:[]}}function createStreamResult(e){let t=[{type:`stream-start`,warnings:e.warnings}];e.response!==void 0&&t.push({...e.response,type:`response-metadata`});let n=0;for(let r of e.content){if(r.type===`text`){let e=`mock-text-${n}`;n+=1,t.push({id:e,type:`text-start`}),r.text.length>0&&t.push({delta:r.text,id:e,type:`text-delta`}),t.push({id:e,type:`text-end`});continue}r.type===`tool-call`&&t.push(r)}return t.push({finishReason:e.finishReason,type:`finish`,usage:e.usage}),{stream:new ReadableStream({start(e){for(let n of t)e.enqueue(n);e.close()}})}}function countUserMessages(e){return e.prompt.filter(e=>e.role===`user`).length}function countToolResults(e){return extractToolResults(e).length}function estimateTokens(e){return Math.max(1,Math.ceil(e.trim().length/4))}function formatValue(e){if(typeof e==`string`)return e;try{return JSON.stringify(e)??String(e)}catch{return String(e)}}export{mockModel};

package/dist/src/evals/output.d.ts

@@ -0,0 +1,3 @@
+import type { EveEvalTaskResult } from "#evals/types.js";
+/** Structured null is an output, not a reason to fall back to reply text. */
+export declare function outputOf(result: Pick<EveEvalTaskResult, "data" | "reply"> | undefined): unknown;

package/dist/src/evals/output.js

@@ -0,0 +1 @@
+function outputOf(e){return e===void 0?null:e.data===void 0?e.reply??null:e.data}export{outputOf};

package/dist/src/evals/reported-tool-name.js

@@ -1 +1 @@
-import{TASK_CANCEL_TOOL_NAME,TASK_WAIT_TOOL_NAME}from"#protocol/task-tools.js";import{REPLY_TOOL_NAME}from"#protocol/reply-tool.js";import{CALL_TOOL_NAME,SEARCH_TOOL_NAME,SKILL_TOOL_NAME}from"#protocol/catalog-tools.js";const UNREPORTED_TOOL_NAMES=new Map([[CALL_TOOL_NAME,`Calls through ${CALL_TOOL_NAME} are reported under the entry's name; assert on the tool or agent it reaches.`],[SKILL_TOOL_NAME,`Calls through ${SKILL_TOOL_NAME} are reported as skill loads; use t.loadedSkill(name).`],[`eve__execute`,`eve__execute isn't a tool eve registers (it's reserved for code mode); assert on the tool or agent a call reaches, or use t.loadedSkill(name).`]]),RETIRED_TOOL_NAMES=new Map([[`connection_execute`,"assert on the connection tool's own name, `<connection>__<tool>`"],[`connection_search`,`use ${SEARCH_TOOL_NAME}`],[`final_output`,`use ${REPLY_TOOL_NAME}`],[`load_skill`,`use t.loadedSkill(name)`],[`task_cancel`,`use ${TASK_CANCEL_TOOL_NAME}`],[`task_wait`,`use ${TASK_WAIT_TOOL_NAME}`]]);function assertReportedToolName(e,t){let n=UNREPORTED_TOOL_NAMES.get(e);if(n!==void 0)throw TypeError(n);let r=RETIRED_TOOL_NAMES.get(e);if(r!==void 0&&t!==void 0&&!(t.dynamic||t.static.includes(e)))throw TypeError(`Tool name "${e}" was retired; ${r}.`)}export{assertReportedToolName};
+import{TASK_CANCEL_TOOL_NAME,TASK_WAIT_TOOL_NAME}from"#protocol/task-tools.js";import{REPLY_TOOL_NAME}from"#protocol/reply-tool.js";import{CALL_TOOL_NAME,SEARCH_TOOL_NAME,SKILL_TOOL_NAME}from"#protocol/catalog-tools.js";const UNREPORTED_TOOL_NAMES=new Map([[CALL_TOOL_NAME,`Calls through ${CALL_TOOL_NAME} are reported under the entry's name; assert on the tool or agent it reaches.`],[SKILL_TOOL_NAME,`Calls through ${SKILL_TOOL_NAME} are reported as skill loads; use t.calledSkill(name).`],[`eve__execute`,`eve__execute isn't a tool eve registers (it's reserved for code mode); assert on the tool or agent a call reaches, or use t.calledSkill(name).`]]),RETIRED_TOOL_NAMES=new Map([[`connection_execute`,"assert on the connection tool's own name, `<connection>__<tool>`"],[`connection_search`,`use ${SEARCH_TOOL_NAME}`],[`final_output`,`use ${REPLY_TOOL_NAME}`],[`load_skill`,`use t.calledSkill(name)`],[`task_cancel`,`use ${TASK_CANCEL_TOOL_NAME}`],[`task_wait`,`use ${TASK_WAIT_TOOL_NAME}`]]);function assertReportedToolName(e,t){let n=UNREPORTED_TOOL_NAMES.get(e);if(n!==void 0)throw TypeError(n);let r=RETIRED_TOOL_NAMES.get(e);if(r!==void 0&&t!==void 0&&!(t.dynamic||t.static.includes(e)))throw TypeError(`Tool name "${e}" was retired; ${r}.`)}export{assertReportedToolName};

package/dist/src/evals/reporters/index.d.ts

@@ -1,5 +1,5 @@
-export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js";
-export { Console, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js";
-export { Datadog, type DatadogReporterConfig } from "#evals/runner/reporters/datadog.js";
-export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
+export { braintrustReporter, type BraintrustReporterConfig, } from "#evals/runner/reporters/braintrust.js";
+export { consoleReporter, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js";
+export { datadogReporter, type DatadogReporterConfig } from "#evals/runner/reporters/datadog.js";
+export { junitReporter, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
 export type { EvalReporter, EveEvalCompleteContext, EveEvalSessionStartEvent, EveEvalStartEvent, } from "#evals/runner/reporters/types.js";

package/dist/src/evals/reporters/index.js

@@ -1 +1 @@
-import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{Braintrust}from"#evals/runner/reporters/braintrust.js";import{Datadog}from"#evals/runner/reporters/datadog.js";export{Braintrust,Console,Datadog,JUnit};
+import{consoleReporter}from"#evals/runner/reporters/console.js";import{junitReporter}from"#evals/runner/reporters/junit.js";import{braintrustReporter}from"#evals/runner/reporters/braintrust.js";import{datadogReporter}from"#evals/runner/reporters/datadog.js";export{braintrustReporter,consoleReporter,datadogReporter,junitReporter};

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{mkdir,writeFile}from"node:fs/promises";import{dirname,join}from"node:path";function resolveArtifactDirectory(e){let t=new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19);return join(e,`.eve`,`evals`,t)}async function writeArtifacts(e,t){let n=join(e,`evals`);await mkdir(n,{recursive:!0}),await writeFile(join(e,`summary.json`),JSON.stringify(buildSummaryArtifact(t),null,2));let r=t.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(e,`results.jsonl`),`${r}\n`),await Promise.all(t.results.map(async e=>{let t=join(n,artifactBaseName(e));await mkdir(dirname(t),{recursive:!0}),await writeFile(`${t}.json`,JSON.stringify(buildEvalArtifact(e),null,2));let r=e.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(`${t}.events.ndjson`,`${r}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,errored:e.errored,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,attempt:e.attempt,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt}}function artifactBaseName(e){let t=sanitizeArtifactPath(e.id);return e.attempt===void 0?t:`${t}.attempt-${e.attempt}`}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(`${t}.events.ndjson`,`${r}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,outcome:e.result.outcome,assertions:e.assertions.map(e=>({name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,errored:e.errored,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,outcome:e.result.outcome,data:e.result.data,reply:e.result.reply,sessionEnded:e.result.sessionEnded,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,attempt:e.attempt,result:{outcome:e.result.outcome,error:e.result.error,data:e.result.data,reply:e.result.reply,sessionEnded:e.result.sessionEnded,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt}}function artifactBaseName(e){let t=sanitizeArtifactPath(e.id);return e.attempt===void 0?t:`${t}.attempt-${e.attempt}`}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.d.ts

@@ -1,48 +1,16 @@
-import type { MessageStreamEvent } from "#protocol/message.js";
-import type { InputRequest } from "#shared/input.js";
-import type { TokenUsage } from "#shared/token-usage.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
 import type { EveEvalDerivedFacts } from "#evals/types.js";
-/**
- * Options for {@link deriveRunFacts}.
- */
 export interface DeriveRunFactsOptions {
     /** Session id stamped onto every derived tool call, skill load, and subagent call. */
     readonly sessionId?: string;
+    /** Session events through this turn: cumulative usage and reusable task/child metadata. */
+    readonly usageEvents?: readonly SessionStreamEvent[];
 }
 /**
- * Extracts derived execution facts from a completed run's stream events.
- *
- * Tool calls and skill loads pair each `actions.requested` entry with its
- * matching `action.result` by call id. Subagent calls are the calls to agent tasks,
- * paired with their `task.settled` the same way. These facts power checks,
- * scorers, and reporters.
- */
-export declare function deriveRunFacts(events: readonly MessageStreamEvent[], options?: DeriveRunFactsOptions): EveEvalDerivedFacts;
-/**
- * The last event in `events` (in stream order) that carries the session's
- * running usage total (`session.waiting`, `turn.waiting`, `session.failed`, or
- * `session.completed`), with its time, so an eval that captured a session more
- * than once can keep the latest total by time rather than by capture order.
- * `usage` is undefined when that event reported none.
- */
-export declare function latestUsage(events: readonly MessageStreamEvent[]): {
-    readonly usage: TokenUsage | undefined;
-    readonly at: number;
-} | undefined;
-/**
- * Returns empty derived facts, used when a case produced no events
- * (execution errors, transport failures).
+ * What a segment of a session's events did: its tool calls, skill loads and subagent calls, the
+ * requests it opened, its content and models, read from the session's tables as the segment
+ * leaves them. A call counts when the segment requested or settled it, as an approved call
+ * requested in an earlier segment is; a subagent call when the segment started it on its task.
  */
+export declare function deriveRunFacts(events: readonly SessionStreamEvent[], options?: DeriveRunFactsOptions): EveEvalDerivedFacts;
 export declare function createEmptyDerivedFacts(): EveEvalDerivedFacts;
-/**
- * Input requests the last started turn raised, or a child relayed, that nobody
- * has answered yet. A partial answer settles only the requests it answers; the
- * rest stay open.
- *
- * An eval may watch how a request ends outside the session (for example
- * through `t.target`), so two kinds of open request don't count: the session's
- * own requests from earlier turns, and answered ones whose responder still has
- * to sign in. Relayed requests keep the child's `turnId`, so they never match
- * the parent's turn and are kept by `callId`/`taskId` instead.
- */
-export declare function openInputRequests(events: readonly MessageStreamEvent[]): InputRequest[];

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-import{foldSession,initialSessionProjection,openInputs}from"#protocol/session-projection.js";const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=new Map,o=[],s=new Map,c=[],l=[],u=-1,d=0,f=0,p=new Set,m,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(u,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:u+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`subagent-call`||e.kind===`remote-agent-call`?ensureToolCall(e.callId,e.name,e.input):e.kind===`load-skill`&&a.set(e.callId,{output:void 0,sessionId:n,skill:e.name,status:`pending`,turnIndex:Math.max(u,0)});break;case`action.result`:{let{result:e,status:n}=t.data,r=e.kind===`tool-result`?ensureToolCall(e.callId,e.toolName,{}):e.kind===`subagent-result`?i.get(e.callId):void 0;r!==void 0&&(r.output=e.output,r.status=n);let o=e.kind===`load-skill-result`?a.get(e.callId):void 0;o!==void 0&&(o.output=e.output,o.status=n);break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;o.push({callId:e,name:r,taskId:i,turnIndex:Math.max(u,0)});break}case`task.settled`:s.set(t.data.callId,t.data);break;case`agent.started`:c.push(t.data);break;case`input.requested`:l.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(d+=1);break;case`reasoning.completed`:f+=1;break;case`step.started`:p.add(t.data.modelId);break;case`session.failed`:m=t.data.code}let h=deriveSubagentCalls({agentCalls:o,agentSessions:c,sessionId:n,settledTaskCalls:s});return{toolCalls:r,toolCallCount:r.length,skillLoads:[...a.values()],subagentCalls:h,subagentCallCount:h.length,inputRequests:l,parked:endedParkedOnInput(e),messageCount:d,reasoningBlockCount:f,models:[...p],usage:latestUsage(e)?.usage,failureCode:m}}function latestUsage(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(n!==void 0)switch(n.type){case`session.waiting`:case`turn.waiting`:case`session.failed`:return{usage:n.data.usage,at:Date.parse(n.meta.at)};case`session.completed`:return{usage:n.data?.usage,at:Date.parse(n.meta.at)}}}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,skillLoads:[],subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function openInputRequests(e){let t=initialSessionProjection(),n;for(let r of e)t=foldSession(t,r),r.type===`turn.started`&&(n=r.data.turnId);let r=new Set(Object.values(t.candidates).filter(e=>e.outcome===`pending`).map(e=>e.requestId));return openInputs(t).filter(e=>n===void 0||e.turnId===n||e.callId!==void 0||e.taskId!==void 0).filter(e=>!r.has(e.request.requestId)).map(e=>e.request)}function endedParkedOnInput(e){let t=e.at(-1);return t!==void 0&&PARKING_EVENT_TYPES.has(t.type)&&openInputRequests(e).length>0}export{createEmptyDerivedFacts,deriveRunFacts,latestUsage,openInputRequests};
+import{emptySessionView,foldReceivedEvent}from"#protocol/session-projection/fold.js";import{interaction}from"#protocol/session-projection/reader.js";import{callOutputSource}from"#protocol/session-projection/selectors.js";import{isJsonObjectValue}from"#shared/json.js";import{ownerOf}from"#protocol/session-projection/tree.js";function deriveRunFacts(e,t){let n=t?.sessionId,r=emptySessionView(),i=[...t?.usageEvents??e].sort((e,t)=>e.meta.position.line-t.meta.position.line||e.meta.position.index-t.meta.position.index);for(let e of i)foldReceivedEvent(r,e);let a=new Map,o=new Map,s=[],c=new Map,l=new Set,u=0,d=0,f,indexOf=e=>Math.max((e===void 0?void 0:c.get(e))??c.size-1,0);for(let t of e)switch(t.type){case`turn.started`:c.has(t.data.turnId)||c.set(t.data.turnId,c.size);break;case`call.requested`:a.has(t.data.callId)||a.set(t.data.callId,indexOf(t.scope?.turnId));break;case`call.settled`:a.has(t.data.callId)||a.set(t.data.callId,indexOf(void 0));break;case`call.started`:{let{callId:e,taskId:n}=t.data;n!==void 0&&r.tasks[n]?.kind===`agent`&&!o.has(e)&&o.set(e,indexOf(t.scope?.turnId));break}case`interaction.opened`:{let e=interaction(r,t.data.interactionId);e!==void 0&&s.push(e);break}case`content.completed`:t.data.kind===`text`&&t.data.phase===`reply`?u+=1:t.data.kind===`reasoning`&&(d+=1);break;case`model.started`:l.add(t.data.modelId);break;case`turn.settled`:case`session.ended`:t.data.outcome===`failed`&&(f=t.data.error?.code)}let settled=e=>e.status===`settled`?{output:callOutputSource(r,e.callId)?.output,status:e.outcome??`failed`}:{output:void 0,status:`pending`},p=[],m=[];for(let[e,t]of a){let i=r.calls[e];if(i===void 0)continue;if(i.capability.kind===`skill`){m.push({sessionId:n,skill:i.capability.name,turnIndex:t,...settled(i)});continue}let a=isJsonObjectValue(i.input)?i.input:{};p.push({callId:e,input:a,name:i.capability.name,sessionId:n,turnIndex:t,...settled(i)})}let h=[...o].flatMap(([e,t])=>{let i=r.calls[e],a=i?.taskId===void 0?void 0:r.tasks[i.taskId];return i===void 0||a===void 0?[]:[{callId:e,childSessionId:Object.values(r.children).filter(e=>ownerOf(r,{family:`child`,id:e.sessionId},`task`)===a.taskId).at(-1)?.sessionId,name:a.name,sessionId:n,turnIndex:t,...settled(i)}]}),g=Object.keys(r.usage.byKind).length>0?r.usage.total:void 0;return{toolCalls:p,toolCallCount:p.length,skillLoads:m,subagentCalls:h,subagentCallCount:h.length,interactions:s,parked:endedParkedOnInput(e),messageCount:u,reasoningBlockCount:d,models:[...l],usage:g,failureCode:f}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,skillLoads:[],subagentCalls:[],subagentCallCount:0,interactions:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(n?.type===`turn.paused`)return n.data.awaiting.some(e=>`interactionId`in e);if(n?.type===`turn.settled`||n?.type===`session.ended`)return!1}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(e){let{evaluation:t,target:n,client:r}=e,i=e.startedAt??new Date().toISOString(),a,o=[],s,c;try{let i=await executeTask({client:r,evaluation:t,setupContext:e.setupContext,onLog:e.onLog,onSessionStart:e.onSessionStart,target:n,timeoutMs:e.timeoutMs??t.timeoutMs});a=i.result,o=i.assertions,s=i.error,c=i.skipReason}catch(e){s=toErrorMessage(e),a={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts(),traceContexts:[]}}let l=computeEvalVerdict({error:s,assertions:o,skipReason:c});return{id:t.id,result:a,assertions:o,verdict:l,error:s,skipReason:c,startedAt:i,completedAt:new Date().toISOString()}}export{executeEval};
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(e){let{evaluation:t,target:n,client:r}=e,i=e.startedAt??new Date().toISOString(),a,o=[],s,c;try{let i=await executeTask({client:r,evaluation:t,setupContext:e.setupContext,onLog:e.onLog,onSessionStart:e.onSessionStart,target:n,timeoutMs:e.timeoutMs??t.timeoutMs});a=i.result,o=i.assertions,s=i.error,c=i.skipReason}catch(e){s=toErrorMessage(e),a={data:void 0,reply:void 0,sessionEnded:!1,status:`failed`,events:[],derived:createEmptyDerivedFacts(),traceContexts:[]}}let l=computeEvalVerdict({error:s,assertions:o,skipReason:c});return{id:t.id,result:a,assertions:o,verdict:l,error:s,skipReason:c,startedAt:i,completedAt:new Date().toISOString()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -1,5 +1,5 @@
 import type { Client } from "#client/client.js";
-import type { AssertionResult, EveEval, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
+import type { AssertionResult, EveEval, EveEvalDerivedFacts, EveEvalSessionResult, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
 import type { EvalSessionStartedEvent } from "#evals/session.js";
 /**
  * Options for executing one eval's task.
@@ -33,4 +33,6 @@ interface ExecuteTaskResult {
  * against the completed task result.
  */
 export declare function executeTask(options: ExecuteTaskOptions): Promise<ExecuteTaskResult>;
+/** The facts of every session an eval read, each session's handles read as one stream. */
+export declare function combineDerivedFacts(sessions: readonly EveEvalSessionResult[]): EveEvalDerivedFacts;
 export {};

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle,targetTools}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts,latestUsage}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector(targetTools(r)),s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.skillLoads),r=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,skillLoads:n,subagentCalls:r,subagentCallCount:r.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:a}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=new Map;for(let t of e){let e=latestUsage(t.events),r=n.get(t.sessionId);(r===void 0||e!==void 0&&e.at>=r.at)&&n.set(t.sessionId,e??r)}let r=[...n].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n?.usage]);if(r.length===0||!r.every(e=>e!==void 0))return;let i=r.reduce(addTokenUsage);if(r.every(e=>e.costUsd!==void 0))return i;let{costUsd:a,...o}=i;return o}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle,targetTools}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts,deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage-math.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector(targetTools(r)),s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events);return{data:e.turn?.data,outcome:e.turn?.outcome,error:e.turn?.error,reply:e.turn?.reply,view:e.turn?.view,sessionEnded:e.turn?.sessionEnded??!1,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function derivedBySession(e){let t=[],n=new Map;for(let r of e){let e=r.sessionId===void 0?void 0:n.get(r.sessionId);if(e!==void 0){e.push(r);continue}let i=[r];t.push(i),r.sessionId!==void 0&&n.set(r.sessionId,i)}return t.map(e=>{if(e.length===1)return e[0].derived;let t=new Map;for(let n of e)for(let e of n.events){let{index:n,line:r}=e.meta.position;t.set(`${r}:${n}`,e)}let n=[...t.values()].sort((e,t)=>e.meta.position.line-t.meta.position.line||e.meta.position.index-t.meta.position.index);return deriveRunFacts(n,{sessionId:e[0].sessionId})})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=derivedBySession(e),n=t.flatMap(e=>e.toolCalls),r=t.flatMap(e=>e.skillLoads),i=t.flatMap(e=>e.subagentCalls),a=t.flatMap(e=>e.interactions),o=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:n,toolCallCount:n.length,skillLoads:r,subagentCalls:i,subagentCallCount:i.length,interactions:a,parked:t.some(e=>e.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:o}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`child.opened`?[e.data.sessionId]:[]))),n=new Map;for(let t of e){let e=lastUsageRecordedAt(t.events)??-1/0,r=n.get(t.sessionId);(r===void 0||e>=r.at)&&n.set(t.sessionId,{at:e,usage:t.derived.usage})}let r=[...n].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n.usage]);if(r.length===0||!r.every(e=>e!==void 0))return;let i=r.reduce(addTokenUsage);if(r.every(e=>e.costUsd!==void 0))return i;let{costUsd:a,...o}=i;return o}function lastUsageRecordedAt(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(n?.type===`usage.recorded`)return Date.parse(n.meta.at)}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{combineDerivedFacts,executeTask};

package/dist/src/evals/runner/reporters/braintrust.d.ts

@@ -26,4 +26,4 @@ export interface BraintrustReporterConfig {
  * missing. `config` is optional and defaults to an empty object (the project
  * name then falls back to the first observed eval id).
  */
-export declare function Braintrust(config?: BraintrustReporterConfig): EvalReporter;
+export declare function braintrustReporter(config?: BraintrustReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,exportedAssertionName);for(let[e,t]of r)t.score!==void 0&&(n[e]=t.score);let i=e.assertions.filter(e=>e.passed===!1).map(e=>({...e})),a={...t?.metadata,eveAttempt:e.attempt,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:formatAttemptLabel(e.id,e.attempt),input:t?.description??``,output:e.result.output??``,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
-`))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};
+import{outputOf}from"#evals/output.js";import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";function braintrustReporter(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,exportedAssertionName);for(let[e,t]of r)t.score!==void 0&&(n[e]=t.score);let i=e.assertions.filter(e=>e.passed===!1).map(e=>({...e})),a={...t?.metadata,eveAttempt:e.attempt,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveAwaitingInput:e.result.status===`awaiting-input`,eveSessionEnded:e.result.sessionEnded,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:formatAttemptLabel(e.id,e.attempt),input:t?.description??``,output:outputOf(e.result)??``,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+`))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{braintrustReporter};

package/dist/src/evals/runner/reporters/console.d.ts

@@ -11,4 +11,4 @@ export interface ConsoleReporterConfig {
 /**
  * Creates a {@link ConsoleReporter}.
  */
-export declare function Console(config?: ConsoleReporterConfig): EvalReporter;
+export declare function consoleReporter(config?: ConsoleReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import pc from"#compiled/picocolors/index.js";import{exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureDetailLines,formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function Console(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(e){this.#e=e?.log??console.log,this.#t=pc.createColors(e?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.flatMap(e=>e.score===void 0?[]:[this.#i(e.name,e.score)]).join(`  `),u=formatAttemptLabel(e.id,e.attempt),d=[s,this.#t.dim(u),c,l].filter(Boolean).join(`  `);this.#e(d);for(let e of t)if(e.passed===!1){this.#e(`  ${this.#t.red(`✗ ${formatAssertionFailureHeadline(e)}`)}`);for(let t of formatAssertionFailureDetailLines(e))this.#e(`    ${this.#t.red(t)}`)}for(let e of groupAssertionScores(o,exportedAssertionName).values()){if(e.assertions.every(t=>t.score===e.score))continue;let t=e.assertions[0].name;this.#e(`  ${this.#t.yellow(`⚠ ${t} recorded ${e.assertions.length} times with different scores; Braintrust and Datadog keep only the lowest. Add .label() to tell them apart.`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=aggregateAttempts(a);if(l.length>0){this.#e(``),this.#e(this.#t.bold(`Repeatability:`));for(let{id:e,passed:t,total:n}of l){let r=`${t}/${n} passed`,i=t===n?this.#t.green(r):t===0?this.#t.red(r):this.#t.yellow(r);this.#e(`  ${i}  ${this.#t.dim(e)}`)}}let u=this.#o(a);if(u.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of u){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let d=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(d)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e){let e=n.assertions.filter(e=>e.severity===`soft`);for(let[n,r]of groupAssertionScores(e,e=>e.name)){if(r.score===void 0)continue;let e=t.get(n);e?(e.sum+=r.score,e.count+=1):t.set(n,{sum:r.score,count:1})}}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function aggregateAttempts(e){let t=new Map;for(let n of e){if(n.attempt===void 0||n.verdict!==`passed`&&n.verdict!==`failed`)continue;let e=t.get(n.id)??{passed:0,total:0};e.total+=1,n.verdict===`passed`&&(e.passed+=1),t.set(n.id,e)}return[...t.entries()].map(([e,t])=>({id:e,...t}))}function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{Console};
+import pc from"#compiled/picocolors/index.js";import{exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureDetailLines,formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function consoleReporter(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(e){this.#e=e?.log??console.log,this.#t=pc.createColors(e?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.flatMap(e=>e.score===void 0?[]:[this.#i(e.name,e.score)]).join(`  `),u=formatAttemptLabel(e.id,e.attempt),d=[s,this.#t.dim(u),c,l].filter(Boolean).join(`  `);this.#e(d);for(let e of t)if(e.passed===!1){this.#e(`  ${this.#t.red(`✗ ${formatAssertionFailureHeadline(e)}`)}`);for(let t of formatAssertionFailureDetailLines(e))this.#e(`    ${this.#t.red(t)}`)}for(let e of groupAssertionScores(o,exportedAssertionName).values()){if(e.assertions.every(t=>t.score===e.score))continue;let t=e.assertions[0].name;this.#e(`  ${this.#t.yellow(`⚠ ${t} recorded ${e.assertions.length} times with different scores; Braintrust and Datadog keep only the lowest. Add .label() to tell them apart.`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=aggregateAttempts(a);if(l.length>0){this.#e(``),this.#e(this.#t.bold(`Repeatability:`));for(let{id:e,passed:t,total:n}of l){let r=`${t}/${n} passed`,i=t===n?this.#t.green(r):t===0?this.#t.red(r):this.#t.yellow(r);this.#e(`  ${i}  ${this.#t.dim(e)}`)}}let u=this.#o(a);if(u.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of u){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let d=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(d)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e){let e=n.assertions.filter(e=>e.severity===`soft`);for(let[n,r]of groupAssertionScores(e,e=>e.name)){if(r.score===void 0)continue;let e=t.get(n);e?(e.sum+=r.score,e.count+=1):t.set(n,{sum:r.score,count:1})}}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function aggregateAttempts(e){let t=new Map;for(let n of e){if(n.attempt===void 0||n.verdict!==`passed`&&n.verdict!==`failed`)continue;let e=t.get(n.id)??{passed:0,total:0};e.total+=1,n.verdict===`passed`&&(e.passed+=1),t.set(n.id,e)}return[...t.entries()].map(([e,t])=>({id:e,...t}))}function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{consoleReporter};

package/dist/src/evals/runner/reporters/datadog.d.ts

@@ -45,4 +45,4 @@ export interface DatadogReporterConfig {
  * package and `DD_API_KEY`/`DD_APP_KEY` credentials unless `config.client` is
  * provided.
  */
-export declare function Datadog(config?: DatadogReporterConfig): EvalReporter;
+export declare function datadogReporter(config?: DatadogReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/datadog.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{resolveRuntimeTraceLinks}from"#evals/runner/reporters/datadog-runtime-trace-links.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
-`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[],s=new Set;for(let t of e){let e=this.#l(t.id);if(s.has(e))continue;s.add(e);let n=this.#u(t),r=a.get(e);if(r===void 0){o.push({inputData:n.input,metadata:n.metadata,...n.expectedOutput===null?{}:{expectedOutput:n.expectedOutput}});continue}let c=i.records()[r];c&&!isDatasetRecordEqual(c,n)&&i.update(r,n)}o.length>0&&i.addRecords(o),await i.push();let c=i.id();if(!c)throw Error(`Datadog dataset '${r}' has no id after push().`);let l=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&l.set(t,e.id)}let u={id:c,name:i.name()},d=i.version()??i.latestVersion();d!==null&&(u.version=d),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:u});for(let t of e){let e=this.#l(t.id),n=l.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t),e.attempt!==void 0&&(r.runIteration=e.attempt);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>e.passed===!1).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(exportedAssertionName(e)));for(let[e,i]of r){if(i.score===void 0)continue;let r=i.assertions.flatMap(e=>e.passed===void 0?[]:[e.passed]);t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,...r.length>0&&{assertion_passed:String(r.every(e=>e))},assertion_count:String(i.assertions.length)}})}return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{Datadog};
+import{outputOf}from"#evals/output.js";import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{resolveRuntimeTraceLinks}from"#evals/runner/reporters/datadog-runtime-trace-links.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function datadogReporter(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
+`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[],s=new Set;for(let t of e){let e=this.#l(t.id);if(s.has(e))continue;s.add(e);let n=this.#u(t),r=a.get(e);if(r===void 0){o.push({inputData:n.input,metadata:n.metadata,...n.expectedOutput===null?{}:{expectedOutput:n.expectedOutput}});continue}let c=i.records()[r];c&&!isDatasetRecordEqual(c,n)&&i.update(r,n)}o.length>0&&i.addRecords(o),await i.push();let c=i.id();if(!c)throw Error(`Datadog dataset '${r}' has no id after push().`);let l=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&l.set(t,e.id)}let u={id:c,name:i.name()},d=i.version()??i.latestVersion();d!==null&&(u.version=d),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:u});for(let t of e){let e=this.#l(t.id),n=l.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(outputOf(e.result))),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t),e.attempt!==void 0&&(r.runIteration=e.attempt);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`delivery.consumed`)return t.data.parts;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveAwaitingInput:e.result.status===`awaiting-input`,eveSessionEnded:e.result.sessionEnded});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>e.passed===!1).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(exportedAssertionName(e)));for(let[e,i]of r){if(i.score===void 0)continue;let r=i.assertions.flatMap(e=>e.passed===void 0?[]:[e.passed]);t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,...r.length>0&&{assertion_passed:String(r.every(e=>e))},assertion_count:String(i.assertions.length)}})}return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{datadogReporter};

package/dist/src/evals/runner/reporters/junit.d.ts

@@ -7,4 +7,4 @@ export interface JUnitReporterConfig {
  * Creates a reporter that writes one JUnit XML testsuite for an eval run.
  * Each eval becomes one `<testcase>` named by its path-derived id.
  */
-export declare function JUnit(config: JUnitReporterConfig): EvalReporter;
+export declare function junitReporter(config: JUnitReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/junit.js

@@ -1,4 +1,4 @@
-import{mkdir,writeFile}from"node:fs/promises";import{dirname}from"node:path";import{truncateDiagnostic}from"#evals/diagnostics.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(e){let t=renderJUnit(e,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,t)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+import{mkdir,writeFile}from"node:fs/promises";import{dirname}from"node:path";import{truncateDiagnostic}from"#evals/diagnostics.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function junitReporter(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(e){let t=renderJUnit(e,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,t)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
 `)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(formatAttemptLabel(e.id,e.attempt))}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
 `);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
-`)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.filter(e=>e.passed===!1);return t.length>0?truncateDiagnostic(t.map(formatAssertionFailureHeadline).join(`; `),1e3):e.verdict===`scored`?`score below threshold`:e.verdict}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};
+`)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.filter(e=>e.passed===!1);return t.length>0?truncateDiagnostic(t.map(formatAssertionFailureHeadline).join(`; `),1e3):e.verdict===`scored`?`score below threshold`:e.verdict}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{junitReporter};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -1,4 +1,4 @@
-import type { RuntimeTraceContext } from "#protocol/message.js";
+import type { TraceContext } from "#protocol/session-events/families/session.js";
 import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget, EveEvalTraceContext } from "#evals/types.js";
 /** Context delivered when one eval is scheduled for execution. */
 export interface EveEvalStartEvent {
@@ -12,7 +12,7 @@ export interface EveEvalStartEvent {
 export interface EveEvalSessionStartEvent extends EveEvalStartEvent {
     readonly primary: boolean;
     readonly sessionId: string;
-    readonly traceContext: RuntimeTraceContext;
+    readonly traceContext: TraceContext;
 }
 /** Additional run context delivered with a completed eval result. */
 export interface EveEvalCompleteContext {

package/dist/src/evals/runner/run-evals.d.ts

@@ -38,7 +38,7 @@ export interface RunEvalsOptions {
  *
  * Run-level reporters observe every eval. Eval-defined reporters observe
  * only the evals that reference them; a reporter instance shared by several
- * evals (e.g. one `Braintrust()` passed to every entry of an array export)
+ * evals (e.g. one `braintrustReporter()` passed to every entry of an array export)
  * is deduplicated and observes all of its evals as one group.
  */
 export declare function runEvals(options: RunEvalsOptions): Promise<EveEvalRunSummary>;

package/dist/src/evals/session-cleanup.d.ts

@@ -1,5 +1,5 @@
 interface CleanupSession {
-    readonly sessionId: string | undefined;
+    readonly id: string | undefined;
     cleanup(signal: AbortSignal): Promise<void>;
 }
 /** Settles each distinct eval-owned root once. */

package/dist/src/evals/session-cleanup.js

@@ -1 +1 @@
-async function cleanupEvalSessions(e,t){let n=new Map;for(let t of e){let e=t.sessionId;e!==void 0&&!n.has(e)&&n.set(e,t)}return await Promise.allSettled([...n.values()].map(async e=>await e.cleanup(t)))}export{cleanupEvalSessions};
+async function cleanupEvalSessions(e,t){let n=new Map;for(let t of e){let e=t.id;e!==void 0&&!n.has(e)&&n.set(e,t)}return await Promise.allSettled([...n.values()].map(async e=>await e.cleanup(t)))}export{cleanupEvalSessions};

package/dist/src/evals/session-content.d.ts

@@ -1,5 +1,5 @@
-import type { MessageStreamEvent } from "#protocol/message.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
 /** Formats the user and assistant messages observed in one eval session. */
-export declare function formatEvalTranscript(events: readonly MessageStreamEvent[]): string;
+export declare function formatEvalTranscript(events: readonly SessionStreamEvent[]): string;
 /** Infers the media type used when an eval attaches a local file. */
 export declare function inferMediaType(filePath: string): string;

package/dist/src/evals/session-content.js

@@ -1,3 +1,4 @@
-import{extname}from"node:path";function formatEvalTranscript(e){let t=[];for(let n of e)n.type===`message.received`?t.push(`User:\n${n.data.message}`):n.type===`message.completed`&&t.push(`Assistant:\n${n.data.message}`);return t.join(`
+import{extname}from"node:path";import{replyTextOf}from"#public/channels/reply.js";function formatEvalTranscript(e){let t=[],n;for(let r of e){let e=r.meta?.position?.line;if((r.type!==`delivery.consumed`||n?.line!==e)&&(n=void 0),r.type===`delivery.consumed`){n??={at:t.length,line:e};let i=(r.data.history??[]).map(({role:e,text:t})=>`${e===`user`?`User`:`Assistant`}:\n${t}`);t.splice(n.at,0,...i),n.at+=i.length;let a=r.data.parts.flatMap(e=>e.kind===`text`?[e.text]:[]);a.length>0&&t.push(`User:\n${a.join(`
+`)}`)}else if(r.type===`content.completed`){let e=replyTextOf(r.data);e!==void 0&&t.push(`Assistant:\n${e}`)}}return t.join(`
 
 `)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{formatEvalTranscript,inferMediaType};

package/dist/src/evals/session-manager.d.ts

@@ -2,7 +2,7 @@ import type { CreateSessionOptions, SendTurnInput, SendTurnOptions } from "#clie
 import type { Client } from "#client/client.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
 import { EvalSessionDriver, type EvalSessionStartedEvent } from "#evals/session.js";
-import type { EveEvalLiveTurn, EveEvalSessionResult } from "#evals/types.js";
+import type { EveEvalSessionResult } from "#evals/types.js";
 export declare class EvalSessionManager {
     #private;
     constructor(input: {
@@ -15,12 +15,9 @@ export declare class EvalSessionManager {
     /** A recorded stub failure always fails the eval, even if the agent recovers. */
     verifyStubs(): Promise<void>;
     send(message: SendTurnInput["message"], options?: SendTurnOptions): Promise<import("#evals/types.js").EveEvalTurn>;
-    attachSession(sessionId: string, options?: {
+    attach(sessionId: string, options?: {
         readonly startIndex?: number;
-    }): Promise<EvalSessionDriver>;
-    watchTurn(sessionId: string, options?: {
-        readonly startIndex?: number;
-    }): EveEvalLiveTurn;
+    }): EvalSessionDriver;
     snapshots(): readonly EveEvalSessionResult[];
     lastTurnSession(): EvalSessionDriver | undefined;
     hasActivity(): boolean;

package/dist/src/evals/session-manager.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{createEveSessionStubsRoutePath}from"#protocol/routes.js";import{getMessageResponseReceipt}from"#client/message-response.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{EvalSessionDriver}from"#evals/session.js";import{cleanupEvalSessions}from"#evals/session-cleanup.js";const log=createLogger(`eve.evals`);var EvalSessionManager=class{#e;#t;#n;#r;#i=[];#a=[];#o;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#r=e.onSessionStart,this.#t=e.signal}async session(e={}){let t=Date.now(),{session:n}=await this.#e.sessions.create({...e,signal:e.signal??this.#t}),r={startedAt:t,notReadyMs:0,acceptedAt:Date.now()};e.stubs!==void 0&&this.#a.push({sessionId:n.state.sessionId,headers:e.headers,ruleIds:e.stubs.map(e=>e.id)});let i=this.#s(n);return i.recordSend(`prewarm`,r),i}async verifyStubs(){for(let e of this.#a){let t=await this.#e.fetch(createEveSessionStubsRoutePath(e.sessionId),{headers:e.headers,signal:this.#t});if(!t.ok)throw Error(`Could not verify tool stubs for session ${e.sessionId} (HTTP ${t.status}).`);let n=await t.json();if(typeof n!=`object`||!n||!(`error`in n))throw Error(`Invalid tool stub verification response.`);if(n.error!==null)throw Error(`Tool stubbing failed: ${String(n.error)}`);if(!(`matchedRuleIds`in n)||!Array.isArray(n.matchedRuleIds)||!n.matchedRuleIds.every(e=>typeof e==`string`))throw Error(`Invalid tool stub verification response.`);let r=n.matchedRuleIds,i=e.ruleIds.filter(e=>!r.includes(e));i.length>0&&log.warn(`Tool stubs did not match any calls: ${i.join(`, `)}.`,{sessionId:e.sessionId})}}async send(e,t={}){let{session:n,response:r}=await this.#e.sessions.create({turnPolicy:`queue`,...t,message:e,signal:t.signal??this.#t}),i=this.#s(n);return i.recordSend(`create`,getMessageResponseReceipt(r)),await i.consume(r,e).result()}async attachSession(e,t){let n=this.#c(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#c(e,t).watchTurn(t)}snapshots(){return this.#i.map(e=>e.snapshot())}lastTurnSession(){return this.#o}hasActivity(){return this.#i.length>0}async cleanup(e){return await cleanupEvalSessions(this.#i,e)}#s(e){let t=new EvalSessionDriver({collector:this.#n,onSessionStart:this.#r,onTurn:e=>{this.#o=e},primary:this.#i.length===0,session:e,signal:this.#t});return this.#i.push(t),t}#c(e,t){return this.#s(this.#e.sessions.attach(e,{streamIndex:t?.startIndex??0}))}};export{EvalSessionManager};
+import{createLogger}from"#internal/logging.js";import{getDeliveryReceipt}from"#client/delivery.js";import{createEveSessionStubsRoutePath}from"#protocol/routes.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{EvalSessionDriver}from"#evals/session.js";import{cleanupEvalSessions}from"#evals/session-cleanup.js";const log=createLogger(`eve.evals`);var EvalSessionManager=class{#e;#t;#n;#r;#i=[];#a=[];#o;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#r=e.onSessionStart,this.#t=e.signal}async session(e={}){let t=Date.now(),{session:n}=await this.#e.sessions.create({...e,signal:e.signal??this.#t}),r={startedAt:t,notReadyMs:0,acceptedAt:Date.now()};e.stubs!==void 0&&this.#a.push({sessionId:n.state.sessionId,headers:e.headers,ruleIds:e.stubs.map(e=>e.id)});let i=this.#s(n);return i.recordSend(`prewarm`,r),i}async verifyStubs(){for(let e of this.#a){let t=await this.#e.fetch(createEveSessionStubsRoutePath(e.sessionId),{headers:e.headers,signal:this.#t});if(!t.ok)throw Error(`Could not verify tool stubs for session ${e.sessionId} (HTTP ${t.status}).`);let n=await t.json();if(typeof n!=`object`||!n||!(`error`in n))throw Error(`Invalid tool stub verification response.`);if(n.error!==null)throw Error(`Tool stubbing failed: ${String(n.error)}`);if(!(`matchedRuleIds`in n)||!Array.isArray(n.matchedRuleIds)||!n.matchedRuleIds.every(e=>typeof e==`string`))throw Error(`Invalid tool stub verification response.`);let r=n.matchedRuleIds,i=e.ruleIds.filter(e=>!r.includes(e));i.length>0&&log.warn(`Tool stubs did not match any calls: ${i.join(`, `)}.`,{sessionId:e.sessionId})}}async send(e,t={}){let{session:n,delivery:r}=await this.#e.sessions.create({turnPolicy:`queue`,...t,message:e,signal:t.signal??this.#t}),i=this.#s(n);return i.recordSend(`create`,getDeliveryReceipt(r)),await i.consume(r.stream({signal:t.signal??this.#t}),e,{deliveryId:r.deliveryId}).result()}attach(e,t){return this.#c(e,t)}snapshots(){return this.#i.map(e=>e.snapshot())}lastTurnSession(){return this.#o}hasActivity(){return this.#i.length>0}async cleanup(e){return await cleanupEvalSessions(this.#i,e)}#s(e,t){let n=new EvalSessionDriver({collector:this.#n,history:t,onSessionStart:this.#r,onTurn:e=>{this.#o=e},primary:this.#i.length===0,session:e,signal:this.#t});return this.#i.push(n),n}#c(e,t){let n=t?.startIndex??0,r=this.#i.filter(t=>t.id===e).flatMap(e=>e.events).filter(e=>e.meta.position.line<n);return this.#s(this.#e.sessions.attach(e,{streamIndex:n}),r)}};export{EvalSessionManager};

package/dist/src/evals/session.d.ts

@@ -1,16 +1,21 @@
-import { type ClientSendReceipt } from "#client/message-response.js";
-import type { ClientSession } from "#client/session.js";
-import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions } from "#client/types.js";
-import type { MessageStreamEvent, RuntimeTraceContext, AgentStartedStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
-import type { InputRequest, InputResponse } from "#shared/input.js";
+import type { Delivery } from "#shared/delivery.js";
+import { type SessionView } from "#protocol/session-projection/public-view.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
+import { type ClientSendReceipt } from "#client/delivery.js";
+import { type ClientSession } from "#client/session.js";
+import type { ClientSessionState, SendTurnInput, SendTurnOptions, RespondTurnOptions } from "#client/types.js";
+import type { TraceContext } from "#protocol/session-events/families/session.js";
+import type { Answer } from "#shared/input.js";
+import type { Interaction } from "#protocol/session-projection/reader.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
-import type { EveEvalAgentSession, EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSendTiming, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
-import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
+import type { EveEvalAgentSession, EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSendTiming, EveEvalSession, EveEvalSessionResult, EveEvalTurn, EveEvalWatchOptions } from "#evals/types.js";
+import type { EveEvalInteractionMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
  */
 export declare class EveEvalTurnFailedError extends Error {
-    readonly event: (TurnFailureStreamEvent & MessageStreamEvent) | undefined;
+    /** The `turn.settled` or `session.ended` that reported the failure. */
+    readonly event: SessionStreamEvent | undefined;
     readonly turn: EveEvalTurn;
     constructor(turn: EveEvalTurn);
 }
@@ -20,7 +25,7 @@ export interface EvalSessionStartedEvent {
     readonly primary: boolean;
     readonly sessionId: string;
     readonly startedAt: string;
-    readonly traceContext: RuntimeTraceContext;
+    readonly traceContext: TraceContext;
 }
 export declare class EvalSessionDriver implements EveEvalSession {
     #private;
@@ -31,35 +36,39 @@ export declare class EvalSessionDriver implements EveEvalSession {
         readonly session: ClientSession;
         readonly onTurn: (session: EvalSessionDriver) => void;
         readonly signal?: AbortSignal;
+        /** What this eval already read of the session before this driver attached. */
+        readonly history?: readonly SessionStreamEvent[];
     });
-    get events(): readonly MessageStreamEvent[];
+    get events(): readonly SessionStreamEvent[];
+    get view(): SessionView;
     get transcript(): string;
     get lastInput(): string;
     get lastTurn(): EveEvalTurn | undefined;
-    get pendingInputRequests(): readonly InputRequest[];
-    get sessionId(): string;
+    get pendingInteractions(): readonly Interaction[];
+    get id(): string;
     get state(): ClientSessionState;
-    cancel(): Promise<CancelSessionResult>;
+    cancel(): Promise<Delivery>;
     compact(): Promise<EveEvalTurn>;
-    agent(started: AgentStartedStreamEvent): EveEvalAgentSession;
+    agent(opened: Extract<SessionStreamEvent, {
+        type: "child.opened";
+    }>): EveEvalAgentSession;
     /** @internal */
     cleanup(signal: AbortSignal): Promise<void>;
-    requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
-    respond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalTurn>;
-    startRespond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
+    requireInteraction(filter?: EveEvalInteractionMatchOptions): Interaction;
+    respond(responses: readonly Answer[], options?: RespondTurnOptions): Promise<EveEvalTurn>;
+    startRespond(responses: readonly Answer[], options?: RespondTurnOptions): Promise<EveEvalLiveTurn>;
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(message: SendTurnInput["message"], options?: SendTurnOptions): Promise<EveEvalTurn>;
     start(message: string, options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
     /** @internal Keeps when a request to this session was sent and accepted. */
     recordSend(kind: EveEvalSendTiming["kind"], receipt: ClientSendReceipt | undefined): void;
     /** @internal */
-    consume(events: AsyncIterable<MessageStreamEvent>, message?: SendTurnInput["message"]): EveEvalLiveTurn;
-    sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
-    readTurn(options?: {
-        readonly startIndex?: number;
-    }): Promise<EveEvalTurn>;
-    watchTurn(options?: {
-        readonly startIndex?: number;
+    consume(events: AsyncIterable<SessionStreamEvent>, message?: SendTurnInput["message"], ending?: {
+        readonly deliveryId?: string;
+        readonly endsWith?: (event: SessionStreamEvent) => boolean;
     }): EveEvalLiveTurn;
+    sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
+    nextTurn(options?: EveEvalWatchOptions): Promise<EveEvalTurn>;
+    watch(options?: EveEvalWatchOptions): EveEvalLiveTurn;
     snapshot(): EveEvalSessionResult;
 }