eve 0.83.0 → 0.83.1

package/dist/src/context/dynamic-tool-lifecycle.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{contextStorage}from"#context/container.js";import{SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,StepDynamicToolMetadataKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{serializeInputSchema,serializeOutputSchema}from"#tools/schema.js";import{parseJsonObject}from"#shared/json.js";import{assertNotConnectionOwned}from"#connections/ownership.js";import{eveNamespaceReservation}from"#protocol/runtime-tools.js";import{ALLOWED_DYNAMIC_TOOL_EVENTS}from"#dynamic/definition.js";import{isCurrentDynamicToolMetadata,toCurrentDynamicToolMetadataList}from"#context/dynamic-tool-metadata.js";import{clearDurableDynamicCallbacks,hasUnregisteredDurableDynamicCallbacks,readDurableDynamicToolCallbacks,registerDurableDynamicCallback}from"#tools/durable-callbacks.js";import{hasSchemaValidator}from"#tools/durable-schema.js";import{ConnectionRegistryKey}from"#context/providers/connection-key.js";import{TOOL_SLUG_PATTERN,TOOL_SLUG_RULE}from"#discover/grammar.js";import{BundleKey}from"#runtime/sessions/runtime-context-keys.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";import{isFrameworkTool}from"#tools/provided/framework-tool.js";import{isWorkflowToolDefinition}from"#tools/workflow-definition.js";import{isBrandedToolEntry}from"#tools/dynamic.js";import{workflowIdForHandling}from"#runtime/subagents/workflow-reference.js";const log=createLogger(`dynamic-tools`);function qualifyDynamicToolNames(e,t,n){let r=Object.keys(n);if(r.length===0)return[];if(t){let t=r[0];return[{name:e.slug,entryKey:t,entry:n[t]}]}let i=e.extensionNamespace===void 0?``:`${e.extensionNamespace}__`;return r.map(t=>{let r=`${i}${t}`;if(!TOOL_SLUG_PATTERN.test(r))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned illegal tool name "${r}". ${TOOL_SLUG_RULE}`);let a=eveNamespaceReservation(r);if(a!==void 0)throw Error(`Dynamic tool resolver "${e.logicalPath}" returned the reserved tool name "${r}". ${a}; rename the map key.`);return{entry:n[t],entryKey:t,name:r}})}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicToolMetadataKey;case`turn.started`:return TurnDynamicToolMetadataKey;case`step.started`:return StepDynamicToolMetadataKey;default:return}}function readDynamicToolResult(e,t){let assertOrdinaryTool=t=>{if(isWorkflowToolDefinition(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" cannot return defineWorkflowTool(). Workflow tools must be static tools; use defineTool() for dynamic entries.`)};if(assertOrdinaryTool(t),isBrandedToolEntry(t))return{entries:{_single:t},isSingle:!0};if(typeof t!=`object`||!t||Array.isArray(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" must return defineTool(), a map of defineTool() values, or null.`);let n={};for(let[r,i]of Object.entries(t)){if(assertOrdinaryTool(i),!isBrandedToolEntry(i))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned "${r}" without defineTool(). Wrap every dynamic tool entry in defineTool().`);n[r]=i}return{entries:n,isSingle:!1}}function validateReference(e){let t=e.phase===`inputSchema`||e.phase===`outputSchema`,n=t?`Write the schema expression inline in defineTool() or use defineDurableSchema() for a provider package.`:`Author the callback inline in transformed source or use an eve durable callback helper.`;if(e.stamped===void 0){if(e.required)throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" does not have a durable descriptor. `+n);return}let r=Object.keys(e.stamped).filter(e=>e!==`closure`&&e!==`callback`);if(r.includes(`stepId`))throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" was persisted by a pre-release eve version that identified callbacks by build offset. Start a new session to re-resolve it.`);if(r.length>0)throw Error(`Dynamic tool "${e.name}" has invalid ${e.phase} callback metadata: unknown key(s) ${r.join(`, `)}.`);if(typeof e.stamped.callback!=`function`)throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" does not have a durable descriptor. `+n);let i;try{i=parseJsonObject(e.stamped.closure)}catch(n){throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" has a non-serializable capture. ${toErrorMessage(n)}`+(t?` Inline the schema construction or reference a stable module-level schema; put only JSON data in its closure.`:``))}return registerDurableDynamicCallback({callback:e.stamped.callback,phase:e.phase,owner:e.owner}),{closure:i}}function validateDurableDynamicToolCallbacks(e,t,n){let r=readDurableDynamicToolCallbacks(t)??{},i=Object.keys(r).filter(e=>e!==`execute`&&e!==`inputSchema`&&e!==`outputSchema`&&e!==`label`&&e!==`approvalKey`&&e!==`approvalPrompt`&&e!==`approvalRequest`&&e!==`approvalResponse`&&e!==`toModelOutput`);if(i.length>0)throw Error(`Dynamic tool "${e}" has unknown durable callback phase(s): ${i.join(`, `)}.`);let a=t.approval!==void 0,o=t.approval!==void 0&&typeof t.approval!=`function`?t.approval:void 0,s=o?.prompt!==void 0,c=o?.response!==void 0,l=validateReference({name:e,owner:n,phase:`execute`,stamped:r.execute,required:!0}),u=validateReference({name:e,owner:n,phase:`labelComplete`,stamped:r.label?.complete,required:!1}),d=validateReference({name:e,owner:n,phase:`labelDelta`,stamped:r.label?.delta,required:!1}),f=validateReference({name:e,owner:n,phase:`labelStart`,stamped:r.label?.start,required:!1}),p=validateReference({name:e,owner:n,phase:`approvalKey`,stamped:r.approvalKey,required:t.approvalKey!==void 0}),m=validateReference({name:e,owner:n,phase:`approvalPrompt`,stamped:r.approvalPrompt,required:s}),h=validateReference({name:e,owner:n,phase:`approvalRequest`,stamped:r.approvalRequest,required:a}),g=validateReference({name:e,owner:n,phase:`approvalResponse`,stamped:r.approvalResponse,required:c}),_=validateReference({name:e,owner:n,phase:`toModelOutput`,stamped:r.toModelOutput,required:t.toModelOutput!==void 0}),v=validateReference({name:e,owner:n,phase:`inputSchema`,stamped:r.inputSchema,required:hasSchemaValidator(t.inputSchema)}),y=validateReference({name:e,owner:n,phase:`outputSchema`,stamped:r.outputSchema,required:hasSchemaValidator(t.outputSchema)}),b={execute:l};return v!==void 0&&(b.inputSchema=v),y!==void 0&&(b.outputSchema=y),(u!==void 0||d!==void 0||f!==void 0)&&(b.label={complete:u,delta:d,start:f}),p!==void 0&&(b.approvalKey=p),m!==void 0&&(b.approvalPrompt=m),h!==void 0&&(b.approvalRequest=h),g!==void 0&&(b.approvalResponse=g),_!==void 0&&(b.toModelOutput=_),b}function createMetadata(e){return{availableInSubagents:e.entry.availableInSubagents,deferred:e.entry.deferred,callbacks:validateDurableDynamicToolCallbacks(e.name,e.entry,{sessionId:e.sessionId,scope:e.scope,resolverSlug:e.resolver.slug,entryKey:e.entryKey,name:e.name}),description:e.entry.description,frameworkTool:isFrameworkTool(e.entry)||void 0,endsTurn:readDynamicEndsTurn(e.name,e.entry.endsTurn)||void 0,entryKey:e.entryKey,inputSchema:serializeInputSchema(e.entry.inputSchema),name:e.name,outputSchema:serializeOutputSchema(e.entry.outputSchema),resolverSlug:e.resolver.slug}}function readDynamicEndsTurn(e,t){if(typeof t==`function`)throw Error(`Dynamic tool "${e}" sets endsTurn to a function, which dynamic tools do not support. Set endsTurn to true or false; the resolver runs for each event, so it can decide then.`);return t===!0}function workflowToolNames(e){let t=e.get(BundleKey);if(t===void 0)return new Set;let{subagentRegistry:n,toolRegistry:r}=t;return new Set([...[...r.toolsByName.values()].filter(({prepared:e})=>workflowIdForHandling(e.behavior?.handling)!==void 0).map(({prepared:e})=>e.name),...n.preparedTools.map(e=>e.name),...n.dynamicResolvers.map(e=>e.name)])}async function resolveToolsFromEvent(e,t,n,r){let i=e.require(SessionIdKey),a=n.type.split(`.`)[0],o=await Promise.allSettled(t.map(async t=>{let o=t.events[n.type];if(o===void 0)return null;clearDurableDynamicCallbacks(i,{scope:a,resolverSlug:t.slug});try{let s=await o(n,buildResolveContext(e,r));if(s==null)return null;let{entries:c,isSingle:l}=readDynamicToolResult(t,s),u=qualifyDynamicToolNames(t,l,c),d=e.get(ConnectionRegistryKey)?.getConnectionNames()??[],f=workflowToolNames(e);for(let{name:e}of u)if(assertNotConnectionOwned({connectionNames:d,name:e,remedy:`Rename the map key.`,subject:`Dynamic tool`}),f.has(e))throw Error(`Dynamic tool "${e}" from resolver "${t.logicalPath}" collides with the workflow tool or agent "${e}". Rename the map key.`);return{metadata:u.map(({name:e,entryKey:n,entry:r})=>createMetadata({entry:r,entryKey:n,name:e,resolver:t,sessionId:i,scope:a})),resolver:t}}catch(e){throw clearDurableDynamicCallbacks(i,{scope:a,resolverSlug:t.slug}),e}})),s=[],c=new Map;for(let e of o){if(e.status===`rejected`){log.error(`Dynamic tool resolver (${n.type}) failed — skipping its complete result.`,{error:toErrorMessage(e.reason)});continue}if(e.value!==null){for(let t of e.value.metadata){let n=c.get(t.name);if(n!==void 0&&n!==e.value.resolver.slug)throw Error(`Dynamic tool "${t.name}" from resolver "${e.value.resolver.slug}" collides with dynamic resolver "${n}". Namespace the map key manually.`);c.set(t.name,e.value.resolver.slug)}s.push(...e.value.metadata)}}return{metadata:s}}const resolvedStepTools=new WeakMap;function stepCoordinate(e){return`${e.data.turnId}:${String(e.data.stepIndex)}`}function storeResolvedStepTools(e){e.ctx.set(StepDynamicToolMetadataKey,e.metadata);let t=stepCoordinate(e.event);resolvedStepTools.set(e.ctx,{coordinate:t,metadata:e.metadata})}async function resolveStepDynamicTools(e){let t=stepCoordinate(e.event),n=resolvedStepTools.get(e.ctx);if(n?.coordinate===t){e.ctx.set(StepDynamicToolMetadataKey,n.metadata);return}let r=e.resolvers.filter(e=>e.eventNames.includes(`step.started`)),{metadata:i}=r.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,r,e.event,e.messages);storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:i})}async function preparePersistedStepDynamicToolMetadata(e){let t=e.ctx.get(StepDynamicToolMetadataKey)??[],n=t.filter(isCurrentDynamicToolMetadata);if(n.length===t.length&&!hasUnregisteredDurableDynamicCallbacks(n,{sessionId:e.ctx.require(SessionIdKey),scope:`step`})){n.length>0&&storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:n});return}await resolveStepDynamicTools(e);let r=e.ctx.get(StepDynamicToolMetadataKey)??[];storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:toCurrentDynamicToolMetadataList(t,r.filter(isCurrentDynamicToolMetadata))})}async function dispatchDynamicToolEvent(e){if(e.event.type===`session.completed`){let t=e.ctx.get(SessionIdKey);t!==void 0&&clearDurableDynamicCallbacks(t);return}if(!ALLOWED_DYNAMIC_TOOL_EVENTS.has(e.event.type))return;if(e.event.type===`step.started`){await resolveStepDynamicTools({...e,event:e.event});return}e.event.type===`turn.started`&&e.ctx.set(StepDynamicToolMetadataKey,[]);let t=e.resolvers.filter(t=>t.eventNames.includes(e.event.type)),{metadata:n}=t.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,t,e.event,e.messages),r=durableKeyForEvent(e.event.type);if(r===void 0)return;if(e.event.type===`session.started`){e.ctx.set(SessionDynamicToolMetadataKey,n);return}let i=new Set(t.map(e=>e.slug)),a=(e.ctx.get(r)??[]).filter(e=>!i.has(e.resolverSlug));e.ctx.set(r,[...a,...n])}async function refreshDynamicSessionToolsForRuntimeRevision(e){let t=e.ctx.get(SessionDynamicToolMetadataKey)??[],n=t.filter(isCurrentDynamicToolMetadata),r=n.length!==t.length,i=e.ctx.get(SessionDynamicToolRuntimeRevisionKey)!==e.runtimeRevision;if(!i&&!r&&!hasUnregisteredDurableDynamicCallbacks(n,{sessionId:e.ctx.require(SessionIdKey),scope:`session`}))return;let a=e.resolvers.filter(e=>e.eventNames.includes(`session.started`)),{metadata:o}=a.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,a,e.event,e.messages);e.ctx.set(SessionDynamicToolMetadataKey,i?o:toCurrentDynamicToolMetadataList(t,o)),e.ctx.set(SessionDynamicToolRuntimeRevisionKey,e.runtimeRevision)}async function rebindMissingCompiledDynamicToolCallbacks(e){let t=e.ctx.get(TurnDynamicToolMetadataKey)??[],n=t.filter(t=>!isCurrentDynamicToolMetadata(t)||hasUnregisteredDurableDynamicCallbacks([t],{sessionId:e.ctx.require(SessionIdKey),scope:`turn`}));if(n.length===0)return;let r=new Set(n.map(e=>e.resolverSlug)),i=e.resolvers.filter(e=>r.has(e.slug)),a=new Set(i.filter(e=>e.rebindMissingCallbacks===!0).map(e=>e.slug)),o=new Set(t.filter(e=>!isCurrentDynamicToolMetadata(e)).map(e=>e.resolverSlug));if(i.length===0){e.ctx.set(TurnDynamicToolMetadataKey,toCurrentDynamicToolMetadataList(t));return}let s=await contextStorage.run(e.ctx,async()=>await resolveToolsFromEvent(e.ctx,i,e.event,e.messages)),c=toCurrentDynamicToolMetadataList(t,s.metadata);e.ctx.set(TurnDynamicToolMetadataKey,c);let l=c.filter(t=>(a.has(t.resolverSlug)||o.has(t.resolverSlug))&&n.some(e=>e.resolverSlug===t.resolverSlug&&e.name===t.name)&&hasUnregisteredDurableDynamicCallbacks([t],{sessionId:e.ctx.require(SessionIdKey),scope:`turn`}));if(l.length>0)throw Error(`Dynamic tool callback rebind did not restore: ${l.map(e=>e.name).join(`, `)}. The tool may have been renamed or removed.`)}export{dispatchDynamicToolEvent,preparePersistedStepDynamicToolMetadata,rebindMissingCompiledDynamicToolCallbacks,refreshDynamicSessionToolsForRuntimeRevision,resolveStepDynamicTools,validateDurableDynamicToolCallbacks};
+import{createLogger}from"#internal/logging.js";import{contextStorage}from"#context/container.js";import{SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,StepDynamicToolMetadataKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{serializeInputSchema,serializeOutputSchema}from"#tools/schema.js";import{parseJsonObject}from"#shared/json.js";import{assertNotConnectionOwned}from"#connections/ownership.js";import{eveNamespaceReservation}from"#protocol/runtime-tools.js";import{ALLOWED_DYNAMIC_TOOL_EVENTS}from"#dynamic/definition.js";import{isCurrentDynamicToolMetadata,toCurrentDynamicToolMetadataList}from"#context/dynamic-tool-metadata.js";import{clearDurableDynamicCallbacks,hasUnregisteredDurableDynamicCallbacks,readDurableDynamicToolCallbacks,registerDurableDynamicCallback}from"#tools/durable-callbacks.js";import{hasSchemaValidator}from"#tools/durable-schema.js";import{ConnectionRegistryKey}from"#context/providers/connection-key.js";import{TOOL_SLUG_PATTERN,TOOL_SLUG_RULE}from"#discover/grammar.js";import{BundleKey}from"#runtime/sessions/runtime-context-keys.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";import{isFrameworkTool}from"#tools/provided/framework-tool.js";import{isWorkflowToolDefinition}from"#tools/workflow-definition.js";import{isBrandedToolEntry}from"#tools/dynamic.js";import{workflowIdForHandling}from"#runtime/subagents/workflow-reference.js";const log=createLogger(`dynamic-tools`);function qualifyDynamicToolNames(e,t,n){let r=Object.keys(n);if(r.length===0)return[];if(t){let t=r[0];return[{name:e.slug,entryKey:t,entry:n[t]}]}let i=e.extensionNamespace===void 0?``:`${e.extensionNamespace}__`;return r.map(t=>{let r=`${i}${t}`;if(!TOOL_SLUG_PATTERN.test(r))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned illegal tool name "${r}". ${TOOL_SLUG_RULE}`);let a=eveNamespaceReservation(r);if(a!==void 0)throw Error(`Dynamic tool resolver "${e.logicalPath}" returned the reserved tool name "${r}". ${a}; rename the map key.`);return{entry:n[t],entryKey:t,name:r}})}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicToolMetadataKey;case`turn.started`:return TurnDynamicToolMetadataKey;case`step.started`:return StepDynamicToolMetadataKey;default:return}}function readDynamicToolResult(e,t){let assertOrdinaryTool=t=>{if(isWorkflowToolDefinition(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" cannot return defineWorkflowTool(). Workflow tools must be static tools; use defineTool() for dynamic entries.`)};if(assertOrdinaryTool(t),isBrandedToolEntry(t))return{entries:{_single:t},isSingle:!0};if(typeof t!=`object`||!t||Array.isArray(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" must return defineTool(), a map of defineTool() values, or null.`);let n={};for(let[r,i]of Object.entries(t)){if(assertOrdinaryTool(i),!isBrandedToolEntry(i))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned "${r}" without defineTool(). Wrap every dynamic tool entry in defineTool().`);n[r]=i}return{entries:n,isSingle:!1}}function validateReference(e){let t=e.phase===`inputSchema`||e.phase===`outputSchema`,n=t?`Write the schema expression inline in defineTool() or use defineDurableSchema() for a provider package.`:`Author the callback inline in transformed source or use an eve durable callback helper.`;if(e.stamped===void 0){if(e.required)throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" does not have a durable descriptor. `+n);return}let r=Object.keys(e.stamped).filter(e=>e!==`closure`&&e!==`callback`);if(r.includes(`stepId`))throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" was persisted by a pre-release eve version that identified callbacks by build offset. Start a new session to re-resolve it.`);if(r.length>0)throw Error(`Dynamic tool "${e.name}" has invalid ${e.phase} callback metadata: unknown key(s) ${r.join(`, `)}.`);if(typeof e.stamped.callback!=`function`)throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" does not have a durable descriptor. `+n);let i;try{i=parseJsonObject(e.stamped.closure)}catch(n){throw Error(`Dynamic tool "${e.name}" callback "${e.phase}" has a non-serializable capture. ${toErrorMessage(n)}`+(t?` Inline the schema construction or reference a stable module-level schema; put only JSON data in its closure.`:``))}return registerDurableDynamicCallback({callback:e.stamped.callback,phase:e.phase,owner:e.owner}),{closure:i}}function validateDurableDynamicToolCallbacks(e,t,n){let r=readDurableDynamicToolCallbacks(t)??{},i=Object.keys(r).filter(e=>e!==`endsTurn`&&e!==`execute`&&e!==`inputSchema`&&e!==`outputSchema`&&e!==`label`&&e!==`approvalKey`&&e!==`approvalPrompt`&&e!==`approvalRequest`&&e!==`approvalResponse`&&e!==`toModelOutput`);if(i.length>0)throw Error(`Dynamic tool "${e}" has unknown durable callback phase(s): ${i.join(`, `)}.`);let a=t.approval!==void 0,o=t.approval!==void 0&&typeof t.approval!=`function`?t.approval:void 0,s=o?.prompt!==void 0,c=o?.response!==void 0,l=validateReference({name:e,owner:n,phase:`execute`,stamped:r.execute,required:!0}),u=validateReference({name:e,owner:n,phase:`labelComplete`,stamped:r.label?.complete,required:!1}),d=validateReference({name:e,owner:n,phase:`labelDelta`,stamped:r.label?.delta,required:!1}),f=validateReference({name:e,owner:n,phase:`labelStart`,stamped:r.label?.start,required:!1}),p=validateReference({name:e,owner:n,phase:`approvalKey`,stamped:r.approvalKey,required:t.approvalKey!==void 0}),m=validateReference({name:e,owner:n,phase:`approvalPrompt`,stamped:r.approvalPrompt,required:s}),h=validateReference({name:e,owner:n,phase:`approvalRequest`,stamped:r.approvalRequest,required:a}),g=validateReference({name:e,owner:n,phase:`approvalResponse`,stamped:r.approvalResponse,required:c}),_=validateReference({name:e,owner:n,phase:`toModelOutput`,stamped:r.toModelOutput,required:t.toModelOutput!==void 0}),v=validateReference({name:e,owner:n,phase:`endsTurn`,stamped:r.endsTurn,required:typeof t.endsTurn==`function`}),y=validateReference({name:e,owner:n,phase:`inputSchema`,stamped:r.inputSchema,required:hasSchemaValidator(t.inputSchema)}),b=validateReference({name:e,owner:n,phase:`outputSchema`,stamped:r.outputSchema,required:hasSchemaValidator(t.outputSchema)}),x={execute:l};return y!==void 0&&(x.inputSchema=y),b!==void 0&&(x.outputSchema=b),(u!==void 0||d!==void 0||f!==void 0)&&(x.label={complete:u,delta:d,start:f}),p!==void 0&&(x.approvalKey=p),m!==void 0&&(x.approvalPrompt=m),h!==void 0&&(x.approvalRequest=h),g!==void 0&&(x.approvalResponse=g),_!==void 0&&(x.toModelOutput=_),v!==void 0&&(x.endsTurn=v),x}function createMetadata(e){return{availableInSubagents:e.entry.availableInSubagents,deferred:e.entry.deferred,callbacks:validateDurableDynamicToolCallbacks(e.name,e.entry,{sessionId:e.sessionId,scope:e.scope,resolverSlug:e.resolver.slug,entryKey:e.entryKey,name:e.name}),description:e.entry.description,frameworkTool:isFrameworkTool(e.entry)||void 0,endsTurn:e.entry.endsTurn===!0||void 0,entryKey:e.entryKey,inputSchema:serializeInputSchema(e.entry.inputSchema),name:e.name,outputSchema:serializeOutputSchema(e.entry.outputSchema),resolverSlug:e.resolver.slug}}function workflowToolNames(e){let t=e.get(BundleKey);if(t===void 0)return new Set;let{subagentRegistry:n,toolRegistry:r}=t;return new Set([...[...r.toolsByName.values()].filter(({prepared:e})=>workflowIdForHandling(e.behavior?.handling)!==void 0).map(({prepared:e})=>e.name),...n.preparedTools.map(e=>e.name),...n.dynamicResolvers.map(e=>e.name)])}async function resolveToolsFromEvent(e,t,n,r){let i=e.require(SessionIdKey),a=n.type.split(`.`)[0],o=await Promise.allSettled(t.map(async t=>{let o=t.events[n.type];if(o===void 0)return null;clearDurableDynamicCallbacks(i,{scope:a,resolverSlug:t.slug});try{let s=await o(n,buildResolveContext(e,r));if(s==null)return null;let{entries:c,isSingle:l}=readDynamicToolResult(t,s),u=qualifyDynamicToolNames(t,l,c),d=e.get(ConnectionRegistryKey)?.getConnectionNames()??[],f=workflowToolNames(e);for(let{name:e}of u)if(assertNotConnectionOwned({connectionNames:d,name:e,remedy:`Rename the map key.`,subject:`Dynamic tool`}),f.has(e))throw Error(`Dynamic tool "${e}" from resolver "${t.logicalPath}" collides with the workflow tool or agent "${e}". Rename the map key.`);return{metadata:u.map(({name:e,entryKey:n,entry:r})=>createMetadata({entry:r,entryKey:n,name:e,resolver:t,sessionId:i,scope:a})),resolver:t}}catch(e){throw clearDurableDynamicCallbacks(i,{scope:a,resolverSlug:t.slug}),e}})),s=[],c=new Map;for(let e of o){if(e.status===`rejected`){log.error(`Dynamic tool resolver (${n.type}) failed — skipping its complete result.`,{error:toErrorMessage(e.reason)});continue}if(e.value!==null){for(let t of e.value.metadata){let n=c.get(t.name);if(n!==void 0&&n!==e.value.resolver.slug)throw Error(`Dynamic tool "${t.name}" from resolver "${e.value.resolver.slug}" collides with dynamic resolver "${n}". Namespace the map key manually.`);c.set(t.name,e.value.resolver.slug)}s.push(...e.value.metadata)}}return{metadata:s}}const resolvedStepTools=new WeakMap;function stepCoordinate(e){return`${e.data.turnId}:${String(e.data.stepIndex)}`}function storeResolvedStepTools(e){e.ctx.set(StepDynamicToolMetadataKey,e.metadata);let t=stepCoordinate(e.event);resolvedStepTools.set(e.ctx,{coordinate:t,metadata:e.metadata})}async function resolveStepDynamicTools(e){let t=stepCoordinate(e.event),n=resolvedStepTools.get(e.ctx);if(n?.coordinate===t){e.ctx.set(StepDynamicToolMetadataKey,n.metadata);return}let r=e.resolvers.filter(e=>e.eventNames.includes(`step.started`)),{metadata:i}=r.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,r,e.event,e.messages);storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:i})}async function preparePersistedStepDynamicToolMetadata(e){let t=e.ctx.get(StepDynamicToolMetadataKey)??[],n=t.filter(isCurrentDynamicToolMetadata);if(n.length===t.length&&!hasUnregisteredDurableDynamicCallbacks(n,{sessionId:e.ctx.require(SessionIdKey),scope:`step`})){n.length>0&&storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:n});return}await resolveStepDynamicTools(e);let r=e.ctx.get(StepDynamicToolMetadataKey)??[];storeResolvedStepTools({ctx:e.ctx,event:e.event,metadata:toCurrentDynamicToolMetadataList(t,r.filter(isCurrentDynamicToolMetadata))})}async function dispatchDynamicToolEvent(e){if(e.event.type===`session.completed`){let t=e.ctx.get(SessionIdKey);t!==void 0&&clearDurableDynamicCallbacks(t);return}if(!ALLOWED_DYNAMIC_TOOL_EVENTS.has(e.event.type))return;if(e.event.type===`step.started`){await resolveStepDynamicTools({...e,event:e.event});return}e.event.type===`turn.started`&&e.ctx.set(StepDynamicToolMetadataKey,[]);let t=e.resolvers.filter(t=>t.eventNames.includes(e.event.type)),{metadata:n}=t.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,t,e.event,e.messages),r=durableKeyForEvent(e.event.type);if(r===void 0)return;if(e.event.type===`session.started`){e.ctx.set(SessionDynamicToolMetadataKey,n);return}let i=new Set(t.map(e=>e.slug)),a=(e.ctx.get(r)??[]).filter(e=>!i.has(e.resolverSlug));e.ctx.set(r,[...a,...n])}async function refreshDynamicSessionToolsForRuntimeRevision(e){let t=e.ctx.get(SessionDynamicToolMetadataKey)??[],n=t.filter(isCurrentDynamicToolMetadata),r=n.length!==t.length,i=e.ctx.get(SessionDynamicToolRuntimeRevisionKey)!==e.runtimeRevision;if(!i&&!r&&!hasUnregisteredDurableDynamicCallbacks(n,{sessionId:e.ctx.require(SessionIdKey),scope:`session`}))return;let a=e.resolvers.filter(e=>e.eventNames.includes(`session.started`)),{metadata:o}=a.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,a,e.event,e.messages);e.ctx.set(SessionDynamicToolMetadataKey,i?o:toCurrentDynamicToolMetadataList(t,o)),e.ctx.set(SessionDynamicToolRuntimeRevisionKey,e.runtimeRevision)}async function rebindMissingCompiledDynamicToolCallbacks(e){let t=e.ctx.get(TurnDynamicToolMetadataKey)??[],n=t.filter(t=>!isCurrentDynamicToolMetadata(t)||hasUnregisteredDurableDynamicCallbacks([t],{sessionId:e.ctx.require(SessionIdKey),scope:`turn`}));if(n.length===0)return;let r=new Set(n.map(e=>e.resolverSlug)),i=e.resolvers.filter(e=>r.has(e.slug)),a=new Set(i.filter(e=>e.rebindMissingCallbacks===!0).map(e=>e.slug)),o=new Set(t.filter(e=>!isCurrentDynamicToolMetadata(e)).map(e=>e.resolverSlug));if(i.length===0){e.ctx.set(TurnDynamicToolMetadataKey,toCurrentDynamicToolMetadataList(t));return}let s=await contextStorage.run(e.ctx,async()=>await resolveToolsFromEvent(e.ctx,i,e.event,e.messages)),c=toCurrentDynamicToolMetadataList(t,s.metadata);e.ctx.set(TurnDynamicToolMetadataKey,c);let l=c.filter(t=>(a.has(t.resolverSlug)||o.has(t.resolverSlug))&&n.some(e=>e.resolverSlug===t.resolverSlug&&e.name===t.name)&&hasUnregisteredDurableDynamicCallbacks([t],{sessionId:e.ctx.require(SessionIdKey),scope:`turn`}));if(l.length>0)throw Error(`Dynamic tool callback rebind did not restore: ${l.map(e=>e.name).join(`, `)}. The tool may have been renamed or removed.`)}export{dispatchDynamicToolEvent,preparePersistedStepDynamicToolMetadata,rebindMissingCompiledDynamicToolCallbacks,refreshDynamicSessionToolsForRuntimeRevision,resolveStepDynamicTools,validateDurableDynamicToolCallbacks};

package/dist/src/context/prewarm.d.ts

@@ -0,0 +1,27 @@
+import type { ContextContainer } from "#context/container.js";
+/**
+ * Why prewarmers run: `"session"` for a session created without a message,
+ * before that message; `"turn"` when a turn's step opens its first model call.
+ */
+export type SessionPrewarmPhase = "session" | "turn";
+/**
+ * Starts session work in the background so it races the model call instead
+ * of delaying it, and whatever needs the result waits only for what remains.
+ *
+ * `start` runs inside a framework context scope and must return without
+ * awaiting the work. It hands the work to a framework provider, which lets a
+ * consumer wait for it and settles it before the scope commits. No turn has
+ * started in the `"session"` phase: auth is the session creator's, not the
+ * first message's.
+ *
+ * Starting early only moves work earlier. Work that fails is redone when it
+ * is needed, as it would have been without prewarming, so it must leave
+ * nothing that the lazy path cannot reuse or session end cannot clean up.
+ */
+export interface SessionPrewarmer {
+    /** Names the prewarmer in logs. */
+    readonly name: string;
+    start(ctx: ContextContainer, phase: SessionPrewarmPhase): void;
+}
+/** Starts every prewarmer; returns without waiting for their work. */
+export declare function startSessionPrewarmers(ctx: ContextContainer, phase: SessionPrewarmPhase): void;

package/dist/src/context/prewarm.js

@@ -0,0 +1 @@
+import{createLogger}from"#internal/logging.js";import{SessionIdKey}from"#context/keys.js";import{sandboxPrewarmer}from"#context/providers/sandbox.js";const log=createLogger(`context.prewarm`),sessionPrewarmers=[sandboxPrewarmer];function startSessionPrewarmers(e,t){for(let n of sessionPrewarmers)try{n.start(e,t)}catch(r){log.warn(`session prewarmer failed to start; its work runs when needed`,{error:r,phase:t,prewarmer:n.name,sessionId:e.get(SessionIdKey)})}}export{startSessionPrewarmers};

package/dist/src/context/providers/sandbox.d.ts

@@ -1,3 +1,10 @@
 import type { SandboxAccess } from "#sandbox/state.js";
 import type { FrameworkContextProvider } from "#context/provider.js";
+import type { SessionPrewarmer } from "#context/prewarm.js";
+/**
+ * Starts a prewarmed session's sandbox, and resumes a sandbox the session
+ * already recorded when a turn begins. A turn never starts a sandbox: every
+ * agent has one, and most turns may not need it.
+ */
+export declare const sandboxPrewarmer: SessionPrewarmer;
 export declare const sandboxProvider: FrameworkContextProvider<SandboxAccess>;

package/dist/src/context/providers/sandbox.js

@@ -1 +1 @@
-import{createLogger,logError}from"#internal/logging.js";import{SandboxKey,SandboxTerminalCleanupKey,SessionIdKey}from"#context/keys.js";import{BundleKey,ChannelKey}from"#runtime/sessions/runtime-context-keys.js";import{getActiveRuntimeNode}from"#context/node.js";import{ensureSandboxAccess}from"#execution/sandbox/ensure.js";const log=createLogger(`context.sandbox`),sandboxProvider={key:SandboxKey,async create(e,t){let n=e.get(BundleKey);if(n===void 0)return;let r=getActiveRuntimeNode(e),i=r.sandboxRegistry,a=e.require(SessionIdKey),o=e.get(ChannelKey)?.state,s=o?.parentSandboxState,c=i.sandbox?.definition.kind===`parent`,l=o?.sandboxSessionId,u=c||l!==void 0,d=u?l??a:a,f=await ensureSandboxAccess({compiledArtifactsSource:n.compiledArtifactsSource,nodeId:r.nodeId,ownsSandbox:!u,registry:i,sessionId:d,state:t.sandboxState??(u?s:void 0)??null});return e.setVirtualContext(SandboxTerminalCleanupKey,e=>f.end(e)),{value:f}},async commit(e,t){let n=await e.captureState();return{...t,sandboxState:n}},async dispose(e){await e.release().catch(e=>{logError(log,`failed to close a step's sandbox connection`,e)})}};export{sandboxProvider};
+import{createLogger,logError}from"#internal/logging.js";import{SandboxKey,SandboxTerminalCleanupKey,SessionIdKey}from"#context/keys.js";import{BundleKey,ChannelKey}from"#runtime/sessions/runtime-context-keys.js";import{getActiveRuntimeNode}from"#context/node.js";import{ensureSandboxAccess}from"#execution/sandbox/ensure.js";const log=createLogger(`context.sandbox`),sandboxPrewarmer={name:`sandbox`,start(e,t){let n=e.get(SandboxKey);t===`session`?n?.prepare():n?.resumeRecorded()}},sandboxProvider={key:SandboxKey,async create(e,t){let n=e.get(BundleKey);if(n===void 0)return;let r=getActiveRuntimeNode(e),i=r.sandboxRegistry,a=e.require(SessionIdKey),o=e.get(ChannelKey)?.state,s=o?.parentSandboxState,c=i.sandbox?.definition.kind===`parent`,l=o?.sandboxSessionId,u=c||l!==void 0,d=u?l??a:a,f=await ensureSandboxAccess({compiledArtifactsSource:n.compiledArtifactsSource,nodeId:r.nodeId,ownsSandbox:!u,registry:i,sessionId:d,state:t.sandboxState??(u?s:void 0)??null});return e.setVirtualContext(SandboxTerminalCleanupKey,e=>f.end(e)),{value:f}},async commit(e,t){await e.settle();let n=await e.captureState();return{...t,sandboxState:n}},async dispose(e){await e.release().catch(e=>{logError(log,`failed to close a step's sandbox connection`,e)})}};export{sandboxPrewarmer,sandboxProvider};

package/dist/src/context/run-step.js

@@ -1 +1 @@
-import{contextStorage}from"#context/container.js";import{connectionProvider}from"#context/providers/connection.js";import{sandboxProvider}from"#context/providers/sandbox.js";import{sessionProvider}from"#context/providers/session.js";import{toolStubProvider}from"#context/providers/tool-stubs.js";const frameworkProviders=[sessionProvider,toolStubProvider,connectionProvider,sandboxProvider];async function withContextScope(e,t,n){let r=[];e.clearVirtualContext();try{for(let n of frameworkProviders){let i=await n.create(e,t);i!==void 0&&(e.setVirtualContext(n.key,i.value),r.push(n))}let i=await contextStorage.run(e,()=>n(t)),a=i.session;for(let t of r)t.commit!==void 0&&(a=await t.commit(e.require(t.key),a));return a===i.session?i:{result:i.result,session:a}}catch(t){let n=[];for(let i of r.toReversed())if(i.rollback!==void 0)try{await i.rollback(e.require(i.key),t)}catch(e){n.push(e)}throw n.length>0?AggregateError([t,...n],`Framework context rollback did not complete.`,{cause:t}):t}finally{for(let t of r.toReversed())await t.dispose?.(e.require(t.key))}}async function runStep(e,t,n){let r=await withContextScope(e,t,async e=>{let t=await n(e);return{result:t,session:t.session}});return{...r.result,session:r.session}}export{runStep,withContextScope};
+import{contextStorage}from"#context/container.js";import{sandboxProvider}from"#context/providers/sandbox.js";import{connectionProvider}from"#context/providers/connection.js";import{sessionProvider}from"#context/providers/session.js";import{toolStubProvider}from"#context/providers/tool-stubs.js";const frameworkProviders=[sessionProvider,toolStubProvider,connectionProvider,sandboxProvider];async function withContextScope(e,t,n){let r=[];e.clearVirtualContext();try{for(let n of frameworkProviders){let i=await n.create(e,t);i!==void 0&&(e.setVirtualContext(n.key,i.value),r.push(n))}let i=await contextStorage.run(e,()=>n(t)),a=i.session;for(let t of r)t.commit!==void 0&&(a=await t.commit(e.require(t.key),a));return a===i.session?i:{result:i.result,session:a}}catch(t){let n=[];for(let i of r.toReversed())if(i.rollback!==void 0)try{await i.rollback(e.require(i.key),t)}catch(e){n.push(e)}throw n.length>0?AggregateError([t,...n],`Framework context rollback did not complete.`,{cause:t}):t}finally{for(let t of r.toReversed())await t.dispose?.(e.require(t.key))}}async function runStep(e,t,n){let r=await withContextScope(e,t,async e=>{let t=await n(e);return{result:t,session:t.session}});return{...r.result,session:r.session}}export{runStep,withContextScope};

package/dist/src/discover/lib.d.ts

@@ -7,8 +7,8 @@ import type { ProjectSource, ProjectSourceEntry } from "#discover/project-source
  */
 export declare const DISCOVER_LIB_DIRECTORY_INVALID = "discover/lib-directory-invalid";
 /**
- * Diagnostic emitted when discovery finds an unsupported entry under the
- * authored `lib/` tree.
+ * Diagnostic emitted when discovery finds an entry under the authored `lib/`
+ * tree that is neither a regular file nor a directory.
  */
 export declare const DISCOVER_LIB_ENTRY_UNSUPPORTED = "discover/lib-entry-unsupported";
 /**
@@ -28,7 +28,8 @@ interface DiscoverLibSourcesResult {
 }
 /**
  * Discovers module-only helper sources under `lib/` without executing
- * authored modules.
+ * authored modules. Other files, such as JSON data imported by a helper, are
+ * left to the bundler.
  */
 export declare function discoverLibSources(input: DiscoverLibSourcesInput): Promise<DiscoverLibSourcesResult>;
 export {};

package/dist/src/discover/lib.js

@@ -1 +1 @@
-import{discoverNamedSourceDirectory}from"#discover/grammar.js";const DISCOVER_LIB_DIRECTORY_INVALID=`discover/lib-directory-invalid`,DISCOVER_LIB_ENTRY_UNSUPPORTED=`discover/lib-entry-unsupported`;async function discoverLibSources(e){let t=await discoverNamedSourceDirectory({directoryName:`lib`,invalidDirectoryCode:DISCOVER_LIB_DIRECTORY_INVALID,invalidDirectoryMessage:`Expected "${e.agentRoot}/lib" to be a directory of authored helper modules.`,recursive:!0,rootEntries:e.rootEntries,rootPath:e.agentRoot,source:e.source,unsupportedEntryCode:DISCOVER_LIB_ENTRY_UNSUPPORTED,unsupportedEntryMessage:e=>`Expected "${e}" to be a supported authored module within "lib/".`,unsupportedFileCode:DISCOVER_LIB_ENTRY_UNSUPPORTED,unsupportedFileMessage:e=>`Expected "${e}" to be a supported authored module within "lib/".`});return{diagnostics:t.diagnostics,lib:t.sources}}export{DISCOVER_LIB_DIRECTORY_INVALID,DISCOVER_LIB_ENTRY_UNSUPPORTED,discoverLibSources};
+import{discoverNamedSourceDirectory}from"#discover/grammar.js";const DISCOVER_LIB_DIRECTORY_INVALID=`discover/lib-directory-invalid`,DISCOVER_LIB_ENTRY_UNSUPPORTED=`discover/lib-entry-unsupported`;async function discoverLibSources(e){let t=await discoverNamedSourceDirectory({directoryName:`lib`,invalidDirectoryCode:DISCOVER_LIB_DIRECTORY_INVALID,invalidDirectoryMessage:`Expected "${e.agentRoot}/lib" to be a directory of authored helper modules.`,recursive:!0,rootEntries:e.rootEntries,rootPath:e.agentRoot,source:e.source,unsupportedEntryCode:DISCOVER_LIB_ENTRY_UNSUPPORTED,unsupportedEntryMessage:e=>`Expected "${e}" to be a regular file or directory within "lib/".`});return{diagnostics:t.diagnostics,lib:t.sources}}export{DISCOVER_LIB_DIRECTORY_INVALID,DISCOVER_LIB_ENTRY_UNSUPPORTED,discoverLibSources};

package/dist/src/discover/named-source-directory.d.ts

@@ -45,8 +45,9 @@ interface DiscoverNamedSourceDirectoryBaseInput {
      * When set, emit this diagnostic for any leaf file that is neither a
      * supported authored module nor (when allowed) a markdown file. When
      * unset, unrecognized leaf files are silently ignored — used by
-     * `tools/`, `channels/`, and `hooks/` so authors can drop incidental
-     * files into those directories without build errors.
+     * `tools/`, `channels/`, `hooks/`, and `lib/` so authors can drop
+     * incidental files (or JSON data imported by a module) into those
+     * directories without build errors.
      */
     unsupportedFileCode?: string;
     unsupportedFileMessage?: (sourcePath: string, directoryName: string) => string;

package/dist/src/evals/cli/eval.d.ts

@@ -3,6 +3,7 @@ export interface EvalCliOptions {
     url?: string;
     timeout?: string;
     maxConcurrency?: string;
+    repeat?: string;
     json?: boolean;
     junit?: string;
     skipReport?: boolean;
@@ -10,6 +11,7 @@ export interface EvalCliOptions {
     list?: boolean;
     tag?: string[];
     excludeTag?: string[];
+    keepState?: boolean;
     verbose?: boolean;
 }
 type EvalCliLogger = {

package/dist/src/evals/cli/eval.js

@@ -1,4 +1,4 @@
-import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{basename,join}from"node:path";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{noDevelopmentExtensions}from"#compiler/development-extensions.js";import{EVE_DEV_STATE_DIR_ENV,EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{shutdownActiveSandboxConnections}from"#execution/sandbox/active-connections.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{createEvalStateDirectory,removeEvalStateDirectory}from"#evals/cli/eval-state-directory.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n,r=process.cwd()){await loadDevelopmentEnvironmentFiles(r);let i=e.length>0?e:void 0,a=await discoverAndImportEvals(r,i);if(a.length===0){let e=await findMisplacedEvalDirs(r);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
+import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{basename,join,relative}from"node:path";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{noDevelopmentExtensions}from"#compiler/development-extensions.js";import{EVE_DEV_STATE_DIR_ENV,EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{shutdownActiveSandboxConnections}from"#execution/sandbox/active-connections.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{createEvalStateDirectory,removeEvalStateDirectory}from"#evals/cli/eval-state-directory.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n,r=process.cwd()){await loadDevelopmentEnvironmentFiles(r);let i=e.length>0?e:void 0,a=await discoverAndImportEvals(r,i);if(a.length===0){let e=await findMisplacedEvalDirs(r);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
 `+e.map(e=>`  - ${e}`).join(`
 `)+`
-eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):i?n.error(`No evals found matching: ${i.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let o=t.tag??[],s=t.excludeTag??[],c=filterEvalsByTags({evaluations:a,includeTags:o,excludeTags:[]});if(c.length===0){n.error(`No evals matched the provided tags (${o.join(`, `)}).`),process.exitCode=2;return}let l=filterEvalsByTags({evaluations:c,includeTags:[],excludeTags:s});if(t.list===!0){printEvalList(l,t.json===!0,n);return}if(l.length===0){n.log(`All ${c.length} matching evals are excluded by tags (${s.join(`, `)}); nothing to run.`);return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let f;try{f=await discoverEvalConfig(r)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h,g,_;try{if(_=await f.setup?.(),t.url)g=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:r}),h=await resolveEvalTargetHandle({client:g,expectedAgentName:await readExpectedAgentName(r),kind:`remote`,url:t.url});else{let e=randomUUID();process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=e,m=await createEvalStateDirectory(r,e),process.env[EVE_DEV_STATE_DIR_ENV]=m,p=createDevelopmentServer(r,{developmentExtensions:noDevelopmentExtensions(),host:`127.0.0.1`,port:0});let t=await p.start();g=await createEvalClient({kind:`local`,url:t.url}),h=await resolveEvalTargetHandle({client:g,expectedAgentName:await readExpectedAgentName(r),kind:`local`,url:t.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:l,config:f,setupContext:_,target:h,client:g,appRoot:r,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{for(let e of[()=>p?.close(),()=>p&&shutdownActiveSandboxConnections({log:e=>n.error(e)}),()=>m&&removeEvalStateDirectory(m),()=>f.teardown?.(_)])try{await e()}catch(e){n.error(`Eval cleanup failed: ${e instanceof Error?e.message:String(e)}`),process.exitCode=1}}let v=typeof process.exitCode==`number`?process.exitCode:0;await flushStandardStreams(),process.exit(v)}async function flushStandardStreams(){await Promise.all([flushStream(process.stdout),flushStream(process.stderr)])}async function flushStream(e){await new Promise(t=>{let n=!1,settle=()=>{n||(n=!0,setImmediate(()=>{e.off(`error`,settle),t()}))};e.once(`error`,settle);try{e.write(``,settle)}catch{settle()}})}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let t=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof t.name==`string`&&t.name.length>0?t.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};
+eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):i?n.error(`No evals found matching: ${i.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let o=t.tag??[],s=t.excludeTag??[],c=filterEvalsByTags({evaluations:a,includeTags:o,excludeTags:[]});if(c.length===0){n.error(`No evals matched the provided tags (${o.join(`, `)}).`),process.exitCode=2;return}let l=filterEvalsByTags({evaluations:c,includeTags:[],excludeTags:s});if(t.list===!0){printEvalList(l,t.json===!0,n);return}if(l.length===0){n.log(`All ${c.length} matching evals are excluded by tags (${s.join(`, `)}); nothing to run.`);return}let u,d,f;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`),f=parsePositiveInteger(t.repeat,`--repeat`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p;try{p=await discoverEvalConfig(r)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let m,h,g,_,v;try{if(v=await p.setup?.(),t.url)t.keepState===!0&&n.error(`--keep-state has no effect with --url: the remote server keeps its own state.`),_=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:r}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`remote`,url:t.url});else{let e=randomUUID();process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=e,h=await createEvalStateDirectory(r,e),process.env[EVE_DEV_STATE_DIR_ENV]=h,m=createDevelopmentServer(r,{developmentExtensions:noDevelopmentExtensions(),host:`127.0.0.1`,port:0});let t=await m.start();_=await createEvalClient({kind:`local`,url:t.url}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`local`,url:t.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:l,config:p,setupContext:v,target:g,client:_,appRoot:r,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,repeat:f,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{for(let e of[()=>m?.close(),()=>m&&shutdownActiveSandboxConnections({log:e=>n.error(e)}),async()=>{if(h!==void 0){if(t.keepState!==!0)return await removeEvalStateDirectory(h);n.error(`Kept the eval server's state in ${relative(r,h)}.`)}},()=>p.teardown?.(v)])try{await e()}catch(e){n.error(`Eval cleanup failed: ${e instanceof Error?e.message:String(e)}`),process.exitCode=1}}let y=typeof process.exitCode==`number`?process.exitCode:0;await flushStandardStreams(),process.exit(y)}async function flushStandardStreams(){await Promise.all([flushStream(process.stdout),flushStream(process.stderr)])}async function flushStream(e){await new Promise(t=>{let n=!1,settle=()=>{n||(n=!0,setImmediate(()=>{e.off(`error`,settle),t()}))};e.once(`error`,settle);try{e.write(``,settle)}catch{settle()}})}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let t=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof t.name==`string`&&t.name.length>0?t.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};

package/dist/src/evals/define-eval-config.d.ts

@@ -5,11 +5,12 @@ import type { EveEvalConfig, EveEvalConfigInput } from "#evals/types.js";
  *
  * Exactly one `evals.config.ts` is required. It supplies the optional default
  * `judge` model for `t.judge(...)` assertions (so individual evals need not
- * repeat it), run-level `reporters`, `maxConcurrency` and `timeoutMs` defaults,
- * and optional run-wide `setup` and `teardown`. CLI flags (`--max-concurrency`,
- * `--timeout`) and per-eval values take precedence over the config defaults.
+ * repeat it), run-level `reporters`, `maxConcurrency`, `timeoutMs`, and `repeat`
+ * defaults, and optional run-wide `setup` and `teardown`. CLI flags
+ * (`--max-concurrency`, `--timeout`, `--repeat`) and per-eval values take
+ * precedence over the config defaults.
  *
- * Throws on invalid input: a non-positive or non-integer `maxConcurrency`,
+ * Throws on invalid input: a non-positive or non-integer `maxConcurrency` or `repeat`,
  * a negative or non-finite `timeoutMs`, non-array `reporters`, or a
  * non-function `setup` or `teardown`.
  */

package/dist/src/evals/define-eval-config.js

@@ -1 +1 @@
-function defineEvalConfig(e){return validateEvalConfigInput(e),{...e,_tag:`EveEvalConfig`}}function validateEvalConfigInput(e){if(e.setup!==void 0&&typeof e.setup!=`function`)throw Error("Eval config `setup` must be a function.");if(e.teardown!==void 0&&typeof e.teardown!=`function`)throw Error("Eval config `teardown` must be a function.");if(e.maxConcurrency!==void 0&&(!Number.isInteger(e.maxConcurrency)||e.maxConcurrency<1))throw Error("Eval config `maxConcurrency` must be a positive integer.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval config `timeoutMs` must be a non-negative finite number.");if(e.reporters!==void 0&&!Array.isArray(e.reporters))throw Error("Eval config `reporters` must be an array of reporters.")}export{defineEvalConfig};
+function defineEvalConfig(e){return validateEvalConfigInput(e),{...e,_tag:`EveEvalConfig`}}function validateEvalConfigInput(e){if(e.setup!==void 0&&typeof e.setup!=`function`)throw Error("Eval config `setup` must be a function.");if(e.teardown!==void 0&&typeof e.teardown!=`function`)throw Error("Eval config `teardown` must be a function.");if(e.maxConcurrency!==void 0&&(!Number.isInteger(e.maxConcurrency)||e.maxConcurrency<1))throw Error("Eval config `maxConcurrency` must be a positive integer.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval config `timeoutMs` must be a non-negative finite number.");if(e.repeat!==void 0&&(!Number.isInteger(e.repeat)||e.repeat<1))throw Error("Eval config `repeat` must be a positive integer.");if(e.reporters!==void 0&&!Array.isArray(e.reporters))throw Error("Eval config `reporters` must be an array of reporters.")}export{defineEvalConfig};

package/dist/src/evals/index.d.ts

@@ -6,6 +6,6 @@ export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js"
 export type { InputRequest } from "#shared/input.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, ScoreInput, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSendTiming, EveEvalSessionClock, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, ScoreInput, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";
 export type { ToolStub, ToolStubOutcome } from "#tool-stubs/types.js";

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{mkdir,writeFile}from"node:fs/promises";import{dirname,join}from"node:path";function resolveArtifactDirectory(e){let t=new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19);return join(e,`.eve`,`evals`,t)}async function writeArtifacts(e,t){let n=join(e,`evals`);await mkdir(n,{recursive:!0}),await writeFile(join(e,`summary.json`),JSON.stringify(buildSummaryArtifact(t),null,2));let r=t.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
-`);await writeFile(join(e,`results.jsonl`),`${r}\n`),await Promise.all(t.results.map(async e=>{let t=join(n,`${sanitizeArtifactPath(e.id)}.json`);await mkdir(dirname(t),{recursive:!0}),await writeFile(t,JSON.stringify(buildEvalArtifact(e),null,2));let r=e.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(n,`${sanitizeArtifactPath(e.id)}.events.ndjson`),`${r}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,errored:e.errored,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(e,`results.jsonl`),`${r}\n`),await Promise.all(t.results.map(async e=>{let t=join(n,artifactBaseName(e));await mkdir(dirname(t),{recursive:!0}),await writeFile(`${t}.json`,JSON.stringify(buildEvalArtifact(e),null,2));let r=e.result.events.map(e=>JSON.stringify(e)).join(`
+`);await writeFile(`${t}.events.ndjson`,`${r}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,errored:e.errored,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,attempt:e.attempt,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,attempt:e.attempt,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason,startedAt:e.startedAt,completedAt:e.completedAt}}function artifactBaseName(e){let t=sanitizeArtifactPath(e.id);return e.attempt===void 0?t:`${t}.attempt-${e.attempt}`}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/attempt-label.d.ts

@@ -0,0 +1,2 @@
+/** Labels an eval result by id, suffixed with `#<attempt>` when the run repeats evals. */
+export declare function formatAttemptLabel(id: string, attempt: number | undefined): string;

package/dist/src/evals/runner/attempt-label.js

@@ -0,0 +1 @@
+function formatAttemptLabel(e,t){return t===void 0?e:`${e}#${t}`}export{formatAttemptLabel};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,exportedAssertionName);for(let[e,t]of r)t.score!==void 0&&(n[e]=t.score);let i=e.assertions.filter(e=>e.passed===!1).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output??``,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,exportedAssertionName);for(let[e,t]of r)t.score!==void 0&&(n[e]=t.score);let i=e.assertions.filter(e=>e.passed===!1).map(e=>({...e})),a={...t?.metadata,eveAttempt:e.attempt,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:formatAttemptLabel(e.id,e.attempt),input:t?.description??``,output:e.result.output??``,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import pc from"#compiled/picocolors/index.js";import{exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{formatAssertionFailureDetailLines,formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function Console(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(e){this.#e=e?.log??console.log,this.#t=pc.createColors(e?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.flatMap(e=>e.score===void 0?[]:[this.#i(e.name,e.score)]).join(`  `),u=[s,this.#t.dim(e.id),c,l].filter(Boolean).join(`  `);this.#e(u);for(let e of t)if(e.passed===!1){this.#e(`  ${this.#t.red(`✗ ${formatAssertionFailureHeadline(e)}`)}`);for(let t of formatAssertionFailureDetailLines(e))this.#e(`    ${this.#t.red(t)}`)}for(let e of groupAssertionScores(o,exportedAssertionName).values()){if(e.assertions.every(t=>t.score===e.score))continue;let t=e.assertions[0].name;this.#e(`  ${this.#t.yellow(`⚠ ${t} recorded ${e.assertions.length} times with different scores; Braintrust and Datadog keep only the lowest. Add .label() to tell them apart.`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e){let e=n.assertions.filter(e=>e.severity===`soft`);for(let[n,r]of groupAssertionScores(e,e=>e.name)){if(r.score===void 0)continue;let e=t.get(n);e?(e.sum+=r.score,e.count+=1):t.set(n,{sum:r.score,count:1})}}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{Console};
+import pc from"#compiled/picocolors/index.js";import{exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureDetailLines,formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function Console(e={}){return new ConsoleReporter(e)}var ConsoleReporter=class{#e;#t;constructor(e){this.#e=e?.log??console.log,this.#t=pc.createColors(e?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r,skipReason:i}=e,a=t.filter(e=>e.severity===`gate`),o=t.filter(e=>e.severity===`soft`),s=this.#n(n),c=a.length>0?this.#r(a.filter(e=>e.passed).length,a.length):``,l=o.flatMap(e=>e.score===void 0?[]:[this.#i(e.name,e.score)]).join(`  `),u=formatAttemptLabel(e.id,e.attempt),d=[s,this.#t.dim(u),c,l].filter(Boolean).join(`  `);this.#e(d);for(let e of t)if(e.passed===!1){this.#e(`  ${this.#t.red(`✗ ${formatAssertionFailureHeadline(e)}`)}`);for(let t of formatAssertionFailureDetailLines(e))this.#e(`    ${this.#t.red(t)}`)}for(let e of groupAssertionScores(o,exportedAssertionName).values()){if(e.assertions.every(t=>t.score===e.score))continue;let t=e.assertions[0].name;this.#e(`  ${this.#t.yellow(`⚠ ${t} recorded ${e.assertions.length} times with different scores; Braintrust and Datadog keep only the lowest. Add .label() to tell them apart.`)}`)}r&&this.#e(`  ${this.#t.red(r)}`),i&&this.#e(`  ${this.#t.dim(i)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=aggregateAttempts(a);if(l.length>0){this.#e(``),this.#e(this.#t.bold(`Repeatability:`));for(let{id:e,passed:t,total:n}of l){let r=`${t}/${n} passed`,i=t===n?this.#t.green(r):t===0?this.#t.red(r):this.#t.yellow(r);this.#e(`  ${i}  ${this.#t.dim(e)}`)}}let u=this.#o(a);if(u.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of u){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let d=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(d)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`–`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e){let e=n.assertions.filter(e=>e.severity===`soft`);for(let[n,r]of groupAssertionScores(e,e=>e.name)){if(r.score===void 0)continue;let e=t.get(n);e?(e.sum+=r.score,e.count+=1):t.set(n,{sum:r.score,count:1})}}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function aggregateAttempts(e){let t=new Map;for(let n of e){if(n.attempt===void 0||n.verdict!==`passed`&&n.verdict!==`failed`)continue;let e=t.get(n.id)??{passed:0,total:0};e.total+=1,n.verdict===`passed`&&(e.passed+=1),t.set(n.id,e)}return[...t.entries()].map(([e,t])=>({id:e,...t}))}function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{Console};

package/dist/src/evals/runner/reporters/datadog.js

@@ -1,2 +1,2 @@
 import{composeAssertionScoreMetadata,exportedAssertionName,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{resolveRuntimeTraceLinks}from"#evals/runner/reporters/datadog-runtime-trace-links.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
-`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>e.passed===!1).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(exportedAssertionName(e)));for(let[e,i]of r){if(i.score===void 0)continue;let r=i.assertions.flatMap(e=>e.passed===void 0?[]:[e.passed]);t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,...r.length>0&&{assertion_passed:String(r.every(e=>e))},assertion_count:String(i.assertions.length)}})}return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{Datadog};
+`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[],s=new Set;for(let t of e){let e=this.#l(t.id);if(s.has(e))continue;s.add(e);let n=this.#u(t),r=a.get(e);if(r===void 0){o.push({inputData:n.input,metadata:n.metadata,...n.expectedOutput===null?{}:{expectedOutput:n.expectedOutput}});continue}let c=i.records()[r];c&&!isDatasetRecordEqual(c,n)&&i.update(r,n)}o.length>0&&i.addRecords(o),await i.push();let c=i.id();if(!c)throw Error(`Datadog dataset '${r}' has no id after push().`);let l=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&l.set(t,e.id)}let u={id:c,name:i.name()},d=i.version()??i.latestVersion();d!==null&&(u.version=d),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:u});for(let t of e){let e=this.#l(t.id),n=l.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t),e.attempt!==void 0&&(r.runIteration=e.attempt);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>e.passed===!1).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(exportedAssertionName(e)));for(let[e,i]of r){if(i.score===void 0)continue;let r=i.assertions.flatMap(e=>e.passed===void 0?[]:[e.passed]);t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,...r.length>0&&{assertion_passed:String(r.every(e=>e))},assertion_count:String(i.assertions.length)}})}return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{Datadog};

package/dist/src/evals/runner/reporters/junit.js

@@ -1,4 +1,4 @@
-import{mkdir,writeFile}from"node:fs/promises";import{dirname}from"node:path";import{truncateDiagnostic}from"#evals/diagnostics.js";import{formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(e){let t=renderJUnit(e,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,t)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
-`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
+import{mkdir,writeFile}from"node:fs/promises";import{dirname}from"node:path";import{truncateDiagnostic}from"#evals/diagnostics.js";import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{formatAssertionFailureHeadline}from"#evals/runner/reporters/assertion-diagnostics.js";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(e){let t=renderJUnit(e,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,t)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(formatAttemptLabel(e.id,e.attempt))}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
 `);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
 `)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.filter(e=>e.passed===!1);return t.length>0?truncateDiagnostic(t.map(formatAssertionFailureHeadline).join(`; `),1e3):e.verdict===`scored`?`score below threshold`:e.verdict}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -3,6 +3,8 @@ import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget, EveEvalT
 /** Context delivered when one eval is scheduled for execution. */
 export interface EveEvalStartEvent {
     readonly evaluation: EveEval;
+    /** 1-based attempt number, present only when the run repeats evals. */
+    readonly attempt?: number;
     readonly startedAt: string;
     readonly target: EveEvalTarget;
 }

package/dist/src/evals/runner/run-evals.d.ts

@@ -24,6 +24,11 @@ export interface RunEvalsOptions {
     readonly maxConcurrency?: number;
     /** Overrides every eval's `timeoutMs` when set (CLI `--timeout`). */
     readonly timeoutMs?: number;
+    /**
+     * Times to run every eval. Must be a positive integer. Overrides the config
+     * `repeat`; defaults to 1 when neither is set.
+     */
+    readonly repeat?: number;
     /** Receives verbose activity lines as evals run. */
     readonly onEvalLog?: (evalId: string, message: string) => void;
 }

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(e){let{config:t,target:n,client:r,appRoot:i}=e,a=e.maxConcurrency??t.maxConcurrency??8;if(!Number.isInteger(a)||a<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(e.maxConcurrency??t.maxConcurrency)}.`);let o=e.evaluations.map(e=>applyConfigDefaults(e,t)),s=new Date().toISOString(),c=buildReporterBindings({...e,evaluations:o});for(let e of c)await e.reporter.onRunStart(o.filter(t=>e.evalIds.has(t.id)),n);let l=[],u=[...o],d=new Set,f=Promise.resolve(),enqueueReporterCallback=(e,t)=>{f=f.then(async()=>{for(let n of c)n.evalIds.has(e.id)&&await t(n.reporter)})};for(;u.length>0||d.size>0;){for(;u.length>0&&d.size<a;){let t=u.shift();if(t===void 0)break;let i=new Date().toISOString();enqueueReporterCallback(t,async e=>{await e.onEvalStart?.({evaluation:t,startedAt:i,target:n})});let a=(async()=>{let a=await executeEval({client:r,evaluation:t,setupContext:e.setupContext,onLog:e.onEvalLog===void 0?void 0:n=>e.onEvalLog?.(t.id,n),onSessionStart:e=>{enqueueReporterCallback(t,async r=>{await r.onSessionStart?.({evaluation:t,primary:e.primary,sessionId:e.sessionId,startedAt:e.startedAt,target:n,traceContext:e.traceContext})})},startedAt:i,target:n,timeoutMs:e.timeoutMs});l.push(a);for(let n of a.result.sessions??[]){if(n.sessionId===void 0)continue;let r=n.primary?`primary session`:`secondary session`;e.onEvalLog?.(t.id,`workflow run id (${r}): ${n.sessionId}`)}enqueueReporterCallback(t,async e=>{await e.onEvalComplete(a,{evaluation:t,target:n,traceContexts:a.result.traceContexts})})})().finally(()=>{d.delete(a)});d.add(a)}d.size>0&&await Promise.race(d)}await f;let p=new Map(o.map((e,t)=>[e.id,t]));l.sort((e,t)=>(p.get(e.id)??0)-(p.get(t.id)??0));let m=buildSummary(n,l,s),h=resolveArtifactDirectory(i);await writeArtifacts(h,m);for(let e of c)await e.reporter.onRunComplete(scopeSummary(m,e.evalIds));return m}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{formatAttemptLabel}from"#evals/runner/attempt-label.js";import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(e){let{config:t,target:n,client:r,appRoot:i}=e,a=e.maxConcurrency??t.maxConcurrency??8;if(!Number.isInteger(a)||a<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(e.maxConcurrency??t.maxConcurrency)}.`);let o=e.repeat??t.repeat??1;if(!Number.isInteger(o)||o<1)throw Error(`Eval repeat must be a positive integer; got ${String(e.repeat??t.repeat)}.`);let s=e.evaluations.map(e=>applyConfigDefaults(e,t)),c=new Date().toISOString(),l=buildReporterBindings({...e,evaluations:s});for(let e of l)await e.reporter.onRunStart(s.filter(t=>e.evalIds.has(t.id)),n);let u=[],d=s.flatMap(e=>Array.from({length:o},(t,n)=>({evaluation:e,attempt:o>1?n+1:void 0}))),f=new Set,p=Promise.resolve(),enqueueReporterCallback=(e,t)=>{p=p.then(async()=>{for(let n of l)n.evalIds.has(e.id)&&await t(n.reporter)})};for(;d.length>0||f.size>0;){for(;d.length>0&&f.size<a;){let t=d.shift();if(t===void 0)break;let{evaluation:i,attempt:a}=t,o=formatAttemptLabel(i.id,a),s=new Date().toISOString();enqueueReporterCallback(i,async e=>{await e.onEvalStart?.({evaluation:i,attempt:a,startedAt:s,target:n})});let c=(async()=>{let t=await executeEval({client:r,evaluation:i,setupContext:e.setupContext,onLog:e.onEvalLog===void 0?void 0:t=>e.onEvalLog?.(o,t),onSessionStart:e=>{enqueueReporterCallback(i,async t=>{await t.onSessionStart?.({evaluation:i,attempt:a,primary:e.primary,sessionId:e.sessionId,startedAt:e.startedAt,target:n,traceContext:e.traceContext})})},startedAt:s,target:n,timeoutMs:e.timeoutMs}),c=a===void 0?t:{...t,attempt:a};u.push(c);for(let t of c.result.sessions??[]){if(t.sessionId===void 0)continue;let n=t.primary?`primary session`:`secondary session`;e.onEvalLog?.(o,`workflow run id (${n}): ${t.sessionId}`)}enqueueReporterCallback(i,async e=>{await e.onEvalComplete(c,{evaluation:i,target:n,traceContexts:c.result.traceContexts})})})().finally(()=>{f.delete(c)});f.add(c)}f.size>0&&await Promise.race(f)}await p;let m=new Map(s.map((e,t)=>[e.id,t]));u.sort((e,t)=>(m.get(e.id)??0)-(m.get(t.id)??0)||(e.attempt??0)-(t.attempt??0));let h=buildSummary(n,u,c),g=resolveArtifactDirectory(i);await writeArtifacts(g,h);for(let e of l)await e.reporter.onRunComplete(scopeSummary(h,e.evalIds));return h}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/session-manager.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{createEveSessionStubsRoutePath}from"#protocol/routes.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{EvalSessionDriver}from"#evals/session.js";import{cleanupEvalSessions}from"#evals/session-cleanup.js";const log=createLogger(`eve.evals`);var EvalSessionManager=class{#e;#t;#n;#r;#i=[];#a=[];#o;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#r=e.onSessionStart,this.#t=e.signal}async session(e={}){let{session:t}=await this.#e.sessions.create({...e,signal:e.signal??this.#t});return e.stubs!==void 0&&this.#a.push({sessionId:t.state.sessionId,headers:e.headers,ruleIds:e.stubs.map(e=>e.id)}),this.#s(t)}async verifyStubs(){for(let e of this.#a){let t=await this.#e.fetch(createEveSessionStubsRoutePath(e.sessionId),{headers:e.headers,signal:this.#t});if(!t.ok)throw Error(`Could not verify tool stubs for session ${e.sessionId} (HTTP ${t.status}).`);let n=await t.json();if(typeof n!=`object`||!n||!(`error`in n))throw Error(`Invalid tool stub verification response.`);if(n.error!==null)throw Error(`Tool stubbing failed: ${String(n.error)}`);if(!(`matchedRuleIds`in n)||!Array.isArray(n.matchedRuleIds)||!n.matchedRuleIds.every(e=>typeof e==`string`))throw Error(`Invalid tool stub verification response.`);let r=n.matchedRuleIds,i=e.ruleIds.filter(e=>!r.includes(e));i.length>0&&log.warn(`Tool stubs did not match any calls: ${i.join(`, `)}.`,{sessionId:e.sessionId})}}async send(e,t={}){let{session:n,response:r}=await this.#e.sessions.create({turnPolicy:`queue`,...t,message:e,signal:t.signal??this.#t});return await this.#s(n).consume(r,e).result()}async attachSession(e,t){let n=this.#c(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#c(e,t).watchTurn(t)}snapshots(){return this.#i.map(e=>e.snapshot())}lastTurnSession(){return this.#o}hasActivity(){return this.#i.length>0}async cleanup(e){return await cleanupEvalSessions(this.#i,e)}#s(e){let t=new EvalSessionDriver({collector:this.#n,onSessionStart:this.#r,onTurn:e=>{this.#o=e},primary:this.#i.length===0,session:e,signal:this.#t});return this.#i.push(t),t}#c(e,t){return this.#s(this.#e.sessions.attach(e,{streamIndex:t?.startIndex??0}))}};export{EvalSessionManager};
+import{createLogger}from"#internal/logging.js";import{createEveSessionStubsRoutePath}from"#protocol/routes.js";import{getMessageResponseReceipt}from"#client/message-response.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{EvalSessionDriver}from"#evals/session.js";import{cleanupEvalSessions}from"#evals/session-cleanup.js";const log=createLogger(`eve.evals`);var EvalSessionManager=class{#e;#t;#n;#r;#i=[];#a=[];#o;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#r=e.onSessionStart,this.#t=e.signal}async session(e={}){let t=Date.now(),{session:n}=await this.#e.sessions.create({...e,signal:e.signal??this.#t}),r={startedAt:t,notReadyMs:0,acceptedAt:Date.now()};e.stubs!==void 0&&this.#a.push({sessionId:n.state.sessionId,headers:e.headers,ruleIds:e.stubs.map(e=>e.id)});let i=this.#s(n);return i.recordSend(`prewarm`,r),i}async verifyStubs(){for(let e of this.#a){let t=await this.#e.fetch(createEveSessionStubsRoutePath(e.sessionId),{headers:e.headers,signal:this.#t});if(!t.ok)throw Error(`Could not verify tool stubs for session ${e.sessionId} (HTTP ${t.status}).`);let n=await t.json();if(typeof n!=`object`||!n||!(`error`in n))throw Error(`Invalid tool stub verification response.`);if(n.error!==null)throw Error(`Tool stubbing failed: ${String(n.error)}`);if(!(`matchedRuleIds`in n)||!Array.isArray(n.matchedRuleIds)||!n.matchedRuleIds.every(e=>typeof e==`string`))throw Error(`Invalid tool stub verification response.`);let r=n.matchedRuleIds,i=e.ruleIds.filter(e=>!r.includes(e));i.length>0&&log.warn(`Tool stubs did not match any calls: ${i.join(`, `)}.`,{sessionId:e.sessionId})}}async send(e,t={}){let{session:n,response:r}=await this.#e.sessions.create({turnPolicy:`queue`,...t,message:e,signal:t.signal??this.#t}),i=this.#s(n);return i.recordSend(`create`,getMessageResponseReceipt(r)),await i.consume(r,e).result()}async attachSession(e,t){let n=this.#c(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#c(e,t).watchTurn(t)}snapshots(){return this.#i.map(e=>e.snapshot())}lastTurnSession(){return this.#o}hasActivity(){return this.#i.length>0}async cleanup(e){return await cleanupEvalSessions(this.#i,e)}#s(e){let t=new EvalSessionDriver({collector:this.#n,onSessionStart:this.#r,onTurn:e=>{this.#o=e},primary:this.#i.length===0,session:e,signal:this.#t});return this.#i.push(t),t}#c(e,t){return this.#s(this.#e.sessions.attach(e,{streamIndex:t?.startIndex??0}))}};export{EvalSessionManager};

package/dist/src/evals/session.d.ts

@@ -1,9 +1,10 @@
+import { type ClientSendReceipt } from "#client/message-response.js";
 import type { ClientSession } from "#client/session.js";
 import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions } from "#client/types.js";
 import type { MessageStreamEvent, RuntimeTraceContext, AgentStartedStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#shared/input.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
-import type { EveEvalAgentSession, EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+import type { EveEvalAgentSession, EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSendTiming, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
 import type { EveEvalInputRequestMatchOptions } from "#evals/match.js";
 /**
  * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
@@ -49,6 +50,8 @@ export declare class EvalSessionDriver implements EveEvalSession {
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(message: SendTurnInput["message"], options?: SendTurnOptions): Promise<EveEvalTurn>;
     start(message: string, options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
+    /** @internal Keeps when a request to this session was sent and accepted. */
+    recordSend(kind: EveEvalSendTiming["kind"], receipt: ClientSendReceipt | undefined): void;
     /** @internal */
     consume(events: AsyncIterable<MessageStreamEvent>, message?: SendTurnInput["message"]): EveEvalLiveTurn;
     sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;

package/dist/src/evals/session.js

@@ -1,2 +1,2 @@
-import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
-`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{getMessageResponseReceipt}from"#client/message-response.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o=[];#s=[];#c;#l;#u=[];#d=new Set;#f;#p=!1;#m=[];constructor(e){this.#i=e.collector,this.#l=e.onSessionStart,this.#c=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#y()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#y()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#f}get pendingInputRequests(){return this.#m}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#m.length===0&&this.#b(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#m.filter(t=>inputRequestMatches(t,e));return(this.#m.length!==1||t.length!==1)&&this.#b(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#m.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#h({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#h({...t,inputResponses:e})}async respondAll(e){let t=this.#m;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#h({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#h({turnPolicy:`queue`,...t,message:e})}async#h(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.recordSend(t===void 0?`message`:`respond`,getMessageResponseReceipt(i)),this.consume(i,n)}recordSend(e,t){t!==void 0&&this.#s.push({kind:e,...t})}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#g(n,e),record:(e,r)=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
+`)),this.#v(n,e,r)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#c,sessionId:e,state:this.#e.state,traceContexts:[...this.#u],clock:{receivedAt:[...this.#o],sends:[...this.#s]}}}#g(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#d.has(r)||(this.#d.add(r),this.#u.push(n),!this.#p&&(this.#p=!0,this.#l?.({primary:this.#c,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#_(e){this.#a.push(...e.events),this.#o.push(...e.receivedAt),this.#m=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#f=n,this.#t(this),n}#v(e,t,n){let r=summarizeTurnEvents(t);return this.#_({data:extractCompletedResult(t),events:t,inputRequests:r.inputRequests,message:r.message,receivedAt:n,sessionId:e,status:r.status})}#y(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#f),status:this.#f?.status??`completed`}}#b(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=[];#r=new Set;#i;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#a(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#i!==void 0)throw this.#i;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#r.add(n)})}async#a(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n),this.#n.push(Date.now());let e=i.observe(n);if(t(n),this.#o(n),isTurnFailureEvent(n)&&this.#s(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#s(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t,this.#n)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#s(t),e}}#o(e){for(let t of this.#r)t.matches(e)&&(this.#r.delete(t),t.resolve(e))}#s(e){if(this.#i===void 0){this.#i=e;for(let t of this.#r)t.reject(e);this.#r.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};

package/dist/src/evals/types.d.ts

@@ -111,6 +111,34 @@ export interface EveEvalSessionResult {
     readonly state: ClientSessionState | undefined;
     /** Distinct trace contexts observed for this session, in stream order. */
     readonly traceContexts: readonly RuntimeTraceContext[];
+    /**
+     * When the eval client sent to this session and received its events, for
+     * measuring eve's own overhead against the server's `meta.at` times.
+     */
+    readonly clock?: EveEvalSessionClock;
+}
+/** Client-clock timing for one captured session, in epoch milliseconds. */
+export interface EveEvalSessionClock {
+    /** When each entry of `events` arrived at the eval client, index-aligned. */
+    readonly receivedAt: readonly number[];
+    /** Every request the eval sent to this session, in send order. */
+    readonly sends: readonly EveEvalSendTiming[];
+}
+/**
+ * One request the eval sent: `prewarm` created the session without a message,
+ * `create` created it with its first message (whose turn is the session's
+ * first), `message` sent a message, and `respond` answered input requests.
+ */
+export interface EveEvalSendTiming {
+    readonly kind: "prewarm" | "create" | "message" | "respond";
+    /** The delivery the server accepted, which the events of its turn carry. */
+    readonly deliveryId?: string;
+    /** The first attempt was issued. */
+    readonly startedAt: number;
+    /** Retrying `409 session_not_ready` before the accepted attempt was issued. */
+    readonly notReadyMs: number;
+    /** The accepted response arrived. */
+    readonly acceptedAt: number;
 }
 /** Trace context attributed to one session involved in an eval. */
 export interface EveEvalTraceContext extends RuntimeTraceContext {
@@ -570,6 +598,8 @@ export type EveEvalVerdict = "passed" | "failed" | "scored" | "skipped";
  */
 export interface EveEvalResult {
     readonly id: string;
+    /** 1-based attempt number, present only when the run repeats evals. */
+    readonly attempt?: number;
     readonly result: EveEvalTaskResult;
     /** Every assertion recorded by the eval's `test(t)`, in record order. */
     readonly assertions: readonly AssertionResult[];
@@ -641,6 +671,12 @@ export interface EveEvalConfigInput<TContext = unknown> {
      * overrides it, and `eve eval --timeout` overrides both.
      */
     readonly timeoutMs?: number;
+    /**
+     * Number of times to run every eval, to measure repeatability. Must be a
+     * positive integer. `eve eval --repeat` overrides it; defaults to 1. Each
+     * attempt is a separate result tagged with its 1-based `attempt`.
+     */
+    readonly repeat?: number;
 }
 /**
  * Validated eval run configuration returned by `defineEvalConfig()`. The

package/dist/src/execution/dynamic-workflow/tool.js

@@ -1 +1 @@
-import{runJsProgram}from"#execution/dynamic-workflow/workflow.js";async function runWorkflowProgramTask(e,t){"use workflow";return runJsProgram(e.js,t,{maxSubagents:e.maxSubagents??0})}runWorkflowProgramTask.workflowId=`workflow//eve@0.83.0//runWorkflowProgramTask`;export{runWorkflowProgramTask};
+import{runJsProgram}from"#execution/dynamic-workflow/workflow.js";async function runWorkflowProgramTask(e,t){"use workflow";return runJsProgram(e.js,t,{maxSubagents:e.maxSubagents??0})}runWorkflowProgramTask.workflowId=`workflow//eve@0.83.1//runWorkflowProgramTask`;export{runWorkflowProgramTask};

package/dist/src/execution/sandbox/ensure.d.ts

@@ -35,6 +35,14 @@ export interface EnsuredSandboxAccess extends SandboxAccess {
      * leaves that connection behind.
      */
     release(): Promise<void>;
+    /**
+     * Resumes the sandbox the session recorded, in the background, so a turn
+     * that needs it waits only for the rest. Never starts one: does nothing
+     * when nothing is recorded, and leaves a gone sandbox for `get()`.
+     */
+    resumeRecorded(): void;
+    /** Waits for the work `prepare()` and `resumeRecorded()` queued; never throws. */
+    settle(): Promise<void>;
 }
 export declare function ensureSandboxAccess(input: EnsureSandboxAccessInput): Promise<EnsuredSandboxAccess>;
 export {};