eve 0.34.0 → 0.36.0

package/dist/src/compiler/normalize-extension.js

@@ -1 +1 @@
-import{join,relative}from"node:path";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";async function compileExtensionContributions(e){let{mount:t,consumerAgentRoot:n}=e,r={externalDependencies:e.externalDependencies},i=await composeManifestContributions({manifest:t.manifest,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext:${t.namespace}`,role:`extension`});if(t.overrides===void 0)return i.contributions;let a=await composeManifestContributions({manifest:t.overrides,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext-override:${t.namespace}`,role:`override`});return applyOverrideDisables({merged:mergeContributions(a.contributions,i.contributions),disables:a.disabledToolTargets,extensionToolNames:new Set(i.contributions.tools.map(e=>e.name)),extensionDynamicToolSlugs:new Set(i.contributions.dynamicTools.map(e=>e.slug)),namespace:t.namespace})}function applyOverrideDisables(e){if(e.disables.length===0)return e.merged;let t=e.namespace.length+2,n=new Set;for(let r of e.disables){if(!e.extensionToolNames.has(r.name)&&!e.extensionDynamicToolSlugs.has(r.name)){let n=[...e.extensionToolNames,...e.extensionDynamicToolSlugs].map(e=>e.slice(t)).sort();throw Error(`The override "agent/extensions/${e.namespace}/${r.logicalPath}" calls disableTool(), but the "${e.namespace}" extension contributes no tool named "${r.name.slice(t)}". It contributes: ${n.length>0?n.join(`, `):`(no tools)`}.`)}n.add(r.name)}return{...e.merged,tools:e.merged.tools.filter(e=>!n.has(e.name)),dynamicTools:e.merged.dynamicTools.filter(e=>!n.has(e.slug))}}async function composeManifestContributions(o){let{manifest:s,namespace:c,consumerAgentRoot:l,options:u,sourceIdScope:d,role:f}=o,p=s.agentRoot,m=`${c}__`,scopeSourceId=e=>`${d}:${e}`,rebase=n=>relative(l,join(p,n)).replaceAll(`\\`,`/`),h=[],g=[],_=[];for(let e of s.tools){let t=await compileToolEntry(p,e,u);if(t.kind===`tool`)h.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`dynamic-tool`)g.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`workflow-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} enables the Workflow tool, but the Workflow tool is the consuming agent's to enable, not an extension's. Remove it.`);else if(t.kind===`web-search-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} configures web search, but the web search provider is the consuming agent's to configure, not an extension's. Remove it.`);else if(f===`extension`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} calls disableTool(), but an extension cannot disable framework tools — that is the consuming agent's to own. Remove it.`);else _.push({name:`${m}${t.name}`,logicalPath:e.logicalPath})}let v=s.hooks.map(e=>{let t=compileHookEntry(e);return{...t,slug:`${m}${t.slug}`,sourceId:scopeSourceId(t.sourceId),logicalPath:rebase(t.logicalPath)}}),y=[],b=[];for(let e of s.skills){let t=await compileSkillSource(p,e,u);t.kind===`skill`?y.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):b.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}let x=(await Promise.all(s.connections.map(e=>compileConnectionDefinition(p,e,u)))).map(e=>({...e,connectionName:`${m}${e.connectionName}`,sourceId:scopeSourceId(e.sourceId),logicalPath:rebase(e.logicalPath)})),S=[],C=[];for(let e of s.instructions){let t=await compileInstructionsEntry(p,e,u);t.kind===`instructions`?C.push(t.definition.markdown):S.push({...t.definition,slug:`${m}${t.definition.slug}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}return{contributions:{tools:h,dynamicTools:g,hooks:v,skills:y,dynamicSkills:b,dynamicInstructions:S,connections:x,instructionFragments:C},disabledToolTargets:_}}function describeExtensionSource(e,t,n){return e===`override`?`The override "agent/extensions/${t}/${n}"`:`The "${t}" extension's "${n}"`}function mergeContributions(e,t){return{tools:dedupeBy([...e.tools,...t.tools],e=>e.name),dynamicTools:dedupeBy([...e.dynamicTools,...t.dynamicTools],e=>e.slug),connections:dedupeBy([...e.connections,...t.connections],e=>e.connectionName),skills:dedupeBy([...e.skills,...t.skills],e=>e.name),hooks:[...e.hooks,...t.hooks],dynamicSkills:[...e.dynamicSkills,...t.dynamicSkills],dynamicInstructions:[...e.dynamicInstructions,...t.dynamicInstructions],instructionFragments:[...e.instructionFragments,...t.instructionFragments]}}function dedupeBy(e,t){let n=new Set,r=[];for(let i of e){let e=t(i);n.has(e)||(n.add(e),r.push(i))}return r}export{applyOverrideDisables,compileExtensionContributions,mergeContributions};
+import{join,relative}from"node:path";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";async function compileExtensionContributions(e){let{mount:t,consumerAgentRoot:n}=e,r={externalDependencies:e.externalDependencies},i=await composeManifestContributions({manifest:t.manifest,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext:${t.namespace}`,role:`extension`});if(t.overrides===void 0)return i.contributions;let a=await composeManifestContributions({manifest:t.overrides,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext-override:${t.namespace}`,role:`override`});return applyOverrideDisables({merged:mergeContributions(a.contributions,i.contributions),disables:a.disabledToolTargets,extensionToolNames:new Set(i.contributions.tools.map(e=>e.name)),extensionDynamicToolSlugs:new Set(i.contributions.dynamicTools.map(e=>e.slug)),namespace:t.namespace})}function applyOverrideDisables(e){if(e.disables.length===0)return e.merged;let t=e.namespace.length+2,n=new Set;for(let r of e.disables){if(!e.extensionToolNames.has(r.name)&&!e.extensionDynamicToolSlugs.has(r.name)){let n=[...e.extensionToolNames,...e.extensionDynamicToolSlugs].map(e=>e.slice(t)).sort();throw Error(`The override "agent/extensions/${e.namespace}/${r.logicalPath}" calls disableTool(), but the "${e.namespace}" extension contributes no tool named "${r.name.slice(t)}". It contributes: ${n.length>0?n.join(`, `):`(no tools)`}.`)}n.add(r.name)}return{...e.merged,tools:e.merged.tools.filter(e=>!n.has(e.name)),dynamicTools:e.merged.dynamicTools.filter(e=>!n.has(e.slug))}}async function composeManifestContributions(o){let{manifest:s,namespace:c,consumerAgentRoot:l,options:u,sourceIdScope:d,role:f}=o,p=s.agentRoot,m=`${c}__`,scopeSourceId=e=>`${d}:${e}`,rebase=n=>relative(l,join(p,n)).replaceAll(`\\`,`/`),h=[],g=[],_=[];for(let e of s.tools){let t=await compileToolEntry(p,e,u);if(t.kind===`tool`)h.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`dynamic-tool`)g.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`workflow-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} enables the Workflow tool, but the Workflow tool is the consuming agent's to enable, not an extension's. Remove it.`);else if(t.kind===`web-search-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} configures web search, but the web search provider is the consuming agent's to configure, not an extension's. Remove it.`);else if(f===`extension`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} calls disableTool(), but an extension cannot disable framework tools — that is the consuming agent's to own. Remove it.`);else _.push({name:`${m}${t.name}`,logicalPath:e.logicalPath})}let v=s.hooks.map(e=>{let t=compileHookEntry(e);return{...t,slug:`${m}${t.slug}`,sourceId:scopeSourceId(t.sourceId),logicalPath:rebase(t.logicalPath)}}),y=[],b=[];for(let e of s.skills){let t=await compileSkillSource(p,e,u);t.kind===`skill`?y.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):b.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}let x=(await Promise.all(s.connections.map(e=>compileConnectionDefinition(p,e,u)))).map(e=>({...e,connectionName:`${m}${e.connectionName}`,sourceId:scopeSourceId(e.sourceId),logicalPath:rebase(e.logicalPath)})),S=[],C=[];for(let e of s.instructions){let t=await compileInstructionsEntry(p,e,u);t.kind===`instructions`?C.push({...t.definition,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):S.push({...t.definition,slug:`${m}${t.definition.slug}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}return{contributions:{tools:h,dynamicTools:g,hooks:v,skills:y,dynamicSkills:b,dynamicInstructions:S,connections:x,instructions:C},disabledToolTargets:_}}function describeExtensionSource(e,t,n){return e===`override`?`The override "agent/extensions/${t}/${n}"`:`The "${t}" extension's "${n}"`}function mergeContributions(e,t){return{tools:dedupeBy([...e.tools,...t.tools],e=>e.name),dynamicTools:dedupeBy([...e.dynamicTools,...t.dynamicTools],e=>e.slug),connections:dedupeBy([...e.connections,...t.connections],e=>e.connectionName),skills:dedupeBy([...e.skills,...t.skills],e=>e.name),hooks:[...e.hooks,...t.hooks],dynamicSkills:[...e.dynamicSkills,...t.dynamicSkills],dynamicInstructions:[...e.dynamicInstructions,...t.dynamicInstructions],instructions:[...e.instructions,...t.instructions]}}function dedupeBy(e,t){let n=new Set,r=[];for(let i of e){let e=t(i);n.has(e)||(n.add(e),r.push(i))}return r}export{applyOverrideDisables,compileExtensionContributions,mergeContributions};

package/dist/src/compiler/normalize-instructions.d.ts

@@ -21,7 +21,7 @@ export type CompiledInstructionsEntry = {
  * by the runtime.
  *
  * Module-backed static instructions sources execute once at build time —
- * the resulting markdown is captured into the compiled manifest. There is
+ * the resulting content is captured into the compiled manifest. There is
  * no per-session re-evaluation at runtime.
  *
  * Module-backed dynamic instructions (exporting `defineDynamic`) are

package/dist/src/compiler/normalize-instructions.js

@@ -1 +1 @@
-import{stripLogicalPathExtension}from"#discover/filesystem.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{assertResolverOnlyDynamicSentinel,isDynamicSentinel}from"#shared/dynamic-tool-definition.js";async function compileInstructionsEntry(i,a,o={}){if(a.sourceKind===`markdown`){let n=normalizeInstructionsDefinition(a.definition,`Expected the compiled instructions definition at "${a.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(a.logicalPath),logicalPath:a.logicalPath,markdown:n.markdown,sourceId:a.sourceId,sourceKind:a.sourceKind}}}let s=await loadModuleBackedDefinition({agentRoot:i,externalDependencies:o.externalDependencies,kind:`instructions`,source:a});if(isDynamicSentinel(s)){assertResolverOnlyDynamicSentinel(s,`Expected the instructions export "${a.exportName??`default`}" from "${a.logicalPath}" to match the public eve shape.`);let t=stripLogicalPathExtension(a.logicalPath).replace(/^instructions\//,``);return{kind:`dynamic-instructions`,definition:{eventNames:Object.keys(s.events),exportName:a.exportName,logicalPath:a.logicalPath,slug:t,sourceId:a.sourceId,sourceKind:`module`}}}let c=normalizeInstructionsDefinition(s,`Expected the instructions export "${a.exportName??`default`}" from "${a.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(a.logicalPath),logicalPath:a.logicalPath,markdown:c.markdown,sourceId:a.sourceId,sourceKind:a.sourceKind}}}async function compileInstructions(e,t,n={}){let r=await compileInstructionsEntry(e,t,n);if(r.kind===`dynamic-instructions`)throw Error(`Expected static instructions from "${t.logicalPath}" but got a dynamic resolver. Use compileInstructionsEntry instead.`);return r.definition}export{compileInstructions,compileInstructionsEntry};
+import{stripLogicalPathExtension}from"#discover/filesystem.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,assertResolverOnlyDynamicSentinel,isDynamicSentinel}from"#shared/dynamic-tool-definition.js";async function compileInstructionsEntry(a,o,s={}){if(o.sourceKind===`markdown`){let n=normalizeInstructionsDefinition(o.definition,`Expected the compiled instructions definition at "${o.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(o.logicalPath),logicalPath:o.logicalPath,content:n.content,role:n.role,sourceId:o.sourceId,sourceKind:o.sourceKind}}}let c=await loadModuleBackedDefinition({agentRoot:a,externalDependencies:s.externalDependencies,kind:`instructions`,source:o});if(isDynamicSentinel(c)){assertResolverOnlyDynamicSentinel(c,`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to match the public eve shape.`);let t=Object.keys(c.events),n=t.find(e=>!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(e));if(n!==void 0)throw Error(`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to use only "session.started" or "turn.started" events. Unsupported event: "${n}".`);let i=stripLogicalPathExtension(o.logicalPath).replace(/^instructions\//,``);return{kind:`dynamic-instructions`,definition:{eventNames:t,exportName:o.exportName,logicalPath:o.logicalPath,slug:i,sourceId:o.sourceId,sourceKind:`module`}}}let l=normalizeInstructionsDefinition(c,`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(o.logicalPath),logicalPath:o.logicalPath,content:l.content,role:l.role,sourceId:o.sourceId,sourceKind:o.sourceKind}}}async function compileInstructions(e,t,n={}){let r=await compileInstructionsEntry(e,t,n);if(r.kind===`dynamic-instructions`)throw Error(`Expected static instructions from "${t.logicalPath}" but got a dynamic resolver. Use compileInstructionsEntry instead.`);return r.definition}export{compileInstructions,compileInstructionsEntry};

package/dist/src/compiler/normalize-manifest.js

@@ -1,3 +1 @@
-import{compileChannelDefinition}from"#compiler/normalize-channel.js";import{ROOT_COMPILED_AGENT_NODE_ID,createCompiledAgentManifest,createCompiledAgentNodeManifest,createCompiledAgentResources}from"#compiler/manifest.js";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";import{mountRefNamespace,packageStateNamespace}from"#discover/extensions.js";import{createCompiledRuntimeModelCatalogLoader}from"#compiler/model-catalog.js";import{compileAgentConfig}from"#compiler/normalize-agent-config.js";import{compileExtensionContributions}from"#compiler/normalize-extension.js";import{compileSandboxDefinition}from"#compiler/normalize-sandbox.js";import{compileScheduleDefinition}from"#compiler/normalize-schedule.js";import{compileSubagentGraph}from"#compiler/normalize-subagent.js";async function compileAgentManifest(e){let r={modelCatalog:createCompiledRuntimeModelCatalogLoader(e.appRoot)},i=await compileAgentNodeManifest(e,r),a=await compileSubagentGraph({appRoot:e.appRoot,compileAgentNodeManifest,compileAgentResources,context:r,externalDependencies:i.config.build?.externalDependencies??[],parentNodeId:ROOT_COMPILED_AGENT_NODE_ID,subagents:e.subagents});return createCompiledAgentManifest({...i,extensionMounts:i.extensionMounts,remoteAgents:a.remoteAgents,subagentEdges:a.edges,subagents:a.nodes})}async function compileAgentNodeManifest(e,t,n={}){let i=Object.hasOwn(n,`agentConfigDefinition`)?await compileAgentConfig(e,t,{definition:n.agentConfigDefinition}):await compileAgentConfig(e,t);if(n.allowWorkflowConfig===!1&&i.experimental?.workflow!==void 0)throw Error(`Workflow runtime configuration is only supported on the root agent config. Remove "experimental.workflow" from "${e.agentId}".`);let a=mergeExternalDependencies(n.externalDependencies,i.build?.externalDependencies),o=a.length===0?i:{...i,build:{...i.build,externalDependencies:a}};return createCompiledAgentNodeManifest({...await compileAgentResources(e,t,{externalDependencies:a}),config:o})}async function compileAgentResources(t,n,r={}){let s=[...r.externalDependencies??[]],c=await Promise.all(t.tools.map(e=>compileToolEntry(t.agentRoot,e,{externalDependencies:s}))),l=[],u=[],d=[],f,p;for(let e of c)e.kind===`tool`?l.push(e.definition):e.kind===`dynamic-tool`?u.push(e.definition):e.kind===`workflow-tool`?f={maxSubagents:e.maxSubagents}:e.kind===`web-search-tool`?p=e.provider:d.push(e.name);let m=(await Promise.all(t.channels.map(n=>compileChannelDefinition(t.agentRoot,n,{externalDependencies:s})))).flat(),h=await Promise.all(t.skills.map(e=>compileSkillSource(t.agentRoot,e,{externalDependencies:s}))),g=[],_=[];for(let e of h)e.kind===`skill`?g.push(e.definition):_.push(e.definition);let v=await Promise.all(t.instructions.map(e=>compileInstructionsEntry(t.agentRoot,e,{externalDependencies:s}))),y=[],b=[];for(let e of v)e.kind===`instructions`?y.push(e.definition):b.push(e.definition);let x=await Promise.all(t.connections.map(e=>compileConnectionDefinition(t.agentRoot,e,{externalDependencies:s}))),S=t.hooks.map(e=>compileHookEntry(e)),C=await Promise.all(t.schedules.map(e=>compileScheduleDefinition(t.agentRoot,e,{externalDependencies:s}))),w=new Set(l.map(e=>e.name)),T=new Set(u.map(e=>e.slug)),E=new Set(x.map(e=>e.connectionName)),D=new Set(g.map(e=>e.name)),O=[];for(let e of[...t.resolvedExtensions].sort((e,t)=>e.namespace.localeCompare(t.namespace))){let r=await compileExtensionContributions({mount:e,context:n,consumerAgentRoot:t.agentRoot,externalDependencies:s});for(let e of r.tools)w.has(e.name)||(w.add(e.name),l.push(e));for(let e of r.dynamicTools)T.has(e.slug)||(T.add(e.slug),u.push(e));for(let e of r.connections)E.has(e.connectionName)||(E.add(e.connectionName),x.push(e));for(let e of r.skills)D.has(e.name)||(D.add(e.name),g.push(e));S.push(...r.hooks),_.push(...r.dynamicSkills),b.push(...r.dynamicInstructions),O.push(...r.instructionFragments)}let k=[...y.map(e=>e.markdown),...O],A=k.length===0?void 0:y.length===1&&O.length===0?y[0]:{name:`instructions`,logicalPath:`instructions`,markdown:k.join(`
-
-`),sourceId:y[0]?.sourceId??`instructions`,sourceKind:`module`};return createCompiledAgentResources({agentRoot:t.agentRoot,appRoot:t.appRoot,channels:m,extensionMounts:compileExtensionMounts(t),connections:x,diagnosticsSummary:t.diagnosticsSummary,disabledFrameworkTools:d,workflowTool:f,webSearchProvider:p,dynamicSkills:_,dynamicTools:u,hooks:S,sandbox:t.sandbox===null?null:await compileSandboxDefinition(t.agentRoot,t.sandbox,{externalDependencies:s}),sandboxWorkspaces:t.sandboxWorkspaces.map(e=>({logicalPath:e.logicalPath,rootEntries:[...e.rootEntries],sourceId:e.sourceId,sourcePath:e.sourcePath})),schedules:C,dynamicInstructions:b,skills:g,instructions:A,tools:l})}function compileExtensionMounts(e){return e.resolvedExtensions.map(t=>{let n=e.extensions.find(e=>mountRefNamespace(e.logicalPath)===t.namespace);return{namespace:t.namespace,packageName:t.packageName,packageNamespace:packageStateNamespace(t.packageName),sourceRoot:t.sourceRoot,mountSourceId:n?.sourceId??`extensions/${t.namespace}`,mountLogicalPath:n?.logicalPath??`extensions/${t.namespace}`}})}function mergeExternalDependencies(...e){let t=new Set;for(let n of e)for(let e of n??[])t.add(e);return[...t]}export{compileAgentManifest};
+import{compileChannelDefinition}from"#compiler/normalize-channel.js";import{ROOT_COMPILED_AGENT_NODE_ID,createCompiledAgentManifest,createCompiledAgentNodeManifest,createCompiledAgentResources}from"#compiler/manifest.js";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";import{mountRefNamespace,packageStateNamespace}from"#discover/extensions.js";import{createCompiledRuntimeModelCatalogLoader}from"#compiler/model-catalog.js";import{compileAgentConfig}from"#compiler/normalize-agent-config.js";import{compileExtensionContributions}from"#compiler/normalize-extension.js";import{compileSandboxDefinition}from"#compiler/normalize-sandbox.js";import{compileScheduleDefinition}from"#compiler/normalize-schedule.js";import{compileSubagentGraph}from"#compiler/normalize-subagent.js";async function compileAgentManifest(e){let r={modelCatalog:createCompiledRuntimeModelCatalogLoader(e.appRoot)},i=await compileAgentNodeManifest(e,r),a=await compileSubagentGraph({appRoot:e.appRoot,compileAgentNodeManifest,compileAgentResources,context:r,externalDependencies:i.config.build?.externalDependencies??[],parentNodeId:ROOT_COMPILED_AGENT_NODE_ID,subagents:e.subagents});return createCompiledAgentManifest({...i,extensionMounts:i.extensionMounts,remoteAgents:a.remoteAgents,subagentEdges:a.edges,subagents:a.nodes})}async function compileAgentNodeManifest(e,t,n={}){let i=Object.hasOwn(n,`agentConfigDefinition`)?await compileAgentConfig(e,t,{definition:n.agentConfigDefinition}):await compileAgentConfig(e,t);if(n.allowWorkflowConfig===!1&&i.experimental?.workflow!==void 0)throw Error(`Workflow runtime configuration is only supported on the root agent config. Remove "experimental.workflow" from "${e.agentId}".`);let a=mergeExternalDependencies(n.externalDependencies,i.build?.externalDependencies),o=a.length===0?i:{...i,build:{...i.build,externalDependencies:a}};return createCompiledAgentNodeManifest({...await compileAgentResources(e,t,{externalDependencies:a}),config:o})}async function compileAgentResources(t,n,r={}){let s=[...r.externalDependencies??[]],c=await Promise.all(t.tools.map(e=>compileToolEntry(t.agentRoot,e,{externalDependencies:s}))),l=[],u=[],d=[],f,p;for(let e of c)e.kind===`tool`?l.push(e.definition):e.kind===`dynamic-tool`?u.push(e.definition):e.kind===`workflow-tool`?f={maxSubagents:e.maxSubagents}:e.kind===`web-search-tool`?p=e.provider:d.push(e.name);let m=(await Promise.all(t.channels.map(n=>compileChannelDefinition(t.agentRoot,n,{externalDependencies:s})))).flat(),h=await Promise.all(t.skills.map(e=>compileSkillSource(t.agentRoot,e,{externalDependencies:s}))),g=[],_=[];for(let e of h)e.kind===`skill`?g.push(e.definition):_.push(e.definition);let v=await Promise.all(t.instructions.map(e=>compileInstructionsEntry(t.agentRoot,e,{externalDependencies:s}))),y=[],b=[];for(let e of v)e.kind===`instructions`?y.push(e.definition):b.push(e.definition);let x=await Promise.all(t.connections.map(e=>compileConnectionDefinition(t.agentRoot,e,{externalDependencies:s}))),S=t.hooks.map(e=>compileHookEntry(e)),C=await Promise.all(t.schedules.map(e=>compileScheduleDefinition(t.agentRoot,e,{externalDependencies:s}))),w=new Set(l.map(e=>e.name)),T=new Set(u.map(e=>e.slug)),E=new Set(x.map(e=>e.connectionName)),D=new Set(g.map(e=>e.name)),O=[];for(let e of[...t.resolvedExtensions].sort((e,t)=>e.namespace.localeCompare(t.namespace))){let r=await compileExtensionContributions({mount:e,context:n,consumerAgentRoot:t.agentRoot,externalDependencies:s});for(let e of r.tools)w.has(e.name)||(w.add(e.name),l.push(e));for(let e of r.dynamicTools)T.has(e.slug)||(T.add(e.slug),u.push(e));for(let e of r.connections)E.has(e.connectionName)||(E.add(e.connectionName),x.push(e));for(let e of r.skills)D.has(e.name)||(D.add(e.name),g.push(e));S.push(...r.hooks),_.push(...r.dynamicSkills),b.push(...r.dynamicInstructions),O.push(...r.instructions)}let k=[...y,...O];return createCompiledAgentResources({agentRoot:t.agentRoot,appRoot:t.appRoot,channels:m,extensionMounts:compileExtensionMounts(t),connections:x,diagnosticsSummary:t.diagnosticsSummary,disabledFrameworkTools:d,workflowTool:f,webSearchProvider:p,dynamicSkills:_,dynamicTools:u,hooks:S,sandbox:t.sandbox===null?null:await compileSandboxDefinition(t.agentRoot,t.sandbox,{externalDependencies:s}),sandboxWorkspaces:t.sandboxWorkspaces.map(e=>({logicalPath:e.logicalPath,rootEntries:[...e.rootEntries],sourceId:e.sourceId,sourcePath:e.sourcePath})),schedules:C,dynamicInstructions:b,skills:g,instructions:k,tools:l})}function compileExtensionMounts(e){return e.resolvedExtensions.map(t=>{let n=e.extensions.find(e=>mountRefNamespace(e.logicalPath)===t.namespace);return{namespace:t.namespace,packageName:t.packageName,packageNamespace:packageStateNamespace(t.packageName),sourceRoot:t.sourceRoot,mountSourceId:n?.sourceId??`extensions/${t.namespace}`,mountLogicalPath:n?.logicalPath??`extensions/${t.namespace}`}})}function mergeExternalDependencies(...e){let t=new Set;for(let n of e)for(let e of n??[])t.add(e);return[...t]}export{compileAgentManifest};

package/dist/src/context/dynamic-instruction-lifecycle.d.ts

@@ -1,7 +1,7 @@
 import type { ModelMessage, SystemModelMessage } from "ai";
 import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
 import type { ResolvedDynamicInstructionsResolver } from "#runtime/types.js";
-import type { ContextContainer } from "#context/container.js";
+import type { AlsContext, ContextContainer } from "#context/container.js";
 import type { ContextKey } from "#context/key.js";
 /**
  * Builds the flattened system messages from session + turn durable keys.
@@ -10,6 +10,10 @@ import type { ContextKey } from "#context/key.js";
 export declare function buildDynamicInstructionMessages(ctx: {
     get<T>(key: ContextKey<T>): T | undefined;
 }): SystemModelMessage[];
+/** Starts the instructions-only virtual context for one lifecycle preamble. */
+export declare function prepareDynamicInstructionPreamble(ctx: AlsContext, messages: readonly ModelMessage[]): void;
+/** Drains user-role results and clears the instructions-only virtual context. */
+export declare function drainDynamicInstructionUserMessages(ctx: AlsContext): ModelMessage[];
 /**
  * Dispatches a stream event to dynamic instruction resolvers.
  *

package/dist/src/context/dynamic-instruction-lifecycle.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{SessionDynamicInstructionsKey,TurnDynamicInstructionsKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,isBrandedInstructionsEntry}from"#shared/dynamic-tool-definition.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";const log=createLogger(`dynamic-instructions`);function lowerToSystemMessage(e){let t=e.markdown.trim();if(t.length!==0)return{role:`system`,content:t}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicInstructionsKey;case`turn.started`:return TurnDynamicInstructionsKey;default:return}}function buildDynamicInstructionMessages(e){let r=e.get(SessionDynamicInstructionsKey)??{},i=e.get(TurnDynamicInstructionsKey)??{};return[...Object.values(r).flat(),...Object.values(i).flat()]}async function dispatchDynamicInstructionEvent(e){let{ctx:t,resolvers:n,event:a,messages:o}=e;if(!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(a.type))return;let s=n.filter(e=>e.eventNames.includes(a.type));if(s.length===0)return;let c=durableKeyForEvent(a.type);if(c===void 0)return;let l=buildResolveContext(t,o),u=await Promise.allSettled(s.map(async e=>{let t=e.events[a.type];if(t===void 0)return null;let n=await t(a,l);return n==null?{resolver:e,message:void 0}:isBrandedInstructionsEntry(n)?{resolver:e,message:lowerToSystemMessage(n)}:(log.error(`Dynamic instructions resolver "${e.slug}" returned an unbranded value — wrap with defineInstructions().`),null)})),d={...t.get(c)};for(let e of u){if(e.status===`rejected`){log.error(`Dynamic instructions resolver (${a.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,message:n}=e.value;n===void 0?delete d[t.slug]:d[t.slug]=[n]}t.set(c,d)}export{buildDynamicInstructionMessages,dispatchDynamicInstructionEvent};
+import{createLogger}from"#internal/logging.js";import{DynamicInstructionResolveMessagesKey,PendingDynamicInstructionUserMessagesKey,SessionDynamicInstructionsKey,TurnDynamicInstructionsKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,isBrandedInstructionsEntry}from"#shared/dynamic-tool-definition.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";const log=createLogger(`dynamic-instructions`);function lowerInstruction(e){let t=normalizeInstructionsDefinition(e,`Expected dynamic instructions to match the public eve shape.`),n=t.content.trim();if(n.length!==0)return t.role===`system`?{role:`system`,message:{role:`system`,content:n}}:{role:`user`,message:{role:`user`,content:n}}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicInstructionsKey;case`turn.started`:return TurnDynamicInstructionsKey;default:return}}function buildDynamicInstructionMessages(e){let t=e.get(SessionDynamicInstructionsKey)??{},n=e.get(TurnDynamicInstructionsKey)??{};return[...Object.values(t).flat(),...Object.values(n).flat()]}function prepareDynamicInstructionPreamble(e,r){e.setVirtualContext(DynamicInstructionResolveMessagesKey,[...r]),e.setVirtualContext(PendingDynamicInstructionUserMessagesKey,[])}function drainDynamicInstructionUserMessages(e){let r=[...e.get(PendingDynamicInstructionUserMessagesKey)??[]];return e.delete(DynamicInstructionResolveMessagesKey),e.delete(PendingDynamicInstructionUserMessagesKey),r}async function dispatchDynamicInstructionEvent(e){let{ctx:r,resolvers:a,event:o,messages:s}=e;if(!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(o.type))return;o.type===`turn.started`&&r.set(TurnDynamicInstructionsKey,{});let c=a.filter(e=>e.eventNames.includes(o.type));if(c.length===0)return;let l=durableKeyForEvent(o.type);if(l===void 0)return;let u=r.get(DynamicInstructionResolveMessagesKey)??s,d=r.get(PendingDynamicInstructionUserMessagesKey)??[],f=buildResolveContext(r,[...u,...d]),p=await Promise.allSettled(c.map(async e=>{let t=e.events[o.type];if(t===void 0)return null;let n=await t(o,f);if(n==null)return{resolver:e,instruction:void 0};if(!isBrandedInstructionsEntry(n))return log.error(`Dynamic instructions resolver "${e.slug}" returned an unbranded value — wrap with defineInstructions().`),null;try{return{resolver:e,instruction:lowerInstruction(n)}}catch(t){return log.error(`Dynamic instructions resolver "${e.slug}" returned an invalid value.`,{error:toErrorMessage(t)}),null}})),m={...r.get(l)};for(let e of p){if(e.status===`rejected`){log.error(`Dynamic instructions resolver (${o.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,instruction:i}=e.value;delete m[t.slug],i?.role===`system`?m[t.slug]=[i.message]:i?.role===`user`&&r.setVirtualContext(PendingDynamicInstructionUserMessagesKey,[...r.get(PendingDynamicInstructionUserMessagesKey)??[],i.message])}r.set(l,m)}export{buildDynamicInstructionMessages,dispatchDynamicInstructionEvent,drainDynamicInstructionUserMessages,prepareDynamicInstructionPreamble};

package/dist/src/context/keys.d.ts

@@ -3,11 +3,11 @@
  * tier. Codec-carrying keys (`ChannelKey`, `BundleKey`) live in
  * `#runtime/sessions/runtime-context-keys.ts`.
  */
-import type { LanguageModel, SystemModelMessage } from "ai";
+import type { LanguageModel, ModelMessage, SystemModelMessage } from "ai";
 import type { JsonObject } from "#shared/json.js";
 import type { ChannelInstrumentationProjection, ChannelDeliveryMetadata, SessionAuthContext, SessionCallback, SessionCapabilities, SessionParent, SessionTraceContext, SessionTurn } from "#channel/types.js";
 import { ContextKey } from "#context/key.js";
-import type { InstrumentationChannelDeliveryRef } from "#harness/instrumentation-lifecycle.js";
+import type { InstrumentationChannelDeliveryRef } from "#harness/instrumentation/lifecycle.js";
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
 import type { DynamicSubagentAgentConfig } from "#runtime/subagents/dynamic-agent-config.js";
 import type { DynamicRemoteAgentConfig } from "#runtime/subagents/dynamic-remote-agent-config.js";
@@ -164,3 +164,7 @@ export declare const SessionDynamicInstructionsKey: ContextKey<Record<string, re
  * resolvers). Keyed by resolver slug. Replaced each turn.
  */
 export declare const TurnDynamicInstructionsKey: ContextKey<Record<string, readonly SystemModelMessage[]>>;
+/** Existing history exposed only to instructions resolvers during a preamble. */
+export declare const DynamicInstructionResolveMessagesKey: ContextKey<readonly ModelMessage[]>;
+/** User-role results waiting to be committed immediately after a preamble. */
+export declare const PendingDynamicInstructionUserMessagesKey: ContextKey<readonly ModelMessage[]>;

package/dist/src/context/keys.js

@@ -1 +1 @@
-import{ContextKey}from"#context/key.js";const AuthKey=new ContextKey(`eve.auth`),InitiatorAuthKey=new ContextKey(`eve.initiatorAuth`),SessionIdKey=new ContextKey(`eve.sessionId`),ContinuationTokenKey=new ContextKey(`eve.continuationToken`),ChannelRequestIdKey=new ContextKey(`eve.channelRequestId`),ChannelDeliveryKey=new ContextKey(`eve.channelDelivery`),ActiveChannelDeliveriesKey=new ContextKey(`eve.activeChannelDeliveries`),ChannelInstrumentationKey=new ContextKey(`eve.channelInstrumentation`),ModeKey=new ContextKey(`eve.mode`),ParentSessionKey=new ContextKey(`eve.parentSession`),ParentTraceContextKey=new ContextKey(`eve.parentTraceContext`),SubagentDepthKey=new ContextKey(`eve.subagentDepth`),CapabilitiesKey=new ContextKey(`eve.capabilities`),SessionCallbackKey=new ContextKey(`eve.sessionCallback`),SessionKey=new ContextKey(`eve.session`),SandboxKey=new ContextKey(`eve.sandbox`),SessionDynamicModelReferenceKey=new ContextKey(`eve.sessionDynamicModelReference`),TurnDynamicModelReferenceKey=new ContextKey(`eve.turnDynamicModelReference`),RuntimeModelMetadataCacheKey=new ContextKey(`eve.runtimeModelMetadataCache`),LiveStepDynamicModelSelectionKey=new ContextKey(`eve.liveStepDynamicModelSelection`),SessionDynamicToolMetadataKey=new ContextKey(`eve.sessionDynamicToolMetadata`),SessionDynamicToolRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicToolRuntimeRevision`),TurnDynamicToolMetadataKey=new ContextKey(`eve.turnDynamicToolMetadata`),LiveStepToolsKey=new ContextKey(`eve.liveStepTools`),SessionDynamicSubagentSelectionsKey=new ContextKey(`eve.sessionDynamicSubagentSelections`),TurnDynamicSubagentSelectionsKey=new ContextKey(`eve.turnDynamicSubagentSelections`),SessionDynamicSubagentRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicSubagentRuntimeRevision`),DynamicSubagentAgentConfigKey=new ContextKey(`eve.dynamicSubagentAgentConfig`),DynamicSkillManifestKey=new ContextKey(`eve.dynamicSkillManifest`),SessionDynamicInstructionsKey=new ContextKey(`eve.sessionDynamicInstructions`),TurnDynamicInstructionsKey=new ContextKey(`eve.turnDynamicInstructions`);export{ActiveChannelDeliveriesKey,AuthKey,CapabilitiesKey,ChannelDeliveryKey,ChannelInstrumentationKey,ChannelRequestIdKey,ContinuationTokenKey,DynamicSkillManifestKey,DynamicSubagentAgentConfigKey,InitiatorAuthKey,LiveStepDynamicModelSelectionKey,LiveStepToolsKey,ModeKey,ParentSessionKey,ParentTraceContextKey,RuntimeModelMetadataCacheKey,SandboxKey,SessionCallbackKey,SessionDynamicInstructionsKey,SessionDynamicModelReferenceKey,SessionDynamicSubagentRuntimeRevisionKey,SessionDynamicSubagentSelectionsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,SessionKey,SubagentDepthKey,TurnDynamicInstructionsKey,TurnDynamicModelReferenceKey,TurnDynamicSubagentSelectionsKey,TurnDynamicToolMetadataKey};
+import{ContextKey}from"#context/key.js";const AuthKey=new ContextKey(`eve.auth`),InitiatorAuthKey=new ContextKey(`eve.initiatorAuth`),SessionIdKey=new ContextKey(`eve.sessionId`),ContinuationTokenKey=new ContextKey(`eve.continuationToken`),ChannelRequestIdKey=new ContextKey(`eve.channelRequestId`),ChannelDeliveryKey=new ContextKey(`eve.channelDelivery`),ActiveChannelDeliveriesKey=new ContextKey(`eve.activeChannelDeliveries`),ChannelInstrumentationKey=new ContextKey(`eve.channelInstrumentation`),ModeKey=new ContextKey(`eve.mode`),ParentSessionKey=new ContextKey(`eve.parentSession`),ParentTraceContextKey=new ContextKey(`eve.parentTraceContext`),SubagentDepthKey=new ContextKey(`eve.subagentDepth`),CapabilitiesKey=new ContextKey(`eve.capabilities`),SessionCallbackKey=new ContextKey(`eve.sessionCallback`),SessionKey=new ContextKey(`eve.session`),SandboxKey=new ContextKey(`eve.sandbox`),SessionDynamicModelReferenceKey=new ContextKey(`eve.sessionDynamicModelReference`),TurnDynamicModelReferenceKey=new ContextKey(`eve.turnDynamicModelReference`),RuntimeModelMetadataCacheKey=new ContextKey(`eve.runtimeModelMetadataCache`),LiveStepDynamicModelSelectionKey=new ContextKey(`eve.liveStepDynamicModelSelection`),SessionDynamicToolMetadataKey=new ContextKey(`eve.sessionDynamicToolMetadata`),SessionDynamicToolRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicToolRuntimeRevision`),TurnDynamicToolMetadataKey=new ContextKey(`eve.turnDynamicToolMetadata`),LiveStepToolsKey=new ContextKey(`eve.liveStepTools`),SessionDynamicSubagentSelectionsKey=new ContextKey(`eve.sessionDynamicSubagentSelections`),TurnDynamicSubagentSelectionsKey=new ContextKey(`eve.turnDynamicSubagentSelections`),SessionDynamicSubagentRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicSubagentRuntimeRevision`),DynamicSubagentAgentConfigKey=new ContextKey(`eve.dynamicSubagentAgentConfig`),DynamicSkillManifestKey=new ContextKey(`eve.dynamicSkillManifest`),SessionDynamicInstructionsKey=new ContextKey(`eve.sessionDynamicInstructions`),TurnDynamicInstructionsKey=new ContextKey(`eve.turnDynamicInstructions`),DynamicInstructionResolveMessagesKey=new ContextKey(`eve.dynamicInstructionResolveMessages`),PendingDynamicInstructionUserMessagesKey=new ContextKey(`eve.pendingDynamicInstructionUserMessages`);export{ActiveChannelDeliveriesKey,AuthKey,CapabilitiesKey,ChannelDeliveryKey,ChannelInstrumentationKey,ChannelRequestIdKey,ContinuationTokenKey,DynamicInstructionResolveMessagesKey,DynamicSkillManifestKey,DynamicSubagentAgentConfigKey,InitiatorAuthKey,LiveStepDynamicModelSelectionKey,LiveStepToolsKey,ModeKey,ParentSessionKey,ParentTraceContextKey,PendingDynamicInstructionUserMessagesKey,RuntimeModelMetadataCacheKey,SandboxKey,SessionCallbackKey,SessionDynamicInstructionsKey,SessionDynamicModelReferenceKey,SessionDynamicSubagentRuntimeRevisionKey,SessionDynamicSubagentSelectionsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,SessionKey,SubagentDepthKey,TurnDynamicInstructionsKey,TurnDynamicModelReferenceKey,TurnDynamicSubagentSelectionsKey,TurnDynamicToolMetadataKey};

package/dist/src/discover/manifest.d.ts

@@ -11,7 +11,7 @@ export declare const AGENT_SOURCE_MANIFEST_KIND = "eve-agent-discovery-manifest"
 /**
  * Current manifest schema version.
  */
-export declare const AGENT_SOURCE_MANIFEST_VERSION = 12;
+export declare const AGENT_SOURCE_MANIFEST_VERSION = 13;
 /**
  * Channel source reference preserved by the discovery manifest.
  */

package/dist/src/discover/manifest.js

@@ -1 +1 @@
-import{basename,relative,resolve}from"node:path";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";import{normalizeLogicalPath}from"#discover/filesystem.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=12;function createAgentSourceManifest(e){let t=resolve(e.appRoot),i=resolve(e.agentRoot),o={agentId:e.agentId??deriveAgentIdFromRoots(t,i,e.packageName),agentRoot:i,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:12,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(o.configModule=e.configModule),o}function deriveAgentIdFromRoots(n,r,a){let o=normalizeLogicalPath(relative(n,r));return o===``||o===`.`||o===`agent`?a??basename(n):basename(r)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};
+import{basename,relative,resolve}from"node:path";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";import{normalizeLogicalPath}from"#discover/filesystem.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=13;function createAgentSourceManifest(e){let t=resolve(e.appRoot),i=resolve(e.agentRoot),o={agentId:e.agentId??deriveAgentIdFromRoots(t,i,e.packageName),agentRoot:i,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:13,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(o.configModule=e.configModule),o}function deriveAgentIdFromRoots(n,r,a){let o=normalizeLogicalPath(relative(n,r));return o===``||o===`.`||o===`agent`?a??basename(n):basename(r)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};

package/dist/src/evals/context.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import"#evals/session.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(n){let i=n.collector,a=``,primary=()=>n.manager.primary,replyMessage=()=>n.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:n.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},cancel:()=>primary().cancel(),requireInputRequest:e=>primary().requireInputRequest(e),respond:(e,t)=>primary().respond(e,t),respondAll:e=>primary().respondAll(e),send:(e,t)=>(a=typeof e==`string`?e:``,primary().send(e,t)),start:(e,t)=>(a=e,primary().start(e,t)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:n.signal,target:n.target,get reply(){return replyMessage()},log:n.log,sleep:e=>sleep(e,n.signal),newSession:()=>n.manager.newSession(),...createScopedAssertions(i,{timing:`final`,select:e=>e}),check:(e,t)=>recordCheck(i,e,t),require:(e,t)=>requireCheck(i,e,t),skip:e=>{throw e.trim().length===0?Error(`skip() requires a non-empty reason.`):i.hasEntries||n.manager.hasActivity()?Error(`skip() must be called before sending messages or recording assertions.`):new EvalSkipped(e)},judge:o},collector:i}}async function requireCheck(e,t,r){let i=r.gate(r.threshold);if(!await e.recordRequirement({name:i.name,threshold:i.threshold,score:()=>evaluateAssertion(i,t)}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:()=>evaluateAssertion(n,t)})}async function evaluateAssertion(e,t){return e.evaluate===void 0?{score:await e.score(t)}:await e.evaluate(t)}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};
+import"#evals/assertions/collector.js";import"#evals/session.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(n){let i=n.collector,a=``,primary=()=>n.manager.primary,replyMessage=()=>n.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:n.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},cancel:()=>primary().cancel(),requireInputRequest:e=>primary().requireInputRequest(e),respond:(e,t)=>primary().respond(e,t),startRespond:(e,t)=>primary().startRespond(e,t),respondAll:e=>primary().respondAll(e),send:(e,t)=>(a=typeof e==`string`?e:``,primary().send(e,t)),start:(e,t)=>(a=e,primary().start(e,t)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:n.signal,target:n.target,get reply(){return replyMessage()},log:n.log,sleep:e=>sleep(e,n.signal),newSession:()=>n.manager.newSession(),...createScopedAssertions(i,{timing:`final`,select:e=>e}),check:(e,t)=>recordCheck(i,e,t),require:(e,t)=>requireCheck(i,e,t),skip:e=>{throw e.trim().length===0?Error(`skip() requires a non-empty reason.`):i.hasEntries||n.manager.hasActivity()?Error(`skip() must be called before sending messages or recording assertions.`):new EvalSkipped(e)},judge:o},collector:i}}async function requireCheck(e,t,r){let i=r.gate(r.threshold);if(!await e.recordRequirement({name:i.name,threshold:i.threshold,score:()=>evaluateAssertion(i,t)}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:()=>evaluateAssertion(n,t)})}async function evaluateAssertion(e,t){return e.evaluate===void 0?{score:await e.score(t)}:await e.evaluate(t)}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};

package/dist/src/evals/index.d.ts

@@ -2,9 +2,9 @@ export { defineEval } from "#evals/define-eval.js";
 export { defineEvalConfig } from "#evals/define-eval-config.js";
 export { EveEvalTurnFailedError } from "#evals/session.js";
 export { mockModel } from "#evals/mock-model.js";
-export type { RuntimeIdentity } from "#protocol/message.js";
+export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";

package/dist/src/evals/reporters/index.d.ts

@@ -1,4 +1,4 @@
 export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js";
 export { Console, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js";
 export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
-export type { EvalReporter } from "#evals/runner/reporters/types.js";
+export type { EvalReporter, EveEvalCompleteContext, EveEvalSessionStartEvent, EveEvalStartEvent, } from "#evals/runner/reporters/types.js";

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{mkdir,writeFile}from"node:fs/promises";import{dirname,join}from"node:path";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/execute-eval.d.ts

@@ -1,5 +1,6 @@
 import type { Client } from "#client/client.js";
 import type { EveEval, EveEvalResult, EveEvalTargetHandle } from "#evals/types.js";
+import type { EvalSessionStartedEvent } from "#evals/session.js";
 /**
  * Options for executing one eval.
  */
@@ -7,9 +8,13 @@ export interface ExecuteEvalOptions {
     readonly evaluation: EveEval;
     /** Receives `t.log` lines as the eval runs (used by `--verbose`). */
     readonly onLog?: (message: string) => void;
+    /** Receives the first trace context observed for each session. */
+    readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
     readonly target: EveEvalTargetHandle;
     /** Overrides the eval's own `timeoutMs` when set (CLI `--timeout`). */
     readonly timeoutMs?: number;
+    /** Runner-owned start time, shared with reporter lifecycle events. */
+    readonly startedAt?: string;
     /**
      * Pre-configured client for communicating with the eve agent.
      * The CLI constructs this once with the appropriate auth and headers,

package/dist/src/evals/runner/execute-eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=n.startedAt??new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,onSessionStart:n.onSessionStart,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts(),traceContexts:[]}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -1,5 +1,6 @@
 import type { Client } from "#client/client.js";
 import type { AssertionResult, EveEval, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
+import { type EvalSessionStartedEvent } from "#evals/session.js";
 /**
  * Options for executing one eval's task.
  */
@@ -8,6 +9,8 @@ interface ExecuteTaskOptions {
     readonly evaluation: EveEval;
     /** Receives each `t.log` line as it is written (used by `--verbose`). */
     readonly onLog?: (message: string) => void;
+    /** Receives the first trace context observed for each session. */
+    readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
     readonly target: EveEvalTargetHandle;
     readonly timeoutMs?: number;
 }

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await runUntilAborted(s.test(h),u)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}async function runUntilAborted(e,t){t.throwIfAborted();let onAbort,n=new Promise((e,n)=>{onAbort=()=>n(t.reason),t.addEventListener(`abort`,onAbort,{once:!0})});try{await Promise.race([e,n])}finally{onAbort!==void 0&&t.removeEventListener(`abort`,onAbort)}}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,onSessionStart:a.onSessionStart,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await runUntilAborted(s.test(h),u)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}async function runUntilAborted(e,t){t.throwIfAborted();let onAbort,n=new Promise((e,n)=>{onAbort=()=>n(t.reason),t.addEventListener(`abort`,onAbort,{once:!0})});try{await Promise.race([e,n])}finally{onAbort!==void 0&&t.removeEventListener(`abort`,onAbort)}}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -1,4 +1,23 @@
-import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js";
+import type { RuntimeTraceContext } from "#protocol/message.js";
+import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget, EveEvalTraceContext } from "#evals/types.js";
+/** Context delivered when one eval is scheduled for execution. */
+export interface EveEvalStartEvent {
+    readonly evaluation: EveEval;
+    readonly startedAt: string;
+    readonly target: EveEvalTarget;
+}
+/** Context delivered once the runner observes a session's first trace. */
+export interface EveEvalSessionStartEvent extends EveEvalStartEvent {
+    readonly primary: boolean;
+    readonly sessionId: string;
+    readonly traceContext: RuntimeTraceContext;
+}
+/** Additional run context delivered with a completed eval result. */
+export interface EveEvalCompleteContext {
+    readonly evaluation: EveEval;
+    readonly target: EveEvalTarget;
+    readonly traceContexts: readonly EveEvalTraceContext[];
+}
 /**
  * Reporter lifecycle interface. The runner calls these methods at defined
  * points during an eval run. Methods may return a promise for reporters
@@ -13,11 +32,18 @@ export interface EvalReporter {
      * this reporter observes.
      */
     onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): void | Promise<void>;
+    /** The runner calls this when an observed eval is scheduled. */
+    onEvalStart?(event: EveEvalStartEvent): void | Promise<void>;
+    /**
+     * The runner calls this once per session when its first trace context is
+     * observed. Sessions without trace instrumentation do not trigger it.
+     */
+    onSessionStart?(event: EveEvalSessionStartEvent): void | Promise<void>;
     /**
      * The runner calls this after each observed eval completes, with its
      * checks, scores, and verdict.
      */
-    onEvalComplete(result: EveEvalResult): void | Promise<void>;
+    onEvalComplete(result: EveEvalResult, context?: EveEvalCompleteContext): void | Promise<void>;
     /**
      * The runner calls this once when the run finishes, with the aggregated
      * summary of the evals this reporter observes.

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve(),enqueueReporterCallback=(e,t)=>{h=h.then(async()=>{for(let n of d)n.evalIds.has(e.id)&&await t(n.reporter)})};for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=new Date().toISOString();enqueueReporterCallback(e,async n=>{await n.onEvalStart?.({evaluation:e,startedAt:t,target:a})});let i=(async()=>{let i=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),onSessionStart:t=>{enqueueReporterCallback(e,async n=>{await n.onSessionStart?.({evaluation:e,primary:t.primary,sessionId:t.sessionId,startedAt:t.startedAt,target:a,traceContext:t.traceContext})})},startedAt:t,target:a,timeoutMs:r.timeoutMs});f.push(i),enqueueReporterCallback(e,async t=>{await t.onEvalComplete(i,{evaluation:e,target:a,traceContexts:i.result.traceContexts})})})().finally(()=>{m.delete(i)});m.add(i)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/session.d.ts

@@ -1,7 +1,7 @@
 import type { Client } from "#client/client.js";
 import type { ClientSession } from "#client/session.js";
 import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions } from "#client/types.js";
-import type { MessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
+import type { MessageStreamEvent, RuntimeTraceContext, TurnFailureStreamEvent } from "#protocol/message.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
 import { AssertionCollector } from "#evals/assertions/collector.js";
 import type { EveEvalAssertions, EveEvalLiveTurn, EveEvalOutputAssertions, EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
@@ -16,11 +16,19 @@ export declare class EveEvalTurnFailedError extends Error {
 }
 export interface EvalSessionDriver extends EveEvalAssertions, EveEvalOutputAssertions {
 }
+export interface EvalSessionStartedEvent {
+    readonly primary: boolean;
+    readonly sessionId: string;
+    readonly startedAt: string;
+    readonly traceContext: RuntimeTraceContext;
+}
 export declare class EvalSessionDriver implements EveEvalSession {
     #private;
     constructor(input: {
         readonly client: Client;
         readonly collector: AssertionCollector;
+        readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
+        readonly primary: boolean;
         readonly session?: ClientSession;
         readonly signal?: AbortSignal;
     });
@@ -32,6 +40,7 @@ export declare class EvalSessionDriver implements EveEvalSession {
     cancel(): Promise<CancelSessionResult>;
     requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     respond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalTurn>;
+    startRespond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
     respondAll(optionId: string): Promise<EveEvalTurn>;
     send(message: SendTurnInput["message"], options?: SendTurnOptions): Promise<EveEvalTurn>;
     start(message: string, options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
@@ -42,13 +51,14 @@ export declare class EvalSessionDriver implements EveEvalSession {
     watchTurn(options?: {
         readonly startIndex?: number;
     }, sessionId?: string): EveEvalLiveTurn;
-    snapshot(primary: boolean): EveEvalSessionResult;
+    snapshot(): EveEvalSessionResult;
 }
 export declare class EvalSessionManager {
     #private;
     constructor(input: {
         readonly client: Client;
         readonly collector?: AssertionCollector;
+        readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
         readonly signal?: AbortSignal;
     });
     get primary(): EvalSessionDriver;

package/dist/src/evals/session.js

@@ -1 +1 @@
-import{readFile}from"node:fs/promises";import{basename,extname}from"node:path";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{summarizeTurnEvents}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r;#i=[];#a;#o=[];constructor(e){this.#e=e.client,this.#r=e.collector,this.#t=e.session,this.#n=e.signal,Object.assign(this,createScopedAssertions(this.#r,{timing:`snapshot`,select:()=>this.#u()}),createOutputAssertions(this.#r,{timing:`snapshot`,select:()=>this.#u()}))}get events(){return this.#i}get lastTurn(){return this.#a}get pendingInputRequests(){return this.#o}get sessionId(){return this.#t?.state.sessionId??this.#a?.sessionId}get state(){return this.#t?.state}async cancel(){if(this.#t===void 0)throw Error(`Eval session has not started.`);return await this.#t.cancel()}requireInputRequest(e={}){this.#o.length===0&&this.#d(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#o.filter(t=>inputRequestMatches(t,e));return(this.#o.length!==1||t.length!==1)&&this.#d(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#o.length} pending request(s)`),this.#r.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#s({...t,inputResponses:e})).result()}async respondAll(e){let t=this.#o;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#s({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#s({turnPolicy:`queue`,...t,message:e})}async#s(e){let t=attachSignal(e,this.#n),n;if(this.#t===void 0){if(t.message===void 0)throw Error(`Eval session has not started.`);let e=await this.#e.sessions.create({...t,message:t.message});this.#t=e.session,n=e.response}else{let{inputResponses:e,message:r,...i}=t;n=e===void 0?await this.#t.send(r,i):await this.#t.respond(e,i)}return new EvalLiveTurn({events:n,record:e=>this.#l(n.sessionId,e),session:this,sessionId:n.sessionId})}async sendFile(n,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:n});return await this.send(a)}async readTurn(e){let t=this.sessionId;return await this.watchTurn(e,requireSessionId(t)).result()}watchTurn(e,t=requireSessionId(this.sessionId)){if(this.#t===void 0)throw Error(`Eval session has not started.`);return new EvalLiveTurn({events:this.#t.stream({signal:this.#n,startIndex:e?.startIndex}),record:e=>this.#l(t,e),session:this,sessionId:t})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#i,{sessionId:t}),events:[...this.#i],primary:e,sessionId:t,state:this.#t?.state}}#c(e){this.#i.push(...e.events),this.#o=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#r,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#a=n,n}#l(e,t){let n=summarizeTurnEvents(t);return this.#c({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#u(){let e=this.sessionId;return{derived:deriveRunFacts(this.#i,{sessionId:e}),events:[...this.#i],output:outputOf(this.#a),status:this.#a?.status??`completed`}}#d(e,t){throw this.#r.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t){try{let n=!1;for await(let t of e)if(this.#t.push(t),this.#a(t),isTurnFailureEvent(t)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${t.type}).`)),isCurrentTurnBoundaryEvent(t)){n=!0,this.#o(Error(`Session ${this.sessionId} reached ${t.type} before the expected event.`));break}if(!n)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return t(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r=[];#i;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#t=e.signal}get primary(){return this.#i??=this.#a(),this.#i}newSession(){return this.#a()}async attachSession(e,t){let n=this.#o(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#o(e,t).watchTurn(t,e)}snapshots(){return this.#r.map(e=>e.snapshot(e===this.#i))}lastTurnSession(){return this.#i?.lastTurn===void 0?this.#r.find(e=>e.lastTurn!==void 0):this.#i}hasActivity(){return this.#r.length>0}#a(){let e=new EvalSessionDriver({client:this.#e,collector:this.#n,signal:this.#t});return this.#r.push(e),e}#o(e,t){let n=new EvalSessionDriver({client:this.#e,collector:this.#n,session:this.#e.sessions.attach(e,{streamIndex:t?.startIndex??0}),signal:this.#t});return this.#r.push(n),n}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{basename,extname}from"node:path";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{summarizeTurnEvents}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r;#i=[];#a;#o;#s=[];#c=new Set;#l;#u=!1;#d=[];constructor(e){this.#e=e.client,this.#r=e.collector,this.#o=e.onSessionStart,this.#a=e.primary,this.#t=e.session,this.#n=e.signal,Object.assign(this,createScopedAssertions(this.#r,{timing:`snapshot`,select:()=>this.#g()}),createOutputAssertions(this.#r,{timing:`snapshot`,select:()=>this.#g()}))}get events(){return this.#i}get lastTurn(){return this.#l}get pendingInputRequests(){return this.#d}get sessionId(){return this.#t?.state.sessionId??this.#l?.sessionId}get state(){return this.#t?.state}async cancel(){if(this.#t===void 0)throw Error(`Eval session has not started.`);return await this.#t.cancel()}requireInputRequest(e={}){this.#d.length===0&&this.#_(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#d.filter(t=>inputRequestMatches(t,e));return(this.#d.length!==1||t.length!==1)&&this.#_(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#d.length} pending request(s)`),this.#r.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#f({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#f({...t,inputResponses:e})}async respondAll(e){let t=this.#d;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#f({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#f({turnPolicy:`queue`,...t,message:e})}async#f(e){let t=attachSignal(e,this.#n),n;if(this.#t===void 0){if(t.message===void 0)throw Error(`Eval session has not started.`);let e=await this.#e.sessions.create({...t,message:t.message});this.#t=e.session,n=e.response}else{let{inputResponses:e,message:r,...i}=t;n=e===void 0?await this.#t.send(r,i):await this.#t.respond(e,i)}return new EvalLiveTurn({events:n,observe:e=>this.#p(n.sessionId,e),record:e=>this.#h(n.sessionId,e),session:this,sessionId:n.sessionId})}async sendFile(n,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:n});return await this.send(a)}async readTurn(e){let t=this.sessionId;return await this.watchTurn(e,requireSessionId(t)).result()}watchTurn(e,t=requireSessionId(this.sessionId)){if(this.#t===void 0)throw Error(`Eval session has not started.`);return new EvalLiveTurn({events:this.#t.stream({signal:this.#n,startIndex:e?.startIndex}),observe:e=>this.#p(t,e),record:e=>this.#h(t,e),session:this,sessionId:t})}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#i,{sessionId:e}),events:[...this.#i],primary:this.#a,sessionId:e,state:this.#t?.state,traceContexts:[...this.#s]}}#p(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#c.has(r)||(this.#c.add(r),this.#s.push(n),!this.#u&&(this.#u=!0,this.#o?.({primary:this.#a,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#m(e){this.#i.push(...e.events),this.#d=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#r,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#l=n,n}#h(e,t){let n=summarizeTurnEvents(t);return this.#m({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#g(){let e=this.sessionId;return{derived:deriveRunFacts(this.#i,{sessionId:e}),events:[...this.#i],output:outputOf(this.#l),status:this.#l?.status??`completed`}}#_(e,t){throw this.#r.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let a=!1;for await(let n of e)if(this.#t.push(n),t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),isCurrentTurnBoundaryEvent(n)){a=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}if(!a)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r;#i=[];#a;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#r=e.onSessionStart,this.#t=e.signal}get primary(){return this.#a??=this.#o(!0),this.#a}newSession(){return this.#o(!1)}async attachSession(e,t){let n=this.#s(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#s(e,t).watchTurn(t,e)}snapshots(){return this.#i.map(e=>e.snapshot())}lastTurnSession(){return this.#a?.lastTurn===void 0?this.#i.find(e=>e.lastTurn!==void 0):this.#a}hasActivity(){return this.#i.length>0}#o(e){let t=new EvalSessionDriver({client:this.#e,collector:this.#n,onSessionStart:this.#r,primary:e,signal:this.#t});return this.#i.push(t),t}#s(e,t){let n=new EvalSessionDriver({client:this.#e,collector:this.#n,onSessionStart:this.#r,primary:!1,session:this.#e.sessions.attach(e,{streamIndex:t?.startIndex??0}),signal:this.#t});return this.#i.push(n),n}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};

package/dist/src/evals/target.js

@@ -1 +1 @@
-import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);return createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url})}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(t){let n=t.delegate,i=t.client,fetchTarget=async(e,t)=>{if(n!==void 0)return await n.fetch(e,t);if(i===void 0)throw Error(`Eval target cannot fetch without a client.`);return await i.fetch(e,t)};return{capabilities:t.capabilities,kind:t.kind,url:t.url,async attachSession(e,a){if(t.sessions!==void 0)return await t.sessions.attachSession(e,a);if(n!==void 0)return await n.attachSession(e,a);if(i===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:i}).attachSession(e,a)},watchTurn(e,a){if(t.sessions!==void 0)return t.sessions.watchTurn(e,a);if(n!==void 0)return n.watchTurn(e,a);if(i===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:i}).watchTurn(e,a)},async dispatchSchedule(n){if(!t.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let r=await fetchTarget(createEveDevDispatchSchedulePath(n),{method:`POST`});if(!r.ok){let e=await readResponseBodySafely(r);throw Error(`Schedule dispatch failed: ${r.status} ${r.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await r.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,n){let r=Date.now()+6e4,a;for(;Date.now()<r;)try{await e.health();return}catch(e){a=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${n}.`+(a===void 0?``:` Last error: ${a}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==1)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle};
+import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);return createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url})}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(t){let n=t.delegate,i=t.client,fetchTarget=async(e,t)=>{if(n!==void 0)return await n.fetch(e,t);if(i===void 0)throw Error(`Eval target cannot fetch without a client.`);return await i.fetch(e,t)};return{capabilities:t.capabilities,kind:t.kind,url:t.url,async attachSession(e,a){if(t.sessions!==void 0)return await t.sessions.attachSession(e,a);if(n!==void 0)return await n.attachSession(e,a);if(i===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:i}).attachSession(e,a)},watchTurn(e,a){if(t.sessions!==void 0)return t.sessions.watchTurn(e,a);if(n!==void 0)return n.watchTurn(e,a);if(i===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:i}).watchTurn(e,a)},async dispatchSchedule(n){if(!t.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let r=await fetchTarget(createEveDevDispatchSchedulePath(n),{method:`POST`});if(!r.ok){let e=await readResponseBodySafely(r);throw Error(`Schedule dispatch failed: ${r.status} ${r.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await r.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,n){let r=Date.now()+6e4,a;for(;Date.now()<r;)try{await e.health();return}catch(e){a=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${n}.`+(a===void 0?``:` Last error: ${a}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==2)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle};

package/dist/src/evals/types.d.ts

@@ -1,6 +1,6 @@
 import type { LanguageModel } from "ai";
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
-import type { RuntimeIdentity, MessageStreamEvent } from "#protocol/message.js";
+import type { RuntimeIdentity, RuntimeTraceContext, MessageStreamEvent } from "#protocol/message.js";
 import type { CancelSessionResult, ClientSessionState, SendTurnInput, SendTurnOptions } from "#client/types.js";
 import type { InputRequest, InputResponse } from "#runtime/input/types.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
@@ -74,6 +74,13 @@ export interface EveEvalSessionResult {
     readonly primary: boolean;
     readonly sessionId?: string;
     readonly state: ClientSessionState | undefined;
+    /** Distinct trace contexts observed for this session, in stream order. */
+    readonly traceContexts: readonly RuntimeTraceContext[];
+}
+/** Trace context attributed to one session involved in an eval. */
+export interface EveEvalTraceContext extends RuntimeTraceContext {
+    readonly primary: boolean;
+    readonly sessionId: string;
 }
 /**
  * Full result of executing one eval against an eve agent.
@@ -106,6 +113,8 @@ export interface EveEvalTaskResult {
      * Present when the eve server populates the event with its runtime metadata.
      */
     readonly runtimeIdentity?: RuntimeIdentity;
+    /** Distinct trace contexts observed across every captured session. */
+    readonly traceContexts: readonly EveEvalTraceContext[];
 }
 /**
  * How a failing assertion affects the verdict. A `"gate"` is a hard
@@ -247,6 +256,8 @@ export interface EveEvalSessionDriver {
     requireInputRequest(filter?: EveEvalInputRequestMatchOptions): InputRequest;
     /** Resolve specific pending requests and run the resumed turn. */
     respond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalTurn>;
+    /** Start a response turn without waiting for its boundary. */
+    startRespond(responses: readonly InputResponse[], options?: SendTurnOptions): Promise<EveEvalLiveTurn>;
     /** Resolve every pending request with the same option id. */
     respondAll(optionId: string): Promise<EveEvalTurn>;
     /** Send one turn through this session. */

package/dist/src/execution/node-step.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{UNSPECIFIED_INPUT_SCHEMA}from"#shared/tool-schema.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";import{resolveRuntimeModelReference}from"#runtime/agent/resolve-model.js";import{createHarnessDelegationToolDefinition}from"#execution/delegation-tool.js";import{PERSISTENT_SUBAGENT_TOOL_INPUT_SCHEMA,SUBAGENT_TOOL_INPUT_SCHEMA}from"#runtime/subagents/registry.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";import{dispatchDynamicModelEvent}from"#context/dynamic-model-lifecycle.js";import{createToolLoopHarness}from"#harness/tool-loop.js";import{getInstrumentationRuntime}from"#harness/instrumentation-runtime.js";import{AGENT_TOOL_DESCRIPTION,AGENT_TOOL_NAME,isImplicitAgentToolAvailable}from"#runtime/framework-tools/agent.js";import{findRegisteredRuntimeTool}from"#runtime/tools/registry.js";import{preserveFrameworkStateOnCompaction}from"#execution/compaction.js";const log=createLogger(`execution.node-step`);function createExecutionNodeStep(e){let t=createRuntimeModelResolver(e.modelResolutionScope),n=e.node.turnAgent.dynamicModel===void 0?void 0:createRuntimeDynamicModelEventDispatcher(e.modelResolutionScope,e.node.turnAgent.dynamicModel),r=createNodeHarnessTools({node:e.node}),i=getInstrumentationRuntime(),a=createToolLoopHarness({abortSignal:e.abortSignal,capabilities:e.capabilities,clearOnly:e.clearOnly,compactOnly:e.compactOnly,workflow:e.node.agent.workflowTool!==void 0,workflowMaxSubagents:e.workflowMaxSubagents,webSearchProvider:e.node.agent.webSearchProvider,handleEvent:e.handleEvent,instrumentation:i,mode:e.mode,onCompaction:preserveFrameworkStateOnCompaction,persistentSubagentSessions:e.node.agent.config?.experimental?.subagentPersistentSessions===!0,dispatchDynamicModelEvent:n,resolveModel:t,runtimeIdentity:buildRuntimeIdentity(e.node),tools:r});return i===void 0?a:async(e,t)=>{try{return await a(e,t)}finally{await i.forceFlush()}}}function buildRuntimeIdentity(e){let n=resolveInstalledPackageInfo(),r={agentId:e.turnAgent.id,agentName:e.agent.config?.name,eveVersion:n.version},i=process.env.VERCEL_GIT_COMMIT_SHA?.trim(),a=process.env.VERCEL_GIT_COMMIT_REF?.trim(),o=process.env.VERCEL_DEPLOYMENT_CREATED_AT?.trim();return i||a||o?{...r,build:{deployedAt:o||void 0,gitBranch:a||void 0,gitSha:i||void 0}}:r}function createRuntimeModelResolver(e){return t=>resolveRuntimeModelReference(t,e)}function createRuntimeDynamicModelEventDispatcher(e,t){return n=>dispatchDynamicModelEvent({ctx:n.ctx,dynamicModel:t,event:n.event,messages:n.messages,scope:e})}function createNodeHarnessTools(e){let t=new Map;for(let n of e.node.turnAgent.tools){let r=resolveHarnessToolDefinition({node:e.node,tool:n});r!==null&&t.set(n.name,r)}return isImplicitAgentToolAvailable({disabledFrameworkTools:e.node.agent.disabledFrameworkTools,hasAuthoredAgentTool:t.has(AGENT_TOOL_NAME),nodeId:e.node.nodeId})&&t.set(AGENT_TOOL_NAME,{description:AGENT_TOOL_DESCRIPTION,inputSchema:e.node.agent.config?.experimental?.subagentPersistentSessions===!0?PERSISTENT_SUBAGENT_TOOL_INPUT_SCHEMA:SUBAGENT_TOOL_INPUT_SCHEMA,name:AGENT_TOOL_NAME,runtimeAction:{kind:`subagent-call`,nodeId:e.node.nodeId,subagentName:AGENT_TOOL_NAME}}),t}function resolveHarnessToolDefinition(e){if(e.tool.kind===`subagent`||e.tool.kind===`remote`)return createHarnessDelegationToolDefinition(e.tool);let t=findRegisteredRuntimeTool(e.node.toolRegistry,e.tool.name);if(t===null)return log.warn(`declared tool is not registered — omitting from toolset`,{toolName:e.tool.name,nodeId:e.node.nodeId}),null;let r=t.definition,i=r.sourceId.startsWith(`eve:`),o=r.execute;return{approvalKey:r.approvalKey,description:r.description,execute:resolveAuthoredExecute({isFrameworkTool:i,rawExecute:o,scope:r.name}),frameworkAction:i&&r.name===LOAD_SKILL_TOOL_NAME?`load-skill`:void 0,inputSchema:r.inputSchema??UNSPECIFIED_INPUT_SCHEMA,name:r.name,approval:r.approval,outputSchema:r.outputSchema,toModelOutput:r.toModelOutput}}function resolveAuthoredExecute(e){let{isFrameworkTool:t,rawExecute:n,scope:i}=e;return n===void 0?void 0:t?n:createToolExecuteWithAuth({execute:n,scope:i})}export{buildRuntimeIdentity,createExecutionNodeStep,createNodeHarnessTools};
+import{createLogger}from"#internal/logging.js";import{resolveInstalledPackageInfo}from"#internal/application/package.js";import{UNSPECIFIED_INPUT_SCHEMA}from"#shared/tool-schema.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";import{resolveRuntimeModelReference}from"#runtime/agent/resolve-model.js";import{createHarnessDelegationToolDefinition}from"#execution/delegation-tool.js";import{PERSISTENT_SUBAGENT_TOOL_INPUT_SCHEMA,SUBAGENT_TOOL_INPUT_SCHEMA}from"#runtime/subagents/registry.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";import{dispatchDynamicModelEvent}from"#context/dynamic-model-lifecycle.js";import{createToolLoopHarness}from"#harness/tool-loop.js";import{getInstrumentationRuntime}from"#harness/instrumentation/runtime.js";import{AGENT_TOOL_DESCRIPTION,AGENT_TOOL_NAME,isImplicitAgentToolAvailable}from"#runtime/framework-tools/agent.js";import{findRegisteredRuntimeTool}from"#runtime/tools/registry.js";import{preserveFrameworkStateOnCompaction}from"#execution/compaction.js";const log=createLogger(`execution.node-step`);function createExecutionNodeStep(e){let t=createRuntimeModelResolver(e.modelResolutionScope),n=e.node.turnAgent.dynamicModel===void 0?void 0:createRuntimeDynamicModelEventDispatcher(e.modelResolutionScope,e.node.turnAgent.dynamicModel),r=createNodeHarnessTools({node:e.node}),i=getInstrumentationRuntime(),a=createToolLoopHarness({abortSignal:e.abortSignal,capabilities:e.capabilities,clearOnly:e.clearOnly,compactOnly:e.compactOnly,workflow:e.node.agent.workflowTool!==void 0,workflowMaxSubagents:e.workflowMaxSubagents,webSearchProvider:e.node.agent.webSearchProvider,handleEvent:e.handleEvent,instrumentation:i,mode:e.mode,onCompaction:preserveFrameworkStateOnCompaction,persistentSubagentSessions:e.node.agent.config?.experimental?.subagentPersistentSessions===!0,dispatchDynamicModelEvent:n,resolveModel:t,runtimeIdentity:buildRuntimeIdentity(e.node),tools:r});return i===void 0?a:async(e,t)=>{try{return await a(e,t)}finally{await i.forceFlush()}}}function buildRuntimeIdentity(e){let n=resolveInstalledPackageInfo(),r={agentId:e.turnAgent.id,agentName:e.agent.config?.name,eveVersion:n.version},i=process.env.VERCEL_GIT_COMMIT_SHA?.trim(),a=process.env.VERCEL_GIT_COMMIT_REF?.trim(),o=process.env.VERCEL_DEPLOYMENT_CREATED_AT?.trim();return i||a||o?{...r,build:{deployedAt:o||void 0,gitBranch:a||void 0,gitSha:i||void 0}}:r}function createRuntimeModelResolver(e){return t=>resolveRuntimeModelReference(t,e)}function createRuntimeDynamicModelEventDispatcher(e,t){return n=>dispatchDynamicModelEvent({ctx:n.ctx,dynamicModel:t,event:n.event,messages:n.messages,scope:e})}function createNodeHarnessTools(e){let t=new Map;for(let n of e.node.turnAgent.tools){let r=resolveHarnessToolDefinition({node:e.node,tool:n});r!==null&&t.set(n.name,r)}return isImplicitAgentToolAvailable({disabledFrameworkTools:e.node.agent.disabledFrameworkTools,hasAuthoredAgentTool:t.has(AGENT_TOOL_NAME),nodeId:e.node.nodeId})&&t.set(AGENT_TOOL_NAME,{description:AGENT_TOOL_DESCRIPTION,inputSchema:e.node.agent.config?.experimental?.subagentPersistentSessions===!0?PERSISTENT_SUBAGENT_TOOL_INPUT_SCHEMA:SUBAGENT_TOOL_INPUT_SCHEMA,name:AGENT_TOOL_NAME,runtimeAction:{kind:`subagent-call`,nodeId:e.node.nodeId,subagentName:AGENT_TOOL_NAME}}),t}function resolveHarnessToolDefinition(e){if(e.tool.kind===`subagent`||e.tool.kind===`remote`)return createHarnessDelegationToolDefinition(e.tool);let t=findRegisteredRuntimeTool(e.node.toolRegistry,e.tool.name);if(t===null)return log.warn(`declared tool is not registered — omitting from toolset`,{toolName:e.tool.name,nodeId:e.node.nodeId}),null;let r=t.definition,i=r.sourceId.startsWith(`eve:`),o=r.execute;return{approvalKey:r.approvalKey,description:r.description,execute:resolveAuthoredExecute({isFrameworkTool:i,rawExecute:o,scope:r.name}),frameworkAction:i&&r.name===LOAD_SKILL_TOOL_NAME?`load-skill`:void 0,inputSchema:r.inputSchema??UNSPECIFIED_INPUT_SCHEMA,name:r.name,approval:r.approval,outputSchema:r.outputSchema,toModelOutput:r.toModelOutput}}function resolveAuthoredExecute(e){let{isFrameworkTool:t,rawExecute:n,scope:i}=e;return n===void 0?void 0:t?n:createToolExecuteWithAuth({execute:n,scope:i})}export{buildRuntimeIdentity,createExecutionNodeStep,createNodeHarnessTools};

package/dist/src/execution/session.js

@@ -1,3 +1,3 @@
-import{formatAvailableSkillsSection}from"#execution/skills/instructions.js";const DEFAULT_ROOT_MAX_INPUT_TOKENS_PER_SESSION=4e7;function createCompactionConfig(e={}){let t=e.thresholdPercent??.9,n={recentWindowSize:10,threshold:e.contextWindowTokens===void 0?1e5:Math.max(1,Math.floor(e.contextWindowTokens*t)),thresholdPercent:t};return e.lastKnownInputTokens===void 0?n:{...n,lastKnownInputTokens:e.lastKnownInputTokens,lastKnownPromptMessageCount:e.lastKnownPromptMessageCount}}function createSession(e){let{turnAgent:t}=e,n=createSessionToolDefinitions(t),r={agent:createSessionAgent(t,createSessionSystemPrompt({turnAgent:t}),n),compaction:createCompactionConfig({contextWindowTokens:t.model?.contextWindowTokens,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:e.continuationToken,history:[],sessionId:e.sessionId};return e.rootSessionId!==void 0&&(r.rootSessionId=e.rootSessionId),r.limits=resolveSessionLimits(e),e.outputSchema!==void 0&&(r.outputSchema=e.outputSchema),e.subagentDepth!==void 0&&(r.subagentDepth=e.subagentDepth),e.workflowMaxSubagents!==void 0&&(r.workflowMaxSubagents=e.workflowMaxSubagents),r}function createSessionAgent(e,t,n){let r={compactionModelReference:e.compactionModel,reasoning:e.reasoning,system:t,tools:n};if(e.model!==void 0)return{...r,modelReference:e.model};if(e.dynamicModel!==void 0)return{...r,dynamicModel:!0};throw Error(`Cannot create a session before dynamic subagent config is selected.`)}function refreshSessionFromTurnAgent(e){return{...e.session,agent:createSessionAgent(e.turnAgent,createSessionSystemPrompt({turnAgent:e.turnAgent}),createSessionToolDefinitions(e.turnAgent)),compaction:createCompactionConfig({contextWindowTokens:e.turnAgent.model?.contextWindowTokens,lastKnownInputTokens:e.session.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.session.compaction.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent})}}function createSessionSystemPrompt(t){let n=formatAvailableSkillsSection(t.turnAgent.availableSkills??[]);return(n===null?t.turnAgent.instructions:[...t.turnAgent.instructions,n]).join(`
+import{formatAvailableSkillsSection}from"#execution/skills/instructions.js";const DEFAULT_ROOT_MAX_INPUT_TOKENS_PER_SESSION=4e7;function createCompactionConfig(e={}){let t=e.thresholdPercent??.9,n={recentWindowSize:10,threshold:e.contextWindowTokens===void 0?1e5:Math.max(1,Math.floor(e.contextWindowTokens*t)),thresholdPercent:t};return e.lastKnownInputTokens===void 0?n:{...n,lastKnownInputTokens:e.lastKnownInputTokens,lastKnownPromptMessageCount:e.lastKnownPromptMessageCount}}function createSession(e){let{turnAgent:t}=e,n=createSessionToolDefinitions(t),r={agent:createSessionAgent(t,createSessionSystemPrompt({turnAgent:t}),n),compaction:createCompactionConfig({contextWindowTokens:t.model?.contextWindowTokens,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:e.continuationToken,history:[...t.initialMessages??[]],sessionId:e.sessionId};return e.rootSessionId!==void 0&&(r.rootSessionId=e.rootSessionId),r.limits=resolveSessionLimits(e),e.outputSchema!==void 0&&(r.outputSchema=e.outputSchema),e.subagentDepth!==void 0&&(r.subagentDepth=e.subagentDepth),e.workflowMaxSubagents!==void 0&&(r.workflowMaxSubagents=e.workflowMaxSubagents),r}function createSessionAgent(e,t,n){let r={compactionModelReference:e.compactionModel,reasoning:e.reasoning,system:t,tools:n};if(e.model!==void 0)return{...r,modelReference:e.model};if(e.dynamicModel!==void 0)return{...r,dynamicModel:!0};throw Error(`Cannot create a session before dynamic subagent config is selected.`)}function refreshSessionFromTurnAgent(e){return{...e.session,agent:createSessionAgent(e.turnAgent,createSessionSystemPrompt({turnAgent:e.turnAgent}),createSessionToolDefinitions(e.turnAgent)),compaction:createCompactionConfig({contextWindowTokens:e.turnAgent.model?.contextWindowTokens,lastKnownInputTokens:e.session.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.session.compaction.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent})}}function createSessionSystemPrompt(t){let n=formatAvailableSkillsSection(t.turnAgent.availableSkills??[]);return(n===null?t.turnAgent.instructions:[...t.turnAgent.instructions,n]).join(`
 
 `)}function mintSubagentContinuationToken(e){return`subagent:${e??crypto.randomUUID()}`}function projectToDurableSession(e){let t={agent:{system:e.agent.system},continuationToken:e.continuationToken,history:e.history,sessionId:e.sessionId};return(e.compaction.lastKnownInputTokens!==void 0||e.compaction.lastKnownPromptMessageCount!==void 0)&&(t.compaction={lastKnownInputTokens:e.compaction.lastKnownInputTokens,lastKnownPromptMessageCount:e.compaction.lastKnownPromptMessageCount}),e.rootSessionId!==void 0&&(t.rootSessionId=e.rootSessionId),e.limits!==void 0&&(t.limits=e.limits),e.outputSchema!==void 0&&(t.outputSchema=e.outputSchema),e.sandboxState!==void 0&&(t.sandboxState=e.sandboxState),e.state!==void 0&&(t.state=e.state),e.subagentDepth!==void 0&&(t.subagentDepth=e.subagentDepth),e.workflowMaxSubagents!==void 0&&(t.workflowMaxSubagents=e.workflowMaxSubagents),t}function hydrateDurableSession(e){let{durable:t,turnAgent:n}=e,r=createSessionToolDefinitions(n),i={agent:createSessionAgent(n,t.agent.system,r),compaction:createCompactionConfig({contextWindowTokens:n.model?.contextWindowTokens,lastKnownInputTokens:t.compaction?.lastKnownInputTokens,lastKnownPromptMessageCount:t.compaction?.lastKnownPromptMessageCount,thresholdPercent:e.compactionOverrides?.thresholdPercent}),continuationToken:t.continuationToken,history:t.history,sessionId:t.sessionId};return t.rootSessionId!==void 0&&(i.rootSessionId=t.rootSessionId),t.limits!==void 0&&(i.limits=t.limits),t.outputSchema!==void 0&&(i.outputSchema=t.outputSchema),t.sandboxState!==void 0&&(i.sandboxState=t.sandboxState),t.state!==void 0&&(i.state=t.state),t.subagentDepth!==void 0&&(i.subagentDepth=t.subagentDepth),t.workflowMaxSubagents!==void 0&&(i.workflowMaxSubagents=t.workflowMaxSubagents),i}function createSessionToolDefinitions(e){return e.tools.map(e=>({description:e.description??``,inputSchema:e.inputSchema,name:e.name,outputSchema:e.outputSchema}))}function resolveSessionLimits(e){let n=e.subagentDepth!==void 0&&e.subagentDepth>0,r=resolveSessionTokenLimit({authored:e.limits?.maxInputTokensPerSession,fallback:n?void 0:DEFAULT_ROOT_MAX_INPUT_TOKENS_PER_SESSION}),i=resolveSessionTokenLimit({authored:e.limits?.maxOutputTokensPerSession,fallback:void 0}),a={};return r!==void 0&&(a.maxInputTokensPerSession=r),i!==void 0&&(a.maxOutputTokensPerSession=i),a}function resolveSessionTokenLimit(e){return e.authored===!1?void 0:e.authored??e.fallback}export{DEFAULT_ROOT_MAX_INPUT_TOKENS_PER_SESSION,createCompactionConfig,createSession,hydrateDurableSession,mintSubagentContinuationToken,projectToDurableSession,refreshSessionFromTurnAgent};