eve 0.33.3 → 0.35.0

package/dist/src/compiler/normalize-extension.d.ts

@@ -1,5 +1,5 @@
 import type { ResolvedExtensionMount } from "#discover/manifest.js";
-import type { CompiledConnectionDefinition, CompiledDynamicInstructionsDefinition, CompiledDynamicSkillDefinition, CompiledDynamicToolDefinition, CompiledHookDefinition, CompiledSkillDefinition, CompiledToolDefinition } from "#compiler/manifest.js";
+import type { CompiledConnectionDefinition, CompiledDynamicInstructionsDefinition, CompiledDynamicSkillDefinition, CompiledDynamicToolDefinition, CompiledHookDefinition, CompiledInstructionsDefinition, CompiledSkillDefinition, CompiledToolDefinition } from "#compiler/manifest.js";
 import type { ManifestCompileContext } from "#compiler/normalize-helpers.js";
 /**
  * Contributions one mounted extension composes into the consuming agent,
@@ -13,7 +13,7 @@ export interface CompiledExtensionContributions {
     readonly dynamicSkills: CompiledDynamicSkillDefinition[];
     readonly dynamicInstructions: CompiledDynamicInstructionsDefinition[];
     readonly connections: CompiledConnectionDefinition[];
-    readonly instructionFragments: string[];
+    readonly instructions: CompiledInstructionsDefinition[];
 }
 /**
  * Compiles one mounted extension's source tree and namespaces its
@@ -59,7 +59,7 @@ export declare function applyOverrideDisables(input: {
  * composed name. Named contributions (tools, connections, skills, dynamic
  * tools) dedup by their model-facing identifier so an override shadows the
  * extension's same-named entry; unnamed contributions (hooks, dynamic skills,
- * dynamic instructions, instruction fragments) simply concatenate.
+ * dynamic instructions, static instructions) simply concatenate.
  *
  * Exported for unit testing: passing the consumer overrides as `primary` and
  * the extension's own contributions as `secondary` yields consumer-wins

package/dist/src/compiler/normalize-extension.js

@@ -1 +1 @@
-import{join,relative}from"node:path";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";async function compileExtensionContributions(e){let{mount:t,consumerAgentRoot:n}=e,r={externalDependencies:e.externalDependencies},i=await composeManifestContributions({manifest:t.manifest,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext:${t.namespace}`,role:`extension`});if(t.overrides===void 0)return i.contributions;let a=await composeManifestContributions({manifest:t.overrides,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext-override:${t.namespace}`,role:`override`});return applyOverrideDisables({merged:mergeContributions(a.contributions,i.contributions),disables:a.disabledToolTargets,extensionToolNames:new Set(i.contributions.tools.map(e=>e.name)),extensionDynamicToolSlugs:new Set(i.contributions.dynamicTools.map(e=>e.slug)),namespace:t.namespace})}function applyOverrideDisables(e){if(e.disables.length===0)return e.merged;let t=e.namespace.length+2,n=new Set;for(let r of e.disables){if(!e.extensionToolNames.has(r.name)&&!e.extensionDynamicToolSlugs.has(r.name)){let n=[...e.extensionToolNames,...e.extensionDynamicToolSlugs].map(e=>e.slice(t)).sort();throw Error(`The override "agent/extensions/${e.namespace}/${r.logicalPath}" calls disableTool(), but the "${e.namespace}" extension contributes no tool named "${r.name.slice(t)}". It contributes: ${n.length>0?n.join(`, `):`(no tools)`}.`)}n.add(r.name)}return{...e.merged,tools:e.merged.tools.filter(e=>!n.has(e.name)),dynamicTools:e.merged.dynamicTools.filter(e=>!n.has(e.slug))}}async function composeManifestContributions(o){let{manifest:s,namespace:c,consumerAgentRoot:l,options:u,sourceIdScope:d,role:f}=o,p=s.agentRoot,m=`${c}__`,scopeSourceId=e=>`${d}:${e}`,rebase=n=>relative(l,join(p,n)).replaceAll(`\\`,`/`),h=[],g=[],_=[];for(let e of s.tools){let t=await compileToolEntry(p,e,u);if(t.kind===`tool`)h.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`dynamic-tool`)g.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`workflow-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} enables the Workflow tool, but the Workflow tool is the consuming agent's to enable, not an extension's. Remove it.`);else if(t.kind===`web-search-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} configures web search, but the web search provider is the consuming agent's to configure, not an extension's. Remove it.`);else if(f===`extension`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} calls disableTool(), but an extension cannot disable framework tools — that is the consuming agent's to own. Remove it.`);else _.push({name:`${m}${t.name}`,logicalPath:e.logicalPath})}let v=s.hooks.map(e=>{let t=compileHookEntry(e);return{...t,slug:`${m}${t.slug}`,sourceId:scopeSourceId(t.sourceId),logicalPath:rebase(t.logicalPath)}}),y=[],b=[];for(let e of s.skills){let t=await compileSkillSource(p,e,u);t.kind===`skill`?y.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):b.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}let x=(await Promise.all(s.connections.map(e=>compileConnectionDefinition(p,e,u)))).map(e=>({...e,connectionName:`${m}${e.connectionName}`,sourceId:scopeSourceId(e.sourceId),logicalPath:rebase(e.logicalPath)})),S=[],C=[];for(let e of s.instructions){let t=await compileInstructionsEntry(p,e,u);t.kind===`instructions`?C.push(t.definition.markdown):S.push({...t.definition,slug:`${m}${t.definition.slug}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}return{contributions:{tools:h,dynamicTools:g,hooks:v,skills:y,dynamicSkills:b,dynamicInstructions:S,connections:x,instructionFragments:C},disabledToolTargets:_}}function describeExtensionSource(e,t,n){return e===`override`?`The override "agent/extensions/${t}/${n}"`:`The "${t}" extension's "${n}"`}function mergeContributions(e,t){return{tools:dedupeBy([...e.tools,...t.tools],e=>e.name),dynamicTools:dedupeBy([...e.dynamicTools,...t.dynamicTools],e=>e.slug),connections:dedupeBy([...e.connections,...t.connections],e=>e.connectionName),skills:dedupeBy([...e.skills,...t.skills],e=>e.name),hooks:[...e.hooks,...t.hooks],dynamicSkills:[...e.dynamicSkills,...t.dynamicSkills],dynamicInstructions:[...e.dynamicInstructions,...t.dynamicInstructions],instructionFragments:[...e.instructionFragments,...t.instructionFragments]}}function dedupeBy(e,t){let n=new Set,r=[];for(let i of e){let e=t(i);n.has(e)||(n.add(e),r.push(i))}return r}export{applyOverrideDisables,compileExtensionContributions,mergeContributions};
+import{join,relative}from"node:path";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";async function compileExtensionContributions(e){let{mount:t,consumerAgentRoot:n}=e,r={externalDependencies:e.externalDependencies},i=await composeManifestContributions({manifest:t.manifest,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext:${t.namespace}`,role:`extension`});if(t.overrides===void 0)return i.contributions;let a=await composeManifestContributions({manifest:t.overrides,namespace:t.namespace,consumerAgentRoot:n,options:r,sourceIdScope:`ext-override:${t.namespace}`,role:`override`});return applyOverrideDisables({merged:mergeContributions(a.contributions,i.contributions),disables:a.disabledToolTargets,extensionToolNames:new Set(i.contributions.tools.map(e=>e.name)),extensionDynamicToolSlugs:new Set(i.contributions.dynamicTools.map(e=>e.slug)),namespace:t.namespace})}function applyOverrideDisables(e){if(e.disables.length===0)return e.merged;let t=e.namespace.length+2,n=new Set;for(let r of e.disables){if(!e.extensionToolNames.has(r.name)&&!e.extensionDynamicToolSlugs.has(r.name)){let n=[...e.extensionToolNames,...e.extensionDynamicToolSlugs].map(e=>e.slice(t)).sort();throw Error(`The override "agent/extensions/${e.namespace}/${r.logicalPath}" calls disableTool(), but the "${e.namespace}" extension contributes no tool named "${r.name.slice(t)}". It contributes: ${n.length>0?n.join(`, `):`(no tools)`}.`)}n.add(r.name)}return{...e.merged,tools:e.merged.tools.filter(e=>!n.has(e.name)),dynamicTools:e.merged.dynamicTools.filter(e=>!n.has(e.slug))}}async function composeManifestContributions(o){let{manifest:s,namespace:c,consumerAgentRoot:l,options:u,sourceIdScope:d,role:f}=o,p=s.agentRoot,m=`${c}__`,scopeSourceId=e=>`${d}:${e}`,rebase=n=>relative(l,join(p,n)).replaceAll(`\\`,`/`),h=[],g=[],_=[];for(let e of s.tools){let t=await compileToolEntry(p,e,u);if(t.kind===`tool`)h.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`dynamic-tool`)g.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)});else if(t.kind===`workflow-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} enables the Workflow tool, but the Workflow tool is the consuming agent's to enable, not an extension's. Remove it.`);else if(t.kind===`web-search-tool`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} configures web search, but the web search provider is the consuming agent's to configure, not an extension's. Remove it.`);else if(f===`extension`)throw Error(`${describeExtensionSource(f,c,e.logicalPath)} calls disableTool(), but an extension cannot disable framework tools — that is the consuming agent's to own. Remove it.`);else _.push({name:`${m}${t.name}`,logicalPath:e.logicalPath})}let v=s.hooks.map(e=>{let t=compileHookEntry(e);return{...t,slug:`${m}${t.slug}`,sourceId:scopeSourceId(t.sourceId),logicalPath:rebase(t.logicalPath)}}),y=[],b=[];for(let e of s.skills){let t=await compileSkillSource(p,e,u);t.kind===`skill`?y.push({...t.definition,name:`${m}${t.definition.name}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):b.push({...t.definition,slug:`${m}${t.definition.slug}`,extensionNamespace:c,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}let x=(await Promise.all(s.connections.map(e=>compileConnectionDefinition(p,e,u)))).map(e=>({...e,connectionName:`${m}${e.connectionName}`,sourceId:scopeSourceId(e.sourceId),logicalPath:rebase(e.logicalPath)})),S=[],C=[];for(let e of s.instructions){let t=await compileInstructionsEntry(p,e,u);t.kind===`instructions`?C.push({...t.definition,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)}):S.push({...t.definition,slug:`${m}${t.definition.slug}`,sourceId:scopeSourceId(t.definition.sourceId),logicalPath:rebase(t.definition.logicalPath)})}return{contributions:{tools:h,dynamicTools:g,hooks:v,skills:y,dynamicSkills:b,dynamicInstructions:S,connections:x,instructions:C},disabledToolTargets:_}}function describeExtensionSource(e,t,n){return e===`override`?`The override "agent/extensions/${t}/${n}"`:`The "${t}" extension's "${n}"`}function mergeContributions(e,t){return{tools:dedupeBy([...e.tools,...t.tools],e=>e.name),dynamicTools:dedupeBy([...e.dynamicTools,...t.dynamicTools],e=>e.slug),connections:dedupeBy([...e.connections,...t.connections],e=>e.connectionName),skills:dedupeBy([...e.skills,...t.skills],e=>e.name),hooks:[...e.hooks,...t.hooks],dynamicSkills:[...e.dynamicSkills,...t.dynamicSkills],dynamicInstructions:[...e.dynamicInstructions,...t.dynamicInstructions],instructions:[...e.instructions,...t.instructions]}}function dedupeBy(e,t){let n=new Set,r=[];for(let i of e){let e=t(i);n.has(e)||(n.add(e),r.push(i))}return r}export{applyOverrideDisables,compileExtensionContributions,mergeContributions};

package/dist/src/compiler/normalize-instructions.d.ts

@@ -21,7 +21,7 @@ export type CompiledInstructionsEntry = {
  * by the runtime.
  *
  * Module-backed static instructions sources execute once at build time —
- * the resulting markdown is captured into the compiled manifest. There is
+ * the resulting content is captured into the compiled manifest. There is
  * no per-session re-evaluation at runtime.
  *
  * Module-backed dynamic instructions (exporting `defineDynamic`) are

package/dist/src/compiler/normalize-instructions.js

@@ -1 +1 @@
-import{stripLogicalPathExtension}from"#discover/filesystem.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{assertResolverOnlyDynamicSentinel,isDynamicSentinel}from"#shared/dynamic-tool-definition.js";async function compileInstructionsEntry(i,a,o={}){if(a.sourceKind===`markdown`){let n=normalizeInstructionsDefinition(a.definition,`Expected the compiled instructions definition at "${a.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(a.logicalPath),logicalPath:a.logicalPath,markdown:n.markdown,sourceId:a.sourceId,sourceKind:a.sourceKind}}}let s=await loadModuleBackedDefinition({agentRoot:i,externalDependencies:o.externalDependencies,kind:`instructions`,source:a});if(isDynamicSentinel(s)){assertResolverOnlyDynamicSentinel(s,`Expected the instructions export "${a.exportName??`default`}" from "${a.logicalPath}" to match the public eve shape.`);let t=stripLogicalPathExtension(a.logicalPath).replace(/^instructions\//,``);return{kind:`dynamic-instructions`,definition:{eventNames:Object.keys(s.events),exportName:a.exportName,logicalPath:a.logicalPath,slug:t,sourceId:a.sourceId,sourceKind:`module`}}}let c=normalizeInstructionsDefinition(s,`Expected the instructions export "${a.exportName??`default`}" from "${a.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(a.logicalPath),logicalPath:a.logicalPath,markdown:c.markdown,sourceId:a.sourceId,sourceKind:a.sourceKind}}}async function compileInstructions(e,t,n={}){let r=await compileInstructionsEntry(e,t,n);if(r.kind===`dynamic-instructions`)throw Error(`Expected static instructions from "${t.logicalPath}" but got a dynamic resolver. Use compileInstructionsEntry instead.`);return r.definition}export{compileInstructions,compileInstructionsEntry};
+import{stripLogicalPathExtension}from"#discover/filesystem.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,assertResolverOnlyDynamicSentinel,isDynamicSentinel}from"#shared/dynamic-tool-definition.js";async function compileInstructionsEntry(a,o,s={}){if(o.sourceKind===`markdown`){let n=normalizeInstructionsDefinition(o.definition,`Expected the compiled instructions definition at "${o.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(o.logicalPath),logicalPath:o.logicalPath,content:n.content,role:n.role,sourceId:o.sourceId,sourceKind:o.sourceKind}}}let c=await loadModuleBackedDefinition({agentRoot:a,externalDependencies:s.externalDependencies,kind:`instructions`,source:o});if(isDynamicSentinel(c)){assertResolverOnlyDynamicSentinel(c,`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to match the public eve shape.`);let t=Object.keys(c.events),n=t.find(e=>!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(e));if(n!==void 0)throw Error(`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to use only "session.started" or "turn.started" events. Unsupported event: "${n}".`);let i=stripLogicalPathExtension(o.logicalPath).replace(/^instructions\//,``);return{kind:`dynamic-instructions`,definition:{eventNames:t,exportName:o.exportName,logicalPath:o.logicalPath,slug:i,sourceId:o.sourceId,sourceKind:`module`}}}let l=normalizeInstructionsDefinition(c,`Expected the instructions export "${o.exportName??`default`}" from "${o.logicalPath}" to match the public eve shape.`);return{kind:`instructions`,definition:{name:stripLogicalPathExtension(o.logicalPath),logicalPath:o.logicalPath,content:l.content,role:l.role,sourceId:o.sourceId,sourceKind:o.sourceKind}}}async function compileInstructions(e,t,n={}){let r=await compileInstructionsEntry(e,t,n);if(r.kind===`dynamic-instructions`)throw Error(`Expected static instructions from "${t.logicalPath}" but got a dynamic resolver. Use compileInstructionsEntry instead.`);return r.definition}export{compileInstructions,compileInstructionsEntry};

package/dist/src/compiler/normalize-manifest.js

@@ -1,3 +1 @@
-import{compileChannelDefinition}from"#compiler/normalize-channel.js";import{ROOT_COMPILED_AGENT_NODE_ID,createCompiledAgentManifest,createCompiledAgentNodeManifest,createCompiledAgentResources}from"#compiler/manifest.js";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";import{mountRefNamespace,packageStateNamespace}from"#discover/extensions.js";import{createCompiledRuntimeModelCatalogLoader}from"#compiler/model-catalog.js";import{compileAgentConfig}from"#compiler/normalize-agent-config.js";import{compileExtensionContributions}from"#compiler/normalize-extension.js";import{compileSandboxDefinition}from"#compiler/normalize-sandbox.js";import{compileScheduleDefinition}from"#compiler/normalize-schedule.js";import{compileSubagentGraph}from"#compiler/normalize-subagent.js";async function compileAgentManifest(e){let r={modelCatalog:createCompiledRuntimeModelCatalogLoader(e.appRoot)},i=await compileAgentNodeManifest(e,r),a=await compileSubagentGraph({appRoot:e.appRoot,compileAgentNodeManifest,compileAgentResources,context:r,externalDependencies:i.config.build?.externalDependencies??[],parentNodeId:ROOT_COMPILED_AGENT_NODE_ID,subagents:e.subagents});return createCompiledAgentManifest({...i,extensionMounts:i.extensionMounts,remoteAgents:a.remoteAgents,subagentEdges:a.edges,subagents:a.nodes})}async function compileAgentNodeManifest(e,t,n={}){let i=Object.hasOwn(n,`agentConfigDefinition`)?await compileAgentConfig(e,t,{definition:n.agentConfigDefinition}):await compileAgentConfig(e,t);if(n.allowWorkflowConfig===!1&&i.experimental?.workflow!==void 0)throw Error(`Workflow runtime configuration is only supported on the root agent config. Remove "experimental.workflow" from "${e.agentId}".`);let a=mergeExternalDependencies(n.externalDependencies,i.build?.externalDependencies),o=a.length===0?i:{...i,build:{...i.build,externalDependencies:a}};return createCompiledAgentNodeManifest({...await compileAgentResources(e,t,{externalDependencies:a}),config:o})}async function compileAgentResources(t,n,r={}){let s=[...r.externalDependencies??[]],c=await Promise.all(t.tools.map(e=>compileToolEntry(t.agentRoot,e,{externalDependencies:s}))),l=[],u=[],d=[],f,p;for(let e of c)e.kind===`tool`?l.push(e.definition):e.kind===`dynamic-tool`?u.push(e.definition):e.kind===`workflow-tool`?f={maxSubagents:e.maxSubagents}:e.kind===`web-search-tool`?p=e.provider:d.push(e.name);let m=(await Promise.all(t.channels.map(n=>compileChannelDefinition(t.agentRoot,n,{externalDependencies:s})))).flat(),h=await Promise.all(t.skills.map(e=>compileSkillSource(t.agentRoot,e,{externalDependencies:s}))),g=[],_=[];for(let e of h)e.kind===`skill`?g.push(e.definition):_.push(e.definition);let v=await Promise.all(t.instructions.map(e=>compileInstructionsEntry(t.agentRoot,e,{externalDependencies:s}))),y=[],b=[];for(let e of v)e.kind===`instructions`?y.push(e.definition):b.push(e.definition);let x=await Promise.all(t.connections.map(e=>compileConnectionDefinition(t.agentRoot,e,{externalDependencies:s}))),S=t.hooks.map(e=>compileHookEntry(e)),C=await Promise.all(t.schedules.map(e=>compileScheduleDefinition(t.agentRoot,e,{externalDependencies:s}))),w=new Set(l.map(e=>e.name)),T=new Set(u.map(e=>e.slug)),E=new Set(x.map(e=>e.connectionName)),D=new Set(g.map(e=>e.name)),O=[];for(let e of[...t.resolvedExtensions].sort((e,t)=>e.namespace.localeCompare(t.namespace))){let r=await compileExtensionContributions({mount:e,context:n,consumerAgentRoot:t.agentRoot,externalDependencies:s});for(let e of r.tools)w.has(e.name)||(w.add(e.name),l.push(e));for(let e of r.dynamicTools)T.has(e.slug)||(T.add(e.slug),u.push(e));for(let e of r.connections)E.has(e.connectionName)||(E.add(e.connectionName),x.push(e));for(let e of r.skills)D.has(e.name)||(D.add(e.name),g.push(e));S.push(...r.hooks),_.push(...r.dynamicSkills),b.push(...r.dynamicInstructions),O.push(...r.instructionFragments)}let k=[...y.map(e=>e.markdown),...O],A=k.length===0?void 0:y.length===1&&O.length===0?y[0]:{name:`instructions`,logicalPath:`instructions`,markdown:k.join(`
-
-`),sourceId:y[0]?.sourceId??`instructions`,sourceKind:`module`};return createCompiledAgentResources({agentRoot:t.agentRoot,appRoot:t.appRoot,channels:m,extensionMounts:compileExtensionMounts(t),connections:x,diagnosticsSummary:t.diagnosticsSummary,disabledFrameworkTools:d,workflowTool:f,webSearchProvider:p,dynamicSkills:_,dynamicTools:u,hooks:S,sandbox:t.sandbox===null?null:await compileSandboxDefinition(t.agentRoot,t.sandbox,{externalDependencies:s}),sandboxWorkspaces:t.sandboxWorkspaces.map(e=>({logicalPath:e.logicalPath,rootEntries:[...e.rootEntries],sourceId:e.sourceId,sourcePath:e.sourcePath})),schedules:C,dynamicInstructions:b,skills:g,instructions:A,tools:l})}function compileExtensionMounts(e){return e.resolvedExtensions.map(t=>{let n=e.extensions.find(e=>mountRefNamespace(e.logicalPath)===t.namespace);return{namespace:t.namespace,packageName:t.packageName,packageNamespace:packageStateNamespace(t.packageName),sourceRoot:t.sourceRoot,mountSourceId:n?.sourceId??`extensions/${t.namespace}`,mountLogicalPath:n?.logicalPath??`extensions/${t.namespace}`}})}function mergeExternalDependencies(...e){let t=new Set;for(let n of e)for(let e of n??[])t.add(e);return[...t]}export{compileAgentManifest};
+import{compileChannelDefinition}from"#compiler/normalize-channel.js";import{ROOT_COMPILED_AGENT_NODE_ID,createCompiledAgentManifest,createCompiledAgentNodeManifest,createCompiledAgentResources}from"#compiler/manifest.js";import{compileConnectionDefinition}from"#compiler/normalize-connection.js";import{compileHookEntry}from"#compiler/normalize-hook.js";import{compileInstructionsEntry}from"#compiler/normalize-instructions.js";import{compileSkillSource}from"#compiler/normalize-skill.js";import{compileToolEntry}from"#compiler/normalize-tool.js";import{mountRefNamespace,packageStateNamespace}from"#discover/extensions.js";import{createCompiledRuntimeModelCatalogLoader}from"#compiler/model-catalog.js";import{compileAgentConfig}from"#compiler/normalize-agent-config.js";import{compileExtensionContributions}from"#compiler/normalize-extension.js";import{compileSandboxDefinition}from"#compiler/normalize-sandbox.js";import{compileScheduleDefinition}from"#compiler/normalize-schedule.js";import{compileSubagentGraph}from"#compiler/normalize-subagent.js";async function compileAgentManifest(e){let r={modelCatalog:createCompiledRuntimeModelCatalogLoader(e.appRoot)},i=await compileAgentNodeManifest(e,r),a=await compileSubagentGraph({appRoot:e.appRoot,compileAgentNodeManifest,compileAgentResources,context:r,externalDependencies:i.config.build?.externalDependencies??[],parentNodeId:ROOT_COMPILED_AGENT_NODE_ID,subagents:e.subagents});return createCompiledAgentManifest({...i,extensionMounts:i.extensionMounts,remoteAgents:a.remoteAgents,subagentEdges:a.edges,subagents:a.nodes})}async function compileAgentNodeManifest(e,t,n={}){let i=Object.hasOwn(n,`agentConfigDefinition`)?await compileAgentConfig(e,t,{definition:n.agentConfigDefinition}):await compileAgentConfig(e,t);if(n.allowWorkflowConfig===!1&&i.experimental?.workflow!==void 0)throw Error(`Workflow runtime configuration is only supported on the root agent config. Remove "experimental.workflow" from "${e.agentId}".`);let a=mergeExternalDependencies(n.externalDependencies,i.build?.externalDependencies),o=a.length===0?i:{...i,build:{...i.build,externalDependencies:a}};return createCompiledAgentNodeManifest({...await compileAgentResources(e,t,{externalDependencies:a}),config:o})}async function compileAgentResources(t,n,r={}){let s=[...r.externalDependencies??[]],c=await Promise.all(t.tools.map(e=>compileToolEntry(t.agentRoot,e,{externalDependencies:s}))),l=[],u=[],d=[],f,p;for(let e of c)e.kind===`tool`?l.push(e.definition):e.kind===`dynamic-tool`?u.push(e.definition):e.kind===`workflow-tool`?f={maxSubagents:e.maxSubagents}:e.kind===`web-search-tool`?p=e.provider:d.push(e.name);let m=(await Promise.all(t.channels.map(n=>compileChannelDefinition(t.agentRoot,n,{externalDependencies:s})))).flat(),h=await Promise.all(t.skills.map(e=>compileSkillSource(t.agentRoot,e,{externalDependencies:s}))),g=[],_=[];for(let e of h)e.kind===`skill`?g.push(e.definition):_.push(e.definition);let v=await Promise.all(t.instructions.map(e=>compileInstructionsEntry(t.agentRoot,e,{externalDependencies:s}))),y=[],b=[];for(let e of v)e.kind===`instructions`?y.push(e.definition):b.push(e.definition);let x=await Promise.all(t.connections.map(e=>compileConnectionDefinition(t.agentRoot,e,{externalDependencies:s}))),S=t.hooks.map(e=>compileHookEntry(e)),C=await Promise.all(t.schedules.map(e=>compileScheduleDefinition(t.agentRoot,e,{externalDependencies:s}))),w=new Set(l.map(e=>e.name)),T=new Set(u.map(e=>e.slug)),E=new Set(x.map(e=>e.connectionName)),D=new Set(g.map(e=>e.name)),O=[];for(let e of[...t.resolvedExtensions].sort((e,t)=>e.namespace.localeCompare(t.namespace))){let r=await compileExtensionContributions({mount:e,context:n,consumerAgentRoot:t.agentRoot,externalDependencies:s});for(let e of r.tools)w.has(e.name)||(w.add(e.name),l.push(e));for(let e of r.dynamicTools)T.has(e.slug)||(T.add(e.slug),u.push(e));for(let e of r.connections)E.has(e.connectionName)||(E.add(e.connectionName),x.push(e));for(let e of r.skills)D.has(e.name)||(D.add(e.name),g.push(e));S.push(...r.hooks),_.push(...r.dynamicSkills),b.push(...r.dynamicInstructions),O.push(...r.instructions)}let k=[...y,...O];return createCompiledAgentResources({agentRoot:t.agentRoot,appRoot:t.appRoot,channels:m,extensionMounts:compileExtensionMounts(t),connections:x,diagnosticsSummary:t.diagnosticsSummary,disabledFrameworkTools:d,workflowTool:f,webSearchProvider:p,dynamicSkills:_,dynamicTools:u,hooks:S,sandbox:t.sandbox===null?null:await compileSandboxDefinition(t.agentRoot,t.sandbox,{externalDependencies:s}),sandboxWorkspaces:t.sandboxWorkspaces.map(e=>({logicalPath:e.logicalPath,rootEntries:[...e.rootEntries],sourceId:e.sourceId,sourcePath:e.sourcePath})),schedules:C,dynamicInstructions:b,skills:g,instructions:k,tools:l})}function compileExtensionMounts(e){return e.resolvedExtensions.map(t=>{let n=e.extensions.find(e=>mountRefNamespace(e.logicalPath)===t.namespace);return{namespace:t.namespace,packageName:t.packageName,packageNamespace:packageStateNamespace(t.packageName),sourceRoot:t.sourceRoot,mountSourceId:n?.sourceId??`extensions/${t.namespace}`,mountLogicalPath:n?.logicalPath??`extensions/${t.namespace}`}})}function mergeExternalDependencies(...e){let t=new Set;for(let n of e)for(let e of n??[])t.add(e);return[...t]}export{compileAgentManifest};

package/dist/src/context/build-dynamic-tools.d.ts

@@ -1,4 +1,5 @@
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import type { HarnessToolMap } from "#harness/types.js";
 import type { ContextKey } from "#context/key.js";
 /**
  * Builds live dynamic tool definitions. Narrower scopes appear first
@@ -8,6 +9,12 @@ import type { ContextKey } from "#context/key.js";
  * `LiveStepToolsKey`). Session/turn tools are replayed from durable
  * metadata via the bundler's registered step functions.
  */
+export declare function buildResponseAuthorizationTools(input: {
+    readonly authoredTools: HarnessToolMap;
+    readonly context?: {
+        get<T>(key: ContextKey<T>): T | undefined;
+    };
+}): HarnessToolMap;
 export declare function buildDynamicTools(ctx: {
     get<T>(key: ContextKey<T>): T | undefined;
 }): readonly HarnessToolDefinition[];

package/dist/src/context/build-dynamic-tools.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{LiveStepToolsKey,SessionDynamicToolMetadataKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toInputSchema,toOutputSchema}from"#shared/tool-schema.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";const log=createLogger(`dynamic-tools`);function lookupStepFunction(e){try{let t=globalThis[Symbol.for(`@workflow/core//registeredSteps`)];return t===void 0?null:t.get(e)||null}catch{return null}}function replayTools(e){let t=[];for(let n of e){if(!n.executeStepFnName||!n.closureVars){log.warn(`Dynamic tool "${n.name}" has no registered step function — skipping on this step. The bundler transform may not have processed this tool file.`);continue}let e=lookupStepFunction(n.executeStepFnName);if(!e){log.warn(`Dynamic tool "${n.name}" references step function "${n.executeStepFnName}" which is not registered — skipping on this step.`);continue}t.push({description:n.description,execute:createToolExecuteWithAuth({scope:n.name,execute:(t,r)=>e(n.closureVars,t,r)}),inputSchema:toInputSchema(n.inputSchema),name:n.name,approval:buildReplayedApproval(n),outputSchema:toOutputSchema(n.outputSchema)})}return t}function buildReplayedApproval(e){if(e.approvalStepFnName===void 0)return;let t=lookupStepFunction(e.approvalStepFnName);return t===null?(log.warn(`Dynamic tool "${e.name}" references approval function "${e.approvalStepFnName}" which is not registered — requiring approval by default.`),()=>`user-approval`):async n=>await t(e.closureVars??{},n)}function buildDynamicTools(e){let i=e.get(LiveStepToolsKey)??[],a=replayTools(e.get(TurnDynamicToolMetadataKey)??[]),o=replayTools(e.get(SessionDynamicToolMetadataKey)??[]);return[...i,...a,...o]}export{buildDynamicTools};
+import{createLogger}from"#internal/logging.js";import{LiveStepToolsKey,SessionDynamicToolMetadataKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toInputSchema,toOutputSchema}from"#shared/tool-schema.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";const log=createLogger(`dynamic-tools`);function lookupStepFunction(e){try{let t=globalThis[Symbol.for(`@workflow/core//registeredSteps`)];return t===void 0?null:t.get(e)||null}catch{return null}}function replayTools(e){let t=[];for(let n of e){if(!n.executeStepFnName||!n.closureVars){log.warn(`Dynamic tool "${n.name}" has no registered step function — skipping on this step. The bundler transform may not have processed this tool file.`);continue}let e=lookupStepFunction(n.executeStepFnName);if(!e){log.warn(`Dynamic tool "${n.name}" references step function "${n.executeStepFnName}" which is not registered — skipping on this step.`);continue}t.push({description:n.description,execute:createToolExecuteWithAuth({scope:n.name,execute:(t,r)=>e(n.closureVars,t,r)}),inputSchema:toInputSchema(n.inputSchema),name:n.name,approval:buildReplayedApproval(n),outputSchema:toOutputSchema(n.outputSchema)})}return t}function buildReplayedApproval(e){if(e.approvalStepFnName===void 0)return;let t=lookupStepFunction(e.approvalStepFnName);if(t===null)return log.warn(`Dynamic tool "${e.name}" references approval function "${e.approvalStepFnName}" which is not registered — requiring approval by default.`),()=>`user-approval`;let policy=async n=>await t(e.closureVars??{},n);if(e.approvalResponseStepFnName===void 0)return policy;let n=lookupStepFunction(e.approvalResponseStepFnName);return n===null?(log.warn(`Dynamic tool "${e.name}" references response authorizer "${e.approvalResponseStepFnName}" which is not registered — rejecting responses.`),{request:policy,response:async()=>({reason:`Approval response authorization is temporarily unavailable.`,status:`rejected`})}):{request:policy,response:async t=>await n(e.closureVars??{},t)}}function buildResponseAuthorizationTools(e){let t=new Map;for(let n of e.context===void 0?[]:buildDynamicTools(e.context))t.has(n.name)||t.set(n.name,n);for(let[n,r]of e.authoredTools)t.has(n)||t.set(n,r);return t}function buildDynamicTools(e){let i=e.get(LiveStepToolsKey)??[],a=replayTools(e.get(TurnDynamicToolMetadataKey)??[]),o=replayTools(e.get(SessionDynamicToolMetadataKey)??[]);return[...i,...a,...o]}export{buildDynamicTools,buildResponseAuthorizationTools};

package/dist/src/context/dynamic-instruction-lifecycle.d.ts

@@ -1,7 +1,7 @@
 import type { ModelMessage, SystemModelMessage } from "ai";
 import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
 import type { ResolvedDynamicInstructionsResolver } from "#runtime/types.js";
-import type { ContextContainer } from "#context/container.js";
+import type { AlsContext, ContextContainer } from "#context/container.js";
 import type { ContextKey } from "#context/key.js";
 /**
  * Builds the flattened system messages from session + turn durable keys.
@@ -10,6 +10,10 @@ import type { ContextKey } from "#context/key.js";
 export declare function buildDynamicInstructionMessages(ctx: {
     get<T>(key: ContextKey<T>): T | undefined;
 }): SystemModelMessage[];
+/** Starts the instructions-only virtual context for one lifecycle preamble. */
+export declare function prepareDynamicInstructionPreamble(ctx: AlsContext, messages: readonly ModelMessage[]): void;
+/** Drains user-role results and clears the instructions-only virtual context. */
+export declare function drainDynamicInstructionUserMessages(ctx: AlsContext): ModelMessage[];
 /**
  * Dispatches a stream event to dynamic instruction resolvers.
  *

package/dist/src/context/dynamic-instruction-lifecycle.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{SessionDynamicInstructionsKey,TurnDynamicInstructionsKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,isBrandedInstructionsEntry}from"#shared/dynamic-tool-definition.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";const log=createLogger(`dynamic-instructions`);function lowerToSystemMessage(e){let t=e.markdown.trim();if(t.length!==0)return{role:`system`,content:t}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicInstructionsKey;case`turn.started`:return TurnDynamicInstructionsKey;default:return}}function buildDynamicInstructionMessages(e){let r=e.get(SessionDynamicInstructionsKey)??{},i=e.get(TurnDynamicInstructionsKey)??{};return[...Object.values(r).flat(),...Object.values(i).flat()]}async function dispatchDynamicInstructionEvent(e){let{ctx:t,resolvers:n,event:a,messages:o}=e;if(!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(a.type))return;let s=n.filter(e=>e.eventNames.includes(a.type));if(s.length===0)return;let c=durableKeyForEvent(a.type);if(c===void 0)return;let l=buildResolveContext(t,o),u=await Promise.allSettled(s.map(async e=>{let t=e.events[a.type];if(t===void 0)return null;let n=await t(a,l);return n==null?{resolver:e,message:void 0}:isBrandedInstructionsEntry(n)?{resolver:e,message:lowerToSystemMessage(n)}:(log.error(`Dynamic instructions resolver "${e.slug}" returned an unbranded value — wrap with defineInstructions().`),null)})),d={...t.get(c)};for(let e of u){if(e.status===`rejected`){log.error(`Dynamic instructions resolver (${a.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,message:n}=e.value;n===void 0?delete d[t.slug]:d[t.slug]=[n]}t.set(c,d)}export{buildDynamicInstructionMessages,dispatchDynamicInstructionEvent};
+import{createLogger}from"#internal/logging.js";import{DynamicInstructionResolveMessagesKey,PendingDynamicInstructionUserMessagesKey,SessionDynamicInstructionsKey,TurnDynamicInstructionsKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{normalizeInstructionsDefinition}from"#internal/authored-definition/core.js";import{ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,isBrandedInstructionsEntry}from"#shared/dynamic-tool-definition.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";const log=createLogger(`dynamic-instructions`);function lowerInstruction(e){let t=normalizeInstructionsDefinition(e,`Expected dynamic instructions to match the public eve shape.`),n=t.content.trim();if(n.length!==0)return t.role===`system`?{role:`system`,message:{role:`system`,content:n}}:{role:`user`,message:{role:`user`,content:n}}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicInstructionsKey;case`turn.started`:return TurnDynamicInstructionsKey;default:return}}function buildDynamicInstructionMessages(e){let t=e.get(SessionDynamicInstructionsKey)??{},n=e.get(TurnDynamicInstructionsKey)??{};return[...Object.values(t).flat(),...Object.values(n).flat()]}function prepareDynamicInstructionPreamble(e,r){e.setVirtualContext(DynamicInstructionResolveMessagesKey,[...r]),e.setVirtualContext(PendingDynamicInstructionUserMessagesKey,[])}function drainDynamicInstructionUserMessages(e){let r=[...e.get(PendingDynamicInstructionUserMessagesKey)??[]];return e.delete(DynamicInstructionResolveMessagesKey),e.delete(PendingDynamicInstructionUserMessagesKey),r}async function dispatchDynamicInstructionEvent(e){let{ctx:r,resolvers:a,event:o,messages:s}=e;if(!ALLOWED_DYNAMIC_INSTRUCTION_EVENTS.has(o.type))return;o.type===`turn.started`&&r.set(TurnDynamicInstructionsKey,{});let c=a.filter(e=>e.eventNames.includes(o.type));if(c.length===0)return;let l=durableKeyForEvent(o.type);if(l===void 0)return;let u=r.get(DynamicInstructionResolveMessagesKey)??s,d=r.get(PendingDynamicInstructionUserMessagesKey)??[],f=buildResolveContext(r,[...u,...d]),p=await Promise.allSettled(c.map(async e=>{let t=e.events[o.type];if(t===void 0)return null;let n=await t(o,f);if(n==null)return{resolver:e,instruction:void 0};if(!isBrandedInstructionsEntry(n))return log.error(`Dynamic instructions resolver "${e.slug}" returned an unbranded value — wrap with defineInstructions().`),null;try{return{resolver:e,instruction:lowerInstruction(n)}}catch(t){return log.error(`Dynamic instructions resolver "${e.slug}" returned an invalid value.`,{error:toErrorMessage(t)}),null}})),m={...r.get(l)};for(let e of p){if(e.status===`rejected`){log.error(`Dynamic instructions resolver (${o.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,instruction:i}=e.value;delete m[t.slug],i?.role===`system`?m[t.slug]=[i.message]:i?.role===`user`&&r.setVirtualContext(PendingDynamicInstructionUserMessagesKey,[...r.get(PendingDynamicInstructionUserMessagesKey)??[],i.message])}r.set(l,m)}export{buildDynamicInstructionMessages,dispatchDynamicInstructionEvent,drainDynamicInstructionUserMessages,prepareDynamicInstructionPreamble};

package/dist/src/context/dynamic-tool-lifecycle.d.ts

@@ -17,6 +17,13 @@ export declare function replayDynamicSessionTools(metadata: readonly DurableDyna
  * scope-appropriate durable key. The tool-loop calls
  * {@link buildDynamicTools} to assemble the effective toolset.
  */
+/** Resolves step-scoped tools once for one internal policy/model pass. */
+export declare function resolveStepDynamicTools(input: {
+    readonly ctx: ContextContainer;
+    readonly resolvers: readonly ResolvedDynamicToolResolver[];
+    readonly event: UnstampedMessageStreamEvent;
+    readonly messages: readonly ModelMessage[];
+}): Promise<void>;
 export declare function dispatchDynamicToolEvent(input: {
     readonly ctx: ContextContainer;
     readonly resolvers: readonly ResolvedDynamicToolResolver[];

package/dist/src/context/dynamic-tool-lifecycle.js

@@ -1 +1 @@
-import{createLogger}from"#internal/logging.js";import{LiveStepToolsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{serializeInputSchema,serializeOutputSchema,toInputSchema,toOutputSchema}from"#shared/tool-schema.js";import{ALLOWED_DYNAMIC_TOOL_EVENTS,isBrandedToolEntry}from"#shared/dynamic-tool-definition.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";const log=createLogger(`dynamic-tools`);function toHarnessToolDefinition(e,t){return{description:t.description,execute:createToolExecuteWithAuth({scope:e,execute:(e,n)=>t.execute(e,n)}),inputSchema:toInputSchema(t.inputSchema),name:e,approval:t.approval,outputSchema:toOutputSchema(t.outputSchema),...t.toModelOutput===void 0?{}:{toModelOutput:t.toModelOutput}}}function qualifyDynamicToolNames(e,t,n){let r=Object.keys(n),i=[];if(r.length===0)return i;if(t)return i.push({name:e.slug,entryKey:r[0],entry:n[r[0]]}),i;let a=e.extensionNamespace===void 0?``:`${e.extensionNamespace}__`;for(let e of r)i.push({name:`${a}${e}`,entryKey:e,entry:n[e]});return i}function replayDynamicSessionTools(e,t){let n=[];for(let t of e){if(!t.executeStepFnName||!t.closureVars){log.warn(`Dynamic tool "${t.name}" has no registered step function — skipping on this step. The bundler transform may not have processed this tool file.`);continue}let e=lookupStepFunction(t.executeStepFnName);if(!e){log.warn(`Dynamic tool "${t.name}" references step function "${t.executeStepFnName}" which is not registered — skipping on this step.`);continue}n.push({description:t.description,execute:createToolExecuteWithAuth({scope:t.name,execute:(n,r)=>e(t.closureVars,n,r)}),inputSchema:toInputSchema(t.inputSchema),name:t.name,outputSchema:toOutputSchema(t.outputSchema)})}return n}function getStepRegistry(){let e=Symbol.for(`@workflow/core//registeredSteps`),t=globalThis,n=t[e];return n===void 0&&(n=new Map,t[e]=n),n}function lookupStepFunction(e){try{return getStepRegistry().get(e)||null}catch{return null}}function registerStepFunction(e,t){getStepRegistry().set(e,t)}function safeSerialize(e){try{return JSON.parse(JSON.stringify(e))}catch{return{}}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicToolMetadataKey;case`turn.started`:return TurnDynamicToolMetadataKey;default:return}}function readDynamicToolResult(e,t){if(isBrandedToolEntry(t))return{entries:{_single:t},isSingle:!0};if(typeof t!=`object`||!t||Array.isArray(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" must return defineTool(), a map of defineTool() values, or null.`);let n={};for(let[r,i]of Object.entries(t)){if(!isBrandedToolEntry(i))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned "${r}" without defineTool(). Wrap every dynamic tool entry in defineTool().`);n[r]=i}return{entries:n,isSingle:!1}}async function resolveToolsFromEvent(e,t,n,r){let i=await Promise.allSettled(t.map(async t=>{let i=t.events[n.type];if(i===void 0)return null;let a=await i(n,buildResolveContext(e,r));if(a==null)return null;let{entries:o,isSingle:s}=readDynamicToolResult(t,a);return{resolver:t,entries:o,isSingle:s}})),c=[],l=[],u=new Map;for(let e of i){if(e.status===`rejected`){log.error(`Dynamic tool resolver (${n.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,entries:r,isSingle:i}=e.value,d=qualifyDynamicToolNames(t,i,r);for(let{name:e,entryKey:r,entry:i}of d){let a=u.get(e);if(a!==void 0&&a!==t.slug)throw Error(`Dynamic tool "${e}" from resolver "${t.slug}" collides with dynamic resolver "${a}". Namespace the map key manually, e.g. "${t.slug}__${e}".`);if(u.set(e,t.slug),l.push(toHarnessToolDefinition(e,i)),n.type===`step.started`)continue;let d=`__executeStepFn`in i?i.__executeStepFn:void 0,f=`__closureVars`in i?i.__closureVars:void 0,p=d?.stepId,m=f===void 0?void 0:safeSerialize(f);if(p===void 0){let e=`eve:framework-dynamic:${t.slug}:${r}`,n=i.execute.bind(i);registerStepFunction(e,(e,t,r)=>n(t,r)),p=e,m={}}let h;if(i.approval!==void 0){h=`eve:dynamic-tool-approval:${t.slug}:${r}`;let e=i.approval.bind(i);registerStepFunction(h,(t,n)=>e(n))}c.push({name:e,description:i.description,inputSchema:serializeInputSchema(i.inputSchema),outputSchema:serializeOutputSchema(i.outputSchema),resolverSlug:t.slug,entryKey:r,executeStepFnName:p,approvalStepFnName:h,closureVars:m})}}return{metadata:c,liveTools:l}}async function dispatchDynamicToolEvent(e){let{ctx:r,resolvers:i,event:a,messages:o}=e;if(!ALLOWED_DYNAMIC_TOOL_EVENTS.has(a.type))return;let s=i.filter(e=>e.eventNames.includes(a.type));if(s.length===0){a.type===`session.started`&&r.set(SessionDynamicToolMetadataKey,[]);return}let{metadata:c,liveTools:l}=await resolveToolsFromEvent(r,s,a,o);if(a.type===`step.started`){r.setVirtualContext(LiveStepToolsKey,l);return}let d=durableKeyForEvent(a.type);if(d===void 0)return;if(a.type===`session.started`){r.set(SessionDynamicToolMetadataKey,c);return}let f=new Set(s.map(e=>e.slug)),p=(r.get(d)??[]).filter(e=>!f.has(e.resolverSlug));r.set(d,[...p,...c])}async function refreshDynamicSessionToolsForRuntimeRevision(e){if(e.ctx.get(SessionDynamicToolRuntimeRevisionKey)===e.runtimeRevision)return;let t=e.resolvers.filter(e=>e.eventNames.includes(`session.started`)),{metadata:i}=t.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,t,e.event,e.messages);e.ctx.set(SessionDynamicToolMetadataKey,i),e.ctx.set(SessionDynamicToolRuntimeRevisionKey,e.runtimeRevision)}export{dispatchDynamicToolEvent,refreshDynamicSessionToolsForRuntimeRevision,replayDynamicSessionTools};
+import{createLogger}from"#internal/logging.js";import{LiveStepToolsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,TurnDynamicToolMetadataKey}from"#context/keys.js";import{toErrorMessage}from"#shared/errors.js";import{serializeInputSchema,serializeOutputSchema,toInputSchema,toOutputSchema}from"#shared/tool-schema.js";import{ALLOWED_DYNAMIC_TOOL_EVENTS,isBrandedToolEntry}from"#shared/dynamic-tool-definition.js";import{createToolExecuteWithAuth}from"#execution/tool-auth.js";import{buildResolveContext}from"#context/dynamic-resolve-context.js";import{resolveApprovalPolicy}from"#public/definitions/approval.js";const log=createLogger(`dynamic-tools`);function toHarnessToolDefinition(e,t){return{description:t.description,execute:createToolExecuteWithAuth({scope:e,execute:(e,n)=>t.execute(e,n)}),inputSchema:toInputSchema(t.inputSchema),name:e,approval:t.approval,outputSchema:toOutputSchema(t.outputSchema),...t.toModelOutput===void 0?{}:{toModelOutput:t.toModelOutput}}}function qualifyDynamicToolNames(e,t,n){let r=Object.keys(n),i=[];if(r.length===0)return i;if(t)return i.push({name:e.slug,entryKey:r[0],entry:n[r[0]]}),i;let a=e.extensionNamespace===void 0?``:`${e.extensionNamespace}__`;for(let e of r)i.push({name:`${a}${e}`,entryKey:e,entry:n[e]});return i}function replayDynamicSessionTools(e,t){let n=[];for(let t of e){if(!t.executeStepFnName||!t.closureVars){log.warn(`Dynamic tool "${t.name}" has no registered step function — skipping on this step. The bundler transform may not have processed this tool file.`);continue}let e=lookupStepFunction(t.executeStepFnName);if(!e){log.warn(`Dynamic tool "${t.name}" references step function "${t.executeStepFnName}" which is not registered — skipping on this step.`);continue}n.push({description:t.description,execute:createToolExecuteWithAuth({scope:t.name,execute:(n,r)=>e(t.closureVars,n,r)}),inputSchema:toInputSchema(t.inputSchema),name:t.name,outputSchema:toOutputSchema(t.outputSchema)})}return n}function getStepRegistry(){let e=Symbol.for(`@workflow/core//registeredSteps`),t=globalThis,n=t[e];return n===void 0&&(n=new Map,t[e]=n),n}function lookupStepFunction(e){try{return getStepRegistry().get(e)||null}catch{return null}}function registerStepFunction(e,t){getStepRegistry().set(e,t)}function safeSerialize(e){try{return JSON.parse(JSON.stringify(e))}catch{return{}}}function durableKeyForEvent(e){switch(e){case`session.started`:return SessionDynamicToolMetadataKey;case`turn.started`:return TurnDynamicToolMetadataKey;default:return}}function readDynamicToolResult(e,t){if(isBrandedToolEntry(t))return{entries:{_single:t},isSingle:!0};if(typeof t!=`object`||!t||Array.isArray(t))throw Error(`Dynamic tool resolver "${e.logicalPath}" must return defineTool(), a map of defineTool() values, or null.`);let n={};for(let[r,i]of Object.entries(t)){if(!isBrandedToolEntry(i))throw Error(`Dynamic tool resolver "${e.logicalPath}" returned "${r}" without defineTool(). Wrap every dynamic tool entry in defineTool().`);n[r]=i}return{entries:n,isSingle:!1}}async function resolveToolsFromEvent(e,t,n,r){let i=await Promise.allSettled(t.map(async t=>{let i=t.events[n.type];if(i===void 0)return null;let a=await i(n,buildResolveContext(e,r));if(a==null)return null;let{entries:o,isSingle:s}=readDynamicToolResult(t,a);return{resolver:t,entries:o,isSingle:s}})),c=[],l=[],u=new Map;for(let e of i){if(e.status===`rejected`){log.error(`Dynamic tool resolver (${n.type}) threw — skipping.`,{error:toErrorMessage(e.reason)});continue}if(e.value===null)continue;let{resolver:t,entries:r,isSingle:i}=e.value,d=qualifyDynamicToolNames(t,i,r);for(let{name:e,entryKey:r,entry:i}of d){let a=u.get(e);if(a!==void 0&&a!==t.slug)throw Error(`Dynamic tool "${e}" from resolver "${t.slug}" collides with dynamic resolver "${a}". Namespace the map key manually, e.g. "${t.slug}__${e}".`);if(u.set(e,t.slug),l.push(toHarnessToolDefinition(e,i)),n.type===`step.started`)continue;let d=`__executeStepFn`in i?i.__executeStepFn:void 0,f=`__closureVars`in i?i.__closureVars:void 0,p=d?.stepId,m=f===void 0?void 0:safeSerialize(f);if(p===void 0){let e=`eve:framework-dynamic:${t.slug}:${r}`,n=i.execute.bind(i);registerStepFunction(e,(e,t,r)=>n(t,r)),p=e,m={}}let h,g;if(i.approval!==void 0){h=`eve:dynamic-tool-approval:${t.slug}:${r}`;let e=resolveApprovalPolicy(i.approval).bind(i);registerStepFunction(h,(t,n)=>e(n));let n=typeof i.approval==`function`?void 0:i.approval.response;n!==void 0&&(g=`eve:dynamic-tool-approval-response:${t.slug}:${r}`,registerStepFunction(g,(e,t)=>n(t)))}c.push({name:e,description:i.description,inputSchema:serializeInputSchema(i.inputSchema),outputSchema:serializeOutputSchema(i.outputSchema),resolverSlug:t.slug,entryKey:r,executeStepFnName:p,approvalStepFnName:h,approvalResponseStepFnName:g,closureVars:m})}}return{metadata:c,liveTools:l}}const resolvedStepTools=new WeakMap;async function resolveStepDynamicTools(e){let n=`data`in e.event?e.event.data:void 0,r=typeof n?.turnId==`string`&&typeof n.stepIndex==`number`?`${n.turnId}:${String(n.stepIndex)}`:void 0,i=resolvedStepTools.get(e.ctx);if(r!==void 0&&i?.coordinate===r){e.ctx.setVirtualContext(LiveStepToolsKey,i.tools);return}let a=e.resolvers.filter(e=>e.eventNames.includes(`step.started`)),{liveTools:o}=a.length===0?{liveTools:[]}:await resolveToolsFromEvent(e.ctx,a,e.event,e.messages);e.ctx.setVirtualContext(LiveStepToolsKey,o),r!==void 0&&resolvedStepTools.set(e.ctx,{coordinate:r,tools:o})}async function dispatchDynamicToolEvent(e){let{ctx:t,resolvers:r,event:i,messages:a}=e;if(!ALLOWED_DYNAMIC_TOOL_EVENTS.has(i.type))return;if(i.type===`step.started`){await resolveStepDynamicTools(e);return}let o=r.filter(e=>e.eventNames.includes(i.type));if(o.length===0){i.type===`session.started`&&t.set(SessionDynamicToolMetadataKey,[]);return}let{metadata:s}=await resolveToolsFromEvent(t,o,i,a),c=durableKeyForEvent(i.type);if(c===void 0)return;if(i.type===`session.started`){t.set(SessionDynamicToolMetadataKey,s);return}let l=new Set(o.map(e=>e.slug)),d=(t.get(c)??[]).filter(e=>!l.has(e.resolverSlug));t.set(c,[...d,...s])}async function refreshDynamicSessionToolsForRuntimeRevision(e){if(e.ctx.get(SessionDynamicToolRuntimeRevisionKey)===e.runtimeRevision)return;let t=e.resolvers.filter(e=>e.eventNames.includes(`session.started`)),{metadata:i}=t.length===0?{metadata:[]}:await resolveToolsFromEvent(e.ctx,t,e.event,e.messages);e.ctx.set(SessionDynamicToolMetadataKey,i),e.ctx.set(SessionDynamicToolRuntimeRevisionKey,e.runtimeRevision)}export{dispatchDynamicToolEvent,refreshDynamicSessionToolsForRuntimeRevision,replayDynamicSessionTools,resolveStepDynamicTools};

package/dist/src/context/keys.d.ts

@@ -3,10 +3,11 @@
  * tier. Codec-carrying keys (`ChannelKey`, `BundleKey`) live in
  * `#runtime/sessions/runtime-context-keys.ts`.
  */
-import type { LanguageModel, SystemModelMessage } from "ai";
+import type { LanguageModel, ModelMessage, SystemModelMessage } from "ai";
 import type { JsonObject } from "#shared/json.js";
-import type { ChannelInstrumentationProjection, SessionAuthContext, SessionCallback, SessionCapabilities, SessionParent, SessionTraceContext, SessionTurn } from "#channel/types.js";
+import type { ChannelInstrumentationProjection, ChannelDeliveryMetadata, SessionAuthContext, SessionCallback, SessionCapabilities, SessionParent, SessionTraceContext, SessionTurn } from "#channel/types.js";
 import { ContextKey } from "#context/key.js";
+import type { InstrumentationChannelDeliveryRef } from "#harness/instrumentation/lifecycle.js";
 import type { HarnessToolDefinition } from "#harness/execute-tool.js";
 import type { DynamicSubagentAgentConfig } from "#runtime/subagents/dynamic-agent-config.js";
 import type { DynamicRemoteAgentConfig } from "#runtime/subagents/dynamic-remote-agent-config.js";
@@ -44,6 +45,16 @@ export declare const InitiatorAuthKey: ContextKey<SessionAuthContext | null>;
 export declare const SessionIdKey: ContextKey<string>;
 export declare const ContinuationTokenKey: ContextKey<string>;
 export declare const ChannelRequestIdKey: ContextKey<string>;
+export declare const ChannelDeliveryKey: ContextKey<ChannelDeliveryMetadata>;
+export interface ActiveChannelDelivery {
+    readonly agentName?: string;
+    readonly delivery: InstrumentationChannelDeliveryRef;
+    readonly rootSessionId: string;
+    readonly sequence: number;
+    readonly sessionId: string;
+    readonly turnId: string;
+}
+export declare const ActiveChannelDeliveriesKey: ContextKey<readonly ActiveChannelDelivery[]>;
 export declare const ChannelInstrumentationKey: ContextKey<ChannelInstrumentationProjection>;
 export declare const ModeKey: ContextKey<RunMode>;
 export declare const ParentSessionKey: ContextKey<SessionParent>;
@@ -90,6 +101,7 @@ export interface DurableDynamicToolMetadata {
     readonly entryKey: string;
     readonly executeStepFnName?: string;
     readonly approvalStepFnName?: string;
+    readonly approvalResponseStepFnName?: string;
     readonly closureVars?: Record<string, unknown>;
 }
 /**
@@ -152,3 +164,7 @@ export declare const SessionDynamicInstructionsKey: ContextKey<Record<string, re
  * resolvers). Keyed by resolver slug. Replaced each turn.
  */
 export declare const TurnDynamicInstructionsKey: ContextKey<Record<string, readonly SystemModelMessage[]>>;
+/** Existing history exposed only to instructions resolvers during a preamble. */
+export declare const DynamicInstructionResolveMessagesKey: ContextKey<readonly ModelMessage[]>;
+/** User-role results waiting to be committed immediately after a preamble. */
+export declare const PendingDynamicInstructionUserMessagesKey: ContextKey<readonly ModelMessage[]>;

package/dist/src/context/keys.js

@@ -1 +1 @@
-import{ContextKey}from"#context/key.js";const AuthKey=new ContextKey(`eve.auth`),InitiatorAuthKey=new ContextKey(`eve.initiatorAuth`),SessionIdKey=new ContextKey(`eve.sessionId`),ContinuationTokenKey=new ContextKey(`eve.continuationToken`),ChannelRequestIdKey=new ContextKey(`eve.channelRequestId`),ChannelInstrumentationKey=new ContextKey(`eve.channelInstrumentation`),ModeKey=new ContextKey(`eve.mode`),ParentSessionKey=new ContextKey(`eve.parentSession`),ParentTraceContextKey=new ContextKey(`eve.parentTraceContext`),SubagentDepthKey=new ContextKey(`eve.subagentDepth`),CapabilitiesKey=new ContextKey(`eve.capabilities`),SessionCallbackKey=new ContextKey(`eve.sessionCallback`),SessionKey=new ContextKey(`eve.session`),SandboxKey=new ContextKey(`eve.sandbox`),SessionDynamicModelReferenceKey=new ContextKey(`eve.sessionDynamicModelReference`),TurnDynamicModelReferenceKey=new ContextKey(`eve.turnDynamicModelReference`),RuntimeModelMetadataCacheKey=new ContextKey(`eve.runtimeModelMetadataCache`),LiveStepDynamicModelSelectionKey=new ContextKey(`eve.liveStepDynamicModelSelection`),SessionDynamicToolMetadataKey=new ContextKey(`eve.sessionDynamicToolMetadata`),SessionDynamicToolRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicToolRuntimeRevision`),TurnDynamicToolMetadataKey=new ContextKey(`eve.turnDynamicToolMetadata`),LiveStepToolsKey=new ContextKey(`eve.liveStepTools`),SessionDynamicSubagentSelectionsKey=new ContextKey(`eve.sessionDynamicSubagentSelections`),TurnDynamicSubagentSelectionsKey=new ContextKey(`eve.turnDynamicSubagentSelections`),SessionDynamicSubagentRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicSubagentRuntimeRevision`),DynamicSubagentAgentConfigKey=new ContextKey(`eve.dynamicSubagentAgentConfig`),DynamicSkillManifestKey=new ContextKey(`eve.dynamicSkillManifest`),SessionDynamicInstructionsKey=new ContextKey(`eve.sessionDynamicInstructions`),TurnDynamicInstructionsKey=new ContextKey(`eve.turnDynamicInstructions`);export{AuthKey,CapabilitiesKey,ChannelInstrumentationKey,ChannelRequestIdKey,ContinuationTokenKey,DynamicSkillManifestKey,DynamicSubagentAgentConfigKey,InitiatorAuthKey,LiveStepDynamicModelSelectionKey,LiveStepToolsKey,ModeKey,ParentSessionKey,ParentTraceContextKey,RuntimeModelMetadataCacheKey,SandboxKey,SessionCallbackKey,SessionDynamicInstructionsKey,SessionDynamicModelReferenceKey,SessionDynamicSubagentRuntimeRevisionKey,SessionDynamicSubagentSelectionsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,SessionKey,SubagentDepthKey,TurnDynamicInstructionsKey,TurnDynamicModelReferenceKey,TurnDynamicSubagentSelectionsKey,TurnDynamicToolMetadataKey};
+import{ContextKey}from"#context/key.js";const AuthKey=new ContextKey(`eve.auth`),InitiatorAuthKey=new ContextKey(`eve.initiatorAuth`),SessionIdKey=new ContextKey(`eve.sessionId`),ContinuationTokenKey=new ContextKey(`eve.continuationToken`),ChannelRequestIdKey=new ContextKey(`eve.channelRequestId`),ChannelDeliveryKey=new ContextKey(`eve.channelDelivery`),ActiveChannelDeliveriesKey=new ContextKey(`eve.activeChannelDeliveries`),ChannelInstrumentationKey=new ContextKey(`eve.channelInstrumentation`),ModeKey=new ContextKey(`eve.mode`),ParentSessionKey=new ContextKey(`eve.parentSession`),ParentTraceContextKey=new ContextKey(`eve.parentTraceContext`),SubagentDepthKey=new ContextKey(`eve.subagentDepth`),CapabilitiesKey=new ContextKey(`eve.capabilities`),SessionCallbackKey=new ContextKey(`eve.sessionCallback`),SessionKey=new ContextKey(`eve.session`),SandboxKey=new ContextKey(`eve.sandbox`),SessionDynamicModelReferenceKey=new ContextKey(`eve.sessionDynamicModelReference`),TurnDynamicModelReferenceKey=new ContextKey(`eve.turnDynamicModelReference`),RuntimeModelMetadataCacheKey=new ContextKey(`eve.runtimeModelMetadataCache`),LiveStepDynamicModelSelectionKey=new ContextKey(`eve.liveStepDynamicModelSelection`),SessionDynamicToolMetadataKey=new ContextKey(`eve.sessionDynamicToolMetadata`),SessionDynamicToolRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicToolRuntimeRevision`),TurnDynamicToolMetadataKey=new ContextKey(`eve.turnDynamicToolMetadata`),LiveStepToolsKey=new ContextKey(`eve.liveStepTools`),SessionDynamicSubagentSelectionsKey=new ContextKey(`eve.sessionDynamicSubagentSelections`),TurnDynamicSubagentSelectionsKey=new ContextKey(`eve.turnDynamicSubagentSelections`),SessionDynamicSubagentRuntimeRevisionKey=new ContextKey(`eve.sessionDynamicSubagentRuntimeRevision`),DynamicSubagentAgentConfigKey=new ContextKey(`eve.dynamicSubagentAgentConfig`),DynamicSkillManifestKey=new ContextKey(`eve.dynamicSkillManifest`),SessionDynamicInstructionsKey=new ContextKey(`eve.sessionDynamicInstructions`),TurnDynamicInstructionsKey=new ContextKey(`eve.turnDynamicInstructions`),DynamicInstructionResolveMessagesKey=new ContextKey(`eve.dynamicInstructionResolveMessages`),PendingDynamicInstructionUserMessagesKey=new ContextKey(`eve.pendingDynamicInstructionUserMessages`);export{ActiveChannelDeliveriesKey,AuthKey,CapabilitiesKey,ChannelDeliveryKey,ChannelInstrumentationKey,ChannelRequestIdKey,ContinuationTokenKey,DynamicInstructionResolveMessagesKey,DynamicSkillManifestKey,DynamicSubagentAgentConfigKey,InitiatorAuthKey,LiveStepDynamicModelSelectionKey,LiveStepToolsKey,ModeKey,ParentSessionKey,ParentTraceContextKey,PendingDynamicInstructionUserMessagesKey,RuntimeModelMetadataCacheKey,SandboxKey,SessionCallbackKey,SessionDynamicInstructionsKey,SessionDynamicModelReferenceKey,SessionDynamicSubagentRuntimeRevisionKey,SessionDynamicSubagentSelectionsKey,SessionDynamicToolMetadataKey,SessionDynamicToolRuntimeRevisionKey,SessionIdKey,SessionKey,SubagentDepthKey,TurnDynamicInstructionsKey,TurnDynamicModelReferenceKey,TurnDynamicSubagentSelectionsKey,TurnDynamicToolMetadataKey};

package/dist/src/discover/filesystem.d.ts

@@ -15,7 +15,7 @@ export type DirectoryEntryType = "directory" | "file" | "other";
 /**
  * Classified root-level agent entry.
  */
-export type AgentRootEntryKind = "agent-config-module" | "channels-directory" | "connections-directory" | "extensions-directory" | "hooks-directory" | "ignored-directory" | "instructions-directory" | "instructions-markdown" | "instructions-module" | "lib-directory" | "sandbox-directory" | "schedules-directory" | "skills-directory" | "system-markdown" | "system-module" | "tools-directory" | "unknown" | "subagents-directory";
+export type AgentRootEntryKind = "agent-config-module" | "channels-directory" | "connections-directory" | "extensions-directory" | "hooks-directory" | "ignored-directory" | "instrumentation-directory" | "instructions-directory" | "instructions-markdown" | "instructions-module" | "lib-directory" | "sandbox-directory" | "schedules-directory" | "skills-directory" | "system-markdown" | "system-module" | "tools-directory" | "unknown" | "subagents-directory";
 /**
  * Classified local-subagent root entry.
  */

package/dist/src/discover/filesystem.js

@@ -1 +1 @@
-import{sep}from"node:path";const SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS=[`.cts`,`.mts`,`.cjs`,`.mjs`,`.ts`,`.js`],PROJECT_MARKER_FILE_NAMES=[`package.json`,`vercel.json`],PROJECT_MARKER_FILE_NAME_SET=new Set(PROJECT_MARKER_FILE_NAMES),GENERATED_AGENT_DIRECTORY_NAMES=new Set([`.devtools`,`.eve`,`.next`,`.output`,`.vercel`,`node_modules`]);function getDirectoryEntryType(e){return e.isDirectory()?`directory`:e.isFile()?`file`:`other`}function isProjectMarkerEntry(e,t){return t===`file`&&PROJECT_MARKER_FILE_NAME_SET.has(e)}function classifyAgentRootEntry(e,t){if(t===`file`)return matchesSupportedModuleBaseName(e,`agent`)?`agent-config-module`:e.toLowerCase()===`instructions.md`?`instructions-markdown`:matchesSupportedModuleBaseName(e,`instructions`)?`instructions-module`:e.toLowerCase()===`system.md`?`system-markdown`:matchesSupportedModuleBaseName(e,`system`)?`system-module`:`unknown`;if(t===`directory`){if(GENERATED_AGENT_DIRECTORY_NAMES.has(e))return`ignored-directory`;if(e===`channels`)return`channels-directory`;if(e===`connections`)return`connections-directory`;if(e===`extensions`)return`extensions-directory`;if(e===`hooks`)return`hooks-directory`;if(e===`instructions`)return`instructions-directory`;if(e===`lib`)return`lib-directory`;if(e===`skills`)return`skills-directory`;if(e===`sandbox`)return`sandbox-directory`;if(e===`tools`)return`tools-directory`;if(e===`schedules`)return`schedules-directory`;if(e===`subagents`)return`subagents-directory`}return`unknown`}function classifyLocalSubagentEntry(e,t){if(t===`file`)return matchesSupportedModuleBaseName(e,`agent`)?`agent-config-module`:e.toLowerCase()===`instructions.md`?`instructions-markdown`:matchesSupportedModuleBaseName(e,`instructions`)?`instructions-module`:e.toLowerCase()===`system.md`?`system-markdown`:matchesSupportedModuleBaseName(e,`system`)?`system-module`:`unknown`;if(t===`directory`){if(GENERATED_AGENT_DIRECTORY_NAMES.has(e))return`ignored-directory`;if(e===`connections`)return`connections-directory`;if(e===`extensions`)return`extensions-directory`;if(e===`hooks`)return`hooks-directory`;if(e===`instructions`)return`instructions-directory`;if(e===`lib`)return`lib-directory`;if(e===`sandbox`)return`sandbox-directory`;if(e===`skills`)return`skills-directory`;if(e===`tools`)return`tools-directory`;if(e===`subagents`)return`subagents-directory`;if(e===`schedules`)return`invalid-schedules-directory`}return`unknown`}function classifySkillPackageEntry(e,t){if(t===`file`)return e.toLowerCase()===`skill.md`?`skill-markdown`:`skill-resource`;if(t===`directory`){if(e===`scripts`)return`skill-scripts-directory`;if(e===`references`)return`skill-references-directory`;if(e===`assets`)return`skill-assets-directory`}return`skill-resource`}function classifySkillsDirectoryEntry(e,t){if(t===`directory`)return`skill-package-directory`;if(t===`file`){if(isTypeScriptDeclarationFileName(e))return`ignored-declaration`;if(e.toLowerCase().endsWith(`.md`))return`flat-skill-markdown`;if(getSupportedModuleBaseName(e)!==null)return`flat-skill-module`}return`unknown`}function normalizeLogicalPath(t){return t.replaceAll(sep,`/`).replace(/^\.\//,``).replace(/^\/+/,``)}function getSupportedModuleBaseName(e){if(isTypeScriptDeclarationFileName(e))return null;for(let n of SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS)if(e.endsWith(n)&&e.length>n.length)return e.slice(0,-n.length);return null}function isTypeScriptDeclarationFileName(e){return/\.d\.(?:cts|mts|ts)$/.test(e)}function matchesSupportedModuleBaseName(e,t){return getSupportedModuleBaseName(e)===t}function stripLogicalPathExtension(e){let t=normalizeLogicalPath(e),n=t.lastIndexOf(`/`),r=t.lastIndexOf(`.`);return r===-1||r<n?t:t.slice(0,r)}export{PROJECT_MARKER_FILE_NAMES,SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS,classifyAgentRootEntry,classifyLocalSubagentEntry,classifySkillPackageEntry,classifySkillsDirectoryEntry,getDirectoryEntryType,getSupportedModuleBaseName,isProjectMarkerEntry,isTypeScriptDeclarationFileName,matchesSupportedModuleBaseName,normalizeLogicalPath,stripLogicalPathExtension};
+import{sep}from"node:path";const SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS=[`.cts`,`.mts`,`.cjs`,`.mjs`,`.ts`,`.js`],PROJECT_MARKER_FILE_NAMES=[`package.json`,`vercel.json`],PROJECT_MARKER_FILE_NAME_SET=new Set(PROJECT_MARKER_FILE_NAMES),GENERATED_AGENT_DIRECTORY_NAMES=new Set([`.devtools`,`.eve`,`.next`,`.output`,`.vercel`,`node_modules`]);function getDirectoryEntryType(e){return e.isDirectory()?`directory`:e.isFile()?`file`:`other`}function isProjectMarkerEntry(e,t){return t===`file`&&PROJECT_MARKER_FILE_NAME_SET.has(e)}function classifyAgentRootEntry(e,t){if(t===`file`)return matchesSupportedModuleBaseName(e,`agent`)?`agent-config-module`:e.toLowerCase()===`instructions.md`?`instructions-markdown`:matchesSupportedModuleBaseName(e,`instructions`)?`instructions-module`:e.toLowerCase()===`system.md`?`system-markdown`:matchesSupportedModuleBaseName(e,`system`)?`system-module`:`unknown`;if(t===`directory`){if(GENERATED_AGENT_DIRECTORY_NAMES.has(e))return`ignored-directory`;if(e===`channels`)return`channels-directory`;if(e===`connections`)return`connections-directory`;if(e===`extensions`)return`extensions-directory`;if(e===`hooks`)return`hooks-directory`;if(e===`instructions`)return`instructions-directory`;if(e===`instrumentation`)return`instrumentation-directory`;if(e===`lib`)return`lib-directory`;if(e===`skills`)return`skills-directory`;if(e===`sandbox`)return`sandbox-directory`;if(e===`tools`)return`tools-directory`;if(e===`schedules`)return`schedules-directory`;if(e===`subagents`)return`subagents-directory`}return`unknown`}function classifyLocalSubagentEntry(e,t){if(t===`file`)return matchesSupportedModuleBaseName(e,`agent`)?`agent-config-module`:e.toLowerCase()===`instructions.md`?`instructions-markdown`:matchesSupportedModuleBaseName(e,`instructions`)?`instructions-module`:e.toLowerCase()===`system.md`?`system-markdown`:matchesSupportedModuleBaseName(e,`system`)?`system-module`:`unknown`;if(t===`directory`){if(GENERATED_AGENT_DIRECTORY_NAMES.has(e))return`ignored-directory`;if(e===`connections`)return`connections-directory`;if(e===`extensions`)return`extensions-directory`;if(e===`hooks`)return`hooks-directory`;if(e===`instructions`)return`instructions-directory`;if(e===`lib`)return`lib-directory`;if(e===`sandbox`)return`sandbox-directory`;if(e===`skills`)return`skills-directory`;if(e===`tools`)return`tools-directory`;if(e===`subagents`)return`subagents-directory`;if(e===`schedules`)return`invalid-schedules-directory`}return`unknown`}function classifySkillPackageEntry(e,t){if(t===`file`)return e.toLowerCase()===`skill.md`?`skill-markdown`:`skill-resource`;if(t===`directory`){if(e===`scripts`)return`skill-scripts-directory`;if(e===`references`)return`skill-references-directory`;if(e===`assets`)return`skill-assets-directory`}return`skill-resource`}function classifySkillsDirectoryEntry(e,t){if(t===`directory`)return`skill-package-directory`;if(t===`file`){if(isTypeScriptDeclarationFileName(e))return`ignored-declaration`;if(e.toLowerCase().endsWith(`.md`))return`flat-skill-markdown`;if(getSupportedModuleBaseName(e)!==null)return`flat-skill-module`}return`unknown`}function normalizeLogicalPath(t){return t.replaceAll(sep,`/`).replace(/^\.\//,``).replace(/^\/+/,``)}function getSupportedModuleBaseName(e){if(isTypeScriptDeclarationFileName(e))return null;for(let n of SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS)if(e.endsWith(n)&&e.length>n.length)return e.slice(0,-n.length);return null}function isTypeScriptDeclarationFileName(e){return/\.d\.(?:cts|mts|ts)$/.test(e)}function matchesSupportedModuleBaseName(e,t){return getSupportedModuleBaseName(e)===t}function stripLogicalPathExtension(e){let t=normalizeLogicalPath(e),n=t.lastIndexOf(`/`),r=t.lastIndexOf(`.`);return r===-1||r<n?t:t.slice(0,r)}export{PROJECT_MARKER_FILE_NAMES,SUPPORTED_AUTHORED_MODULE_FILE_EXTENSIONS,classifyAgentRootEntry,classifyLocalSubagentEntry,classifySkillPackageEntry,classifySkillsDirectoryEntry,getDirectoryEntryType,getSupportedModuleBaseName,isProjectMarkerEntry,isTypeScriptDeclarationFileName,matchesSupportedModuleBaseName,normalizeLogicalPath,stripLogicalPathExtension};

package/dist/src/discover/manifest.d.ts

@@ -11,7 +11,7 @@ export declare const AGENT_SOURCE_MANIFEST_KIND = "eve-agent-discovery-manifest"
 /**
  * Current manifest schema version.
  */
-export declare const AGENT_SOURCE_MANIFEST_VERSION = 12;
+export declare const AGENT_SOURCE_MANIFEST_VERSION = 13;
 /**
  * Channel source reference preserved by the discovery manifest.
  */

package/dist/src/discover/manifest.js

@@ -1 +1 @@
-import{basename,relative,resolve}from"node:path";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";import{normalizeLogicalPath}from"#discover/filesystem.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=12;function createAgentSourceManifest(e){let t=resolve(e.appRoot),i=resolve(e.agentRoot),o={agentId:e.agentId??deriveAgentIdFromRoots(t,i,e.packageName),agentRoot:i,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:12,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(o.configModule=e.configModule),o}function deriveAgentIdFromRoots(n,r,a){let o=normalizeLogicalPath(relative(n,r));return o===``||o===`.`||o===`agent`?a??basename(n):basename(r)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};
+import{basename,relative,resolve}from"node:path";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";import{normalizeLogicalPath}from"#discover/filesystem.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=13;function createAgentSourceManifest(e){let t=resolve(e.appRoot),i=resolve(e.agentRoot),o={agentId:e.agentId??deriveAgentIdFromRoots(t,i,e.packageName),agentRoot:i,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:13,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(o.configModule=e.configModule),o}function deriveAgentIdFromRoots(n,r,a){let o=normalizeLogicalPath(relative(n,r));return o===``||o===`.`||o===`agent`?a??basename(n):basename(r)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};

package/dist/src/evals/cli/eval.js

@@ -1,4 +1,4 @@
-import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{basename,join}from"node:path";import{createDevelopmentServer}from"#internal/nitro/host.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,n,r){let s=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(s);let c=e.length>0?e:void 0,l=await discoverAndImportEvals(s,c);if(l.length===0){let e=await findMisplacedEvalDirs(s);e.length>0?r.error(`No evals found under evals/, but eval files are present inside agent/:
+import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{basename,join}from"node:path";import{createDevelopmentServer}from"#internal/nitro/host.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(t,r,i){let s=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(s);let c=t.length>0?t:void 0,l=await discoverAndImportEvals(s,c);if(l.length===0){let e=await findMisplacedEvalDirs(s);e.length>0?i.error(`No evals found under evals/, but eval files are present inside agent/:
 `+e.map(e=>`  - ${e}`).join(`
 `)+`
-eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):c?r.error(`No evals found matching: ${c.join(`, `)}`):r.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let u=n.tag??[],d=n.excludeTag??[],f=filterEvalsByTags({evaluations:l,includeTags:u,excludeTags:[]});if(f.length===0){r.error(`No evals matched the provided tags (${u.join(`, `)}).`),process.exitCode=2;return}let p=filterEvalsByTags({evaluations:f,includeTags:[],excludeTags:d});if(n.list===!0){printEvalList(p,n.json===!0,r);return}if(p.length===0){r.log(`All ${f.length} matching evals are excluded by tags (${d.join(`, `)}); nothing to run.`);return}let m,h;try{m=parsePositiveInteger(n.maxConcurrency,`--max-concurrency`),h=parseNonNegativeInteger(n.timeout,`--timeout`)}catch(e){r.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let g;try{g=await discoverEvalConfig(s)}catch(e){r.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let _,v,y;try{if(n.url)y=await createEvalClient({kind:`remote`,url:n.url},{workspaceRoot:s}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:n.url});else{_=createDevelopmentServer(s,{host:`127.0.0.1`,port:0});let e=await _.start();y=await createEvalClient({kind:`local`,url:e.url}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:e.url})}let e=n.json===!0?[]:[Console()];n.junit!==void 0&&e.push(JUnit({filePath:n.junit}));let t=await runEvals({evaluations:p,config:g,target:v,client:y,appRoot:s,reporters:e,includeEvalReporters:n.skipReport!==!0,maxConcurrency:m,timeoutMs:h,onEvalLog:n.verbose===!0?(e,t)=>r.log(`[${e}] ${t}`):void 0});n.json&&r.log(JSON.stringify(t,null,2));let a=t.failed>0,o=n.strict===!0&&t.scored>0;(a||o)&&(process.exitCode=1)}finally{_&&(await _.close(),await shutdownActiveSandboxHandles({log:e=>r.error(e)}))}let b=typeof process.exitCode==`number`?process.exitCode:0;process.exit(b)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(t){try{let i=JSON.parse(await readFile(join(t,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(t)}catch{return basename(t)}}export{runEvalCommand};
+eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):c?i.error(`No evals found matching: ${c.join(`, `)}`):i.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let u=r.tag??[],d=r.excludeTag??[],f=filterEvalsByTags({evaluations:l,includeTags:u,excludeTags:[]});if(f.length===0){i.error(`No evals matched the provided tags (${u.join(`, `)}).`),process.exitCode=2;return}let p=filterEvalsByTags({evaluations:f,includeTags:[],excludeTags:d});if(r.list===!0){printEvalList(p,r.json===!0,i);return}if(p.length===0){i.log(`All ${f.length} matching evals are excluded by tags (${d.join(`, `)}); nothing to run.`);return}let m,h;try{m=parsePositiveInteger(r.maxConcurrency,`--max-concurrency`),h=parseNonNegativeInteger(r.timeout,`--timeout`)}catch(e){i.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let g;try{g=await discoverEvalConfig(s)}catch(e){i.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let _,v,y;try{if(r.url)y=await createEvalClient({kind:`remote`,url:r.url},{workspaceRoot:s}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:r.url});else{process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=randomUUID(),_=createDevelopmentServer(s,{host:`127.0.0.1`,port:0});let t=await _.start();y=await createEvalClient({kind:`local`,url:t.url}),v=await resolveEvalTargetHandle({client:y,expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:t.url})}let t=r.json===!0?[]:[Console()];r.junit!==void 0&&t.push(JUnit({filePath:r.junit}));let n=await runEvals({evaluations:p,config:g,target:v,client:y,appRoot:s,reporters:t,includeEvalReporters:r.skipReport!==!0,maxConcurrency:m,timeoutMs:h,onEvalLog:r.verbose===!0?(e,t)=>i.log(`[${e}] ${t}`):void 0});r.json&&i.log(JSON.stringify(n,null,2));let a=n.failed>0,o=r.strict===!0&&n.scored>0;(a||o)&&(process.exitCode=1)}finally{_&&(await _.close(),await shutdownActiveSandboxHandles({log:e=>i.error(e)}))}let b=typeof process.exitCode==`number`?process.exitCode:0;process.exit(b)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let n=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof n.name==`string`&&n.name.length>0?n.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};

package/dist/src/evals/index.d.ts

@@ -2,9 +2,9 @@ export { defineEval } from "#evals/define-eval.js";
 export { defineEvalConfig } from "#evals/define-eval-config.js";
 export { EveEvalTurnFailedError } from "#evals/session.js";
 export { mockModel } from "#evals/mock-model.js";
-export type { RuntimeIdentity } from "#protocol/message.js";
+export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalActionStatus, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";

package/dist/src/evals/reporters/index.d.ts

@@ -1,4 +1,4 @@
 export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js";
 export { Console, type ConsoleReporterConfig } from "#evals/runner/reporters/console.js";
 export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
-export type { EvalReporter } from "#evals/runner/reporters/types.js";
+export type { EvalReporter, EveEvalCompleteContext, EveEvalSessionStartEvent, EveEvalStartEvent, } from "#evals/runner/reporters/types.js";

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
 import{mkdir,writeFile}from"node:fs/promises";import{dirname,join}from"node:path";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
 `);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,threshold:e.threshold,passed:e.passed,message:e.message,metadata:e.metadata})),error:e.error,skipReason:e.skipReason,traceContexts:e.result.traceContexts}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason,traceContexts:e.result.traceContexts}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,runtimeIdentity:e.result.runtimeIdentity,sessions:e.result.sessions,traceContexts:e.result.traceContexts},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return r.push(s),i.set(e,s),s},ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={callId:e,name:t,status:`pending`,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`&&ensureToolCall(e.callId,e.toolName,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}else if(e.kind===`subagent-result`){let t=ensureSubagentCall(e.callId,e.subagentName);t.output=t.output??e.output,t.status=n}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);e.childSessionId=t.data.childSessionId,t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output,e.status===`pending`&&(e.status=`completed`);break}case`input.requested`:s.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(t,n){let r=n?.sessionId,i=[],a=new Map,o=[],s=new Map,c=[],l=-1,u=0,d=0,f,ensureToolCall=(e,t,n)=>{let o=a.get(e);if(o!==void 0)return o;let s={name:t,input:n,output:void 0,status:`pending`,turnIndex:Math.max(l,0),sessionId:r};return i.push(s),a.set(e,s),s},ensureSubagentCall=(e,t)=>{let n=s.get(e);if(n!==void 0)return n;let i={callId:e,name:t,status:`pending`,turnIndex:Math.max(l,0),sessionId:r};return o.push(i),s.set(e,i),i};for(let n of t)switch(n.type){case`turn.started`:l+=1;break;case`actions.requested`:for(let t of n.data.actions)t.kind===`tool-call`?ensureToolCall(t.callId,t.toolName,t.input):t.kind===`load-skill`&&ensureToolCall(t.callId,LOAD_SKILL_TOOL_NAME,t.input);break;case`action.result`:{let{result:e,status:t}=n.data;if(e.kind===`tool-result`){let n=ensureToolCall(e.callId,e.toolName,{});n.output=e.output,n.status=t}else if(e.kind===`subagent-result`){let n=ensureSubagentCall(e.callId,e.subagentName);n.output=n.output??e.output,n.status=t}break}case`subagent.called`:{let e=ensureSubagentCall(n.data.callId,n.data.name);e.childSessionId=n.data.childSessionId,n.data.remote!==void 0&&(e.remoteUrl=n.data.remote.url);break}case`subagent.started`:ensureSubagentCall(n.data.callId,n.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(n.data.callId,n.data.subagentName);e.output=n.data.output,e.status===`pending`&&(e.status=`completed`);break}case`input.requested`:c.push(...n.data.requests);for(let e of n.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:n.data.finishReason!==`tool-calls`&&(u+=1);break;case`reasoning.completed`:d+=1;break;case`session.failed`:f=n.data.code;break}return{toolCalls:i,toolCallCount:i.length,subagentCalls:o,subagentCallCount:o.length,inputRequests:c,parked:endedParkedOnInput(t),messageCount:u,reasoningBlockCount:d,failureCode:f}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(e){for(let n=e.length-1;n>=0;n--){let r=e[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-eval.d.ts

@@ -1,5 +1,6 @@
 import type { Client } from "#client/client.js";
 import type { EveEval, EveEvalResult, EveEvalTargetHandle } from "#evals/types.js";
+import type { EvalSessionStartedEvent } from "#evals/session.js";
 /**
  * Options for executing one eval.
  */
@@ -7,9 +8,13 @@ export interface ExecuteEvalOptions {
     readonly evaluation: EveEval;
     /** Receives `t.log` lines as the eval runs (used by `--verbose`). */
     readonly onLog?: (message: string) => void;
+    /** Receives the first trace context observed for each session. */
+    readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
     readonly target: EveEvalTargetHandle;
     /** Overrides the eval's own `timeoutMs` when set (CLI `--timeout`). */
     readonly timeoutMs?: number;
+    /** Runner-owned start time, shared with reporter lifecycle events. */
+    readonly startedAt?: string;
     /**
      * Pre-configured client for communicating with the eve agent.
      * The CLI constructs this once with the appropriate auth and headers,

package/dist/src/evals/runner/execute-eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(n){let{evaluation:r,target:i,client:a}=n,o=n.startedAt??new Date().toISOString(),s,c=[],l,u;try{let e=await executeTask({client:a,evaluation:r,onLog:n.onLog,onSessionStart:n.onSessionStart,target:i,timeoutMs:n.timeoutMs??r.timeoutMs});s=e.result,c=e.assertions,l=e.error,u=e.skipReason}catch(t){l=toErrorMessage(t),s={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts(),traceContexts:[]}}let d=computeEvalVerdict({error:l,assertions:c,skipReason:u});return{id:r.id,result:s,assertions:c,verdict:d,error:l,skipReason:u,startedAt:o,completedAt:new Date().toISOString()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -1,5 +1,6 @@
 import type { Client } from "#client/client.js";
 import type { AssertionResult, EveEval, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
+import { type EvalSessionStartedEvent } from "#evals/session.js";
 /**
  * Options for executing one eval's task.
  */
@@ -8,6 +9,8 @@ interface ExecuteTaskOptions {
     readonly evaluation: EveEval;
     /** Receives each `t.log` line as it is written (used by `--verbose`). */
     readonly onLog?: (message: string) => void;
+    /** Receives the first trace context observed for each session. */
+    readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;
     readonly target: EveEvalTargetHandle;
     readonly timeoutMs?: number;
 }

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await runUntilAborted(s.test(h),u)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}async function runUntilAborted(e,t){t.throwIfAborted();let onAbort,n=new Promise((e,n)=>{onAbort=()=>n(t.reason),t.addEventListener(`abort`,onAbort,{once:!0})});try{await Promise.race([e,n])}finally{onAbort!==void 0&&t.removeEventListener(`abort`,onAbort)}}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(a){let{client:o,evaluation:s,target:c,timeoutMs:l}=a,u=l===void 0?neverAbortSignal():AbortSignal.timeout(l),d=new AssertionCollector,f=new EvalSessionManager({client:o,collector:d,onSessionStart:a.onSessionStart,signal:u}),p=scopeEvalTargetHandle(c,{sessions:f}),m=[],{context:h}=createEvalContext({collector:d,manager:f,target:p,signal:u,judge:s.judge,log:e=>{m.push(e),a.onLog?.(e)}}),g,_;try{await runUntilAborted(s.test(h),u)}catch(t){t instanceof EvalSkipped?_=t.reason:t instanceof EvalRequirementFailed||(g=toErrorMessage(t))}let v=buildTaskResult({logs:m,sessions:f.snapshots(),turn:f.lastTurnSession()?.lastTurn});return{result:v,assertions:await d.finalize(v),error:g,skipReason:_}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:i}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}async function runUntilAborted(e,t){t.throwIfAborted();let onAbort,n=new Promise((e,n)=>{onAbort=()=>n(t.reason),t.addEventListener(`abort`,onAbort,{once:!0})});try{await Promise.race([e,n])}finally{onAbort!==void 0&&t.removeEventListener(`abort`,onAbort)}}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=new Map;for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name,i=(r.get(e)??0)+1;r.set(e,i);let a=i===1?e:`${e}#${i}`;n[a]=t.score}let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -1,4 +1,23 @@
-import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js";
+import type { RuntimeTraceContext } from "#protocol/message.js";
+import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget, EveEvalTraceContext } from "#evals/types.js";
+/** Context delivered when one eval is scheduled for execution. */
+export interface EveEvalStartEvent {
+    readonly evaluation: EveEval;
+    readonly startedAt: string;
+    readonly target: EveEvalTarget;
+}
+/** Context delivered once the runner observes a session's first trace. */
+export interface EveEvalSessionStartEvent extends EveEvalStartEvent {
+    readonly primary: boolean;
+    readonly sessionId: string;
+    readonly traceContext: RuntimeTraceContext;
+}
+/** Additional run context delivered with a completed eval result. */
+export interface EveEvalCompleteContext {
+    readonly evaluation: EveEval;
+    readonly target: EveEvalTarget;
+    readonly traceContexts: readonly EveEvalTraceContext[];
+}
 /**
  * Reporter lifecycle interface. The runner calls these methods at defined
  * points during an eval run. Methods may return a promise for reporters
@@ -13,11 +32,18 @@ export interface EvalReporter {
      * this reporter observes.
      */
     onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): void | Promise<void>;
+    /** The runner calls this when an observed eval is scheduled. */
+    onEvalStart?(event: EveEvalStartEvent): void | Promise<void>;
+    /**
+     * The runner calls this once per session when its first trace context is
+     * observed. Sessions without trace instrumentation do not trigger it.
+     */
+    onSessionStart?(event: EveEvalSessionStartEvent): void | Promise<void>;
     /**
      * The runner calls this after each observed eval completes, with its
      * checks, scores, and verdict.
      */
-    onEvalComplete(result: EveEvalResult): void | Promise<void>;
+    onEvalComplete(result: EveEvalResult, context?: EveEvalCompleteContext): void | Promise<void>;
     /**
      * The runner calls this once when the run finishes, with the aggregated
      * summary of the evals this reporter observes.

package/dist/src/evals/runner/run-evals.js

@@ -1 +1 @@
-import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve(),enqueueReporterCallback=(e,t)=>{h=h.then(async()=>{for(let n of d)n.evalIds.has(e.id)&&await t(n.reporter)})};for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=new Date().toISOString();enqueueReporterCallback(e,async n=>{await n.onEvalStart?.({evaluation:e,startedAt:t,target:a})});let i=(async()=>{let i=await executeEval({client:o,evaluation:e,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),onSessionStart:t=>{enqueueReporterCallback(e,async n=>{await n.onSessionStart?.({evaluation:e,primary:t.primary,sessionId:t.sessionId,startedAt:t.startedAt,target:a,traceContext:t.traceContext})})},startedAt:t,target:a,timeoutMs:r.timeoutMs});f.push(i),enqueueReporterCallback(e,async t=>{await t.onEvalComplete(i,{evaluation:e,target:a,traceContexts:i.result.traceContexts})})})().finally(()=>{m.delete(i)});m.add(i)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};