eve 0.75.1 → 0.76.2

package/dist/src/discover/grammar.js

@@ -1 +1 @@
-import{join}from"node:path";import{getDirectoryEntryType,isAuthoredTestPath}from"#discover/filesystem.js";import{createDiscoverErrorDiagnostic,createDiscoverWarningDiagnostic}from"#discover/diagnostics.js";import{createModuleSourceRef}from"#discover/manifest.js";import{collectFlatSlotCandidates}from"#discover/slots.js";import{lowerInstructionsMarkdown}from"#internal/helpers/markdown.js";import{discoverMarkdownSource}from"#discover/markdown.js";import{discoverNamedSourceDirectory,discoverNamedSourceDirectory as discoverNamedSourceDirectory$1}from"#discover/named-source-directory.js";const DISCOVER_MODULE_SLOT_COLLISION=`discover/module-slot-collision`,DISCOVER_REQUIRED_INSTRUCTIONS_MISSING=`discover/required-instructions-missing`,DISCOVER_DEPRECATED_SYSTEM_SLOT=`discover/deprecated-system-slot`,DISCOVER_SLOT_COLLISION=`discover/slot-collision`,DISCOVER_TOOLS_DIRECTORY_INVALID=`discover/tools-directory-invalid`,DISCOVER_HOOKS_DIRECTORY_INVALID=`discover/hooks-directory-invalid`,DISCOVER_CHANNELS_DIRECTORY_INVALID=`discover/channels-directory-invalid`,DISCOVER_EXTENSIONS_DIRECTORY_INVALID=`discover/extensions-directory-invalid`,DISCOVER_TOOL_NAME_INVALID=`discover/tool-name-invalid`,DISCOVER_SANDBOX_DIRECTORY_INVALID=`discover/sandbox-directory-invalid`,DISCOVER_HOOK_NAME_INVALID=`discover/hook-name-invalid`,DISCOVER_EXTENSION_NAME_INVALID=`discover/extension-name-invalid`,TOOL_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,CONNECTION_SLUG_PATTERN=/^[a-z][a-z0-9-]{0,63}$/,CHANNEL_SLUG_PATTERN=/^(\.?[a-z][a-z0-9-]{0,63}|\[[a-zA-Z][a-zA-Z0-9_]{0,63}\])$/,HOOK_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,EXTENSION_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,DISCOVER_UNSUPPORTED_DIRECTORY=`discover/unsupported-directory`;async function readSortedDirectoryEntries(e,t,n={}){let r=(await e.readDirectory(t)).filter(e=>n.includeTests||!isAuthoredTestPath(e.name));return r.sort((e,t)=>e.name.localeCompare(t.name)),r}async function discoverInstructionsSource(e){let t=e.rootEntries.some(e=>e.name===`instructions`&&e.isDirectory()),n=await discoverSlotSource({markdownFileName:`instructions.md`,moduleBaseName:`instructions`,rootEntries:e.rootEntries,rootPath:e.rootPath,slotLabel:`instructions`,source:e.source});if(t){let t=await discoverNamedSourceDirectory$1({allowMarkdown:!0,directoryName:`instructions`,invalidDirectoryCode:`discover/instructions-directory-invalid`,invalidDirectoryMessage:`Expected "${join(e.rootPath,`instructions`)}" to be a directory of authored instructions.`,markdownLowerer:e=>lowerInstructionsMarkdown(e),recursive:!1,rootEntries:e.rootEntries,rootPath:e.rootPath,source:e.source}),r=[...t.sources];return n.source!==void 0&&r.unshift(n.source),{diagnostics:[...n.diagnostics,...t.diagnostics],instructions:r}}if(n.diagnostics.length>0||n.source!==void 0)return{diagnostics:n.diagnostics,instructions:n.source===void 0?[]:[n.source]};let r=await discoverSlotSource({markdownFileName:`system.md`,moduleBaseName:`system`,rootEntries:e.rootEntries,rootPath:e.rootPath,slotLabel:`system`,source:e.source});if(r.source!==void 0){let t=r.source.sourceKind===`markdown`?`system.md`:r.source.logicalPath;return{diagnostics:[createDiscoverWarningDiagnostic({code:DISCOVER_DEPRECATED_SYSTEM_SLOT,message:`The "${t}" slot is deprecated. Rename it to "${t.replace(/^system/,`instructions`)}" — the runtime still loads the legacy slot for now, but support will be removed in a future release.`,sourcePath:join(e.rootPath,t)}),...r.diagnostics],instructions:[r.source]}}return r.diagnostics.length>0?{diagnostics:r.diagnostics,instructions:[]}:e.required===!1?{diagnostics:[],instructions:[]}:{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_REQUIRED_INSTRUCTIONS_MISSING,message:`Expected authored instructions at "instructions.md", "instructions.ts", "instructions.cts", "instructions.mts", "instructions.js", "instructions.cjs", "instructions.mjs", or "instructions/" directory.`,sourcePath:e.rootPath})],instructions:[]}}async function discoverSlotSource(e){let t=collectFlatSlotCandidates(e.rootEntries,{markdownFileName:e.markdownFileName,moduleBaseName:e.moduleBaseName});if(t.markdownFileName!==void 0&&t.moduleFileNames.length>0)return{diagnostics:[createSlotCollisionDiagnostic(e.rootPath,e.slotLabel,[t.markdownFileName,...t.moduleFileNames])]};if(t.moduleFileNames.length>1)return{diagnostics:[createModuleSlotCollisionDiagnostic(e.rootPath,e.slotLabel,t.moduleFileNames)]};if(t.markdownFileName!==void 0)return{diagnostics:[],source:await discoverMarkdownSource({logicalPath:e.markdownFileName,lower:lowerInstructionsMarkdown,source:e.source,sourcePath:join(e.rootPath,t.markdownFileName)})};let[n]=t.moduleFileNames;return n===void 0?{diagnostics:[]}:{diagnostics:[],source:createModuleSourceRef({logicalPath:n})}}function discoverFlatModuleSource(e){let t=collectFlatSlotCandidates(e.rootEntries,{moduleBaseName:e.slotName});if(t.moduleFileNames.length>1)return{diagnostics:[createModuleSlotCollisionDiagnostic(e.rootPath,e.slotName,t.moduleFileNames)]};let[n]=t.moduleFileNames;return n===void 0?e.missingDiagnostic===void 0?{diagnostics:[]}:{diagnostics:[createDiscoverErrorDiagnostic({code:e.missingDiagnostic.code,message:e.missingDiagnostic.message,sourcePath:e.rootPath})]}:{diagnostics:[],module:createModuleSourceRef({logicalPath:n})}}function createToolNameDiagnostic(e,t){return TOOL_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_TOOL_NAME_INVALID,message:`Tool filename "${e}" is not a legal tool name. Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createConnectionNameDiagnostic(e,t){return CONNECTION_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:`discover/connection-name-invalid`,message:`Connection filename "${e}" is not a legal connection name. Expected lowercase ASCII letters, digits, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createChannelNameDiagnostic(e,t){return CHANNEL_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:`discover/channel-name-invalid`,message:`Channel path segment "${e}" is not a legal channel name. Expected lowercase kebab-case (\`my-channel\`), optionally with a leading dot (\`.well-known\`), or a path parameter form (\`[sessionId]\`).`,sourcePath:t})}function createHookNameDiagnostic(e,t){return HOOK_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_HOOK_NAME_INVALID,message:`Hook path segment "${e}" is not a legal hook name. Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createExtensionNameDiagnostic(e,t){return EXTENSION_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_EXTENSION_NAME_INVALID,message:`Extension mount filename "${e}" is not a legal extension namespace. Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createUnsupportedRootDirectoryDiagnostics(e){return e.rootEntries.flatMap(t=>!t.isDirectory()||e.classifyEntry(t.name,getDirectoryEntryType(t))!==`unknown`?[]:[createDiscoverWarningDiagnostic({code:DISCOVER_UNSUPPORTED_DIRECTORY,message:e.createUnsupportedDirectoryMessage(t.name),sourcePath:join(e.rootPath,t.name)})])}function createSlotCollisionDiagnostic(e,t,n){return createDiscoverErrorDiagnostic({code:DISCOVER_SLOT_COLLISION,message:`Found conflicting authored sources for "${t}": ${formatQuotedFileList(n)}.`,sourcePath:e})}function createModuleSlotCollisionDiagnostic(e,t,n){return createDiscoverErrorDiagnostic({code:DISCOVER_MODULE_SLOT_COLLISION,message:`Found multiple authored module sources for "${t}": ${formatQuotedFileList(n)}.`,sourcePath:e})}function formatQuotedFileList(e){return e.map(e=>`"${e}"`).join(`, `)}export{CONNECTION_SLUG_PATTERN,DISCOVER_CHANNELS_DIRECTORY_INVALID,DISCOVER_DEPRECATED_SYSTEM_SLOT,DISCOVER_EXTENSIONS_DIRECTORY_INVALID,DISCOVER_EXTENSION_NAME_INVALID,DISCOVER_HOOKS_DIRECTORY_INVALID,DISCOVER_HOOK_NAME_INVALID,DISCOVER_MODULE_SLOT_COLLISION,DISCOVER_REQUIRED_INSTRUCTIONS_MISSING,DISCOVER_SANDBOX_DIRECTORY_INVALID,DISCOVER_SLOT_COLLISION,DISCOVER_TOOLS_DIRECTORY_INVALID,DISCOVER_TOOL_NAME_INVALID,DISCOVER_UNSUPPORTED_DIRECTORY,TOOL_SLUG_PATTERN,createChannelNameDiagnostic,createConnectionNameDiagnostic,createExtensionNameDiagnostic,createHookNameDiagnostic,createModuleSlotCollisionDiagnostic,createSlotCollisionDiagnostic,createToolNameDiagnostic,createUnsupportedRootDirectoryDiagnostics,discoverFlatModuleSource,discoverInstructionsSource,discoverNamedSourceDirectory,readSortedDirectoryEntries};
+import{join}from"node:path";import{getDirectoryEntryType,isAuthoredTestPath}from"#discover/filesystem.js";import{createDiscoverErrorDiagnostic,createDiscoverWarningDiagnostic}from"#discover/diagnostics.js";import{eveNamespaceReservation}from"#protocol/runtime-tools.js";import{createModuleSourceRef}from"#discover/manifest.js";import{collectFlatSlotCandidates}from"#discover/slots.js";import{lowerInstructionsMarkdown}from"#internal/helpers/markdown.js";import{discoverMarkdownSource}from"#discover/markdown.js";import{discoverNamedSourceDirectory,discoverNamedSourceDirectory as discoverNamedSourceDirectory$1}from"#discover/named-source-directory.js";const DISCOVER_MODULE_SLOT_COLLISION=`discover/module-slot-collision`,DISCOVER_REQUIRED_INSTRUCTIONS_MISSING=`discover/required-instructions-missing`,DISCOVER_DEPRECATED_SYSTEM_SLOT=`discover/deprecated-system-slot`,DISCOVER_SLOT_COLLISION=`discover/slot-collision`,DISCOVER_TOOLS_DIRECTORY_INVALID=`discover/tools-directory-invalid`,DISCOVER_HOOKS_DIRECTORY_INVALID=`discover/hooks-directory-invalid`,DISCOVER_CHANNELS_DIRECTORY_INVALID=`discover/channels-directory-invalid`,DISCOVER_EXTENSIONS_DIRECTORY_INVALID=`discover/extensions-directory-invalid`,DISCOVER_TOOL_NAME_INVALID=`discover/tool-name-invalid`,DISCOVER_SANDBOX_DIRECTORY_INVALID=`discover/sandbox-directory-invalid`,DISCOVER_HOOK_NAME_INVALID=`discover/hook-name-invalid`,DISCOVER_EXTENSION_NAME_INVALID=`discover/extension-name-invalid`,TOOL_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,TOOL_SLUG_RULE=`Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,SKILL_NAME_PATTERN=/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/,SKILL_NAME_RULE=`Expected ASCII letters, digits, underscores, and dashes only, starting with a letter or digit, up to 64 characters.`,CONNECTION_SLUG_PATTERN=/^[a-z][a-z0-9-]{0,63}$/,CHANNEL_SLUG_PATTERN=/^(\.?[a-z][a-z0-9-]{0,63}|\[[a-zA-Z][a-zA-Z0-9_]{0,63}\])$/,HOOK_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,EXTENSION_SLUG_PATTERN=/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/,DISCOVER_UNSUPPORTED_DIRECTORY=`discover/unsupported-directory`;async function readSortedDirectoryEntries(e,t,n={}){let r=(await e.readDirectory(t)).filter(e=>n.includeTests||!isAuthoredTestPath(e.name));return r.sort((e,t)=>e.name.localeCompare(t.name)),r}async function discoverInstructionsSource(e){let t=e.rootEntries.some(e=>e.name===`instructions`&&e.isDirectory()),n=await discoverSlotSource({markdownFileName:`instructions.md`,moduleBaseName:`instructions`,rootEntries:e.rootEntries,rootPath:e.rootPath,slotLabel:`instructions`,source:e.source});if(t){let t=await discoverNamedSourceDirectory$1({allowMarkdown:!0,directoryName:`instructions`,invalidDirectoryCode:`discover/instructions-directory-invalid`,invalidDirectoryMessage:`Expected "${join(e.rootPath,`instructions`)}" to be a directory of authored instructions.`,markdownLowerer:e=>lowerInstructionsMarkdown(e),recursive:!1,rootEntries:e.rootEntries,rootPath:e.rootPath,source:e.source}),r=[...t.sources];return n.source!==void 0&&r.unshift(n.source),{diagnostics:[...n.diagnostics,...t.diagnostics],instructions:r}}if(n.diagnostics.length>0||n.source!==void 0)return{diagnostics:n.diagnostics,instructions:n.source===void 0?[]:[n.source]};let r=await discoverSlotSource({markdownFileName:`system.md`,moduleBaseName:`system`,rootEntries:e.rootEntries,rootPath:e.rootPath,slotLabel:`system`,source:e.source});if(r.source!==void 0){let t=r.source.sourceKind===`markdown`?`system.md`:r.source.logicalPath;return{diagnostics:[createDiscoverWarningDiagnostic({code:DISCOVER_DEPRECATED_SYSTEM_SLOT,message:`The "${t}" slot is deprecated. Rename it to "${t.replace(/^system/,`instructions`)}" — the runtime still loads the legacy slot for now, but support will be removed in a future release.`,sourcePath:join(e.rootPath,t)}),...r.diagnostics],instructions:[r.source]}}return r.diagnostics.length>0?{diagnostics:r.diagnostics,instructions:[]}:e.required===!1?{diagnostics:[],instructions:[]}:{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_REQUIRED_INSTRUCTIONS_MISSING,message:`Expected authored instructions at "instructions.md", "instructions.ts", "instructions.cts", "instructions.mts", "instructions.js", "instructions.cjs", "instructions.mjs", or "instructions/" directory.`,sourcePath:e.rootPath})],instructions:[]}}async function discoverSlotSource(e){let t=collectFlatSlotCandidates(e.rootEntries,{markdownFileName:e.markdownFileName,moduleBaseName:e.moduleBaseName});if(t.markdownFileName!==void 0&&t.moduleFileNames.length>0)return{diagnostics:[createSlotCollisionDiagnostic(e.rootPath,e.slotLabel,[t.markdownFileName,...t.moduleFileNames])]};if(t.moduleFileNames.length>1)return{diagnostics:[createModuleSlotCollisionDiagnostic(e.rootPath,e.slotLabel,t.moduleFileNames)]};if(t.markdownFileName!==void 0)return{diagnostics:[],source:await discoverMarkdownSource({logicalPath:e.markdownFileName,lower:lowerInstructionsMarkdown,source:e.source,sourcePath:join(e.rootPath,t.markdownFileName)})};let[n]=t.moduleFileNames;return n===void 0?{diagnostics:[]}:{diagnostics:[],source:createModuleSourceRef({logicalPath:n})}}function discoverFlatModuleSource(e){let t=collectFlatSlotCandidates(e.rootEntries,{moduleBaseName:e.slotName});if(t.moduleFileNames.length>1)return{diagnostics:[createModuleSlotCollisionDiagnostic(e.rootPath,e.slotName,t.moduleFileNames)]};let[n]=t.moduleFileNames;return n===void 0?e.missingDiagnostic===void 0?{diagnostics:[]}:{diagnostics:[createDiscoverErrorDiagnostic({code:e.missingDiagnostic.code,message:e.missingDiagnostic.message,sourcePath:e.rootPath})]}:{diagnostics:[],module:createModuleSourceRef({logicalPath:n})}}function createToolNameDiagnostic(e,t){return TOOL_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_TOOL_NAME_INVALID,message:`Tool filename "${e}" is not a legal tool name. ${TOOL_SLUG_RULE}`,sourcePath:t})}function createConnectionNameDiagnostic(e,t){return CONNECTION_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:`discover/connection-name-invalid`,message:`Connection filename "${e}" is not a legal connection name. Expected lowercase ASCII letters, digits, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createChannelNameDiagnostic(e,t){return CHANNEL_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:`discover/channel-name-invalid`,message:`Channel path segment "${e}" is not a legal channel name. Expected lowercase kebab-case (\`my-channel\`), optionally with a leading dot (\`.well-known\`), or a path parameter form (\`[sessionId]\`).`,sourcePath:t})}function createHookNameDiagnostic(e,t){return HOOK_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_HOOK_NAME_INVALID,message:`Hook path segment "${e}" is not a legal hook name. Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t})}function createExtensionNameDiagnostic(e,t){let n=eveNamespaceReservation(e);return n===void 0?EXTENSION_SLUG_PATTERN.test(e)?null:createDiscoverErrorDiagnostic({code:DISCOVER_EXTENSION_NAME_INVALID,message:`Extension mount filename "${e}" is not a legal extension namespace. Expected ASCII letters, digits, underscores, and dashes only, starting with a letter, up to 64 characters.`,sourcePath:t}):createDiscoverErrorDiagnostic({code:DISCOVER_EXTENSION_NAME_INVALID,message:`Extension mount "${e}" uses the reserved name "${e}". Rename it; ${n}.`,sourcePath:t})}function createUnsupportedRootDirectoryDiagnostics(e){return e.rootEntries.flatMap(t=>!t.isDirectory()||e.classifyEntry(t.name,getDirectoryEntryType(t))!==`unknown`?[]:[createDiscoverWarningDiagnostic({code:DISCOVER_UNSUPPORTED_DIRECTORY,message:e.createUnsupportedDirectoryMessage(t.name),sourcePath:join(e.rootPath,t.name)})])}function createSlotCollisionDiagnostic(e,t,n){return createDiscoverErrorDiagnostic({code:DISCOVER_SLOT_COLLISION,message:`Found conflicting authored sources for "${t}": ${formatQuotedFileList(n)}.`,sourcePath:e})}function createModuleSlotCollisionDiagnostic(e,t,n){return createDiscoverErrorDiagnostic({code:DISCOVER_MODULE_SLOT_COLLISION,message:`Found multiple authored module sources for "${t}": ${formatQuotedFileList(n)}.`,sourcePath:e})}function formatQuotedFileList(e){return e.map(e=>`"${e}"`).join(`, `)}export{CONNECTION_SLUG_PATTERN,DISCOVER_CHANNELS_DIRECTORY_INVALID,DISCOVER_DEPRECATED_SYSTEM_SLOT,DISCOVER_EXTENSIONS_DIRECTORY_INVALID,DISCOVER_EXTENSION_NAME_INVALID,DISCOVER_HOOKS_DIRECTORY_INVALID,DISCOVER_HOOK_NAME_INVALID,DISCOVER_MODULE_SLOT_COLLISION,DISCOVER_REQUIRED_INSTRUCTIONS_MISSING,DISCOVER_SANDBOX_DIRECTORY_INVALID,DISCOVER_SLOT_COLLISION,DISCOVER_TOOLS_DIRECTORY_INVALID,DISCOVER_TOOL_NAME_INVALID,DISCOVER_UNSUPPORTED_DIRECTORY,SKILL_NAME_PATTERN,SKILL_NAME_RULE,TOOL_SLUG_PATTERN,TOOL_SLUG_RULE,createChannelNameDiagnostic,createConnectionNameDiagnostic,createExtensionNameDiagnostic,createHookNameDiagnostic,createModuleSlotCollisionDiagnostic,createSlotCollisionDiagnostic,createToolNameDiagnostic,createUnsupportedRootDirectoryDiagnostics,discoverFlatModuleSource,discoverInstructionsSource,discoverNamedSourceDirectory,readSortedDirectoryEntries};

package/dist/src/discover/manifest.d.ts

@@ -254,6 +254,7 @@ export interface CreateAgentSourceManifestInput {
  */
 interface CreateSkillPackageSourceRefInput {
     assetsPath?: string;
+    deferred?: boolean;
     description: string;
     license?: string;
     logicalPath: string;

package/dist/src/discover/manifest.js

@@ -1 +1 @@
-import{basename,relative,resolve}from"node:path";import{normalizeLogicalPath}from"#discover/filesystem.js";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=15;function createAgentSourceManifest(e){let t=resolve(e.appRoot),n=resolve(e.agentRoot),r={agentId:e.agentId??deriveAgentIdFromRoots(t,n,e.packageName),agentRoot:n,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],memories:[...e.memories??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:15,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(r.configModule=e.configModule),r}function deriveAgentIdFromRoots(e,t,n){let r=normalizeLogicalPath(relative(e,t));return r===``||r===`.`||r===`agent`?n??basename(e):basename(t)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};
+import{basename,relative,resolve}from"node:path";import{normalizeLogicalPath}from"#discover/filesystem.js";import{summarizeDiscoverDiagnostics}from"#discover/diagnostics.js";const AGENT_SOURCE_MANIFEST_KIND=`eve-agent-discovery-manifest`,AGENT_SOURCE_MANIFEST_VERSION=15;function createAgentSourceManifest(e){let t=resolve(e.appRoot),n=resolve(e.agentRoot),r={agentId:e.agentId??deriveAgentIdFromRoots(t,n,e.packageName),agentRoot:n,appRoot:t,channels:[...e.channels??[]],connections:[...e.connections??[]],diagnosticsSummary:summarizeDiscoverDiagnostics(e.diagnostics??[]),extensions:[...e.extensions??[]],resolvedExtensions:[...e.resolvedExtensions??[]],hooks:[...e.hooks??[]],memories:[...e.memories??[]],instructions:[...e.instructions??[]],lib:[...e.lib??[]],kind:AGENT_SOURCE_MANIFEST_KIND,sandbox:e.sandbox??null,sandboxWorkspaces:[...e.sandboxWorkspaces??[]],schedules:[...e.schedules??[]],skills:[...e.skills??[]],tools:[...e.tools??[]],version:15,subagents:[...e.subagents??[]]};return e.configModule!==void 0&&(r.configModule=e.configModule),r}function deriveAgentIdFromRoots(e,t,n){let r=normalizeLogicalPath(relative(e,t));return r===``||r===`.`||r===`agent`?n??basename(e):basename(t)}function createPathDerivedSourceId(e){return normalizeLogicalPath(e)}function createModuleSourceRef(e){let t=normalizeLogicalPath(e.logicalPath),n={sourceKind:`module`,logicalPath:t,sourceId:e.sourceId??createPathDerivedSourceId(t)};return e.exportName!==void 0&&(n.exportName=e.exportName),n}function createConnectionSourceRef(e){return{...createModuleSourceRef(e),connectionName:e.connectionName}}function createLocalSubagentSourceRef(e){let t=normalizeLogicalPath(e.logicalPath);return{entryPath:e.entryPath,logicalPath:t,manifest:e.manifest,rootPath:e.rootPath,sourceId:e.sourceId??createPathDerivedSourceId(t),subagentId:e.subagentId}}function createSkillPackageSourceRef(e){return{assetsPath:e.assetsPath,deferred:e.deferred,description:e.description,license:e.license,logicalPath:normalizeLogicalPath(e.logicalPath),markdown:e.markdown,metadata:e.metadata===void 0?void 0:{...e.metadata},name:e.name,referencesPath:e.referencesPath,rootPath:e.rootPath,scriptsPath:e.scriptsPath,skillFilePath:e.skillFilePath,skillId:e.skillId,sourceId:e.sourceId,sourceKind:`skill-package`}}export{AGENT_SOURCE_MANIFEST_KIND,AGENT_SOURCE_MANIFEST_VERSION,createAgentSourceManifest,createConnectionSourceRef,createLocalSubagentSourceRef,createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef,deriveAgentIdFromRoots};

package/dist/src/discover/skills.d.ts

@@ -5,6 +5,7 @@ export declare const DISCOVER_SKILL_COLLISION = "discover/skill-collision";
 export declare const DISCOVER_SKILL_ENTRY_NOT_DIRECTORY = "discover/skill-entry-not-directory";
 export declare const DISCOVER_SKILL_FRONTMATTER_INVALID = "discover/skill-frontmatter-invalid";
 export declare const DISCOVER_SKILL_MARKDOWN_MISSING = "discover/skill-markdown-missing";
+export declare const DISCOVER_SKILL_NAME_INVALID = "discover/skill-name-invalid";
 /**
  * Input for discovering authored skills.
  */

package/dist/src/discover/skills.js

@@ -1 +1 @@
-import{join,relative,resolve}from"node:path";import{classifySkillPackageEntry,classifySkillsDirectoryEntry,getDirectoryEntryType,getSupportedModuleBaseName,normalizeLogicalPath}from"#discover/filesystem.js";import{toErrorMessage}from"#shared/errors.js";import{createDiskProjectSource}from"#discover/project-source.js";import{createDiscoverErrorDiagnostic}from"#discover/diagnostics.js";import{readSortedDirectoryEntries}from"#discover/grammar.js";import{createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef}from"#discover/manifest.js";import{lowerSkillMarkdown}from"#internal/helpers/markdown.js";const DISCOVER_SKILL_COLLISION=`discover/skill-collision`,DISCOVER_SKILL_ENTRY_NOT_DIRECTORY=`discover/skill-entry-not-directory`,DISCOVER_SKILL_FRONTMATTER_INVALID=`discover/skill-frontmatter-invalid`,DISCOVER_SKILL_MARKDOWN_MISSING=`discover/skill-markdown-missing`;async function discoverSkills(e){let t=e.source??createDiskProjectSource(),n=resolve(e.agentRoot),r=resolve(e.skillsDirectoryPath??join(n,`skills`)),i=normalizeLogicalPath(e.skillsLogicalPath??relative(n,r)),a=await t.stat(r);if(a===`missing`)return{diagnostics:[],skills:[]};if(a!==`directory`)return{diagnostics:[createDiscoverErrorDiagnostic({code:`discover/skills-directory-invalid`,message:`Expected "${r}" to be a directory of authored skills.`,sourcePath:r})],skills:[]};let o=[],s=new Set,c=new Map,l=await readSortedDirectoryEntries(t,r);for(let e of l){let n=await discoverOneSkill({entryName:e.name,entryType:getDirectoryEntryType(e),skillsDirectoryPath:r,skillsLogicalPath:i,source:t});if(o.push(...n.diagnostics),n.skill===null||n.skillId===null||s.has(n.skillId))continue;let a=c.get(n.skillId);if(a!==void 0){o.push(createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_COLLISION,message:`Found conflicting authored skill sources for "${n.skillId}": "${a.logicalPath}" and "${n.logicalPath}".`,sourcePath:join(r,n.skillId)})),s.add(n.skillId),c.delete(n.skillId);continue}c.set(n.skillId,{logicalPath:n.logicalPath,skill:n.skill})}return{diagnostics:o,skills:[...c.values()].map(e=>e.skill)}}async function discoverOneSkill(e){let t=join(e.skillsDirectoryPath,e.entryName);switch(classifySkillsDirectoryEntry(e.entryName,e.entryType)){case`skill-package-directory`:return discoverPackagedSkill({logicalSkillsPath:e.skillsLogicalPath,skillId:e.entryName,skillRootPath:t,source:e.source});case`flat-skill-markdown`:return discoverFlatMarkdownSkill({logicalSkillsPath:e.skillsLogicalPath,skillFileName:e.entryName,skillFilePath:t,source:e.source});case`flat-skill-module`:return discoverFlatModuleSkill({logicalSkillsPath:e.skillsLogicalPath,skillFileName:e.entryName});case`ignored-declaration`:case`ignored-source-map`:return{diagnostics:[],logicalPath:normalizeLogicalPath(join(e.skillsLogicalPath,e.entryName)),skill:null,skillId:null};default:return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_ENTRY_NOT_DIRECTORY,message:`Expected "${t}" to be a skill directory containing SKILL.md or a flat ".md", ".ts", ".cts", ".mts", ".js", ".cjs", or ".mjs" skill file.`,sourcePath:t})],logicalPath:normalizeLogicalPath(join(e.skillsLogicalPath,e.entryName)),skill:null,skillId:null}}}async function discoverPackagedSkill(e){let t=(await readSortedDirectoryEntries(e.source,e.skillRootPath)).find(e=>e.isFile()&&e.name.toLowerCase()===`skill.md`)?.name,n=join(e.skillRootPath,t??`SKILL.md`),r=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillId,t??`SKILL.md`));if(t===void 0)return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_MARKDOWN_MISSING,message:`Expected "${n}" to exist for the "${e.skillId}" skill.`,sourcePath:e.skillRootPath})],logicalPath:r,skill:null,skillId:null};let i;try{i=lowerSkillMarkdown(await e.source.readTextFile(n))}catch(e){return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_FRONTMATTER_INVALID,message:formatSkillDiscoveryError(n,e),sourcePath:n})],logicalPath:r,skill:null,skillId:null}}let a=await discoverSkillPackagePaths(e.source,e.skillRootPath),o={description:i.description,logicalPath:r,markdown:i.markdown,name:e.skillId,rootPath:e.skillRootPath,skillFilePath:n,skillId:e.skillId,sourceId:createPathDerivedSourceId(r)};return a.assetsPath!==void 0&&(o.assetsPath=a.assetsPath),i.license!==void 0&&(o.license=i.license),i.metadata!==void 0&&(o.metadata=i.metadata),a.referencesPath!==void 0&&(o.referencesPath=a.referencesPath),a.scriptsPath!==void 0&&(o.scriptsPath=a.scriptsPath),{diagnostics:[],logicalPath:r,skill:createSkillPackageSourceRef(o),skillId:e.skillId}}async function discoverFlatMarkdownSkill(e){let t=stripMarkdownExtension(e.skillFileName),n=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillFileName)),r;try{r=lowerSkillMarkdown(await e.source.readTextFile(e.skillFilePath),{slug:t})}catch(t){return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_FRONTMATTER_INVALID,message:formatSkillDiscoveryError(e.skillFilePath,t),sourcePath:e.skillFilePath})],logicalPath:n,skill:null,skillId:null}}return{diagnostics:[],logicalPath:n,skill:{definition:r,sourceKind:`markdown`,logicalPath:n,sourceId:createPathDerivedSourceId(n)},skillId:t}}async function discoverFlatModuleSkill(e){let t=getSupportedModuleBaseName(e.skillFileName),n=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillFileName));return t===null?{diagnostics:[],logicalPath:n,skill:null,skillId:null}:{diagnostics:[],logicalPath:n,skill:createModuleSourceRef({logicalPath:n}),skillId:t}}async function discoverSkillPackagePaths(e,t){let n=await e.readDirectory(t),r={};for(let e of n)if(e.isDirectory())switch(classifySkillPackageEntry(e.name,getDirectoryEntryType(e))){case`skill-assets-directory`:r.assetsPath=join(t,e.name);break;case`skill-references-directory`:r.referencesPath=join(t,e.name);break;case`skill-scripts-directory`:r.scriptsPath=join(t,e.name)}return r}function formatSkillDiscoveryError(e,t){return`Invalid authored skill frontmatter in "${e}": ${toErrorMessage(t)}`}function stripMarkdownExtension(e){return e.toLowerCase().endsWith(`.md`)?e.slice(0,-3):e}export{DISCOVER_SKILL_COLLISION,DISCOVER_SKILL_ENTRY_NOT_DIRECTORY,DISCOVER_SKILL_FRONTMATTER_INVALID,DISCOVER_SKILL_MARKDOWN_MISSING,discoverSkills};
+import{join,relative,resolve}from"node:path";import{classifySkillPackageEntry,classifySkillsDirectoryEntry,getDirectoryEntryType,getSupportedModuleBaseName,normalizeLogicalPath}from"#discover/filesystem.js";import{toErrorMessage}from"#shared/errors.js";import{createDiskProjectSource}from"#discover/project-source.js";import{createDiscoverErrorDiagnostic}from"#discover/diagnostics.js";import{SKILL_NAME_PATTERN,SKILL_NAME_RULE,readSortedDirectoryEntries}from"#discover/grammar.js";import{createModuleSourceRef,createPathDerivedSourceId,createSkillPackageSourceRef}from"#discover/manifest.js";import{lowerSkillMarkdown}from"#internal/helpers/markdown.js";const DISCOVER_SKILL_COLLISION=`discover/skill-collision`,DISCOVER_SKILL_ENTRY_NOT_DIRECTORY=`discover/skill-entry-not-directory`,DISCOVER_SKILL_FRONTMATTER_INVALID=`discover/skill-frontmatter-invalid`,DISCOVER_SKILL_MARKDOWN_MISSING=`discover/skill-markdown-missing`,DISCOVER_SKILL_NAME_INVALID=`discover/skill-name-invalid`;async function discoverSkills(e){let t=e.source??createDiskProjectSource(),n=resolve(e.agentRoot),r=resolve(e.skillsDirectoryPath??join(n,`skills`)),i=normalizeLogicalPath(e.skillsLogicalPath??relative(n,r)),a=await t.stat(r);if(a===`missing`)return{diagnostics:[],skills:[]};if(a!==`directory`)return{diagnostics:[createDiscoverErrorDiagnostic({code:`discover/skills-directory-invalid`,message:`Expected "${r}" to be a directory of authored skills.`,sourcePath:r})],skills:[]};let o=[],s=new Set,c=new Map,l=await readSortedDirectoryEntries(t,r);for(let e of l){let n=await discoverOneSkill({entryName:e.name,entryType:getDirectoryEntryType(e),skillsDirectoryPath:r,skillsLogicalPath:i,source:t});if(o.push(...n.diagnostics),n.skill===null||n.skillId===null)continue;if(!SKILL_NAME_PATTERN.test(n.skillId)){o.push(createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_NAME_INVALID,message:`Skill "${n.logicalPath}" has an illegal name "${n.skillId}". ${SKILL_NAME_RULE}`,sourcePath:join(r,e.name)}));continue}if(s.has(n.skillId))continue;let a=c.get(n.skillId);if(a!==void 0){o.push(createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_COLLISION,message:`Found conflicting authored skill sources for "${n.skillId}": "${a.logicalPath}" and "${n.logicalPath}".`,sourcePath:join(r,n.skillId)})),s.add(n.skillId),c.delete(n.skillId);continue}c.set(n.skillId,{logicalPath:n.logicalPath,skill:n.skill})}return{diagnostics:o,skills:[...c.values()].map(e=>e.skill)}}async function discoverOneSkill(e){let t=join(e.skillsDirectoryPath,e.entryName);switch(classifySkillsDirectoryEntry(e.entryName,e.entryType)){case`skill-package-directory`:return discoverPackagedSkill({logicalSkillsPath:e.skillsLogicalPath,skillId:e.entryName,skillRootPath:t,source:e.source});case`flat-skill-markdown`:return discoverFlatMarkdownSkill({logicalSkillsPath:e.skillsLogicalPath,skillFileName:e.entryName,skillFilePath:t,source:e.source});case`flat-skill-module`:return discoverFlatModuleSkill({logicalSkillsPath:e.skillsLogicalPath,skillFileName:e.entryName});case`ignored-declaration`:case`ignored-source-map`:return{diagnostics:[],logicalPath:normalizeLogicalPath(join(e.skillsLogicalPath,e.entryName)),skill:null,skillId:null};default:return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_ENTRY_NOT_DIRECTORY,message:`Expected "${t}" to be a skill directory containing SKILL.md or a flat ".md", ".ts", ".cts", ".mts", ".js", ".cjs", or ".mjs" skill file.`,sourcePath:t})],logicalPath:normalizeLogicalPath(join(e.skillsLogicalPath,e.entryName)),skill:null,skillId:null}}}async function discoverPackagedSkill(e){let t=(await readSortedDirectoryEntries(e.source,e.skillRootPath)).find(e=>e.isFile()&&e.name.toLowerCase()===`skill.md`)?.name,n=join(e.skillRootPath,t??`SKILL.md`),r=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillId,t??`SKILL.md`));if(t===void 0)return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_MARKDOWN_MISSING,message:`Expected "${n}" to exist for the "${e.skillId}" skill.`,sourcePath:e.skillRootPath})],logicalPath:r,skill:null,skillId:null};let i;try{i=lowerSkillMarkdown(await e.source.readTextFile(n))}catch(e){return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_FRONTMATTER_INVALID,message:formatSkillDiscoveryError(n,e),sourcePath:n})],logicalPath:r,skill:null,skillId:null}}let a=await discoverSkillPackagePaths(e.source,e.skillRootPath),o={description:i.description,logicalPath:r,markdown:i.markdown,name:e.skillId,rootPath:e.skillRootPath,skillFilePath:n,skillId:e.skillId,sourceId:createPathDerivedSourceId(r)};return a.assetsPath!==void 0&&(o.assetsPath=a.assetsPath),i.deferred!==void 0&&(o.deferred=i.deferred),i.license!==void 0&&(o.license=i.license),i.metadata!==void 0&&(o.metadata=i.metadata),a.referencesPath!==void 0&&(o.referencesPath=a.referencesPath),a.scriptsPath!==void 0&&(o.scriptsPath=a.scriptsPath),{diagnostics:[],logicalPath:r,skill:createSkillPackageSourceRef(o),skillId:e.skillId}}async function discoverFlatMarkdownSkill(e){let t=stripMarkdownExtension(e.skillFileName),n=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillFileName)),r;try{r=lowerSkillMarkdown(await e.source.readTextFile(e.skillFilePath),{slug:t})}catch(t){return{diagnostics:[createDiscoverErrorDiagnostic({code:DISCOVER_SKILL_FRONTMATTER_INVALID,message:formatSkillDiscoveryError(e.skillFilePath,t),sourcePath:e.skillFilePath})],logicalPath:n,skill:null,skillId:null}}return{diagnostics:[],logicalPath:n,skill:{definition:r,sourceKind:`markdown`,logicalPath:n,sourceId:createPathDerivedSourceId(n)},skillId:t}}async function discoverFlatModuleSkill(e){let t=getSupportedModuleBaseName(e.skillFileName),n=normalizeLogicalPath(join(e.logicalSkillsPath,e.skillFileName));return t===null?{diagnostics:[],logicalPath:n,skill:null,skillId:null}:{diagnostics:[],logicalPath:n,skill:createModuleSourceRef({logicalPath:n}),skillId:t}}async function discoverSkillPackagePaths(e,t){let n=await e.readDirectory(t),r={};for(let e of n)if(e.isDirectory())switch(classifySkillPackageEntry(e.name,getDirectoryEntryType(e))){case`skill-assets-directory`:r.assetsPath=join(t,e.name);break;case`skill-references-directory`:r.referencesPath=join(t,e.name);break;case`skill-scripts-directory`:r.scriptsPath=join(t,e.name)}return r}function formatSkillDiscoveryError(e,t){return`Invalid authored skill frontmatter in "${e}": ${toErrorMessage(t)}`}function stripMarkdownExtension(e){return e.toLowerCase().endsWith(`.md`)?e.slice(0,-3):e}export{DISCOVER_SKILL_COLLISION,DISCOVER_SKILL_ENTRY_NOT_DIRECTORY,DISCOVER_SKILL_FRONTMATTER_INVALID,DISCOVER_SKILL_MARKDOWN_MISSING,DISCOVER_SKILL_NAME_INVALID,discoverSkills};

package/dist/src/dynamic/definition.d.ts

@@ -1,5 +1,5 @@
 import type { ModelMessage } from "ai";
-import type { SessionAuth } from "#context/keys.js";
+import type { SessionAuth, SessionPredecessor } from "#context/keys.js";
 import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
 import type { ConversationContext } from "#shared/conversation-context.js";
 /**
@@ -35,6 +35,12 @@ export interface DynamicResolveContext {
         readonly id: string;
         readonly auth: SessionAuth;
         readonly schedule?: import("#context/session-schedule.js").SessionSchedule;
+        /**
+         * Present when eve started this session in place of a stranded session,
+         * one that another eve version built. It names the earlier session, whose recorded stream
+         * `sessions.attach(predecessor.sessionId)` from `eve/server` reads.
+         */
+        readonly predecessor?: SessionPredecessor;
     };
     /** Channel metadata for the request that triggered this resolve. */
     readonly channel: {
@@ -81,8 +87,9 @@ export type DynamicSentinel<TResult = unknown> = {
  * handlers. It is shared across tools, skills, connections, and agent definitions;
  * the directory it is authored in (not this function) decides what each
  * handler must return and which events are honored. The file's path-derived
- * slug names the single-entry case; a `Record<string, ...>` return names
- * entries `slug__key`. Return `null` to contribute nothing for that event.
+ * slug names the single-entry case; a `Record<string, ...>` return names each
+ * entry by its bare key, prefixed with the mount namespace for an extension's
+ * resolver. Return `null` to contribute nothing for that event.
  *
  * Per-slot return shape:
  * - `agent/tools/`: return a single `defineTool(...)`, a
@@ -121,9 +128,10 @@ export type DynamicSentinel<TResult = unknown> = {
  *
  * A single return is named after the file slug. A map names each entry by its
  * bare key — there is no automatic slug prefix, so namespace keys yourself
- * (e.g. `team__playbook`) when a bare name might collide. A dynamic tool/skill
- * whose name matches an authored one overrides it; two dynamic resolvers
- * emitting the same name is an error.
+ * (e.g. `team__playbook`) when a bare name might collide. Tool names must match
+ * the tool filename charset, and a name under a connection's `<name>__` prefix
+ * is rejected. A dynamic tool/skill whose name matches an authored one
+ * overrides it; two dynamic resolvers emitting the same name is an error.
  */
 export declare function defineDynamic<const TEvents extends DynamicEvents>(definition: {
     readonly events: TEvents;

package/dist/src/evals/assertions/collector.d.ts

@@ -1,4 +1,5 @@
 import type { AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalTaskResult } from "#evals/types.js";
+import type { TargetTools } from "#evals/target.js";
 import type { EveEvalAssertionSubject } from "#evals/assertions/run.js";
 export type AssertionOutcome = AssertionEvaluation;
 /**
@@ -18,6 +19,9 @@ export interface RunAssertion {
  */
 export declare class AssertionCollector {
     #private;
+    /** The target's root agent tools, which tool-name assertions are checked against. */
+    readonly tools: TargetTools | undefined;
+    constructor(tools?: TargetTools);
     /** Whether the eval has already recorded an assertion. */
     get hasEntries(): boolean;
     /** Register a deferred assertion against a turn or session scope. */

package/dist/src/evals/assertions/collector.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{formatAssertionName}from"#evals/diagnostics.js";var AssertionCollector=class{#e=[];#t=[];get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){let r={baseName:e.name,name:e.name,severity:n,threshold:void 0,kind:`deferred`,spec:e,selectSubject:t,score:0,errored:!1};return this.#e.push(r),makeHandle(r)}recordValue(e){let t={baseName:e.name,name:e.name,severity:e.severity,threshold:e.threshold,kind:`resolved`,score:0,errored:!1};this.#e.push(t);let n=settleEntry(t,e.score);return this.#t.push(n),makeHandle(t)}recordOutcome(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:void 0,kind:`resolved`,score:e.outcome.score,message:e.outcome.message,metadata:e.outcome.metadata,errored:!1};return this.#e.push(t),computePassed(t.severity,t.threshold,t.score,t.errored)}async recordRequirement(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:e.threshold,kind:`resolved`,score:0,errored:!1};return this.#e.push(t),await settleEntry(t,e.score),computePassed(t.severity,t.threshold,t.score,t.errored)}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push({name:n.name,score:n.score,severity:n.severity,threshold:n.threshold,passed:computePassed(n.severity,n.threshold,n.score,n.errored),errored:n.errored,message:n.message,metadata:n.metadata})}return t}};function computePassed(e,t,n,r){if(r)return!1;let i=t??(e===`gate`?1:void 0);return i===void 0||n>=i}async function settleEntry(e,t){try{let n=await t();e.score=n.score,e.message=n.message,e.metadata=n.metadata}catch(t){e.score=0,e.severity=`gate`,e.threshold=void 0,e.message=toErrorMessage(t),e.errored=!0}}function makeHandle(e){let t={gate(n){return e.severity=`gate`,e.threshold=n,t},soft(n){return e.severity=`soft`,e.threshold=n,t},atLeast(n){return e.severity=`soft`,e.threshold=n,t},label(n){return e.name=formatAssertionName(e.baseName,n),t}};return t}export{AssertionCollector};
+import{toErrorMessage}from"#shared/errors.js";import{formatAssertionName}from"#evals/diagnostics.js";var AssertionCollector=class{#e=[];#t=[];tools;constructor(e){this.tools=e}get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){let r={baseName:e.name,name:e.name,severity:n,threshold:void 0,kind:`deferred`,spec:e,selectSubject:t,score:0,errored:!1};return this.#e.push(r),makeHandle(r)}recordValue(e){let t={baseName:e.name,name:e.name,severity:e.severity,threshold:e.threshold,kind:`resolved`,score:0,errored:!1};this.#e.push(t);let n=settleEntry(t,e.score);return this.#t.push(n),makeHandle(t)}recordOutcome(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:void 0,kind:`resolved`,score:e.outcome.score,message:e.outcome.message,metadata:e.outcome.metadata,errored:!1};return this.#e.push(t),computePassed(t.severity,t.threshold,t.score,t.errored)}async recordRequirement(e){let t={baseName:e.name,name:e.name,severity:`gate`,threshold:e.threshold,kind:`resolved`,score:0,errored:!1};return this.#e.push(t),await settleEntry(t,e.score),computePassed(t.severity,t.threshold,t.score,t.errored)}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push({name:n.name,score:n.score,severity:n.severity,threshold:n.threshold,passed:computePassed(n.severity,n.threshold,n.score,n.errored),errored:n.errored,message:n.message,metadata:n.metadata})}return t}};function computePassed(e,t,n,r){if(r)return!1;let i=t??(e===`gate`?1:void 0);return i===void 0||n>=i}async function settleEntry(e,t){try{let n=await t();e.score=n.score,e.message=n.message,e.metadata=n.metadata}catch(t){e.score=0,e.severity=`gate`,e.threshold=void 0,e.message=toErrorMessage(t),e.errored=!0}}function makeHandle(e){let t={gate(n){return e.severity=`gate`,e.threshold=n,t},soft(n){return e.severity=`soft`,e.threshold=n,t},atLeast(n){return e.severity=`soft`,e.threshold=n,t},label(n){return e.name=formatAssertionName(e.baseName,n),t}};return t}export{AssertionCollector};

package/dist/src/evals/assertions/run.d.ts

@@ -32,9 +32,8 @@ export declare function messageIncludes(token: string | RegExp): RunAssertion;
  */
 export declare function calledTool(name: string, options?: EveEvalToolCallMatchOptions): RunAssertion;
 /**
- * Sugar over {@link calledTool} for the framework `load_skill` tool: asserts a
- * skill with id `skill` was loaded. `output`/`status`/`count` constrain the
- * matching call exactly as for `calledTool`.
+ * Asserts the skill `skill` was loaded. `output`/`status`/`count` constrain
+ * the matching load exactly as for `calledTool`.
  */
 export declare function loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): RunAssertion;
 /**

package/dist/src/evals/assertions/run.js

@@ -1,2 +1,2 @@
-import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&t.push(n.data.message);return t.join(`
-`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return e.name===void 0?`load-skill-result`:`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};
+import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function loadedSkill(e,t={}){return validateCount(t.count),{name:`loadedSkill(${e})`,evaluate(n){let r=n.derived.skillLoads.filter(t=>t.skill===e).filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,r.length))return{score:1,metadata:{matchingLoads:r.length}};let i=n.derived.skillLoads.map(e=>e.skill).join(`, `);return fail(`expected ${describeCount(t.count)} matching load(s) of "${e}", found ${r.length}; observed skill loads: [${i}]`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&t.push(n.data.message);return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};

package/dist/src/evals/assertions/scoped.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";function createScopedAssertions(e,t){let n=createRecorder(e,t);return{succeeded:()=>n(RunAssertions.succeeded()),parked:()=>n(RunAssertions.parked()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(e,t)),loadedSkill:(e,t)=>n(RunAssertions.loadedSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(e)),toolOrder:e=>n(RunAssertions.toolOrder(e)),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledSubagent:(e,t)=>n(RunAssertions.calledSubagent(e,t)),noFailedActions:()=>n(RunAssertions.noFailedActions()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};
+import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";function createScopedAssertions(e,t){let n=createRecorder(e,t),toolName=t=>(assertReportedToolName(t,e.tools),t);return{succeeded:()=>n(RunAssertions.succeeded()),parked:()=>n(RunAssertions.parked()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(toolName(e),t)),loadedSkill:(e,t)=>n(RunAssertions.loadedSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(toolName(e))),toolOrder:e=>n(RunAssertions.toolOrder(e.map(toolName))),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledSubagent:(e,t)=>n(RunAssertions.calledSubagent(e,t)),noFailedActions:()=>n(RunAssertions.noFailedActions()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};

package/dist/src/evals/index.d.ts

@@ -6,6 +6,6 @@ export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js"
 export type { InputRequest } from "#shared/input.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";
 export type { ToolStub, ToolStubOutcome } from "#tool-stubs/types.js";

package/dist/src/evals/match.d.ts

@@ -38,9 +38,9 @@ export interface EveEvalToolCallMatchOptions {
     readonly count?: EveEvalCountMatcher;
 }
 /**
- * Constraints applied to a `load_skill` call by `t.loadedSkill`. Identical to
- * {@link EveEvalToolCallMatchOptions} without `input`, which the helper fixes to
- * the loaded skill id.
+ * Constraints applied to a skill load by `t.loadedSkill`. Identical to
+ * {@link EveEvalToolCallMatchOptions} without `input`, since the skill name is
+ * the whole input.
  */
 export type EveEvalSkillLoadMatchOptions = Omit<EveEvalToolCallMatchOptions, "input">;
 /**
@@ -92,10 +92,13 @@ export type EveEvalEventMatch<TType extends MessageStreamEvent["type"] = Message
  */
 export declare function matchesValue(matcher: unknown, value: unknown): boolean;
 /**
- * Returns true when one derived tool call satisfies the `input`/`output`/
- * lifecycle constraints (the `count` option is the caller's concern).
+ * Returns true when one derived tool call or skill load satisfies the
+ * `input`/`output`/lifecycle constraints (the `count` option is the caller's
+ * concern).
  */
-export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
+export declare function toolCallMatches(call: Pick<EveEvalToolCall, "output" | "status"> & {
+    readonly input?: JsonObject;
+}, options: EveEvalToolCallMatchOptions): boolean;
 /**
  * Returns true when one derived subagent call satisfies the `remoteUrl`/
  * `output` constraints.

package/dist/src/evals/reported-tool-name.d.ts

@@ -0,0 +1,9 @@
+import type { TargetTools } from "#evals/target.js";
+/**
+ * Throws for a tool name no call is reported under, so an assertion on it can't
+ * pass or fail without saying why. A retired name outside the `eve` namespace
+ * is still an ordinary tool name, so it passes when the root agent declares a
+ * tool by that name, has a dynamic tool resolver that might return one, or its
+ * tools are unknown.
+ */
+export declare function assertReportedToolName(name: string, tools: TargetTools | undefined): void;

package/dist/src/evals/reported-tool-name.js

@@ -0,0 +1 @@
+import{TASK_CANCEL_TOOL_NAME,TASK_WAIT_TOOL_NAME}from"#protocol/task-tools.js";import{REPLY_TOOL_NAME}from"#protocol/reply-tool.js";import{CALL_TOOL_NAME,SEARCH_TOOL_NAME,SKILL_TOOL_NAME}from"#protocol/catalog-tools.js";const UNREPORTED_TOOL_NAMES=new Map([[CALL_TOOL_NAME,`Calls through ${CALL_TOOL_NAME} are reported under the entry's name; assert on the tool or agent it reaches.`],[SKILL_TOOL_NAME,`Calls through ${SKILL_TOOL_NAME} are reported as skill loads; use t.loadedSkill(name).`],[`eve__execute`,`eve__execute isn't a tool eve registers (it's reserved for code mode); assert on the tool or agent a call reaches, or use t.loadedSkill(name).`]]),RETIRED_TOOL_NAMES=new Map([[`connection_execute`,"assert on the connection tool's own name, `<connection>__<tool>`"],[`connection_search`,`use ${SEARCH_TOOL_NAME}`],[`final_output`,`use ${REPLY_TOOL_NAME}`],[`load_skill`,`use t.loadedSkill(name)`],[`task_cancel`,`use ${TASK_CANCEL_TOOL_NAME}`],[`task_wait`,`use ${TASK_WAIT_TOOL_NAME}`]]);function assertReportedToolName(e,t){let n=UNREPORTED_TOOL_NAMES.get(e);if(n!==void 0)throw TypeError(n);let r=RETIRED_TOOL_NAMES.get(e);if(r!==void 0&&t!==void 0&&!(t.dynamic||t.static.includes(e)))throw TypeError(`Tool name "${e}" was retired; ${r}.`)}export{assertReportedToolName};

package/dist/src/evals/runner/derive-run-facts.d.ts

@@ -4,14 +4,14 @@ import type { EveEvalDerivedFacts } from "#evals/types.js";
  * Options for {@link deriveRunFacts}.
  */
 export interface DeriveRunFactsOptions {
-    /** Session id stamped onto every derived tool and subagent call. */
+    /** Session id stamped onto every derived tool call, skill load, and subagent call. */
     readonly sessionId?: string;
 }
 /**
  * Extracts derived execution facts from a completed run's stream events.
  *
- * Tool calls pair each `actions.requested` entry with its matching
- * `action.result` by call id. Subagent calls are the calls to agent tasks,
+ * Tool calls and skill loads pair each `actions.requested` entry with its
+ * matching `action.result` by call id. Subagent calls are the calls to agent tasks,
  * paired with their `task.settled` the same way. These facts power checks,
  * scorers, and reporters.
  */

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=[],l=-1,u=0,d=0,f=new Set,p,m,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(l,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:l+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&ensureToolCall(e.callId,LOAD_SKILL_TOOL_NAME,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;a.push({callId:e,name:r,taskId:i,turnIndex:Math.max(l,0)});break}case`task.settled`:o.set(t.data.callId,t.data);break;case`agent.started`:s.push(t.data);break;case`input.requested`:c.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(u+=1);break;case`reasoning.completed`:d+=1;break;case`step.started`:f.add(t.data.modelId);break;case`session.waiting`:case`turn.waiting`:p=t.data.usage;break;case`session.completed`:p=t.data?.usage;break;case`session.failed`:p=t.data.usage,m=t.data.code}let h=deriveSubagentCalls({agentCalls:a,agentSessions:s,sessionId:n,settledTaskCalls:o});return{toolCalls:r,toolCallCount:r.length,subagentCalls:h,subagentCallCount:h.length,inputRequests:c,parked:endedParkedOnInput(e),messageCount:u,reasoningBlockCount:d,models:[...f],usage:p,failureCode:m}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||PARKING_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=new Map,o=[],s=new Map,c=[],l=[],u=-1,d=0,f=0,p=new Set,m,h,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(u,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:u+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&a.set(e.callId,{output:void 0,sessionId:n,skill:e.name,status:`pending`,turnIndex:Math.max(u,0)});break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}let r=e.kind===`load-skill-result`?a.get(e.callId):void 0;r!==void 0&&(r.output=e.output,r.status=n);break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;o.push({callId:e,name:r,taskId:i,turnIndex:Math.max(u,0)});break}case`task.settled`:s.set(t.data.callId,t.data);break;case`agent.started`:c.push(t.data);break;case`input.requested`:l.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(d+=1);break;case`reasoning.completed`:f+=1;break;case`step.started`:p.add(t.data.modelId);break;case`session.waiting`:case`turn.waiting`:m=t.data.usage;break;case`session.completed`:m=t.data?.usage;break;case`session.failed`:m=t.data.usage,h=t.data.code}let g=deriveSubagentCalls({agentCalls:o,agentSessions:c,sessionId:n,settledTaskCalls:s});return{toolCalls:r,toolCallCount:r.length,skillLoads:[...a.values()],subagentCalls:g,subagentCallCount:g.length,inputRequests:l,parked:endedParkedOnInput(e),messageCount:d,reasoningBlockCount:f,models:[...p],usage:m,failureCode:h}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,skillLoads:[],subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||PARKING_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector,s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:i}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=[...new Map(e.map(e=>[e.sessionId,e.derived.usage]))].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n]);if(n.every(e=>e!==void 0))return n.reduce(addTokenUsage)}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle,targetTools}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector(targetTools(r)),s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.skillLoads),r=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,skillLoads:n,subagentCalls:r,subagentCallCount:r.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:a}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=[...new Map(e.map(e=>[e.sessionId,e.derived.usage]))].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n]);if(n.every(e=>e!==void 0))return n.reduce(addTokenUsage)}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,e=>e.severity===`gate`?`gate:${e.name}`:e.name);for(let[e,t]of r)n[e]=t.score;let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,e=>e.severity===`gate`?`gate:${e.name}`:e.name);for(let[e,t]of r)n[e]=t.score;let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/datadog-runtime-trace-links.d.ts

@@ -0,0 +1,11 @@
+import type { EveEvalTraceContext } from "#evals/types.js";
+/** Identifies one runtime trace by the IDs Datadog's APM-to-LLMObs indexer assigns. */
+export interface ExperimentRuntimeTraceLink {
+    readonly relation: "experiment_runtime";
+    readonly traceId: string;
+    readonly spanId: string;
+    readonly sessionId: string;
+    readonly primary: boolean;
+}
+/** Converts sampled runtime trace contexts to Datadog-indexed links; skips the rest. */
+export declare function resolveRuntimeTraceLinks(traceContexts: readonly EveEvalTraceContext[]): ExperimentRuntimeTraceLink[];

package/dist/src/evals/runner/reporters/datadog-runtime-trace-links.js

@@ -0,0 +1 @@
+import{createHash}from"node:crypto";const DATADOG_LLMOBS_TRACE_ID_NAMESPACE=Buffer.from(`f47ac10b58cc4372a5670e02b2c3d479`,`hex`),W3C_TRACE_ID_PATTERN=/^[0-9a-f]{32}$/iu,W3C_SPAN_ID_PATTERN=/^[0-9a-f]{16}$/iu;function resolveRuntimeTraceLinks(e){return e.flatMap(e=>{if(!(e.traceFlags&1))return[];let t=toDatadogLlmobsTraceId(e.traceId),n=toDatadogLlmobsSpanId(e.spanId);return t===void 0||n===void 0?[]:[{relation:`experiment_runtime`,traceId:t,spanId:n,sessionId:e.sessionId,primary:e.primary}]})}function toDatadogLlmobsTraceId(e){let t=e.toLowerCase();if(!W3C_TRACE_ID_PATTERN.test(t)||/^0+$/u.test(t))return;let n=t.slice(-16).padStart(32,`0`),r=createHash(`sha1`).update(DATADOG_LLMOBS_TRACE_ID_NAMESPACE).update(n).digest().subarray(0,16);return r[6]=r[6]&15|80,r[8]=r[8]&63|128,r.toString(`hex`)}function toDatadogLlmobsSpanId(e){if(W3C_SPAN_ID_PATTERN.test(e)&&!/^0+$/u.test(e))return BigInt(`0x${e}`).toString(10)}export{resolveRuntimeTraceLinks};

package/dist/src/evals/runner/reporters/datadog.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
-`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);if(Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked}),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(e.severity===`gate`?`gate_${e.name}`:e.name));for(let[e,i]of r)t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,assertion_passed:String(i.assertions.every(e=>e.passed)),assertion_count:String(i.assertions.length)}});return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(0,r-n)}export{Datadog};
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{resolveRuntimeTraceLinks}from"#evals/runner/reporters/datadog-runtime-trace-links.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
+`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(e.severity===`gate`?`gate_${e.name}`:e.name));for(let[e,i]of r)t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,assertion_passed:String(i.assertions.every(e=>e.passed)),assertion_count:String(i.assertions.length)}});return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{Datadog};

package/dist/src/evals/session.js

@@ -1,2 +1,2 @@
-import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
-`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
+`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};

package/dist/src/evals/target.d.ts

@@ -7,6 +7,13 @@ export declare function resolveEvalTargetHandle(input: {
     readonly kind: "local" | "remote";
     readonly url: string;
 }): Promise<EveEvalTargetHandle>;
+/** The root agent's static tool names, and whether it has a dynamic tool resolver. */
+export interface TargetTools {
+    readonly dynamic: boolean;
+    readonly static: readonly string[];
+}
+/** The tools of a target resolved from `/eve/v1/info`; undefined for a hand-built handle. */
+export declare function targetTools(target: EveEvalTargetHandle): TargetTools | undefined;
 export declare function createEvalTargetHandle(input: {
     readonly capabilities: EveEvalTargetCapabilities;
     readonly client: Client;

package/dist/src/evals/target.js

@@ -1 +1 @@
-import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);return createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url})}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,async attachSession(r,i){if(e.sessions!==void 0)return await e.sessions.attachSession(r,i);if(t!==void 0)return await t.attachSession(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:n}).attachSession(r,i)},watchTurn(r,i){if(e.sessions!==void 0)return e.sessions.watchTurn(r,i);if(t!==void 0)return t.watchTurn(r,i);if(n===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:n}).watchTurn(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle};
+import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);let n=createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url});return toolsByTarget.set(n,{dynamic:t.tools.dynamic.length>0,static:t.tools.static.map(e=>e.name)}),n}const toolsByTarget=new WeakMap;function targetTools(e){return toolsByTarget.get(e)}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,async attachSession(r,i){if(e.sessions!==void 0)return await e.sessions.attachSession(r,i);if(t!==void 0)return await t.attachSession(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:n}).attachSession(r,i)},watchTurn(r,i){if(e.sessions!==void 0)return e.sessions.watchTurn(r,i);if(t!==void 0)return t.watchTurn(r,i);if(n===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:n}).watchTurn(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle,targetTools};

package/dist/src/evals/types.d.ts

@@ -29,6 +29,21 @@ export interface EveEvalToolCall {
     /** Owning session id, when the runner knows it. */
     readonly sessionId?: string;
 }
+/**
+ * One skill load extracted from the captured stream, pairing the `load-skill`
+ * request with its matching `load-skill-result`.
+ */
+export interface EveEvalSkillLoad {
+    /** The loaded skill's name. */
+    readonly skill: string;
+    /** The skill's instructions; `undefined` when the load never resolved. */
+    readonly output: JsonValue | undefined;
+    readonly status: EveEvalActionStatus;
+    /** Zero-based index of the turn the load happened in. */
+    readonly turnIndex: number;
+    /** Owning session id, when the runner knows it. */
+    readonly sessionId?: string;
+}
 /**
  * One call to an agent task extracted from the captured stream: its
  * `task.started`, joined with its `task.settled` and the task's
@@ -58,6 +73,7 @@ export interface EveEvalSubagentCall {
 export interface EveEvalDerivedFacts {
     readonly toolCalls: readonly EveEvalToolCall[];
     readonly toolCallCount: number;
+    readonly skillLoads: readonly EveEvalSkillLoad[];
     readonly subagentCalls: readonly EveEvalSubagentCall[];
     readonly subagentCallCount: number;
     /** Every HITL input request raised during the run (`input.requested`). */
@@ -209,7 +225,7 @@ export interface EveEvalAssertions {
     parked(): AssertionHandle;
     messageIncludes(token: string | RegExp): AssertionHandle;
     calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
-    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
+    /** Asserts a completed load of `skill`, constrained like `calledTool`. */
     loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
     notCalledTool(name: string): AssertionHandle;
     /** Asserts that tool requests appeared in order, allowing unrelated requests between them. */
@@ -330,7 +346,8 @@ export interface EveEvalTurn extends EveEvalAssertions, EveEvalOutputAssertions
 export interface EveEvalJudgeConfig {
     /** Decision model ID or instance. Defaults to the model used by `eve/ai` decide. */
     readonly model?: DecisionModel;
-    readonly modelOptions?: AgentModelOptionsDefinition;
+    /** Judge calls carry provider options only; eve places no prompt-cache breakpoints on them. */
+    readonly modelOptions?: Pick<AgentModelOptionsDefinition, "providerOptions">;
 }
 /** JSON content accepted as decision state, instructions, or rubric descriptions. */
 export type JudgeInput = string | JsonObject | readonly JsonValue[];