eve 0.75.1 → 0.76.0

package/dist/src/evals/assertions/run.js

@@ -1,2 +1,2 @@
-import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&t.push(n.data.message);return t.join(`
-`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return e.name===void 0?`load-skill-result`:`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};
+import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function loadedSkill(e,t={}){return validateCount(t.count),{name:`loadedSkill(${e})`,evaluate(n){let r=n.derived.skillLoads.filter(t=>t.skill===e).filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,r.length))return{score:1,metadata:{matchingLoads:r.length}};let i=n.derived.skillLoads.map(e=>e.skill).join(`, `);return fail(`expected ${describeCount(t.count)} matching load(s) of "${e}", found ${r.length}; observed skill loads: [${i}]`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&t.push(n.data.message);return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};

package/dist/src/evals/assertions/scoped.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";function createScopedAssertions(e,t){let n=createRecorder(e,t);return{succeeded:()=>n(RunAssertions.succeeded()),parked:()=>n(RunAssertions.parked()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(e,t)),loadedSkill:(e,t)=>n(RunAssertions.loadedSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(e)),toolOrder:e=>n(RunAssertions.toolOrder(e)),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledSubagent:(e,t)=>n(RunAssertions.calledSubagent(e,t)),noFailedActions:()=>n(RunAssertions.noFailedActions()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};
+import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";function createScopedAssertions(e,t){let n=createRecorder(e,t),toolName=t=>(assertReportedToolName(t,e.tools),t);return{succeeded:()=>n(RunAssertions.succeeded()),parked:()=>n(RunAssertions.parked()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(toolName(e),t)),loadedSkill:(e,t)=>n(RunAssertions.loadedSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(toolName(e))),toolOrder:e=>n(RunAssertions.toolOrder(e.map(toolName))),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledSubagent:(e,t)=>n(RunAssertions.calledSubagent(e,t)),noFailedActions:()=>n(RunAssertions.noFailedActions()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};

package/dist/src/evals/index.d.ts

@@ -6,6 +6,6 @@ export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js"
 export type { InputRequest } from "#shared/input.js";
 export type { CancelSessionResult } from "#client/types.js";
 export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, } from "#evals/types.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";
 export type { ToolStub, ToolStubOutcome } from "#tool-stubs/types.js";

package/dist/src/evals/match.d.ts

@@ -38,9 +38,9 @@ export interface EveEvalToolCallMatchOptions {
     readonly count?: EveEvalCountMatcher;
 }
 /**
- * Constraints applied to a `load_skill` call by `t.loadedSkill`. Identical to
- * {@link EveEvalToolCallMatchOptions} without `input`, which the helper fixes to
- * the loaded skill id.
+ * Constraints applied to a skill load by `t.loadedSkill`. Identical to
+ * {@link EveEvalToolCallMatchOptions} without `input`, since the skill name is
+ * the whole input.
  */
 export type EveEvalSkillLoadMatchOptions = Omit<EveEvalToolCallMatchOptions, "input">;
 /**
@@ -92,10 +92,13 @@ export type EveEvalEventMatch<TType extends MessageStreamEvent["type"] = Message
  */
 export declare function matchesValue(matcher: unknown, value: unknown): boolean;
 /**
- * Returns true when one derived tool call satisfies the `input`/`output`/
- * lifecycle constraints (the `count` option is the caller's concern).
+ * Returns true when one derived tool call or skill load satisfies the
+ * `input`/`output`/lifecycle constraints (the `count` option is the caller's
+ * concern).
  */
-export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
+export declare function toolCallMatches(call: Pick<EveEvalToolCall, "output" | "status"> & {
+    readonly input?: JsonObject;
+}, options: EveEvalToolCallMatchOptions): boolean;
 /**
  * Returns true when one derived subagent call satisfies the `remoteUrl`/
  * `output` constraints.

package/dist/src/evals/reported-tool-name.d.ts

@@ -0,0 +1,9 @@
+import type { TargetTools } from "#evals/target.js";
+/**
+ * Throws for a tool name no call is reported under, so an assertion on it can't
+ * pass or fail without saying why. A retired name outside the `eve` namespace
+ * is still an ordinary tool name, so it passes when the root agent declares a
+ * tool by that name, has a dynamic tool resolver that might return one, or its
+ * tools are unknown.
+ */
+export declare function assertReportedToolName(name: string, tools: TargetTools | undefined): void;

package/dist/src/evals/reported-tool-name.js

@@ -0,0 +1 @@
+import{TASK_CANCEL_TOOL_NAME,TASK_WAIT_TOOL_NAME}from"#protocol/task-tools.js";import{REPLY_TOOL_NAME}from"#protocol/reply-tool.js";import{CALL_TOOL_NAME,SEARCH_TOOL_NAME,SKILL_TOOL_NAME}from"#protocol/catalog-tools.js";const UNREPORTED_TOOL_NAMES=new Map([[CALL_TOOL_NAME,`Calls through ${CALL_TOOL_NAME} are reported under the entry's name; assert on the tool or agent it reaches.`],[SKILL_TOOL_NAME,`Calls through ${SKILL_TOOL_NAME} are reported as skill loads; use t.loadedSkill(name).`],[`eve__execute`,`eve__execute isn't a tool eve registers (it's reserved for code mode); assert on the tool or agent a call reaches, or use t.loadedSkill(name).`]]),RETIRED_TOOL_NAMES=new Map([[`connection_execute`,"assert on the connection tool's own name, `<connection>__<tool>`"],[`connection_search`,`use ${SEARCH_TOOL_NAME}`],[`final_output`,`use ${REPLY_TOOL_NAME}`],[`load_skill`,`use t.loadedSkill(name)`],[`task_cancel`,`use ${TASK_CANCEL_TOOL_NAME}`],[`task_wait`,`use ${TASK_WAIT_TOOL_NAME}`]]);function assertReportedToolName(e,t){let n=UNREPORTED_TOOL_NAMES.get(e);if(n!==void 0)throw TypeError(n);let r=RETIRED_TOOL_NAMES.get(e);if(r!==void 0&&t!==void 0&&!(t.dynamic||t.static.includes(e)))throw TypeError(`Tool name "${e}" was retired; ${r}.`)}export{assertReportedToolName};

package/dist/src/evals/runner/derive-run-facts.d.ts

@@ -4,14 +4,14 @@ import type { EveEvalDerivedFacts } from "#evals/types.js";
  * Options for {@link deriveRunFacts}.
  */
 export interface DeriveRunFactsOptions {
-    /** Session id stamped onto every derived tool and subagent call. */
+    /** Session id stamped onto every derived tool call, skill load, and subagent call. */
     readonly sessionId?: string;
 }
 /**
  * Extracts derived execution facts from a completed run's stream events.
  *
- * Tool calls pair each `actions.requested` entry with its matching
- * `action.result` by call id. Subagent calls are the calls to agent tasks,
+ * Tool calls and skill loads pair each `actions.requested` entry with its
+ * matching `action.result` by call id. Subagent calls are the calls to agent tasks,
  * paired with their `task.settled` the same way. These facts power checks,
  * scorers, and reporters.
  */

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=[],l=-1,u=0,d=0,f=new Set,p,m,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(l,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:l+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&ensureToolCall(e.callId,LOAD_SKILL_TOOL_NAME,e.input);break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;a.push({callId:e,name:r,taskId:i,turnIndex:Math.max(l,0)});break}case`task.settled`:o.set(t.data.callId,t.data);break;case`agent.started`:s.push(t.data);break;case`input.requested`:c.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(u+=1);break;case`reasoning.completed`:d+=1;break;case`step.started`:f.add(t.data.modelId);break;case`session.waiting`:case`turn.waiting`:p=t.data.usage;break;case`session.completed`:p=t.data?.usage;break;case`session.failed`:p=t.data.usage,m=t.data.code}let h=deriveSubagentCalls({agentCalls:a,agentSessions:s,sessionId:n,settledTaskCalls:o});return{toolCalls:r,toolCallCount:r.length,subagentCalls:h,subagentCallCount:h.length,inputRequests:c,parked:endedParkedOnInput(e),messageCount:u,reasoningBlockCount:d,models:[...f],usage:p,failureCode:m}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||PARKING_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};
+const PARKING_EVENT_TYPES=new Set([`turn.waiting`,`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=new Map,o=[],s=new Map,c=[],l=[],u=-1,d=0,f=0,p=new Set,m,h,ensureToolCall=(e,t,a)=>{let o=i.get(e);if(o!==void 0)return o;let s={name:t,input:a,output:void 0,status:`pending`,turnIndex:Math.max(u,0),sessionId:n};return r.push(s),i.set(e,s),s};for(let t of e)switch(t.type){case`turn.started`:u+=1;break;case`actions.requested`:for(let e of t.data.actions)e.kind===`tool-call`?ensureToolCall(e.callId,e.toolName,e.input):e.kind===`load-skill`&&a.set(e.callId,{output:void 0,sessionId:n,skill:e.name,status:`pending`,turnIndex:Math.max(u,0)});break;case`action.result`:{let{result:e,status:n}=t.data;if(e.kind===`tool-result`){let t=ensureToolCall(e.callId,e.toolName,{});t.output=e.output,t.status=n}let r=e.kind===`load-skill-result`?a.get(e.callId):void 0;r!==void 0&&(r.output=e.output,r.status=n);break}case`task.started`:{let{callId:e,kind:n,name:r,taskId:i}=t.data;if(n!==`agent`)break;o.push({callId:e,name:r,taskId:i,turnIndex:Math.max(u,0)});break}case`task.settled`:s.set(t.data.callId,t.data);break;case`agent.started`:c.push(t.data);break;case`input.requested`:l.push(...t.data.requests);for(let e of t.data.requests)ensureToolCall(e.action.callId,e.action.toolName,e.action.input);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(d+=1);break;case`reasoning.completed`:f+=1;break;case`step.started`:p.add(t.data.modelId);break;case`session.waiting`:case`turn.waiting`:m=t.data.usage;break;case`session.completed`:m=t.data?.usage;break;case`session.failed`:m=t.data.usage,h=t.data.code}let g=deriveSubagentCalls({agentCalls:o,agentSessions:c,sessionId:n,settledTaskCalls:s});return{toolCalls:r,toolCallCount:r.length,skillLoads:[...a.values()],subagentCalls:g,subagentCallCount:g.length,inputRequests:l,parked:endedParkedOnInput(e),messageCount:d,reasoningBlockCount:f,models:[...p],usage:m,failureCode:h}}function deriveSubagentCalls(e){let t=new Map;for(let n of e.agentSessions)n.taskId!==void 0&&t.set(n.taskId,n);return e.agentCalls.map(n=>{let r=t.get(n.taskId),i=e.settledTaskCalls.get(n.callId);return{callId:n.callId,childSessionId:r?.sessionId,name:n.name,output:i?.output,remoteUrl:r?.remote?.url,sessionId:e.sessionId,status:i?.status??`working`,turnIndex:n.turnIndex}})}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,skillLoads:[],subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0,models:[]}}function endedParkedOnInput(e){for(let t=e.length-1;t>=0;t--){let n=e[t];if(!(n===void 0||PARKING_EVENT_TYPES.has(n.type)))return n.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/execute-task.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector,s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),r=e.flatMap(e=>e.derived.inputRequests),i=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:r,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:i}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=[...new Map(e.map(e=>[e.sessionId,e.derived.usage]))].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n]);if(n.every(e=>e!==void 0))return n.reduce(addTokenUsage)}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};
+import{toErrorMessage}from"#shared/errors.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{scopeEvalTargetHandle,targetTools}from"#evals/target.js";import{EvalSessionManager}from"#evals/session-manager.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";import{runUntilAborted}from"#evals/abort.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{addTokenUsage}from"#shared/token-usage.js";import{createEvalContext}from"#evals/context.js";async function executeTask(e){let{client:t,evaluation:n,target:r,timeoutMs:i}=e,a=i===void 0?neverAbortSignal():AbortSignal.timeout(i),o=new AssertionCollector(targetTools(r)),s=new EvalSessionManager({client:t,collector:o,onSessionStart:e.onSessionStart,signal:a}),c=scopeEvalTargetHandle(r,{sessions:s}),l=[],{context:u}=createEvalContext({setupContext:e.setupContext,collector:o,manager:s,target:c,signal:a,judge:n.judge,log:t=>{l.push(t),e.onLog?.(t)}}),d,f;try{await runUntilAborted(n.test(u),a),await runUntilAborted(s.verifyStubs(),a)}catch(e){e instanceof EvalSkipped?f=e.reason:e instanceof EvalRequirementFailed||(d=toErrorMessage(e))}if(i!==void 0&&a.aborted){let e=(await s.cleanup(AbortSignal.timeout(5e3))).flatMap(e=>e.status===`rejected`?[toErrorMessage(e.reason)]:[]);if(e.length>0){let t=`Eval timeout cleanup failed: ${e.join(`; `)}`;d=d===void 0?t:`${d}\n${t}`}}let p=buildTaskResult({logs:l,sessions:s.snapshots(),turn:s.lastTurnSession()?.lastTurn});return{result:p,assertions:await o.finalize(p),error:d,skipReason:f}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:e.turn?.data===void 0?n:e.turn.data,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t),traceContexts:collectTraceContexts(e.sessions)}}function collectTraceContexts(e){return e.flatMap(e=>{let t=e.sessionId;return t===void 0?[]:e.traceContexts.map(n=>({...n,primary:e.primary,sessionId:t}))})}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.skillLoads),r=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,skillLoads:n,subagentCalls:r,subagentCallCount:r.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),models:[...new Set(e.flatMap(e=>e.derived.models))],usage:evalUsage(e),failureCode:a}}function evalUsage(e){let t=new Set(e.flatMap(e=>e.events.flatMap(e=>e.type===`agent.started`?[e.data.sessionId]:[]))),n=[...new Map(e.map(e=>[e.sessionId,e.derived.usage]))].flatMap(([e,n])=>e!==void 0&&t.has(e)?[]:[n]);if(n.every(e=>e!==void 0))return n.reduce(addTokenUsage)}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,e=>e.severity===`gate`?`gate:${e.name}`:e.name);for(let[e,t]of r)n[e]=t.score;let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(e,t){let n=await loadBraintrustSdk();this.#t=n;let r=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let t of e)this.#r.set(t.id,t);let i=resolveTags(e,t),a=resolveExperimentMetadata(e,t);this.#n=await n.init({project:this.#e.projectName??e[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:i,metadata:a,noExitFlush:!0,setCurrent:!1,repoInfo:r.sha?{commit:r.sha,branch:r.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={},r=groupAssertionScores(e.assertions,e=>e.severity===`gate`?`gate:${e.name}`:e.name);for(let[e,t]of r)n[e]=t.score;let i=e.assertions.filter(e=>!e.passed).map(e=>({...e})),a={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked,eveAssertionScores:composeAssertionScoreMetadata(e.assertions)};e.result.traceContexts.length>0&&(a.eveTraceIds=[...new Set(e.result.traceContexts.map(e=>e.traceId))],a.eveTraceContexts=e.result.traceContexts),i.length>0&&(a.eveFailedAssertions=i),e.result.derived.failureCode&&(a.eveFailureCode=e.result.derived.failureCode);let o={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:a,metrics:o,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
 `))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/datadog-runtime-trace-links.d.ts

@@ -0,0 +1,11 @@
+import type { EveEvalTraceContext } from "#evals/types.js";
+/** Identifies one runtime trace by the IDs Datadog's APM-to-LLMObs indexer assigns. */
+export interface ExperimentRuntimeTraceLink {
+    readonly relation: "experiment_runtime";
+    readonly traceId: string;
+    readonly spanId: string;
+    readonly sessionId: string;
+    readonly primary: boolean;
+}
+/** Converts sampled runtime trace contexts to Datadog-indexed links; skips the rest. */
+export declare function resolveRuntimeTraceLinks(traceContexts: readonly EveEvalTraceContext[]): ExperimentRuntimeTraceLink[];

package/dist/src/evals/runner/reporters/datadog-runtime-trace-links.js

@@ -0,0 +1 @@
+import{createHash}from"node:crypto";const DATADOG_LLMOBS_TRACE_ID_NAMESPACE=Buffer.from(`f47ac10b58cc4372a5670e02b2c3d479`,`hex`),W3C_TRACE_ID_PATTERN=/^[0-9a-f]{32}$/iu,W3C_SPAN_ID_PATTERN=/^[0-9a-f]{16}$/iu;function resolveRuntimeTraceLinks(e){return e.flatMap(e=>{if(!(e.traceFlags&1))return[];let t=toDatadogLlmobsTraceId(e.traceId),n=toDatadogLlmobsSpanId(e.spanId);return t===void 0||n===void 0?[]:[{relation:`experiment_runtime`,traceId:t,spanId:n,sessionId:e.sessionId,primary:e.primary}]})}function toDatadogLlmobsTraceId(e){let t=e.toLowerCase();if(!W3C_TRACE_ID_PATTERN.test(t)||/^0+$/u.test(t))return;let n=t.slice(-16).padStart(32,`0`),r=createHash(`sha1`).update(DATADOG_LLMOBS_TRACE_ID_NAMESPACE).update(n).digest().subarray(0,16);return r[6]=r[6]&15|80,r[8]=r[8]&63|128,r.toString(`hex`)}function toDatadogLlmobsSpanId(e){if(W3C_SPAN_ID_PATTERN.test(e)&&!/^0+$/u.test(e))return BigInt(`0x${e}`).toString(10)}export{resolveRuntimeTraceLinks};

package/dist/src/evals/runner/reporters/datadog.js

@@ -1,2 +1,2 @@
-import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
-`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);if(Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked}),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(e.severity===`gate`?`gate_${e.name}`:e.name));for(let[e,i]of r)t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,assertion_passed:String(i.assertions.every(e=>e.passed)),assertion_count:String(i.assertions.length)}});return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(0,r-n)}export{Datadog};
+import{composeAssertionScoreMetadata,groupAssertionScores}from"#evals/runner/reporters/assertion-scores.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";import{resolveRuntimeTraceLinks}from"#evals/runner/reporters/datadog-runtime-trace-links.js";import{MISSING_DATASET_API_MESSAGE,getDatadogDataset,resolveDatadogClient,toDatadogJsonRecord,toDatadogJsonValue,toOptionalDatadogJsonValue}from"#evals/runner/reporters/datadog-client.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t;#n=new Map;#r=new Map;#i;#a;#o;#s;constructor(e){this.#e=e,this.#t={recordInputs:e.recordInputs??!0,recordOutputs:e.recordOutputs??!0,recordExpectedOutputs:e.recordExpectedOutputs??!0,recordAssertionDetails:e.recordAssertionDetails??!0,recordErrors:e.recordErrors??!0}}async onRunStart(e,t){this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0,this.#n.clear();for(let t of e)this.#n.set(t.id,t);if(this.#r=composeDatasetRecordKeys(e),!this.#e.client&&(!process.env.DD_API_KEY||!process.env.DD_APP_KEY)){(this.#e.log??console.log)(`Datadog reporting skipped: set DD_API_KEY and DD_APP_KEY to upload eval results.
+`);return}let n=resolveProjectName(this.#e,e),r=await resolveDatadogClient(this.#e,n),i=resolveLocalGitMetadata(process.cwd()),a=this.#e.experimentName??defaultExperimentName(),o=resolveExperimentMetadata(e,t);i.sha&&(o.eveGitCommit=i.sha,o.eveGitBranch=i.branch),Object.assign(o,this.#e.metadata);let s=this.#t.recordInputs?`${n} evals`:`${a} dataset`,c={name:a,projectName:n,description:this.#e.description,dataset:{name:this.#e.datasetName??s},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(o),config:toDatadogJsonRecord(this.#e.config)};if(this.#t.recordInputs){if(!r.createDataset||!r.pullDataset)throw Error(MISSING_DATASET_API_MESSAGE);this.#i=r,this.#a=c;return}this.#o=await r.startExperiment(c)}async onEvalComplete(e){!this.#t.recordInputs&&this.#o&&await this.#d(e)}async onRunComplete(e){try{if(this.#t.recordInputs&&await this.#c(e.results),!this.#o)return;let t=e.failed>0||e.errored>0;await this.#o.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#s&&n(`Datadog dataset URL: ${this.#s}\n`);let r=this.#o.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#i=void 0,this.#a=void 0,this.#o=void 0,this.#s=void 0}}async#c(e){let t=this.#i,n=this.#a;if(!t||!n)return;let r=n.dataset?.name??`${n.projectName} evals`,i=await getDatadogDataset(t,r,{projectName:n.projectName,description:this.#e.description??`eve eval inputs.`}),a=new Map;for(let[e,t]of i.records().entries()){let n=t.metadata[DATASET_RECORD_KEY_METADATA];typeof n==`string`&&a.set(n,e)}let o=[];for(let t of e){let e=this.#u(t),n=a.get(this.#l(t.id));if(n===void 0){o.push({inputData:e.input,metadata:e.metadata,...e.expectedOutput===null?{}:{expectedOutput:e.expectedOutput}});continue}let r=i.records()[n];r&&!isDatasetRecordEqual(r,e)&&i.update(n,e)}o.length>0&&i.addRecords(o),await i.push();let s=i.id();if(!s)throw Error(`Datadog dataset '${r}' has no id after push().`);let c=new Map;for(let e of i.records()){let t=e.metadata[DATASET_RECORD_KEY_METADATA];typeof t==`string`&&e.id&&c.set(t,e.id)}let l={id:s,name:i.name()},u=i.version()??i.latestVersion();u!==null&&(l.version=u),this.#s=i.url()??void 0,this.#o=await t.startExperiment({...n,dataset:l});for(let t of e){let e=this.#l(t.id),n=c.get(e);if(!n)throw Error(`Datadog dataset '${r}' has no record id for '${e}' after push().`);await this.#d(t,n)}}#l(e){return this.#r.get(e)??e}#u(e){let t=this.#n.get(e.id),n=this.#t.recordExpectedOutputs?toOptionalDatadogJsonValue(resolveExpectedOutput(t)):void 0;return{input:toDatadogJsonValue(resolveInput(e,t)),expectedOutput:n??null,metadata:{[DATASET_RECORD_KEY_METADATA]:this.#l(e.id),eveEvalId:e.id}}}async#d(e,t){if(!this.#o)return;let n=this.#n.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#t.recordAssertionDetails)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#t.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#t.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#t.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#t.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#o.submitSpan(r);await this.#o.submitEvaluationMetrics(i,resolveEvaluationMetrics(e))}};const EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`],DATASET_RECORD_KEY_METADATA=`eveRecordKey`;function composeDatasetRecordKeys(e){let t=new Map,n=new Map;for(let r of e){let e=r.description?.trim()||r.id,i=(n.get(e)??0)+1;n.set(e,i),t.set(r.id,i===1?e:`${e} #${i}`)}return t}function isDatasetRecordEqual(e,t){return JSON.stringify(e.input)===JSON.stringify(t.input)&&JSON.stringify(e.expectedOutput??null)===JSON.stringify(t.expectedOutput)&&JSON.stringify(e.metadata)===JSON.stringify(t.metadata)}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveSkillLoads:e.result.derived.skillLoads.map(e=>e.skill),eveParked:e.result.derived.parked});let i=resolveRuntimeTraceLinks(e.result.traceContexts);if(i.length>0&&(r.experimentRuntimeTraceLinks=i),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t),r.eveAssertionScores=composeAssertionScoreMetadata(e.assertions)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e){let t=[],n=new Set(BUILT_IN_METRIC_LABELS),r=groupAssertionScores(e.assertions,e=>toDatadogMetricLabel(e.severity===`gate`?`gate_${e.name}`:e.name));for(let[e,i]of r)t.push({label:reserveDatadogMetricLabel(e,n),value:i.score,tags:{assertion_severity:i.assertions[0]?.severity??`soft`,assertion_passed:String(i.assertions.every(e=>e.passed)),assertion_count:String(i.assertions.length)}});return t.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),t}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(1,r-n)}export{Datadog};

package/dist/src/evals/session.js

@@ -1,2 +1,2 @@
-import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
-`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};
+import{readFile}from"node:fs/promises";import{basename}from"node:path";import{TurnSegment,summarizeTurnEvents}from"#client/session-utils.js";import{isTurnFailureEvent}from"#protocol/message.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async compact(){if((await this.#e.compact()).status!==`accepted`)throw Error(`compact() found no active session for "${this.sessionId}".`);return await this.readTurn()}agent(e){let t=this.#e.agent(e),n=this.#r;return{name:t.name,sessionId:t.sessionId,stream:(e={})=>t.stream({...e,signal:e.signal??n}),taskId:t.taskId}}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
+`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1,i=new TurnSegment;for await(let n of e){this.#t.push(n);let e=i.observe(n);if(t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),e){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){assertReportedToolName(e,this.#e.tools);let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};

package/dist/src/evals/target.d.ts

@@ -7,6 +7,13 @@ export declare function resolveEvalTargetHandle(input: {
     readonly kind: "local" | "remote";
     readonly url: string;
 }): Promise<EveEvalTargetHandle>;
+/** The root agent's static tool names, and whether it has a dynamic tool resolver. */
+export interface TargetTools {
+    readonly dynamic: boolean;
+    readonly static: readonly string[];
+}
+/** The tools of a target resolved from `/eve/v1/info`; undefined for a hand-built handle. */
+export declare function targetTools(target: EveEvalTargetHandle): TargetTools | undefined;
 export declare function createEvalTargetHandle(input: {
     readonly capabilities: EveEvalTargetCapabilities;
     readonly client: Client;

package/dist/src/evals/target.js

@@ -1 +1 @@
-import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);return createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url})}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,async attachSession(r,i){if(e.sessions!==void 0)return await e.sessions.attachSession(r,i);if(t!==void 0)return await t.attachSession(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:n}).attachSession(r,i)},watchTurn(r,i){if(e.sessions!==void 0)return e.sessions.watchTurn(r,i);if(t!==void 0)return t.watchTurn(r,i);if(n===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:n}).watchTurn(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle};
+import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{stripNpmPackageScope}from"#shared/package-name.js";import{EvalSessionManager}from"#evals/session-manager.js";import{setTimeout}from"node:timers/promises";async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&!matchesExpectedAgentName(e.expectedAgentName,t.agent.name))throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);let n=createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url});return toolsByTarget.set(n,{dynamic:t.tools.dynamic.length>0,static:t.tools.static.map(e=>e.name)}),n}const toolsByTarget=new WeakMap;function targetTools(e){return toolsByTarget.get(e)}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,sessions:t.sessions,url:e.url})}function createHandle(e){let t=e.delegate,n=e.client,fetchTarget=async(e,r)=>{if(t!==void 0)return await t.fetch(e,r);if(n===void 0)throw Error(`Eval target cannot fetch without a client.`);return await n.fetch(e,r)};return{capabilities:e.capabilities,kind:e.kind,url:e.url,async attachSession(r,i){if(e.sessions!==void 0)return await e.sessions.attachSession(r,i);if(t!==void 0)return await t.attachSession(r,i);if(n===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:n}).attachSession(r,i)},watchTurn(r,i){if(e.sessions!==void 0)return e.sessions.watchTurn(r,i);if(t!==void 0)return t.watchTurn(r,i);if(n===void 0)throw Error(`Eval target cannot watch sessions without a client.`);return new EvalSessionManager({client:n}).watchTurn(r,i)},async dispatchSchedule(t){if(!e.capabilities.devRoutes)throw Error(`target.dispatchSchedule() requires a target with dev routes enabled.`);let n=await fetchTarget(createEveDevDispatchSchedulePath(t),{method:`POST`});if(!n.ok){let e=await readResponseBodySafely(n);throw Error(`Schedule dispatch failed: ${n.status} ${n.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await n.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`}}function matchesExpectedAgentName(e,t){return t===e||t===stripNpmPackageScope(e)}async function waitForTargetHealth(e,t){let n=Date.now()+6e4,r;for(;Date.now()<n;)try{await e.health();return}catch(e){r=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${t}.`+(r===void 0?``:` Last error: ${r}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==5)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle,targetTools};

package/dist/src/evals/types.d.ts

@@ -29,6 +29,21 @@ export interface EveEvalToolCall {
     /** Owning session id, when the runner knows it. */
     readonly sessionId?: string;
 }
+/**
+ * One skill load extracted from the captured stream, pairing the `load-skill`
+ * request with its matching `load-skill-result`.
+ */
+export interface EveEvalSkillLoad {
+    /** The loaded skill's name. */
+    readonly skill: string;
+    /** The skill's instructions; `undefined` when the load never resolved. */
+    readonly output: JsonValue | undefined;
+    readonly status: EveEvalActionStatus;
+    /** Zero-based index of the turn the load happened in. */
+    readonly turnIndex: number;
+    /** Owning session id, when the runner knows it. */
+    readonly sessionId?: string;
+}
 /**
  * One call to an agent task extracted from the captured stream: its
  * `task.started`, joined with its `task.settled` and the task's
@@ -58,6 +73,7 @@ export interface EveEvalSubagentCall {
 export interface EveEvalDerivedFacts {
     readonly toolCalls: readonly EveEvalToolCall[];
     readonly toolCallCount: number;
+    readonly skillLoads: readonly EveEvalSkillLoad[];
     readonly subagentCalls: readonly EveEvalSubagentCall[];
     readonly subagentCallCount: number;
     /** Every HITL input request raised during the run (`input.requested`). */
@@ -209,7 +225,7 @@ export interface EveEvalAssertions {
     parked(): AssertionHandle;
     messageIncludes(token: string | RegExp): AssertionHandle;
     calledTool(name: string, options?: EveEvalToolCallMatchOptions): AssertionHandle;
-    /** Sugar for `calledTool("load_skill", { input: { skill }, ... })`. */
+    /** Asserts a completed load of `skill`, constrained like `calledTool`. */
     loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): AssertionHandle;
     notCalledTool(name: string): AssertionHandle;
     /** Asserts that tool requests appeared in order, allowing unrelated requests between them. */

package/dist/src/execution/agent-sessions/remote.js

@@ -1 +1 @@
-import{z}from"#compiled/zod/index.js";import{decisionToTraceContentCeiling}from"#shared/forwarded-trace-policy.js";import{expectFunction,expectObjectRecord}from"#internal/authored-module.js";import{createEveCallbackRoutePath,createEveSessionCancelRoutePath,createEveSessionResetRoutePath,createEveSessionRoutePath}from"#protocol/routes.js";import{CancelTurnResponseSchema}from"#protocol/cancel-turn.js";import{ResetResponseSchema}from"#protocol/reset-session.js";import{formatTraceparent,writeAgentDispatchTracestate}from"#protocol/traceparent.js";import{writeForwardedAudienceBaggage,writeForwardedParentSessionBaggage}from"#protocol/baggage.js";import{writeConversationBaggage}from"#tracing/conversation-context.js";import{REMOTE_AGENT_PROTOCOL_MISMATCH,REMOTE_AGENT_PROTOCOL_VERSION,formatRemoteAgentProtocolMismatch,readRemoteAgentProtocolVersion}from"#protocol/remote-agent-protocol.js";import{FatalError}from"#compiled/@workflow/core/index.js";import{createWorkflowCallbackUrl}from"#execution/workflow-callback-url.js";import{createRemoteAgentRouteUrl}from"#subagents/remote-route-url.js";import{formatSubagentInput}from"#subagents/invocation.js";const CreateSessionResponseSchema=z.object({ok:z.literal(!0),protocolVersion:z.number().optional(),sessionId:z.string().min(1),status:z.literal(`accepted`)});async function startRemoteAgentSession(e){let t=e.parent?.continuationToken??e.session.continuationToken;if(!t)throw Error(`Cannot dispatch remote agent without a parent continuation token.`);if(!e.callbackBaseUrl)throw Error(`Cannot dispatch remote agent without a callback base URL.`);let n=buildForwardedPrincipalField(e),r={capabilities:e.capabilities??{},callback:{callId:e.action.callId,subagentName:e.action.remoteAgentName,token:t,url:createWorkflowCallbackUrl(e.callbackBaseUrl,createEveCallbackRoutePath(t))},message:formatRemoteAgentCallInputMessage({action:e.action,remote:e.remote}),outputSchema:e.action.input.outputSchema,protocolVersion:REMOTE_AGENT_PROTOCOL_VERSION};n!==void 0&&(r.forwardedPrincipal=n),e.operationId!==void 0&&(r.operationId=e.operationId);let i=await resolveRemoteAgentRequestHeaders(e.remote),a=formatTraceparent(e.parent?.traceContext);a!==void 0&&setHeader(i,`traceparent`,a),setHeader(i,`tracestate`,writeAgentDispatchTracestate(readHeader(i,`tracestate`),e.parent?.traceContext));let o=writeForwardedAudienceBaggage(readHeader(i,`baggage`),buildForwardedTraceAssertion({forwardedPrincipal:n,originAudience:e.originAudience,parentTraceContext:e.parent?.traceContext,traceparent:a})),s=writeConversationBaggage(o,e.parent?.conversationId);setHeader(i,`baggage`,writeForwardedParentSessionBaggage(s,e.parent?.lineage));let c=await fetch(createRemoteAgentSessionUrl(e.remote),{body:JSON.stringify(r),headers:{"content-type":`application/json`,...i},method:`POST`});if(!c.ok)throw await throwIfRemoteAgentProtocolMismatch(c,e.action.remoteAgentName),Error(`Remote agent "${e.action.remoteAgentName}" create-session request failed with HTTP ${c.status}.`);let l;try{l=await c.json()}catch{throw Error(`Remote agent "${e.action.remoteAgentName}" create-session response was not valid JSON.`)}let u=CreateSessionResponseSchema.safeParse(l);if(!u.success)throw Error(`Remote agent "${e.action.remoteAgentName}" create-session response was invalid.`);let d=readRemoteAgentProtocolVersion(u.data.protocolVersion);if(d!==REMOTE_AGENT_PROTOCOL_VERSION)throw new FatalError(formatRemoteAgentProtocolMismatch({name:e.action.remoteAgentName,receiverVersion:d}));return{sessionId:u.data.sessionId}}async function throwIfRemoteAgentProtocolMismatch(e,t){let n;try{n=await e.json()}catch{return}if(typeof n!=`object`||!n||Reflect.get(n,`code`)!==REMOTE_AGENT_PROTOCOL_MISMATCH)return;let r=readRemoteAgentProtocolVersion(Reflect.get(n,`protocolVersion`));throw new FatalError(formatRemoteAgentProtocolMismatch({name:t,receiverVersion:r}))}function buildForwardedTraceAssertion(e){let t=decisionToTraceContentCeiling(e.parentTraceContext?.decision);if(e.forwardedPrincipal!==void 0&&e.parentTraceContext!==void 0&&(e.parentTraceContext.traceFlags&1)==1&&e.traceparent!==void 0&&t!==void 0)return{ceiling:t,originAudience:e.originAudience??`unknown`}}async function respondToRemoteAgentSession(e){let t=await fetch(createRemoteAgentRouteUrl(e.remote.url,createEveSessionRoutePath(e.remote.sessionId)),{body:JSON.stringify({inputResponses:e.responses,...e.remote.forwardPrincipal===!0&&e.auth!=null&&{forwardedPrincipal:{current:e.auth}}}),headers:{"content-type":`application/json`,...e.headers},method:`POST`,redirect:`error`});if(!t.ok)throw Error(`Remote agent "${e.remote.name}" input answer failed with HTTP ${t.status}.`)}async function continueRemoteAgentSession(e){let t=buildForwardedPrincipalField(e),n={callback:e.callback,message:e.message,outputSchema:e.outputSchema};t!==void 0&&(n.forwardedPrincipal=t);let r=await fetch(createRemoteAgentContinueUrl(e.remote,e.sessionId),{body:JSON.stringify(n),headers:{"content-type":`application/json`,...await resolveRemoteAgentRequestHeaders(e.remote)},method:`POST`});if(!r.ok){let n=r.status===400&&t!==void 0?` The receiver may support forwarded principals only on session creation; upgrade it before retrying.`:``;throw Error(`Remote agent "${e.remote.name}" continue-session request failed with HTTP ${r.status}.${n}`)}}function buildForwardedPrincipalField(e){if(e.remote.forwardPrincipal!==!0||e.auth===null||e.auth===void 0)return;let t={current:e.auth};return e.initiatorAuth!==null&&e.initiatorAuth!==void 0&&(t.initiator=e.initiatorAuth),t}async function cancelRemoteAgentTurn(e){let t=await fetch(createRemoteAgentCancelTurnUrl(e.remote,e.sessionId),{headers:await resolveRemoteAgentRequestHeaders(e.remote),method:`POST`});if(!t.ok)throw Error(`Remote agent "${e.remote.name}" cancel-turn request failed with HTTP ${t.status}.`);let n;try{n=await t.json()}catch{throw Error(`Remote agent "${e.remote.name}" cancel-turn response was not valid JSON.`)}let r=CancelTurnResponseSchema.safeParse(n);if(!r.success||r.data.status===`accepted`&&r.data.sessionId!==e.sessionId)throw Error(`Remote agent "${e.remote.name}" cancel-turn response was invalid.`);return r.data.status===`accepted`?{sessionId:r.data.sessionId,status:`accepted`}:{status:`no_active_turn`}}function readHeader(e,t){let n=Object.keys(e).find(e=>e.toLowerCase()===t);return n===void 0?void 0:e[n]}function setHeader(e,t,n){for(let n of Object.keys(e))n.toLowerCase()===t&&delete e[n];n!==void 0&&(e[t]=n)}async function resetRemoteAgentSession(e){let t=await resolveRemoteAgentRequestHeaders(e.remote),n=await fetch(createRemoteAgentRouteUrl(e.remote.url,createEveSessionResetRoutePath(e.sessionId)),{body:JSON.stringify({reason:e.reason}),headers:{"content-type":`application/json`,...t},method:`POST`});if(!n.ok)throw Error(`Remote agent "${e.remote.name}" reset-session request failed with HTTP ${n.status}.`);let r=ResetResponseSchema.safeParse(await n.json());if(!r.success||r.data.status===`reset`&&r.data.previousSessionId!==e.sessionId)throw Error(`Remote agent "${e.remote.name}" reset-session response was invalid.`);return r.data}function resolveRemoteAgentForAction(e){let t=e.registry.get(e.nodeId)?.definition;if(e.dynamicRemoteAgent!==void 0){if(t===void 0)throw new FatalError(`Missing remote agent "${e.remoteAgentName}" in runtime registry.`);let n=resolveDynamicRemoteAgentCredentials(e.dynamicRemoteAgent),r=e.dynamicRemoteAgent,i={description:r.description,kind:`remote`,logicalPath:t.logicalPath,name:e.remoteAgentName,nodeId:e.nodeId,path:r.path,sourceId:t.sourceId,sourceKind:`module`,url:r.url};return r.forwardPrincipal!==void 0&&(i.forwardPrincipal=r.forwardPrincipal),n.auth!==void 0&&(i.auth=n.auth),n.headers!==void 0&&(i.headers=n.headers),i}if(t?.kind!==`remote`)throw new FatalError(`Missing remote agent "${e.remoteAgentName}" in runtime registry.`);return t}async function resolveRemoteAgentStreamHeaders(e){if(e.resolverId===void 0)return{};let t=new Set([e.bundle.graph.root,...e.bundle.graph.nodesByNodeId.values()]);for(let n of t){let t=n.subagentRegistry.subagentsByNodeId.get(e.resolverId)?.definition;if(t!==void 0){if(t.kind!==`remote`||t.name!==e.name||t.url!==e.url)throw Error(`Remote child stream resolver does not match the authored remote agent.`);return await resolveRemoteAgentRequestHeaders(t)}}return await resolveRemoteAgentRequestHeaders(resolveDynamicRemoteAgentCredentials({credentialsStepId:e.resolverId,description:``,path:``,url:e.url}))}function resolveDynamicRemoteAgentCredentials(e){if(e.credentialsStepId===void 0)return{};let t=getStepRegistry().get(e.credentialsStepId);if(t===void 0)throw Error(`Dynamic remote subagent credentials function "${e.credentialsStepId}" is not registered.`);let n=expectObjectRecord(t(),`Dynamic remote subagent credentials are invalid.`),r={};return n.auth!==void 0&&(r.auth=expectFunction(n.auth,`Dynamic remote subagent auth is invalid.`)),n.headers!==void 0&&(r.headers=resolveDynamicRemoteAgentHeaders(n.headers)),r}function resolveDynamicRemoteAgentHeaders(e){if(typeof e==`function`)return e;let t=expectObjectRecord(e,`Dynamic remote subagent headers are invalid.`);for(let e of Object.values(t))if(typeof e!=`string`)throw Error(`Dynamic remote subagent headers are invalid.`);return t}function getStepRegistry(){let e=Symbol.for(`@workflow/core//registeredSteps`),t=globalThis,n=t[e];return n===void 0&&(n=new Map,t[e]=n),n}function createRemoteAgentSessionUrl(e){return createRemoteAgentRouteUrl(e.url,e.path)}function createRemoteAgentCancelTurnUrl(e,t){return createRemoteAgentRouteUrl(e.url,createEveSessionCancelRoutePath(t))}function createRemoteAgentContinueUrl(e,t){return createRemoteAgentRouteUrl(e.url,createEveSessionRoutePath(t))}async function resolveRemoteAgentRequestHeaders(e){let t={};return e.headers!==void 0&&Object.assign(t,typeof e.headers==`function`?await e.headers():e.headers),e.auth!==void 0&&Object.assign(t,(await e.auth()).headers),t}function formatRemoteAgentCallInputMessage(e){let t=typeof e.action.input.message==`string`?e.action.input.message:``;return formatSubagentInput({description:e.remote.description,message:t,name:e.action.remoteAgentName,type:`remote`}).message}export{cancelRemoteAgentTurn,continueRemoteAgentSession,resetRemoteAgentSession,resolveRemoteAgentForAction,resolveRemoteAgentStreamHeaders,respondToRemoteAgentSession,startRemoteAgentSession};
+import{z}from"#compiled/zod/index.js";import{decisionToTraceContentCeiling}from"#shared/forwarded-trace-policy.js";import{expectFunction,expectObjectRecord}from"#internal/authored-module.js";import{createEveCallbackRoutePath,createEveSessionCancelRoutePath,createEveSessionResetRoutePath,createEveSessionRoutePath}from"#protocol/routes.js";import{CancelTurnResponseSchema}from"#protocol/cancel-turn.js";import{ResetResponseSchema}from"#protocol/reset-session.js";import{formatTraceparent,writeAgentDispatchTracestate}from"#protocol/traceparent.js";import{writeForwardedAudienceBaggage,writeForwardedParentSessionBaggage}from"#protocol/baggage.js";import{writeConversationBaggage}from"#tracing/conversation-context.js";import{REMOTE_AGENT_PROTOCOL_MISMATCH,REMOTE_AGENT_PROTOCOL_VERSION,UNVERSIONED_REMOTE_AGENT_PROTOCOL,formatRemoteAgentProtocolMismatch,readRemoteAgentProtocolVersion}from"#protocol/remote-agent-protocol.js";import{FatalError}from"#compiled/@workflow/core/index.js";import{createWorkflowCallbackUrl}from"#execution/workflow-callback-url.js";import{createRemoteAgentRouteUrl}from"#subagents/remote-route-url.js";import{formatSubagentInput}from"#subagents/invocation.js";const CreateSessionResponseSchema=z.object({ok:z.literal(!0),protocolVersion:z.number().optional(),sessionId:z.string().min(1),status:z.literal(`accepted`)}),UNKNOWN_REMOTE_OUTCOME=`The request may have completed on the remote agent. Do not retry automatically; check the remote outcome before sending the request again.`;async function startRemoteAgentSession(e){let t=e.parent?.continuationToken??e.session.continuationToken;if(!t)throw Error(`Cannot dispatch remote agent without a parent continuation token.`);if(!e.callbackBaseUrl)throw Error(`Cannot dispatch remote agent without a callback base URL.`);let n=buildForwardedPrincipalField(e),r={capabilities:e.capabilities??{},callback:{callId:e.action.callId,subagentName:e.action.remoteAgentName,token:t,url:createWorkflowCallbackUrl(e.callbackBaseUrl,createEveCallbackRoutePath(t))},message:formatRemoteAgentCallInputMessage({action:e.action,remote:e.remote}),outputSchema:e.action.input.outputSchema,protocolVersion:REMOTE_AGENT_PROTOCOL_VERSION};n!==void 0&&(r.forwardedPrincipal=n),e.operationId!==void 0&&(r.operationId=e.operationId);let i=await resolveRemoteAgentRequestHeaders(e.remote),a=formatTraceparent(e.parent?.traceContext);a!==void 0&&setHeader(i,`traceparent`,a),setHeader(i,`tracestate`,writeAgentDispatchTracestate(readHeader(i,`tracestate`),e.parent?.traceContext));let o=writeForwardedAudienceBaggage(readHeader(i,`baggage`),buildForwardedTraceAssertion({forwardedPrincipal:n,originAudience:e.originAudience,parentTraceContext:e.parent?.traceContext,traceparent:a})),s=writeConversationBaggage(o,e.parent?.conversationId);setHeader(i,`baggage`,writeForwardedParentSessionBaggage(s,e.parent?.lineage));let c=await fetch(createRemoteAgentSessionUrl(e.remote),{body:JSON.stringify(r),headers:{"content-type":`application/json`,...i},method:`POST`});if(!c.ok)throw await throwIfRemoteAgentProtocolMismatch(c,e.action.remoteAgentName),Error(`Remote agent "${e.action.remoteAgentName}" create-session request failed with HTTP ${c.status}.`);let l;try{l=await c.json()}catch{throw new FatalError(`Remote agent "${e.action.remoteAgentName}" create-session response was not valid JSON. ${UNKNOWN_REMOTE_OUTCOME}`)}let u=CreateSessionResponseSchema.safeParse(l);if(!u.success)throw new FatalError(`Remote agent "${e.action.remoteAgentName}" create-session response was invalid. ${UNKNOWN_REMOTE_OUTCOME}`);let d=readRemoteAgentProtocolVersion(u.data.protocolVersion);if(d!==REMOTE_AGENT_PROTOCOL_VERSION&&d!==UNVERSIONED_REMOTE_AGENT_PROTOCOL)throw new FatalError(`${formatRemoteAgentProtocolMismatch({name:e.action.remoteAgentName,receiverVersion:d})} ${UNKNOWN_REMOTE_OUTCOME} Remote session: ${u.data.sessionId}.`);return{sessionId:u.data.sessionId}}async function throwIfRemoteAgentProtocolMismatch(e,t){let n;try{n=await e.json()}catch{return}if(typeof n!=`object`||!n||Reflect.get(n,`code`)!==REMOTE_AGENT_PROTOCOL_MISMATCH)return;let r=readRemoteAgentProtocolVersion(Reflect.get(n,`protocolVersion`));throw new FatalError(formatRemoteAgentProtocolMismatch({name:t,receiverVersion:r}))}function buildForwardedTraceAssertion(e){let t=decisionToTraceContentCeiling(e.parentTraceContext?.decision);if(e.forwardedPrincipal!==void 0&&e.parentTraceContext!==void 0&&(e.parentTraceContext.traceFlags&1)==1&&e.traceparent!==void 0&&t!==void 0)return{ceiling:t,originAudience:e.originAudience??`unknown`}}async function respondToRemoteAgentSession(e){let t=await fetch(createRemoteAgentRouteUrl(e.remote.url,createEveSessionRoutePath(e.remote.sessionId)),{body:JSON.stringify({inputResponses:e.responses,...e.remote.forwardPrincipal===!0&&e.auth!=null&&{forwardedPrincipal:{current:e.auth}}}),headers:{"content-type":`application/json`,...e.headers},method:`POST`,redirect:`error`});if(!t.ok)throw Error(`Remote agent "${e.remote.name}" input answer failed with HTTP ${t.status}.`)}async function continueRemoteAgentSession(e){let t=buildForwardedPrincipalField(e),n={callback:e.callback,message:e.message,outputSchema:e.outputSchema};t!==void 0&&(n.forwardedPrincipal=t);let r=await fetch(createRemoteAgentContinueUrl(e.remote,e.sessionId),{body:JSON.stringify(n),headers:{"content-type":`application/json`,...await resolveRemoteAgentRequestHeaders(e.remote)},method:`POST`});if(!r.ok){let n=r.status===400&&t!==void 0?` The receiver may support forwarded principals only on session creation; upgrade it before retrying.`:``;throw Error(`Remote agent "${e.remote.name}" continue-session request failed with HTTP ${r.status}.${n}`)}}function buildForwardedPrincipalField(e){if(e.remote.forwardPrincipal!==!0||e.auth===null||e.auth===void 0)return;let t={current:e.auth};return e.initiatorAuth!==null&&e.initiatorAuth!==void 0&&(t.initiator=e.initiatorAuth),t}async function cancelRemoteAgentTurn(e){let t=await fetch(createRemoteAgentCancelTurnUrl(e.remote,e.sessionId),{headers:await resolveRemoteAgentRequestHeaders(e.remote),method:`POST`});if(!t.ok)throw Error(`Remote agent "${e.remote.name}" cancel-turn request failed with HTTP ${t.status}.`);let n;try{n=await t.json()}catch{throw Error(`Remote agent "${e.remote.name}" cancel-turn response was not valid JSON.`)}let r=CancelTurnResponseSchema.safeParse(n);if(!r.success||r.data.status===`accepted`&&r.data.sessionId!==e.sessionId)throw Error(`Remote agent "${e.remote.name}" cancel-turn response was invalid.`);return r.data.status===`accepted`?{sessionId:r.data.sessionId,status:`accepted`}:{status:`no_active_turn`}}function readHeader(e,t){let n=Object.keys(e).find(e=>e.toLowerCase()===t);return n===void 0?void 0:e[n]}function setHeader(e,t,n){for(let n of Object.keys(e))n.toLowerCase()===t&&delete e[n];n!==void 0&&(e[t]=n)}async function resetRemoteAgentSession(e){let t=await resolveRemoteAgentRequestHeaders(e.remote),n=await fetch(createRemoteAgentRouteUrl(e.remote.url,createEveSessionResetRoutePath(e.sessionId)),{body:JSON.stringify({reason:e.reason}),headers:{"content-type":`application/json`,...t},method:`POST`});if(!n.ok)throw Error(`Remote agent "${e.remote.name}" reset-session request failed with HTTP ${n.status}.`);let r=ResetResponseSchema.safeParse(await n.json());if(!r.success||r.data.status===`reset`&&r.data.previousSessionId!==e.sessionId)throw Error(`Remote agent "${e.remote.name}" reset-session response was invalid.`);return r.data}function resolveRemoteAgentForAction(e){let t=e.registry.get(e.nodeId)?.definition;if(e.dynamicRemoteAgent!==void 0){if(t===void 0)throw new FatalError(`Missing remote agent "${e.remoteAgentName}" in runtime registry.`);let n=resolveDynamicRemoteAgentCredentials(e.dynamicRemoteAgent),r=e.dynamicRemoteAgent,i={description:r.description,kind:`remote`,logicalPath:t.logicalPath,name:e.remoteAgentName,nodeId:e.nodeId,path:r.path,sourceId:t.sourceId,sourceKind:`module`,url:r.url};return r.forwardPrincipal!==void 0&&(i.forwardPrincipal=r.forwardPrincipal),n.auth!==void 0&&(i.auth=n.auth),n.headers!==void 0&&(i.headers=n.headers),i}if(t?.kind!==`remote`)throw new FatalError(`Missing remote agent "${e.remoteAgentName}" in runtime registry.`);return t}async function resolveRemoteAgentStreamHeaders(e){if(e.resolverId===void 0)return{};let t=new Set([e.bundle.graph.root,...e.bundle.graph.nodesByNodeId.values()]);for(let n of t){let t=n.subagentRegistry.subagentsByNodeId.get(e.resolverId)?.definition;if(t!==void 0){if(t.kind!==`remote`||t.name!==e.name||t.url!==e.url)throw Error(`Remote child stream resolver does not match the authored remote agent.`);return await resolveRemoteAgentRequestHeaders(t)}}return await resolveRemoteAgentRequestHeaders(resolveDynamicRemoteAgentCredentials({credentialsStepId:e.resolverId,description:``,path:``,url:e.url}))}function resolveDynamicRemoteAgentCredentials(e){if(e.credentialsStepId===void 0)return{};let t=getStepRegistry().get(e.credentialsStepId);if(t===void 0)throw Error(`Dynamic remote subagent credentials function "${e.credentialsStepId}" is not registered.`);let n=expectObjectRecord(t(),`Dynamic remote subagent credentials are invalid.`),r={};return n.auth!==void 0&&(r.auth=expectFunction(n.auth,`Dynamic remote subagent auth is invalid.`)),n.headers!==void 0&&(r.headers=resolveDynamicRemoteAgentHeaders(n.headers)),r}function resolveDynamicRemoteAgentHeaders(e){if(typeof e==`function`)return e;let t=expectObjectRecord(e,`Dynamic remote subagent headers are invalid.`);for(let e of Object.values(t))if(typeof e!=`string`)throw Error(`Dynamic remote subagent headers are invalid.`);return t}function getStepRegistry(){let e=Symbol.for(`@workflow/core//registeredSteps`),t=globalThis,n=t[e];return n===void 0&&(n=new Map,t[e]=n),n}function createRemoteAgentSessionUrl(e){return createRemoteAgentRouteUrl(e.url,e.path)}function createRemoteAgentCancelTurnUrl(e,t){return createRemoteAgentRouteUrl(e.url,createEveSessionCancelRoutePath(t))}function createRemoteAgentContinueUrl(e,t){return createRemoteAgentRouteUrl(e.url,createEveSessionRoutePath(t))}async function resolveRemoteAgentRequestHeaders(e){let t={};return e.headers!==void 0&&Object.assign(t,typeof e.headers==`function`?await e.headers():e.headers),e.auth!==void 0&&Object.assign(t,(await e.auth()).headers),t}function formatRemoteAgentCallInputMessage(e){let t=typeof e.action.input.message==`string`?e.action.input.message:``;return formatSubagentInput({description:e.remote.description,message:t,name:e.action.remoteAgentName,type:`remote`}).message}export{cancelRemoteAgentTurn,continueRemoteAgentSession,resetRemoteAgentSession,resolveRemoteAgentForAction,resolveRemoteAgentStreamHeaders,respondToRemoteAgentSession,startRemoteAgentSession};

package/dist/src/execution/catalog/connection-auth.d.ts

@@ -0,0 +1,41 @@
+/** Sign-in shared by connection tool calls, searches, and connecting a connection. */
+import type { ConnectionRegistry } from "#runtime/connections/registry-types.js";
+import { createAuthorizationExecution, type ScopedAuthorization } from "#runtime/connections/scoped-authorization.js";
+import type { ResolvedConnectionDefinition } from "#runtime/types.js";
+import { type ConnectionClient, type ConnectionToolMetadata } from "#shared/connection-types.js";
+/** A connection ready for one call, with any sign-in the user finished completed. */
+export interface ConnectionSession {
+    /** Completes and starts sign-in for this call, and guards against a sign-in loop. */
+    readonly auth: ReturnType<typeof createAuthorizationExecution>;
+    readonly client: ConnectionClient;
+    /** The connection's interactive sign-in, when it supports one. */
+    readonly scoped: ScopedAuthorization | undefined;
+}
+/**
+ * Completes the sign-in the user finished for `connection`, if any, before a
+ * search, a sign-in, or a tool call uses its client. It never prompts; a caller
+ * that needs sign-in starts it through `auth`.
+ */
+export declare function completeConnectionSignIn(registry: ConnectionRegistry, connection: ResolvedConnectionDefinition): Promise<ConnectionSession>;
+/** A connection's tools, or why they can't be listed. */
+export type ConnectionListing = {
+    readonly tools: readonly ConnectionToolMetadata[];
+}
+/** Listable once the user signs in; `error` is the listing's request for sign-in. */
+ | {
+    readonly authorization: ScopedAuthorization;
+    readonly error: unknown;
+} | {
+    readonly failure: string;
+};
+export declare function listConnectionTools(connection: ResolvedConnectionDefinition, { auth, client, scoped }: ConnectionSession): Promise<ConnectionListing>;
+/**
+ * A connection's tools, or a timeout error once listing them takes longer than
+ * {@link LISTING_TIMEOUT_MS}, so one hung server can't stall a step. Clients
+ * share one in-flight listing and take no signal, so this stops waiting rather
+ * than cancelling: a late result fills only that client's cache, and the race
+ * handles a late rejection.
+ */
+export declare function listToolMetadata(connectionName: string, client: ConnectionClient): Promise<readonly ConnectionToolMetadata[]>;
+/** Why a connection's tools could not be listed, for a failure other than a needed sign-in. */
+export declare function listingFailureMessage(connectionName: string, error: unknown): string;

package/dist/src/execution/catalog/connection-auth.js

@@ -0,0 +1 @@
+import{createLogger}from"#internal/logging.js";import{toErrorMessage}from"#shared/errors.js";import{CALL_TOOL_NAME}from"#protocol/catalog-tools.js";import{isConnectionAuthorizationFailedError,isConnectionAuthorizationRequiredError}from"#connections/errors.js";import{getAuthorizationResults}from"#harness/authorization.js";import{resolveConnectionAuthorization}from"#runtime/connections/resolve-authorization.js";import{createAuthorizationExecution}from"#runtime/connections/scoped-authorization.js";import{supportsInteractiveAuthorization}from"#shared/connection-types.js";const log=createLogger(`framework.catalog-connections`);async function completeConnectionSignIn(e,t){assertPendingSignInInstance(t);let n=await interactiveAuthorization(t),r=createAuthorizationExecution();n!==void 0&&await r.complete(n);let i=e.getClient(t.connectionName);return n!==void 0&&r.isJustAuthorized(n)&&await i.close(),{auth:r,client:i,scoped:n}}async function listConnectionTools(e,{auth:t,client:n,scoped:r}){let i=e.connectionName;try{return{tools:await listToolMetadata(i,n)}}catch(e){return isConnectionAuthorizationRequiredError(e)?r===void 0?{failure:`"${i}" requires authorization and cannot start interactive sign-in.`}:t.isJustAuthorized(r)?{failure:`Authorization failed for "${i}": the service rejected the token immediately after authorization.`}:{authorization:r,error:e}:(log.warn(`failed to load connection tools`,{connection:i,error:e}),{failure:listingFailureMessage(i,e)})}}const LISTING_TIMEOUT_MS=1e4;var ListingTimeoutError=class extends Error{};async function listToolMetadata(e,t){let n,r=new Promise((t,r)=>{n=setTimeout(()=>{r(new ListingTimeoutError(`"${e}" did not list its tools within ${LISTING_TIMEOUT_MS/1e3}s. Try again later.`))},LISTING_TIMEOUT_MS)});try{return await Promise.race([t.getToolMetadata(),r])}finally{clearTimeout(n)}}function listingFailureMessage(e,t){return t instanceof ListingTimeoutError?t.message:isConnectionAuthorizationFailedError(t)?`Authorization failed for "${e}": ${t.message}`:`Failed to load tools for "${e}": ${toErrorMessage(t)}`}async function interactiveAuthorization(e){let t=await resolveConnectionAuthorization(e);if(t!==void 0&&supportsInteractiveAuthorization(t))return{scope:e.connectionName,instanceId:e.instanceId,connection:{url:e.url??``},authorization:t}}function assertPendingSignInInstance(e){for(let t of getAuthorizationResults())if(t.name===e.connectionName&&t.instanceId!==void 0&&t.instanceId!==e.instanceId)throw Error(`Authorization for "${t.name}" cannot complete because its resolved connection changed while sign-in was pending. Sign in again with ${CALL_TOOL_NAME}({ name: "${t.name}" }).`)}export{completeConnectionSignIn,listConnectionTools,listToolMetadata,listingFailureMessage};

package/dist/src/execution/catalog/connection-entry.d.ts

@@ -0,0 +1,16 @@
+/**
+ * A connection's catalog entries. `<connection>__<tool>` calls one of its
+ * tools: it applies the connection's approval under the entry's name and runs
+ * an approved call only against the connection instance it was approved for.
+ * The connection's own name signs the user in, so its tools become listable.
+ */
+import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import type { ConnectionRegistry } from "#runtime/connections/registry-types.js";
+import type { ResolvedConnectionDefinition } from "#runtime/types.js";
+/** The entry for one tool of one connection instance in `registry`. */
+export declare function connectionEntry(registry: ConnectionRegistry, connection: ResolvedConnectionDefinition, toolName: string): HarnessToolDefinition;
+/**
+ * The entry for a connection's own name: it signs the user in when listing the
+ * connection's tools needs that, then points the model at `eve__search`.
+ */
+export declare function connectionSignInEntry(registry: ConnectionRegistry, connection: ResolvedConnectionDefinition): HarnessToolDefinition;

package/dist/src/execution/catalog/connection-entry.js

@@ -0,0 +1 @@
+import{completeConnectionSignIn,listConnectionTools,listToolMetadata,listingFailureMessage}from"./connection-auth.js";import{closestNames}from"./rank.js";import{connectionToolSignature}from"./signatures.js";import{isObject}from"#shared/guards.js";import{loadContext}from"#context/container.js";import{displayProperName,displayTitle}from"#shared/display-name.js";import{SEARCH_TOOL_NAME}from"#protocol/catalog-tools.js";import{defineJsonSchema,refineJsonSchema}from"#tools/schema.js";import{connectionToolName}from"#connections/ownership.js";import{ContextKey}from"#context/key.js";import{resolveApprovalPolicy}from"#approval/definition.js";import{isApprovalRecheck}from"#harness/approval-recheck.js";import{isConnectionAuthorizationRequiredError}from"#connections/errors.js";import{connectionToolModelOutput,toConnectionToolResult}from"#runtime/connections/tool-result.js";const CONNECTION_CHANGED_MESSAGE=`The connection for this tool call changed or is unavailable. Request a new tool call and approval.`,ConnectionApprovalPinsKey=new ContextKey(`eve.connectionApprovalPins`);function instanceKey(e){return e.instanceId??``}function connectionEntry(e,t,n){let r=`${displayProperName(t.connectionName)}: ${displayTitle(n)}`;return{approval:connectionApproval(t),deferred:!0,description:``,execute:(r,i)=>callConnectionTool(e,t,n,r,i),inputSchema:refineJsonSchema({},r=>checkInput(e,t,n,r)),label:{start:()=>r},name:connectionToolName(t.connectionName,n),toModelOutput:connectionToolModelOutput}}function connectionSignInEntry(e,t){let n=displayProperName(t.connectionName),r=`Sign in to use the ${n} tools`;return{deferred:!0,description:t.description===``?`${r}.`:`${r}: ${t.description}`,execute:()=>connect(e,t),inputSchema:defineJsonSchema({type:`object`,properties:{},additionalProperties:!1}),label:{start:()=>`Sign in to ${n}`},name:t.connectionName}}async function connect(e,t){let n=await completeConnectionSignIn(e,t),r=await listConnectionTools(t,n);if(`failure`in r)throw Error(r.failure);if(`authorization`in r)return await n.auth.handleError(r.error,r.authorization);let i=displayProperName(t.connectionName),a=`${SEARCH_TOOL_NAME}({ query: "${t.connectionName}__" })`;return n.scoped!==void 0&&n.auth.isJustAuthorized(n.scoped)?`Signed in to ${i}. Find the ${i} tools with ${a}.`:`The ${i} tools are available. Find them with ${a}.`}async function callConnectionTool(e,t,n,r,i){releaseApprovalPin(i.toolCallId,t);let a=await completeConnectionSignIn(e,t),{auth:o,client:s,scoped:c}=a,l=await listConnectionTools(t,a);if(`failure`in l)throw Error(l.failure);if(`authorization`in l)return await o.handleError(l.error,l.authorization);let u=await checkCall(t,l.tools,n,r);if(`issues`in u)throw Error(issuesMessage(u.issues));let{input:d,tool:f}=u,p;try{p=await s.executeTool(f.name,d,{abortSignal:i.abortSignal,callId:i.toolCallId})}catch(e){if(isConnectionAuthorizationRequiredError(e))return await o.handleError(e,c);throw e}let m=toConnectionToolResult(t.protocol,f,p);if(!m.ok)throw Error(m.error);return m.value}async function checkInput(e,t,n,r){let i;try{i=await listToolMetadata(t.connectionName,e.getClient(t.connectionName))}catch(e){return isConnectionAuthorizationRequiredError(e)?{value:r}:{issues:[{message:listingFailureMessage(t.connectionName,e)}]}}let a=await checkCall(t,i,n,r);return`issues`in a?{issues:a.issues}:{value:a.input}}async function checkCall(e,t,n,r){let i=t.find(e=>e.name===n);if(i===void 0)return{issues:[{message:unknownToolMessage(e,n,t)}]};let a=await defineJsonSchema(i.inputSchema)[`~standard`].validate(isObject(r)?r:{});return a.issues===void 0?{input:a.value,tool:i}:{issues:[...a.issues,{message:`Signature: ${connectionToolSignature(e,i)}`}]}}function issuesMessage(e){return e.map(({message:e,path:t=[]})=>{let n=t.map(e=>String(typeof e==`object`?e.key:e)).join(`.`);return n.length>0?`${n}: ${e}`:e}).join(` `)}function unknownToolMessage(e,t,n){let{connectionName:r}=e,i=closestNames(t,n).map(e=>connectionToolName(r,e));return`Connection "${r}" has no tool named "${t}".${i.length>0?` Closest tools: ${i.join(`, `)}.`:` Find its tools with ${SEARCH_TOOL_NAME}({ query: "${r}__" }).`}`}function connectionApproval(e){let{approval:t}=e;if(t===void 0)return;let n=resolveApprovalPolicy(t),request=t=>requestApproval(e,n,t),r=typeof t==`function`?void 0:t.response;return r===void 0?request:{request,response:t=>authorizeApprovalResponse(e,r,t)}}async function requestApproval(e,t,n){isApprovalRecheck(n)&&!matchesApprovalPin(n.callId,e)&&markApprovalPinChanged(n.callId);let r=await t(n);return!isApprovalRecheck(n)&&parksForApproval(r)&&pinApprovedInstance(n.callId,e),r}function parksForApproval(e){return e===!0||e===`user-approval`||typeof e==`object`&&e.type===`user-approval`}async function authorizeApprovalResponse(e,t,n){return matchesApprovalPin(n.request.callId,e)?await t(n):{reason:CONNECTION_CHANGED_MESSAGE,status:`rejected`}}function pinApprovedInstance(e,t){let n=loadContext();n.get(ConnectionApprovalPinsKey)?.[e]===void 0&&n.set(ConnectionApprovalPinsKey,(n={})=>Object.fromEntries([...Object.entries(n).slice(-49),[e,instanceKey(t)]]))}function matchesApprovalPin(e,t){let n=loadContext().get(ConnectionApprovalPinsKey)?.[e];return n!==void 0&&n===instanceKey(t)}function markApprovalPinChanged(e){loadContext().set(ConnectionApprovalPinsKey,(t={})=>({...t,[e]:`\0changed`}))}function releaseApprovalPin(e,t){let n=loadContext(),r=n.get(ConnectionApprovalPinsKey);if(r?.[e]===void 0)return;let{[e]:i,...a}=r;if(n.set(ConnectionApprovalPinsKey,a),i!==instanceKey(t))throw Error(CONNECTION_CHANGED_MESSAGE)}export{connectionEntry,connectionSignInEntry};

package/dist/src/execution/catalog/listing.d.ts

@@ -0,0 +1,9 @@
+import type { Announcement } from "#harness/announcements.js";
+import type { StepCatalog } from "./step-catalog.js";
+/**
+ * Announces the catalog: the kinds of deferred entries, their namespaces, and
+ * the connections, then the whole listing again whenever that changes.
+ * Connection tools stay unlisted, since listing them needs a network call and
+ * maybe a sign-in. `announced` holds the values announced so far.
+ */
+export declare function catalogAnnouncements(catalog: StepCatalog, announced: Readonly<Record<string, string>> | undefined): Readonly<Record<string, Announcement>>;

package/dist/src/execution/catalog/listing.js

@@ -0,0 +1,3 @@
+import{compareCodeUnits}from"./rank.js";import{CALL_TOOL_NAME,SEARCH_TOOL_NAME,SKILL_TOOL_NAME}from"#protocol/catalog-tools.js";import{isAgentTool}from"#execution/tasks/tool-entry-point.js";const ANNOUNCEMENT_KEY=`catalog`;function catalogAnnouncements(e,t){let n=listCatalog(e);if(t?.[ANNOUNCEMENT_KEY]===void 0&&isEmpty(n))return{};let r=reachableNames(e);return{[ANNOUNCEMENT_KEY]:{value:JSON.stringify(n),render:e=>renderCatalogAnnouncement(parseListing(e),n,r)}}}function listCatalog(e){let t=[...e.deferred.values()],n=[];t.some(e=>!isAgentTool(e))&&n.push(`tools`),t.some(isAgentTool)&&n.push(`agents`),[...e.skills.values()].some(({deferred:e})=>e)&&n.push(`skills`);let r=capped([...namespaceCounts(e)].sort(([e,t],[n,r])=>r-t||compareCodeUnits(e,n)).map(([e])=>e),20);return{connections:capped(e.connections.map(({connectionName:e,description:t})=>({description:t,name:e})).sort((e,t)=>compareCodeUnits(e.name,t.name)),20),kinds:n,namespaces:{...r,items:[...r.items].sort(compareCodeUnits)}}}function namespaceCounts(e){let t=new Set(e.connections.map(({connectionName:e})=>e)),n=new Map;for(let r of deferredNames(e)){let e=namespaceOf(r);e!==``&&!t.has(e)&&n.set(e,(n.get(e)??0)+1)}return n}function reachableNames(e){return new Set([...[...e.entries.keys(),...e.skills.keys()].map(namespaceOf),...e.connections.map(({connectionName:e})=>e)])}function namespaceOf(e){return e.slice(0,Math.max(0,e.indexOf(`__`)))}function deferredNames(e){return[...[...e.deferred.values()].map(({name:e})=>e),...[...e.skills.values()].filter(({deferred:e})=>e).map(({name:e})=>e)]}function capped(e,t){return{items:e.slice(0,t),more:e.length>t}}function renderCatalogAnnouncement(e,t,n){if(isEmpty(t))return`The catalog changed. It is empty now: ${SEARCH_TOOL_NAME} finds nothing.`;if(e===void 0||isEmpty(e))return renderListing(t);let r=[...e.namespaces.items,...e.connections.items.map(({name:e})=>e)].filter(e=>!n.has(e));return[`The catalog changed.`,renderListing(t),...r.length>0?[`No longer available: ${r.join(`, `)}`]:[]].join(`
+`)}function renderListing({connections:e,kinds:t,namespaces:n}){let r=t.length===0?`Your connections have more tools`:`You have more ${joinWords(t)}`,i=t.includes(`skills`)?t.length===1&&e.items.length===0?`Load them with ${SKILL_TOOL_NAME}({ name }).`:`Call tools with ${CALL_TOOL_NAME}({ name, input }) and load skills with ${SKILL_TOOL_NAME}({ name }).`:`Call them with ${CALL_TOOL_NAME}({ name, input }).`,a=[`${r} than are loaded here. Before saying you have no tool for a task, look for one with ${SEARCH_TOOL_NAME}, which searches your own catalog, not the web. ${i}`];if(n.items.length>0){let e=[...n.items,...n.more?[`and more`]:[]];a.push(`Namespaces, whose entries are named <namespace>__<name>; search one with "<namespace>__": ${e.join(`, `)}`)}return e.items.length>0&&a.push(`Connections, whose tools are named <connection>__<tool>; search one connection's tools with "<connection>__":`,...e.items.map(formatConnection),...e.more?[`- and more`]:[]),a.join(`
+`)}function joinWords(e){return e.length<=2?e.join(` and `):`${e.slice(0,-1).join(`, `)}, and ${e.at(-1)}`}function formatConnection(e){return e.description.length===0?`- ${e.name}`:`- ${e.name}: ${e.description}`}function isEmpty(e){return e.kinds.length===0&&e.connections.items.length===0}function parseListing(e){if(e!==void 0)try{return JSON.parse(e)}catch{return}}export{catalogAnnouncements};

package/dist/src/execution/catalog/rank.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Ranking for `eve__search` and for `eve__tool`'s and `eve__skill`'s suggestions when a
+ * name is unknown.
+ * Query and text split into lowercase words (camelCase, acronyms, and
+ * punctuation are boundaries), so `create issue`, `createIssue`, and
+ * `create_issue` are the same words. Matches rank in tiers:
+ *
+ * 1. Exact: the query's words are the full name's words.
+ * 2. Exact without namespace: they are the name after its last `__`, so
+ *    `create_issue` finds `linear__create_issue` and `sre__create_issue`.
+ * 3. Connection: they are the connection's name, or its first whole words,
+ *    so `linear` lists the `linear` connection's tools, while `git` leaves
+ *    `github`'s tools to the name prefix tier.
+ * 4. Name prefix: the full name's words, or those after its last `__`, start
+ *    with the query's words, the last of which may be partial, so
+ *    `create_iss` finds `create_issue` and `status page incidents` finds
+ *    `sre__status_page_incidents_list`.
+ * 5. Keyword: any other match. A query word matches a word when either is a
+ *    prefix of the other, so `repo` finds `repositories` and `issues` finds
+ *    `issue`, and each query word adds the weight of every field it matches.
+ *    One-letter and filler words such as `a` and `the` don't score when the
+ *    query has other words, so `a` doesn't find `assign`.
+ *
+ * Within a tier, the keyword score orders matches, then connection, then name.
+ */
+/** What the ranker reads from one catalog entry. */
+export interface RankCandidate {
+    /** The entry's own name; a connection tool's name without its connection prefix. */
+    readonly name: string;
+    readonly description: string;
+    readonly inputSchema?: Readonly<Record<string, unknown>>;
+    readonly connection?: {
+        readonly description: string;
+        readonly name: string;
+    };
+}
+/** Candidates matching `query`, best first. */
+export declare function rankCandidates<T extends RankCandidate>(query: string, candidates: readonly T[]): T[];
+/** Whether `query` has no searchable words, such as `"^"` or `"  "`. */
+export declare function isEmptyQuery(query: string): boolean;
+/** Names of the candidates closest to an unknown `name`, best first. */
+export declare function closestNames(name: string, candidates: readonly RankCandidate[]): string[];
+/** Orders by UTF-16 code unit, the same in every locale. */
+export declare function compareCodeUnits(a: string, b: string): number;

package/dist/src/execution/catalog/rank.js

@@ -0,0 +1 @@
+import{isObject}from"#shared/guards.js";import{connectionToolName}from"#connections/ownership.js";function rankCandidates(e,t){let n=tokenize(e),r=scoringTerms(n);return t.map(e=>({candidate:e,...rankCandidate(n,r,e)})).filter(e=>e.score>0).sort((e,t)=>e.tier-t.tier||t.score-e.score||compareCodeUnits(e.candidate.connection?.name??``,t.candidate.connection?.name??``)||compareCodeUnits(e.candidate.name,t.candidate.name)).map(e=>e.candidate)}function isEmptyQuery(e){return tokenize(e).length===0}function closestNames(e,t){return isEmptyQuery(e)?[]:rankCandidates(e,t).slice(0,5).map(e=>e.name)}const Tier={exact:0,exactWithoutNamespace:1,connection:2,namePrefix:3,keyword:4};function rankCandidate(e,t,n){let r=n.connection===void 0?n.name:connectionToolName(n.connection.name,n.name),i=tokenize(r),a=r.lastIndexOf(`__`),o=tokenize(a<0?r:r.slice(a+2)),s=sameWords(i,e)?Tier.exact:sameWords(o,e)?Tier.exactWithoutNamespace:n.connection!==void 0&&startsWithWholeWords(tokenize(n.connection.name),e)?Tier.connection:startsWithWords(i,e)||startsWithWords(o,e)?Tier.namePrefix:Tier.keyword;return{score:scoreFields(t,candidateFields(n)),tier:s}}const FILLER_WORDS=new Set([`a`,`an`,`and`,`for`,`in`,`is`,`me`,`my`,`of`,`or`,`please`,`the`,`to`,`with`]);function scoringTerms(e){let t=e.filter(e=>e.length>1&&!FILLER_WORDS.has(e));return t.length>0?t:e}function sameWords(e,t){return e.length===t.length&&startsWithWholeWords(e,t)}function startsWithWholeWords(e,t){return t.length<=e.length&&t.every((t,n)=>e[n]===t)}function startsWithWords(e,t){return t.length<=e.length&&t.every((n,r)=>r===t.length-1?e[r].startsWith(n):e[r]===n)}function candidateFields(e){let t=isObject(e.inputSchema?.properties)?Object.entries(e.inputSchema.properties):[],n=t.flatMap(([,e])=>isObject(e)&&typeof e.description==`string`?[e.description]:[]);return[[tokenize(e.name),6],[tokenize(e.connection?.name??``),4],[t.flatMap(([e])=>tokenize(e)),3],[proseWords(e.description),2],[n.flatMap(proseWords),1],[proseWords(e.connection?.description??``),1]]}function scoreFields(e,t){let n=0;for(let r of e)for(let[e,i]of t)e.some(e=>matchesWord(e,r))&&(n+=i);return n}function matchesWord(e,t){return e.startsWith(t)||e.length>=3&&t.startsWith(e)}function tokenize(e){return e.replaceAll(/([A-Z]+)([A-Z][a-z])/gu,`$1 $2`).replaceAll(/([a-z0-9])([A-Z])/gu,`$1 $2`).toLowerCase().split(/[^a-z0-9]+/u).filter(e=>e!==``)}function proseWords(e){return tokenize(e).filter(e=>e.length>1)}function compareCodeUnits(e,t){return e<t?-1:+(e>t)}export{closestNames,compareCodeUnits,isEmptyQuery,rankCandidates};