experimental-ash 0.26.0 → 0.27.0

package/dist/src/evals/loaders/json.js

@@ -1 +1 @@
-import{isAbsolute as e,resolve as t}from"node:path";import{readFile as n}from"node:fs/promises";async function r(r){let i=await n(e(r)?r:t(process.cwd(),r),`utf-8`);return JSON.parse(i)}export{r as loadJson};
+import{isAbsolute,resolve}from"node:path";import{readFile}from"node:fs/promises";async function loadJson(e){let t=await readFile(isAbsolute(e)?e:resolve(process.cwd(),e),`utf-8`);return JSON.parse(t)}export{loadJson};

package/dist/src/evals/loaders/yaml.js

@@ -1 +1 @@
-import{i as e}from"../../chunks/chunk-DSjMdhoD.js";import{t}from"../../chunks/gray-matter-D-9jHwOT.js";import{isAbsolute as n,resolve as r}from"node:path";import{readFile as i}from"node:fs/promises";var a=e(t(),1);async function o(e){let t=await i(n(e)?e:r(process.cwd(),e),`utf-8`);return(0,a.default)(t.trimStart().startsWith(`---`)?t:`---\n${t}\n---`).data??{}}export{o as loadYaml};
+import{isAbsolute,resolve}from"node:path";import{readFile}from"node:fs/promises";import matter from"#compiled/gray-matter/index.js";async function loadYaml(e){let t=await readFile(isAbsolute(e)?e:resolve(process.cwd(),e),`utf-8`);return matter(t.trimStart().startsWith(`---`)?t:`---\n${t}\n---`).data??{}}export{loadYaml};

package/dist/src/evals/reporters/index.js

@@ -1 +1 @@
-import{Braintrust as e}from"../runner/reporters/braintrust.js";export{e as Braintrust};
+import{Braintrust}from"#evals/runner/reporters/braintrust.js";export{Braintrust};

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
-import{join as e}from"node:path";import{mkdir as t,writeFile as n}from"node:fs/promises";function r(t,n){return e(t,`.ash`,`evals`,`${new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19)}-${n}`)}async function i(r,i){let l=e(r,`cases`);await t(l,{recursive:!0}),await n(e(r,`summary.json`),JSON.stringify(a(i),null,2));let u=i.cases.map(e=>JSON.stringify(o(e))).join(`
-`);await n(e(r,`results.jsonl`),`${u}\n`),await Promise.all(i.cases.map(async t=>{let r=c(t.case.id);await n(e(l,`${r}.json`),JSON.stringify(s(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await n(e(l,`${r}.events.ndjson`),`${i}\n`)}))}function a(e){return{suite:e.suite,target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,errored:e.errored,totalCases:e.cases.length,cases:e.cases.map(e=>({id:e.case.id,status:e.result.status,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}))}}function o(e){return{caseId:e.case.id,status:e.result.status,output:e.result.output,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}}function s(e){return{case:{id:e.case.id,input:e.case.input,expected:e.case.expected,tags:e.case.tags,metadata:e.case.metadata},result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,derived:e.result.derived},scores:e.scores,error:e.error}}function c(e){return e.replace(/[^a-zA-Z0-9_-]/g,`_`)}export{r as resolveArtifactDirectory,i as writeArtifacts};
+import{join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(t,n){return join(t,`.ash`,`evals`,`${new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19)}-${n}`)}async function writeArtifacts(r,i){let a=join(r,`cases`);await mkdir(a,{recursive:!0}),await writeFile(join(r,`summary.json`),JSON.stringify(buildSummaryArtifact(i),null,2));let o=i.cases.map(e=>JSON.stringify(buildCaseResultLine(e))).join(`
+`);await writeFile(join(r,`results.jsonl`),`${o}\n`),await Promise.all(i.cases.map(async t=>{let r=sanitizeFileName(t.case.id);await writeFile(join(a,`${r}.json`),JSON.stringify(buildCaseArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
+`);await writeFile(join(a,`${r}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{suite:e.suite,target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,errored:e.errored,totalCases:e.cases.length,cases:e.cases.map(e=>({id:e.case.id,status:e.result.status,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}))}}function buildCaseResultLine(e){return{caseId:e.case.id,status:e.result.status,output:e.result.output,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}}function buildCaseArtifact(e){return{case:{id:e.case.id,input:e.case.input,expected:e.case.expected,tags:e.case.tags,metadata:e.case.metadata},result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,derived:e.result.derived},scores:e.scores,error:e.error}}function sanitizeFileName(e){return e.replace(/[^a-zA-Z0-9_-]/g,`_`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-function e(e){let n=[],r=[],i=0,a=0,o;for(let t of e)switch(t.type){case`actions.requested`:for(let e of t.data.actions)`toolName`in e&&typeof e.toolName==`string`&&n.push(e.toolName);break;case`subagent.called`:r.push(t.data.name);break;case`subagent.started`:r.push(t.data.subagentName);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(i+=1);break;case`reasoning.completed`:a+=1;break;case`session.failed`:o=t.data.code;break}return{toolCalls:n,toolCallCount:n.length,subagentCalls:t(r),subagentCallCount:r.length,messageCount:i,reasoningBlockCount:a,failureCode:o}}function t(e){return[...new Set(e)]}export{e as deriveRunFacts};
+function deriveRunFacts(e){let t=[],n=[],r=0,i=0,a;for(let o of e)switch(o.type){case`actions.requested`:for(let e of o.data.actions)`toolName`in e&&typeof e.toolName==`string`&&t.push(e.toolName);break;case`subagent.called`:n.push(o.data.name);break;case`subagent.started`:n.push(o.data.subagentName);break;case`message.completed`:o.data.finishReason!==`tool-calls`&&(r+=1);break;case`reasoning.completed`:i+=1;break;case`session.failed`:a=o.data.code;break}return{toolCalls:t,toolCallCount:t.length,subagentCalls:deduplicateStrings(n),subagentCallCount:n.length,messageCount:r,reasoningBlockCount:i,failureCode:a}}function deduplicateStrings(e){return[...new Set(e)]}export{deriveRunFacts};

package/dist/src/evals/runner/discover.js

@@ -1 +1 @@
-import{t as e}from"../../chunks/authored-module-loader-DcCfCiBm.js";import{join as t,relative as n}from"node:path";import{readdir as r}from"node:fs/promises";const i=`.eval.ts`;async function a(e){let r=t(e,`evals`),i=[];try{await u(r,i)}catch(e){if(d(e))return[];throw e}return i.sort((e,t)=>n(r,e).localeCompare(n(r,t))),i}function o(e,r){let a=n(t(e,`evals`),r).split(/[\\/]/u).join(`/`);return a.endsWith(i)?a.slice(0,-8):a}async function s(t,n){let r=(await e(n)).default;if(!l(r))throw Error(`Suite file "${n}" does not export a valid AshEvalSuite as its default export. Use defineEvalSuite() to create the suite.`);return{...r,id:o(t,n)}}async function c(e,t){let n=await a(e);if(n.length===0)return[];let r=[];for(let i of n){let n=await s(e,i);(t===void 0||t.length===0||t.includes(n.id))&&r.push(n)}return r}function l(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`AshEvalSuite`}async function u(e,n){let a=await r(e,{withFileTypes:!0});for(let r of a){let a=t(e,r.name);r.isDirectory()?await u(a,n):r.isFile()&&r.name.endsWith(i)&&n.push(a)}}function d(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{c as discoverAndImportSuites,a as discoverSuiteFiles,s as importSuiteFile};
+import{join,relative}from"node:path";import{readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";var EVAL_SUITE_GLOB_SUFFIX=`.eval.ts`;async function discoverSuiteFiles(n){let r=join(n,`evals`),i=[];try{await collectSuiteFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveSuiteId(n,r){let a=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return a.endsWith(EVAL_SUITE_GLOB_SUFFIX)?a.slice(0,-EVAL_SUITE_GLOB_SUFFIX.length):a}async function importSuiteFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default;if(!isAshEvalSuiteDefinition(n))throw Error(`Suite file "${t}" does not export a valid AshEvalSuite as its default export. Use defineEvalSuite() to create the suite.`);return{...n,id:deriveSuiteId(e,t)}}async function discoverAndImportSuites(e,t){let n=await discoverSuiteFiles(e);if(n.length===0)return[];let r=[];for(let i of n){let n=await importSuiteFile(e,i);(t===void 0||t.length===0||t.includes(n.id))&&r.push(n)}return r}function isAshEvalSuiteDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`AshEvalSuite`}async function collectSuiteFiles(t,r){let a=await readdir(t,{withFileTypes:!0});for(let n of a){let a=join(t,n.name);n.isDirectory()?await collectSuiteFiles(a,r):n.isFile()&&n.name.endsWith(EVAL_SUITE_GLOB_SUFFIX)&&r.push(a)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportSuites,discoverSuiteFiles,importSuiteFile};

package/dist/src/evals/runner/execute-case.js

@@ -1 +1 @@
-import{deriveRunFacts as e}from"./derive-run-facts.js";async function t(t){let{client:a,testCase:o,task:s,timeoutMs:c}=t,l=a.session(),u=n(o,s),d=c===void 0?void 0:AbortSignal.timeout(c),f=[],p;for(let e of u)p=await(await l.sendMessage(e,{signal:d})).result(),f.push(...p.events);if(!p)return i();let m=p.message??null,h=e(f),g=r(f),_={output:m,finalMessage:m,sessionId:p.sessionId,status:p.status,events:f,derived:h,runtimeIdentity:g};return s?.parseOutput!==void 0&&(_.output=s.parseOutput(_)),_}function n(e,t){return t?.messages===void 0?t?.prompt===void 0?[typeof e.input==`string`?e.input:JSON.stringify(e.input)]:[t.prompt(e)]:t.messages(e)}function r(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function i(){return{output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}export{t as executeCase};
+import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";async function executeCase(t){let{client:n,testCase:r,task:i,timeoutMs:a}=t,o=n.session(),s=resolveMessages(r,i),c=a===void 0?void 0:AbortSignal.timeout(a),l=[],u;for(let e of s)u=await(await o.sendMessage(e,{signal:c})).result(),l.push(...u.events);if(!u)return createEmptyResult();let d=u.message??null,f=deriveRunFacts(l),p=extractRuntimeIdentity(l),m={output:d,finalMessage:d,sessionId:u.sessionId,status:u.status,events:l,derived:f,runtimeIdentity:p};return i?.parseOutput!==void 0&&(m.output=i.parseOutput(m)),m}function resolveMessages(e,t){return t?.messages===void 0?t?.prompt===void 0?[typeof e.input==`string`?e.input:JSON.stringify(e.input)]:[t.prompt(e)]:t.messages(e)}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function createEmptyResult(){return{output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}export{executeCase};

package/dist/src/evals/runner/execute-suite.js

@@ -1 +1 @@
-import{t as e}from"../../chunks/errors-HYWjHxV6.js";import{resolveArtifactDirectory as t,writeArtifacts as n}from"./artifacts.js";import{executeCase as r}from"./execute-case.js";async function i(e){let{suite:r,target:i,reporters:o,appRoot:c,client:l}=e,u=new Date().toISOString();for(let e of o)await e.onSuiteStart(r,i);let d=await r.load(),f=r.maxConcurrency??8,p=[],m=[...d],h=new Set;for(;m.length>0||h.size>0;){for(;m.length>0&&h.size<f;){let e=m.shift();if(e===void 0)break;let t=(async()=>{let t=await a(r,e,l);p.push(t);for(let e of o)await e.onCaseComplete(t)})().finally(()=>{h.delete(t)});h.add(t)}h.size>0&&await Promise.race(h)}let g=new Date().toISOString(),_={suite:r.id,target:i,cases:p,startedAt:u,completedAt:g,passed:p.filter(e=>!e.error&&e.scores.every(e=>s(e,r.thresholds))).length,errored:p.filter(e=>e.error!==void 0).length};await n(t(c,r.id),_);for(let e of o)await e.onSuiteComplete(_);return _}async function a(t,n,i){let a,s;try{a=await r({client:i,testCase:n,task:t.task,timeoutMs:t.timeoutMs})}catch(t){s=e(t),a={output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}let c=await o(t,n,a);return{case:n,result:a,scores:c,error:s}}async function o(e,t,n){let r={input:typeof t.input==`string`?t.input:JSON.stringify(t.input),output:String(n.output??``),expected:t.expected===void 0?void 0:String(t.expected),model:e.model,modelOptions:e.modelOptions,case:t,result:n},i=[];for(let t of e.scores){let e=await t(r);e!==null&&i.push(e)}return i}function s(e,t){if(e.score===null)return!0;let n=t?.[e.name]??1;return e.score>=n}export{i as executeSuite};
+import{toErrorMessage}from"#shared/errors.js";import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeCase}from"#evals/runner/execute-case.js";var DEFAULT_MAX_CONCURRENCY=8;async function executeSuite(e){let{suite:r,target:a,reporters:o,appRoot:s,client:c}=e,l=new Date().toISOString();for(let e of o)await e.onSuiteStart(r,a);let u=await r.load(),d=r.maxConcurrency??DEFAULT_MAX_CONCURRENCY,f=[],p=[...u],m=new Set;for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<d;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeSingleCase(r,e,c);f.push(t);for(let e of o)await e.onCaseComplete(t)})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}let h=new Date().toISOString(),g={suite:r.id,target:a,cases:f,startedAt:l,completedAt:h,passed:f.filter(e=>!e.error&&e.scores.every(e=>meetsThreshold(e,r.thresholds))).length,errored:f.filter(e=>e.error!==void 0).length};await writeArtifacts(resolveArtifactDirectory(s,r.id),g);for(let e of o)await e.onSuiteComplete(g);return g}async function executeSingleCase(t,n,i){let a,o;try{a=await executeCase({client:i,testCase:n,task:t.task,timeoutMs:t.timeoutMs})}catch(t){o=toErrorMessage(t),a={output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}let s=await runScorers(t,n,a);return{case:n,result:a,scores:s,error:o}}async function runScorers(e,t,n){let r={input:typeof t.input==`string`?t.input:JSON.stringify(t.input),output:String(n.output??``),expected:t.expected===void 0?void 0:String(t.expected),model:e.model,modelOptions:e.modelOptions,case:t,result:n},i=[];for(let t of e.scores){let e=await t(r);e!==null&&i.push(e)}return i}function meetsThreshold(e,t){if(e.score===null)return!0;let n=t?.[e.name]??1;return e.score>=n}export{executeSuite};

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata as e}from"../resolve-git-metadata.js";function t(e={}){return new n(e)}var n=class{#e;#t;#n;#r;constructor(e){this.#e=e}async onSuiteStart(t,n){let o=await r();this.#t=o;let s=e(process.cwd());this.#r=t.metadata;let c=i(t,n),l=a(t,n);this.#n=await o.init({project:this.#e.projectName??t.id,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:c,metadata:l,noExitFlush:!0,setCurrent:!1,repoInfo:s.sha?{commit:s.sha,branch:s.branch}:null})}onCaseComplete(e){if(!this.#n)return;let t={};for(let n of e.scores)n.score!==null&&(t[n.name]=n.score);let n={...this.#r,...e.case.metadata,ashSessionId:e.result.sessionId,ashStatus:e.result.status,ashToolCalls:e.result.derived.toolCalls,ashSubagentCalls:e.result.derived.subagentCalls};e.result.derived.failureCode&&(n.ashFailureCode=e.result.derived.failureCode);let r={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.case.id,input:e.case.input,output:e.result.output,expected:e.case.expected,error:e.error??void 0,scores:t,metadata:n,metrics:r,tags:e.case.tags?[...e.case.tags]:void 0})}async onSuiteComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function r(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
-`))}}function i(e,t){let n=[`ash`,`suite:${e.id}`,`target:${t.kind}`];return Array.isArray(e.tags)&&n.push(...e.tags),n}function a(e,t){return{ashSuiteId:e.id,ashTargetKind:t.kind,ashTargetUrl:t.url,ashTimestamp:new Date().toISOString()}}export{t as Braintrust};
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r;constructor(e){this.#e=e}async onSuiteStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r=t.metadata;let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t.id,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onCaseComplete(e){if(!this.#n)return;let t={};for(let n of e.scores)n.score!==null&&(t[n.name]=n.score);let n={...this.#r,...e.case.metadata,ashSessionId:e.result.sessionId,ashStatus:e.result.status,ashToolCalls:e.result.derived.toolCalls,ashSubagentCalls:e.result.derived.subagentCalls};e.result.derived.failureCode&&(n.ashFailureCode=e.result.derived.failureCode);let r={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.case.id,input:e.case.input,output:e.result.output,expected:e.case.expected,error:e.error??void 0,scores:t,metadata:n,metrics:r,tags:e.case.tags?[...e.case.tags]:void 0})}async onSuiteComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}},BRAINTRUST_PACKAGE=`braintrust`;async function loadBraintrustSdk(){try{return await import(BRAINTRUST_PACKAGE)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+`))}}function resolveTags(e,t){let n=[`ash`,`suite:${e.id}`,`target:${t.kind}`];return Array.isArray(e.tags)&&n.push(...e.tags),n}function resolveExperimentMetadata(e,t){return{ashSuiteId:e.id,ashTargetKind:t.kind,ashTargetUrl:t.url,ashTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import{i as e}from"../../../chunks/chunk-DSjMdhoD.js";import{t}from"../../../chunks/picocolors-aAkqW4On.js";var n=e(t(),1),r=class{#e;#t;#n;constructor(e){this.#e=e?.log??console.log,this.#t=n.default.createColors(e?.color??!!process.stdout.isTTY)}onSuiteStart(e,t){this.#n=e.thresholds,this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVAL`))} ${this.#t.bold(e.id)}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onCaseComplete(e){let{case:t,scores:n,error:r}=e,i=n.every(e=>{if(e.score===null)return!0;let t=this.#n?.[e.name]??1;return e.score>=t}),a=r?this.#t.red(`✗`):i?this.#t.green(`✓`):this.#t.yellow(`○`),o=n.map(e=>this.#r(e.name,e.score)).join(`  `),s=[a,this.#t.dim(t.id),o].filter(Boolean).join(`  `);this.#e(s),r&&this.#e(`  ${this.#t.red(r)}`)}onSuiteComplete(e){this.#e(``);let{passed:t,errored:n,cases:r}=e,o=r.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} errored`));let c=o-t-n;c>0&&s.push(this.#t.yellow(`${c} scored`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let l=this.#i(r);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#r(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} cases)`)}`)}}let u=i(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${a(u)}`)),this.#e(``)}#r(e,t){if(t===null)return this.#t.dim(`${e}: n/a`);let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#i(e){let t=new Map;for(let n of e)for(let e of n.scores){if(e.score===null)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function i(e,t){return new Date(t).getTime()-new Date(e).getTime()}function a(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{r as ConsoleReporter};
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;#n;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onSuiteStart(e,t){this.#n=e.thresholds,this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVAL`))} ${this.#t.bold(e.id)}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onCaseComplete(e){let{case:t,scores:n,error:r}=e,i=n.every(e=>{if(e.score===null)return!0;let t=this.#n?.[e.name]??1;return e.score>=t}),a=r?this.#t.red(`✗`):i?this.#t.green(`✓`):this.#t.yellow(`○`),o=n.map(e=>this.#r(e.name,e.score)).join(`  `),s=[a,this.#t.dim(t.id),o].filter(Boolean).join(`  `);this.#e(s),r&&this.#e(`  ${this.#t.red(r)}`)}onSuiteComplete(e){this.#e(``);let{passed:t,errored:n,cases:r}=e,i=r.length,a=[];t>0&&a.push(this.#t.green(`${t} passed`)),n>0&&a.push(this.#t.red(`${n} errored`));let o=i-t-n;o>0&&a.push(this.#t.yellow(`${o} scored`)),this.#e(`${this.#t.bold(`Results:`)} ${a.join(`, `)} ${this.#t.dim(`(${i} total)`)}`);let s=this.#i(r);if(s.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of s){let r=this.#r(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} cases)`)}`)}}let c=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(c)}`)),this.#e(``)}#r(e,t){if(t===null)return this.#t.dim(`${e}: n/a`);let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#i(e){let t=new Map;for(let n of e)for(let e of n.scores){if(e.score===null)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/resolve-git-metadata.js

@@ -1 +1 @@
-import{execSync as e}from"node:child_process";function t(e){let t=n(`git rev-parse HEAD`,e);return{branch:n(`git branch --show-current`,e)||void 0,sha:t||void 0}}function n(t,n){try{return e(t,{cwd:n,encoding:`utf8`,stdio:[`pipe`,`pipe`,`pipe`],timeout:5e3}).trim()}catch{return}}export{t as resolveLocalGitMetadata};
+import{execSync}from"node:child_process";function resolveLocalGitMetadata(e){let t=execGitCommand(`git rev-parse HEAD`,e);return{branch:execGitCommand(`git branch --show-current`,e)||void 0,sha:t||void 0}}function execGitCommand(t,n){try{return execSync(t,{cwd:n,encoding:`utf8`,stdio:[`pipe`,`pipe`,`pipe`],timeout:5e3}).trim()}catch{return}}export{resolveLocalGitMetadata};

package/dist/src/evals/scorers/autoevals-client.js

@@ -1,2 +1,2 @@
-import"../../chunks/jsdist-BkeWZ7EZ.js";import{generateText as e,jsonSchema as t}from"ai";function n(e){return{chat:{completions:{create:t=>r(t,e)}}}}async function r(t,n){let r=o(t.tools),a=await e({model:n.languageModel,messages:i(t.messages??[]),tools:Object.keys(r).length>0?r:void 0,toolChoice:s(t.tool_choice),providerOptions:n.providerOptions}),c=a.toolCalls.map(e=>({id:e.toolCallId,type:`function`,function:{name:e.toolName,arguments:JSON.stringify(e.input??{})}}));return{choices:[{index:0,finish_reason:c.length>0?`tool_calls`:`stop`,message:{role:`assistant`,content:a.text||null,tool_calls:c.length>0?c:void 0}}]}}function i(e){return e.map(e=>{let t=a(e.content);switch(e.role){case`assistant`:return{role:`assistant`,content:t};case`developer`:case`system`:return{role:`system`,content:t};default:return{role:`user`,content:t}}})}function a(e){return e==null?``:typeof e==`string`?e:e.map(e=>e.text??``).filter(Boolean).join(`
-`)}function o(e){let n={};for(let r of e??[])r.type!==`function`||r.function?.name===void 0||(n[r.function.name]={description:r.function.description,inputSchema:t(r.function.parameters??{})});return n}function s(e){if(e!==void 0)return typeof e==`string`?e:{type:`tool`,toolName:e.function.name}}export{n as createAutoevalsClient};
+import"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{generateText,jsonSchema}from"ai";function createAutoevalsClient(e){return{chat:{completions:{create:t=>createChatCompletion(t,e)}}}}async function createChatCompletion(t,n){let r=convertTools(t.tools),i=await generateText({model:n.languageModel,messages:convertMessages(t.messages??[]),tools:Object.keys(r).length>0?r:void 0,toolChoice:convertToolChoice(t.tool_choice),providerOptions:n.providerOptions}),a=i.toolCalls.map(e=>({id:e.toolCallId,type:`function`,function:{name:e.toolName,arguments:JSON.stringify(e.input??{})}}));return{choices:[{index:0,finish_reason:a.length>0?`tool_calls`:`stop`,message:{role:`assistant`,content:i.text||null,tool_calls:a.length>0?a:void 0}}]}}function convertMessages(e){return e.map(e=>{let t=contentToText(e.content);switch(e.role){case`assistant`:return{role:`assistant`,content:t};case`developer`:case`system`:return{role:`system`,content:t};default:return{role:`user`,content:t}}})}function contentToText(e){return e==null?``:typeof e==`string`?e:e.map(e=>e.text??``).filter(Boolean).join(`
+`)}function convertTools(e){let n={};for(let r of e??[])r.type!==`function`||r.function?.name===void 0||(n[r.function.name]={description:r.function.description,inputSchema:jsonSchema(r.function.parameters??{})});return n}function convertToolChoice(e){if(e!==void 0)return typeof e==`string`?e:{type:`tool`,toolName:e.function.name}}export{createAutoevalsClient};

package/dist/src/evals/scorers/autoevals.js

@@ -1 +1 @@
-import{n as e}from"../../chunks/chunk-DSjMdhoD.js";import{t}from"../../chunks/runtime-model-Dh0Nz64z.js";import{a as n,i as r,n as i,t as a}from"../../chunks/jsdist-BkeWZ7EZ.js";import{createAutoevalsClient as o}from"./autoevals-client.js";import"ai";var s=e({closedQA:()=>d,factuality:()=>c,sql:()=>u,summary:()=>l});function c(e={}){return f(i,e)}function l(e={}){return f(n,e)}function u(e={}){return f(r,e)}function d(e){return async n=>{let r=e.model??n.model;return m(await a({input:n.input,output:n.output,criteria:e.criteria,model:t(r),client:o({languageModel:r,providerOptions:p(n,e.modelOptions)})}))}}function f(e,n){return async r=>{let i=n.model??r.model;return m(await e({input:r.input,output:r.output,expected:r.expected,model:t(i),client:o({languageModel:i,providerOptions:p(r,n.modelOptions)})}))}}function p(e,t){let n=e.modelOptions?.providerOptions,r=t?.providerOptions;if(!(n===void 0&&r===void 0))return{...n,...r}}function m(e){return{name:e.name,score:e.score,metadata:e.metadata}}export{d as closedQA,c as factuality,u as sql,l as summary,s as t};
+import{ClosedQA,Factuality,Sql,Summary}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import"ai";import{createAutoevalsClient}from"#evals/scorers/autoevals-client.js";function factuality(e={}){return languageScorer(Factuality,e)}function summary(e={}){return languageScorer(Summary,e)}function sql(e={}){return languageScorer(Sql,e)}function closedQA(t){return async n=>{let r=t.model??n.model;return toScorerResult(await ClosedQA({input:n.input,output:n.output,criteria:t.criteria,model:formatLanguageModelGatewayId(r),client:createAutoevalsClient({languageModel:r,providerOptions:mergeProviderOptions(n,t.modelOptions)})}))}}function languageScorer(e,t){return async n=>{let r=t.model??n.model;return toScorerResult(await e({input:n.input,output:n.output,expected:n.expected,model:formatLanguageModelGatewayId(r),client:createAutoevalsClient({languageModel:r,providerOptions:mergeProviderOptions(n,t.modelOptions)})}))}}function mergeProviderOptions(e,t){let n=e.modelOptions?.providerOptions,r=t?.providerOptions;if(!(n===void 0&&r===void 0))return{...n,...r}}function toScorerResult(e){return{name:e.name,score:e.score,metadata:e.metadata}}export{closedQA,factuality,sql,summary};

package/dist/src/evals/scorers/json.js

@@ -1 +1 @@
-import{n as e}from"../../chunks/chunk-DSjMdhoD.js";var t=e({deepEqual:()=>n});function n(){return({case:e,result:t})=>e.expected===void 0?null:{name:`json.deepEqual`,score:+(r(t.output)===r(e.expected))}}function r(e){return JSON.stringify(e,(e,t)=>{if(typeof t==`object`&&t&&!Array.isArray(t)){let e={};for(let n of Object.keys(t).sort())e[n]=t[n];return e}return t})}export{n as deepEqual,t};
+function deepEqual(){return({case:e,result:t})=>e.expected===void 0?null:{name:`json.deepEqual`,score:+(stableStringify(t.output)===stableStringify(e.expected))}}function stableStringify(e){return JSON.stringify(e,(e,t)=>{if(typeof t==`object`&&t&&!Array.isArray(t)){let e={};for(let n of Object.keys(t).sort())e[n]=t[n];return e}return t})}export{deepEqual};

package/dist/src/evals/scorers/run.js

@@ -1 +1 @@
-import{n as e}from"../../chunks/chunk-DSjMdhoD.js";var t=e({didNotFail:()=>n,maxToolCalls:()=>a,usedNoTools:()=>i,usedTool:()=>r});function n(){return({result:e})=>({name:`run.didNotFail`,score:e.status===`failed`?0:1})}function r(e){return({result:t})=>{let n=+!!t.derived.toolCalls.includes(e);return{name:`run.usedTool(${e})`,score:n,metadata:{expectedTool:e,actualToolCalls:t.derived.toolCalls}}}}function i(){return({result:e})=>({name:`run.usedNoTools`,score:+(e.derived.toolCallCount===0),metadata:{toolCallCount:e.derived.toolCallCount}})}function a(e){return({result:t})=>{let n=+(t.derived.toolCallCount<=e);return{name:`run.maxToolCalls(${e})`,score:n,metadata:{maxAllowed:e,actualToolCallCount:t.derived.toolCallCount}}}}export{n as didNotFail,a as maxToolCalls,t,i as usedNoTools,r as usedTool};
+function didNotFail(){return({result:e})=>({name:`run.didNotFail`,score:e.status===`failed`?0:1})}function usedTool(e){return({result:t})=>{let n=+!!t.derived.toolCalls.includes(e);return{name:`run.usedTool(${e})`,score:n,metadata:{expectedTool:e,actualToolCalls:t.derived.toolCalls}}}}function usedNoTools(){return({result:e})=>({name:`run.usedNoTools`,score:+(e.derived.toolCallCount===0),metadata:{toolCallCount:e.derived.toolCallCount}})}function maxToolCalls(e){return({result:t})=>{let n=+(t.derived.toolCallCount<=e);return{name:`run.maxToolCalls(${e})`,score:n,metadata:{maxAllowed:e,actualToolCallCount:t.derived.toolCallCount}}}}export{didNotFail,maxToolCalls,usedNoTools,usedTool};

package/dist/src/evals/scorers/sql.js

@@ -1 +1 @@
-import{n as e}from"../../chunks/chunk-DSjMdhoD.js";var t=e({exactNormalized:()=>n});function n(){return({output:e,expected:t})=>{if(t===void 0)return null;let n=r(e),i=r(t);return{name:`sql.exactNormalized`,score:+(n===i),metadata:{normalizedOutput:n,normalizedExpected:i}}}}function r(e){return e.toLowerCase().replace(/\s+/g,` `).trim().replace(/;+$/,``)}export{n as exactNormalized,t};
+function exactNormalized(){return({output:e,expected:t})=>{if(t===void 0)return null;let n=normalizeSql(e),r=normalizeSql(t);return{name:`sql.exactNormalized`,score:+(n===r),metadata:{normalizedOutput:n,normalizedExpected:r}}}}function normalizeSql(e){return e.toLowerCase().replace(/\s+/g,` `).trim().replace(/;+$/,``)}export{exactNormalized};

package/dist/src/evals/scorers/text.js

@@ -1 +1 @@
-import{n as e}from"../../chunks/chunk-DSjMdhoD.js";import{r as t}from"../../chunks/jsdist-BkeWZ7EZ.js";var n=e({exact:()=>r,includes:()=>i,levenshtein:()=>a});function r(){return({output:e,expected:t})=>t===void 0?null:{name:`text.exact`,score:+(e===t)}}function i(){return({output:e,expected:t})=>t===void 0?null:{name:`text.includes`,score:+!!e.includes(t)}}function a(){return async({output:e,expected:n})=>n===void 0?null:{name:`text.levenshtein`,score:(await t({output:e,expected:n})).score}}export{r as exact,i as includes,a as levenshtein,n as t};
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";function exact(){return({output:e,expected:t})=>t===void 0?null:{name:`text.exact`,score:+(e===t)}}function includes(){return({output:e,expected:t})=>t===void 0?null:{name:`text.includes`,score:+!!e.includes(t)}}function levenshtein(){return async({output:t,expected:n})=>n===void 0?null:{name:`text.levenshtein`,score:(await Levenshtein({output:t,expected:n})).score}}export{exact,includes,levenshtein};

package/dist/src/evals/scores/index.js

@@ -1 +1 @@
-import{t as e}from"../scorers/autoevals.js";import{t}from"../scorers/json.js";import{t as n}from"../scorers/run.js";import{t as r}from"../scorers/sql.js";import{t as i}from"../scorers/text.js";const a=i,o=t,s=r,c=n,l=e;export{l as Autoevals,o as Json,c as Run,s as Sql,a as Text};
+import*as JsonScorers from"#evals/scorers/json.js";import*as RunScorers from"#evals/scorers/run.js";import*as SqlScorers from"#evals/scorers/sql.js";import*as TextScorers from"#evals/scorers/text.js";import*as AutoevalsScorers from"#evals/scorers/autoevals.js";var Text=TextScorers,Json=JsonScorers,Sql=SqlScorers,Run=RunScorers,Autoevals=AutoevalsScorers;export{Autoevals,Json,Run,Sql,Text};

package/dist/src/execution/authorization-challenge-defaults.js

@@ -1,60 +1 @@
-/**
- * Pure helpers that fill in sensible defaults on the
- * `ConnectionAuthorizationChallenge` returned by an authored
- * `startAuthorization` callback.
- *
- * Kept standalone so the orchestrator can stay focused on durable-step
- * plumbing and so the defaulting rules can be unit-tested without a
- * workflow runtime.
- */
-/**
- * Fills in a default `instructions` string when the authored
- * challenge didn't supply one.
- *
- * Most authors only return `{ url }` from `startAuthorization` — the
- * verbatim IdP consent URL is enough to drive the flow. The UI that
- * renders `connection.authorization_required` still wants a short
- * call to action next to the button, so the runtime synthesizes one
- * from the connection's filesystem-derived name (`"linear"`,
- * `"notion"`) rather than making every connection author repeat the
- * same boilerplate.
- *
- * The short `connectionName` is used deliberately — not the rich
- * `description` field, which is written for the model and can be a
- * full sentence. A description like `"Linear workspace — issues,
- * projects, cycles, and comments."` would splice into
- * `"Authorize Linear workspace — issues, projects, cycles, and
- * comments. in your browser to continue."`, which reads poorly.
- * `connectionName` is short and token-like by construction
- * (AGENTS.md rule 21), so it splices cleanly.
- *
- * The first character is upper-cased so the default reads as prose
- * (`"Authorize Notion…"`, not `"Authorize notion…"`). This is a
- * best-effort cosmetic pass — it will not magically produce
- * `GitHub`, `AWS`, or `npm`. Connections that want exact brand
- * casing should set `challenge.instructions` explicitly (directly or
- * via a Connect helper's `instructions` option).
- *
- * Connections that want a custom prompt (localization, brand-specific
- * wording) set `challenge.instructions` explicitly and it passes
- * through untouched.
- */
-export function withDefaultAuthorizationInstructions(challenge, connectionName) {
-    if (challenge.instructions !== undefined)
-        return challenge;
-    return {
-        ...challenge,
-        instructions: `Authorize ${capitalizeFirst(connectionName)} in your browser to continue.`,
-    };
-}
-/**
- * Upper-cases the first character of an ASCII connection name, leaving
- * the rest untouched. Connection names are filesystem-derived and
- * expected to be ASCII per AGENTS.md rule 21, so locale-aware case
- * mapping is unnecessary.
- */
-function capitalizeFirst(value) {
-    if (value.length === 0)
-        return value;
-    return value.charAt(0).toUpperCase() + value.slice(1);
-}
+function withDefaultAuthorizationInstructions(e,t){return e.instructions===void 0?{...e,instructions:`Authorize ${capitalizeFirst(t)} in your browser to continue.`}:e}function capitalizeFirst(e){return e.length===0?e:e.charAt(0).toUpperCase()+e.slice(1)}export{withDefaultAuthorizationInstructions};

package/dist/src/execution/await-authorization-orchestrator.js

@@ -1,192 +1 @@
-/**
- * Workflow-body orchestrator for the interactive OAuth
- * await-authorization-and-splice flow.
- *
- * The workflow body owns hook creation and waiting; model-facing side
- * effects stay inside durable steps.
- */
-import { createHook, getWorkflowMetadata } from "#compiled/@workflow/core/index.js";
-import { getHarnessEmissionState } from "#harness/emission.js";
-import { createAshConnectionCallbackRoutePath } from "#protocol/routes.js";
-import { completeAuthorizationForConnectionStep, emitConnectionAuthorizationPendingStep, resolvePendingToolCallsStep, startAuthorizationForConnectionStep, } from "#execution/connection-auth-steps.js";
-/**
- * Orchestrates one await-authorization-and-splice cycle for all
- * interactive-authorization connections that have pending tool calls.
- * Connections that only implement `getToken` are ignored here — they
- * surface via `drainPendingConnectionAuthorizations` on the harness
- * step boundary and rely on out-of-band auth.
- *
- * Error behaviour: one connection's failure does not block the
- * others. Connections that fail `startAuthorization` or
- * `completeAuthorization` surface a structured error result on the
- * spliced tool call rather than an exception in the workflow body.
- */
-export async function awaitAuthorizationAndResolve(input) {
-    const emissionState = getHarnessEmissionState(input.session);
-    const connectionNames = uniqueConnectionNames(input.pendingToolCalls);
-    if (connectionNames.length === 0) {
-        return {
-            serializedContext: input.serializedContext,
-            session: input.session,
-        };
-    }
-    // Mint hooks up front so their tokens, and therefore the framework
-    // callback URLs we hand to the IdP, are stable across replays.
-    // `createHook` is a workflow-body API; calling it here (rather than
-    // inside a step) is what makes the token deterministic.
-    //
-    // The framework owns the public callback URL: instead of redirecting
-    // the IdP at Workflow's raw
-    // `/.well-known/workflow/v1/webhook/:token` endpoint, we build a
-    // namespaced Ash route at
-    // `/ash/v1/connections/:name/callback/:token`. The
-    // matching framework route handler projects the inbound request and
-    // resumes this hook via `resumeHook(token, payload)`. Owning the
-    // callback in Ash lets the framework decide delivery policy (auth,
-    // throttling, logging) without leaking generic workflow primitives.
-    const callbackBaseUrl = trimTrailingSlash(getWorkflowMetadata().url);
-    const hooks = connectionNames.map((name) => {
-        const hook = createHook();
-        return {
-            connectionName: name,
-            hook,
-            webhookUrl: `${callbackBaseUrl}${createAshConnectionCallbackRoutePath(name, hook.token)}`,
-        };
-    });
-    // Run every `startAuthorization` inside its own durable step,
-    // parallelised across connections. Each step emits the matching
-    // `connection.authorization_required` event before returning.
-    const startResults = await Promise.all(hooks.map(async ({ connectionName, hook, webhookUrl }) => ({
-        connectionName,
-        hook,
-        webhookUrl,
-        start: await startAuthorizationForConnectionStep({
-            connectionName,
-            emissionState,
-            parentWritable: input.parentWritable,
-            serializedContext: input.serializedContext,
-            webhookUrl,
-        }),
-    })));
-    const successfulStarts = startResults.flatMap((r) => r.start.ok
-        ? [
-            {
-                connectionName: r.connectionName,
-                hook: r.hook,
-                principal: r.start.principal,
-                // Forward the start step's post-emit serialized ctx so the
-                // matching `completeAuthorization` step's emit sees the
-                // channel-state writes (e.g. tracked Slack message ts) the
-                // `connection.authorization_required` handler made.
-                serializedContext: r.start.serializedContext,
-                state: r.start.state,
-                webhookUrl: r.webhookUrl,
-            },
-        ]
-        : []);
-    // Announce that the turn is now waiting on webhook callbacks.
-    // Emission skipped when every `startAuthorization` failed.
-    if (successfulStarts.length > 0) {
-        await emitConnectionAuthorizationPendingStep({
-            connectionNames: successfulStarts.map((s) => s.connectionName),
-            emissionState,
-            parentWritable: input.parentWritable,
-            serializedContext: input.serializedContext,
-        });
-    }
-    // Suspend the workflow until each hook is resumed by the framework
-    // callback route, then run `completeAuthorization` inside a durable
-    // step per connection. `hook.dispose()` runs in a `finally` so a
-    // throw inside `awaitHookRequest` or the step does not leak the
-    // hook.
-    const completeResults = await Promise.all(successfulStarts.map(async ({ connectionName, hook, principal, serializedContext, state, webhookUrl }) => {
-        try {
-            const request = await awaitHookRequest(hook);
-            const complete = await completeAuthorizationForConnectionStep({
-                connectionName,
-                emissionState,
-                parentWritable: input.parentWritable,
-                principal,
-                request,
-                serializedContext,
-                state,
-                webhookUrl,
-            });
-            return { complete, connectionName, principal };
-        }
-        finally {
-            hook.dispose();
-        }
-    }));
-    const tokens = {};
-    const principals = {};
-    const failedConnections = {};
-    const resolvedConnectionNames = [];
-    // `startAuthorization` failures are terminal for the current cycle —
-    // the author's callback threw and there is no authorization state to
-    // complete. Surface them as non-retryable.
-    for (const r of startResults) {
-        if (!r.start.ok) {
-            failedConnections[r.connectionName] = {
-                reason: r.start.reason,
-                retryable: false,
-            };
-        }
-    }
-    for (const r of completeResults) {
-        if (r.complete.ok) {
-            tokens[r.connectionName] = r.complete.token;
-            principals[r.connectionName] = r.principal;
-            resolvedConnectionNames.push(r.connectionName);
-        }
-        else {
-            failedConnections[r.connectionName] = {
-                reason: r.complete.reason,
-                retryable: r.complete.retryable,
-            };
-        }
-    }
-    return resolvePendingToolCallsStep({
-        failedConnections,
-        parentWritable: input.parentWritable,
-        pendingCalls: input.pendingToolCalls,
-        principals,
-        resolvedConnectionNames,
-        serializedContext: input.serializedContext,
-        session: input.session,
-        tokens,
-    });
-}
-function uniqueConnectionNames(pendingCalls) {
-    const seen = new Set();
-    for (const call of pendingCalls) {
-        const names = call.kind === "connection-execute" ? [call.connectionName] : call.connectionNames;
-        for (const name of names) {
-            seen.add(name);
-        }
-    }
-    return [...seen];
-}
-/**
- * Awaits the first payload delivered to `hook` via `resumeHook`, using
- * its async iterator. Using the iterator (rather than `await hook`
- * directly) lets us return early once we have the request we need
- * without relying on the `Thenable` surface that resolves to only the
- * first payload.
- *
- * The framework callback route projects the inbound IdP request into
- * an {@link AuthorizationCallbackRequest} before resuming, so this
- * function returns that JSON-serializable shape directly; no further
- * projection is needed.
- */
-async function awaitHookRequest(hook) {
-    const iterator = hook[Symbol.asyncIterator]();
-    const next = await iterator.next();
-    if (next.done === true || next.value === undefined) {
-        throw new Error("Connection callback hook closed before delivering a request.");
-    }
-    return next.value;
-}
-function trimTrailingSlash(value) {
-    return value.endsWith("/") ? value.slice(0, -1) : value;
-}
+import{createAshConnectionCallbackRoutePath}from"#protocol/routes.js";import{createHook,getWorkflowMetadata}from"#compiled/@workflow/core/index.js";import{getHarnessEmissionState}from"#harness/emission.js";import{completeAuthorizationForConnectionStep,emitConnectionAuthorizationPendingStep,resolvePendingToolCallsStep,startAuthorizationForConnectionStep}from"#execution/connection-auth-steps.js";async function awaitAuthorizationAndResolve(s){let c=getHarnessEmissionState(s.session),l=uniqueConnectionNames(s.pendingToolCalls);if(l.length===0)return{serializedContext:s.serializedContext,session:s.session};let u=trimTrailingSlash(getWorkflowMetadata().url),d=l.map(n=>{let r=createHook();return{connectionName:n,hook:r,webhookUrl:`${u}${createAshConnectionCallbackRoutePath(n,r.token)}`}}),f=await Promise.all(d.map(async({connectionName:e,hook:t,webhookUrl:n})=>({connectionName:e,hook:t,webhookUrl:n,start:await startAuthorizationForConnectionStep({connectionName:e,emissionState:c,parentWritable:s.parentWritable,serializedContext:s.serializedContext,webhookUrl:n})}))),p=f.flatMap(e=>e.start.ok?[{connectionName:e.connectionName,hook:e.hook,principal:e.start.principal,serializedContext:e.start.serializedContext,state:e.start.state,webhookUrl:e.webhookUrl}]:[]);p.length>0&&await emitConnectionAuthorizationPendingStep({connectionNames:p.map(e=>e.connectionName),emissionState:c,parentWritable:s.parentWritable,serializedContext:s.serializedContext});let m=await Promise.all(p.map(async({connectionName:e,hook:t,principal:n,serializedContext:r,state:i,webhookUrl:a})=>{try{let o=await awaitHookRequest(t);return{complete:await completeAuthorizationForConnectionStep({connectionName:e,emissionState:c,parentWritable:s.parentWritable,principal:n,request:o,serializedContext:r,state:i,webhookUrl:a}),connectionName:e,principal:n}}finally{t.dispose()}})),h={},g={},_={},v=[];for(let e of f)e.start.ok||(_[e.connectionName]={reason:e.start.reason,retryable:!1});for(let e of m)e.complete.ok?(h[e.connectionName]=e.complete.token,g[e.connectionName]=e.principal,v.push(e.connectionName)):_[e.connectionName]={reason:e.complete.reason,retryable:e.complete.retryable};return resolvePendingToolCallsStep({failedConnections:_,parentWritable:s.parentWritable,pendingCalls:s.pendingToolCalls,principals:g,resolvedConnectionNames:v,serializedContext:s.serializedContext,session:s.session,tokens:h})}function uniqueConnectionNames(e){let t=new Set;for(let n of e){let e=n.kind===`connection-execute`?[n.connectionName]:n.connectionNames;for(let n of e)t.add(n)}return[...t]}async function awaitHookRequest(e){let t=await e[Symbol.asyncIterator]().next();if(t.done===!0||t.value===void 0)throw Error(`Connection callback hook closed before delivering a request.`);return t.value}function trimTrailingSlash(e){return e.endsWith(`/`)?e.slice(0,-1):e}export{awaitAuthorizationAndResolve};

package/dist/src/execution/await-authorization-splice.js

@@ -1,66 +1 @@
-/**
- * Splices real tool results over {@link CONNECTION_AUTHORIZATION_PLACEHOLDER}
- * outputs in the session history after an interactive authorization
- * flow completes.
- */
-import { isConnectionAuthorizationPlaceholder } from "#runtime/framework-tools/pending-connection-tool-calls.js";
-/**
- * Produces a new {@link HarnessSession} whose history has had every
- * placeholder tool-result output replaced with the corresponding entry
- * in `resolutions`.
- *
- * Tool-result entries whose `toolCallId` is not present in
- * `resolutions`, or whose current output is not a placeholder, are
- * left untouched. The original history array and messages are not
- * mutated — all modified messages are cloned.
- */
-export function splicePendingToolResults(session, resolutions) {
-    if (Object.keys(resolutions).length === 0) {
-        return session;
-    }
-    let mutated = false;
-    const nextHistory = session.history.map((message) => {
-        if (message.role !== "tool")
-            return message;
-        if (typeof message.content === "string")
-            return message;
-        if (!Array.isArray(message.content))
-            return message;
-        let messageChanged = false;
-        const nextContent = message.content.map((part) => {
-            if (!isToolResultPartLike(part))
-                return part;
-            const resolution = resolutions[part.toolCallId];
-            if (resolution === undefined)
-                return part;
-            if (!isPlaceholderOutput(part.output))
-                return part;
-            messageChanged = true;
-            return { ...part, output: toJsonOutput(resolution) };
-        });
-        if (!messageChanged)
-            return message;
-        mutated = true;
-        return { ...message, content: nextContent };
-    });
-    if (!mutated)
-        return session;
-    return { ...session, history: nextHistory };
-}
-function isToolResultPartLike(part) {
-    if (typeof part !== "object" || part === null)
-        return false;
-    const p = part;
-    return p.type === "tool-result" && typeof p.toolCallId === "string" && p.output !== undefined;
-}
-function isPlaceholderOutput(output) {
-    if (typeof output !== "object" || output === null)
-        return false;
-    const typed = output;
-    if (typed.type !== "json")
-        return false;
-    return isConnectionAuthorizationPlaceholder(typed.value);
-}
-function toJsonOutput(value) {
-    return { type: "json", value };
-}
+import{isConnectionAuthorizationPlaceholder}from"#runtime/framework-tools/pending-connection-tool-calls.js";function splicePendingToolResults(e,t){if(Object.keys(t).length===0)return e;let n=!1,r=e.history.map(e=>{if(e.role!==`tool`||typeof e.content==`string`||!Array.isArray(e.content))return e;let r=!1,i=e.content.map(e=>{if(!isToolResultPartLike(e))return e;let n=t[e.toolCallId];return n===void 0||!isPlaceholderOutput(e.output)?e:(r=!0,{...e,output:toJsonOutput(n)})});return r?(n=!0,{...e,content:i}):e});return n?{...e,history:r}:e}function isToolResultPartLike(e){if(typeof e!=`object`||!e)return!1;let t=e;return t.type===`tool-result`&&typeof t.toolCallId==`string`&&t.output!==void 0}function isPlaceholderOutput(t){if(typeof t!=`object`||!t)return!1;let n=t;return n.type===`json`?isConnectionAuthorizationPlaceholder(n.value):!1}function toJsonOutput(e){return{type:`json`,value:e}}export{splicePendingToolResults};