eve 0.6.0-beta.9 → 0.7.2

package/dist/src/evals/runner/discover.d.ts

@@ -1,20 +1,41 @@
-import type { EveEvalSuite } from "#evals/types.js";
+import type { EveEval, EveEvalConfig } from "#evals/types.js";
 /**
- * Discovers eval suite files under `<appRoot>/evals/` by recursively
+ * Discovers eval files under `<appRoot>/evals/` by recursively
  * scanning for files matching `*.eval.ts`.
  *
  * Returns absolute paths sorted alphabetically by relative path.
  */
-export declare function discoverSuiteFiles(appRoot: string): Promise<string[]>;
+export declare function discoverEvalFiles(appRoot: string): Promise<string[]>;
 /**
- * Imports a discovered suite file and stamps the path-derived id onto
- * the suite definition.
+ * Returns true when `evalId` matches one of the requested filters. A filter
+ * matches its exact eval id or any eval nested under it, so `"runtime"`
+ * matches both `evals/runtime.eval.ts` and every eval in `evals/runtime/`
+ * (and every entry of an array-exported `evals/runtime.eval.ts`).
+ */
+export declare function matchesEvalFilter(evalId: string, filters: readonly string[]): boolean;
+/**
+ * Imports a discovered eval file and stamps the path-derived id(s) onto
+ * the eval definition(s).
+ *
+ * The file must `export default` either one `EveEvalDefinition` (produced
+ * by `defineEval()`) or an array of them. A single definition derives its
+ * id from the file path; array entries derive `<file-id>/<index>` ids with
+ * the index zero-padded to four digits (e.g. `"weather/0000"`).
+ */
+export declare function importEvalFile(appRoot: string, filePath: string): Promise<EveEval[]>;
+/**
+ * Discovers and imports all eval files, optionally filtering by eval id.
+ * Filters match exactly or by directory prefix (see {@link matchesEvalFilter}).
  *
- * The file must `export default` an `EveEvalSuiteDefinition` (produced
- * by `defineEvalSuite()`).
+ * Throws when two files derive the same eval id (e.g. an array-exported
+ * `evals/weather.eval.ts` colliding with `evals/weather/0000.eval.ts`).
  */
-export declare function importSuiteFile(appRoot: string, filePath: string): Promise<EveEvalSuite>;
+export declare function discoverAndImportEvals(appRoot: string, evalIds?: readonly string[]): Promise<EveEval[]>;
 /**
- * Discovers and imports all suite files, optionally filtering by suite ID.
+ * Discovers and imports the required `evals/evals.config.ts` run-wide
+ * configuration (produced by `defineEvalConfig()`).
+ *
+ * Throws when the file is missing or does not default-export a valid
+ * `EveEvalConfig`.
  */
-export declare function discoverAndImportSuites(appRoot: string, suiteIds?: readonly string[]): Promise<EveEvalSuite[]>;
+export declare function discoverEvalConfig(appRoot: string): Promise<EveEvalConfig>;

package/dist/src/evals/runner/discover.js

@@ -1 +1 @@
-import{join,relative}from"node:path";import{readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_SUITE_GLOB_SUFFIX=`.eval.ts`;async function discoverSuiteFiles(n){let r=join(n,`evals`),i=[];try{await collectSuiteFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveSuiteId(n,r){let a=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return a.endsWith(EVAL_SUITE_GLOB_SUFFIX)?a.slice(0,-8):a}async function importSuiteFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default;if(!isEveEvalSuiteDefinition(n))throw Error(`Suite file "${t}" does not export a valid EveEvalSuite as its default export. Use defineEvalSuite() to create the suite.`);return{...n,id:deriveSuiteId(e,t)}}async function discoverAndImportSuites(e,t){let n=await discoverSuiteFiles(e);if(n.length===0)return[];let r=[];for(let i of n){let n=await importSuiteFile(e,i);(t===void 0||t.length===0||t.includes(n.id))&&r.push(n)}return r}function isEveEvalSuiteDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalSuite`}async function collectSuiteFiles(t,r){let a=await readdir(t,{withFileTypes:!0});for(let n of a){let a=join(t,n.name);n.isDirectory()?await collectSuiteFiles(a,r):n.isFile()&&n.name.endsWith(EVAL_SUITE_GLOB_SUFFIX)&&r.push(a)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportSuites,discoverSuiteFiles,importSuiteFile};
+import{join,relative}from"node:path";import{access,readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_FILE_SUFFIX=`.eval.ts`,EVAL_CONFIG_FILE=`evals.config.ts`;async function discoverEvalFiles(n){let r=join(n,`evals`),i=[];try{await collectEvalFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveEvalId(n,r){let i=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return i.endsWith(EVAL_FILE_SUFFIX)?i.slice(0,-8):i}function matchesEvalFilter(e,t){return t.length===0?!0:t.some(t=>e===t||e.startsWith(`${t}/`))}async function importEvalFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default,r=deriveEvalId(e,t);if(isEveEvalDefinition(n))return[{...n,id:r}];if(Array.isArray(n))return n.map((e,n)=>{if(!isEveEvalDefinition(e))throw Error(`Eval file "${t}" exports an array whose entry at index ${n} is not a valid EveEval. Use defineEval() to create every entry.`);return{...e,id:`${r}/${String(n).padStart(4,`0`)}`}});throw Error(`Eval file "${t}" does not export a valid EveEval (or array of EveEvals) as its default export. Use defineEval() to create the eval.`)}async function discoverAndImportEvals(e,t){let n=await discoverEvalFiles(e);if(n.length===0)return[];let r=t??[],i=[],a=new Map;for(let t of n)for(let n of await importEvalFile(e,t)){let e=a.get(n.id);if(e!==void 0)throw Error(`Duplicate eval id "${n.id}" derived from both "${e}" and "${t}".`);a.set(n.id,t),matchesEvalFilter(n.id,r)&&i.push(n)}return i}async function discoverEvalConfig(t){let r=join(t,`evals`,EVAL_CONFIG_FILE);try{await access(r)}catch(e){throw isNoEntryError(e)?Error(`Missing required eval config at evals/${EVAL_CONFIG_FILE}. Create it with defineEvalConfig({}) (optionally \`{ judge: { model } }\` to set the default judge model for \`t.judge.*\` assertions).`):e}let a=(await loadAuthoredModuleNamespace(r)).default;if(!isEveEvalConfig(a))throw Error(`Eval config "evals/${EVAL_CONFIG_FILE}" must default-export a defineEvalConfig() value.`);return a}function isEveEvalConfig(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalConfig`}function isEveEvalDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEval`}async function collectEvalFiles(t,n){let i=await readdir(t,{withFileTypes:!0});for(let r of i){let i=join(t,r.name);r.isDirectory()?await collectEvalFiles(i,n):r.isFile()&&r.name.endsWith(EVAL_FILE_SUFFIX)&&n.push(i)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportEvals,discoverEvalConfig,discoverEvalFiles,importEvalFile,matchesEvalFilter};

package/dist/src/evals/runner/execute-eval.d.ts

@@ -0,0 +1,25 @@
+import type { Client } from "#client/client.js";
+import type { EveEval, EveEvalResult, EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Options for executing one eval.
+ */
+export interface ExecuteEvalOptions {
+    readonly evaluation: EveEval;
+    readonly failOnSkip?: boolean;
+    /** Receives `t.log` lines as the eval runs (used by `--verbose`). */
+    readonly onLog?: (message: string) => void;
+    readonly target: EveEvalTargetHandle;
+    /** Overrides the eval's own `timeoutMs` when set (CLI `--timeout`). */
+    readonly timeoutMs?: number;
+    /**
+     * Pre-configured client for communicating with the Eve agent.
+     * The CLI constructs this once with the appropriate auth and headers,
+     * and every eval creates fresh sessions from it.
+     */
+    readonly client: Client;
+}
+/**
+ * Executes one eval end to end: verifies requirements, runs `test(t)`,
+ * collects its assertions, and computes the verdict.
+ */
+export declare function executeEval(options: ExecuteEvalOptions): Promise<EveEvalResult>;

package/dist/src/evals/runner/execute-eval.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{findUnmetRequirements,formatUnmetRequirements}from"#evals/requirements.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(i){let{evaluation:a,target:o,client:s}=i,c=new Date().toISOString(),l=a.requires??[],u=findUnmetRequirements(l,o);if(u.length>0)return createRequirementResult(a,u,{failOnSkip:i.failOnSkip===!0,startedAt:c});let d,f=[],p;try{let e=await executeTask({client:s,evaluation:a,onLog:i.onLog,requirements:l,target:o,timeoutMs:i.timeoutMs??a.timeoutMs});d=e.result,f=e.assertions,p=e.error}catch(n){p=toErrorMessage(n),d={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let m=computeEvalVerdict({error:p,assertions:f});return{id:a.id,result:d,assertions:f,verdict:m,error:p,startedAt:c,completedAt:new Date().toISOString()}}function createRequirementResult(e,t,n){let r=formatUnmetRequirements(t);return{id:e.id,result:createEmptyTaskResult(),assertions:[],verdict:n.failOnSkip?`failed`:`skipped`,skipReason:r,startedAt:n.startedAt,completedAt:new Date().toISOString()}}function createEmptyTaskResult(){return{output:null,finalMessage:null,status:`completed`,events:[],derived:createEmptyDerivedFacts()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -0,0 +1,31 @@
+import type { Client } from "#client/client.js";
+import type { AssertionResult, EveEval, EveEvalRequirement, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
+/**
+ * Options for executing one eval's task.
+ */
+interface ExecuteTaskOptions {
+    readonly client: Client;
+    readonly evaluation: EveEval;
+    /** Receives each `t.log` line as it is written (used by `--verbose`). */
+    readonly onLog?: (message: string) => void;
+    readonly requirements?: readonly EveEvalRequirement[];
+    readonly target: EveEvalTargetHandle;
+    readonly timeoutMs?: number;
+}
+/**
+ * Task result plus the assertions the eval's `test(t)` recorded. `error` is
+ * set when the `test` body threw (e.g. a failed `expectOk()` or a bespoke
+ * `throw`); the partial run is still captured so recorded assertions report.
+ */
+export interface ExecuteTaskResult {
+    readonly result: EveEvalTaskResult;
+    readonly assertions: readonly AssertionResult[];
+    readonly error?: string;
+}
+/**
+ * Executes one eval's `test(t)` against an Eve agent target: drives the
+ * session(s), captures the run, then finalizes the recorded assertions
+ * against the completed task result.
+ */
+export declare function executeTask(options: ExecuteTaskOptions): Promise<ExecuteTaskResult>;
+export {};

package/dist/src/evals/runner/execute-task.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(r){let{client:a,evaluation:o,target:s,timeoutMs:c}=r,l=c===void 0?neverAbortSignal():AbortSignal.timeout(c),u=new EvalSessionManager({client:a,signal:l}),d=scopeEvalTargetHandle(s,{requirements:r.requirements??[],sessions:u}),f=[],{context:p,collector:m}=createEvalContext({manager:u,target:d,signal:l,judge:o.judge,log:e=>{f.push(e),r.onLog?.(e)}}),h;try{await o.test(p)}catch(t){h=toErrorMessage(t)}let g=buildTaskResult({logs:f,sessions:u.snapshots(),turn:u.lastTurnSession()?.lastTurn});return{result:g,assertions:await m.finalize(g),error:h}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:n,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:a}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.d.ts

@@ -6,7 +6,7 @@ import type { EvalReporter } from "#evals/runner/reporters/types.js";
 export interface BraintrustReporterConfig {
     /** Braintrust project id. Maps to the `projectId` init option. */
     readonly projectId?: string;
-    /** Braintrust project name. Defaults to the suite id when omitted. */
+    /** Braintrust project name. Defaults to the eval id when omitted. */
     readonly projectName?: string;
     /** Name for the created experiment. When omitted, Braintrust names it. */
     readonly experimentName?: string;
@@ -19,9 +19,11 @@ export interface BraintrustReporterConfig {
 }
 /**
  * Creates an {@link EvalReporter} that uploads eval results to Braintrust as an
- * experiment. Add it to a suite's `reporters` array. Requires the optional
- * `braintrust` peer package; the reporter throws a descriptive error when the
- * suite starts if it is missing. `config` is optional and defaults to an empty
- * object (the project name then falls back to the suite id).
+ * experiment. Add it to an eval's `reporters` array; share one instance
+ * across several evals (e.g. every entry of an array export) to group them
+ * into a single experiment. Requires the optional `braintrust` peer package;
+ * the reporter throws a descriptive error when the run starts if it is
+ * missing. `config` is optional and defaults to an empty object (the project
+ * name then falls back to the first observed eval id).
  */
 export declare function Braintrust(config?: BraintrustReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/braintrust.js

@@ -1,2 +1,2 @@
-import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r;constructor(e){this.#e=e}async onSuiteStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r=t.metadata;let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t.id,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onCaseComplete(e){if(!this.#n)return;let t={};for(let n of e.scores)n.score!==null&&(t[n.name]=n.score);for(let n of e.checks)t[`check:${n.name}`]=+!!n.passed;let n=e.checks.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),r={...this.#r,...e.case.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};n.length>0&&(r.eveFailedChecks=n),e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode);let i={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.case.id,input:e.case.input,output:e.result.output,expected:e.case.expected,error:e.error??void 0,scores:t,metadata:r,metrics:i,tags:e.case.tags?[...e.case.tags]:void 0})}async onSuiteComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
-`))}}function resolveTags(e,t){let n=[`eve`,`suite:${e.id}`,`target:${t.kind}`];return Array.isArray(e.tags)&&n.push(...e.tags),n}function resolveExperimentMetadata(e,t){return{eveSuiteId:e.id,eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r=new Map;constructor(e){this.#e=e}async onRunStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r.clear();for(let e of t)this.#r.set(e.id,e);let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t[0]?.id??`eve evals`,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onEvalComplete(e){if(!this.#n)return;let t=this.#r.get(e.id),n={};for(let t of e.assertions){let e=t.severity===`gate`?`gate:${t.name}`:t.name;n[e]=t.score}let r=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message})),i={...t?.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked};r.length>0&&(i.eveFailedAssertions=r),e.result.derived.failureCode&&(i.eveFailureCode=e.result.derived.failureCode);let a={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.id,input:t?.description??``,output:e.result.output,error:e.error??void 0,scores:n,metadata:i,metrics:a,tags:t?.tags?[...t.tags]:void 0})}async onRunComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+`))}}function resolveTags(e,t){let n=new Set([`eve`,`target:${t.kind}`]);for(let t of e){n.add(`eval:${t.id}`);for(let e of t.tags??[])n.add(e)}return[...n]}function resolveExperimentMetadata(e,t){return{eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.d.ts

@@ -1,4 +1,4 @@
-import type { EveEvalCaseResult, EveEvalSuite, EveEvalSuiteResult, EveEvalTarget } from "#evals/types.js";
+import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js";
 import type { EvalReporter } from "#evals/runner/reporters/types.js";
 /**
  * Console reporter that prints eval progress and results to stdout.
@@ -9,7 +9,7 @@ export declare class ConsoleReporter implements EvalReporter {
         log?: (message: string) => void;
         color?: boolean;
     });
-    onSuiteStart(suite: EveEvalSuite, target: EveEvalTarget): void;
-    onCaseComplete(caseResult: EveEvalCaseResult): void;
-    onSuiteComplete(suiteResult: EveEvalSuiteResult): void;
+    onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): void;
+    onEvalComplete(result: EveEvalResult): void;
+    onRunComplete(summary: EveEvalRunSummary): void;
 }

package/dist/src/evals/runner/reporters/console.js

@@ -1 +1 @@
-import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onSuiteStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVAL`))} ${this.#t.bold(e.id)}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onCaseComplete(e){let{case:t,checks:n,scores:r,verdict:i,error:a}=e,o=this.#n(i),s=n.length>0?this.#r(n.filter(e=>e.passed).length,n.length):``,c=r.map(e=>this.#i(e.name,e.score)).join(`  `),l=[o,this.#t.dim(t.id),s,c].filter(Boolean).join(`  `);this.#e(l),e.skipReason!==void 0&&this.#e(`  ${this.#t.dim(`skipped: ${e.skipReason}`)}`);for(let e of n){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}a&&this.#e(`  ${this.#t.red(a)}`)}onSuiteComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,cases:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 cases`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Checks:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} cases)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`-`)}}#r(e,t){let n=`checks ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){if(t===null)return this.#t.dim(`${e}: n/a`);let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of r.checks)e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.scores){if(e.score===null)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onRunStart(e,t){this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVALS`))} ${this.#t.bold(String(e.length))}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onEvalComplete(e){let{assertions:t,verdict:n,error:r}=e,i=t.filter(e=>e.severity===`gate`),a=t.filter(e=>e.severity===`soft`),o=this.#n(n),s=i.length>0?this.#r(i.filter(e=>e.passed).length,i.length):``,c=a.map(e=>this.#i(e.name,e.score)).join(`  `),l=[o,this.#t.dim(e.id),s,c].filter(Boolean).join(`  `);this.#e(l),e.skipReason!==void 0&&this.#e(`  ${this.#t.dim(`skipped: ${e.skipReason}`)}`);for(let e of t){if(e.passed)continue;let t=e.message===void 0?``:`: ${e.message}`;this.#e(`  ${this.#t.red(`✗ ${e.name}${t}`)}`)}r&&this.#e(`  ${this.#t.red(r)}`)}onRunComplete(e){this.#e(``);let{passed:t,failed:n,scored:r,skipped:i,results:a}=e,o=a.length,s=[];t>0&&s.push(this.#t.green(`${t} passed`)),n>0&&s.push(this.#t.red(`${n} failed`)),r>0&&s.push(this.#t.yellow(`${r} scored`)),i>0&&s.push(this.#t.dim(`${i} skipped`)),s.length===0&&s.push(this.#t.dim(`0 evals`)),this.#e(`${this.#t.bold(`Results:`)} ${s.join(`, `)} ${this.#t.dim(`(${o} total)`)}`);let c=this.#a(a);if(c.total>0){let e=this.#t.green(`${c.passed} passed`),t=c.failed>0?`, ${this.#t.red(`${c.failed} failed`)}`:``;this.#e(`${this.#t.bold(`Gates:`)} ${e}${t}`)}let l=this.#o(a);if(l.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of l){let r=this.#i(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} evals)`)}`)}}let u=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(u)}`)),this.#e(``)}#n(e){switch(e){case`passed`:return this.#t.green(`✓`);case`failed`:return this.#t.red(`✗`);case`scored`:return this.#t.yellow(`○`);case`skipped`:return this.#t.dim(`-`)}}#r(e,t){let n=`gates ${e}/${t}`;return e===t?this.#t.green(n):this.#t.red(n)}#i(e,t){let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#a(e){let t=0,n=0;for(let r of e)for(let e of gatesOf(r))e.passed?t+=1:n+=1;return{passed:t,failed:n,total:t+n}}#o(e){let t=new Map;for(let n of e)for(let e of n.assertions){if(e.severity!==`soft`)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function gatesOf(e){return e.assertions.filter(e=>e.severity===`gate`)}function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/reporters/junit.d.ts

@@ -0,0 +1,10 @@
+import type { EvalReporter } from "#evals/runner/reporters/types.js";
+export interface JUnitReporterConfig {
+    readonly filePath: string;
+    readonly suiteName?: string;
+}
+/**
+ * Creates a reporter that writes one JUnit XML testsuite for an eval run.
+ * Each eval becomes one `<testcase>` named by its path-derived id.
+ */
+export declare function JUnit(config: JUnitReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/junit.js

@@ -0,0 +1,4 @@
+import{dirname}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function JUnit(e){return new JUnitReporter(e)}var JUnitReporter=class{#e;constructor(e){this.#e=e}onRunStart(){}onEvalComplete(){}async onRunComplete(r){let i=renderJUnit(r,{suiteName:this.#e.suiteName});await mkdir(dirname(this.#e.filePath),{recursive:!0}),await writeFile(this.#e.filePath,i)}};function renderJUnit(e,t){let n=e.failed+e.scored,r=e.results.map(renderTestCase);return[`<?xml version="1.0" encoding="UTF-8"?>`,`<testsuite name="${escapeXml(t.suiteName??`eve evals`)}" tests="${e.results.length}" failures="${n}" skipped="${e.skipped}" time="${formatSeconds(durationSeconds(e))}">`,...r,`</testsuite>`,``].join(`
+`)}function renderTestCase(e){let t=`classname="eve.eval" name="${escapeXml(e.id)}" time="${formatSeconds(durationSeconds(e))}"`;if(e.verdict===`passed`)return`  <testcase ${t}/>`;if(e.verdict===`skipped`)return[`  <testcase ${t}>`,`    <skipped message="${escapeXml(e.skipReason??`skipped`)}"/>`,`  </testcase>`].join(`
+`);let n=failureMessage(e);return[`  <testcase ${t}>`,`    <failure message="${escapeXml(n)}">${escapeXml(JSON.stringify(buildFailureDetail(e),null,2))}</failure>`,`  </testcase>`].join(`
+`)}function buildFailureDetail(e){return{verdict:e.verdict,error:e.error,assertions:e.assertions,logs:e.result.logs}}function failureMessage(e){if(e.error!==void 0)return e.error;let t=e.assertions.find(e=>!e.passed);return t===void 0?e.verdict===`scored`?`score below threshold`:e.skipReason??e.verdict:t.message===void 0?t.name:`${t.name}: ${t.message}`}function durationSeconds(e){let t=new Date(e.completedAt).getTime()-new Date(e.startedAt).getTime();return Math.max(0,t/1e3)}function formatSeconds(e){return e.toFixed(3)}function escapeXml(e){return e.replace(/&/g,`&amp;`).replace(/"/g,`&quot;`).replace(/'/g,`&apos;`).replace(/</g,`&lt;`).replace(/>/g,`&gt;`)}export{JUnit};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -1,20 +1,26 @@
-import type { EveEvalCaseResult, EveEvalSuite, EveEvalSuiteResult, EveEvalTarget } from "#evals/types.js";
+import type { EveEval, EveEvalResult, EveEvalRunSummary, EveEvalTarget } from "#evals/types.js";
 /**
  * Reporter lifecycle interface. The runner calls these methods at defined
- * points during suite execution. Methods may return a promise for reporters
+ * points during an eval run. Methods may return a promise for reporters
  * that perform asynchronous work (e.g. uploading to a remote service).
+ *
+ * Run-level reporters (console, JUnit) observe every eval in the run.
+ * Eval-defined reporters observe only the evals that reference them.
  */
 export interface EvalReporter {
     /**
-     * The runner calls this once when a suite run begins.
+     * The runner calls this once before any eval executes, with the evals
+     * this reporter observes.
      */
-    onSuiteStart(suite: EveEvalSuite, target: EveEvalTarget): void | Promise<void>;
+    onRunStart(evaluations: readonly EveEval[], target: EveEvalTarget): void | Promise<void>;
     /**
-     * The runner calls this after each case completes, with its scores.
+     * The runner calls this after each observed eval completes, with its
+     * checks, scores, and verdict.
      */
-    onCaseComplete(caseResult: EveEvalCaseResult): void | Promise<void>;
+    onEvalComplete(result: EveEvalResult): void | Promise<void>;
     /**
-     * The runner calls this once when a suite run finishes, with the aggregated result.
+     * The runner calls this once when the run finishes, with the aggregated
+     * summary of the evals this reporter observes.
      */
-    onSuiteComplete(suiteResult: EveEvalSuiteResult): void | Promise<void>;
+    onRunComplete(summary: EveEvalRunSummary): void | Promise<void>;
 }

package/dist/src/evals/runner/run-evals.d.ts

@@ -0,0 +1,38 @@
+import type { Client } from "#client/client.js";
+import type { EveEval, EveEvalConfig, EveEvalRunSummary, EveEvalTargetHandle } from "#evals/types.js";
+import type { EvalReporter } from "#evals/runner/reporters/types.js";
+/**
+ * Options for executing a set of evals as one run.
+ */
+export interface RunEvalsOptions {
+    readonly evaluations: readonly EveEval[];
+    /** Run-wide configuration from `evals.config.ts` (defaults shared by every eval). */
+    readonly config: EveEvalConfig;
+    readonly target: EveEvalTargetHandle;
+    readonly client: Client;
+    readonly appRoot: string;
+    /** Run-level reporters (console, JUnit) that observe every eval. */
+    readonly reporters: readonly EvalReporter[];
+    /** When false, eval-defined and config `reporters` are ignored (CLI `--skip-report`). */
+    readonly includeEvalReporters?: boolean;
+    readonly failOnSkip?: boolean;
+    /**
+     * Maximum number of evals executing at once. Must be a positive integer.
+     * Overrides the config `maxConcurrency`; defaults to 8 when neither is set.
+     */
+    readonly maxConcurrency?: number;
+    /** Overrides every eval's `timeoutMs` when set (CLI `--timeout`). */
+    readonly timeoutMs?: number;
+    /** Receives `ctx.log` lines as evals run (used by `--verbose`). */
+    readonly onEvalLog?: (evalId: string, message: string) => void;
+}
+/**
+ * Executes every eval with bounded concurrency, drives reporters, writes
+ * run artifacts under `.eve/evals/`, and returns the aggregated summary.
+ *
+ * Run-level reporters observe every eval. Eval-defined reporters observe
+ * only the evals that reference them; a reporter instance shared by several
+ * evals (e.g. one `Braintrust()` passed to every entry of an array export)
+ * is deduplicated and observes all of its evals as one group.
+ */
+export declare function runEvals(options: RunEvalsOptions): Promise<EveEvalRunSummary>;

package/dist/src/evals/runner/run-evals.js

@@ -0,0 +1 @@
+import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(r){let{config:i,target:a,client:o,appRoot:s}=r,c=r.maxConcurrency??i.maxConcurrency??8;if(!Number.isInteger(c)||c<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(r.maxConcurrency??i.maxConcurrency)}.`);let l=r.evaluations.map(e=>applyConfigDefaults(e,i)),u=new Date().toISOString(),d=buildReporterBindings({...r,evaluations:l});for(let e of d)await e.reporter.onRunStart(l.filter(t=>e.evalIds.has(t.id)),a);let f=[],p=[...l],m=new Set,h=Promise.resolve();for(;p.length>0||m.size>0;){for(;p.length>0&&m.size<c;){let e=p.shift();if(e===void 0)break;let t=(async()=>{let t=await executeEval({client:o,evaluation:e,failOnSkip:r.failOnSkip,onLog:r.onEvalLog===void 0?void 0:t=>r.onEvalLog?.(e.id,t),target:a,timeoutMs:r.timeoutMs});f.push(t),h=h.then(async()=>{for(let e of d)e.evalIds.has(t.id)&&await e.reporter.onEvalComplete(t)})})().finally(()=>{m.delete(t)});m.add(t)}m.size>0&&await Promise.race(m)}await h;let g=new Map(l.map((e,t)=>[e.id,t]));f.sort((e,t)=>(g.get(e.id)??0)-(g.get(t.id)??0));let _=buildSummary(a,f,u);await writeArtifacts(resolveArtifactDirectory(s),_);for(let e of d)await e.reporter.onRunComplete(scopeSummary(_,e.evalIds));return _}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){return e.judge!==void 0||t.judge===void 0?e:{...e,judge:t.judge}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0&&e.skipReason===void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0&&e.skipReason===void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};

package/dist/src/evals/runner/verdict.d.ts

@@ -1,18 +1,13 @@
-import type { EveEvalCaseVerdict, EveEvalCheckResult, EveEvalScorerResult } from "#evals/types.js";
+import type { AssertionResult, EveEvalVerdict } from "#evals/types.js";
 /**
- * Returns true when a scorer result meets its threshold. Null scores
- * are treated as passing (the scorer could not produce a result).
- * Scorers without a configured threshold default to 1.0 (exact match).
+ * Computes the per-eval verdict from the execution outcome and recorded
+ * assertions. An execution error or a failed gate assertion is a hard
+ * failure; a below-threshold soft assertion at worst demotes the eval to
+ * `"scored"` (which `eve eval --strict` later promotes to a failing exit
+ * code). Soft assertions without a threshold are tracked-only and never
+ * demote the verdict.
  */
-export declare function meetsThreshold(result: EveEvalScorerResult, thresholds?: Readonly<Record<string, number>>): boolean;
-/**
- * Computes the per-case verdict from execution outcome, check results, and
- * scores: execution errors and failed checks are hard failures; scores stay
- * soft and at worst demote the case to `"scored"`.
- */
-export declare function computeCaseVerdict(input: {
+export declare function computeEvalVerdict(input: {
     readonly error?: string;
-    readonly checks: readonly EveEvalCheckResult[];
-    readonly scores: readonly EveEvalScorerResult[];
-    readonly thresholds?: Readonly<Record<string, number>>;
-}): EveEvalCaseVerdict;
+    readonly assertions: readonly AssertionResult[];
+}): EveEvalVerdict;

package/dist/src/evals/runner/verdict.js

@@ -1 +1 @@
-function meetsThreshold(e,t){if(e.score===null)return!0;let n=t?.[e.name]??1;return e.score>=n}function computeCaseVerdict(e){return e.error!==void 0||e.checks.some(e=>!e.passed)?`failed`:e.scores.some(t=>!meetsThreshold(t,e.thresholds))?`scored`:`passed`}export{computeCaseVerdict,meetsThreshold};
+function computeEvalVerdict(e){if(e.error!==void 0)return`failed`;let t=!1;for(let n of e.assertions)if(!n.passed){if(n.severity===`gate`)return`failed`;t=!0}return t?`scored`:`passed`}export{computeEvalVerdict};

package/dist/src/evals/session.d.ts

@@ -0,0 +1,52 @@
+import type { Client } from "#client/client.js";
+import type { ClientSession } from "#client/session.js";
+import type { SendTurnInput, SessionState } from "#client/types.js";
+import type { HandleMessageStreamEvent, TurnFailureStreamEvent } from "#protocol/message.js";
+import type { InputRequest, InputResponse } from "#runtime/input/types.js";
+import type { EveEvalSession, EveEvalSessionResult, EveEvalTurn } from "#evals/types.js";
+/**
+ * Error thrown by {@link EveEvalTurn.expectOk} when a turn failed.
+ */
+export declare class EveEvalTurnFailedError extends Error {
+    readonly event: TurnFailureStreamEvent | undefined;
+    readonly turn: EveEvalTurn;
+    constructor(turn: EveEvalTurn);
+}
+export declare class EvalSessionDriver implements EveEvalSession {
+    #private;
+    constructor(input: {
+        readonly session: ClientSession;
+        readonly signal?: AbortSignal;
+    });
+    get events(): readonly HandleMessageStreamEvent[];
+    get lastTurn(): EveEvalTurn | undefined;
+    get pendingInputRequests(): readonly InputRequest[];
+    get sessionId(): string | undefined;
+    get state(): SessionState;
+    expectInputRequests(filter?: {
+        readonly display?: InputRequest["display"];
+        readonly toolName?: string;
+    }): readonly InputRequest[];
+    respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
+    respondAll(optionId: string): Promise<EveEvalTurn>;
+    send(input: SendTurnInput): Promise<EveEvalTurn>;
+    sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
+    readTurn(options?: {
+        readonly startIndex?: number;
+    }): Promise<EveEvalTurn>;
+    snapshot(primary: boolean): EveEvalSessionResult;
+}
+export declare class EvalSessionManager {
+    #private;
+    constructor(input: {
+        readonly client: Client;
+        readonly signal?: AbortSignal;
+    });
+    get primary(): EvalSessionDriver;
+    newSession(): EvalSessionDriver;
+    attachSession(sessionId: string, options?: {
+        readonly startIndex?: number;
+    }): Promise<EvalSessionDriver>;
+    snapshots(): readonly EveEvalSessionResult[];
+    lastTurnSession(): EvalSessionDriver | undefined;
+}

package/dist/src/evals/session.js

@@ -0,0 +1 @@
+import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=[];#r;#i=[];constructor(e){this.#e=e.session,this.#t=e.signal}get events(){return this.#n}get lastTurn(){return this.#r}get pendingInputRequests(){return this.#i}get sessionId(){return this.#e.state.sessionId??this.#r?.sessionId}get state(){return this.#e.state}expectInputRequests(e){if(this.#i.length===0)throw Error(`Expected pending input requests, but the last turn did not park.`);let t=this.#i.filter(t=>inputRequestMatches(t,e));if(t.length===0)throw Error(`No pending input requests matched ${formatInputRequestFilter(e)}.`);return t}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.expectInputRequests();for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#a({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#a({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:t,status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#n,{sessionId:t}),events:[...this.#n],primary:e,sessionId:t,state:this.#e.state}}#a(e){this.#n.push(...e.events),this.#i=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({data:e.data,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId??this.sessionId??``,status:e.status,toolCalls:t.toolCalls});return this.#r=n,n}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}},EvalSessionManager=class{#e;#t;#n=[];#r;constructor(e){this.#e=e.client,this.#t=e.signal}get primary(){return this.#r??=this.#i(),this.#r}newSession(){return this.#i()}async attachSession(e,t){let n=new EvalSessionDriver({session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#n.push(n),await n.readTurn(t),n}snapshots(){return this.#n.map(e=>e.snapshot(e===this.#r))}lastTurnSession(){return this.#r?.lastTurn===void 0?this.#n.find(e=>e.lastTurn!==void 0):this.#r}#i(){let e=new EvalSessionDriver({session:this.#e.session(),signal:this.#t});return this.#n.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function inputRequestMatches(e,t){return t===void 0?!0:t.display!==void 0&&e.display!==t.display?!1:t.toolName===void 0?!0:e.action.kind===`tool-call`&&e.action.toolName===t.toolName}function formatInputRequestFilter(e){return e===void 0?`{}`:JSON.stringify(e)}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};

package/dist/src/evals/target.d.ts

@@ -0,0 +1,23 @@
+import { Client } from "#client/client.js";
+import { EvalSessionManager } from "#evals/session.js";
+import type { EveEvalRequirement, EveEvalTargetCapabilities, EveEvalTargetHandle } from "#evals/types.js";
+export declare class EveEvalRequirementError extends Error {
+    readonly requirement: EveEvalRequirement;
+    constructor(requirement: EveEvalRequirement, message: string);
+}
+export declare function resolveEvalTargetHandle(input: {
+    readonly client: Client;
+    readonly expectedAgentName?: string;
+    readonly kind: "local" | "remote";
+    readonly url: string;
+}): Promise<EveEvalTargetHandle>;
+export declare function createEvalTargetHandle(input: {
+    readonly capabilities: EveEvalTargetCapabilities;
+    readonly client: Client;
+    readonly kind: "local" | "remote";
+    readonly url: string;
+}): EveEvalTargetHandle;
+export declare function scopeEvalTargetHandle(target: EveEvalTargetHandle, input: {
+    readonly requirements: readonly EveEvalRequirement[];
+    readonly sessions?: EvalSessionManager;
+}): EveEvalTargetHandle;

package/dist/src/evals/target.js

@@ -0,0 +1 @@
+import{createEveDevDispatchSchedulePath}from"#protocol/routes.js";import{toErrorMessage}from"#shared/errors.js";import"#client/client.js";import{EvalSessionManager}from"#evals/session.js";import{setTimeout}from"node:timers/promises";var EveEvalRequirementError=class extends Error{requirement;constructor(e,t){super(t),this.name=`EveEvalRequirementError`,this.requirement=e}};async function resolveEvalTargetHandle(e){await waitForTargetHealth(e.client,e.url);let t=await e.client.info();if(assertAgentInfoShape(t,e.url),e.expectedAgentName!==void 0&&t.agent.name!==e.expectedAgentName)throw Error(`Expected eval target ${JSON.stringify(e.expectedAgentName)} at ${e.url}, but ${JSON.stringify(t.agent.name)} is responding there.`);return createEvalTargetHandle({capabilities:capabilitiesFromInfo(t),client:e.client,kind:e.kind,url:e.url})}function createEvalTargetHandle(e){return createHandle({capabilities:e.capabilities,client:e.client,kind:e.kind,requireDeclared:!1,requirements:[],sessions:void 0,url:e.url})}function scopeEvalTargetHandle(e,t){return createHandle({capabilities:e.capabilities,client:void 0,delegate:e,kind:e.kind,requireDeclared:!0,requirements:t.requirements,sessions:t.sessions,url:e.url})}function createHandle(t){let r=new Set(t.requirements),i=t.delegate,a=t.client,fetchTarget=async(e,t)=>{if(i!==void 0)return await i.fetch(e,t);if(a===void 0)throw Error(`Eval target cannot fetch without a client.`);return await a.fetch(e,t)};return{capabilities:t.capabilities,kind:t.kind,url:t.url,async attachSession(e,r){if(t.sessions!==void 0)return await t.sessions.attachSession(e,r);if(i!==void 0)return await i.attachSession(e,r);if(a===void 0)throw Error(`Eval target cannot attach sessions without a client.`);return await new EvalSessionManager({client:a}).attachSession(e,r)},async dispatchSchedule(n){if(t.requireDeclared&&!r.has(`devRoutes`))throw new EveEvalRequirementError(`devRoutes`,'target.dispatchSchedule() requires declaring `requires: ["devRoutes"]` on the eval or case.');if(!t.capabilities.devRoutes)throw new EveEvalRequirementError(`devRoutes`,`target.dispatchSchedule() requires a target with dev routes enabled.`);let i=await fetchTarget(createEveDevDispatchSchedulePath(n),{method:`POST`});if(!i.ok){let e=await readResponseBodySafely(i);throw Error(`Schedule dispatch failed: ${i.status} ${i.statusText}`+(e.length>0?`, ${e}`:``))}return parseScheduleDispatchResult(await i.json())},async fetch(e,t){return await fetchTarget(e,t)}}}function capabilitiesFromInfo(e){return{devRoutes:e.capabilities?.devRoutes??e.mode===`development`,mockModels:e.capabilities?.mockModels??!1}}async function waitForTargetHealth(e,n){let i=Date.now()+6e4,a;for(;Date.now()<i;)try{await e.health();return}catch(e){a=toErrorMessage(e),await setTimeout(250)}throw Error(`Timed out waiting for eval target health at ${n}.`+(a===void 0?``:` Last error: ${a}`))}function assertAgentInfoShape(e,t){if(e.kind!==`eve-agent-info`||e.version!==1)throw Error(`Eval target ${t} returned an unrecognized /eve/v1/info payload.`)}function parseScheduleDispatchResult(e){if(typeof e!=`object`||!e||!(`scheduleId`in e)||typeof e.scheduleId!=`string`||!(`sessionIds`in e)||!Array.isArray(e.sessionIds)||e.sessionIds.some(e=>typeof e!=`string`))throw Error(`Schedule dispatch returned an unexpected response shape: ${JSON.stringify(e)}`);return{scheduleId:e.scheduleId,sessionIds:[...e.sessionIds]}}async function readResponseBodySafely(e){try{return(await e.text()).trim()}catch{return``}}export{EveEvalRequirementError,createEvalTargetHandle,resolveEvalTargetHandle,scopeEvalTargetHandle};