eve 0.6.0-beta.8 → 0.7.0

package/dist/src/evals/assertions/run.js

@@ -0,0 +1,2 @@
+import{deepEquals,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function completed(){return{name:`completed`,evaluate(e){return e.status===`failed`?fail(failureDetail(`run failed`,e.derived.failureCode)):e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS}}}function waiting(){return{name:`waiting`,evaluate(e){return e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`)}}}function didNotFail(){return{name:`didNotFail`,evaluate(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?PASS:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let r=joinCompletedMessages(t.events);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(r)}`)}}}function calledTool(e,t={}){return{name:`calledTool(${e})`,evaluate(n){let i=n.derived.toolCalls.filter(t=>t.name===e),a=i.filter(e=>toolCallMatches(e,t));if(t.times===void 0?a.length>0:a.length===t.times)return{score:1,metadata:{matchingCalls:a.length}};let o=i.length>0?`observed ${e} calls: ${i.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`;return fail(`${t.times===void 0?`expected a matching call to "${e}"`:`expected exactly ${t.times} matching call(s), found ${a.length}`}; ${o}`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=t.derived.toolCalls.map(e=>e.name),r=0;for(let t of n)if(t===e[r]&&(r+=1),r===e.length)break;return r===e.length?PASS:fail(`missing "${e[r]}" after [${e.slice(0,r).join(`, `)}]; observed order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(e=>e.data.result.kind===`tool-result`?e.data.result.toolName:e.data.result.kind);return fail(`${t.length} failed action(s): ${n.join(`, `)}`)}}}function calledSubagent(e,n={}){return{name:`calledSubagent(${e})`,evaluate(r){let a=r.derived.subagentCalls.filter(t=>t.name===e);return a.filter(e=>subagentCallMatches(e,n)).length>0?PASS:a.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but no call matched the constraints`,{observedSubagentCalls:a})}}}function event(e,t){return{name:`event(${t})`,evaluate(n){return e(n.events)?PASS:fail(`event predicate "${t}" did not hold`)}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}export{calledSubagent,calledTool,completed,didNotFail,event,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,outputEquals,outputMatches,toolOrder,usedNoTools,waiting};

package/dist/src/evals/autoevals-client.js

@@ -0,0 +1,2 @@
+import"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{generateText,jsonSchema}from"ai";function createAutoevalsClient(e){return{chat:{completions:{create:t=>createChatCompletion(t,e)}}}}async function createChatCompletion(t,n){let r=convertTools(t.tools),i=await generateText({model:n.languageModel,messages:convertMessages(t.messages??[]),tools:Object.keys(r).length>0?r:void 0,toolChoice:convertToolChoice(t.tool_choice),providerOptions:n.providerOptions}),a=i.toolCalls.map(e=>({id:e.toolCallId,type:`function`,function:{name:e.toolName,arguments:JSON.stringify(e.input??{})}}));return{choices:[{index:0,finish_reason:a.length>0?`tool_calls`:`stop`,message:{role:`assistant`,content:i.text||null,tool_calls:a.length>0?a:void 0}}]}}function convertMessages(e){return e.map(e=>{let t=contentToText(e.content);switch(e.role){case`assistant`:return{role:`assistant`,content:t};case`developer`:case`system`:return{role:`system`,content:t};default:return{role:`user`,content:t}}})}function contentToText(e){return e==null?``:typeof e==`string`?e:e.map(e=>e.text??``).filter(Boolean).join(`
+`)}function convertTools(e){let n={};for(let r of e??[])r.type!==`function`||r.function?.name===void 0||(n[r.function.name]={description:r.function.description,inputSchema:jsonSchema(r.function.parameters??{})});return n}function convertToolChoice(e){if(e!==void 0)return typeof e==`string`?e:{type:`tool`,toolName:e.function.name}}export{createAutoevalsClient};

package/dist/src/evals/cli/eval-client.d.ts

@@ -0,0 +1,22 @@
+import { Client } from "#client/client.js";
+import type { ClientOptions } from "#client/types.js";
+import type { EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Resolves the {@link ClientOptions} for an eval target.
+ *
+ * Local targets need no auth. Remote targets connect with the same options
+ * as every other development client (`resolveDevelopmentClientOptions`):
+ * per-request headers carrying the Vercel OIDC trusted-IDP token (which
+ * bypasses Deployment Protection without a per-project secret) plus
+ * `x-vercel-protection-bypass` when `VERCEL_AUTOMATION_BYPASS_SECRET` is
+ * set, and a bearer resolved from the same OIDC cascade.
+ *
+ * `EVE_EVAL_AUTH_TOKEN` overrides the bearer with a static token for
+ * targets whose auth is not OIDC-based.
+ */
+export declare function resolveEvalClientOptions(target: Pick<EveEvalTargetHandle, "kind" | "url">): ClientOptions;
+/**
+ * Creates the Eve {@link Client} for an eval target from
+ * {@link resolveEvalClientOptions}.
+ */
+export declare function createEvalClient(target: Pick<EveEvalTargetHandle, "kind" | "url">): Client;

package/dist/src/evals/cli/eval-client.js

@@ -0,0 +1 @@
+import{resolveDevelopmentClientOptions}from"#services/dev-client/client-options.js";import{Client}from"#client/client.js";function resolveEvalClientOptions(t){if(t.kind===`local`)return{host:t.url};let n=resolveDevelopmentClientOptions(t.url),r=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return r?{...n,auth:{bearer:r}}:n}function createEvalClient(e){return new Client(resolveEvalClientOptions(e))}export{createEvalClient,resolveEvalClientOptions};

package/dist/src/evals/cli/eval.d.ts

@@ -1,11 +1,16 @@
 interface EvalCliOptions {
-    suite?: string[];
     url?: string;
     timeout?: string;
     maxConcurrency?: string;
     json?: boolean;
-    all?: boolean;
+    junit?: string;
     skipReport?: boolean;
+    strict?: boolean;
+    list?: boolean;
+    mockModels?: boolean;
+    noSkips?: boolean;
+    tag?: string[];
+    verbose?: boolean;
 }
 type EvalCliLogger = {
     log(message: string): void;
@@ -13,6 +18,10 @@ type EvalCliLogger = {
 };
 /**
  * Runs the `eve eval` command with already-parsed Commander options.
+ *
+ * Exit codes: `0` when every executed eval passed its gate assertions (and
+ * soft thresholds under `--strict`), `1` when any eval failed, `2` for runner
+ * or configuration errors (no evals discovered, no evals matching filters).
  */
-export declare function runEvalCommand(options: EvalCliOptions, logger: EvalCliLogger): Promise<void>;
+export declare function runEvalCommand(evalIds: readonly string[], options: EvalCliOptions, logger: EvalCliLogger): Promise<void>;
 export {};

package/dist/src/evals/cli/eval.js

@@ -1 +1 @@
-import{VERCEL_PROTECTION_BYPASS_HEADER}from"#services/dev-client/request-headers.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{Client}from"#client/client.js";import{getVercelOidcToken}from"#compiled/@vercel/oidc/index.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{discoverAndImportSuites}from"#evals/runner/discover.js";import{executeSuite}from"#evals/runner/execute-suite.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";async function runEvalCommand(e,n){let r=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(r);let s=e.suite,c=await discoverAndImportSuites(r,s);if(c.length===0){s&&s.length>0?n.error(`No suites found matching: ${s.join(`, `)}`):n.error(`No eval suites found. Create suite files under evals/ with the *.eval.ts extension.`),process.exitCode=1;return}let l,u;e.url?u={kind:`remote`,url:e.url}:(l=await startDevelopmentServer(r,{host:`127.0.0.1`,port:0}),u={kind:`local`,url:l.url});let d=createEvalClient(u);try{let t=[];for(let n of c){let i=applyCliOverrides(n,e),a=buildReporters(i,{json:e.json===!0,skipReport:e.skipReport===!0}),o=await executeSuite({suite:i,target:u,reporters:a,appRoot:r,client:d});t.push(o)}e.json&&n.log(JSON.stringify(t,null,2)),t.some(e=>e.errored>0)&&(process.exitCode=1)}finally{l&&await l.close()}process.exit(process.exitCode??0)}function createEvalClient(t){if(t.kind===`local`)return new Client({host:t.url});let r={},i=process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim();return i&&(r[VERCEL_PROTECTION_BYPASS_HEADER]=i),new Client({auth:resolveRemoteBearerAuth(),headers:Object.keys(r).length>0?r:void 0,host:t.url})}function resolveRemoteBearerAuth(){let e=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return e?{bearer:e}:{bearer:resolveOidcBearerToken}}async function resolveOidcBearerToken(){try{let e=(await getVercelOidcToken()).trim();if(e.length>0)return e}catch{}return process.env.VERCEL_OIDC_TOKEN?.trim()??``}function applyCliOverrides(e,t){let n=t.maxConcurrency?Number.parseInt(t.maxConcurrency,10):void 0,r=t.timeout?Number.parseInt(t.timeout,10):void 0;if(n===void 0&&r===void 0)return e;let i={...e};return n!==void 0&&(i.maxConcurrency=n),r!==void 0&&(i.timeoutMs=r),i}function buildReporters(e,t){let n=t.json?[]:[new ConsoleReporter];return!t.skipReport&&e.reporters&&n.push(...e.reporters),n}export{runEvalCommand};
+import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{resolveApplicationRoot}from"#internal/application/paths.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{EVE_MOCK_AUTHORED_MODELS_ENV}from"#runtime/agent/resolve-model.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let s=resolveApplicationRoot();if(t.url&&t.mockModels===!0){n.error(`--mock-models cannot be used with --url. The target's mock-model state is discovered from /eve/v1/info.`),process.exitCode=2;return}let c=t.mockModels===!0?setEnvForRun(EVE_MOCK_AUTHORED_MODELS_ENV,`1`):void 0,l;try{loadDevelopmentEnvironmentFiles(s);let r=e.length>0?e:void 0,a=await discoverAndImportEvals(s,r);if(a.length===0){r?n.error(`No evals found matching: ${r.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let c=filterEvalsByTag(a,t.tag??[]);if(c.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(c,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(s)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m;try{t.url?m=await resolveEvalTargetHandle({client:createEvalClient({kind:`remote`,url:t.url}),expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:t.url}):(p=await startDevelopmentServer(s,{host:`127.0.0.1`,port:0}),m=await resolveEvalTargetHandle({client:createEvalClient({kind:`local`,url:p.url}),expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:p.url}));let e=createEvalClient(m),r=t.json===!0?[]:[new ConsoleReporter];t.junit!==void 0&&r.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:c,config:f,target:m,client:e,appRoot:s,reporters:r,includeEvalReporters:t.skipReport!==!0,failOnSkip:t.noSkips===!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{p&&await p.close()}l=typeof process.exitCode==`number`?process.exitCode:0}finally{c?.()}l!==void 0&&process.exit(l)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}function setEnvForRun(e,t){let n=process.env[e];return process.env[e]=t,()=>{n===void 0?delete process.env[e]:process.env[e]=n}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};

package/dist/src/evals/context.d.ts

@@ -0,0 +1,19 @@
+import { EvalSessionManager } from "#evals/session.js";
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalContext, EveEvalJudgeConfig, EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Builds the `EveEvalContext` (`t`) for one eval run, wiring the session
+ * manager (driving), the assertion collector (recording), and the judge
+ * namespace. Returns the collector so the runner can {@link
+ * AssertionCollector.finalize} it against the completed task result.
+ */
+export declare function createEvalContext(deps: {
+    readonly manager: EvalSessionManager;
+    readonly target: EveEvalTargetHandle;
+    readonly signal: AbortSignal;
+    readonly judge: EveEvalJudgeConfig | undefined;
+    readonly log: (message: string) => void;
+}): {
+    readonly context: EveEvalContext;
+    readonly collector: AssertionCollector;
+};

package/dist/src/evals/context.js

@@ -0,0 +1 @@
+import"#evals/session.js";import{AssertionCollector}from"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{buildJudgeContext}from"#evals/judge.js";function createEvalContext(r){let i=new AssertionCollector,a=``,primary=()=>r.manager.primary,replyMessage=()=>r.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:r.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},expectInputRequests:e=>primary().expectInputRequests(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:r.signal,target:r.target,get reply(){return replyMessage()},log:r.log,newSession:()=>r.manager.newSession(),completed:()=>i.recordRun(RunAssertions.completed()),didNotFail:()=>i.recordRun(RunAssertions.didNotFail()),waiting:()=>i.recordRun(RunAssertions.waiting()),messageIncludes:e=>i.recordRun(RunAssertions.messageIncludes(e)),calledTool:(e,n)=>i.recordRun(RunAssertions.calledTool(e,n)),notCalledTool:e=>i.recordRun(RunAssertions.notCalledTool(e)),toolOrder:e=>i.recordRun(RunAssertions.toolOrder(e)),usedNoTools:()=>i.recordRun(RunAssertions.usedNoTools()),maxToolCalls:e=>i.recordRun(RunAssertions.maxToolCalls(e)),calledSubagent:(e,n)=>i.recordRun(RunAssertions.calledSubagent(e,n)),noFailedActions:()=>i.recordRun(RunAssertions.noFailedActions()),event:(e,n)=>i.recordRun(RunAssertions.event(e,n)),outputEquals:e=>i.recordRun(RunAssertions.outputEquals(e)),outputMatches:e=>i.recordRun(RunAssertions.outputMatches(e)),check:(e,t)=>recordCheck(i,e,t),judge:o},collector:i}}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:async()=>({score:await n.score(t)})})}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}export{createEvalContext};

package/dist/src/evals/define-eval-config.d.ts

@@ -0,0 +1,16 @@
+import type { EveEvalConfig, EveEvalConfigInput } from "#evals/types.js";
+/**
+ * Defines the run-wide configuration shared by every eval, authored as the
+ * default export of `evals.config.ts` at the root of the `evals/` directory.
+ *
+ * Exactly one `evals.config.ts` is required. It supplies the optional default
+ * `judge` model for `t.judge.*` assertions (so individual evals need not
+ * repeat it), optional run-level `reporters`, a default `maxConcurrency`, and a
+ * default `timeoutMs`. CLI flags (`--max-concurrency`, `--timeout`) and
+ * per-eval values take precedence over the config defaults.
+ *
+ * Throws on invalid input: a `judge` without a `model`, a non-positive or
+ * non-integer `maxConcurrency`, a negative or non-finite `timeoutMs`, or a
+ * non-array `reporters`.
+ */
+export declare function defineEvalConfig(input: EveEvalConfigInput): EveEvalConfig;

package/dist/src/evals/define-eval-config.js

@@ -0,0 +1 @@
+function defineEvalConfig(e){return validateEvalConfigInput(e),{...e,_tag:`EveEvalConfig`}}function validateEvalConfigInput(e){if(e.judge!==void 0&&(e.judge.model===void 0||e.judge.model===null))throw Error("Eval config `judge` requires a `model`. It is the default judge model for `t.judge.*` assertions across every eval.");if(e.maxConcurrency!==void 0&&(!Number.isInteger(e.maxConcurrency)||e.maxConcurrency<1))throw Error("Eval config `maxConcurrency` must be a positive integer.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval config `timeoutMs` must be a non-negative finite number.");if(e.reporters!==void 0&&!Array.isArray(e.reporters))throw Error("Eval config `reporters` must be an array of reporters.")}export{defineEvalConfig};

package/dist/src/evals/define-eval.d.ts

@@ -0,0 +1,20 @@
+import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";
+/**
+ * Defines one Eve eval. Each eval file is exactly one case: an imperative
+ * `test(t)` function that drives the agent (`t.send`, `t.respond`, …) and
+ * asserts on what it produced (`t.completed()`, `t.check(...)`,
+ * `t.judge.autoevals.*`). Organize related evals with directory nesting under
+ * `evals/`, or default-export an array of evals to fan one file out over a
+ * dataset.
+ *
+ * A `judge` is optional: `t.judge.*` assertions fall back to the `judge`
+ * declared in `evals.config.ts` unless this eval overrides it. The judge model
+ * is used solely for scoring, never for the agent under test. Eval identity is
+ * derived from the `evals/<path>.eval.ts` file path by the discovery layer, so
+ * authoring `id` or `name` throws.
+ *
+ * Throws on invalid input: a missing `test` function, a removed legacy key
+ * (`input`/`run`/`checks`/`scores`/`expected`/`thresholds`/`parseOutput`/
+ * `model`), or a negative or non-finite `timeoutMs`.
+ */
+export declare function defineEval(input: EveEvalInput): EveEvalDefinition;

package/dist/src/evals/define-eval.js

@@ -0,0 +1 @@
+function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.completed()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(validateRequirements(e.requires,"Eval `requires`"),e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}function validateRequirements(e,t){if(e!==void 0){if(!Array.isArray(e))throw Error(`${t} must be an array of requirement strings.`);for(let n of e)if(!(n===`mockModels`||n===`devRoutes`)&&!(n.startsWith(`env:`)&&n.length>4))throw Error(`${t} contains unsupported requirement "${n}". Expected "mockModels", "devRoutes", or "env:<NAME>".`)}}export{defineEval};

package/dist/src/evals/expect/index.d.ts

@@ -0,0 +1,25 @@
+import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
+import type { Assertion } from "#evals/types.js";
+export type { Assertion, AssertionHandle, AssertionSeverity } from "#evals/types.js";
+/**
+ * Passes when the value (coerced to a string) contains `substring`. A hard
+ * gate by default. Apply with `t.check(value, includes("..."))`.
+ */
+export declare function includes(substring: string): Assertion;
+/**
+ * Passes when the value deep-equals `expected` (exact structural equality).
+ * A hard gate by default.
+ */
+export declare function equals(expected: unknown): Assertion;
+/**
+ * Passes when the value validates against a Standard Schema (e.g. a Zod
+ * schema). A hard gate by default.
+ */
+export declare function matches(schema: StandardSchemaV1): Assertion;
+/**
+ * Scores normalized character-level Levenshtein similarity between the value
+ * and `expected` (1 = identical, 0 = entirely different). Soft by default —
+ * tracked unless you set a bar with `.atLeast(...)`. Use it for fuzzy
+ * comparison when exact match is too strict but a judge model is overkill.
+ */
+export declare function similarity(expected: string): Assertion;

package/dist/src/evals/expect/index.js

@@ -0,0 +1 @@
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>+!!String(t??``).includes(e)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,similarity};

package/dist/src/evals/index.d.ts

@@ -1,3 +1,8 @@
-export { defineEvalSuite } from "#evals/define-eval-suite.js";
+export { defineEval } from "#evals/define-eval.js";
+export { defineEvalConfig } from "#evals/define-eval-config.js";
+export { EveEvalTurnFailedError } from "#evals/session.js";
+export { EveEvalRequirementError } from "#evals/target.js";
 export type { RuntimeIdentity } from "#protocol/message.js";
-export type { EveEvalCase, EveEvalCaseResult, EveEvalDerivedFacts, EveEvalScorer, EveEvalScorerArgs, EveEvalScorerResult, EveEvalSuite, EveEvalSuiteDefinition, EveEvalSuiteInput, EveEvalSuiteResult, EveEvalTarget, EveEvalTask, EveEvalTaskResult, } from "#evals/types.js";
+export type { InputRequest } from "#runtime/input/types.js";
+export type { EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
+export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalRequirement, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";

package/dist/src/evals/index.js

@@ -1 +1 @@
-import{defineEvalSuite}from"#evals/define-eval-suite.js";export{defineEvalSuite};
+import{EveEvalRequirementError}from"#evals/target.js";import{EveEvalTurnFailedError}from"#evals/session.js";import{defineEval}from"#evals/define-eval.js";import{defineEvalConfig}from"#evals/define-eval-config.js";export{EveEvalRequirementError,EveEvalTurnFailedError,defineEval,defineEvalConfig};

package/dist/src/evals/judge.d.ts

@@ -0,0 +1,20 @@
+import type { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalJudgeConfig, JudgeContext } from "#evals/types.js";
+/**
+ * Dependencies the judge namespace closes over: where to record assertions,
+ * how to resolve the default graded value (`t.reply`) and the original prompt
+ * (the autoevals `input`), and the eval/config judge model.
+ */
+export interface JudgeDeps {
+    readonly collector: AssertionCollector;
+    readonly getReply: () => unknown;
+    readonly getInput: () => string;
+    readonly judge: EveEvalJudgeConfig | undefined;
+}
+/**
+ * Builds the `t.judge` namespace bound to the resolved judge model. Each
+ * grader records a soft assertion (override with `.atLeast`/`.gate`) that
+ * fires the model call immediately; the collector awaits it before the
+ * verdict.
+ */
+export declare function buildJudgeContext(deps: JudgeDeps): JudgeContext;

package/dist/src/evals/judge.js

@@ -0,0 +1 @@
+import{ClosedQA,Factuality,Sql,Summary}from"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import{createAutoevalsClient}from"#evals/autoevals-client.js";function buildJudgeContext(a){function grade(e,t,n){let r=t?.on??a.getReply(),o=String(r??``),s=a.getInput(),c=t?.model??a.judge?.model,l=t?.modelOptions??a.judge?.modelOptions;return a.collector.recordValue({name:e,severity:`soft`,score:async()=>{if(c===void 0)throw Error(`${e} needs a judge model. Set \`judge\` on the eval or in evals.config.ts, or pass { model } to the call.`);return{score:(await n({input:s,output:o,model:c,modelOptions:l})).score??0,metadata:{judge:formatLanguageModelGatewayId(c)}}}})}return{autoevals:{factuality:(e,n)=>grade(`judge.autoevals.factuality`,n,({input:n,output:r,model:i,modelOptions:a})=>Factuality({input:n,output:r,expected:e,...client(i,a)})),summarizes:(e,t)=>grade(`judge.autoevals.summarizes`,t,({input:t,output:n,model:i,modelOptions:a})=>Summary({input:t,output:n,expected:e,...client(i,a)})),closedQA:(t,n)=>grade(`judge.autoevals.closedQA`,n,({input:n,output:r,model:i,modelOptions:a})=>ClosedQA({input:n,output:r,criteria:t,...client(i,a)})),sql:(e,t)=>grade(`judge.autoevals.sql`,t,({input:t,output:r,model:i,modelOptions:a})=>Sql({input:t,output:r,expected:e,...client(i,a)}))}}}function client(e,t){return{model:formatLanguageModelGatewayId(e),client:createAutoevalsClient({languageModel:e,providerOptions:t?.providerOptions})}}export{buildJudgeContext};

package/dist/src/evals/match.d.ts

@@ -0,0 +1,67 @@
+import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
+/**
+ * One matcher accepted by the assertion options (`t.calledTool`,
+ * `t.calledSubagent`):
+ *
+ * - a **literal** is compared structurally; objects partial-deep-match (every
+ *   key in the matcher must match the observed value, recursively, and nested
+ *   values are matchers themselves), arrays match element-wise, primitives
+ *   compare with `Object.is`
+ * - a **RegExp** tests string values directly and the JSON serialization of
+ *   anything else
+ * - a **function** receives the observed value and either returns a boolean
+ *   verdict, or returns a resolved expected value that is then compared like a
+ *   literal — e.g. `(o) => o === process.env.EVE_WEATHER_AGENT_HOST`. To assert
+ *   a literal boolean field, use the literal directly; boolean returns are
+ *   always treated as verdicts.
+ */
+export type EveEvalValueMatcher<T = unknown> = T | RegExp | ((value: T) => unknown);
+/**
+ * Constraints applied to tool calls by `t.calledTool`. All provided
+ * constraints must hold for a call to match.
+ */
+export interface EveEvalToolCallMatchOptions {
+    /** Partial-deep matcher over the call input (see {@link EveEvalValueMatcher}). */
+    readonly input?: EveEvalValueMatcher;
+    /** Matcher over the call output. */
+    readonly output?: EveEvalValueMatcher;
+    /** Required error state of matching calls. */
+    readonly isError?: boolean;
+    /** Exact number of matching calls required. Defaults to "at least one". */
+    readonly times?: number;
+}
+/**
+ * Constraints applied to subagent calls by `t.calledSubagent`.
+ */
+export interface EveEvalSubagentCallMatchOptions {
+    /** Matcher over the `subagent.called` remote URL. */
+    readonly remoteUrl?: EveEvalValueMatcher;
+    /** Matcher over the `subagent.completed` output. */
+    readonly output?: EveEvalValueMatcher;
+}
+/**
+ * Returns true when the observed value satisfies a matcher (literal, RegExp,
+ * or function — see {@link EveEvalValueMatcher}).
+ */
+export declare function matchesValue(matcher: unknown, value: unknown): boolean;
+/**
+ * Returns true when one derived tool call satisfies the `input`/`output`/
+ * `isError` constraints (the `times` count is the caller's concern).
+ */
+export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
+/**
+ * Returns true when one derived subagent call satisfies the `remoteUrl`/
+ * `output` constraints.
+ */
+export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions): boolean;
+/**
+ * Strict structural equality used by `t.outputEquals`: unlike matcher
+ * comparison, objects must carry exactly the same keys on both sides.
+ */
+export declare function deepEquals(a: unknown, b: unknown): boolean;
+/**
+ * Tests a RegExp without carrying `lastIndex` state between calls. Matcher
+ * patterns are reused across tool calls and across every case in an eval, so
+ * a `g`/`y`-flagged pattern would otherwise return order-dependent results.
+ */
+export declare function testRegExp(pattern: RegExp, text: string): boolean;

package/dist/src/evals/match.js

@@ -0,0 +1 @@
+function matchesValue(e,t){if(e instanceof RegExp)return testRegExpAgainst(e,t);if(typeof e==`function`){let n=e(t);return typeof n==`boolean`?n:typeof n==`function`?Object.is(n,t):matchesValue(n,t)}return Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||t.isError!==void 0&&e.isError!==t.isError)}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};

package/dist/src/evals/reporters/index.d.ts

@@ -1,2 +1,3 @@
 export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js";
+export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
 export type { EvalReporter } from "#evals/runner/reporters/types.js";

package/dist/src/evals/reporters/index.js

@@ -1 +1 @@
-import{Braintrust}from"#evals/runner/reporters/braintrust.js";export{Braintrust};
+import{JUnit}from"#evals/runner/reporters/junit.js";import{Braintrust}from"#evals/runner/reporters/braintrust.js";export{Braintrust,JUnit};

package/dist/src/evals/requirements.d.ts

@@ -0,0 +1,3 @@
+import type { EveEvalRequirement, EveEvalTarget } from "#evals/types.js";
+export declare function findUnmetRequirements(requirements: readonly EveEvalRequirement[], target: EveEvalTarget, env?: NodeJS.ProcessEnv): readonly EveEvalRequirement[];
+export declare function formatUnmetRequirements(requirements: readonly EveEvalRequirement[]): string;

package/dist/src/evals/requirements.js

@@ -0,0 +1 @@
+function findUnmetRequirements(e,t,n=process.env){let r=[];for(let i of new Set(e))requirementMet(i,t,n)||r.push(i);return r}function formatUnmetRequirements(e){return e.length===0?``:`Unmet eval requirement${e.length===1?``:`s`}: ${e.map(formatRequirement).join(`, `)}`}function requirementMet(e,t,n){if(e===`mockModels`)return t.capabilities.mockModels;if(e===`devRoutes`)return t.capabilities.devRoutes;let r=e.slice(4);return r.length>0&&n[r]!==void 0}function formatRequirement(e){return e===`mockModels`?`"mockModels" (local evals can enable this with --mock-models)`:e===`devRoutes`?`"devRoutes"`:`"${e}"`}export{findUnmetRequirements,formatUnmetRequirements};

package/dist/src/evals/runner/artifacts.d.ts

@@ -1,11 +1,12 @@
-import type { EveEvalSuiteResult } from "#evals/types.js";
+import type { EveEvalRunSummary } from "#evals/types.js";
 /**
- * Resolves the artifact output directory for one suite run.
+ * Resolves the artifact output directory for one `eve eval` run.
  *
- * Layout: `.eve/evals/<timestamp>-<suite-id>/`
+ * Layout: `.eve/evals/<timestamp>/`
  */
-export declare function resolveArtifactDirectory(appRoot: string, suiteId: string): string;
+export declare function resolveArtifactDirectory(appRoot: string): string;
 /**
- * Writes all artifacts for a completed suite run.
+ * Writes all artifacts for a completed eval run: a run summary, a JSONL
+ * results index, and per-eval detail/event files under `evals/`.
  */
-export declare function writeArtifacts(artifactDir: string, suiteResult: EveEvalSuiteResult): Promise<void>;
+export declare function writeArtifacts(artifactDir: string, summary: EveEvalRunSummary): Promise<void>;

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
-import{join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(t,n){return join(t,`.eve`,`evals`,`${new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19)}-${n}`)}async function writeArtifacts(r,i){let a=join(r,`cases`);await mkdir(a,{recursive:!0}),await writeFile(join(r,`summary.json`),JSON.stringify(buildSummaryArtifact(i),null,2));let o=i.cases.map(e=>JSON.stringify(buildCaseResultLine(e))).join(`
-`);await writeFile(join(r,`results.jsonl`),`${o}\n`),await Promise.all(i.cases.map(async t=>{let r=sanitizeFileName(t.case.id);await writeFile(join(a,`${r}.json`),JSON.stringify(buildCaseArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(a,`${r}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{suite:e.suite,target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,errored:e.errored,totalCases:e.cases.length,cases:e.cases.map(e=>({id:e.case.id,status:e.result.status,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}))}}function buildCaseResultLine(e){return{caseId:e.case.id,status:e.result.status,output:e.result.output,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error}}function buildCaseArtifact(e){return{case:{id:e.case.id,input:e.case.input,expected:e.case.expected,tags:e.case.tags,metadata:e.case.metadata},result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,derived:e.result.derived},scores:e.scores,error:e.error}}function sanitizeFileName(e){return e.replace(/[^a-zA-Z0-9_-]/g,`_`)}export{resolveArtifactDirectory,writeArtifacts};
+import{dirname,join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
+`);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};

package/dist/src/evals/runner/derive-run-facts.d.ts

@@ -1,9 +1,23 @@
 import type { HandleMessageStreamEvent } from "#protocol/message.js";
 import type { EveEvalDerivedFacts } from "#evals/types.js";
+/**
+ * Options for {@link deriveRunFacts}.
+ */
+export interface DeriveRunFactsOptions {
+    /** Session id stamped onto every derived tool and subagent call. */
+    readonly sessionId?: string;
+}
 /**
  * Extracts derived execution facts from a completed run's stream events.
  *
- * These facts summarize tool usage, subagent calls, message counts, and
- * failure information for scoring and reporting.
+ * Tool calls pair each `actions.requested` entry with its matching
+ * `action.result` by call id; subagent calls join `subagent.called` /
+ * `subagent.started` with `subagent.completed` the same way. These facts
+ * power checks, scorers, and reporters.
+ */
+export declare function deriveRunFacts(events: readonly HandleMessageStreamEvent[], options?: DeriveRunFactsOptions): EveEvalDerivedFacts;
+/**
+ * Returns empty derived facts, used when a case produced no events
+ * (execution errors, transport failures).
  */
-export declare function deriveRunFacts(events: readonly HandleMessageStreamEvent[]): EveEvalDerivedFacts;
+export declare function createEmptyDerivedFacts(): EveEvalDerivedFacts;

package/dist/src/evals/runner/derive-run-facts.js

@@ -1 +1 @@
-function deriveRunFacts(e){let t=[],n=[],r=0,i=0,a;for(let o of e)switch(o.type){case`actions.requested`:for(let e of o.data.actions)`toolName`in e&&typeof e.toolName==`string`&&t.push(e.toolName);break;case`subagent.called`:n.push(o.data.name);break;case`subagent.started`:n.push(o.data.subagentName);break;case`message.completed`:o.data.finishReason!==`tool-calls`&&(r+=1);break;case`reasoning.completed`:i+=1;break;case`session.failed`:a=o.data.code;break}return{toolCalls:t,toolCallCount:t.length,subagentCalls:deduplicateStrings(n),subagentCallCount:n.length,messageCount:r,reasoningBlockCount:i,failureCode:a}}function deduplicateStrings(e){return[...new Set(e)]}export{deriveRunFacts};
+const TURN_EPILOGUE_EVENT_TYPES=new Set([`turn.completed`,`session.waiting`,`session.completed`]);function deriveRunFacts(e,t){let n=t?.sessionId,r=[],i=new Map,a=[],o=new Map,s=[],c=-1,l=0,u=0,d,ensureSubagentCall=(e,t)=>{let r=o.get(e);if(r!==void 0)return r;let i={name:t,isError:!1,turnIndex:Math.max(c,0),sessionId:n};return a.push(i),o.set(e,i),i};for(let t of e)switch(t.type){case`turn.started`:c+=1;break;case`actions.requested`:for(let e of t.data.actions){if(e.kind!==`tool-call`)continue;let t={name:e.toolName,input:e.input,output:void 0,isError:!1,turnIndex:Math.max(c,0),sessionId:n};r.push(t),i.set(e.callId,t)}break;case`action.result`:{let{result:e,status:n}=t.data,r=n===`failed`||e.isError===!0;if(e.kind===`tool-result`){let t=i.get(e.callId);t!==void 0&&(t.output=e.output,t.isError=r)}else if(e.kind===`subagent-result`){let t=o.get(e.callId);t!==void 0&&(t.output=t.output??e.output,t.isError=r)}break}case`subagent.called`:{let e=ensureSubagentCall(t.data.callId,t.data.name);t.data.remote!==void 0&&(e.remoteUrl=t.data.remote.url);break}case`subagent.started`:ensureSubagentCall(t.data.callId,t.data.subagentName);break;case`subagent.completed`:{let e=ensureSubagentCall(t.data.callId,t.data.subagentName);e.output=t.data.output;break}case`input.requested`:s.push(...t.data.requests);break;case`message.completed`:t.data.finishReason!==`tool-calls`&&(l+=1);break;case`reasoning.completed`:u+=1;break;case`session.failed`:d=t.data.code;break}return{toolCalls:r,toolCallCount:r.length,subagentCalls:a,subagentCallCount:a.length,inputRequests:s,parked:endedParkedOnInput(e),messageCount:l,reasoningBlockCount:u,failureCode:d}}function createEmptyDerivedFacts(){return{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,inputRequests:[],parked:!1,messageCount:0,reasoningBlockCount:0}}function endedParkedOnInput(t){for(let n=t.length-1;n>=0;n--){let r=t[n];if(!(r===void 0||TURN_EPILOGUE_EVENT_TYPES.has(r.type)))return r.type===`input.requested`}return!1}export{createEmptyDerivedFacts,deriveRunFacts};

package/dist/src/evals/runner/discover.d.ts

@@ -1,20 +1,41 @@
-import type { EveEvalSuite } from "#evals/types.js";
+import type { EveEval, EveEvalConfig } from "#evals/types.js";
 /**
- * Discovers eval suite files under `<appRoot>/evals/` by recursively
+ * Discovers eval files under `<appRoot>/evals/` by recursively
  * scanning for files matching `*.eval.ts`.
  *
  * Returns absolute paths sorted alphabetically by relative path.
  */
-export declare function discoverSuiteFiles(appRoot: string): Promise<string[]>;
+export declare function discoverEvalFiles(appRoot: string): Promise<string[]>;
 /**
- * Imports a discovered suite file and stamps the path-derived id onto
- * the suite definition.
+ * Returns true when `evalId` matches one of the requested filters. A filter
+ * matches its exact eval id or any eval nested under it, so `"runtime"`
+ * matches both `evals/runtime.eval.ts` and every eval in `evals/runtime/`
+ * (and every entry of an array-exported `evals/runtime.eval.ts`).
+ */
+export declare function matchesEvalFilter(evalId: string, filters: readonly string[]): boolean;
+/**
+ * Imports a discovered eval file and stamps the path-derived id(s) onto
+ * the eval definition(s).
+ *
+ * The file must `export default` either one `EveEvalDefinition` (produced
+ * by `defineEval()`) or an array of them. A single definition derives its
+ * id from the file path; array entries derive `<file-id>/<index>` ids with
+ * the index zero-padded to four digits (e.g. `"weather/0000"`).
+ */
+export declare function importEvalFile(appRoot: string, filePath: string): Promise<EveEval[]>;
+/**
+ * Discovers and imports all eval files, optionally filtering by eval id.
+ * Filters match exactly or by directory prefix (see {@link matchesEvalFilter}).
  *
- * The file must `export default` an `EveEvalSuiteDefinition` (produced
- * by `defineEvalSuite()`).
+ * Throws when two files derive the same eval id (e.g. an array-exported
+ * `evals/weather.eval.ts` colliding with `evals/weather/0000.eval.ts`).
  */
-export declare function importSuiteFile(appRoot: string, filePath: string): Promise<EveEvalSuite>;
+export declare function discoverAndImportEvals(appRoot: string, evalIds?: readonly string[]): Promise<EveEval[]>;
 /**
- * Discovers and imports all suite files, optionally filtering by suite ID.
+ * Discovers and imports the required `evals/evals.config.ts` run-wide
+ * configuration (produced by `defineEvalConfig()`).
+ *
+ * Throws when the file is missing or does not default-export a valid
+ * `EveEvalConfig`.
  */
-export declare function discoverAndImportSuites(appRoot: string, suiteIds?: readonly string[]): Promise<EveEvalSuite[]>;
+export declare function discoverEvalConfig(appRoot: string): Promise<EveEvalConfig>;

package/dist/src/evals/runner/discover.js

@@ -1 +1 @@
-import{join,relative}from"node:path";import{readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_SUITE_GLOB_SUFFIX=`.eval.ts`;async function discoverSuiteFiles(n){let r=join(n,`evals`),i=[];try{await collectSuiteFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveSuiteId(n,r){let a=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return a.endsWith(EVAL_SUITE_GLOB_SUFFIX)?a.slice(0,-8):a}async function importSuiteFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default;if(!isEveEvalSuiteDefinition(n))throw Error(`Suite file "${t}" does not export a valid EveEvalSuite as its default export. Use defineEvalSuite() to create the suite.`);return{...n,id:deriveSuiteId(e,t)}}async function discoverAndImportSuites(e,t){let n=await discoverSuiteFiles(e);if(n.length===0)return[];let r=[];for(let i of n){let n=await importSuiteFile(e,i);(t===void 0||t.length===0||t.includes(n.id))&&r.push(n)}return r}function isEveEvalSuiteDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalSuite`}async function collectSuiteFiles(t,r){let a=await readdir(t,{withFileTypes:!0});for(let n of a){let a=join(t,n.name);n.isDirectory()?await collectSuiteFiles(a,r):n.isFile()&&n.name.endsWith(EVAL_SUITE_GLOB_SUFFIX)&&r.push(a)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportSuites,discoverSuiteFiles,importSuiteFile};
+import{join,relative}from"node:path";import{access,readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_FILE_SUFFIX=`.eval.ts`,EVAL_CONFIG_FILE=`evals.config.ts`;async function discoverEvalFiles(n){let r=join(n,`evals`),i=[];try{await collectEvalFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveEvalId(n,r){let i=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return i.endsWith(EVAL_FILE_SUFFIX)?i.slice(0,-8):i}function matchesEvalFilter(e,t){return t.length===0?!0:t.some(t=>e===t||e.startsWith(`${t}/`))}async function importEvalFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default,r=deriveEvalId(e,t);if(isEveEvalDefinition(n))return[{...n,id:r}];if(Array.isArray(n))return n.map((e,n)=>{if(!isEveEvalDefinition(e))throw Error(`Eval file "${t}" exports an array whose entry at index ${n} is not a valid EveEval. Use defineEval() to create every entry.`);return{...e,id:`${r}/${String(n).padStart(4,`0`)}`}});throw Error(`Eval file "${t}" does not export a valid EveEval (or array of EveEvals) as its default export. Use defineEval() to create the eval.`)}async function discoverAndImportEvals(e,t){let n=await discoverEvalFiles(e);if(n.length===0)return[];let r=t??[],i=[],a=new Map;for(let t of n)for(let n of await importEvalFile(e,t)){let e=a.get(n.id);if(e!==void 0)throw Error(`Duplicate eval id "${n.id}" derived from both "${e}" and "${t}".`);a.set(n.id,t),matchesEvalFilter(n.id,r)&&i.push(n)}return i}async function discoverEvalConfig(t){let r=join(t,`evals`,EVAL_CONFIG_FILE);try{await access(r)}catch(e){throw isNoEntryError(e)?Error(`Missing required eval config at evals/${EVAL_CONFIG_FILE}. Create it with defineEvalConfig({}) (optionally \`{ judge: { model } }\` to set the default judge model for \`t.judge.*\` assertions).`):e}let a=(await loadAuthoredModuleNamespace(r)).default;if(!isEveEvalConfig(a))throw Error(`Eval config "evals/${EVAL_CONFIG_FILE}" must default-export a defineEvalConfig() value.`);return a}function isEveEvalConfig(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalConfig`}function isEveEvalDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEval`}async function collectEvalFiles(t,n){let i=await readdir(t,{withFileTypes:!0});for(let r of i){let i=join(t,r.name);r.isDirectory()?await collectEvalFiles(i,n):r.isFile()&&r.name.endsWith(EVAL_FILE_SUFFIX)&&n.push(i)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportEvals,discoverEvalConfig,discoverEvalFiles,importEvalFile,matchesEvalFilter};

package/dist/src/evals/runner/execute-eval.d.ts

@@ -0,0 +1,25 @@
+import type { Client } from "#client/client.js";
+import type { EveEval, EveEvalResult, EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Options for executing one eval.
+ */
+export interface ExecuteEvalOptions {
+    readonly evaluation: EveEval;
+    readonly failOnSkip?: boolean;
+    /** Receives `t.log` lines as the eval runs (used by `--verbose`). */
+    readonly onLog?: (message: string) => void;
+    readonly target: EveEvalTargetHandle;
+    /** Overrides the eval's own `timeoutMs` when set (CLI `--timeout`). */
+    readonly timeoutMs?: number;
+    /**
+     * Pre-configured client for communicating with the Eve agent.
+     * The CLI constructs this once with the appropriate auth and headers,
+     * and every eval creates fresh sessions from it.
+     */
+    readonly client: Client;
+}
+/**
+ * Executes one eval end to end: verifies requirements, runs `test(t)`,
+ * collects its assertions, and computes the verdict.
+ */
+export declare function executeEval(options: ExecuteEvalOptions): Promise<EveEvalResult>;

package/dist/src/evals/runner/execute-eval.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{executeTask}from"#evals/runner/execute-task.js";import{findUnmetRequirements,formatUnmetRequirements}from"#evals/requirements.js";import{computeEvalVerdict}from"#evals/runner/verdict.js";async function executeEval(i){let{evaluation:a,target:o,client:s}=i,c=new Date().toISOString(),l=a.requires??[],u=findUnmetRequirements(l,o);if(u.length>0)return createRequirementResult(a,u,{failOnSkip:i.failOnSkip===!0,startedAt:c});let d,f=[],p;try{let e=await executeTask({client:s,evaluation:a,onLog:i.onLog,requirements:l,target:o,timeoutMs:i.timeoutMs??a.timeoutMs});d=e.result,f=e.assertions,p=e.error}catch(n){p=toErrorMessage(n),d={output:null,finalMessage:null,status:`failed`,events:[],derived:createEmptyDerivedFacts()}}let m=computeEvalVerdict({error:p,assertions:f});return{id:a.id,result:d,assertions:f,verdict:m,error:p,startedAt:c,completedAt:new Date().toISOString()}}function createRequirementResult(e,t,n){let r=formatUnmetRequirements(t);return{id:e.id,result:createEmptyTaskResult(),assertions:[],verdict:n.failOnSkip?`failed`:`skipped`,skipReason:r,startedAt:n.startedAt,completedAt:new Date().toISOString()}}function createEmptyTaskResult(){return{output:null,finalMessage:null,status:`completed`,events:[],derived:createEmptyDerivedFacts()}}export{executeEval};

package/dist/src/evals/runner/execute-task.d.ts

@@ -0,0 +1,31 @@
+import type { Client } from "#client/client.js";
+import type { AssertionResult, EveEval, EveEvalRequirement, EveEvalTargetHandle, EveEvalTaskResult } from "#evals/types.js";
+/**
+ * Options for executing one eval's task.
+ */
+interface ExecuteTaskOptions {
+    readonly client: Client;
+    readonly evaluation: EveEval;
+    /** Receives each `t.log` line as it is written (used by `--verbose`). */
+    readonly onLog?: (message: string) => void;
+    readonly requirements?: readonly EveEvalRequirement[];
+    readonly target: EveEvalTargetHandle;
+    readonly timeoutMs?: number;
+}
+/**
+ * Task result plus the assertions the eval's `test(t)` recorded. `error` is
+ * set when the `test` body threw (e.g. a failed `expectOk()` or a bespoke
+ * `throw`); the partial run is still captured so recorded assertions report.
+ */
+export interface ExecuteTaskResult {
+    readonly result: EveEvalTaskResult;
+    readonly assertions: readonly AssertionResult[];
+    readonly error?: string;
+}
+/**
+ * Executes one eval's `test(t)` against an Eve agent target: drives the
+ * session(s), captures the run, then finalizes the recorded assertions
+ * against the completed task result.
+ */
+export declare function executeTask(options: ExecuteTaskOptions): Promise<ExecuteTaskResult>;
+export {};

package/dist/src/evals/runner/execute-task.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";import{scopeEvalTargetHandle}from"#evals/target.js";import{EvalSessionManager}from"#evals/session.js";import{createEmptyDerivedFacts}from"#evals/runner/derive-run-facts.js";import{createEvalContext}from"#evals/context.js";async function executeTask(r){let{client:a,evaluation:o,target:s,timeoutMs:c}=r,l=c===void 0?neverAbortSignal():AbortSignal.timeout(c),u=new EvalSessionManager({client:a,signal:l}),d=scopeEvalTargetHandle(s,{requirements:r.requirements??[],sessions:u}),f=[],{context:p,collector:m}=createEvalContext({manager:u,target:d,signal:l,judge:o.judge,log:e=>{f.push(e),r.onLog?.(e)}}),h;try{await o.test(p)}catch(t){h=toErrorMessage(t)}let g=buildTaskResult({logs:f,sessions:u.snapshots(),turn:u.lastTurnSession()?.lastTurn});return{result:g,assertions:await m.finalize(g),error:h}}function buildTaskResult(e){let t=e.sessions.flatMap(e=>e.events),n=e.turn?.message??null;return{output:n,finalMessage:n,sessionId:selectPrimarySessionId(e.sessions),status:e.turn?.status??`completed`,events:t,logs:e.logs,derived:combineDerivedFacts(e.sessions),sessions:e.sessions,runtimeIdentity:extractRuntimeIdentity(t)}}function combineDerivedFacts(e){if(e.length===0)return createEmptyDerivedFacts();let t=e.flatMap(e=>e.derived.toolCalls),n=e.flatMap(e=>e.derived.subagentCalls),i=e.flatMap(e=>e.derived.inputRequests),a=e.find(e=>e.derived.failureCode!==void 0)?.derived.failureCode;return{toolCalls:t,toolCallCount:t.length,subagentCalls:n,subagentCallCount:n.length,inputRequests:i,parked:e.some(e=>e.derived.parked),messageCount:sum(e,e=>e.derived.messageCount),reasoningBlockCount:sum(e,e=>e.derived.reasoningBlockCount),failureCode:a}}function selectPrimarySessionId(e){return e.find(e=>e.primary)?.sessionId??e[0]?.sessionId}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function sum(e,t){return e.reduce((e,n)=>e+t(n),0)}function neverAbortSignal(){return new AbortController().signal}export{executeTask};

package/dist/src/evals/runner/reporters/braintrust.d.ts

@@ -6,7 +6,7 @@ import type { EvalReporter } from "#evals/runner/reporters/types.js";
 export interface BraintrustReporterConfig {
     /** Braintrust project id. Maps to the `projectId` init option. */
     readonly projectId?: string;
-    /** Braintrust project name. Defaults to the suite id when omitted. */
+    /** Braintrust project name. Defaults to the eval id when omitted. */
     readonly projectName?: string;
     /** Name for the created experiment. When omitted, Braintrust names it. */
     readonly experimentName?: string;
@@ -19,9 +19,11 @@ export interface BraintrustReporterConfig {
 }
 /**
  * Creates an {@link EvalReporter} that uploads eval results to Braintrust as an
- * experiment. Add it to a suite's `reporters` array. Requires the optional
- * `braintrust` peer package; the reporter throws a descriptive error when the
- * suite starts if it is missing. `config` is optional and defaults to an empty
- * object (the project name then falls back to the suite id).
+ * experiment. Add it to an eval's `reporters` array; share one instance
+ * across several evals (e.g. every entry of an array export) to group them
+ * into a single experiment. Requires the optional `braintrust` peer package;
+ * the reporter throws a descriptive error when the run starts if it is
+ * missing. `config` is optional and defaults to an empty object (the project
+ * name then falls back to the first observed eval id).
  */
 export declare function Braintrust(config?: BraintrustReporterConfig): EvalReporter;