eve 0.6.0-beta.9 → 0.7.2

package/dist/src/compiler/normalize-tool.d.ts

@@ -1,5 +1,6 @@
 import type { ModuleSourceRef } from "#shared/source-ref.js";
 import type { CompiledToolDefinition, CompiledDynamicToolDefinition } from "#compiler/manifest.js";
+import { type ModuleBackedDefinitionLoadOptions } from "#compiler/normalize-helpers.js";
 /**
  * Compiled tool entry produced from one authored `tools/*.ts` file.
  *
@@ -31,4 +32,4 @@ export type CompiledToolEntry = {
  * directories into a slug-safe single segment. Authored `name` fields
  * are rejected by the normalizer.
  */
-export declare function compileToolEntry(agentRoot: string, source: ModuleSourceRef): Promise<CompiledToolEntry>;
+export declare function compileToolEntry(agentRoot: string, source: ModuleSourceRef, options?: ModuleBackedDefinitionLoadOptions): Promise<CompiledToolEntry>;

package/dist/src/compiler/normalize-tool.js

@@ -1 +1 @@
-import{stripLogicalPathExtension}from"#discover/filesystem.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{normalizeToolDefinition}from"#internal/authored-definition/schema-backed.js";async function compileToolEntry(e,t){let n=normalizeToolDefinition(await loadModuleBackedDefinition({agentRoot:e,kind:`tool`,source:t}),`Expected the tool export "${t.exportName??`default`}" from "${t.logicalPath}" to match the public Eve shape.`),r=stripLogicalPathExtension(t.logicalPath).replace(/^tools\//,``).replaceAll(`/`,`-`);return n.kind===`disabled`?{kind:`disabled`,name:r}:n.kind===`enable-workflow`?{kind:`enable-workflow`}:n.kind===`dynamic-tool`?{kind:`dynamic-tool`,definition:{eventNames:[...n.eventNames],exportName:t.exportName,logicalPath:t.logicalPath,slug:r,sourceId:t.sourceId,sourceKind:`module`}}:{kind:`tool`,definition:{description:n.definition.description,exportName:t.exportName,inputSchema:n.definition.inputSchema??null,logicalPath:t.logicalPath,name:r,outputSchema:n.definition.outputSchema,sourceId:t.sourceId,sourceKind:`module`}}}export{compileToolEntry};
+import{stripLogicalPathExtension}from"#discover/filesystem.js";import{loadModuleBackedDefinition}from"#compiler/normalize-helpers.js";import{normalizeToolDefinition}from"#internal/authored-definition/schema-backed.js";async function compileToolEntry(e,t,n={}){let r=normalizeToolDefinition(await loadModuleBackedDefinition({agentRoot:e,externalDependencies:n.externalDependencies,kind:`tool`,source:t}),`Expected the tool export "${t.exportName??`default`}" from "${t.logicalPath}" to match the public Eve shape.`),i=stripLogicalPathExtension(t.logicalPath).replace(/^tools\//,``).replaceAll(`/`,`-`);return r.kind===`disabled`?{kind:`disabled`,name:i}:r.kind===`enable-workflow`?{kind:`enable-workflow`}:r.kind===`dynamic-tool`?{kind:`dynamic-tool`,definition:{eventNames:[...r.eventNames],exportName:t.exportName,logicalPath:t.logicalPath,slug:i,sourceId:t.sourceId,sourceKind:`module`}}:{kind:`tool`,definition:{description:r.definition.description,exportName:t.exportName,inputSchema:r.definition.inputSchema??null,logicalPath:t.logicalPath,name:i,outputSchema:r.definition.outputSchema,sourceId:t.sourceId,sourceKind:`module`}}}export{compileToolEntry};

package/dist/src/compiler/workspace-resources.js

@@ -1 +1 @@
-import{join,posix}from"node:path";import{createHash}from"node:crypto";import{cp,mkdir,readFile,readdir,rm}from"node:fs/promises";import{normalizeLogicalPath}from"#discover/filesystem.js";import{ROOT_COMPILED_AGENT_NODE_ID,deriveResourceRootEntries}from"#compiler/manifest.js";import{normalizeSkillPackage,writeSkillPackageDirectory}from"#shared/skill-package.js";const RESOURCES_DIRECTORY=`workspace-resources`;async function materializeWorkspaceResources(t){let n=join(t.compileDirectoryPath,RESOURCES_DIRECTORY);await rm(n,{force:!0,recursive:!0});let r=await materializeNode({nodeId:ROOT_COMPILED_AGENT_NODE_ID,resourcesRoot:n,manifest:t.manifest}),i=await Promise.all(t.manifest.subagents.map(async e=>({...e,agent:await materializeNode({nodeId:e.nodeId,resourcesRoot:n,manifest:e.agent})})));return{...r,kind:t.manifest.kind,subagentEdges:t.manifest.subagentEdges,subagents:i,version:t.manifest.version}}function createResourceRoot(t,n,r){return{contentHash:r,logicalPath:normalizeLogicalPath(join(RESOURCES_DIRECTORY,n)),rootEntries:deriveResourceRootEntries({sandboxWorkspaces:t.sandboxWorkspaces,skills:t.skills})}}async function materializeNode(t){for(let e of t.manifest.sandboxWorkspaces)if(e.rootEntries.some(e=>e===`skills/`||e===`skills`))throw Error(`Sandbox workspace "${e.logicalPath}" cannot define "skills" because Eve manages that workspace entry.`);let n=join(t.resourcesRoot,t.nodeId);await mkdir(n,{recursive:!0});for(let e of t.manifest.sandboxWorkspaces)await cp(e.sourcePath,n,{recursive:!0});for(let e of t.manifest.skills)await materializeSkill({nodeRoot:n,skill:e});let a=await heveWorkspaceResourceRoot(n);return{...t.manifest,skills:t.manifest.skills.map(stripSkillPackageFiles),workspaceResourceRoot:createResourceRoot(t.manifest,t.nodeId,a)}}async function materializeSkill(t){let n=join(t.nodeRoot,`skills`,t.skill.name);if(t.skill.sourceKind===`skill-package`){await cp(t.skill.rootPath,n,{recursive:!0});return}await writeSkillPackageDirectory({rootPath:t.nodeRoot,skill:normalizeSkillPackage(t.skill)})}function stripSkillPackageFiles(e){let{files:t,...n}=e;return n}async function heveWorkspaceResourceRoot(e){let t=await listWorkspaceResourceFiles({logicalDirectoryPath:`.`,sourceDirectoryPath:e});t.sort((e,t)=>e.logicalPath.localeCompare(t.logicalPath));let r=createHash(`sha256`);r.update(`eve-workspace-resource-root-v1\0`);for(let e of t){let t=await readFile(e.sourcePath);r.update(e.logicalPath),r.update(`\0`),r.update(String(t.byteLength)),r.update(`\0`),r.update(t),r.update(`\0`)}return r.digest(`hex`)}async function listWorkspaceResourceFiles(n){let r=[],i=await readdir(n.sourceDirectoryPath,{withFileTypes:!0});for(let a of i){if(!a.isDirectory()&&!a.isFile())continue;let i=join(n.sourceDirectoryPath,a.name),o=posix.join(n.logicalDirectoryPath,a.name);if(a.isDirectory()){r.push(...await listWorkspaceResourceFiles({logicalDirectoryPath:o,sourceDirectoryPath:i}));continue}r.push({logicalPath:o,sourcePath:i})}return r}export{materializeWorkspaceResources};
+import{join,posix}from"node:path";import{cp,mkdir,readFile,readdir,rm}from"node:fs/promises";import{createHash}from"node:crypto";import{normalizeLogicalPath}from"#discover/filesystem.js";import{ROOT_COMPILED_AGENT_NODE_ID,deriveResourceRootEntries}from"#compiler/manifest.js";import{normalizeSkillPackage,writeSkillPackageDirectory}from"#shared/skill-package.js";const RESOURCES_DIRECTORY=`workspace-resources`;async function materializeWorkspaceResources(t){let n=join(t.compileDirectoryPath,RESOURCES_DIRECTORY);await rm(n,{force:!0,recursive:!0});let r=await materializeNode({nodeId:ROOT_COMPILED_AGENT_NODE_ID,resourcesRoot:n,manifest:t.manifest}),i=await Promise.all(t.manifest.subagents.map(async e=>({...e,agent:await materializeNode({nodeId:e.nodeId,resourcesRoot:n,manifest:e.agent})})));return{...r,kind:t.manifest.kind,subagentEdges:t.manifest.subagentEdges,subagents:i,version:t.manifest.version}}function createResourceRoot(t,n,r){return{contentHash:r,logicalPath:normalizeLogicalPath(join(RESOURCES_DIRECTORY,n)),rootEntries:deriveResourceRootEntries({sandboxWorkspaces:t.sandboxWorkspaces,skills:t.skills})}}async function materializeNode(t){for(let e of t.manifest.sandboxWorkspaces)if(e.rootEntries.some(e=>e===`skills/`||e===`skills`))throw Error(`Sandbox workspace "${e.logicalPath}" cannot define "skills" because Eve manages that workspace entry.`);let i=join(t.resourcesRoot,t.nodeId);await mkdir(i,{recursive:!0});for(let e of t.manifest.sandboxWorkspaces)await cp(e.sourcePath,i,{recursive:!0});for(let e of t.manifest.skills)await materializeSkill({nodeRoot:i,skill:e});let a=await hashWorkspaceResourceRoot(i);return{...t.manifest,skills:t.manifest.skills.map(stripSkillPackageFiles),workspaceResourceRoot:createResourceRoot(t.manifest,t.nodeId,a)}}async function materializeSkill(t){let r=join(t.nodeRoot,`skills`,t.skill.name);if(t.skill.sourceKind===`skill-package`){await cp(t.skill.rootPath,r,{recursive:!0});return}await writeSkillPackageDirectory({rootPath:t.nodeRoot,skill:normalizeSkillPackage(t.skill)})}function stripSkillPackageFiles(e){let{files:t,...n}=e;return n}async function hashWorkspaceResourceRoot(e){let t=await listWorkspaceResourceFiles({logicalDirectoryPath:`.`,sourceDirectoryPath:e});t.sort((e,t)=>e.logicalPath.localeCompare(t.logicalPath));let n=createHash(`sha256`);n.update(`eve-workspace-resource-root-v1\0`);for(let e of t){let t=await readFile(e.sourcePath);n.update(e.logicalPath),n.update(`\0`),n.update(String(t.byteLength)),n.update(`\0`),n.update(t),n.update(`\0`)}return n.digest(`hex`)}async function listWorkspaceResourceFiles(n){let r=[],i=await readdir(n.sourceDirectoryPath,{withFileTypes:!0});for(let a of i){if(!a.isDirectory()&&!a.isFile())continue;let i=join(n.sourceDirectoryPath,a.name),o=posix.join(n.logicalDirectoryPath,a.name);if(a.isDirectory()){r.push(...await listWorkspaceResourceFiles({logicalDirectoryPath:o,sourceDirectoryPath:i}));continue}r.push({logicalPath:o,sourcePath:i})}return r}export{materializeWorkspaceResources};

package/dist/src/context/node.d.ts

@@ -4,4 +4,4 @@ import type { ContextReader } from "#context/provider.js";
  * context. The bundle is already resolved to the correct node (root or
  * subagent) at run start.
  */
-export declare function getActiveRuntimeNode(ctx: ContextReader): import("../runtime/graph.js").ResolvedRuntimeAgentNode;
+export declare function getActiveRuntimeNode(ctx: ContextReader): import("../runtime/graph.ts").ResolvedRuntimeAgentNode;

package/dist/src/evals/assertions/collector.d.ts

@@ -0,0 +1,43 @@
+import type { AssertionHandle, AssertionResult, AssertionSeverity, EveEvalTaskResult } from "#evals/types.js";
+/**
+ * Outcome of evaluating one assertion: a 0–1 score (boolean assertions use
+ * exactly 0 or 1) with optional human-readable detail and metadata.
+ */
+export interface AssertionOutcome {
+    readonly score: number;
+    readonly message?: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+/**
+ * A run-level assertion (e.g. `t.completed()`), evaluated lazily against the
+ * final task result after `test(t)` returns. The evaluation is deferred so
+ * the assertion always sees the complete run regardless of call order.
+ */
+export interface RunAssertion {
+    readonly name: string;
+    evaluate(result: EveEvalTaskResult): AssertionOutcome | Promise<AssertionOutcome>;
+}
+/**
+ * Collects the assertions recorded by an eval's `test(t)`. Run-level
+ * assertions register a deferred spec; value/judge assertions evaluate their
+ * captured value immediately (the value is ephemeral) and register the pending
+ * promise. {@link finalize} resolves everything against the final result and
+ * produces the ordered {@link AssertionResult} list the verdict reads.
+ */
+export declare class AssertionCollector {
+    #private;
+    /** Register a run-level assertion evaluated against the final result. */
+    recordRun(spec: RunAssertion, severity?: AssertionSeverity): AssertionHandle;
+    /** Register a value/judge assertion, evaluating the captured value now. */
+    recordValue(input: {
+        readonly name: string;
+        readonly severity: AssertionSeverity;
+        readonly threshold?: number;
+        readonly score: () => Promise<AssertionOutcome>;
+    }): AssertionHandle;
+    /**
+     * Awaits every pending value/judge assertion, evaluates the deferred
+     * run-level assertions against `result`, and returns the recorded results.
+     */
+    finalize(result: EveEvalTaskResult): Promise<readonly AssertionResult[]>;
+}

package/dist/src/evals/assertions/collector.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";var AssertionCollector=class{#e=[];#t=[];recordRun(e,t=`gate`){let n={name:e.name,severity:t,threshold:void 0,kind:`deferred`,spec:e,score:0,failed:!1};return this.#e.push(n),makeHandle(n,Promise.resolve())}recordValue(t){let n={name:t.name,severity:t.severity,threshold:t.threshold,kind:`resolved`,score:0,failed:!1};this.#e.push(n);let r=t.score().then(e=>{n.score=e.score,n.message=e.message,n.metadata=e.metadata}).catch(t=>{n.score=0,n.severity=`gate`,n.threshold=void 0,n.message=toErrorMessage(t),n.failed=!0});return this.#t.push(r),makeHandle(n,r)}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push({name:n.name,score:n.score,severity:n.severity,threshold:n.threshold,passed:computePassed(n.severity,n.threshold,n.score,n.failed),message:n.message,metadata:n.metadata})}return t}};function computePassed(e,t,n,r){if(r)return!1;let i=t??(e===`gate`?1:void 0);return i===void 0||n>=i}function makeHandle(e,t){let n={gate(t){return e.severity=`gate`,e.threshold=t,n},soft(t){return e.severity=`soft`,e.threshold=t,n},atLeast(t){return e.severity=`soft`,e.threshold=t,n},then(e,n){return t.then(e,n)}};return n}export{AssertionCollector};

package/dist/src/evals/assertions/run.d.ts

@@ -0,0 +1,72 @@
+import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
+import type { HandleMessageStreamEvent } from "#protocol/message.js";
+import { type EveEvalSubagentCallMatchOptions, type EveEvalToolCallMatchOptions } from "#evals/match.js";
+import type { RunAssertion } from "#evals/assertions/collector.js";
+/**
+ * Asserts the run ran to completion: it did not fail and did not park on an
+ * unanswered HITL input request.
+ */
+export declare function completed(): RunAssertion;
+/**
+ * Asserts the run ended parked on HITL input. For approval gates and
+ * ask-question flows.
+ */
+export declare function waiting(): RunAssertion;
+/**
+ * Asserts the run did not fail: terminal status is not `"failed"` and no
+ * `turn.failed` / `step.failed` events were emitted. Parked runs pass; use
+ * {@link completed} to also reject parking.
+ */
+export declare function didNotFail(): RunAssertion;
+/**
+ * Asserts the joined assistant message text contains `token` (substring for
+ * strings, `test` for RegExps).
+ */
+export declare function messageIncludes(token: string | RegExp): RunAssertion;
+/**
+ * Asserts a tool call with `name` happened. Options constrain the call
+ * further: `input` partial-deep-matches, `output` matches the result,
+ * `isError` constrains error state, and `times` requires an exact count.
+ */
+export declare function calledTool(name: string, options?: EveEvalToolCallMatchOptions): RunAssertion;
+/**
+ * Asserts no tool call with `name` happened.
+ */
+export declare function notCalledTool(name: string): RunAssertion;
+/**
+ * Asserts the named tools were called in the given order (subsequence match:
+ * other calls may interleave).
+ */
+export declare function toolOrder(names: readonly string[]): RunAssertion;
+/**
+ * Asserts the run made no tool calls at all.
+ */
+export declare function usedNoTools(): RunAssertion;
+/**
+ * Asserts the run made at most `max` tool calls.
+ */
+export declare function maxToolCalls(max: number): RunAssertion;
+/**
+ * Asserts no action result (tool, subagent, or skill) reported a failure.
+ */
+export declare function noFailedActions(): RunAssertion;
+/**
+ * Asserts a subagent delegation to `name` occurred. `remoteUrl` matches the
+ * `subagent.called` remote metadata, `output` matches the `subagent.completed`
+ * output.
+ */
+export declare function calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): RunAssertion;
+/**
+ * Escape hatch: asserts an arbitrary predicate over the full typed event
+ * stream. `label` names the assertion in reports.
+ */
+export declare function event(predicate: (events: readonly HandleMessageStreamEvent[]) => boolean, label: string): RunAssertion;
+/**
+ * Asserts `result.output` (the final assistant message) deep-equals `value`.
+ */
+export declare function outputEquals(value: unknown): RunAssertion;
+/**
+ * Asserts `result.output` validates against a Standard Schema (e.g. a Zod
+ * schema).
+ */
+export declare function outputMatches(schema: StandardSchemaV1): RunAssertion;

package/dist/src/evals/assertions/run.js

@@ -0,0 +1,2 @@
+import{deepEquals,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function completed(){return{name:`completed`,evaluate(e){return e.status===`failed`?fail(failureDetail(`run failed`,e.derived.failureCode)):e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS}}}function waiting(){return{name:`waiting`,evaluate(e){return e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`)}}}function didNotFail(){return{name:`didNotFail`,evaluate(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?PASS:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let r=joinCompletedMessages(t.events);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(r)}`)}}}function calledTool(e,t={}){return{name:`calledTool(${e})`,evaluate(n){let i=n.derived.toolCalls.filter(t=>t.name===e),a=i.filter(e=>toolCallMatches(e,t));if(t.times===void 0?a.length>0:a.length===t.times)return{score:1,metadata:{matchingCalls:a.length}};let o=i.length>0?`observed ${e} calls: ${i.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`;return fail(`${t.times===void 0?`expected a matching call to "${e}"`:`expected exactly ${t.times} matching call(s), found ${a.length}`}; ${o}`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=t.derived.toolCalls.map(e=>e.name),r=0;for(let t of n)if(t===e[r]&&(r+=1),r===e.length)break;return r===e.length?PASS:fail(`missing "${e[r]}" after [${e.slice(0,r).join(`, `)}]; observed order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(e=>e.data.result.kind===`tool-result`?e.data.result.toolName:e.data.result.kind);return fail(`${t.length} failed action(s): ${n.join(`, `)}`)}}}function calledSubagent(e,n={}){return{name:`calledSubagent(${e})`,evaluate(r){let a=r.derived.subagentCalls.filter(t=>t.name===e);return a.filter(e=>subagentCallMatches(e,n)).length>0?PASS:a.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but no call matched the constraints`,{observedSubagentCalls:a})}}}function event(e,t){return{name:`event(${t})`,evaluate(n){return e(n.events)?PASS:fail(`event predicate "${t}" did not hold`)}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}export{calledSubagent,calledTool,completed,didNotFail,event,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,outputEquals,outputMatches,toolOrder,usedNoTools,waiting};

package/dist/src/evals/autoevals-client.js

@@ -0,0 +1,2 @@
+import"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{generateText,jsonSchema}from"ai";function createAutoevalsClient(e){return{chat:{completions:{create:t=>createChatCompletion(t,e)}}}}async function createChatCompletion(t,n){let r=convertTools(t.tools),i=await generateText({model:n.languageModel,messages:convertMessages(t.messages??[]),tools:Object.keys(r).length>0?r:void 0,toolChoice:convertToolChoice(t.tool_choice),providerOptions:n.providerOptions}),a=i.toolCalls.map(e=>({id:e.toolCallId,type:`function`,function:{name:e.toolName,arguments:JSON.stringify(e.input??{})}}));return{choices:[{index:0,finish_reason:a.length>0?`tool_calls`:`stop`,message:{role:`assistant`,content:i.text||null,tool_calls:a.length>0?a:void 0}}]}}function convertMessages(e){return e.map(e=>{let t=contentToText(e.content);switch(e.role){case`assistant`:return{role:`assistant`,content:t};case`developer`:case`system`:return{role:`system`,content:t};default:return{role:`user`,content:t}}})}function contentToText(e){return e==null?``:typeof e==`string`?e:e.map(e=>e.text??``).filter(Boolean).join(`
+`)}function convertTools(e){let n={};for(let r of e??[])r.type!==`function`||r.function?.name===void 0||(n[r.function.name]={description:r.function.description,inputSchema:jsonSchema(r.function.parameters??{})});return n}function convertToolChoice(e){if(e!==void 0)return typeof e==`string`?e:{type:`tool`,toolName:e.function.name}}export{createAutoevalsClient};

package/dist/src/evals/cli/eval-client.d.ts

@@ -0,0 +1,22 @@
+import { Client } from "#client/client.js";
+import type { ClientOptions } from "#client/types.js";
+import type { EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Resolves the {@link ClientOptions} for an eval target.
+ *
+ * Local targets need no auth. Remote targets connect with the same options
+ * as every other development client (`resolveDevelopmentClientOptions`):
+ * per-request headers carrying the Vercel OIDC trusted-IDP token (which
+ * bypasses Deployment Protection without a per-project secret) plus
+ * `x-vercel-protection-bypass` when `VERCEL_AUTOMATION_BYPASS_SECRET` is
+ * set, and a bearer resolved from the same OIDC cascade.
+ *
+ * `EVE_EVAL_AUTH_TOKEN` overrides the bearer with a static token for
+ * targets whose auth is not OIDC-based.
+ */
+export declare function resolveEvalClientOptions(target: Pick<EveEvalTargetHandle, "kind" | "url">): ClientOptions;
+/**
+ * Creates the Eve {@link Client} for an eval target from
+ * {@link resolveEvalClientOptions}.
+ */
+export declare function createEvalClient(target: Pick<EveEvalTargetHandle, "kind" | "url">): Client;

package/dist/src/evals/cli/eval-client.js

@@ -0,0 +1 @@
+import{resolveDevelopmentClientOptions}from"#services/dev-client/client-options.js";import{Client}from"#client/client.js";function resolveEvalClientOptions(t){if(t.kind===`local`)return{host:t.url};let n=resolveDevelopmentClientOptions(t.url),r=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return r?{...n,auth:{bearer:r}}:n}function createEvalClient(e){return new Client(resolveEvalClientOptions(e))}export{createEvalClient,resolveEvalClientOptions};

package/dist/src/evals/cli/eval.d.ts

@@ -3,11 +3,14 @@ interface EvalCliOptions {
     timeout?: string;
     maxConcurrency?: string;
     json?: boolean;
+    junit?: string;
     skipReport?: boolean;
     strict?: boolean;
     list?: boolean;
+    mockModels?: boolean;
+    noSkips?: boolean;
     tag?: string[];
-    case?: string[];
+    verbose?: boolean;
 }
 type EvalCliLogger = {
     log(message: string): void;
@@ -16,9 +19,9 @@ type EvalCliLogger = {
 /**
  * Runs the `eve eval` command with already-parsed Commander options.
  *
- * Exit codes: `0` when every executed case passed its checks (and score
- * thresholds under `--strict`), `1` when any case failed, `2` for runner or
- * configuration errors (no suites discovered, no cases matching filters).
+ * Exit codes: `0` when every executed eval passed its gate assertions (and
+ * soft thresholds under `--strict`), `1` when any eval failed, `2` for runner
+ * or configuration errors (no evals discovered, no evals matching filters).
  */
-export declare function runEvalCommand(suiteIds: readonly string[], options: EvalCliOptions, logger: EvalCliLogger): Promise<void>;
+export declare function runEvalCommand(evalIds: readonly string[], options: EvalCliOptions, logger: EvalCliLogger): Promise<void>;
 export {};

package/dist/src/evals/cli/eval.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{VERCEL_PROTECTION_BYPASS_HEADER}from"#services/dev-client/request-headers.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{Client}from"#client/client.js";import{getVercelOidcToken}from"#compiled/@vercel/oidc/index.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{discoverAndImportSuites}from"#evals/runner/discover.js";import{executeSuite}from"#evals/runner/execute-suite.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";async function runEvalCommand(e,t,r){let i=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(i);let l=e.length>0?e:void 0,u=await discoverAndImportSuites(i,l);if(u.length===0){l?r.error(`No suites found matching: ${l.join(`, `)}`):r.error(`No eval suites found. Create suite files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let d=await loadAndFilterSuites(u,t,r);if(d===void 0){process.exitCode=2;return}if(t.list===!0){printSuiteList(d,t.json===!0,r);return}let f,p;t.url?p={kind:`remote`,url:t.url}:(f=await startDevelopmentServer(i,{host:`127.0.0.1`,port:0}),p={kind:`local`,url:f.url});let m=createEvalClient(p);try{let e=[];for(let n of d){let r=applyCliOverrides(n,t),a=buildReporters(r,{json:t.json===!0,skipReport:t.skipReport===!0}),o=await executeSuite({suite:r,target:p,reporters:a,appRoot:i,client:m});e.push(o)}t.json&&r.log(JSON.stringify(e,null,2));let n=e.some(e=>e.failed>0),a=t.strict===!0&&e.some(e=>e.scored>0);(n||a)&&(process.exitCode=1)}finally{f&&await f.close()}process.exit(process.exitCode??0)}async function loadAndFilterSuites(t,n,r){let i=n.tag??[],a=n.case??[],o=i.length>0||a.length>0,s=[];for(let n of t){let t;try{t=await n.load()}catch(t){r.error(`Failed to load cases for suite "${n.id}": ${toErrorMessage(t)}`);return}let c=t.filter(e=>!(a.length>0&&!a.includes(e.id)||i.length>0&&!carriesTag(n,e,i)));o&&c.length===0||s.push({suite:n,cases:c})}if(s.length===0){r.error(`No cases matched the provided filters (${describeFilters(i,a)}).`);return}return s}function carriesTag(e,t,n){let r=e.tags?.some(e=>n.includes(e))??!1,i=t.tags?.some(e=>n.includes(e))??!1;return r||i}function describeFilters(e,t){let n=[];return e.length>0&&n.push(`tags: ${e.join(`, `)}`),t.length>0&&n.push(`cases: ${t.join(`, `)}`),n.join(`; `)}function printSuiteList(e,t,n){if(t){let t=e.map(({suite:e,cases:t})=>({id:e.id,description:e.description,tags:e.tags,cases:t.map(e=>({id:e.id,tags:e.tags}))}));n.log(JSON.stringify(t,null,2));return}for(let{suite:t,cases:r}of e){let e=t.description===void 0?``:` — ${t.description}`;n.log(`${t.id}${e} (${r.length} case${r.length===1?``:`s`})`);for(let e of r){let t=e.tags!==void 0&&e.tags.length>0?` [${e.tags.join(`, `)}]`:``;n.log(`  ${e.id}${t}`)}}}function createEvalClient(e){if(e.kind===`local`)return new Client({host:e.url});let n={},i=process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim();return i&&(n[VERCEL_PROTECTION_BYPASS_HEADER]=i),new Client({auth:resolveRemoteBearerAuth(),headers:Object.keys(n).length>0?n:void 0,host:e.url})}function resolveRemoteBearerAuth(){let e=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return e?{bearer:e}:{bearer:resolveOidcBearerToken}}async function resolveOidcBearerToken(){try{let e=(await getVercelOidcToken()).trim();if(e.length>0)return e}catch{}return process.env.VERCEL_OIDC_TOKEN?.trim()??``}function applyCliOverrides(e,t){let n=t.maxConcurrency?Number.parseInt(t.maxConcurrency,10):void 0,r=t.timeout?Number.parseInt(t.timeout,10):void 0,i={...e.suite,load:()=>Promise.resolve([...e.cases])};return n!==void 0&&(i.maxConcurrency=n),r!==void 0&&(i.timeoutMs=r),i}function buildReporters(e,t){let n=t.json?[]:[new ConsoleReporter];return!t.skipReport&&e.reporters&&n.push(...e.reporters),n}export{runEvalCommand};
+import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{resolveApplicationRoot}from"#internal/application/paths.js";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{EVE_MOCK_AUTHORED_MODELS_ENV}from"#runtime/agent/resolve-model.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let s=resolveApplicationRoot();if(t.url&&t.mockModels===!0){n.error(`--mock-models cannot be used with --url. The target's mock-model state is discovered from /eve/v1/info.`),process.exitCode=2;return}let c=t.mockModels===!0?setEnvForRun(EVE_MOCK_AUTHORED_MODELS_ENV,`1`):void 0,l;try{loadDevelopmentEnvironmentFiles(s);let r=e.length>0?e:void 0,a=await discoverAndImportEvals(s,r);if(a.length===0){r?n.error(`No evals found matching: ${r.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let c=filterEvalsByTag(a,t.tag??[]);if(c.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(c,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(s)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m;try{t.url?m=await resolveEvalTargetHandle({client:createEvalClient({kind:`remote`,url:t.url}),expectedAgentName:await readExpectedAgentName(s),kind:`remote`,url:t.url}):(p=await startDevelopmentServer(s,{host:`127.0.0.1`,port:0}),m=await resolveEvalTargetHandle({client:createEvalClient({kind:`local`,url:p.url}),expectedAgentName:await readExpectedAgentName(s),kind:`local`,url:p.url}));let e=createEvalClient(m),r=t.json===!0?[]:[new ConsoleReporter];t.junit!==void 0&&r.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:c,config:f,target:m,client:e,appRoot:s,reporters:r,includeEvalReporters:t.skipReport!==!0,failOnSkip:t.noSkips===!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{p&&await p.close()}l=typeof process.exitCode==`number`?process.exitCode:0}finally{c?.()}l!==void 0&&process.exit(l)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}function setEnvForRun(e,t){let n=process.env[e];return process.env[e]=t,()=>{n===void 0?delete process.env[e]:process.env[e]=n}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};

package/dist/src/evals/context.d.ts

@@ -0,0 +1,19 @@
+import { EvalSessionManager } from "#evals/session.js";
+import { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalContext, EveEvalJudgeConfig, EveEvalTargetHandle } from "#evals/types.js";
+/**
+ * Builds the `EveEvalContext` (`t`) for one eval run, wiring the session
+ * manager (driving), the assertion collector (recording), and the judge
+ * namespace. Returns the collector so the runner can {@link
+ * AssertionCollector.finalize} it against the completed task result.
+ */
+export declare function createEvalContext(deps: {
+    readonly manager: EvalSessionManager;
+    readonly target: EveEvalTargetHandle;
+    readonly signal: AbortSignal;
+    readonly judge: EveEvalJudgeConfig | undefined;
+    readonly log: (message: string) => void;
+}): {
+    readonly context: EveEvalContext;
+    readonly collector: AssertionCollector;
+};

package/dist/src/evals/context.js

@@ -0,0 +1 @@
+import"#evals/session.js";import{AssertionCollector}from"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{buildJudgeContext}from"#evals/judge.js";function createEvalContext(r){let i=new AssertionCollector,a=``,primary=()=>r.manager.primary,replyMessage=()=>r.manager.lastTurnSession()?.lastTurn?.message??null,o=buildJudgeContext({collector:i,getReply:replyMessage,getInput:()=>a,judge:r.judge});return{context:{get events(){return primary().events},get pendingInputRequests(){return primary().pendingInputRequests},get state(){return primary().state},get sessionId(){return primary().sessionId},expectInputRequests:e=>primary().expectInputRequests(e),respond:(...e)=>primary().respond(...e),respondAll:e=>primary().respondAll(e),send:e=>(a=promptText(e),primary().send(e)),sendFile:(e,t,n)=>(a=e,primary().sendFile(e,t,n)),signal:r.signal,target:r.target,get reply(){return replyMessage()},log:r.log,newSession:()=>r.manager.newSession(),completed:()=>i.recordRun(RunAssertions.completed()),didNotFail:()=>i.recordRun(RunAssertions.didNotFail()),waiting:()=>i.recordRun(RunAssertions.waiting()),messageIncludes:e=>i.recordRun(RunAssertions.messageIncludes(e)),calledTool:(e,n)=>i.recordRun(RunAssertions.calledTool(e,n)),notCalledTool:e=>i.recordRun(RunAssertions.notCalledTool(e)),toolOrder:e=>i.recordRun(RunAssertions.toolOrder(e)),usedNoTools:()=>i.recordRun(RunAssertions.usedNoTools()),maxToolCalls:e=>i.recordRun(RunAssertions.maxToolCalls(e)),calledSubagent:(e,n)=>i.recordRun(RunAssertions.calledSubagent(e,n)),noFailedActions:()=>i.recordRun(RunAssertions.noFailedActions()),event:(e,n)=>i.recordRun(RunAssertions.event(e,n)),outputEquals:e=>i.recordRun(RunAssertions.outputEquals(e)),outputMatches:e=>i.recordRun(RunAssertions.outputMatches(e)),check:(e,t)=>recordCheck(i,e,t),judge:o},collector:i}}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:async()=>({score:await n.score(t)})})}function promptText(e){if(typeof e==`string`)return e;let t=e.message;return typeof t==`string`?t:``}export{createEvalContext};

package/dist/src/evals/define-eval-config.d.ts

@@ -0,0 +1,16 @@
+import type { EveEvalConfig, EveEvalConfigInput } from "#evals/types.js";
+/**
+ * Defines the run-wide configuration shared by every eval, authored as the
+ * default export of `evals.config.ts` at the root of the `evals/` directory.
+ *
+ * Exactly one `evals.config.ts` is required. It supplies the optional default
+ * `judge` model for `t.judge.*` assertions (so individual evals need not
+ * repeat it), optional run-level `reporters`, a default `maxConcurrency`, and a
+ * default `timeoutMs`. CLI flags (`--max-concurrency`, `--timeout`) and
+ * per-eval values take precedence over the config defaults.
+ *
+ * Throws on invalid input: a `judge` without a `model`, a non-positive or
+ * non-integer `maxConcurrency`, a negative or non-finite `timeoutMs`, or a
+ * non-array `reporters`.
+ */
+export declare function defineEvalConfig(input: EveEvalConfigInput): EveEvalConfig;

package/dist/src/evals/define-eval-config.js

@@ -0,0 +1 @@
+function defineEvalConfig(e){return validateEvalConfigInput(e),{...e,_tag:`EveEvalConfig`}}function validateEvalConfigInput(e){if(e.judge!==void 0&&(e.judge.model===void 0||e.judge.model===null))throw Error("Eval config `judge` requires a `model`. It is the default judge model for `t.judge.*` assertions across every eval.");if(e.maxConcurrency!==void 0&&(!Number.isInteger(e.maxConcurrency)||e.maxConcurrency<1))throw Error("Eval config `maxConcurrency` must be a positive integer.");if(e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval config `timeoutMs` must be a non-negative finite number.");if(e.reporters!==void 0&&!Array.isArray(e.reporters))throw Error("Eval config `reporters` must be an array of reporters.")}export{defineEvalConfig};

package/dist/src/evals/define-eval.d.ts

@@ -0,0 +1,20 @@
+import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";
+/**
+ * Defines one Eve eval. Each eval file is exactly one case: an imperative
+ * `test(t)` function that drives the agent (`t.send`, `t.respond`, …) and
+ * asserts on what it produced (`t.completed()`, `t.check(...)`,
+ * `t.judge.autoevals.*`). Organize related evals with directory nesting under
+ * `evals/`, or default-export an array of evals to fan one file out over a
+ * dataset.
+ *
+ * A `judge` is optional: `t.judge.*` assertions fall back to the `judge`
+ * declared in `evals.config.ts` unless this eval overrides it. The judge model
+ * is used solely for scoring, never for the agent under test. Eval identity is
+ * derived from the `evals/<path>.eval.ts` file path by the discovery layer, so
+ * authoring `id` or `name` throws.
+ *
+ * Throws on invalid input: a missing `test` function, a removed legacy key
+ * (`input`/`run`/`checks`/`scores`/`expected`/`thresholds`/`parseOutput`/
+ * `model`), or a negative or non-finite `timeoutMs`.
+ */
+export declare function defineEval(input: EveEvalInput): EveEvalDefinition;

package/dist/src/evals/define-eval.js

@@ -0,0 +1 @@
+function defineEval(e){return validateEvalInput(e),{...e,_tag:`EveEval`}}function validateEvalInput(e){if(`id`in e)throw Error("Eval must not specify `id`. Eval identity is derived from the file path under evals/.");if(`name`in e)throw Error("Eval must not specify `name`. Eval identity is derived from the file path under evals/.");if(rejectLegacyKey(e,`input`,"Send the prompt inside `test`: `async test(t) { await t.send(...) }`."),rejectLegacyKey(e,`run`,"Rename `run` to `test`; it receives the same context `t`."),rejectLegacyKey(e,`checks`,"Assert inline inside `test` (e.g. `t.completed()`, `t.calledTool(...)`)."),rejectLegacyKey(e,`scores`,"Use soft assertions inside `test`: `t.check(...).atLeast(n)` or `t.judge.autoevals.*`."),rejectLegacyKey(e,`expected`,"Pass the reference value to the assertion (e.g. `t.check(t.reply, includes(value))`)."),rejectLegacyKey(e,`thresholds`,"Put the threshold on the assertion: `.atLeast(n)`."),rejectLegacyKey(e,`parseOutput`,"Read the value you want inside `test` and assert on it directly."),rejectLegacyKey(e,`model`,"Rename `model` to `judge: { model }`."),rejectLegacyKey(e,`modelOptions`,"Move it under `judge: { model, modelOptions }`."),rejectLegacyKey(e,`cases`,"Each eval file is one case; default-export an array of `defineEval(...)` for datasets."),typeof e.test!=`function`)throw Error("Eval requires a `test(t)` function.");if(validateRequirements(e.requires,"Eval `requires`"),e.timeoutMs!==void 0&&(e.timeoutMs<0||!Number.isFinite(e.timeoutMs)))throw Error("Eval `timeoutMs` must be a non-negative finite number.")}function rejectLegacyKey(e,t,n){if(t in e)throw Error(`Eval \`${t}\` is no longer supported. ${n}`)}function validateRequirements(e,t){if(e!==void 0){if(!Array.isArray(e))throw Error(`${t} must be an array of requirement strings.`);for(let n of e)if(!(n===`mockModels`||n===`devRoutes`)&&!(n.startsWith(`env:`)&&n.length>4))throw Error(`${t} contains unsupported requirement "${n}". Expected "mockModels", "devRoutes", or "env:<NAME>".`)}}export{defineEval};

package/dist/src/evals/expect/index.d.ts

@@ -0,0 +1,25 @@
+import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
+import type { Assertion } from "#evals/types.js";
+export type { Assertion, AssertionHandle, AssertionSeverity } from "#evals/types.js";
+/**
+ * Passes when the value (coerced to a string) contains `substring`. A hard
+ * gate by default. Apply with `t.check(value, includes("..."))`.
+ */
+export declare function includes(substring: string): Assertion;
+/**
+ * Passes when the value deep-equals `expected` (exact structural equality).
+ * A hard gate by default.
+ */
+export declare function equals(expected: unknown): Assertion;
+/**
+ * Passes when the value validates against a Standard Schema (e.g. a Zod
+ * schema). A hard gate by default.
+ */
+export declare function matches(schema: StandardSchemaV1): Assertion;
+/**
+ * Scores normalized character-level Levenshtein similarity between the value
+ * and `expected` (1 = identical, 0 = entirely different). Soft by default —
+ * tracked unless you set a bar with `.atLeast(...)`. Use it for fuzzy
+ * comparison when exact match is too strict but a judge model is overkill.
+ */
+export declare function similarity(expected: string): Assertion;

package/dist/src/evals/expect/index.js

@@ -0,0 +1 @@
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{deepEquals}from"#evals/match.js";function makeAssertion(e){return{name:e.name,severity:e.severity,threshold:e.threshold,score:e.score,gate(t){return makeAssertion({...e,severity:`gate`,threshold:t})},soft(t){return makeAssertion({...e,severity:`soft`,threshold:t})},atLeast(t){return makeAssertion({...e,severity:`soft`,threshold:t})}}}function includes(e){return makeAssertion({name:`includes(${e})`,severity:`gate`,score:t=>+!!String(t??``).includes(e)})}function equals(e){return makeAssertion({name:`equals`,severity:`gate`,score:n=>+!!deepEquals(n,e)})}function matches(e){return makeAssertion({name:`matches`,severity:`gate`,score:async t=>{let n=await e[`~standard`].validate(t);return+(!(`issues`in n)||n.issues===void 0)}})}function similarity(t){return makeAssertion({name:`similarity`,severity:`soft`,score:async n=>(await Levenshtein({output:String(n??``),expected:t})).score??0})}export{equals,includes,matches,similarity};

package/dist/src/evals/index.d.ts

@@ -1,4 +1,8 @@
-export { defineEvalSuite } from "#evals/define-eval-suite.js";
+export { defineEval } from "#evals/define-eval.js";
+export { defineEvalConfig } from "#evals/define-eval-config.js";
+export { EveEvalTurnFailedError } from "#evals/session.js";
+export { EveEvalRequirementError } from "#evals/target.js";
 export type { RuntimeIdentity } from "#protocol/message.js";
 export type { InputRequest } from "#runtime/input/types.js";
-export type { EveEvalCase, EveEvalCaseResult, EveEvalCaseVerdict, EveEvalCheck, EveEvalCheckArgs, EveEvalCheckResult, EveEvalDerivedFacts, EveEvalScorer, EveEvalScorerArgs, EveEvalScorerResult, EveEvalSubagentCall, EveEvalSuite, EveEvalSuiteDefinition, EveEvalSuiteInput, EveEvalSuiteResult, EveEvalTarget, EveEvalTask, EveEvalTaskResult, EveEvalToolCall, } from "#evals/types.js";
+export type { EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
+export type { Assertion, AssertionHandle, AssertionResult, AssertionSeverity, AutoevalsJudges, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSessionResult, EveEvalRequirement, EveEvalScheduleDispatchResult, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalDefinition, EveEvalInput, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalToolCall, EveEvalTurn, EveEvalVerdict, JudgeContext, JudgeOpts, } from "#evals/types.js";

package/dist/src/evals/index.js

@@ -1 +1 @@
-import{defineEvalSuite}from"#evals/define-eval-suite.js";export{defineEvalSuite};
+import{EveEvalRequirementError}from"#evals/target.js";import{EveEvalTurnFailedError}from"#evals/session.js";import{defineEval}from"#evals/define-eval.js";import{defineEvalConfig}from"#evals/define-eval-config.js";export{EveEvalRequirementError,EveEvalTurnFailedError,defineEval,defineEvalConfig};

package/dist/src/evals/judge.d.ts

@@ -0,0 +1,20 @@
+import type { AssertionCollector } from "#evals/assertions/collector.js";
+import type { EveEvalJudgeConfig, JudgeContext } from "#evals/types.js";
+/**
+ * Dependencies the judge namespace closes over: where to record assertions,
+ * how to resolve the default graded value (`t.reply`) and the original prompt
+ * (the autoevals `input`), and the eval/config judge model.
+ */
+export interface JudgeDeps {
+    readonly collector: AssertionCollector;
+    readonly getReply: () => unknown;
+    readonly getInput: () => string;
+    readonly judge: EveEvalJudgeConfig | undefined;
+}
+/**
+ * Builds the `t.judge` namespace bound to the resolved judge model. Each
+ * grader records a soft assertion (override with `.atLeast`/`.gate`) that
+ * fires the model call immediately; the collector awaits it before the
+ * verdict.
+ */
+export declare function buildJudgeContext(deps: JudgeDeps): JudgeContext;

package/dist/src/evals/judge.js

@@ -0,0 +1 @@
+import{ClosedQA,Factuality,Sql,Summary}from"../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import{createAutoevalsClient}from"#evals/autoevals-client.js";function buildJudgeContext(a){function grade(e,t,n){let r=t?.on??a.getReply(),o=String(r??``),s=a.getInput(),c=t?.model??a.judge?.model,l=t?.modelOptions??a.judge?.modelOptions;return a.collector.recordValue({name:e,severity:`soft`,score:async()=>{if(c===void 0)throw Error(`${e} needs a judge model. Set \`judge\` on the eval or in evals.config.ts, or pass { model } to the call.`);return{score:(await n({input:s,output:o,model:c,modelOptions:l})).score??0,metadata:{judge:formatLanguageModelGatewayId(c)}}}})}return{autoevals:{factuality:(e,n)=>grade(`judge.autoevals.factuality`,n,({input:n,output:r,model:i,modelOptions:a})=>Factuality({input:n,output:r,expected:e,...client(i,a)})),summarizes:(e,t)=>grade(`judge.autoevals.summarizes`,t,({input:t,output:n,model:i,modelOptions:a})=>Summary({input:t,output:n,expected:e,...client(i,a)})),closedQA:(t,n)=>grade(`judge.autoevals.closedQA`,n,({input:n,output:r,model:i,modelOptions:a})=>ClosedQA({input:n,output:r,criteria:t,...client(i,a)})),sql:(e,t)=>grade(`judge.autoevals.sql`,t,({input:t,output:r,model:i,modelOptions:a})=>Sql({input:t,output:r,expected:e,...client(i,a)}))}}}function client(e,t){return{model:formatLanguageModelGatewayId(e),client:createAutoevalsClient({languageModel:e,providerOptions:t?.providerOptions})}}export{buildJudgeContext};

package/dist/src/evals/{checks/match.d.ts → match.d.ts}

@@ -1,7 +1,7 @@
-import type { EveEvalCheckArgs, EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
+import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
 /**
- * One matcher accepted by the built-in checks' options (`Checks.toolCalled`,
- * `Checks.subagentCalled`, `Run.usedTool`):
+ * One matcher accepted by the assertion options (`t.calledTool`,
+ * `t.calledSubagent`):
  *
  * - a **literal** is compared structurally; objects partial-deep-match (every
  *   key in the matcher must match the observed value, recursively, and nested
@@ -9,17 +9,16 @@ import type { EveEvalCheckArgs, EveEvalSubagentCall, EveEvalToolCall } from "#ev
  *   compare with `Object.is`
  * - a **RegExp** tests string values directly and the JSON serialization of
  *   anything else
- * - a **function** receives the observed value (plus the check args when
- *   invoked from a check) and either returns a boolean verdict, or returns a
- *   resolved expected value that is then compared like a literal — e.g.
- *   `() => process.env.EVE_WEATHER_AGENT_HOST` compares against a
- *   runner-assigned value. To assert a literal boolean field, use the literal
- *   directly; boolean returns are always treated as verdicts.
+ * - a **function** receives the observed value and either returns a boolean
+ *   verdict, or returns a resolved expected value that is then compared like a
+ *   literal — e.g. `(o) => o === process.env.EVE_WEATHER_AGENT_HOST`. To assert
+ *   a literal boolean field, use the literal directly; boolean returns are
+ *   always treated as verdicts.
  */
-export type EveEvalValueMatcher<T = unknown> = T | RegExp | ((value: T, args?: EveEvalCheckArgs) => unknown);
+export type EveEvalValueMatcher<T = unknown> = T | RegExp | ((value: T) => unknown);
 /**
- * Constraints applied to tool calls by `Checks.toolCalled` and
- * `Run.usedTool`. All provided constraints must hold for a call to match.
+ * Constraints applied to tool calls by `t.calledTool`. All provided
+ * constraints must hold for a call to match.
  */
 export interface EveEvalToolCallMatchOptions {
     /** Partial-deep matcher over the call input (see {@link EveEvalValueMatcher}). */
@@ -32,7 +31,7 @@ export interface EveEvalToolCallMatchOptions {
     readonly times?: number;
 }
 /**
- * Constraints applied to subagent calls by `Checks.subagentCalled`.
+ * Constraints applied to subagent calls by `t.calledSubagent`.
  */
 export interface EveEvalSubagentCallMatchOptions {
     /** Matcher over the `subagent.called` remote URL. */
@@ -44,25 +43,25 @@ export interface EveEvalSubagentCallMatchOptions {
  * Returns true when the observed value satisfies a matcher (literal, RegExp,
  * or function — see {@link EveEvalValueMatcher}).
  */
-export declare function matchesValue(matcher: unknown, value: unknown, args?: EveEvalCheckArgs): boolean;
+export declare function matchesValue(matcher: unknown, value: unknown): boolean;
 /**
  * Returns true when one derived tool call satisfies the `input`/`output`/
  * `isError` constraints (the `times` count is the caller's concern).
  */
-export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions, args?: EveEvalCheckArgs): boolean;
+export declare function toolCallMatches(call: EveEvalToolCall, options: EveEvalToolCallMatchOptions): boolean;
 /**
  * Returns true when one derived subagent call satisfies the `remoteUrl`/
  * `output` constraints.
  */
-export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions, args?: EveEvalCheckArgs): boolean;
+export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions): boolean;
 /**
- * Strict structural equality used by `Checks.outputEquals`: unlike matcher
+ * Strict structural equality used by `t.outputEquals`: unlike matcher
  * comparison, objects must carry exactly the same keys on both sides.
  */
 export declare function deepEquals(a: unknown, b: unknown): boolean;
 /**
  * Tests a RegExp without carrying `lastIndex` state between calls. Matcher
- * patterns are reused across tool calls and across every case in a suite, so
+ * patterns are reused across tool calls and across every case in an eval, so
  * a `g`/`y`-flagged pattern would otherwise return order-dependent results.
  */
 export declare function testRegExp(pattern: RegExp, text: string): boolean;

package/dist/src/evals/match.js

@@ -0,0 +1 @@
+function matchesValue(e,t){if(e instanceof RegExp)return testRegExpAgainst(e,t);if(typeof e==`function`){let n=e(t);return typeof n==`boolean`?n:typeof n==`function`?Object.is(n,t):matchesValue(n,t)}return Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||t.isError!==void 0&&e.isError!==t.isError)}function subagentCallMatches(e,t){return!(t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length?n.every(n=>deepEquals(e[n],t[n])):!1}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};

package/dist/src/evals/reporters/index.d.ts

@@ -1,2 +1,3 @@
 export { Braintrust, type BraintrustReporterConfig } from "#evals/runner/reporters/braintrust.js";
+export { JUnit, type JUnitReporterConfig } from "#evals/runner/reporters/junit.js";
 export type { EvalReporter } from "#evals/runner/reporters/types.js";

package/dist/src/evals/reporters/index.js

@@ -1 +1 @@
-import{Braintrust}from"#evals/runner/reporters/braintrust.js";export{Braintrust};
+import{JUnit}from"#evals/runner/reporters/junit.js";import{Braintrust}from"#evals/runner/reporters/braintrust.js";export{Braintrust,JUnit};

package/dist/src/evals/requirements.d.ts

@@ -0,0 +1,3 @@
+import type { EveEvalRequirement, EveEvalTarget } from "#evals/types.js";
+export declare function findUnmetRequirements(requirements: readonly EveEvalRequirement[], target: EveEvalTarget, env?: NodeJS.ProcessEnv): readonly EveEvalRequirement[];
+export declare function formatUnmetRequirements(requirements: readonly EveEvalRequirement[]): string;

package/dist/src/evals/requirements.js

@@ -0,0 +1 @@
+function findUnmetRequirements(e,t,n=process.env){let r=[];for(let i of new Set(e))requirementMet(i,t,n)||r.push(i);return r}function formatUnmetRequirements(e){return e.length===0?``:`Unmet eval requirement${e.length===1?``:`s`}: ${e.map(formatRequirement).join(`, `)}`}function requirementMet(e,t,n){if(e===`mockModels`)return t.capabilities.mockModels;if(e===`devRoutes`)return t.capabilities.devRoutes;let r=e.slice(4);return r.length>0&&n[r]!==void 0}function formatRequirement(e){return e===`mockModels`?`"mockModels" (local evals can enable this with --mock-models)`:e===`devRoutes`?`"devRoutes"`:`"${e}"`}export{findUnmetRequirements,formatUnmetRequirements};

package/dist/src/evals/runner/artifacts.d.ts

@@ -1,11 +1,12 @@
-import type { EveEvalSuiteResult } from "#evals/types.js";
+import type { EveEvalRunSummary } from "#evals/types.js";
 /**
- * Resolves the artifact output directory for one suite run.
+ * Resolves the artifact output directory for one `eve eval` run.
  *
- * Layout: `.eve/evals/<timestamp>-<suite-id>/`
+ * Layout: `.eve/evals/<timestamp>/`
  */
-export declare function resolveArtifactDirectory(appRoot: string, suiteId: string): string;
+export declare function resolveArtifactDirectory(appRoot: string): string;
 /**
- * Writes all artifacts for a completed suite run.
+ * Writes all artifacts for a completed eval run: a run summary, a JSONL
+ * results index, and per-eval detail/event files under `evals/`.
  */
-export declare function writeArtifacts(artifactDir: string, suiteResult: EveEvalSuiteResult): Promise<void>;
+export declare function writeArtifacts(artifactDir: string, summary: EveEvalRunSummary): Promise<void>;

package/dist/src/evals/runner/artifacts.js

@@ -1,3 +1,3 @@
-import{join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(t,n){return join(t,`.eve`,`evals`,`${new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19)}-${n}`)}async function writeArtifacts(r,i){let a=join(r,`cases`);await mkdir(a,{recursive:!0}),await writeFile(join(r,`summary.json`),JSON.stringify(buildSummaryArtifact(i),null,2));let o=i.cases.map(e=>JSON.stringify(buildCaseResultLine(e))).join(`
-`);await writeFile(join(r,`results.jsonl`),`${o}\n`),await Promise.all(i.cases.map(async t=>{let r=sanitizeFileName(t.case.id);await writeFile(join(a,`${r}.json`),JSON.stringify(buildCaseArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
-`);await writeFile(join(a,`${r}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{suite:e.suite,target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalCases:e.cases.length,cases:e.cases.map(e=>({id:e.case.id,verdict:e.verdict,status:e.result.status,checks:e.checks.map(e=>({name:e.name,passed:e.passed})),scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error,skipReason:e.skipReason}))}}function buildCaseResultLine(e){return{caseId:e.case.id,verdict:e.verdict,status:e.result.status,output:e.result.output,checks:e.checks,scores:e.scores.map(e=>({name:e.name,score:e.score})),error:e.error,skipReason:e.skipReason}}function buildCaseArtifact(e){return{case:{id:e.case.id,input:e.case.input,expected:e.case.expected,tags:e.case.tags,metadata:e.case.metadata},result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,derived:e.result.derived},verdict:e.verdict,checks:e.checks,scores:e.scores,error:e.error,skipReason:e.skipReason}}function sanitizeFileName(e){return e.replace(/[^a-zA-Z0-9_-]/g,`_`)}export{resolveArtifactDirectory,writeArtifacts};
+import{dirname,join}from"node:path";import{mkdir,writeFile}from"node:fs/promises";function resolveArtifactDirectory(e){return join(e,`.eve`,`evals`,new Date().toISOString().replace(/[:.]/g,`-`).slice(0,19))}async function writeArtifacts(t,n){let r=join(t,`evals`);await mkdir(r,{recursive:!0}),await writeFile(join(t,`summary.json`),JSON.stringify(buildSummaryArtifact(n),null,2));let i=n.results.map(e=>JSON.stringify(buildResultLine(e))).join(`
+`);await writeFile(join(t,`results.jsonl`),`${i}\n`),await Promise.all(n.results.map(async t=>{let n=join(r,`${sanitizeArtifactPath(t.id)}.json`);await mkdir(dirname(n),{recursive:!0}),await writeFile(n,JSON.stringify(buildEvalArtifact(t),null,2));let i=t.result.events.map(e=>JSON.stringify(e)).join(`
+`);await writeFile(join(r,`${sanitizeArtifactPath(t.id)}.events.ndjson`),`${i}\n`)}))}function buildSummaryArtifact(e){return{target:e.target,startedAt:e.startedAt,completedAt:e.completedAt,passed:e.passed,failed:e.failed,scored:e.scored,skipped:e.skipped,errored:e.errored,totalEvals:e.results.length,evals:e.results.map(e=>({id:e.id,verdict:e.verdict,status:e.result.status,assertions:e.assertions.map(e=>({name:e.name,score:e.score,severity:e.severity,passed:e.passed})),error:e.error,skipReason:e.skipReason}))}}function buildResultLine(e){return{id:e.id,verdict:e.verdict,status:e.result.status,output:e.result.output,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function buildEvalArtifact(e){return{id:e.id,result:{output:e.result.output,finalMessage:e.result.finalMessage,sessionId:e.result.sessionId,status:e.result.status,logs:e.result.logs,derived:e.result.derived,sessions:e.result.sessions},verdict:e.verdict,assertions:e.assertions,error:e.error,skipReason:e.skipReason}}function sanitizeArtifactPath(e){return e.split(`/`).map(e=>e.replace(/[^a-zA-Z0-9_-]/g,`_`)).join(`/`)}export{resolveArtifactDirectory,writeArtifacts};