eve 0.5.4 → 0.6.0-beta.3

package/dist/src/evals/runner/execute-case.d.ts

@@ -0,0 +1,23 @@
+import type { Client } from "#client/client.js";
+import type { EveEvalCase, EveEvalTask, EveEvalTaskResult } from "#evals/types.js";
+/**
+ * Options for executing one eval case.
+ */
+interface ExecuteCaseOptions {
+    readonly client: Client;
+    readonly testCase: EveEvalCase;
+    readonly task?: EveEvalTask;
+    readonly timeoutMs?: number;
+}
+/**
+ * Executes one eval case against an Eve agent target.
+ *
+ * The runner owns session lifecycle, stream capture, and derived metadata.
+ * The suite only declares how to derive messages and parse outputs.
+ *
+ * For multi-turn cases (`task.messages`), events from all turns are
+ * accumulated so derived facts reflect the full conversation. The output
+ * and status come from the final turn only.
+ */
+export declare function executeCase(options: ExecuteCaseOptions): Promise<EveEvalTaskResult>;
+export {};

package/dist/src/evals/runner/execute-case.js

@@ -0,0 +1 @@
+import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";async function executeCase(t){let{client:n,testCase:r,task:i,timeoutMs:a}=t,o=n.session(),s=resolveMessages(r,i),c=a===void 0?void 0:AbortSignal.timeout(a),l=[],u;for(let e of s)u=await(await o.send({message:e,signal:c})).result(),l.push(...u.events);if(!u)return createEmptyResult();let d=u.message??null,f=deriveRunFacts(l),p=extractRuntimeIdentity(l),m={output:d,finalMessage:d,sessionId:u.sessionId,status:u.status,events:l,derived:f,runtimeIdentity:p};return i?.parseOutput!==void 0&&(m.output=i.parseOutput(m)),m}function resolveMessages(e,t){return t?.messages===void 0?t?.prompt===void 0?[typeof e.input==`string`?e.input:JSON.stringify(e.input)]:[t.prompt(e)]:t.messages(e)}function extractRuntimeIdentity(e){for(let t of e)if(t.type===`session.started`&&t.data.runtime!==void 0)return t.data.runtime}function createEmptyResult(){return{output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}export{executeCase};

package/dist/src/evals/runner/execute-suite.d.ts

@@ -0,0 +1,24 @@
+import type { Client } from "#client/client.js";
+import type { EveEvalSuite, EveEvalSuiteResult, EveEvalTarget } from "#evals/types.js";
+import type { EvalReporter } from "#evals/runner/reporters/types.js";
+/**
+ * Options for executing one eval suite.
+ */
+interface ExecuteSuiteOptions {
+    readonly suite: EveEvalSuite;
+    readonly target: EveEvalTarget;
+    readonly reporters: readonly EvalReporter[];
+    readonly appRoot: string;
+    /**
+     * Pre-configured client for communicating with the Eve agent.
+     * The CLI constructs this once with the appropriate auth and headers,
+     * and every case creates a fresh session from it.
+     */
+    readonly client: Client;
+}
+/**
+ * Executes all cases in one eval suite, runs scorers, writes artifacts, and
+ * reports results.
+ */
+export declare function executeSuite(options: ExecuteSuiteOptions): Promise<EveEvalSuiteResult>;
+export {};

package/dist/src/evals/runner/execute-suite.js

@@ -0,0 +1 @@
+import{toErrorMessage}from"#shared/errors.js";import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeCase}from"#evals/runner/execute-case.js";async function executeSuite(e){let{suite:r,target:i,reporters:a,appRoot:o,client:s}=e,c=new Date().toISOString();for(let e of a)await e.onSuiteStart(r,i);let l=await r.load(),u=r.maxConcurrency??8,d=[],f=[...l],p=new Set;for(;f.length>0||p.size>0;){for(;f.length>0&&p.size<u;){let e=f.shift();if(e===void 0)break;let t=(async()=>{let t=await executeSingleCase(r,e,s);d.push(t);for(let e of a)await e.onCaseComplete(t)})().finally(()=>{p.delete(t)});p.add(t)}p.size>0&&await Promise.race(p)}let m=new Date().toISOString(),h={suite:r.id,target:i,cases:d,startedAt:c,completedAt:m,passed:d.filter(e=>!e.error&&e.scores.every(e=>meetsThreshold(e,r.thresholds))).length,errored:d.filter(e=>e.error!==void 0).length};await writeArtifacts(resolveArtifactDirectory(o,r.id),h);for(let e of a)await e.onSuiteComplete(h);return h}async function executeSingleCase(t,n,i){let a,o;try{a=await executeCase({client:i,testCase:n,task:t.task,timeoutMs:t.timeoutMs})}catch(t){o=toErrorMessage(t),a={output:null,finalMessage:null,status:`failed`,events:[],derived:{toolCalls:[],toolCallCount:0,subagentCalls:[],subagentCallCount:0,messageCount:0,reasoningBlockCount:0}}}let s=await runScorers(t,n,a);return{case:n,result:a,scores:s,error:o}}async function runScorers(e,t,n){let r={input:typeof t.input==`string`?t.input:JSON.stringify(t.input),output:String(n.output??``),expected:t.expected===void 0?void 0:String(t.expected),model:e.model,modelOptions:e.modelOptions,case:t,result:n},i=[];for(let t of e.scores){let e=await t(r);e!==null&&i.push(e)}return i}function meetsThreshold(e,t){if(e.score===null)return!0;let n=t?.[e.name]??1;return e.score>=n}export{executeSuite};

package/dist/src/evals/runner/reporters/braintrust.d.ts

@@ -0,0 +1,27 @@
+import type { EvalReporter } from "#evals/runner/reporters/types.js";
+/**
+ * Configuration for the Braintrust reporter. Every field is optional and maps
+ * onto the corresponding Braintrust SDK `init` option.
+ */
+export interface BraintrustReporterConfig {
+    /** Braintrust project id. Maps to the `projectId` init option. */
+    readonly projectId?: string;
+    /** Braintrust project name. Defaults to the suite id when omitted. */
+    readonly projectName?: string;
+    /** Name for the created experiment. When omitted, Braintrust names it. */
+    readonly experimentName?: string;
+    /** Name of an existing experiment to compare results against (the diff base). */
+    readonly baseExperimentName?: string;
+    /** Id of an existing experiment to compare results against (the diff base). */
+    readonly baseExperimentId?: string;
+    /** When true, update a matching existing experiment instead of creating one. */
+    readonly update?: boolean;
+}
+/**
+ * Creates an {@link EvalReporter} that uploads eval results to Braintrust as an
+ * experiment. Add it to a suite's `reporters` array. Requires the optional
+ * `braintrust` peer package; the reporter throws a descriptive error when the
+ * suite starts if it is missing. `config` is optional and defaults to an empty
+ * object (the project name then falls back to the suite id).
+ */
+export declare function Braintrust(config?: BraintrustReporterConfig): EvalReporter;

package/dist/src/evals/runner/reporters/braintrust.js

@@ -0,0 +1,2 @@
+import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Braintrust(e={}){return new BraintrustReporter(e)}var BraintrustReporter=class{#e;#t;#n;#r;constructor(e){this.#e=e}async onSuiteStart(t,n){let r=await loadBraintrustSdk();this.#t=r;let i=resolveLocalGitMetadata(process.cwd());this.#r=t.metadata;let a=resolveTags(t,n),o=resolveExperimentMetadata(t,n);this.#n=await r.init({project:this.#e.projectName??t.id,projectId:this.#e.projectId,experiment:this.#e.experimentName,baseExperiment:this.#e.baseExperimentName,baseExperimentId:this.#e.baseExperimentId,update:this.#e.update,tags:a,metadata:o,noExitFlush:!0,setCurrent:!1,repoInfo:i.sha?{commit:i.sha,branch:i.branch}:null})}onCaseComplete(e){if(!this.#n)return;let t={};for(let n of e.scores)n.score!==null&&(t[n.name]=n.score);let n={...this.#r,...e.case.metadata,eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveToolCalls:e.result.derived.toolCalls,eveSubagentCalls:e.result.derived.subagentCalls};e.result.derived.failureCode&&(n.eveFailureCode=e.result.derived.failureCode);let r={toolCallCount:e.result.derived.toolCallCount,subagentCallCount:e.result.derived.subagentCallCount,messageCount:e.result.derived.messageCount,reasoningBlockCount:e.result.derived.reasoningBlockCount};this.#n.log({id:e.case.id,input:e.case.input,output:e.result.output,expected:e.case.expected,error:e.error??void 0,scores:t,metadata:n,metrics:r,tags:e.case.tags?[...e.case.tags]:void 0})}async onSuiteComplete(e){if(this.#n)try{this.#t&&await this.#t.flush();let e=await this.#n.summarize();e.experimentUrl&&console.log(`Braintrust experiment: ${e.experimentUrl}\n\n`)}finally{await this.#n.close(),this.#n=void 0,this.#t=void 0}}};async function loadBraintrustSdk(){try{return await import(`braintrust`)}catch{throw Error([`The 'braintrust' package is required for Braintrust reporting but was not found.`,``,`Install it with:`,`  npm install braintrust`].join(`
+`))}}function resolveTags(e,t){let n=[`eve`,`suite:${e.id}`,`target:${t.kind}`];return Array.isArray(e.tags)&&n.push(...e.tags),n}function resolveExperimentMetadata(e,t){return{eveSuiteId:e.id,eveTargetKind:t.kind,eveTargetUrl:t.url,eveTimestamp:new Date().toISOString()}}export{Braintrust};

package/dist/src/evals/runner/reporters/console.d.ts

@@ -0,0 +1,15 @@
+import type { EveEvalCaseResult, EveEvalSuite, EveEvalSuiteResult, EveEvalTarget } from "#evals/types.js";
+import type { EvalReporter } from "#evals/runner/reporters/types.js";
+/**
+ * Console reporter that prints eval progress and results to stdout.
+ */
+export declare class ConsoleReporter implements EvalReporter {
+    #private;
+    constructor(options?: {
+        log?: (message: string) => void;
+        color?: boolean;
+    });
+    onSuiteStart(suite: EveEvalSuite, target: EveEvalTarget): void;
+    onCaseComplete(caseResult: EveEvalCaseResult): void;
+    onSuiteComplete(suiteResult: EveEvalSuiteResult): void;
+}

package/dist/src/evals/runner/reporters/console.js

@@ -0,0 +1 @@
+import picocolors from"#compiled/picocolors/index.js";var ConsoleReporter=class{#e;#t;#n;constructor(t){this.#e=t?.log??console.log,this.#t=picocolors.createColors(t?.color??!!process.stdout.isTTY)}onSuiteStart(e,t){this.#n=e.thresholds,this.#e(``),this.#e(`${this.#t.bold(this.#t.cyan(`EVAL`))} ${this.#t.bold(e.id)}`),this.#e(`${this.#t.dim(`target`)} ${t.kind===`local`?this.#t.green(t.url):this.#t.blue(t.url)}`),this.#e(``)}onCaseComplete(e){let{case:t,scores:n,error:r}=e,i=n.every(e=>{if(e.score===null)return!0;let t=this.#n?.[e.name]??1;return e.score>=t}),a=r?this.#t.red(`✗`):i?this.#t.green(`✓`):this.#t.yellow(`○`),o=n.map(e=>this.#r(e.name,e.score)).join(`  `),s=[a,this.#t.dim(t.id),o].filter(Boolean).join(`  `);this.#e(s),r&&this.#e(`  ${this.#t.red(r)}`)}onSuiteComplete(e){this.#e(``);let{passed:t,errored:n,cases:r}=e,i=r.length,a=[];t>0&&a.push(this.#t.green(`${t} passed`)),n>0&&a.push(this.#t.red(`${n} errored`));let o=i-t-n;o>0&&a.push(this.#t.yellow(`${o} scored`)),this.#e(`${this.#t.bold(`Results:`)} ${a.join(`, `)} ${this.#t.dim(`(${i} total)`)}`);let s=this.#i(r);if(s.length>0){this.#e(``);for(let{name:e,avg:t,count:n}of s){let r=this.#r(e,t);this.#e(`  ${r} ${this.#t.dim(`(${n} cases)`)}`)}}let c=computeDurationMs(e.startedAt,e.completedAt);this.#e(``),this.#e(this.#t.dim(`Completed in ${formatDuration(c)}`)),this.#e(``)}#r(e,t){if(t===null)return this.#t.dim(`${e}: n/a`);let n=`${e}: ${Math.round(t*100)}%`;return t===1?this.#t.green(n):t===0?this.#t.red(n):this.#t.yellow(n)}#i(e){let t=new Map;for(let n of e)for(let e of n.scores){if(e.score===null)continue;let n=t.get(e.name);n?(n.sum+=e.score,n.count+=1):t.set(e.name,{sum:e.score,count:1})}return[...t.entries()].map(([e,{sum:t,count:n}])=>({name:e,avg:t/n,count:n}))}};function computeDurationMs(e,t){return new Date(t).getTime()-new Date(e).getTime()}function formatDuration(e){return e<1e3?`${e}ms`:e<6e4?`${(e/1e3).toFixed(1)}s`:`${Math.floor(e/6e4)}m ${(e%6e4/1e3).toFixed(0)}s`}export{ConsoleReporter};

package/dist/src/evals/runner/reporters/types.d.ts

@@ -0,0 +1,20 @@
+import type { EveEvalCaseResult, EveEvalSuite, EveEvalSuiteResult, EveEvalTarget } from "#evals/types.js";
+/**
+ * Reporter lifecycle interface. The runner calls these methods at defined
+ * points during suite execution. Methods may return a promise for reporters
+ * that perform asynchronous work (e.g. uploading to a remote service).
+ */
+export interface EvalReporter {
+    /**
+     * The runner calls this once when a suite run begins.
+     */
+    onSuiteStart(suite: EveEvalSuite, target: EveEvalTarget): void | Promise<void>;
+    /**
+     * The runner calls this after each case completes, with its scores.
+     */
+    onCaseComplete(caseResult: EveEvalCaseResult): void | Promise<void>;
+    /**
+     * The runner calls this once when a suite run finishes, with the aggregated result.
+     */
+    onSuiteComplete(suiteResult: EveEvalSuiteResult): void | Promise<void>;
+}

package/dist/src/evals/runner/reporters/types.js

@@ -0,0 +1 @@
+export{};

package/dist/src/evals/runner/resolve-git-metadata.d.ts

@@ -0,0 +1,16 @@
+interface GitMetadata {
+    readonly branch?: string;
+    readonly sha?: string;
+}
+/**
+ * Resolves local git metadata for the eval run context.
+ *
+ * Used to populate `repoInfo` on the Braintrust experiment so the dashboard
+ * shows which sha/branch produced the run. This describes the eval code,
+ * not the remote target.
+ *
+ * Returns an empty object when git is unavailable or the directory is
+ * not a git repository.
+ */
+export declare function resolveLocalGitMetadata(cwd: string): GitMetadata;
+export {};

package/dist/src/evals/runner/resolve-git-metadata.js

@@ -0,0 +1 @@
+import{execSync}from"node:child_process";function resolveLocalGitMetadata(e){let t=execGitCommand(`git rev-parse HEAD`,e);return{branch:execGitCommand(`git branch --show-current`,e)||void 0,sha:t||void 0}}function execGitCommand(t,n){try{return execSync(t,{cwd:n,encoding:`utf8`,stdio:[`pipe`,`pipe`,`pipe`],timeout:5e3}).trim()}catch{return}}export{resolveLocalGitMetadata};

package/dist/src/evals/scorers/autoevals-client.d.ts

@@ -0,0 +1,14 @@
+import { generateText, type LanguageModel } from "ai";
+import { Factuality } from "autoevals";
+/**
+ * The OpenAI-shaped client surface autoevals expects. Extracted from the
+ * library so we don't take a direct dependency on `openai` types.
+ */
+type AutoevalsClient = NonNullable<Parameters<typeof Factuality>[0]["client"]>;
+type ProviderOptions = Parameters<typeof generateText>[0]["providerOptions"];
+interface AutoevalsClientConfig {
+    readonly languageModel: LanguageModel;
+    readonly providerOptions?: ProviderOptions;
+}
+export declare function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient;
+export {};

package/dist/src/evals/scorers/autoevals-client.js

@@ -0,0 +1,2 @@
+import"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import{generateText,jsonSchema}from"ai";function createAutoevalsClient(e){return{chat:{completions:{create:t=>createChatCompletion(t,e)}}}}async function createChatCompletion(t,n){let r=convertTools(t.tools),i=await generateText({model:n.languageModel,messages:convertMessages(t.messages??[]),tools:Object.keys(r).length>0?r:void 0,toolChoice:convertToolChoice(t.tool_choice),providerOptions:n.providerOptions}),a=i.toolCalls.map(e=>({id:e.toolCallId,type:`function`,function:{name:e.toolName,arguments:JSON.stringify(e.input??{})}}));return{choices:[{index:0,finish_reason:a.length>0?`tool_calls`:`stop`,message:{role:`assistant`,content:i.text||null,tool_calls:a.length>0?a:void 0}}]}}function convertMessages(e){return e.map(e=>{let t=contentToText(e.content);switch(e.role){case`assistant`:return{role:`assistant`,content:t};case`developer`:case`system`:return{role:`system`,content:t};default:return{role:`user`,content:t}}})}function contentToText(e){return e==null?``:typeof e==`string`?e:e.map(e=>e.text??``).filter(Boolean).join(`
+`)}function convertTools(e){let n={};for(let r of e??[])r.type!==`function`||r.function?.name===void 0||(n[r.function.name]={description:r.function.description,inputSchema:jsonSchema(r.function.parameters??{})});return n}function convertToolChoice(e){if(e!==void 0)return typeof e==`string`?e:{type:`tool`,toolName:e.function.name}}export{createAutoevalsClient};

package/dist/src/evals/scorers/autoevals.d.ts

@@ -0,0 +1,58 @@
+import { type LanguageModel } from "ai";
+import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
+import type { EveEvalScorer } from "#evals/types.js";
+/**
+ * Optional per-scorer overrides for model-backed Autoevals scorers. When
+ * omitted, the suite's `model` and `modelOptions` apply.
+ */
+export interface AutoevalsModelScorerOptions {
+    /** Judge model for this scorer. Overrides the suite judge model. */
+    readonly model?: LanguageModel;
+    /** Provider options for this scorer. Eve merges them over the suite's `modelOptions`. */
+    readonly modelOptions?: AgentModelOptionsDefinition;
+}
+/**
+ * Options for {@link closedQA}. Extends {@link AutoevalsModelScorerOptions}
+ * with the required free-form `criteria` the judge applies.
+ */
+export interface AutoevalsClosedQAOptions extends AutoevalsModelScorerOptions {
+    /**
+     * Natural-language criterion the judge model evaluates the output against.
+     *
+     * @example "The answer must mention the refund policy."
+     */
+    readonly criteria: string;
+}
+/**
+ * Scores output factuality against the case `expected` value using the
+ * Autoevals Factuality grader. Requires a per-case `expected` and a judge
+ * model (the suite `model`/`modelOptions`, overridable via `options`).
+ * Returns a graded score between 0 and 1.
+ */
+export declare function factuality(options?: AutoevalsModelScorerOptions): EveEvalScorer;
+/**
+ * Scores whether `output` summarizes the case `input` better than the case
+ * `expected` summary, using the Autoevals Summary grader. Requires a per-case
+ * `expected` and a judge model (the suite `model`/`modelOptions`, overridable
+ * via `options`). Returns a graded score between 0 and 1.
+ */
+export declare function summary(options?: AutoevalsModelScorerOptions): EveEvalScorer;
+/**
+ * Scores semantic equivalence between the `output` SQL and the case `expected`
+ * SQL using the model-judged Autoevals Sql grader. Requires a per-case
+ * `expected` and a judge model (the suite `model`/`modelOptions`, overridable
+ * via `options`). For a deterministic check, use `Sql.exactNormalized()`
+ * instead.
+ */
+export declare function sql(options?: AutoevalsModelScorerOptions): EveEvalScorer;
+/**
+ * Judges whether `output` meets a required free-form natural-language
+ * `criteria` using a judge model.
+ *
+ * Use this when "correct" cannot be expressed as exact or fuzzy match against
+ * an `expected` value (for example, "must cite a source" or "must apologize
+ * before refusing"). The `criteria` is the only required field; the judge model
+ * comes from the suite `model`/`modelOptions`, overridable via `options`.
+ * Per-case `expected` is ignored.
+ */
+export declare function closedQA(options: AutoevalsClosedQAOptions): EveEvalScorer;

package/dist/src/evals/scorers/autoevals.js

@@ -0,0 +1 @@
+import{ClosedQA,Factuality,Sql,Summary}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";import"ai";import{formatLanguageModelGatewayId}from"#internal/runtime-model.js";import{createAutoevalsClient}from"#evals/scorers/autoevals-client.js";function factuality(e={}){return languageScorer(Factuality,e)}function summary(e={}){return languageScorer(Summary,e)}function sql(e={}){return languageScorer(Sql,e)}function closedQA(t){return async n=>{let r=t.model??n.model;return toScorerResult(await ClosedQA({input:n.input,output:n.output,criteria:t.criteria,model:formatLanguageModelGatewayId(r),client:createAutoevalsClient({languageModel:r,providerOptions:mergeProviderOptions(n,t.modelOptions)})}))}}function languageScorer(e,t){return async n=>{let r=t.model??n.model;return toScorerResult(await e({input:n.input,output:n.output,expected:n.expected,model:formatLanguageModelGatewayId(r),client:createAutoevalsClient({languageModel:r,providerOptions:mergeProviderOptions(n,t.modelOptions)})}))}}function mergeProviderOptions(e,t){let n=e.modelOptions?.providerOptions,r=t?.providerOptions;if(!(n===void 0&&r===void 0))return{...n,...r}}function toScorerResult(e){return{name:e.name,score:e.score,metadata:e.metadata}}export{closedQA,factuality,sql,summary};

package/dist/src/evals/scorers/json.d.ts

@@ -0,0 +1,10 @@
+import type { EveEvalScorer } from "#evals/types.js";
+/**
+ * Scores 1 when the task output deep-equals the case expected value (via
+ * `JSON.stringify` with sorted keys), 0 otherwise. Returns `null` when
+ * expected is absent.
+ *
+ * Uses `case.expected` and `result.output` for the raw values rather than
+ * the coerced string fields.
+ */
+export declare function deepEqual(): EveEvalScorer;

package/dist/src/evals/scorers/json.js

@@ -0,0 +1 @@
+function deepEqual(){return({case:e,result:t})=>e.expected===void 0?null:{name:`json.deepEqual`,score:+(stableStringify(t.output)===stableStringify(e.expected))}}function stableStringify(e){return JSON.stringify(e,(e,t)=>{if(typeof t==`object`&&t&&!Array.isArray(t)){let e={};for(let n of Object.keys(t).sort())e[n]=t[n];return e}return t})}export{deepEqual};

package/dist/src/evals/scorers/run.d.ts

@@ -0,0 +1,18 @@
+import type { EveEvalScorer } from "#evals/types.js";
+/**
+ * Scores 1 when the run did not fail, 0 when it did.
+ */
+export declare function didNotFail(): EveEvalScorer;
+/**
+ * Scores 1 when the run called a tool with the given name at least once,
+ * 0 otherwise.
+ */
+export declare function usedTool(name: string): EveEvalScorer;
+/**
+ * Scores 1 when the run made no tool calls at all, 0 otherwise.
+ */
+export declare function usedNoTools(): EveEvalScorer;
+/**
+ * Scores 1 when the run made at most `max` tool calls, 0 otherwise.
+ */
+export declare function maxToolCalls(max: number): EveEvalScorer;

package/dist/src/evals/scorers/run.js

@@ -0,0 +1 @@
+function didNotFail(){return({result:e})=>({name:`run.didNotFail`,score:e.status===`failed`?0:1})}function usedTool(e){return({result:t})=>{let n=+!!t.derived.toolCalls.includes(e);return{name:`run.usedTool(${e})`,score:n,metadata:{expectedTool:e,actualToolCalls:t.derived.toolCalls}}}}function usedNoTools(){return({result:e})=>({name:`run.usedNoTools`,score:+(e.derived.toolCallCount===0),metadata:{toolCallCount:e.derived.toolCallCount}})}function maxToolCalls(e){return({result:t})=>{let n=+(t.derived.toolCallCount<=e);return{name:`run.maxToolCalls(${e})`,score:n,metadata:{maxAllowed:e,actualToolCallCount:t.derived.toolCallCount}}}}export{didNotFail,maxToolCalls,usedNoTools,usedTool};

package/dist/src/evals/scorers/sql.d.ts

@@ -0,0 +1,9 @@
+import type { EveEvalScorer } from "#evals/types.js";
+/**
+ * Scores 1 when the normalized SQL of `output` matches the normalized
+ * SQL of `expected`, 0 otherwise. Returns `null` when expected is absent.
+ *
+ * Normalization collapses whitespace, strips trailing semicolons, and
+ * lowercases the SQL string.
+ */
+export declare function exactNormalized(): EveEvalScorer;

package/dist/src/evals/scorers/sql.js

@@ -0,0 +1 @@
+function exactNormalized(){return({output:e,expected:t})=>{if(t===void 0)return null;let n=normalizeSql(e),r=normalizeSql(t);return{name:`sql.exactNormalized`,score:+(n===r),metadata:{normalizedOutput:n,normalizedExpected:r}}}}function normalizeSql(e){return e.toLowerCase().replace(/\s+/g,` `).trim().replace(/;+$/,``)}export{exactNormalized};

package/dist/src/evals/scorers/text.d.ts

@@ -0,0 +1,18 @@
+import type { EveEvalScorer } from "#evals/types.js";
+/**
+ * Scores 1 when `output` exactly equals `expected` (string comparison),
+ * 0 otherwise. Returns `null` when expected is absent.
+ */
+export declare function exact(): EveEvalScorer;
+/**
+ * Scores 1 when `output` contains `expected` as a substring,
+ * 0 otherwise. Returns `null` when expected is absent.
+ */
+export declare function includes(): EveEvalScorer;
+/**
+ * Scores by normalized character-level Levenshtein similarity
+ * (1 = identical, 0 = entirely different). Use it for fuzzy comparison
+ * when exact match is too strict but you do not want to call an LLM
+ * judge or embedding model. Returns `null` when expected is absent.
+ */
+export declare function levenshtein(): EveEvalScorer;

package/dist/src/evals/scorers/text.js

@@ -0,0 +1 @@
+import{Levenshtein}from"../../node_modules/.pnpm/autoevals@0.0.132_ws@8.21.0/node_modules/autoevals/jsdist/index.js";function exact(){return({output:e,expected:t})=>t===void 0?null:{name:`text.exact`,score:+(e===t)}}function includes(){return({output:e,expected:t})=>t===void 0?null:{name:`text.includes`,score:+!!e.includes(t)}}function levenshtein(){return async({output:t,expected:n})=>n===void 0?null:{name:`text.levenshtein`,score:(await Levenshtein({output:t,expected:n})).score}}export{exact,includes,levenshtein};

package/dist/src/evals/scores/index.d.ts

@@ -0,0 +1,72 @@
+import * as JsonScorers from "#evals/scorers/json.js";
+import * as RunScorers from "#evals/scorers/run.js";
+import * as SqlScorers from "#evals/scorers/sql.js";
+import * as TextScorers from "#evals/scorers/text.js";
+import * as AutoevalsScorers from "#evals/scorers/autoevals.js";
+export type { AutoevalsClosedQAOptions, AutoevalsModelScorerOptions, } from "#evals/scorers/autoevals.js";
+/**
+ * Text scorers for exact, substring, and fuzzy character-level matching.
+ *
+ * @example
+ * ```ts
+ * import { Text } from "eve/evals/scores";
+ *
+ * Text.exact()
+ * Text.includes()
+ * Text.levenshtein()
+ * ```
+ */
+export declare const Text: typeof TextScorers;
+/**
+ * JSON scorers for structural comparison.
+ *
+ * @example
+ * ```ts
+ * import { Json } from "eve/evals/scores";
+ *
+ * Json.deepEqual()
+ * ```
+ */
+export declare const Json: typeof JsonScorers;
+/**
+ * SQL scorers for normalized query comparison.
+ *
+ * @example
+ * ```ts
+ * import { Sql } from "eve/evals/scores";
+ *
+ * Sql.exactNormalized()
+ * ```
+ */
+export declare const Sql: typeof SqlScorers;
+/**
+ * Run-derived scorers for execution and tool-usage checks.
+ *
+ * @example
+ * ```ts
+ * import { Run } from "eve/evals/scores";
+ *
+ * Run.didNotFail()
+ * Run.usedTool("get_weather")
+ * Run.usedNoTools()
+ * Run.maxToolCalls(5)
+ * ```
+ */
+export declare const Run: typeof RunScorers;
+/**
+ * Eve-owned wrappers around Braintrust Autoevals scorers.
+ *
+ * String model IDs route through the Vercel AI Gateway; AI SDK model
+ * instances run directly.
+ *
+ * @example
+ * ```ts
+ * import { Autoevals } from "eve/evals/scores";
+ *
+ * Autoevals.factuality()
+ * Autoevals.summary()
+ * Autoevals.sql()
+ * Autoevals.closedQA({ criteria: "Must cite a source." })
+ * ```
+ */
+export declare const Autoevals: typeof AutoevalsScorers;

package/dist/src/evals/scores/index.js

@@ -0,0 +1 @@
+import*as JsonScorers from"#evals/scorers/json.js";import*as RunScorers from"#evals/scorers/run.js";import*as SqlScorers from"#evals/scorers/sql.js";import*as TextScorers from"#evals/scorers/text.js";import*as AutoevalsScorers from"#evals/scorers/autoevals.js";const Text=TextScorers,Json=JsonScorers,Sql=SqlScorers,Run=RunScorers,Autoevals=AutoevalsScorers;export{Autoevals,Json,Run,Sql,Text};