eve 0.83.2 → 0.84.0

package/dist/src/dynamic/definition.d.ts

@@ -1,141 +1,131 @@
-import type { ModelMessage } from "ai";
-import type { SessionAuth, SessionPredecessor } from "#context/keys.js";
-import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
+import type { InstrumentationChannel } from "#shared/instrumentation-channel.js";
+import type { HistoryMessage as ModelMessage } from "#shared/history-message.js";
+import type { SessionAuth, SessionParent, SessionPredecessor } from "#context/keys.js";
+import type { SessionSchedule } from "#context/session-schedule.js";
+import type { UserPart } from "#protocol/session-events/envelope.js";
+import type { SessionView } from "#protocol/session-projection/session-view.js";
 import type { ConversationContext } from "#shared/conversation-context.js";
 /**
- * Stream event types allowed for dynamic tool resolvers. Dispatch
- * supports any event; this extract restricts the public surface until
- * more events are validated.
+ * The session's latest turn, running or not. Its `id` keys a memo that runs once per turn: it
+ * stays the same through the turn's tool calls, pauses, and steering, until the next turn starts.
  */
-export type DynamicToolEventName = Extract<UnstampedMessageStreamEvent["type"], "session.started" | "turn.started" | "step.started">;
-export declare const ALLOWED_DYNAMIC_TOOL_EVENTS: ReadonlySet<string>;
+export interface DynamicTurn {
+    readonly id: string;
+    readonly status: "running" | "paused" | "settled";
+    /** The parts of the messages the turn opened with. Messages that steer it later don't count. */
+    readonly input: readonly UserPart[];
+}
 /**
- * Instructions and skills are restricted to session/turn boundaries.
- * Keeping their resolved context stable within a turn avoids changing the
- * model input between tool-loop steps.
+ * One run of a memo or a hook handler. A retry of the same run, after a process failed
+ * before its step was saved, has the same `idempotencyKey`.
+ *
+ * A memo's `idempotencyKey` is stable per session, definition, and memo key: it also repeats
+ * when the same key runs again after a deployment change discarded its result, or after an
+ * evaluation that didn't use it. A service that honors the key answers those runs with its first
+ * response; when a run needs fresh data, send a key of your own instead.
  */
-export declare const ALLOWED_DYNAMIC_INSTRUCTION_EVENTS: ReadonlySet<string>;
-export declare const ALLOWED_DYNAMIC_SKILL_EVENTS: ReadonlySet<string>;
-export declare const ALLOWED_DYNAMIC_CONNECTION_EVENTS: ReadonlySet<string>;
+export interface Occurrence {
+    /** Pass it to an external service, so a retried effect doesn't happen twice. */
+    readonly idempotencyKey: string;
+    readonly abortSignal: AbortSignal;
+}
 /**
- * Context passed to a dynamic resolver's event handler.
- *
- * Exposes read-only session identity, auth, and channel metadata. State
- * is not exposed here; resolvers read it through `defineState` handles or
- * the session context inside tool `execute` functions.
+ * A memo key: any JSON value, compared as canonical JSON, such as a tenant ID or
+ * `["tables", ctx.session.id]`. `undefined` reads as `null`.
  */
-export interface DynamicResolveContext {
-    /** Active cancellation signal when resolving a dynamic model. */
-    readonly abortSignal?: AbortSignal;
-    /** Effective model for this resolver, or `null` before dynamic model selection. */
-    readonly model: {
-        readonly id: string;
-    } | null;
+export type DynamicKey = unknown;
+/**
+ * Runs `fn` the first time the session sees `key` from this definition, keeps the JSON it returns,
+ * and returns the kept result for the same key afterwards, in any process. Only the keys the
+ * latest successful evaluation used stay kept, and a deployment change discards them: a key used
+ * again after either runs `fn` again. Code outside a memo runs again at every evaluation.
+ */
+export type Memo = <T>(key: DynamicKey, fn: (occurrence: Occurrence) => T | Promise<T>) => Promise<T>;
+/** The session as of its latest commit: what every dynamic value and hook handler reads. */
+export interface SessionFrame {
     readonly session: {
         readonly id: string;
         readonly auth: SessionAuth;
-        readonly schedule?: import("#context/session-schedule.js").SessionSchedule;
+        /** Present only for scheduled work. */
+        readonly schedule?: SessionSchedule;
         /**
-         * Present when eve started this session in place of a stranded session,
-         * one that another eve version built. It names the earlier session, whose recorded stream
+         * Present when eve started this session in place of a stranded session, one that another eve
+         * version built. It names the earlier session, whose recorded stream
          * `sessions.attach(predecessor.sessionId)` from `eve/server` reads.
          */
         readonly predecessor?: SessionPredecessor;
+        /**
+         * Present only in a delegated session. `sessionId`, `callId`, and `turn` name the immediate
+         * parent and the call that opened this session; `rootSessionId` names the top-level session.
+         */
+        readonly parent?: SessionParent;
     };
     /** Channel metadata for the request that triggered this resolve. */
-    readonly channel: {
-        /** Channel type that produced the request (e.g. `"slack"`, `"http"`), when known. */
-        readonly kind?: string;
-        /** Channel-owned resume handle for the conversation, when the channel supplies one. */
-        readonly continuationToken?: string;
-        /** Free-form channel-specific metadata attached to the request. */
-        readonly metadata?: Readonly<Record<string, unknown>>;
-    };
+    readonly channel: InstrumentationChannel;
     /**
      * Immutable classification and execution context for the active conversation.
      * Absent on sessions persisted before this context key existed.
      */
     readonly conversation?: ConversationContext;
-    /** Conversation history visible at this resolve point, oldest first. */
-    readonly messages: readonly ModelMessage[];
+    /** The session's tables, as a client folds them from the stream. */
+    readonly view: SessionView;
+    /** The session's latest turn, or `null` before its first. */
+    readonly turn: DynamicTurn | null;
+    /** The time of the session's latest commit, so it reads the same in every process. */
+    readonly now: Date;
 }
-/**
- * Base event handler map accepted by `defineDynamic`. Intentionally
- * wide so it accepts both tool-returning and skill-returning handlers:
- * the slot directory (tools/ vs skills/) determines the required return,
- * validated at runtime by the respective resolver.
- */
-export type DynamicEvents<TResult = unknown> = {
-    readonly [K in DynamicToolEventName]?: (event: unknown, ctx: DynamicResolveContext) => TResult | Promise<TResult>;
-};
-type DynamicEventMapHandler<TEvents extends DynamicEvents> = Extract<NonNullable<TEvents[keyof TEvents]>, (...args: never[]) => unknown>;
-type DynamicEventMapResult<TEvents extends DynamicEvents> = Awaited<ReturnType<DynamicEventMapHandler<TEvents>>>;
-/**
- * Marker discriminator for a `defineDynamic({ events })` export.
- */
+/** What a dynamic agent `model` reads. The model isn't chosen yet, so there's no `ctx.model`. */
+export interface DynamicModelContext extends SessionFrame {
+    readonly memo: Memo;
+    /** The conversation history as of the latest commit, assembled when something reads it. */
+    messages(): Promise<readonly ModelMessage[]>;
+}
+/** What a dynamic value reads: the session as of its latest commit, and the model chosen for it. */
+export interface DynamicContext extends DynamicModelContext {
+    /** The model chosen for the session. */
+    readonly model: {
+        readonly id: string;
+    };
+}
+/** Marker discriminator for a `defineDynamic()` export. */
 export declare const DYNAMIC_SENTINEL_KIND: "eve:dynamic";
-/**
- * Return value of `defineDynamic`: the runtime shape of a dynamic export,
- * stamped with a sentinel kind the compiler/normalizer detects.
- */
-export type DynamicSentinel<TResult = unknown> = {
+/** A `defineDynamic()` value, which the compiler detects by its `kind`. */
+export interface DynamicSentinel<TResult = unknown, TContext = DynamicContext> {
     readonly kind: typeof DYNAMIC_SENTINEL_KIND;
-    readonly events: DynamicEvents<TResult>;
-};
+    readonly evaluate: (ctx: TContext) => TResult | Promise<TResult>;
+}
+/** `defineDynamic()` typed for the folder it's authored in. */
+export type DefineDynamic<TResult, TContext = DynamicContext> = (fn: (ctx: TContext) => TResult | Promise<TResult>) => DynamicSentinel<TResult, TContext>;
 /**
- * Defines a dynamic resolver evaluated at runtime from stream-event
- * handlers. It is shared across tools, skills, connections, and agent definitions;
- * the directory it is authored in (not this function) decides what each
- * handler must return and which events are honored. The file's path-derived
- * slug names the single-entry case; a `Record<string, ...>` return names each
- * entry by its bare key, prefixed with the mount namespace for an extension's
- * resolver. Return `null` to contribute nothing for that event.
- *
- * Per-slot return shape:
- * - `agent/tools/`: return a single `defineTool(...)`, a
- *   `Record<string, defineTool(...)>`, or `null`.
- * - `agent/skills/`: return a single `defineSkill(...)`, a
- *   `Record<string, defineSkill(...)>`, or `null`.
- * - `agent/connections/`: return one connection definition, a
- *   `Record<string, connection definition>`, or `null`.
- * - `agent/subagents/<name>/agent.ts`: return `defineAgent(...)` to configure
- *   and expose the subagent, or `null` to omit it.
+ * Defines a value that depends on the session: a function of the session as of its latest
+ * commit. eve calls it in each step that needs the value, before each model call today, and
+ * installs what it returns, or nothing for `null`. The folder or field decides what it returns:
+ * tools in `agent/tools/`, skills in `agent/skills/`, a model in an agent's `model` field.
  *
- * Per-slot events: tools resolvers run at `session.started`,
- * `turn.started`, and `step.started`. Skills and connection resolvers run only
- * at `session.started` and `turn.started`; the runtime never invokes a
- * handler keyed on `step.started` in that slot. Dynamic subagents run at
- * `session.started` and `turn.started` only.
+ * Work that shouldn't repeat goes in `ctx.memo(key, fn)`: `fn` runs once per key, and a later
+ * evaluation, in any process, gets the result it kept.
  *
  * ```ts
- * import { defineDynamic, defineTool } from "eve/tools";
- * import { z } from "zod";
+ * import { defineDynamic } from "eve/tools";
  *
- * export default defineDynamic({
- *   events: {
- *     "session.started": async (event, ctx) => ({
- *       export: defineTool({
- *         description: "Export data",
- *         inputSchema: z.object({ format: z.string() }),
- *         async execute(input) {
- *           return doExport(input.format);
- *         },
- *       }),
- *     }),
- *   },
+ * export default defineDynamic(async (ctx) => {
+ *   const tenantId = ctx.session.auth.current?.attributes.tenantId;
+ *   if (!tenantId) return null;
+ *   const rows = await ctx.memo(tenantId, () => db.crmTools(tenantId));
+ *   return Object.fromEntries(rows.map((row) => [row.slug, crmTool(row)]));
  * });
  * ```
- *
- * A single return is named after the file slug. A map names each entry by its
- * bare key — there is no automatic slug prefix, so namespace keys yourself
- * (e.g. `team__playbook`) when a bare name might collide. Tool names must match
- * the tool filename charset, and a name under a connection's `<name>__` prefix
- * is rejected. A dynamic tool/skill whose name matches an authored one
- * overrides it; two dynamic resolvers emitting the same name is an error.
  */
-export declare function defineDynamic<const TEvents extends DynamicEvents>(definition: {
-    readonly events: TEvents;
-}): DynamicSentinel<DynamicEventMapResult<TEvents>>;
-export declare function assertResolverOnlyDynamicSentinel(sentinel: DynamicSentinel, message: string): void;
+export declare function defineDynamic<TResult, TContext extends DynamicModelContext = DynamicContext>(fn: (ctx: TContext) => TResult | Promise<TResult>): DynamicSentinel<TResult, TContext>;
+/**
+ * Marks a definition eve generates, such as a memory slot's tools: its function receives eve's
+ * own evaluation context, with the step's state.
+ */
+export declare function asFrameworkDefinition<T extends object>(definition: T): T;
+export declare function isFrameworkDefinition(value: unknown): boolean;
 export declare function isDynamicSentinel(value: unknown): value is DynamicSentinel;
-export {};
+/**
+ * Throws unless `sentinel` is a `defineDynamic(fn)` value: the function and nothing beside it.
+ * The compiler and the runtime loader both use it, so every source agrees.
+ */
+export declare function assertDynamicSentinelKeys(sentinel: DynamicSentinel, message: string): void;

package/dist/src/dynamic/definition.js

@@ -1 +1 @@
-import{stampDefinitionKey}from"#internal/authored-definition/source-identity.js";const ALLOWED_DYNAMIC_TOOL_EVENTS=new Set([`session.started`,`turn.started`,`step.started`]),ALLOWED_DYNAMIC_INSTRUCTION_EVENTS=new Set([`session.started`,`turn.started`]),ALLOWED_DYNAMIC_SKILL_EVENTS=new Set([`session.started`,`turn.started`]),ALLOWED_DYNAMIC_CONNECTION_EVENTS=new Set([`session.started`,`turn.started`]),DYNAMIC_SENTINEL_KIND=`eve:dynamic`;function defineDynamic(e){let t={kind:DYNAMIC_SENTINEL_KIND,events:e.events};return stampDefinitionKey(t,`dynamic:${Object.keys(e.events).join(`,`)}`),t}function assertResolverOnlyDynamicSentinel(e,t){let n=Object.keys(e).filter(e=>e!==`events`&&e!==`kind`);if(n.length>0)throw Error(`${t} Unknown key(s): ${n.join(`, `)}.`)}function isDynamicSentinel(e){return typeof e==`object`&&!!e&&e.kind===`eve:dynamic`}export{ALLOWED_DYNAMIC_CONNECTION_EVENTS,ALLOWED_DYNAMIC_INSTRUCTION_EVENTS,ALLOWED_DYNAMIC_SKILL_EVENTS,ALLOWED_DYNAMIC_TOOL_EVENTS,DYNAMIC_SENTINEL_KIND,assertResolverOnlyDynamicSentinel,defineDynamic,isDynamicSentinel};
+import{stampDefinitionKey}from"#internal/authored-definition/source-identity.js";const DYNAMIC_SENTINEL_KIND=`eve:dynamic`;function defineDynamic(e){assertDynamicFunction(e,`defineDynamic()`);let t={evaluate:e,kind:DYNAMIC_SENTINEL_KIND};return stampDefinitionKey(t,`dynamic`),t}function assertDynamicFunction(e,t){if(typeof e!=`function`)throw typeof e==`object`&&e&&`events`in e?Error(`${t} takes a function of the session, not an events map: defineDynamic(async (ctx) => value). Key work that should run once per session, turn, or tenant with ctx.memo(key, fn).`):Error(`${t} takes a function: defineDynamic(async (ctx) => value).`)}const FRAMEWORK_DEFINITION=Symbol.for(`eve.framework-definition`);function asFrameworkDefinition(e){return Object.defineProperty(e,FRAMEWORK_DEFINITION,{value:!0}),e}function isFrameworkDefinition(e){return typeof e==`object`&&!!e&&FRAMEWORK_DEFINITION in e}function isDynamicSentinel(e){return typeof e==`object`&&!!e&&e.kind===`eve:dynamic`}function assertDynamicSentinelKeys(e,t){assertDynamicFunction(e.evaluate??e,`${t} defineDynamic()`);let n=Object.keys(e).filter(e=>e!==`kind`&&e!==`evaluate`);if(n.length>0)throw Error(`${t} defineDynamic() takes only a function; unknown key(s): ${n.join(`, `)}. Packaging configuration such as build.externalDependencies belongs on the root agent.ts.`)}export{DYNAMIC_SENTINEL_KIND,asFrameworkDefinition,assertDynamicSentinelKeys,defineDynamic,isDynamicSentinel,isFrameworkDefinition};

package/dist/src/evals/assertions/collector.js

@@ -1 +1 @@
-import{toErrorMessage}from"#shared/errors.js";import{formatAssertionName}from"#evals/diagnostics.js";var AssertionCollector=class{#e=[];#t=[];tools;constructor(e){this.tools=e}get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){return makeHandle(this.#n({name:e.name,severity:n,kind:`deferred`,spec:e,selectSubject:t}))}recordValue(e){let t=this.#n({...e,kind:`resolved`});return this.#t.push(settleEntry(t,e.score)),makeHandle(t)}recordScore(e){return this.recordValue({name:`score`,severity:`soft`,score:async()=>toOutcome(await e)})}recordOutcome(e){let t=this.#n({name:e.name,severity:`gate`,kind:`resolved`});return t.score=e.outcome.score,t.message=e.outcome.message,t.metadata=e.outcome.metadata,computePassed(t)===!0}async recordRequirement(e){let t=this.#n({...e,severity:`gate`,kind:`resolved`});return await settleEntry(t,e.score),computePassed(t)===!0}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??e);n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push(toResult(n))}return t}#n(e){let t={baseName:e.name,name:e.name,key:e.key,severity:e.severity,threshold:e.threshold,kind:e.kind,spec:e.spec,selectSubject:e.selectSubject,score:void 0,errored:!1};return this.#e.push(t),t}};function toResult(e){return{name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.errored?void 0:ruleThreshold(e),passed:computePassed(e),errored:e.errored,message:e.message,metadata:e.metadata}}function ruleThreshold(e){return e.threshold??(e.severity===`gate`?1:void 0)}function computePassed(e){if(e.errored)return!1;let t=ruleThreshold(e);if(t!==void 0)return e.score!==void 0&&e.score>=t}function toOutcome(e){let t=typeof e==`number`?{score:e}:e;if(typeof t.score!=`number`||!(t.score>=0&&t.score<=1))throw TypeError(`Scores must be numbers from 0 to 1; received ${String(t.score)}.`);return t}async function settleEntry(e,t){try{let n=await t();e.score=n.score,e.message=n.message,e.metadata=n.metadata}catch(t){e.severity=`gate`,e.threshold=void 0,e.message=toErrorMessage(t),e.errored=!0}}function makeHandle(e){let t={gate(n){return e.severity=`gate`,e.threshold=n,t},soft(n){return e.severity=`soft`,e.threshold=n,t},atLeast(n){return e.severity=`soft`,e.threshold=n,t},label(n){return e.name=formatAssertionName(e.baseName,n),e.key=n.trim(),t}};return t}export{AssertionCollector};
+import{toErrorMessage}from"#shared/errors.js";import{outputOf}from"#evals/output.js";import{formatAssertionName}from"#evals/diagnostics.js";var AssertionCollector=class{#e=[];#t=[];tools;constructor(e){this.tools=e}get hasEntries(){return this.#e.length>0}recordScoped(e,t,n=`gate`){return makeHandle(this.#n({name:e.name,severity:n,kind:`deferred`,spec:e,selectSubject:t}))}recordValue(e){let t=this.#n({...e,kind:`resolved`});return this.#t.push(settleEntry(t,e.score)),makeHandle(t)}recordScore(e){return this.recordValue({name:`score`,severity:`soft`,score:async()=>toOutcome(await e)})}recordOutcome(e){let t=this.#n({name:e.name,severity:`gate`,kind:`resolved`});return t.score=e.outcome.score,t.message=e.outcome.message,t.metadata=e.outcome.metadata,computePassed(t)===!0}async recordRequirement(e){let t=this.#n({...e,severity:`gate`,kind:`resolved`});return await settleEntry(t,e.score),computePassed(t)===!0}async finalize(e){await Promise.all(this.#t);let t=[];for(let n of this.#e){if(n.kind===`deferred`&&n.spec!==void 0){let t=await n.spec.evaluate(n.selectSubject?.(e)??{...e,output:outputOf(e)});n.score=t.score,n.message=t.message,n.metadata=t.metadata}t.push(toResult(n))}return t}#n(e){let t={baseName:e.name,name:e.name,key:e.key,severity:e.severity,threshold:e.threshold,kind:e.kind,spec:e.spec,selectSubject:e.selectSubject,score:void 0,errored:!1};return this.#e.push(t),t}};function toResult(e){return{name:e.name,key:e.key,score:e.score,severity:e.severity,threshold:e.errored?void 0:ruleThreshold(e),passed:computePassed(e),errored:e.errored,message:e.message,metadata:e.metadata}}function ruleThreshold(e){return e.threshold??(e.severity===`gate`?1:void 0)}function computePassed(e){if(e.errored)return!1;let t=ruleThreshold(e);if(t!==void 0)return e.score!==void 0&&e.score>=t}function toOutcome(e){let t=typeof e==`number`?{score:e}:e;if(typeof t.score!=`number`||!(t.score>=0&&t.score<=1))throw TypeError(`Scores must be numbers from 0 to 1; received ${String(t.score)}.`);return t}async function settleEntry(e,t){try{let n=await t();e.score=n.score,e.message=n.message,e.metadata=n.metadata}catch(t){e.severity=`gate`,e.threshold=void 0,e.message=toErrorMessage(t),e.errored=!0}}function makeHandle(e){let t={gate(n){return e.severity=`gate`,e.threshold=n,t},soft(n){return e.severity=`soft`,e.threshold=n,t},atLeast(n){return e.severity=`soft`,e.threshold=n,t},label(n){return e.name=formatAssertionName(e.baseName,n),e.key=n.trim(),t}};return t}export{AssertionCollector};

package/dist/src/evals/assertions/run.d.ts

@@ -1,14 +1,15 @@
+import type { RequestStatus } from "#protocol/session-projection/request-status.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
-import type { MessageStreamEvent } from "#protocol/message.js";
 import { type EveEvalSkillLoadMatchOptions, type EveEvalSubagentCallMatchOptions, type EveEvalToolCallMatchOptions } from "#evals/match.js";
 import type { EveEvalEventMatch } from "#evals/match.js";
 import type { RunAssertion } from "#evals/assertions/collector.js";
 /** Minimal captured scope consumed by deterministic eval assertions. */
 export interface EveEvalAssertionSubject {
     readonly derived: import("#evals/types.js").EveEvalDerivedFacts;
-    readonly events: readonly MessageStreamEvent[];
+    readonly events: readonly SessionStreamEvent[];
     readonly output: unknown;
-    readonly status: "completed" | "failed" | "waiting";
+    readonly status: RequestStatus | "working";
 }
 /**
  * Asserts the run finished successfully: it did not fail and did not park on an
@@ -18,7 +19,7 @@ export declare function succeeded(): RunAssertion;
 /**
  * Asserts the run ended cleanly parked on HITL input.
  */
-export declare function parked(): RunAssertion;
+export declare function awaitingInput(): RunAssertion;
 /**
  * Asserts the joined assistant message text contains `token` (substring for
  * strings, `test` for RegExps).
@@ -35,7 +36,7 @@ export declare function calledTool(name: string, options?: EveEvalToolCallMatchO
  * Asserts the skill `skill` was loaded. `output`/`status`/`count` constrain
  * the matching load exactly as for `calledTool`.
  */
-export declare function loadedSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): RunAssertion;
+export declare function calledSkill(skill: string, options?: EveEvalSkillLoadMatchOptions): RunAssertion;
 /**
  * Asserts no tool call with `name` happened.
  */
@@ -54,21 +55,21 @@ export declare function usedNoTools(): RunAssertion;
  */
 export declare function maxToolCalls(max: number): RunAssertion;
 /**
- * Asserts no action result (tool, subagent, or skill) reported a failure.
+ * Asserts no call reported a failure.
  */
-export declare function noFailedActions(): RunAssertion;
+export declare function noFailedCalls(): RunAssertion;
 /**
  * Asserts a call to the agent tool `name` occurred. Each call to an agent
- * task counts once: its status and `output` come from `task.settled`, and the
- * session and remote metadata from the task's `agent.started` once its
+ * task counts once: its status and `output` come from `call.settled`, and the
+ * session and remote metadata from the task's `child.opened` once its
  * session opened.
  */
-export declare function calledSubagent(name: string, options?: EveEvalSubagentCallMatchOptions): RunAssertion;
+export declare function calledAgent(name: string, options?: EveEvalSubagentCallMatchOptions): RunAssertion;
 /**
  * Escape hatch: asserts an arbitrary predicate over the full typed event
  * stream. `label` names the assertion in reports.
  */
-export declare function eventsSatisfy(label: string, predicate: (events: readonly MessageStreamEvent[]) => boolean): RunAssertion;
+export declare function eventsSatisfy(label: string, predicate: (events: readonly SessionStreamEvent[]) => boolean): RunAssertion;
 /** Asserts a typed stream event exists, optionally constraining its count. */
 export declare function typedEvent(matcher: EveEvalEventMatch): RunAssertion;
 /** Asserts no typed stream event matches. */

package/dist/src/evals/assertions/run.js

@@ -1,2 +1,2 @@
-import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function loadedSkill(e,t={}){return validateCount(t.count),{name:`loadedSkill(${e})`,evaluate(n){let r=n.derived.skillLoads.filter(t=>t.skill===e).filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,r.length))return{score:1,metadata:{matchingLoads:r.length}};let i=n.derived.skillLoads.map(e=>e.skill).join(`, `);return fail(`expected ${describeCount(t.count)} matching load(s) of "${e}", found ${r.length}; observed skill loads: [${i}]`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&t.push(n.data.message);return t.join(`
-`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};
+import{failureOf}from"#client/session-utils.js";import{replyTextOf}from"#public/channels/reply.js";import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on unanswered input request(s)`):PASS:t}}}function awaitingInput(){return{name:`awaitingInput`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),i=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};let a=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,o;return o=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${i.length}`:`expected matching call count to satisfy the count predicate, found ${i.length}`,fail(`${o}; ${a}`)}}}function calledSkill(e,t={}){return validateCount(t.count),{name:`calledSkill(${e})`,evaluate(n){let r=n.derived.skillLoads.filter(t=>t.skill===e).filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,r.length))return{score:1,metadata:{matchingLoads:r.length}};let i=n.derived.skillLoads.map(e=>e.skill).join(`, `);return fail(`expected ${describeCount(t.count)} matching load(s) of "${e}", found ${r.length}; observed skill loads: [${i}]`)}}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedCalls(){return{name:`noFailedCalls`,evaluate(e){let t=e.events.filter(e=>e.type===`call.settled`&&e.data.outcome===`failed`);if(t.length===0)return PASS;let n=t.map(formatFailedCallResult);return fail(`${t.length} failed call(s): ${n.join(`; `)}`,{failedCalls:t.map(e=>({callId:e.data.callId,error:e.data.error,output:e.data.output,status:e.data.outcome}))})}}}function calledAgent(e,t={}){return validateCount(t.count),{name:`calledAgent(${e})`,evaluate(n){let r=n.derived.subagentCalls.filter(t=>t.name===e),i=r.filter(e=>subagentCallMatches(e,t));if(matchesCount(t.count,i.length))return{score:1,metadata:{matchingCalls:i.length}};if(r.length===0){let t=n.derived.subagentCalls.map(e=>e.name);return fail(`subagent "${e}" was never called; observed: [${t.join(`, `)}]`,{observedSubagentCalls:n.derived.subagentCalls})}return fail(`subagent "${e}" was called but ${i.length} call(s) matched the constraints`,{observedSubagentCalls:r})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));if(matchesCount(e.count,n.length))return{score:1,metadata:{matchingEvents:n.length}};let r=describeCount(e.count);return fail(`expected ${r} matching ${e.type} event(s), found ${n.length}; observed: [${t.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(t){let n=t.events.filter(t=>eventMatches(t,e));return n.length===0?PASS:fail(`expected no matching ${e.type} events, found ${n.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(t){let n=-1;for(let r of e){let e=t.events.flatMap((e,t)=>eventMatches(e,r)?[t]:[]);if(!matchesCount(r.count,e.length)){let t=describeCount(r.count);return fail(`expected ${t} matching ${r.type} event(s), found ${e.length}`)}let i=e[0];if(i===void 0||i<=n)return fail(`event group ${r.type} did not occur after the previous group`);n=e[e.length-1]??n}return PASS}}}function outputEquals(e){return{name:`outputEquals`,evaluate(t){return deepEquals(t.output,e)?PASS:fail(`output ${truncate(JSON.stringify(t.output))} does not equal expected ${truncate(JSON.stringify(e))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);if(!(`issues`in n)||n.issues===void 0)return PASS;let r=n.issues.map(e=>e.message).join(`; `);return fail(`output failed schema validation: ${r}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e){if(n.type!==`content.completed`)continue;let e=replyTextOf(n.data);e!==void 0&&t.push(e)}return t.join(`
+`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedCallResult(e){let{callId:t,outcome:n,output:r,error:i}=e.data;return[`callId=${t}`,`status=${n}`,i===void 0?void 0:`error=${i.message}`,`output=${truncate(formatUnknown(r))}`].filter(e=>e!==void 0).join(` `)}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));for(let t of e.events){let e=failureOf(t);if(e!==void 0)return fail(`${t.type} (${e.code}): ${e.message}`)}}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(e!==void 0&&typeof e!=`function`&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)t.type===`call.requested`&&t.data.capability.kind!==`skill`&&append(t.data.callId,t.data.capability.name);return t}export{awaitingInput,calledAgent,calledSkill,calledTool,eventOrder,eventsSatisfy,maxToolCalls,messageIncludes,noFailedCalls,notCalledTool,notEvent,outputEquals,outputMatches,succeeded,toolOrder,typedEvent,usedNoTools};

package/dist/src/evals/assertions/scoped.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";function createScopedAssertions(e,t){let n=createRecorder(e,t),toolName=t=>(assertReportedToolName(t,e.tools),t);return{succeeded:()=>n(RunAssertions.succeeded()),parked:()=>n(RunAssertions.parked()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(toolName(e),t)),loadedSkill:(e,t)=>n(RunAssertions.loadedSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(toolName(e))),toolOrder:e=>n(RunAssertions.toolOrder(e.map(toolName))),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledSubagent:(e,t)=>n(RunAssertions.calledSubagent(e,t)),noFailedActions:()=>n(RunAssertions.noFailedActions()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};
+import"#evals/assertions/collector.js";import*as RunAssertions from"#evals/assertions/run.js";import{assertReportedToolName}from"#evals/reported-tool-name.js";function createScopedAssertions(e,t){let n=createRecorder(e,t),toolName=t=>(assertReportedToolName(t,e.tools),t);return{succeeded:()=>n(RunAssertions.succeeded()),awaitingInput:()=>n(RunAssertions.awaitingInput()),messageIncludes:e=>n(RunAssertions.messageIncludes(e)),calledTool:(e,t)=>n(RunAssertions.calledTool(toolName(e),t)),calledSkill:(e,t)=>n(RunAssertions.calledSkill(e,t)),notCalledTool:e=>n(RunAssertions.notCalledTool(toolName(e))),toolOrder:e=>n(RunAssertions.toolOrder(e.map(toolName))),usedNoTools:()=>n(RunAssertions.usedNoTools()),maxToolCalls:e=>n(RunAssertions.maxToolCalls(e)),calledAgent:(e,t)=>n(RunAssertions.calledAgent(e,t)),noFailedCalls:()=>n(RunAssertions.noFailedCalls()),event:(e,t)=>n(RunAssertions.typedEvent({...t,type:e})),notEvent:(e,t)=>n(RunAssertions.notEvent({...t,type:e})),eventOrder:e=>n(RunAssertions.eventOrder(e)),eventsSatisfy:(e,t)=>n(RunAssertions.eventsSatisfy(e,t))}}function createOutputAssertions(e,t){let n=createRecorder(e,t);return{outputEquals:e=>n(RunAssertions.outputEquals(e)),outputMatches:e=>n(RunAssertions.outputMatches(e))}}function createRecorder(e,t){return t.timing===`final`?n=>e.recordScoped(n,t.select):n=>{let r=t.select();return e.recordScoped(n,()=>r)}}export{createOutputAssertions,createScopedAssertions};

package/dist/src/evals/cli/eval-client.js

@@ -1 +1 @@
-import{resolveDevelopmentClientOptions}from"#services/dev-client/client-options.js";import{resolveVerifiedRemoteDevelopmentClient}from"#setup/verified-remote-client.js";import{Client}from"#client/client.js";import{readVercelProjectLink}from"#internal/vercel/project-link.js";function resolveEvalClientOptions(e){if(e.kind===`local`)return{host:e.url};let t=resolveDevelopmentClientOptions(e.url),n=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return n?{...t,auth:{bearer:n},redirect:`manual`}:t}async function createEvalClient(e,t={}){let n=resolveEvalClientOptions(e);if(e.kind===`local`||n.auth!==void 0||t.workspaceRoot===void 0)return new Client(n);let r=process.env.VERCEL_ORG_ID?.trim(),i=process.env.VERCEL_PROJECT_ID?.trim(),a=r&&i?{orgId:r,projectId:i}:await readVercelProjectLink(t.workspaceRoot),{options:o}=await resolveVerifiedRemoteDevelopmentClient({projectSource:a,serverUrl:e.url,workspaceRoot:t.workspaceRoot,deps:t.deps});return new Client({...n,...o})}export{createEvalClient,resolveEvalClientOptions};
+import{resolveVerifiedRemoteDevelopmentClient}from"#setup/verified-remote-client.js";import{resolveDevelopmentClientOptions}from"#services/dev-client/client-options.js";import{Client}from"#client/client.js";import{readVercelProjectLink}from"#internal/vercel/project-link.js";function resolveEvalClientOptions(e){if(e.kind===`local`)return{baseUrl:e.url};let t=resolveDevelopmentClientOptions(e.url),n=process.env.EVE_EVAL_AUTH_TOKEN?.trim();return n?{...t,auth:{bearer:n},redirect:`manual`}:t}async function createEvalClient(e,t={}){let n=resolveEvalClientOptions(e);if(e.kind===`local`||n.auth!==void 0||t.workspaceRoot===void 0)return new Client(n);let r=process.env.VERCEL_ORG_ID?.trim(),i=process.env.VERCEL_PROJECT_ID?.trim(),a=r&&i?{orgId:r,projectId:i}:await readVercelProjectLink(t.workspaceRoot),{options:o}=await resolveVerifiedRemoteDevelopmentClient({projectSource:a,serverUrl:e.url,workspaceRoot:t.workspaceRoot,deps:t.deps});return new Client({...n,...o})}export{createEvalClient,resolveEvalClientOptions};

package/dist/src/evals/cli/eval.js

@@ -1,4 +1,4 @@
-import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{basename,join,relative}from"node:path";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{noDevelopmentExtensions}from"#compiler/development-extensions.js";import{EVE_DEV_STATE_DIR_ENV,EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{shutdownActiveSandboxConnections}from"#execution/sandbox/active-connections.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{createEvalStateDirectory,removeEvalStateDirectory}from"#evals/cli/eval-state-directory.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n,r=process.cwd()){await loadDevelopmentEnvironmentFiles(r);let i=e.length>0?e:void 0,a=await discoverAndImportEvals(r,i);if(a.length===0){let e=await findMisplacedEvalDirs(r);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
+import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{EVE_DEV_STATE_DIR_ENV,EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{basename,join,relative}from"node:path";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{noDevelopmentExtensions}from"#compiler/development-extensions.js";import{shutdownActiveSandboxConnections}from"#execution/sandbox/active-connections.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{createEvalStateDirectory,removeEvalStateDirectory}from"#evals/cli/eval-state-directory.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{consoleReporter}from"#evals/runner/reporters/console.js";import{junitReporter}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n,r=process.cwd()){await loadDevelopmentEnvironmentFiles(r);let i=e.length>0?e:void 0,a=await discoverAndImportEvals(r,i);if(a.length===0){let e=await findMisplacedEvalDirs(r);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
 `+e.map(e=>`  - ${e}`).join(`
 `)+`
-eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):i?n.error(`No evals found matching: ${i.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let o=t.tag??[],s=t.excludeTag??[],c=filterEvalsByTags({evaluations:a,includeTags:o,excludeTags:[]});if(c.length===0){n.error(`No evals matched the provided tags (${o.join(`, `)}).`),process.exitCode=2;return}let l=filterEvalsByTags({evaluations:c,includeTags:[],excludeTags:s});if(t.list===!0){printEvalList(l,t.json===!0,n);return}if(l.length===0){n.log(`All ${c.length} matching evals are excluded by tags (${s.join(`, `)}); nothing to run.`);return}let u,d,f;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`),f=parsePositiveInteger(t.repeat,`--repeat`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p;try{p=await discoverEvalConfig(r)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let m,h,g,_,v;try{if(v=await p.setup?.(),t.url)t.keepState===!0&&n.error(`--keep-state has no effect with --url: the remote server keeps its own state.`),_=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:r}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`remote`,url:t.url});else{let e=randomUUID();process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=e,h=await createEvalStateDirectory(r,e),process.env[EVE_DEV_STATE_DIR_ENV]=h,m=createDevelopmentServer(r,{developmentExtensions:noDevelopmentExtensions(),host:`127.0.0.1`,port:0});let t=await m.start();_=await createEvalClient({kind:`local`,url:t.url}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`local`,url:t.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:l,config:p,setupContext:v,target:g,client:_,appRoot:r,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,repeat:f,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{for(let e of[()=>m?.close(),()=>m&&shutdownActiveSandboxConnections({log:e=>n.error(e)}),async()=>{if(h!==void 0){if(t.keepState!==!0)return await removeEvalStateDirectory(h);n.error(`Kept the eval server's state in ${relative(r,h)}.`)}},()=>p.teardown?.(v)])try{await e()}catch(e){n.error(`Eval cleanup failed: ${e instanceof Error?e.message:String(e)}`),process.exitCode=1}}let y=typeof process.exitCode==`number`?process.exitCode:0;await flushStandardStreams(),process.exit(y)}async function flushStandardStreams(){await Promise.all([flushStream(process.stdout),flushStream(process.stderr)])}async function flushStream(e){await new Promise(t=>{let n=!1,settle=()=>{n||(n=!0,setImmediate(()=>{e.off(`error`,settle),t()}))};e.once(`error`,settle);try{e.write(``,settle)}catch{settle()}})}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let t=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof t.name==`string`&&t.name.length>0?t.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};
+eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):i?n.error(`No evals found matching: ${i.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let o=t.tag??[],s=t.excludeTag??[],c=filterEvalsByTags({evaluations:a,includeTags:o,excludeTags:[]});if(c.length===0){n.error(`No evals matched the provided tags (${o.join(`, `)}).`),process.exitCode=2;return}let l=filterEvalsByTags({evaluations:c,includeTags:[],excludeTags:s});if(t.list===!0){printEvalList(l,t.json===!0,n);return}if(l.length===0){n.log(`All ${c.length} matching evals are excluded by tags (${s.join(`, `)}); nothing to run.`);return}let u,d,f;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`),f=parsePositiveInteger(t.repeat,`--repeat`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p;try{p=await discoverEvalConfig(r)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let m,h,g,_,v;try{if(v=await p.setup?.(),t.url)t.keepState===!0&&n.error(`--keep-state has no effect with --url: the remote server keeps its own state.`),_=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:r}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`remote`,url:t.url});else{let e=randomUUID();process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=e,h=await createEvalStateDirectory(r,e),process.env[EVE_DEV_STATE_DIR_ENV]=h,m=createDevelopmentServer(r,{developmentExtensions:noDevelopmentExtensions(),host:`127.0.0.1`,port:0});let t=await m.start();_=await createEvalClient({kind:`local`,url:t.url}),g=await resolveEvalTargetHandle({client:_,expectedAgentName:await readExpectedAgentName(r),kind:`local`,url:t.url})}let e=t.json===!0?[]:[consoleReporter()];t.junit!==void 0&&e.push(junitReporter({filePath:t.junit}));let i=await runEvals({evaluations:l,config:p,setupContext:v,target:g,client:_,appRoot:r,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,repeat:f,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{for(let e of[()=>m?.close(),()=>m&&shutdownActiveSandboxConnections({log:e=>n.error(e)}),async()=>{if(h!==void 0){if(t.keepState!==!0)return await removeEvalStateDirectory(h);n.error(`Kept the eval server's state in ${relative(r,h)}.`)}},()=>p.teardown?.(v)])try{await e()}catch(e){n.error(`Eval cleanup failed: ${e instanceof Error?e.message:String(e)}`),process.exitCode=1}}let y=typeof process.exitCode==`number`?process.exitCode:0;await flushStandardStreams(),process.exit(y)}async function flushStandardStreams(){await Promise.all([flushStream(process.stdout),flushStream(process.stderr)])}async function flushStream(e){await new Promise(t=>{let n=!1,settle=()=>{n||(n=!0,setImmediate(()=>{e.off(`error`,settle),t()}))};e.once(`error`,settle);try{e.write(``,settle)}catch{settle()}})}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let t=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof t.name==`string`&&t.name.length>0?t.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};

package/dist/src/evals/context.js

@@ -1 +1 @@
-import"#evals/assertions/collector.js";import"#evals/session-manager.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(e){let t=e.collector,n=buildJudgeContext({collector:t,getReply:()=>e.manager.lastTurnSession()?.lastTurn?.message??null,getInput:()=>e.manager.lastTurnSession()?.lastInput??``,judge:e.judge,signal:e.signal});return{context:{session:t=>e.manager.session(t),send:(t,n)=>e.manager.send(t,n),context:e.setupContext,signal:e.signal,target:e.target,log:e.log,sleep:t=>sleep(t,e.signal),...createScopedAssertions(t,{timing:`final`,select:e=>e}),check:(e,n)=>recordCheck(t,e,n),score:e=>t.recordScore(e),require:(e,n)=>requireCheck(t,e,n),skip:n=>{throw n.trim().length===0?Error(`skip() requires a non-empty reason.`):t.hasEntries||e.manager.hasActivity()?Error(`skip() must be called before creating sessions, sending messages, or recording assertions.`):new EvalSkipped(n)},judge:n},collector:t}}async function requireCheck(e,t,n){let r=n.gate(n.threshold);if(!await e.recordRequirement({name:r.name,threshold:r.threshold,score:()=>evaluateAssertion(r,t)}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:()=>evaluateAssertion(n,t)})}async function evaluateAssertion(e,t){return e.evaluate===void 0?{score:await e.score(t)}:await e.evaluate(t)}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};
+import{outputOf}from"#evals/output.js";import"#evals/assertions/collector.js";import"#evals/session-manager.js";import{createScopedAssertions}from"#evals/assertions/scoped.js";import{buildJudgeContext}from"#evals/judge.js";import{EvalRequirementFailed,EvalSkipped}from"#evals/control-flow.js";function createEvalContext(e){let t=e.collector,n=buildJudgeContext({collector:t,getReply:()=>e.manager.lastTurnSession()?.lastTurn?.reply??null,getInput:()=>e.manager.lastTurnSession()?.lastInput??``,judge:e.judge,signal:e.signal});return{context:{session:t=>e.manager.session(t),send:(t,n)=>e.manager.send(t,n),context:e.setupContext,abortSignal:e.signal,target:e.target,log:e.log,sleep:t=>sleep(t,e.signal),...createScopedAssertions(t,{timing:`final`,select:e=>({...e,output:outputOf(e)})}),check:(e,n)=>recordCheck(t,e,n),score:e=>t.recordScore(e),require:(e,n)=>requireCheck(t,e,n),skip:n=>{throw n.trim().length===0?Error(`skip() requires a non-empty reason.`):t.hasEntries||e.manager.hasActivity()?Error(`skip() must be called before creating sessions, sending messages, or recording assertions.`):new EvalSkipped(n)},judge:n},collector:t}}async function requireCheck(e,t,n){let r=n.gate(n.threshold);if(!await e.recordRequirement({name:r.name,threshold:r.threshold,score:()=>evaluateAssertion(r,t)}))throw new EvalRequirementFailed;return t}function recordCheck(e,t,n){return e.recordValue({name:n.name,severity:n.severity,threshold:n.threshold,score:()=>evaluateAssertion(n,t)})}async function evaluateAssertion(e,t){return e.evaluate===void 0?{score:await e.score(t)}:await e.evaluate(t)}function sleep(e=1e3,t){if(!Number.isFinite(e)||e<0)throw Error(`sleep() duration must be a non-negative finite number.`);return t?.aborted?Promise.reject(t.reason):new Promise((n,r)=>{let i=setTimeout(n,e);t?.addEventListener(`abort`,()=>{clearTimeout(i),r(t.reason)},{once:!0})})}export{createEvalContext};

package/dist/src/evals/expect/index.d.ts

@@ -1,6 +1,5 @@
 import type { StandardSchemaV1 } from "#compiled/@standard-schema/spec/index.js";
 import type { Assertion } from "#evals/types.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionSeverity, } from "#evals/types.js";
 /**
  * Passes when the value (coerced to a string) contains a substring or matches
  * a RegExp. A hard gate by default.

package/dist/src/evals/index.d.ts

@@ -2,10 +2,7 @@ export { defineEval } from "#evals/define-eval.js";
 export { defineEvalConfig } from "#evals/define-eval-config.js";
 export { EveEvalTurnFailedError } from "#evals/session.js";
 export { mockModel } from "#evals/mock-model.js";
-export type { RuntimeIdentity, RuntimeTraceContext } from "#protocol/message.js";
-export type { InputRequest } from "#shared/input.js";
-export type { CancelSessionResult } from "#client/types.js";
-export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInputRequestMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
-export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSendTiming, EveEvalSessionClock, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, ScoreInput, } from "#evals/types.js";
+export type { EveEvalCountMatcher, EveEvalEventMatch, EveEvalInteractionMatchOptions, EveEvalValueMatcher, EveEvalToolCallMatchOptions, EveEvalSkillLoadMatchOptions, EveEvalSubagentCallMatchOptions, } from "#evals/match.js";
+export type { Assertion, AssertionEvaluation, AssertionHandle, AssertionResult, AssertionSeverity, EveEvalActionStatus, EveEvalAgentSession, EveEvalAssertions, EveEvalContext, EveEvalDerivedFacts, EveEvalJudgeConfig, EveEvalRunSummary, EveEvalSession, EveEvalSendTiming, EveEvalSessionClock, EveEvalSessionResult, EveEvalScheduleDispatchResult, EveEvalSkillLoad, EveEvalSubagentCall, EveEval, EveEvalConfig, EveEvalConfigInput, EveEvalConfigContext, EveEvalDefinition, EveEvalInput, EveEvalLiveTurn, EveEvalWatchOptions, EveEvalResult, EveEvalTarget, EveEvalTargetCapabilities, EveEvalTargetHandle, EveEvalTaskResult, EveEvalTraceContext, EveEvalToolCall, EveEvalTurn, EveEvalStreamEvent, EveEvalWaitForEventOptions, EveEvalVerdict, JudgeBatch, JudgeInput, JudgeQuestion, JudgeContext, JudgeOpts, ScoreInput, } from "#evals/types.js";
 export type { MockModelMessage, MockModelOptions, MockModelRequest, MockModelResponder, MockModelResponse, MockModelTool, MockModelToolCall, MockModelToolResult, MockModelUsage, } from "#evals/mock-model.js";
 export type { ToolStub, ToolStubOutcome } from "#tool-stubs/types.js";

package/dist/src/evals/judge.js

@@ -1,2 +1,2 @@
-import{formatDiagnosticValue,toDiagnosticMetadataValue}from"#evals/diagnostics.js";import{decide}from"#ai/decide.js";import{runUntilAborted}from"#evals/abort.js";function buildJudgeContext(e){function judge(t,n){let r=typeof t==`object`&&`questions`in t,i=r?t.questions:{judgment:typeof t==`string`?{type:`boolean`,instructions:t}:t},a=Object.entries(i);if(a.length===0)throw Error(`Judge questions must be a non-empty map.`);let o=r&&t.state!==void 0?t.state:{input:e.getInput(),output:!r&&n?.on!==void 0?n.on:e.getReply()??``},s=n?.model??e.judge?.model,c=n?.modelOptions??e.judge?.modelOptions,l=runUntilAborted((async()=>{let t=structuredClone(o),n=structuredClone(i),r=Object.fromEntries(Object.entries(n).map(([e,t])=>{if(t.type!==`choice`)return[e,t];let{expected:n,...r}=t;if(typeof n!=`string`||!Object.hasOwn(t.criteria,n))throw Error(`Judge question "${e}" expected must name an option in criteria.`);return[e,r]}));return{result:await decide({model:s,state:isJsonArray(t)?[{type:`json`,value:t}]:t,questions:r,providerOptions:c?.providerOptions,abortSignal:e.signal}),state:t,questions:n}})(),e.signal),u=Object.fromEntries(a.map(([t,n])=>[t,e.collector.recordValue({name:`judge.${n.type}${r?`.${t}`:``}`,severity:`soft`,score:async()=>{let{result:e,state:n,questions:i}=await l,a=i[t],o=e.answers[t],s=judgeScore(a,o);return{score:s,message:formatJudgeDetail(n,a,o),metadata:{judge:e.response.modelId,questionId:t,question:a,state:n,answer:o,score:s,usage:e.usage,usageScope:r?`batch`:`assertion`,warnings:e.warnings,rounding:e.rounding,providerMetadata:e.providerMetadata,response:toDiagnosticMetadataValue(e.response)}}}})]));return r?u:u.judgment}return judge}function isJsonArray(e){return Array.isArray(e)}function formatJudgeDetail(e,t,n){return[[`state`,e],[`question`,t],[`answer`,n]].map(([e,t])=>`${e}: ${formatDiagnosticValue(t)}`).join(`
+import{decideWithMetadata}from"#ai/decision-adapter.js";import{formatDiagnosticValue,toDiagnosticMetadataValue}from"#evals/diagnostics.js";import{runUntilAborted}from"#evals/abort.js";function buildJudgeContext(e){function judge(t,n){let r=typeof t==`object`&&`questions`in t,i=r?t.questions:{judgment:typeof t==`string`?{type:`boolean`,instructions:t}:t},a=Object.entries(i);if(a.length===0)throw Error(`Judge questions must be a non-empty map.`);let o=r&&t.state!==void 0?t.state:{input:e.getInput(),output:!r&&n?.on!==void 0?n.on:e.getReply()??``},s=n?.model??e.judge?.model,c=n?.modelOptions??e.judge?.modelOptions,l=runUntilAborted((async()=>{let t=structuredClone(o),n=structuredClone(i),r=Object.fromEntries(Object.entries(n).map(([e,t])=>{if(t.type!==`choice`)return[e,t];let{expected:n,...r}=t;if(typeof n!=`string`||!Object.hasOwn(t.criteria,n))throw Error(`Judge question "${e}" expected must name an option in criteria.`);return[e,r]}));return{result:await decideWithMetadata({model:s,state:isJsonArray(t)?[{type:`json`,value:t}]:t,questions:r,providerOptions:c?.providerOptions,signal:e.signal}),state:t,questions:n}})(),e.signal),u=Object.fromEntries(a.map(([t,n])=>[t,e.collector.recordValue({name:`judge.${n.type}${r?`.${t}`:``}`,severity:`soft`,score:async()=>{let{result:e,state:n,questions:i}=await l,a=i[t],o=e.answers[t],s=judgeScore(a,o);return{score:s,message:formatJudgeDetail(n,a,o),metadata:{judge:e.response.modelId,questionId:t,question:a,state:n,answer:o,score:s,usage:e.usage,usageScope:r?`batch`:`assertion`,warnings:e.warnings,rounding:e.rounding,providerMetadata:e.providerMetadata,response:toDiagnosticMetadataValue(e.response)}}}})]));return r?u:u.judgment}return judge}function isJsonArray(e){return Array.isArray(e)}function formatJudgeDetail(e,t,n){return[[`state`,e],[`question`,t],[`answer`,n]].map(([e,t])=>`${e}: ${formatDiagnosticValue(t)}`).join(`
 `)}function judgeScore(e,t){switch(t.type){case`boolean`:return t.probability;case`score`:if(e.type===`score`)return t.score/(e.criteria.length-1);break;case`choice`:if(e.type===`choice`)return+(t.choice===e.expected)}throw Error(`Decision answer does not match the judge question.`)}export{buildJudgeContext};

package/dist/src/evals/match.d.ts

@@ -1,10 +1,11 @@
-import type { MessageStreamEvent } from "#protocol/message.js";
-import type { InputRequest } from "#shared/input.js";
+import type { SessionStreamEvent } from "#protocol/session-event.js";
+import type { Scope } from "#protocol/session-events/envelope.js";
+import type { Interaction } from "#protocol/session-projection/reader.js";
 import type { JsonObject, JsonValue } from "#shared/json.js";
 import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
 /**
  * One matcher accepted by the assertion options (`t.calledTool`,
- * `t.calledSubagent`):
+ * `t.calledAgent`):
  *
  * - a **literal** is compared structurally; objects partial-deep-match (every
  *   key in the matcher must match the observed value, recursively, and nested
@@ -38,51 +39,53 @@ export interface EveEvalToolCallMatchOptions {
     readonly count?: EveEvalCountMatcher;
 }
 /**
- * Constraints applied to a skill load by `t.loadedSkill`. Identical to
+ * Constraints applied to a skill load by `t.calledSkill`. Identical to
  * {@link EveEvalToolCallMatchOptions} without `input`, since the skill name is
  * the whole input.
  */
 export type EveEvalSkillLoadMatchOptions = Omit<EveEvalToolCallMatchOptions, "input">;
 /**
- * Constraints applied to subagent calls by `t.calledSubagent`.
+ * Constraints applied to subagent calls by `t.calledAgent`.
  */
 export interface EveEvalSubagentCallMatchOptions {
     /** Matcher over the agent tool call id. */
     readonly callId?: EveEvalValueMatcher<string | undefined>;
     /** Matcher over the id of the agent's session. */
     readonly childSessionId?: EveEvalValueMatcher<string | undefined>;
-    /** Matcher over the `agent.started` remote URL. */
-    readonly remoteUrl?: EveEvalValueMatcher<string | undefined>;
-    /** Matcher over the output of the call's `task.settled`. */
+    /** Matcher over the output of the call's `call.settled`. */
     readonly output?: EveEvalValueMatcher;
     /** Required lifecycle outcome. Defaults to `"completed"`. */
     readonly status?: EveEvalSubagentCall["status"];
     /** Constraint over the matching delegation count. Defaults to "at least one". */
     readonly count?: EveEvalCountMatcher;
 }
-/** Constraints accepted by `requireInputRequest`. */
-export interface EveEvalInputRequestMatchOptions {
+/** Constraints accepted by `requireInteraction`. */
+export interface EveEvalInteractionMatchOptions {
     /** Matcher over the request's display hint. */
-    readonly display?: EveEvalValueMatcher<InputRequest["display"]>;
+    readonly display?: EveEvalValueMatcher<Interaction["request"]["display"]>;
     /** Partial-deep matcher over a tool-call action's input. */
     readonly input?: EveEvalValueMatcher<JsonObject>;
     /** Matcher over the complete option-id list in request order. */
     readonly optionIds?: EveEvalValueMatcher<readonly string[]>;
     /** Matcher over the request prompt. */
     readonly prompt?: EveEvalValueMatcher<string>;
-    /** Required tool name for a tool-call action. */
+    /** Required kind: `approval`, `question`, or `budget`. */
+    readonly kind?: Interaction["request"]["kind"];
+    /** Required name of the tool the request asks about. */
     readonly toolName?: string;
 }
 /** One typed stream-event matcher used by scoped event assertions. */
-export type EveEvalEventMatch<TType extends MessageStreamEvent["type"] = MessageStreamEvent["type"]> = TType extends MessageStreamEvent["type"] ? {
+export type EveEvalEventMatch<TType extends SessionStreamEvent["type"] = SessionStreamEvent["type"]> = TType extends SessionStreamEvent["type"] ? {
     /** Stream-event type to match. */
     readonly type: TType;
     /** Partial-deep matcher over the event data. */
-    readonly data?: EveEvalDeepMatcher<Extract<MessageStreamEvent, {
+    readonly data?: EveEvalDeepMatcher<Extract<SessionStreamEvent, {
         type: TType;
     }> extends {
         data: infer TData;
     } ? TData : never>;
+    /** Partial-deep matcher over the event scope: the turn, task, run, or change it belongs to. */
+    readonly scope?: EveEvalDeepMatcher<Scope>;
     /** Constraint over the matching event count. Defaults to "at least one". */
     readonly count?: EveEvalCountMatcher;
 } : never;
@@ -100,14 +103,14 @@ export declare function toolCallMatches(call: Pick<EveEvalToolCall, "output" | "
     readonly input?: JsonObject;
 }, options: EveEvalToolCallMatchOptions): boolean;
 /**
- * Returns true when one derived subagent call satisfies the `remoteUrl`/
+ * Returns true when one derived subagent call satisfies the `callId`, `childSessionId`, and
  * `output` constraints.
  */
 export declare function subagentCallMatches(call: EveEvalSubagentCall, options: EveEvalSubagentCallMatchOptions): boolean;
 /** Returns true when one HITL request satisfies every supplied constraint. */
-export declare function inputRequestMatches(request: InputRequest, options: EveEvalInputRequestMatchOptions): boolean;
+export declare function interactionMatches(request: Interaction, options: EveEvalInteractionMatchOptions): boolean;
 /** Returns true when one stream event satisfies a typed event matcher. */
-export declare function eventMatches(event: MessageStreamEvent, matcher: EveEvalEventMatch): boolean;
+export declare function eventMatches(event: SessionStreamEvent, matcher: EveEvalEventMatch): boolean;
 /**
  * Strict structural equality used by scoped `outputEquals`: unlike matcher
  * comparison, objects must carry exactly the same keys on both sides.

package/dist/src/evals/match.js

@@ -1 +1 @@
-function matchesValue(e,t){return e instanceof RegExp?testRegExpAgainst(e,t):typeof e==`function`?e(t):Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function subagentCallMatches(e,t){return!(t.callId!==void 0&&!matchesValue(t.callId,e.callId)||t.childSessionId!==void 0&&!matchesValue(t.childSessionId,e.childSessionId)||t.remoteUrl!==void 0&&!matchesValue(t.remoteUrl,e.remoteUrl)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function inputRequestMatches(e,t){if(t.display!==void 0&&!matchesValue(t.display,e.display)||t.prompt!==void 0&&!matchesValue(t.prompt,e.prompt))return!1;if(t.optionIds!==void 0){let n=(e.options??[]).map(e=>e.id);if(!matchesValue(t.optionIds,n))return!1}return!(t.toolName!==void 0&&(e.action.kind!==`tool-call`||e.action.toolName!==t.toolName)||t.input!==void 0&&(e.action.kind!==`tool-call`||!matchesValue(t.input,e.action.input)))}function eventMatches(e,t){return e.type===t.type&&(t.data===void 0||matchesValue(t.data,`data`in e?e.data:void 0))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length&&n.every(n=>deepEquals(e[n],t[n]))}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,eventMatches,inputRequestMatches,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};
+function matchesValue(e,t){return e instanceof RegExp?testRegExpAgainst(e,t):typeof e==`function`?e(t):Array.isArray(e)?!Array.isArray(t)||t.length!==e.length?!1:e.every((e,n)=>matchesValue(e,t[n])):isPlainObject(e)?isPlainObject(t)?Object.entries(e).every(([e,n])=>matchesValue(n,t[e])):!1:Object.is(e,t)}function toolCallMatches(e,t){return!(t.input!==void 0&&!matchesValue(t.input,e.input)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function subagentCallMatches(e,t){return!(t.callId!==void 0&&!matchesValue(t.callId,e.callId)||t.childSessionId!==void 0&&!matchesValue(t.childSessionId,e.childSessionId)||t.output!==void 0&&!matchesValue(t.output,e.output)||e.status!==(t.status??`completed`))}function interactionMatches(e,t){if(t.display!==void 0&&!matchesValue(t.display,e.request.display)||t.prompt!==void 0&&!matchesValue(t.prompt,e.request.prompt)||t.kind!==void 0&&e.request.kind!==t.kind)return!1;if(t.optionIds!==void 0){let n=(e.request.options??[]).map(e=>e.id);if(!matchesValue(t.optionIds,n))return!1}return!(t.toolName!==void 0&&e.call?.name!==t.toolName||t.input!==void 0&&(e.call===void 0||!matchesValue(t.input,e.call.input)))}function eventMatches(e,t){return e.type===t.type&&(t.data===void 0||matchesValue(t.data,`data`in e?e.data:void 0))&&(t.scope===void 0||matchesValue(t.scope,e.scope??{}))}function deepEquals(e,t){if(Object.is(e,t))return!0;if(Array.isArray(e)||Array.isArray(t))return!Array.isArray(e)||!Array.isArray(t)||e.length!==t.length?!1:e.every((e,n)=>deepEquals(e,t[n]));if(isPlainObject(e)&&isPlainObject(t)){let n=Object.keys(e),r=Object.keys(t);return n.length===r.length&&n.every(n=>deepEquals(e[n],t[n]))}return!1}function testRegExpAgainst(e,t){if(typeof t==`string`)return testRegExp(e,t);if(t===void 0)return!1;let n=JSON.stringify(t);return n!==void 0&&testRegExp(e,n)}function testRegExp(e,t){return(e.global||e.sticky)&&(e.lastIndex=0),e.test(t)}function isPlainObject(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}export{deepEquals,eventMatches,interactionMatches,matchesValue,subagentCallMatches,testRegExp,toolCallMatches};