@ultimat3/ai 29.4.0 → 30.1.0

package/src/context-overflow.ts

@@ -0,0 +1,86 @@
+// Single responsibility: telling a provider's "the prompt outgrew the window" refusal apart from
+// every other non-2xx — the one predicate, the one table of phrases, and the class it raises.
+// Both transports call `providerFailure`; the code and title stay in ./error-codes with the rest.
+
+import { UltimateError } from '@ultimat3/core';
+import { BILLED_META, billedCall } from './billed';
+import { costOf } from './cost';
+import './error-codes';
+import { AiTransportError } from './errors';
+import type { ModelId } from './models';
+import { ZERO_USAGE } from './wire';
+
+/** The statuses a context-window refusal arrives with: 400 on every wire, 413 behind some proxies. */
+const OVERFLOW_STATUSES: ReadonlySet<number> = new Set([400, 413]);
+
+/**
+ * What a provider's message says when the request outgrew the model's window, lower-cased. One
+ * row per phrase as each endpoint words it — a phrase, never a status alone: a 400 is also a bad
+ * tool schema, and that one must stay `X_AI_PROVIDER_UNAVAILABLE` with its own fix line.
+ */
+export const CONTEXT_OVERFLOW_PHRASES: readonly string[] = [
+  // Anthropic: "prompt is too long: 210000 tokens > 200000 maximum".
+  'prompt is too long',
+  // Anthropic: "input length and `max_tokens` exceed context limit: …".
+  'exceed context limit',
+  // OpenAI, vLLM, OpenRouter: "This model's maximum context length is 128000 tokens. …".
+  'maximum context length',
+  // OpenAI's error code, which some proxies put in the message.
+  'context_length_exceeded',
+  // OpenAI Responses-era wording: "Your input exceeds the context window of this model."
+  'exceeds the context window',
+  // Mistral: "… too large for model with 32768 maximum context length".
+  'too large for model with',
+  // Gemini's OpenAI-compatible endpoint.
+  'exceeds the maximum number of tokens allowed',
+];
+
+/** Whether a provider's non-2xx answer is the request being longer than the model's window. */
+export function isContextOverflow(status: number | undefined, detail: string): boolean {
+  if (status === undefined || !OVERFLOW_STATUSES.has(status)) return false;
+  const said = detail.toLowerCase();
+  return CONTEXT_OVERFLOW_PHRASES.some((phrase) => said.includes(phrase));
+}
+
+/**
+ * The provider refused the request as over the model's context window. Its own code, not the
+ * transport's: the transport code is `retryable`, and every retry of this one re-sends the same
+ * request to the same refusal. A refused request billed nothing, and says so on `meta.billed` so
+ * a run's meter settles it instead of counting a call whose bill it cannot know.
+ */
+export class AiContextOverflowError extends UltimateError {
+  constructor(input: {
+    readonly provider: string;
+    readonly model: ModelId;
+    readonly status: number;
+    readonly detail: string;
+  }) {
+    super({
+      code: 'X_AI_CONTEXT_OVERFLOW',
+      cause: `provider "${input.provider}" refused a request to model "${input.model}" as longer than its context window (${input.status}): ${input.detail}`,
+      fix: 'compact: { at: 0.6, keepTurns: 2 }   # on the agent — or shorten the prompt, or route to a model whose registerModel row has a larger contextWindow',
+      meta: {
+        provider: input.provider,
+        model: input.model,
+        status: input.status,
+        detail: input.detail,
+        [BILLED_META]: billedCall(ZERO_USAGE, costOf(input.model, ZERO_USAGE)),
+      },
+    });
+  }
+}
+
+/** The error a provider's non-2xx answer becomes: an overflow when it says so, else the transport's. */
+export function providerFailure(input: {
+  readonly provider: string;
+  readonly model: ModelId;
+  readonly status: number;
+  /** Already scrubbed of the credential (`withoutKey`). */
+  readonly detail: string;
+  readonly envVar: string;
+}): UltimateError {
+  if (isContextOverflow(input.status, input.detail)) {
+    return new AiContextOverflowError(input);
+  }
+  return new AiTransportError(input);
+}

package/src/error-codes.ts

@@ -0,0 +1,95 @@
+// The X_* codes owned by @ultimat3/ai, and nothing else: the list, its titles and the one
+// registration call. `bun run new-error-code --package ai` writes here. The classes that throw
+// these codes live in `./errors` and its siblings; `./errors` imports this file for its
+// registration, so a module that can construct one of them has the code registered.
+import { registerErrorCodes } from '@ultimat3/core';
+
+export const AI_ERROR_CODES = [
+  'X_AI_PROVIDER_UNAVAILABLE',
+  'X_AI_KEY_MISSING',
+  'X_AI_REQUEST_INVALID',
+  'X_AI_BUDGET_EXCEEDED',
+  'X_AI_GATEWAY_MISSING',
+  'X_AI_PROMPT_VERSION',
+  'X_AI_MODEL_UNKNOWN',
+  'X_AI_PROMPT_SECRET',
+  'X_LLM_OUTPUT_INVALID',
+  'X_LLM_REFUSED',
+  'X_LLM_TRUNCATED',
+  'X_LLM_STREAM_INVALID',
+  'X_AGENT_MAX_TURNS',
+  'X_AGENT_TOOL_UNEXPOSED',
+  'X_HIVE_EMPTY',
+  'X_EVAL_THRESHOLD',
+  'X_EVAL_BASELINE_MISSING',
+  'X_EVAL_BASELINE_INVALID',
+  'X_EVAL_MISSING',
+  'X_EVAL_RECORDING',
+  'X_VECTOR_DIM_MISMATCH',
+  'X_VECTOR_SCOPE_WIDENED',
+  'X_AI_EMBEDDER_INVALID',
+  'X_VECTOR_UNSCOPED',
+  'X_AI_CONTENT_UNSUPPORTED',
+  'X_AI_MODEL_UNRESOLVED',
+  'X_AGENT_TOOL_LOOP',
+  'X_AGENT_DEADLINE',
+  'X_LLM_TOOL_ARGUMENTS_INVALID',
+  'X_AGENT_TRANSCRIPT_INVALID',
+  'X_AI_PROVIDER_UNEXPECTED',
+  'X_AGENT_TOOL_UNDECLARED',
+  'X_AGENT_CONTEXT_EXHAUSTED',
+  'X_AI_SCRIPT_EXHAUSTED',
+  'X_EVAL_TOOL_UNDOUBLED',
+  'X_AI_CONTEXT_OVERFLOW',
+] as const;
+
+export type AiErrorCode = (typeof AI_ERROR_CODES)[number];
+
+export const AI_ERROR_TITLES: Readonly<Record<AiErrorCode, string>> = {
+  X_AI_PROVIDER_UNAVAILABLE: 'the model provider is unreachable',
+  X_AI_KEY_MISSING: 'the provider API key is not set',
+  X_AI_REQUEST_INVALID: 'the provider would reject this request',
+  X_AI_BUDGET_EXCEEDED: 'a model call would exceed its budget',
+  X_AI_GATEWAY_MISSING: 'an llm() action ran with no gateway installed',
+  X_AI_PROMPT_VERSION: 'prompt version or slots are wrong',
+  X_AI_MODEL_UNKNOWN: 'a model id nothing registered in the catalogue',
+  X_AI_PROMPT_SECRET: 'a Secret was about to be rendered into a prompt',
+  X_LLM_OUTPUT_INVALID: 'structured output failed its schema on the answer and the repair turn',
+  X_LLM_REFUSED: 'the model declined the request',
+  X_LLM_TRUNCATED: 'the answer hit its maxTokens ceiling before it was complete',
+  X_LLM_STREAM_INVALID: 'a streamed answer failed its output schema, and a stream cannot repair',
+  X_AGENT_MAX_TURNS: 'an agent hit its turn ceiling without answering',
+  X_AGENT_TOOL_UNEXPOSED:
+    'an agent lists a tool that is neither in-process nor an MCP-exposed action',
+  X_HIVE_EMPTY: 'a hive split produced no members, so the run did nothing',
+  X_EVAL_THRESHOLD: 'an eval scored below its tolerance',
+  X_EVAL_BASELINE_MISSING: 'an eval has no recorded baseline to gate against',
+  X_EVAL_BASELINE_INVALID: 'a recorded baseline cannot be read',
+  X_EVAL_MISSING: 'a prompt has no eval',
+  X_EVAL_RECORDING: 'the gate ran with baseline recording switched on',
+  X_VECTOR_DIM_MISMATCH: 'embedding dimensions differ from the store',
+  X_VECTOR_SCOPE_WIDENED: 'a derived vector scope tried to leave its tenant',
+  X_AI_EMBEDDER_INVALID: 'an Embedder returned fewer vectors than texts it was given',
+  X_VECTOR_UNSCOPED: 'a vector store was read with no tenant bound inside an org request',
+  X_AI_CONTENT_UNSUPPORTED: 'an image or document block the role, model or wire format cannot take',
+  X_AI_MODEL_UNRESOLVED: 'a model call named no model',
+  X_AGENT_TOOL_LOOP: 'an agent repeated one tool call with one result and was stopped',
+  X_AGENT_DEADLINE: 'an agent ran past its declared deadline and was stopped',
+  X_LLM_TOOL_ARGUMENTS_INVALID: 'a finished turn called a tool with arguments that are not JSON',
+  X_AGENT_TRANSCRIPT_INVALID: 'an agent was asked to continue from a transcript it cannot read',
+  X_AI_PROVIDER_UNEXPECTED:
+    'a provider ended a turn with an unknown reason and an incomplete answer',
+  X_AGENT_TOOL_UNDECLARED: 'an agent run offered a tool its declaration does not list',
+  X_AGENT_CONTEXT_EXHAUSTED: "an agent's next request would not fit its model's context window",
+  X_AI_SCRIPT_EXHAUSTED: 'a scriptedProvider was called more times than it has turns',
+  X_EVAL_TOOL_UNDOUBLED: 'an agent eval called a tool with no double',
+  X_AI_CONTEXT_OVERFLOW: 'a provider refused a request as longer than the model context window',
+};
+
+// Titles must be registered for `format()` to render the contract's first line. Unconditional and
+// in one call: every code above is owned here, so a second package claiming one is a real conflict
+// that has to surface as X_ERROR_CODE_DUPLICATE rather than resolve to whoever imported first.
+// `X_FORBIDDEN` (policy's) and `X_CURRENCY_MISMATCH` (money's) are thrown here but never titled.
+registerErrorCodes(
+  Object.fromEntries(Object.entries(AI_ERROR_TITLES).map(([code, title]) => [code, { title }])),
+);

package/src/errors-retry.ts

@@ -1,10 +1,10 @@
-// The retry classification for this package's codes, apart from ./errors only because one file has
-// one job and that catalogue is at its ceiling. The codes, their titles and the single
-// `registerErrorCodes` call stay in ./errors — one owner, one registration.
+// The retry classification for this package's codes, apart from ./error-codes only because one
+// file has one job. The codes, their titles and the single `registerErrorCodes` call stay in
+// ./error-codes — one owner, one registration.
 
 import type { ErrorRetry } from '@ultimat3/core';
 import { registerErrorRetry } from '@ultimat3/core';
-import type { AiErrorCode } from './errors';
+import type { AiErrorCode } from './error-codes';
 
 /**
  * May a client run this model call again? Unclassified means `terminal` — core fails closed — and
@@ -58,6 +58,10 @@ export const AI_ERROR_RETRY = {
   // A run whose transcript outgrew the window outgrows it again: the attempt starts from the same
   // input, takes the same turns and reaches the same size. The fix is `compact` or the model.
   X_AGENT_CONTEXT_EXHAUSTED: 'terminal',
+  // A provider refused the request as over the window: the same request overflows on every
+  // attempt and every provider. Was the transport code, `retryable`, until 2026-11 — an `agentJob`
+  // re-bought every turn of the run to reach the same 400 three times.
+  X_AI_CONTEXT_OVERFLOW: 'terminal',
 } as const satisfies Readonly<Partial<Record<AiErrorCode, ErrorRetry>>>;
 
 // `Partial`, so the table may be a subset — but every key is still checked against the declared set,

package/src/errors.ts

@@ -1,91 +1,10 @@
-// The X_* codes owned by @ultimat3/ai. Budget and threshold failures name the exact knob to
-// change, because the caller is often a CI job with no human attached.
+// The error classes for @ultimat3/ai's core codes. Budget and threshold failures name the exact
+// knob to change, because the caller is often a CI job with no human attached.
 
-import { registerErrorCodes, UltimateError } from '@ultimat3/core';
-
-export const AI_ERROR_CODES = [
-  'X_AI_PROVIDER_UNAVAILABLE',
-  'X_AI_KEY_MISSING',
-  'X_AI_REQUEST_INVALID',
-  'X_AI_BUDGET_EXCEEDED',
-  'X_AI_GATEWAY_MISSING',
-  'X_AI_PROMPT_VERSION',
-  'X_AI_MODEL_UNKNOWN',
-  'X_AI_PROMPT_SECRET',
-  'X_LLM_OUTPUT_INVALID',
-  'X_LLM_REFUSED',
-  'X_LLM_TRUNCATED',
-  'X_LLM_STREAM_INVALID',
-  'X_AGENT_MAX_TURNS',
-  'X_AGENT_TOOL_UNEXPOSED',
-  'X_HIVE_EMPTY',
-  'X_EVAL_THRESHOLD',
-  'X_EVAL_BASELINE_MISSING',
-  'X_EVAL_BASELINE_INVALID',
-  'X_EVAL_MISSING',
-  'X_EVAL_RECORDING',
-  'X_VECTOR_DIM_MISMATCH',
-  'X_VECTOR_SCOPE_WIDENED',
-  'X_AI_EMBEDDER_INVALID',
-  'X_VECTOR_UNSCOPED',
-  'X_AI_CONTENT_UNSUPPORTED',
-  'X_AI_MODEL_UNRESOLVED',
-  'X_AGENT_TOOL_LOOP',
-  'X_AGENT_DEADLINE',
-  'X_LLM_TOOL_ARGUMENTS_INVALID',
-  'X_AGENT_TRANSCRIPT_INVALID',
-  'X_AI_PROVIDER_UNEXPECTED',
-  'X_AGENT_TOOL_UNDECLARED',
-  'X_AGENT_CONTEXT_EXHAUSTED',
-] as const;
-
-export type AiErrorCode = (typeof AI_ERROR_CODES)[number];
-
-export const AI_ERROR_TITLES: Readonly<Record<AiErrorCode, string>> = {
-  X_AI_PROVIDER_UNAVAILABLE: 'the model provider is unreachable',
-  X_AI_KEY_MISSING: 'the provider API key is not set',
-  X_AI_REQUEST_INVALID: 'the provider would reject this request',
-  X_AI_BUDGET_EXCEEDED: 'a model call would exceed its budget',
-  X_AI_GATEWAY_MISSING: 'an llm() action ran with no gateway installed',
-  X_AI_PROMPT_VERSION: 'prompt version or slots are wrong',
-  X_AI_MODEL_UNKNOWN: 'a model id nothing registered in the catalogue',
-  X_AI_PROMPT_SECRET: 'a Secret was about to be rendered into a prompt',
-  X_LLM_OUTPUT_INVALID: 'structured output failed its schema on the answer and the repair turn',
-  X_LLM_REFUSED: 'the model declined the request',
-  X_LLM_TRUNCATED: 'the answer hit its maxTokens ceiling before it was complete',
-  X_LLM_STREAM_INVALID: 'a streamed answer failed its output schema, and a stream cannot repair',
-  X_AGENT_MAX_TURNS: 'an agent hit its turn ceiling without answering',
-  X_AGENT_TOOL_UNEXPOSED:
-    'an agent lists a tool that is neither in-process nor an MCP-exposed action',
-  X_HIVE_EMPTY: 'a hive split produced no members, so the run did nothing',
-  X_EVAL_THRESHOLD: 'an eval scored below its tolerance',
-  X_EVAL_BASELINE_MISSING: 'an eval has no recorded baseline to gate against',
-  X_EVAL_BASELINE_INVALID: 'a recorded baseline cannot be read',
-  X_EVAL_MISSING: 'a prompt has no eval',
-  X_EVAL_RECORDING: 'the gate ran with baseline recording switched on',
-  X_VECTOR_DIM_MISMATCH: 'embedding dimensions differ from the store',
-  X_VECTOR_SCOPE_WIDENED: 'a derived vector scope tried to leave its tenant',
-  X_AI_EMBEDDER_INVALID: 'an Embedder returned fewer vectors than texts it was given',
-  X_VECTOR_UNSCOPED: 'a vector store was read with no tenant bound inside an org request',
-  X_AI_CONTENT_UNSUPPORTED: 'an image or document block the role, model or wire format cannot take',
-  X_AI_MODEL_UNRESOLVED: 'a model call named no model',
-  X_AGENT_TOOL_LOOP: 'an agent repeated one tool call with one result and was stopped',
-  X_AGENT_DEADLINE: 'an agent ran past its declared deadline and was stopped',
-  X_LLM_TOOL_ARGUMENTS_INVALID: 'a finished turn called a tool with arguments that are not JSON',
-  X_AGENT_TRANSCRIPT_INVALID: 'an agent was asked to continue from a transcript it cannot read',
-  X_AI_PROVIDER_UNEXPECTED:
-    'a provider ended a turn with an unknown reason and an incomplete answer',
-  X_AGENT_TOOL_UNDECLARED: 'an agent run offered a tool its declaration does not list',
-  X_AGENT_CONTEXT_EXHAUSTED: "an agent's next request would not fit its model's context window",
-};
-
-// Titles must be registered for `format()` to render the contract's first line. Unconditional and
-// in one call: every code above is owned here, so a second package claiming one is a real conflict
-// that has to surface as X_ERROR_CODE_DUPLICATE rather than resolve to whoever imported first.
-// `X_FORBIDDEN` (policy's) and `X_CURRENCY_MISMATCH` (money's) are thrown here but never titled.
-registerErrorCodes(
-  Object.fromEntries(Object.entries(AI_ERROR_TITLES).map(([code, title]) => [code, { title }])),
-);
+import { UltimateError } from '@ultimat3/core';
+// The codes, their titles and the single registration call are `./error-codes`, imported here for
+// that side effect.
+import './error-codes';
 
 // No `docs:` on the subclasses below. `UltimateError` fills it from `describeErrorCode(code).docs`,
 // which is `@ultimat3/core`'s `ERROR_DOCS_URL` — one page for every code, never one per code, because

package/src/eval-agent.ts

@@ -0,0 +1,100 @@
+// The agent arm of `defineEval`: one case is one whole `agent.run()` — its tools, its turns — under
+// the gateway the eval was handed, with every tool answered by the eval's double. Split from
+// ./evals so the prompt arm and the gate stay one file each.
+
+import type { AnyAction, InvokeOptions } from '@ultimat3/action';
+import type { Actor } from '@ultimat3/core';
+import { assert, serviceActor, stringField } from '@ultimat3/core';
+import type { AgentRun } from './agent-continue';
+import { agentFactsOf } from './agent-facts';
+import type { Gateway } from './gateway';
+import type { Prompt } from './prompt';
+import { describePrompts } from './prompt';
+import type { ToolDouble } from './runtime';
+import { withEvalScope } from './runtime';
+import type { EvalRun, Scorer } from './scorers';
+
+/** What the agent arm needs of an `agent()` action: its name, and `.run()`. */
+export interface EvaluatedAgent<I = unknown> extends AnyAction {
+  run(input: I, opts?: InvokeOptions): Promise<AgentRun<unknown>>;
+}
+
+/** One agent case: the agent's own input, plus what a good answer looks like. */
+export interface AgentEvalCase<I = unknown> {
+  readonly name: string;
+  readonly input: I;
+  /** Reference answer, or the substring/JSON the scorers check for. */
+  readonly expected?: string;
+}
+
+export interface AgentEvalInput<I = unknown> {
+  readonly name: string;
+  /** An `agent()` action. Its prompt is the one this eval covers, and its hash keys the baseline. */
+  readonly agent: EvaluatedAgent<I>;
+  readonly cases: readonly AgentEvalCase<I>[];
+  /**
+   * One double per tool the model may call. An eval NEVER runs a real tool: a call to a tool with
+   * no double here ends the run as `X_EVAL_TOOL_UNDOUBLED`.
+   */
+  readonly tools: Readonly<Record<string, ToolDouble>>;
+  readonly scorers: readonly Scorer[];
+  /** As on the prompt arm: `import.meta.resolve('./name.baseline.json')`. */
+  readonly baseline: string;
+  readonly tolerance: number;
+  /** Who each case runs as — for a policy that needs one. Default: the `eval` service actor. */
+  readonly actor?: Actor;
+}
+
+/** What one case produced, before it is scored. `run` is the agent arm's. */
+export type CaseOutcome =
+  | { readonly ok: true; readonly output: string; readonly run?: EvalRun }
+  | { readonly ok: false; readonly code: string };
+
+/** The default identity of an agent eval: a service, never a person a policy would mistake it for. */
+export const EVAL_ACTOR: Actor = serviceActor({ id: 'eval' });
+
+/** The code that means the EVAL is wrong rather than the agent: never scored, always thrown. */
+const MISDECLARED = 'X_EVAL_TOOL_UNDOUBLED';
+
+/**
+ * The prompt an agent runs on, read off the agent registry. An action `agent()` did not build — or
+ * a `named()` twin, which the registry never sees — has none, and an eval of it would cover nothing.
+ */
+export function agentPrompt(agent: AnyAction, evalName: string): Prompt {
+  const ref = agentFactsOf(agent)?.prompt;
+  const prompt = ref === undefined ? undefined : describePrompts().find((p) => p.ref === ref);
+  assert(
+    prompt !== undefined,
+    `defineEval "${evalName}" names an agent whose prompt is not registered: an action agent() did not build, a named() twin, or a prompt reset since`,
+    `registerAction('<name>', agent({ … })) and pass that action to defineEval({ agent })`,
+  );
+  return prompt;
+}
+
+/**
+ * Run one case. A failed run — a loop, `X_AGENT_MAX_TURNS`, an invalid answer — is an OUTCOME the
+ * eval scores 0 and reports by code, so a regression never crashes the suite. An undoubled tool
+ * call is the one failure rethrown: the eval is misdeclared, and scoring it would let a recording
+ * accept a case that tried to run a real tool.
+ */
+export async function runAgentCase(
+  input: AgentEvalInput,
+  testCase: AgentEvalCase,
+  gateway: Gateway,
+): Promise<CaseOutcome> {
+  const scope = { eval: input.name, gateway, tools: input.tools, calls: [] };
+  try {
+    const run = await withEvalScope(scope, () =>
+      input.agent.run(testCase.input, { actor: input.actor ?? EVAL_ACTOR }),
+    );
+    return {
+      ok: true,
+      output: JSON.stringify(run.output) ?? 'null',
+      run: { turns: run.turns, toolCalls: [...scope.calls], usage: run.usage },
+    };
+  } catch (error) {
+    const code = stringField(error, 'code');
+    if (code === MISDECLARED) throw error;
+    return { ok: false, code: code ?? 'uncoded' };
+  }
+}

package/src/eval-errors.ts

@@ -1,6 +1,6 @@
-// The five X_EVAL_* codes, apart from ./errors only because one file has one job and the catalogue
+// The six X_EVAL_* codes, apart from ./errors only because one file has one job and the catalogue
 // outgrew its ceiling. The codes themselves, their titles and the single `registerErrorCodes` call
-// stay in ./errors — one owner, one registration, one place a duplicate can surface.
+// stay in ./error-codes — one owner, one registration, one place a duplicate can surface.
 
 import { UltimateError } from '@ultimat3/core';
 
@@ -73,11 +73,37 @@ export class EvalBaselineInvalidError extends UltimateError {
  * money and answers users, so the gate fails on it exactly like an untyped module.
  */
 export class EvalMissingError extends UltimateError {
-  constructor(input: { prompt: string; id: string }) {
+  /** `agent`: the agent the prompt drives, so the fix names the arm that runs its tools and turns. */
+  constructor(input: { prompt: string; id: string; agent?: string }) {
+    const arm =
+      input.agent === undefined
+        ? 'prompt, cases, scorers'
+        : `agent: ${input.agent}, cases, tools, scorers`;
     super({
       code: 'X_EVAL_MISSING',
-      cause: `prompt "${input.prompt}" has no eval`,
-      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
+      cause:
+        input.agent === undefined
+          ? `prompt "${input.prompt}" has no eval`
+          : `prompt "${input.prompt}" has no eval, and agent ${input.agent} runs on it`,
+      fix: `defineEval({ name: '${input.id}', ${arm}, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
+    });
+  }
+}
+
+/**
+ * An agent eval's model called a tool the eval declares no double for. An eval NEVER runs a real
+ * tool — a case that reached one in CI would write — so the run ends here rather than reading the
+ * call as a tool failure the model could route around, or scoring it 0 and letting a recording
+ * accept it. Thrown out of `run` / `assert`: the eval is misdeclared, not the agent regressed.
+ */
+export class EvalToolUndoubledError extends UltimateError {
+  constructor(input: { eval: string; tool: string; doubled: readonly string[] }) {
+    const doubled = input.doubled.length === 0 ? 'none' : input.doubled.join(', ');
+    super({
+      code: 'X_EVAL_TOOL_UNDOUBLED',
+      cause: `agent eval "${input.eval}" ran a case whose model called ${input.tool}, which has no double (doubled: ${doubled}); an eval never runs a real tool`,
+      fix: `tools: { ${input.tool}: (input) => ({ /* what the tool answers */ }) }   # on defineEval({ name: '${input.eval}' })`,
+      meta: { eval: input.eval, tool: input.tool },
     });
   }
 }