@ultimat3/ai 1.2.0 → 2.0.0

package/src/budget.ts

@@ -62,10 +62,20 @@ export function estimateSpend(request: GenerateRequest): SpendEstimate {
 /** Where cross-request counters live. Swap for Redis in a multi-process deployment. */
 export interface BudgetStore {
   spent(key: string): Promise<number> | number;
+  /** `tokens` may be NEGATIVE: releasing a reservation the call never spent is a credit. */
   add(key: string, tokens: number): Promise<void> | void;
   reset(key?: string): Promise<void> | void;
 }
 
+/**
+ * What `reserve` debited, so `record` can reconcile it against the provider's real counts and
+ * `release` can give it back. Held by the caller rather than the ledger because one ledger serves
+ * every concurrent call in a request, and each one owns its own reservation.
+ */
+export interface BudgetReservation {
+  readonly tokens: number;
+}
+
 export class MemoryBudgetStore implements BudgetStore {
   private readonly counters = new Map<string, number>();
 
@@ -108,6 +118,23 @@ export class BudgetLedger {
   private requestTokens = 0;
   private costMinor = 0;
   private readonly currency: string;
+  /**
+   * The ledger this one was `derive`d from, or `undefined` for a scope's root. Set by `derive`
+   * rather than taken through `BudgetLedgerInput`, so the chain is always the derivation and a
+   * caller cannot build a cycle out of it.
+   *
+   * Without it a derived ledger reported to nobody: `llm()` derives one per call, so the ambient
+   * ledger `gateway.scope()` installed counted zero tokens and zero cost however many calls ran
+   * inside it, and its `request` ceiling was re-granted in full to every one of them.
+   */
+  private parent: BudgetLedger | undefined;
+  /**
+   * Reservations take turns. Check-then-debit spans an `await store.spent()`, and three callers
+   * interleaving inside it is the bypass this ledger exists to close — one event loop, so a
+   * promise chain IS the lock. A store shared across PROCESSES needs an atomic increment of its
+   * own; this closes the parallelism inside one.
+   */
+  private turnstile: Promise<unknown> = Promise.resolve();
 
   constructor(input: BudgetLedgerInput) {
     this.limits = input.limits;
@@ -118,12 +145,40 @@ export class BudgetLedger {
   }
 
   /**
-   * Check an estimate against every applicable scope BEFORE the call. Throws on the first
-   * scope that cannot cover it, naming that scope, so the fix line points at one knob rather
-   * than four. Nothing is debited here: `record` does that with the provider's real counts.
+   * Check an estimate against every applicable scope BEFORE the call, then DEBIT it. Throws on
+   * the first scope that cannot cover it, naming that scope, so the fix line points at one knob
+   * rather than four.
+   *
+   * The debit is what makes the ceiling hold under parallelism. Checking without debiting meant
+   * three concurrent calls under one ledger all read `spent() === 0`, all passed, and all three
+   * recorded against a ceiling only one of them fitted — an "un-bypassable" org budget bypassed
+   * by `Promise.all`. `record` replaces the estimate with the real counts; `release` gives it
+   * back when the call never happened.
    */
-  async reserve(estimate: SpendEstimate): Promise<void> {
-    this.assertScope('request', this.limits.request, this.requestTokens, estimate.tokens);
+  async reserve(estimate: SpendEstimate): Promise<BudgetReservation> {
+    // The ROOT's turnstile, not this ledger's: reservations under one scope take turns even when
+    // each call derived its own ledger, which is every `llm()` call. A per-ledger queue serialised
+    // nothing once `derive` existed — `Promise.all` of three derived ledgers all read the chain
+    // before any of them debited it.
+    const gate = this.rootLedger();
+    const turn = gate.turnstile.then(() => this.reserveNow(estimate));
+    // Chained on a settled shadow: one refusal must not reject every reservation queued behind it.
+    gate.turnstile = turn.catch(() => undefined);
+    return await turn;
+  }
+
+  private rootLedger(): BudgetLedger {
+    let ledger: BudgetLedger = this;
+    while (ledger.parent !== undefined) ledger = ledger.parent;
+    return ledger;
+  }
+
+  private async reserveNow(estimate: SpendEstimate): Promise<BudgetReservation> {
+    // Every ledger in the chain, because each keeps its own counter and the tightest limit is not
+    // always the one with the most spent against it.
+    for (let l: BudgetLedger | undefined = this; l !== undefined; l = l.parent) {
+      l.assertScope('request', l.limits.request, l.requestTokens, estimate.tokens);
+    }
     // Per call, so nothing is "already spent" against it.
     this.assertScope('tokensIn', this.limits.tokensIn, 0, estimate.inputTokens);
     if (this.limits.actor !== undefined && this.actorKey !== undefined) {
@@ -135,6 +190,14 @@ export class BudgetLedger {
       this.assertScope(`org:${this.orgKey}`, this.limits.org, spent, estimate.tokens);
     }
     this.assertCost(estimate.cost);
+    await this.debit(estimate.tokens);
+    return { tokens: estimate.tokens };
+  }
+
+  /** Give a reservation back: a provider that threw, a stream abandoned before `done`. */
+  async release(reservation: BudgetReservation | undefined): Promise<void> {
+    if (reservation === undefined) return;
+    await this.debit(-reservation.tokens);
   }
 
   /**
@@ -143,7 +206,7 @@ export class BudgetLedger {
    * an `llm()` action must not be able to widen the actor or org ceiling it runs inside.
    */
   derive(limits: BudgetLimits): BudgetLedger {
-    return new BudgetLedger({
+    const child = new BudgetLedger({
       limits: {
         ...pick('request', tighterNumber(this.limits.request, limits.request)),
         ...pick('tokensIn', tighterNumber(this.limits.tokensIn, limits.tokensIn)),
@@ -156,13 +219,37 @@ export class BudgetLedger {
       store: this.store,
       currency: this.currency,
     });
+    child.parent = this;
+    return child;
+  }
+
+  /**
+   * Debit ACTUAL usage after the call, replacing the estimate `reserve` worked from — so only
+   * the DIFFERENCE lands here. Called without the reservation it behaves as it always did and
+   * debits the full amount, which double-counts a reserved call: pass the handle `reserve`
+   * returned.
+   */
+  async record(usage: TokenUsage, cost: Money, reservation?: BudgetReservation): Promise<void> {
+    // Up the chain, because `derive` copies the currency: a scope's reported cost is its own
+    // calls plus every call made under a ledger derived from it.
+    for (let l: BudgetLedger | undefined = this; l !== undefined; l = l.parent) {
+      l.costMinor += cost.minor;
+    }
+    await this.debit(totalTokens(usage) - (reservation?.tokens ?? 0));
   }
 
-  /** Debit ACTUAL usage after the call, replacing the estimate `reserve` worked from. */
-  async record(usage: TokenUsage, cost: Money): Promise<void> {
-    const tokens = totalTokens(usage);
-    this.requestTokens += tokens;
-    this.costMinor += cost.minor;
+  /**
+   * The one write path. Negative credits a release or an over-estimate back.
+   *
+   * The in-memory counters walk the chain; the STORE is written once, by the ledger the call was
+   * made on. A child shares its parent's store and identity keys, so debiting through the parent
+   * as well would bill the actor and the org twice for one call.
+   */
+  private async debit(tokens: number): Promise<void> {
+    if (tokens === 0) return;
+    for (let l: BudgetLedger | undefined = this; l !== undefined; l = l.parent) {
+      l.requestTokens += tokens;
+    }
     if (this.actorKey !== undefined) await this.store.add(this.actorKey, tokens);
     if (this.orgKey !== undefined) await this.store.add(this.orgKey, tokens);
   }

package/src/error-body.ts

@@ -0,0 +1,44 @@
+// Single responsibility: what a provider's own failure body says, and the credential that must
+// never survive into it.
+//
+// Shared by both transports rather than copied: every endpoint this package speaks to reports a
+// failure in the same `{ error: { message } }` envelope, and every one of them carries a key in a
+// header a proxy can echo into its own 4xx body. Two copies of either rule is two behaviours to
+// keep in step — and the scrub was on one provider only until 2026-08.
+
+import { REDACTED } from '@ultimat3/core';
+
+/** Enough of an error body to name the field that was wrong, not enough to fill a log. */
+const DETAIL_LIMIT = 300;
+
+/**
+ * The provider's own message, when it sent one — it names the offending field, we name the fix.
+ * Falls back to the raw text because a proxy or a gateway timeout page is not JSON and is still
+ * the best evidence there is.
+ */
+export async function detailOf(response: Response): Promise<string> {
+  const body = await response.text().catch(() => '');
+  try {
+    const parsed: unknown = JSON.parse(body);
+    if (typeof parsed === 'object' && parsed !== null) {
+      const error = (parsed as Record<string, unknown>)['error'];
+      if (typeof error === 'object' && error !== null) {
+        const message = (error as Record<string, unknown>)['message'];
+        if (typeof message === 'string') return message.slice(0, DETAIL_LIMIT);
+      }
+    }
+  } catch {
+    // Not JSON — a proxy or a gateway timeout page. The raw text is still the best evidence.
+  }
+  return body === '' ? response.statusText : body.slice(0, DETAIL_LIMIT);
+}
+
+/**
+ * Every occurrence of the credential replaced with `[redacted]`. Cheap, and the one leak path:
+ * a proxy that echoes the request headers into its own 400 body puts the key in an error, and an
+ * error reaches a log index, a span and an HTTP problem document.
+ */
+export function withoutKey(detail: string, apiKey: string): string {
+  if (apiKey === '') return detail;
+  return detail.split(apiKey).join(REDACTED);
+}

package/src/errors.ts

@@ -10,9 +10,14 @@ export const AI_ERROR_CODES = [
   'X_AI_BUDGET_EXCEEDED',
   'X_AI_GATEWAY_MISSING',
   'X_AI_PROMPT_VERSION',
+  'X_AI_MODEL_UNKNOWN',
+  'X_AI_PROMPT_SECRET',
   'X_LLM_OUTPUT_INVALID',
   'X_LLM_REFUSED',
   'X_LLM_TRUNCATED',
+  'X_LLM_STREAM_INVALID',
+  'X_AGENT_MAX_TURNS',
+  'X_AGENT_TOOL_UNEXPOSED',
   'X_EVAL_THRESHOLD',
   'X_EVAL_BASELINE_MISSING',
   'X_EVAL_BASELINE_INVALID',
@@ -32,9 +37,14 @@ export const AI_ERROR_TITLES: Readonly<Record<AiErrorCode, string>> = {
   X_AI_BUDGET_EXCEEDED: 'a model call would exceed its budget',
   X_AI_GATEWAY_MISSING: 'an llm() action ran with no gateway installed',
   X_AI_PROMPT_VERSION: 'prompt version or slots are wrong',
+  X_AI_MODEL_UNKNOWN: 'a model id nothing registered in the catalogue',
+  X_AI_PROMPT_SECRET: 'a Secret was about to be rendered into a prompt',
   X_LLM_OUTPUT_INVALID: 'structured output failed its schema on the answer and the repair turn',
   X_LLM_REFUSED: 'the model declined the request',
   X_LLM_TRUNCATED: 'the answer hit its maxTokens ceiling before it was complete',
+  X_LLM_STREAM_INVALID: 'a streamed answer failed its output schema, and a stream cannot repair',
+  X_AGENT_MAX_TURNS: 'an agent hit its turn ceiling without answering',
+  X_AGENT_TOOL_UNEXPOSED: 'an agent lists a tool that is not an MCP-exposed action',
   X_EVAL_THRESHOLD: 'an eval scored below its tolerance',
   X_EVAL_BASELINE_MISSING: 'an eval has no recorded baseline to gate against',
   X_EVAL_BASELINE_INVALID: 'a recorded baseline cannot be read',
@@ -108,6 +118,63 @@ export class AiGatewayMissingError extends UltimateError {
   }
 }
 
+/**
+ * A model id nothing put in the catalogue. This is what replaced the closed `ModelId` union: the
+ * union made a company's own model id inexpressible, so the only way past `tsc` was to claim a
+ * Claude id — and then `costOf` priced an internal model at Anthropic list rates and the budget
+ * ledger reserved against a number belonging to a model nobody ran. A wrong id is still refused;
+ * it is refused HERE, at the first read of the spec, instead of by making a right one impossible.
+ */
+export class AiModelUnknownError extends UltimateError {
+  constructor(input: { model: string; registered: readonly string[] }) {
+    super({
+      code: 'X_AI_MODEL_UNKNOWN',
+      cause:
+        `model "${input.model}" has no registered spec, so nothing can price it ` +
+        `(registered: ${input.registered.length > 0 ? input.registered.join(', ') : 'none'})`,
+      // The `errors` gate blanks every interpolation, so the literal half alone has to name the
+      // call. Which ids ARE registered is a fact of the failure, and cause is where facts live.
+      fix: 'registerModel({ id, contextWindow, maxOutput, inputPerMillion, outputPerMillion, cacheMinimumTokens, reasoning }) at boot, before configureAi',
+      docs: docsFor('X_AI_MODEL_UNKNOWN'),
+      meta: { model: input.model },
+    });
+  }
+}
+
+/**
+ * A `Secret` reached a prompt variable. `Secret` redacts by VALUE, so this would not have leaked
+ * — it would have rendered `[redacted]` into the template and asked the model to reason about it,
+ * which is a prompt that reads fine and means something else. The same class of failure as an
+ * unfilled `{{slot}}`, and refused for the same reason: loudly, before a token is spent.
+ */
+export class AiPromptSecretError extends UltimateError {
+  constructor(input: { ref: string; keys: readonly string[] }) {
+    super({
+      code: 'X_AI_PROMPT_SECRET',
+      cause: `prompt "${input.ref}" was given a Secret in vars(): ${input.keys.join(', ')}`,
+      fix: 'drop the key from vars() and from the template, or revealSecret(value) in vars() if the model genuinely has to read it',
+      docs: docsFor('X_AI_PROMPT_SECRET'),
+    });
+  }
+}
+
+/**
+ * A streamed answer did not satisfy its `output` schema. Distinct from `X_LLM_OUTPUT_INVALID`
+ * because there is no repair turn to have failed: the consumer has already read the tokens, and
+ * replaying a second answer over the top is two answers to one question. So a stream gets one
+ * attempt, and the fix is either a looser schema or the non-streaming call that CAN repair.
+ */
+export class LlmStreamInvalidError extends UltimateError {
+  constructor(input: { prompt: string; issues: string }) {
+    super({
+      code: 'X_LLM_STREAM_INVALID',
+      cause: `streamed answer to prompt "${input.prompt}" failed its output schema: ${input.issues}`,
+      fix: 'call the action instead of .stream() when the answer must satisfy a structured schema — a stream has already delivered its tokens and cannot take a repair turn',
+      docs: docsFor('X_LLM_STREAM_INVALID'),
+    });
+  }
+}
+
 /**
  * The model's answer failed the action's `output` schema on the first turn AND on the repair
  * turn that followed. Two failures is a disagreement between the prompt and the schema, not a
@@ -126,6 +193,50 @@ export class LlmOutputInvalidError extends UltimateError {
   }
 }
 
+/**
+ * An `agent()` ran out of turns with no answer. Never a partial one: the loop's whole contract is
+ * that it either satisfies `output` or says it did not, and a half-finished transcript returned as
+ * a result is a model's working notes presented as a decision.
+ *
+ * Reaching the ceiling almost always means the loop has no exit condition — a tool that answers
+ * the same thing every turn, or a prompt that never tells the model to finish. Raising the
+ * ceiling on that spends more money on the same non-answer, which is why the fix names the prompt
+ * before it names the number.
+ */
+export class AgentMaxTurnsError extends UltimateError {
+  constructor(input: { agent: string; turns: number; calls: number }) {
+    super({
+      code: 'X_AGENT_MAX_TURNS',
+      cause:
+        `agent "${input.agent}" used all ${input.turns} turns and ${input.calls} tool calls ` +
+        `without calling the respond tool`,
+      fix: 'tell the template when to stop and answer through the respond tool, then bump its version — raise maxTurns only once the run demonstrably converges',
+      docs: docsFor('X_AGENT_MAX_TURNS'),
+      meta: { agent: input.agent, turns: input.turns },
+    });
+  }
+}
+
+/**
+ * An `agent()` lists an action that is not an MCP-exposed tool. Refused at DECLARATION rather
+ * than filtered at the call, because a silently dropped tool is the worst of both: the
+ * declaration reads as if the model can call it, and the model is never offered it.
+ *
+ * `isMcpExposed` is the one predicate — an in-app agent and an external MCP client see exactly
+ * the same catalogue, which is what keeps "there is no second authz system" true of the catalogue
+ * too.
+ */
+export class AgentToolUnexposedError extends UltimateError {
+  constructor(input: { agent: string; tools: readonly string[] }) {
+    super({
+      code: 'X_AGENT_TOOL_UNEXPOSED',
+      cause: `agent "${input.agent}" lists tools no MCP surface exposes: ${input.tools.join(', ')}`,
+      fix: 'add mcp: { expose: true } to the action named in cause, or drop it from the agent tools list',
+      docs: docsFor('X_AGENT_TOOL_UNEXPOSED'),
+    });
+  }
+}
+
 /**
  * The provider's safety classifiers declined the request. A refusal is a 200 with no answer in
  * it, so it has to become an error HERE or it becomes an empty string somewhere downstream that
@@ -137,8 +248,13 @@ export class LlmRefusedError extends UltimateError {
   constructor(input: {
     prompt: string;
     model: string;
-    /** A blessed model that is NOT the one that refused — the fix has to be pasteable. */
-    alternative: string;
+    /**
+     * A blessed model MORE capable than the one that refused, or `undefined` when the refusal
+     * came from the most capable one this build knows. Retrying a refusal on a weaker model is
+     * the one retry that cannot help, so the fix line drops the suggestion rather than inventing
+     * a downgrade.
+     */
+    alternative: string | undefined;
     category: string | undefined;
     explanation: string | undefined;
   }) {
@@ -148,7 +264,10 @@ export class LlmRefusedError extends UltimateError {
         `model "${input.model}" declined prompt "${input.prompt}"` +
         `${input.category === undefined ? '' : ` (${input.category})`}` +
         `${input.explanation === undefined ? '' : `: ${input.explanation}`}`,
-      fix: `set model: '${input.alternative}' on the llm() declaration, or edit the template in definePrompt('${input.prompt}') and bump its version`,
+      fix:
+        input.alternative === undefined
+          ? `edit the template in definePrompt('${input.prompt}') and bump its version — no blessed model is more capable than '${input.model}'`
+          : `set model: '${input.alternative}' on the llm() declaration, or edit the template in definePrompt('${input.prompt}') and bump its version`,
       docs: docsFor('X_LLM_REFUSED'),
       meta: { model: input.model, category: input.category },
     });
@@ -201,100 +320,15 @@ export class AiPromptRenderError extends UltimateError {
   }
 }
 
-/**
- * An eval scored further below its recorded baseline than its tolerance allows. The gate is the
- * DROP, not an absolute number — a model that got marginally worse everywhere is not the same
- * event as a prompt edit that broke one case, and only the second one is anybody's fault.
- *
- * This is a test failure, not a warning.
- */
-export class EvalThresholdError extends UltimateError {
-  constructor(input: {
-    eval: string;
-    score: number;
-    baseline: number;
-    tolerance: number;
-    promptVersion: string;
-    regressed: readonly string[];
-  }) {
-    super({
-      code: 'X_EVAL_THRESHOLD',
-      cause:
-        `eval "${input.eval}" scored ${input.score.toFixed(3)} against a recorded baseline of ` +
-        `${input.baseline.toFixed(3)} (tolerance ${input.tolerance.toFixed(3)}) on prompt ` +
-        `version ${input.promptVersion}; regressed: ${input.regressed.join(', ')}`,
-      fix: `x test ${input.eval} to see per-case scores, then fix the prompt — or ULTIMATE_EVAL_RECORD=1 x test eval to accept the new numbers as a reviewed diff`,
-      docs: docsFor('X_EVAL_THRESHOLD'),
-    });
-  }
-}
-
-/**
- * An eval declared a baseline that has never been recorded. Not a pass: an eval with nothing to
- * compare against gates on nothing, and a step that cannot fail is a step that is not running.
- */
-export class EvalBaselineMissingError extends UltimateError {
-  constructor(input: { eval: string; path: string; reason: string; fix?: string }) {
-    super({
-      code: 'X_EVAL_BASELINE_MISSING',
-      cause: `eval "${input.eval}" gates against ${input.path}, which ${input.reason}`,
-      fix: input.fix ?? `ULTIMATE_EVAL_RECORD=1 x test eval, then commit ${input.path}`,
-      docs: docsFor('X_EVAL_BASELINE_MISSING'),
-    });
-  }
-}
-
-/** A recorded baseline that cannot be read. Never treated as absent — that would erase a gate. */
-export class EvalBaselineInvalidError extends UltimateError {
-  constructor(input: { path: string; problem: string }) {
-    super({
-      code: 'X_EVAL_BASELINE_INVALID',
-      cause: `the recorded baseline ${input.path} ${input.problem}`,
-      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
-      docs: docsFor('X_EVAL_BASELINE_INVALID'),
-    });
-  }
-}
-
-/**
- * A registered prompt that no eval names. An unevaluated prompt is untested code that costs
- * money and answers users, so the gate fails on it exactly like an untyped module.
- */
-export class EvalMissingError extends UltimateError {
-  constructor(input: { prompt: string; id: string }) {
-    super({
-      code: 'X_EVAL_MISSING',
-      cause: `prompt "${input.prompt}" has no eval`,
-      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
-      docs: docsFor('X_EVAL_MISSING'),
-    });
-  }
-}
-
-/**
- * The gate ran with baseline recording switched on. Recording makes every eval write the numbers
- * it just measured and pass, so a `x verify` that inherited the flag reports green over scores
- * nothing compared — and rewrites the committed baselines on its way through, which is the half
- * a red step alone would not undo. Recording is a deliberate, reviewable diff, never a gate run.
- */
-export class EvalRecordingError extends UltimateError {
-  constructor(input: { env: string }) {
-    super({
-      code: 'X_EVAL_RECORDING',
-      cause: `${input.env} is set, so every eval would re-record its baseline instead of gating on it`,
-      fix: `env -u ${input.env} x verify`,
-      docs: docsFor('X_EVAL_RECORDING'),
-    });
-  }
-}
-
 /** A vector's length does not match the store's declared dimension. */
 export class VectorDimMismatchError extends UltimateError {
   constructor(input: { store: string; expected: number; received: number }) {
     super({
       code: 'X_VECTOR_DIM_MISMATCH',
       cause: `store "${input.store}" expects ${input.expected} dimensions, got ${input.received}`,
-      fix: 'use the same embedder that created the store, or x ai reindex to rebuild it',
+      // Not `x ai reindex`: that command is PLANNED and throws, so a fix line naming it sends an
+      // operator to a wall. A fix has to be performable today, which here means app code.
+      fix: 'use the same embedder that created the store, or re-embed every record at the new width and upsert it',
       docs: docsFor('X_VECTOR_DIM_MISMATCH'),
     });
   }
@@ -330,7 +364,7 @@ export class EmbedderDimMismatchError extends UltimateError {
       cause:
         `embedder "${input.embedder}" is declared with ${input.expected} dimensions but the ` +
         `provider returned ${input.received}`,
-      fix: `set dimension: ${input.received} on the embedder, then x ai reindex to rebuild the store`,
+      fix: `set dimension: ${input.received} on the embedder, then re-embed every record at that width and upsert it`,
       docs: docsFor('X_VECTOR_DIM_MISMATCH'),
     });
   }
@@ -393,13 +427,23 @@ export class AiRequestInvalidError extends UltimateError {
 export class AiTransportError extends UltimateError {
   readonly status: number | undefined;
 
-  constructor(input: { provider: string; status?: number | undefined; detail: string }) {
+  constructor(input: {
+    provider: string;
+    status?: number | undefined;
+    detail: string;
+    /**
+     * The env var holding THIS provider's key. Passed on the HTTP path, where a 401 has one fix:
+     * a hardcoded `ANTHROPIC_API_KEY` was the whole fix line, so an OpenAI-format endpoint
+     * rejecting a key sent an operator to set a variable it never reads.
+     */
+    envVar?: string | undefined;
+  }) {
     super({
       code: 'X_AI_PROVIDER_UNAVAILABLE',
       cause: `provider "${input.provider}" ${
         input.status === undefined ? 'failed' : `returned ${input.status}`
       }: ${input.detail}`,
-      fix: fixForStatus(input.status),
+      fix: fixForStatus(input.status, input.envVar),
       docs: docsFor('X_AI_PROVIDER_UNAVAILABLE'),
       meta: { provider: input.provider, status: input.status },
     });
@@ -407,9 +451,11 @@ export class AiTransportError extends UltimateError {
   }
 }
 
-function fixForStatus(status: number | undefined): string {
+function fixForStatus(status: number | undefined, envVar: string | undefined): string {
   if (status === 401 || status === 403) {
-    return 'export ANTHROPIC_API_KEY=<key> with a key that is active for this model';
+    return envVar === undefined
+      ? 'export the API key env var of the provider named in cause, with a key that is active for this model'
+      : `export ${envVar}=<key> with a key that is active for this model`;
   }
   if (status === 429) {
     return 'lower concurrency or raise the provider rate limit; the gateway already backs off';

package/src/eval-baseline.ts

@@ -9,7 +9,7 @@
 // not a measurement.
 
 import { isAbsolute } from 'node:path';
-import { EvalBaselineInvalidError, EvalBaselineMissingError } from './errors';
+import { EvalBaselineInvalidError, EvalBaselineMissingError } from './eval-errors';
 
 export interface EvalBaseline {
   readonly eval: string;

package/src/eval-errors.ts

@@ -0,0 +1,98 @@
+// The five X_EVAL_* codes, apart from ./errors only because one file has one job and the catalogue
+// outgrew its ceiling. The codes themselves, their titles and the single `registerErrorCodes` call
+// stay in ./errors — one owner, one registration, one place a duplicate can surface.
+
+import { UltimateError } from '@ultimat3/core';
+import type { AiErrorCode } from './errors';
+
+const docsFor = (code: AiErrorCode): string => `https://ultimate.dev/errors/${code}`;
+
+/**
+ * An eval scored further below its recorded baseline than its tolerance allows. The gate is the
+ * DROP, not an absolute number — a model that got marginally worse everywhere is not the same
+ * event as a prompt edit that broke one case, and only the second one is anybody's fault.
+ *
+ * This is a test failure, not a warning.
+ */
+export class EvalThresholdError extends UltimateError {
+  constructor(input: {
+    eval: string;
+    score: number;
+    baseline: number;
+    tolerance: number;
+    promptVersion: string;
+    regressed: readonly string[];
+  }) {
+    super({
+      code: 'X_EVAL_THRESHOLD',
+      cause:
+        `eval "${input.eval}" scored ${input.score.toFixed(3)} against a recorded baseline of ` +
+        `${input.baseline.toFixed(3)} (tolerance ${input.tolerance.toFixed(3)}) on prompt ` +
+        `version ${input.promptVersion}; regressed: ${input.regressed.join(', ')}`,
+      // `x test eval --filter <name>`, never `x test <name>`: `x test`'s positional is a TestType,
+      // so the eval's own name there is `X_CLI_BAD_FLAG` ("not a test type") — a fix line that
+      // cannot be run is axiom 4 broken at the one moment it is needed.
+      fix: `x test eval --filter ${input.eval} to see per-case scores, then fix the prompt — or ULTIMATE_EVAL_RECORD=1 x test eval to accept the new numbers as a reviewed diff`,
+      docs: docsFor('X_EVAL_THRESHOLD'),
+    });
+  }
+}
+
+/**
+ * An eval declared a baseline that has never been recorded. Not a pass: an eval with nothing to
+ * compare against gates on nothing, and a step that cannot fail is a step that is not running.
+ */
+export class EvalBaselineMissingError extends UltimateError {
+  constructor(input: { eval: string; path: string; reason: string; fix?: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_MISSING',
+      cause: `eval "${input.eval}" gates against ${input.path}, which ${input.reason}`,
+      fix: input.fix ?? `ULTIMATE_EVAL_RECORD=1 x test eval, then commit ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_MISSING'),
+    });
+  }
+}
+
+/** A recorded baseline that cannot be read. Never treated as absent — that would erase a gate. */
+export class EvalBaselineInvalidError extends UltimateError {
+  constructor(input: { path: string; problem: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_INVALID',
+      cause: `the recorded baseline ${input.path} ${input.problem}`,
+      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_INVALID'),
+    });
+  }
+}
+
+/**
+ * A registered prompt that no eval names. An unevaluated prompt is untested code that costs
+ * money and answers users, so the gate fails on it exactly like an untyped module.
+ */
+export class EvalMissingError extends UltimateError {
+  constructor(input: { prompt: string; id: string }) {
+    super({
+      code: 'X_EVAL_MISSING',
+      cause: `prompt "${input.prompt}" has no eval`,
+      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
+      docs: docsFor('X_EVAL_MISSING'),
+    });
+  }
+}
+
+/**
+ * The gate ran with baseline recording switched on. Recording makes every eval write the numbers
+ * it just measured and pass, so a `x verify` that inherited the flag reports green over scores
+ * nothing compared — and rewrites the committed baselines on its way through, which is the half
+ * a red step alone would not undo. Recording is a deliberate, reviewable diff, never a gate run.
+ */
+export class EvalRecordingError extends UltimateError {
+  constructor(input: { env: string }) {
+    super({
+      code: 'X_EVAL_RECORDING',
+      cause: `${input.env} is set, so every eval would re-record its baseline instead of gating on it`,
+      fix: `env -u ${input.env} x verify`,
+      docs: docsFor('X_EVAL_RECORDING'),
+    });
+  }
+}

package/src/evals.ts

@@ -11,7 +11,6 @@
 // Every result is filed against a prompt's content hash, so a score is always attributable
 // to an exact prompt rather than "whatever was in main that day".
 
-import { EvalBaselineMissingError, EvalThresholdError } from './errors';
 import type { EvalBaseline, Regression } from './eval-baseline';
 import {
   baselinePath,
@@ -21,6 +20,7 @@ import {
   regressionsAgainst,
   writeBaseline,
 } from './eval-baseline';
+import { EvalBaselineMissingError, EvalThresholdError } from './eval-errors';
 import type { Gateway } from './gateway';
 import type { Prompt, PromptVars } from './prompt';
 import { describePrompts } from './prompt';