@ultimat3/ai 23.0.0 → 25.0.0

package/src/eval-errors.ts

@@ -61,7 +61,9 @@ export class EvalBaselineInvalidError extends UltimateError {
     super({
       code: 'X_EVAL_BASELINE_INVALID',
       cause: `the recorded baseline ${input.path} ${input.problem}`,
-      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
+      // The command is the record run; the path is named behind the `#`, where it is read and
+      // never run — `x test eval` takes no path, and a file name may carry `$(…)` (sweep 1c).
+      fix: `ULTIMATE_EVAL_RECORD=1 x test eval   # re-records ${input.path}`,
     });
   }
 }

package/src/evals.ts

@@ -187,7 +187,7 @@ async function runEval<V extends PromptVars>(
       ...(input.prompt.system !== undefined ? { system: input.prompt.system } : {}),
       ...(input.prompt.model !== undefined ? { model: input.prompt.model } : {}),
       ...(input.prompt.effort !== undefined ? { effort: input.prompt.effort } : {}),
-      // `thinking` too, and not because it is tidy: `contentHash` covers it, so the baseline is
+      // `thinking` too, and not because it is tidy: `promptHash` covers it, so the baseline is
       // filed under a hash describing a configuration the measurement did not use. `llm()` and
       // `agent()` have always sent both.
       ...(input.prompt.thinking !== undefined ? { thinking: input.prompt.thinking } : {}),

package/src/failure-disclosure.ts

@@ -0,0 +1,90 @@
+// The one rule for what of a thrown value a remote reader may see: an LLM tool result and a hive
+// member's reason both reach someone outside this process, so both ask this file — and the half
+// it withholds goes to the log as content-free facts (code, name, frames) — never its text.
+
+import type { Logger } from '@ultimat3/core';
+import { hasPublicCause, isUltimateError, stringField } from '@ultimat3/core';
+import { statusFor } from '@ultimat3/http';
+
+/**
+ * What may be shown. `code` is absent for an uncoded throw — a foreign `.message` is the server's
+ * (a pg `Key (email)=(…)`), so nothing of it is carried. `cause` is absent when it is withheld:
+ * the caller substitutes its own fixed sentence, because who is reading decides the wording.
+ */
+export interface DisclosedFailure {
+  readonly code: string | undefined;
+  readonly cause: string | undefined;
+  readonly fix: string | undefined;
+}
+
+/**
+ * Read STRUCTURALLY and totally — `stringField`, never `typeof e.code` or `.message`: the value is
+ * whatever an app's handler, its driver or its SDK threw, so each read is a getter call or a
+ * `Proxy` trap, and this runs inside a catch block with nothing left to answer if a probe raises.
+ *
+ * A 5xx code's cause is shown only when core's `hasPublicCause` allows it — the rule a production
+ * problem document follows (`@ultimat3/http`'s `toProblem`). Hidden, the developer's `fix` goes
+ * with it (a driver writes it from the statement it failed on); a `callerFix` an `UltimateError`
+ * built in this process is authored for a remote reader and stays.
+ */
+export function discloseFailure(
+  error: unknown,
+  logger: Logger | undefined,
+  where: string,
+): DisclosedFailure {
+  const code = stringField(error, 'code');
+  if (code === undefined) return withheld(error, logger, where, undefined, undefined);
+  if (statusFor(code) >= 500 && !hasPublicCause(code)) {
+    const callerFix = isUltimateError(error) ? stringField(error, 'callerFix') : undefined;
+    return withheld(error, logger, where, code, callerFix);
+  }
+  const fix = stringField(error, 'fix');
+  return {
+    code,
+    cause: stringField(error, 'cause') ?? 'unknown',
+    fix: fix === undefined || fix === '' ? undefined : fix,
+  };
+}
+
+/**
+ * A stack's FRAMES and nothing else: every line that is a frame (`    at …`), none that is not.
+ * The message is the first line — and, when it spans several, the lines after it until the first
+ * frame — so keeping only frame-shaped lines drops all of it, whatever it held. A frame is code
+ * position, which carries no caller data.
+ */
+const FRAME = /^\s+at\s/;
+
+function framesOf(error: unknown): string | null {
+  const stack = stringField(error, 'stack');
+  if (stack === undefined) return null;
+  const lines = stack.split('\n');
+  // Skip to the first frame line that FOLLOWS the message: a message may itself contain text
+  // shaped like a frame, and only lines after the message block are the stack.
+  const message = stringField(error, 'message') ?? '';
+  const skip = message === '' ? 1 : message.split('\n').length;
+  const frames = lines.slice(skip).filter((line) => FRAME.test(line));
+  return frames.length === 0 ? null : frames.join('\n');
+}
+
+/**
+ * The withheld half, logged as FACTS ONLY — the code, the error's name, and its frames. Never the
+ * message and never the cause: the logger redacts by field NAME, not by content, so a pg
+ * `Key (email)=(ceo@corp.com)` logged as text reached the logs whole. What identifies the failure
+ * is the code and where it was thrown, both of which survive this.
+ */
+function withheld(
+  error: unknown,
+  logger: Logger | undefined,
+  where: string,
+  code: string | undefined,
+  fix: string | undefined,
+): DisclosedFailure {
+  logger?.error('ai.failure.withheld', {
+    where,
+    code: code ?? 'uncoded',
+    // `name`, read structurally: what kind of throw it was (`Error`, `TypeError`, a driver's own).
+    name: stringField(error, 'name') ?? typeof error,
+    frames: framesOf(error),
+  });
+  return { code, cause: undefined, fix };
+}

package/src/gateway.ts

@@ -7,18 +7,26 @@
 import type { Random } from '@ultimat3/core';
 import {
   backoffDelay,
+  fingerprint,
   finiteCount,
   isRetryableStatus,
   isUltimateError,
+  logger,
   renderThrowable,
   stringField,
 } from '@ultimat3/core';
 import type { Money } from '@ultimat3/money';
-import type { BudgetLimits, BudgetStore } from './budget';
-import { BudgetLedger, currentBudget, estimateSpend, withBudget } from './budget';
+import type { BudgetKeys, BudgetLimits, BudgetStore } from './budget';
+import {
+  assertFiniteLimits,
+  BudgetLedger,
+  currentBudget,
+  estimateSpend,
+  withBudget,
+} from './budget';
 import { AiProviderUnavailableError } from './errors';
+import { resolveModel } from './model-resolve';
 import type { ModelId } from './models';
-import { DEFAULT_MODEL } from './models';
 import type { GenerateRequest, GenerateResult, Provider, StreamChunk } from './provider';
 
 /**
@@ -32,7 +40,7 @@ export interface GatewayCache {
 
 /**
  * A gateway's retry budget. NOT core's `RetryPolicy`, and deliberately still its own declaration:
- * these three field names are what an app writes in `createGateway({ retry })`, so renaming them
+ * these three field names are what an app writes in `providerGateway({ retry })`, so renaming them
  * onto `base`/`max`/`jitter` would break every caller for no behaviour. What WAS a duplicate is the
  * arithmetic, and that is gone — `backoffMs` is core's `backoffDelay` with this shape mapped onto
  * it, so there is one curve in the framework and one place a jitter bug can live.
@@ -45,12 +53,16 @@ export interface RetryPolicy {
   readonly maxDelayMs: number;
 }
 
-export const DEFAULT_RETRY: RetryPolicy = { attempts: 3, baseDelayMs: 500, maxDelayMs: 8_000 };
+export const DEFAULT_GATEWAY_RETRY: RetryPolicy = {
+  attempts: 3,
+  baseDelayMs: 500,
+  maxDelayMs: 8_000,
+};
 
 /** The one code `attempt` may collect into. Every other coded refusal is the caller's answer. */
 const PROVIDER_UNAVAILABLE = 'X_AI_PROVIDER_UNAVAILABLE';
 
-export interface CreateGatewayInput {
+export interface ProviderGatewayInput {
   /** Tried in order for a given model. First provider that lists the model wins. */
   readonly providers: readonly Provider[];
   readonly budget?: BudgetLimits;
@@ -70,6 +82,13 @@ export interface CreateGatewayInput {
 }
 
 export interface Gateway {
+  /**
+   * The model a call with none declared runs on — `providerGateway({ defaultModel })`. Read by
+   * `llm()` and `agent()` after the declaration and its prompt, so the app's choice is the one
+   * fallback; absent, a call naming no model is refused (`X_AI_MODEL_UNRESOLVED`) — never routed to
+   * a vendor the framework picked.
+   */
+  readonly defaultModel?: ModelId | undefined;
   generate(request: GenerateRequest): Promise<GenerateResult>;
   stream(request: GenerateRequest): AsyncIterable<StreamChunk>;
   /** Open a budget scope. Nested gateway calls inside `fn` share one ledger. */
@@ -78,59 +97,63 @@ export interface Gateway {
   spent(): Promise<Money>;
   /**
    * A ledger over this gateway's own `budget`, for a call made with no `scope()` open — which is
-   * every call `llm()` and `agent()` make. Optional so a hand-written gateway still satisfies the
-   * interface; absent, such a call runs under the ceilings its own declaration sets and no others.
+   * every call `llm()`, `agent()` and `hive()` make, each passing `budgetKeysFor(ctx.actor)` so the
+   * `actor` and `org` ceilings count the caller. Required: a gateway without it ran every such call
+   * under no gateway ceiling at all.
    */
-  callLedger?(): BudgetLedger;
+  callLedger(keys: BudgetKeys): BudgetLedger;
 }
 
-export function createGateway(input: CreateGatewayInput): Gateway {
+export function providerGateway(input: ProviderGatewayInput): Gateway {
   return new GatewayImpl(input);
 }
 
 class GatewayImpl implements Gateway {
-  private readonly config: CreateGatewayInput;
+  private readonly config: ProviderGatewayInput;
   private readonly retry: RetryPolicy;
   private readonly sleep: (ms: number) => Promise<void>;
   /** Left `undefined` rather than defaulted, so `backoffDelay` owns the one fallback to `Math.random`. */
   private readonly random: Random | undefined;
 
-  constructor(config: CreateGatewayInput) {
+  get defaultModel(): ModelId | undefined {
+    return this.config.defaultModel;
+  }
+
+  constructor(config: ProviderGatewayInput) {
     this.config = config;
-    this.retry = config.retry ?? DEFAULT_RETRY;
+    this.retry = config.retry ?? DEFAULT_GATEWAY_RETRY;
     // `attempts` is the retry loop's only exit condition and nothing screened it: `attempt <= NaN`
     // is false on the first comparison, so `attempt()` calls no provider at all and raises
     // `X_AI_PROVIDER_UNAVAILABLE` with an EMPTY attempt list — measured, "no provider could serve
-    // model claude-opus-5 ()" for a provider that was never asked. A floor of 1 because the field
+    // model <id> ()" for a provider that was never asked. A floor of 1 because the field
     // is documented as total attempts INCLUDING the first, so zero of them is not a policy.
     // `baseDelayMs` and `maxDelayMs` are deliberately not screened here: `backoffDelay` refuses a
     // non-finite one already, and a NEGATIVE base is clamped to a zero wait on purpose
     // (`gateway-backoff.test.ts` pins it), which a count check here would start refusing.
-    finiteCount('createGateway', 'retry.attempts', this.retry.attempts, 1);
+    finiteCount('gateway', 'retry.attempts', this.retry.attempts, 1);
+    // Screened here, under the config's own key, rather than by the ledger `callLedger` builds:
+    // that refused on the first CALL and named `the AI budget`, not the `gateway` to edit.
+    if (config.budget !== undefined) assertFiniteLimits(config.budget, 'gateway', 'budget.');
     this.sleep = config.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
     this.random = config.random;
   }
 
   /**
    * `budget` held with no scope open. It was enforced ONLY inside `scope()`, which nothing in the
-   * framework calls — so `createGateway({ budget: { request: 100 } })` capped no call any app made.
-   * No actor or org key here: those scopes need `scope()` to name whose window they are.
+   * framework calls — so `providerGateway({ budget: { request: 100 } })` capped no call any app made.
+   * A raw `generate` / `stream` outside any scope passes no keys: it has no caller to count.
    */
-  callLedger(): BudgetLedger {
+  callLedger(keys: BudgetKeys): BudgetLedger {
     return new BudgetLedger({
       limits: this.config.budget ?? {},
+      ...(keys.actorKey !== undefined ? { actorKey: keys.actorKey } : {}),
+      ...(keys.orgKey !== undefined ? { orgKey: keys.orgKey } : {}),
       ...(this.config.budgetStore !== undefined ? { store: this.config.budgetStore } : {}),
     });
   }
 
-  scope<T>(input: { actorKey?: string; orgKey?: string }, fn: () => Promise<T>): Promise<T> {
-    const ledger = new BudgetLedger({
-      limits: this.config.budget ?? {},
-      ...(input.actorKey !== undefined ? { actorKey: input.actorKey } : {}),
-      ...(input.orgKey !== undefined ? { orgKey: input.orgKey } : {}),
-      ...(this.config.budgetStore !== undefined ? { store: this.config.budgetStore } : {}),
-    });
-    return withBudget(ledger, fn);
+  scope<T>(input: BudgetKeys, fn: () => Promise<T>): Promise<T> {
+    return withBudget(this.callLedger(input), fn);
   }
 
   async spent(): Promise<Money> {
@@ -140,10 +163,10 @@ class GatewayImpl implements Gateway {
   }
 
   async generate(request: GenerateRequest): Promise<GenerateResult> {
-    const model = request.model ?? this.config.defaultModel ?? DEFAULT_MODEL;
+    const model = resolveModel('gateway', request.model, this.config.defaultModel);
     const resolved: GenerateRequest = { ...request, model, maxTokens: ceilingOf(request) };
 
-    const cacheKey = cacheKeyFor(resolved);
+    const cacheKey = promptCacheKey(resolved);
     const cached = await this.config.cache?.get(cacheKey);
     if (cached !== undefined) {
       // A cache hit costs nothing, so it is deliberately NOT debited from the budget.
@@ -153,7 +176,7 @@ class GatewayImpl implements Gateway {
     // Reserve against the ESTIMATE before spending anything — tokens AND money, since a
     // cheap-in-tokens call on an expensive model is still a cost cap the app declared.
     // `record` below replaces the estimate with the provider's real counts.
-    const ledger = currentBudget() ?? this.callLedger();
+    const ledger = currentBudget() ?? this.callLedger({});
     // The estimate is DEBITED here, not merely checked: three concurrent calls under one ledger
     // all read the same `spent()` otherwise, all pass, and all three record against a ceiling
     // only one of them fitted.
@@ -170,14 +193,24 @@ class GatewayImpl implements Gateway {
     await ledger?.record(result.usage, result.cost, reservation);
     // A refusal is not an answer, so it is not cached. Storing one would keep serving a decision
     // the classifier might not make twice, long after the prompt that provoked it was fixed.
-    if (result.stopReason !== 'refusal') {
+    if (result.stopReason !== 'refusal') await this.remember(cacheKey, result);
+    return result;
+  }
+
+  /**
+   * The cache write, AFTER the call was paid for and recorded. A throw here must not turn that
+   * answer into a failure: the caller would retry and pay for it twice. Logged, and the answer kept.
+   */
+  private async remember(cacheKey: string, result: GenerateResult): Promise<void> {
+    try {
       await this.config.cache?.set(cacheKey, JSON.stringify(result));
+    } catch (error) {
+      logger.warn('ai.cache.write_failed', { cacheKey, error: renderThrowable(error) });
     }
-    return result;
   }
 
   async *stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
-    const model = request.model ?? this.config.defaultModel ?? DEFAULT_MODEL;
+    const model = resolveModel('gateway', request.model, this.config.defaultModel);
     const resolved: GenerateRequest = { ...request, model, maxTokens: ceilingOf(request) };
 
     // Routed BEFORE the reservation, not after it. `providerFor` throws for a registered model no
@@ -196,7 +229,7 @@ class GatewayImpl implements Gateway {
     // `generate`, or reconnects itself and knows what it has already shown.
     const provider = this.providerFor(model);
 
-    const ledger = currentBudget() ?? this.callLedger();
+    const ledger = currentBudget() ?? this.callLedger({});
     const reservation = await ledger?.reserve(estimateSpend(resolved));
     let settled = false;
     try {
@@ -368,17 +401,20 @@ export function isRetryable(error: unknown): boolean {
 
 /**
  * Cache key. Every field that changes the answer is in it — a key that ignores `effort` or
- * `system` would serve one prompt's answer for another.
+ * `system` would serve one prompt's answer for another — and each tool WHOLE: every `llm()` tool
+ * is named `respond`, so a key over names alone served an answer shaped for an old `output` schema
+ * after the schema changed. Core's `fingerprint`, so key order never splits one request in two and
+ * the width is the one every other sharing key in the framework has.
  */
-export function cacheKeyFor(request: GenerateRequest): string {
-  return JSON.stringify({
+export function promptCacheKey(request: GenerateRequest): string {
+  return fingerprint({
     model: request.model,
     system: request.system ?? '',
     messages: request.messages,
     maxTokens: request.maxTokens,
     effort: request.effort ?? 'high',
     thinking: request.thinking ?? 'adaptive',
-    tools: request.tools?.map((t) => t.name) ?? [],
+    tools: request.tools ?? [],
     stop: request.stopSequences ?? [],
   });
 }

package/src/hive-pool.ts

@@ -5,7 +5,8 @@
 // happens to the siblings when one throws". Nothing here knows what a model is.
 
 import type { Ctx } from '@ultimat3/core';
-import { isThrownError, isUltimateError, stringField, withChildContext } from '@ultimat3/core';
+import { withChildContext } from '@ultimat3/core';
+import { discloseFailure } from './failure-disclosure';
 import type { HiveMember, HiveMemberError } from './hive-result';
 import { SKIPPED_ABORTED, SKIPPED_NO_INPUT } from './hive-result';
 
@@ -57,7 +58,7 @@ export async function runPool<I, O>(input: PoolInput<I, O>): Promise<readonly Hi
         );
         members[index] = { status: 'ok', index, value };
       } catch (error) {
-        members[index] = { status: 'failed', index, ...failureOf(error) };
+        members[index] = { status: 'failed', index, ...failureOf(error, ctx, index) };
         if (input.onMemberError === 'abort') controller.abort();
       }
     }
@@ -72,19 +73,26 @@ export async function runPool<I, O>(input: PoolInput<I, O>): Promise<readonly Hi
 }
 
 /**
- * What a member threw, as two data fields — never as an error's `cause:`, which is why the thrown
- * value is read structurally and never interpolated. A foreign throw gets `'unknown'` rather than
- * an invented `X_` code: a code nothing declares is a code no `x errors explain` can answer.
+ * What the caller may read of a member's throw, as two data fields — never as an error's `cause:`.
+ * The reason ships in the hive action's 200 answer, so it is `discloseFailure`'s verdict, the one
+ * the tool result gets: a 5xx cause or a foreign `.message` (a pg `Key (email)=(…)`) is withheld
+ * and logged whole. A throw with no code gets `'unknown'` rather than an invented `X_` code: a code
+ * nothing declares is a code no `x errors explain` can answer.
  */
-function failureOf(error: unknown): { readonly code: string; readonly reason: string } {
-  if (isUltimateError(error)) return { code: error.code, reason: error.cause };
-  // `isThrownError` and `stringField`, never `error instanceof Error` and `.message`: a member
-  // is an app's action, so the value is one the framework did not build — `instanceof` runs a
-  // `Proxy`'s `getPrototypeOf` trap and `.message` is a getter call. A throw HERE would take the
-  // whole hive down with it, which is the one outcome the three arms exist to prevent.
-  const message = stringField(error, 'message');
-  if (isThrownError(error) && message !== undefined && message !== '') {
-    return { code: 'unknown', reason: message };
-  }
-  return { code: 'unknown', reason: 'the member threw a value that is not an Error' };
+function failureOf(
+  error: unknown,
+  ctx: Ctx,
+  index: number,
+): { readonly code: string; readonly reason: string } {
+  // `fix` is not carried: `reason` has always been the cause alone, and a member's caller acts on
+  // the code. What `discloseFailure` withholds stays withheld either way.
+  const shown = discloseFailure(error, ctx.logger, `hive member ${index}`);
+  return { code: shown.code ?? 'unknown', reason: shown.cause ?? HIDDEN_MEMBER_CAUSE };
 }
+
+/**
+ * What a caller reads in place of a reason it may not see. Fixed, so nothing of the throw rides on
+ * it — the hive's answer goes wherever its caller sends it, a model's context included.
+ */
+export const HIDDEN_MEMBER_CAUSE =
+  'the member failed inside the server; the details are withheld from this answer; the server log records its code and where it failed';

package/src/hive-result.ts

@@ -23,7 +23,10 @@ export type HiveMember<O> =
   | {
       readonly status: 'failed';
       readonly index: number;
-      /** The `UltimateError` code the member threw, or `'unknown'` for a foreign throw. */
+      /**
+       * The code the member threw — read structurally, so a foreign throw carrying a string `code`
+       * (a driver's SQLSTATE) reports it — or `'unknown'` for a throw with none.
+       */
       readonly code: string;
       readonly reason: string;
     }

package/src/hive.ts

@@ -5,8 +5,8 @@
  * `@ultimat3/core`, and it is a list rather than a sentence because the ordinal a file writes for
  * itself is wrong the day the next one lands and no file can see the others. A fan-out is still
  * one server-authoritative operation with an input schema, an output schema and a policy, so this
- * returns an `action` and inherits `.tool()`, `.openapi()`, `.client()`, `.job()`, `.contract()`
- * and its manifest row without a line here.
+ * returns an `action` and inherits `.openapi()`, `.client()`, `.job()`, `.contract()`, its MCP
+ * tool and its manifest row without a line here.
  *
  * It exists because the alternative is a hand-rolled `Promise.all` over `agent()` calls, and that
  * loop gets four things wrong every time: it takes the actor from somewhere other than the request,
@@ -20,12 +20,13 @@ import type { Ctx } from '@ultimat3/core';
 import { finiteCount, throwIfAborted, withSpan } from '@ultimat3/core';
 import type { AnySchema, InferInput, InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
 import type { BudgetLimits } from './budget';
-import { BudgetLedger, currentBudget, withBudget } from './budget';
+import { BudgetLedger, budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget';
 import { HiveEmptyError } from './hive-errors';
 import { runPool } from './hive-pool';
 import type { HiveMember, HiveMemberError, HiveOutput, HiveResult } from './hive-result';
 import { hiveResultSchema } from './hive-result';
 import type { LlmBudget } from './llm';
+import { installedGateway } from './runtime';
 
 /**
  * Members in flight at once when the declaration omits one. Small and deliberately arbitrary — it
@@ -131,7 +132,14 @@ async function run<
       'hive.concurrency': width,
       'hive.on_member_error': def.onMemberError,
     });
-    const ledger = (currentBudget() ?? new BudgetLedger({ limits: {} })).derive(limitsOf(def));
+    // Rooted as `llm()` and `agent()` root: the gateway's ceilings, keyed on the caller. An empty
+    // root dropped every gateway ceiling for every member. No gateway installed is a hive of plain
+    // actions, which spends nothing — a model member is refused `X_AI_GATEWAY_MISSING` on its own.
+    const root =
+      currentBudget() ??
+      installedGateway()?.callLedger(budgetKeysFor(args.ctx.actor)) ??
+      new BudgetLedger({ limits: {} });
+    const ledger = root.derive(limitsOf(def));
     const result = await withBudget(ledger, () =>
       runPool<InferInput<MIn>, InferOutput<MOut>>({
         inputs,
@@ -183,7 +191,9 @@ function limitsOf<
     ...(budget?.tokensIn === undefined
       ? {}
       : { tokensIn: finiteCount(SUBJECT, 'budget.tokensIn', budget.tokensIn) }),
-    ...(budget?.costPerCall === undefined ? {} : { costPerCall: budget.costPerCall }),
+    ...(budget?.costPerCall === undefined
+      ? {}
+      : { costPerCall: screenedMoney(SUBJECT, 'budget.costPerCall', budget.costPerCall) }),
     // The ledger's `request` scope accumulates across every call made under it, which for a hive
     // under `withBudget` is every member's every turn.
     ...(budget?.tokensPerRun === undefined