@ultimat3/ai 29.4.0 → 30.1.0

package/src/gateway.ts

@@ -4,17 +4,8 @@
 // budgeted, and cost-accounted in integer minor units. Everything an app does with a model
 // goes through here, so budgets and accounting cannot be bypassed by a stray fetch.
 
-import type { Random } from '@ultimat3/core';
-import {
-  backoffDelay,
-  fingerprint,
-  finiteCount,
-  isRetryableStatus,
-  isUltimateError,
-  logger,
-  renderThrowable,
-  stringField,
-} from '@ultimat3/core';
+import type { Breaker, Random } from '@ultimat3/core';
+import { fingerprint, finiteCount, logger, renderThrowable } from '@ultimat3/core';
 import type { Money } from '@ultimat3/money';
 import { billedOf } from './billed';
 import type { BudgetKeys, BudgetLimits, BudgetReservation, BudgetStore } from './budget';
@@ -25,11 +16,18 @@ import {
   estimateSpend,
   withBudget,
 } from './budget';
-import { AiProviderUnavailableError } from './errors';
+import type { GatewayBreakerOption } from './gateway-providers';
+import {
+  admittedProvider,
+  attemptProviders,
+  providerBreakers,
+  streamVerdict,
+} from './gateway-providers';
 import { resolveModel } from './model-resolve';
 import type { ModelId } from './models';
 import type { GenerateRequest, GenerateResult, Provider, StreamChunk } from './provider';
 import type { SamplingWire } from './sampling';
+import { ZERO_USAGE } from './wire';
 
 /**
  * Response cache. Structurally satisfied by `@ultimat3/cache`'s memo/LRU tiers; declared as
@@ -61,8 +59,8 @@ export const DEFAULT_GATEWAY_RETRY: RetryPolicy = {
   maxDelayMs: 8_000,
 };
 
-/** The one code `attempt` may collect into. Every other coded refusal is the caller's answer. */
-const PROVIDER_UNAVAILABLE = 'X_AI_PROVIDER_UNAVAILABLE';
+export type { GatewayBreakerOption } from './gateway-providers';
+export { backoffMs, DEFAULT_GATEWAY_BREAKER, isRetryable } from './gateway-providers';
 
 export interface ProviderGatewayInput {
   /** Tried in order for a given model. First provider that lists the model wins. */
@@ -81,6 +79,12 @@ export interface ProviderGatewayInput {
    * green. Production never passes one.
    */
   readonly random?: Random | undefined;
+  /**
+   * One breaker per provider, on by default (`{ failures: 5, cooldownMs: 30_000 }`): a provider
+   * whose retries ran out five passes running is skipped — no call, no sleep — until one probe
+   * after the cooldown answers. Per process. `(provider) => breaker({ … })` to choose your own, `'off'` declines.
+   */
+  readonly breaker?: GatewayBreakerOption | undefined;
 }
 
 export interface Gateway {
@@ -124,6 +128,7 @@ class GatewayImpl implements Gateway {
   private readonly sleep: (ms: number) => Promise<void>;
   /** Left `undefined` rather than defaulted, so `backoffDelay` owns the one fallback to `Math.random`. */
   private readonly random: Random | undefined;
+  private readonly gateOf: (provider: Provider) => Breaker | undefined;
 
   get defaultModel(): ModelId | undefined {
     return this.config.defaultModel;
@@ -146,6 +151,7 @@ class GatewayImpl implements Gateway {
     if (config.budget !== undefined) assertFiniteLimits(config.budget, 'gateway', 'budget.');
     this.sleep = config.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
     this.random = config.random;
+    this.gateOf = providerBreakers(config.breaker);
   }
 
   /**
@@ -179,8 +185,9 @@ class GatewayImpl implements Gateway {
     const cacheKey = promptCacheKey(resolved);
     const cached = await this.config.cache?.get(cacheKey);
     if (cached !== undefined) {
-      // A cache hit costs nothing, so it is deliberately NOT debited from the budget.
-      return JSON.parse(cached) as GenerateResult;
+      // A cache hit costs nothing, so it is deliberately NOT debited from the budget, and says so:
+      // carrying the stored figures, a meter summing the run billed it a second time.
+      return freeOf(JSON.parse(cached) as GenerateResult);
     }
 
     // Reserve against the ESTIMATE before spending anything — tokens AND money, since a
@@ -196,7 +203,15 @@ class GatewayImpl implements Gateway {
 
     let result: GenerateResult;
     try {
-      result = await this.attempt(model, (provider) => provider.generate(resolved));
+      result = await attemptProviders({
+        model,
+        candidates: this.config.providers.filter((p) => p.models.includes(model)),
+        retry: this.retry,
+        sleep: this.sleep,
+        random: this.random,
+        gateOf: this.gateOf,
+        call: (provider) => provider.generate(resolved),
+      });
     } catch (error) {
       // A call that never landed must not go on holding its reservation — and one that DID land
       // and still failed (`billedOf`: the provider answered with something unusable) is recorded
@@ -251,15 +266,28 @@ class GatewayImpl implements Gateway {
     // single provider serving this model and that call stands or throws; `attempt`'s backoff and
     // its fallback across providers belong to `generate` alone. A caller that wants either uses
     // `generate`, or reconnects itself and knows what it has already shown.
-    const provider = this.providerFor(model);
+    const { provider, ticket } = admittedProvider(model, this.config.providers, this.gateOf);
+    const gate = this.gateOf(provider);
+    let answered = false;
 
     const ledger = currentBudget() ?? this.callLedger({});
-    const reservation = await ledger?.reserve(
-      estimateSpend(resolved, this.marksPromptCache(model)),
-    );
+    let reservation: BudgetReservation | undefined;
+    try {
+      reservation = await ledger?.reserve(estimateSpend(resolved, this.marksPromptCache(model)));
+    } catch (error) {
+      // A refused reservation sent nothing: the half-open probe `admittedProvider` took goes back,
+      // or the next healthy call would wait out a whole extra cooldown.
+      gate?.released(ticket);
+      throw error;
+    }
     let settled = false;
     try {
       for await (const chunk of provider.stream(resolved)) {
+        // The first chunk is the provider's verdict for its breaker: it answered.
+        if (!answered) {
+          answered = true;
+          streamVerdict(gate, ticket, undefined);
+        }
         if (chunk.type !== 'done') {
           yield chunk;
           continue;
@@ -272,6 +300,10 @@ class GatewayImpl implements Gateway {
         yield { type: 'done', result: { ...chunk.result, provider: provider.name } };
       }
     } catch (error) {
+      if (!answered) {
+        answered = true;
+        streamVerdict(gate, ticket, error);
+      }
       // Thrown after the provider reported usage: the bill is recorded, exactly as `generate` does.
       // Only when nothing settled yet — a provider that throws AFTER its `done` chunk was already
       // recorded, and releasing again would credit the ledger a second time.
@@ -284,6 +316,8 @@ class GatewayImpl implements Gateway {
       // A stream its consumer abandoned never reached `done` — so nothing reconciled the
       // reservation and it would hold the ceiling for the rest of the window.
       if (!settled) await ledger?.release(reservation);
+      // Abandoned before a chunk: the probe goes back, counting nothing.
+      if (!answered) gate?.released(ticket);
     }
   }
 
@@ -299,73 +333,6 @@ class GatewayImpl implements Gateway {
       (p) => p.models.includes(model) && p.marksPromptCache?.(model) === true,
     );
   }
-
-  private providerFor(model: ModelId): Provider {
-    const provider = this.config.providers.find((p) => p.models.includes(model));
-    if (provider === undefined) {
-      throw new AiProviderUnavailableError({
-        model,
-        attempts: this.config.providers.map((p) => `${p.name}: does not serve ${model}`),
-        unserved: true,
-      });
-    }
-    return provider;
-  }
-
-  /**
-   * Try every provider that serves `model`, retrying each on a retryable failure with
-   * exponential backoff plus full jitter (jitter matters: synchronised retries from N
-   * workers reproduce the rate limit they are backing off from).
-   *
-   * Fallback here is across PROVIDERS serving one model, never across models — a silent model
-   * swap changes what answered, what it cost and which eval baseline the answer belongs to. The
-   * provider that did answer is stamped onto the result, so the fallback that DOES exist reaches
-   * the span instead of being invisible.
-   */
-  private async attempt(
-    model: ModelId,
-    call: (provider: Provider) => Promise<GenerateResult>,
-  ): Promise<GenerateResult> {
-    const candidates = this.config.providers.filter((p) => p.models.includes(model));
-    const failures: string[] = [];
-    if (candidates.length === 0) {
-      throw new AiProviderUnavailableError({
-        model,
-        attempts: [`no provider serves ${model}`],
-        unserved: true,
-      });
-    }
-
-    for (const provider of candidates) {
-      for (let attempt = 1; attempt <= this.retry.attempts; attempt += 1) {
-        try {
-          return { ...(await call(provider)), provider: provider.name };
-        } catch (error) {
-          // A coded refusal that is NOT `X_AI_PROVIDER_UNAVAILABLE` reaches the caller verbatim.
-          // Those are raised locally, before the socket opens — a missing credential, a control
-          // the model does not accept — so the same rejection is waiting on every provider and
-          // every attempt, which is the reason a 400 is not retried three lines below. Collecting
-          // one into `X_AI_PROVIDER_UNAVAILABLE` discards its runnable `fix:` and answers the same
-          // failure a different way from `stream`, which does not route through here at all.
-          // A transport failure keeps the old path: `AiTransportError` IS
-          // `X_AI_PROVIDER_UNAVAILABLE`, so "provider one 503'd, provider two timed out" still
-          // collects across the candidates. `stringField` rather than `error.code`, because the
-          // value came from an app's `Provider` and a property read on it can trap.
-          if (isUltimateError(error) && stringField(error, 'code') !== PROVIDER_UNAVAILABLE) {
-            throw error;
-          }
-          // `renderThrowable`, never `error.message` or `String(error)`: this line becomes the
-          // `cause` of `X_AI_PROVIDER_UNAVAILABLE`, and a renderer that throws replaces the coded
-          // refusal with a `TypeError` nothing downstream can catch by code. It bounds the text
-          // too — a provider's 1MB body is not a cause.
-          failures.push(`${provider.name}#${attempt}: ${renderThrowable(error)}`);
-          if (!isRetryable(error) || attempt === this.retry.attempts) break;
-          await this.sleep(backoffMs(this.retry, attempt, this.random));
-        }
-      }
-    }
-    throw new AiProviderUnavailableError({ model, attempts: failures });
-  }
 }
 
 /** A failed call's reservation: reconciled against its bill when the failure carries one, else given back. */
@@ -379,6 +346,14 @@ async function settleFailure(
   await ledger?.record(billed.usage, billed.cost, reservation);
 }
 
+/** The stored answer as a hit delivers it: no token spent, no money, in the currency it named. */
+const freeOf = (stored: GenerateResult): GenerateResult => ({
+  ...stored,
+  usage: ZERO_USAGE,
+  cost: { ...stored.cost, minor: 0 },
+  ...(stored.costExact === undefined ? {} : { costExact: { ...stored.costExact, minor: 0 } }),
+});
+
 /**
  * The request's completion ceiling, screened at the one seam every model call in an app passes
  * through — `llm()`, `agent()`, an eval, a judge and a hand-built `generate()` alike.
@@ -394,71 +369,6 @@ function ceilingOf(request: GenerateRequest): number {
   return finiteCount('the AI gateway', 'maxTokens', request.maxTokens, 1);
 }
 
-/**
- * Full jitter: a uniform pick from [0, exponential], capped BEFORE the roll.
- *
- * The arithmetic is core's `backoffDelay` — this is the mapping from the gateway's own field names
- * onto it, and nothing else. Two things came with the delegation and neither is cosmetic: the
- * result is ROUNDED where this floored it (a shift of at most 1ms, and the same rounding
- * `@ultimat3/jobs` and `@ultimat3/realtime` already use), and a policy carrying a `NaN` — which is
- * what `Number(process.env.…)` answers for an unset variable — is REFUSED rather than clamped.
- * This paragraph said "waits 0" until 2026-08-26, which was the safe answer to the wrong question
- * and had already stopped being true: a schedule of zeroes still spins, it just spins on purpose,
- * so core's `backoffDelay` refuses a non-finite bound before it clamps and this gateway inherits
- * the refusal with the curve. `gateway-backoff.test.ts` pins both halves — the refusal, and the
- * negative base that IS still clamped to zero.
- */
-export function backoffMs(policy: RetryPolicy, attempt: number, random?: Random): number {
-  return backoffDelay({
-    attempt,
-    base: policy.baseDelayMs,
-    max: policy.maxDelayMs,
-    curve: 'exponential',
-    jitter: 'full',
-    random,
-  });
-}
-
-/**
- * Retryable = the request was well formed and the provider was momentarily unable. A 400 is
- * never retried: the same body produces the same rejection and only burns the budget.
- *
- * The status half is core's `isRetryableStatus`, which is WIDER than the `429 || >= 500` this
- * gateway shipped: 408, 409 and 425 join it. Each is transient by construction — the server gave up
- * waiting for a body it never fully read, a concurrent writer won the round, the handshake was not
- * finished — so not retrying them was a gap rather than a policy, and one narrower table in one
- * package was how it stayed invisible. The `code` branch has no equivalent in core and stays here:
- * that table is HTTP status only, and a socket that timed out never produced one.
- */
-/**
- * A transport that never produced a response. Node spells these `ETIMEDOUT`/`ECONNRESET`; Bun's
- * `fetch` spells a refused, dropped or unopenable connection `ConnectionRefused`,
- * `ConnectionClosed` and `FailedToOpenSocket` — which this list did not know, so on the one runtime
- * the framework runs on a provider mid-restart was never retried.
- */
-const RETRYABLE_TRANSPORT_CODES: ReadonlySet<string> = new Set([
-  'ETIMEDOUT',
-  'ECONNRESET',
-  'ConnectionRefused',
-  'ConnectionClosed',
-  'FailedToOpenSocket',
-]);
-
-export function isRetryable(error: unknown): boolean {
-  if (typeof error !== 'object' || error === null) return false;
-  // A `Provider` is the APP's object, so the value it rejected with is one the framework did not
-  // build: `e.status` is a getter call and, on a `Proxy`, a trap. A value that fights being read
-  // cannot be SHOWN to be retryable, and this runs inside the catch block that has nothing left
-  // to answer with — so it fails closed rather than raising.
-  try {
-    const e = error as { status?: unknown; code?: unknown };
-    if (typeof e.status === 'number') return isRetryableStatus(e.status);
-    return typeof e.code === 'string' && RETRYABLE_TRANSPORT_CODES.has(e.code);
-  } catch {
-    return false;
-  }
-}
-
 /**
  * Cache key. Every field that changes the answer is in it — a key that ignores `effort` or
  * `system` would serve one prompt's answer for another — and each tool WHOLE: every `llm()` tool

package/src/hive-errors.ts

@@ -1,6 +1,6 @@
 // The X_HIVE_* codes, apart from ./errors only because one file has one job and that catalogue is
 // already at its ceiling — the same split `eval-errors.ts` made. The codes, their titles and the
-// single `registerErrorCodes` call stay in ./errors: one owner, one registration, one place a
+// single `registerErrorCodes` call stay in ./error-codes: one owner, one registration, one place a
 // duplicate can surface.
 
 import { UltimateError } from '@ultimat3/core';

package/src/in-process-tool.ts

@@ -60,18 +60,27 @@ export function inProcessTool<TInput extends StandardSchemaV1>(
     inputJsonSchema: toWireSchema(def.input),
     inProcess: true,
     ...(def.discloseErrors === undefined ? {} : { discloseErrors: def.discloseErrors }),
+    parseInput: (raw) => parseToolInput(def, raw),
     async run({ input, actor }) {
-      const parsed = await validateAsync(def.input, input);
-      if (parsed.issues !== undefined) {
-        // The line only: the structured issues are `@ultimat3/action`'s own shape, for a form.
-        throw new InputInvalidError(
-          def.name,
-          formatIssues(parsed.issues).join('; '),
-          undefined,
-          'tool',
-        );
-      }
-      return def.execute({ input: parsed.value, actor });
+      return def.execute({ input: await parseToolInput(def, input), actor });
     },
   };
 }
+
+/** The input parse `run` makes, alone — also what an agent eval holds this tool's double to. */
+async function parseToolInput<TInput extends StandardSchemaV1>(
+  def: InProcessToolDef<TInput>,
+  raw: unknown,
+): Promise<InferOutput<TInput>> {
+  const parsed = await validateAsync(def.input, raw);
+  if (parsed.issues !== undefined) {
+    // The line only: the structured issues are `@ultimat3/action`'s own shape, for a form.
+    throw new InputInvalidError(
+      def.name,
+      formatIssues(parsed.issues).join('; '),
+      undefined,
+      'tool',
+    );
+  }
+  return parsed.value;
+}

package/src/index.ts

@@ -8,6 +8,7 @@ export { agent } from './agent';
 export type { AgentCompaction, AgentCompactOption, AgentCompactRule } from './agent-compact-def';
 export type {
   AgentAction,
+  AgentCheckpoint,
   AgentContinuation,
   AgentRun,
   AgentTranscript,
@@ -81,6 +82,7 @@ export {
 } from './content-blocks';
 export type { ContentRefusal } from './content-errors';
 export { AiContentUnsupportedError } from './content-errors';
+export { AiContextOverflowError } from './context-overflow';
 export { COST_SCALE_DIGITS, costScale, exactCostOf, roundUpCost } from './cost';
 export type { EchoProvider, EchoProviderInput } from './echo-provider';
 export { echoProvider } from './echo-provider';
@@ -93,12 +95,11 @@ export {
   normalizeVector,
   wordTokens,
 } from './embeddings';
-export type { AiErrorCode } from './errors';
+export type { AiErrorCode } from './error-codes';
+export { AI_ERROR_CODES, AI_ERROR_TITLES } from './error-codes';
 export {
   AgentMaxTurnsError,
   AgentToolUnexposedError,
-  AI_ERROR_CODES,
-  AI_ERROR_TITLES,
   AiBudgetExceededError,
   AiGatewayMissingError,
   AiKeyMissingError,
@@ -133,14 +134,19 @@ export {
   EvalMissingError,
   EvalRecordingError,
   EvalThresholdError,
+  EvalToolUndoubledError,
 } from './eval-errors';
 export type {
+  AgentEvalCase,
+  AgentEvalInput,
   CaseResult,
   DefineEvalInput,
   Eval,
   EvalCase,
   EvalFact,
   EvalResult,
+  EvaluatedAgent,
+  PromptEvalInput,
 } from './evals';
 export {
   baselineFrom,
@@ -150,9 +156,16 @@ export {
   resetEvals,
 } from './evals';
 export type { AiFetch } from './fetch-seam';
-export type { Gateway, GatewayCache, ProviderGatewayInput, RetryPolicy } from './gateway';
+export type {
+  Gateway,
+  GatewayBreakerOption,
+  GatewayCache,
+  ProviderGatewayInput,
+  RetryPolicy,
+} from './gateway';
 export {
   backoffMs,
+  DEFAULT_GATEWAY_BREAKER,
   DEFAULT_GATEWAY_RETRY,
   isRetryable,
   promptCacheKey,
@@ -165,7 +178,7 @@ export type { HiveMember, HiveMemberError, HiveOutput, HiveResult } from './hive
 export type { InProcessToolDef } from './in-process-tool';
 export { inProcessTool } from './in-process-tool';
 export type { LlmAction, LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm';
-export { llm } from './llm';
+export { LlmTextChunk, llm } from './llm';
 export type { LlmCache, LlmScopeArgs, LlmSemanticCache } from './llm-cache';
 export type { LlmStreamChunk } from './llm-stream';
 export type { ModelSource } from './model-resolve';
@@ -220,6 +233,8 @@ export type {
   PromptCacheHint,
   PromptCacheOption,
   PromptCacheRule,
+  ProviderCacheRow,
+  ProviderPromptCache,
   SystemParts,
   SystemPrompt,
 } from './prompt-cache';
@@ -275,7 +290,8 @@ export {
 export { assertNoSecrets } from './redaction';
 export type { RemoteEmbedder, RemoteEmbedderInput } from './remote-embedder';
 export { remoteEmbedder } from './remote-embedder';
-export type { AiRuntimeInput, Redactor } from './runtime';
+export { meteredRun } from './run-meter';
+export type { AiRuntimeInput, Redactor, ToolDouble } from './runtime';
 export {
   aiEmbedder,
   aiGateway,
@@ -284,8 +300,9 @@ export {
   semanticCacheFor,
 } from './runtime';
 export type { Sampling } from './sampling';
-export type { Scorer } from './scorers';
+export type { EvalRun, EvalToolCall, Scorer, ScorerInput } from './scorers';
 export {
+  calledTool,
   contains,
   exact,
   jsonSchemaValid,
@@ -293,6 +310,13 @@ export {
   llmJudge,
   numericTolerance,
 } from './scorers';
+export type {
+  ScriptedCall,
+  ScriptedProvider,
+  ScriptedProviderInput,
+  ScriptedTurn,
+} from './scripted-provider';
+export { AiScriptExhaustedError, scriptedProvider } from './scripted-provider';
 export type {
   AgentTool,
   JsonSchema,

package/src/llm.ts

@@ -21,17 +21,22 @@
  * same rules.
  */
 
-import type { Action, InvokeOptions } from '@ultimat3/action';
-import { action } from '@ultimat3/action';
+import type { InvokeOptions, StreamAction } from '@ultimat3/action';
+import { action, chunksRead } from '@ultimat3/action';
 import type { Ctx, Span, SpanAttributes } from '@ultimat3/core';
 import { finiteCount, withSpan } from '@ultimat3/core';
 import type { InferInput, InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
-import { formatIssues, validateAsync } from '@ultimat3/schema';
+import { formatIssues, t, validateAsync } from '@ultimat3/schema';
 import { runTotal } from './agent-usage';
 import { LlmTruncatedError } from './answer-errors';
 import type { BudgetLimits } from './budget';
 import { budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget';
-import { LlmOutputInvalidError, LlmRefusedError, LlmStreamInvalidError } from './errors';
+import {
+  AiTransportError,
+  LlmOutputInvalidError,
+  LlmRefusedError,
+  LlmStreamInvalidError,
+} from './errors';
 import type { Gateway } from './gateway';
 import { openCache } from './llm-cache';
 import type { LlmBudget, LlmDef } from './llm-def';
@@ -63,13 +68,21 @@ const DEFAULT_MAX_TOKENS = 4_096;
 
 export type { LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm-def';
 
+/**
+ * What `llm()`'s route streams to a caller asking for `text/event-stream`: the answer's text, as it
+ * arrives. `thinking` never crosses the wire — reasoning is for the server's own `.stream()`
+ * consumer to decide about, not for a browser to render by default. The final frame is `done`,
+ * the value that satisfied `output`.
+ */
+export const LlmTextChunk = t.object({ type: t.literal('text'), text: t.string });
+
 /**
  * An `action` with one extra way to be called. Every projection an action has, it has — its MCP tool,
  * `.openapi()`, `.client()`, `.job()`, `.contract()` — plus a transport for the case an action's
  * single return value cannot serve: text on a screen before the answer is finished.
  */
 export interface LlmAction<TInput extends StandardSchemaV1, TOutput extends StandardSchemaV1>
-  extends Action<TInput, TOutput> {
+  extends StreamAction<TInput, TOutput, typeof LlmTextChunk> {
   /**
    * The same call, delivered as it arrives. Runs the action's policy, input parse, budget scope,
    * semantic cache and span exactly as calling it would — the invocation IS an ordinary one,
@@ -106,12 +119,30 @@ export function llm<
   // a call that cannot answer.
   finiteCount('llm', 'maxTokens', def.maxTokens ?? DEFAULT_MAX_TOKENS, 1);
   const sampling = screenSampling('llm', def);
-  const built = action<TInput, TOutput>({
+  const built = action<TInput, TOutput, typeof LlmTextChunk>({
     input: def.input,
     output: def.output,
     policy: def.policy,
     ...(def.mcp === undefined ? {} : { mcp: def.mcp }),
-    handle: (args) => generate(def, respond, sampling, args),
+    // Declared, so the route answers `text/event-stream` when asked. Drained (MCP, `.job()`, JSON,
+    // a direct call) it is the non-streaming call, repair turn included; only a caller READING the
+    // chunks gets the one-attempt streamed turn, exactly as server-side `.stream()` does.
+    stream: LlmTextChunk,
+    async *handle(args) {
+      if (!chunksRead()) return await generate(def, respond, sampling, args);
+      const chunks = llmStream<InferOutput<TOutput>>((sink) =>
+        withLlmSink(sink, () => generate(def, respond, sampling, args)),
+      );
+      for await (const chunk of chunks) {
+        if (chunk.type === 'text') yield { type: 'text' as const, text: chunk.text };
+        else if (chunk.type === 'done') return chunk.value;
+      }
+      // `llmStream` ends with `done` or throws; reaching here is a sink that closed early.
+      throw new AiTransportError({
+        provider: 'gateway',
+        detail: 'the streamed answer ended without its value',
+      });
+    },
   });
   return streamable(built);
 }
@@ -126,7 +157,7 @@ export function llm<
  * The original is captured first, or the override would call itself.
  */
 function streamable<TInput extends StandardSchemaV1, TOutput extends StandardSchemaV1>(
-  target: Action<TInput, TOutput>,
+  target: StreamAction<TInput, TOutput, typeof LlmTextChunk>,
 ): LlmAction<TInput, TOutput> {
   const rename = target.named.bind(target);
   const self: LlmAction<TInput, TOutput> = Object.assign(target, {

package/src/model-resolve.ts

@@ -7,7 +7,13 @@ import type { ModelId } from './models';
 import { aiGateway } from './runtime';
 
 /** Where a model was resolved. Bounded, and the first word of the refusal's cause. */
-export type ModelSite = 'llm' | 'agent' | 'gateway' | 'provider' | 'echo-provider';
+export type ModelSite =
+  | 'llm'
+  | 'agent'
+  | 'gateway'
+  | 'provider'
+  | 'echo-provider'
+  | 'scripted-provider';
 
 /** The three places, spelled once so the refusal and the docs cannot drift apart. */
 const PLACES =

package/src/openai-body.ts

@@ -9,10 +9,13 @@ import { AiRequestInvalidError } from './errors';
 import type { Effort, ModelId, ThinkingMode } from './models';
 import { modelSpec } from './models';
 import { toOpenAiMessages, toOpenAiTools, toolChoiceFor } from './openai-messages';
+import type { PromptCacheRule, ProviderCacheRow } from './prompt-cache';
 import { OPENAI_CACHE_KEY_MAX, promptCacheRule, SESSION_KEY_MAX } from './prompt-cache';
 import type { GenerateRequest } from './provider';
 import { samplingBody } from './sampling';
 
+type CacheRule = Pick<PromptCacheRule, 'marks' | 'session'>;
+
 export interface ChatCompletionBodyInput {
   readonly request: GenerateRequest;
   /** Already resolved — the gateway picks the model, the provider never guesses mid-request. */
@@ -22,6 +25,8 @@ export interface ChatCompletionBodyInput {
   readonly stream: boolean;
   /** The endpoint, for `PROMPT_CACHE_RULES` — which host this is decides what a cache hint sends. */
   readonly baseUrl?: string | undefined;
+  /** The provider's own screened rows (`promptCache.rules`), read before the table. */
+  readonly cacheRows?: readonly ProviderCacheRow[];
 }
 
 /**
@@ -38,7 +43,7 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record<strin
   const spec = modelSpec(model);
   assertMediaContent(request.messages, { provider, model });
   const hint = request.promptCache;
-  const rule = promptCacheRule('openai', input.baseUrl, model);
+  const rule = promptCacheRule('openai', input.baseUrl, model, input.cacheRows);
   const marked = hint !== undefined && rule.marks === 'parts';
   const body: Record<string, unknown> = {
     model,
@@ -71,15 +76,17 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record<strin
   return body;
 }
 
-/** The headers a cache hint adds for this endpoint and model: OpenRouter's stickiness key, or none. */
+/**
+ * The headers a cache hint adds under this request's resolved rule: OpenRouter's stickiness key,
+ * or none.
+ */
 export function sessionHeaders(
   request: GenerateRequest,
-  model: ModelId,
-  baseUrl: string | undefined,
+  rule: CacheRule,
 ): Readonly<Record<string, string>> {
   const session = request.promptCache?.session;
   if (session === undefined) return {};
-  if (promptCacheRule('openai', baseUrl, model).session !== 'x-session-id') return {};
+  if (rule.session !== 'x-session-id') return {};
   return { 'x-session-id': sessionKey(session) };
 }