@tangle-network/agent-app 0.43.68 → 0.43.70

package/README.md

@@ -16,7 +16,7 @@ The substrate packages — `@tangle-network/agent-runtime`, `agent-eval`, `agent
 - **Bounded tool loop** — `runAppToolLoop` / `streamAppToolLoop`: stream a turn → collect tool calls → dispatch → fold results back → re-run, capped. These are 1:1 aliases of `@tangle-network/agent-runtime`'s `runToolLoop` / `streamToolLoop` (the engine owns the loop; this package adds no logic on that path). Substrate-free behind a `streamTurn` seam, so it drives a sandboxed agent, a Worker, or an in-browser copilot unchanged.
 - **Assembled chat vertical** — `createChatTurnRoutes` wires auth → thread/message store → streaming turn with buffered replay → uploads → sidecar question answering into one route factory, over `authorize` / `produce` / `store` / `interactions` seams. No hand-rolled orchestration. See [`examples/chat-app.md`](./examples/chat-app.md).
 - **Sandbox-optional** — the same tools, billing, eval, and loop work without a container. A `fetch`-only adapter maps any OpenAI-compatible stream (Tangle Router, tcloud) into the loop. See [`examples/browser-copilot.md`](./examples/browser-copilot.md).
-- **Resumable turns (sandbox-free path)** — for a browser/edge copilot streaming the Router directly, buffer a turn so a dropped tab loses nothing and a reconnecting client replays the tail. **Sandbox products don't need this** — the sandbox SDK already buffers + replays sessions (server-side reconnect via `box.streamPrompt`'s `lastEventId`; browser-direct via `box.mintScopedToken()` + `SessionGatewayClient`). See [`examples/resumable-turns.md`](./examples/resumable-turns.md).
+- **Resumable turns** — buffer a turn so a dropped tab loses nothing and a reconnecting client replays the tail. Two niches: a browser/edge copilot streaming the Router directly (no sandbox session to attach to), and any DETACHED sandbox run a browser must tail (`dispatchPrompt({ detach: true })` / `driveTurn` execute on a lane the session gateway never sees). **An INTERACTIVE sandbox turn doesn't need it** — drive it on the message lane (`box.session(id).sendMessage()`) and attach the tab with `box.mintScopedToken()` + `SessionGatewayClient`, or let the worker resume with `box.streamPrompt('', { executionId, lastEventId })`. See [`examples/resumable-turns.md`](./examples/resumable-turns.md).
 - **Composes the engine, never forks it** — `/eval` re-exports `@tangle-network/agent-eval`'s verifier; `/integrations` wraps the hub; `/tangle` and `/billing` take the tcloud client as a structural contract. Engines are **peer dependencies** — you pin the version, nothing is bundled.
 - **ESM, typed, zero runtime deps** in the substrate-free modules (`/runtime`, `/web`, `/crypto`, `/redact`, `/stream`). Ships with `.d.ts` and npm [provenance](https://www.npmjs.com/package/@tangle-network/agent-app#provenance).
 

package/dist/chat-routes/index.d.ts

@@ -5,6 +5,7 @@ import { ChatTurnIdentity, ChatTurnProducer } from '@tangle-network/agent-runtim
 import { InteractionAnswerRoute, InteractionAnswerRouteOptions } from '../interactions/index.js';
 import { PersistedChatMessageForTurn } from '../stream/index.js';
 import { d as TurnEventStore } from '../turn-buffer-DGnAPKwa.js';
+import { M as ModelFailoverAttempt } from '../failover-H0x12kY3.js';
 export { D as DEFAULT_STALE_TURN_LOCK_GRACE_MS, a as DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS, R as ReconcileStaleTurnLockOptions, b as ReconcileStaleTurnLockResult, S as StaleTurnLockSandboxProbeResult, c as StaleTurnLockSessionProbeResult, r as reconcileStaleTurnLock } from '../stale-turn-lock-DucQzvXu.js';
 import { J as JsonRecord } from '../stream-normalizer-QYUl4vnl.js';
 import { SandboxExecChannel, PromptInputPart } from '../sandbox/index.js';
@@ -298,7 +299,22 @@ interface ChatTurnRouteProducer extends ChatTurnProducer {
      *  parts until the turn completes. */
     draftParts?(): Array<Record<string, unknown>>;
     usage?(): ChatTurnUsage;
+    /** The model that SERVED the turn. With failover wired this is the model that
+     *  actually answered, which is not necessarily the one the caller preferred —
+     *  read it after the stream drains, never before. */
     model?: string;
+    /** Model-failover attribution, when the producer supports it. Reported onto
+     *  the usage/billing receipt so a downgrade is never silent. */
+    modelFailover?(): ChatTurnModelFailover;
+}
+/** Which model served, and what it took to get there. */
+interface ChatTurnModelFailover {
+    /** The model that served the turn. */
+    model?: string;
+    /** Every model tried, in order, with the reason each was abandoned. */
+    attempts: ModelFailoverAttempt[];
+    /** True when the preferred model did not serve. */
+    usedFallback: boolean;
 }
 /** Resolve authorization status and context for a chat turn including tenant and user identification */
 type ChatTurnAuthorization<TContext> = {
@@ -410,6 +426,13 @@ interface ChatTurnLifecycleComplete<TContext> extends ChatTurnLifecycleBase<TCon
     finalText: string;
     usage: ChatTurnUsage;
     durationMs: number;
+    /** The model that SERVED the turn — the fallback's id when failover moved it.
+     *  `usage` is that model's, so telemetry that splits cost or quality by model
+     *  must key on this and not on the requested one. */
+    model?: string;
+    /** Attribution for a downgrade: which models were tried and why each failed.
+     *  `undefined` when the producer reports no failover support. */
+    modelFailover?: ChatTurnModelFailover;
 }
 /** Represent an error occurring during a chat turn lifecycle with context and duration information */
 interface ChatTurnLifecycleError<TContext> extends ChatTurnLifecycleBase<TContext> {
@@ -516,6 +539,13 @@ interface CreateChatTurnRoutesOptions<TContext = void> {
         context: TContext;
         failed: boolean;
         failureReason?: string;
+        /** The model that SERVED this turn. With failover wired it may differ from
+         *  the requested one, so a product that bills or scores per model MUST read
+         *  it here rather than assuming the model it asked for. */
+        model?: string;
+        /** Present when the producer supports failover: the full attempt trail, and
+         *  `usedFallback` — the flag that makes a silent downgrade impossible. */
+        modelFailover?: ChatTurnModelFailover;
     }): Promise<void>;
     /** Per-event side channel (product broadcast). The turn-buffer tap is
      *  already wired; this runs in addition. */
@@ -566,6 +596,115 @@ interface ChatTurnRoutes {
 /** Build chat turn routes to handle and validate incoming chat requests with optional logging */
 declare function createChatTurnRoutes<TContext = void>(options: CreateChatTurnRoutesOptions<TContext>): ChatTurnRoutes;
 
+/**
+ * Model failover for a STREAMING turn.
+ *
+ * `/model-resolution`'s `runWithModelFailover` already owns the policy: walk a
+ * chain, classify a resolved-or-thrown signal, re-throw a non-outage failure
+ * immediately, and carry the attempt trail. This module does NOT re-implement
+ * any of that — it composes it. What it adds is the one thing a whole-call
+ * primitive cannot express, because a stream fails PARTWAY:
+ *
+ *   **A turn may only fail over before its first client-visible byte.**
+ *
+ * Once a text delta, tool call, or ask has reached the browser (and the
+ * persisted transcript), restarting on another model would duplicate the
+ * answer. So each attempt is probed: open the stream, pull events into a small
+ * buffer, and decide at the first meaningful event whether this model is
+ * serving. Committing replays the buffer and hands the live iterator through;
+ * abandoning discards the buffer (those events describe the dead model's
+ * session — including its `step-finish` usage, which must never be billed) and
+ * lets `runWithModelFailover` walk to the next model.
+ *
+ * The classification itself is `isUpstreamUnavailable` verbatim, so this path
+ * inherits the measured facts from the 2026-07-25 outage — above all that an
+ * outage is NOT always a thrown error: the sandbox RESOLVES a terminal `error`
+ * event carrying `{ errorCode: 'provider_inference_unavailable' }`, which a
+ * classifier inspecting only `catch` misses entirely. That resolved shape is
+ * the whole reason the breakage went unnoticed, so it is classified here first.
+ *
+ * Conservative by construction:
+ * - A terminal failure that is NOT an outage (400, bad schema, content filter)
+ *   COMMITS rather than failing over — it surfaces to the user exactly as it
+ *   does today. Those fail identically on every model; walking the chain would
+ *   only multiply latency and spend to reach the same error.
+ * - A clean stream that simply produced nothing is NOT retried. An empty answer
+ *   is not evidence of a dead upstream, and a silent re-roll on another model is
+ *   precisely the unattributable downgrade this work exists to prevent.
+ * - A chain of length 1 costs nothing: one attempt, no extra call, no added
+ *   latency, byte-identical to no failover at all.
+ */
+
+/**
+ * True when `event` puts content in front of the user (or in the persisted
+ * transcript), making a restart on another model unsafe.
+ *
+ * Deliberately an allow-list of KNOWN-INERT types rather than a deny-list: an
+ * unrecognized event commits. Getting this wrong in the safe direction costs a
+ * missed failover; getting it wrong the other way duplicates a user's answer.
+ */
+declare function isCommittingSandboxEvent(event: unknown): boolean;
+/** A terminal failure event, classified. `outage` decides failover vs surface. */
+interface TerminalFailure {
+    outage: boolean;
+    reason: string;
+    code?: string;
+}
+/**
+ * Classify a terminal failure event. Returns `null` for any non-terminal event.
+ *
+ * The RESOLVED shape is checked first and deliberately: the sandbox reports an
+ * upstream outage by resolving `{ success: false, errorCode:
+ * 'provider_inference_unavailable' }` inside a terminal `error` event's `data`,
+ * never by throwing. Both `data` and the whole record are offered to
+ * `isUpstreamUnavailable` so a payload nested either way is caught.
+ */
+declare function classifyTerminalFailure(event: unknown): TerminalFailure | null;
+/** Open the raw turn stream for one specific model. */
+type OpenModelStream = (args: {
+    model: string;
+    /** 1 for the preferred model, 2 for the first fallback, and so on. */
+    attempt: number;
+}) => AsyncIterable<unknown> | Promise<AsyncIterable<unknown>>;
+/** Fired when a model is abandoned and the next one is about to be tried. */
+interface ModelFallbackInfo {
+    from: string;
+    to: string;
+    reason: string;
+}
+/** Define inputs for streaming a turn across a model failover chain */
+interface ModelFailoverStreamOptions {
+    /** Preferred model first, then fallbacks in descending preference. */
+    models: readonly string[];
+    open: OpenModelStream;
+    /** Override the commit-point rule. Default {@link isCommittingSandboxEvent}. */
+    isCommitting?: (event: unknown) => boolean;
+    onFallback?: (info: ModelFallbackInfo) => void;
+    log?: (message: string, meta?: Record<string, unknown>) => void;
+}
+/** The failover-wrapped stream plus the attribution every consumer needs. */
+interface ModelFailoverStreamHandle {
+    events: AsyncGenerator<unknown, void, unknown>;
+    /** The model that actually served. `undefined` until the first pull resolves it. */
+    servingModel(): string | undefined;
+    /** Every model tried, in order, with the reason each was abandoned. */
+    attempts(): ModelFailoverAttempt[];
+    /** True when the preferred model did not serve — the attributability signal. */
+    usedFallback(): boolean;
+}
+/**
+ * Wrap `open` in reactive model failover, streaming from the first model in
+ * `models` that reaches its commit point.
+ *
+ * Zero added latency on the happy path: the preferred model is opened first and,
+ * the moment it emits anything meaningful, its events flow straight through.
+ *
+ * @throws ModelFailoverExhaustedError when every model's upstream is down, and
+ *         re-throws a non-outage error from the FIRST model without walking the
+ *         chain (both behaviors inherited from `runWithModelFailover`).
+ */
+declare function streamWithModelFailover(options: ModelFailoverStreamOptions): ModelFailoverStreamHandle;
+
 /**
  * Sandbox lane: bridge a raw sandbox event stream (`streamSandboxPrompt`) into
  * the `ChatTurnProducer` shape agent-runtime's `handleChatTurn` consumes AND
@@ -608,10 +747,46 @@ type FilePartPromotionOutcome = {
 };
 /** Define options for producing sandbox chat events with rendering and interaction controls */
 interface SandboxChatProducerOptions {
-    /** The raw sandbox event stream (e.g. `streamSandboxPrompt(...)`). */
-    events: AsyncIterable<unknown>;
-    /** Recorded on the persisted assistant message. */
+    /** The raw sandbox event stream (e.g. `streamSandboxPrompt(...)`).
+     *
+     *  An ALREADY-OPEN stream is bound to one model and cannot be reopened, so
+     *  this form can never fail over. Prefer {@link openEvents}; exactly one of
+     *  the two is required. */
+    events?: AsyncIterable<unknown>;
+    /** Open the raw sandbox stream FOR A GIVEN MODEL — the failover-capable form
+     *  of {@link events}. The callback receives the model to run and returns the
+     *  same `AsyncIterable` `events` would have been (typically
+     *  `streamSandboxPrompt(shell, box, prompt, { ...opts, model })`).
+     *
+     *  Wiring this turns failover ON with no further flag: whenever the resolved
+     *  chain (`model` + {@link fallbackModels}) holds more than one entry, a model
+     *  whose upstream is dead is abandoned BEFORE its first client-visible byte
+     *  and the next one is tried. A one-entry chain opens exactly once — no extra
+     *  call, no added latency, byte-identical to today.
+     *
+     *  Requires {@link model}: failover has to know which model it is running. */
+    openEvents?: OpenModelStream;
+    /** Recorded on the persisted assistant message. When failover moves the turn
+     *  to another model, the producer's `model` reports the model that ACTUALLY
+     *  served, not this preferred one — see {@link modelFailover}. */
     model?: string;
+    /** Models to try, in order, when `model`'s upstream is dead. Product config,
+     *  never a shell default: agent-app cannot know which ids are live, and a
+     *  baked list would rot into exactly the stale-liveness bug this guards
+     *  against. Empty/omitted → one attempt, today's behavior.
+     *
+     *  Pick these deliberately. A same-family fallback is NOT automatically safe:
+     *  `gemini-2.5-flash` produced zero persisted deliverables in 2 of 3 measured
+     *  runs on a med-legal filing flow where `gemini-2.5-pro` succeeded. That is
+     *  why every fallback is surfaced (persisted `model`, a transcript notice, and
+     *  `modelFailover()`) instead of being applied silently. */
+    fallbackModels?: readonly string[];
+    /** Opt out of failover while still using {@link openEvents}. `false` collapses
+     *  the chain to `model` alone. */
+    modelFailover?: false;
+    /** Fired when a model is abandoned mid-chain (telemetry/alerting). The user
+     *  already sees a transcript notice; this is for the operator. */
+    onModelFallback?: (info: ModelFallbackInfo) => void;
     /** Which ask kinds the product renders a card for. Anything else is
      *  auto-declined (see `declineInteraction`) so the run never hangs in the
      *  broker waiting on a card no client will show. Default: question/plan.
@@ -705,9 +880,30 @@ interface DetachedTurnOptions {
     scopeId: string;
     /** The raw sandbox event stream for this turn (e.g. `streamSandboxPrompt`).
      *  Ownership of the box, prompt, tooling, and attachments stays with the
-     *  caller — this only projects the stream. */
-    events: AsyncIterable<unknown>;
-    /** Recorded on the persisted assistant message + usage receipt. */
+     *  caller — this only projects the stream.
+     *
+     *  An already-open stream is bound to one model and cannot fail over. Prefer
+     *  {@link openEvents}; exactly one of the two is required. */
+    events?: AsyncIterable<unknown>;
+    /** Open the raw sandbox stream FOR A GIVEN MODEL — the failover-capable form
+     *  of {@link events}. Wiring it turns failover on with no further flag
+     *  whenever {@link fallbackModels} is non-empty. Requires {@link model}.
+     *
+     *  An autonomous run is the case that most needs this: nobody is watching to
+     *  notice a dead upstream and retry by hand, so without failover the mission
+     *  step or queue job simply fails. Forwarded to the producer. */
+    openEvents?: SandboxChatProducerOptions['openEvents'];
+    /** Models to try, in order, when `model`'s upstream is dead. Product config —
+     *  see the producer's note on why a same-family fallback is not automatically
+     *  safe and why every fallback is surfaced. */
+    fallbackModels?: SandboxChatProducerOptions['fallbackModels'];
+    /** Opt out of failover while still using {@link openEvents}. */
+    modelFailover?: false;
+    /** Fired when a model is abandoned mid-chain (telemetry/alerting). */
+    onModelFallback?: SandboxChatProducerOptions['onModelFallback'];
+    /** The PREFERRED model. Recorded on the persisted assistant message + usage
+     *  receipt — unless failover moved the turn, in which case the model that
+     *  actually served is recorded instead and surfaced on the result. */
     model?: string;
     /** Per-flush buffer coalescer. Default `coalesceDeltas`. */
     coalesce?: (events: unknown[]) => unknown[];
@@ -786,6 +982,15 @@ interface DetachedTurnResult {
      *  `null` when the turn produced nothing and the row was retracted. Absent
      *  when the caller owns persistence (today's behavior). */
     messageId?: string | null;
+    /** The model that SERVED the turn — the fallback's id when failover moved it.
+     *  A caller that bills or scores per model must read this, not the model it
+     *  requested. */
+    model?: string;
+    /** True when the preferred model did not serve. Makes an autonomous
+     *  downgrade — which no human watched happen — attributable after the fact. */
+    usedModelFallback?: boolean;
+    /** Every model tried, in order, with the reason each was abandoned. */
+    modelAttempts?: ModelFailoverAttempt[];
 }
 /**
  * Stream a detached turn into the live turn-event buffer, durably.
@@ -1311,4 +1516,4 @@ interface PromoteAgentFilePartOptions {
 /** Promote a part of an agent file with optional byte limits and MIME type detection */
 declare function promoteAgentFilePart(options: PromoteAgentFilePartOptions): Promise<PromoteFilePartResult>;
 
-export { type AssistantDraftSnapshot, type AssistantDraftStore, type AssistantDraftWriter, type AssistantDraftWriterOptions, type AssistantRowValues, type AttachmentPathArgs, type AttachmentPathCheck, type AttachmentReadResult, type AttachmentUploadAuthorization, type AttachmentWriteResult, type BuildDispatchPartsInput, ChatAttachmentKind, type ChatRouteDurableProjection, type ChatRouteDurableProjectionLogger, type ChatTurnAuthorization, type ChatTurnAuthorizeArgs, ChatTurnFilePartInput, type ChatTurnGateResult, type ChatTurnHeartbeat, type ChatTurnInputPatch, type ChatTurnLifecycle, type ChatTurnLifecycleComplete, type ChatTurnLifecycleError, type ChatTurnLifecycleStart, type ChatTurnLock, type ChatTurnLockResult, type ChatTurnMessageStore, ChatTurnPartInput, type ChatTurnProduceArgs, ChatTurnRequestPayload, type ChatTurnRouteProducer, type ChatTurnRoutes, type ChatTurnUsage, type CreateAttachmentUploadRouteOptions, type CreateChatTurnRoutesOptions, type CreateUploadRouteOptions, type DetachedTurnFinal, type DetachedTurnOptions, type DetachedTurnParts, type DetachedTurnResult, type DispatchPartsOutcome, type DraftPersistenceTuning, type DraftStoredMessage, type FilePartPromotionOutcome, PROMOTE_MAX_FILE_BYTES, type PromoteAgentFilePartOptions, type PromoteFilePartResult, PromptInputPart, type RawAgentFilePart, type ReadAttachmentFn, type ReadSandboxMentionFn, type ResolveChatAttachmentsOptions, type ResolveChatAttachmentsResult, type SandboxChatProducerOptions, type SandboxUploadSink, UPLOAD_INLINE_MAX_BYTES, UPLOAD_MAX_FILE_BYTES, type UploadAuthorization, type UploadedChatFile, type WriteAttachmentFn, assistantRowIdForTurn, buildDispatchParts, bytesToBase64, createAssistantDraftWriter, createAttachmentUploadRoute, createChatTurnRoutes, createSandboxChatProducer, createUploadRoute, defaultValidateAttachmentPath, isDraftContentEvent, promoteAgentFilePart, resolveChatAttachments, runDetachedTurn, sanitizeUploadFilename, sniffMimeFromName, storeSupportsDraftPersistence, withDurableChatProjection };
+export { type AssistantDraftSnapshot, type AssistantDraftStore, type AssistantDraftWriter, type AssistantDraftWriterOptions, type AssistantRowValues, type AttachmentPathArgs, type AttachmentPathCheck, type AttachmentReadResult, type AttachmentUploadAuthorization, type AttachmentWriteResult, type BuildDispatchPartsInput, ChatAttachmentKind, type ChatRouteDurableProjection, type ChatRouteDurableProjectionLogger, type ChatTurnAuthorization, type ChatTurnAuthorizeArgs, ChatTurnFilePartInput, type ChatTurnGateResult, type ChatTurnHeartbeat, type ChatTurnInputPatch, type ChatTurnLifecycle, type ChatTurnLifecycleComplete, type ChatTurnLifecycleError, type ChatTurnLifecycleStart, type ChatTurnLock, type ChatTurnLockResult, type ChatTurnMessageStore, type ChatTurnModelFailover, ChatTurnPartInput, type ChatTurnProduceArgs, ChatTurnRequestPayload, type ChatTurnRouteProducer, type ChatTurnRoutes, type ChatTurnUsage, type CreateAttachmentUploadRouteOptions, type CreateChatTurnRoutesOptions, type CreateUploadRouteOptions, type DetachedTurnFinal, type DetachedTurnOptions, type DetachedTurnParts, type DetachedTurnResult, type DispatchPartsOutcome, type DraftPersistenceTuning, type DraftStoredMessage, type FilePartPromotionOutcome, type ModelFailoverStreamHandle, type ModelFailoverStreamOptions, type ModelFallbackInfo, type OpenModelStream, PROMOTE_MAX_FILE_BYTES, type PromoteAgentFilePartOptions, type PromoteFilePartResult, PromptInputPart, type RawAgentFilePart, type ReadAttachmentFn, type ReadSandboxMentionFn, type ResolveChatAttachmentsOptions, type ResolveChatAttachmentsResult, type SandboxChatProducerOptions, type SandboxUploadSink, UPLOAD_INLINE_MAX_BYTES, UPLOAD_MAX_FILE_BYTES, type UploadAuthorization, type UploadedChatFile, type WriteAttachmentFn, assistantRowIdForTurn, buildDispatchParts, bytesToBase64, classifyTerminalFailure, createAssistantDraftWriter, createAttachmentUploadRoute, createChatTurnRoutes, createSandboxChatProducer, createUploadRoute, defaultValidateAttachmentPath, isCommittingSandboxEvent, isDraftContentEvent, promoteAgentFilePart, resolveChatAttachments, runDetachedTurn, sanitizeUploadFilename, sniffMimeFromName, storeSupportsDraftPersistence, streamWithModelFailover, withDurableChatProjection };

package/dist/chat-routes/index.js

@@ -17,6 +17,11 @@ import {
   DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
   reconcileStaleTurnLock
 } from "../chunk-CCVWR33L.js";
+import {
+  buildModelChain,
+  isUpstreamUnavailable,
+  runWithModelFailover
+} from "../chunk-KFTRTOT2.js";
 import {
   parseJsonObjectBody
 } from "../chunk-PTUMBMVH.js";
@@ -480,6 +485,7 @@ function createChatTurnRoutes(options) {
             error: terminalError ?? lastFailureData ?? new Error("chat turn failed")
           });
         } else {
+          const failoverInfo = producer?.modelFailover?.();
           await lifecycle.onTurnComplete?.({
             identity,
             executionId,
@@ -487,7 +493,9 @@ function createChatTurnRoutes(options) {
             context,
             durationMs,
             finalText: producer?.finalText() ?? "",
-            usage: producer?.usage?.() ?? {}
+            usage: producer?.usage?.() ?? {},
+            ...producer?.model ? { model: producer.model } : {},
+            ...failoverInfo ? { modelFailover: failoverInfo } : {}
           });
         }
       } catch (err) {
@@ -634,13 +642,18 @@ function createChatTurnRoutes(options) {
             // (not a throw) still lands here — so surface `runFailed` so the
             // product skips billing an errored turn instead of marking it
             // complete with empty text.
-            onTurnComplete: ({ identity: turnIdentity, finalText }) => options.onTurnComplete({
-              identity: turnIdentity,
-              finalText,
-              context,
-              failed: runFailed,
-              ...runFailed ? { failureReason: failureReasonOf(lastFailureData) } : {}
-            })
+            onTurnComplete: ({ identity: turnIdentity, finalText }) => {
+              const failoverInfo = producer?.modelFailover?.();
+              return options.onTurnComplete({
+                identity: turnIdentity,
+                finalText,
+                context,
+                failed: runFailed,
+                ...runFailed ? { failureReason: failureReasonOf(lastFailureData) } : {},
+                ...producer?.model ? { model: producer.model } : {},
+                ...failoverInfo ? { modelFailover: failoverInfo } : {}
+              });
+            }
           } : {},
           ...options.traceFlush ? { traceFlush: () => options.traceFlush(context) } : {}
         }
@@ -779,6 +792,130 @@ function concatStreams(streams) {
   });
 }
 
+// src/chat-routes/model-failover-stream.ts
+var TERMINAL_FAILURE_TYPES = /* @__PURE__ */ new Set(["error", "session.run.failed"]);
+var NON_COMMITTING_PART_TYPES = /* @__PURE__ */ new Set(["step-start", "step-finish"]);
+function isCommittingSandboxEvent(event) {
+  const record = asRecord(event);
+  if (!record) return false;
+  const type = asString(record.type) ?? "";
+  if (!type) return false;
+  if (type === "message.part.updated") {
+    const part = asRecord(asRecord(record.data)?.part);
+    const partType = asString(part?.type) ?? "";
+    if (NON_COMMITTING_PART_TYPES.has(partType)) return false;
+    if (partType === "text" || partType === "reasoning") {
+      const text = asString(part?.text) ?? asString(part?.content) ?? "";
+      return text.length > 0;
+    }
+    return true;
+  }
+  if (type === "start" || type === "execution.started" || type === "status" || type === "model-processing" || type === "session.created" || type === "session.updated" || type === "session.idle" || type === "step-start" || type === "step-finish" || type === "turn" || type === "warning") {
+    return false;
+  }
+  return true;
+}
+function classifyTerminalFailure(event) {
+  const record = asRecord(event);
+  if (!record) return null;
+  const type = asString(record.type) ?? "";
+  if (!TERMINAL_FAILURE_TYPES.has(type)) return null;
+  const data = asRecord(record.data);
+  const outage = isUpstreamUnavailable(data) || isUpstreamUnavailable(record);
+  const reason = asString(data?.message) ?? asString(data?.error) ?? asString(data?.reason) ?? asString(record.message) ?? `sandbox stream reported ${type}`;
+  const code = asString(data?.errorCode) ?? asString(data?.code);
+  return { outage, reason, ...code ? { code } : {} };
+}
+async function closeIterator(iterator, log) {
+  try {
+    await iterator.return?.();
+  } catch (err) {
+    log?.("[chat-routes] abandoning a failed model stream threw", {
+      error: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
+function streamWithModelFailover(options) {
+  const committing = options.isCommitting ?? isCommittingSandboxEvent;
+  let serving;
+  let trail = [];
+  let fellBack = false;
+  let attemptIndex = 0;
+  const probe = async (model) => {
+    attemptIndex += 1;
+    const source = await options.open({ model, attempt: attemptIndex });
+    const iterator = source[Symbol.asyncIterator]();
+    const buffered = [];
+    for (; ; ) {
+      const next = await iterator.next();
+      if (next.done) {
+        return { committed: true, buffered, iterator: null };
+      }
+      const event = next.value;
+      const failure = classifyTerminalFailure(event);
+      if (failure?.outage) {
+        await closeIterator(iterator, options.log);
+        return {
+          committed: false,
+          error: failure.reason,
+          ...failure.code ? { errorCode: failure.code } : {}
+        };
+      }
+      buffered.push(event);
+      if (failure) return { committed: true, buffered, iterator };
+      if (committing(event)) return { committed: true, buffered, iterator };
+    }
+  };
+  const events = (async function* () {
+    let handle;
+    try {
+      const outcome = await runWithModelFailover({
+        models: options.models,
+        run: probe,
+        // The probe has already classified the raw payload with
+        // `isUpstreamUnavailable`; this reads its verdict rather than
+        // re-classifying a wrapper object, so the two can never disagree.
+        isUnavailableResult: (result) => result.committed === false,
+        onFallback: (attempt, nextModel) => {
+          const info = {
+            from: attempt.model,
+            to: nextModel,
+            reason: attempt.reason ?? "upstream unavailable"
+          };
+          options.log?.("[chat-routes] model upstream unavailable; falling over", { ...info });
+          options.onFallback?.(info);
+        }
+      });
+      serving = outcome.model;
+      trail = outcome.attempts;
+      fellBack = outcome.usedFallback;
+      handle = outcome.value;
+    } catch (err) {
+      const attempts = err?.attempts;
+      if (Array.isArray(attempts)) trail = attempts;
+      throw err;
+    }
+    for (const event of handle.buffered) yield event;
+    const live = handle.iterator;
+    if (!live) return;
+    try {
+      for (; ; ) {
+        const next = await live.next();
+        if (next.done) return;
+        yield next.value;
+      }
+    } finally {
+      await closeIterator(live, options.log);
+    }
+  })();
+  return {
+    events,
+    servingModel: () => serving,
+    attempts: () => trail,
+    usedFallback: () => fellBack
+  };
+}
+
 // src/chat-routes/sandbox-producer.ts
 function textDelta(tracker, key, part, rawDelta) {
   const explicit = typeof rawDelta === "string" ? rawDelta : void 0;
@@ -885,6 +1022,40 @@ function sandboxStreamFailureDiagnostic(error) {
 function createSandboxChatProducer(options) {
   const log = options.log ?? ((message, meta) => console.error(message, meta ?? ""));
   const renderable = options.isRenderableInteraction ?? isRenderableInteractionKind;
+  if (options.openEvents && !options.model) {
+    throw new Error(
+      "createSandboxChatProducer: `openEvents` requires `model` \u2014 failover must know which model it is running"
+    );
+  }
+  if (!options.openEvents && !options.events) {
+    throw new Error("createSandboxChatProducer: pass `openEvents` (failover-capable) or `events`");
+  }
+  const chain = options.model ? buildModelChain(options.model, options.modelFailover === false ? [] : options.fallbackModels ?? []) : [];
+  const pendingModelNotices = [];
+  let modelNoticeCount = 0;
+  let failover;
+  let source;
+  if (options.openEvents) {
+    failover = streamWithModelFailover({
+      models: chain,
+      open: options.openEvents,
+      log,
+      onFallback: (info) => {
+        modelNoticeCount += 1;
+        pendingModelNotices.push({
+          id: `model-fallback-${modelNoticeCount}`,
+          // Named models on both sides: a quality regression after a downgrade
+          // must be attributable to the model that actually answered.
+          text: `${info.from} was unavailable (${info.reason}) \u2014 answered with ${info.to} instead.`
+        });
+        options.onModelFallback?.(info);
+      }
+    });
+    source = failover.events;
+  } else {
+    source = options.events;
+  }
+  const servingModel = () => failover?.servingModel() ?? options.model;
   let fullText = "";
   const partOrder = [];
   const partMap = /* @__PURE__ */ new Map();
@@ -931,9 +1102,18 @@ function createSandboxChatProducer(options) {
       yield { type: "usage", usage: { promptTokens, completionTokens } };
     }
   }
+  function* drainModelNotices() {
+    while (pendingModelNotices.length > 0) {
+      const queued = pendingModelNotices.shift();
+      const notice = noticePart("warning", queued.id, queued.text);
+      recordPersistedPart(notice, void 0, noticePartKey(notice.id));
+      yield { type: "notice", id: notice.id, noticeKind: "warning", text: queued.text };
+    }
+  }
   async function* stream() {
     try {
-      for await (const raw of options.events) {
+      for await (const raw of source) {
+        yield* drainModelNotices();
         const record = asRecord(raw);
         if (!record || typeof record.type !== "string") continue;
         const normalized = normalizeToolEvent({ type: record.type, data: asRecord(record.data) });
@@ -1155,6 +1335,7 @@ ${errorContent}` : errorContent;
         }
         yield toProducerWireEvent(record);
       }
+      yield* drainModelNotices();
     } catch (streamErr) {
       const diagnostic = sandboxStreamFailureDiagnostic(streamErr);
       log("[chat-routes] sandbox stream failed", {
@@ -1193,7 +1374,19 @@ ${diagnostic.userMessage}` : diagnostic.userMessage;
     // phantom failures the final write then reverses.
     draftParts: () => draftAssistantParts(partOrder, partMap, fullText),
     usage: () => usage,
-    ...options.model ? { model: options.model } : {}
+    // A GETTER, not a captured value: `turn-routes` reads `producer.model` at
+    // draft-snapshot and persist time, both of which happen after the stream
+    // resolved which model serves. A plain property would freeze the PREFERRED
+    // model into the row and make a downgrade unattributable — the exact
+    // failure mode this work exists to prevent.
+    get model() {
+      return servingModel();
+    },
+    modelFailover: () => ({
+      model: servingModel(),
+      attempts: failover?.attempts() ?? [],
+      usedFallback: failover?.usedFallback() ?? false
+    })
   };
 }
 
@@ -1219,6 +1412,7 @@ function cachedResultFrom(final) {
 async function runDetachedTurn(opts) {
   const { store, turnId, scopeId } = opts;
   let producer;
+  const servingModel = () => producer?.model ?? opts.model;
   let draft;
   if (opts.persist) {
     const { store: persistStore, threadId, messageId, transformText, ...tuning } = opts.persist;
@@ -1236,13 +1430,19 @@ async function runDetachedTurn(opts) {
         content: producer.finalText?.() ?? "",
         ...producer.draftParts ? { parts: producer.draftParts() } : {},
         ...producer.usage ? { usage: producer.usage() } : {},
-        ...opts.model ? { model: opts.model } : {}
+        ...servingModel() ? { model: servingModel() } : {}
       } : null,
       ...transformText ? { transformText } : {},
       ...opts.log ? { log: opts.log } : {}
     });
   }
-  const settleRow = async (result) => {
+  const settleRow = async (base) => {
+    const info = producer?.modelFailover?.();
+    const result = {
+      ...base,
+      ...servingModel() ? { model: servingModel() } : {},
+      ...info ? { usedModelFallback: info.usedFallback, modelAttempts: info.attempts } : {}
+    };
     if (!draft) return result;
     const transform = opts.persist?.transformText;
     const content = transform ? await transform(result.text) : result.text;
@@ -1259,7 +1459,7 @@ async function runDetachedTurn(opts) {
     const values = {
       content,
       ...parts2.length > 0 ? { parts: parts2 } : {},
-      ...opts.model ? { model: opts.model } : {},
+      ...result.model ? { model: result.model } : {},
       ...result.usage.inputTokens !== void 0 ? { inputTokens: result.usage.inputTokens } : {},
       ...result.usage.outputTokens !== void 0 ? { outputTokens: result.usage.outputTokens } : {},
       ...result.usage.reasoningTokens !== void 0 ? { reasoningTokens: result.usage.reasoningTokens } : {},
@@ -1310,8 +1510,11 @@ async function runDetachedTurn(opts) {
   });
   await tap.onEvent({ type: "turn", turnId });
   producer = createSandboxChatProducer({
-    events: opts.events,
+    ...opts.openEvents ? { openEvents: opts.openEvents } : { events: opts.events },
     model: opts.model,
+    ...opts.fallbackModels ? { fallbackModels: opts.fallbackModels } : {},
+    ...opts.modelFailover === false ? { modelFailover: false } : {},
+    ...opts.onModelFallback ? { onModelFallback: opts.onModelFallback } : {},
     isRenderableInteraction: opts.isRenderableInteraction,
     declineInteraction: opts.declineInteraction,
     promoteFilePart: opts.promoteFilePart,
@@ -2064,6 +2267,7 @@ export {
   bytesToBase64,
   chatTurnRequestInit,
   checkAttachmentType,
+  classifyTerminalFailure,
   createAssistantDraftWriter,
   createAttachmentUploadRoute,
   createChatTurnRoutes,
@@ -2073,6 +2277,7 @@ export {
   defaultValidateAttachmentPath,
   fileMentionsToParts,
   formatBytes,
+  isCommittingSandboxEvent,
   isDraftContentEvent,
   mediaTypeForMentionPath,
   mentionKindForPath,
@@ -2088,6 +2293,7 @@ export {
   sniffBinary,
   sniffMimeFromName,
   storeSupportsDraftPersistence,
+  streamWithModelFailover,
   validateSandboxMentionPath,
   withDurableChatProjection
 };