agents 0.28.1 → 0.29.0

package/dist/chat/index.d.ts

@@ -9,7 +9,7 @@ import {
   _ as FiberRecoveryContext$1,
   at as isPositionlessAgentToolChunk,
   z as AgentToolEventState
-} from "../agent-routing-BMvWTFEp.js";
+} from "../agent-routing-CDbpVjYq.js";
 import { _ as TaskStep, m as TaskRunOptions } from "../tasks-E89KdwkS.js";
 import {
   n as ClientToolSchema,
@@ -34,7 +34,7 @@ import {
   r as AgentToolProgressEmitResult,
   s as interceptAgentToolBroadcast,
   t as AgentToolBroadcastHooks
-} from "../agent-tools-ohoewW-B.js";
+} from "../agent-tools-C8DH8Yii.js";
 import { l as SessionMessage } from "../types-B7LojTe4.js";
 import { ModelMessage, UIMessage } from "ai";
 import { Connection, FiberRecoveryContext } from "agents";
@@ -578,9 +578,11 @@ type ChatRecoveryExhaustedContext = Pick<
    * - `work_budget_exceeded` — the turn kept producing content but exceeded the
    *   configured `maxRecoveryWork` runaway-loop budget.
    * - `recovery_aborted` — the caller's `shouldKeepRecovering` hook returned `false`.
-   * - `out_of_memory` — recovery attempts kept hitting a Durable Object
-   *   memory-limit reset (the isolate exceeded its 128 MB limit) until the
-   *   tight `maxOomRetries` budget drained (#1825).
+   * - `interrupted_attempts_exceeded` — the turn's attempts kept dying before
+   *   they finished (an isolate reset such as a memory-limit reset or a
+   *   deploy) until `maxInterruptedAttempts` of them had been interrupted.
+   * - `out_of_memory` — the alarm memory-limit circuit breaker sealed the
+   *   recovery after repeated Durable Object memory-limit resets (#1825).
    * - `stable_timeout` — a recovery attempt kept timing out waiting for the
    *   isolate to reach stable state until the budget drained (extreme churn).
    * - `max_recovery_window_exceeded` — DEPRECATED. The old absolute incident-age
@@ -620,7 +622,8 @@ type ChatRecoveryProgressContext = {
  * Configuration for durable chat recovery, which is always enabled for chat
  * turns. `true` uses these defaults: `maxAttempts: 10`,
  * `stableTimeoutMs: 10_000`, `noProgressTimeoutMs: 300_000` (5 min),
- * `maxRecoveryWork: 10000`, and a generic terminal message. Assign an object to
+ * `maxRecoveryWork: 10000`, `maxInterruptedAttempts: 5`, and a generic
+ * terminal message. Assign an object to
  * tune recovery without disabling its durable bookkeeping.
  *
  * **Apply this as a class field or in the constructor — never assign it in
@@ -642,8 +645,10 @@ type ChatRecoveryConfig =
        * How long an incident may go WITHOUT forward progress before it is
        * sealed with `reason="no_progress_timeout"`. This is the primary
        * stuck-turn bound. It **resets on every progress-bearing attempt**, so a
-       * turn that keeps producing content survives unbounded interruption while
-       * a genuinely idle turn is sealed within the window. Defaults to 5 min.
+       * turn that keeps producing content is never sealed by this bound, while
+       * a genuinely idle turn is sealed within the window. Repeated
+       * interruption is bounded separately by `maxInterruptedAttempts`.
+       * Defaults to 5 min.
        */
       noProgressTimeoutMs?: number;
       /**
@@ -670,16 +675,29 @@ type ChatRecoveryConfig =
        */
       maxRecoveryWork?: number;
       /**
-       * Tight retry budget for the specific case of a Durable Object isolate
-       * exceeding its memory limit and being reset mid-turn. An OOM is usually
-       * deterministic (the turn's working set no longer fits in 128 MB), so
-       * re-running re-OOMs; but a single OOM can be a transient spike, so
-       * recovery retries this many times before sealing with
-       * `reason="out_of_memory"`. Counts only attempts that ended in an OOM (not
-       * total attempts), so a turn interrupted by deploys is unaffected.
-       * Defaults to `3`. Set `0` to seal on the first OOM. Much tighter than
-       * `maxRecoveryWork` because an OOM is attributable and each re-run is
-       * expensive (it re-runs the model).
+       * How many of one turn's attempts may be interrupted before recovery
+       * gives up with `reason="interrupted_attempts_exceeded"`. An attempt is
+       * interrupted when its isolate dies before the attempt finishes: a
+       * Durable Object memory-limit reset, a deploy, or any other reset. The
+       * original turn counts, so the default of `5` runs the model at most five
+       * times for a turn that dies every time.
+       *
+       * Each attempt records itself before it runs anything heavy, and the
+       * next attempt counts a record left by an isolate that died, because
+       * the isolate that died cannot record anything. Streamed output never
+       * resets the count: a turn that re-runs and dies again produces output
+       * too (#2613). A memory-limit reset and a
+       * deploy look the same from the next isolate, so a turn interrupted by
+       * this many deploys in a row is also sealed. Values below `1` are read
+       * as `1`; `Infinity` disables the bound.
+       */
+      maxInterruptedAttempts?: number;
+      /**
+       * @deprecated Use `maxInterruptedAttempts`. When that is unset, this is
+       * read as `maxInterruptedAttempts: maxOomRetries + 1`, which keeps the
+       * old number of retries. The budget now counts every interrupted
+       * attempt, not only the ones that ended in an observed memory-limit
+       * reset, because a reset isolate cannot record what killed it.
        */
       maxOomRetries?: number;
       /**
@@ -709,7 +727,7 @@ type ResolvedChatRecoveryConfig = {
   terminalMessage: string;
   noProgressTimeoutMs: number;
   maxRecoveryWork: number;
-  maxOomRetries: number;
+  maxInterruptedAttempts: number;
   shouldKeepRecovering?: (
     ctx: ChatRecoveryProgressContext
   ) => boolean | Promise<boolean>;
@@ -1366,6 +1384,36 @@ declare function unwrapChatFiberSnapshot<Kind extends string>(
  * {@link chatRecoveryIncidentId}).
  */
 type ChatRecoveryKind = "retry" | "continue";
+/**
+ * One recovery attempt in flight, recorded on the incident before the attempt
+ * does anything heavy. `isolate` identifies the object instance that started
+ * it; `token` identifies the attempt. An isolate killed by a memory-limit
+ * reset cannot write anything, so it cannot clear this record: a later
+ * isolate that finds a record it did not write knows that attempt died
+ * (#2613).
+ */
+type ChatRecoveryAttempt = {
+  readonly isolate: string;
+  readonly token: string;
+};
+/**
+ * Why an incident evaluation is running.
+ *
+ * - `interrupted`: a later isolate is replaying a turn run whose isolate died
+ *   before the run finished. The replay is itself an attempt, recorded as
+ *   `attempt`.
+ * - `stalled`: a live isolate caught a stalled or failed stream. Nothing
+ *   died, so nothing is counted; transient retries are bounded by
+ *   `transientRetries` instead.
+ */
+type ChatRecoveryTrigger =
+  | {
+      readonly kind: "interrupted";
+      readonly attempt: ChatRecoveryAttempt;
+    }
+  | {
+      readonly kind: "stalled";
+    };
 /**
  * Durable per-incident recovery record.
  *
@@ -1416,17 +1464,25 @@ type ChatRecoveryIncident = {
    */
   workBaseline?: number;
   /**
-   * Count of recovery attempts for this incident that ended in a Durable Object
-   * memory-limit reset (the isolate exceeded its 128 MB limit — see
-   * `isDurableObjectMemoryLimitReset`). An OOM is a poison signal: re-running the
-   * same memory-heavy turn deterministically re-OOMs, so unlike a deploy/eviction
-   * it must NOT credit forward progress and must be bounded by a tight,
-   * OOM-specific budget (`maxOomRetries`) rather than the generic attempt cap
-   * (which resets on progress). Bumped by `ChatRecoveryEngine.recordOomAndDecide`
-   * when a recovery callback observes an OOM; exceeding `maxOomRetries` seals the
-   * incident with `reason="out_of_memory"` (#1825). Optional for backward-compat.
+   * How many of this turn's attempts were interrupted: their isolate died
+   * before they finished. The original turn's death is counted when its run
+   * is first replayed; every later death is found through the
+   * `attemptsInFlight` entries it left behind. Never counted on the failure
+   * path, because an isolate killed by a memory-limit reset cannot write
+   * (#2613), and never reset by progress, because a turn that re-runs and
+   * dies again streams output too. Reaching `maxInterruptedAttempts` seals the
+   * incident with `reason="interrupted_attempts_exceeded"`. Optional for
+   * backward-compat.
+   */
+  interruptedAttempts?: number;
+  /**
+   * The recovery attempts currently running for this turn. Each is written
+   * and confirmed before it does any heavy work and removed when it
+   * finishes, so a replay and a callback that overlap are tracked apart. An
+   * entry written by another isolate means that isolate died mid-attempt;
+   * however many entries it left, it counts as one interruption.
    */
-  oomAttempts?: number;
+  attemptsInFlight?: ChatRecoveryAttempt[];
   /**
    * Recoveries scheduled for this incident after a transient or rate-limited
    * stream error. Drives the retry backoff and is capped by `maxAttempts`.
@@ -1501,18 +1557,14 @@ declare const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
  */
 declare const DEFAULT_CHAT_RECOVERY_MAX_WORK = 10000;
 /**
- * Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
- * (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
- * working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
- * But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
- * across the global scope / noisy neighbors), so recovery retries a small number
- * of times before sealing with `reason="out_of_memory"` rather than abandoning a
- * turn that one more attempt might have completed. Far tighter than the generic
- * `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
- * expensive (it re-runs the model). Counts attempts that ended in an OOM, not
- * total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
- */
-declare const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
+ * Default number of interrupted attempts a turn may accumulate before recovery
+ * seals it with `reason="interrupted_attempts_exceeded"` (#2613). The original
+ * turn counts, so a turn that dies every time runs the model at most five
+ * times. A deterministic memory-limit reset dies every time; a deploy rarely
+ * lands on the same turn five times in a row, so five tolerates churn while
+ * still bounding a crash loop.
+ */
+declare const DEFAULT_CHAT_RECOVERY_MAX_INTERRUPTED_ATTEMPTS = 5;
 declare const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 10000;
 /**
  * Delay before retrying a recovery that timed out waiting for stable state.
@@ -1739,7 +1791,8 @@ type EvaluateChatRecoveryIncidentInput = {
     recoveryRootRequestId?: string | null;
     latestUserMessageId?: string | null;
     recoveryKind: ChatRecoveryKind;
-  } /** Fully-resolved recovery config. */;
+  } /** Why this evaluation is running; decides whether it counts an interruption. */;
+  trigger: ChatRecoveryTrigger /** Fully-resolved recovery config. */;
   config: ResolvedChatRecoveryConfig /** The existing incident for this identity, or `null` if this is fresh. */;
   existing: ChatRecoveryIncident | null /** Current value of the durable monotonic progress counter. */;
   currentProgress: number;
@@ -1817,6 +1870,13 @@ interface BeginChatRecoveryIncidentInput {
   recoveryRootRequestId?: string | null;
   latestUserMessageId?: string | null;
   recoveryKind: ChatRecoveryKind;
+  /**
+   * Why the incident is being opened: the replay of a turn run whose isolate
+   * died (`interrupted`), or a stall a live isolate caught (`stalled`). Only
+   * a replay can count against `maxInterruptedAttempts`, and only a replay
+   * records itself as the attempt in flight.
+   */
+  trigger: "interrupted" | "stalled";
   /** Test-only clock injection for deterministic debounce/window timing. */
   nowMs?: number;
 }
@@ -1824,6 +1884,12 @@ interface BeginChatRecoveryIncidentResult {
   incident: ChatRecoveryIncident;
   config: ResolvedChatRecoveryConfig;
   exhausted: boolean;
+  /**
+   * The attempt this opening recorded as in flight (an `interrupted` trigger
+   * on an incident that is not sealed), for the caller to finish. `null`
+   * when nothing was recorded.
+   */
+  attempt: ChatRecoveryAttempt | null;
 }
 /**
  * Package-specific host operations the engine drives during incident
@@ -1878,6 +1944,14 @@ interface ChatRecoveryAdapter {
   listActiveIncidents?(): Promise<ChatRecoveryIncident[]>;
   /** Persist the evaluated incident under `key`. */
   putIncident(key: string, incident: ChatRecoveryIncident): Promise<void>;
+  /**
+   * Resolve once every write issued so far is durably confirmed (Durable
+   * Object `storage.sync()`). {@link ChatRecoveryEngine.beginIncident} waits
+   * on it before returning: the attempt that follows can run in the same
+   * invocation, and a memory-limit reset discards writes that were not yet
+   * confirmed, which would lose the interruption count (#2613).
+   */
+  syncStorage(): Promise<void>;
   /**
    * Delete the incident record under `key`. The engine calls this on the
    * terminal `completed` transition (a completed recovery is never retried, so
@@ -2098,8 +2172,32 @@ interface ChatFiberWakeHooks<TClassify> {
  * this class owns the surrounding sequence and its ordering invariants.
  */
 declare class ChatRecoveryEngine {
+  #private;
   private readonly adapter;
   constructor(adapter: ChatRecoveryAdapter);
+  /**
+   * Run one recovery attempt under the interruption budget (#2613): a
+   * `_chatRecoveryContinue` / `_chatRecoveryRetry` callback and the turn it
+   * starts. Before `run` does anything heavy, the attempt counts any earlier
+   * attempt that died (entries another isolate left in `attemptsInFlight`)
+   * and records itself, confirmed with `syncStorage`. Once
+   * `maxInterruptedAttempts` attempts have died the turn is sealed with
+   * `interrupted_attempts_exceeded` and `run` is never called. The entry is
+   * removed when `run` settles, whatever its outcome.
+   *
+   * This catches deaths the replay of an interrupted turn run cannot see, such
+   * as a reset before the callback starts its turn, which leaves no turn run
+   * to replay. A callback for a sealed incident does nothing: the turn was
+   * already terminalized, and running it would re-run the model. A payload
+   * without an incident, or one whose incident is gone, runs untracked.
+   *
+   * @param input - The callback being run, its payload, and the work.
+   */
+  runRecoveryAttempt(input: {
+    callback: ChatRecoveryScheduleCallback;
+    data: Record<string, unknown> | undefined;
+    run: () => Promise<void>;
+  }): Promise<void>;
   /**
    * Open or re-evaluate the recovery incident for `input`, persist the result,
    * and broadcast its lifecycle events. Returns the incident, the resolved
@@ -2223,34 +2321,6 @@ declare class ChatRecoveryEngine {
   cancelScheduledRecovery(requestId: string): Promise<boolean>;
   /** Whether the user cancelled this incident's scheduled recovery. */
   isRecoveryCancelled(incidentId: string | undefined): Promise<boolean>;
-  /**
-   * Record that a recovery callback observed a Durable Object memory-limit reset
-   * (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
-   * and decide what to do next (#1825).
-   *
-   * Bumps the incident's durable `oomAttempts` counter, then:
-   *  - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
-   *    reschedule of the SAME callback (same machinery as
-   *    {@link rescheduleAfterStableTimeout}: the executing attempt settles
-   *    only after the callback returns, so an idempotent reschedule would
-   *    dedup onto that doomed attempt) and returns `"rescheduled"`. The small
-   *    delay lets a transient memory spike clear before the re-run;
-   *  - otherwise leaves the incremented count persisted (so a begin-path
-   *    re-evaluation agrees) and returns `"exhausted"` — the caller then
-   *    terminalizes via the give-up path with `reason="out_of_memory"`.
-   *
-   * Returns `"exhausted"` when there is no incident to track against (no id /
-   * record gone): an OOM we cannot bound must seal rather than loop. Unlike a
-   * stable-state retry this is gated by the OOM-specific budget, NOT the generic
-   * attempt cap — re-running an OOM streams a little "progress" that would
-   * otherwise reset the attempt cap forever (the #1825 loop).
-   */
-  recordOomAndDecide(input: {
-    incidentId: string | undefined;
-    callback: ChatRecoveryScheduleCallback;
-    data: Record<string, unknown> | undefined;
-    maxOomRetries: number;
-  }): Promise<"rescheduled" | "exhausted">;
   /**
    * Give up on a recovery turn whose retry budget drained, terminalizing it so
    * it can never become an eternal spinner (#1645). The shared spine both
@@ -3704,6 +3774,7 @@ export {
   type ChatFiberWakeHooks,
   type ChatProtocolEvent,
   type ChatRecoveryAdapter,
+  type ChatRecoveryAttempt,
   type ChatRecoveryCodec,
   type ChatRecoveryConfig,
   type ChatRecoveryContext,
@@ -3720,6 +3791,7 @@ export {
   type ChatRecoveryTaskHooks,
   type ChatRecoveryTaskInput,
   type ChatRecoveryTaskReason,
+  type ChatRecoveryTrigger,
   type ChatResponseResult,
   ChatStreamStalledError,
   type ChatStreamStatus,
@@ -3738,7 +3810,7 @@ export {
   type ContinuationSpec,
   ContinuationState,
   DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS,
-  DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES,
+  DEFAULT_CHAT_RECOVERY_MAX_INTERRUPTED_ATTEMPTS,
   DEFAULT_CHAT_RECOVERY_MAX_WORK,
   DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS,
   DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS,