agents 0.17.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,10 @@
1
1
  import { n as AgentEmail } from "./internal_context-Dg4Cgjcu.js";
2
- import { t as RetryOptions } from "./retries-CwlpAGet.js";
2
+ import { t as RetryOptions } from "./retries-CvHJwSuh.js";
3
3
  import {
4
4
  n as Observability,
5
5
  r as ObservabilityEvent,
6
6
  s as MCPObservabilityEvent
7
- } from "./index-CcbnKkNh.js";
7
+ } from "./index-BRnybD6X.js";
8
8
  import { t as AgentMcpOAuthProvider } from "./do-oauth-client-provider-D4ZwyBDu.js";
9
9
  import {
10
10
  _ as WorkflowEventPayload,
@@ -2349,6 +2349,15 @@ declare const DEFAULT_AGENT_STATIC_OPTIONS: {
2349
2349
  agentToolReattachMaxWindowMs: number;
2350
2350
  detachedMaxBudgetMs: number;
2351
2351
  detachedNoProgressBudgetMs: number;
2352
+ /**
2353
+ * Consecutive alarm invocations that may end in a Durable Object memory-limit
2354
+ * reset (the isolate exceeded its 128 MB limit) before the alarm-boundary
2355
+ * circuit breaker stops the platform's auto-retry loop and seals the looping
2356
+ * work (#1825). A small budget tolerates a genuinely transient memory spike;
2357
+ * a deterministic OOM (the work's footprint, not the platform, is the cause)
2358
+ * is bounded here regardless of whether the in-DO recovery budgets could run.
2359
+ */
2360
+ maxAlarmMemoryLimitStrikes: number;
2352
2361
  };
2353
2362
  /**
2354
2363
  * Configuration options for the Agent.
@@ -2436,6 +2445,20 @@ interface AgentStaticOptions {
2436
2445
  * `detached: { noProgressBudgetMs }`.
2437
2446
  */
2438
2447
  detachedNoProgressBudgetMs?: number;
2448
+ /**
2449
+ * Consecutive alarm invocations that may end in a Durable Object memory-limit
2450
+ * reset (the isolate exceeded its 128 MB limit) before the alarm-boundary
2451
+ * circuit breaker stops the platform's auto-retry loop and seals the looping
2452
+ * recovery work (#1825). Default: 3. Set to `0` to seal on the first such
2453
+ * reset. This is the universal backstop for the case where the in-DO recovery
2454
+ * budgets (`chatRecovery.maxOomRetries` / `maxRecoveryWork`) can't engage
2455
+ * because the OOM bypasses them — e.g. it is thrown before the budget code
2456
+ * runs, or its own writes also OOM. The boundary handler runs at the outermost
2457
+ * alarm frame, after the heavy turn has unwound and GC has reclaimed its
2458
+ * footprint, so its small seal/purge writes can land where mid-turn writes
2459
+ * could not.
2460
+ */
2461
+ maxAlarmMemoryLimitStrikes?: number;
2439
2462
  }
2440
2463
  declare function getCurrentAgent<
2441
2464
  T extends Agent<Cloudflare.Env> = Agent<Cloudflare.Env>
@@ -3420,6 +3443,66 @@ declare class Agent<
3420
3443
  * See {@link https://developers.cloudflare.com/agents/api-reference/schedule-tasks/}
3421
3444
  */
3422
3445
  alarm(): Promise<void>;
3446
+ /**
3447
+ * The alarm body: PartyServer init + due-schedule processing + housekeeping +
3448
+ * next-alarm arm. Extracted from {@link alarm} so the memory-limit circuit
3449
+ * breaker can wrap it at the outermost frame (see {@link alarm}).
3450
+ */
3451
+ private _cf_runAlarmBody;
3452
+ /**
3453
+ * Durable storage key for the alarm memory-limit strike counter (#1825).
3454
+ */
3455
+ private static readonly _CF_OOM_ALARM_STRIKES_KEY;
3456
+ /**
3457
+ * The schedule row id currently executing in the alarm loop, so the
3458
+ * memory-limit circuit breaker can purge the exact looping row (#1825).
3459
+ * `undefined` outside a callback (e.g. an OOM from `super.alarm()`/onStart).
3460
+ */
3461
+ private _cf_executingScheduleRowId?;
3462
+ /**
3463
+ * The schedule-callback names whose alarm rows drive a recovery loop that can
3464
+ * deterministically OOM. The base agent has none; chat hosts (`Think`,
3465
+ * `AIChatAgent`) override this to return their recovery continuation callbacks
3466
+ * so the circuit breaker can surgically back them off / purge them WITHOUT
3467
+ * disturbing unrelated scheduled tasks. See {@link _cf_handleAlarmMemoryLimitReset}.
3468
+ */
3469
+ protected _cf_recoveryAlarmCallbacks(): string[];
3470
+ /**
3471
+ * Hook for a host to terminalize ("seal") any in-flight recovery work as an
3472
+ * out-of-memory exhaustion when the alarm circuit breaker trips at its strike
3473
+ * budget (#1825). Runs at the outermost alarm frame (post-unwind, so writes
3474
+ * can land). Default: no-op. Chat hosts override to fire `onExhausted` + the
3475
+ * terminal banner and persist the sealed incident.
3476
+ */
3477
+ protected _cf_sealMemoryLimitedRecovery(): Promise<void>;
3478
+ /**
3479
+ * Clear the durable memory-limit strike counter after a clean alarm so the
3480
+ * circuit breaker counts CONSECUTIVE resets rather than lifetime ones
3481
+ * (#1825). Reads first (cheap, usually cached) and only writes when a strike
3482
+ * is actually recorded, so the common no-strike path costs no write.
3483
+ * Best-effort: a stale strike only costs one extra tolerated spike later.
3484
+ */
3485
+ private _cf_clearAlarmMemoryLimitStrikes;
3486
+ /**
3487
+ * Alarm-boundary circuit breaker for Durable Object memory-limit resets
3488
+ * (#1825). The in-DO recovery budgets (`chatRecovery.maxOomRetries` /
3489
+ * `maxRecoveryWork`) only engage if their code runs AND its writes land; a
3490
+ * severe OOM can defeat both — thrown before the budget runs (boot hydration),
3491
+ * or its own small writes also OOM under memory pressure. In that case the
3492
+ * error reaches {@link alarm} and, unhandled, the platform auto-retries the
3493
+ * alarm indefinitely (re-running the doomed, billable turn each cycle).
3494
+ *
3495
+ * This runs at the OUTERMOST frame: the heavy turn has unwound and GC has
3496
+ * reclaimed its footprint, so the small writes here can land where mid-turn
3497
+ * ones (e.g. give-up's incident read) OOMed. A durable strike counter tolerates
3498
+ * a few resets (a transient spike may clear), backing off the recovery rows so
3499
+ * the retry is not a hot loop. At the `maxAlarmMemoryLimitStrikes` budget it
3500
+ * seals the recovery work and purges the looping rows so the loop — and the
3501
+ * bill — stops. Each step is best-effort: even these tiny writes can OOM, but
3502
+ * swallowing (not re-throwing) still halts the platform's auto-retry, and a
3503
+ * later wake re-arms legitimate schedules.
3504
+ */
3505
+ private _cf_handleAlarmMemoryLimitReset;
3423
3506
  /**
3424
3507
  * Intercept incoming HTTP/WS requests whose URL contains a
3425
3508
  * `/sub/{child-class}/{child-name}` marker and forward them to
@@ -5305,4 +5388,4 @@ export {
5305
5388
  EmailRoutingOptions as z,
5306
5389
  ElicitResult$1 as zt
5307
5390
  };
5308
- //# sourceMappingURL=agent-tool-types-Cd1TZPfB.d.ts.map
5391
+ //# sourceMappingURL=agent-tool-types-CNyE1iz_.d.ts.map
@@ -24,7 +24,7 @@ import {
24
24
  w as RunAgentToolOptions,
25
25
  x as ChatCapableAgentClass,
26
26
  y as AgentToolStoredChunk
27
- } from "./agent-tool-types-Cd1TZPfB.js";
27
+ } from "./agent-tool-types-CNyE1iz_.js";
28
28
  export {
29
29
  AGENT_TOOL_MILESTONE_PART,
30
30
  AGENT_TOOL_PROGRESS_PART,
@@ -4,7 +4,7 @@ import {
4
4
  o as AgentToolEventMessage,
5
5
  s as AgentToolEventState,
6
6
  y as AgentToolStoredChunk
7
- } from "./agent-tool-types-Cd1TZPfB.js";
7
+ } from "./agent-tool-types-CNyE1iz_.js";
8
8
 
9
9
  //#region src/chat/agent-tools.d.ts
10
10
  type AgentToolProgressEmitResult = "emitted" | "coalesced" | "inactive";
@@ -116,4 +116,4 @@ export {
116
116
  interceptAgentToolBroadcast as s,
117
117
  AgentToolBroadcastHooks as t
118
118
  };
119
- //# sourceMappingURL=agent-tools-_E8wxUIK.d.ts.map
119
+ //# sourceMappingURL=agent-tools-CSnyGvJ2.d.ts.map
@@ -19,7 +19,7 @@ import {
19
19
  w as RunAgentToolOptions,
20
20
  x as ChatCapableAgentClass,
21
21
  y as AgentToolStoredChunk
22
- } from "./agent-tool-types-Cd1TZPfB.js";
22
+ } from "./agent-tool-types-CNyE1iz_.js";
23
23
  import { Tool } from "ai";
24
24
 
25
25
  //#region src/agent-tools.d.ts
@@ -4,7 +4,7 @@ import {
4
4
  a as AgentToolEvent,
5
5
  o as AgentToolEventMessage,
6
6
  s as AgentToolEventState
7
- } from "../agent-tool-types-Cd1TZPfB.js";
7
+ } from "../agent-tool-types-CNyE1iz_.js";
8
8
  import {
9
9
  n as ClientToolSchema,
10
10
  r as createToolsFromClientSchemas,
@@ -18,7 +18,7 @@ import {
18
18
  r as AgentToolProgressEmitResult,
19
19
  s as interceptAgentToolBroadcast,
20
20
  t as AgentToolBroadcastHooks
21
- } from "../agent-tools-_E8wxUIK.js";
21
+ } from "../agent-tools-CSnyGvJ2.js";
22
22
  import { JSONSchema7, UIMessage } from "ai";
23
23
  import { Connection } from "agents";
24
24
 
@@ -457,6 +457,9 @@ type ChatRecoveryExhaustedContext = Pick<
457
457
  * - `work_budget_exceeded` — the turn kept producing content but exceeded the
458
458
  * configured `maxRecoveryWork` runaway-loop budget.
459
459
  * - `recovery_aborted` — the caller's `shouldKeepRecovering` hook returned `false`.
460
+ * - `out_of_memory` — recovery attempts kept hitting a Durable Object
461
+ * memory-limit reset (the isolate exceeded its 128 MB limit) until the
462
+ * tight `maxOomRetries` budget drained (#1825).
460
463
  * - `stable_timeout` — a recovery attempt kept timing out waiting for the
461
464
  * isolate to reach stable state until the budget drained (extreme churn).
462
465
  * - `max_recovery_window_exceeded` — DEPRECATED. The old absolute incident-age
@@ -494,7 +497,7 @@ type ChatRecoveryProgressContext = {
494
497
  /**
495
498
  * Configuration for durable chat recovery. `true` uses these defaults:
496
499
  * `maxAttempts: 10`, `stableTimeoutMs: 10_000`, `noProgressTimeoutMs: 300_000`
497
- * (5 min), `maxRecoveryWork: Infinity`, and a generic terminal message.
500
+ * (5 min), `maxRecoveryWork: 1000`, and a generic terminal message.
498
501
  *
499
502
  * **Apply this as a class field or in the constructor — never assign it in
500
503
  * `onStart()`.** On every wake the SDK evaluates recovery budgets (and may seal
@@ -522,12 +525,31 @@ type ChatRecoveryConfig =
522
525
  /**
523
526
  * Runaway-loop guard. Maximum recovery WORK — produced content/tool units
524
527
  * since the incident began — before a still-progressing turn is sealed
525
- * with `reason="work_budget_exceeded"`. Defaults to `Infinity` (no cap):
526
- * the SDK never terminates a progressing turn on its own. Set a finite
527
- * value (or use `shouldKeepRecovering`) to bound a loop that keeps
528
- * emitting content but never converges.
528
+ * with `reason="work_budget_exceeded"`. Defaults to `1000`: a generous
529
+ * backstop that bounds wasted re-run cost when a turn keeps emitting a
530
+ * little content but never converges (e.g. an isolate that OOMs mid-stream
531
+ * on every recovery #1825 — which otherwise resets the attempt cap and
532
+ * no-progress window forever). Work only accrues from the first
533
+ * interruption until the turn completes, so a normal interrupted turn
534
+ * never approaches it. A very long agentic turn under heavy interruption
535
+ * that legitimately needs more can raise this (or set `Infinity` to
536
+ * disable the framework cap and bound the runaway via `shouldKeepRecovering`
537
+ * instead).
529
538
  */
530
539
  maxRecoveryWork?: number;
540
+ /**
541
+ * Tight retry budget for the specific case of a Durable Object isolate
542
+ * exceeding its memory limit and being reset mid-turn. An OOM is usually
543
+ * deterministic (the turn's working set no longer fits in 128 MB), so
544
+ * re-running re-OOMs; but a single OOM can be a transient spike, so
545
+ * recovery retries this many times before sealing with
546
+ * `reason="out_of_memory"`. Counts only attempts that ended in an OOM (not
547
+ * total attempts), so a turn interrupted by deploys is unaffected.
548
+ * Defaults to `3`. Set `0` to seal on the first OOM. Much tighter than
549
+ * `maxRecoveryWork` because an OOM is attributable and each re-run is
550
+ * expensive (it re-runs the model).
551
+ */
552
+ maxOomRetries?: number;
531
553
  /**
532
554
  * Caller policy consulted on each recovery attempt from the second
533
555
  * onward — it is NOT called on the first detection (the attempt that
@@ -555,6 +577,7 @@ type ResolvedChatRecoveryConfig = {
555
577
  terminalMessage: string;
556
578
  noProgressTimeoutMs: number;
557
579
  maxRecoveryWork: number;
580
+ maxOomRetries: number;
558
581
  shouldKeepRecovering?: (
559
582
  ctx: ChatRecoveryProgressContext
560
583
  ) => boolean | Promise<boolean>;
@@ -2094,6 +2117,18 @@ type ChatRecoveryIncident = {
2094
2117
  * from an older build is never falsely sealed.
2095
2118
  */
2096
2119
  workBaseline?: number;
2120
+ /**
2121
+ * Count of recovery attempts for this incident that ended in a Durable Object
2122
+ * memory-limit reset (the isolate exceeded its 128 MB limit — see
2123
+ * `isDurableObjectMemoryLimitReset`). An OOM is a poison signal: re-running the
2124
+ * same memory-heavy turn deterministically re-OOMs, so unlike a deploy/eviction
2125
+ * it must NOT credit forward progress and must be bounded by a tight,
2126
+ * OOM-specific budget (`maxOomRetries`) rather than the generic attempt cap
2127
+ * (which resets on progress). Bumped by `ChatRecoveryEngine.recordOomAndDecide`
2128
+ * when a recovery callback observes an OOM; exceeding `maxOomRetries` seals the
2129
+ * incident with `reason="out_of_memory"` (#1825). Optional for backward-compat.
2130
+ */
2131
+ oomAttempts?: number;
2097
2132
  };
2098
2133
  declare const CHAT_RECOVERY_INCIDENT_KEY_PREFIX = "cf:chat-recovery:incident:";
2099
2134
  /**
@@ -2123,12 +2158,42 @@ declare const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
2123
2158
  */
2124
2159
  declare const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
2125
2160
  /**
2126
- * Runaway-loop guard default. `Infinity` = no SDK-imposed work cap: a turn that
2127
- * keeps making forward progress is never terminated by the framework on its own
2128
- * (rfc-chat-recovery-work-budget). Integrators bound a content-emitting runaway
2129
- * by setting `maxRecoveryWork` or a `shouldKeepRecovering` predicate.
2130
- */
2131
- declare const DEFAULT_CHAT_RECOVERY_MAX_WORK: number;
2161
+ * Runaway-loop guard default the framework-imposed backstop on cumulative
2162
+ * recovery WORK (produced content/tool units) since an incident opened.
2163
+ *
2164
+ * Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
2165
+ * *mechanism* but no default cap, so a progressing turn was never terminated on
2166
+ * its own. Production issue #1825 showed that this is a footgun: an isolate that
2167
+ * OOMs mid-stream still credits a little progress before it dies, which resets
2168
+ * BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
2169
+ * every wake — and a fast crash loop (each attempt inside the alarm-debounce
2170
+ * window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
2171
+ * ONLY instrument whose meter still climbs across such a loop is disabled, so
2172
+ * recovery re-runs the turn (and its LLM calls) forever.
2173
+ *
2174
+ * A finite default closes that loop out of the box: work climbs regardless of
2175
+ * debounce/progress resets, so a content-emitting runaway is always sealed with
2176
+ * `reason="work_budget_exceeded"`. The value is deliberately generous — it
2177
+ * bounds wasted re-run cost without clipping a normal interrupted turn (work
2178
+ * only accrues from the first interruption until the turn completes, after which
2179
+ * the incident is deleted). A very long agentic turn under heavy interruption
2180
+ * that legitimately needs more should raise `maxRecoveryWork` (or set it to
2181
+ * `Infinity` to restore the pre-#1825 unbounded behavior).
2182
+ */
2183
+ declare const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1000;
2184
+ /**
2185
+ * Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
2186
+ * (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
2187
+ * working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
2188
+ * But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
2189
+ * across the global scope / noisy neighbors), so recovery retries a small number
2190
+ * of times before sealing with `reason="out_of_memory"` rather than abandoning a
2191
+ * turn that one more attempt might have completed. Far tighter than the generic
2192
+ * `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
2193
+ * expensive (it re-runs the model). Counts attempts that ended in an OOM, not
2194
+ * total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
2195
+ */
2196
+ declare const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
2132
2197
  declare const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 10000;
2133
2198
  /**
2134
2199
  * Delay before retrying a recovery that timed out waiting for stable state.
@@ -2182,6 +2247,21 @@ declare function sweepStaleChatRecoveryIncidents(
2182
2247
  storage: Pick<DurableObjectStorage, "list" | "delete">,
2183
2248
  now: number
2184
2249
  ): Promise<void>;
2250
+ /**
2251
+ * List the persisted recovery incidents that are still live (status
2252
+ * `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
2253
+ * (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
2254
+ * (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
2255
+ * Lists by the incident key prefix so the storage layout stays encapsulated.
2256
+ */
2257
+ declare function listActiveChatRecoveryIncidents(
2258
+ storage: Pick<DurableObjectStorage, "list">
2259
+ ): Promise<
2260
+ {
2261
+ key: string;
2262
+ incident: ChatRecoveryIncident;
2263
+ }[]
2264
+ >;
2185
2265
  /**
2186
2266
  * Summarize a child agent's persisted recovery incidents for the parent's
2187
2267
  * agent-tool reattach decision: `"in-progress"` if any incident is still live
@@ -2794,6 +2874,34 @@ declare class ChatRecoveryEngine {
2794
2874
  data: Record<string, unknown> | undefined;
2795
2875
  fallbackMaxAttempts: number;
2796
2876
  }): Promise<boolean>;
2877
+ /**
2878
+ * Record that a recovery callback observed a Durable Object memory-limit reset
2879
+ * (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
2880
+ * and decide what to do next (#1825).
2881
+ *
2882
+ * Bumps the incident's durable `oomAttempts` counter, then:
2883
+ * - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
2884
+ * reschedule of the SAME callback (same machinery as
2885
+ * {@link rescheduleAfterStableTimeout}: the executing one-shot row is
2886
+ * deleted only after the callback returns, so an idempotent reschedule
2887
+ * would dedup onto that doomed row) and returns `"rescheduled"`. The small
2888
+ * delay lets a transient memory spike clear before the re-run;
2889
+ * - otherwise leaves the incremented count persisted (so a begin-path
2890
+ * re-evaluation agrees) and returns `"exhausted"` — the caller then
2891
+ * terminalizes via the give-up path with `reason="out_of_memory"`.
2892
+ *
2893
+ * Returns `"exhausted"` when there is no incident to track against (no id /
2894
+ * record gone): an OOM we cannot bound must seal rather than loop. Unlike a
2895
+ * stable-state retry this is gated by the OOM-specific budget, NOT the generic
2896
+ * attempt cap — re-running an OOM streams a little "progress" that would
2897
+ * otherwise reset the attempt cap forever (the #1825 loop).
2898
+ */
2899
+ recordOomAndDecide(input: {
2900
+ incidentId: string | undefined;
2901
+ callback: ChatRecoveryScheduleCallback;
2902
+ data: Record<string, unknown> | undefined;
2903
+ maxOomRetries: number;
2904
+ }): Promise<"rescheduled" | "exhausted">;
2797
2905
  /**
2798
2906
  * Give up on a recovery turn whose retry budget drained, terminalizing it so
2799
2907
  * it can never become an eternal spinner (#1645). The shared spine both
@@ -3179,6 +3287,7 @@ export {
3179
3287
  type ContinuationSpec,
3180
3288
  ContinuationState,
3181
3289
  DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS,
3290
+ DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES,
3182
3291
  DEFAULT_CHAT_RECOVERY_MAX_WORK,
3183
3292
  DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS,
3184
3293
  DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS,
@@ -3254,6 +3363,7 @@ export {
3254
3363
  interceptAgentToolBroadcast,
3255
3364
  isReplayChunk,
3256
3365
  iterateWithStallWatchdog,
3366
+ listActiveChatRecoveryIncidents,
3257
3367
  normalizeToolInput,
3258
3368
  parseProtocolMessage,
3259
3369
  partAwaitsClientInteraction,
@@ -2445,12 +2445,42 @@ const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
2445
2445
  */
2446
2446
  const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
2447
2447
  /**
2448
- * Runaway-loop guard default. `Infinity` = no SDK-imposed work cap: a turn that
2449
- * keeps making forward progress is never terminated by the framework on its own
2450
- * (rfc-chat-recovery-work-budget). Integrators bound a content-emitting runaway
2451
- * by setting `maxRecoveryWork` or a `shouldKeepRecovering` predicate.
2452
- */
2453
- const DEFAULT_CHAT_RECOVERY_MAX_WORK = Number.POSITIVE_INFINITY;
2448
+ * Runaway-loop guard default the framework-imposed backstop on cumulative
2449
+ * recovery WORK (produced content/tool units) since an incident opened.
2450
+ *
2451
+ * Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
2452
+ * *mechanism* but no default cap, so a progressing turn was never terminated on
2453
+ * its own. Production issue #1825 showed that this is a footgun: an isolate that
2454
+ * OOMs mid-stream still credits a little progress before it dies, which resets
2455
+ * BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
2456
+ * every wake — and a fast crash loop (each attempt inside the alarm-debounce
2457
+ * window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
2458
+ * ONLY instrument whose meter still climbs across such a loop is disabled, so
2459
+ * recovery re-runs the turn (and its LLM calls) forever.
2460
+ *
2461
+ * A finite default closes that loop out of the box: work climbs regardless of
2462
+ * debounce/progress resets, so a content-emitting runaway is always sealed with
2463
+ * `reason="work_budget_exceeded"`. The value is deliberately generous — it
2464
+ * bounds wasted re-run cost without clipping a normal interrupted turn (work
2465
+ * only accrues from the first interruption until the turn completes, after which
2466
+ * the incident is deleted). A very long agentic turn under heavy interruption
2467
+ * that legitimately needs more should raise `maxRecoveryWork` (or set it to
2468
+ * `Infinity` to restore the pre-#1825 unbounded behavior).
2469
+ */
2470
+ const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1e3;
2471
+ /**
2472
+ * Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
2473
+ * (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
2474
+ * working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
2475
+ * But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
2476
+ * across the global scope / noisy neighbors), so recovery retries a small number
2477
+ * of times before sealing with `reason="out_of_memory"` rather than abandoning a
2478
+ * turn that one more attempt might have completed. Far tighter than the generic
2479
+ * `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
2480
+ * expensive (it re-runs the model). Counts attempts that ended in an OOM, not
2481
+ * total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
2482
+ */
2483
+ const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
2454
2484
  const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 1e4;
2455
2485
  /**
2456
2486
  * Delay before retrying a recovery that timed out waiting for stable state.
@@ -2497,6 +2527,7 @@ function resolveChatRecoveryConfig(raw) {
2497
2527
  terminalMessage: custom?.terminalMessage ?? "The assistant was interrupted and could not recover. Please try again.",
2498
2528
  noProgressTimeoutMs: Math.max(0, Math.floor(custom?.noProgressTimeoutMs ?? 3e5)),
2499
2529
  maxRecoveryWork: typeof custom?.maxRecoveryWork === "number" && custom.maxRecoveryWork >= 0 ? custom.maxRecoveryWork : DEFAULT_CHAT_RECOVERY_MAX_WORK,
2530
+ maxOomRetries: typeof custom?.maxOomRetries === "number" && custom.maxOomRetries >= 0 ? Math.floor(custom.maxOomRetries) : 3,
2500
2531
  ...custom?.shouldKeepRecovering ? { shouldKeepRecovering: custom.shouldKeepRecovering } : {},
2501
2532
  ...custom?.onExhausted ? { onExhausted: custom.onExhausted } : {}
2502
2533
  };
@@ -2539,6 +2570,22 @@ async function sweepStaleChatRecoveryIncidents(storage, now) {
2539
2570
  for (let i = 0; i < staleKeys.length; i += 128) await storage.delete(staleKeys.slice(i, i + 128));
2540
2571
  }
2541
2572
  /**
2573
+ * List the persisted recovery incidents that are still live (status
2574
+ * `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
2575
+ * (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
2576
+ * (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
2577
+ * Lists by the incident key prefix so the storage layout stays encapsulated.
2578
+ */
2579
+ async function listActiveChatRecoveryIncidents(storage) {
2580
+ const entries = await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX });
2581
+ const active = [];
2582
+ for (const [key, incident] of entries) if (incident && (incident.status === "detected" || incident.status === "scheduled" || incident.status === "attempting")) active.push({
2583
+ key,
2584
+ incident
2585
+ });
2586
+ return active;
2587
+ }
2588
+ /**
2542
2589
  * Summarize a child agent's persisted recovery incidents for the parent's
2543
2590
  * agent-tool reattach decision: `"in-progress"` if any incident is still live
2544
2591
  * (detected/scheduled/attempting), else `"failed"` if any terminalized
@@ -2725,10 +2772,12 @@ async function evaluateChatRecoveryIncident(input) {
2725
2772
  const progress = Math.max(prevProgress, currentProgress);
2726
2773
  const work = progress - workBaseline;
2727
2774
  const workBudgetExceeded = existing != null && Number.isFinite(config.maxRecoveryWork) && work > config.maxRecoveryWork;
2775
+ const oomAttempts = existing?.oomAttempts ?? 0;
2776
+ const oomBudgetExceeded = existing != null && !awaitingClientInteraction && oomAttempts > config.maxOomRetries;
2728
2777
  const debounced = existing != null && !madeProgress && now - existing.lastAttemptAt < 3e4;
2729
2778
  const attempt = madeProgress ? 1 : debounced ? existing?.attempt ?? 1 : (existing?.attempt ?? 0) + 1;
2730
2779
  let abortedByCaller = false;
2731
- if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && attempt <= config.maxAttempts) try {
2780
+ if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && !oomBudgetExceeded && attempt <= config.maxAttempts) try {
2732
2781
  const ctx = {
2733
2782
  incidentId,
2734
2783
  requestId: identity.requestId,
@@ -2743,7 +2792,7 @@ async function evaluateChatRecoveryIncident(input) {
2743
2792
  } catch (error) {
2744
2793
  input.onShouldKeepRecoveringError?.(error);
2745
2794
  }
2746
- const exhausted = !awaitingClientInteraction && (noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
2795
+ const exhausted = !awaitingClientInteraction && (oomBudgetExceeded || noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
2747
2796
  const incident = {
2748
2797
  incidentId,
2749
2798
  requestId: identity.requestId,
@@ -2757,7 +2806,8 @@ async function evaluateChatRecoveryIncident(input) {
2757
2806
  lastProgressAt,
2758
2807
  progress,
2759
2808
  workBaseline,
2760
- ...exhausted ? { reason: workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
2809
+ ...oomAttempts > 0 ? { oomAttempts } : {},
2810
+ ...exhausted ? { reason: oomBudgetExceeded ? "out_of_memory" : workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
2761
2811
  };
2762
2812
  const events = [];
2763
2813
  if (!existing) events.push({
@@ -3031,6 +3081,54 @@ var ChatRecoveryEngine = class {
3031
3081
  return true;
3032
3082
  }
3033
3083
  /**
3084
+ * Record that a recovery callback observed a Durable Object memory-limit reset
3085
+ * (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
3086
+ * and decide what to do next (#1825).
3087
+ *
3088
+ * Bumps the incident's durable `oomAttempts` counter, then:
3089
+ * - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
3090
+ * reschedule of the SAME callback (same machinery as
3091
+ * {@link rescheduleAfterStableTimeout}: the executing one-shot row is
3092
+ * deleted only after the callback returns, so an idempotent reschedule
3093
+ * would dedup onto that doomed row) and returns `"rescheduled"`. The small
3094
+ * delay lets a transient memory spike clear before the re-run;
3095
+ * - otherwise leaves the incremented count persisted (so a begin-path
3096
+ * re-evaluation agrees) and returns `"exhausted"` — the caller then
3097
+ * terminalizes via the give-up path with `reason="out_of_memory"`.
3098
+ *
3099
+ * Returns `"exhausted"` when there is no incident to track against (no id /
3100
+ * record gone): an OOM we cannot bound must seal rather than loop. Unlike a
3101
+ * stable-state retry this is gated by the OOM-specific budget, NOT the generic
3102
+ * attempt cap — re-running an OOM streams a little "progress" that would
3103
+ * otherwise reset the attempt cap forever (the #1825 loop).
3104
+ */
3105
+ async recordOomAndDecide(input) {
3106
+ const { adapter } = this;
3107
+ if (!input.incidentId) return "exhausted";
3108
+ const key = chatRecoveryIncidentKey(input.incidentId);
3109
+ const incident = await adapter.getIncident(key);
3110
+ if (!incident) return "exhausted";
3111
+ const oomAttempts = (incident.oomAttempts ?? 0) + 1;
3112
+ if (oomAttempts > input.maxOomRetries) {
3113
+ await adapter.putIncident(key, {
3114
+ ...incident,
3115
+ oomAttempts,
3116
+ lastAttemptAt: adapter.now(),
3117
+ reason: "out_of_memory"
3118
+ });
3119
+ return "exhausted";
3120
+ }
3121
+ await adapter.putIncident(key, {
3122
+ ...incident,
3123
+ oomAttempts,
3124
+ status: "scheduled",
3125
+ lastAttemptAt: adapter.now(),
3126
+ reason: "oom_retry"
3127
+ });
3128
+ await adapter.scheduleRecovery(input.callback, input.data ?? {}, "stable_timeout_retry", 3);
3129
+ return "rescheduled";
3130
+ }
3131
+ /**
3034
3132
  * Give up on a recovery turn whose retry budget drained, terminalizing it so
3035
3133
  * it can never become an eternal spinner (#1645). The shared spine both
3036
3134
  * packages repeated verbatim:
@@ -3305,6 +3403,6 @@ async function* iterateWithStallWatchdog(source, timeoutMs, onStall) {
3305
3403
  }
3306
3404
  }
3307
3405
  //#endregion
3308
- export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
3406
+ export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, listActiveChatRecoveryIncidents, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
3309
3407
 
3310
3408
  //# sourceMappingURL=index.js.map