agents 0.17.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-tool-types-Cd1TZPfB.d.ts → agent-tool-types-CNyE1iz_.d.ts} +86 -3
- package/dist/agent-tool-types.d.ts +1 -1
- package/dist/{agent-tools-_E8wxUIK.d.ts → agent-tools-CSnyGvJ2.d.ts} +2 -2
- package/dist/agent-tools.d.ts +1 -1
- package/dist/chat/index.d.ts +123 -13
- package/dist/chat/index.js +108 -10
- package/dist/chat/index.js.map +1 -1
- package/dist/chat-sdk/index.d.ts +1 -1
- package/dist/client.d.ts +1 -1
- package/dist/{index-CcbnKkNh.d.ts → index-BRnybD6X.d.ts} +10 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.js +135 -4
- package/dist/index.js.map +1 -1
- package/dist/mcp/client.d.ts +1 -1
- package/dist/mcp/index.d.ts +1 -1
- package/dist/observability/index.d.ts +1 -1
- package/dist/react.d.ts +1 -1
- package/dist/{retries-CwlpAGet.d.ts → retries-CvHJwSuh.d.ts} +15 -6
- package/dist/retries.d.ts +8 -6
- package/dist/retries.js +44 -1
- package/dist/retries.js.map +1 -1
- package/dist/serializable.d.ts +1 -1
- package/dist/sub-routing.d.ts +1 -1
- package/dist/workflows.d.ts +1 -1
- package/docs/chat-agents.md +22 -13
- package/package.json +1 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { n as AgentEmail } from "./internal_context-Dg4Cgjcu.js";
|
|
2
|
-
import { t as RetryOptions } from "./retries-
|
|
2
|
+
import { t as RetryOptions } from "./retries-CvHJwSuh.js";
|
|
3
3
|
import {
|
|
4
4
|
n as Observability,
|
|
5
5
|
r as ObservabilityEvent,
|
|
6
6
|
s as MCPObservabilityEvent
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-BRnybD6X.js";
|
|
8
8
|
import { t as AgentMcpOAuthProvider } from "./do-oauth-client-provider-D4ZwyBDu.js";
|
|
9
9
|
import {
|
|
10
10
|
_ as WorkflowEventPayload,
|
|
@@ -2349,6 +2349,15 @@ declare const DEFAULT_AGENT_STATIC_OPTIONS: {
|
|
|
2349
2349
|
agentToolReattachMaxWindowMs: number;
|
|
2350
2350
|
detachedMaxBudgetMs: number;
|
|
2351
2351
|
detachedNoProgressBudgetMs: number;
|
|
2352
|
+
/**
|
|
2353
|
+
* Consecutive alarm invocations that may end in a Durable Object memory-limit
|
|
2354
|
+
* reset (the isolate exceeded its 128 MB limit) before the alarm-boundary
|
|
2355
|
+
* circuit breaker stops the platform's auto-retry loop and seals the looping
|
|
2356
|
+
* work (#1825). A small budget tolerates a genuinely transient memory spike;
|
|
2357
|
+
* a deterministic OOM (the work's footprint, not the platform, is the cause)
|
|
2358
|
+
* is bounded here regardless of whether the in-DO recovery budgets could run.
|
|
2359
|
+
*/
|
|
2360
|
+
maxAlarmMemoryLimitStrikes: number;
|
|
2352
2361
|
};
|
|
2353
2362
|
/**
|
|
2354
2363
|
* Configuration options for the Agent.
|
|
@@ -2436,6 +2445,20 @@ interface AgentStaticOptions {
|
|
|
2436
2445
|
* `detached: { noProgressBudgetMs }`.
|
|
2437
2446
|
*/
|
|
2438
2447
|
detachedNoProgressBudgetMs?: number;
|
|
2448
|
+
/**
|
|
2449
|
+
* Consecutive alarm invocations that may end in a Durable Object memory-limit
|
|
2450
|
+
* reset (the isolate exceeded its 128 MB limit) before the alarm-boundary
|
|
2451
|
+
* circuit breaker stops the platform's auto-retry loop and seals the looping
|
|
2452
|
+
* recovery work (#1825). Default: 3. Set to `0` to seal on the first such
|
|
2453
|
+
* reset. This is the universal backstop for the case where the in-DO recovery
|
|
2454
|
+
* budgets (`chatRecovery.maxOomRetries` / `maxRecoveryWork`) can't engage
|
|
2455
|
+
* because the OOM bypasses them — e.g. it is thrown before the budget code
|
|
2456
|
+
* runs, or its own writes also OOM. The boundary handler runs at the outermost
|
|
2457
|
+
* alarm frame, after the heavy turn has unwound and GC has reclaimed its
|
|
2458
|
+
* footprint, so its small seal/purge writes can land where mid-turn writes
|
|
2459
|
+
* could not.
|
|
2460
|
+
*/
|
|
2461
|
+
maxAlarmMemoryLimitStrikes?: number;
|
|
2439
2462
|
}
|
|
2440
2463
|
declare function getCurrentAgent<
|
|
2441
2464
|
T extends Agent<Cloudflare.Env> = Agent<Cloudflare.Env>
|
|
@@ -3420,6 +3443,66 @@ declare class Agent<
|
|
|
3420
3443
|
* See {@link https://developers.cloudflare.com/agents/api-reference/schedule-tasks/}
|
|
3421
3444
|
*/
|
|
3422
3445
|
alarm(): Promise<void>;
|
|
3446
|
+
/**
|
|
3447
|
+
* The alarm body: PartyServer init + due-schedule processing + housekeeping +
|
|
3448
|
+
* next-alarm arm. Extracted from {@link alarm} so the memory-limit circuit
|
|
3449
|
+
* breaker can wrap it at the outermost frame (see {@link alarm}).
|
|
3450
|
+
*/
|
|
3451
|
+
private _cf_runAlarmBody;
|
|
3452
|
+
/**
|
|
3453
|
+
* Durable storage key for the alarm memory-limit strike counter (#1825).
|
|
3454
|
+
*/
|
|
3455
|
+
private static readonly _CF_OOM_ALARM_STRIKES_KEY;
|
|
3456
|
+
/**
|
|
3457
|
+
* The schedule row id currently executing in the alarm loop, so the
|
|
3458
|
+
* memory-limit circuit breaker can purge the exact looping row (#1825).
|
|
3459
|
+
* `undefined` outside a callback (e.g. an OOM from `super.alarm()`/onStart).
|
|
3460
|
+
*/
|
|
3461
|
+
private _cf_executingScheduleRowId?;
|
|
3462
|
+
/**
|
|
3463
|
+
* The schedule-callback names whose alarm rows drive a recovery loop that can
|
|
3464
|
+
* deterministically OOM. The base agent has none; chat hosts (`Think`,
|
|
3465
|
+
* `AIChatAgent`) override this to return their recovery continuation callbacks
|
|
3466
|
+
* so the circuit breaker can surgically back them off / purge them WITHOUT
|
|
3467
|
+
* disturbing unrelated scheduled tasks. See {@link _cf_handleAlarmMemoryLimitReset}.
|
|
3468
|
+
*/
|
|
3469
|
+
protected _cf_recoveryAlarmCallbacks(): string[];
|
|
3470
|
+
/**
|
|
3471
|
+
* Hook for a host to terminalize ("seal") any in-flight recovery work as an
|
|
3472
|
+
* out-of-memory exhaustion when the alarm circuit breaker trips at its strike
|
|
3473
|
+
* budget (#1825). Runs at the outermost alarm frame (post-unwind, so writes
|
|
3474
|
+
* can land). Default: no-op. Chat hosts override to fire `onExhausted` + the
|
|
3475
|
+
* terminal banner and persist the sealed incident.
|
|
3476
|
+
*/
|
|
3477
|
+
protected _cf_sealMemoryLimitedRecovery(): Promise<void>;
|
|
3478
|
+
/**
|
|
3479
|
+
* Clear the durable memory-limit strike counter after a clean alarm so the
|
|
3480
|
+
* circuit breaker counts CONSECUTIVE resets rather than lifetime ones
|
|
3481
|
+
* (#1825). Reads first (cheap, usually cached) and only writes when a strike
|
|
3482
|
+
* is actually recorded, so the common no-strike path costs no write.
|
|
3483
|
+
* Best-effort: a stale strike only costs one extra tolerated spike later.
|
|
3484
|
+
*/
|
|
3485
|
+
private _cf_clearAlarmMemoryLimitStrikes;
|
|
3486
|
+
/**
|
|
3487
|
+
* Alarm-boundary circuit breaker for Durable Object memory-limit resets
|
|
3488
|
+
* (#1825). The in-DO recovery budgets (`chatRecovery.maxOomRetries` /
|
|
3489
|
+
* `maxRecoveryWork`) only engage if their code runs AND its writes land; a
|
|
3490
|
+
* severe OOM can defeat both — thrown before the budget runs (boot hydration),
|
|
3491
|
+
* or its own small writes also OOM under memory pressure. In that case the
|
|
3492
|
+
* error reaches {@link alarm} and, unhandled, the platform auto-retries the
|
|
3493
|
+
* alarm indefinitely (re-running the doomed, billable turn each cycle).
|
|
3494
|
+
*
|
|
3495
|
+
* This runs at the OUTERMOST frame: the heavy turn has unwound and GC has
|
|
3496
|
+
* reclaimed its footprint, so the small writes here can land where mid-turn
|
|
3497
|
+
* ones (e.g. give-up's incident read) OOMed. A durable strike counter tolerates
|
|
3498
|
+
* a few resets (a transient spike may clear), backing off the recovery rows so
|
|
3499
|
+
* the retry is not a hot loop. At the `maxAlarmMemoryLimitStrikes` budget it
|
|
3500
|
+
* seals the recovery work and purges the looping rows so the loop — and the
|
|
3501
|
+
* bill — stops. Each step is best-effort: even these tiny writes can OOM, but
|
|
3502
|
+
* swallowing (not re-throwing) still halts the platform's auto-retry, and a
|
|
3503
|
+
* later wake re-arms legitimate schedules.
|
|
3504
|
+
*/
|
|
3505
|
+
private _cf_handleAlarmMemoryLimitReset;
|
|
3423
3506
|
/**
|
|
3424
3507
|
* Intercept incoming HTTP/WS requests whose URL contains a
|
|
3425
3508
|
* `/sub/{child-class}/{child-name}` marker and forward them to
|
|
@@ -5305,4 +5388,4 @@ export {
|
|
|
5305
5388
|
EmailRoutingOptions as z,
|
|
5306
5389
|
ElicitResult$1 as zt
|
|
5307
5390
|
};
|
|
5308
|
-
//# sourceMappingURL=agent-tool-types-
|
|
5391
|
+
//# sourceMappingURL=agent-tool-types-CNyE1iz_.d.ts.map
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
o as AgentToolEventMessage,
|
|
5
5
|
s as AgentToolEventState,
|
|
6
6
|
y as AgentToolStoredChunk
|
|
7
|
-
} from "./agent-tool-types-
|
|
7
|
+
} from "./agent-tool-types-CNyE1iz_.js";
|
|
8
8
|
|
|
9
9
|
//#region src/chat/agent-tools.d.ts
|
|
10
10
|
type AgentToolProgressEmitResult = "emitted" | "coalesced" | "inactive";
|
|
@@ -116,4 +116,4 @@ export {
|
|
|
116
116
|
interceptAgentToolBroadcast as s,
|
|
117
117
|
AgentToolBroadcastHooks as t
|
|
118
118
|
};
|
|
119
|
-
//# sourceMappingURL=agent-tools-
|
|
119
|
+
//# sourceMappingURL=agent-tools-CSnyGvJ2.d.ts.map
|
package/dist/agent-tools.d.ts
CHANGED
package/dist/chat/index.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
a as AgentToolEvent,
|
|
5
5
|
o as AgentToolEventMessage,
|
|
6
6
|
s as AgentToolEventState
|
|
7
|
-
} from "../agent-tool-types-
|
|
7
|
+
} from "../agent-tool-types-CNyE1iz_.js";
|
|
8
8
|
import {
|
|
9
9
|
n as ClientToolSchema,
|
|
10
10
|
r as createToolsFromClientSchemas,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
r as AgentToolProgressEmitResult,
|
|
19
19
|
s as interceptAgentToolBroadcast,
|
|
20
20
|
t as AgentToolBroadcastHooks
|
|
21
|
-
} from "../agent-tools-
|
|
21
|
+
} from "../agent-tools-CSnyGvJ2.js";
|
|
22
22
|
import { JSONSchema7, UIMessage } from "ai";
|
|
23
23
|
import { Connection } from "agents";
|
|
24
24
|
|
|
@@ -457,6 +457,9 @@ type ChatRecoveryExhaustedContext = Pick<
|
|
|
457
457
|
* - `work_budget_exceeded` — the turn kept producing content but exceeded the
|
|
458
458
|
* configured `maxRecoveryWork` runaway-loop budget.
|
|
459
459
|
* - `recovery_aborted` — the caller's `shouldKeepRecovering` hook returned `false`.
|
|
460
|
+
* - `out_of_memory` — recovery attempts kept hitting a Durable Object
|
|
461
|
+
* memory-limit reset (the isolate exceeded its 128 MB limit) until the
|
|
462
|
+
* tight `maxOomRetries` budget drained (#1825).
|
|
460
463
|
* - `stable_timeout` — a recovery attempt kept timing out waiting for the
|
|
461
464
|
* isolate to reach stable state until the budget drained (extreme churn).
|
|
462
465
|
* - `max_recovery_window_exceeded` — DEPRECATED. The old absolute incident-age
|
|
@@ -494,7 +497,7 @@ type ChatRecoveryProgressContext = {
|
|
|
494
497
|
/**
|
|
495
498
|
* Configuration for durable chat recovery. `true` uses these defaults:
|
|
496
499
|
* `maxAttempts: 10`, `stableTimeoutMs: 10_000`, `noProgressTimeoutMs: 300_000`
|
|
497
|
-
* (5 min), `maxRecoveryWork:
|
|
500
|
+
* (5 min), `maxRecoveryWork: 1000`, and a generic terminal message.
|
|
498
501
|
*
|
|
499
502
|
* **Apply this as a class field or in the constructor — never assign it in
|
|
500
503
|
* `onStart()`.** On every wake the SDK evaluates recovery budgets (and may seal
|
|
@@ -522,12 +525,31 @@ type ChatRecoveryConfig =
|
|
|
522
525
|
/**
|
|
523
526
|
* Runaway-loop guard. Maximum recovery WORK — produced content/tool units
|
|
524
527
|
* since the incident began — before a still-progressing turn is sealed
|
|
525
|
-
* with `reason="work_budget_exceeded"`. Defaults to `
|
|
526
|
-
*
|
|
527
|
-
*
|
|
528
|
-
*
|
|
528
|
+
* with `reason="work_budget_exceeded"`. Defaults to `1000`: a generous
|
|
529
|
+
* backstop that bounds wasted re-run cost when a turn keeps emitting a
|
|
530
|
+
* little content but never converges (e.g. an isolate that OOMs mid-stream
|
|
531
|
+
* on every recovery — #1825 — which otherwise resets the attempt cap and
|
|
532
|
+
* no-progress window forever). Work only accrues from the first
|
|
533
|
+
* interruption until the turn completes, so a normal interrupted turn
|
|
534
|
+
* never approaches it. A very long agentic turn under heavy interruption
|
|
535
|
+
* that legitimately needs more can raise this (or set `Infinity` to
|
|
536
|
+
* disable the framework cap and bound the runaway via `shouldKeepRecovering`
|
|
537
|
+
* instead).
|
|
529
538
|
*/
|
|
530
539
|
maxRecoveryWork?: number;
|
|
540
|
+
/**
|
|
541
|
+
* Tight retry budget for the specific case of a Durable Object isolate
|
|
542
|
+
* exceeding its memory limit and being reset mid-turn. An OOM is usually
|
|
543
|
+
* deterministic (the turn's working set no longer fits in 128 MB), so
|
|
544
|
+
* re-running re-OOMs; but a single OOM can be a transient spike, so
|
|
545
|
+
* recovery retries this many times before sealing with
|
|
546
|
+
* `reason="out_of_memory"`. Counts only attempts that ended in an OOM (not
|
|
547
|
+
* total attempts), so a turn interrupted by deploys is unaffected.
|
|
548
|
+
* Defaults to `3`. Set `0` to seal on the first OOM. Much tighter than
|
|
549
|
+
* `maxRecoveryWork` because an OOM is attributable and each re-run is
|
|
550
|
+
* expensive (it re-runs the model).
|
|
551
|
+
*/
|
|
552
|
+
maxOomRetries?: number;
|
|
531
553
|
/**
|
|
532
554
|
* Caller policy consulted on each recovery attempt from the second
|
|
533
555
|
* onward — it is NOT called on the first detection (the attempt that
|
|
@@ -555,6 +577,7 @@ type ResolvedChatRecoveryConfig = {
|
|
|
555
577
|
terminalMessage: string;
|
|
556
578
|
noProgressTimeoutMs: number;
|
|
557
579
|
maxRecoveryWork: number;
|
|
580
|
+
maxOomRetries: number;
|
|
558
581
|
shouldKeepRecovering?: (
|
|
559
582
|
ctx: ChatRecoveryProgressContext
|
|
560
583
|
) => boolean | Promise<boolean>;
|
|
@@ -2094,6 +2117,18 @@ type ChatRecoveryIncident = {
|
|
|
2094
2117
|
* from an older build is never falsely sealed.
|
|
2095
2118
|
*/
|
|
2096
2119
|
workBaseline?: number;
|
|
2120
|
+
/**
|
|
2121
|
+
* Count of recovery attempts for this incident that ended in a Durable Object
|
|
2122
|
+
* memory-limit reset (the isolate exceeded its 128 MB limit — see
|
|
2123
|
+
* `isDurableObjectMemoryLimitReset`). An OOM is a poison signal: re-running the
|
|
2124
|
+
* same memory-heavy turn deterministically re-OOMs, so unlike a deploy/eviction
|
|
2125
|
+
* it must NOT credit forward progress and must be bounded by a tight,
|
|
2126
|
+
* OOM-specific budget (`maxOomRetries`) rather than the generic attempt cap
|
|
2127
|
+
* (which resets on progress). Bumped by `ChatRecoveryEngine.recordOomAndDecide`
|
|
2128
|
+
* when a recovery callback observes an OOM; exceeding `maxOomRetries` seals the
|
|
2129
|
+
* incident with `reason="out_of_memory"` (#1825). Optional for backward-compat.
|
|
2130
|
+
*/
|
|
2131
|
+
oomAttempts?: number;
|
|
2097
2132
|
};
|
|
2098
2133
|
declare const CHAT_RECOVERY_INCIDENT_KEY_PREFIX = "cf:chat-recovery:incident:";
|
|
2099
2134
|
/**
|
|
@@ -2123,12 +2158,42 @@ declare const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
|
|
|
2123
2158
|
*/
|
|
2124
2159
|
declare const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
|
|
2125
2160
|
/**
|
|
2126
|
-
* Runaway-loop guard default
|
|
2127
|
-
*
|
|
2128
|
-
*
|
|
2129
|
-
*
|
|
2130
|
-
|
|
2131
|
-
|
|
2161
|
+
* Runaway-loop guard default — the framework-imposed backstop on cumulative
|
|
2162
|
+
* recovery WORK (produced content/tool units) since an incident opened.
|
|
2163
|
+
*
|
|
2164
|
+
* Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
|
|
2165
|
+
* *mechanism* but no default cap, so a progressing turn was never terminated on
|
|
2166
|
+
* its own. Production issue #1825 showed that this is a footgun: an isolate that
|
|
2167
|
+
* OOMs mid-stream still credits a little progress before it dies, which resets
|
|
2168
|
+
* BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
|
|
2169
|
+
* every wake — and a fast crash loop (each attempt inside the alarm-debounce
|
|
2170
|
+
* window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
|
|
2171
|
+
* ONLY instrument whose meter still climbs across such a loop is disabled, so
|
|
2172
|
+
* recovery re-runs the turn (and its LLM calls) forever.
|
|
2173
|
+
*
|
|
2174
|
+
* A finite default closes that loop out of the box: work climbs regardless of
|
|
2175
|
+
* debounce/progress resets, so a content-emitting runaway is always sealed with
|
|
2176
|
+
* `reason="work_budget_exceeded"`. The value is deliberately generous — it
|
|
2177
|
+
* bounds wasted re-run cost without clipping a normal interrupted turn (work
|
|
2178
|
+
* only accrues from the first interruption until the turn completes, after which
|
|
2179
|
+
* the incident is deleted). A very long agentic turn under heavy interruption
|
|
2180
|
+
* that legitimately needs more should raise `maxRecoveryWork` (or set it to
|
|
2181
|
+
* `Infinity` to restore the pre-#1825 unbounded behavior).
|
|
2182
|
+
*/
|
|
2183
|
+
declare const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1000;
|
|
2184
|
+
/**
|
|
2185
|
+
* Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
|
|
2186
|
+
* (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
|
|
2187
|
+
* working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
|
|
2188
|
+
* But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
|
|
2189
|
+
* across the global scope / noisy neighbors), so recovery retries a small number
|
|
2190
|
+
* of times before sealing with `reason="out_of_memory"` rather than abandoning a
|
|
2191
|
+
* turn that one more attempt might have completed. Far tighter than the generic
|
|
2192
|
+
* `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
|
|
2193
|
+
* expensive (it re-runs the model). Counts attempts that ended in an OOM, not
|
|
2194
|
+
* total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
|
|
2195
|
+
*/
|
|
2196
|
+
declare const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
|
|
2132
2197
|
declare const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 10000;
|
|
2133
2198
|
/**
|
|
2134
2199
|
* Delay before retrying a recovery that timed out waiting for stable state.
|
|
@@ -2182,6 +2247,21 @@ declare function sweepStaleChatRecoveryIncidents(
|
|
|
2182
2247
|
storage: Pick<DurableObjectStorage, "list" | "delete">,
|
|
2183
2248
|
now: number
|
|
2184
2249
|
): Promise<void>;
|
|
2250
|
+
/**
|
|
2251
|
+
* List the persisted recovery incidents that are still live (status
|
|
2252
|
+
* `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
|
|
2253
|
+
* (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
|
|
2254
|
+
* (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
|
|
2255
|
+
* Lists by the incident key prefix so the storage layout stays encapsulated.
|
|
2256
|
+
*/
|
|
2257
|
+
declare function listActiveChatRecoveryIncidents(
|
|
2258
|
+
storage: Pick<DurableObjectStorage, "list">
|
|
2259
|
+
): Promise<
|
|
2260
|
+
{
|
|
2261
|
+
key: string;
|
|
2262
|
+
incident: ChatRecoveryIncident;
|
|
2263
|
+
}[]
|
|
2264
|
+
>;
|
|
2185
2265
|
/**
|
|
2186
2266
|
* Summarize a child agent's persisted recovery incidents for the parent's
|
|
2187
2267
|
* agent-tool reattach decision: `"in-progress"` if any incident is still live
|
|
@@ -2794,6 +2874,34 @@ declare class ChatRecoveryEngine {
|
|
|
2794
2874
|
data: Record<string, unknown> | undefined;
|
|
2795
2875
|
fallbackMaxAttempts: number;
|
|
2796
2876
|
}): Promise<boolean>;
|
|
2877
|
+
/**
|
|
2878
|
+
* Record that a recovery callback observed a Durable Object memory-limit reset
|
|
2879
|
+
* (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
|
|
2880
|
+
* and decide what to do next (#1825).
|
|
2881
|
+
*
|
|
2882
|
+
* Bumps the incident's durable `oomAttempts` counter, then:
|
|
2883
|
+
* - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
|
|
2884
|
+
* reschedule of the SAME callback (same machinery as
|
|
2885
|
+
* {@link rescheduleAfterStableTimeout}: the executing one-shot row is
|
|
2886
|
+
* deleted only after the callback returns, so an idempotent reschedule
|
|
2887
|
+
* would dedup onto that doomed row) and returns `"rescheduled"`. The small
|
|
2888
|
+
* delay lets a transient memory spike clear before the re-run;
|
|
2889
|
+
* - otherwise leaves the incremented count persisted (so a begin-path
|
|
2890
|
+
* re-evaluation agrees) and returns `"exhausted"` — the caller then
|
|
2891
|
+
* terminalizes via the give-up path with `reason="out_of_memory"`.
|
|
2892
|
+
*
|
|
2893
|
+
* Returns `"exhausted"` when there is no incident to track against (no id /
|
|
2894
|
+
* record gone): an OOM we cannot bound must seal rather than loop. Unlike a
|
|
2895
|
+
* stable-state retry this is gated by the OOM-specific budget, NOT the generic
|
|
2896
|
+
* attempt cap — re-running an OOM streams a little "progress" that would
|
|
2897
|
+
* otherwise reset the attempt cap forever (the #1825 loop).
|
|
2898
|
+
*/
|
|
2899
|
+
recordOomAndDecide(input: {
|
|
2900
|
+
incidentId: string | undefined;
|
|
2901
|
+
callback: ChatRecoveryScheduleCallback;
|
|
2902
|
+
data: Record<string, unknown> | undefined;
|
|
2903
|
+
maxOomRetries: number;
|
|
2904
|
+
}): Promise<"rescheduled" | "exhausted">;
|
|
2797
2905
|
/**
|
|
2798
2906
|
* Give up on a recovery turn whose retry budget drained, terminalizing it so
|
|
2799
2907
|
* it can never become an eternal spinner (#1645). The shared spine both
|
|
@@ -3179,6 +3287,7 @@ export {
|
|
|
3179
3287
|
type ContinuationSpec,
|
|
3180
3288
|
ContinuationState,
|
|
3181
3289
|
DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS,
|
|
3290
|
+
DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES,
|
|
3182
3291
|
DEFAULT_CHAT_RECOVERY_MAX_WORK,
|
|
3183
3292
|
DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS,
|
|
3184
3293
|
DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS,
|
|
@@ -3254,6 +3363,7 @@ export {
|
|
|
3254
3363
|
interceptAgentToolBroadcast,
|
|
3255
3364
|
isReplayChunk,
|
|
3256
3365
|
iterateWithStallWatchdog,
|
|
3366
|
+
listActiveChatRecoveryIncidents,
|
|
3257
3367
|
normalizeToolInput,
|
|
3258
3368
|
parseProtocolMessage,
|
|
3259
3369
|
partAwaitsClientInteraction,
|
package/dist/chat/index.js
CHANGED
|
@@ -2445,12 +2445,42 @@ const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
|
|
|
2445
2445
|
*/
|
|
2446
2446
|
const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
|
|
2447
2447
|
/**
|
|
2448
|
-
* Runaway-loop guard default
|
|
2449
|
-
*
|
|
2450
|
-
*
|
|
2451
|
-
*
|
|
2452
|
-
|
|
2453
|
-
|
|
2448
|
+
* Runaway-loop guard default — the framework-imposed backstop on cumulative
|
|
2449
|
+
* recovery WORK (produced content/tool units) since an incident opened.
|
|
2450
|
+
*
|
|
2451
|
+
* Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
|
|
2452
|
+
* *mechanism* but no default cap, so a progressing turn was never terminated on
|
|
2453
|
+
* its own. Production issue #1825 showed that this is a footgun: an isolate that
|
|
2454
|
+
* OOMs mid-stream still credits a little progress before it dies, which resets
|
|
2455
|
+
* BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
|
|
2456
|
+
* every wake — and a fast crash loop (each attempt inside the alarm-debounce
|
|
2457
|
+
* window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
|
|
2458
|
+
* ONLY instrument whose meter still climbs across such a loop is disabled, so
|
|
2459
|
+
* recovery re-runs the turn (and its LLM calls) forever.
|
|
2460
|
+
*
|
|
2461
|
+
* A finite default closes that loop out of the box: work climbs regardless of
|
|
2462
|
+
* debounce/progress resets, so a content-emitting runaway is always sealed with
|
|
2463
|
+
* `reason="work_budget_exceeded"`. The value is deliberately generous — it
|
|
2464
|
+
* bounds wasted re-run cost without clipping a normal interrupted turn (work
|
|
2465
|
+
* only accrues from the first interruption until the turn completes, after which
|
|
2466
|
+
* the incident is deleted). A very long agentic turn under heavy interruption
|
|
2467
|
+
* that legitimately needs more should raise `maxRecoveryWork` (or set it to
|
|
2468
|
+
* `Infinity` to restore the pre-#1825 unbounded behavior).
|
|
2469
|
+
*/
|
|
2470
|
+
const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1e3;
|
|
2471
|
+
/**
|
|
2472
|
+
* Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
|
|
2473
|
+
* (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
|
|
2474
|
+
* working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
|
|
2475
|
+
* But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
|
|
2476
|
+
* across the global scope / noisy neighbors), so recovery retries a small number
|
|
2477
|
+
* of times before sealing with `reason="out_of_memory"` rather than abandoning a
|
|
2478
|
+
* turn that one more attempt might have completed. Far tighter than the generic
|
|
2479
|
+
* `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
|
|
2480
|
+
* expensive (it re-runs the model). Counts attempts that ended in an OOM, not
|
|
2481
|
+
* total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
|
|
2482
|
+
*/
|
|
2483
|
+
const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
|
|
2454
2484
|
const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 1e4;
|
|
2455
2485
|
/**
|
|
2456
2486
|
* Delay before retrying a recovery that timed out waiting for stable state.
|
|
@@ -2497,6 +2527,7 @@ function resolveChatRecoveryConfig(raw) {
|
|
|
2497
2527
|
terminalMessage: custom?.terminalMessage ?? "The assistant was interrupted and could not recover. Please try again.",
|
|
2498
2528
|
noProgressTimeoutMs: Math.max(0, Math.floor(custom?.noProgressTimeoutMs ?? 3e5)),
|
|
2499
2529
|
maxRecoveryWork: typeof custom?.maxRecoveryWork === "number" && custom.maxRecoveryWork >= 0 ? custom.maxRecoveryWork : DEFAULT_CHAT_RECOVERY_MAX_WORK,
|
|
2530
|
+
maxOomRetries: typeof custom?.maxOomRetries === "number" && custom.maxOomRetries >= 0 ? Math.floor(custom.maxOomRetries) : 3,
|
|
2500
2531
|
...custom?.shouldKeepRecovering ? { shouldKeepRecovering: custom.shouldKeepRecovering } : {},
|
|
2501
2532
|
...custom?.onExhausted ? { onExhausted: custom.onExhausted } : {}
|
|
2502
2533
|
};
|
|
@@ -2539,6 +2570,22 @@ async function sweepStaleChatRecoveryIncidents(storage, now) {
|
|
|
2539
2570
|
for (let i = 0; i < staleKeys.length; i += 128) await storage.delete(staleKeys.slice(i, i + 128));
|
|
2540
2571
|
}
|
|
2541
2572
|
/**
|
|
2573
|
+
* List the persisted recovery incidents that are still live (status
|
|
2574
|
+
* `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
|
|
2575
|
+
* (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
|
|
2576
|
+
* (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
|
|
2577
|
+
* Lists by the incident key prefix so the storage layout stays encapsulated.
|
|
2578
|
+
*/
|
|
2579
|
+
async function listActiveChatRecoveryIncidents(storage) {
|
|
2580
|
+
const entries = await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX });
|
|
2581
|
+
const active = [];
|
|
2582
|
+
for (const [key, incident] of entries) if (incident && (incident.status === "detected" || incident.status === "scheduled" || incident.status === "attempting")) active.push({
|
|
2583
|
+
key,
|
|
2584
|
+
incident
|
|
2585
|
+
});
|
|
2586
|
+
return active;
|
|
2587
|
+
}
|
|
2588
|
+
/**
|
|
2542
2589
|
* Summarize a child agent's persisted recovery incidents for the parent's
|
|
2543
2590
|
* agent-tool reattach decision: `"in-progress"` if any incident is still live
|
|
2544
2591
|
* (detected/scheduled/attempting), else `"failed"` if any terminalized
|
|
@@ -2725,10 +2772,12 @@ async function evaluateChatRecoveryIncident(input) {
|
|
|
2725
2772
|
const progress = Math.max(prevProgress, currentProgress);
|
|
2726
2773
|
const work = progress - workBaseline;
|
|
2727
2774
|
const workBudgetExceeded = existing != null && Number.isFinite(config.maxRecoveryWork) && work > config.maxRecoveryWork;
|
|
2775
|
+
const oomAttempts = existing?.oomAttempts ?? 0;
|
|
2776
|
+
const oomBudgetExceeded = existing != null && !awaitingClientInteraction && oomAttempts > config.maxOomRetries;
|
|
2728
2777
|
const debounced = existing != null && !madeProgress && now - existing.lastAttemptAt < 3e4;
|
|
2729
2778
|
const attempt = madeProgress ? 1 : debounced ? existing?.attempt ?? 1 : (existing?.attempt ?? 0) + 1;
|
|
2730
2779
|
let abortedByCaller = false;
|
|
2731
|
-
if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && attempt <= config.maxAttempts) try {
|
|
2780
|
+
if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && !oomBudgetExceeded && attempt <= config.maxAttempts) try {
|
|
2732
2781
|
const ctx = {
|
|
2733
2782
|
incidentId,
|
|
2734
2783
|
requestId: identity.requestId,
|
|
@@ -2743,7 +2792,7 @@ async function evaluateChatRecoveryIncident(input) {
|
|
|
2743
2792
|
} catch (error) {
|
|
2744
2793
|
input.onShouldKeepRecoveringError?.(error);
|
|
2745
2794
|
}
|
|
2746
|
-
const exhausted = !awaitingClientInteraction && (noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
|
|
2795
|
+
const exhausted = !awaitingClientInteraction && (oomBudgetExceeded || noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
|
|
2747
2796
|
const incident = {
|
|
2748
2797
|
incidentId,
|
|
2749
2798
|
requestId: identity.requestId,
|
|
@@ -2757,7 +2806,8 @@ async function evaluateChatRecoveryIncident(input) {
|
|
|
2757
2806
|
lastProgressAt,
|
|
2758
2807
|
progress,
|
|
2759
2808
|
workBaseline,
|
|
2760
|
-
...
|
|
2809
|
+
...oomAttempts > 0 ? { oomAttempts } : {},
|
|
2810
|
+
...exhausted ? { reason: oomBudgetExceeded ? "out_of_memory" : workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
|
|
2761
2811
|
};
|
|
2762
2812
|
const events = [];
|
|
2763
2813
|
if (!existing) events.push({
|
|
@@ -3031,6 +3081,54 @@ var ChatRecoveryEngine = class {
|
|
|
3031
3081
|
return true;
|
|
3032
3082
|
}
|
|
3033
3083
|
/**
|
|
3084
|
+
* Record that a recovery callback observed a Durable Object memory-limit reset
|
|
3085
|
+
* (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
|
|
3086
|
+
* and decide what to do next (#1825).
|
|
3087
|
+
*
|
|
3088
|
+
* Bumps the incident's durable `oomAttempts` counter, then:
|
|
3089
|
+
* - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
|
|
3090
|
+
* reschedule of the SAME callback (same machinery as
|
|
3091
|
+
* {@link rescheduleAfterStableTimeout}: the executing one-shot row is
|
|
3092
|
+
* deleted only after the callback returns, so an idempotent reschedule
|
|
3093
|
+
* would dedup onto that doomed row) and returns `"rescheduled"`. The small
|
|
3094
|
+
* delay lets a transient memory spike clear before the re-run;
|
|
3095
|
+
* - otherwise leaves the incremented count persisted (so a begin-path
|
|
3096
|
+
* re-evaluation agrees) and returns `"exhausted"` — the caller then
|
|
3097
|
+
* terminalizes via the give-up path with `reason="out_of_memory"`.
|
|
3098
|
+
*
|
|
3099
|
+
* Returns `"exhausted"` when there is no incident to track against (no id /
|
|
3100
|
+
* record gone): an OOM we cannot bound must seal rather than loop. Unlike a
|
|
3101
|
+
* stable-state retry this is gated by the OOM-specific budget, NOT the generic
|
|
3102
|
+
* attempt cap — re-running an OOM streams a little "progress" that would
|
|
3103
|
+
* otherwise reset the attempt cap forever (the #1825 loop).
|
|
3104
|
+
*/
|
|
3105
|
+
async recordOomAndDecide(input) {
|
|
3106
|
+
const { adapter } = this;
|
|
3107
|
+
if (!input.incidentId) return "exhausted";
|
|
3108
|
+
const key = chatRecoveryIncidentKey(input.incidentId);
|
|
3109
|
+
const incident = await adapter.getIncident(key);
|
|
3110
|
+
if (!incident) return "exhausted";
|
|
3111
|
+
const oomAttempts = (incident.oomAttempts ?? 0) + 1;
|
|
3112
|
+
if (oomAttempts > input.maxOomRetries) {
|
|
3113
|
+
await adapter.putIncident(key, {
|
|
3114
|
+
...incident,
|
|
3115
|
+
oomAttempts,
|
|
3116
|
+
lastAttemptAt: adapter.now(),
|
|
3117
|
+
reason: "out_of_memory"
|
|
3118
|
+
});
|
|
3119
|
+
return "exhausted";
|
|
3120
|
+
}
|
|
3121
|
+
await adapter.putIncident(key, {
|
|
3122
|
+
...incident,
|
|
3123
|
+
oomAttempts,
|
|
3124
|
+
status: "scheduled",
|
|
3125
|
+
lastAttemptAt: adapter.now(),
|
|
3126
|
+
reason: "oom_retry"
|
|
3127
|
+
});
|
|
3128
|
+
await adapter.scheduleRecovery(input.callback, input.data ?? {}, "stable_timeout_retry", 3);
|
|
3129
|
+
return "rescheduled";
|
|
3130
|
+
}
|
|
3131
|
+
/**
|
|
3034
3132
|
* Give up on a recovery turn whose retry budget drained, terminalizing it so
|
|
3035
3133
|
* it can never become an eternal spinner (#1645). The shared spine both
|
|
3036
3134
|
* packages repeated verbatim:
|
|
@@ -3305,6 +3403,6 @@ async function* iterateWithStallWatchdog(source, timeoutMs, onStall) {
|
|
|
3305
3403
|
}
|
|
3306
3404
|
}
|
|
3307
3405
|
//#endregion
|
|
3308
|
-
export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
|
|
3406
|
+
export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, listActiveChatRecoveryIncidents, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
|
|
3309
3407
|
|
|
3310
3408
|
//# sourceMappingURL=index.js.map
|