agents 0.17.0 → 0.17.3

package/dist/chat/index.js

@@ -1,5 +1,5 @@
 import { i as normalizeToolInput, n as getPartialStreamText, r as isReplayChunk, t as applyChunkToParts } from "../message-builder-BymO4N_D.js";
-import { i as interceptAgentToolBroadcast, n as applyAgentToolEvent, r as createAgentToolEventState, t as AgentToolProgressEmitter } from "../agent-tools-BXlsuX0d.js";
+import { i as interceptAgentToolBroadcast, n as applyAgentToolEvent, r as createAgentToolEventState, t as AgentToolProgressEmitter } from "../agent-tools-y7zLfw4Q.js";
 import { t as truncateToolOutput } from "../tool-output-truncation-CNnnGZQ3.js";
 import { n as transition, r as StreamAccumulator, t as MessageType } from "../wire-types-nflOzNuU.js";
 import { jsonSchema, tool } from "ai";
@@ -2445,12 +2445,42 @@ const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
 */
 const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
 /**
-* Runaway-loop guard default. `Infinity` = no SDK-imposed work cap: a turn that
-* keeps making forward progress is never terminated by the framework on its own
-* (rfc-chat-recovery-work-budget). Integrators bound a content-emitting runaway
-* by setting `maxRecoveryWork` or a `shouldKeepRecovering` predicate.
-*/
-const DEFAULT_CHAT_RECOVERY_MAX_WORK = Number.POSITIVE_INFINITY;
+* Runaway-loop guard default — the framework-imposed backstop on cumulative
+* recovery WORK (produced content/tool units) since an incident opened.
+*
+* Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
+* *mechanism* but no default cap, so a progressing turn was never terminated on
+* its own. Production issue #1825 showed that this is a footgun: an isolate that
+* OOMs mid-stream still credits a little progress before it dies, which resets
+* BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
+* every wake — and a fast crash loop (each attempt inside the alarm-debounce
+* window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
+* ONLY instrument whose meter still climbs across such a loop is disabled, so
+* recovery re-runs the turn (and its LLM calls) forever.
+*
+* A finite default closes that loop out of the box: work climbs regardless of
+* debounce/progress resets, so a content-emitting runaway is always sealed with
+* `reason="work_budget_exceeded"`. The value is deliberately generous — it
+* bounds wasted re-run cost without clipping a normal interrupted turn (work
+* only accrues from the first interruption until the turn completes, after which
+* the incident is deleted). A very long agentic turn under heavy interruption
+* that legitimately needs more should raise `maxRecoveryWork` (or set it to
+* `Infinity` to restore the pre-#1825 unbounded behavior).
+*/
+const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1e3;
+/**
+* Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
+* (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
+* working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
+* But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
+* across the global scope / noisy neighbors), so recovery retries a small number
+* of times before sealing with `reason="out_of_memory"` rather than abandoning a
+* turn that one more attempt might have completed. Far tighter than the generic
+* `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
+* expensive (it re-runs the model). Counts attempts that ended in an OOM, not
+* total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
+*/
+const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
 const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 1e4;
 /**
 * Delay before retrying a recovery that timed out waiting for stable state.
@@ -2497,6 +2527,7 @@ function resolveChatRecoveryConfig(raw) {
 		terminalMessage: custom?.terminalMessage ?? "The assistant was interrupted and could not recover. Please try again.",
 		noProgressTimeoutMs: Math.max(0, Math.floor(custom?.noProgressTimeoutMs ?? 3e5)),
 		maxRecoveryWork: typeof custom?.maxRecoveryWork === "number" && custom.maxRecoveryWork >= 0 ? custom.maxRecoveryWork : DEFAULT_CHAT_RECOVERY_MAX_WORK,
+		maxOomRetries: typeof custom?.maxOomRetries === "number" && custom.maxOomRetries >= 0 ? Math.floor(custom.maxOomRetries) : 3,
 		...custom?.shouldKeepRecovering ? { shouldKeepRecovering: custom.shouldKeepRecovering } : {},
 		...custom?.onExhausted ? { onExhausted: custom.onExhausted } : {}
 	};
@@ -2539,6 +2570,22 @@ async function sweepStaleChatRecoveryIncidents(storage, now) {
 	for (let i = 0; i < staleKeys.length; i += 128) await storage.delete(staleKeys.slice(i, i + 128));
 }
 /**
+* List the persisted recovery incidents that are still live (status
+* `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
+* (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
+* (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
+* Lists by the incident key prefix so the storage layout stays encapsulated.
+*/
+async function listActiveChatRecoveryIncidents(storage) {
+	const entries = await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX });
+	const active = [];
+	for (const [key, incident] of entries) if (incident && (incident.status === "detected" || incident.status === "scheduled" || incident.status === "attempting")) active.push({
+		key,
+		incident
+	});
+	return active;
+}
+/**
 * Summarize a child agent's persisted recovery incidents for the parent's
 * agent-tool reattach decision: `"in-progress"` if any incident is still live
 * (detected/scheduled/attempting), else `"failed"` if any terminalized
@@ -2725,10 +2772,12 @@ async function evaluateChatRecoveryIncident(input) {
 	const progress = Math.max(prevProgress, currentProgress);
 	const work = progress - workBaseline;
 	const workBudgetExceeded = existing != null && Number.isFinite(config.maxRecoveryWork) && work > config.maxRecoveryWork;
+	const oomAttempts = existing?.oomAttempts ?? 0;
+	const oomBudgetExceeded = existing != null && !awaitingClientInteraction && oomAttempts > config.maxOomRetries;
 	const debounced = existing != null && !madeProgress && now - existing.lastAttemptAt < 3e4;
 	const attempt = madeProgress ? 1 : debounced ? existing?.attempt ?? 1 : (existing?.attempt ?? 0) + 1;
 	let abortedByCaller = false;
-	if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && attempt <= config.maxAttempts) try {
+	if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && !oomBudgetExceeded && attempt <= config.maxAttempts) try {
 		const ctx = {
 			incidentId,
 			requestId: identity.requestId,
@@ -2743,7 +2792,7 @@ async function evaluateChatRecoveryIncident(input) {
 	} catch (error) {
 		input.onShouldKeepRecoveringError?.(error);
 	}
-	const exhausted = !awaitingClientInteraction && (noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
+	const exhausted = !awaitingClientInteraction && (oomBudgetExceeded || noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
 	const incident = {
 		incidentId,
 		requestId: identity.requestId,
@@ -2757,7 +2806,8 @@ async function evaluateChatRecoveryIncident(input) {
 		lastProgressAt,
 		progress,
 		workBaseline,
-		...exhausted ? { reason: workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
+		...oomAttempts > 0 ? { oomAttempts } : {},
+		...exhausted ? { reason: oomBudgetExceeded ? "out_of_memory" : workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
 	};
 	const events = [];
 	if (!existing) events.push({
@@ -3031,6 +3081,54 @@ var ChatRecoveryEngine = class {
 		return true;
 	}
 	/**
+	* Record that a recovery callback observed a Durable Object memory-limit reset
+	* (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
+	* and decide what to do next (#1825).
+	*
+	* Bumps the incident's durable `oomAttempts` counter, then:
+	*  - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
+	*    reschedule of the SAME callback (same machinery as
+	*    {@link rescheduleAfterStableTimeout}: the executing one-shot row is
+	*    deleted only after the callback returns, so an idempotent reschedule
+	*    would dedup onto that doomed row) and returns `"rescheduled"`. The small
+	*    delay lets a transient memory spike clear before the re-run;
+	*  - otherwise leaves the incremented count persisted (so a begin-path
+	*    re-evaluation agrees) and returns `"exhausted"` — the caller then
+	*    terminalizes via the give-up path with `reason="out_of_memory"`.
+	*
+	* Returns `"exhausted"` when there is no incident to track against (no id /
+	* record gone): an OOM we cannot bound must seal rather than loop. Unlike a
+	* stable-state retry this is gated by the OOM-specific budget, NOT the generic
+	* attempt cap — re-running an OOM streams a little "progress" that would
+	* otherwise reset the attempt cap forever (the #1825 loop).
+	*/
+	async recordOomAndDecide(input) {
+		const { adapter } = this;
+		if (!input.incidentId) return "exhausted";
+		const key = chatRecoveryIncidentKey(input.incidentId);
+		const incident = await adapter.getIncident(key);
+		if (!incident) return "exhausted";
+		const oomAttempts = (incident.oomAttempts ?? 0) + 1;
+		if (oomAttempts > input.maxOomRetries) {
+			await adapter.putIncident(key, {
+				...incident,
+				oomAttempts,
+				lastAttemptAt: adapter.now(),
+				reason: "out_of_memory"
+			});
+			return "exhausted";
+		}
+		await adapter.putIncident(key, {
+			...incident,
+			oomAttempts,
+			status: "scheduled",
+			lastAttemptAt: adapter.now(),
+			reason: "oom_retry"
+		});
+		await adapter.scheduleRecovery(input.callback, input.data ?? {}, "stable_timeout_retry", 3);
+		return "rescheduled";
+	}
+	/**
 	* Give up on a recovery turn whose retry budget drained, terminalizing it so
 	* it can never become an eternal spinner (#1645). The shared spine both
 	* packages repeated verbatim:
@@ -3305,6 +3403,6 @@ async function* iterateWithStallWatchdog(source, timeoutMs, onStall) {
 	}
 }
 //#endregion
-export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
+export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, listActiveChatRecoveryIncidents, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
 
 //# sourceMappingURL=index.js.map