agents 0.13.3 → 0.14.0

package/dist/index.js

@@ -5,7 +5,7 @@ import { createHeaderBasedEmailResolver, signAgentHeaders } from "./email.js";
 import { i as _classPrivateFieldInitSpec, n as _classPrivateFieldSet2, t as _classPrivateFieldGet2 } from "./classPrivateFieldGet2-Evpt0SEr.js";
 import { SUB_PREFIX, getSubAgentByName, parseSubAgentPath, routeSubAgentRequest } from "./sub-routing.js";
 import { isErrorRetryable, tryN, validateRetryOptions } from "./retries.js";
-import { o as RPC_DO_PREFIX, r as MCPConnectionState, s as DisposableStore, t as MCPClientManager } from "./client-D1kFXo80.js";
+import { a as MCPConnectionState, c as RPC_DO_PREFIX, i as normalizeServerId, l as DisposableStore, n as MCP_SERVER_ID_MAX_LENGTH, t as MCPClientManager } from "./client-NradHZZz.js";
 import { DurableObjectOAuthClientProvider } from "./mcp/do-oauth-client-provider.js";
 import { genericObservability } from "./observability/index.js";
 import { AsyncLocalStorage } from "node:async_hooks";
@@ -114,6 +114,9 @@ function getNextCronTime(cron) {
 	return parseCronExpression(cron).getNextDate();
 }
 const DEFAULT_KEEP_ALIVE_INTERVAL_MS = 3e4;
+const DEFAULT_AGENT_TOOL_RECOVERY_TIMEOUT_MS = 2e3;
+const DEFAULT_AGENT_TOOL_RECOVERY_TOTAL_TIMEOUT_MS = 5e3;
+const DEFAULT_AGENT_TOOL_REATTACH_TIMEOUT_MS = 12e4;
 const SUB_AGENT_IDENTITY_VERSION_LEGACY = "legacy";
 const SUB_AGENT_IDENTITY_VERSION_PATH_V2 = "path-v2";
 const SUB_AGENT_IDENTITY_PATH_V2_PREFIX = "cf-agents:v2:";
@@ -262,7 +265,17 @@ const DEFAULT_AGENT_STATIC_OPTIONS = {
 		maxAttempts: 3,
 		baseDelayMs: 100,
 		maxDelayMs: 3e3
-	}
+	},
+	/** Timeout for internal framework fiber recovery hooks. */
+	fiberRecoveryHookTimeoutMs: 1e4,
+	/** Soft deadline for one interrupted-fiber recovery scan. */
+	fiberRecoveryScanDeadlineMs: 1e4,
+	/**
+	* Maximum age of an unmanaged interrupted-fiber row before recovery gives
+	* up. Bounds repeated retries of a `onFiberRecovered()` hook that keeps
+	* throwing so a poison row cannot re-trigger forever across boots.
+	*/
+	fiberRecoveryMaxAgeMs: 1440 * 60 * 1e3
 };
 /**
 * Parse the raw `retry_options` TEXT column from a SQLite row into a
@@ -285,6 +298,19 @@ function resolveRetryConfig(taskRetry, defaults) {
 		maxDelayMs: taskRetry?.maxDelayMs ?? defaults.maxDelayMs
 	};
 }
+/**
+* Whether an error is a Durable Object reset caused by a code update (deploy).
+*
+* This is a transient, environmental failure: the invocation started on a
+* superseded isolate, so every `ctx.storage` op throws this for the entire
+* life of the invocation (code never reloads mid-invocation) — but the next
+* fresh invocation runs the new code and succeeds. workerd surfaces it as a
+* plain `Error` with this message, so a message match is the only signal.
+*/
+function isDurableObjectCodeUpdateReset(error) {
+	const message = error instanceof Error ? error.message : typeof error === "string" ? error : "";
+	return /reset because its code was updated/i.test(message);
+}
 function getCurrentAgent() {
 	const store = __DO_NOT_USE_WILL_BREAK__agentContext.getStore();
 	if (!store) return {
@@ -382,7 +408,10 @@ var Agent = class Agent extends Server {
 				maxAttempts: userRetry?.maxAttempts ?? DEFAULT_AGENT_STATIC_OPTIONS.retry.maxAttempts,
 				baseDelayMs: userRetry?.baseDelayMs ?? DEFAULT_AGENT_STATIC_OPTIONS.retry.baseDelayMs,
 				maxDelayMs: userRetry?.maxDelayMs ?? DEFAULT_AGENT_STATIC_OPTIONS.retry.maxDelayMs
-			}
+			},
+			fiberRecoveryHookTimeoutMs: ctor.options?.fiberRecoveryHookTimeoutMs ?? DEFAULT_AGENT_STATIC_OPTIONS.fiberRecoveryHookTimeoutMs,
+			fiberRecoveryScanDeadlineMs: ctor.options?.fiberRecoveryScanDeadlineMs ?? DEFAULT_AGENT_STATIC_OPTIONS.fiberRecoveryScanDeadlineMs,
+			fiberRecoveryMaxAgeMs: ctor.options?.fiberRecoveryMaxAgeMs ?? DEFAULT_AGENT_STATIC_OPTIONS.fiberRecoveryMaxAgeMs
 		};
 		return this._cachedOptions;
 	}
@@ -885,7 +914,7 @@ var Agent = class Agent extends Server {
 					this.broadcastMcpServers();
 					this._checkOrphanedWorkflows();
 					await this._checkRunFibers();
-					const recoveredAgentToolFinishes = await this._reconcileAgentToolRuns({ deferFinishHooks: true });
+					const startupAgentToolRunIds = this._agentToolRunRecoveryRunIds();
 					this._insideOnStart = true;
 					this._warnedScheduleInOnStart.clear();
 					let result;
@@ -894,7 +923,7 @@ var Agent = class Agent extends Server {
 					} finally {
 						this._insideOnStart = false;
 					}
-					await this._runDeferredAgentToolFinishHooks(recoveredAgentToolFinishes);
+					this._scheduleAgentToolRunRecovery({ runIds: startupAgentToolRunIds });
 					return result;
 				});
 			});
@@ -2386,20 +2415,67 @@ var Agent = class Agent extends Server {
 			return null;
 		}
 	}
+	_fiberRecoveryPayload(ctx, managedRow, startedAt) {
+		return {
+			fiberId: ctx.id,
+			fiberName: ctx.name,
+			managed: managedRow !== null,
+			recoveryReason: ctx.recoveryReason,
+			elapsedMs: startedAt === void 0 ? void 0 : Date.now() - startedAt
+		};
+	}
+	async _withFiberRecoveryTimeout(ctx, operation) {
+		const timeoutMs = this._resolvedOptions.fiberRecoveryHookTimeoutMs;
+		if (timeoutMs <= 0) return operation();
+		let timer;
+		try {
+			return await Promise.race([operation(), new Promise((_, reject) => {
+				timer = setTimeout(() => {
+					reject(/* @__PURE__ */ new Error(`Fiber recovery hook timed out after ${timeoutMs}ms for "${ctx.name}" (${ctx.id})`));
+				}, timeoutMs);
+			})]);
+		} finally {
+			if (timer !== void 0) clearTimeout(timer);
+		}
+	}
+	_recordFiberRecoveryFailure(ctx, managedRow, error, startedAt, reason = "handler_error") {
+		const errorMessage = this._fiberErrorMessage(error);
+		const completedAt = Date.now();
+		if (managedRow) {
+			this.sql`
+        UPDATE cf_agents_fibers
+        SET status = 'error',
+            error_message = ${errorMessage},
+            completed_at = ${completedAt}
+        WHERE fiber_id = ${ctx.id}
+          AND status = 'interrupted'
+      `;
+			this._notifyManagedFiberTerminal(ctx.id);
+		}
+		this._emit("fiber:recovery:failed", {
+			...this._fiberRecoveryPayload(ctx, managedRow, startedAt),
+			error: errorMessage,
+			reason
+		});
+	}
 	async _runFiberRecoveryHook(ctx, managedRow) {
+		const startedAt = Date.now();
+		this._emit("fiber:recovery:attempt", this._fiberRecoveryPayload(ctx, managedRow));
 		try {
-			if (!await this._handleInternalFiberRecovery(ctx)) {
+			const handled = await this._withFiberRecoveryTimeout(ctx, () => this._handleInternalFiberRecovery(ctx));
+			if (!handled) {
 				const recoveryResult = await this.onFiberRecovered(ctx);
 				if (managedRow && recoveryResult) this._applyManagedFiberRecoveryResult(ctx.id, recoveryResult);
 			}
+			this._emit("fiber:recovery:handled", {
+				...this._fiberRecoveryPayload(ctx, managedRow, startedAt),
+				status: handled ? "internal" : managedRow ? "managed" : "user"
+			});
+			return true;
 		} catch (e) {
-			if (managedRow) this.sql`
-          UPDATE cf_agents_fibers
-          SET error_message = ${this._fiberErrorMessage(e)}
-          WHERE fiber_id = ${ctx.id}
-            AND status = 'interrupted'
-        `;
+			this._recordFiberRecoveryFailure(ctx, managedRow, e, startedAt);
 			console.error(`[Agent] Fiber recovery failed for "${ctx.name}" (${ctx.id}):`, e);
+			return false;
 		}
 	}
 	_fiberInspectionFromRow(row) {
@@ -2684,6 +2760,12 @@ var Agent = class Agent extends Server {
       INSERT INTO cf_agents_runs (id, name, snapshot, created_at)
       VALUES (${id}, ${name}, NULL, ${Date.now()})
     `;
+		const startedAt = Date.now();
+		this._emit("fiber:run:started", {
+			fiberId: id,
+			fiberName: name,
+			managed: options?.managed === true
+		});
 		this._runFiberActiveFibers.add(id);
 		const writeSnapshot = (data) => {
 			const snapshot = JSON.stringify(data);
@@ -2722,12 +2804,25 @@ var Agent = class Agent extends Server {
 					snapshot: null
 				}));
 				options?.beforeRunCleanup?.({ ok: true });
+				this._emit("fiber:run:completed", {
+					fiberId: id,
+					fiberName: name,
+					managed: options?.managed === true,
+					elapsedMs: Date.now() - startedAt
+				});
 				return result;
 			} catch (error) {
 				options?.beforeRunCleanup?.({
 					ok: false,
 					error
 				});
+				this._emit("fiber:run:failed", {
+					fiberId: id,
+					fiberName: name,
+					managed: options?.managed === true,
+					error: this._fiberErrorMessage(error),
+					elapsedMs: Date.now() - startedAt
+				});
 				throw error;
 			}
 		} finally {
@@ -2777,18 +2872,42 @@ var Agent = class Agent extends Server {
 	async _checkRunFibers() {
 		if (this._runFiberRecoveryInProgress) return;
 		this._runFiberRecoveryInProgress = true;
+		const scanStartedAt = Date.now();
+		const scanDeadlineMs = this._resolvedOptions.fiberRecoveryScanDeadlineMs;
+		const fiberRecoveryMaxAgeMs = this._resolvedOptions.fiberRecoveryMaxAgeMs;
 		try {
 			const rows = this.sql`SELECT id, name, snapshot, created_at FROM cf_agents_runs`;
 			for (const row of rows) {
+				if (scanDeadlineMs > 0 && Date.now() - scanStartedAt > scanDeadlineMs) {
+					this._emit("fiber:recovery:skipped", {
+						fiberId: row.id,
+						fiberName: row.name,
+						reason: "scan_deadline_exceeded",
+						elapsedMs: Date.now() - scanStartedAt
+					});
+					break;
+				}
 				if (this._runFiberActiveFibers.has(row.id)) continue;
 				const snapshot = this._parseFiberRecoverySnapshot(row.id, row.snapshot);
 				const ctx = {
 					id: row.id,
 					name: row.name,
 					snapshot,
-					createdAt: row.created_at
+					createdAt: row.created_at,
+					recoveryReason: "interrupted"
 				};
 				const managedRow = this._readFiber(row.id);
+				this._emit("fiber:recovery:detected", {
+					...this._fiberRecoveryPayload(ctx, managedRow),
+					elapsedMs: Date.now() - row.created_at
+				});
+				this._emit("fiber:run:interrupted", {
+					fiberId: row.id,
+					fiberName: row.name,
+					managed: managedRow !== null,
+					recoveryReason: "interrupted",
+					elapsedMs: Date.now() - row.created_at
+				});
 				if (managedRow) {
 					if (this._isTerminalFiberStatus(managedRow.status)) {
 						this.sql`DELETE FROM cf_agents_runs WHERE id = ${row.id}`;
@@ -2808,8 +2927,17 @@ var Agent = class Agent extends Server {
 					ctx.metadata = this._parseFiberJsonObject(managedRow.metadata_json);
 					ctx.status = "interrupted";
 				}
-				await this._runFiberRecoveryHook(ctx, managedRow);
-				this.sql`DELETE FROM cf_agents_runs WHERE id = ${row.id}`;
+				const recovered = await this._runFiberRecoveryHook(ctx, managedRow);
+				const tooOld = fiberRecoveryMaxAgeMs > 0 && Date.now() - row.created_at > fiberRecoveryMaxAgeMs;
+				if (recovered || managedRow || tooOld) {
+					if (!recovered && !managedRow && tooOld) this._emit("fiber:recovery:skipped", {
+						fiberId: row.id,
+						fiberName: row.name,
+						reason: "max_age_exceeded",
+						elapsedMs: Date.now() - row.created_at
+					});
+					this.sql`DELETE FROM cf_agents_runs WHERE id = ${row.id}`;
+				}
 				if (managedRow) this._notifyManagedFiberTerminal(row.id);
 			}
 			const ledgerOnlyRows = this.sql`
@@ -2822,6 +2950,16 @@ var Agent = class Agent extends Server {
           AND r.id IS NULL
       `;
 			for (const row of ledgerOnlyRows) {
+				if (scanDeadlineMs > 0 && Date.now() - scanStartedAt > scanDeadlineMs) {
+					this._emit("fiber:recovery:skipped", {
+						fiberId: row.fiber_id,
+						fiberName: row.name,
+						reason: "scan_deadline_exceeded",
+						elapsedMs: Date.now() - scanStartedAt,
+						managed: true
+					});
+					break;
+				}
 				if (this._runFiberActiveFibers.has(row.fiber_id)) continue;
 				const snapshot = this._parseFiberRecoverySnapshot(row.fiber_id, row.snapshot);
 				const completedAt = Date.now();
@@ -2832,15 +2970,28 @@ var Agent = class Agent extends Server {
           WHERE fiber_id = ${row.fiber_id}
             AND status IN ('pending', 'running')
         `;
-				await this._runFiberRecoveryHook({
+				const ctx = {
 					id: row.fiber_id,
 					name: row.name,
 					snapshot,
 					createdAt: row.created_at,
 					idempotencyKey: row.idempotency_key ?? void 0,
 					metadata: this._parseFiberJsonObject(row.metadata_json),
-					status: "interrupted"
-				}, row);
+					status: "interrupted",
+					recoveryReason: "interrupted"
+				};
+				this._emit("fiber:recovery:detected", {
+					...this._fiberRecoveryPayload(ctx, row),
+					elapsedMs: Date.now() - row.created_at
+				});
+				this._emit("fiber:run:interrupted", {
+					fiberId: row.fiber_id,
+					fiberName: row.name,
+					managed: true,
+					recoveryReason: "interrupted",
+					elapsedMs: Date.now() - row.created_at
+				});
+				await this._runFiberRecoveryHook(ctx, row);
 				this._notifyManagedFiberTerminal(row.fiber_id);
 			}
 		} finally {
@@ -2998,6 +3149,8 @@ var Agent = class Agent extends Server {
 				});
 				return;
 			}
+			const isOneShotSchedule = row.type === "delayed" || row.type === "scheduled";
+			const shouldDeferReset = (error) => isOneShotSchedule && isDurableObjectCodeUpdateReset(error);
 			try {
 				this._emit("schedule:execute", {
 					callback: row.callback,
@@ -3013,9 +3166,14 @@ var Agent = class Agent extends Server {
 					await callback.bind(this)(parsedPayload, row);
 				}, {
 					baseDelayMs,
-					maxDelayMs
+					maxDelayMs,
+					shouldRetry: (error) => !shouldDeferReset(error)
 				});
 			} catch (e) {
+				if (shouldDeferReset(e)) {
+					console.warn(`Deferring scheduled callback "${row.callback}" to a fresh invocation after a Durable Object code-update reset; the one-shot row is preserved and the alarm will re-run on new code.`);
+					throw e;
+				}
 				console.error(`error executing callback "${row.callback}" after ${maxAttempts} attempts`, e);
 				this._emit("schedule:error", {
 					callback: row.callback,
@@ -3835,7 +3993,7 @@ var Agent = class Agent extends Server {
 		const agentType = cls.name;
 		const existing = this._readAgentToolRun(runId);
 		if (existing) {
-			if (this._isAgentToolTerminal(existing.status)) {
+			if (existing.status === "completed" || existing.status === "error" || existing.status === "aborted") {
 				if (existing.status === "completed" && existing.output_json == null) try {
 					const child = await this.subAgent(cls, runId);
 					const inspection = await this._asAgentToolChildAdapter(child).inspectAgentToolRun(runId);
@@ -3847,7 +4005,19 @@ var Agent = class Agent extends Server {
 				} catch {}
 				return this._resultFromAgentToolRow(existing);
 			}
-			return await this._replayAndInterruptAgentToolRun(existing, "Agent tool run was still running, but live-tail reattachment is not supported in this runtime.");
+			try {
+				const child = await this.subAgent(cls, runId);
+				const adapter = this._asAgentToolChildAdapter(child);
+				const reattach = await this._reattachAgentToolRunToTerminal(adapter, existing, 1);
+				if (reattach.result) {
+					await this._finishAgentToolRun(this._agentToolRunInfoFromRow(existing), reattach.result, {
+						sequence: reattach.sequence,
+						completedAt: reattach.completedAt
+					});
+					return reattach.result;
+				}
+			} catch {}
+			return await this._replayAndInterruptAgentToolRun(existing, "Agent tool run was still running and could not be re-attached to a terminal result within the recovery budget.");
 		}
 		const displayOrder = options.displayOrder ?? 0;
 		const inputPreview = options.inputPreview ?? this._defaultAgentToolPreview(options.input);
@@ -4127,7 +4297,7 @@ var Agent = class Agent extends Server {
           error_message = ${result.error ?? null},
           completed_at = ${completedAt}
       WHERE run_id = ${runId}
-        AND status NOT IN ('completed', 'error', 'aborted', 'interrupted')
+        AND status NOT IN ('completed', 'error', 'aborted')
     `;
 		if (result.status === "completed" && result.output !== void 0) this.sql`
         UPDATE cf_agent_tool_runs
@@ -4179,7 +4349,12 @@ var Agent = class Agent extends Server {
 	}
 	async _broadcastAgentToolStoredChunks(row, sequence, replay, connection) {
 		const child = await this._cf_resolveSubAgent(row.agent_type, row.run_id);
-		const chunks = await this._asAgentToolChildAdapter(child).getAgentToolChunks(row.run_id);
+		const adapter = this._asAgentToolChildAdapter(child);
+		return this._broadcastAgentToolStoredChunksFromAdapter(adapter, row, sequence, replay, connection);
+	}
+	async _broadcastAgentToolStoredChunksFromAdapter(adapter, row, sequence, replay, connection, timeoutMs) {
+		const chunks = await this._getAgentToolChunksForRecovery(adapter, row.run_id, timeoutMs);
+		if (!chunks) return sequence;
 		return this._broadcastAgentToolChunks(row.parent_tool_call_id ?? void 0, row.run_id, chunks, sequence, replay, connection);
 	}
 	async _forwardAgentToolStream(stream, parentToolCallId, runId, sequence, signal) {
@@ -4188,11 +4363,17 @@ var Agent = class Agent extends Server {
 		const reader = stream.getReader();
 		const decoder = new TextDecoder();
 		let bufferedBytes = "";
+		let aborted = false;
+		let resolveAbort;
+		const abortPromise = new Promise((resolve) => {
+			resolveAbort = resolve;
+		});
 		let abortListener;
 		if (signal) {
-			abortListener = () => {};
+			abortListener = () => resolveAbort?.();
 			signal.addEventListener("abort", abortListener, { once: true });
 		}
+		let forwardedSinceProgress = false;
 		try {
 			const forwardChunk = (chunk) => {
 				this._broadcastAgentToolEvent(parentToolCallId, next++, {
@@ -4200,6 +4381,7 @@ var Agent = class Agent extends Server {
 					runId,
 					body: chunk.body
 				});
+				forwardedSinceProgress = true;
 			};
 			const forwardLine = (line) => {
 				try {
@@ -4221,14 +4403,17 @@ var Agent = class Agent extends Server {
 				}
 			};
 			while (true) {
-				let readResult;
-				try {
-					readResult = await reader.read();
-				} catch (error) {
-					if (signal?.aborted) break;
-					throw error;
+				const readPromise = reader.read();
+				readPromise.catch(() => {});
+				const raced = await Promise.race([readPromise.then((result) => ({
+					kind: "read",
+					result
+				})), abortPromise.then(() => ({ kind: "abort" }))]);
+				if (raced.kind === "abort") {
+					aborted = true;
+					break;
 				}
-				const { done, value } = readResult;
+				const { done, value } = raced.result;
 				if (done) {
 					bufferedBytes += decoder.decode();
 					flushBufferedBytes(true);
@@ -4238,13 +4423,42 @@ var Agent = class Agent extends Server {
 					bufferedBytes += decoder.decode(value, { stream: true });
 					flushBufferedBytes();
 				} else forwardChunk(value);
+				if (forwardedSinceProgress) {
+					forwardedSinceProgress = false;
+					try {
+						await this._onAgentToolStreamProgress();
+					} catch {}
+				}
 			}
 		} finally {
 			if (abortListener && signal) signal.removeEventListener("abort", abortListener);
-			reader.releaseLock();
+			if (!aborted) try {
+				reader.releaseLock();
+			} catch {}
 		}
 		return next;
 	}
+	/**
+	* Hook invoked by `_forwardAgentToolStream` after a child produces output that
+	* was forwarded to the parent's connections. Forwarding a sub-agent's stream
+	* is genuine forward progress for the *parent* turn (the parent is
+	* orchestrating the child), so chat-recovery subclasses (Think / AIChatAgent)
+	* override this to advance their recovery progress marker.
+	*
+	* Without it, a parent whose turn merely `await`s a sub-agent banks zero
+	* progress of its own, so under deploy churn the parent's no-progress recovery
+	* window exhausts and abandons the turn as `interrupted` — even though the
+	* child is healthily streaming and ultimately completes (observed in the
+	* `deploy-churn --mode subagent` harness: `attempt 6/6, stable_timeout,
+	* progress: 1`).
+	*
+	* Called ONLY after at least one chunk was actually forwarded — never merely
+	* because a child is attached — so a silent / hung child still lets the parent
+	* exhaust on its own timer. The base Agent has no recovery budget, so this is
+	* a no-op; subclasses should throttle the (durable) bump since this can be
+	* called repeatedly while a child streams.
+	*/
+	async _onAgentToolStreamProgress() {}
 	_broadcastAgentToolTerminal(parentToolCallId, sequence, result, replay, connection) {
 		if (result.status === "completed") this._broadcastAgentToolEvent(parentToolCallId, sequence, {
 			kind: "finished",
@@ -4291,6 +4505,58 @@ var Agent = class Agent extends Server {
 		await this._finishAgentToolRun(this._agentToolRunInfoFromRow(row), result, { sequence });
 		return result;
 	}
+	/**
+	* Re-attach to a still-running child agent-tool run and tail it to its real
+	* terminal result, instead of abandoning it as `interrupted` (#1630). The
+	* child is a separate facet with its own `chatRecovery`, so resolving it via
+	* the adapter wakes it and lets it self-complete the interrupted turn; we tail
+	* its live stream (forwarding chunks to the parent's connections) until it
+	* reaches terminal, then inspect for the collected result.
+	*
+	* Bounded by {@link DEFAULT_AGENT_TOOL_REATTACH_TIMEOUT_MS}: a child that keeps
+	* advancing toward terminal within the window is collected; a genuinely hung
+	* child returns `{ result: undefined }` once the budget elapses so the caller
+	* falls back to `interrupted` and recovery can never block forever.
+	*
+	* Returns the terminal `result` (and `completedAt`) when the child reaches a
+	* terminal status within the budget, plus the advanced broadcast `sequence`.
+	* Returns `{ result: undefined }` when there is no `tailAgentToolRun` adapter,
+	* the budget is exhausted, or the child is still non-terminal.
+	*/
+	async _reattachAgentToolRunToTerminal(adapter, row, sequence, budgetMs = DEFAULT_AGENT_TOOL_REATTACH_TIMEOUT_MS) {
+		if (typeof adapter.tailAgentToolRun !== "function") return { sequence };
+		const controller = new AbortController();
+		let timeoutId;
+		if (budgetMs > 0 && Number.isFinite(budgetMs)) timeoutId = setTimeout(() => controller.abort(), budgetMs);
+		else if (budgetMs <= 0) controller.abort();
+		this._emit("agent_tool:recovery:reattach", {
+			runId: row.run_id,
+			agentType: row.agent_type,
+			budgetMs
+		});
+		let nextSequence = sequence;
+		if (!controller.signal.aborted) try {
+			let afterSequence = -1;
+			try {
+				const existing = await adapter.getAgentToolChunks(row.run_id);
+				const last = existing[existing.length - 1];
+				if (last) afterSequence = last.sequence;
+			} catch {}
+			const stream = await adapter.tailAgentToolRun(row.run_id, { afterSequence });
+			nextSequence = await this._forwardAgentToolStream(stream, row.parent_tool_call_id ?? void 0, row.run_id, nextSequence, controller.signal);
+		} catch {}
+		if (timeoutId !== void 0) clearTimeout(timeoutId);
+		let inspection = null;
+		try {
+			inspection = await adapter.inspectAgentToolRun(row.run_id);
+		} catch {}
+		if (inspection && inspection.status !== "running" && inspection.status !== "starting") return {
+			sequence: nextSequence,
+			result: this._terminalResultFromInspection(row.agent_type, inspection),
+			completedAt: inspection.completedAt
+		};
+		return { sequence: nextSequence };
+	}
 	async _replayAgentToolRuns(connection) {
 		const rows = this.sql`
       SELECT run_id, parent_tool_call_id, agent_type, input_preview, status,
@@ -4323,6 +4589,10 @@ var Agent = class Agent extends Server {
 		}
 	}
 	async _reconcileAgentToolRuns(options) {
+		const reattachTimeoutMs = options?.reattachTimeoutMs ?? DEFAULT_AGENT_TOOL_REATTACH_TIMEOUT_MS;
+		const startedAt = Date.now();
+		const totalTimeoutMs = options?.totalRecoveryTimeoutMs ?? DEFAULT_AGENT_TOOL_RECOVERY_TOTAL_TIMEOUT_MS;
+		const deadlineAt = totalTimeoutMs > 0 ? startedAt + totalTimeoutMs : Number.POSITIVE_INFINITY;
 		const deferredFinishes = [];
 		const rows = this.sql`
       SELECT run_id, parent_tool_call_id, agent_type, input_preview, status,
@@ -4332,43 +4602,158 @@ var Agent = class Agent extends Server {
       WHERE status IN ('starting', 'running')
       ORDER BY started_at ASC
     `;
-		for (const row of rows) {
-			let sequence = 1;
-			let completedAt;
-			let result;
-			try {
-				const child = await this._cf_resolveSubAgent(row.agent_type, row.run_id);
-				const inspection = await this._asAgentToolChildAdapter(child).inspectAgentToolRun(row.run_id);
-				try {
-					sequence = await this._broadcastAgentToolStoredChunks(row, sequence);
-				} catch {}
-				if (!inspection || inspection.status === "running" || inspection.status === "starting") result = {
+		const runIds = options?.runIds !== void 0 ? new Set(options.runIds) : void 0;
+		const recoveryRows = rows.filter((row) => !runIds || runIds.has(row.run_id));
+		this._emit("agent_tool:recovery:begin", {
+			runCount: recoveryRows.length,
+			totalTimeoutMs
+		});
+		const finalizeRow = async (row, result, sequence, completedAt) => {
+			this._emit("agent_tool:recovery:row", {
+				runId: row.run_id,
+				agentType: row.agent_type,
+				status: result.status,
+				reason: result.error,
+				elapsedMs: Date.now() - startedAt
+			});
+			const deferredFinish = await this._finishAgentToolRun(this._agentToolRunInfoFromRow(row), result, {
+				sequence,
+				completedAt,
+				deferFinishHook: options?.deferFinishHooks
+			});
+			if (deferredFinish) deferredFinishes.push(deferredFinish);
+		};
+		const reattachQueue = [];
+		for (const row of recoveryRows) {
+			const sequence = 1;
+			const remainingMs = deadlineAt - Date.now();
+			if (remainingMs <= 0) {
+				this._emit("agent_tool:recovery:deadline", {
+					runId: row.run_id,
+					agentType: row.agent_type,
+					elapsedMs: Date.now() - startedAt
+				});
+				await finalizeRow(row, {
 					runId: row.run_id,
 					agentType: row.agent_type,
 					status: "interrupted",
-					error: "Agent tool run was still running, but live-tail reattachment is not supported in this runtime."
-				};
-				else {
-					result = this._terminalResultFromInspection(row.agent_type, inspection);
-					completedAt = inspection.completedAt;
-				}
-			} catch {
-				result = {
+					error: "Agent tool run recovery deadline exceeded."
+				}, sequence, void 0);
+				continue;
+			}
+			const childTimeout = options?.childInspectionTimeoutMs ?? DEFAULT_AGENT_TOOL_RECOVERY_TIMEOUT_MS;
+			const boundedChildTimeout = childTimeout > 0 ? Math.min(childTimeout, remainingMs) : remainingMs;
+			const recovery = await this._inspectAgentToolRunForRecovery(row, sequence, boundedChildTimeout);
+			if (recovery.status !== "inspected") {
+				await finalizeRow(row, {
 					runId: row.run_id,
 					agentType: row.agent_type,
 					status: "interrupted",
-					error: "Agent tool run could not be inspected during parent recovery."
-				};
+					error: recovery.status === "timed-out" ? "Agent tool run inspection timed out during parent recovery." : "Agent tool run could not be inspected during parent recovery."
+				}, sequence, void 0);
+				continue;
 			}
-			const deferredFinish = await this._finishAgentToolRun(this._agentToolRunInfoFromRow(row), result, {
-				sequence,
-				completedAt,
-				deferFinishHook: options?.deferFinishHooks
-			});
-			if (deferredFinish) deferredFinishes.push(deferredFinish);
+			const inspection = recovery.inspection;
+			const stillRunning = !inspection || inspection.status === "running" || inspection.status === "starting";
+			if (stillRunning && typeof recovery.adapter.tailAgentToolRun === "function") {
+				reattachQueue.push({
+					row,
+					adapter: recovery.adapter
+				});
+				continue;
+			}
+			let sequenceAfterReplay = sequence;
+			try {
+				sequenceAfterReplay = await this._broadcastAgentToolStoredChunksFromAdapter(recovery.adapter, row, sequence, void 0, void 0, boundedChildTimeout);
+			} catch {}
+			if (stillRunning) await finalizeRow(row, {
+				runId: row.run_id,
+				agentType: row.agent_type,
+				status: "interrupted",
+				error: "Agent tool run was still running, but live-tail reattachment is not supported in this runtime."
+			}, sequenceAfterReplay, void 0);
+			else await finalizeRow(row, this._terminalResultFromInspection(row.agent_type, inspection), sequenceAfterReplay, inspection.completedAt);
 		}
+		await Promise.all(reattachQueue.map(async ({ row, adapter }) => {
+			const reattach = await this._reattachAgentToolRunToTerminal(adapter, row, 1, reattachTimeoutMs);
+			await finalizeRow(row, reattach.result ?? {
+				runId: row.run_id,
+				agentType: row.agent_type,
+				status: "interrupted",
+				error: "Agent tool run was still running and did not reach a terminal result within the re-attach budget."
+			}, reattach.sequence, reattach.completedAt);
+		}));
+		this._emit("agent_tool:recovery:complete", {
+			runCount: recoveryRows.length,
+			elapsedMs: Date.now() - startedAt
+		});
 		return deferredFinishes;
 	}
+	async _inspectAgentToolRunForRecovery(row, _sequence, timeoutMs = DEFAULT_AGENT_TOOL_RECOVERY_TIMEOUT_MS) {
+		const inspect = (async () => {
+			const child = await this._cf_resolveSubAgent(row.agent_type, row.run_id);
+			const adapter = this._asAgentToolChildAdapter(child);
+			return {
+				status: "inspected",
+				adapter,
+				inspection: await adapter.inspectAgentToolRun(row.run_id)
+			};
+		})().catch(() => ({ status: "failed" }));
+		if (timeoutMs <= 0) return inspect;
+		let timeoutId;
+		const timeout = new Promise((resolve) => {
+			timeoutId = setTimeout(() => {
+				resolve({ status: "timed-out" });
+			}, timeoutMs);
+		});
+		const result = await Promise.race([inspect, timeout]);
+		if (timeoutId !== void 0) clearTimeout(timeoutId);
+		return result;
+	}
+	_scheduleAgentToolRunRecovery(options) {
+		if (this._agentToolRunRecoveryPromise) return this._agentToolRunRecoveryPromise;
+		if (options?.runIds && options.runIds.length === 0) return Promise.resolve();
+		const recovery = (async () => {
+			await new Promise((resolve) => setTimeout(resolve, 0));
+			const recoveredAgentToolFinishes = await this._reconcileAgentToolRuns({
+				deferFinishHooks: true,
+				childInspectionTimeoutMs: options?.childInspectionTimeoutMs,
+				totalRecoveryTimeoutMs: options?.totalRecoveryTimeoutMs,
+				reattachTimeoutMs: options?.reattachTimeoutMs,
+				runIds: options?.runIds
+			});
+			await this._runDeferredAgentToolFinishHooks(recoveredAgentToolFinishes);
+		})().catch(async (error) => {
+			this._emit("agent_tool:recovery:failed", { error: error instanceof Error ? error.message : String(error) });
+			try {
+				await this.onError(error);
+			} catch {}
+		}).finally(() => {
+			this._agentToolRunRecoveryPromise = void 0;
+		});
+		this._agentToolRunRecoveryPromise = recovery;
+		this.ctx.waitUntil(recovery);
+		return recovery;
+	}
+	_agentToolRunRecoveryRunIds() {
+		return this.sql`
+      SELECT run_id
+      FROM cf_agent_tool_runs
+      WHERE status IN ('starting', 'running')
+      ORDER BY started_at ASC
+    `.map((row) => row.run_id);
+	}
+	async _getAgentToolChunksForRecovery(adapter, runId, timeoutMs) {
+		const chunks = adapter.getAgentToolChunks(runId).catch(() => void 0);
+		if (timeoutMs === void 0 || timeoutMs <= 0) return chunks;
+		let timeoutId;
+		const timeout = new Promise((resolve) => {
+			timeoutId = setTimeout(() => resolve(void 0), timeoutMs);
+		});
+		const result = await Promise.race([chunks, timeout]);
+		if (timeoutId !== void 0) clearTimeout(timeoutId);
+		return result;
+	}
 	/**
 	* Shared facet resolution — takes a CamelCase class name string
 	* (matching `ctx.exports`) rather than a class reference. Both
@@ -5360,7 +5745,26 @@ var Agent = class Agent extends Server {
 	async addMcpServer(serverName, urlOrBinding, callbackHostOrOptions, agentsPrefix, options) {
 		const isHttpTransport = typeof urlOrBinding === "string";
 		const normalizedUrl = isHttpTransport ? new URL(urlOrBinding).href : void 0;
-		const existingServer = this.mcp.listServers().find((s) => s.name === serverName && (!isHttpTransport || new URL(s.server_url).href === normalizedUrl));
+		let requestedId;
+		if (typeof callbackHostOrOptions === "object" && callbackHostOrOptions !== null && typeof callbackHostOrOptions.id === "string") {
+			const rawId = callbackHostOrOptions.id;
+			requestedId = normalizeServerId(rawId);
+		}
+		const allServers = this.mcp.listServers();
+		const existingServer = allServers.find((s) => s.name === serverName && (!isHttpTransport || new URL(s.server_url).href === normalizedUrl));
+		if (requestedId) {
+			const idConflict = allServers.find((s) => {
+				if (s.id !== requestedId) return false;
+				if (s.name !== serverName) return true;
+				if (isHttpTransport) return new URL(s.server_url).href !== normalizedUrl;
+				return false;
+			});
+			if (idConflict) throw new Error(`MCP server id "${requestedId}" is already in use by server "${idConflict.name}" (${idConflict.server_url}). Stable ids must be unique per (name, url).`);
+			if (existingServer && existingServer.id !== requestedId) {
+				await this.mcp.migrateServerId(existingServer.id, requestedId, this.name);
+				existingServer.id = requestedId;
+			}
+		}
 		if (existingServer && this.mcp.mcpConnections[existingServer.id]) {
 			const conn = this.mcp.mcpConnections[existingServer.id];
 			if (conn.connectionState === MCPConnectionState.AUTHENTICATING && conn.options.transport.authProvider?.authUrl) return {
@@ -5377,7 +5781,7 @@ var Agent = class Agent extends Server {
 		if (typeof urlOrBinding !== "string") {
 			const rpcOpts = callbackHostOrOptions;
 			const normalizedName = serverName.toLowerCase().replace(/\s+/g, "-");
-			const reconnectId = existingServer?.id;
+			const reconnectId = requestedId ?? existingServer?.id;
 			const { id } = await this.mcp.connect(`${RPC_DO_PREFIX}${normalizedName}`, {
 				reconnect: reconnectId ? { id: reconnectId } : void 0,
 				transport: {
@@ -5434,7 +5838,7 @@ var Agent = class Agent extends Server {
 			const normalizedHost = resolvedCallbackHost.replace(/\/$/, "");
 			callbackUrl = resolvedCallbackPath ? `${normalizedHost}/${resolvedCallbackPath.replace(/^\//, "")}` : `${normalizedHost}/${resolvedAgentsPrefix}/${camelCaseToKebabCase(this._ParentClass.name)}/${this.name}/callback`;
 		}
-		const id = nanoid(8);
+		const id = requestedId ?? nanoid(8);
 		let authProvider;
 		if (callbackUrl) {
 			authProvider = this.createMcpOAuthProvider(callbackUrl);
@@ -5766,6 +6170,6 @@ var StreamingResponse = class {
 	}
 };
 //#endregion
-export { Agent, DEFAULT_AGENT_STATIC_OPTIONS, DurableObjectOAuthClientProvider, MessageType, SUB_PREFIX, SqlError, StreamingResponse, __DO_NOT_USE_WILL_BREAK__agentContext, callable, createHeaderBasedEmailResolver, getAgentByName, getCurrentAgent, getSubAgentByName, parseSubAgentPath, routeAgentEmail, routeAgentRequest, routeSubAgentRequest, unstable_callable };
+export { Agent, DEFAULT_AGENT_STATIC_OPTIONS, DurableObjectOAuthClientProvider, MCP_SERVER_ID_MAX_LENGTH, MessageType, SUB_PREFIX, SqlError, StreamingResponse, __DO_NOT_USE_WILL_BREAK__agentContext, callable, createHeaderBasedEmailResolver, getAgentByName, getCurrentAgent, getSubAgentByName, normalizeServerId, parseSubAgentPath, routeAgentEmail, routeAgentRequest, routeSubAgentRequest, unstable_callable };
 
 //# sourceMappingURL=index.js.map