agents 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -8
- package/dist/{agent-tool-types-CTw3UJUP.d.ts → agent-tool-types-CNyE1iz_.d.ts} +671 -138
- package/dist/agent-tool-types.d.ts +34 -18
- package/dist/agent-tool-types.js +20 -1
- package/dist/agent-tool-types.js.map +1 -0
- package/dist/agent-tools-BXlsuX0d.js +304 -0
- package/dist/agent-tools-BXlsuX0d.js.map +1 -0
- package/dist/agent-tools-CSnyGvJ2.d.ts +119 -0
- package/dist/agent-tools.d.ts +24 -18
- package/dist/agent-tools.js.map +1 -1
- package/dist/ai-chat-agent.d.ts +1 -1
- package/dist/ai-chat-agent.js +1 -2
- package/dist/ai-chat-agent.js.map +1 -1
- package/dist/ai-chat-v5-migration.d.ts +1 -1
- package/dist/ai-chat-v5-migration.js +1 -2
- package/dist/ai-chat-v5-migration.js.map +1 -1
- package/dist/ai-react.d.ts +1 -1
- package/dist/ai-react.js +1 -2
- package/dist/ai-react.js.map +1 -1
- package/dist/ai-types.d.ts +1 -7
- package/dist/ai-types.js +2 -8
- package/dist/ai-types.js.map +1 -1
- package/dist/browser/ai.js +1 -1
- package/dist/browser/index.js +1 -1
- package/dist/chat/index.d.ts +2033 -77
- package/dist/chat/index.js +1828 -245
- package/dist/chat/index.js.map +1 -1
- package/dist/chat/react.d.ts +602 -0
- package/dist/chat/react.js +1506 -0
- package/dist/chat/react.js.map +1 -0
- package/dist/chat-sdk/index.d.ts +4 -4
- package/dist/{classPrivateFieldGet2-CZ7QjTXN.js → classPrivateFieldGet2-DZBYAB34.js} +5 -5
- package/dist/{classPrivateMethodInitSpec-D-0__zd9.js → classPrivateMethodInitSpec-qMjJ6sHQ.js} +2 -2
- package/dist/{client-BXJ9n2f7.js → client-BZ-B3NhC.js} +2 -2
- package/dist/client-BZ-B3NhC.js.map +1 -0
- package/dist/client-tools-aIBO0Fk7.d.ts +53 -0
- package/dist/client.d.ts +76 -57
- package/dist/client.js +33 -5
- package/dist/client.js.map +1 -1
- package/dist/{connector-CrKhowfD.js → connector-CdldGF3h.js} +5 -5
- package/dist/{connector-CrKhowfD.js.map → connector-CdldGF3h.js.map} +1 -1
- package/dist/email.js +1 -1
- package/dist/email.js.map +1 -1
- package/dist/{index-B7IbEeze.d.ts → index-BRnybD6X.d.ts} +197 -14
- package/dist/index.d.ts +95 -73
- package/dist/index.js +694 -25
- package/dist/index.js.map +1 -1
- package/dist/mcp/client.d.ts +18 -14
- package/dist/mcp/client.js +1 -1
- package/dist/mcp/index.d.ts +30 -30
- package/dist/mcp/index.js +7 -7
- package/dist/mcp/index.js.map +1 -1
- package/dist/{agent-tools-3zLG7MgA.js → message-builder-BymO4N_D.js} +45 -126
- package/dist/message-builder-BymO4N_D.js.map +1 -0
- package/dist/observability/index.d.ts +1 -1
- package/dist/observability/index.js +4 -2
- package/dist/observability/index.js.map +1 -1
- package/dist/react.d.ts +123 -110
- package/dist/react.js +44 -11
- package/dist/react.js.map +1 -1
- package/dist/{retries-CwlpAGet.d.ts → retries-CvHJwSuh.d.ts} +15 -6
- package/dist/retries.d.ts +8 -6
- package/dist/retries.js +44 -1
- package/dist/retries.js.map +1 -1
- package/dist/serializable.d.ts +1 -1
- package/dist/skills/compile.js +1 -1
- package/dist/skills/compile.js.map +1 -1
- package/dist/skills/index.js +5 -5
- package/dist/skills/index.js.map +1 -1
- package/dist/sub-routing.d.ts +6 -6
- package/dist/utils.js +1 -1
- package/dist/utils.js.map +1 -1
- package/dist/vite.js +5 -5
- package/dist/vite.js.map +1 -1
- package/dist/wire-types-nflOzNuU.js +240 -0
- package/dist/wire-types-nflOzNuU.js.map +1 -0
- package/dist/{workflow-types-SrZK_o9p.d.ts → workflow-types-Baz_PO5v.d.ts} +42 -22
- package/dist/workflow-types.d.ts +25 -21
- package/dist/workflow-types.js.map +1 -1
- package/dist/workflows.d.ts +22 -21
- package/dist/workflows.js +31 -7
- package/dist/workflows.js.map +1 -1
- package/docs/adding-to-existing-project.md +450 -0
- package/docs/agent-class.md +503 -0
- package/docs/agent-tools.md +552 -0
- package/docs/browse-the-web.md +430 -0
- package/docs/callable-methods.md +627 -0
- package/docs/chat-agents.md +1696 -0
- package/docs/chat-sdk.md +181 -0
- package/docs/client-sdk.md +520 -0
- package/docs/client-tools-continuation.md +177 -0
- package/docs/codemode.md +440 -0
- package/docs/configuration.md +775 -0
- package/docs/cross-domain-authentication.md +171 -0
- package/docs/durable-execution.md +537 -0
- package/docs/email.md +663 -0
- package/docs/get-current-agent.md +204 -0
- package/docs/getting-started.md +305 -0
- package/docs/http-websockets.md +668 -0
- package/docs/human-in-the-loop.md +661 -0
- package/docs/index.md +151 -0
- package/docs/long-running-agents.md +730 -0
- package/docs/mcp-client.md +620 -0
- package/docs/mcp-servers.md +526 -0
- package/docs/mcp-transports.md +308 -0
- package/docs/migration-to-ai-sdk-v5.md +96 -0
- package/docs/migration-to-ai-sdk-v6.md +163 -0
- package/docs/observability.md +261 -0
- package/docs/push-notifications.md +367 -0
- package/docs/queue.md +329 -0
- package/docs/readonly-connections.md +278 -0
- package/docs/resumable-streaming.md +127 -0
- package/docs/retries.md +444 -0
- package/docs/routing.md +749 -0
- package/docs/scheduling.md +898 -0
- package/docs/securing-mcp-servers.md +359 -0
- package/docs/server-driven-messages.md +477 -0
- package/docs/sessions.md +1024 -0
- package/docs/state.md +512 -0
- package/docs/sub-agents.md +389 -0
- package/docs/webhooks.md +604 -0
- package/docs/workflows.md +877 -0
- package/package.json +41 -14
- package/dist/agent-tools-3zLG7MgA.js.map +0 -1
- package/dist/agent-tools-DZhI5F6Q.d.ts +0 -14
- package/dist/client-BXJ9n2f7.js.map +0 -1
package/dist/chat/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { i as normalizeToolInput, n as getPartialStreamText, r as isReplayChunk, t as applyChunkToParts } from "../message-builder-BymO4N_D.js";
|
|
2
|
+
import { i as interceptAgentToolBroadcast, n as applyAgentToolEvent, r as createAgentToolEventState, t as AgentToolProgressEmitter } from "../agent-tools-BXlsuX0d.js";
|
|
2
3
|
import { t as truncateToolOutput } from "../tool-output-truncation-CNnnGZQ3.js";
|
|
4
|
+
import { n as transition, r as StreamAccumulator, t as MessageType } from "../wire-types-nflOzNuU.js";
|
|
3
5
|
import { jsonSchema, tool } from "ai";
|
|
4
6
|
import { nanoid } from "nanoid";
|
|
5
7
|
//#region src/chat/sanitize.ts
|
|
@@ -59,44 +61,63 @@ function stripOpenAIMetadata(part, metadataKey) {
|
|
|
59
61
|
}
|
|
60
62
|
/**
|
|
61
63
|
* Enforce SQLite row size limits by compacting tool outputs and text parts
|
|
62
|
-
* when a serialized message exceeds the safety threshold (1.8MB).
|
|
64
|
+
* when a serialized message exceeds the safety threshold (1.8MB). Shared by
|
|
65
|
+
* `@cloudflare/ai-chat` and `@cloudflare/think` so both compact identically.
|
|
63
66
|
*
|
|
64
67
|
* Compaction strategy:
|
|
65
|
-
* 1. Compact tool outputs over 1KB
|
|
66
|
-
*
|
|
68
|
+
* 1. Compact tool outputs over 1KB with {@link truncateToolOutput}, preserving
|
|
69
|
+
* the structured output shape, and annotate `metadata.compactedToolOutputs`
|
|
70
|
+
* with the compacted tool-call IDs.
|
|
71
|
+
* 2. If still too big, truncate text parts from oldest to newest, annotating
|
|
72
|
+
* `metadata.compactedTextParts` with the truncated part indices.
|
|
67
73
|
*/
|
|
68
|
-
function enforceRowSizeLimit(message) {
|
|
74
|
+
function enforceRowSizeLimit(message, options) {
|
|
69
75
|
let json = JSON.stringify(message);
|
|
70
76
|
let size = byteLength(json);
|
|
71
77
|
if (size <= 18e5) return message;
|
|
72
|
-
if (message.role !== "assistant")
|
|
78
|
+
if (message.role !== "assistant") {
|
|
79
|
+
options?.warn?.(`Non-assistant message ${message.id} is ${size} bytes, exceeds row limit. Truncating text parts.`);
|
|
80
|
+
return truncateTextParts(message);
|
|
81
|
+
}
|
|
82
|
+
options?.warn?.(`Message ${message.id} is ${size} bytes, compacting tool outputs to fit SQLite row limit`);
|
|
83
|
+
const compactedToolCallIds = [];
|
|
73
84
|
const compactedParts = message.parts.map((part) => {
|
|
74
85
|
if ("output" in part && "toolCallId" in part && "state" in part && part.state === "output-available") {
|
|
75
86
|
const output = part.output;
|
|
76
87
|
const truncated = truncateToolOutput(output, 1e3);
|
|
77
|
-
if (truncated.truncated)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
88
|
+
if (truncated.truncated) {
|
|
89
|
+
compactedToolCallIds.push(part.toolCallId);
|
|
90
|
+
return {
|
|
91
|
+
...part,
|
|
92
|
+
output: truncated.output
|
|
93
|
+
};
|
|
94
|
+
}
|
|
81
95
|
}
|
|
82
96
|
return part;
|
|
83
97
|
});
|
|
84
|
-
|
|
98
|
+
const result = {
|
|
85
99
|
...message,
|
|
86
100
|
parts: compactedParts
|
|
87
101
|
};
|
|
102
|
+
if (compactedToolCallIds.length > 0) result.metadata = {
|
|
103
|
+
...result.metadata ?? {},
|
|
104
|
+
compactedToolOutputs: compactedToolCallIds
|
|
105
|
+
};
|
|
88
106
|
json = JSON.stringify(result);
|
|
89
107
|
size = byteLength(json);
|
|
90
108
|
if (size <= 18e5) return result;
|
|
109
|
+
options?.warn?.(`Message ${message.id} still ${size} bytes after tool compaction, truncating text parts`);
|
|
91
110
|
return truncateTextParts(result);
|
|
92
111
|
}
|
|
93
112
|
function truncateTextParts(message) {
|
|
113
|
+
const compactedTextPartIndices = [];
|
|
94
114
|
const parts = [...message.parts];
|
|
95
115
|
for (let i = 0; i < parts.length; i++) {
|
|
96
116
|
const part = parts[i];
|
|
97
117
|
if (part.type === "text" && "text" in part) {
|
|
98
118
|
const text = part.text;
|
|
99
119
|
if (text.length > 1e3) {
|
|
120
|
+
compactedTextPartIndices.push(i);
|
|
100
121
|
parts[i] = {
|
|
101
122
|
...part,
|
|
102
123
|
text: `[Text truncated for storage (${text.length} chars). First 500 chars: ${text.slice(0, 500)}...]`
|
|
@@ -109,139 +130,17 @@ function truncateTextParts(message) {
|
|
|
109
130
|
}
|
|
110
131
|
}
|
|
111
132
|
}
|
|
112
|
-
|
|
133
|
+
const result = {
|
|
113
134
|
...message,
|
|
114
135
|
parts
|
|
115
136
|
};
|
|
137
|
+
if (compactedTextPartIndices.length > 0) result.metadata = {
|
|
138
|
+
...result.metadata ?? {},
|
|
139
|
+
compactedTextParts: compactedTextPartIndices
|
|
140
|
+
};
|
|
141
|
+
return result;
|
|
116
142
|
}
|
|
117
143
|
//#endregion
|
|
118
|
-
//#region src/chat/stream-accumulator.ts
|
|
119
|
-
function asMetadata(value) {
|
|
120
|
-
if (value != null && typeof value === "object" && !Array.isArray(value)) return value;
|
|
121
|
-
}
|
|
122
|
-
var StreamAccumulator = class {
|
|
123
|
-
constructor(options) {
|
|
124
|
-
this.messageId = options.messageId;
|
|
125
|
-
this._isContinuation = options.continuation ?? false;
|
|
126
|
-
this.parts = options.existingParts ? [...options.existingParts] : [];
|
|
127
|
-
this.metadata = options.existingMetadata ? { ...options.existingMetadata } : void 0;
|
|
128
|
-
}
|
|
129
|
-
applyChunk(chunk) {
|
|
130
|
-
const handled = applyChunkToParts(this.parts, chunk);
|
|
131
|
-
if (chunk.type === "tool-approval-request" && chunk.toolCallId) return {
|
|
132
|
-
handled,
|
|
133
|
-
action: {
|
|
134
|
-
type: "tool-approval-request",
|
|
135
|
-
toolCallId: chunk.toolCallId
|
|
136
|
-
}
|
|
137
|
-
};
|
|
138
|
-
if ((chunk.type === "tool-output-available" || chunk.type === "tool-output-error") && chunk.toolCallId) {
|
|
139
|
-
if (!this.parts.some((p) => "toolCallId" in p && p.toolCallId === chunk.toolCallId)) return {
|
|
140
|
-
handled,
|
|
141
|
-
action: {
|
|
142
|
-
type: "cross-message-tool-update",
|
|
143
|
-
updateType: chunk.type === "tool-output-available" ? "output-available" : "output-error",
|
|
144
|
-
toolCallId: chunk.toolCallId,
|
|
145
|
-
output: chunk.output,
|
|
146
|
-
errorText: chunk.errorText,
|
|
147
|
-
preliminary: chunk.preliminary
|
|
148
|
-
}
|
|
149
|
-
};
|
|
150
|
-
}
|
|
151
|
-
if (!handled) switch (chunk.type) {
|
|
152
|
-
case "start": {
|
|
153
|
-
if (chunk.messageId != null && !this._isContinuation) this.messageId = chunk.messageId;
|
|
154
|
-
const startMeta = asMetadata(chunk.messageMetadata);
|
|
155
|
-
if (startMeta) this.metadata = this.metadata ? {
|
|
156
|
-
...this.metadata,
|
|
157
|
-
...startMeta
|
|
158
|
-
} : { ...startMeta };
|
|
159
|
-
return {
|
|
160
|
-
handled: true,
|
|
161
|
-
action: {
|
|
162
|
-
type: "start",
|
|
163
|
-
messageId: chunk.messageId,
|
|
164
|
-
metadata: startMeta
|
|
165
|
-
}
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
case "finish": {
|
|
169
|
-
const finishMeta = asMetadata(chunk.messageMetadata);
|
|
170
|
-
if (finishMeta) this.metadata = this.metadata ? {
|
|
171
|
-
...this.metadata,
|
|
172
|
-
...finishMeta
|
|
173
|
-
} : { ...finishMeta };
|
|
174
|
-
return {
|
|
175
|
-
handled: true,
|
|
176
|
-
action: {
|
|
177
|
-
type: "finish",
|
|
178
|
-
finishReason: "finishReason" in chunk ? chunk.finishReason : void 0,
|
|
179
|
-
metadata: finishMeta
|
|
180
|
-
}
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
case "message-metadata": {
|
|
184
|
-
const msgMeta = asMetadata(chunk.messageMetadata);
|
|
185
|
-
if (msgMeta) this.metadata = this.metadata ? {
|
|
186
|
-
...this.metadata,
|
|
187
|
-
...msgMeta
|
|
188
|
-
} : { ...msgMeta };
|
|
189
|
-
return {
|
|
190
|
-
handled: true,
|
|
191
|
-
action: {
|
|
192
|
-
type: "message-metadata",
|
|
193
|
-
metadata: msgMeta ?? {}
|
|
194
|
-
}
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
case "finish-step": return { handled: true };
|
|
198
|
-
case "error": return {
|
|
199
|
-
handled: true,
|
|
200
|
-
action: {
|
|
201
|
-
type: "error",
|
|
202
|
-
error: chunk.errorText ?? JSON.stringify(chunk)
|
|
203
|
-
}
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
return { handled };
|
|
207
|
-
}
|
|
208
|
-
/** Snapshot the current state as a UIMessage. */
|
|
209
|
-
toMessage() {
|
|
210
|
-
return {
|
|
211
|
-
id: this.messageId,
|
|
212
|
-
role: "assistant",
|
|
213
|
-
parts: [...this.parts],
|
|
214
|
-
...this.metadata != null && { metadata: this.metadata }
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Merge this accumulator's message into an existing message array.
|
|
219
|
-
* Handles continuation (walk backward for last assistant), replacement
|
|
220
|
-
* (update existing by messageId), or append (new message).
|
|
221
|
-
*/
|
|
222
|
-
mergeInto(messages) {
|
|
223
|
-
let existingIdx = messages.findIndex((m) => m.id === this.messageId);
|
|
224
|
-
if (existingIdx < 0 && this._isContinuation) {
|
|
225
|
-
for (let i = messages.length - 1; i >= 0; i--) if (messages[i].role === "assistant") {
|
|
226
|
-
existingIdx = i;
|
|
227
|
-
break;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
const partialMessage = {
|
|
231
|
-
id: existingIdx >= 0 ? messages[existingIdx].id : this.messageId,
|
|
232
|
-
role: "assistant",
|
|
233
|
-
parts: [...this.parts],
|
|
234
|
-
...this.metadata != null && { metadata: this.metadata }
|
|
235
|
-
};
|
|
236
|
-
if (existingIdx >= 0) {
|
|
237
|
-
const updated = [...messages];
|
|
238
|
-
updated[existingIdx] = partialMessage;
|
|
239
|
-
return updated;
|
|
240
|
-
}
|
|
241
|
-
return [...messages, partialMessage];
|
|
242
|
-
}
|
|
243
|
-
};
|
|
244
|
-
//#endregion
|
|
245
144
|
//#region src/chat/turn-queue.ts
|
|
246
145
|
var TurnQueue = class {
|
|
247
146
|
constructor() {
|
|
@@ -436,71 +335,6 @@ var SubmitConcurrencyController = class {
|
|
|
436
335
|
}
|
|
437
336
|
};
|
|
438
337
|
//#endregion
|
|
439
|
-
//#region src/chat/broadcast-state.ts
|
|
440
|
-
function transition(state, event) {
|
|
441
|
-
switch (event.type) {
|
|
442
|
-
case "clear": return {
|
|
443
|
-
state: { status: "idle" },
|
|
444
|
-
isStreaming: false
|
|
445
|
-
};
|
|
446
|
-
case "resume-fallback": {
|
|
447
|
-
const accumulator = new StreamAccumulator({ messageId: event.messageId });
|
|
448
|
-
return {
|
|
449
|
-
state: {
|
|
450
|
-
status: "observing",
|
|
451
|
-
streamId: event.streamId,
|
|
452
|
-
accumulator
|
|
453
|
-
},
|
|
454
|
-
isStreaming: true
|
|
455
|
-
};
|
|
456
|
-
}
|
|
457
|
-
case "response": {
|
|
458
|
-
let accumulator;
|
|
459
|
-
const isReplayedStart = event.replay === true && event.chunkData?.type === "start";
|
|
460
|
-
if (state.status === "idle" || state.streamId !== event.streamId || isReplayedStart) {
|
|
461
|
-
let messageId = event.messageId;
|
|
462
|
-
let existingParts;
|
|
463
|
-
let existingMetadata;
|
|
464
|
-
if (event.continuation && event.currentMessages) {
|
|
465
|
-
for (let i = event.currentMessages.length - 1; i >= 0; i--) if (event.currentMessages[i].role === "assistant") {
|
|
466
|
-
messageId = event.currentMessages[i].id;
|
|
467
|
-
existingParts = [...event.currentMessages[i].parts];
|
|
468
|
-
if (event.currentMessages[i].metadata != null) existingMetadata = { ...event.currentMessages[i].metadata };
|
|
469
|
-
break;
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
accumulator = new StreamAccumulator({
|
|
473
|
-
messageId,
|
|
474
|
-
continuation: event.continuation,
|
|
475
|
-
existingParts,
|
|
476
|
-
existingMetadata
|
|
477
|
-
});
|
|
478
|
-
} else accumulator = state.accumulator;
|
|
479
|
-
if (event.chunkData) accumulator.applyChunk(event.chunkData);
|
|
480
|
-
let messagesUpdate;
|
|
481
|
-
if (event.done) {
|
|
482
|
-
messagesUpdate = (prev) => accumulator.mergeInto(prev);
|
|
483
|
-
return {
|
|
484
|
-
state: { status: "idle" },
|
|
485
|
-
messagesUpdate,
|
|
486
|
-
isStreaming: false
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
if (event.chunkData && !event.replay) messagesUpdate = (prev) => accumulator.mergeInto(prev);
|
|
490
|
-
else if (event.replayComplete) messagesUpdate = (prev) => accumulator.mergeInto(prev);
|
|
491
|
-
return {
|
|
492
|
-
state: {
|
|
493
|
-
status: "observing",
|
|
494
|
-
streamId: event.streamId,
|
|
495
|
-
accumulator
|
|
496
|
-
},
|
|
497
|
-
messagesUpdate,
|
|
498
|
-
isStreaming: true
|
|
499
|
-
};
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
//#endregion
|
|
504
338
|
//#region src/chat/protocol.ts
|
|
505
339
|
/**
|
|
506
340
|
* Wire protocol message type constants for the cf_agent_chat_* protocol.
|
|
@@ -519,12 +353,34 @@ const CHAT_MESSAGE_TYPES = {
|
|
|
519
353
|
STREAM_RESUME_ACK: "cf_agent_stream_resume_ack",
|
|
520
354
|
STREAM_RESUME_REQUEST: "cf_agent_stream_resume_request",
|
|
521
355
|
STREAM_RESUME_NONE: "cf_agent_stream_resume_none",
|
|
356
|
+
STREAM_PENDING: "cf_agent_stream_pending",
|
|
522
357
|
TOOL_RESULT: "cf_agent_tool_result",
|
|
523
358
|
TOOL_APPROVAL: "cf_agent_tool_approval",
|
|
524
359
|
MESSAGE_UPDATED: "cf_agent_message_updated",
|
|
525
360
|
CHAT_RECOVERING: "cf_agent_chat_recovering"
|
|
526
361
|
};
|
|
527
362
|
//#endregion
|
|
363
|
+
//#region src/chat/connection.ts
|
|
364
|
+
/**
|
|
365
|
+
* Send a message on a connection, swallowing the specific
|
|
366
|
+
* "send after close" error a racing disconnect produces. Returns `true` if the
|
|
367
|
+
* send went out, `false` if the socket was already closed. Any other error
|
|
368
|
+
* rethrows.
|
|
369
|
+
*/
|
|
370
|
+
function sendIfOpen(connection, message) {
|
|
371
|
+
try {
|
|
372
|
+
connection.send(message);
|
|
373
|
+
return true;
|
|
374
|
+
} catch (error) {
|
|
375
|
+
if (isWebSocketClosedSendError(error)) return false;
|
|
376
|
+
throw error;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
/** Whether an error is the "WebSocket send() after close" `TypeError`. */
|
|
380
|
+
function isWebSocketClosedSendError(error) {
|
|
381
|
+
return error instanceof TypeError && error.message.includes("WebSocket send() after close");
|
|
382
|
+
}
|
|
383
|
+
//#endregion
|
|
528
384
|
//#region src/chat/resumable-stream.ts
|
|
529
385
|
/**
|
|
530
386
|
* ResumableStream: Standalone class for buffering, persisting, and replaying
|
|
@@ -576,6 +432,18 @@ const ABANDONED_STREAM_RETENTION_MS = 3600 * 1e3;
|
|
|
576
432
|
/** Shared encoder for UTF-8 byte length measurement */
|
|
577
433
|
const textEncoder = new TextEncoder();
|
|
578
434
|
/**
|
|
435
|
+
* How far ahead (seconds) to schedule the resumable-stream buffer cleanup
|
|
436
|
+
* alarm. Set to the short completion-grace window ({@link COMPLETED_RETENTION_MS},
|
|
437
|
+
* 10m) so a finished buffer is reclaimed promptly. The re-arm-while-reclaimable
|
|
438
|
+
* loop (see {@link cleanupStreamBuffers}) revisits any longer-lived rows — e.g.
|
|
439
|
+
* an abandoned in-flight buffer on its 1h window — by waking again each interval
|
|
440
|
+
* until they age out, then stops. Driving cleanup from an alarm (rather than
|
|
441
|
+
* only piggybacking on the next stream completion) ensures idle/one-off chat
|
|
442
|
+
* DOs still reclaim their buffers without waking forever (#1706). Shared by
|
|
443
|
+
* `AIChatAgent` and `Think`.
|
|
444
|
+
*/
|
|
445
|
+
const STREAM_CLEANUP_DELAY_SECONDS = 600;
|
|
446
|
+
/**
|
|
579
447
|
* A stored row body is either a single chunk body (a JSON object string —
|
|
580
448
|
* legacy per-chunk rows and single-chunk segments) or a packed segment (a JSON
|
|
581
449
|
* array of chunk body strings). Unpack to the individual chunk bodies in order.
|
|
@@ -590,18 +458,6 @@ function unpackSegmentBody(rowBody) {
|
|
|
590
458
|
} catch {}
|
|
591
459
|
return [rowBody];
|
|
592
460
|
}
|
|
593
|
-
function sendIfOpen$1(connection, message) {
|
|
594
|
-
try {
|
|
595
|
-
connection.send(message);
|
|
596
|
-
return true;
|
|
597
|
-
} catch (error) {
|
|
598
|
-
if (isWebSocketClosedSendError$1(error)) return false;
|
|
599
|
-
throw error;
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
function isWebSocketClosedSendError$1(error) {
|
|
603
|
-
return error instanceof TypeError && error.message.includes("WebSocket send() after close");
|
|
604
|
-
}
|
|
605
461
|
function isMissingMetadataColumnError(error) {
|
|
606
462
|
const message = error instanceof Error ? error.message : String(error);
|
|
607
463
|
return (message.includes("message_id") || message.includes("is_continuation")) && (message.toLowerCase().includes("no such column") || message.toLowerCase().includes("has no column named"));
|
|
@@ -834,7 +690,7 @@ var ResumableStream = class ResumableStream {
|
|
|
834
690
|
where stream_id = ${streamId}
|
|
835
691
|
order by chunk_index asc
|
|
836
692
|
`;
|
|
837
|
-
for (const chunk of chunks || []) for (const body of unpackSegmentBody(chunk.body)) if (!sendIfOpen
|
|
693
|
+
for (const chunk of chunks || []) for (const body of unpackSegmentBody(chunk.body)) if (!sendIfOpen(connection, JSON.stringify({
|
|
838
694
|
body,
|
|
839
695
|
done: false,
|
|
840
696
|
id: requestId,
|
|
@@ -843,7 +699,7 @@ var ResumableStream = class ResumableStream {
|
|
|
843
699
|
...continuation && { continuation: true }
|
|
844
700
|
}))) return null;
|
|
845
701
|
if (this._activeStreamId !== streamId) {
|
|
846
|
-
sendIfOpen
|
|
702
|
+
sendIfOpen(connection, JSON.stringify({
|
|
847
703
|
body: "",
|
|
848
704
|
done: true,
|
|
849
705
|
id: requestId,
|
|
@@ -854,7 +710,7 @@ var ResumableStream = class ResumableStream {
|
|
|
854
710
|
return null;
|
|
855
711
|
}
|
|
856
712
|
if (!this._isLive) {
|
|
857
|
-
sendIfOpen
|
|
713
|
+
sendIfOpen(connection, JSON.stringify({
|
|
858
714
|
body: "",
|
|
859
715
|
done: true,
|
|
860
716
|
id: requestId,
|
|
@@ -865,7 +721,7 @@ var ResumableStream = class ResumableStream {
|
|
|
865
721
|
this.complete(streamId);
|
|
866
722
|
return streamId;
|
|
867
723
|
}
|
|
868
|
-
sendIfOpen
|
|
724
|
+
sendIfOpen(connection, JSON.stringify({
|
|
869
725
|
body: "",
|
|
870
726
|
done: false,
|
|
871
727
|
id: requestId,
|
|
@@ -881,7 +737,7 @@ var ResumableStream = class ResumableStream {
|
|
|
881
737
|
if (!stream) return false;
|
|
882
738
|
const continuation = stream.is_continuation === 1;
|
|
883
739
|
if (!this._replayStoredChunks(connection, stream.id, requestId, continuation)) return false;
|
|
884
|
-
return sendIfOpen
|
|
740
|
+
return sendIfOpen(connection, JSON.stringify({
|
|
885
741
|
body: "",
|
|
886
742
|
done: true,
|
|
887
743
|
id: requestId,
|
|
@@ -932,7 +788,7 @@ var ResumableStream = class ResumableStream {
|
|
|
932
788
|
where stream_id = ${streamId}
|
|
933
789
|
order by chunk_index asc
|
|
934
790
|
`;
|
|
935
|
-
for (const chunk of chunks || []) for (const body of unpackSegmentBody(chunk.body)) if (!sendIfOpen
|
|
791
|
+
for (const chunk of chunks || []) for (const body of unpackSegmentBody(chunk.body)) if (!sendIfOpen(connection, JSON.stringify({
|
|
936
792
|
body,
|
|
937
793
|
done: false,
|
|
938
794
|
id: requestId,
|
|
@@ -1119,6 +975,22 @@ var ResumableStream = class ResumableStream {
|
|
|
1119
975
|
}
|
|
1120
976
|
};
|
|
1121
977
|
ResumableStream.CHUNK_MAX_BYTES = 18e5;
|
|
978
|
+
/**
|
|
979
|
+
* The buffer-cleanup alarm body: sweep aged stream buffers, then re-arm only
|
|
980
|
+
* while rows remain so a fully-swept DO stops waking itself. `rearm` schedules
|
|
981
|
+
* the next sweep — it MUST schedule a non-idempotent alarm, because this runs
|
|
982
|
+
* INSIDE the currently-executing one-shot schedule row, which `alarm()` deletes
|
|
983
|
+
* only after it returns; an idempotent reschedule would dedup onto that row and
|
|
984
|
+
* be deleted with it, so the re-arm would silently never fire and buffers that
|
|
985
|
+
* survived this sweep (e.g. a younger turn) would go uncollected. A fresh
|
|
986
|
+
* delayed row survives the deletion. Shared by `AIChatAgent` and `Think`.
|
|
987
|
+
*
|
|
988
|
+
* `@internal`
|
|
989
|
+
*/
|
|
990
|
+
async function cleanupStreamBuffers(stream, rearm) {
|
|
991
|
+
stream.cleanup();
|
|
992
|
+
if (stream.hasReclaimableStreams()) await rearm();
|
|
993
|
+
}
|
|
1122
994
|
//#endregion
|
|
1123
995
|
//#region src/chat/sql-batch.ts
|
|
1124
996
|
/**
|
|
@@ -1206,19 +1078,7 @@ function createToolsFromClientSchemas(clientTools, options) {
|
|
|
1206
1078
|
* The scheduling algorithm (prerequisite chaining, debounce, TurnQueue
|
|
1207
1079
|
* enrollment) stays in the host — this class only manages the data.
|
|
1208
1080
|
*/
|
|
1209
|
-
const MSG_STREAM_RESUME_NONE = CHAT_MESSAGE_TYPES.STREAM_RESUME_NONE;
|
|
1210
|
-
function sendIfOpen(connection, message) {
|
|
1211
|
-
try {
|
|
1212
|
-
connection.send(message);
|
|
1213
|
-
return true;
|
|
1214
|
-
} catch (error) {
|
|
1215
|
-
if (isWebSocketClosedSendError(error)) return false;
|
|
1216
|
-
throw error;
|
|
1217
|
-
}
|
|
1218
|
-
}
|
|
1219
|
-
function isWebSocketClosedSendError(error) {
|
|
1220
|
-
return error instanceof TypeError && error.message.includes("WebSocket send() after close");
|
|
1221
|
-
}
|
|
1081
|
+
const MSG_STREAM_RESUME_NONE$1 = CHAT_MESSAGE_TYPES.STREAM_RESUME_NONE;
|
|
1222
1082
|
var ContinuationState = class {
|
|
1223
1083
|
constructor() {
|
|
1224
1084
|
this.pending = null;
|
|
@@ -1262,7 +1122,7 @@ var ContinuationState = class {
|
|
|
1262
1122
|
* continuation stream to start, then clear the map.
|
|
1263
1123
|
*/
|
|
1264
1124
|
sendResumeNone() {
|
|
1265
|
-
const msg = JSON.stringify({ type: MSG_STREAM_RESUME_NONE });
|
|
1125
|
+
const msg = JSON.stringify({ type: MSG_STREAM_RESUME_NONE$1 });
|
|
1266
1126
|
for (const connection of this.awaitingConnections.values()) sendIfOpen(connection, msg);
|
|
1267
1127
|
this.awaitingConnections.clear();
|
|
1268
1128
|
}
|
|
@@ -1313,6 +1173,266 @@ var ContinuationState = class {
|
|
|
1313
1173
|
}
|
|
1314
1174
|
};
|
|
1315
1175
|
//#endregion
|
|
1176
|
+
//#region src/chat/pre-stream-turns.ts
|
|
1177
|
+
/**
|
|
1178
|
+
* PreStreamTurns — tracks accepted chat turns that have not yet started a
|
|
1179
|
+
* resumable stream, and the connections parked waiting for one.
|
|
1180
|
+
*
|
|
1181
|
+
* The resume handshake can resume an ACTIVE stream (chunks buffering) and can
|
|
1182
|
+
* replay a TERMINAL outcome, but a normal turn spends a window between "request
|
|
1183
|
+
* accepted" and "first chunk produced" in neither state: it is queued, waiting
|
|
1184
|
+
* on `waitForMcpConnections`, debouncing, or simply running async setup inside
|
|
1185
|
+
* `onChatMessage` before a stream object exists. A client that reconnects or
|
|
1186
|
+
* re-mounts in that window used to get `cf_agent_stream_resume_none` and give
|
|
1187
|
+
* up, so the turn the server went on to complete normally never drove the
|
|
1188
|
+
* client's AI-SDK `status` (issue #1784).
|
|
1189
|
+
*
|
|
1190
|
+
* This container lets a host represent that window as resumable: the handshake
|
|
1191
|
+
* parks the reconnecting connection here (and tells it to keep waiting), and the
|
|
1192
|
+
* host flushes the parked connections into the normal `STREAM_RESUMING` path the
|
|
1193
|
+
* moment a stream actually starts — or releases them with `resume_none` if the
|
|
1194
|
+
* turn settles without ever streaming.
|
|
1195
|
+
*
|
|
1196
|
+
* Concurrency-safe under queued turns via an accepted-request set: parked
|
|
1197
|
+
* connections are only released with `resume_none` once EVERY accepted turn has
|
|
1198
|
+
* settled with no active stream, so a client parked during the gap between one
|
|
1199
|
+
* turn finishing and the next starting still resumes onto the next stream.
|
|
1200
|
+
*
|
|
1201
|
+
* Pure data + send-through-callback, mirroring {@link ContinuationState}: the
|
|
1202
|
+
* host owns the actual frame sends and the stream-start / turn-settle wiring.
|
|
1203
|
+
*
|
|
1204
|
+
* @internal Sibling-package support for `@cloudflare/ai-chat` and
|
|
1205
|
+
* `@cloudflare/think`, not a public API.
|
|
1206
|
+
*/
|
|
1207
|
+
const MSG_STREAM_PENDING = CHAT_MESSAGE_TYPES.STREAM_PENDING;
|
|
1208
|
+
const MSG_STREAM_RESUME_NONE = CHAT_MESSAGE_TYPES.STREAM_RESUME_NONE;
|
|
1209
|
+
var PreStreamTurns = class {
|
|
1210
|
+
constructor() {
|
|
1211
|
+
this._accepted = /* @__PURE__ */ new Set();
|
|
1212
|
+
this.awaitingConnections = /* @__PURE__ */ new Map();
|
|
1213
|
+
this._latestRequestId = null;
|
|
1214
|
+
}
|
|
1215
|
+
/** Mark a freshly-accepted turn as in flight (pre-stream). */
|
|
1216
|
+
begin(requestId) {
|
|
1217
|
+
this._accepted.add(requestId);
|
|
1218
|
+
this._latestRequestId = requestId;
|
|
1219
|
+
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Mark an accepted turn as settled. Returns `true` when no accepted turn
|
|
1222
|
+
* remains in flight (the caller should release parked connections if no
|
|
1223
|
+
* stream is active).
|
|
1224
|
+
*/
|
|
1225
|
+
settle(requestId) {
|
|
1226
|
+
this._accepted.delete(requestId);
|
|
1227
|
+
if (this._accepted.size === 0) {
|
|
1228
|
+
this._latestRequestId = null;
|
|
1229
|
+
return true;
|
|
1230
|
+
}
|
|
1231
|
+
return false;
|
|
1232
|
+
}
|
|
1233
|
+
/** Whether any accepted turn is still pre-stream. */
|
|
1234
|
+
hasInFlight() {
|
|
1235
|
+
return this._accepted.size > 0;
|
|
1236
|
+
}
|
|
1237
|
+
/** The request id to advertise in the keep-waiting frame, if known. */
|
|
1238
|
+
get latestRequestId() {
|
|
1239
|
+
return this._latestRequestId;
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Park a reconnecting connection and tell it to keep waiting (so its
|
|
1243
|
+
* transport does not resolve `reconnectToStream` early). No-op when nothing
|
|
1244
|
+
* is in flight. Parked connections are deliberately NOT added to the host's
|
|
1245
|
+
* `pendingResumeConnections` — they must keep receiving any live broadcast —
|
|
1246
|
+
* until the host flushes them through `notifyStreamResuming` on stream start.
|
|
1247
|
+
*/
|
|
1248
|
+
park(connection) {
|
|
1249
|
+
if (!this.hasInFlight()) return false;
|
|
1250
|
+
this.awaitingConnections.set(connection.id, connection);
|
|
1251
|
+
sendIfOpen(connection, JSON.stringify({
|
|
1252
|
+
type: MSG_STREAM_PENDING,
|
|
1253
|
+
...this._latestRequestId ? { id: this._latestRequestId } : {}
|
|
1254
|
+
}));
|
|
1255
|
+
return true;
|
|
1256
|
+
}
|
|
1257
|
+
/** Drop a single connection (e.g. on socket close) without releasing others. */
|
|
1258
|
+
release(connectionId) {
|
|
1259
|
+
this.awaitingConnections.delete(connectionId);
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* A stream has started: hand every parked connection to `notify` (the host's
|
|
1263
|
+
* `notifyStreamResuming`, which sends `STREAM_RESUMING` and excludes the
|
|
1264
|
+
* connection from live broadcast until it ACKs), then clear the awaiting map.
|
|
1265
|
+
* The accepted set is untouched — the turn is still running.
|
|
1266
|
+
*/
|
|
1267
|
+
flushOnStreamStart(notify) {
|
|
1268
|
+
for (const connection of this.awaitingConnections.values()) notify(connection);
|
|
1269
|
+
this.awaitingConnections.clear();
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* Release every parked connection with `STREAM_RESUME_NONE` (the turn settled
|
|
1273
|
+
* without ever starting a stream) and clear the awaiting map. Safe to call
|
|
1274
|
+
* when the map is empty (no-op), so the host can call it liberally from a
|
|
1275
|
+
* turn-settle path.
|
|
1276
|
+
*/
|
|
1277
|
+
releaseAwaiting() {
|
|
1278
|
+
const msg = JSON.stringify({ type: MSG_STREAM_RESUME_NONE });
|
|
1279
|
+
for (const connection of this.awaitingConnections.values()) sendIfOpen(connection, msg);
|
|
1280
|
+
this.awaitingConnections.clear();
|
|
1281
|
+
}
|
|
1282
|
+
/** Drop all state (chat clear / destroy). Does not send any frames. */
|
|
1283
|
+
reset() {
|
|
1284
|
+
this._accepted.clear();
|
|
1285
|
+
this.awaitingConnections.clear();
|
|
1286
|
+
this._latestRequestId = null;
|
|
1287
|
+
}
|
|
1288
|
+
};
|
|
1289
|
+
//#endregion
|
|
1290
|
+
//#region src/chat/auto-continuation-controller.ts
|
|
1291
|
+
var AutoContinuationController = class AutoContinuationController {
|
|
1292
|
+
constructor(host) {
|
|
1293
|
+
this.host = host;
|
|
1294
|
+
this._timer = null;
|
|
1295
|
+
this._barrierActive = false;
|
|
1296
|
+
}
|
|
1297
|
+
/**
|
|
1298
|
+
* Schedule an auto-continuation for a tool result/approval that opted in with
|
|
1299
|
+
* `autoContinue` (#1650). Coalesces rapid sibling results into a single
|
|
1300
|
+
* continuation via the debounce timer; the actual fire is gated by
|
|
1301
|
+
* {@link fireWhenStable}. If a continuation is already running
|
|
1302
|
+
* (`pastCoalesce`), the new result is stored as the deferred follow-up
|
|
1303
|
+
* instead of re-arming.
|
|
1304
|
+
*/
|
|
1305
|
+
schedule(spec) {
|
|
1306
|
+
const c = this.host.continuation;
|
|
1307
|
+
if (c.pending?.pastCoalesce) {
|
|
1308
|
+
c.deferred = {
|
|
1309
|
+
connection: spec.connection,
|
|
1310
|
+
connectionId: spec.connection.id,
|
|
1311
|
+
clientTools: spec.clientTools,
|
|
1312
|
+
body: spec.body,
|
|
1313
|
+
errorPrefix: spec.errorPrefix,
|
|
1314
|
+
prerequisite: null
|
|
1315
|
+
};
|
|
1316
|
+
return;
|
|
1317
|
+
}
|
|
1318
|
+
if (c.pending) {
|
|
1319
|
+
c.pending.connection = spec.connection;
|
|
1320
|
+
c.pending.connectionId = spec.connection.id;
|
|
1321
|
+
c.pending.clientTools = spec.clientTools;
|
|
1322
|
+
c.pending.body = spec.body;
|
|
1323
|
+
c.pending.errorPrefix = spec.errorPrefix;
|
|
1324
|
+
c.awaitingConnections.set(spec.connection.id, spec.connection);
|
|
1325
|
+
this.armTimer();
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
c.pending = {
|
|
1329
|
+
connection: spec.connection,
|
|
1330
|
+
connectionId: spec.connection.id,
|
|
1331
|
+
requestId: this.host.generateRequestId(),
|
|
1332
|
+
clientTools: spec.clientTools,
|
|
1333
|
+
body: spec.body,
|
|
1334
|
+
errorPrefix: spec.errorPrefix,
|
|
1335
|
+
prerequisite: null,
|
|
1336
|
+
pastCoalesce: false
|
|
1337
|
+
};
|
|
1338
|
+
c.awaitingConnections.set(spec.connection.id, spec.connection);
|
|
1339
|
+
this.armTimer();
|
|
1340
|
+
}
|
|
1341
|
+
/**
|
|
1342
|
+
* Re-arm the barrier for a result/approval that arrived WITHOUT `autoContinue`
|
|
1343
|
+
* (#1650). A standalone errored result declines to continue on its own, but in
|
|
1344
|
+
* a parallel batch a SIBLING may already have opted in — and this result can
|
|
1345
|
+
* be the one that completes the batch, so we must re-run the barrier check.
|
|
1346
|
+
* Unlike {@link schedule} this NEVER creates a pending continuation, and
|
|
1347
|
+
* no-ops once the continuation is running (`pastCoalesce`).
|
|
1348
|
+
*/
|
|
1349
|
+
rearmForBatch() {
|
|
1350
|
+
const pending = this.host.continuation.pending;
|
|
1351
|
+
if (!pending || pending.pastCoalesce) return;
|
|
1352
|
+
this.armTimer();
|
|
1353
|
+
}
|
|
1354
|
+
/** (Re)arm the coalesce timer; on fire, run {@link fireWhenStable}. */
|
|
1355
|
+
armTimer() {
|
|
1356
|
+
if (this._timer) clearTimeout(this._timer);
|
|
1357
|
+
this._timer = setTimeout(() => {
|
|
1358
|
+
this._timer = null;
|
|
1359
|
+
if (!this.host.continuation.pending) return;
|
|
1360
|
+
this.fireWhenStable();
|
|
1361
|
+
}, AutoContinuationController.COALESCE_MS);
|
|
1362
|
+
}
|
|
1363
|
+
/**
|
|
1364
|
+
* Fire an auto-continuation, but only once the model's parallel tool-call
|
|
1365
|
+
* batch is fully answered (#1649) and no assistant turn is mid-stream (#1650).
|
|
1366
|
+
* The barrier is event-driven with NO orphan timeout: when the batch is still
|
|
1367
|
+
* incomplete we drain the in-flight applies, re-check, and — if still
|
|
1368
|
+
* incomplete — return WITHOUT firing and WITHOUT holding the isolate, leaving
|
|
1369
|
+
* `continuation.pending` in place. The next sibling's result re-arms the
|
|
1370
|
+
* coalesce timer and re-runs this check; the continuation fires once the final
|
|
1371
|
+
* sibling lands. A true orphan (a sibling that never arrives) simply never
|
|
1372
|
+
* auto-continues — a later user turn / chat recovery repairs the transcript.
|
|
1373
|
+
*/
|
|
1374
|
+
fireWhenStable() {
|
|
1375
|
+
const c = this.host.continuation;
|
|
1376
|
+
if (!c.pending) return;
|
|
1377
|
+
if (c.pending.pastCoalesce) return;
|
|
1378
|
+
if (this._barrierActive) return;
|
|
1379
|
+
if (this.host.isStreamActive()) return;
|
|
1380
|
+
if (!this.host.hasPendingInteraction() && !this.host.hasIncompleteToolBatch()) {
|
|
1381
|
+
this.cancelTimer();
|
|
1382
|
+
this.host.fire();
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
this._barrierActive = true;
|
|
1386
|
+
this.host.keepAliveWhile(() => this.host.drainInteractionApplies()).catch(() => {}).finally(() => {
|
|
1387
|
+
this._barrierActive = false;
|
|
1388
|
+
const pending = c.pending;
|
|
1389
|
+
if (!pending || pending.pastCoalesce) return;
|
|
1390
|
+
if (this.host.isStreamActive()) return;
|
|
1391
|
+
if (this.host.hasIncompleteToolBatch()) return;
|
|
1392
|
+
this.cancelTimer();
|
|
1393
|
+
this.host.fire();
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1396
|
+
/**
|
|
1397
|
+
* Transition the deferred follow-up (stored while a continuation was running)
|
|
1398
|
+
* to pending and re-run the barrier — its batch may still be incomplete (or a
|
|
1399
|
+
* stream active), in which case it parks and re-arms instead of firing blind.
|
|
1400
|
+
*/
|
|
1401
|
+
activateDeferredAndReschedule() {
|
|
1402
|
+
if (!this.host.continuation.activateDeferred(() => this.host.generateRequestId())) return;
|
|
1403
|
+
this.fireWhenStable();
|
|
1404
|
+
}
|
|
1405
|
+
/**
|
|
1406
|
+
* Cancel any still-armed coalesce timer. Called on the fire path so a sibling
|
|
1407
|
+
* result that re-armed it during a barrier wait can't fire a duplicate
|
|
1408
|
+
* continuation after this one starts (#1649 / #1650).
|
|
1409
|
+
*/
|
|
1410
|
+
cancelTimer() {
|
|
1411
|
+
if (this._timer) {
|
|
1412
|
+
clearTimeout(this._timer);
|
|
1413
|
+
this._timer = null;
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
/**
|
|
1417
|
+
* `true` when the barrier is going to fire on its own — its coalesce timer is
|
|
1418
|
+
* still pending or its completeness drain is in progress. The host combines
|
|
1419
|
+
* this with its own pending/`pastCoalesce` checks to decide idle/stable.
|
|
1420
|
+
*/
|
|
1421
|
+
isArmed() {
|
|
1422
|
+
return this._timer !== null || this._barrierActive;
|
|
1423
|
+
}
|
|
1424
|
+
/**
|
|
1425
|
+
* Tear down the controller-owned barrier state (timer + double-fire guard).
|
|
1426
|
+
* Scoped to ONLY this controller's fields — the host clears the rest of its
|
|
1427
|
+
* turn state (stream gate, interaction tail, continuation data) separately.
|
|
1428
|
+
*/
|
|
1429
|
+
reset() {
|
|
1430
|
+
this.cancelTimer();
|
|
1431
|
+
this._barrierActive = false;
|
|
1432
|
+
}
|
|
1433
|
+
};
|
|
1434
|
+
AutoContinuationController.COALESCE_MS = 50;
|
|
1435
|
+
//#endregion
|
|
1316
1436
|
//#region src/chat/abort-registry.ts
|
|
1317
1437
|
/**
|
|
1318
1438
|
* AbortRegistry — manages per-request AbortControllers.
|
|
@@ -1415,6 +1535,58 @@ var AbortRegistry = class {
|
|
|
1415
1535
|
}
|
|
1416
1536
|
};
|
|
1417
1537
|
//#endregion
|
|
1538
|
+
//#region src/chat/async-helpers.ts
|
|
1539
|
+
/**
|
|
1540
|
+
* @internal Small async control-flow helpers shared by the chat hosts
|
|
1541
|
+
* (`@cloudflare/ai-chat` and `@cloudflare/think`) — not a public API. Extracted
|
|
1542
|
+
* so the host idle/stable waits and the interaction-apply completeness drain
|
|
1543
|
+
* stay byte-identical across both. See `design/chat-shared-layer.md`.
|
|
1544
|
+
*/
|
|
1545
|
+
/**
|
|
1546
|
+
* Sentinel returned by {@link awaitWithDeadline} when the deadline elapses
|
|
1547
|
+
* before the awaited promise settles. A single shared symbol so both hosts
|
|
1548
|
+
* compare against the same identity.
|
|
1549
|
+
*/
|
|
1550
|
+
const TIMED_OUT = Symbol("timed-out");
|
|
1551
|
+
/**
|
|
1552
|
+
* Await `promise`, but give up and resolve to {@link TIMED_OUT} once `deadline`
|
|
1553
|
+
* (an absolute `Date.now()` ms timestamp) passes. A `null` deadline waits
|
|
1554
|
+
* indefinitely (the promise is returned unchanged). The timeout timer is always
|
|
1555
|
+
* cleared so it can't pin the isolate awake past resolution.
|
|
1556
|
+
*/
|
|
1557
|
+
async function awaitWithDeadline(promise, deadline) {
|
|
1558
|
+
if (deadline == null) return promise;
|
|
1559
|
+
const remainingMs = Math.max(0, deadline - Date.now());
|
|
1560
|
+
let timer;
|
|
1561
|
+
const result = await Promise.race([promise, new Promise((resolve) => {
|
|
1562
|
+
timer = setTimeout(() => resolve(TIMED_OUT), remainingMs);
|
|
1563
|
+
})]);
|
|
1564
|
+
clearTimeout(timer);
|
|
1565
|
+
return result;
|
|
1566
|
+
}
|
|
1567
|
+
/**
|
|
1568
|
+
* Drain the host's interaction-apply chain so a subsequent completeness check
|
|
1569
|
+
* (e.g. `hasIncompleteToolBatch`) sees every tool result that has ALREADY
|
|
1570
|
+
* arrived.
|
|
1571
|
+
*
|
|
1572
|
+
* Bounded by real apply activity (a storage write each), never a fixed timer:
|
|
1573
|
+
* `getTail` is re-read after every await because a sibling can extend the tail
|
|
1574
|
+
* mid-drain, and the loop stops once the tail stops advancing. Bails early when
|
|
1575
|
+
* `hasPending()` goes false (the pending continuation was cleared by a chat
|
|
1576
|
+
* clear / turn reset) so a stale drain can't hold the isolate awake.
|
|
1577
|
+
*/
|
|
1578
|
+
async function drainInteractionApplies(hasPending, getTail) {
|
|
1579
|
+
let tail = getTail();
|
|
1580
|
+
for (;;) {
|
|
1581
|
+
if (!hasPending()) return;
|
|
1582
|
+
try {
|
|
1583
|
+
await tail;
|
|
1584
|
+
} catch {}
|
|
1585
|
+
if (getTail() === tail) return;
|
|
1586
|
+
tail = getTail();
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
//#endregion
|
|
1418
1590
|
//#region src/chat/tool-state.ts
|
|
1419
1591
|
/**
|
|
1420
1592
|
* Apply a tool part update to a parts array.
|
|
@@ -1561,16 +1733,80 @@ function toolApprovalUpdate(toolCallId, approved) {
|
|
|
1561
1733
|
return {
|
|
1562
1734
|
toolCallId,
|
|
1563
1735
|
matchStates: ["input-available", "approval-requested"],
|
|
1564
|
-
apply: (part) =>
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
...part
|
|
1569
|
-
approved
|
|
1570
|
-
|
|
1571
|
-
|
|
1736
|
+
apply: (part) => {
|
|
1737
|
+
const approval = typeof part.approval === "object" && part.approval !== null && !Array.isArray(part.approval) ? part.approval : void 0;
|
|
1738
|
+
const approvalId = typeof approval?.id === "string" ? approval.id : toolCallId;
|
|
1739
|
+
return {
|
|
1740
|
+
...part,
|
|
1741
|
+
state: approved ? "approval-responded" : "output-denied",
|
|
1742
|
+
approval: {
|
|
1743
|
+
...approval,
|
|
1744
|
+
id: approvalId,
|
|
1745
|
+
approved
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1572
1749
|
};
|
|
1573
1750
|
}
|
|
1751
|
+
/** Extract a tool part's name from its `tool-<name>` / `dynamic-tool` shape. */
|
|
1752
|
+
function toolPartName(record) {
|
|
1753
|
+
const type = typeof record.type === "string" ? record.type : "";
|
|
1754
|
+
if (type === "dynamic-tool") return typeof record.toolName === "string" ? record.toolName : void 0;
|
|
1755
|
+
if (type.startsWith("tool-")) return type.slice(5);
|
|
1756
|
+
}
|
|
1757
|
+
/**
|
|
1758
|
+
* Whether a part is still awaiting a CLIENT interaction that can genuinely
|
|
1759
|
+
* arrive after a restart: an `approval-requested` part (a reconnecting client
|
|
1760
|
+
* replays the approval) or an `input-available` part for a CLIENT tool (the SPA
|
|
1761
|
+
* replays the `tool-result`). A SERVER tool's `input-available` is NOT pending —
|
|
1762
|
+
* its `execute()` died with the isolate.
|
|
1763
|
+
*/
|
|
1764
|
+
function partAwaitsClientInteraction(part, clientResolvable) {
|
|
1765
|
+
if (typeof part !== "object" || part === null || !("state" in part)) return false;
|
|
1766
|
+
const record = part;
|
|
1767
|
+
const state = record.state;
|
|
1768
|
+
if (state === "approval-requested") return true;
|
|
1769
|
+
if (state !== "input-available") return false;
|
|
1770
|
+
const toolName = toolPartName(record);
|
|
1771
|
+
return toolName != null && clientResolvable.has(toolName);
|
|
1772
|
+
}
|
|
1773
|
+
/**
|
|
1774
|
+
* Names of the CLIENT-resolvable tools — the client-provided schemas from the
|
|
1775
|
+
* last request, which have no server `execute`. An interrupted `input-available`
|
|
1776
|
+
* part for one of these can still be resolved by the client replaying a
|
|
1777
|
+
* `tool-result`; a server tool's cannot.
|
|
1778
|
+
*/
|
|
1779
|
+
function clientResolvableToolNames(tools) {
|
|
1780
|
+
const names = /* @__PURE__ */ new Set();
|
|
1781
|
+
for (const tool of tools ?? []) if (tool?.name) names.add(tool.name);
|
|
1782
|
+
return names;
|
|
1783
|
+
}
|
|
1784
|
+
/**
|
|
1785
|
+
* `true` when the latest assistant message is mid-batch: it carries at least
|
|
1786
|
+
* one settled tool result AND at least one tool call/approval still awaiting a
|
|
1787
|
+
* client result. That is the #1649 signature — the model fanned out parallel
|
|
1788
|
+
* tool calls and only some have been answered. Scoped to the leaf (the step the
|
|
1789
|
+
* continuation answers) so an unrelated dangling tool in an earlier message
|
|
1790
|
+
* doesn't block a legitimate follow-up continuation.
|
|
1791
|
+
*/
|
|
1792
|
+
function hasIncompleteToolBatch(messages) {
|
|
1793
|
+
let leaf;
|
|
1794
|
+
for (let i = messages.length - 1; i >= 0; i--) if (messages[i].role === "assistant") {
|
|
1795
|
+
leaf = messages[i];
|
|
1796
|
+
break;
|
|
1797
|
+
}
|
|
1798
|
+
if (!leaf) return false;
|
|
1799
|
+
let hasPending = false;
|
|
1800
|
+
let hasSettled = false;
|
|
1801
|
+
for (const part of leaf.parts) {
|
|
1802
|
+
const record = part;
|
|
1803
|
+
const state = record.state;
|
|
1804
|
+
if (state === "input-available" || state === "approval-requested") hasPending = true;
|
|
1805
|
+
else if (typeof record.type === "string" && (record.type.startsWith("tool-") || record.type === "dynamic-tool") && (state === "output-available" || state === "output-error" || state === "output-denied" || state === "approval-responded")) hasSettled = true;
|
|
1806
|
+
if (hasPending && hasSettled) return true;
|
|
1807
|
+
}
|
|
1808
|
+
return false;
|
|
1809
|
+
}
|
|
1574
1810
|
//#endregion
|
|
1575
1811
|
//#region src/chat/parse-protocol.ts
|
|
1576
1812
|
/**
|
|
@@ -1682,6 +1918,48 @@ function resolveToolMergeId(message, serverMessages) {
|
|
|
1682
1918
|
return message;
|
|
1683
1919
|
}
|
|
1684
1920
|
/**
|
|
1921
|
+
* Merge a freshly-reconstructed orphaned partial onto the assistant message
|
|
1922
|
+
* that already owns its target id (the orphan-persist **(c)** step).
|
|
1923
|
+
*
|
|
1924
|
+
* Used by hosts whose store can hold an assistant row for the SAME id BEFORE
|
|
1925
|
+
* the stream finalizes — e.g. an early persist at tool-approval time, or a
|
|
1926
|
+
* continuation resuming the prior assistant message. On recovery the engine
|
|
1927
|
+
* replays the same chunks, so a naive append would leave two parts per tool
|
|
1928
|
+
* call. The merge therefore:
|
|
1929
|
+
*
|
|
1930
|
+
* - keeps ALL existing parts (the persisted row is authoritative for tool
|
|
1931
|
+
* parts that had a client result applied IN PLACE — that result lives only
|
|
1932
|
+
* in storage, never in the chunk stream, so a whole-message replace would
|
|
1933
|
+
* clobber it);
|
|
1934
|
+
* - appends only the reconstructed parts whose `toolCallId` is NOT already
|
|
1935
|
+
* present (dedup by tool-call identity);
|
|
1936
|
+
* - overlays the incoming metadata onto the existing metadata (incoming wins
|
|
1937
|
+
* on conflicts), falling back to whichever side is present.
|
|
1938
|
+
*
|
|
1939
|
+
* The result carries the INCOMING message's id/role (the caller has already
|
|
1940
|
+
* resolved the incoming id to the existing row's id via the (b) target-id
|
|
1941
|
+
* step), so it is safe to write straight back through `updateMessage`.
|
|
1942
|
+
*
|
|
1943
|
+
* Hosts whose orphan persist only ever runs at stream finalize (no early/
|
|
1944
|
+
* mid-stream row for the same id) never hit the merge branch and don't need
|
|
1945
|
+
* this — a plain append/replace is already dedup-safe because the shared
|
|
1946
|
+
* reconstruction (`StreamAccumulator` / `applyChunkToParts`) is idempotent by
|
|
1947
|
+
* `toolCallId`.
|
|
1948
|
+
*/
|
|
1949
|
+
function reconcileOrphanPartial(existing, incoming) {
|
|
1950
|
+
const existingToolCallIds = new Set(existing.parts.filter((p) => "toolCallId" in p).map((p) => p.toolCallId));
|
|
1951
|
+
const newParts = incoming.parts.filter((p) => !("toolCallId" in p && existingToolCallIds.has(p.toolCallId)));
|
|
1952
|
+
const merged = {
|
|
1953
|
+
...incoming,
|
|
1954
|
+
parts: [...existing.parts, ...newParts]
|
|
1955
|
+
};
|
|
1956
|
+
if (existing.metadata) merged.metadata = incoming.metadata ? {
|
|
1957
|
+
...existing.metadata,
|
|
1958
|
+
...incoming.metadata
|
|
1959
|
+
} : existing.metadata;
|
|
1960
|
+
return merged;
|
|
1961
|
+
}
|
|
1962
|
+
/**
|
|
1685
1963
|
* Content key for assistant messages used for dedup of identical short replies.
|
|
1686
1964
|
* Returns JSON of sanitized parts, or undefined for non-assistant messages.
|
|
1687
1965
|
*/
|
|
@@ -1770,6 +2048,126 @@ function findMessageByToolCallId(messages, toolCallId) {
|
|
|
1770
2048
|
}
|
|
1771
2049
|
}
|
|
1772
2050
|
//#endregion
|
|
2051
|
+
//#region src/chat/repair-transcript.ts
|
|
2052
|
+
/**
|
|
2053
|
+
* Whether a tool part already has a settled result the provider accepts, so it
|
|
2054
|
+
* must NOT be re-repaired into an errored result.
|
|
2055
|
+
*
|
|
2056
|
+
* Single source of truth for the terminal tool states. Mirrors the AI SDK's
|
|
2057
|
+
* terminal states: `convertToModelMessages` emits a `tool-result` for
|
|
2058
|
+
* `output-available`, `output-error`, AND `output-denied` (a user-denied
|
|
2059
|
+
* approval — its denial reason becomes the tool-result). Omitting any of these
|
|
2060
|
+
* makes repair re-flip the part every turn — clobbering a real `errorText` /
|
|
2061
|
+
* denial with the generic "interrupted" message.
|
|
2062
|
+
*/
|
|
2063
|
+
function toolPartHasSettledResult(record) {
|
|
2064
|
+
if ("output" in record || "result" in record) return true;
|
|
2065
|
+
const state = typeof record.state === "string" ? record.state : "";
|
|
2066
|
+
return state === "output-available" || state === "output-error" || state === "output-denied";
|
|
2067
|
+
}
|
|
2068
|
+
/**
|
|
2069
|
+
* Repair interrupted tool calls and normalize malformed tool inputs across a
|
|
2070
|
+
* transcript. Behavior mirrors `@cloudflare/think`'s original
|
|
2071
|
+
* `_repairToolTranscriptParts`:
|
|
2072
|
+
*
|
|
2073
|
+
* - a tool part with NO settled result and state `approval-responded` is kept
|
|
2074
|
+
* verbatim (an approved server tool waiting for its continuation to run
|
|
2075
|
+
* `execute()` — not abandoned);
|
|
2076
|
+
* - a tool part with NO settled result for which `shouldRepair` returns false
|
|
2077
|
+
* is kept verbatim (a part still awaiting a CLIENT interaction; see option);
|
|
2078
|
+
* - any other tool part with no settled result is normalized then handed to
|
|
2079
|
+
* `repairPart` (default: flipped to an errored result);
|
|
2080
|
+
* - a tool part WITH a settled result only has its `input` normalized.
|
|
2081
|
+
*
|
|
2082
|
+
* Messages with no changed part keep their original object reference so callers
|
|
2083
|
+
* can cheaply detect what to persist.
|
|
2084
|
+
*/
|
|
2085
|
+
function repairInterruptedToolParts(messages, options) {
|
|
2086
|
+
const isSettled = options.isSettled ?? toolPartHasSettledResult;
|
|
2087
|
+
const normalizeInput = options.normalizeInput ?? normalizeToolInput;
|
|
2088
|
+
const { repairPart } = options;
|
|
2089
|
+
const shouldRepair = options.shouldRepair ?? (() => true);
|
|
2090
|
+
let removedToolCalls = 0;
|
|
2091
|
+
let normalizedInputs = 0;
|
|
2092
|
+
const toolCallIds = [];
|
|
2093
|
+
const repaired = [];
|
|
2094
|
+
for (const message of messages) {
|
|
2095
|
+
const parts = [];
|
|
2096
|
+
let messageChanged = false;
|
|
2097
|
+
for (const part of message.parts) {
|
|
2098
|
+
const record = part;
|
|
2099
|
+
const toolCallId = typeof record.toolCallId === "string" ? record.toolCallId : void 0;
|
|
2100
|
+
if (!(typeof record.type === "string" && (record.type.startsWith("tool-") || record.type === "dynamic-tool") && toolCallId)) {
|
|
2101
|
+
parts.push(part);
|
|
2102
|
+
continue;
|
|
2103
|
+
}
|
|
2104
|
+
if (!isSettled(record)) {
|
|
2105
|
+
if ((typeof record.state === "string" ? record.state : "") === "approval-responded") {
|
|
2106
|
+
parts.push(part);
|
|
2107
|
+
continue;
|
|
2108
|
+
}
|
|
2109
|
+
if (!shouldRepair(part)) {
|
|
2110
|
+
parts.push(part);
|
|
2111
|
+
continue;
|
|
2112
|
+
}
|
|
2113
|
+
const normalized = normalizeInput("input" in record ? record.input : void 0);
|
|
2114
|
+
parts.push(repairPart({
|
|
2115
|
+
...part,
|
|
2116
|
+
input: normalized.input
|
|
2117
|
+
}));
|
|
2118
|
+
if (normalized.changed) normalizedInputs++;
|
|
2119
|
+
removedToolCalls++;
|
|
2120
|
+
messageChanged = true;
|
|
2121
|
+
toolCallIds.push(toolCallId);
|
|
2122
|
+
continue;
|
|
2123
|
+
}
|
|
2124
|
+
const normalized = normalizeInput("input" in record ? record.input : void 0);
|
|
2125
|
+
if (normalized.changed) {
|
|
2126
|
+
parts.push({
|
|
2127
|
+
...part,
|
|
2128
|
+
input: normalized.input
|
|
2129
|
+
});
|
|
2130
|
+
normalizedInputs++;
|
|
2131
|
+
messageChanged = true;
|
|
2132
|
+
continue;
|
|
2133
|
+
}
|
|
2134
|
+
parts.push(part);
|
|
2135
|
+
}
|
|
2136
|
+
repaired.push(messageChanged ? {
|
|
2137
|
+
...message,
|
|
2138
|
+
parts
|
|
2139
|
+
} : message);
|
|
2140
|
+
}
|
|
2141
|
+
return {
|
|
2142
|
+
messages: repaired,
|
|
2143
|
+
removedToolCalls,
|
|
2144
|
+
normalizedInputs,
|
|
2145
|
+
toolCallIds
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
//#endregion
|
|
2149
|
+
//#region src/chat/orphan-persist.ts
|
|
2150
|
+
/**
|
|
2151
|
+
* Reconstruct a message from `chunks` and upsert it via the store. Returns
|
|
2152
|
+
* `true` when a write happened (so a caller that broadcasts after — Think — can
|
|
2153
|
+
* gate its broadcast on it), `false` when there was nothing to persist (no
|
|
2154
|
+
* parts, or `prepare` returned `null`).
|
|
2155
|
+
*/
|
|
2156
|
+
async function persistReconstructedOrphan(chunks, options) {
|
|
2157
|
+
if (chunks.length === 0) return false;
|
|
2158
|
+
const accumulator = new StreamAccumulator({ messageId: options.fallbackId });
|
|
2159
|
+
for (const chunk of chunks) try {
|
|
2160
|
+
accumulator.applyChunk(JSON.parse(chunk.body));
|
|
2161
|
+
} catch {}
|
|
2162
|
+
if (accumulator.parts.length === 0) return false;
|
|
2163
|
+
const prepared = options.prepare(accumulator.toMessage());
|
|
2164
|
+
if (prepared === null) return false;
|
|
2165
|
+
const existing = await options.store.getMessage(prepared.id);
|
|
2166
|
+
if (existing) await options.store.updateMessage(options.merge(existing, prepared));
|
|
2167
|
+
else await options.store.appendMessage(prepared);
|
|
2168
|
+
return true;
|
|
2169
|
+
}
|
|
2170
|
+
//#endregion
|
|
1773
2171
|
//#region src/chat/recovery.ts
|
|
1774
2172
|
function createChatFiberSnapshot({ kind, requestId, recoveryRootRequestId, continuation, messages, lastBody, lastClientTools }) {
|
|
1775
2173
|
const latestMessage = messages.length > 0 ? messages[messages.length - 1] : void 0;
|
|
@@ -1820,6 +2218,1191 @@ function unwrapChatFiberSnapshot(key, value, expectedKind) {
|
|
|
1820
2218
|
};
|
|
1821
2219
|
}
|
|
1822
2220
|
//#endregion
|
|
1823
|
-
|
|
2221
|
+
//#region src/chat/recovery-codec.ts
|
|
2222
|
+
/**
|
|
2223
|
+
* `ChatRecoveryCodec` — the streaming-codec seam the recovery engine replays an
|
|
2224
|
+
* interrupted turn's durable buffer through to reconstruct its partial assistant
|
|
2225
|
+
* state. The engine and hosts only ever see the wire-agnostic `RecoveryPartial`
|
|
2226
|
+
* shape (`{ text, parts }`); the codec owns the chunk-vocabulary differences.
|
|
2227
|
+
*
|
|
2228
|
+
* Two implementations exist today: {@link AISDKRecoveryCodec} (AI SDK SSE chunks,
|
|
2229
|
+
* used by `@cloudflare/ai-chat` and `@cloudflare/think`) and `PiRecoveryCodec`
|
|
2230
|
+
* (the pi `AgentEvent` vocabulary, in the `experimental/pi-recovery` fixture).
|
|
2231
|
+
* Formalizing the interface here is the proof that the codec — not the engine —
|
|
2232
|
+
* carries the chunk-shape contract.
|
|
2233
|
+
*
|
|
2234
|
+
* @internal Shared chat-recovery internals; not a public API.
|
|
2235
|
+
*/
|
|
2236
|
+
/**
|
|
2237
|
+
* Whether a reconstructed AI SDK `UIMessage` parts array carries any settled
|
|
2238
|
+
* (provider-accepted) tool result — the completed, often non-idempotent work
|
|
2239
|
+
* that a `{ persist: false }` recovery return would otherwise silently discard
|
|
2240
|
+
* (#1631). A part counts as settled when it is a tool part (`tool-*` /
|
|
2241
|
+
* `dynamic-tool`) carrying an `output`/`result`, or whose state reached a
|
|
2242
|
+
* terminal `output-{available,error,denied}`.
|
|
2243
|
+
*
|
|
2244
|
+
* This is the AI SDK codec's implementation of the per-vocabulary "did this
|
|
2245
|
+
* partial settle a tool?" question. It lives with {@link AISDKRecoveryCodec}
|
|
2246
|
+
* (not in the engine) because the codec owns the part vocabulary — the engine
|
|
2247
|
+
* only ever reads the precomputed `RecoveryPartial.hasSettledToolResults`
|
|
2248
|
+
* boolean and never names a part type. Foreign codecs (e.g. AG-UI) compute the
|
|
2249
|
+
* same boolean from their own chunk vocabulary without producing AI SDK parts.
|
|
2250
|
+
*/
|
|
2251
|
+
function partialHasSettledToolResults(parts) {
|
|
2252
|
+
return parts.some((part) => {
|
|
2253
|
+
const record = part;
|
|
2254
|
+
const type = typeof record.type === "string" ? record.type : "";
|
|
2255
|
+
if (!(type.startsWith("tool-") || type === "dynamic-tool")) return false;
|
|
2256
|
+
if ("output" in record || "result" in record) return true;
|
|
2257
|
+
const state = typeof record.state === "string" ? record.state : "";
|
|
2258
|
+
return state === "output-available" || state === "output-error" || state === "output-denied";
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
/**
|
|
2262
|
+
* The single, host-agnostic rule for crediting recovery forward progress from a
|
|
2263
|
+
* stored stream chunk — the convergence of what `AIChatAgent` and `Think`
|
|
2264
|
+
* previously each decided on their own (ai-chat keyed on chunk type only; Think
|
|
2265
|
+
* keyed on its flush cadence). Both hosts now call this at chunk-store time so
|
|
2266
|
+
* the bump TIMING is identical:
|
|
2267
|
+
*
|
|
2268
|
+
* - a **milestone** ({@link ChatRecoveryCodec.isProgressChunk}) always credits;
|
|
2269
|
+
* - **streaming content** ({@link ChatRecoveryCodec.isStreamingContentChunk})
|
|
2270
|
+
* credits at most once per throttle window, so a long single segment still
|
|
2271
|
+
* registers progress across crashes without writing storage per token;
|
|
2272
|
+
* - anything else never credits.
|
|
2273
|
+
*
|
|
2274
|
+
* Finer than either host's prior cadence in the worst case and never coarser, so
|
|
2275
|
+
* it can only delay/avoid a false `no_progress_timeout`, never hasten give-up.
|
|
2276
|
+
*/
|
|
2277
|
+
function shouldCreditStreamProgress(input) {
|
|
2278
|
+
const { codec, type, throttle, now } = input;
|
|
2279
|
+
if (codec.isProgressChunk(type)) return true;
|
|
2280
|
+
if (codec.isStreamingContentChunk(type)) return throttle.shouldCredit(now);
|
|
2281
|
+
return false;
|
|
2282
|
+
}
|
|
2283
|
+
/**
|
|
2284
|
+
* The AI SDK codec: replays SSE chunk bodies through {@link getPartialStreamText}
|
|
2285
|
+
* (`applyChunkToParts` under the hood). Stateless — share the
|
|
2286
|
+
* {@link aiSdkRecoveryCodec} singleton rather than constructing per call.
|
|
2287
|
+
*/
|
|
2288
|
+
var AISDKRecoveryCodec = class {
|
|
2289
|
+
toRecoveryPartial(bodies) {
|
|
2290
|
+
const { text, parts } = getPartialStreamText(bodies.map((body) => ({ body })));
|
|
2291
|
+
return {
|
|
2292
|
+
text,
|
|
2293
|
+
parts,
|
|
2294
|
+
hasSettledToolResults: partialHasSettledToolResults(parts)
|
|
2295
|
+
};
|
|
2296
|
+
}
|
|
2297
|
+
isProgressChunk(type) {
|
|
2298
|
+
return type === "text-start" || type === "reasoning-start" || type === "tool-input-available" || type === "tool-output-available" || type === "tool-output-error" || type === "tool-output-denied";
|
|
2299
|
+
}
|
|
2300
|
+
isStreamingContentChunk(type) {
|
|
2301
|
+
return type === "text-delta" || type === "reasoning-delta" || type === "tool-input-delta";
|
|
2302
|
+
}
|
|
2303
|
+
};
|
|
2304
|
+
/** Shared stateless {@link AISDKRecoveryCodec} instance. */
|
|
2305
|
+
const aiSdkRecoveryCodec = new AISDKRecoveryCodec();
|
|
2306
|
+
//#endregion
|
|
2307
|
+
//#region src/chat/resume-handshake.ts
|
|
2308
|
+
/**
|
|
2309
|
+
* Drives the server side of the stream-resume protocol over a
|
|
2310
|
+
* {@link ResumeHandshakeHost}. Construct once per agent (the host wires its
|
|
2311
|
+
* `ResumableStream` / `ContinuationState` / pending set in) and call the three
|
|
2312
|
+
* public methods from the host's existing onConnect / onMessage wiring, so
|
|
2313
|
+
* handler registration timing stays host-owned.
|
|
2314
|
+
*/
|
|
2315
|
+
var ResumeHandshake = class {
|
|
2316
|
+
constructor(host) {
|
|
2317
|
+
this.host = host;
|
|
2318
|
+
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Notify a connection that an active stream can be resumed; it should reply
|
|
2321
|
+
* with `STREAM_RESUME_ACK` to receive the replay.
|
|
2322
|
+
*
|
|
2323
|
+
* A connection can legitimately be notified more than once for the same
|
|
2324
|
+
* request — proactively from onConnect AND in response to its explicit
|
|
2325
|
+
* `STREAM_RESUME_REQUEST` (#1733). This is intentional and must NOT be deduped
|
|
2326
|
+
* here: an explicit request always deserves a response (else the client's
|
|
2327
|
+
* `reconnectToStream` hangs to its timeout with no replay), and the proactive
|
|
2328
|
+
* notify is required for clients that never send a request. The notify is one
|
|
2329
|
+
* tiny frame; the client dedupes its ACK so the buffer is not replayed twice.
|
|
2330
|
+
*/
|
|
2331
|
+
notifyStreamResuming(connection) {
|
|
2332
|
+
const { resumableStream, pendingResumeConnections } = this.host;
|
|
2333
|
+
if (!resumableStream.hasActiveStream()) return;
|
|
2334
|
+
if (sendIfOpen(connection, JSON.stringify({
|
|
2335
|
+
type: CHAT_MESSAGE_TYPES.STREAM_RESUMING,
|
|
2336
|
+
id: resumableStream.activeRequestId
|
|
2337
|
+
}))) pendingResumeConnections.add(connection.id);
|
|
2338
|
+
}
|
|
2339
|
+
/**
|
|
2340
|
+
* Handle a client `STREAM_RESUME_REQUEST`. The client sends this after its
|
|
2341
|
+
* message handler is registered, avoiding the race where a proactive
|
|
2342
|
+
* `STREAM_RESUMING` from onConnect arrives before the handler is ready.
|
|
2343
|
+
*/
|
|
2344
|
+
async handleResumeRequest(connection) {
|
|
2345
|
+
const { resumableStream, continuation, preStream } = this.host;
|
|
2346
|
+
if (resumableStream.hasActiveStream()) if (continuation.activeRequestId === resumableStream.activeRequestId && continuation.activeConnectionId !== null && continuation.activeConnectionId !== connection.id && this._ownerStillPresent(continuation.activeConnectionId)) sendIfOpen(connection, JSON.stringify({ type: CHAT_MESSAGE_TYPES.STREAM_RESUME_NONE }));
|
|
2347
|
+
else this.notifyStreamResuming(connection);
|
|
2348
|
+
else if (continuation.pending !== null && (continuation.pending.connectionId === null || continuation.pending.connectionId === connection.id)) {
|
|
2349
|
+
continuation.awaitingConnections.set(connection.id, connection);
|
|
2350
|
+
this._sendStreamPending(connection, continuation.pending.requestId);
|
|
2351
|
+
} else if (await this._replayTerminalOnResume(connection)) {} else if (preStream?.park(connection)) {} else sendIfOpen(connection, JSON.stringify({ type: CHAT_MESSAGE_TYPES.STREAM_RESUME_NONE }));
|
|
2352
|
+
}
|
|
2353
|
+
/** Send a keep-waiting `STREAM_PENDING` frame (#1784). */
|
|
2354
|
+
_sendStreamPending(connection, requestId) {
|
|
2355
|
+
sendIfOpen(connection, JSON.stringify({
|
|
2356
|
+
type: CHAT_MESSAGE_TYPES.STREAM_PENDING,
|
|
2357
|
+
...requestId ? { id: requestId } : {}
|
|
2358
|
+
}));
|
|
2359
|
+
}
|
|
2360
|
+
/** Whether the active continuation's owner connection is still present. */
|
|
2361
|
+
_ownerStillPresent(connectionId) {
|
|
2362
|
+
return this.host.isConnectionPresent ? this.host.isConnectionPresent(connectionId) : true;
|
|
2363
|
+
}
|
|
2364
|
+
/** Handle a client `STREAM_RESUME_ACK` for `requestId`. */
|
|
2365
|
+
async handleResumeAck(connection, requestId) {
|
|
2366
|
+
const { resumableStream, pendingResumeConnections, responseMessageType } = this.host;
|
|
2367
|
+
pendingResumeConnections.delete(connection.id);
|
|
2368
|
+
if (resumableStream.hasActiveStream() && resumableStream.activeRequestId === requestId) {
|
|
2369
|
+
const orphanedStreamId = resumableStream.replayChunks(connection, resumableStream.activeRequestId);
|
|
2370
|
+
if (orphanedStreamId) await this.host.persistOrphanedStream(orphanedStreamId);
|
|
2371
|
+
} else if (resumableStream.hasActiveStream()) {} else if (await this._replayTerminalOnAck(connection, requestId)) {} else if (!resumableStream.replayCompletedChunksByRequestId(connection, requestId)) sendIfOpen(connection, JSON.stringify({
|
|
2372
|
+
body: "",
|
|
2373
|
+
done: true,
|
|
2374
|
+
id: requestId,
|
|
2375
|
+
type: responseMessageType,
|
|
2376
|
+
replay: true
|
|
2377
|
+
}));
|
|
2378
|
+
}
|
|
2379
|
+
/**
|
|
2380
|
+
* Replay a pending terminal outcome (#1645) over the resume handshake so a
|
|
2381
|
+
* reconnecting client surfaces it exactly like a live exhaustion. The bare
|
|
2382
|
+
* terminal frame is dropped by the client unless it arrives on a resumed
|
|
2383
|
+
* stream — the only path that reaches the transport's stream reader and
|
|
2384
|
+
* becomes `useChat.error` — so we drive `STREAM_RESUMING` here and deliver the
|
|
2385
|
+
* error frame once the client ACKs (see {@link _replayTerminalOnAck}). Returns
|
|
2386
|
+
* `true` if a terminal was pending (and `STREAM_RESUMING` was sent).
|
|
2387
|
+
*/
|
|
2388
|
+
async _replayTerminalOnResume(connection) {
|
|
2389
|
+
const pending = await this.host.pendingChatTerminal();
|
|
2390
|
+
if (!pending) return false;
|
|
2391
|
+
sendIfOpen(connection, JSON.stringify({
|
|
2392
|
+
type: CHAT_MESSAGE_TYPES.STREAM_RESUMING,
|
|
2393
|
+
id: pending.requestId
|
|
2394
|
+
}));
|
|
2395
|
+
return true;
|
|
2396
|
+
}
|
|
2397
|
+
/**
|
|
2398
|
+
* Deliver the pending terminal error frame on the resumed stream the client
|
|
2399
|
+
* ACKed (#1645). The record is retained (not cleared) so concurrent reconnects
|
|
2400
|
+
* (e.g. multiple tabs) each learn the outcome; it is cleared when a later turn
|
|
2401
|
+
* supersedes it.
|
|
2402
|
+
*/
|
|
2403
|
+
async _replayTerminalOnAck(connection, requestId) {
|
|
2404
|
+
const { resumableStream, responseMessageType } = this.host;
|
|
2405
|
+
const pending = await this.host.pendingChatTerminal();
|
|
2406
|
+
if (!pending || pending.requestId !== requestId) return false;
|
|
2407
|
+
if (!resumableStream.replayErroredChunksByRequestId(connection, pending.requestId)) return true;
|
|
2408
|
+
sendIfOpen(connection, JSON.stringify({
|
|
2409
|
+
body: pending.body,
|
|
2410
|
+
done: true,
|
|
2411
|
+
error: true,
|
|
2412
|
+
id: pending.requestId,
|
|
2413
|
+
type: responseMessageType
|
|
2414
|
+
}));
|
|
2415
|
+
return true;
|
|
2416
|
+
}
|
|
2417
|
+
};
|
|
2418
|
+
//#endregion
|
|
2419
|
+
//#region src/chat/recovery-incident.ts
|
|
2420
|
+
const CHAT_RECOVERY_INCIDENT_KEY_PREFIX = "cf:chat-recovery:incident:";
|
|
2421
|
+
/**
|
|
2422
|
+
* Durable, monotonic forward-progress counter for recovery budget resets.
|
|
2423
|
+
* Bumped at production time when new content is streamed, so it reflects
|
|
2424
|
+
* genuinely new content and is immune to reconnects/re-persists; never
|
|
2425
|
+
* recomputed from the (compactable) transcript.
|
|
2426
|
+
*/
|
|
2427
|
+
const CHAT_RECOVERY_PROGRESS_KEY = "cf:chat-recovery:progress";
|
|
2428
|
+
/**
|
|
2429
|
+
* Durable record of an in-progress recovery so a "recovering…" status (#1620)
|
|
2430
|
+
* can be broadcast live and survive the set/clear happening in different
|
|
2431
|
+
* isolates (a continuation runs in a later alarm invocation).
|
|
2432
|
+
*/
|
|
2433
|
+
const CHAT_RECOVERING_KEY = "cf:chat:recovering";
|
|
2434
|
+
/**
|
|
2435
|
+
* Durable record of the last turn that ended in a terminal error / abandoned
|
|
2436
|
+
* recovery (#1645). Replayed on the next reconnect via the resume handshake;
|
|
2437
|
+
* cleared when a later turn supersedes it.
|
|
2438
|
+
*/
|
|
2439
|
+
const CHAT_LAST_TERMINAL_KEY = "cf:chat:last-terminal";
|
|
2440
|
+
/**
|
|
2441
|
+
* Secondary backstop only. The primary recovery bound is the no-progress wall
|
|
2442
|
+
* clock; with alarm debounce this cap rarely binds (it catches a pathological
|
|
2443
|
+
* tight alarm-loop). Kept high so the no-progress window seals first under
|
|
2444
|
+
* normal deploy cadence (#1637).
|
|
2445
|
+
*/
|
|
2446
|
+
const DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS = 10;
|
|
2447
|
+
/**
|
|
2448
|
+
* Runaway-loop guard default — the framework-imposed backstop on cumulative
|
|
2449
|
+
* recovery WORK (produced content/tool units) since an incident opened.
|
|
2450
|
+
*
|
|
2451
|
+
* Originally `Infinity` (rfc-chat-recovery-work-budget): the SDK shipped the
|
|
2452
|
+
* *mechanism* but no default cap, so a progressing turn was never terminated on
|
|
2453
|
+
* its own. Production issue #1825 showed that this is a footgun: an isolate that
|
|
2454
|
+
* OOMs mid-stream still credits a little progress before it dies, which resets
|
|
2455
|
+
* BOTH progress-keyed bounds (the attempt cap and the no-progress window) on
|
|
2456
|
+
* every wake — and a fast crash loop (each attempt inside the alarm-debounce
|
|
2457
|
+
* window) pins the attempt counter too. With `maxRecoveryWork = Infinity` the
|
|
2458
|
+
* ONLY instrument whose meter still climbs across such a loop is disabled, so
|
|
2459
|
+
* recovery re-runs the turn (and its LLM calls) forever.
|
|
2460
|
+
*
|
|
2461
|
+
* A finite default closes that loop out of the box: work climbs regardless of
|
|
2462
|
+
* debounce/progress resets, so a content-emitting runaway is always sealed with
|
|
2463
|
+
* `reason="work_budget_exceeded"`. The value is deliberately generous — it
|
|
2464
|
+
* bounds wasted re-run cost without clipping a normal interrupted turn (work
|
|
2465
|
+
* only accrues from the first interruption until the turn completes, after which
|
|
2466
|
+
* the incident is deleted). A very long agentic turn under heavy interruption
|
|
2467
|
+
* that legitimately needs more should raise `maxRecoveryWork` (or set it to
|
|
2468
|
+
* `Infinity` to restore the pre-#1825 unbounded behavior).
|
|
2469
|
+
*/
|
|
2470
|
+
const DEFAULT_CHAT_RECOVERY_MAX_WORK = 1e3;
|
|
2471
|
+
/**
|
|
2472
|
+
* Tight, OOM-specific retry budget (#1825). A Durable Object memory-limit reset
|
|
2473
|
+
* (`isDurableObjectMemoryLimitReset`) is usually deterministic — the turn's
|
|
2474
|
+
* working set no longer fits in the isolate's 128 MB — so re-running it re-OOMs.
|
|
2475
|
+
* But a single OOM CAN be a transient spike (the isolate's 128 MB is shared
|
|
2476
|
+
* across the global scope / noisy neighbors), so recovery retries a small number
|
|
2477
|
+
* of times before sealing with `reason="out_of_memory"` rather than abandoning a
|
|
2478
|
+
* turn that one more attempt might have completed. Far tighter than the generic
|
|
2479
|
+
* `maxRecoveryWork` backstop because an OOM is attributable and re-running it is
|
|
2480
|
+
* expensive (it re-runs the model). Counts attempts that ended in an OOM, not
|
|
2481
|
+
* total attempts, so a turn interrupted by deploys (no OOM) is unaffected.
|
|
2482
|
+
*/
|
|
2483
|
+
const DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES = 3;
|
|
2484
|
+
const DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS = 1e4;
|
|
2485
|
+
/**
|
|
2486
|
+
* Delay before retrying a recovery that timed out waiting for stable state.
|
|
2487
|
+
* Gives an actively-churning isolate (e.g. a deploy in flight) time to settle.
|
|
2488
|
+
*/
|
|
2489
|
+
const CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS = 3;
|
|
2490
|
+
const DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE = "The assistant was interrupted and could not recover. Please try again.";
|
|
2491
|
+
/**
|
|
2492
|
+
* Incidents that have not seen a new attempt within this window are assumed
|
|
2493
|
+
* abandoned and swept so durable storage does not grow without bound.
|
|
2494
|
+
*/
|
|
2495
|
+
const CHAT_RECOVERY_INCIDENT_TTL_MS = 3600 * 1e3;
|
|
2496
|
+
/** Max keys per Durable Object KV `delete([...])` call. */
|
|
2497
|
+
const KV_DELETE_MAX_KEYS = 128;
|
|
2498
|
+
/**
|
|
2499
|
+
* PRIMARY recovery bound (#1637): seal an incident that has made no forward
|
|
2500
|
+
* progress for this long. Keyed to `lastProgressAt`, which resets on every
|
|
2501
|
+
* progress-bearing attempt — so a turn that keeps producing content survives
|
|
2502
|
+
* deploy churn indefinitely, while a genuinely stuck turn dies within 5 min.
|
|
2503
|
+
*/
|
|
2504
|
+
const DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS = 300 * 1e3;
|
|
2505
|
+
/**
|
|
2506
|
+
* Alarm debounce: recovery alarms bunched within this window collapse into a
|
|
2507
|
+
* single attempt. A deploy rollout drops/reconnects the socket several times
|
|
2508
|
+
* over ~11–22s; without this, one logical deploy would burn several attempts.
|
|
2509
|
+
*/
|
|
2510
|
+
const CHAT_RECOVERY_ALARM_DEBOUNCE_MS = 30 * 1e3;
|
|
2511
|
+
/**
|
|
2512
|
+
* Staleness bound for the live "recovering…" flag (#1620). A flag older than
|
|
2513
|
+
* this is treated as abandoned so it can neither pin the indicator on forever
|
|
2514
|
+
* nor suppress a genuinely-new recovering signal. NOT a recovery budget.
|
|
2515
|
+
*/
|
|
2516
|
+
const CHAT_RECOVERING_FLAG_TTL_MS = 900 * 1e3;
|
|
2517
|
+
/**
|
|
2518
|
+
* Resolve a raw `chatRecovery` config field into the fully-defaulted form the
|
|
2519
|
+
* engine reasons about. Identical defaulting in both packages today.
|
|
2520
|
+
*/
|
|
2521
|
+
function resolveChatRecoveryConfig(raw) {
|
|
2522
|
+
const custom = typeof raw === "object" && raw !== null ? raw : void 0;
|
|
2523
|
+
return {
|
|
2524
|
+
enabled: raw !== false,
|
|
2525
|
+
maxAttempts: Math.max(1, Math.floor(custom?.maxAttempts ?? 10)),
|
|
2526
|
+
stableTimeoutMs: Math.max(0, Math.floor(custom?.stableTimeoutMs ?? 1e4)),
|
|
2527
|
+
terminalMessage: custom?.terminalMessage ?? "The assistant was interrupted and could not recover. Please try again.",
|
|
2528
|
+
noProgressTimeoutMs: Math.max(0, Math.floor(custom?.noProgressTimeoutMs ?? 3e5)),
|
|
2529
|
+
maxRecoveryWork: typeof custom?.maxRecoveryWork === "number" && custom.maxRecoveryWork >= 0 ? custom.maxRecoveryWork : DEFAULT_CHAT_RECOVERY_MAX_WORK,
|
|
2530
|
+
maxOomRetries: typeof custom?.maxOomRetries === "number" && custom.maxOomRetries >= 0 ? Math.floor(custom.maxOomRetries) : 3,
|
|
2531
|
+
...custom?.shouldKeepRecovering ? { shouldKeepRecovering: custom.shouldKeepRecovering } : {},
|
|
2532
|
+
...custom?.onExhausted ? { onExhausted: custom.onExhausted } : {}
|
|
2533
|
+
};
|
|
2534
|
+
}
|
|
2535
|
+
/**
|
|
2536
|
+
* Stable identifier for a recovery incident.
|
|
2537
|
+
*
|
|
2538
|
+
* `recoveryKind` is intentionally NOT part of the identity: a single
|
|
2539
|
+
* interrupted turn can flip between "retry" (no chunks persisted) and
|
|
2540
|
+
* "continue" (partial chunks exist) across restarts, and the attempt budget
|
|
2541
|
+
* must be shared so recovery stays bounded by `maxAttempts`. This formula is a
|
|
2542
|
+
* cutover invariant.
|
|
2543
|
+
*/
|
|
2544
|
+
function chatRecoveryIncidentId(input) {
|
|
2545
|
+
return [input.recoveryRootRequestId ?? input.requestId, input.latestUserMessageId ?? ""].join(":");
|
|
2546
|
+
}
|
|
2547
|
+
/** Durable storage key for an incident record. */
|
|
2548
|
+
function chatRecoveryIncidentKey(incidentId) {
|
|
2549
|
+
return `${CHAT_RECOVERY_INCIDENT_KEY_PREFIX}${encodeURIComponent(incidentId)}`;
|
|
2550
|
+
}
|
|
2551
|
+
/**
|
|
2552
|
+
* Select incident keys that have been inactive past the TTL. Pure over a map of
|
|
2553
|
+
* stored incidents; the caller performs the batched delete.
|
|
2554
|
+
*/
|
|
2555
|
+
function selectStaleIncidentKeys(entries, now) {
|
|
2556
|
+
const staleKeys = [];
|
|
2557
|
+
for (const [key, incident] of entries) if (now - (incident?.lastAttemptAt ?? incident?.firstSeenAt ?? 0) > 36e5) staleKeys.push(key);
|
|
2558
|
+
return staleKeys;
|
|
2559
|
+
}
|
|
2560
|
+
/**
|
|
2561
|
+
* Sweep recovery incidents inactive past the TTL from durable storage. Lists by
|
|
2562
|
+
* the incident key prefix, selects stale keys (`selectStaleIncidentKeys`), and
|
|
2563
|
+
* batch-deletes them — the DO KV `delete([...])` accepts up to
|
|
2564
|
+
* `KV_DELETE_MAX_KEYS` per call, collapsing N awaited round-trips into
|
|
2565
|
+
* ceil(N / 128). Shared by `AIChatAgent` and `Think` so the sweep policy lives in
|
|
2566
|
+
* one place. See `design/rfc-chat-recovery-foundation.md`.
|
|
2567
|
+
*/
|
|
2568
|
+
async function sweepStaleChatRecoveryIncidents(storage, now) {
|
|
2569
|
+
const staleKeys = selectStaleIncidentKeys(await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX }), now);
|
|
2570
|
+
for (let i = 0; i < staleKeys.length; i += 128) await storage.delete(staleKeys.slice(i, i + 128));
|
|
2571
|
+
}
|
|
2572
|
+
/**
|
|
2573
|
+
* List the persisted recovery incidents that are still live (status
|
|
2574
|
+
* `detected` / `scheduled` / `attempting`) — i.e. NOT yet terminalized
|
|
2575
|
+
* (`exhausted` / `failed`). Used by the alarm-boundary OOM circuit breaker
|
|
2576
|
+
* (#1825) to find the incident(s) it must seal when the in-DO budgets could not.
|
|
2577
|
+
* Lists by the incident key prefix so the storage layout stays encapsulated.
|
|
2578
|
+
*/
|
|
2579
|
+
async function listActiveChatRecoveryIncidents(storage) {
|
|
2580
|
+
const entries = await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX });
|
|
2581
|
+
const active = [];
|
|
2582
|
+
for (const [key, incident] of entries) if (incident && (incident.status === "detected" || incident.status === "scheduled" || incident.status === "attempting")) active.push({
|
|
2583
|
+
key,
|
|
2584
|
+
incident
|
|
2585
|
+
});
|
|
2586
|
+
return active;
|
|
2587
|
+
}
|
|
2588
|
+
/**
|
|
2589
|
+
* Summarize a child agent's persisted recovery incidents for the parent's
|
|
2590
|
+
* agent-tool reattach decision: `"in-progress"` if any incident is still live
|
|
2591
|
+
* (detected/scheduled/attempting), else `"failed"` if any terminalized
|
|
2592
|
+
* (exhausted/failed), else `"none"`. In-progress takes precedence so a parent
|
|
2593
|
+
* never gives up on a child that is still recovering. Shared by `AIChatAgent`
|
|
2594
|
+
* and `Think`. See `design/rfc-chat-recovery-foundation.md`.
|
|
2595
|
+
*/
|
|
2596
|
+
async function classifyAgentToolChildRecovery(storage) {
|
|
2597
|
+
const entries = await storage.list({ prefix: CHAT_RECOVERY_INCIDENT_KEY_PREFIX });
|
|
2598
|
+
let failed = false;
|
|
2599
|
+
for (const incident of entries.values()) {
|
|
2600
|
+
if (incident.status === "detected" || incident.status === "scheduled" || incident.status === "attempting") return "in-progress";
|
|
2601
|
+
if (incident.status === "exhausted" || incident.status === "failed") failed = true;
|
|
2602
|
+
}
|
|
2603
|
+
return failed ? "failed" : "none";
|
|
2604
|
+
}
|
|
2605
|
+
/**
|
|
2606
|
+
* Read the durable monotonic recovery-progress counter (0 when unset). The value
|
|
2607
|
+
* feeds the no-progress budget decision; shared by `AIChatAgent` and `Think`.
|
|
2608
|
+
*/
|
|
2609
|
+
async function readChatRecoveryProgress(storage) {
|
|
2610
|
+
return await storage.get("cf:chat-recovery:progress") ?? 0;
|
|
2611
|
+
}
|
|
2612
|
+
/**
|
|
2613
|
+
* Advance the durable recovery-progress counter by one. Called when genuinely new
|
|
2614
|
+
* content is durably flushed (real, reconnect-immune forward progress); shared by
|
|
2615
|
+
* `AIChatAgent` and `Think`.
|
|
2616
|
+
*/
|
|
2617
|
+
async function bumpChatRecoveryProgress(storage) {
|
|
2618
|
+
const current = await storage.get("cf:chat-recovery:progress") ?? 0;
|
|
2619
|
+
await storage.put(CHAT_RECOVERY_PROGRESS_KEY, current + 1);
|
|
2620
|
+
}
|
|
2621
|
+
/**
|
|
2622
|
+
* Throttle window for crediting a parent turn's recovery progress from forwarded
|
|
2623
|
+
* sub-agent (agent-tool) stream chunks (N9). Forwarding a child's chunks IS
|
|
2624
|
+
* forward progress for the parent, but the credit must not write storage per
|
|
2625
|
+
* token.
|
|
2626
|
+
*/
|
|
2627
|
+
const AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS = 5e3;
|
|
2628
|
+
/**
|
|
2629
|
+
* Per-isolate throttle gate for agent-tool stream-progress crediting (N9). The
|
|
2630
|
+
* `_lastBumpAt` clock is in-memory, so it resets per isolate and the first
|
|
2631
|
+
* forwarded chunk after a restart always credits. `shouldCredit(now)` returns
|
|
2632
|
+
* `true` at most once per `AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS` window and
|
|
2633
|
+
* records the time on each credit. Shared by `AIChatAgent` and `Think`.
|
|
2634
|
+
*/
|
|
2635
|
+
var AgentToolStreamProgressThrottle = class {
|
|
2636
|
+
constructor() {
|
|
2637
|
+
this._lastBumpAt = 0;
|
|
2638
|
+
}
|
|
2639
|
+
shouldCredit(now) {
|
|
2640
|
+
if (now - this._lastBumpAt < 5e3) return false;
|
|
2641
|
+
this._lastBumpAt = now;
|
|
2642
|
+
return true;
|
|
2643
|
+
}
|
|
2644
|
+
};
|
|
2645
|
+
/**
|
|
2646
|
+
* Throttle window for crediting recovery progress from mid-segment streaming
|
|
2647
|
+
* content (text/reasoning/tool-input deltas). A milestone chunk credits
|
|
2648
|
+
* unconditionally; deltas credit at most once per window so a long single
|
|
2649
|
+
* segment registers forward progress across crashes without writing storage per
|
|
2650
|
+
* token. 5s is far finer than the 300s no-progress budget, so any crash gap
|
|
2651
|
+
* longer than this window over an actively-streaming segment still credits.
|
|
2652
|
+
*/
|
|
2653
|
+
const CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS = 5e3;
|
|
2654
|
+
/**
|
|
2655
|
+
* Per-isolate throttle gate for crediting recovery progress from mid-segment
|
|
2656
|
+
* streaming-content chunks — the delta arm of {@link shouldCreditStreamProgress}.
|
|
2657
|
+
* The `_lastBumpAt` clock is in-memory, so it resets per isolate and the first
|
|
2658
|
+
* delta after a restart always credits. Shared by `AIChatAgent` and `Think`.
|
|
2659
|
+
*/
|
|
2660
|
+
var StreamProgressCreditThrottle = class {
|
|
2661
|
+
constructor() {
|
|
2662
|
+
this._lastBumpAt = 0;
|
|
2663
|
+
}
|
|
2664
|
+
shouldCredit(now) {
|
|
2665
|
+
if (now - this._lastBumpAt < 5e3) return false;
|
|
2666
|
+
this._lastBumpAt = now;
|
|
2667
|
+
return true;
|
|
2668
|
+
}
|
|
2669
|
+
};
|
|
2670
|
+
/**
|
|
2671
|
+
* Persist a durable record of the last terminal turn so a client that
|
|
2672
|
+
* (re)connects after the turn ended still learns its outcome (#1645). Kept
|
|
2673
|
+
* until a later turn supersedes it ({@link clearChatTerminal}); a single record
|
|
2674
|
+
* is sufficient because only the most recent terminal is relevant.
|
|
2675
|
+
*/
|
|
2676
|
+
async function recordChatTerminal(storage, requestId, body) {
|
|
2677
|
+
await storage.put(CHAT_LAST_TERMINAL_KEY, {
|
|
2678
|
+
requestId,
|
|
2679
|
+
body
|
|
2680
|
+
});
|
|
2681
|
+
}
|
|
2682
|
+
/** Clear the durable terminal record once a later turn supersedes it (#1645). */
|
|
2683
|
+
async function clearChatTerminal(storage) {
|
|
2684
|
+
await storage.delete(CHAT_LAST_TERMINAL_KEY);
|
|
2685
|
+
}
|
|
2686
|
+
/** Read the pending terminal record, or `null` if none is stored (#1645). */
|
|
2687
|
+
async function pendingChatTerminal(storage) {
|
|
2688
|
+
return await storage.get("cf:chat:last-terminal") ?? null;
|
|
2689
|
+
}
|
|
2690
|
+
/**
|
|
2691
|
+
* Build the on-connect "recovering…" replay frame (#1620), or `null` when no
|
|
2692
|
+
* (non-stale) recovery is in progress. A client that connects between recovery
|
|
2693
|
+
* attempts (no active stream) reads the turn as working rather than frozen. A
|
|
2694
|
+
* record older than the flag TTL is treated as abandoned (its terminal-clear
|
|
2695
|
+
* never ran) and skipped, so a dead recovery can't show "recovering…" forever.
|
|
2696
|
+
* `messageType` is the package's recovering wire-type enum.
|
|
2697
|
+
*/
|
|
2698
|
+
async function buildChatRecoveringFrame(storage, messageType, now) {
|
|
2699
|
+
const recovering = await storage.get(CHAT_RECOVERING_KEY);
|
|
2700
|
+
if (!recovering || now - (recovering.at ?? 0) >= 9e5) return null;
|
|
2701
|
+
return {
|
|
2702
|
+
type: messageType,
|
|
2703
|
+
recovering: true,
|
|
2704
|
+
...recovering.requestId ? { id: recovering.requestId } : {}
|
|
2705
|
+
};
|
|
2706
|
+
}
|
|
2707
|
+
/**
|
|
2708
|
+
* Set or clear the live "recovering…" status (#1620). Persists a durable record
|
|
2709
|
+
* (so set/clear stay consistent across the isolates a recovery spans) and
|
|
2710
|
+
* broadcasts a recovering frame — but only on a genuine transition, so a
|
|
2711
|
+
* deploy/reconnect storm (which re-detects recovery many times) doesn't spam
|
|
2712
|
+
* the wire. A flag older than the TTL is stale: the owning incident was
|
|
2713
|
+
* abandoned without a terminal (e.g. the DO went idle before recovery could
|
|
2714
|
+
* resolve), so it is treated as not-recovering and can neither pin the
|
|
2715
|
+
* indicator on forever nor suppress a genuinely-new recovering signal.
|
|
2716
|
+
* `messageType` is the package's recovering wire-type enum; `broadcast` is the
|
|
2717
|
+
* package's chat-broadcast wrapper.
|
|
2718
|
+
*/
|
|
2719
|
+
async function setChatRecovering(active, requestId, deps) {
|
|
2720
|
+
const { storage, messageType, broadcast, now } = deps;
|
|
2721
|
+
const existing = await storage.get(CHAT_RECOVERING_KEY);
|
|
2722
|
+
const activeExisting = existing && now - (existing.at ?? 0) < 9e5;
|
|
2723
|
+
if (active) {
|
|
2724
|
+
if (activeExisting) return;
|
|
2725
|
+
await storage.put(CHAT_RECOVERING_KEY, {
|
|
2726
|
+
...requestId ? { requestId } : {},
|
|
2727
|
+
at: now
|
|
2728
|
+
});
|
|
2729
|
+
} else {
|
|
2730
|
+
if (!existing) return;
|
|
2731
|
+
await storage.delete(CHAT_RECOVERING_KEY);
|
|
2732
|
+
requestId = requestId ?? existing.requestId;
|
|
2733
|
+
}
|
|
2734
|
+
broadcast({
|
|
2735
|
+
type: messageType,
|
|
2736
|
+
recovering: active,
|
|
2737
|
+
...requestId ? { id: requestId } : {}
|
|
2738
|
+
});
|
|
2739
|
+
}
|
|
2740
|
+
/**
|
|
2741
|
+
* Compute the next recovery incident and budget decision.
|
|
2742
|
+
*
|
|
2743
|
+
* This is the durable recovery budget — a faithful extraction of
|
|
2744
|
+
* `_beginChatRecoveryIncident` from both `AIChatAgent` and `Think`. The
|
|
2745
|
+
* instruments are decoupled by what they catch:
|
|
2746
|
+
*
|
|
2747
|
+
* - STUCK — no-progress window: `lastProgressAt` resets on every
|
|
2748
|
+
* progress-bearing attempt, so a turn that keeps producing content survives
|
|
2749
|
+
* churn indefinitely; a stuck turn is sealed after `noProgressTimeoutMs`.
|
|
2750
|
+
* - DEBOUNCE — alarms bunched within `CHAT_RECOVERY_ALARM_DEBOUNCE_MS` collapse
|
|
2751
|
+
* into one attempt, so a single rollout's reconnect storm isn't N attempts.
|
|
2752
|
+
* - ALARM-LOOP — the attempt cap (resets on progress) catches a tight
|
|
2753
|
+
* no-progress alarm loop.
|
|
2754
|
+
* - RUNAWAY — the work budget seals a loop that keeps emitting content but
|
|
2755
|
+
* never converges. Keyed to WORK done, not wall-clock. Defaults to no cap.
|
|
2756
|
+
* - CALLER — `shouldKeepRecovering` lets the integrator express a
|
|
2757
|
+
* token/cost/step budget the SDK should not hardcode. Consulted only when no
|
|
2758
|
+
* hard bound has already sealed the incident, and never on first detection.
|
|
2759
|
+
*
|
|
2760
|
+
* A turn parked on a pending client interaction is budget-free: every bound is
|
|
2761
|
+
* suppressed and the no-progress clock kept fresh.
|
|
2762
|
+
*/
|
|
2763
|
+
async function evaluateChatRecoveryIncident(input) {
|
|
2764
|
+
const { identity, config, existing, currentProgress, awaitingClientInteraction, now } = input;
|
|
2765
|
+
const incidentId = chatRecoveryIncidentId(identity);
|
|
2766
|
+
const recoveryRootRequestId = identity.recoveryRootRequestId ?? identity.requestId;
|
|
2767
|
+
const prevProgress = existing?.progress ?? 0;
|
|
2768
|
+
const madeProgress = existing != null && currentProgress > prevProgress;
|
|
2769
|
+
const lastProgressAt = madeProgress || awaitingClientInteraction ? now : existing?.lastProgressAt ?? existing?.firstSeenAt ?? now;
|
|
2770
|
+
const noProgressExceeded = existing != null && !awaitingClientInteraction && now - lastProgressAt > config.noProgressTimeoutMs;
|
|
2771
|
+
const workBaseline = existing?.workBaseline ?? currentProgress;
|
|
2772
|
+
const progress = Math.max(prevProgress, currentProgress);
|
|
2773
|
+
const work = progress - workBaseline;
|
|
2774
|
+
const workBudgetExceeded = existing != null && Number.isFinite(config.maxRecoveryWork) && work > config.maxRecoveryWork;
|
|
2775
|
+
const oomAttempts = existing?.oomAttempts ?? 0;
|
|
2776
|
+
const oomBudgetExceeded = existing != null && !awaitingClientInteraction && oomAttempts > config.maxOomRetries;
|
|
2777
|
+
const debounced = existing != null && !madeProgress && now - existing.lastAttemptAt < 3e4;
|
|
2778
|
+
const attempt = madeProgress ? 1 : debounced ? existing?.attempt ?? 1 : (existing?.attempt ?? 0) + 1;
|
|
2779
|
+
let abortedByCaller = false;
|
|
2780
|
+
if (existing != null && !awaitingClientInteraction && config.shouldKeepRecovering && !noProgressExceeded && !workBudgetExceeded && !oomBudgetExceeded && attempt <= config.maxAttempts) try {
|
|
2781
|
+
const ctx = {
|
|
2782
|
+
incidentId,
|
|
2783
|
+
requestId: identity.requestId,
|
|
2784
|
+
recoveryRootRequestId,
|
|
2785
|
+
attempt,
|
|
2786
|
+
maxAttempts: config.maxAttempts,
|
|
2787
|
+
recoveryKind: identity.recoveryKind,
|
|
2788
|
+
work,
|
|
2789
|
+
ageMs: now - (existing.firstSeenAt ?? now)
|
|
2790
|
+
};
|
|
2791
|
+
abortedByCaller = await config.shouldKeepRecovering(ctx) === false;
|
|
2792
|
+
} catch (error) {
|
|
2793
|
+
input.onShouldKeepRecoveringError?.(error);
|
|
2794
|
+
}
|
|
2795
|
+
const exhausted = !awaitingClientInteraction && (oomBudgetExceeded || noProgressExceeded || workBudgetExceeded || abortedByCaller || attempt > config.maxAttempts);
|
|
2796
|
+
const incident = {
|
|
2797
|
+
incidentId,
|
|
2798
|
+
requestId: identity.requestId,
|
|
2799
|
+
recoveryRootRequestId,
|
|
2800
|
+
recoveryKind: identity.recoveryKind,
|
|
2801
|
+
attempt,
|
|
2802
|
+
maxAttempts: config.maxAttempts,
|
|
2803
|
+
status: exhausted ? "exhausted" : "attempting",
|
|
2804
|
+
firstSeenAt: existing?.firstSeenAt ?? now,
|
|
2805
|
+
lastAttemptAt: now,
|
|
2806
|
+
lastProgressAt,
|
|
2807
|
+
progress,
|
|
2808
|
+
workBaseline,
|
|
2809
|
+
...oomAttempts > 0 ? { oomAttempts } : {},
|
|
2810
|
+
...exhausted ? { reason: oomBudgetExceeded ? "out_of_memory" : workBudgetExceeded ? "work_budget_exceeded" : noProgressExceeded ? "no_progress_timeout" : abortedByCaller ? "recovery_aborted" : "max_attempts_exceeded" } : {}
|
|
2811
|
+
};
|
|
2812
|
+
const events = [];
|
|
2813
|
+
if (!existing) events.push({
|
|
2814
|
+
type: "chat:recovery:detected",
|
|
2815
|
+
incidentId,
|
|
2816
|
+
requestId: identity.requestId,
|
|
2817
|
+
attempt,
|
|
2818
|
+
maxAttempts: config.maxAttempts,
|
|
2819
|
+
recoveryKind: identity.recoveryKind
|
|
2820
|
+
});
|
|
2821
|
+
events.push({
|
|
2822
|
+
type: "chat:recovery:attempt",
|
|
2823
|
+
incidentId,
|
|
2824
|
+
requestId: identity.requestId,
|
|
2825
|
+
attempt,
|
|
2826
|
+
maxAttempts: config.maxAttempts,
|
|
2827
|
+
recoveryKind: identity.recoveryKind
|
|
2828
|
+
});
|
|
2829
|
+
return {
|
|
2830
|
+
incident,
|
|
2831
|
+
exhausted,
|
|
2832
|
+
events
|
|
2833
|
+
};
|
|
2834
|
+
}
|
|
2835
|
+
//#endregion
|
|
2836
|
+
//#region src/chat/recovery-engine.ts
|
|
2837
|
+
/**
|
|
2838
|
+
* Resolve the `schedule()` idempotency option for a recovery schedule. Single
|
|
2839
|
+
* source of truth for both packages; see {@link ChatRecoveryScheduleReason} for
|
|
2840
|
+
* the rationale behind each case.
|
|
2841
|
+
*
|
|
2842
|
+
* This is a cutover invariant: flipping either case silently breaks deploy-storm
|
|
2843
|
+
* dedup (initial) or stalls stable-timeout retries (reschedule), and neither is
|
|
2844
|
+
* caught by a type error — only by the recovery suites.
|
|
2845
|
+
*/
|
|
2846
|
+
function chatRecoverySchedulePolicy(reason) {
|
|
2847
|
+
return { idempotent: reason === "initial" };
|
|
2848
|
+
}
|
|
2849
|
+
/**
|
|
2850
|
+
* Drives the shared recovery orchestration over a {@link ChatRecoveryAdapter}.
|
|
2851
|
+
* The incident *budget math* lives in the pure `evaluateChatRecoveryIncident`;
|
|
2852
|
+
* this class owns the surrounding sequence and its ordering invariants.
|
|
2853
|
+
*/
|
|
2854
|
+
var ChatRecoveryEngine = class {
|
|
2855
|
+
constructor(adapter) {
|
|
2856
|
+
this.adapter = adapter;
|
|
2857
|
+
}
|
|
2858
|
+
/**
|
|
2859
|
+
* Open or re-evaluate the recovery incident for `input`, persist the result,
|
|
2860
|
+
* and broadcast its lifecycle events. Returns the incident, the resolved
|
|
2861
|
+
* config, and whether the budget is now exhausted.
|
|
2862
|
+
*/
|
|
2863
|
+
/**
|
|
2864
|
+
* Dispatch a recovered fiber to the package's non-chat handler (the
|
|
2865
|
+
* messenger/workflow seam) before any chat-recovery processing. Returns `true`
|
|
2866
|
+
* when the package consumed the fiber — the caller must then skip chat
|
|
2867
|
+
* recovery for it. The engine owns the *ordering* (this runs before the
|
|
2868
|
+
* chat-fiber gate); the *behavior* is adapter-owned. No-op (`false`) when the
|
|
2869
|
+
* adapter omits {@link ChatRecoveryAdapter.tryHandleNonChatFiberRecovery}.
|
|
2870
|
+
*/
|
|
2871
|
+
async handleNonChatFiber(ctx) {
|
|
2872
|
+
return await this.adapter.tryHandleNonChatFiberRecovery?.(ctx) ?? false;
|
|
2873
|
+
}
|
|
2874
|
+
/**
|
|
2875
|
+
* The shared wake-recovery LIFECYCLE for an interrupted chat fiber. Both
|
|
2876
|
+
* packages drove this exact frame; the divergent organs are the
|
|
2877
|
+
* {@link ChatFiberWakeHooks}. In order:
|
|
2878
|
+
*
|
|
2879
|
+
* 1. non-chat dispatch ({@link handleNonChatFiber}) FIRST, then the chat-fiber
|
|
2880
|
+
* name gate — a non-chat fiber is never misread as an orphaned chat turn;
|
|
2881
|
+
* 2. parse the request id, unwrap the snapshot, resolve the orphaned stream +
|
|
2882
|
+
* reconstruct its partial;
|
|
2883
|
+
* 3. classify the turn (retry/continue + package detail) and open the incident;
|
|
2884
|
+
* 4. if the budget is already exhausted, persist the settled partial (so
|
|
2885
|
+
* non-idempotent tool results are not discarded — #1631) and terminalize
|
|
2886
|
+
* BEFORE consulting `onChatRecovery`;
|
|
2887
|
+
* 5. otherwise, inside a `failed`-on-throw guard: invoke `onChatRecovery`,
|
|
2888
|
+
* apply the shared persist gate (base eligibility AND `persist !== false ||
|
|
2889
|
+
* settled tool results`), complete the live stream, then hand the
|
|
2890
|
+
* retry/continue/skip DECISION to {@link ChatFiberWakeHooks.dispatchRecoveredTurn}.
|
|
2891
|
+
*
|
|
2892
|
+
* Returns `true` when the fiber was a chat (or non-chat) recovery the engine
|
|
2893
|
+
* handled, `false` when it was not a chat fiber (the caller keeps looking). Any
|
|
2894
|
+
* throw after the incident opens flips it to `failed` so it is never left
|
|
2895
|
+
* leaking in `attempting`.
|
|
2896
|
+
*/
|
|
2897
|
+
async handleChatFiberRecovery(ctx, wake) {
|
|
2898
|
+
const { adapter } = this;
|
|
2899
|
+
if (await this.handleNonChatFiber(ctx)) return true;
|
|
2900
|
+
const chatPrefix = wake.chatFiberPrefix();
|
|
2901
|
+
if (!ctx.name.startsWith(chatPrefix)) return false;
|
|
2902
|
+
const requestId = ctx.name.slice(chatPrefix.length);
|
|
2903
|
+
const { snapshot, recoveryData } = wake.unwrapRecoverySnapshot(ctx);
|
|
2904
|
+
const { streamId, streamStillActive, streamStatus } = adapter.resolveRecoveryStream(requestId);
|
|
2905
|
+
const partial = streamId ? adapter.getPartialStreamText(streamId) : {
|
|
2906
|
+
text: "",
|
|
2907
|
+
parts: [],
|
|
2908
|
+
hasSettledToolResults: false
|
|
2909
|
+
};
|
|
2910
|
+
const { recoveryKind, detail } = await wake.classifyRecoveredTurn({
|
|
2911
|
+
snapshot,
|
|
2912
|
+
requestId,
|
|
2913
|
+
streamId,
|
|
2914
|
+
partial,
|
|
2915
|
+
streamStillActive,
|
|
2916
|
+
streamStatus
|
|
2917
|
+
});
|
|
2918
|
+
const recoveryRootRequestId = snapshot?.recoveryRootRequestId ?? requestId;
|
|
2919
|
+
const { incident, config, exhausted } = await this.beginIncident({
|
|
2920
|
+
requestId,
|
|
2921
|
+
recoveryRootRequestId,
|
|
2922
|
+
latestUserMessageId: snapshot?.latestUserMessageId,
|
|
2923
|
+
recoveryKind
|
|
2924
|
+
});
|
|
2925
|
+
if (exhausted) {
|
|
2926
|
+
if (await this._shouldPersistOrphanedPartial(wake, {
|
|
2927
|
+
streamId,
|
|
2928
|
+
streamStillActive,
|
|
2929
|
+
streamStatus,
|
|
2930
|
+
snapshot,
|
|
2931
|
+
options: void 0,
|
|
2932
|
+
partial
|
|
2933
|
+
})) await wake.persistOrphanedStream(streamId);
|
|
2934
|
+
await adapter.exhaustChatRecovery(incident, config, partial, streamId, ctx.createdAt);
|
|
2935
|
+
return true;
|
|
2936
|
+
}
|
|
2937
|
+
try {
|
|
2938
|
+
const options = await wake.invokeOnChatRecovery?.({
|
|
2939
|
+
incident,
|
|
2940
|
+
recoveryKind,
|
|
2941
|
+
recoveryRootRequestId,
|
|
2942
|
+
requestId,
|
|
2943
|
+
streamId,
|
|
2944
|
+
partial,
|
|
2945
|
+
snapshot,
|
|
2946
|
+
recoveryData,
|
|
2947
|
+
createdAt: ctx.createdAt
|
|
2948
|
+
}) ?? {};
|
|
2949
|
+
if (await this._shouldPersistOrphanedPartial(wake, {
|
|
2950
|
+
streamId,
|
|
2951
|
+
streamStillActive,
|
|
2952
|
+
streamStatus,
|
|
2953
|
+
snapshot,
|
|
2954
|
+
options,
|
|
2955
|
+
partial
|
|
2956
|
+
})) await wake.persistOrphanedStream(streamId);
|
|
2957
|
+
if (streamStillActive) await wake.completeRecoveredStream(streamId);
|
|
2958
|
+
await wake.dispatchRecoveredTurn({
|
|
2959
|
+
incident,
|
|
2960
|
+
config,
|
|
2961
|
+
recoveryKind,
|
|
2962
|
+
options,
|
|
2963
|
+
snapshot,
|
|
2964
|
+
requestId,
|
|
2965
|
+
recoveryRootRequestId,
|
|
2966
|
+
streamId,
|
|
2967
|
+
streamStatus,
|
|
2968
|
+
detail
|
|
2969
|
+
});
|
|
2970
|
+
return true;
|
|
2971
|
+
} catch (error) {
|
|
2972
|
+
await this.updateIncident(incident.incidentId, "failed", error instanceof Error ? error.message : String(error));
|
|
2973
|
+
throw error;
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
/**
|
|
2977
|
+
* The shared persist gate: base eligibility (the package's
|
|
2978
|
+
* {@link ChatFiberWakeHooks.shouldPersistOrphanedPartial}) AND the
|
|
2979
|
+
* never-drop-settled-work clause `options.persist !== false ||
|
|
2980
|
+
* partial.hasSettledToolResults`. `options: undefined` (the exhausted branch)
|
|
2981
|
+
* collapses the clause to the base gate. The clause lives here — not in each
|
|
2982
|
+
* package — because settled-work preservation is a cross-package invariant
|
|
2983
|
+
* (#1631), and the codec (not the engine) decides whether a partial carries
|
|
2984
|
+
* settled tool work, so the engine stays wire-vocabulary-agnostic.
|
|
2985
|
+
*/
|
|
2986
|
+
async _shouldPersistOrphanedPartial(wake, input) {
|
|
2987
|
+
return await wake.shouldPersistOrphanedPartial({
|
|
2988
|
+
streamId: input.streamId,
|
|
2989
|
+
streamStillActive: input.streamStillActive,
|
|
2990
|
+
streamStatus: input.streamStatus,
|
|
2991
|
+
snapshot: input.snapshot
|
|
2992
|
+
}) && (input.options?.persist !== false || input.partial.hasSettledToolResults);
|
|
2993
|
+
}
|
|
2994
|
+
async beginIncident(input) {
|
|
2995
|
+
const { adapter } = this;
|
|
2996
|
+
const config = adapter.resolveConfig();
|
|
2997
|
+
const key = chatRecoveryIncidentKey(chatRecoveryIncidentId(input));
|
|
2998
|
+
const now = input.nowMs ?? adapter.now();
|
|
2999
|
+
await adapter.sweepStaleIncidents(now);
|
|
3000
|
+
const existing = await adapter.getIncident(key);
|
|
3001
|
+
adapter.ensureInteractionStateLoaded?.();
|
|
3002
|
+
const { incident, exhausted, events } = await evaluateChatRecoveryIncident({
|
|
3003
|
+
identity: input,
|
|
3004
|
+
config,
|
|
3005
|
+
existing,
|
|
3006
|
+
currentProgress: await adapter.readProgress(),
|
|
3007
|
+
awaitingClientInteraction: adapter.isAwaitingClientInteraction?.() ?? false,
|
|
3008
|
+
now,
|
|
3009
|
+
onShouldKeepRecoveringError: (error) => adapter.onShouldKeepRecoveringError?.(error)
|
|
3010
|
+
});
|
|
3011
|
+
await adapter.putIncident(key, incident);
|
|
3012
|
+
for (const event of events) adapter.emitRecoveryEvent(event);
|
|
3013
|
+
return {
|
|
3014
|
+
incident,
|
|
3015
|
+
config,
|
|
3016
|
+
exhausted
|
|
3017
|
+
};
|
|
3018
|
+
}
|
|
3019
|
+
/**
|
|
3020
|
+
* Schedule a recovery continuation/retry: the transition + emit + enqueue
|
|
3021
|
+
* triplet both packages repeat at every fiber-recovery and stall-routing
|
|
3022
|
+
* decision. In order:
|
|
3023
|
+
*
|
|
3024
|
+
* 1. transition the incident to `scheduled` (persist + drive the #1620
|
|
3025
|
+
* "recovering…" status) via {@link updateIncident};
|
|
3026
|
+
* 2. emit `chat:recovery:scheduled`; and
|
|
3027
|
+
* 3. enqueue the callback through the adapter's idempotent schedule.
|
|
3028
|
+
*
|
|
3029
|
+
* `recoveryKind` is passed explicitly (not read off the incident) because a
|
|
3030
|
+
* caller can legitimately report a different kind than the incident was opened
|
|
3031
|
+
* with — e.g. `AIChatAgent`'s lost-partial branch opens a `continue` incident
|
|
3032
|
+
* but schedules (and reports) a `retry`. `requestId` always matches
|
|
3033
|
+
* `incident.requestId` (the evaluation rewrites it to the current attempt), so
|
|
3034
|
+
* it is read from the incident.
|
|
3035
|
+
*/
|
|
3036
|
+
async scheduleRecovery(input) {
|
|
3037
|
+
const { incident } = input;
|
|
3038
|
+
await this.updateIncident(incident.incidentId, "scheduled");
|
|
3039
|
+
this.adapter.emitRecoveryEvent({
|
|
3040
|
+
type: "chat:recovery:scheduled",
|
|
3041
|
+
incidentId: incident.incidentId,
|
|
3042
|
+
requestId: incident.requestId,
|
|
3043
|
+
attempt: incident.attempt,
|
|
3044
|
+
maxAttempts: incident.maxAttempts,
|
|
3045
|
+
recoveryKind: input.recoveryKind
|
|
3046
|
+
});
|
|
3047
|
+
await this.adapter.scheduleRecovery(input.callback, input.data, input.reason ?? "initial", 0);
|
|
3048
|
+
}
|
|
3049
|
+
/**
|
|
3050
|
+
* Reschedule a recovery continuation/retry that timed out waiting for stable
|
|
3051
|
+
* state, INSIDE the currently-executing one-shot schedule row. Reads the
|
|
3052
|
+
* incident; if it is still under the attempt cap, bumps `attempt`, marks it
|
|
3053
|
+
* `scheduled` with `reason:"stable_timeout_retry"`, and issues a delayed,
|
|
3054
|
+
* NON-idempotent schedule (`alarm()` deletes the executing row only after this
|
|
3055
|
+
* returns, so an idempotent reschedule would dedup onto that doomed row and
|
|
3056
|
+
* never fire — see {@link chatRecoverySchedulePolicy}).
|
|
3057
|
+
*
|
|
3058
|
+
* Returns `true` when a retry was scheduled, `false` when there is no incident
|
|
3059
|
+
* (no id / record gone) or the attempt budget is already spent — in which case
|
|
3060
|
+
* the caller falls through to the give-up path. Deliberately bypasses the
|
|
3061
|
+
* `evaluateChatRecoveryIncident` budget (this is a coarse stable-state retry,
|
|
3062
|
+
* not a fresh interruption) and {@link updateIncident} (no `scheduled` event /
|
|
3063
|
+
* recovering-flag churn on a same-turn reschedule).
|
|
3064
|
+
*/
|
|
3065
|
+
async rescheduleAfterStableTimeout(input) {
|
|
3066
|
+
const { adapter } = this;
|
|
3067
|
+
if (!input.incidentId) return false;
|
|
3068
|
+
const key = chatRecoveryIncidentKey(input.incidentId);
|
|
3069
|
+
const incident = await adapter.getIncident(key);
|
|
3070
|
+
if (!incident) return false;
|
|
3071
|
+
const attempt = incident.attempt ?? 0;
|
|
3072
|
+
if (attempt >= (incident.maxAttempts ?? input.fallbackMaxAttempts)) return false;
|
|
3073
|
+
await adapter.putIncident(key, {
|
|
3074
|
+
...incident,
|
|
3075
|
+
attempt: attempt + 1,
|
|
3076
|
+
status: "scheduled",
|
|
3077
|
+
lastAttemptAt: adapter.now(),
|
|
3078
|
+
reason: "stable_timeout_retry"
|
|
3079
|
+
});
|
|
3080
|
+
await adapter.scheduleRecovery(input.callback, input.data ?? {}, "stable_timeout_retry", 3);
|
|
3081
|
+
return true;
|
|
3082
|
+
}
|
|
3083
|
+
/**
|
|
3084
|
+
* Record that a recovery callback observed a Durable Object memory-limit reset
|
|
3085
|
+
* (the isolate exceeded its 128 MB limit — `isDurableObjectMemoryLimitReset`)
|
|
3086
|
+
* and decide what to do next (#1825).
|
|
3087
|
+
*
|
|
3088
|
+
* Bumps the incident's durable `oomAttempts` counter, then:
|
|
3089
|
+
* - if it is still within `maxOomRetries`, issues a delayed, NON-idempotent
|
|
3090
|
+
* reschedule of the SAME callback (same machinery as
|
|
3091
|
+
* {@link rescheduleAfterStableTimeout}: the executing one-shot row is
|
|
3092
|
+
* deleted only after the callback returns, so an idempotent reschedule
|
|
3093
|
+
* would dedup onto that doomed row) and returns `"rescheduled"`. The small
|
|
3094
|
+
* delay lets a transient memory spike clear before the re-run;
|
|
3095
|
+
* - otherwise leaves the incremented count persisted (so a begin-path
|
|
3096
|
+
* re-evaluation agrees) and returns `"exhausted"` — the caller then
|
|
3097
|
+
* terminalizes via the give-up path with `reason="out_of_memory"`.
|
|
3098
|
+
*
|
|
3099
|
+
* Returns `"exhausted"` when there is no incident to track against (no id /
|
|
3100
|
+
* record gone): an OOM we cannot bound must seal rather than loop. Unlike a
|
|
3101
|
+
* stable-state retry this is gated by the OOM-specific budget, NOT the generic
|
|
3102
|
+
* attempt cap — re-running an OOM streams a little "progress" that would
|
|
3103
|
+
* otherwise reset the attempt cap forever (the #1825 loop).
|
|
3104
|
+
*/
|
|
3105
|
+
async recordOomAndDecide(input) {
|
|
3106
|
+
const { adapter } = this;
|
|
3107
|
+
if (!input.incidentId) return "exhausted";
|
|
3108
|
+
const key = chatRecoveryIncidentKey(input.incidentId);
|
|
3109
|
+
const incident = await adapter.getIncident(key);
|
|
3110
|
+
if (!incident) return "exhausted";
|
|
3111
|
+
const oomAttempts = (incident.oomAttempts ?? 0) + 1;
|
|
3112
|
+
if (oomAttempts > input.maxOomRetries) {
|
|
3113
|
+
await adapter.putIncident(key, {
|
|
3114
|
+
...incident,
|
|
3115
|
+
oomAttempts,
|
|
3116
|
+
lastAttemptAt: adapter.now(),
|
|
3117
|
+
reason: "out_of_memory"
|
|
3118
|
+
});
|
|
3119
|
+
return "exhausted";
|
|
3120
|
+
}
|
|
3121
|
+
await adapter.putIncident(key, {
|
|
3122
|
+
...incident,
|
|
3123
|
+
oomAttempts,
|
|
3124
|
+
status: "scheduled",
|
|
3125
|
+
lastAttemptAt: adapter.now(),
|
|
3126
|
+
reason: "oom_retry"
|
|
3127
|
+
});
|
|
3128
|
+
await adapter.scheduleRecovery(input.callback, input.data ?? {}, "stable_timeout_retry", 3);
|
|
3129
|
+
return "rescheduled";
|
|
3130
|
+
}
|
|
3131
|
+
/**
|
|
3132
|
+
* Give up on a recovery turn whose retry budget drained, terminalizing it so
|
|
3133
|
+
* it can never become an eternal spinner (#1645). The shared spine both
|
|
3134
|
+
* packages repeated verbatim:
|
|
3135
|
+
*
|
|
3136
|
+
* 1. resolve config + the incident key from `data.incidentId`;
|
|
3137
|
+
* 2. best-effort READ the stored incident — a failed read is tolerated
|
|
3138
|
+
* (reported via `onGiveUpBookkeepingError("read", …)`) and the incident is
|
|
3139
|
+
* synthesized, because the read backs only the re-entry guard, not the
|
|
3140
|
+
* terminal UX;
|
|
3141
|
+
* 3. re-entry guard: a `stored.status === "exhausted"` record means
|
|
3142
|
+
* terminalization already fired, so a duplicate stale alarm returns without
|
|
3143
|
+
* re-broadcasting the banner;
|
|
3144
|
+
* 4. build the exhausted incident (reuse `stored`, or synthesize a minimal one
|
|
3145
|
+
* so a swept/missing record STILL terminalizes through `onExhausted`);
|
|
3146
|
+
* 5. resolve the orphaned stream id + partial;
|
|
3147
|
+
* 6. terminalize via `exhaustChatRecovery` — BEFORE sealing. The terminal
|
|
3148
|
+
* writes can reject with a platform transient in the deploy/storage window
|
|
3149
|
+
* a give-up runs in (#1730); letting that throw propagate is deliberate, so
|
|
3150
|
+
* `Agent._executeScheduleCallback` defers the one-shot row and the WHOLE
|
|
3151
|
+
* give-up re-runs on a healthy isolate. Sealing first would arm the
|
|
3152
|
+
* re-entry guard and turn that re-run into a no-op, dropping the durable
|
|
3153
|
+
* terminal record. The re-run is idempotent (terminal writes overwrite the
|
|
3154
|
+
* same key); a second banner is the documented at-least-once edge; and
|
|
3155
|
+
* 7. best-effort SEAL write so the re-entry guard sees `exhausted` on a
|
|
3156
|
+
* duplicate alarm — a failed seal (reported via
|
|
3157
|
+
* `onGiveUpBookkeepingError("seal", …)`) costs at most one re-delivered
|
|
3158
|
+
* banner.
|
|
3159
|
+
*
|
|
3160
|
+
* The two packages diverged only in parameters the caller supplies:
|
|
3161
|
+
* `reason` (`Think` passes `stable_timeout` | `recovery_error`; `AIChatAgent`
|
|
3162
|
+
* always `stable_timeout`) and the root-id chain (`Think` includes
|
|
3163
|
+
* `recoveredRequestId`; `AIChatAgent` never sets it, so the unified chain
|
|
3164
|
+
* collapses identically). Exactly-once terminalization rests on the re-entry
|
|
3165
|
+
* guard alone in `AIChatAgent`; `Think` additionally short-circuits duplicate
|
|
3166
|
+
* alarms earlier in its durable-submission layer.
|
|
3167
|
+
*/
|
|
3168
|
+
async exhaustRecoveryGiveUp(input) {
|
|
3169
|
+
const { adapter } = this;
|
|
3170
|
+
const config = adapter.resolveConfig();
|
|
3171
|
+
const incidentKey = input.data?.incidentId ? chatRecoveryIncidentKey(input.data.incidentId) : null;
|
|
3172
|
+
let stored = null;
|
|
3173
|
+
if (incidentKey) try {
|
|
3174
|
+
stored = await adapter.getIncident(incidentKey);
|
|
3175
|
+
} catch (readError) {
|
|
3176
|
+
adapter.onGiveUpBookkeepingError("read", readError);
|
|
3177
|
+
}
|
|
3178
|
+
if (stored?.status === "exhausted") return;
|
|
3179
|
+
const rootRequestId = input.data?.originalRequestId ?? input.data?.recoveredRequestId ?? adapter.activeChatRecoveryRootRequestId() ?? stored?.recoveryRootRequestId ?? stored?.requestId ?? "";
|
|
3180
|
+
const incident = stored ? {
|
|
3181
|
+
...stored,
|
|
3182
|
+
status: "exhausted",
|
|
3183
|
+
reason: input.reason
|
|
3184
|
+
} : {
|
|
3185
|
+
incidentId: input.data?.incidentId ?? crypto.randomUUID(),
|
|
3186
|
+
requestId: rootRequestId,
|
|
3187
|
+
recoveryRootRequestId: rootRequestId,
|
|
3188
|
+
recoveryKind: input.callback === "_chatRecoveryRetry" ? "retry" : "continue",
|
|
3189
|
+
attempt: config.maxAttempts,
|
|
3190
|
+
maxAttempts: config.maxAttempts,
|
|
3191
|
+
status: "exhausted",
|
|
3192
|
+
firstSeenAt: adapter.now(),
|
|
3193
|
+
lastAttemptAt: adapter.now(),
|
|
3194
|
+
reason: input.reason
|
|
3195
|
+
};
|
|
3196
|
+
const { streamId } = adapter.resolveRecoveryStream(incident.recoveryRootRequestId ?? incident.requestId);
|
|
3197
|
+
const partial = streamId ? adapter.getPartialStreamText(streamId) : {
|
|
3198
|
+
text: "",
|
|
3199
|
+
parts: [],
|
|
3200
|
+
hasSettledToolResults: false
|
|
3201
|
+
};
|
|
3202
|
+
await adapter.exhaustChatRecovery(incident, config, partial, streamId, incident.firstSeenAt);
|
|
3203
|
+
if (incidentKey) try {
|
|
3204
|
+
await adapter.putIncident(incidentKey, incident);
|
|
3205
|
+
} catch (writeError) {
|
|
3206
|
+
adapter.onGiveUpBookkeepingError("seal", writeError);
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
/**
|
|
3210
|
+
* Apply a status transition to the recovery incident `incidentId`:
|
|
3211
|
+
*
|
|
3212
|
+
* - `completed` → drop the record (terminal, never retried);
|
|
3213
|
+
* - any other status → persist the new status (and `reason`), so the attempt
|
|
3214
|
+
* budget survives restarts until the TTL sweep reclaims it;
|
|
3215
|
+
* - emit the matching `completed`/`skipped`/`failed` lifecycle event; and
|
|
3216
|
+
* - drive the live "recovering…" status (#1620): `scheduled` marks it active
|
|
3217
|
+
* (keyed by the recovery-root request id), terminal states clear it.
|
|
3218
|
+
*
|
|
3219
|
+
* No-op when `incidentId` is undefined or the record is already gone. This is
|
|
3220
|
+
* the transition twin of {@link beginIncident}: all I/O is adapter-owned, the
|
|
3221
|
+
* engine owns only the state-machine shape.
|
|
3222
|
+
*/
|
|
3223
|
+
async updateIncident(incidentId, status, reason) {
|
|
3224
|
+
if (!incidentId) return;
|
|
3225
|
+
const { adapter } = this;
|
|
3226
|
+
const key = chatRecoveryIncidentKey(incidentId);
|
|
3227
|
+
const incident = await adapter.getIncident(key);
|
|
3228
|
+
if (!incident) return;
|
|
3229
|
+
if (status === "completed") await adapter.deleteIncident(key);
|
|
3230
|
+
else await adapter.putIncident(key, {
|
|
3231
|
+
...incident,
|
|
3232
|
+
status,
|
|
3233
|
+
...reason ? { reason } : {}
|
|
3234
|
+
});
|
|
3235
|
+
const eventType = status === "completed" ? "chat:recovery:completed" : status === "skipped" ? "chat:recovery:skipped" : status === "failed" ? "chat:recovery:failed" : void 0;
|
|
3236
|
+
if (eventType) adapter.emitRecoveryEvent({
|
|
3237
|
+
type: eventType,
|
|
3238
|
+
incidentId,
|
|
3239
|
+
requestId: incident.requestId,
|
|
3240
|
+
attempt: incident.attempt,
|
|
3241
|
+
maxAttempts: incident.maxAttempts,
|
|
3242
|
+
recoveryKind: incident.recoveryKind,
|
|
3243
|
+
...reason ? { reason } : {}
|
|
3244
|
+
});
|
|
3245
|
+
if (status === "scheduled") await adapter.setRecovering(true, incident.recoveryRootRequestId ?? incident.requestId);
|
|
3246
|
+
else if (status === "completed" || status === "skipped" || status === "failed") await adapter.setRecovering(false);
|
|
3247
|
+
}
|
|
3248
|
+
};
|
|
3249
|
+
/**
|
|
3250
|
+
* Build the `ChatRecoveryExhaustedContext` delivered to `onExhausted` and the
|
|
3251
|
+
* `chat:recovery:exhausted` event. Pure field-mapping shared by both packages;
|
|
3252
|
+
* the `reason` falls back to `max_attempts_exceeded` when the incident did not
|
|
3253
|
+
* record a more specific cause.
|
|
3254
|
+
*/
|
|
3255
|
+
function buildChatRecoveryExhaustedContext(input) {
|
|
3256
|
+
const { incident, config } = input;
|
|
3257
|
+
return {
|
|
3258
|
+
incidentId: incident.incidentId,
|
|
3259
|
+
requestId: incident.requestId,
|
|
3260
|
+
recoveryRootRequestId: incident.recoveryRootRequestId ?? incident.requestId,
|
|
3261
|
+
attempt: incident.attempt,
|
|
3262
|
+
maxAttempts: incident.maxAttempts,
|
|
3263
|
+
recoveryKind: incident.recoveryKind,
|
|
3264
|
+
streamId: input.streamId,
|
|
3265
|
+
createdAt: input.createdAt,
|
|
3266
|
+
partialText: input.partialText,
|
|
3267
|
+
partialParts: input.partialParts,
|
|
3268
|
+
reason: incident.reason ?? "max_attempts_exceeded",
|
|
3269
|
+
terminalMessage: config.terminalMessage
|
|
3270
|
+
};
|
|
3271
|
+
}
|
|
3272
|
+
/**
|
|
3273
|
+
* Run the shared exhaustion notification: emit `chat:recovery:exhausted`, then
|
|
3274
|
+
* invoke the caller's `onExhausted` hook. A throwing hook is swallowed (and
|
|
3275
|
+
* reported via `onError`) so it can NEVER prevent the caller from delivering
|
|
3276
|
+
* terminal UX — a tested invariant in both packages. The terminal record /
|
|
3277
|
+
* banner / submission writes that follow are intentionally package-owned (their
|
|
3278
|
+
* ordering legitimately diverges), so they are NOT part of this helper.
|
|
3279
|
+
*/
|
|
3280
|
+
async function notifyChatRecoveryExhausted(ctx, hooks) {
|
|
3281
|
+
hooks.emit(ctx);
|
|
3282
|
+
try {
|
|
3283
|
+
await hooks.onExhausted?.(ctx);
|
|
3284
|
+
} catch (error) {
|
|
3285
|
+
hooks.onError(error);
|
|
3286
|
+
}
|
|
3287
|
+
}
|
|
3288
|
+
/**
|
|
3289
|
+
* The complete give-up choreography from a single call: build the exhausted
|
|
3290
|
+
* context, fire the shared notification ({@link notifyChatRecoveryExhausted}),
|
|
3291
|
+
* then hand that context to the host's `terminalize` step. Folds the
|
|
3292
|
+
* `buildChatRecoveryExhaustedContext` → `notifyChatRecoveryExhausted` → host
|
|
3293
|
+
* terminalize sequence that every host's `_exhaustChatRecovery` repeated.
|
|
3294
|
+
*
|
|
3295
|
+
* What this OWNS (the invariant, so it cannot drift per host):
|
|
3296
|
+
* - the notification ALWAYS runs before any terminal write, and
|
|
3297
|
+
* - a throwing `onExhausted` can NEVER block terminal delivery — it is swallowed
|
|
3298
|
+
* via `onError` (a tested invariant in both published packages).
|
|
3299
|
+
*
|
|
3300
|
+
* What it deliberately does NOT own: the terminal-record / broadcast /
|
|
3301
|
+
* recovering-clear writes — their exact set diverges per host (both
|
|
3302
|
+
* `AIChatAgent` and `Think` broadcast the banner first so it survives a storage
|
|
3303
|
+
* write that rejects mid-deploy; `Think` additionally writes a submission row)
|
|
3304
|
+
* — see {@link ChatRecoveryAdapter.exhaustChatRecovery}. The host expresses
|
|
3305
|
+
* those writes inside `terminalize`. A `terminalize` that throws DOES propagate,
|
|
3306
|
+
* so the whole give-up re-runs on a healthy isolate (#1730); see
|
|
3307
|
+
* {@link ChatRecoveryEngine.exhaustRecoveryGiveUp}.
|
|
3308
|
+
*
|
|
3309
|
+
* `partialParts` is passed explicitly (not derived from a `RecoveryPartial`) so a
|
|
3310
|
+
* foreign-vocabulary host can pass `[]` rather than fabricate AI-SDK parts — the
|
|
3311
|
+
* engine seam stays parts-vocabulary-agnostic.
|
|
3312
|
+
*/
|
|
3313
|
+
async function runChatRecoveryExhaustion(input, hooks) {
|
|
3314
|
+
const ctx = buildChatRecoveryExhaustedContext({
|
|
3315
|
+
incident: input.incident,
|
|
3316
|
+
config: input.config,
|
|
3317
|
+
partialText: input.partialText,
|
|
3318
|
+
partialParts: input.partialParts,
|
|
3319
|
+
streamId: input.streamId,
|
|
3320
|
+
createdAt: input.createdAt
|
|
3321
|
+
});
|
|
3322
|
+
await notifyChatRecoveryExhausted(ctx, {
|
|
3323
|
+
emit: hooks.emit,
|
|
3324
|
+
onExhausted: hooks.onExhausted,
|
|
3325
|
+
onError: hooks.onError
|
|
3326
|
+
});
|
|
3327
|
+
await hooks.terminalize(ctx);
|
|
3328
|
+
}
|
|
3329
|
+
//#endregion
|
|
3330
|
+
//#region src/chat/stall-watchdog.ts
|
|
3331
|
+
/**
|
|
3332
|
+
* Shared inactivity watchdog for UI-message streams.
|
|
3333
|
+
*
|
|
3334
|
+
* A model/transport stream can park indefinitely without ever throwing (a hung
|
|
3335
|
+
* provider, a wedged transport). Left unguarded, the consumer read-loop waits
|
|
3336
|
+
* forever. {@link iterateWithStallWatchdog} wraps such a stream so that a gap of
|
|
3337
|
+
* `timeoutMs` between chunks aborts the upstream and throws
|
|
3338
|
+
* {@link ChatStreamStalledError}, letting the consumer route the stall into
|
|
3339
|
+
* bounded recovery (#1626) — a transient hang is retried within the existing
|
|
3340
|
+
* recovery budget — while genuine in-band errors stay terminal.
|
|
3341
|
+
*
|
|
3342
|
+
* @internal Sibling-package support for `@cloudflare/ai-chat` and
|
|
3343
|
+
* `@cloudflare/think`, not a public API. See
|
|
3344
|
+
* `design/rfc-chat-recovery-foundation.md`.
|
|
3345
|
+
*/
|
|
3346
|
+
/**
|
|
3347
|
+
* Thrown by {@link iterateWithStallWatchdog} when the inactivity watchdog fires
|
|
3348
|
+
* (a model/transport stream that parks without ever throwing). Distinct from
|
|
3349
|
+
* in-band model/stream errors so the read-loop catch can route a stall into
|
|
3350
|
+
* bounded recovery (#1626) — a transient hang is retried within the existing
|
|
3351
|
+
* recovery budget — while genuine errors stay terminal.
|
|
3352
|
+
*/
|
|
3353
|
+
var ChatStreamStalledError = class extends Error {
|
|
3354
|
+
constructor(message) {
|
|
3355
|
+
super(message);
|
|
3356
|
+
this.isChatStreamStall = true;
|
|
3357
|
+
this.name = "ChatStreamStalledError";
|
|
3358
|
+
}
|
|
3359
|
+
};
|
|
3360
|
+
/**
|
|
3361
|
+
* Wrap a UI-message stream with an inactivity watchdog. If no chunk arrives
|
|
3362
|
+
* within `timeoutMs`, `onStall` runs (aborting the upstream model stream) and
|
|
3363
|
+
* the iterator throws, so the consumer loop exits with a terminal error
|
|
3364
|
+
* instead of parking forever on a hung provider/transport. `timeoutMs <= 0`
|
|
3365
|
+
* passes the source through untouched.
|
|
3366
|
+
*/
|
|
3367
|
+
async function* iterateWithStallWatchdog(source, timeoutMs, onStall) {
|
|
3368
|
+
if (!(timeoutMs > 0)) {
|
|
3369
|
+
yield* source;
|
|
3370
|
+
return;
|
|
3371
|
+
}
|
|
3372
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
3373
|
+
let selfAborted = false;
|
|
3374
|
+
try {
|
|
3375
|
+
while (true) {
|
|
3376
|
+
let timer;
|
|
3377
|
+
let stalled = false;
|
|
3378
|
+
const stall = new Promise((_, reject) => {
|
|
3379
|
+
timer = setTimeout(() => {
|
|
3380
|
+
stalled = true;
|
|
3381
|
+
reject(new ChatStreamStalledError(`Chat stream stalled: no activity for ${timeoutMs}ms; the turn was aborted by the stall watchdog.`));
|
|
3382
|
+
}, timeoutMs);
|
|
3383
|
+
});
|
|
3384
|
+
const nextPromise = iterator.next();
|
|
3385
|
+
nextPromise.catch(() => {});
|
|
3386
|
+
let next;
|
|
3387
|
+
try {
|
|
3388
|
+
next = await Promise.race([nextPromise, stall]);
|
|
3389
|
+
} catch (err) {
|
|
3390
|
+
if (stalled) {
|
|
3391
|
+
selfAborted = true;
|
|
3392
|
+
onStall();
|
|
3393
|
+
}
|
|
3394
|
+
throw err;
|
|
3395
|
+
} finally {
|
|
3396
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
3397
|
+
}
|
|
3398
|
+
if (next.done) return;
|
|
3399
|
+
yield next.value;
|
|
3400
|
+
}
|
|
3401
|
+
} finally {
|
|
3402
|
+
if (!selfAborted) await iterator.return?.(void 0).catch(() => {});
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
//#endregion
|
|
3406
|
+
export { AGENT_TOOL_STREAM_PROGRESS_BUMP_THROTTLE_MS, AbortRegistry, AgentToolProgressEmitter, AgentToolStreamProgressThrottle, AutoContinuationController, CHAT_LAST_TERMINAL_KEY, CHAT_MESSAGE_TYPES, CHAT_RECOVERING_FLAG_TTL_MS, CHAT_RECOVERING_KEY, CHAT_RECOVERY_ALARM_DEBOUNCE_MS, CHAT_RECOVERY_INCIDENT_KEY_PREFIX, CHAT_RECOVERY_INCIDENT_TTL_MS, CHAT_RECOVERY_PROGRESS_KEY, CHAT_RECOVERY_STABLE_RETRY_DELAY_SECONDS, CHAT_STREAM_PROGRESS_CREDIT_THROTTLE_MS, ChatRecoveryEngine, ChatStreamStalledError, ContinuationState, DEFAULT_CHAT_RECOVERY_MAX_ATTEMPTS, DEFAULT_CHAT_RECOVERY_MAX_OOM_RETRIES, DEFAULT_CHAT_RECOVERY_MAX_WORK, DEFAULT_CHAT_RECOVERY_NO_PROGRESS_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_STABLE_TIMEOUT_MS, DEFAULT_CHAT_RECOVERY_TERMINAL_MESSAGE, KV_DELETE_MAX_KEYS, MAX_BOUND_PARAMS, MessageType, PreStreamTurns, ROW_MAX_BYTES, ResumableStream, ResumeHandshake, STREAM_CLEANUP_DELAY_SECONDS, StreamAccumulator, StreamProgressCreditThrottle, SubmitConcurrencyController, TIMED_OUT, TurnQueue, aiSdkRecoveryCodec, applyAgentToolEvent, applyChunkToParts, applyToolUpdate, awaitWithDeadline, transition as broadcastTransition, buildChatRecoveringFrame, buildInClauseStrings, bumpChatRecoveryProgress, byteLength, chatRecoverySchedulePolicy, classifyAgentToolChildRecovery, cleanupStreamBuffers, clearChatTerminal, clientResolvableToolNames, createAgentToolEventState, createChatFiberSnapshot, createToolsFromClientSchemas, crossMessageToolResultUpdate, drainInteractionApplies, enforceRowSizeLimit, hasIncompleteToolBatch, interceptAgentToolBroadcast, isReplayChunk, iterateWithStallWatchdog, listActiveChatRecoveryIncidents, normalizeToolInput, parseProtocolMessage, partAwaitsClientInteraction, pausedExecutionUpdate, pendingChatTerminal, persistReconstructedOrphan, readChatRecoveryProgress, reconcileMessages, reconcileOrphanPartial, recordChatTerminal, repairInterruptedToolParts, resolveChatRecoveryConfig, resolveToolMergeId, runChatRecoveryExhaustion, sanitizeMessage, sendIfOpen, setChatRecovering, shouldCreditStreamProgress, sweepStaleChatRecoveryIncidents, toolApprovalUpdate, toolPartHasSettledResult, toolResultUpdate, unwrapChatFiberSnapshot, wrapChatFiberSnapshot };
|
|
1824
3407
|
|
|
1825
3408
|
//# sourceMappingURL=index.js.map
|