@superblocksteam/gateway 2.0.161 → 2.0.162-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -39
- package/dist/agents/resolve-orchestrator-url.d.ts +18 -0
- package/dist/agents/resolve-orchestrator-url.js +64 -1
- package/dist/agents/resolve-orchestrator-url.js.map +1 -1
- package/dist/capabilities/lifecycle.d.ts +46 -29
- package/dist/capabilities/lifecycle.js +1731 -843
- package/dist/capabilities/lifecycle.js.map +1 -1
- package/dist/capabilities/query-integration.d.ts +14 -0
- package/dist/capabilities/query-integration.js +172 -0
- package/dist/capabilities/query-integration.js.map +1 -0
- package/dist/capabilities/source-files-archive.d.ts +11 -0
- package/dist/capabilities/source-files-archive.js +54 -0
- package/dist/capabilities/source-files-archive.js.map +1 -0
- package/dist/capabilities/types.d.ts +197 -119
- package/dist/capabilities/types.js +45 -6
- package/dist/capabilities/types.js.map +1 -1
- package/dist/capture/browser-contract.d.ts +15 -5
- package/dist/capture/browser-contract.js +13 -3
- package/dist/capture/browser-contract.js.map +1 -1
- package/dist/capture/browser-instructions.d.ts +3 -2
- package/dist/capture/browser-instructions.js +3 -2
- package/dist/capture/browser-instructions.js.map +1 -1
- package/dist/capture/mode.d.ts +8 -1
- package/dist/capture/mode.js +22 -2
- package/dist/capture/mode.js.map +1 -1
- package/dist/config.d.ts +22 -23
- package/dist/config.js +13 -12
- package/dist/config.js.map +1 -1
- package/dist/deps.d.ts +7 -7
- package/dist/index.d.ts +2 -4
- package/dist/index.js +2 -4
- package/dist/index.js.map +1 -1
- package/dist/integrations/read-only-postgres-query.d.ts +2 -0
- package/dist/integrations/read-only-postgres-query.js +164 -0
- package/dist/integrations/read-only-postgres-query.js.map +1 -0
- package/dist/main.js +1 -1
- package/dist/main.js.map +1 -1
- package/dist/playwright/ensure-chromium.d.ts +5 -7
- package/dist/playwright/ensure-chromium.js +13 -13
- package/dist/playwright/ensure-chromium.js.map +1 -1
- package/dist/preview/capture-screenshot.d.ts +30 -4
- package/dist/preview/capture-screenshot.js +67 -17
- package/dist/preview/capture-screenshot.js.map +1 -1
- package/dist/preview/viewer-url.js +3 -1
- package/dist/preview/viewer-url.js.map +1 -1
- package/dist/process/fault-barrier.js +1 -1
- package/dist/process/fault-barrier.js.map +1 -1
- package/dist/sabs/agent-facing-text.d.ts +2 -0
- package/dist/sabs/agent-facing-text.js +56 -2
- package/dist/sabs/agent-facing-text.js.map +1 -1
- package/dist/sabs/app-state.d.ts +126 -0
- package/dist/sabs/app-state.js +332 -0
- package/dist/sabs/app-state.js.map +1 -0
- package/dist/sabs/awaited-decision.d.ts +49 -0
- package/dist/sabs/awaited-decision.js +129 -0
- package/dist/sabs/awaited-decision.js.map +1 -0
- package/dist/sabs/editor-client-methods.d.ts +23 -5
- package/dist/sabs/editor-client-methods.js +45 -4
- package/dist/sabs/editor-client-methods.js.map +1 -1
- package/dist/sabs/editor-socket.d.ts +85 -0
- package/dist/sabs/editor-socket.js +67 -0
- package/dist/sabs/editor-socket.js.map +1 -0
- package/dist/sabs/session-peer.d.ts +119 -82
- package/dist/sabs/session-peer.js +10 -1
- package/dist/sabs/session-peer.js.map +1 -1
- package/dist/sabs/streamed-reply.d.ts +45 -0
- package/dist/sabs/streamed-reply.js +125 -0
- package/dist/sabs/streamed-reply.js.map +1 -0
- package/dist/sabs/turn-collector.d.ts +50 -35
- package/dist/sabs/turn-collector.js +110 -176
- package/dist/sabs/turn-collector.js.map +1 -1
- package/dist/sabs/websocket-session-peer.d.ts +104 -69
- package/dist/sabs/websocket-session-peer.js +1333 -477
- package/dist/sabs/websocket-session-peer.js.map +1 -1
- package/dist/server/client.d.ts +41 -9
- package/dist/server/client.js +126 -21
- package/dist/server/client.js.map +1 -1
- package/dist/start.d.ts +1 -1
- package/dist/start.js +30 -8
- package/dist/start.js.map +1 -1
- package/dist/stores/memory.d.ts +46 -0
- package/dist/stores/memory.js +163 -0
- package/dist/stores/memory.js.map +1 -0
- package/dist/stores/types.d.ts +92 -0
- package/dist/stores/types.js +11 -0
- package/dist/stores/types.js.map +1 -0
- package/dist/telemetry/mcp-client.d.ts +6 -1
- package/dist/telemetry/mcp-client.js +100 -4
- package/dist/telemetry/mcp-client.js.map +1 -1
- package/dist/telemetry/metrics.d.ts +12 -3
- package/dist/telemetry/metrics.js +60 -14
- package/dist/telemetry/metrics.js.map +1 -1
- package/dist/telemetry/runtime.js +4 -6
- package/dist/telemetry/runtime.js.map +1 -1
- package/dist/transports/mcp/admin-tools.d.ts +8 -0
- package/dist/transports/mcp/admin-tools.js +115 -4
- package/dist/transports/mcp/admin-tools.js.map +1 -1
- package/dist/transports/mcp/app-status-html.d.ts +4 -8
- package/dist/transports/mcp/app-status-html.js +336 -128
- package/dist/transports/mcp/app-status-html.js.map +1 -1
- package/dist/transports/mcp/client-presentation.d.ts +16 -0
- package/dist/transports/mcp/client-presentation.js +13 -0
- package/dist/transports/mcp/client-presentation.js.map +1 -0
- package/dist/transports/mcp/cowork-editor-url.d.ts +6 -0
- package/dist/transports/mcp/cowork-editor-url.js +10 -0
- package/dist/transports/mcp/cowork-editor-url.js.map +1 -0
- package/dist/transports/mcp/decision-card-html.d.ts +26 -0
- package/dist/transports/mcp/decision-card-html.js +876 -0
- package/dist/transports/mcp/decision-card-html.js.map +1 -0
- package/dist/transports/mcp/decision-elicitation.d.ts +42 -0
- package/dist/transports/mcp/decision-elicitation.js +59 -0
- package/dist/transports/mcp/decision-elicitation.js.map +1 -1
- package/dist/transports/mcp/editor-document-probe.d.ts +4 -0
- package/dist/transports/mcp/editor-document-probe.js +35 -0
- package/dist/transports/mcp/editor-document-probe.js.map +1 -0
- package/dist/transports/mcp/editor-integration-setup-url.d.ts +14 -0
- package/dist/transports/mcp/editor-integration-setup-url.js +29 -0
- package/dist/transports/mcp/editor-integration-setup-url.js.map +1 -0
- package/dist/transports/mcp/format-tool-content.d.ts +13 -18
- package/dist/transports/mcp/format-tool-content.js +172 -13
- package/dist/transports/mcp/format-tool-content.js.map +1 -1
- package/dist/transports/mcp/instructions/index.d.ts +23 -0
- package/dist/transports/mcp/instructions/index.js +81 -0
- package/dist/transports/mcp/instructions/index.js.map +1 -0
- package/dist/transports/mcp/instructions/result.d.ts +18 -0
- package/dist/transports/mcp/instructions/result.js +58 -0
- package/dist/transports/mcp/instructions/result.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/ask-user.d.ts +2 -0
- package/dist/transports/mcp/instructions/tools/ask-user.js +20 -0
- package/dist/transports/mcp/instructions/tools/ask-user.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/build-app.d.ts +4 -0
- package/dist/transports/mcp/instructions/tools/build-app.js +54 -0
- package/dist/transports/mcp/instructions/tools/build-app.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/check-app-progress.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/check-app-progress.js +79 -0
- package/dist/transports/mcp/instructions/tools/check-app-progress.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/check-publish-progress.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/check-publish-progress.js +23 -0
- package/dist/transports/mcp/instructions/tools/check-publish-progress.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/copy.d.ts +18 -0
- package/dist/transports/mcp/instructions/tools/copy.js +49 -0
- package/dist/transports/mcp/instructions/tools/copy.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/create-integration.d.ts +4 -0
- package/dist/transports/mcp/instructions/tools/create-integration.js +41 -0
- package/dist/transports/mcp/instructions/tools/create-integration.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/edit-app.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/edit-app.js +18 -0
- package/dist/transports/mcp/instructions/tools/edit-app.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/get-app.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/get-app.js +49 -0
- package/dist/transports/mcp/instructions/tools/get-app.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/get-integration-metadata.d.ts +2 -0
- package/dist/transports/mcp/instructions/tools/get-integration-metadata.js +13 -0
- package/dist/transports/mcp/instructions/tools/get-integration-metadata.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/index.d.ts +7 -0
- package/dist/transports/mcp/instructions/tools/index.js +31 -0
- package/dist/transports/mcp/instructions/tools/index.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/publish-app.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/publish-app.js +23 -0
- package/dist/transports/mcp/instructions/tools/publish-app.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/start-app.d.ts +3 -0
- package/dist/transports/mcp/instructions/tools/start-app.js +22 -0
- package/dist/transports/mcp/instructions/tools/start-app.js.map +1 -0
- package/dist/transports/mcp/instructions/tools/upload-artifact.d.ts +2 -0
- package/dist/transports/mcp/instructions/tools/upload-artifact.js +13 -0
- package/dist/transports/mcp/instructions/tools/upload-artifact.js.map +1 -0
- package/dist/transports/mcp/mcp-app-brand-css.d.ts +1 -0
- package/dist/transports/mcp/mcp-app-brand-css.js +180 -0
- package/dist/transports/mcp/mcp-app-brand-css.js.map +1 -0
- package/dist/transports/mcp/mount.d.ts +14 -1
- package/dist/transports/mcp/mount.js +729 -172
- package/dist/transports/mcp/mount.js.map +1 -1
- package/dist/transports/mcp/native-browser-presence.d.ts +45 -0
- package/dist/transports/mcp/native-browser-presence.js +158 -0
- package/dist/transports/mcp/native-browser-presence.js.map +1 -0
- package/dist/transports/mcp/plan-approval.d.ts +44 -0
- package/dist/transports/mcp/plan-approval.js +179 -0
- package/dist/transports/mcp/plan-approval.js.map +1 -0
- package/dist/transports/mcp/session-directory.d.ts +7 -0
- package/dist/transports/mcp/session-directory.js +18 -0
- package/dist/transports/mcp/session-directory.js.map +1 -0
- package/dist/transports/mcp/tool-names.d.ts +2 -0
- package/dist/transports/mcp/tool-names.js +2 -0
- package/dist/transports/mcp/tool-names.js.map +1 -0
- package/package.json +16 -7
- package/skills/superblocks-build/SKILL.md +59 -0
- package/skills/superblocks-import/SKILL.md +125 -0
- package/dist/capabilities/import-prompt.d.ts +0 -11
- package/dist/capabilities/import-prompt.js +0 -96
- package/dist/capabilities/import-prompt.js.map +0 -1
- package/dist/capabilities/persisted-progress.d.ts +0 -46
- package/dist/capabilities/persisted-progress.js +0 -246
- package/dist/capabilities/persisted-progress.js.map +0 -1
- package/dist/events/cursor.d.ts +0 -43
- package/dist/events/cursor.js +0 -78
- package/dist/events/cursor.js.map +0 -1
- package/dist/events/memory-event-store.d.ts +0 -34
- package/dist/events/memory-event-store.js +0 -110
- package/dist/events/memory-event-store.js.map +0 -1
- package/dist/events/merge.d.ts +0 -23
- package/dist/events/merge.js +0 -97
- package/dist/events/merge.js.map +0 -1
- package/dist/events/normalized-collector.d.ts +0 -62
- package/dist/events/normalized-collector.js +0 -156
- package/dist/events/normalized-collector.js.map +0 -1
- package/dist/events/schema.d.ts +0 -9
- package/dist/events/schema.js +0 -93
- package/dist/events/schema.js.map +0 -1
- package/dist/events/snapshot.d.ts +0 -32
- package/dist/events/snapshot.js +0 -57
- package/dist/events/snapshot.js.map +0 -1
- package/dist/events/stream-key.d.ts +0 -2
- package/dist/events/stream-key.js +0 -31
- package/dist/events/stream-key.js.map +0 -1
- package/dist/events/types.d.ts +0 -179
- package/dist/events/types.js +0 -66
- package/dist/events/types.js.map +0 -1
- package/dist/resume/memory-progress-store.d.ts +0 -39
- package/dist/resume/memory-progress-store.js +0 -82
- package/dist/resume/memory-progress-store.js.map +0 -1
- package/dist/resume/memory-recent-app-store.d.ts +0 -14
- package/dist/resume/memory-recent-app-store.js +0 -27
- package/dist/resume/memory-recent-app-store.js.map +0 -1
- package/dist/resume/memory-turn-store.d.ts +0 -18
- package/dist/resume/memory-turn-store.js +0 -73
- package/dist/resume/memory-turn-store.js.map +0 -1
- package/dist/resume/progress-key.d.ts +0 -21
- package/dist/resume/progress-key.js +0 -58
- package/dist/resume/progress-key.js.map +0 -1
- package/dist/resume/stores.d.ts +0 -14
- package/dist/resume/stores.js +0 -18
- package/dist/resume/stores.js.map +0 -1
- package/dist/resume/types.d.ts +0 -124
- package/dist/resume/types.js +0 -13
- package/dist/resume/types.js.map +0 -1
|
@@ -1,37 +1,23 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { WebSocket } from "ws";
|
|
3
3
|
import { createISocketClient, ISocket, SocketHeartbeat, } from "@superblocksteam/shared";
|
|
4
|
-
import { pickOrchestratorBaseUrl } from "../agents/resolve-orchestrator-url.js";
|
|
5
|
-
import {
|
|
4
|
+
import { canonicalizeOrchestratorBaseUrl, pickOrchestratorBaseUrl, } from "../agents/resolve-orchestrator-url.js";
|
|
5
|
+
import { gatewayDebug } from "../debug.js";
|
|
6
6
|
import { pickEditorProfileKey } from "../integrations/profile-key.js";
|
|
7
7
|
import { HttpOrchestratorClient, } from "../orchestrator/client.js";
|
|
8
|
+
import { recordCheckpointOutcome } from "../telemetry/metrics.js";
|
|
9
|
+
import { sanitizeAgentFacingText } from "./agent-facing-text.js";
|
|
10
|
+
import { AppState } from "./app-state.js";
|
|
8
11
|
import { buildBrowserContext } from "./browser-context.js";
|
|
9
12
|
import { createEditorClientMethods, loadIntegrationsForPrompt, } from "./editor-client-methods.js";
|
|
13
|
+
import { EditorSocket } from "./editor-socket.js";
|
|
10
14
|
import { appendRecentToolRun, buildRecentToolRun } from "./recent-tool-runs.js";
|
|
15
|
+
import { PromptNotSentError, } from "./session-peer.js";
|
|
11
16
|
import { DEFAULT_SETTLE_MS, TurnCollector } from "./turn-collector.js";
|
|
12
|
-
/**
|
|
13
|
-
* A prompt reaches Clark over the dev server's socket, not over HTTP: the
|
|
14
|
-
* sandbox exposes no HTTP route that starts a generation. This is the same
|
|
15
|
-
* call the editor makes when a user submits a prompt in the browser.
|
|
16
|
-
*/
|
|
17
|
-
const GENERATE_METHOD = "editor.aiGenerateWithQueue";
|
|
18
|
-
/**
|
|
19
|
-
* Commits the files Clark wrote as a draft and creates the checkpoint. The
|
|
20
|
-
* browser editor fires this automatically once files are in sync; a headless
|
|
21
|
-
* peer that skips it leaves the app uncommitted, so the build only finishes
|
|
22
|
-
* when a human finally opens the edit URL.
|
|
23
|
-
*/
|
|
24
|
-
const ACCEPT_DRAFT_METHOD = "editor.aiAcceptDraft";
|
|
25
|
-
const RELOAD_CHAT_METHOD = "editor.aiChatGetMessages";
|
|
26
17
|
const RELEASE_TURN_LOCK_METHOD = "editor.releaseTurnLock";
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
*
|
|
31
|
-
* Deliberately sent without `isUserActive`: the sandbox counts that as a person
|
|
32
|
-
* at a keyboard, and a headless peer is not one.
|
|
33
|
-
*/
|
|
34
|
-
const PING_METHOD = "editor.ping";
|
|
18
|
+
function editorStructureScopeKey(applicationId, branch) {
|
|
19
|
+
return `${applicationId}\u0000${branch?.trim() ?? ""}`;
|
|
20
|
+
}
|
|
35
21
|
/**
|
|
36
22
|
* Whole budget for connecting and getting the prompt acknowledged, so a
|
|
37
23
|
* session that accepts the socket but never answers cannot strand the MCP call
|
|
@@ -42,6 +28,8 @@ const PING_METHOD = "editor.ping";
|
|
|
42
28
|
* was failing on a tighter budget only to succeed on the retry.
|
|
43
29
|
*/
|
|
44
30
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
31
|
+
/** Playwright capture (45s + 2.5s settle) plus one held-socket RPC budget. */
|
|
32
|
+
const DEFAULT_CHECKPOINT_SCREENSHOT_TIMEOUT_MS = 180_000;
|
|
45
33
|
/**
|
|
46
34
|
* The sandbox binds its HTTP port before it attaches the socket handler, and
|
|
47
35
|
* an upgrade arriving in that window gets no response at all rather than a
|
|
@@ -58,10 +46,36 @@ const CONNECT_RETRY_DELAY_MS = 500;
|
|
|
58
46
|
* which is the right threshold for a diagnostic and the wrong one for ending a
|
|
59
47
|
* turn: a sandbox that misses two beats and then answers is recovering, not
|
|
60
48
|
* gone. This is the second gate — the silence has to outlast it, and any beat
|
|
61
|
-
* that succeeds clears it.
|
|
62
|
-
* is a couple of minutes, against a ten-minute turn budget.
|
|
49
|
+
* that succeeds clears it.
|
|
63
50
|
*/
|
|
64
51
|
const DEFAULT_BEAT_LOST_GRACE_MS = 60_000;
|
|
52
|
+
const DEFAULT_BEAT_HEALTH_TIMEOUT_MS = 3_000;
|
|
53
|
+
async function liveEditIsHealthy(input) {
|
|
54
|
+
if (!input.sessionToken) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const response = await fetch(new URL("/_sb_health", input.liveEditUrl).toString(), {
|
|
59
|
+
headers: {
|
|
60
|
+
Authorization: `Bearer ${input.sessionToken}`,
|
|
61
|
+
},
|
|
62
|
+
signal: AbortSignal.timeout(DEFAULT_BEAT_HEALTH_TIMEOUT_MS),
|
|
63
|
+
});
|
|
64
|
+
if (!response.ok) {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
const body = await response.json();
|
|
68
|
+
return (typeof body === "object" &&
|
|
69
|
+
body !== null &&
|
|
70
|
+
"applicationId" in body &&
|
|
71
|
+
body.applicationId === input.applicationId &&
|
|
72
|
+
"status" in body &&
|
|
73
|
+
body.status === "healthy");
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
65
79
|
/**
|
|
66
80
|
* Consecutive upgrade rejections from one host before asking where the live
|
|
67
81
|
* edit went. More than one, so an ingress route that briefly answers for a pod
|
|
@@ -125,18 +139,21 @@ const DEFAULT_HEARTBEAT_TIMEOUT_MS = 15 * 1000;
|
|
|
125
139
|
* healthy round trip looks like at the instant a socket closes.
|
|
126
140
|
*/
|
|
127
141
|
const MIN_REPORTED_BEAT_WAIT_MS = 1_000;
|
|
128
|
-
/**
|
|
129
|
-
* How long to wait after Clark acks the prompt for the generation turn to
|
|
130
|
-
* reach a settled terminal state (completed after quiet settle, cancelled,
|
|
131
|
-
* or needs_decision).
|
|
132
|
-
*/
|
|
133
|
-
const DEFAULT_TURN_TIMEOUT_MS = 10 * 60 * 1000;
|
|
134
142
|
/**
|
|
135
143
|
* What a caller is told when the live edit backing its turn is gone. The cause
|
|
136
144
|
* is appended by whichever path noticed: a close code and heartbeat account
|
|
137
145
|
* when the socket dropped, nothing when SABS said so outright.
|
|
138
146
|
*/
|
|
139
147
|
const LIVE_EDIT_TERMINATED_REPLY = "The Superblocks live-edit session terminated before the turn finished, so nothing was built. Send the prompt again to start it over.";
|
|
148
|
+
function tagRetryableLiveEditDeath(turn, outcome) {
|
|
149
|
+
if (outcome.generationStatus !== "live_edit_terminated") {
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (outcome.retryable === false || outcome.recoveryAction) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
turn.retryAction ??= "retry_build";
|
|
156
|
+
}
|
|
140
157
|
/**
|
|
141
158
|
* How long a turn waits for Clark's first push before suspecting the sandbox is
|
|
142
159
|
* not reporting into this socket.
|
|
@@ -164,8 +181,7 @@ const DEFAULT_RECONNECT_GRACE_MS = 20_000;
|
|
|
164
181
|
* How many times one turn goes back for a live edit that keeps dropping it.
|
|
165
182
|
*
|
|
166
183
|
* A sandbox that closes every socket it accepts is not one a turn can finish
|
|
167
|
-
* on,
|
|
168
|
-
* for the whole turn budget.
|
|
184
|
+
* on, so reconnect attempts are capped independently.
|
|
169
185
|
*/
|
|
170
186
|
const DEFAULT_RECONNECT_ATTEMPTS = 3;
|
|
171
187
|
/**
|
|
@@ -173,6 +189,7 @@ const DEFAULT_RECONNECT_ATTEMPTS = 3;
|
|
|
173
189
|
* Clark's files before giving up on checkpointing them.
|
|
174
190
|
*/
|
|
175
191
|
const DEFAULT_READY_WAIT_MS = 30_000;
|
|
192
|
+
const PENDING_START_POLL_INTERVAL_MS = 250;
|
|
176
193
|
/**
|
|
177
194
|
* The role the dev server classifies this connection as. A gateway peer gets
|
|
178
195
|
* Clark's pushes for the turns it starts and its own sandbox-liveness signal,
|
|
@@ -202,86 +219,11 @@ function liveEditOf(source) {
|
|
|
202
219
|
url: source.liveEditUrl,
|
|
203
220
|
};
|
|
204
221
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
const DEFAULT_FAILED_START_TTL_MS = 30 * 60 * 1000;
|
|
211
|
-
/**
|
|
212
|
-
* Bound on distinct apps holding a dead start at once, so a pathological caller
|
|
213
|
-
* cannot grow this without limit.
|
|
214
|
-
*/
|
|
215
|
-
const MAX_FAILED_STARTS = 256;
|
|
216
|
-
/**
|
|
217
|
-
* Dead starts, scoped to the caller whose prompt died and bounded in both age
|
|
218
|
-
* and count.
|
|
219
|
-
*
|
|
220
|
-
* Scoped because a connect failure is not a fact about the application:
|
|
221
|
-
* `connectLiveEdit` is where Superblocks authorizes the caller, and Superblocks
|
|
222
|
-
* authorizes per application, so a caller with no rights to this one still
|
|
223
|
-
* reaches the failure path. Keyed any wider, its record is what everyone who
|
|
224
|
-
* *does* have rights reads as the app's status - by application alone that was
|
|
225
|
-
* every organization, and by organization it was every colleague. The record
|
|
226
|
-
* exists so the caller who just failed gets an honest answer and can retry, and
|
|
227
|
-
* that caller is who it is keyed to.
|
|
228
|
-
*
|
|
229
|
-
* Bounded because nothing else evicts: an app whose start failed and that is
|
|
230
|
-
* never retried would otherwise hold its entry for the life of the process.
|
|
231
|
-
*/
|
|
232
|
-
class FailedStarts {
|
|
233
|
-
entries = new Map();
|
|
234
|
-
ttlMs;
|
|
235
|
-
maxEntries;
|
|
236
|
-
constructor(ttlMs, maxEntries) {
|
|
237
|
-
this.ttlMs = ttlMs ?? DEFAULT_FAILED_START_TTL_MS;
|
|
238
|
-
this.maxEntries = maxEntries ?? MAX_FAILED_STARTS;
|
|
239
|
-
}
|
|
240
|
-
set(scope, turn) {
|
|
241
|
-
const key = this.keyOf(scope);
|
|
242
|
-
// Re-inserted rather than updated in place, so this key becomes the newest
|
|
243
|
-
// and the eviction below stays in insertion order.
|
|
244
|
-
this.entries.delete(key);
|
|
245
|
-
this.entries.set(key, { at: Date.now(), turn });
|
|
246
|
-
for (const oldest of this.entries.keys()) {
|
|
247
|
-
if (this.entries.size <= this.maxEntries) {
|
|
248
|
-
break;
|
|
249
|
-
}
|
|
250
|
-
this.entries.delete(oldest);
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
get(scope) {
|
|
254
|
-
const key = this.keyOf(scope);
|
|
255
|
-
const entry = this.entries.get(key);
|
|
256
|
-
if (!entry) {
|
|
257
|
-
return undefined;
|
|
258
|
-
}
|
|
259
|
-
if (Date.now() - entry.at >= this.ttlMs) {
|
|
260
|
-
this.entries.delete(key);
|
|
261
|
-
return undefined;
|
|
262
|
-
}
|
|
263
|
-
return entry.turn;
|
|
264
|
-
}
|
|
265
|
-
/**
|
|
266
|
-
* Forget every caller's record for this application. The reads are scoped,
|
|
267
|
-
* but acceptance is not: a prompt Superblocks took means the app has a live
|
|
268
|
-
* turn, and nobody's dead start outranks that.
|
|
269
|
-
*/
|
|
270
|
-
clear(applicationId) {
|
|
271
|
-
for (const key of [...this.entries.keys()]) {
|
|
272
|
-
if (key.endsWith(`\u0000${applicationId}`)) {
|
|
273
|
-
this.entries.delete(key);
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
/**
|
|
278
|
-
* Separated by NUL because no id can contain one. Any printable separator is
|
|
279
|
-
* a character an id could carry, and an id that carried it would key two
|
|
280
|
-
* different callers to the same entry.
|
|
281
|
-
*/
|
|
282
|
-
keyOf(scope) {
|
|
283
|
-
return `${scope.organizationId}\u0000${scope.userId}\u0000${scope.applicationId}`;
|
|
284
|
-
}
|
|
222
|
+
function isBrowserObserverEvent(event) {
|
|
223
|
+
return ((event.kind === "message" &&
|
|
224
|
+
event.message.gatewayObserverSource === "browser") ||
|
|
225
|
+
(event.kind === "generation_state" &&
|
|
226
|
+
event.gatewayObserverSource === "browser"));
|
|
285
227
|
}
|
|
286
228
|
export function buildSessionUrl(input) {
|
|
287
229
|
// liveEditUrl is server-sourced (connectLiveEdit), not caller-supplied, so a
|
|
@@ -333,6 +275,15 @@ function isHostRejection(error) {
|
|
|
333
275
|
function isTimeout(error) {
|
|
334
276
|
return /timed out|timeout/i.test(error.message ?? "");
|
|
335
277
|
}
|
|
278
|
+
function isRemoteRejection(error) {
|
|
279
|
+
// ISocket reserves 7-10 for timeout, connection close, abort, and a request
|
|
280
|
+
// made after close. Every other numbered code arrived in a response.
|
|
281
|
+
return (error.code !== undefined &&
|
|
282
|
+
error.code !== 7 &&
|
|
283
|
+
error.code !== 8 &&
|
|
284
|
+
error.code !== 9 &&
|
|
285
|
+
error.code !== 10);
|
|
286
|
+
}
|
|
336
287
|
function isSocketOpen(ws) {
|
|
337
288
|
return ws.readyState === WebSocket.OPEN;
|
|
338
289
|
}
|
|
@@ -362,6 +313,54 @@ function describeSocketError(error) {
|
|
|
362
313
|
return String(error);
|
|
363
314
|
}
|
|
364
315
|
}
|
|
316
|
+
function checkpointFailure(error) {
|
|
317
|
+
const detail = typeof error === "object" && error !== null
|
|
318
|
+
? error
|
|
319
|
+
: undefined;
|
|
320
|
+
const rawMessage = error instanceof Error
|
|
321
|
+
? error.message
|
|
322
|
+
: typeof detail?.message === "string" && detail.message.length > 0
|
|
323
|
+
? detail.message
|
|
324
|
+
: describeSocketError(error);
|
|
325
|
+
const message = sanitizeAgentFacingText(rawMessage).slice(0, 500);
|
|
326
|
+
const code = typeof detail?.code === "number" ? detail.code : undefined;
|
|
327
|
+
const status = typeof detail?.status === "number" ? detail.status : undefined;
|
|
328
|
+
const category = status === 408 || status === 504 || /timed out|timeout/i.test(message)
|
|
329
|
+
? "timeout"
|
|
330
|
+
: status === 401 ||
|
|
331
|
+
status === 403 ||
|
|
332
|
+
code === 401 ||
|
|
333
|
+
code === 403 ||
|
|
334
|
+
/\b(?:401|403)\b|unauthoriz|forbidden/i.test(message)
|
|
335
|
+
? "authentication"
|
|
336
|
+
: status === 409 || code === 409 || /\b409\b|conflict/i.test(message)
|
|
337
|
+
? "conflict"
|
|
338
|
+
: "downstream";
|
|
339
|
+
return {
|
|
340
|
+
category,
|
|
341
|
+
...(code === undefined ? {} : { code }),
|
|
342
|
+
message,
|
|
343
|
+
...(status === undefined ? {} : { status }),
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
function normalizeCheckpointWarning(value) {
|
|
347
|
+
if (typeof value !== "object" || value === null) {
|
|
348
|
+
return undefined;
|
|
349
|
+
}
|
|
350
|
+
const warning = value;
|
|
351
|
+
if (warning.code !== "draft_state_cleanup_failed" ||
|
|
352
|
+
typeof warning.message !== "string") {
|
|
353
|
+
return undefined;
|
|
354
|
+
}
|
|
355
|
+
const message = sanitizeAgentFacingText(warning.message).trim().slice(0, 500);
|
|
356
|
+
if (!message) {
|
|
357
|
+
return undefined;
|
|
358
|
+
}
|
|
359
|
+
return {
|
|
360
|
+
code: "draft_state_cleanup_failed",
|
|
361
|
+
message,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
365
364
|
/**
|
|
366
365
|
* What the socket said on its way out, appended to the disconnect reply.
|
|
367
366
|
*
|
|
@@ -442,21 +441,30 @@ function describeOutstandingBeat(heartbeat, at) {
|
|
|
442
441
|
return ` A beat sent ${seconds}s earlier had still not been answered.`;
|
|
443
442
|
}
|
|
444
443
|
function describeSettledBeats(heartbeat, at) {
|
|
444
|
+
if (heartbeat.refusals > 0) {
|
|
445
|
+
const beats = heartbeat.refusals === 1 ? "beat" : "beats";
|
|
446
|
+
// The refusal's own reason rather than `lastError`: an unanswered beat
|
|
447
|
+
// after the refusal overwrites that with its own cause while leaving
|
|
448
|
+
// `refusals` standing, so reading it here would put a timeout's message in
|
|
449
|
+
// the peer's mouth.
|
|
450
|
+
const stated = heartbeat.lastRefusalError ?? heartbeat.lastError;
|
|
451
|
+
const cause = stated ? ` (${stated})` : "";
|
|
452
|
+
// Reported ahead of the silence counts, and alongside them rather than
|
|
453
|
+
// instead of them. A refusal is an answer: the connection worked and the
|
|
454
|
+
// peer named its own reason for the turn dying, which no inferred silence
|
|
455
|
+
// outranks. But a refusal clears nothing, so "refused, then went quiet" is
|
|
456
|
+
// a different death from "refusing right up to the close" and the caller
|
|
457
|
+
// needs both halves to tell them apart.
|
|
458
|
+
const silence = heartbeat.failures > 0
|
|
459
|
+
? ` ${heartbeat.failures} later ${heartbeat.failures === 1 ? "beat" : "beats"} went unanswered.`
|
|
460
|
+
: "";
|
|
461
|
+
return ` ${heartbeat.refusals} liveness ${beats} came back refused${cause}.${silence}`;
|
|
462
|
+
}
|
|
445
463
|
if (heartbeat.failures > 0) {
|
|
446
464
|
const beats = heartbeat.failures === 1 ? "beat" : "beats";
|
|
447
465
|
const cause = heartbeat.lastError ? ` (${heartbeat.lastError})` : "";
|
|
448
466
|
return ` ${heartbeat.failures} liveness ${beats} failed before it${cause}.`;
|
|
449
467
|
}
|
|
450
|
-
if (heartbeat.refusals > 0) {
|
|
451
|
-
const beats = heartbeat.refusals === 1 ? "beat" : "beats";
|
|
452
|
-
const cause = heartbeat.lastError ? ` (${heartbeat.lastError})` : "";
|
|
453
|
-
// A refusal is an answer, so the connection worked and the peer named its
|
|
454
|
-
// own reason for the turn dying. Reported before a lapse because a refusal
|
|
455
|
-
// does not clear one: an earlier excused silence would otherwise outlive
|
|
456
|
-
// the beat that explained itself, and describing the socket as merely
|
|
457
|
-
// quiet throws away the only account the caller can act on.
|
|
458
|
-
return ` ${heartbeat.refusals} liveness ${beats} came back refused${cause}.`;
|
|
459
|
-
}
|
|
460
468
|
if (heartbeat.lapses > 0) {
|
|
461
469
|
const beats = heartbeat.lapses === 1 ? "beat" : "beats";
|
|
462
470
|
// A silent request channel on a connection that was demonstrably carrying
|
|
@@ -534,6 +542,41 @@ async function withinDeadline(lookup, deadline) {
|
|
|
534
542
|
}
|
|
535
543
|
}
|
|
536
544
|
}
|
|
545
|
+
async function beforePromptDeadline(work, deadline, signal) {
|
|
546
|
+
if (signal?.aborted) {
|
|
547
|
+
throw new PromptNotSentError("Prompt forward cancelled before the live-edit request was sent", signal.reason);
|
|
548
|
+
}
|
|
549
|
+
const remaining = deadline - Date.now();
|
|
550
|
+
if (remaining <= 0) {
|
|
551
|
+
throw new PromptNotSentError("Prompt forward timed out before the live-edit request was sent");
|
|
552
|
+
}
|
|
553
|
+
let timer;
|
|
554
|
+
let onAbort;
|
|
555
|
+
try {
|
|
556
|
+
return await Promise.race([
|
|
557
|
+
work,
|
|
558
|
+
new Promise((_resolve, reject) => {
|
|
559
|
+
timer = setTimeout(() => reject(new PromptNotSentError("Prompt forward timed out before the live-edit request was sent")), remaining);
|
|
560
|
+
}),
|
|
561
|
+
...(signal
|
|
562
|
+
? [
|
|
563
|
+
new Promise((_resolve, reject) => {
|
|
564
|
+
onAbort = () => reject(new PromptNotSentError("Prompt forward cancelled before the live-edit request was sent", signal.reason));
|
|
565
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
566
|
+
}),
|
|
567
|
+
]
|
|
568
|
+
: []),
|
|
569
|
+
]);
|
|
570
|
+
}
|
|
571
|
+
finally {
|
|
572
|
+
if (timer) {
|
|
573
|
+
clearTimeout(timer);
|
|
574
|
+
}
|
|
575
|
+
if (onAbort) {
|
|
576
|
+
signal?.removeEventListener("abort", onAbort);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
}
|
|
537
580
|
async function open(url, attemptTimeoutMs, deadline, onSocketError,
|
|
538
581
|
/**
|
|
539
582
|
* Where the live edit is now, asked for only once and only when the host in
|
|
@@ -589,9 +632,20 @@ nextHost) {
|
|
|
589
632
|
}
|
|
590
633
|
throw new Error(`Prompt forward timed out: ${current.host} did not accept a live-edit socket (${lastError instanceof Error ? lastError.message : String(lastError)})${current.host !== url.host ? ` after moving from ${url.host}` : ""}`);
|
|
591
634
|
}
|
|
635
|
+
/**
|
|
636
|
+
* The typed `editor.*` surface for a held session.
|
|
637
|
+
*
|
|
638
|
+
* Built per call rather than held on the session because the edit-scoped token
|
|
639
|
+
* travels with the request, not with the socket: a session outlives the token
|
|
640
|
+
* a given turn was authorized with.
|
|
641
|
+
*/
|
|
642
|
+
function editorSocket(held, sessionToken) {
|
|
643
|
+
return new EditorSocket(held.client, () => sessionToken ? `Bearer ${sessionToken}` : undefined);
|
|
644
|
+
}
|
|
592
645
|
export class WebSocketSessionPeer {
|
|
593
646
|
options;
|
|
594
647
|
held = new Map();
|
|
648
|
+
promptStarts = new Map();
|
|
595
649
|
/**
|
|
596
650
|
* Every server -> editor method the dev server has called on this peer.
|
|
597
651
|
* Diagnostic only: it answers "did that push ever arrive" without guessing
|
|
@@ -609,64 +663,98 @@ export class WebSocketSessionPeer {
|
|
|
609
663
|
* prompt for the same app replaces it.
|
|
610
664
|
*/
|
|
611
665
|
turns = new Map();
|
|
612
|
-
/**
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
/**
|
|
618
|
-
* Last needs_decision payload (plan or multi_choice) per application, so
|
|
619
|
-
* approve/refine via edit_app can reuse the pending plan after
|
|
620
|
-
* check_app_progress drained the finished turn.
|
|
621
|
-
*/
|
|
622
|
-
pendingDecisions = new Map();
|
|
623
|
-
/**
|
|
624
|
-
* Applications that have dispatched at least one BUILD turn. Survives
|
|
625
|
-
* clearing the pending plan, which happens as soon as an approved build
|
|
626
|
-
* starts — the point of the latch is the turns that come after it.
|
|
627
|
-
*/
|
|
628
|
-
buildModeApplications = new Set();
|
|
629
|
-
/**
|
|
630
|
-
* Turns that died before Superblocks ever saw them, so no {@link ActiveTurn}
|
|
631
|
-
* was ever created to hold the outcome. Read by
|
|
632
|
-
* {@link WebSocketSessionPeer.turnState} and
|
|
633
|
-
* {@link WebSocketSessionPeer.awaitTurn} after the live map, and cleared when
|
|
634
|
-
* a prompt for the app is accepted.
|
|
635
|
-
*/
|
|
636
|
-
failedStarts;
|
|
666
|
+
/** Browser-owned turns observed over an authenticated held session. */
|
|
667
|
+
observedTurns = new Map();
|
|
668
|
+
editorStructureByScope = new Map();
|
|
669
|
+
/** Facts about an application that outlive any one turn. */
|
|
670
|
+
appState;
|
|
637
671
|
constructor(options = {}) {
|
|
638
672
|
this.options = options;
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
673
|
+
this.appState =
|
|
674
|
+
options.appState ??
|
|
675
|
+
new AppState({
|
|
676
|
+
...(options.failedStartTtlMs === undefined
|
|
677
|
+
? {}
|
|
678
|
+
: { failedStartTtlMs: options.failedStartTtlMs }),
|
|
679
|
+
...(options.maxFailedStarts === undefined
|
|
680
|
+
? {}
|
|
681
|
+
: { maxFailedStarts: options.maxFailedStarts }),
|
|
682
|
+
...(options.pendingStartTtlMs === undefined
|
|
683
|
+
? {}
|
|
684
|
+
: { pendingStartTtlMs: options.pendingStartTtlMs }),
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
/** The per-application facts this peer writes, for the capability layer. */
|
|
688
|
+
state() {
|
|
689
|
+
return this.appState;
|
|
690
|
+
}
|
|
691
|
+
recordPendingStart(input) {
|
|
692
|
+
return this.appState.recordPendingStart(input, {
|
|
693
|
+
...(input.idempotencyKey === undefined
|
|
694
|
+
? {}
|
|
695
|
+
: { idempotencyKey: input.idempotencyKey }),
|
|
696
|
+
...(input.retryAction ? { retryAction: input.retryAction } : {}),
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
clearPendingStart(input) {
|
|
700
|
+
this.appState.clearPendingStart(input, input.idempotencyKey, undefined, input.pendingStartToken);
|
|
701
|
+
}
|
|
702
|
+
async waitForPendingTurn(input) {
|
|
703
|
+
for (;;) {
|
|
704
|
+
const turn = this.turns.get(input.applicationId);
|
|
705
|
+
if (turn &&
|
|
706
|
+
turn.revision > input.afterRevision &&
|
|
707
|
+
(input.idempotencyKey === undefined ||
|
|
708
|
+
turn.idempotencyKey === input.idempotencyKey) &&
|
|
709
|
+
(input.pendingStartToken === undefined ||
|
|
710
|
+
turn.pendingStartToken === input.pendingStartToken) &&
|
|
711
|
+
(input.scope.branch === undefined || input.scope.branch === turn.branch)) {
|
|
712
|
+
return turn;
|
|
713
|
+
}
|
|
714
|
+
if (input.signal?.aborted ||
|
|
715
|
+
!this.appState.pendingStart(input.scope) ||
|
|
716
|
+
Date.now() >= input.deadlineAtMs) {
|
|
717
|
+
return undefined;
|
|
718
|
+
}
|
|
719
|
+
await new Promise((resolve) => {
|
|
720
|
+
const settle = () => {
|
|
721
|
+
clearTimeout(timer);
|
|
722
|
+
input.signal?.removeEventListener("abort", settle);
|
|
723
|
+
resolve();
|
|
724
|
+
};
|
|
725
|
+
const timer = setTimeout(settle, Math.min(PENDING_START_POLL_INTERVAL_MS, Math.max(0, input.deadlineAtMs - Date.now())));
|
|
726
|
+
input.signal?.addEventListener("abort", settle, { once: true });
|
|
727
|
+
});
|
|
728
|
+
}
|
|
643
729
|
}
|
|
644
730
|
/**
|
|
645
731
|
* Plan waiting on human approve/refine for this application, if any.
|
|
646
732
|
*/
|
|
647
|
-
pendingPlan(applicationId) {
|
|
648
|
-
return this.
|
|
733
|
+
pendingPlan(applicationId, requestedBranch) {
|
|
734
|
+
return this.appState.pendingPlan(applicationId, requestedBranch);
|
|
649
735
|
}
|
|
650
736
|
/**
|
|
651
737
|
* Full needs_decision payload waiting on a human for this application.
|
|
652
738
|
*/
|
|
653
|
-
pendingDecision(applicationId) {
|
|
654
|
-
return this.
|
|
739
|
+
pendingDecision(applicationId, requestedBranch) {
|
|
740
|
+
return this.appState.pendingDecision(applicationId, requestedBranch);
|
|
655
741
|
}
|
|
656
742
|
/**
|
|
657
743
|
* Remember a turn that never reached Superblocks. See
|
|
658
744
|
* {@link SessionPeer.recordFailedStart}.
|
|
659
745
|
*/
|
|
660
746
|
recordFailedStart(input) {
|
|
747
|
+
const pendingCleared = this.appState.clearPendingStart(input, input.idempotencyKey, undefined, input.pendingStartToken);
|
|
748
|
+
if (input.pendingStartRequired && !pendingCleared) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
661
751
|
const running = this.turns.get(input.applicationId);
|
|
662
|
-
if (running &&
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
//
|
|
667
|
-
//
|
|
668
|
-
// with nothing left in memory to say which. Skipping keeps every stored
|
|
669
|
-
// record older than any finished turn it can be read beside.
|
|
752
|
+
if (running &&
|
|
753
|
+
!running.finished &&
|
|
754
|
+
input.branch === running.branch &&
|
|
755
|
+
!(input.pendingStartRequired && pendingCleared)) {
|
|
756
|
+
// This failed prompt did not end the turn already running on its branch.
|
|
757
|
+
// Keeping its newer revision would hide that turn's eventual outcome.
|
|
670
758
|
console.warn("gateway clark turn never started, but one is running", {
|
|
671
759
|
applicationId: input.applicationId,
|
|
672
760
|
organizationId: input.organizationId,
|
|
@@ -681,81 +769,99 @@ export class WebSocketSessionPeer {
|
|
|
681
769
|
reason: input.reason,
|
|
682
770
|
userId: input.userId,
|
|
683
771
|
});
|
|
684
|
-
this.
|
|
685
|
-
|
|
772
|
+
this.appState.recordFailedStart(input, {
|
|
773
|
+
...(input.decision ? { decision: input.decision } : {}),
|
|
774
|
+
generationStatus: input.generationStatus ?? "live_edit_terminated",
|
|
686
775
|
reply: input.reason,
|
|
776
|
+
...(input.recoveryAction
|
|
777
|
+
? { recoveryAction: input.recoveryAction }
|
|
778
|
+
: {}),
|
|
687
779
|
...(input.retryAction ? { retryAction: input.retryAction } : {}),
|
|
780
|
+
...(input.retryable === false ? { retryable: false } : {}),
|
|
688
781
|
// Nothing ran, so nothing is viewable. Reporting "ready" here is how a
|
|
689
782
|
// caller comes to show the untouched starter template as the app.
|
|
690
783
|
viewStatus: "unknown",
|
|
691
|
-
});
|
|
784
|
+
}, input.revision);
|
|
692
785
|
}
|
|
693
786
|
/**
|
|
694
787
|
* Where this application's turn stands for this caller, read without draining
|
|
695
788
|
* the activity an {@link WebSocketSessionPeer.awaitTurn} poll is owed.
|
|
696
789
|
*
|
|
697
|
-
*
|
|
698
|
-
*
|
|
699
|
-
*
|
|
700
|
-
* already finished: `turns` keeps finished turns, so without this a follow-up
|
|
701
|
-
* whose connect failed would be answered by the previous turn's outcome and
|
|
702
|
-
* its reply, reporting an edit as landed that never ran. And it is read rather
|
|
703
|
-
* than superseded by deleting from `turns`, because that map is keyed by
|
|
704
|
-
* application while a dead start belongs to one caller - deleting would erase
|
|
705
|
-
* a colleague's outcome to answer this caller's question.
|
|
706
|
-
*
|
|
707
|
-
* Reading recency out of the order rather than out of a clock holds because
|
|
708
|
-
* of the two rules either side of this one: accepting a prompt clears every
|
|
709
|
-
* caller's record for the app, and no record is made while a turn is running.
|
|
710
|
-
* Together they leave every stored record newer than any finished turn it can
|
|
711
|
-
* be read beside, so "the record wins" is not a guess about which came first.
|
|
790
|
+
* A running turn is current. Once it finishes, the newer of that turn and an
|
|
791
|
+
* unscoped failed start wins; explicit branch reads filter before comparing.
|
|
792
|
+
* Revisions preserve that ordering even when both events share a clock tick.
|
|
712
793
|
*/
|
|
713
794
|
turnState(input) {
|
|
714
|
-
const
|
|
715
|
-
|
|
795
|
+
const storedTurn = this.latestTurn(input.applicationId);
|
|
796
|
+
const turn = storedTurn &&
|
|
797
|
+
input.branch !== undefined &&
|
|
798
|
+
input.branch !== storedTurn.branch
|
|
799
|
+
? undefined
|
|
800
|
+
: storedTurn;
|
|
801
|
+
if (this.appState.pendingStart(input)) {
|
|
716
802
|
return { phase: "running" };
|
|
717
803
|
}
|
|
718
|
-
const failed = this.
|
|
804
|
+
const failed = this.appState.failedStart(input, turn?.revision);
|
|
719
805
|
if (failed) {
|
|
720
806
|
return {
|
|
807
|
+
...(failed.branch === undefined ? {} : { branch: failed.branch }),
|
|
721
808
|
generationStatus: failed.generationStatus,
|
|
722
809
|
phase: "finished",
|
|
810
|
+
...(failed.recoveryAction
|
|
811
|
+
? { recoveryAction: failed.recoveryAction }
|
|
812
|
+
: {}),
|
|
813
|
+
...(failed.retryable === false ? { reply: failed.reply } : {}),
|
|
723
814
|
...(failed.retryAction ? { retryAction: failed.retryAction } : {}),
|
|
815
|
+
...(failed.retryable === false ? { retryable: false } : {}),
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
if (turn && !turn.finished) {
|
|
819
|
+
return {
|
|
820
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
821
|
+
phase: "running",
|
|
724
822
|
};
|
|
725
823
|
}
|
|
726
824
|
if (!turn?.finished) {
|
|
727
825
|
return undefined;
|
|
728
826
|
}
|
|
729
827
|
return {
|
|
828
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
730
829
|
...(turn.finished.checkpointCommitId
|
|
731
830
|
? { checkpointCommitId: turn.finished.checkpointCommitId }
|
|
732
831
|
: {}),
|
|
832
|
+
...(turn.finished.checkpointFailure
|
|
833
|
+
? { checkpointFailure: turn.finished.checkpointFailure }
|
|
834
|
+
: {}),
|
|
835
|
+
...(turn.finished.checkpointWarning
|
|
836
|
+
? { checkpointWarning: turn.finished.checkpointWarning }
|
|
837
|
+
: {}),
|
|
733
838
|
generationStatus: turn.finished.generationStatus,
|
|
734
839
|
phase: "finished",
|
|
735
840
|
...(turn.retryAction ? { retryAction: turn.retryAction } : {}),
|
|
841
|
+
...(turn.finished.retryable === false
|
|
842
|
+
? { retryable: false }
|
|
843
|
+
: {}),
|
|
736
844
|
};
|
|
737
845
|
}
|
|
738
846
|
clearPendingPlan(applicationId) {
|
|
739
|
-
this.
|
|
740
|
-
this.pendingDecisions.delete(applicationId);
|
|
847
|
+
this.appState.clearPending(applicationId);
|
|
741
848
|
}
|
|
742
849
|
clearPendingDecision(applicationId) {
|
|
743
|
-
this.
|
|
744
|
-
this.pendingDecisions.delete(applicationId);
|
|
745
|
-
}
|
|
746
|
-
markBuildMode(applicationId) {
|
|
747
|
-
this.buildModeApplications.add(applicationId);
|
|
748
|
-
}
|
|
749
|
-
buildMode(applicationId) {
|
|
750
|
-
return this.buildModeApplications.has(applicationId);
|
|
850
|
+
this.appState.clearPending(applicationId);
|
|
751
851
|
}
|
|
752
852
|
isHeld(applicationId) {
|
|
753
853
|
const entry = this.held.get(applicationId);
|
|
754
854
|
return Boolean(entry && isSocketOpen(entry.ws));
|
|
755
855
|
}
|
|
756
|
-
|
|
856
|
+
heldContext(applicationId) {
|
|
857
|
+
const entry = this.held.get(applicationId);
|
|
858
|
+
return entry && isSocketOpen(entry.ws)
|
|
859
|
+
? { ...(entry.branchName ? { branch: entry.branchName } : {}) }
|
|
860
|
+
: undefined;
|
|
861
|
+
}
|
|
862
|
+
libraryBrowserContext(applicationId, requestedBranch) {
|
|
757
863
|
const held = this.held.get(applicationId);
|
|
758
|
-
if (!held) {
|
|
864
|
+
if (!held || requestedBranch !== held.branchName) {
|
|
759
865
|
return undefined;
|
|
760
866
|
}
|
|
761
867
|
return this.buildHeldBrowserContext(held);
|
|
@@ -768,15 +874,18 @@ export class WebSocketSessionPeer {
|
|
|
768
874
|
if (!input.userId.trim()) {
|
|
769
875
|
throw new Error("Live-edit hold failed: userId is required to identify who the session is held for");
|
|
770
876
|
}
|
|
877
|
+
const expectedTurn = this.turns.get(input.applicationId);
|
|
878
|
+
this.assertReadOnlyHoldCurrent(input.applicationId, expectedTurn);
|
|
771
879
|
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
772
880
|
const deadline = Date.now() + timeoutMs;
|
|
773
881
|
const held = await this.ensureHeld({
|
|
774
882
|
applicationId: input.applicationId,
|
|
775
883
|
branchName: input.branch,
|
|
884
|
+
getLiveEditStatus: input.getLiveEditStatus,
|
|
776
885
|
keepalive: input.keepalive,
|
|
777
886
|
liveEdit: liveEditOf(input),
|
|
778
887
|
userId: input.userId,
|
|
779
|
-
}, deadline);
|
|
888
|
+
}, deadline, expectedTurn);
|
|
780
889
|
if (input.apiKey) {
|
|
781
890
|
held.apiKey = input.apiKey;
|
|
782
891
|
}
|
|
@@ -788,14 +897,25 @@ export class WebSocketSessionPeer {
|
|
|
788
897
|
await this.resolveHeldOrchestrator(held);
|
|
789
898
|
}
|
|
790
899
|
release(applicationId) {
|
|
791
|
-
this.buildModeApplications.delete(applicationId);
|
|
792
900
|
const entry = this.held.get(applicationId);
|
|
793
901
|
if (!entry) {
|
|
794
902
|
return;
|
|
795
903
|
}
|
|
904
|
+
const turn = this.turns.get(applicationId);
|
|
905
|
+
if (turn?.held === entry && !turn.finished) {
|
|
906
|
+
this.cancelTurn(turn, "This turn was cancelled because its live-edit session was released.");
|
|
907
|
+
}
|
|
796
908
|
this.drop(entry);
|
|
797
909
|
}
|
|
798
910
|
closeAll() {
|
|
911
|
+
for (const turn of this.turns.values()) {
|
|
912
|
+
if (!turn.finished) {
|
|
913
|
+
this.cancelTurn(turn, "This turn was cancelled because the gateway shut down.");
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
for (const applicationId of [...this.observedTurns.keys()]) {
|
|
917
|
+
this.forgetObservedTurn(applicationId);
|
|
918
|
+
}
|
|
799
919
|
for (const applicationId of [...this.held.keys()]) {
|
|
800
920
|
this.release(applicationId);
|
|
801
921
|
}
|
|
@@ -812,17 +932,19 @@ export class WebSocketSessionPeer {
|
|
|
812
932
|
if (started.status === "auth_required") {
|
|
813
933
|
return started;
|
|
814
934
|
}
|
|
815
|
-
const turnTimeoutMs = this.options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
|
|
816
|
-
const readyWaitMs = this.options.readyWaitMs ?? DEFAULT_READY_WAIT_MS;
|
|
817
|
-
// The turn's own budget already bounds this; the extra slack covers the
|
|
818
|
-
// checkpoint that follows Clark's terminal state.
|
|
819
|
-
const deadline = Date.now() + turnTimeoutMs + readyWaitMs + 1_000;
|
|
820
935
|
for (;;) {
|
|
936
|
+
if (input.signal?.aborted) {
|
|
937
|
+
throw new Error("Superblocks turn wait aborted");
|
|
938
|
+
}
|
|
821
939
|
const update = await this.awaitTurn({
|
|
822
940
|
applicationId: input.applicationId,
|
|
941
|
+
branch: input.branch,
|
|
823
942
|
signal: input.signal,
|
|
824
|
-
waitMs:
|
|
943
|
+
waitMs: this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
825
944
|
});
|
|
945
|
+
if (input.signal?.aborted) {
|
|
946
|
+
throw new Error("Superblocks turn wait aborted");
|
|
947
|
+
}
|
|
826
948
|
if (update.status === "finished") {
|
|
827
949
|
const { activity: _activity, elapsedMs: _elapsedMs, status: _status, ...finished } = update;
|
|
828
950
|
return {
|
|
@@ -833,11 +955,26 @@ export class WebSocketSessionPeer {
|
|
|
833
955
|
status: "accepted",
|
|
834
956
|
};
|
|
835
957
|
}
|
|
836
|
-
if (update.status === "unknown"
|
|
958
|
+
if (update.status === "unknown") {
|
|
837
959
|
throw new Error("Prompt forward failed: the turn ended without reporting an outcome");
|
|
838
960
|
}
|
|
839
961
|
}
|
|
840
962
|
}
|
|
963
|
+
claimPromptLane(applicationId) {
|
|
964
|
+
const ready = this.promptStarts.get(applicationId) ?? Promise.resolve();
|
|
965
|
+
let release;
|
|
966
|
+
const hold = new Promise((resolve) => {
|
|
967
|
+
release = resolve;
|
|
968
|
+
});
|
|
969
|
+
const current = ready.then(() => hold);
|
|
970
|
+
this.promptStarts.set(applicationId, current);
|
|
971
|
+
void current.then(() => {
|
|
972
|
+
if (this.promptStarts.get(applicationId) === current) {
|
|
973
|
+
this.promptStarts.delete(applicationId);
|
|
974
|
+
}
|
|
975
|
+
});
|
|
976
|
+
return { ready, release: () => void ready.then(release) };
|
|
977
|
+
}
|
|
841
978
|
async startPrompt(input) {
|
|
842
979
|
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
843
980
|
const settleMs = this.options.settleMs ?? DEFAULT_SETTLE_MS;
|
|
@@ -846,119 +983,188 @@ export class WebSocketSessionPeer {
|
|
|
846
983
|
// gateway never had one. Say which it is, without quoting the credential.
|
|
847
984
|
const segments = String(input.sessionToken ?? "").split(".").length;
|
|
848
985
|
if (segments !== 3) {
|
|
849
|
-
throw new
|
|
986
|
+
throw new PromptNotSentError(`Prompt forward failed: the live-edit credential is not a JWT (${segments} segment(s)); the session will reject it`);
|
|
850
987
|
}
|
|
851
988
|
if (!input.userId?.trim()) {
|
|
852
|
-
throw new
|
|
989
|
+
throw new PromptNotSentError("Prompt forward failed: userId is required to identify who the turn is running for");
|
|
853
990
|
}
|
|
854
991
|
const userId = input.userId;
|
|
855
|
-
// Connecting and waiting for the acknowledgement share one budget. Giving
|
|
856
|
-
// each the full timeout let a slow sandbox spend twice it, and the caller
|
|
857
|
-
// that is waiting on the tool call gives up long before that.
|
|
858
992
|
const deadline = Date.now() + timeoutMs;
|
|
859
|
-
const
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
keepalive: input.keepalive,
|
|
863
|
-
liveEdit: liveEditOf(input),
|
|
864
|
-
...(input.resolveLiveEdit
|
|
865
|
-
? { resolveLiveEdit: input.resolveLiveEdit }
|
|
866
|
-
: {}),
|
|
867
|
-
userId,
|
|
868
|
-
}, deadline);
|
|
869
|
-
if (input.apiKey) {
|
|
870
|
-
held.apiKey = input.apiKey;
|
|
871
|
-
}
|
|
872
|
-
if (input.organizationId) {
|
|
873
|
-
held.organizationId = input.organizationId;
|
|
874
|
-
}
|
|
875
|
-
held.sessionToken = input.sessionToken;
|
|
876
|
-
await this.resolveHeldProfileKey(held);
|
|
877
|
-
await this.resolveHeldOrchestrator(held);
|
|
878
|
-
// A turn already running for this app is over as far as callers are
|
|
879
|
-
// concerned: its outcome belongs to the prompt being replaced.
|
|
880
|
-
this.supersedeTurn(input.applicationId);
|
|
881
|
-
this.clearPendingDecision(input.applicationId);
|
|
882
|
-
const turn = {
|
|
883
|
-
abort: new AbortController(),
|
|
884
|
-
activity: [],
|
|
885
|
-
collector: undefined,
|
|
886
|
-
delivered: 0,
|
|
887
|
-
heardAtLastReconnect: 0,
|
|
888
|
-
held,
|
|
889
|
-
listener: () => undefined,
|
|
890
|
-
normalized: undefined,
|
|
891
|
-
recentRuns: [],
|
|
892
|
-
reconnects: 0,
|
|
893
|
-
...(input.attachments?.length ? { retryAction: "retry_import" } : {}),
|
|
894
|
-
startedAt: Date.now(),
|
|
895
|
-
turnId: randomUUID(),
|
|
896
|
-
unsubscribe: () => undefined,
|
|
897
|
-
waiters: new Set(),
|
|
898
|
-
};
|
|
899
|
-
const report = (message) => {
|
|
900
|
-
this.note(turn, message);
|
|
901
|
-
input.onProgress?.(message);
|
|
902
|
-
};
|
|
903
|
-
const collector = new TurnCollector(report, {
|
|
904
|
-
initialSyncStatus: held.syncStatus,
|
|
905
|
-
settleMs,
|
|
906
|
-
});
|
|
907
|
-
turn.collector = collector;
|
|
908
|
-
const normalized = new NormalizedTurnCollector({
|
|
909
|
-
collector,
|
|
910
|
-
onAuthRequired: (reason) => report(`Superblocks needs an interactive authorization (${reason})`),
|
|
911
|
-
onSessionStatus: (status) => {
|
|
912
|
-
if (status === "terminated") {
|
|
913
|
-
this.abandonTurnOnSessionTerminated(input.applicationId, turn);
|
|
914
|
-
}
|
|
915
|
-
},
|
|
916
|
-
turnId: turn.turnId,
|
|
917
|
-
});
|
|
918
|
-
turn.normalized = normalized;
|
|
919
|
-
// Both transports go through the normalized collector rather than into
|
|
920
|
-
// the collector directly: during the transition Clark's activity arrives
|
|
921
|
-
// on the socket and relayed through SABS, and the turn must not count the
|
|
922
|
-
// same message twice because it heard it twice.
|
|
923
|
-
const listener = (event) => normalized.fromSocket(event);
|
|
924
|
-
turn.listener = listener;
|
|
925
|
-
held.listeners.add(listener);
|
|
926
|
-
// The session the socket actually landed on, which is not the one the
|
|
927
|
-
// prompt named if the live edit had moved. The held record is the only
|
|
928
|
-
// authority: it was seeded from this prompt's pair and has since been
|
|
929
|
-
// replaced by anything the connect learned, so falling back to the
|
|
930
|
-
// prompt's id could only reinstate one the socket moved away from.
|
|
931
|
-
turn.unsubscribe = this.subscribeToPushes(held.liveEdit.id, normalized);
|
|
993
|
+
const revision = input.revision ?? this.appState.markTurnStarted();
|
|
994
|
+
const lane = this.claimPromptLane(input.applicationId);
|
|
995
|
+
let held;
|
|
932
996
|
try {
|
|
933
|
-
|
|
997
|
+
await beforePromptDeadline(lane.ready, deadline, input.signal);
|
|
998
|
+
try {
|
|
999
|
+
const requestedLiveEdit = liveEditOf(input);
|
|
1000
|
+
const current = this.held.get(input.applicationId);
|
|
1001
|
+
const liveEdit = current?.userId === userId &&
|
|
1002
|
+
current.movedFrom === requestedLiveEdit.url &&
|
|
1003
|
+
isSocketOpen(current.ws)
|
|
1004
|
+
? current.liveEdit
|
|
1005
|
+
: requestedLiveEdit;
|
|
1006
|
+
const heldInput = {
|
|
1007
|
+
applicationId: input.applicationId,
|
|
1008
|
+
branchName: input.branch,
|
|
1009
|
+
getLiveEditStatus: input.getLiveEditStatus,
|
|
1010
|
+
keepalive: input.keepalive,
|
|
1011
|
+
liveEdit,
|
|
1012
|
+
...(input.resolveLiveEdit
|
|
1013
|
+
? { resolveLiveEdit: input.resolveLiveEdit }
|
|
1014
|
+
: {}),
|
|
1015
|
+
userId,
|
|
1016
|
+
};
|
|
1017
|
+
const opening = this.connect(heldInput, deadline, false);
|
|
1018
|
+
try {
|
|
1019
|
+
held = await beforePromptDeadline(opening, deadline, input.signal);
|
|
1020
|
+
if (liveEdit !== requestedLiveEdit) {
|
|
1021
|
+
held.movedFrom = requestedLiveEdit.url;
|
|
1022
|
+
}
|
|
1023
|
+
if (current?.userId === userId &&
|
|
1024
|
+
current.liveEdit.url === held.liveEdit.url) {
|
|
1025
|
+
held.syncStatus = current.syncStatus;
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
catch (error) {
|
|
1029
|
+
void opening.then((stale) => {
|
|
1030
|
+
if (stale.provisional && !stale.dropping) {
|
|
1031
|
+
this.drop(stale);
|
|
1032
|
+
}
|
|
1033
|
+
}, () => undefined);
|
|
1034
|
+
throw error;
|
|
1035
|
+
}
|
|
1036
|
+
if (input.apiKey) {
|
|
1037
|
+
held.apiKey = input.apiKey;
|
|
1038
|
+
}
|
|
1039
|
+
if (input.organizationId) {
|
|
1040
|
+
held.organizationId = input.organizationId;
|
|
1041
|
+
}
|
|
1042
|
+
held.sessionToken = input.sessionToken;
|
|
1043
|
+
await beforePromptDeadline(this.resolveHeldProfileKey(held), deadline, input.signal);
|
|
1044
|
+
await beforePromptDeadline(this.resolveHeldOrchestrator(held), deadline, input.signal);
|
|
1045
|
+
}
|
|
1046
|
+
catch (error) {
|
|
1047
|
+
throw error instanceof PromptNotSentError
|
|
1048
|
+
? error
|
|
1049
|
+
: new PromptNotSentError(describeSocketError(error), error);
|
|
1050
|
+
}
|
|
1051
|
+
let integrations = [];
|
|
1052
|
+
const editorContext = this.editorContextFor(held);
|
|
1053
|
+
if (editorContext) {
|
|
1054
|
+
try {
|
|
1055
|
+
integrations = await beforePromptDeadline(loadIntegrationsForPrompt(editorContext), deadline, input.signal);
|
|
1056
|
+
}
|
|
1057
|
+
catch (error) {
|
|
1058
|
+
if (error instanceof PromptNotSentError) {
|
|
1059
|
+
throw error;
|
|
1060
|
+
}
|
|
1061
|
+
console.warn("gateway failed to prefill prompt integrations", {
|
|
1062
|
+
applicationId: input.applicationId,
|
|
1063
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
await beforePromptDeadline(Promise.resolve(), deadline, input.signal);
|
|
1068
|
+
if (input.pendingStartRequired &&
|
|
1069
|
+
(input.organizationId === undefined ||
|
|
1070
|
+
input.idempotencyKey === undefined ||
|
|
1071
|
+
input.pendingStartToken === undefined ||
|
|
1072
|
+
!this.appState.pendingStart({
|
|
1073
|
+
applicationId: input.applicationId,
|
|
1074
|
+
...(input.branch === undefined ? {} : { branch: input.branch }),
|
|
1075
|
+
organizationId: input.organizationId,
|
|
1076
|
+
userId,
|
|
1077
|
+
}, input.idempotencyKey, input.pendingStartToken))) {
|
|
1078
|
+
throw new PromptNotSentError("A newer start attempt replaced this one before its prompt was sent");
|
|
1079
|
+
}
|
|
1080
|
+
const turn = {
|
|
1081
|
+
abort: new AbortController(),
|
|
1082
|
+
activity: [],
|
|
1083
|
+
...(held.branchName === undefined ? {} : { branch: held.branchName }),
|
|
1084
|
+
collector: undefined,
|
|
1085
|
+
delivered: 0,
|
|
1086
|
+
heardAtLastReconnect: 0,
|
|
1087
|
+
held,
|
|
1088
|
+
...(input.idempotencyKey === undefined
|
|
1089
|
+
? {}
|
|
1090
|
+
: { idempotencyKey: input.idempotencyKey }),
|
|
1091
|
+
listener: () => undefined,
|
|
1092
|
+
...(input.pendingStartToken === undefined
|
|
1093
|
+
? {}
|
|
1094
|
+
: { pendingStartToken: input.pendingStartToken }),
|
|
1095
|
+
recentRuns: [],
|
|
1096
|
+
reconnects: 0,
|
|
1097
|
+
revision,
|
|
1098
|
+
...(input.attachments?.length ? { retryAction: "retry_build" } : {}),
|
|
1099
|
+
startedAt: Date.now(),
|
|
1100
|
+
turnId: randomUUID(),
|
|
1101
|
+
waiters: new Set(),
|
|
1102
|
+
};
|
|
1103
|
+
const report = (message) => {
|
|
1104
|
+
this.note(turn, message);
|
|
1105
|
+
input.onProgress?.(message);
|
|
1106
|
+
};
|
|
1107
|
+
const collector = new TurnCollector(report, {
|
|
1108
|
+
initialSyncStatus: held.syncStatus,
|
|
1109
|
+
settleMs,
|
|
1110
|
+
});
|
|
1111
|
+
const hasDefaultStructure = this.editorStructureByScope.get(editorStructureScopeKey(input.applicationId, held.branchName));
|
|
1112
|
+
if (hasDefaultStructure) {
|
|
1113
|
+
collector.handle({
|
|
1114
|
+
hasDefaultStructure,
|
|
1115
|
+
kind: "editor_structure",
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
turn.collector = collector;
|
|
1119
|
+
const bufferedEvents = [];
|
|
1120
|
+
let accepted = false;
|
|
1121
|
+
const listener = (event) => {
|
|
1122
|
+
if (accepted) {
|
|
1123
|
+
collector.handle(event);
|
|
1124
|
+
}
|
|
1125
|
+
else {
|
|
1126
|
+
bufferedEvents.push(event);
|
|
1127
|
+
}
|
|
1128
|
+
};
|
|
1129
|
+
turn.listener = listener;
|
|
1130
|
+
held.pendingTurn = turn;
|
|
1131
|
+
held.listeners.add(listener);
|
|
1132
|
+
if (held.provisional) {
|
|
1133
|
+
held.heartbeat.start();
|
|
1134
|
+
}
|
|
1135
|
+
return await this.sendPrompt({
|
|
1136
|
+
acceptBufferedEvents: (generationId) => {
|
|
1137
|
+
collector.nameGeneration(generationId);
|
|
1138
|
+
accepted = true;
|
|
1139
|
+
for (const event of bufferedEvents) {
|
|
1140
|
+
collector.handle(event);
|
|
1141
|
+
}
|
|
1142
|
+
bufferedEvents.length = 0;
|
|
1143
|
+
},
|
|
1144
|
+
held,
|
|
1145
|
+
input,
|
|
1146
|
+
integrations,
|
|
1147
|
+
provisional: Boolean(held.provisional),
|
|
1148
|
+
turn,
|
|
1149
|
+
});
|
|
934
1150
|
}
|
|
935
1151
|
catch (error) {
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
1152
|
+
if (held?.provisional && !held.dropping) {
|
|
1153
|
+
this.drop(held);
|
|
1154
|
+
}
|
|
939
1155
|
throw error;
|
|
940
1156
|
}
|
|
1157
|
+
finally {
|
|
1158
|
+
lane.release();
|
|
1159
|
+
}
|
|
941
1160
|
}
|
|
942
1161
|
/**
|
|
943
1162
|
* Sends the prompt on a session already holding a subscribed turn, and hands
|
|
944
1163
|
* the turn to {@link runTurn} once the session confirms it took it on.
|
|
945
1164
|
*/
|
|
946
1165
|
async sendPrompt(args) {
|
|
947
|
-
const { held, input,
|
|
1166
|
+
const { held, input, integrations, provisional, turn } = args;
|
|
948
1167
|
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
949
|
-
let integrations = [];
|
|
950
|
-
const editorContext = this.editorContextFor(held);
|
|
951
|
-
if (editorContext) {
|
|
952
|
-
try {
|
|
953
|
-
integrations = await loadIntegrationsForPrompt(editorContext);
|
|
954
|
-
}
|
|
955
|
-
catch (error) {
|
|
956
|
-
console.warn("gateway failed to prefill prompt integrations", {
|
|
957
|
-
applicationId: input.applicationId,
|
|
958
|
-
message: error instanceof Error ? error.message : String(error),
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
1168
|
const browserContext = this.browserContextFor(held);
|
|
963
1169
|
let acknowledgment;
|
|
964
1170
|
try {
|
|
@@ -966,7 +1172,7 @@ export class WebSocketSessionPeer {
|
|
|
966
1172
|
// as the RPC payload's `jwt` and once as the ISocket Authorization
|
|
967
1173
|
// header. The session verifies the header; the AI service re-reads the
|
|
968
1174
|
// payload. Dropping either breaks a different hop.
|
|
969
|
-
acknowledgment = await held.
|
|
1175
|
+
acknowledgment = await editorSocket(held, input.sessionToken).aiGenerateWithQueue({
|
|
970
1176
|
...(browserContext ? { browserContext } : {}),
|
|
971
1177
|
jwt: input.sessionToken,
|
|
972
1178
|
llmConfig: {
|
|
@@ -981,12 +1187,17 @@ export class WebSocketSessionPeer {
|
|
|
981
1187
|
thinkingEnabled: false,
|
|
982
1188
|
unifiedProviderEnabled: true,
|
|
983
1189
|
},
|
|
984
|
-
mode: input.mode ?? "PLAN",
|
|
1190
|
+
mode: (input.mode ?? "PLAN"),
|
|
985
1191
|
prompt: input.prompt,
|
|
986
1192
|
promptContext: {
|
|
987
1193
|
attachments: input.attachments ?? [],
|
|
988
1194
|
entities: [],
|
|
989
|
-
|
|
1195
|
+
// Sent without `metadata` on purpose: it is the expensive
|
|
1196
|
+
// per-integration detail, and Clark asks for it per integration
|
|
1197
|
+
// through `aiGetIntegration`. The shared request type does not
|
|
1198
|
+
// model that yet, so the narrowing is stated here rather than
|
|
1199
|
+
// widening every field of the payload.
|
|
1200
|
+
integrations: integrations,
|
|
990
1201
|
},
|
|
991
1202
|
...(input.planContext
|
|
992
1203
|
? {
|
|
@@ -1006,24 +1217,48 @@ export class WebSocketSessionPeer {
|
|
|
1006
1217
|
...(input.responseMetadata
|
|
1007
1218
|
? { responseMetadata: input.responseMetadata }
|
|
1008
1219
|
: {}),
|
|
1009
|
-
...(input.importSource ? { importSource: input.importSource } : {}),
|
|
1010
1220
|
queueMode: "interrupt",
|
|
1011
1221
|
requestId: turn.turnId,
|
|
1012
1222
|
userPreferences: { suggestNextSteps: false },
|
|
1013
|
-
}
|
|
1223
|
+
});
|
|
1224
|
+
this.assertHeadlessTurnBound({ acknowledgment, turn });
|
|
1014
1225
|
}
|
|
1015
1226
|
catch (error) {
|
|
1016
|
-
|
|
1227
|
+
this.discardUnstartedTurn(turn);
|
|
1017
1228
|
const rejection = error;
|
|
1018
1229
|
if (isAuthRejection(rejection)) {
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1230
|
+
if (provisional) {
|
|
1231
|
+
this.drop(held);
|
|
1232
|
+
}
|
|
1233
|
+
else {
|
|
1234
|
+
this.armIdle(held);
|
|
1235
|
+
this.armKeepalive(held);
|
|
1236
|
+
}
|
|
1022
1237
|
return {
|
|
1023
1238
|
message: "Interactive auth required for datasource access",
|
|
1024
1239
|
status: "auth_required",
|
|
1025
1240
|
};
|
|
1026
1241
|
}
|
|
1242
|
+
if (isRemoteRejection(rejection)) {
|
|
1243
|
+
if (provisional) {
|
|
1244
|
+
this.drop(held);
|
|
1245
|
+
}
|
|
1246
|
+
else {
|
|
1247
|
+
this.armIdle(held);
|
|
1248
|
+
this.armKeepalive(held);
|
|
1249
|
+
}
|
|
1250
|
+
throw new PromptNotSentError(`Prompt forward failed: ${rejection.message ?? "unknown socket error"}`, error);
|
|
1251
|
+
}
|
|
1252
|
+
else {
|
|
1253
|
+
const priorHeld = this.held.get(input.applicationId);
|
|
1254
|
+
this.supersedeTurn(input.applicationId);
|
|
1255
|
+
if (priorHeld) {
|
|
1256
|
+
this.drop(priorHeld);
|
|
1257
|
+
}
|
|
1258
|
+
if (held !== priorHeld) {
|
|
1259
|
+
this.drop(held);
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1027
1262
|
if (isTimeout(rejection)) {
|
|
1028
1263
|
throw new Error(`Prompt forward timed out: the live-edit session accepted the connection but did not acknowledge the prompt within ${timeoutMs}ms (${rejection.message}). The sandbox is usually still waking up when this happens, so retrying the same prompt normally works.`);
|
|
1029
1264
|
}
|
|
@@ -1031,21 +1266,37 @@ export class WebSocketSessionPeer {
|
|
|
1031
1266
|
? error
|
|
1032
1267
|
: new Error(`Prompt forward failed: ${rejection.message ?? "unknown socket error"}`);
|
|
1033
1268
|
}
|
|
1034
|
-
this.
|
|
1035
|
-
|
|
1036
|
-
held,
|
|
1037
|
-
listener,
|
|
1038
|
-
turn,
|
|
1039
|
-
});
|
|
1269
|
+
turn.revision = this.appState.markTurnStarted();
|
|
1270
|
+
args.acceptBufferedEvents(acknowledgment?.generationId);
|
|
1040
1271
|
input.onProgress?.("Superblocks accepted the prompt");
|
|
1041
1272
|
// Only now, with the prompt accepted and a live turn about to go in the
|
|
1042
|
-
// map, is a recorded dead start out of date. Clearing it
|
|
1043
|
-
// claimed instead dropped it before Superblocks had taken
|
|
1044
|
-
// retry that came back needing interactive auth - a return,
|
|
1045
|
-
// nothing re-recorded it - left the app with no turn at all
|
|
1046
|
-
// went back to previewing the starter template.
|
|
1047
|
-
this.
|
|
1273
|
+
// map, is a recorded dead start for this branch out of date. Clearing it
|
|
1274
|
+
// when the slot was claimed instead dropped it before Superblocks had taken
|
|
1275
|
+
// anything, so a retry that came back needing interactive auth - a return,
|
|
1276
|
+
// not a throw, so nothing re-recorded it - left the app with no turn at all
|
|
1277
|
+
// and `get_app` went back to previewing the starter template.
|
|
1278
|
+
this.appState.clearFailedStarts(input.applicationId, turn.branch, turn.revision);
|
|
1279
|
+
this.appState.clearPendingStartsThrough(input.applicationId, turn.revision);
|
|
1280
|
+
// The pending decision goes for the same reason and at the same moment.
|
|
1281
|
+
// Clearing it when the slot was claimed spent the plan on a prompt
|
|
1282
|
+
// Superblocks had not taken: an approve that came back needing interactive
|
|
1283
|
+
// auth left neither a pending plan nor the BUILD latch - which is only set
|
|
1284
|
+
// once a start is accepted - so the retry proposed a plan again instead of
|
|
1285
|
+
// building the one the user had already approved.
|
|
1286
|
+
this.clearPendingDecision(input.applicationId);
|
|
1287
|
+
this.forgetObservedTurn(input.applicationId);
|
|
1288
|
+
const priorHeld = this.held.get(input.applicationId);
|
|
1289
|
+
this.supersedeTurn(input.applicationId);
|
|
1048
1290
|
this.turns.set(input.applicationId, turn);
|
|
1291
|
+
delete held.pendingTurn;
|
|
1292
|
+
if (provisional) {
|
|
1293
|
+
delete held.provisional;
|
|
1294
|
+
this.setHeld(held);
|
|
1295
|
+
held.heartbeat.start();
|
|
1296
|
+
if (priorHeld && priorHeld !== held) {
|
|
1297
|
+
this.drop(priorHeld);
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1049
1300
|
// Deliberately not awaited: the turn outlives the call that started it so a
|
|
1050
1301
|
// client with no progress channel can follow it through awaitTurn.
|
|
1051
1302
|
void this.runTurn({
|
|
@@ -1063,19 +1314,265 @@ export class WebSocketSessionPeer {
|
|
|
1063
1314
|
status: "accepted",
|
|
1064
1315
|
};
|
|
1065
1316
|
}
|
|
1317
|
+
latestTurn(applicationId) {
|
|
1318
|
+
return (this.observedTurns.get(applicationId) ?? this.turns.get(applicationId));
|
|
1319
|
+
}
|
|
1320
|
+
observeEditorEvent(held, event) {
|
|
1321
|
+
if (held.dropping || this.held.get(held.applicationId) !== held) {
|
|
1322
|
+
return;
|
|
1323
|
+
}
|
|
1324
|
+
let observed = this.observedTurns.get(held.applicationId);
|
|
1325
|
+
const message = event.kind === "message" ? event.message : undefined;
|
|
1326
|
+
const userContent = message?.role === "user" &&
|
|
1327
|
+
typeof message.content === "string" &&
|
|
1328
|
+
message.content.trim()
|
|
1329
|
+
? message.content.trim()
|
|
1330
|
+
: undefined;
|
|
1331
|
+
const userMessageId = userContent && typeof message?.id === "string" ? message.id : undefined;
|
|
1332
|
+
if (userMessageId && observed?.userMessageIds.has(userMessageId)) {
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
const activeGeneration = event.kind === "generation_state" &&
|
|
1336
|
+
(event.state === "debugging" ||
|
|
1337
|
+
event.state === "generating" ||
|
|
1338
|
+
event.state === "summarizing");
|
|
1339
|
+
if (userContent || activeGeneration) {
|
|
1340
|
+
this.clearPendingDecision(held.applicationId);
|
|
1341
|
+
}
|
|
1342
|
+
const owned = this.turns.get(held.applicationId);
|
|
1343
|
+
if (owned) {
|
|
1344
|
+
if (!owned.finished && !userContent && !activeGeneration) {
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
this.supersedeTurn(held.applicationId);
|
|
1348
|
+
}
|
|
1349
|
+
const distinctGeneration = activeGeneration &&
|
|
1350
|
+
event.kind === "generation_state" &&
|
|
1351
|
+
Boolean(event.generationId &&
|
|
1352
|
+
observed?.generationId &&
|
|
1353
|
+
event.generationId !== observed.generationId);
|
|
1354
|
+
if (observed &&
|
|
1355
|
+
(observed.held !== held ||
|
|
1356
|
+
distinctGeneration ||
|
|
1357
|
+
(userContent && observed.finished))) {
|
|
1358
|
+
this.forgetObservedTurn(held.applicationId);
|
|
1359
|
+
observed = undefined;
|
|
1360
|
+
this.clearPendingDecision(held.applicationId);
|
|
1361
|
+
}
|
|
1362
|
+
if (observed?.finished) {
|
|
1363
|
+
if (!activeGeneration) {
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
if (event.kind === "generation_state" &&
|
|
1367
|
+
event.generationId &&
|
|
1368
|
+
event.generationId === observed.generationId &&
|
|
1369
|
+
observed.finished.generationStatus !== "needs_decision") {
|
|
1370
|
+
return;
|
|
1371
|
+
}
|
|
1372
|
+
observed = undefined;
|
|
1373
|
+
this.clearPendingDecision(held.applicationId);
|
|
1374
|
+
}
|
|
1375
|
+
if (!observed) {
|
|
1376
|
+
held.beatLoss = undefined;
|
|
1377
|
+
this.clearIdleTimer(held);
|
|
1378
|
+
const activity = [];
|
|
1379
|
+
const next = {
|
|
1380
|
+
abort: new AbortController(),
|
|
1381
|
+
activity,
|
|
1382
|
+
...(held.branchName === undefined ? {} : { branch: held.branchName }),
|
|
1383
|
+
collector: new TurnCollector((message) => this.note(next, message), {
|
|
1384
|
+
...(event.kind === "generation_state" && event.generationId
|
|
1385
|
+
? { generationId: event.generationId }
|
|
1386
|
+
: {}),
|
|
1387
|
+
settleMs: this.options.settleMs ?? DEFAULT_SETTLE_MS,
|
|
1388
|
+
}),
|
|
1389
|
+
delivered: 0,
|
|
1390
|
+
...(event.kind === "generation_state" && event.generationId
|
|
1391
|
+
? { generationId: event.generationId }
|
|
1392
|
+
: {}),
|
|
1393
|
+
held,
|
|
1394
|
+
recentRuns: [],
|
|
1395
|
+
revision: this.appState.markTurnStarted(),
|
|
1396
|
+
startedAt: Date.now(),
|
|
1397
|
+
userMessageIds: new Set(),
|
|
1398
|
+
waiters: new Set(),
|
|
1399
|
+
};
|
|
1400
|
+
observed = next;
|
|
1401
|
+
this.observedTurns.set(held.applicationId, observed);
|
|
1402
|
+
void this.finishObservedTurn(held.applicationId, observed);
|
|
1403
|
+
}
|
|
1404
|
+
if (userMessageId) {
|
|
1405
|
+
observed.userMessageIds.add(userMessageId);
|
|
1406
|
+
}
|
|
1407
|
+
if (event.kind === "generation_state" &&
|
|
1408
|
+
event.generationId &&
|
|
1409
|
+
!observed.generationId) {
|
|
1410
|
+
observed.generationId = event.generationId;
|
|
1411
|
+
observed.collector.nameGeneration(event.generationId);
|
|
1412
|
+
}
|
|
1413
|
+
if (userContent) {
|
|
1414
|
+
this.note(observed, `User in editor: ${userContent}`);
|
|
1415
|
+
}
|
|
1416
|
+
observed.collector.handle(event);
|
|
1417
|
+
}
|
|
1418
|
+
forgetObservedTurn(applicationId) {
|
|
1419
|
+
const observed = this.observedTurns.get(applicationId);
|
|
1420
|
+
if (!observed) {
|
|
1421
|
+
return;
|
|
1422
|
+
}
|
|
1423
|
+
this.observedTurns.delete(applicationId);
|
|
1424
|
+
this.settleObservedTurn(observed, "cancelled");
|
|
1425
|
+
}
|
|
1426
|
+
async finishObservedTurn(applicationId, turn) {
|
|
1427
|
+
let generationStatus;
|
|
1428
|
+
try {
|
|
1429
|
+
generationStatus = await turn.collector.waitUntilTerminal({
|
|
1430
|
+
signal: turn.abort.signal,
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
catch {
|
|
1434
|
+
return;
|
|
1435
|
+
}
|
|
1436
|
+
if (this.observedTurns.get(applicationId) !== turn || turn.finished) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
const decision = turn.collector.decision();
|
|
1440
|
+
if (decision &&
|
|
1441
|
+
(generationStatus === "needs_decision" ||
|
|
1442
|
+
decision.kind === "missing_integrations")) {
|
|
1443
|
+
const reply = turn.collector.reply();
|
|
1444
|
+
this.appState.setPendingDecision(applicationId, {
|
|
1445
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1446
|
+
decision,
|
|
1447
|
+
reply,
|
|
1448
|
+
});
|
|
1449
|
+
if (decision.kind === "plan") {
|
|
1450
|
+
this.appState.setPendingPlan(applicationId, {
|
|
1451
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1452
|
+
...(decision.messageId ? { messageId: decision.messageId } : {}),
|
|
1453
|
+
plan: decision.plan,
|
|
1454
|
+
...(decision.title ? { title: decision.title } : {}),
|
|
1455
|
+
});
|
|
1456
|
+
}
|
|
1457
|
+
else {
|
|
1458
|
+
this.appState.clearPendingPlan(applicationId);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
turn.finished = {
|
|
1462
|
+
reply: turn.collector.reply(),
|
|
1463
|
+
generationStatus,
|
|
1464
|
+
viewStatus: turn.collector.viewableStatus(),
|
|
1465
|
+
...decisionFields(decision),
|
|
1466
|
+
};
|
|
1467
|
+
turn.collector.dispose();
|
|
1468
|
+
this.wake(turn);
|
|
1469
|
+
this.armObservedTurnRelease(applicationId, turn);
|
|
1470
|
+
}
|
|
1471
|
+
settleObservedTurn(turn, generationStatus, reply = turn.collector.reply()) {
|
|
1472
|
+
if (turn.finished) {
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
tagRetryableLiveEditDeath(turn, { generationStatus });
|
|
1476
|
+
turn.finished = {
|
|
1477
|
+
reply,
|
|
1478
|
+
generationStatus,
|
|
1479
|
+
viewStatus: turn.collector.viewableStatus(),
|
|
1480
|
+
...decisionFields(turn.collector.decision()),
|
|
1481
|
+
};
|
|
1482
|
+
turn.collector.dispose();
|
|
1483
|
+
turn.abort.abort();
|
|
1484
|
+
this.wake(turn);
|
|
1485
|
+
}
|
|
1486
|
+
armObservedTurnRelease(applicationId, turn) {
|
|
1487
|
+
if (this.observedTurns.get(applicationId) !== turn ||
|
|
1488
|
+
this.held.get(applicationId) !== turn.held ||
|
|
1489
|
+
!turn.finished) {
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
if (turn.finished.generationStatus === "needs_decision") {
|
|
1493
|
+
this.armIdle(turn.held);
|
|
1494
|
+
}
|
|
1495
|
+
else {
|
|
1496
|
+
this.armObservedPostTurnRelease(turn);
|
|
1497
|
+
}
|
|
1498
|
+
this.armKeepalive(turn.held);
|
|
1499
|
+
}
|
|
1500
|
+
armObservedPostTurnRelease(turn) {
|
|
1501
|
+
const held = turn.held;
|
|
1502
|
+
this.clearTimers(held);
|
|
1503
|
+
const delayMs = this.options.releaseAfterTurnMs ?? DEFAULT_RELEASE_AFTER_TURN_MS;
|
|
1504
|
+
held.idleTimer = setTimeout(() => {
|
|
1505
|
+
if (this.observedTurns.get(held.applicationId) === turn &&
|
|
1506
|
+
this.held.get(held.applicationId) === held) {
|
|
1507
|
+
this.drop(held, { preserveFinishedObserved: true });
|
|
1508
|
+
}
|
|
1509
|
+
}, delayMs);
|
|
1510
|
+
held.idleTimer.unref?.();
|
|
1511
|
+
}
|
|
1066
1512
|
async awaitTurn(input) {
|
|
1067
|
-
const
|
|
1068
|
-
// The
|
|
1069
|
-
|
|
1070
|
-
//
|
|
1071
|
-
//
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1513
|
+
const pollStartedAtMs = Date.now();
|
|
1514
|
+
// The deadline is the end of the poll. Answers with activity or nothing.
|
|
1515
|
+
const deadlineAtMs = pollStartedAtMs + Math.max(0, input.waitMs);
|
|
1516
|
+
// The floor is the minimum time the poll will wait to collect activity for before it answers.
|
|
1517
|
+
// If the turn is finished (ex: needs a user decision), it will answer immediately.
|
|
1518
|
+
const floorAtMs = pollStartedAtMs +
|
|
1519
|
+
Math.min(Math.max(0, input.floorMs ?? 0), Math.max(0, input.waitMs));
|
|
1520
|
+
const storedTurn = this.latestTurn(input.applicationId);
|
|
1521
|
+
// Filter out turns that are not for the requested branch.
|
|
1522
|
+
let turn = storedTurn &&
|
|
1523
|
+
input.branch !== undefined &&
|
|
1524
|
+
input.branch !== storedTurn.branch
|
|
1525
|
+
? undefined
|
|
1526
|
+
: storedTurn;
|
|
1527
|
+
// check if there is a pending turn for the requested branch.
|
|
1528
|
+
const pendingScope = input.organizationId && input.userId
|
|
1529
|
+
? {
|
|
1530
|
+
applicationId: input.applicationId,
|
|
1531
|
+
...(input.branch === undefined ? {} : { branch: input.branch }),
|
|
1532
|
+
organizationId: input.organizationId,
|
|
1533
|
+
userId: input.userId,
|
|
1534
|
+
}
|
|
1535
|
+
: undefined;
|
|
1536
|
+
const pending = pendingScope
|
|
1537
|
+
? this.appState.pendingStart(pendingScope)
|
|
1538
|
+
: undefined;
|
|
1539
|
+
if (pendingScope && pending) {
|
|
1540
|
+
const started = await this.waitForPendingTurn({
|
|
1541
|
+
afterRevision: turn?.revision ?? 0,
|
|
1542
|
+
applicationId: input.applicationId,
|
|
1543
|
+
...(pending.idempotencyKey === undefined
|
|
1544
|
+
? {}
|
|
1545
|
+
: { idempotencyKey: pending.idempotencyKey }),
|
|
1546
|
+
pendingStartToken: pending.token,
|
|
1547
|
+
scope: pendingScope,
|
|
1548
|
+
signal: input.signal,
|
|
1549
|
+
deadlineAtMs,
|
|
1550
|
+
});
|
|
1551
|
+
if (started) {
|
|
1552
|
+
turn = started;
|
|
1553
|
+
}
|
|
1554
|
+
else if (this.appState.pendingStart(pendingScope)) {
|
|
1555
|
+
return { activity: [], elapsedMs: 0, status: "running" };
|
|
1556
|
+
}
|
|
1557
|
+
else {
|
|
1558
|
+
const currentTurn = this.latestTurn(input.applicationId);
|
|
1559
|
+
turn =
|
|
1560
|
+
currentTurn &&
|
|
1561
|
+
input.branch !== undefined &&
|
|
1562
|
+
input.branch !== currentTurn.branch
|
|
1563
|
+
? undefined
|
|
1564
|
+
: currentTurn;
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
// The same recency order as turnState. A poll that cannot say who it is for
|
|
1568
|
+
// reads no failed-start record at all; those records belong to callers.
|
|
1569
|
+
const failed = input.organizationId && input.userId
|
|
1570
|
+
? this.appState.failedStart({
|
|
1075
1571
|
applicationId: input.applicationId,
|
|
1572
|
+
...(input.branch === undefined ? {} : { branch: input.branch }),
|
|
1076
1573
|
organizationId: input.organizationId,
|
|
1077
1574
|
userId: input.userId,
|
|
1078
|
-
})
|
|
1575
|
+
}, turn?.revision)
|
|
1079
1576
|
: undefined;
|
|
1080
1577
|
// A turn that never started took no time, and there is no activity to
|
|
1081
1578
|
// relay. Reporting it as finished is what stops the caller polling a session
|
|
@@ -1084,39 +1581,74 @@ export class WebSocketSessionPeer {
|
|
|
1084
1581
|
return { activity: [], elapsedMs: 0, status: "finished", ...failed };
|
|
1085
1582
|
}
|
|
1086
1583
|
if (!turn) {
|
|
1584
|
+
if (pendingScope && this.appState.pendingStart(pendingScope)) {
|
|
1585
|
+
return { activity: [], elapsedMs: 0, status: "running" };
|
|
1586
|
+
}
|
|
1087
1587
|
return { activity: [], elapsedMs: 0, status: "unknown" };
|
|
1088
1588
|
}
|
|
1089
|
-
|
|
1589
|
+
const remainingWaitMs = Math.max(0, deadlineAtMs - Date.now());
|
|
1590
|
+
if (
|
|
1591
|
+
// If the turn is finished answer immediately.
|
|
1592
|
+
!turn.finished &&
|
|
1593
|
+
!input.signal?.aborted &&
|
|
1594
|
+
(remainingWaitMs > 0 || input.waitMs === 0) &&
|
|
1595
|
+
// Even if there is activity, we will wait until the floor is closed to answer.
|
|
1596
|
+
(turn.delivered >= turn.activity.length || Date.now() < floorAtMs)) {
|
|
1090
1597
|
await new Promise((resolve) => {
|
|
1598
|
+
// Ends the wait and hands the caller whatever the turn has by now.
|
|
1091
1599
|
const settle = () => {
|
|
1092
|
-
clearTimeout(
|
|
1093
|
-
|
|
1600
|
+
clearTimeout(deadlineTimer);
|
|
1601
|
+
clearTimeout(floorTimer);
|
|
1602
|
+
turn.waiters.delete(onWake);
|
|
1094
1603
|
input.signal?.removeEventListener("abort", settle);
|
|
1095
1604
|
resolve();
|
|
1096
1605
|
};
|
|
1097
|
-
|
|
1098
|
-
|
|
1606
|
+
// Called on every wake (any activity)
|
|
1607
|
+
const onWake = () => {
|
|
1608
|
+
// Settle if the turn is finished or the floor is reached.
|
|
1609
|
+
if (turn.finished || Date.now() >= floorAtMs) {
|
|
1610
|
+
settle();
|
|
1611
|
+
}
|
|
1612
|
+
};
|
|
1613
|
+
// Settle if there is activity.
|
|
1614
|
+
// Called when the floor timer expires.
|
|
1615
|
+
const onFloor = () => {
|
|
1616
|
+
if (turn.delivered < turn.activity.length) {
|
|
1617
|
+
settle();
|
|
1618
|
+
}
|
|
1619
|
+
};
|
|
1620
|
+
const deadlineTimer = setTimeout(settle, remainingWaitMs);
|
|
1621
|
+
const floorTimer = setTimeout(onFloor, Math.max(0, floorAtMs - Date.now()));
|
|
1622
|
+
turn.waiters.add(onWake);
|
|
1099
1623
|
input.signal?.addEventListener("abort", settle, { once: true });
|
|
1624
|
+
if (input.signal?.aborted) {
|
|
1625
|
+
settle();
|
|
1626
|
+
}
|
|
1100
1627
|
});
|
|
1101
1628
|
}
|
|
1102
|
-
|
|
1629
|
+
// Now drain the turn and return the activity.
|
|
1630
|
+
const activity = input.signal?.aborted ? [] : this.drain(turn);
|
|
1103
1631
|
const elapsedMs = Date.now() - turn.startedAt;
|
|
1104
1632
|
const recentRuns = turn.recentRuns.length > 0 ? [...turn.recentRuns] : undefined;
|
|
1105
1633
|
if (turn.finished) {
|
|
1106
1634
|
return {
|
|
1107
1635
|
activity,
|
|
1636
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1108
1637
|
elapsedMs,
|
|
1109
1638
|
status: "finished",
|
|
1110
1639
|
...turn.finished,
|
|
1640
|
+
viewStatus: this.currentViewStatus(input.applicationId, turn.branch, turn.finished.viewStatus),
|
|
1111
1641
|
...(turn.retryAction ? { retryAction: turn.retryAction } : {}),
|
|
1112
1642
|
...(recentRuns ? { recentRuns } : {}),
|
|
1113
1643
|
};
|
|
1114
1644
|
}
|
|
1115
1645
|
return {
|
|
1116
1646
|
activity,
|
|
1647
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1117
1648
|
elapsedMs,
|
|
1118
1649
|
status: "running",
|
|
1119
1650
|
...(recentRuns ? { recentRuns } : {}),
|
|
1651
|
+
viewStatus: this.currentViewStatus(input.applicationId, turn.branch, turn.collector.viewableStatus()),
|
|
1120
1652
|
};
|
|
1121
1653
|
}
|
|
1122
1654
|
/**
|
|
@@ -1128,7 +1660,6 @@ export class WebSocketSessionPeer {
|
|
|
1128
1660
|
async runTurn(input) {
|
|
1129
1661
|
const turn = input.turn;
|
|
1130
1662
|
const { collector } = turn;
|
|
1131
|
-
const turnTimeoutMs = this.options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
|
|
1132
1663
|
const readyWaitMs = this.options.readyWaitMs ?? DEFAULT_READY_WAIT_MS;
|
|
1133
1664
|
const report = (message) => {
|
|
1134
1665
|
this.note(turn, message);
|
|
@@ -1137,7 +1668,7 @@ export class WebSocketSessionPeer {
|
|
|
1137
1668
|
// The session the turn opened on, kept only so its listener is detached at
|
|
1138
1669
|
// the end even if the turn moved off it in between.
|
|
1139
1670
|
const opening = turn.held;
|
|
1140
|
-
let
|
|
1671
|
+
let checkpointOutcome;
|
|
1141
1672
|
try {
|
|
1142
1673
|
if (turn.finished) {
|
|
1143
1674
|
return;
|
|
@@ -1178,12 +1709,11 @@ export class WebSocketSessionPeer {
|
|
|
1178
1709
|
}
|
|
1179
1710
|
const generationStatus = await collector.waitUntilTerminal({
|
|
1180
1711
|
signal: turn.abort.signal,
|
|
1181
|
-
timeoutMs: turnTimeoutMs,
|
|
1182
1712
|
});
|
|
1183
1713
|
if (turn.finished) {
|
|
1184
1714
|
return;
|
|
1185
1715
|
}
|
|
1186
|
-
|
|
1716
|
+
gatewayDebug("gateway clark turn states", {
|
|
1187
1717
|
applicationId: input.applicationId,
|
|
1188
1718
|
draftPending: collector.needsDraftAccept(),
|
|
1189
1719
|
generationStatus,
|
|
@@ -1199,26 +1729,41 @@ export class WebSocketSessionPeer {
|
|
|
1199
1729
|
// produced nothing is `no_changes` and never reaches here, so Clark
|
|
1200
1730
|
// answering a question in chat cannot checkpoint the bare template.
|
|
1201
1731
|
if (generationStatus === "completed") {
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1732
|
+
const lane = this.claimPromptLane(input.applicationId);
|
|
1733
|
+
await lane.ready;
|
|
1734
|
+
try {
|
|
1735
|
+
if (turn.finished || this.turns.get(input.applicationId) !== turn) {
|
|
1736
|
+
return;
|
|
1737
|
+
}
|
|
1738
|
+
// TODO(ENG-5922): `turn.held` is a dead session when the turn was kept
|
|
1739
|
+
// alive on the relay, so this accept cannot land and the caller gets a
|
|
1740
|
+
// finished turn with no commit.
|
|
1741
|
+
checkpointOutcome = await this.checkpoint({
|
|
1742
|
+
collector,
|
|
1743
|
+
held: turn.held,
|
|
1744
|
+
onProgress: report,
|
|
1745
|
+
readyWaitMs,
|
|
1746
|
+
sessionToken: input.sessionToken,
|
|
1747
|
+
turnId: turn.turnId,
|
|
1748
|
+
});
|
|
1749
|
+
}
|
|
1750
|
+
finally {
|
|
1751
|
+
lane.release();
|
|
1752
|
+
}
|
|
1214
1753
|
}
|
|
1215
|
-
if (turn.finished) {
|
|
1754
|
+
if (turn.finished || this.turns.get(input.applicationId) !== turn) {
|
|
1216
1755
|
return;
|
|
1217
1756
|
}
|
|
1757
|
+
if (generationStatus === "completed") {
|
|
1758
|
+
this.clearPendingDecision(input.applicationId);
|
|
1759
|
+
}
|
|
1218
1760
|
const decision = collector.decision();
|
|
1219
|
-
if (
|
|
1761
|
+
if (decision &&
|
|
1762
|
+
(generationStatus === "needs_decision" ||
|
|
1763
|
+
decision.kind === "missing_integrations")) {
|
|
1220
1764
|
const reply = collector.reply();
|
|
1221
|
-
this.
|
|
1765
|
+
this.appState.setPendingDecision(input.applicationId, {
|
|
1766
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1222
1767
|
decision,
|
|
1223
1768
|
...(turn.recentRuns.length > 0
|
|
1224
1769
|
? { recentRuns: [...turn.recentRuns] }
|
|
@@ -1226,20 +1771,29 @@ export class WebSocketSessionPeer {
|
|
|
1226
1771
|
reply,
|
|
1227
1772
|
});
|
|
1228
1773
|
if (decision.kind === "plan") {
|
|
1229
|
-
this.
|
|
1774
|
+
this.appState.setPendingPlan(input.applicationId, {
|
|
1775
|
+
...(turn.branch === undefined ? {} : { branch: turn.branch }),
|
|
1230
1776
|
...(decision.messageId ? { messageId: decision.messageId } : {}),
|
|
1231
1777
|
plan: decision.plan,
|
|
1232
1778
|
...(decision.title ? { title: decision.title } : {}),
|
|
1233
1779
|
});
|
|
1234
1780
|
}
|
|
1235
1781
|
else {
|
|
1236
|
-
this.
|
|
1782
|
+
this.appState.clearPendingPlan(input.applicationId);
|
|
1237
1783
|
}
|
|
1238
1784
|
}
|
|
1239
1785
|
// Disconnect / supersede already wrote the authoritative outcome; never
|
|
1240
|
-
// overwrite that with a later collector
|
|
1786
|
+
// overwrite that with a later collector terminal status.
|
|
1241
1787
|
turn.finished ??= {
|
|
1242
|
-
...(
|
|
1788
|
+
...(checkpointOutcome?.status === "saved" && checkpointOutcome.commitId
|
|
1789
|
+
? { checkpointCommitId: checkpointOutcome.commitId }
|
|
1790
|
+
: {}),
|
|
1791
|
+
...(checkpointOutcome?.status === "failed"
|
|
1792
|
+
? { checkpointFailure: checkpointOutcome.failure }
|
|
1793
|
+
: {}),
|
|
1794
|
+
...(checkpointOutcome?.status === "saved" && checkpointOutcome.warning
|
|
1795
|
+
? { checkpointWarning: checkpointOutcome.warning }
|
|
1796
|
+
: {}),
|
|
1243
1797
|
reply: collector.reply(),
|
|
1244
1798
|
generationStatus,
|
|
1245
1799
|
...(turn.recentRuns.length > 0
|
|
@@ -1268,14 +1822,21 @@ export class WebSocketSessionPeer {
|
|
|
1268
1822
|
finally {
|
|
1269
1823
|
opening.listeners.delete(turn.listener);
|
|
1270
1824
|
turn.held.listeners.delete(turn.listener);
|
|
1271
|
-
// The turn is over on both transports before the collector is disposed:
|
|
1272
|
-
// a push arriving afterwards would otherwise be handed to it.
|
|
1273
|
-
turn.unsubscribe();
|
|
1274
1825
|
collector.dispose();
|
|
1275
1826
|
// A superseded turn must not arm timers or keepalive on a held session
|
|
1276
1827
|
// that now belongs to a newer prompt.
|
|
1277
1828
|
const stillMapped = this.turns.get(input.applicationId) === turn;
|
|
1278
|
-
|
|
1829
|
+
// Nor may a turn that outlived its socket. `recoverTurn` keeps a turn
|
|
1830
|
+
// running on the SABS relay after the close took its session out of the
|
|
1831
|
+
// held map, and `turn.held` still points at that dead object. Arming a
|
|
1832
|
+
// keepalive on it leaves an interval nothing can ever clear - the release
|
|
1833
|
+
// timer no-ops precisely because the session is not current - so it goes
|
|
1834
|
+
// on calling connectLiveEdit for the life of the process.
|
|
1835
|
+
const sessionStillHeld = this.held.get(input.applicationId) === turn.held;
|
|
1836
|
+
if (!sessionStillHeld) {
|
|
1837
|
+
this.clearTimers(turn.held);
|
|
1838
|
+
}
|
|
1839
|
+
else if (stillMapped) {
|
|
1279
1840
|
try {
|
|
1280
1841
|
await turn.held.client.requestWithTracing(RELEASE_TURN_LOCK_METHOD, this.turnCorrelator(turn.turnId), {}, `Bearer ${input.sessionToken}`);
|
|
1281
1842
|
}
|
|
@@ -1300,20 +1861,6 @@ export class WebSocketSessionPeer {
|
|
|
1300
1861
|
pushWaitMs() {
|
|
1301
1862
|
return this.options.pushWaitMs ?? DEFAULT_PUSH_WAIT_MS;
|
|
1302
1863
|
}
|
|
1303
|
-
/**
|
|
1304
|
-
* Attaches a turn to the events SABS relays for its live edit, returning the
|
|
1305
|
-
* idempotent detach the turn calls when it ends.
|
|
1306
|
-
*
|
|
1307
|
-
* A turn with no live edit id, or a deployment with no push channel, gets a
|
|
1308
|
-
* detach that does nothing: the socket remains its only source, which is the
|
|
1309
|
-
* behavior from before the channel existed.
|
|
1310
|
-
*/
|
|
1311
|
-
subscribeToPushes(_liveEditId, _normalized) {
|
|
1312
|
-
// Stdio Gateway has no SABS HTTP push ingress (ENG-5797). The held
|
|
1313
|
-
// socket is the only event source; the detach is a no-op so turn
|
|
1314
|
-
// teardown stays the same shape as the parent branch.
|
|
1315
|
-
return () => undefined;
|
|
1316
|
-
}
|
|
1317
1864
|
/**
|
|
1318
1865
|
* Checks that the session took this prompt on as a turn belonging to this
|
|
1319
1866
|
* peer, and gives up on it now if it did not.
|
|
@@ -1321,8 +1868,8 @@ export class WebSocketSessionPeer {
|
|
|
1321
1868
|
* A dev server that predates the gateway role accepts the socket, echoes the
|
|
1322
1869
|
* role it was offered, and then treats the connection as a library socket: the
|
|
1323
1870
|
* prompt is acknowledged and Clark's pushes go to the browser binding
|
|
1324
|
-
* instead. The turn would
|
|
1325
|
-
*
|
|
1871
|
+
* instead. The turn would stay detached, and a rollout with one stale
|
|
1872
|
+
* sandbox in it would look like Clark hanging. Only a
|
|
1326
1873
|
* server that bound the turn can name it back, so the missing handle is the
|
|
1327
1874
|
* mismatch, reported where it is still cheap to act on.
|
|
1328
1875
|
*/
|
|
@@ -1330,9 +1877,7 @@ export class WebSocketSessionPeer {
|
|
|
1330
1877
|
if (input.acknowledgment?.headlessTurn?.turnId === input.turn.turnId) {
|
|
1331
1878
|
return;
|
|
1332
1879
|
}
|
|
1333
|
-
|
|
1334
|
-
this.release(input.held.applicationId);
|
|
1335
|
-
throw new Error(`Prompt forward failed: the live-edit session acknowledged the prompt but did not bind it to this ${GATEWAY_PROTOCOL} peer, so nothing about the turn would report back. The sandbox is running a dev server from before the ${GATEWAY_PROTOCOL} role; restart the sandbox to pick up the current one.`);
|
|
1880
|
+
throw new Error(`Prompt forward failed: the live-edit session acknowledged the prompt but did not bind it to this ${GATEWAY_PROTOCOL} peer, so nothing about the turn would report back. The sandbox is running a dev server from before the ${GATEWAY_PROTOCOL} role. Keep the MCP connection running and retry this app action shortly.`);
|
|
1336
1881
|
}
|
|
1337
1882
|
/**
|
|
1338
1883
|
* The field the session reads to decide which turn a call belongs to. Sent on
|
|
@@ -1446,8 +1991,10 @@ export class WebSocketSessionPeer {
|
|
|
1446
1991
|
};
|
|
1447
1992
|
}
|
|
1448
1993
|
this.carryOver(fresh, previous);
|
|
1994
|
+
fresh.pendingTurn = turn;
|
|
1449
1995
|
fresh.listeners.add(turn.listener);
|
|
1450
1996
|
previous.listeners.delete(turn.listener);
|
|
1997
|
+
fresh.heartbeat.start();
|
|
1451
1998
|
const claim = fresh.sessionToken === undefined
|
|
1452
1999
|
? "refused"
|
|
1453
2000
|
: await this.proveIdentity({
|
|
@@ -1456,18 +2003,23 @@ export class WebSocketSessionPeer {
|
|
|
1456
2003
|
turnId: turn.turnId,
|
|
1457
2004
|
});
|
|
1458
2005
|
// Opening and claiming take seconds, and the turn can settle inside them:
|
|
1459
|
-
// a relayed terminal event,
|
|
2006
|
+
// a relayed terminal event, the turn being cancelled, or a newer prompt
|
|
1460
2007
|
// taking the app. runTurn arms the post-turn timers once, against the
|
|
1461
2008
|
// session the turn held when it ended, so a replacement committed after
|
|
1462
2009
|
// that is a socket nothing will ever close. Nothing has been committed at
|
|
1463
2010
|
// this point but the listener, so there is nothing to undo beyond letting
|
|
1464
2011
|
// the replacement go.
|
|
1465
2012
|
if (turn.finished || turn.held !== previous) {
|
|
2013
|
+
delete fresh.pendingTurn;
|
|
1466
2014
|
fresh.listeners.delete(turn.listener);
|
|
1467
|
-
|
|
2015
|
+
if (fresh.listeners.size === 0 &&
|
|
2016
|
+
this.turns.get(fresh.applicationId)?.held !== fresh) {
|
|
2017
|
+
this.drop(fresh);
|
|
2018
|
+
}
|
|
1468
2019
|
return { kind: "stale" };
|
|
1469
2020
|
}
|
|
1470
2021
|
if (claim === "unreachable") {
|
|
2022
|
+
delete fresh.pendingTurn;
|
|
1471
2023
|
fresh.listeners.delete(turn.listener);
|
|
1472
2024
|
this.drop(fresh);
|
|
1473
2025
|
return {
|
|
@@ -1476,6 +2028,7 @@ export class WebSocketSessionPeer {
|
|
|
1476
2028
|
};
|
|
1477
2029
|
}
|
|
1478
2030
|
if (claim === "refused") {
|
|
2031
|
+
delete fresh.pendingTurn;
|
|
1479
2032
|
fresh.listeners.delete(turn.listener);
|
|
1480
2033
|
// Go back to the socket the turn was on, but only while it is still
|
|
1481
2034
|
// there: a live edit that would not hand the turn over is still pushing
|
|
@@ -1484,22 +2037,17 @@ export class WebSocketSessionPeer {
|
|
|
1484
2037
|
// session would leave the next prompt to discover it.
|
|
1485
2038
|
if (isSocketOpen(previous.ws)) {
|
|
1486
2039
|
previous.listeners.add(turn.listener);
|
|
1487
|
-
this.
|
|
2040
|
+
this.setHeld(previous);
|
|
1488
2041
|
}
|
|
1489
2042
|
this.drop(fresh);
|
|
1490
2043
|
return { kind: "refused" };
|
|
1491
2044
|
}
|
|
2045
|
+
delete fresh.pendingTurn;
|
|
2046
|
+
delete fresh.provisional;
|
|
1492
2047
|
turn.held = fresh;
|
|
2048
|
+
this.setHeld(fresh);
|
|
2049
|
+
fresh.heartbeat.start();
|
|
1493
2050
|
this.drop(previous);
|
|
1494
|
-
// A reconnect that followed the session to another host is still listening
|
|
1495
|
-
// to SABS under the id the turn started with, which is not the one the
|
|
1496
|
-
// moved session relays under. Left alone the turn hears nothing from the
|
|
1497
|
-
// transport that outlives its socket - the same trap the prompt path names
|
|
1498
|
-
// when it subscribes under the id its connect landed on.
|
|
1499
|
-
if (fresh.liveEdit.id && fresh.liveEdit.id !== previous.liveEdit.id) {
|
|
1500
|
-
turn.unsubscribe();
|
|
1501
|
-
turn.unsubscribe = this.subscribeToPushes(fresh.liveEdit.id, turn.normalized);
|
|
1502
|
-
}
|
|
1503
2051
|
return { kind: "took_over" };
|
|
1504
2052
|
}
|
|
1505
2053
|
/**
|
|
@@ -1537,7 +2085,10 @@ export class WebSocketSessionPeer {
|
|
|
1537
2085
|
*/
|
|
1538
2086
|
async proveIdentity(input) {
|
|
1539
2087
|
try {
|
|
1540
|
-
await input.held
|
|
2088
|
+
await editorSocket(input.held, input.sessionToken).aiChatGetMessages({
|
|
2089
|
+
force: false,
|
|
2090
|
+
...this.turnCorrelator(input.turnId),
|
|
2091
|
+
});
|
|
1541
2092
|
return "claimed";
|
|
1542
2093
|
}
|
|
1543
2094
|
catch (error) {
|
|
@@ -1553,9 +2104,10 @@ export class WebSocketSessionPeer {
|
|
|
1553
2104
|
}
|
|
1554
2105
|
}
|
|
1555
2106
|
async reconnect(previous, deadline) {
|
|
1556
|
-
|
|
2107
|
+
return await this.connect({
|
|
1557
2108
|
applicationId: previous.applicationId,
|
|
1558
2109
|
branchName: previous.branchName,
|
|
2110
|
+
getLiveEditStatus: previous.getLiveEditStatus,
|
|
1559
2111
|
keepalive: previous.keepalive,
|
|
1560
2112
|
// Both halves, so a reattach keeps listening on the session it was
|
|
1561
2113
|
// already on: the socket is being replaced, not the live edit.
|
|
@@ -1568,16 +2120,30 @@ export class WebSocketSessionPeer {
|
|
|
1568
2120
|
? { resolveLiveEdit: previous.resolveLiveEdit }
|
|
1569
2121
|
: {}),
|
|
1570
2122
|
userId: previous.userId,
|
|
1571
|
-
}, deadline);
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
this.
|
|
1575
|
-
|
|
1576
|
-
|
|
2123
|
+
}, deadline, false);
|
|
2124
|
+
}
|
|
2125
|
+
setHeld(held) {
|
|
2126
|
+
this.held.set(held.applicationId, held);
|
|
2127
|
+
this.publishEditorStructure(held);
|
|
2128
|
+
}
|
|
2129
|
+
recordEditorStructure(held, hasDefaultStructure) {
|
|
2130
|
+
held.editorHasDefaultStructure = hasDefaultStructure;
|
|
2131
|
+
this.publishEditorStructure(held);
|
|
2132
|
+
}
|
|
2133
|
+
publishEditorStructure(held) {
|
|
2134
|
+
if (held.dropping ||
|
|
2135
|
+
this.held.get(held.applicationId) !== held ||
|
|
2136
|
+
!held.editorHasDefaultStructure) {
|
|
2137
|
+
return;
|
|
1577
2138
|
}
|
|
1578
|
-
|
|
1579
|
-
|
|
2139
|
+
this.editorStructureByScope.set(editorStructureScopeKey(held.applicationId, held.branchName), held.editorHasDefaultStructure);
|
|
2140
|
+
}
|
|
2141
|
+
currentViewStatus(applicationId, branch, fallback) {
|
|
2142
|
+
const hasDefaultStructure = this.editorStructureByScope.get(editorStructureScopeKey(applicationId, branch));
|
|
2143
|
+
if (!hasDefaultStructure) {
|
|
2144
|
+
return fallback;
|
|
1580
2145
|
}
|
|
2146
|
+
return hasDefaultStructure === "no" ? "ready" : "unknown";
|
|
1581
2147
|
}
|
|
1582
2148
|
/** Lines this caller has not been handed yet. */
|
|
1583
2149
|
drain(turn) {
|
|
@@ -1597,7 +2163,7 @@ export class WebSocketSessionPeer {
|
|
|
1597
2163
|
}
|
|
1598
2164
|
/**
|
|
1599
2165
|
* Records the outcome of a turn from outside runTurn, and ends the collector
|
|
1600
|
-
* waits so runTurn cannot overwrite it with a later
|
|
2166
|
+
* waits so runTurn cannot overwrite it with a later terminal status. The
|
|
1601
2167
|
* first outcome written wins: it is the one callers have already been shown.
|
|
1602
2168
|
*/
|
|
1603
2169
|
settleTurn(turn, finished) {
|
|
@@ -1614,30 +2180,88 @@ export class WebSocketSessionPeer {
|
|
|
1614
2180
|
const inSync = await input.collector.waitUntilInSync({
|
|
1615
2181
|
timeoutMs: input.readyWaitMs,
|
|
1616
2182
|
});
|
|
1617
|
-
|
|
2183
|
+
gatewayDebug("gateway clark checkpoint sync wait", {
|
|
1618
2184
|
applicationId: input.held.applicationId,
|
|
1619
2185
|
inSync,
|
|
1620
2186
|
});
|
|
1621
2187
|
if (!inSync) {
|
|
1622
|
-
input.onProgress?.("
|
|
1623
|
-
|
|
2188
|
+
input.onProgress?.("Superblocks' files are still saving; the app will finish in the editor");
|
|
2189
|
+
recordCheckpointOutcome("sync_timeout");
|
|
2190
|
+
return {
|
|
2191
|
+
failure: {
|
|
2192
|
+
category: "timeout",
|
|
2193
|
+
message: "Superblocks' files did not finish saving before the checkpoint deadline",
|
|
2194
|
+
},
|
|
2195
|
+
status: "failed",
|
|
2196
|
+
};
|
|
1624
2197
|
}
|
|
1625
2198
|
input.onProgress?.("Saving a checkpoint");
|
|
1626
2199
|
try {
|
|
1627
|
-
const result = await input.held.
|
|
1628
|
-
screenshotDataUrl: null,
|
|
2200
|
+
const result = await editorSocket(input.held, input.sessionToken).aiAcceptDraft({
|
|
1629
2201
|
silently: true,
|
|
1630
2202
|
...this.turnCorrelator(input.turnId),
|
|
1631
|
-
}
|
|
2203
|
+
});
|
|
1632
2204
|
input.collector.markCheckpointed();
|
|
1633
2205
|
const commitId = result?.commitId?.trim() ? result.commitId : undefined;
|
|
2206
|
+
const warning = commitId
|
|
2207
|
+
? normalizeCheckpointWarning(result?.warning)
|
|
2208
|
+
: undefined;
|
|
2209
|
+
const captureCheckpointScreenshot = this.options.captureCheckpointScreenshot;
|
|
2210
|
+
if (commitId && captureCheckpointScreenshot) {
|
|
2211
|
+
const screenshotTask = (async () => {
|
|
2212
|
+
const budgetMs = this.options.checkpointScreenshotTimeoutMs ??
|
|
2213
|
+
DEFAULT_CHECKPOINT_SCREENSHOT_TIMEOUT_MS;
|
|
2214
|
+
let timer;
|
|
2215
|
+
try {
|
|
2216
|
+
await Promise.race([
|
|
2217
|
+
(async () => {
|
|
2218
|
+
const screenshotDataUrl = await captureCheckpointScreenshot({
|
|
2219
|
+
applicationId: input.held.applicationId,
|
|
2220
|
+
commitId,
|
|
2221
|
+
sessionToken: input.sessionToken,
|
|
2222
|
+
});
|
|
2223
|
+
if (screenshotDataUrl) {
|
|
2224
|
+
await editorSocket(input.held, input.sessionToken).aiUploadCheckpointScreenshot({
|
|
2225
|
+
commitId,
|
|
2226
|
+
screenshotDataUrl,
|
|
2227
|
+
...this.turnCorrelator(input.turnId),
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
})(),
|
|
2231
|
+
new Promise((_, reject) => {
|
|
2232
|
+
timer = setTimeout(() => reject(new Error("checkpoint screenshot timed out")), budgetMs);
|
|
2233
|
+
timer.unref?.();
|
|
2234
|
+
}),
|
|
2235
|
+
]);
|
|
2236
|
+
}
|
|
2237
|
+
catch (error) {
|
|
2238
|
+
console.warn("gateway checkpoint screenshot failed", {
|
|
2239
|
+
applicationId: input.held.applicationId,
|
|
2240
|
+
commitId,
|
|
2241
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2242
|
+
});
|
|
2243
|
+
}
|
|
2244
|
+
finally {
|
|
2245
|
+
if (timer)
|
|
2246
|
+
clearTimeout(timer);
|
|
2247
|
+
}
|
|
2248
|
+
})();
|
|
2249
|
+
input.held.pendingCheckpointScreenshots.add(screenshotTask);
|
|
2250
|
+
void screenshotTask.finally(() => {
|
|
2251
|
+
input.held.pendingCheckpointScreenshots.delete(screenshotTask);
|
|
2252
|
+
this.releasePostTurnIfReady(input.held);
|
|
2253
|
+
});
|
|
2254
|
+
}
|
|
1634
2255
|
// The accept writes draftCommitted on the server. The warm sandbox still
|
|
1635
2256
|
// holds the chat it loaded earlier, so a browser that opens this same
|
|
1636
2257
|
// session and loads chat without force never sees the checkpoint card.
|
|
1637
2258
|
// Force-reloading now puts the server message into the sandbox store.
|
|
1638
2259
|
if (commitId) {
|
|
1639
2260
|
try {
|
|
1640
|
-
await input.held
|
|
2261
|
+
await editorSocket(input.held, input.sessionToken).aiChatGetMessages({
|
|
2262
|
+
force: true,
|
|
2263
|
+
...this.turnCorrelator(input.turnId),
|
|
2264
|
+
});
|
|
1641
2265
|
}
|
|
1642
2266
|
catch (error) {
|
|
1643
2267
|
console.warn("gateway clark chat reload after checkpoint failed", {
|
|
@@ -1646,23 +2270,32 @@ export class WebSocketSessionPeer {
|
|
|
1646
2270
|
});
|
|
1647
2271
|
}
|
|
1648
2272
|
}
|
|
1649
|
-
|
|
2273
|
+
gatewayDebug("gateway clark checkpoint saved", {
|
|
1650
2274
|
applicationId: input.held.applicationId,
|
|
1651
2275
|
commitId: commitId ?? "none",
|
|
1652
2276
|
});
|
|
1653
|
-
input.onProgress?.(
|
|
1654
|
-
|
|
2277
|
+
input.onProgress?.(warning?.message ??
|
|
2278
|
+
(commitId ? "Checkpoint saved" : "App already up to date"));
|
|
2279
|
+
recordCheckpointOutcome(warning ? "cleanup_warning" : commitId ? "saved" : "unchanged");
|
|
2280
|
+
return {
|
|
2281
|
+
...(commitId ? { commitId } : {}),
|
|
2282
|
+
status: "saved",
|
|
2283
|
+
...(warning ? { warning } : {}),
|
|
2284
|
+
};
|
|
1655
2285
|
}
|
|
1656
2286
|
catch (error) {
|
|
2287
|
+
const failure = checkpointFailure(error);
|
|
2288
|
+
recordCheckpointOutcome(`${failure.category}_failure`);
|
|
1657
2289
|
console.warn("gateway clark checkpoint failed", {
|
|
1658
2290
|
applicationId: input.held.applicationId,
|
|
1659
|
-
message:
|
|
2291
|
+
message: `${failure.message}${failure.code === undefined ? "" : ` (code ${failure.code})`}`,
|
|
1660
2292
|
});
|
|
1661
2293
|
input.onProgress?.("Could not save a checkpoint for Superblocks' changes");
|
|
1662
|
-
return
|
|
2294
|
+
return { failure, status: "failed" };
|
|
1663
2295
|
}
|
|
1664
2296
|
}
|
|
1665
|
-
async ensureHeld(input, deadline) {
|
|
2297
|
+
async ensureHeld(input, deadline, expectedTurn) {
|
|
2298
|
+
this.assertReadOnlyHoldCurrent(input.applicationId, expectedTurn);
|
|
1666
2299
|
const existing = this.held.get(input.applicationId);
|
|
1667
2300
|
// A connect that followed a moved session left this record pointing
|
|
1668
2301
|
// somewhere the prompt's own resolve never saw. Overlapping forwards are
|
|
@@ -1681,6 +2314,10 @@ export class WebSocketSessionPeer {
|
|
|
1681
2314
|
}
|
|
1682
2315
|
else {
|
|
1683
2316
|
existing.branchName = input.branchName;
|
|
2317
|
+
// A new prompt is a new turn, and the beat-lost clock measures one
|
|
2318
|
+
// turn's silence. Carrying a running clock across would judge this
|
|
2319
|
+
// prompt on how quiet the socket was before it was sent.
|
|
2320
|
+
existing.beatLoss = undefined;
|
|
1684
2321
|
// The caller resolved the live edit for this prompt, so its id is at
|
|
1685
2322
|
// least as fresh as whatever an earlier prompt left here - but only
|
|
1686
2323
|
// when it named the host this socket is on. A prompt reusing through
|
|
@@ -1696,6 +2333,9 @@ export class WebSocketSessionPeer {
|
|
|
1696
2333
|
if (input.keepalive) {
|
|
1697
2334
|
existing.keepalive = input.keepalive;
|
|
1698
2335
|
}
|
|
2336
|
+
if (input.getLiveEditStatus) {
|
|
2337
|
+
existing.getLiveEditStatus = input.getLiveEditStatus;
|
|
2338
|
+
}
|
|
1699
2339
|
if (input.resolveLiveEdit) {
|
|
1700
2340
|
existing.resolveLiveEdit = input.resolveLiveEdit;
|
|
1701
2341
|
}
|
|
@@ -1707,12 +2347,12 @@ export class WebSocketSessionPeer {
|
|
|
1707
2347
|
// Settle behind the open already in flight, then re-decide: it may have
|
|
1708
2348
|
// produced the socket this caller wants, or failed and left nothing.
|
|
1709
2349
|
await pending.catch(() => undefined);
|
|
1710
|
-
return await this.ensureHeld(input, deadline);
|
|
2350
|
+
return await this.ensureHeld(input, deadline, expectedTurn);
|
|
1711
2351
|
}
|
|
1712
2352
|
if (existing) {
|
|
1713
2353
|
this.drop(existing);
|
|
1714
2354
|
}
|
|
1715
|
-
const connect = this.connect(input, deadline);
|
|
2355
|
+
const connect = this.connect(input, deadline, true, expectedTurn);
|
|
1716
2356
|
this.connecting.set(input.applicationId, connect);
|
|
1717
2357
|
try {
|
|
1718
2358
|
return await connect;
|
|
@@ -1721,7 +2361,7 @@ export class WebSocketSessionPeer {
|
|
|
1721
2361
|
this.connecting.delete(input.applicationId);
|
|
1722
2362
|
}
|
|
1723
2363
|
}
|
|
1724
|
-
async connect(input, deadline) {
|
|
2364
|
+
async connect(input, deadline, publish = true, expectedTurn) {
|
|
1725
2365
|
// Unique per connection. The dev server ignores the disconnect of a peerId
|
|
1726
2366
|
// that is no longer the bound one, so a reattach that closes its
|
|
1727
2367
|
// predecessor afterwards keeps the binding it just took; reusing one id
|
|
@@ -1806,8 +2446,18 @@ export class WebSocketSessionPeer {
|
|
|
1806
2446
|
if (event.kind === "sync_status" && event.status) {
|
|
1807
2447
|
held.syncStatus = event.status;
|
|
1808
2448
|
}
|
|
1809
|
-
|
|
1810
|
-
|
|
2449
|
+
if (event.kind === "editor_structure") {
|
|
2450
|
+
this.recordEditorStructure(held, event.hasDefaultStructure);
|
|
2451
|
+
}
|
|
2452
|
+
if (isBrowserObserverEvent(event)) {
|
|
2453
|
+
this.observeEditorEvent(held, event);
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
if (held.listeners.size > 0) {
|
|
2457
|
+
for (const listener of [...held.listeners]) {
|
|
2458
|
+
listener(event);
|
|
2459
|
+
}
|
|
2460
|
+
return;
|
|
1811
2461
|
}
|
|
1812
2462
|
};
|
|
1813
2463
|
}, (method) => {
|
|
@@ -1819,8 +2469,10 @@ export class WebSocketSessionPeer {
|
|
|
1819
2469
|
if (!current) {
|
|
1820
2470
|
return;
|
|
1821
2471
|
}
|
|
1822
|
-
const turn = this.turns.get(current.applicationId);
|
|
1823
|
-
if (!turn
|
|
2472
|
+
const turn = current.pendingTurn ?? this.turns.get(current.applicationId);
|
|
2473
|
+
if (!turn ||
|
|
2474
|
+
turn.finished ||
|
|
2475
|
+
(turn.held !== current && current.pendingTurn !== turn)) {
|
|
1824
2476
|
return;
|
|
1825
2477
|
}
|
|
1826
2478
|
turn.recentRuns = appendRecentToolRun(turn.recentRuns, buildRecentToolRun(run));
|
|
@@ -1834,15 +2486,19 @@ export class WebSocketSessionPeer {
|
|
|
1834
2486
|
if (!current) {
|
|
1835
2487
|
return;
|
|
1836
2488
|
}
|
|
2489
|
+
const turn = current.pendingTurn ?? this.turns.get(current.applicationId);
|
|
2490
|
+
if (!turn ||
|
|
2491
|
+
turn.finished ||
|
|
2492
|
+
(turn.held !== current && current.pendingTurn !== turn)) {
|
|
2493
|
+
return;
|
|
2494
|
+
}
|
|
1837
2495
|
try {
|
|
1838
|
-
await current.
|
|
2496
|
+
await editorSocket(current, current.sessionToken).aiRespondToToolPermission({
|
|
1839
2497
|
approved: payload.approved,
|
|
1840
|
-
...this.turnCorrelator(
|
|
2498
|
+
...this.turnCorrelator(turn.turnId),
|
|
1841
2499
|
messageId: payload.messageId,
|
|
1842
2500
|
requestId: payload.requestId ?? payload.messageId,
|
|
1843
|
-
}
|
|
1844
|
-
? `Bearer ${current.sessionToken}`
|
|
1845
|
-
: undefined);
|
|
2501
|
+
});
|
|
1846
2502
|
}
|
|
1847
2503
|
catch (error) {
|
|
1848
2504
|
console.warn("gateway auto-approve tool permission failed", {
|
|
@@ -1864,11 +2520,24 @@ export class WebSocketSessionPeer {
|
|
|
1864
2520
|
this.held.delete(input.applicationId);
|
|
1865
2521
|
}
|
|
1866
2522
|
const held = heldRef.current;
|
|
2523
|
+
if (held?.provisional === true || (!held && !publish)) {
|
|
2524
|
+
return;
|
|
2525
|
+
}
|
|
1867
2526
|
// Its own closes - a reattach swapping sockets, a release - leave the
|
|
1868
2527
|
// turn either running elsewhere or already settled.
|
|
1869
2528
|
if (held?.dropping) {
|
|
1870
2529
|
return;
|
|
1871
2530
|
}
|
|
2531
|
+
const observed = held
|
|
2532
|
+
? this.observedTurns.get(held.applicationId)
|
|
2533
|
+
: undefined;
|
|
2534
|
+
if (held && observed?.held === held && !observed.finished) {
|
|
2535
|
+
const applicationEnded = isApplicationClose(close);
|
|
2536
|
+
this.settleObservedTurn(observed, applicationEnded ? "live_edit_terminated" : "detached", applicationEnded
|
|
2537
|
+
? LIVE_EDIT_TERMINATED_REPLY + describeClose(close)
|
|
2538
|
+
: observed.collector.reply());
|
|
2539
|
+
return;
|
|
2540
|
+
}
|
|
1872
2541
|
// An application code is the live edit stating its own end, so there
|
|
1873
2542
|
// is nothing left to get back onto. Any other close is the
|
|
1874
2543
|
// transport's, and a turn is not over because the socket carrying it
|
|
@@ -1894,10 +2563,12 @@ export class WebSocketSessionPeer {
|
|
|
1894
2563
|
client,
|
|
1895
2564
|
heartbeat,
|
|
1896
2565
|
idleTimer: undefined,
|
|
2566
|
+
getLiveEditStatus: input.getLiveEditStatus,
|
|
1897
2567
|
keepalive: input.keepalive,
|
|
1898
2568
|
keepaliveTimer: undefined,
|
|
1899
2569
|
listeners: new Set(),
|
|
1900
2570
|
liveEdit,
|
|
2571
|
+
pendingCheckpointScreenshots: new Set(),
|
|
1901
2572
|
...(input.resolveLiveEdit
|
|
1902
2573
|
? { resolveLiveEdit: input.resolveLiveEdit }
|
|
1903
2574
|
: {}),
|
|
@@ -1905,12 +2576,22 @@ export class WebSocketSessionPeer {
|
|
|
1905
2576
|
? {}
|
|
1906
2577
|
: { movedFrom: input.liveEdit.url }),
|
|
1907
2578
|
peerId,
|
|
2579
|
+
...(!publish ? { provisional: true } : {}),
|
|
1908
2580
|
socket,
|
|
1909
2581
|
userId: input.userId,
|
|
1910
2582
|
ws,
|
|
1911
2583
|
};
|
|
1912
2584
|
heldRef.current = held;
|
|
1913
|
-
|
|
2585
|
+
if (publish) {
|
|
2586
|
+
try {
|
|
2587
|
+
this.assertReadOnlyHoldCurrent(input.applicationId, expectedTurn);
|
|
2588
|
+
}
|
|
2589
|
+
catch (error) {
|
|
2590
|
+
this.drop(held);
|
|
2591
|
+
throw error;
|
|
2592
|
+
}
|
|
2593
|
+
this.setHeld(held);
|
|
2594
|
+
}
|
|
1914
2595
|
// Inbound frames are what separates a dead connection from a request
|
|
1915
2596
|
// channel that stopped answering on a live one.
|
|
1916
2597
|
ws.on("message", () => {
|
|
@@ -1919,9 +2600,19 @@ export class WebSocketSessionPeer {
|
|
|
1919
2600
|
// Beats for as long as the socket is held, including the grace period after
|
|
1920
2601
|
// a turn settles: the checkpoint lands in that window and the sandbox has
|
|
1921
2602
|
// to still be there for it.
|
|
1922
|
-
|
|
2603
|
+
if (publish) {
|
|
2604
|
+
heartbeat.start();
|
|
2605
|
+
}
|
|
1923
2606
|
return held;
|
|
1924
2607
|
}
|
|
2608
|
+
assertReadOnlyHoldCurrent(applicationId, expectedTurn) {
|
|
2609
|
+
const current = this.turns.get(applicationId);
|
|
2610
|
+
const ownsPublishedSession = current && this.held.get(applicationId) === current.held;
|
|
2611
|
+
if (current !== expectedTurn ||
|
|
2612
|
+
(current && (!current.finished || ownsPublishedSession))) {
|
|
2613
|
+
throw new Error("Live-edit hold skipped because the application has an active turn or completed a turn while the session opened");
|
|
2614
|
+
}
|
|
2615
|
+
}
|
|
1925
2616
|
/**
|
|
1926
2617
|
* The liveness beat for one held socket.
|
|
1927
2618
|
*
|
|
@@ -1939,7 +2630,7 @@ export class WebSocketSessionPeer {
|
|
|
1939
2630
|
// beat that never went out can never be recorded as one that landed.
|
|
1940
2631
|
throw new Error("live-edit session went away before the beat was sent");
|
|
1941
2632
|
}
|
|
1942
|
-
await held.
|
|
2633
|
+
await editorSocket(held, held.sessionToken).ping(this.turnCorrelator((held.pendingTurn ?? this.turns.get(held.applicationId))?.turnId));
|
|
1943
2634
|
},
|
|
1944
2635
|
describeError: describeSocketError,
|
|
1945
2636
|
intervalMs: this.options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS,
|
|
@@ -1971,6 +2662,9 @@ export class WebSocketSessionPeer {
|
|
|
1971
2662
|
if (!held) {
|
|
1972
2663
|
return;
|
|
1973
2664
|
}
|
|
2665
|
+
if (held.provisional) {
|
|
2666
|
+
return;
|
|
2667
|
+
}
|
|
1974
2668
|
// The clock measures silence, so anything that is not silence ends it.
|
|
1975
2669
|
// A success and a refusal both prove the peer answered this beat - and
|
|
1976
2670
|
// a refusal must never cost a turn, since it is an application-level
|
|
@@ -1978,7 +2672,7 @@ export class WebSocketSessionPeer {
|
|
|
1978
2672
|
// verdict that inbound traffic explained the quiet, which un-latches
|
|
1979
2673
|
// its alarm too. Only a genuinely unanswered beat keeps the clock.
|
|
1980
2674
|
if (outcome !== "grace" && outcome !== "miss") {
|
|
1981
|
-
held.
|
|
2675
|
+
held.beatLoss = undefined;
|
|
1982
2676
|
return;
|
|
1983
2677
|
}
|
|
1984
2678
|
// Started here rather than from `onLost`, which fires only on the edge
|
|
@@ -1989,14 +2683,20 @@ export class WebSocketSessionPeer {
|
|
|
1989
2683
|
// is still inside the missed-beat allowance, so it keeps a clock that
|
|
1990
2684
|
// is already running without starting one.
|
|
1991
2685
|
if (outcome === "miss") {
|
|
1992
|
-
held.
|
|
2686
|
+
held.beatLoss ??= {
|
|
2687
|
+
at: Date.now(),
|
|
2688
|
+
baseline: this.livenessTurnSnapshot(held),
|
|
2689
|
+
};
|
|
1993
2690
|
}
|
|
1994
|
-
|
|
2691
|
+
const beatLoss = held.beatLoss;
|
|
2692
|
+
if (!beatLoss) {
|
|
1995
2693
|
return;
|
|
1996
2694
|
}
|
|
1997
2695
|
const graceMs = this.options.beatLostGraceMs ?? DEFAULT_BEAT_LOST_GRACE_MS;
|
|
1998
|
-
|
|
1999
|
-
|
|
2696
|
+
const confirmationWindowStartedAt = beatLoss.failedHealthCheck?.at ?? beatLoss.at;
|
|
2697
|
+
if (Date.now() - confirmationWindowStartedAt >= graceMs &&
|
|
2698
|
+
!beatLoss.healthCheck) {
|
|
2699
|
+
beatLoss.healthCheck = this.confirmBeatLostSession(held, account, beatLoss);
|
|
2000
2700
|
}
|
|
2001
2701
|
},
|
|
2002
2702
|
// The sandbox grants liveness on proof of authorization, so a beat sent
|
|
@@ -2012,10 +2712,100 @@ export class WebSocketSessionPeer {
|
|
|
2012
2712
|
timeoutMs: this.options.heartbeatTimeoutMs ?? DEFAULT_HEARTBEAT_TIMEOUT_MS,
|
|
2013
2713
|
});
|
|
2014
2714
|
}
|
|
2715
|
+
livenessTurnSnapshot(held) {
|
|
2716
|
+
const turn = this.turns.get(held.applicationId);
|
|
2717
|
+
const activeTurn = turn?.held === held &&
|
|
2718
|
+
!turn.finished &&
|
|
2719
|
+
!turn.collector.status() &&
|
|
2720
|
+
!turn.collector.isSettling()
|
|
2721
|
+
? turn
|
|
2722
|
+
: undefined;
|
|
2723
|
+
const observed = this.observedTurns.get(held.applicationId);
|
|
2724
|
+
const activeObserved = observed?.held === held &&
|
|
2725
|
+
!observed.finished &&
|
|
2726
|
+
!observed.collector.status() &&
|
|
2727
|
+
!observed.collector.isSettling()
|
|
2728
|
+
? observed
|
|
2729
|
+
: undefined;
|
|
2730
|
+
return {
|
|
2731
|
+
observed: activeObserved,
|
|
2732
|
+
observedLastEventAt: activeObserved?.collector.lastEventAt(),
|
|
2733
|
+
turn: activeTurn,
|
|
2734
|
+
turnActivityLength: activeTurn?.activity.length,
|
|
2735
|
+
turnLastEventAt: activeTurn?.collector.lastEventAt(),
|
|
2736
|
+
};
|
|
2737
|
+
}
|
|
2738
|
+
sameLivenessTurnSnapshot(left, right) {
|
|
2739
|
+
return (left.observed === right.observed &&
|
|
2740
|
+
left.observedLastEventAt === right.observedLastEventAt &&
|
|
2741
|
+
left.turn === right.turn &&
|
|
2742
|
+
left.turnActivityLength === right.turnActivityLength &&
|
|
2743
|
+
left.turnLastEventAt === right.turnLastEventAt);
|
|
2744
|
+
}
|
|
2745
|
+
async confirmBeatLostSession(held, heartbeat, beatLoss) {
|
|
2746
|
+
const applicationId = held.applicationId;
|
|
2747
|
+
const snapshot = this.livenessTurnSnapshot(held);
|
|
2748
|
+
const activeObserved = snapshot.observed;
|
|
2749
|
+
const activeTurn = snapshot.turn;
|
|
2750
|
+
if (!activeTurn && !activeObserved) {
|
|
2751
|
+
if (held.beatLoss === beatLoss) {
|
|
2752
|
+
held.beatLoss = undefined;
|
|
2753
|
+
}
|
|
2754
|
+
return;
|
|
2755
|
+
}
|
|
2756
|
+
const priorSnapshot = beatLoss.failedHealthCheck?.snapshot ?? beatLoss.baseline;
|
|
2757
|
+
const activitySincePriorCheck = !this.sameLivenessTurnSnapshot(priorSnapshot, snapshot);
|
|
2758
|
+
if (activitySincePriorCheck) {
|
|
2759
|
+
held.beatLoss = undefined;
|
|
2760
|
+
return;
|
|
2761
|
+
}
|
|
2762
|
+
const healthy = await liveEditIsHealthy({
|
|
2763
|
+
applicationId,
|
|
2764
|
+
liveEditUrl: held.liveEdit.url,
|
|
2765
|
+
sessionToken: held.sessionToken,
|
|
2766
|
+
});
|
|
2767
|
+
let liveEditStatus;
|
|
2768
|
+
if (!healthy && held.liveEdit.id && held.getLiveEditStatus) {
|
|
2769
|
+
try {
|
|
2770
|
+
liveEditStatus = await held.getLiveEditStatus({
|
|
2771
|
+
liveEditId: held.liveEdit.id,
|
|
2772
|
+
signal: AbortSignal.timeout(DEFAULT_BEAT_HEALTH_TIMEOUT_MS),
|
|
2773
|
+
});
|
|
2774
|
+
}
|
|
2775
|
+
catch {
|
|
2776
|
+
// An unavailable control plane is not evidence that the sandbox died.
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
if (held.dropping ||
|
|
2780
|
+
this.held.get(applicationId) !== held ||
|
|
2781
|
+
held.beatLoss !== beatLoss) {
|
|
2782
|
+
return;
|
|
2783
|
+
}
|
|
2784
|
+
if (healthy ||
|
|
2785
|
+
!this.sameLivenessTurnSnapshot(snapshot, this.livenessTurnSnapshot(held))) {
|
|
2786
|
+
held.beatLoss = undefined;
|
|
2787
|
+
return;
|
|
2788
|
+
}
|
|
2789
|
+
if (liveEditStatus !== "terminated" &&
|
|
2790
|
+
liveEditStatus !== "terminating" &&
|
|
2791
|
+
beatLoss.failedHealthCheck === undefined) {
|
|
2792
|
+
beatLoss.failedHealthCheck = {
|
|
2793
|
+
at: Date.now(),
|
|
2794
|
+
snapshot,
|
|
2795
|
+
};
|
|
2796
|
+
delete beatLoss.healthCheck;
|
|
2797
|
+
return;
|
|
2798
|
+
}
|
|
2799
|
+
held.beatLoss = undefined;
|
|
2800
|
+
this.abandonTurnOnBeatLost(held, heartbeat, activeTurn, activeObserved, liveEditStatus);
|
|
2801
|
+
}
|
|
2015
2802
|
armIdle(held) {
|
|
2016
2803
|
if (held.idleTimer) {
|
|
2017
2804
|
clearTimeout(held.idleTimer);
|
|
2018
2805
|
}
|
|
2806
|
+
// This session is now held for an answer, so a screenshot settling later
|
|
2807
|
+
// must not release it on the finished turn's behalf.
|
|
2808
|
+
held.postTurnReleaseTurn = undefined;
|
|
2019
2809
|
const idleMs = this.options.idleMs ?? DEFAULT_IDLE_MS;
|
|
2020
2810
|
held.idleTimer = setTimeout(() => {
|
|
2021
2811
|
this.release(held.applicationId);
|
|
@@ -2026,8 +2816,8 @@ export class WebSocketSessionPeer {
|
|
|
2026
2816
|
* Ends a turn whose live edit is gone.
|
|
2027
2817
|
*
|
|
2028
2818
|
* The dev server exits on its own when its lock is lost, and it says nothing
|
|
2029
|
-
* on the way out. A turn left waiting on that socket
|
|
2030
|
-
*
|
|
2819
|
+
* on the way out. A turn left waiting on that socket would otherwise never
|
|
2820
|
+
* report an outcome to the caller.
|
|
2031
2821
|
*
|
|
2032
2822
|
* Reached only once the live edit being gone is established, which a close on
|
|
2033
2823
|
* its own never does: either it named its own end in the application range,
|
|
@@ -2101,17 +2891,20 @@ export class WebSocketSessionPeer {
|
|
|
2101
2891
|
/**
|
|
2102
2892
|
* Ends a turn whose socket has stopped answering.
|
|
2103
2893
|
*
|
|
2104
|
-
*
|
|
2894
|
+
* Three gates answer different questions. `SocketHeartbeat`
|
|
2105
2895
|
* declares the beat lost after consecutive genuine misses, having already
|
|
2106
2896
|
* excused a run of unanswered beats and every silence that inbound traffic
|
|
2107
2897
|
* explained - the right threshold for a diagnostic, and too eager for this: a
|
|
2108
2898
|
* sandbox that misses two beats and answers the next was recovering, and its
|
|
2109
2899
|
* turn has to survive that. So the loss also has to outlast
|
|
2110
2900
|
* {@link DEFAULT_BEAT_LOST_GRACE_MS}, and any beat that succeeds clears it.
|
|
2901
|
+
* Finally, the exact live edit must either be terminal in the control plane,
|
|
2902
|
+
* or its direct HTTP health check must fail again after another grace
|
|
2903
|
+
* window. One timeout, 5xx, or inconclusive control-plane result is still
|
|
2904
|
+
* recoverable.
|
|
2111
2905
|
*
|
|
2112
2906
|
* What is left is a request channel that has been silent for minutes on a
|
|
2113
|
-
* socket that never closed, with a turn waiting on it
|
|
2114
|
-
* waits out the whole turn budget.
|
|
2907
|
+
* socket that never closed, with a turn waiting on it indefinitely.
|
|
2115
2908
|
*
|
|
2116
2909
|
* This is the only trigger that reaches a half-open socket. An unsolicited
|
|
2117
2910
|
* close is the ordinary one and never arrives for a connection that stays up
|
|
@@ -2123,24 +2916,47 @@ export class WebSocketSessionPeer {
|
|
|
2123
2916
|
* wedged should also be torn down rather than handed to the next prompt is a
|
|
2124
2917
|
* separate question from whether this turn should keep waiting on it.
|
|
2125
2918
|
*/
|
|
2126
|
-
abandonTurnOnBeatLost(
|
|
2127
|
-
const
|
|
2919
|
+
abandonTurnOnBeatLost(held, heartbeat, turn, observed, liveEditStatus) {
|
|
2920
|
+
const applicationId = held.applicationId;
|
|
2921
|
+
if (held.dropping || this.held.get(applicationId) !== held) {
|
|
2922
|
+
return;
|
|
2923
|
+
}
|
|
2128
2924
|
// The same rule the disconnect and SABS paths use: settle only a turn that
|
|
2129
2925
|
// has not settled itself. A finished turn's socket keeps beating through
|
|
2130
2926
|
// the post-turn grace, so a sandbox reaped in that window must not
|
|
2131
2927
|
// overwrite the outcome the caller already has.
|
|
2132
|
-
if (
|
|
2928
|
+
if (turn &&
|
|
2929
|
+
this.turns.get(applicationId) === turn &&
|
|
2930
|
+
turn.held === held &&
|
|
2931
|
+
!turn.finished &&
|
|
2932
|
+
!turn.collector.status()) {
|
|
2933
|
+
console.warn("gateway clark turn abandoned", {
|
|
2934
|
+
applicationId,
|
|
2935
|
+
beatFailures: heartbeat.failures,
|
|
2936
|
+
beatLapses: heartbeat.lapses,
|
|
2937
|
+
beatsSent: heartbeat.sent,
|
|
2938
|
+
lastBeatError: heartbeat.lastError,
|
|
2939
|
+
liveEditStatus,
|
|
2940
|
+
reason: "the live-edit socket stopped answering",
|
|
2941
|
+
});
|
|
2942
|
+
let confirmation = "The direct live-edit health check failed twice while no conclusive Superblocks control-plane status was available.";
|
|
2943
|
+
if (liveEditStatus === "active" || liveEditStatus === "starting") {
|
|
2944
|
+
confirmation = `The direct live-edit health check failed twice while the Superblocks control plane last reported the exact session ${liveEditStatus}.`;
|
|
2945
|
+
}
|
|
2946
|
+
else if (liveEditStatus === "terminated" ||
|
|
2947
|
+
liveEditStatus === "terminating") {
|
|
2948
|
+
confirmation = `The Superblocks control plane reported the exact live-edit session ${liveEditStatus}.`;
|
|
2949
|
+
}
|
|
2950
|
+
this.settleAsLiveEditTerminated(turn, `${describeHeartbeat(heartbeat, Date.now())} ${confirmation}`);
|
|
2133
2951
|
return;
|
|
2134
2952
|
}
|
|
2135
|
-
|
|
2136
|
-
applicationId
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
});
|
|
2143
|
-
this.settleAsLiveEditTerminated(turn, describeHeartbeat(heartbeat, Date.now()));
|
|
2953
|
+
if (observed &&
|
|
2954
|
+
this.observedTurns.get(applicationId) === observed &&
|
|
2955
|
+
observed.held === held &&
|
|
2956
|
+
!observed.finished) {
|
|
2957
|
+
this.settleObservedTurn(observed, "detached");
|
|
2958
|
+
this.armObservedTurnRelease(applicationId, observed);
|
|
2959
|
+
}
|
|
2144
2960
|
}
|
|
2145
2961
|
/**
|
|
2146
2962
|
* Records a dead live edit as the turn's outcome and ends every wait on it.
|
|
@@ -2148,6 +2964,12 @@ export class WebSocketSessionPeer {
|
|
|
2148
2964
|
* cause that reaches the caller.
|
|
2149
2965
|
*/
|
|
2150
2966
|
settleAsLiveEditTerminated(turn, evidence) {
|
|
2967
|
+
if (turn.finished) {
|
|
2968
|
+
return;
|
|
2969
|
+
}
|
|
2970
|
+
tagRetryableLiveEditDeath(turn, {
|
|
2971
|
+
generationStatus: "live_edit_terminated",
|
|
2972
|
+
});
|
|
2151
2973
|
this.settleTurn(turn, {
|
|
2152
2974
|
reply: LIVE_EDIT_TERMINATED_REPLY + evidence,
|
|
2153
2975
|
generationStatus: "live_edit_terminated",
|
|
@@ -2253,32 +3075,55 @@ export class WebSocketSessionPeer {
|
|
|
2253
3075
|
if (!prior) {
|
|
2254
3076
|
return;
|
|
2255
3077
|
}
|
|
2256
|
-
prior
|
|
2257
|
-
|
|
3078
|
+
this.cancelTurn(prior);
|
|
3079
|
+
}
|
|
3080
|
+
cancelTurn(turn, reply = "This turn was replaced by a newer prompt on the same app, so its work was cancelled.") {
|
|
3081
|
+
turn.finished ??= {
|
|
3082
|
+
reply,
|
|
2258
3083
|
generationStatus: "cancelled",
|
|
2259
|
-
viewStatus:
|
|
3084
|
+
viewStatus: turn.collector.viewableStatus(),
|
|
2260
3085
|
};
|
|
2261
|
-
|
|
2262
|
-
held?.listeners.delete(
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
3086
|
+
turn.held.listeners.delete(turn.listener);
|
|
3087
|
+
this.held.get(turn.held.applicationId)?.listeners.delete(turn.listener);
|
|
3088
|
+
turn.collector.dispose();
|
|
3089
|
+
turn.abort.abort();
|
|
3090
|
+
this.wake(turn);
|
|
3091
|
+
}
|
|
3092
|
+
discardUnstartedTurn(turn) {
|
|
3093
|
+
if (turn.held.pendingTurn === turn) {
|
|
3094
|
+
delete turn.held.pendingTurn;
|
|
3095
|
+
}
|
|
3096
|
+
turn.held.listeners.delete(turn.listener);
|
|
3097
|
+
turn.collector.dispose();
|
|
3098
|
+
turn.abort.abort();
|
|
3099
|
+
this.wake(turn);
|
|
2267
3100
|
}
|
|
2268
3101
|
armPostTurnRelease(held, turn) {
|
|
2269
3102
|
this.clearTimers(held);
|
|
2270
3103
|
const delayMs = this.options.releaseAfterTurnMs ?? DEFAULT_RELEASE_AFTER_TURN_MS;
|
|
3104
|
+
held.postTurnReleaseAt = Date.now() + delayMs;
|
|
3105
|
+
held.postTurnReleaseTurn = turn;
|
|
2271
3106
|
held.idleTimer = setTimeout(() => {
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
return;
|
|
2275
|
-
}
|
|
2276
|
-
if (this.held.get(held.applicationId) === held) {
|
|
2277
|
-
this.drop(held);
|
|
2278
|
-
}
|
|
3107
|
+
held.postTurnReleaseAt = 0;
|
|
3108
|
+
this.releasePostTurnIfReady(held);
|
|
2279
3109
|
}, delayMs);
|
|
2280
3110
|
held.idleTimer.unref?.();
|
|
2281
3111
|
}
|
|
3112
|
+
releasePostTurnIfReady(held) {
|
|
3113
|
+
const turn = held.postTurnReleaseTurn;
|
|
3114
|
+
if (!turn ||
|
|
3115
|
+
Date.now() < (held.postTurnReleaseAt ?? 0) ||
|
|
3116
|
+
held.pendingCheckpointScreenshots.size > 0) {
|
|
3117
|
+
return;
|
|
3118
|
+
}
|
|
3119
|
+
const active = this.turns.get(held.applicationId);
|
|
3120
|
+
if (active && active !== turn && !active.finished) {
|
|
3121
|
+
return;
|
|
3122
|
+
}
|
|
3123
|
+
if (this.held.get(held.applicationId) === held) {
|
|
3124
|
+
this.drop(held);
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
2282
3127
|
armKeepalive(held) {
|
|
2283
3128
|
if (held.keepaliveTimer) {
|
|
2284
3129
|
clearInterval(held.keepaliveTimer);
|
|
@@ -2301,21 +3146,30 @@ export class WebSocketSessionPeer {
|
|
|
2301
3146
|
* belongs to the socket and outlives these, which the post-turn grace period
|
|
2302
3147
|
* resets.
|
|
2303
3148
|
*/
|
|
2304
|
-
|
|
3149
|
+
clearIdleTimer(held) {
|
|
3150
|
+
held.postTurnReleaseTurn = undefined;
|
|
2305
3151
|
if (held.idleTimer) {
|
|
2306
3152
|
clearTimeout(held.idleTimer);
|
|
2307
3153
|
held.idleTimer = undefined;
|
|
2308
3154
|
}
|
|
3155
|
+
}
|
|
3156
|
+
clearTimers(held) {
|
|
3157
|
+
this.clearIdleTimer(held);
|
|
2309
3158
|
if (held.keepaliveTimer) {
|
|
2310
3159
|
clearInterval(held.keepaliveTimer);
|
|
2311
3160
|
held.keepaliveTimer = undefined;
|
|
2312
3161
|
}
|
|
2313
3162
|
}
|
|
2314
|
-
drop(held) {
|
|
3163
|
+
drop(held, options = {}) {
|
|
2315
3164
|
console.error("gateway held session released", {
|
|
2316
3165
|
applicationId: held.applicationId,
|
|
2317
3166
|
});
|
|
2318
3167
|
held.dropping = true;
|
|
3168
|
+
const observed = this.observedTurns.get(held.applicationId);
|
|
3169
|
+
if (observed?.held === held &&
|
|
3170
|
+
!(options.preserveFinishedObserved && observed.finished)) {
|
|
3171
|
+
this.forgetObservedTurn(held.applicationId);
|
|
3172
|
+
}
|
|
2319
3173
|
this.clearTimers(held);
|
|
2320
3174
|
held.heartbeat.stop();
|
|
2321
3175
|
// Only if it is still the current session: a reattach registers its
|
|
@@ -2465,7 +3319,9 @@ export class WebSocketSessionPeer {
|
|
|
2465
3319
|
}
|
|
2466
3320
|
if (this.options.orchestrator || this.options.orchestratorUrl) {
|
|
2467
3321
|
held.orchestrator = this.options.orchestrator;
|
|
2468
|
-
held.orchestratorUrl = this.options.orchestratorUrl
|
|
3322
|
+
held.orchestratorUrl = this.options.orchestratorUrl
|
|
3323
|
+
? canonicalizeOrchestratorBaseUrl(this.options.orchestratorUrl)
|
|
3324
|
+
: this.options.orchestratorUrl;
|
|
2469
3325
|
return;
|
|
2470
3326
|
}
|
|
2471
3327
|
if (!held.apiKey || !held.organizationId || !this.options.server) {
|