@superblocksteam/gateway 2.0.155-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +87 -0
- package/README.md +170 -0
- package/dist/agents/resolve-orchestrator-url.d.ts +22 -0
- package/dist/agents/resolve-orchestrator-url.js +38 -0
- package/dist/agents/resolve-orchestrator-url.js.map +1 -0
- package/dist/capabilities/import-prompt.d.ts +11 -0
- package/dist/capabilities/import-prompt.js +96 -0
- package/dist/capabilities/import-prompt.js.map +1 -0
- package/dist/capabilities/lifecycle.d.ts +73 -0
- package/dist/capabilities/lifecycle.js +2531 -0
- package/dist/capabilities/lifecycle.js.map +1 -0
- package/dist/capabilities/persisted-progress.d.ts +46 -0
- package/dist/capabilities/persisted-progress.js +246 -0
- package/dist/capabilities/persisted-progress.js.map +1 -0
- package/dist/capabilities/types.d.ts +446 -0
- package/dist/capabilities/types.js +47 -0
- package/dist/capabilities/types.js.map +1 -0
- package/dist/capture/browser-contract.d.ts +36 -0
- package/dist/capture/browser-contract.js +24 -0
- package/dist/capture/browser-contract.js.map +1 -0
- package/dist/capture/browser-instructions.d.ts +9 -0
- package/dist/capture/browser-instructions.js +17 -0
- package/dist/capture/browser-instructions.js.map +1 -0
- package/dist/capture/capture-library-probe.d.ts +55 -0
- package/dist/capture/capture-library-probe.js +177 -0
- package/dist/capture/capture-library-probe.js.map +1 -0
- package/dist/capture/capture-library.d.ts +52 -0
- package/dist/capture/capture-library.js +249 -0
- package/dist/capture/capture-library.js.map +1 -0
- package/dist/capture/classify-library-paint.d.ts +21 -0
- package/dist/capture/classify-library-paint.js +13 -0
- package/dist/capture/classify-library-paint.js.map +1 -0
- package/dist/capture/host-mcp-scan.d.ts +16 -0
- package/dist/capture/host-mcp-scan.js +64 -0
- package/dist/capture/host-mcp-scan.js.map +1 -0
- package/dist/capture/library-parent-script.d.ts +34 -0
- package/dist/capture/library-parent-script.js +82 -0
- package/dist/capture/library-parent-script.js.map +1 -0
- package/dist/capture/mode.d.ts +22 -0
- package/dist/capture/mode.js +45 -0
- package/dist/capture/mode.js.map +1 -0
- package/dist/config.d.ts +141 -0
- package/dist/config.js +109 -0
- package/dist/config.js.map +1 -0
- package/dist/debug.d.ts +2 -0
- package/dist/debug.js +9 -0
- package/dist/debug.js.map +1 -0
- package/dist/deps.d.ts +36 -0
- package/dist/deps.js +2 -0
- package/dist/deps.js.map +1 -0
- package/dist/events/cursor.d.ts +43 -0
- package/dist/events/cursor.js +78 -0
- package/dist/events/cursor.js.map +1 -0
- package/dist/events/memory-event-store.d.ts +34 -0
- package/dist/events/memory-event-store.js +110 -0
- package/dist/events/memory-event-store.js.map +1 -0
- package/dist/events/merge.d.ts +23 -0
- package/dist/events/merge.js +97 -0
- package/dist/events/merge.js.map +1 -0
- package/dist/events/normalized-collector.d.ts +62 -0
- package/dist/events/normalized-collector.js +156 -0
- package/dist/events/normalized-collector.js.map +1 -0
- package/dist/events/schema.d.ts +9 -0
- package/dist/events/schema.js +93 -0
- package/dist/events/schema.js.map +1 -0
- package/dist/events/snapshot.d.ts +32 -0
- package/dist/events/snapshot.js +57 -0
- package/dist/events/snapshot.js.map +1 -0
- package/dist/events/stream-key.d.ts +2 -0
- package/dist/events/stream-key.js +31 -0
- package/dist/events/stream-key.js.map +1 -0
- package/dist/events/types.d.ts +179 -0
- package/dist/events/types.js +66 -0
- package/dist/events/types.js.map +1 -0
- package/dist/health/readiness.d.ts +99 -0
- package/dist/health/readiness.js +202 -0
- package/dist/health/readiness.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/integrations/filters.d.ts +8 -0
- package/dist/integrations/filters.js +40 -0
- package/dist/integrations/filters.js.map +1 -0
- package/dist/integrations/map.d.ts +26 -0
- package/dist/integrations/map.js +43 -0
- package/dist/integrations/map.js.map +1 -0
- package/dist/integrations/profile-key.d.ts +19 -0
- package/dist/integrations/profile-key.js +35 -0
- package/dist/integrations/profile-key.js.map +1 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.js +5 -0
- package/dist/main.js.map +1 -0
- package/dist/orchestrator/client.d.ts +62 -0
- package/dist/orchestrator/client.js +160 -0
- package/dist/orchestrator/client.js.map +1 -0
- package/dist/playwright/ensure-chromium.d.ts +26 -0
- package/dist/playwright/ensure-chromium.js +60 -0
- package/dist/playwright/ensure-chromium.js.map +1 -0
- package/dist/preview/capture-screenshot.d.ts +52 -0
- package/dist/preview/capture-screenshot.js +158 -0
- package/dist/preview/capture-screenshot.js.map +1 -0
- package/dist/preview/viewer-url.d.ts +82 -0
- package/dist/preview/viewer-url.js +128 -0
- package/dist/preview/viewer-url.js.map +1 -0
- package/dist/process/fault-barrier.d.ts +55 -0
- package/dist/process/fault-barrier.js +113 -0
- package/dist/process/fault-barrier.js.map +1 -0
- package/dist/resume/memory-progress-store.d.ts +39 -0
- package/dist/resume/memory-progress-store.js +82 -0
- package/dist/resume/memory-progress-store.js.map +1 -0
- package/dist/resume/memory-recent-app-store.d.ts +14 -0
- package/dist/resume/memory-recent-app-store.js +27 -0
- package/dist/resume/memory-recent-app-store.js.map +1 -0
- package/dist/resume/memory-turn-store.d.ts +18 -0
- package/dist/resume/memory-turn-store.js +73 -0
- package/dist/resume/memory-turn-store.js.map +1 -0
- package/dist/resume/progress-key.d.ts +21 -0
- package/dist/resume/progress-key.js +58 -0
- package/dist/resume/progress-key.js.map +1 -0
- package/dist/resume/stores.d.ts +14 -0
- package/dist/resume/stores.js +18 -0
- package/dist/resume/stores.js.map +1 -0
- package/dist/resume/types.d.ts +124 -0
- package/dist/resume/types.js +13 -0
- package/dist/resume/types.js.map +1 -0
- package/dist/sabs/agent-facing-text.d.ts +33 -0
- package/dist/sabs/agent-facing-text.js +125 -0
- package/dist/sabs/agent-facing-text.js.map +1 -0
- package/dist/sabs/browser-context.d.ts +53 -0
- package/dist/sabs/browser-context.js +91 -0
- package/dist/sabs/browser-context.js.map +1 -0
- package/dist/sabs/editor-client-methods.d.ts +147 -0
- package/dist/sabs/editor-client-methods.js +715 -0
- package/dist/sabs/editor-client-methods.js.map +1 -0
- package/dist/sabs/recent-tool-runs.d.ts +14 -0
- package/dist/sabs/recent-tool-runs.js +61 -0
- package/dist/sabs/recent-tool-runs.js.map +1 -0
- package/dist/sabs/session-peer.d.ts +351 -0
- package/dist/sabs/session-peer.js +2 -0
- package/dist/sabs/session-peer.js.map +1 -0
- package/dist/sabs/turn-collector.d.ts +152 -0
- package/dist/sabs/turn-collector.js +573 -0
- package/dist/sabs/turn-collector.js.map +1 -0
- package/dist/sabs/websocket-session-peer.d.ts +514 -0
- package/dist/sabs/websocket-session-peer.js +2487 -0
- package/dist/sabs/websocket-session-peer.js.map +1 -0
- package/dist/security/constant-time.d.ts +9 -0
- package/dist/security/constant-time.js +13 -0
- package/dist/security/constant-time.js.map +1 -0
- package/dist/server/client.d.ts +371 -0
- package/dist/server/client.js +572 -0
- package/dist/server/client.js.map +1 -0
- package/dist/server/identity-reads.d.ts +76 -0
- package/dist/server/identity-reads.js +209 -0
- package/dist/server/identity-reads.js.map +1 -0
- package/dist/start.d.ts +14 -0
- package/dist/start.js +75 -0
- package/dist/start.js.map +1 -0
- package/dist/telemetry/metrics.d.ts +32 -0
- package/dist/telemetry/metrics.js +57 -0
- package/dist/telemetry/metrics.js.map +1 -0
- package/dist/transports/mcp/admin-tools.d.ts +47 -0
- package/dist/transports/mcp/admin-tools.js +164 -0
- package/dist/transports/mcp/admin-tools.js.map +1 -0
- package/dist/transports/mcp/app-status-html.d.ts +15 -0
- package/dist/transports/mcp/app-status-html.js +563 -0
- package/dist/transports/mcp/app-status-html.js.map +1 -0
- package/dist/transports/mcp/decision-elicitation.d.ts +74 -0
- package/dist/transports/mcp/decision-elicitation.js +233 -0
- package/dist/transports/mcp/decision-elicitation.js.map +1 -0
- package/dist/transports/mcp/dispatch-probe.d.ts +24 -0
- package/dist/transports/mcp/dispatch-probe.js +48 -0
- package/dist/transports/mcp/dispatch-probe.js.map +1 -0
- package/dist/transports/mcp/format-tool-content.d.ts +25 -0
- package/dist/transports/mcp/format-tool-content.js +48 -0
- package/dist/transports/mcp/format-tool-content.js.map +1 -0
- package/dist/transports/mcp/mount.d.ts +9 -0
- package/dist/transports/mcp/mount.js +469 -0
- package/dist/transports/mcp/mount.js.map +1 -0
- package/dist/transports/mcp/progress-notifier.d.ts +39 -0
- package/dist/transports/mcp/progress-notifier.js +52 -0
- package/dist/transports/mcp/progress-notifier.js.map +1 -0
- package/package.json +72 -0
|
@@ -0,0 +1,2487 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { WebSocket } from "ws";
|
|
3
|
+
import { createISocketClient, ISocket, SocketHeartbeat, } from "@superblocksteam/shared";
|
|
4
|
+
import { pickOrchestratorBaseUrl } from "../agents/resolve-orchestrator-url.js";
|
|
5
|
+
import { NormalizedTurnCollector } from "../events/normalized-collector.js";
|
|
6
|
+
import { pickEditorProfileKey } from "../integrations/profile-key.js";
|
|
7
|
+
import { HttpOrchestratorClient, } from "../orchestrator/client.js";
|
|
8
|
+
import { buildBrowserContext } from "./browser-context.js";
|
|
9
|
+
import { createEditorClientMethods, loadIntegrationsForPrompt, } from "./editor-client-methods.js";
|
|
10
|
+
import { appendRecentToolRun, buildRecentToolRun } from "./recent-tool-runs.js";
|
|
11
|
+
import { DEFAULT_SETTLE_MS, TurnCollector } from "./turn-collector.js";
|
|
12
|
+
/**
|
|
13
|
+
* A prompt reaches Clark over the dev server's socket, not over HTTP: the
|
|
14
|
+
* sandbox exposes no HTTP route that starts a generation. This is the same
|
|
15
|
+
* call the editor makes when a user submits a prompt in the browser.
|
|
16
|
+
*/
|
|
17
|
+
const GENERATE_METHOD = "editor.aiGenerateWithQueue";
|
|
18
|
+
/**
|
|
19
|
+
* Commits the files Clark wrote as a draft and creates the checkpoint. The
|
|
20
|
+
* browser editor fires this automatically once files are in sync; a headless
|
|
21
|
+
* peer that skips it leaves the app uncommitted, so the build only finishes
|
|
22
|
+
* when a human finally opens the edit URL.
|
|
23
|
+
*/
|
|
24
|
+
const ACCEPT_DRAFT_METHOD = "editor.aiAcceptDraft";
|
|
25
|
+
const RELOAD_CHAT_METHOD = "editor.aiChatGetMessages";
|
|
26
|
+
const RESPOND_TOOL_PERMISSION_METHOD = "editor.aiRespondToToolPermission";
|
|
27
|
+
/**
|
|
28
|
+
* The cheapest authorized call there is, used as the gateway's liveness beat.
|
|
29
|
+
*
|
|
30
|
+
* Deliberately sent without `isUserActive`: the sandbox counts that as a person
|
|
31
|
+
* at a keyboard, and a headless peer is not one.
|
|
32
|
+
*/
|
|
33
|
+
const PING_METHOD = "editor.ping";
|
|
34
|
+
/**
|
|
35
|
+
* Whole budget for connecting and getting the prompt acknowledged, so a
|
|
36
|
+
* session that accepts the socket but never answers cannot strand the MCP call
|
|
37
|
+
* waiting on it.
|
|
38
|
+
*
|
|
39
|
+
* A sandbox that has gone idle since the app was created needs most of a
|
|
40
|
+
* minute before it acknowledges anything, and the first edit after that pause
|
|
41
|
+
* was failing on a tighter budget only to succeed on the retry.
|
|
42
|
+
*/
|
|
43
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
44
|
+
/**
|
|
45
|
+
* The sandbox binds its HTTP port before it attaches the socket handler, and
|
|
46
|
+
* an upgrade arriving in that window gets no response at all rather than a
|
|
47
|
+
* refusal. One long attempt would spend the whole budget waiting on a session
|
|
48
|
+
* that was seconds from ready, so each attempt is short and retried.
|
|
49
|
+
*/
|
|
50
|
+
const DEFAULT_CONNECT_ATTEMPT_TIMEOUT_MS = 3_000;
|
|
51
|
+
const CONNECT_RETRY_DELAY_MS = 500;
|
|
52
|
+
/**
|
|
53
|
+
* How long a lost liveness beat has to stay lost before the turn running on
|
|
54
|
+
* that socket is given up on.
|
|
55
|
+
*
|
|
56
|
+
* The beat is declared lost for *reporting* after a couple of genuine misses,
|
|
57
|
+
* which is the right threshold for a diagnostic and the wrong one for ending a
|
|
58
|
+
* turn: a sandbox that misses two beats and then answers is recovering, not
|
|
59
|
+
* gone. This is the second gate — the silence has to outlast it, and any beat
|
|
60
|
+
* that succeeds clears it. Two missed beats on the default interval plus this
|
|
61
|
+
* is a couple of minutes, against a ten-minute turn budget.
|
|
62
|
+
*/
|
|
63
|
+
const DEFAULT_BEAT_LOST_GRACE_MS = 60_000;
|
|
64
|
+
/**
|
|
65
|
+
* Consecutive upgrade rejections from one host before asking where the live
|
|
66
|
+
* edit went. More than one, so an ingress route that briefly answers for a pod
|
|
67
|
+
* that is not up yet is waited out rather than read as a move.
|
|
68
|
+
*
|
|
69
|
+
* Exported so a test can say "kept trying past the point where it asked"
|
|
70
|
+
* without restating the number.
|
|
71
|
+
*/
|
|
72
|
+
export const HOST_REJECTIONS_BEFORE_RERESOLVE = 2;
|
|
73
|
+
/**
|
|
74
|
+
* How many times one connect may ask where the session went.
|
|
75
|
+
*
|
|
76
|
+
* Bounded because each ask is a Superblocks API round trip, and spaced by a
|
|
77
|
+
* fresh rejection streak so the asks land across the connect window rather
|
|
78
|
+
* than all at the start of it - SABS needs time to notice the pod is gone
|
|
79
|
+
* before it can answer with anywhere different.
|
|
80
|
+
*/
|
|
81
|
+
export const MAX_RERESOLVES_PER_OPEN = 3;
|
|
82
|
+
/** Match AppManager's idle window so a quiet Claude turn does not outlive it. */
|
|
83
|
+
const DEFAULT_IDLE_MS = 30 * 60 * 1000;
|
|
84
|
+
/**
|
|
85
|
+
* Grace period between a turn settling and the gateway socket being dropped.
|
|
86
|
+
*
|
|
87
|
+
* Long enough for the checkpoint and any trailing pushes to land, short enough
|
|
88
|
+
* that the sandbox is not held open by a peer with nothing left to do.
|
|
89
|
+
*
|
|
90
|
+
* Must outlast {@link DEFAULT_SETTLE_MS}. The teardown this release leads to is
|
|
91
|
+
* read as a dead live edit by any turn whose `completed` had not settled yet,
|
|
92
|
+
* so releasing inside the settle window would report healthy builds as
|
|
93
|
+
* failures. `releases the socket only after a completed has had its settle
|
|
94
|
+
* window` holds this.
|
|
95
|
+
*/
|
|
96
|
+
export const DEFAULT_RELEASE_AFTER_TURN_MS = 30_000;
|
|
97
|
+
/**
|
|
98
|
+
* Refresh connectLiveEdit while held so local AppManager lastAccessed does not
|
|
99
|
+
* age out under a long Clark build with no MCP traffic.
|
|
100
|
+
*/
|
|
101
|
+
const DEFAULT_KEEPALIVE_MS = 5 * 60 * 1000;
|
|
102
|
+
/**
|
|
103
|
+
* How often the held socket proves the gateway is still alive.
|
|
104
|
+
*
|
|
105
|
+
* The sandbox expires a gateway peer's liveness signal if nothing refreshes it,
|
|
106
|
+
* so this has to be comfortably inside that window — a build that runs for
|
|
107
|
+
* twenty quiet minutes must not look abandoned — while still letting a peer that
|
|
108
|
+
* really died stop protecting the sandbox promptly.
|
|
109
|
+
*/
|
|
110
|
+
const DEFAULT_HEARTBEAT_MS = 30 * 1000;
|
|
111
|
+
/**
|
|
112
|
+
* How long one beat waits for its answer before it counts as missed.
|
|
113
|
+
*
|
|
114
|
+
* Deliberately far below the prompt budget. `requestWithTracing` takes no
|
|
115
|
+
* per-request deadline, so a beat issued on the held client inherits the
|
|
116
|
+
* socket's own — the whole prompt timeout — and a beat that waits that long
|
|
117
|
+
* cannot report a miss inside any window a caller waits in, nor before three
|
|
118
|
+
* more beats have been sent after it.
|
|
119
|
+
*/
|
|
120
|
+
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 15 * 1000;
|
|
121
|
+
/**
|
|
122
|
+
* Shortest wait for an unanswered beat worth putting in a death report, since
|
|
123
|
+
* the report states it in whole seconds and anything under one is what a
|
|
124
|
+
* healthy round trip looks like at the instant a socket closes.
|
|
125
|
+
*/
|
|
126
|
+
const MIN_REPORTED_BEAT_WAIT_MS = 1_000;
|
|
127
|
+
/**
|
|
128
|
+
* How long to wait after Clark acks the prompt for the generation turn to
|
|
129
|
+
* reach a settled terminal state (completed after quiet settle, cancelled,
|
|
130
|
+
* or needs_decision).
|
|
131
|
+
*/
|
|
132
|
+
const DEFAULT_TURN_TIMEOUT_MS = 10 * 60 * 1000;
|
|
133
|
+
/**
|
|
134
|
+
* What a caller is told when the live edit backing its turn is gone. The cause
|
|
135
|
+
* is appended by whichever path noticed: a close code and heartbeat account
|
|
136
|
+
* when the socket dropped, nothing when SABS said so outright.
|
|
137
|
+
*/
|
|
138
|
+
const LIVE_EDIT_TERMINATED_REPLY = "The Superblocks live-edit session terminated before the turn finished, so nothing was built. Send the prompt again to start it over.";
|
|
139
|
+
/**
|
|
140
|
+
* How long a turn waits for Clark's first push before suspecting the sandbox is
|
|
141
|
+
* not reporting into this socket.
|
|
142
|
+
*
|
|
143
|
+
* The dev server routes a turn's pushes to the peer that asked for it, so the
|
|
144
|
+
* usual cause of silence here is this connection itself — dropped, or opened
|
|
145
|
+
* against a session that had already lost the turn. A healthy turn reports
|
|
146
|
+
* `generating` within a second of the acknowledgment, so this window is
|
|
147
|
+
* silence, not slowness.
|
|
148
|
+
*/
|
|
149
|
+
const DEFAULT_PUSH_WAIT_MS = 10_000;
|
|
150
|
+
/** How long to wait for the flushed pushes after reattaching. */
|
|
151
|
+
const DEFAULT_REATTACH_WAIT_MS = 20_000;
|
|
152
|
+
/**
|
|
153
|
+
* How long a turn keeps trying to get back onto its live edit after a close it
|
|
154
|
+
* did not ask for.
|
|
155
|
+
*
|
|
156
|
+
* A close is not an answer about the sandbox: the same 1006 covers a proxy
|
|
157
|
+
* idle-reap, a load-balancer hiccup and a dev server that died. Reaching the
|
|
158
|
+
* live edit again settles it, and failing to reach it for this long is the
|
|
159
|
+
* evidence the close itself never was.
|
|
160
|
+
*/
|
|
161
|
+
const DEFAULT_RECONNECT_GRACE_MS = 20_000;
|
|
162
|
+
/**
|
|
163
|
+
* How many times one turn goes back for a live edit that keeps dropping it.
|
|
164
|
+
*
|
|
165
|
+
* A sandbox that closes every socket it accepts is not one a turn can finish
|
|
166
|
+
* on, and each attempt is cheap enough that an uncapped retry would chase it
|
|
167
|
+
* for the whole turn budget.
|
|
168
|
+
*/
|
|
169
|
+
const DEFAULT_RECONNECT_ATTEMPTS = 3;
|
|
170
|
+
/**
|
|
171
|
+
* After the job settles, how long to wait for the dev server to finish flushing
|
|
172
|
+
* Clark's files before giving up on checkpointing them.
|
|
173
|
+
*/
|
|
174
|
+
const DEFAULT_READY_WAIT_MS = 30_000;
|
|
175
|
+
/**
|
|
176
|
+
* The role the dev server classifies this connection as. A gateway peer gets
|
|
177
|
+
* Clark's pushes for the turns it starts and its own sandbox-liveness signal,
|
|
178
|
+
* without being counted as a browser editor: it never takes the editor
|
|
179
|
+
* binding, and a user's own tab keeps its presence and its lock.
|
|
180
|
+
*
|
|
181
|
+
* Offering the role is not the same as being given it. A `ws` server echoes the
|
|
182
|
+
* first subprotocol it was offered whether or not it knows the name, so a dev
|
|
183
|
+
* server that predates the role accepts this socket, answers `gateway`, and
|
|
184
|
+
* still classifies it as a library socket that never receives Clark's pushes.
|
|
185
|
+
* The handshake cannot tell the two apart; the acknowledgment can, because only
|
|
186
|
+
* a server that routed the turn to this peer can name the turn back. See
|
|
187
|
+
* {@link WebSocketSessionPeer.headlessTurnHandle}.
|
|
188
|
+
*/
|
|
189
|
+
const GATEWAY_PROTOCOL = "gateway";
|
|
190
|
+
const DEFAULT_LLM = {
|
|
191
|
+
coreModel: "claude-sonnet-4-5",
|
|
192
|
+
provider: "bedrock",
|
|
193
|
+
};
|
|
194
|
+
/**
|
|
195
|
+
* Reads a {@link LiveEdit} off the caller-facing pair, which names the same two
|
|
196
|
+
* things with the prompt's field names.
|
|
197
|
+
*/
|
|
198
|
+
function liveEditOf(source) {
|
|
199
|
+
return {
|
|
200
|
+
...(source.liveEditId ? { id: source.liveEditId } : {}),
|
|
201
|
+
url: source.liveEditUrl,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* How long a dead start stays worth reporting. Past this the caller it was
|
|
206
|
+
* recorded for is not coming back for it, and the record is only holding
|
|
207
|
+
* memory.
|
|
208
|
+
*/
|
|
209
|
+
const DEFAULT_FAILED_START_TTL_MS = 30 * 60 * 1000;
|
|
210
|
+
/**
|
|
211
|
+
* Bound on distinct apps holding a dead start at once, so a pathological caller
|
|
212
|
+
* cannot grow this without limit.
|
|
213
|
+
*/
|
|
214
|
+
const MAX_FAILED_STARTS = 256;
|
|
215
|
+
/**
|
|
216
|
+
* Dead starts, scoped to the caller whose prompt died and bounded in both age
|
|
217
|
+
* and count.
|
|
218
|
+
*
|
|
219
|
+
* Scoped because a connect failure is not a fact about the application:
|
|
220
|
+
* `connectLiveEdit` is where Superblocks authorizes the caller, and Superblocks
|
|
221
|
+
* authorizes per application, so a caller with no rights to this one still
|
|
222
|
+
* reaches the failure path. Keyed any wider, its record is what everyone who
|
|
223
|
+
* *does* have rights reads as the app's status - by application alone that was
|
|
224
|
+
* every organization, and by organization it was every colleague. The record
|
|
225
|
+
* exists so the caller who just failed gets an honest answer and can retry, and
|
|
226
|
+
* that caller is who it is keyed to.
|
|
227
|
+
*
|
|
228
|
+
* Bounded because nothing else evicts: an app whose start failed and that is
|
|
229
|
+
* never retried would otherwise hold its entry for the life of the process.
|
|
230
|
+
*/
|
|
231
|
+
class FailedStarts {
|
|
232
|
+
entries = new Map();
|
|
233
|
+
ttlMs;
|
|
234
|
+
maxEntries;
|
|
235
|
+
constructor(ttlMs, maxEntries) {
|
|
236
|
+
this.ttlMs = ttlMs ?? DEFAULT_FAILED_START_TTL_MS;
|
|
237
|
+
this.maxEntries = maxEntries ?? MAX_FAILED_STARTS;
|
|
238
|
+
}
|
|
239
|
+
set(scope, turn) {
|
|
240
|
+
const key = this.keyOf(scope);
|
|
241
|
+
// Re-inserted rather than updated in place, so this key becomes the newest
|
|
242
|
+
// and the eviction below stays in insertion order.
|
|
243
|
+
this.entries.delete(key);
|
|
244
|
+
this.entries.set(key, { at: Date.now(), turn });
|
|
245
|
+
for (const oldest of this.entries.keys()) {
|
|
246
|
+
if (this.entries.size <= this.maxEntries) {
|
|
247
|
+
break;
|
|
248
|
+
}
|
|
249
|
+
this.entries.delete(oldest);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
get(scope) {
|
|
253
|
+
const key = this.keyOf(scope);
|
|
254
|
+
const entry = this.entries.get(key);
|
|
255
|
+
if (!entry) {
|
|
256
|
+
return undefined;
|
|
257
|
+
}
|
|
258
|
+
if (Date.now() - entry.at >= this.ttlMs) {
|
|
259
|
+
this.entries.delete(key);
|
|
260
|
+
return undefined;
|
|
261
|
+
}
|
|
262
|
+
return entry.turn;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Forget every caller's record for this application. The reads are scoped,
|
|
266
|
+
* but acceptance is not: a prompt Superblocks took means the app has a live
|
|
267
|
+
* turn, and nobody's dead start outranks that.
|
|
268
|
+
*/
|
|
269
|
+
clear(applicationId) {
|
|
270
|
+
for (const key of [...this.entries.keys()]) {
|
|
271
|
+
if (key.endsWith(`\u0000${applicationId}`)) {
|
|
272
|
+
this.entries.delete(key);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Separated by NUL because no id can contain one. Any printable separator is
|
|
278
|
+
* a character an id could carry, and an id that carried it would key two
|
|
279
|
+
* different callers to the same entry.
|
|
280
|
+
*/
|
|
281
|
+
keyOf(scope) {
|
|
282
|
+
return `${scope.organizationId}\u0000${scope.userId}\u0000${scope.applicationId}`;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
export function buildSessionUrl(input) {
|
|
286
|
+
// liveEditUrl is server-sourced (connectLiveEdit), not caller-supplied, so a
|
|
287
|
+
// host allowlist would mostly duplicate the Superblocks API's own trust
|
|
288
|
+
// boundary. The scheme check still matters: anything other than http(s)
|
|
289
|
+
// would let a compromised response smuggle the session token off-protocol.
|
|
290
|
+
const url = new URL("/sb-updates", input.liveEditUrl);
|
|
291
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
292
|
+
throw new Error(`Prompt forward failed: live-edit URL must be http(s), got ${url.protocol}`);
|
|
293
|
+
}
|
|
294
|
+
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
|
295
|
+
url.searchParams.set("peerId", input.peerId);
|
|
296
|
+
if (input.userId) {
|
|
297
|
+
url.searchParams.set("userId", input.userId);
|
|
298
|
+
}
|
|
299
|
+
url.searchParams.set("applicationId", input.applicationId);
|
|
300
|
+
return url;
|
|
301
|
+
}
|
|
302
|
+
function isAuthRejection(error) {
|
|
303
|
+
return error.code === 401 || error.code === 403;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Whether a host answered the upgrade by denying it serves this live edit, as
|
|
307
|
+
* opposed to not answering it.
|
|
308
|
+
*
|
|
309
|
+
* The difference is the whole reason the connect loop retries. A booting
|
|
310
|
+
* sandbox binds its port before attaching the socket handler and drops the
|
|
311
|
+
* upgrade in silence, so a cold start looks like an attempt timeout and is
|
|
312
|
+
* worth waiting out on the same host. A status in the 404 family is something
|
|
313
|
+
* answering on that host and saying the session is not there - which is what a
|
|
314
|
+
* stale session -> host binding looks like from the outside, and no amount of
|
|
315
|
+
* retrying the same host will fix it.
|
|
316
|
+
*
|
|
317
|
+
* 5xx is deliberately not included: that is the session's own host reporting it
|
|
318
|
+
* is not ready yet, which is the retry case, not a move.
|
|
319
|
+
*/
|
|
320
|
+
function isHostRejection(error) {
|
|
321
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
322
|
+
// The wording is ws's own for a non-101 response to the handshake.
|
|
323
|
+
const status = /Unexpected server response: (\d{3})/.exec(message)?.[1];
|
|
324
|
+
return status === "404" || status === "410";
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* ISocket gives up on an unanswered request in two different wordings —
|
|
328
|
+
* "Request timed out after N seconds" when its own no-response timer fires,
|
|
329
|
+
* and "Abort timeout" when the underlying signal aborts first. Both mean the
|
|
330
|
+
* same thing to a caller, and neither says what to do about it.
|
|
331
|
+
*/
|
|
332
|
+
function isTimeout(error) {
|
|
333
|
+
return /timed out|timeout/i.test(error.message ?? "");
|
|
334
|
+
}
|
|
335
|
+
function isSocketOpen(ws) {
|
|
336
|
+
return ws.readyState === WebSocket.OPEN;
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Says what a rejected socket request actually failed on.
|
|
340
|
+
*
|
|
341
|
+
* Every rejection this client produces is a plain `{ code, message }` object
|
|
342
|
+
* and never an `Error` — a refusal from the wire, a no-response timeout, and a
|
|
343
|
+
* closed connection all take that shape. `String()` on one of those yields
|
|
344
|
+
* "[object Object]" and throws away the only account of the cause there is.
|
|
345
|
+
*/
|
|
346
|
+
function describeSocketError(error) {
|
|
347
|
+
if (error instanceof Error) {
|
|
348
|
+
return error.message;
|
|
349
|
+
}
|
|
350
|
+
if (typeof error !== "object" || error === null) {
|
|
351
|
+
return String(error);
|
|
352
|
+
}
|
|
353
|
+
const { code, message } = error;
|
|
354
|
+
if (typeof message === "string" && message.length > 0) {
|
|
355
|
+
return code === undefined ? message : `${message} (code ${code})`;
|
|
356
|
+
}
|
|
357
|
+
try {
|
|
358
|
+
return JSON.stringify(error);
|
|
359
|
+
}
|
|
360
|
+
catch {
|
|
361
|
+
return String(error);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* What the socket said on its way out, appended to the disconnect reply.
|
|
366
|
+
*
|
|
367
|
+
* The close frame is the only account of why a turn died that reaches the
|
|
368
|
+
* caller: 1006 is a proxy or network kill that sent no frame at all, 1001 a
|
|
369
|
+
* sandbox going away, and an application code is a cause the dev server chose
|
|
370
|
+
* to name. This process's console is not visible to whoever asked for the
|
|
371
|
+
* build, so a bare "it disconnected" leaves them nothing to act on.
|
|
372
|
+
*/
|
|
373
|
+
function describeClose(close) {
|
|
374
|
+
const parts = [];
|
|
375
|
+
if (close.code !== undefined) {
|
|
376
|
+
parts.push(`code ${close.code}`);
|
|
377
|
+
}
|
|
378
|
+
if (close.reason) {
|
|
379
|
+
parts.push(close.reason);
|
|
380
|
+
}
|
|
381
|
+
const closed = parts.length > 0 ? ` The session closed with ${parts.join(": ")}.` : "";
|
|
382
|
+
// A death that sends no close frame reports only 1006, which names nothing.
|
|
383
|
+
// The error that arrived just before it is the account of what happened.
|
|
384
|
+
return close.error ? `${closed} Socket error: ${close.error}.` : closed;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Whether the close is the live edit naming its own end.
|
|
388
|
+
*
|
|
389
|
+
* 4000-4999 is the WebSocket private range: nothing but the peer's own code
|
|
390
|
+
* puts a value there, so a close carrying one is a statement - a lost live-edit
|
|
391
|
+
* lock, a deliberate shutdown. Every other code belongs to the transport, and
|
|
392
|
+
* 1006 in particular is what a death with no close frame reports, which is
|
|
393
|
+
* equally what a reaped proxy connection reports. Those say the socket ended;
|
|
394
|
+
* only the application range says the live edit did.
|
|
395
|
+
*/
|
|
396
|
+
function isApplicationClose(close) {
|
|
397
|
+
return close.code !== undefined && close.code >= 4000 && close.code <= 4999;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* How the liveness beat was faring when the socket went, for the reply.
|
|
401
|
+
*
|
|
402
|
+
* Beats landing right up to the close mean the connection was carrying traffic
|
|
403
|
+
* and cannot have been reaped for idleness, which points at the sandbox rather
|
|
404
|
+
* than the path to it. A run of failures says the opposite. The close code
|
|
405
|
+
* distinguishes neither on its own.
|
|
406
|
+
*
|
|
407
|
+
* Several of these counts can stand at once, because only an answered beat
|
|
408
|
+
* clears the account: a refusal answers one beat and leaves every record of
|
|
409
|
+
* what went missing behind it. The order below is therefore most-specific
|
|
410
|
+
* first, and a stated cause outranks an inferred one - a refusal is the only
|
|
411
|
+
* outcome where the peer said why, so it is reported ahead of a silence that
|
|
412
|
+
* merely happened to look excusable.
|
|
413
|
+
*/
|
|
414
|
+
function describeHeartbeat(heartbeat, at) {
|
|
415
|
+
return (describeSettledBeats(heartbeat, at) + describeOutstandingBeat(heartbeat, at));
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* The beat that never got an answer because the socket went first.
|
|
419
|
+
*
|
|
420
|
+
* Appended to whichever clause fired rather than being one of them: an
|
|
421
|
+
* outstanding beat is a fact about the end of the connection, and it stands
|
|
422
|
+
* alongside whatever the settled beats say. A refusal followed by silence is a
|
|
423
|
+
* different death from refusals arriving right up to the close, and the
|
|
424
|
+
* settled counts alone cannot tell those apart - only an answered beat clears
|
|
425
|
+
* them, and this beat was never answered at all.
|
|
426
|
+
*
|
|
427
|
+
* Sub-second waits say nothing and are left out. A beat is in flight for
|
|
428
|
+
* exactly as long as an answer takes, so a close landing inside a healthy round
|
|
429
|
+
* trip would otherwise report a silence of "0s" and indict the request channel
|
|
430
|
+
* for the very exchange that proves it works.
|
|
431
|
+
*/
|
|
432
|
+
function describeOutstandingBeat(heartbeat, at) {
|
|
433
|
+
if (heartbeat.pendingSince === undefined) {
|
|
434
|
+
return "";
|
|
435
|
+
}
|
|
436
|
+
const waitingMs = at - heartbeat.pendingSince;
|
|
437
|
+
if (waitingMs < MIN_REPORTED_BEAT_WAIT_MS) {
|
|
438
|
+
return "";
|
|
439
|
+
}
|
|
440
|
+
const seconds = Math.round(waitingMs / 1000);
|
|
441
|
+
return ` A beat sent ${seconds}s earlier had still not been answered.`;
|
|
442
|
+
}
|
|
443
|
+
function describeSettledBeats(heartbeat, at) {
|
|
444
|
+
if (heartbeat.failures > 0) {
|
|
445
|
+
const beats = heartbeat.failures === 1 ? "beat" : "beats";
|
|
446
|
+
const cause = heartbeat.lastError ? ` (${heartbeat.lastError})` : "";
|
|
447
|
+
return ` ${heartbeat.failures} liveness ${beats} failed before it${cause}.`;
|
|
448
|
+
}
|
|
449
|
+
if (heartbeat.refusals > 0) {
|
|
450
|
+
const beats = heartbeat.refusals === 1 ? "beat" : "beats";
|
|
451
|
+
const cause = heartbeat.lastError ? ` (${heartbeat.lastError})` : "";
|
|
452
|
+
// A refusal is an answer, so the connection worked and the peer named its
|
|
453
|
+
// own reason for the turn dying. Reported before a lapse because a refusal
|
|
454
|
+
// does not clear one: an earlier excused silence would otherwise outlive
|
|
455
|
+
// the beat that explained itself, and describing the socket as merely
|
|
456
|
+
// quiet throws away the only account the caller can act on.
|
|
457
|
+
return ` ${heartbeat.refusals} liveness ${beats} came back refused${cause}.`;
|
|
458
|
+
}
|
|
459
|
+
if (heartbeat.lapses > 0) {
|
|
460
|
+
const beats = heartbeat.lapses === 1 ? "beat" : "beats";
|
|
461
|
+
// A silent request channel on a connection that was demonstrably carrying
|
|
462
|
+
// traffic. Reporting this as a plain failure would point at the network,
|
|
463
|
+
// which is the one thing it rules out.
|
|
464
|
+
return ` ${heartbeat.lapses} liveness ${beats} went unanswered while the socket was still delivering data.`;
|
|
465
|
+
}
|
|
466
|
+
if (heartbeat.lastSuccessAt !== undefined) {
|
|
467
|
+
const seconds = Math.round((at - heartbeat.lastSuccessAt) / 1000);
|
|
468
|
+
return ` The last liveness beat succeeded ${seconds}s earlier.`;
|
|
469
|
+
}
|
|
470
|
+
if (heartbeat.sent > 0) {
|
|
471
|
+
const beats = heartbeat.sent === 1 ? "beat" : "beats";
|
|
472
|
+
return ` ${heartbeat.sent} liveness ${beats} sent, none answered.`;
|
|
473
|
+
}
|
|
474
|
+
return " No liveness beat had run yet.";
|
|
475
|
+
}
|
|
476
|
+
function decisionFields(decision) {
|
|
477
|
+
return decision ? { decision } : {};
|
|
478
|
+
}
|
|
479
|
+
async function attempt(url, timeoutMs, onSocketError) {
|
|
480
|
+
const socket = new WebSocket(url, GATEWAY_PROTOCOL);
|
|
481
|
+
return await new Promise((resolve, reject) => {
|
|
482
|
+
const timer = setTimeout(() => {
|
|
483
|
+
socket.terminate();
|
|
484
|
+
reject(new Error("session is not accepting sockets yet"));
|
|
485
|
+
}, timeoutMs);
|
|
486
|
+
const settle = (outcome) => {
|
|
487
|
+
clearTimeout(timer);
|
|
488
|
+
socket.off("open", onOpen);
|
|
489
|
+
socket.off("error", onError);
|
|
490
|
+
outcome();
|
|
491
|
+
};
|
|
492
|
+
const onOpen = () => settle(() => {
|
|
493
|
+
// ISocket attaches its own handlers a tick later. An 'error' landing in
|
|
494
|
+
// that gap would have no listener, and node throws on an unhandled
|
|
495
|
+
// emitter 'error' rather than reporting it to the caller. Recording it
|
|
496
|
+
// rather than discarding it is what makes a later 1006 explainable.
|
|
497
|
+
socket.on("error", onSocketError);
|
|
498
|
+
resolve(socket);
|
|
499
|
+
});
|
|
500
|
+
const onError = (error) => settle(() => reject(error));
|
|
501
|
+
socket.once("open", onOpen);
|
|
502
|
+
socket.once("error", onError);
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* Waits for a lookup, but never past the deadline the caller is already bound
|
|
507
|
+
* by, and never raises.
|
|
508
|
+
*
|
|
509
|
+
* The connect loop clamps every socket attempt to what is left of its budget;
|
|
510
|
+
* a re-resolve is a Superblocks API round trip made from inside that loop and
|
|
511
|
+
* has to answer to the same clock. A hung one would otherwise hold the loop
|
|
512
|
+
* open for as long as it hangs, and `open` would return later than the
|
|
513
|
+
* deadline it promised by that much. Giving up on it leaves the host already
|
|
514
|
+
* in hand, which is the same fallback a failed lookup gets.
|
|
515
|
+
*/
|
|
516
|
+
async function withinDeadline(lookup, deadline) {
|
|
517
|
+
const remaining = deadline - Date.now();
|
|
518
|
+
if (remaining <= 0) {
|
|
519
|
+
return undefined;
|
|
520
|
+
}
|
|
521
|
+
let timer;
|
|
522
|
+
try {
|
|
523
|
+
return await Promise.race([
|
|
524
|
+
lookup.catch(() => undefined),
|
|
525
|
+
new Promise((resolve) => {
|
|
526
|
+
timer = setTimeout(() => resolve(undefined), remaining);
|
|
527
|
+
}),
|
|
528
|
+
]);
|
|
529
|
+
}
|
|
530
|
+
finally {
|
|
531
|
+
if (timer) {
|
|
532
|
+
clearTimeout(timer);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
async function open(url, attemptTimeoutMs, deadline, onSocketError,
|
|
537
|
+
/**
|
|
538
|
+
* Where the live edit is now, asked for only once and only when the host in
|
|
539
|
+
* hand is denying it. Absent keeps the prompt on the host it was given, and
|
|
540
|
+
* so does a hook that answers with nothing or fails: the host already in hand
|
|
541
|
+
* is the better fallback than abandoning the attempt.
|
|
542
|
+
*/
|
|
543
|
+
nextHost) {
|
|
544
|
+
let current = url;
|
|
545
|
+
let lastError;
|
|
546
|
+
let consecutiveRejections = 0;
|
|
547
|
+
let asks = 0;
|
|
548
|
+
while (Date.now() < deadline) {
|
|
549
|
+
try {
|
|
550
|
+
return await attempt(current, Math.min(attemptTimeoutMs, deadline - Date.now()), onSocketError);
|
|
551
|
+
}
|
|
552
|
+
catch (error) {
|
|
553
|
+
lastError = error;
|
|
554
|
+
consecutiveRejections = isHostRejection(error)
|
|
555
|
+
? consecutiveRejections + 1
|
|
556
|
+
: 0;
|
|
557
|
+
// Asked for only after the host has said no more than once: a single
|
|
558
|
+
// rejection can be an ingress route that exists a moment before the pod
|
|
559
|
+
// behind it does. Asked again only after another full streak, and never
|
|
560
|
+
// more than a few times per open.
|
|
561
|
+
//
|
|
562
|
+
// The spacing is the point, not politeness. The first ask lands about a
|
|
563
|
+
// second after the resolve that produced this host, often before SABS
|
|
564
|
+
// has observed the pod is gone, so it can hand back the same dead
|
|
565
|
+
// binding. Asking once and never again would spend the rest of the
|
|
566
|
+
// budget on a host already known to be denying the session.
|
|
567
|
+
if (nextHost &&
|
|
568
|
+
asks < MAX_RERESOLVES_PER_OPEN &&
|
|
569
|
+
consecutiveRejections >= HOST_REJECTIONS_BEFORE_RERESOLVE) {
|
|
570
|
+
asks += 1;
|
|
571
|
+
// Re-armed: the next ask waits for a fresh streak rather than
|
|
572
|
+
// following on the very next rejection.
|
|
573
|
+
consecutiveRejections = 0;
|
|
574
|
+
const moved = await withinDeadline(nextHost(), deadline);
|
|
575
|
+
if (moved && moved.href !== current.href) {
|
|
576
|
+
current = moved;
|
|
577
|
+
// Straight to the attempt: the delay below is for a host worth
|
|
578
|
+
// asking again, and this is a different one.
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
const remaining = deadline - Date.now();
|
|
583
|
+
if (remaining <= 0) {
|
|
584
|
+
break;
|
|
585
|
+
}
|
|
586
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(CONNECT_RETRY_DELAY_MS, remaining)));
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
throw new Error(`Prompt forward timed out: ${current.host} did not accept a live-edit socket (${lastError instanceof Error ? lastError.message : String(lastError)})${current.host !== url.host ? ` after moving from ${url.host}` : ""}`);
|
|
590
|
+
}
|
|
591
|
+
export class WebSocketSessionPeer {
|
|
592
|
+
options;
|
|
593
|
+
held = new Map();
|
|
594
|
+
/**
|
|
595
|
+
* Every server -> editor method the dev server has called on this peer.
|
|
596
|
+
* Diagnostic only: it answers "did that push ever arrive" without guessing
|
|
597
|
+
* from what the collector chose to keep.
|
|
598
|
+
*/
|
|
599
|
+
inboundMethods = new Set();
|
|
600
|
+
/**
|
|
601
|
+
* In-flight opens, so overlapping prompts for one app share a socket instead
|
|
602
|
+
* of each opening their own and orphaning all but the last.
|
|
603
|
+
*/
|
|
604
|
+
connecting = new Map();
|
|
605
|
+
/**
|
|
606
|
+
* Turns keyed by application. An entry outlives the call that started it and
|
|
607
|
+
* is kept after finishing so a late poll still learns the outcome; the next
|
|
608
|
+
* prompt for the same app replaces it.
|
|
609
|
+
*/
|
|
610
|
+
turns = new Map();
|
|
611
|
+
/**
|
|
612
|
+
* Last plan Clark proposed per application, so approve/refine can re-send
|
|
613
|
+
* planContext without the MCP client echoing the full markdown.
|
|
614
|
+
*/
|
|
615
|
+
pendingPlans = new Map();
|
|
616
|
+
/**
|
|
617
|
+
* Last needs_decision payload (plan or multi_choice) per application, so
|
|
618
|
+
* approve/refine via edit_app can reuse the pending plan after
|
|
619
|
+
* check_app_progress drained the finished turn.
|
|
620
|
+
*/
|
|
621
|
+
pendingDecisions = new Map();
|
|
622
|
+
/**
|
|
623
|
+
* Applications that have dispatched at least one BUILD turn. Survives
|
|
624
|
+
* clearing the pending plan, which happens as soon as an approved build
|
|
625
|
+
* starts — the point of the latch is the turns that come after it.
|
|
626
|
+
*/
|
|
627
|
+
buildModeApplications = new Set();
|
|
628
|
+
/**
|
|
629
|
+
* Turns that died before Superblocks ever saw them, so no {@link ActiveTurn}
|
|
630
|
+
* was ever created to hold the outcome. Read by
|
|
631
|
+
* {@link WebSocketSessionPeer.turnState} and
|
|
632
|
+
* {@link WebSocketSessionPeer.awaitTurn} after the live map, and cleared when
|
|
633
|
+
* a prompt for the app is accepted.
|
|
634
|
+
*/
|
|
635
|
+
failedStarts;
|
|
636
|
+
constructor(options = {}) {
|
|
637
|
+
this.options = options;
|
|
638
|
+
// Built here rather than as a field initializer: those run before the
|
|
639
|
+
// constructor's parameter properties are assigned, so `this.options` would
|
|
640
|
+
// still be undefined.
|
|
641
|
+
this.failedStarts = new FailedStarts(options.failedStartTtlMs, options.maxFailedStarts);
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Plan waiting on human approve/refine for this application, if any.
|
|
645
|
+
*/
|
|
646
|
+
pendingPlan(applicationId) {
|
|
647
|
+
return this.pendingPlans.get(applicationId);
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Full needs_decision payload waiting on a human for this application.
|
|
651
|
+
*/
|
|
652
|
+
pendingDecision(applicationId) {
|
|
653
|
+
return this.pendingDecisions.get(applicationId);
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Remember a turn that never reached Superblocks. See
|
|
657
|
+
* {@link SessionPeer.recordFailedStart}.
|
|
658
|
+
*/
|
|
659
|
+
recordFailedStart(input) {
|
|
660
|
+
const running = this.turns.get(input.applicationId);
|
|
661
|
+
if (running && !running.finished) {
|
|
662
|
+
// A turn is already running for this app, and a second prompt that could
|
|
663
|
+
// not open its own session did not end it. Recording anyway is what makes
|
|
664
|
+
// the precedence in `turnState` unreadable: the same pair of facts - one
|
|
665
|
+
// finished turn, one record - would mean "the record came later" when the
|
|
666
|
+
// turn had already ended and "the turn came later" when it ended after,
|
|
667
|
+
// with nothing left in memory to say which. Skipping keeps every stored
|
|
668
|
+
// record older than any finished turn it can be read beside.
|
|
669
|
+
console.warn("gateway clark turn never started, but one is running", {
|
|
670
|
+
applicationId: input.applicationId,
|
|
671
|
+
organizationId: input.organizationId,
|
|
672
|
+
reason: input.reason,
|
|
673
|
+
userId: input.userId,
|
|
674
|
+
});
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
console.warn("gateway clark turn never started", {
|
|
678
|
+
applicationId: input.applicationId,
|
|
679
|
+
organizationId: input.organizationId,
|
|
680
|
+
reason: input.reason,
|
|
681
|
+
userId: input.userId,
|
|
682
|
+
});
|
|
683
|
+
this.failedStarts.set(input, {
|
|
684
|
+
generationStatus: "live_edit_terminated",
|
|
685
|
+
reply: input.reason,
|
|
686
|
+
// Nothing ran, so nothing is viewable. Reporting "ready" here is how a
|
|
687
|
+
// caller comes to show the untouched starter template as the app.
|
|
688
|
+
viewStatus: "unknown",
|
|
689
|
+
});
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Where this application's turn stands for this caller, read without draining
|
|
693
|
+
* the activity an {@link WebSocketSessionPeer.awaitTurn} poll is owed.
|
|
694
|
+
*
|
|
695
|
+
* Order matters, and it is recency. A turn still *running* is live work and
|
|
696
|
+
* outranks everything - a second prompt failing to open its own session does
|
|
697
|
+
* not end it. Failing that, this caller's own dead start outranks a turn that
|
|
698
|
+
* already finished: `turns` keeps finished turns, so without this a follow-up
|
|
699
|
+
* whose connect failed would be answered by the previous turn's outcome and
|
|
700
|
+
* its reply, reporting an edit as landed that never ran. And it is read rather
|
|
701
|
+
* than superseded by deleting from `turns`, because that map is keyed by
|
|
702
|
+
* application while a dead start belongs to one caller - deleting would erase
|
|
703
|
+
* a colleague's outcome to answer this caller's question.
|
|
704
|
+
*
|
|
705
|
+
* Reading recency out of the order rather than out of a clock holds because
|
|
706
|
+
* of the two rules either side of this one: accepting a prompt clears every
|
|
707
|
+
* caller's record for the app, and no record is made while a turn is running.
|
|
708
|
+
* Together they leave every stored record newer than any finished turn it can
|
|
709
|
+
* be read beside, so "the record wins" is not a guess about which came first.
|
|
710
|
+
*/
|
|
711
|
+
turnState(input) {
|
|
712
|
+
const turn = this.turns.get(input.applicationId);
|
|
713
|
+
if (turn && !turn.finished) {
|
|
714
|
+
return { phase: "running" };
|
|
715
|
+
}
|
|
716
|
+
const failed = this.failedStarts.get(input);
|
|
717
|
+
if (failed) {
|
|
718
|
+
return { generationStatus: failed.generationStatus, phase: "finished" };
|
|
719
|
+
}
|
|
720
|
+
if (!turn?.finished) {
|
|
721
|
+
return undefined;
|
|
722
|
+
}
|
|
723
|
+
return {
|
|
724
|
+
...(turn.finished.checkpointCommitId
|
|
725
|
+
? { checkpointCommitId: turn.finished.checkpointCommitId }
|
|
726
|
+
: {}),
|
|
727
|
+
generationStatus: turn.finished.generationStatus,
|
|
728
|
+
phase: "finished",
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
clearPendingPlan(applicationId) {
|
|
732
|
+
this.pendingPlans.delete(applicationId);
|
|
733
|
+
this.pendingDecisions.delete(applicationId);
|
|
734
|
+
}
|
|
735
|
+
clearPendingDecision(applicationId) {
|
|
736
|
+
this.pendingPlans.delete(applicationId);
|
|
737
|
+
this.pendingDecisions.delete(applicationId);
|
|
738
|
+
}
|
|
739
|
+
markBuildMode(applicationId) {
|
|
740
|
+
this.buildModeApplications.add(applicationId);
|
|
741
|
+
}
|
|
742
|
+
buildMode(applicationId) {
|
|
743
|
+
return this.buildModeApplications.has(applicationId);
|
|
744
|
+
}
|
|
745
|
+
isHeld(applicationId) {
|
|
746
|
+
const entry = this.held.get(applicationId);
|
|
747
|
+
return Boolean(entry && isSocketOpen(entry.ws));
|
|
748
|
+
}
|
|
749
|
+
libraryBrowserContext(applicationId) {
|
|
750
|
+
const held = this.held.get(applicationId);
|
|
751
|
+
if (!held) {
|
|
752
|
+
return undefined;
|
|
753
|
+
}
|
|
754
|
+
return this.buildHeldBrowserContext(held);
|
|
755
|
+
}
|
|
756
|
+
async holdLiveEdit(input) {
|
|
757
|
+
const segments = String(input.sessionToken ?? "").split(".").length;
|
|
758
|
+
if (segments !== 3) {
|
|
759
|
+
throw new Error(`Live-edit hold failed: the live-edit credential is not a JWT (${segments} segment(s)); the session will reject it`);
|
|
760
|
+
}
|
|
761
|
+
if (!input.userId.trim()) {
|
|
762
|
+
throw new Error("Live-edit hold failed: userId is required to identify who the session is held for");
|
|
763
|
+
}
|
|
764
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
765
|
+
const deadline = Date.now() + timeoutMs;
|
|
766
|
+
const held = await this.ensureHeld({
|
|
767
|
+
applicationId: input.applicationId,
|
|
768
|
+
branchName: input.branch,
|
|
769
|
+
keepalive: input.keepalive,
|
|
770
|
+
liveEdit: liveEditOf(input),
|
|
771
|
+
userId: input.userId,
|
|
772
|
+
}, deadline);
|
|
773
|
+
if (input.apiKey) {
|
|
774
|
+
held.apiKey = input.apiKey;
|
|
775
|
+
}
|
|
776
|
+
if (input.organizationId) {
|
|
777
|
+
held.organizationId = input.organizationId;
|
|
778
|
+
}
|
|
779
|
+
held.sessionToken = input.sessionToken;
|
|
780
|
+
await this.resolveHeldProfileKey(held);
|
|
781
|
+
await this.resolveHeldOrchestrator(held);
|
|
782
|
+
}
|
|
783
|
+
release(applicationId) {
|
|
784
|
+
this.buildModeApplications.delete(applicationId);
|
|
785
|
+
const entry = this.held.get(applicationId);
|
|
786
|
+
if (!entry) {
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
this.drop(entry);
|
|
790
|
+
}
|
|
791
|
+
closeAll() {
|
|
792
|
+
for (const applicationId of [...this.held.keys()]) {
|
|
793
|
+
this.release(applicationId);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Sends the prompt and waits for Clark's whole turn.
|
|
798
|
+
*
|
|
799
|
+
* Kept for callers with nowhere to poll from — the HTTP capability routes
|
|
800
|
+
* answer a single request — and implemented on the same engine as
|
|
801
|
+
* {@link startPrompt} so there is one turn lifecycle, not two.
|
|
802
|
+
*/
|
|
803
|
+
async forwardPrompt(input) {
|
|
804
|
+
const started = await this.startPrompt(input);
|
|
805
|
+
if (started.status === "auth_required") {
|
|
806
|
+
return started;
|
|
807
|
+
}
|
|
808
|
+
const turnTimeoutMs = this.options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
|
|
809
|
+
const readyWaitMs = this.options.readyWaitMs ?? DEFAULT_READY_WAIT_MS;
|
|
810
|
+
// The turn's own budget already bounds this; the extra slack covers the
|
|
811
|
+
// checkpoint that follows Clark's terminal state.
|
|
812
|
+
const deadline = Date.now() + turnTimeoutMs + readyWaitMs + 1_000;
|
|
813
|
+
for (;;) {
|
|
814
|
+
const update = await this.awaitTurn({
|
|
815
|
+
applicationId: input.applicationId,
|
|
816
|
+
signal: input.signal,
|
|
817
|
+
waitMs: Math.max(0, deadline - Date.now()),
|
|
818
|
+
});
|
|
819
|
+
if (update.status === "finished") {
|
|
820
|
+
const { activity: _activity, elapsedMs: _elapsedMs, status: _status, ...finished } = update;
|
|
821
|
+
return {
|
|
822
|
+
...finished,
|
|
823
|
+
// See startPrompt, which resolved this: the turn may have run on a
|
|
824
|
+
// live edit the caller never named.
|
|
825
|
+
...(started.liveEditId ? { liveEditId: started.liveEditId } : {}),
|
|
826
|
+
status: "accepted",
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
if (update.status === "unknown" || Date.now() >= deadline) {
|
|
830
|
+
throw new Error("Prompt forward failed: the turn ended without reporting an outcome");
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
async startPrompt(input) {
|
|
835
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
836
|
+
const settleMs = this.options.settleMs ?? DEFAULT_SETTLE_MS;
|
|
837
|
+
// The session rejects a malformed credential with "Invalid token
|
|
838
|
+
// specified", which reads like the token expired rather than like the
|
|
839
|
+
// gateway never had one. Say which it is, without quoting the credential.
|
|
840
|
+
const segments = String(input.sessionToken ?? "").split(".").length;
|
|
841
|
+
if (segments !== 3) {
|
|
842
|
+
throw new Error(`Prompt forward failed: the live-edit credential is not a JWT (${segments} segment(s)); the session will reject it`);
|
|
843
|
+
}
|
|
844
|
+
if (!input.userId?.trim()) {
|
|
845
|
+
throw new Error("Prompt forward failed: userId is required to identify who the turn is running for");
|
|
846
|
+
}
|
|
847
|
+
const userId = input.userId;
|
|
848
|
+
// Connecting and waiting for the acknowledgement share one budget. Giving
|
|
849
|
+
// each the full timeout let a slow sandbox spend twice it, and the caller
|
|
850
|
+
// that is waiting on the tool call gives up long before that.
|
|
851
|
+
const deadline = Date.now() + timeoutMs;
|
|
852
|
+
const held = await this.ensureHeld({
|
|
853
|
+
applicationId: input.applicationId,
|
|
854
|
+
branchName: input.branch,
|
|
855
|
+
keepalive: input.keepalive,
|
|
856
|
+
liveEdit: liveEditOf(input),
|
|
857
|
+
...(input.resolveLiveEdit
|
|
858
|
+
? { resolveLiveEdit: input.resolveLiveEdit }
|
|
859
|
+
: {}),
|
|
860
|
+
userId,
|
|
861
|
+
}, deadline);
|
|
862
|
+
if (input.apiKey) {
|
|
863
|
+
held.apiKey = input.apiKey;
|
|
864
|
+
}
|
|
865
|
+
if (input.organizationId) {
|
|
866
|
+
held.organizationId = input.organizationId;
|
|
867
|
+
}
|
|
868
|
+
held.sessionToken = input.sessionToken;
|
|
869
|
+
await this.resolveHeldProfileKey(held);
|
|
870
|
+
await this.resolveHeldOrchestrator(held);
|
|
871
|
+
// A turn already running for this app is over as far as callers are
|
|
872
|
+
// concerned: its outcome belongs to the prompt being replaced.
|
|
873
|
+
this.supersedeTurn(input.applicationId);
|
|
874
|
+
this.clearPendingDecision(input.applicationId);
|
|
875
|
+
const turn = {
|
|
876
|
+
abort: new AbortController(),
|
|
877
|
+
activity: [],
|
|
878
|
+
collector: undefined,
|
|
879
|
+
delivered: 0,
|
|
880
|
+
heardAtLastReconnect: 0,
|
|
881
|
+
held,
|
|
882
|
+
listener: () => undefined,
|
|
883
|
+
normalized: undefined,
|
|
884
|
+
recentRuns: [],
|
|
885
|
+
reconnects: 0,
|
|
886
|
+
startedAt: Date.now(),
|
|
887
|
+
turnId: randomUUID(),
|
|
888
|
+
unsubscribe: () => undefined,
|
|
889
|
+
waiters: new Set(),
|
|
890
|
+
};
|
|
891
|
+
const report = (message) => {
|
|
892
|
+
this.note(turn, message);
|
|
893
|
+
input.onProgress?.(message);
|
|
894
|
+
};
|
|
895
|
+
const collector = new TurnCollector(report, { settleMs });
|
|
896
|
+
turn.collector = collector;
|
|
897
|
+
const normalized = new NormalizedTurnCollector({
|
|
898
|
+
collector,
|
|
899
|
+
onAuthRequired: (reason) => report(`Superblocks needs an interactive authorization (${reason})`),
|
|
900
|
+
onSessionStatus: (status) => {
|
|
901
|
+
if (status === "terminated") {
|
|
902
|
+
this.abandonTurnOnSessionTerminated(input.applicationId, turn);
|
|
903
|
+
}
|
|
904
|
+
},
|
|
905
|
+
turnId: turn.turnId,
|
|
906
|
+
});
|
|
907
|
+
turn.normalized = normalized;
|
|
908
|
+
// Both transports go through the normalized collector rather than into
|
|
909
|
+
// the collector directly: during the transition Clark's activity arrives
|
|
910
|
+
// on the socket and relayed through SABS, and the turn must not count the
|
|
911
|
+
// same message twice because it heard it twice.
|
|
912
|
+
const listener = (event) => normalized.fromSocket(event);
|
|
913
|
+
turn.listener = listener;
|
|
914
|
+
held.listeners.add(listener);
|
|
915
|
+
// The session the socket actually landed on, which is not the one the
|
|
916
|
+
// prompt named if the live edit had moved. The held record is the only
|
|
917
|
+
// authority: it was seeded from this prompt's pair and has since been
|
|
918
|
+
// replaced by anything the connect learned, so falling back to the
|
|
919
|
+
// prompt's id could only reinstate one the socket moved away from.
|
|
920
|
+
turn.unsubscribe = this.subscribeToPushes(held.liveEdit.id, normalized);
|
|
921
|
+
try {
|
|
922
|
+
return await this.sendPrompt({ held, input, listener, turn });
|
|
923
|
+
}
|
|
924
|
+
catch (error) {
|
|
925
|
+
// Every failure between here and the turn being handed to runTurn ends
|
|
926
|
+
// the turn where it stands, and the push subscription has to go with it.
|
|
927
|
+
turn.unsubscribe();
|
|
928
|
+
throw error;
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
/**
|
|
932
|
+
* Sends the prompt on a session already holding a subscribed turn, and hands
|
|
933
|
+
* the turn to {@link runTurn} once the session confirms it took it on.
|
|
934
|
+
*/
|
|
935
|
+
async sendPrompt(args) {
|
|
936
|
+
const { held, input, listener, turn } = args;
|
|
937
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
938
|
+
let integrations = [];
|
|
939
|
+
const editorContext = this.editorContextFor(held);
|
|
940
|
+
if (editorContext) {
|
|
941
|
+
try {
|
|
942
|
+
integrations = await loadIntegrationsForPrompt(editorContext);
|
|
943
|
+
}
|
|
944
|
+
catch (error) {
|
|
945
|
+
console.warn("gateway failed to prefill prompt integrations", {
|
|
946
|
+
applicationId: input.applicationId,
|
|
947
|
+
message: error instanceof Error ? error.message : String(error),
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
const browserContext = this.browserContextFor(held);
|
|
952
|
+
let acknowledgment;
|
|
953
|
+
try {
|
|
954
|
+
// The editor and the smoke client both send the credential twice: once
|
|
955
|
+
// as the RPC payload's `jwt` and once as the ISocket Authorization
|
|
956
|
+
// header. The session verifies the header; the AI service re-reads the
|
|
957
|
+
// payload. Dropping either breaks a different hop.
|
|
958
|
+
acknowledgment = await held.client.requestWithTracing(GENERATE_METHOD, {
|
|
959
|
+
...(browserContext ? { browserContext } : {}),
|
|
960
|
+
jwt: input.sessionToken,
|
|
961
|
+
llmConfig: {
|
|
962
|
+
coreModel: this.options.coreModel ?? DEFAULT_LLM.coreModel,
|
|
963
|
+
// The ai-service gates Clark's testing tools on this flag *and* a
|
|
964
|
+
// browser context, so the two travel together or not at all.
|
|
965
|
+
...(browserContext ? { flags: { isTestingEnabled: true } } : {}),
|
|
966
|
+
interleavedThinking: false,
|
|
967
|
+
provider: this.options.provider ?? DEFAULT_LLM.provider,
|
|
968
|
+
thinkingBudgetTokens: "0",
|
|
969
|
+
thinkingEnabled: false,
|
|
970
|
+
unifiedProviderEnabled: true,
|
|
971
|
+
},
|
|
972
|
+
mode: input.mode ?? "PLAN",
|
|
973
|
+
prompt: input.prompt,
|
|
974
|
+
promptContext: {
|
|
975
|
+
attachments: input.attachments ?? [],
|
|
976
|
+
entities: [],
|
|
977
|
+
integrations,
|
|
978
|
+
},
|
|
979
|
+
...(input.planContext
|
|
980
|
+
? {
|
|
981
|
+
planContext: {
|
|
982
|
+
...(input.planContext.approved !== undefined
|
|
983
|
+
? { approved: input.planContext.approved }
|
|
984
|
+
: {}),
|
|
985
|
+
...(input.planContext.plan
|
|
986
|
+
? { plan: input.planContext.plan }
|
|
987
|
+
: {}),
|
|
988
|
+
...(input.planContext.title
|
|
989
|
+
? { title: input.planContext.title }
|
|
990
|
+
: {}),
|
|
991
|
+
},
|
|
992
|
+
}
|
|
993
|
+
: {}),
|
|
994
|
+
...(input.responseMetadata
|
|
995
|
+
? { responseMetadata: input.responseMetadata }
|
|
996
|
+
: {}),
|
|
997
|
+
...(input.importSource ? { importSource: input.importSource } : {}),
|
|
998
|
+
queueMode: "interrupt",
|
|
999
|
+
requestId: turn.turnId,
|
|
1000
|
+
userPreferences: { suggestNextSteps: false },
|
|
1001
|
+
}, {}, `Bearer ${input.sessionToken}`);
|
|
1002
|
+
}
|
|
1003
|
+
catch (error) {
|
|
1004
|
+
held.listeners.delete(listener);
|
|
1005
|
+
const rejection = error;
|
|
1006
|
+
if (isAuthRejection(rejection)) {
|
|
1007
|
+
turn.unsubscribe();
|
|
1008
|
+
this.armIdle(held);
|
|
1009
|
+
this.armKeepalive(held);
|
|
1010
|
+
return {
|
|
1011
|
+
message: "Interactive auth required for datasource access",
|
|
1012
|
+
status: "auth_required",
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
if (isTimeout(rejection)) {
|
|
1016
|
+
throw new Error(`Prompt forward timed out: the live-edit session accepted the connection but did not acknowledge the prompt within ${timeoutMs}ms (${rejection.message}). The sandbox is usually still waking up when this happens, so retrying the same prompt normally works.`);
|
|
1017
|
+
}
|
|
1018
|
+
throw error instanceof Error
|
|
1019
|
+
? error
|
|
1020
|
+
: new Error(`Prompt forward failed: ${rejection.message ?? "unknown socket error"}`);
|
|
1021
|
+
}
|
|
1022
|
+
this.assertHeadlessTurnBound({
|
|
1023
|
+
acknowledgment,
|
|
1024
|
+
held,
|
|
1025
|
+
listener,
|
|
1026
|
+
turn,
|
|
1027
|
+
});
|
|
1028
|
+
input.onProgress?.("Superblocks accepted the prompt");
|
|
1029
|
+
// Only now, with the prompt accepted and a live turn about to go in the
|
|
1030
|
+
// map, is a recorded dead start out of date. Clearing it when the slot was
|
|
1031
|
+
// claimed instead dropped it before Superblocks had taken anything, so a
|
|
1032
|
+
// retry that came back needing interactive auth - a return, not a throw, so
|
|
1033
|
+
// nothing re-recorded it - left the app with no turn at all and `get_app`
|
|
1034
|
+
// went back to previewing the starter template.
|
|
1035
|
+
this.failedStarts.clear(input.applicationId);
|
|
1036
|
+
this.turns.set(input.applicationId, turn);
|
|
1037
|
+
// Deliberately not awaited: the turn outlives the call that started it so a
|
|
1038
|
+
// client with no progress channel can follow it through awaitTurn.
|
|
1039
|
+
void this.runTurn({
|
|
1040
|
+
applicationId: input.applicationId,
|
|
1041
|
+
onProgress: input.onProgress,
|
|
1042
|
+
sessionToken: input.sessionToken,
|
|
1043
|
+
turn,
|
|
1044
|
+
});
|
|
1045
|
+
return {
|
|
1046
|
+
activity: this.drain(turn),
|
|
1047
|
+
// Where the prompt ended up, which is not always where it was sent: a
|
|
1048
|
+
// connect that followed a moved session is holding an id the caller's
|
|
1049
|
+
// own resolve never saw, and this turn's events arrive under that one.
|
|
1050
|
+
...(held.liveEdit.id ? { liveEditId: held.liveEdit.id } : {}),
|
|
1051
|
+
status: "accepted",
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
async awaitTurn(input) {
|
|
1055
|
+
const turn = this.turns.get(input.applicationId);
|
|
1056
|
+
// The same recency order as turnState: a turn still running outranks
|
|
1057
|
+
// everything, and this caller's own dead start outranks a turn that already
|
|
1058
|
+
// finished. A finished turn with no dead start against it is reported as
|
|
1059
|
+
// itself, below. A poll that cannot say who it is for reads no record at
|
|
1060
|
+
// all - the records belong to callers, not to the application.
|
|
1061
|
+
const failed = (!turn || turn.finished) && input.organizationId && input.userId
|
|
1062
|
+
? this.failedStarts.get({
|
|
1063
|
+
applicationId: input.applicationId,
|
|
1064
|
+
organizationId: input.organizationId,
|
|
1065
|
+
userId: input.userId,
|
|
1066
|
+
})
|
|
1067
|
+
: undefined;
|
|
1068
|
+
// A turn that never started took no time, and there is no activity to
|
|
1069
|
+
// relay. Reporting it as finished is what stops the caller polling a session
|
|
1070
|
+
// that was never there.
|
|
1071
|
+
if (failed) {
|
|
1072
|
+
return { activity: [], elapsedMs: 0, status: "finished", ...failed };
|
|
1073
|
+
}
|
|
1074
|
+
if (!turn) {
|
|
1075
|
+
return { activity: [], elapsedMs: 0, status: "unknown" };
|
|
1076
|
+
}
|
|
1077
|
+
if (turn.delivered >= turn.activity.length && !turn.finished) {
|
|
1078
|
+
await new Promise((resolve) => {
|
|
1079
|
+
const settle = () => {
|
|
1080
|
+
clearTimeout(timer);
|
|
1081
|
+
turn.waiters.delete(settle);
|
|
1082
|
+
input.signal?.removeEventListener("abort", settle);
|
|
1083
|
+
resolve();
|
|
1084
|
+
};
|
|
1085
|
+
const timer = setTimeout(settle, Math.max(0, input.waitMs));
|
|
1086
|
+
turn.waiters.add(settle);
|
|
1087
|
+
input.signal?.addEventListener("abort", settle, { once: true });
|
|
1088
|
+
});
|
|
1089
|
+
}
|
|
1090
|
+
const activity = this.drain(turn);
|
|
1091
|
+
const elapsedMs = Date.now() - turn.startedAt;
|
|
1092
|
+
const recentRuns = turn.recentRuns.length > 0 ? [...turn.recentRuns] : undefined;
|
|
1093
|
+
if (turn.finished) {
|
|
1094
|
+
return {
|
|
1095
|
+
activity,
|
|
1096
|
+
elapsedMs,
|
|
1097
|
+
status: "finished",
|
|
1098
|
+
...turn.finished,
|
|
1099
|
+
...(recentRuns ? { recentRuns } : {}),
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
return {
|
|
1103
|
+
activity,
|
|
1104
|
+
elapsedMs,
|
|
1105
|
+
status: "running",
|
|
1106
|
+
...(recentRuns ? { recentRuns } : {}),
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
/**
|
|
1110
|
+
* Follows the turn to its end on the held session: terminal state, then the
|
|
1111
|
+
* checkpoint that commits Clark's draft. Never throws — a caller polling for
|
|
1112
|
+
* news has to be told what happened, and the only place left to report a
|
|
1113
|
+
* failure is the turn record itself.
|
|
1114
|
+
*/
|
|
1115
|
+
async runTurn(input) {
|
|
1116
|
+
const turn = input.turn;
|
|
1117
|
+
const { collector } = turn;
|
|
1118
|
+
const turnTimeoutMs = this.options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
|
|
1119
|
+
const readyWaitMs = this.options.readyWaitMs ?? DEFAULT_READY_WAIT_MS;
|
|
1120
|
+
const report = (message) => {
|
|
1121
|
+
this.note(turn, message);
|
|
1122
|
+
input.onProgress?.(message);
|
|
1123
|
+
};
|
|
1124
|
+
// The session the turn opened on, kept only so its listener is detached at
|
|
1125
|
+
// the end even if the turn moved off it in between.
|
|
1126
|
+
const opening = turn.held;
|
|
1127
|
+
let checkpointCommitId;
|
|
1128
|
+
try {
|
|
1129
|
+
if (turn.finished) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
if (!(await collector.waitForClarkPush({
|
|
1133
|
+
signal: turn.abort.signal,
|
|
1134
|
+
timeoutMs: this.pushWaitMs(),
|
|
1135
|
+
}))) {
|
|
1136
|
+
if (turn.finished) {
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
await this.reattach({ report, turn });
|
|
1140
|
+
if (turn.finished) {
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
if (!(await collector.waitForClarkPush({
|
|
1144
|
+
signal: turn.abort.signal,
|
|
1145
|
+
timeoutMs: this.options.reattachWaitMs ?? DEFAULT_REATTACH_WAIT_MS,
|
|
1146
|
+
}))) {
|
|
1147
|
+
if (turn.finished) {
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
console.warn("gateway clark turn detached", {
|
|
1151
|
+
applicationId: input.applicationId,
|
|
1152
|
+
inboundMethods: [...this.inboundMethods].sort(),
|
|
1153
|
+
});
|
|
1154
|
+
turn.finished ??= {
|
|
1155
|
+
reply: collector.reply(),
|
|
1156
|
+
generationStatus: "detached",
|
|
1157
|
+
viewStatus: collector.viewableStatus(),
|
|
1158
|
+
...decisionFields(collector.decision()),
|
|
1159
|
+
};
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
if (turn.finished) {
|
|
1164
|
+
return;
|
|
1165
|
+
}
|
|
1166
|
+
const generationStatus = await collector.waitUntilTerminal({
|
|
1167
|
+
signal: turn.abort.signal,
|
|
1168
|
+
timeoutMs: turnTimeoutMs,
|
|
1169
|
+
});
|
|
1170
|
+
if (turn.finished) {
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
console.info("gateway clark turn states", {
|
|
1174
|
+
applicationId: input.applicationId,
|
|
1175
|
+
draftPending: collector.needsDraftAccept(),
|
|
1176
|
+
generationStatus,
|
|
1177
|
+
inboundMethods: [...this.inboundMethods].sort(),
|
|
1178
|
+
states: collector.stateSequence(),
|
|
1179
|
+
workSignals: collector.workSignals(),
|
|
1180
|
+
});
|
|
1181
|
+
// Any turn that produced work asks for the accept, not only one still
|
|
1182
|
+
// holding `hasDraft: true` when it ends. A build normally does report a
|
|
1183
|
+
// draft, but the dev server sometimes flushes the files itself first, and
|
|
1184
|
+
// `aiAcceptDraft` is what mints the checkpoint either way. Accepting with
|
|
1185
|
+
// nothing pending is a no-op that returns no commit id. A turn that
|
|
1186
|
+
// produced nothing is `no_changes` and never reaches here, so Clark
|
|
1187
|
+
// answering a question in chat cannot checkpoint the bare template.
|
|
1188
|
+
if (generationStatus === "completed") {
|
|
1189
|
+
// TODO(ENG-5922): `turn.held` is a dead session when the turn was kept
|
|
1190
|
+
// alive on the relay, so this accept cannot land and the caller gets a
|
|
1191
|
+
// finished turn with no commit.
|
|
1192
|
+
checkpointCommitId = await this.checkpoint({
|
|
1193
|
+
collector,
|
|
1194
|
+
held: turn.held,
|
|
1195
|
+
onProgress: report,
|
|
1196
|
+
readyWaitMs,
|
|
1197
|
+
sessionToken: input.sessionToken,
|
|
1198
|
+
turnId: turn.turnId,
|
|
1199
|
+
});
|
|
1200
|
+
this.clearPendingDecision(input.applicationId);
|
|
1201
|
+
}
|
|
1202
|
+
if (turn.finished) {
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
const decision = collector.decision();
|
|
1206
|
+
if (generationStatus === "needs_decision" && decision) {
|
|
1207
|
+
const reply = collector.reply();
|
|
1208
|
+
this.pendingDecisions.set(input.applicationId, {
|
|
1209
|
+
decision,
|
|
1210
|
+
...(turn.recentRuns.length > 0
|
|
1211
|
+
? { recentRuns: [...turn.recentRuns] }
|
|
1212
|
+
: {}),
|
|
1213
|
+
reply,
|
|
1214
|
+
});
|
|
1215
|
+
if (decision.kind === "plan") {
|
|
1216
|
+
this.pendingPlans.set(input.applicationId, {
|
|
1217
|
+
...(decision.messageId ? { messageId: decision.messageId } : {}),
|
|
1218
|
+
plan: decision.plan,
|
|
1219
|
+
...(decision.title ? { title: decision.title } : {}),
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
else {
|
|
1223
|
+
this.pendingPlans.delete(input.applicationId);
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
// Disconnect / supersede already wrote the authoritative outcome; never
|
|
1227
|
+
// overwrite that with a later collector timeout or terminal status.
|
|
1228
|
+
turn.finished ??= {
|
|
1229
|
+
...(checkpointCommitId ? { checkpointCommitId } : {}),
|
|
1230
|
+
reply: collector.reply(),
|
|
1231
|
+
generationStatus,
|
|
1232
|
+
...(turn.recentRuns.length > 0
|
|
1233
|
+
? { recentRuns: [...turn.recentRuns] }
|
|
1234
|
+
: {}),
|
|
1235
|
+
viewStatus: collector.viewableStatus(),
|
|
1236
|
+
...decisionFields(decision),
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
catch (error) {
|
|
1240
|
+
if (!turn.finished) {
|
|
1241
|
+
console.warn("gateway clark turn failed", {
|
|
1242
|
+
applicationId: input.applicationId,
|
|
1243
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
// A disconnect / supersede already settled the turn with the reason; the
|
|
1247
|
+
// abort it caused says less than what is already recorded.
|
|
1248
|
+
turn.finished ??= {
|
|
1249
|
+
reply: collector.reply(),
|
|
1250
|
+
generationStatus: "timeout",
|
|
1251
|
+
viewStatus: collector.viewableStatus(),
|
|
1252
|
+
...decisionFields(collector.decision()),
|
|
1253
|
+
};
|
|
1254
|
+
}
|
|
1255
|
+
finally {
|
|
1256
|
+
opening.listeners.delete(turn.listener);
|
|
1257
|
+
turn.held.listeners.delete(turn.listener);
|
|
1258
|
+
// The turn is over on both transports before the collector is disposed:
|
|
1259
|
+
// a push arriving afterwards would otherwise be handed to it.
|
|
1260
|
+
turn.unsubscribe();
|
|
1261
|
+
collector.dispose();
|
|
1262
|
+
// A superseded turn must not arm timers or keepalive on a held session
|
|
1263
|
+
// that now belongs to a newer prompt.
|
|
1264
|
+
const stillMapped = this.turns.get(input.applicationId) === turn;
|
|
1265
|
+
if (stillMapped) {
|
|
1266
|
+
// Held a little longer than the turn so the sandbox startup watchdog and
|
|
1267
|
+
// idle reaper still see an attached peer while the build lands, then let
|
|
1268
|
+
// go: nothing headless is waiting on it after that, and the sandbox
|
|
1269
|
+
// should be free to go idle on the user's own terms.
|
|
1270
|
+
//
|
|
1271
|
+
// A turn that ended on a question is the exception. The human answer is
|
|
1272
|
+
// the next thing that will happen to this app, and the sandbox reaps
|
|
1273
|
+
// itself once no socket is attached — letting go here means the answer,
|
|
1274
|
+
// minutes later, lands on a dev server that has already exited. Hold the
|
|
1275
|
+
// session on the ordinary idle timer instead.
|
|
1276
|
+
if (turn.finished?.generationStatus === "needs_decision") {
|
|
1277
|
+
this.armIdle(turn.held);
|
|
1278
|
+
}
|
|
1279
|
+
else {
|
|
1280
|
+
this.armPostTurnRelease(turn.held, turn);
|
|
1281
|
+
}
|
|
1282
|
+
this.armKeepalive(turn.held);
|
|
1283
|
+
}
|
|
1284
|
+
this.wake(turn);
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
pushWaitMs() {
|
|
1288
|
+
return this.options.pushWaitMs ?? DEFAULT_PUSH_WAIT_MS;
|
|
1289
|
+
}
|
|
1290
|
+
/**
|
|
1291
|
+
* Attaches a turn to the events SABS relays for its live edit, returning the
|
|
1292
|
+
* idempotent detach the turn calls when it ends.
|
|
1293
|
+
*
|
|
1294
|
+
* A turn with no live edit id, or a deployment with no push channel, gets a
|
|
1295
|
+
* detach that does nothing: the socket remains its only source, which is the
|
|
1296
|
+
* behavior from before the channel existed.
|
|
1297
|
+
*/
|
|
1298
|
+
subscribeToPushes(_liveEditId, _normalized) {
|
|
1299
|
+
// Stdio Gateway has no SABS HTTP push ingress (ENG-5797). The held
|
|
1300
|
+
// socket is the only event source; the detach is a no-op so turn
|
|
1301
|
+
// teardown stays the same shape as the parent branch.
|
|
1302
|
+
return () => undefined;
|
|
1303
|
+
}
|
|
1304
|
+
/**
|
|
1305
|
+
* Checks that the session took this prompt on as a turn belonging to this
|
|
1306
|
+
* peer, and gives up on it now if it did not.
|
|
1307
|
+
*
|
|
1308
|
+
* A dev server that predates the gateway role accepts the socket, echoes the
|
|
1309
|
+
* role it was offered, and then treats the connection as a library socket: the
|
|
1310
|
+
* prompt is acknowledged and Clark's pushes go to the browser binding
|
|
1311
|
+
* instead. The turn would spend its whole budget looking detached, and a
|
|
1312
|
+
* rollout with one stale sandbox in it would look like Clark hanging. Only a
|
|
1313
|
+
* server that bound the turn can name it back, so the missing handle is the
|
|
1314
|
+
* mismatch, reported where it is still cheap to act on.
|
|
1315
|
+
*/
|
|
1316
|
+
assertHeadlessTurnBound(input) {
|
|
1317
|
+
if (input.acknowledgment?.headlessTurn?.turnId === input.turn.turnId) {
|
|
1318
|
+
return;
|
|
1319
|
+
}
|
|
1320
|
+
input.held.listeners.delete(input.listener);
|
|
1321
|
+
this.release(input.held.applicationId);
|
|
1322
|
+
throw new Error(`Prompt forward failed: the live-edit session acknowledged the prompt but did not bind it to this ${GATEWAY_PROTOCOL} peer, so nothing about the turn would report back. The sandbox is running a dev server from before the ${GATEWAY_PROTOCOL} role; restart the sandbox to pick up the current one.`);
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* The field the session reads to decide which turn a call belongs to. Sent on
|
|
1326
|
+
* the calls that make up a turn's lifetime, so a turn keeps reporting into
|
|
1327
|
+
* this caller across a reattach and no other session can be handed it.
|
|
1328
|
+
*/
|
|
1329
|
+
turnCorrelator(turnId) {
|
|
1330
|
+
return turnId ? { headlessTurnId: turnId } : {};
|
|
1331
|
+
}
|
|
1332
|
+
/**
|
|
1333
|
+
* Moves a turn onto a fresh socket while the one it is on is still open,
|
|
1334
|
+
* because Clark has stopped pushing to it. A turn that was running unseen
|
|
1335
|
+
* becomes visible again from where it got to.
|
|
1336
|
+
*
|
|
1337
|
+
* The socket it came in on is still the better of the two whenever the
|
|
1338
|
+
* replacement fails to take the turn over: the sandbox is demonstrably alive,
|
|
1339
|
+
* and it is still pushing this turn somewhere. A caller mid-turn has nothing
|
|
1340
|
+
* to gain from an exception here, so there is no failure to report but the
|
|
1341
|
+
* log line.
|
|
1342
|
+
*/
|
|
1343
|
+
async reattach(input) {
|
|
1344
|
+
const previous = input.turn.held;
|
|
1345
|
+
input.report("Reattaching to the Superblocks session");
|
|
1346
|
+
const outcome = await this.takeOverTurn({
|
|
1347
|
+
deadline: Date.now() + (this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
1348
|
+
previous,
|
|
1349
|
+
turn: input.turn,
|
|
1350
|
+
});
|
|
1351
|
+
if (outcome.kind === "took_over" || outcome.kind === "stale") {
|
|
1352
|
+
return;
|
|
1353
|
+
}
|
|
1354
|
+
console.warn("gateway clark reattach failed", {
|
|
1355
|
+
applicationId: previous.applicationId,
|
|
1356
|
+
message: outcome.kind === "unreachable"
|
|
1357
|
+
? outcome.message
|
|
1358
|
+
: "the live-edit session would not hand the turn over",
|
|
1359
|
+
});
|
|
1360
|
+
}
|
|
1361
|
+
/**
|
|
1362
|
+
* Whether this turn gets another go at its live edit, booking it if so.
|
|
1363
|
+
*
|
|
1364
|
+
* The allowance is for a live edit that cannot hold a socket, and a turn that
|
|
1365
|
+
* has heard from Clark since its last reconnect is not that: it is getting
|
|
1366
|
+
* somewhere, so its allowance starts over. Without that, a long build behind
|
|
1367
|
+
* a proxy that reaps every few minutes runs out of goes and is failed for the
|
|
1368
|
+
* very reason this path exists to survive.
|
|
1369
|
+
*/
|
|
1370
|
+
takeReconnectAllowance(turn, heardAt) {
|
|
1371
|
+
if (heardAt > turn.heardAtLastReconnect) {
|
|
1372
|
+
turn.reconnects = 0;
|
|
1373
|
+
}
|
|
1374
|
+
turn.heardAtLastReconnect = heardAt;
|
|
1375
|
+
const allowed = this.options.reconnectAttempts ?? DEFAULT_RECONNECT_ATTEMPTS;
|
|
1376
|
+
if (turn.reconnects >= allowed) {
|
|
1377
|
+
return false;
|
|
1378
|
+
}
|
|
1379
|
+
turn.reconnects += 1;
|
|
1380
|
+
return true;
|
|
1381
|
+
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Opens a replacement connection and asks the live edit to hand `turn` over
|
|
1384
|
+
* to it.
|
|
1385
|
+
*
|
|
1386
|
+
* Everything that belongs to the session rather than to the socket is carried
|
|
1387
|
+
* across first: the credential the claim itself is made with is one of those
|
|
1388
|
+
* things, and so is the orchestrator a mid-turn tool call needs. The turn's
|
|
1389
|
+
* listener goes on the replacement before the claim, so a push that arrives
|
|
1390
|
+
* while it is in flight is not dropped.
|
|
1391
|
+
*
|
|
1392
|
+
* The turn is claimed before the old socket is let go. The session releases a
|
|
1393
|
+
* turn when the connection holding it closes, so closing first leaves the
|
|
1394
|
+
* claim nothing to take over; claiming first makes the old socket's close a
|
|
1395
|
+
* no-op scoped to a peer that is no longer the owner.
|
|
1396
|
+
*
|
|
1397
|
+
* Nothing has moved when this fails: the listener is back where it was and
|
|
1398
|
+
* the replacement is closed, which leaves the caller free to decide what a
|
|
1399
|
+
* socket it could not replace means.
|
|
1400
|
+
*/
|
|
1401
|
+
async takeOverTurn(input) {
|
|
1402
|
+
// One attempt at a time per turn. Both paths that reach for a replacement
|
|
1403
|
+
// answer the same stall - the pushes stop, then the close lands - and the
|
|
1404
|
+
// second cannot see the first from `turn.held`, which the first has not
|
|
1405
|
+
// reassigned yet. Two attempts would each claim the turn on their own
|
|
1406
|
+
// socket and orphan the loser, so the second waits on the first and acts
|
|
1407
|
+
// on what it achieved.
|
|
1408
|
+
const running = input.turn.takingOver;
|
|
1409
|
+
if (running) {
|
|
1410
|
+
return await running;
|
|
1411
|
+
}
|
|
1412
|
+
const attempt = this.attemptTakeOver(input);
|
|
1413
|
+
input.turn.takingOver = attempt;
|
|
1414
|
+
try {
|
|
1415
|
+
return await attempt;
|
|
1416
|
+
}
|
|
1417
|
+
finally {
|
|
1418
|
+
if (input.turn.takingOver === attempt) {
|
|
1419
|
+
input.turn.takingOver = undefined;
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
async attemptTakeOver(input) {
|
|
1424
|
+
const { previous, turn } = input;
|
|
1425
|
+
let fresh;
|
|
1426
|
+
try {
|
|
1427
|
+
fresh = await this.reconnect(previous, input.deadline);
|
|
1428
|
+
}
|
|
1429
|
+
catch (error) {
|
|
1430
|
+
return {
|
|
1431
|
+
kind: "unreachable",
|
|
1432
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1433
|
+
};
|
|
1434
|
+
}
|
|
1435
|
+
this.carryOver(fresh, previous);
|
|
1436
|
+
fresh.listeners.add(turn.listener);
|
|
1437
|
+
previous.listeners.delete(turn.listener);
|
|
1438
|
+
const claim = fresh.sessionToken === undefined
|
|
1439
|
+
? "refused"
|
|
1440
|
+
: await this.proveIdentity({
|
|
1441
|
+
held: fresh,
|
|
1442
|
+
sessionToken: fresh.sessionToken,
|
|
1443
|
+
turnId: turn.turnId,
|
|
1444
|
+
});
|
|
1445
|
+
// Opening and claiming take seconds, and the turn can settle inside them:
|
|
1446
|
+
// a relayed terminal event, its own budget running out, a newer prompt
|
|
1447
|
+
// taking the app. runTurn arms the post-turn timers once, against the
|
|
1448
|
+
// session the turn held when it ended, so a replacement committed after
|
|
1449
|
+
// that is a socket nothing will ever close. Nothing has been committed at
|
|
1450
|
+
// this point but the listener, so there is nothing to undo beyond letting
|
|
1451
|
+
// the replacement go.
|
|
1452
|
+
if (turn.finished || turn.held !== previous) {
|
|
1453
|
+
fresh.listeners.delete(turn.listener);
|
|
1454
|
+
this.drop(fresh);
|
|
1455
|
+
return { kind: "stale" };
|
|
1456
|
+
}
|
|
1457
|
+
if (claim === "unreachable") {
|
|
1458
|
+
fresh.listeners.delete(turn.listener);
|
|
1459
|
+
this.drop(fresh);
|
|
1460
|
+
return {
|
|
1461
|
+
kind: "unreachable",
|
|
1462
|
+
message: "the live-edit session stopped answering on a fresh socket",
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
if (claim === "refused") {
|
|
1466
|
+
fresh.listeners.delete(turn.listener);
|
|
1467
|
+
// Go back to the socket the turn was on, but only while it is still
|
|
1468
|
+
// there: a live edit that would not hand the turn over is still pushing
|
|
1469
|
+
// it to the connection that started it. A replacement for a socket that
|
|
1470
|
+
// already closed has nothing to go back to, and re-registering a dead
|
|
1471
|
+
// session would leave the next prompt to discover it.
|
|
1472
|
+
if (isSocketOpen(previous.ws)) {
|
|
1473
|
+
previous.listeners.add(turn.listener);
|
|
1474
|
+
this.held.set(previous.applicationId, previous);
|
|
1475
|
+
}
|
|
1476
|
+
this.drop(fresh);
|
|
1477
|
+
return { kind: "refused" };
|
|
1478
|
+
}
|
|
1479
|
+
turn.held = fresh;
|
|
1480
|
+
this.drop(previous);
|
|
1481
|
+
// A reconnect that followed the session to another host is still listening
|
|
1482
|
+
// to SABS under the id the turn started with, which is not the one the
|
|
1483
|
+
// moved session relays under. Left alone the turn hears nothing from the
|
|
1484
|
+
// transport that outlives its socket - the same trap the prompt path names
|
|
1485
|
+
// when it subscribes under the id its connect landed on.
|
|
1486
|
+
if (fresh.liveEdit.id && fresh.liveEdit.id !== previous.liveEdit.id) {
|
|
1487
|
+
turn.unsubscribe();
|
|
1488
|
+
turn.unsubscribe = this.subscribeToPushes(fresh.liveEdit.id, turn.normalized);
|
|
1489
|
+
}
|
|
1490
|
+
return { kind: "took_over" };
|
|
1491
|
+
}
|
|
1492
|
+
/**
|
|
1493
|
+
* Carries onto a replacement socket everything that belongs to the session
|
|
1494
|
+
* rather than to the connection: who the turn runs as, what it is allowed to
|
|
1495
|
+
* do, and which orchestrator its tool calls go to. A replacement missing any
|
|
1496
|
+
* of it is connected but cannot act for the caller.
|
|
1497
|
+
*/
|
|
1498
|
+
carryOver(fresh, previous) {
|
|
1499
|
+
fresh.apiKey = previous.apiKey;
|
|
1500
|
+
fresh.branchName = previous.branchName;
|
|
1501
|
+
fresh.orchestrator = previous.orchestrator;
|
|
1502
|
+
fresh.orchestratorUrl = previous.orchestratorUrl;
|
|
1503
|
+
fresh.organizationId = previous.organizationId;
|
|
1504
|
+
fresh.profileKey = previous.profileKey;
|
|
1505
|
+
fresh.profiles = previous.profiles;
|
|
1506
|
+
fresh.sessionToken = previous.sessionToken;
|
|
1507
|
+
}
|
|
1508
|
+
/**
|
|
1509
|
+
* Makes one authenticated read on a freshly opened socket.
|
|
1510
|
+
*
|
|
1511
|
+
* The dev server trusts a peer's identity per connection and does not carry
|
|
1512
|
+
* it over from the peer this one replaced, so it withholds the pushes it
|
|
1513
|
+
* scopes to a caller — Clark's step-by-step task status — until the new
|
|
1514
|
+
* connection has proven who it is. Reading the chat is the cheapest way to
|
|
1515
|
+
* prove it, and it costs a turn nothing when the read fails.
|
|
1516
|
+
*
|
|
1517
|
+
* Naming `turnId` is what makes this call a reattach rather than just an
|
|
1518
|
+
* authenticated read: the session only moves a turn onto a connection that
|
|
1519
|
+
* can say which turn it is taking over.
|
|
1520
|
+
*
|
|
1521
|
+
* Answers whether it worked, because a caller mid-reattach has to decide
|
|
1522
|
+
* between two sockets on it.
|
|
1523
|
+
*/
|
|
1524
|
+
async proveIdentity(input) {
|
|
1525
|
+
try {
|
|
1526
|
+
await input.held.client.requestWithTracing(RELOAD_CHAT_METHOD, { force: false, ...this.turnCorrelator(input.turnId) }, {}, `Bearer ${input.sessionToken}`);
|
|
1527
|
+
return "claimed";
|
|
1528
|
+
}
|
|
1529
|
+
catch (error) {
|
|
1530
|
+
const message = describeSocketError(error);
|
|
1531
|
+
console.warn("gateway clark reattach identity call failed", {
|
|
1532
|
+
applicationId: input.held.applicationId,
|
|
1533
|
+
message,
|
|
1534
|
+
});
|
|
1535
|
+
// A call that went unanswered says the socket opened onto something that
|
|
1536
|
+
// is not servicing requests, which is the same fact a lost liveness beat
|
|
1537
|
+
// reports and nothing like a session declining to hand a turn over.
|
|
1538
|
+
return isTimeout({ message }) ? "unreachable" : "refused";
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
async reconnect(previous, deadline) {
|
|
1542
|
+
const connect = this.connect({
|
|
1543
|
+
applicationId: previous.applicationId,
|
|
1544
|
+
branchName: previous.branchName,
|
|
1545
|
+
keepalive: previous.keepalive,
|
|
1546
|
+
// Both halves, so a reattach keeps listening on the session it was
|
|
1547
|
+
// already on: the socket is being replaced, not the live edit.
|
|
1548
|
+
liveEdit: previous.liveEdit,
|
|
1549
|
+
// The address in hand is one this turn has already been served on, so
|
|
1550
|
+
// a session that moved out from under it denies every attempt made
|
|
1551
|
+
// here. Re-resolving on the way in is not enough for a turn that
|
|
1552
|
+
// outlives its socket.
|
|
1553
|
+
...(previous.resolveLiveEdit
|
|
1554
|
+
? { resolveLiveEdit: previous.resolveLiveEdit }
|
|
1555
|
+
: {}),
|
|
1556
|
+
userId: previous.userId,
|
|
1557
|
+
}, deadline);
|
|
1558
|
+
// Published so a prompt arriving mid-reattach waits for this socket instead
|
|
1559
|
+
// of opening a third one.
|
|
1560
|
+
this.connecting.set(previous.applicationId, connect);
|
|
1561
|
+
try {
|
|
1562
|
+
return await connect;
|
|
1563
|
+
}
|
|
1564
|
+
finally {
|
|
1565
|
+
this.connecting.delete(previous.applicationId);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
/** Lines this caller has not been handed yet. */
|
|
1569
|
+
drain(turn) {
|
|
1570
|
+
const pending = turn.activity.slice(turn.delivered);
|
|
1571
|
+
turn.delivered = turn.activity.length;
|
|
1572
|
+
return pending;
|
|
1573
|
+
}
|
|
1574
|
+
wake(turn) {
|
|
1575
|
+
for (const waiter of [...turn.waiters]) {
|
|
1576
|
+
waiter();
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
/** Adds a line to what the next poll on this turn is handed. */
|
|
1580
|
+
note(turn, message) {
|
|
1581
|
+
turn.activity.push(message);
|
|
1582
|
+
this.wake(turn);
|
|
1583
|
+
}
|
|
1584
|
+
/**
|
|
1585
|
+
* Records the outcome of a turn from outside runTurn, and ends the collector
|
|
1586
|
+
* waits so runTurn cannot overwrite it with a later timeout of its own. The
|
|
1587
|
+
* first outcome written wins: it is the one callers have already been shown.
|
|
1588
|
+
*/
|
|
1589
|
+
settleTurn(turn, finished) {
|
|
1590
|
+
turn.finished ??= finished;
|
|
1591
|
+
turn.abort.abort();
|
|
1592
|
+
this.wake(turn);
|
|
1593
|
+
}
|
|
1594
|
+
/**
|
|
1595
|
+
* Mirrors the browser editor's auto-accept: wait for the dev server to flush
|
|
1596
|
+
* Clark's files, then commit the draft. Failing to checkpoint costs the caller
|
|
1597
|
+
* a committed app, not the generation itself, so this never throws.
|
|
1598
|
+
*/
|
|
1599
|
+
async checkpoint(input) {
|
|
1600
|
+
const inSync = await input.collector.waitUntilInSync({
|
|
1601
|
+
timeoutMs: input.readyWaitMs,
|
|
1602
|
+
});
|
|
1603
|
+
console.info("gateway clark checkpoint sync wait", {
|
|
1604
|
+
applicationId: input.held.applicationId,
|
|
1605
|
+
inSync,
|
|
1606
|
+
});
|
|
1607
|
+
if (!inSync) {
|
|
1608
|
+
input.onProgress?.("Clark's files are still saving; the app will finish in the editor");
|
|
1609
|
+
return undefined;
|
|
1610
|
+
}
|
|
1611
|
+
input.onProgress?.("Saving a checkpoint");
|
|
1612
|
+
try {
|
|
1613
|
+
const result = await input.held.client.requestWithTracing(ACCEPT_DRAFT_METHOD, {
|
|
1614
|
+
screenshotDataUrl: null,
|
|
1615
|
+
silently: true,
|
|
1616
|
+
...this.turnCorrelator(input.turnId),
|
|
1617
|
+
}, {}, `Bearer ${input.sessionToken}`);
|
|
1618
|
+
input.collector.markCheckpointed();
|
|
1619
|
+
const commitId = result?.commitId?.trim() ? result.commitId : undefined;
|
|
1620
|
+
// The accept writes draftCommitted on the server. The warm sandbox still
|
|
1621
|
+
// holds the chat it loaded earlier, so a browser that opens this same
|
|
1622
|
+
// session and loads chat without force never sees the checkpoint card.
|
|
1623
|
+
// Force-reloading now puts the server message into the sandbox store.
|
|
1624
|
+
if (commitId) {
|
|
1625
|
+
try {
|
|
1626
|
+
await input.held.client.requestWithTracing(RELOAD_CHAT_METHOD, { force: true, ...this.turnCorrelator(input.turnId) }, {}, `Bearer ${input.sessionToken}`);
|
|
1627
|
+
}
|
|
1628
|
+
catch (error) {
|
|
1629
|
+
console.warn("gateway clark chat reload after checkpoint failed", {
|
|
1630
|
+
applicationId: input.held.applicationId,
|
|
1631
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
console.info("gateway clark checkpoint saved", {
|
|
1636
|
+
applicationId: input.held.applicationId,
|
|
1637
|
+
commitId: commitId ?? "none",
|
|
1638
|
+
});
|
|
1639
|
+
input.onProgress?.(commitId ? "Checkpoint saved" : "App already up to date");
|
|
1640
|
+
return commitId;
|
|
1641
|
+
}
|
|
1642
|
+
catch (error) {
|
|
1643
|
+
console.warn("gateway clark checkpoint failed", {
|
|
1644
|
+
applicationId: input.held.applicationId,
|
|
1645
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1646
|
+
});
|
|
1647
|
+
input.onProgress?.("Could not save a checkpoint for Superblocks' changes");
|
|
1648
|
+
return undefined;
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
async ensureHeld(input, deadline) {
|
|
1652
|
+
const existing = this.held.get(input.applicationId);
|
|
1653
|
+
// A connect that followed a moved session left this record pointing
|
|
1654
|
+
// somewhere the prompt's own resolve never saw. Overlapping forwards are
|
|
1655
|
+
// ordinary - Clark retries - so the prompt behind the move still names the
|
|
1656
|
+
// host the session left; reading that as a different session would drop
|
|
1657
|
+
// the socket the move just won and dial the host that stopped answering.
|
|
1658
|
+
const sameUrl = existing?.liveEdit.url === input.liveEdit.url;
|
|
1659
|
+
if (existing &&
|
|
1660
|
+
(sameUrl || existing.movedFrom === input.liveEdit.url) &&
|
|
1661
|
+
isSocketOpen(existing.ws)) {
|
|
1662
|
+
// The held map is keyed by applicationId alone. Reusing a socket opened
|
|
1663
|
+
// for a different user would attribute Clark work to the wrong principal
|
|
1664
|
+
// and keep their session JWT on the wire.
|
|
1665
|
+
if (existing.userId !== input.userId) {
|
|
1666
|
+
this.drop(existing);
|
|
1667
|
+
}
|
|
1668
|
+
else {
|
|
1669
|
+
existing.branchName = input.branchName;
|
|
1670
|
+
// The caller resolved the live edit for this prompt, so its id is at
|
|
1671
|
+
// least as fresh as whatever an earlier prompt left here - but only
|
|
1672
|
+
// when it named the host this socket is on. A prompt reusing through
|
|
1673
|
+
// `movedFrom` resolved before the move, so its id is the one the move
|
|
1674
|
+
// replaced, and taking it would undo what the move established. An
|
|
1675
|
+
// absent id is not news either way and must not clear what is here.
|
|
1676
|
+
if (sameUrl && input.liveEdit.id) {
|
|
1677
|
+
existing.liveEdit = {
|
|
1678
|
+
id: input.liveEdit.id,
|
|
1679
|
+
url: existing.liveEdit.url,
|
|
1680
|
+
};
|
|
1681
|
+
}
|
|
1682
|
+
if (input.keepalive) {
|
|
1683
|
+
existing.keepalive = input.keepalive;
|
|
1684
|
+
}
|
|
1685
|
+
if (input.resolveLiveEdit) {
|
|
1686
|
+
existing.resolveLiveEdit = input.resolveLiveEdit;
|
|
1687
|
+
}
|
|
1688
|
+
return existing;
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
const pending = this.connecting.get(input.applicationId);
|
|
1692
|
+
if (pending) {
|
|
1693
|
+
// Settle behind the open already in flight, then re-decide: it may have
|
|
1694
|
+
// produced the socket this caller wants, or failed and left nothing.
|
|
1695
|
+
await pending.catch(() => undefined);
|
|
1696
|
+
return await this.ensureHeld(input, deadline);
|
|
1697
|
+
}
|
|
1698
|
+
if (existing) {
|
|
1699
|
+
this.drop(existing);
|
|
1700
|
+
}
|
|
1701
|
+
const connect = this.connect(input, deadline);
|
|
1702
|
+
this.connecting.set(input.applicationId, connect);
|
|
1703
|
+
try {
|
|
1704
|
+
return await connect;
|
|
1705
|
+
}
|
|
1706
|
+
finally {
|
|
1707
|
+
this.connecting.delete(input.applicationId);
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
async connect(input, deadline) {
|
|
1711
|
+
// Unique per connection. The dev server ignores the disconnect of a peerId
|
|
1712
|
+
// that is no longer the bound one, so a reattach that closes its
|
|
1713
|
+
// predecessor afterwards keeps the binding it just took; reusing one id
|
|
1714
|
+
// would have that close clear it.
|
|
1715
|
+
const peerId = `gateway-held-${input.applicationId}-${randomUUID()}`;
|
|
1716
|
+
const url = buildSessionUrl({
|
|
1717
|
+
applicationId: input.applicationId,
|
|
1718
|
+
liveEditUrl: input.liveEdit.url,
|
|
1719
|
+
peerId,
|
|
1720
|
+
userId: input.userId,
|
|
1721
|
+
});
|
|
1722
|
+
// Where the session turned out to be, which is what the held record has to
|
|
1723
|
+
// remember: the value the prompt arrived with describes where it was looked
|
|
1724
|
+
// for, not where the socket ended up.
|
|
1725
|
+
let liveEdit = input.liveEdit;
|
|
1726
|
+
const resolveLiveEdit = input.resolveLiveEdit;
|
|
1727
|
+
const close = {};
|
|
1728
|
+
const ws = await open(url, this.options.connectAttemptTimeoutMs ??
|
|
1729
|
+
DEFAULT_CONNECT_ATTEMPT_TIMEOUT_MS, deadline, (error) => {
|
|
1730
|
+
close.error = error.message;
|
|
1731
|
+
}, resolveLiveEdit &&
|
|
1732
|
+
(async () => {
|
|
1733
|
+
try {
|
|
1734
|
+
const moved = await resolveLiveEdit();
|
|
1735
|
+
const next = buildSessionUrl({
|
|
1736
|
+
applicationId: input.applicationId,
|
|
1737
|
+
liveEditUrl: moved.liveEditUrl,
|
|
1738
|
+
peerId,
|
|
1739
|
+
userId: input.userId,
|
|
1740
|
+
});
|
|
1741
|
+
// Recorded whether or not the host changed. A resolve that lands
|
|
1742
|
+
// back on the same host is still the freshest word on which live
|
|
1743
|
+
// edit that host is serving. Recorded only once the URL is built,
|
|
1744
|
+
// so a session that cannot be reached does not overwrite the one
|
|
1745
|
+
// the socket may yet connect to. Taken as one value: a resolve that
|
|
1746
|
+
// names a host but not the session on it leaves this with no id
|
|
1747
|
+
// rather than the one from before the move, so the turn falls back
|
|
1748
|
+
// to the socket instead of listening on a live edit that moved.
|
|
1749
|
+
liveEdit = liveEditOf(moved);
|
|
1750
|
+
console.warn("gateway re-resolved a live-edit host", {
|
|
1751
|
+
applicationId: input.applicationId,
|
|
1752
|
+
from: new URL(input.liveEdit.url).host,
|
|
1753
|
+
liveEditId: liveEdit.id,
|
|
1754
|
+
peerId,
|
|
1755
|
+
to: next.host,
|
|
1756
|
+
});
|
|
1757
|
+
return next;
|
|
1758
|
+
}
|
|
1759
|
+
catch (error) {
|
|
1760
|
+
// Reported rather than raised: the connect loop's fallback is the
|
|
1761
|
+
// host already in hand, and losing that to a failed lookup would
|
|
1762
|
+
// turn a recoverable stale binding into no attempt at all.
|
|
1763
|
+
console.warn("gateway could not re-resolve a live-edit host", {
|
|
1764
|
+
applicationId: input.applicationId,
|
|
1765
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1766
|
+
peerId,
|
|
1767
|
+
});
|
|
1768
|
+
return undefined;
|
|
1769
|
+
}
|
|
1770
|
+
}));
|
|
1771
|
+
// Registered before ISocket takes the socket so it runs ahead of ISocket's
|
|
1772
|
+
// own close handler, which is handed neither code nor reason.
|
|
1773
|
+
ws.on("close", (code, reason) => {
|
|
1774
|
+
close.code = code;
|
|
1775
|
+
const text = reason?.toString().trim();
|
|
1776
|
+
if (text) {
|
|
1777
|
+
close.reason = text;
|
|
1778
|
+
}
|
|
1779
|
+
});
|
|
1780
|
+
// Same cast the smoke client uses: ISocket's browser WebSocket type and
|
|
1781
|
+
// the `ws` package's Node type are structurally compatible at runtime but
|
|
1782
|
+
// not assignable under TypeScript's DOM lib.
|
|
1783
|
+
const heldRef = {
|
|
1784
|
+
current: undefined,
|
|
1785
|
+
};
|
|
1786
|
+
const socket = new ISocket(ws, createEditorClientMethods(() => {
|
|
1787
|
+
const listeners = heldRef.current?.listeners;
|
|
1788
|
+
if (!listeners || listeners.size === 0) {
|
|
1789
|
+
return undefined;
|
|
1790
|
+
}
|
|
1791
|
+
return (event) => {
|
|
1792
|
+
for (const listener of [...listeners]) {
|
|
1793
|
+
listener(event);
|
|
1794
|
+
}
|
|
1795
|
+
};
|
|
1796
|
+
}, (method) => {
|
|
1797
|
+
this.inboundMethods.add(method);
|
|
1798
|
+
}, {
|
|
1799
|
+
getContext: () => this.editorContextFor(heldRef.current),
|
|
1800
|
+
onToolRun: (run) => {
|
|
1801
|
+
const current = heldRef.current;
|
|
1802
|
+
if (!current) {
|
|
1803
|
+
return;
|
|
1804
|
+
}
|
|
1805
|
+
const turn = this.turns.get(current.applicationId);
|
|
1806
|
+
if (!turn) {
|
|
1807
|
+
return;
|
|
1808
|
+
}
|
|
1809
|
+
turn.recentRuns = appendRecentToolRun(turn.recentRuns, buildRecentToolRun(run));
|
|
1810
|
+
const mark = run.success ? "passed" : "failed";
|
|
1811
|
+
const line = `API test ${mark}: ${run.name}`;
|
|
1812
|
+
turn.activity.push(line);
|
|
1813
|
+
this.wake(turn);
|
|
1814
|
+
},
|
|
1815
|
+
respondToToolPermission: async (payload) => {
|
|
1816
|
+
const current = heldRef.current;
|
|
1817
|
+
if (!current) {
|
|
1818
|
+
return;
|
|
1819
|
+
}
|
|
1820
|
+
try {
|
|
1821
|
+
await current.client.requestWithTracing(RESPOND_TOOL_PERMISSION_METHOD, {
|
|
1822
|
+
approved: payload.approved,
|
|
1823
|
+
...this.turnCorrelator(this.turns.get(current.applicationId)?.turnId),
|
|
1824
|
+
messageId: payload.messageId,
|
|
1825
|
+
requestId: payload.requestId ?? payload.messageId,
|
|
1826
|
+
}, {}, current.sessionToken
|
|
1827
|
+
? `Bearer ${current.sessionToken}`
|
|
1828
|
+
: undefined);
|
|
1829
|
+
}
|
|
1830
|
+
catch (error) {
|
|
1831
|
+
console.warn("gateway auto-approve tool permission failed", {
|
|
1832
|
+
applicationId: current.applicationId,
|
|
1833
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
},
|
|
1837
|
+
}), [], {
|
|
1838
|
+
onClose: () => {
|
|
1839
|
+
const current = this.held.get(input.applicationId);
|
|
1840
|
+
if (current?.ws === ws) {
|
|
1841
|
+
console.error("gateway held session closed", {
|
|
1842
|
+
applicationId: input.applicationId,
|
|
1843
|
+
requested: Boolean(current.dropping),
|
|
1844
|
+
});
|
|
1845
|
+
this.clearTimers(current);
|
|
1846
|
+
current.heartbeat.stop();
|
|
1847
|
+
this.held.delete(input.applicationId);
|
|
1848
|
+
}
|
|
1849
|
+
const held = heldRef.current;
|
|
1850
|
+
// Its own closes - a reattach swapping sockets, a release - leave the
|
|
1851
|
+
// turn either running elsewhere or already settled.
|
|
1852
|
+
if (held?.dropping) {
|
|
1853
|
+
return;
|
|
1854
|
+
}
|
|
1855
|
+
// An application code is the live edit stating its own end, so there
|
|
1856
|
+
// is nothing left to get back onto. Any other close is the
|
|
1857
|
+
// transport's, and a turn is not over because the socket carrying it
|
|
1858
|
+
// is: the gateway goes and looks before saying so.
|
|
1859
|
+
if (!held || isApplicationClose(close)) {
|
|
1860
|
+
this.abandonTurnOnDisconnect(input.applicationId, close, held?.heartbeat.account);
|
|
1861
|
+
return;
|
|
1862
|
+
}
|
|
1863
|
+
void this.recoverTurn({ close, previous: held });
|
|
1864
|
+
},
|
|
1865
|
+
timeouts: {
|
|
1866
|
+
connectionTimeoutInSeconds: undefined,
|
|
1867
|
+
// Held across prompts, so budget each RPC the full timeout rather than
|
|
1868
|
+
// whatever remained of the connect deadline from the first open.
|
|
1869
|
+
noResponseTimeoutInSeconds: (this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS) / 1000,
|
|
1870
|
+
},
|
|
1871
|
+
});
|
|
1872
|
+
const client = createISocketClient(socket);
|
|
1873
|
+
const heartbeat = this.buildHeartbeat(heldRef);
|
|
1874
|
+
const held = {
|
|
1875
|
+
applicationId: input.applicationId,
|
|
1876
|
+
branchName: input.branchName,
|
|
1877
|
+
client,
|
|
1878
|
+
heartbeat,
|
|
1879
|
+
idleTimer: undefined,
|
|
1880
|
+
keepalive: input.keepalive,
|
|
1881
|
+
keepaliveTimer: undefined,
|
|
1882
|
+
listeners: new Set(),
|
|
1883
|
+
liveEdit,
|
|
1884
|
+
...(input.resolveLiveEdit
|
|
1885
|
+
? { resolveLiveEdit: input.resolveLiveEdit }
|
|
1886
|
+
: {}),
|
|
1887
|
+
...(liveEdit.url === input.liveEdit.url
|
|
1888
|
+
? {}
|
|
1889
|
+
: { movedFrom: input.liveEdit.url }),
|
|
1890
|
+
peerId,
|
|
1891
|
+
socket,
|
|
1892
|
+
userId: input.userId,
|
|
1893
|
+
ws,
|
|
1894
|
+
};
|
|
1895
|
+
heldRef.current = held;
|
|
1896
|
+
this.held.set(input.applicationId, held);
|
|
1897
|
+
// Inbound frames are what separates a dead connection from a request
|
|
1898
|
+
// channel that stopped answering on a live one.
|
|
1899
|
+
ws.on("message", () => {
|
|
1900
|
+
heartbeat.recordReceive();
|
|
1901
|
+
});
|
|
1902
|
+
// Beats for as long as the socket is held, including the grace period after
|
|
1903
|
+
// a turn settles: the checkpoint lands in that window and the sandbox has
|
|
1904
|
+
// to still be there for it.
|
|
1905
|
+
heartbeat.start();
|
|
1906
|
+
return held;
|
|
1907
|
+
}
|
|
1908
|
+
/**
|
|
1909
|
+
* The liveness beat for one held socket.
|
|
1910
|
+
*
|
|
1911
|
+
* Everything about pacing, deadlines and when to believe a silence lives in
|
|
1912
|
+
* {@link SocketHeartbeat}; what belongs here is only what is specific to this
|
|
1913
|
+
* peer - which call to make, what counts as unanswered on this client, and
|
|
1914
|
+
* who to tell.
|
|
1915
|
+
*/
|
|
1916
|
+
buildHeartbeat(heldRef) {
|
|
1917
|
+
return new SocketHeartbeat({
|
|
1918
|
+
beat: async () => {
|
|
1919
|
+
const held = heldRef.current;
|
|
1920
|
+
if (!held?.sessionToken) {
|
|
1921
|
+
// Unreachable behind `shouldBeat`. Thrown rather than returned so a
|
|
1922
|
+
// beat that never went out can never be recorded as one that landed.
|
|
1923
|
+
throw new Error("live-edit session went away before the beat was sent");
|
|
1924
|
+
}
|
|
1925
|
+
await held.client.requestWithTracing(PING_METHOD, this.turnCorrelator(this.turns.get(held.applicationId)?.turnId), {}, `Bearer ${held.sessionToken}`);
|
|
1926
|
+
},
|
|
1927
|
+
describeError: describeSocketError,
|
|
1928
|
+
intervalMs: this.options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS,
|
|
1929
|
+
// This client rejects with a plain `{ code, message }` object rather than
|
|
1930
|
+
// an Error, so a timeout is only recognizable from the rendered message.
|
|
1931
|
+
isUnanswered: (error) => isTimeout({ message: describeSocketError(error) }),
|
|
1932
|
+
maxConsecutiveMisses: this.options.heartbeatMaxMisses,
|
|
1933
|
+
maxConsecutiveUnanswered: this.options.heartbeatMaxUnanswered,
|
|
1934
|
+
onLost: (account) => {
|
|
1935
|
+
const held = heldRef.current;
|
|
1936
|
+
console.warn("gateway liveness beat lost", {
|
|
1937
|
+
applicationId: held?.applicationId,
|
|
1938
|
+
failures: account.failures,
|
|
1939
|
+
lapses: account.lapses,
|
|
1940
|
+
message: account.lastError,
|
|
1941
|
+
});
|
|
1942
|
+
},
|
|
1943
|
+
onOutcome: (outcome, durationMs, account) => {
|
|
1944
|
+
const held = heldRef.current;
|
|
1945
|
+
if (outcome !== "success") {
|
|
1946
|
+
console.warn(`gateway liveness beat ${outcome}`, {
|
|
1947
|
+
applicationId: held?.applicationId,
|
|
1948
|
+
durationMs,
|
|
1949
|
+
failures: account.failures,
|
|
1950
|
+
lapses: account.lapses,
|
|
1951
|
+
message: account.lastError,
|
|
1952
|
+
});
|
|
1953
|
+
}
|
|
1954
|
+
if (!held) {
|
|
1955
|
+
return;
|
|
1956
|
+
}
|
|
1957
|
+
// The clock measures silence, so anything that is not silence ends it.
|
|
1958
|
+
// A success and a refusal both prove the peer answered this beat - and
|
|
1959
|
+
// a refusal must never cost a turn, since it is an application-level
|
|
1960
|
+
// problem on a reachable connection. A lapse is the heartbeat's own
|
|
1961
|
+
// verdict that inbound traffic explained the quiet, which un-latches
|
|
1962
|
+
// its alarm too. Only a genuinely unanswered beat keeps the clock.
|
|
1963
|
+
if (outcome !== "grace" && outcome !== "miss") {
|
|
1964
|
+
held.beatLostAt = undefined;
|
|
1965
|
+
return;
|
|
1966
|
+
}
|
|
1967
|
+
// Started here rather than from `onLost`, which fires only on the edge
|
|
1968
|
+
// into the lost state. A refusal stops this clock without un-latching
|
|
1969
|
+
// that state, so the edge has already been spent and would never come
|
|
1970
|
+
// again - and silence returning after a refusal could then never end
|
|
1971
|
+
// the turn. `miss` is that same edge as the heartbeat draws it: `grace`
|
|
1972
|
+
// is still inside the missed-beat allowance, so it keeps a clock that
|
|
1973
|
+
// is already running without starting one.
|
|
1974
|
+
if (outcome === "miss") {
|
|
1975
|
+
held.beatLostAt ??= Date.now();
|
|
1976
|
+
}
|
|
1977
|
+
if (held.beatLostAt === undefined) {
|
|
1978
|
+
return;
|
|
1979
|
+
}
|
|
1980
|
+
const graceMs = this.options.beatLostGraceMs ?? DEFAULT_BEAT_LOST_GRACE_MS;
|
|
1981
|
+
if (Date.now() - held.beatLostAt >= graceMs) {
|
|
1982
|
+
this.abandonTurnOnBeatLost(held.applicationId, account);
|
|
1983
|
+
}
|
|
1984
|
+
},
|
|
1985
|
+
// The sandbox grants liveness on proof of authorization, so a beat sent
|
|
1986
|
+
// before this session has a credential would be refused; skipping it
|
|
1987
|
+
// costs nothing, because a session with no credential has no turn to
|
|
1988
|
+
// protect yet either.
|
|
1989
|
+
shouldBeat: () => {
|
|
1990
|
+
const held = heldRef.current;
|
|
1991
|
+
return (held !== undefined &&
|
|
1992
|
+
Boolean(held.sessionToken) &&
|
|
1993
|
+
isSocketOpen(held.ws));
|
|
1994
|
+
},
|
|
1995
|
+
timeoutMs: this.options.heartbeatTimeoutMs ?? DEFAULT_HEARTBEAT_TIMEOUT_MS,
|
|
1996
|
+
});
|
|
1997
|
+
}
|
|
1998
|
+
armIdle(held) {
|
|
1999
|
+
if (held.idleTimer) {
|
|
2000
|
+
clearTimeout(held.idleTimer);
|
|
2001
|
+
}
|
|
2002
|
+
const idleMs = this.options.idleMs ?? DEFAULT_IDLE_MS;
|
|
2003
|
+
held.idleTimer = setTimeout(() => {
|
|
2004
|
+
this.release(held.applicationId);
|
|
2005
|
+
}, idleMs);
|
|
2006
|
+
held.idleTimer.unref?.();
|
|
2007
|
+
}
|
|
2008
|
+
/**
|
|
2009
|
+
* Ends a turn whose live edit is gone.
|
|
2010
|
+
*
|
|
2011
|
+
* The dev server exits on its own when its lock is lost, and it says nothing
|
|
2012
|
+
* on the way out. A turn left waiting on that socket burns its entire budget
|
|
2013
|
+
* — ten minutes — before the caller is told anything.
|
|
2014
|
+
*
|
|
2015
|
+
* Reached only once the live edit being gone is established, which a close on
|
|
2016
|
+
* its own never does: either it named its own end in the application range,
|
|
2017
|
+
* or {@link recoverTurn} could not get back to it for the whole reconnect
|
|
2018
|
+
* grace. `live_edit_terminated` is a claim about the sandbox, so it is worth
|
|
2019
|
+
* only as much as the evidence behind it. The close code and heartbeat
|
|
2020
|
+
* account go in the reply, since they are the only account of the cause that
|
|
2021
|
+
* reaches the caller.
|
|
2022
|
+
*/
|
|
2023
|
+
abandonTurnOnDisconnect(applicationId, close = {}, heartbeat) {
|
|
2024
|
+
const turn = this.turns.get(applicationId);
|
|
2025
|
+
if (!turn || turn.finished) {
|
|
2026
|
+
return;
|
|
2027
|
+
}
|
|
2028
|
+
const beats = heartbeat ? describeHeartbeat(heartbeat, Date.now()) : "";
|
|
2029
|
+
console.warn("gateway clark turn abandoned", {
|
|
2030
|
+
applicationId,
|
|
2031
|
+
beatFailures: heartbeat?.failures,
|
|
2032
|
+
beatLapses: heartbeat?.lapses,
|
|
2033
|
+
// A refusal never counts as a failure, so without these two a death that
|
|
2034
|
+
// the reply explains in full - every answer an error, or a silence the
|
|
2035
|
+
// socket's own traffic excused - logs as `beatFailures: 0`, which reads
|
|
2036
|
+
// as a beat that had nothing to report.
|
|
2037
|
+
beatRefusals: heartbeat?.refusals,
|
|
2038
|
+
beatsSent: heartbeat?.sent,
|
|
2039
|
+
beatsUnanswered: heartbeat?.unanswered,
|
|
2040
|
+
closeCode: close.code,
|
|
2041
|
+
closeError: close.error,
|
|
2042
|
+
closeReason: close.reason,
|
|
2043
|
+
lastBeatError: heartbeat?.lastError,
|
|
2044
|
+
lastBeatSuccessAt: heartbeat?.lastSuccessAt,
|
|
2045
|
+
pendingBeatSentAt: heartbeat?.pendingSince,
|
|
2046
|
+
reason: "live-edit session disconnected",
|
|
2047
|
+
});
|
|
2048
|
+
this.settleAsLiveEditTerminated(turn, describeClose(close) + beats);
|
|
2049
|
+
}
|
|
2050
|
+
/**
|
|
2051
|
+
* Ends a turn SABS has just said has no live edit left to run on.
|
|
2052
|
+
*
|
|
2053
|
+
* A socket close is the other trigger for this same outcome, and it is the
|
|
2054
|
+
* slower one: the dev server exits silently when it loses its lock, so the
|
|
2055
|
+
* close can be minutes late or never arrive at all. This event is the control
|
|
2056
|
+
* plane stating the session is gone, which is the same fact without waiting
|
|
2057
|
+
* for a connection to notice it.
|
|
2058
|
+
*
|
|
2059
|
+
* Only for a turn that has not settled itself, though. Every healthy build
|
|
2060
|
+
* ends with its session torn down too, and reading that as a dead live edit
|
|
2061
|
+
* would report each finished app as a failure. A collector that already
|
|
2062
|
+
* settled on a terminal state keeps it; the teardown is the outcome only for
|
|
2063
|
+
* a turn still waiting for one. The disconnect path needs no such guard - an
|
|
2064
|
+
* unsolicited close mid-turn is abnormal in its own right, where a teardown
|
|
2065
|
+
* is what the end of every turn looks like.
|
|
2066
|
+
*
|
|
2067
|
+
* Settled, not merely reached: a terminal state is provisional until the
|
|
2068
|
+
* quiet window closes, because a build that resumes after a false
|
|
2069
|
+
* completed/idle clears it. A teardown inside that window therefore still
|
|
2070
|
+
* ends the turn - which is the honest answer, since the checkpoint that
|
|
2071
|
+
* would follow runs over the socket to the dev server that just died and
|
|
2072
|
+
* could not have landed either way.
|
|
2073
|
+
*/
|
|
2074
|
+
abandonTurnOnSessionTerminated(applicationId, turn) {
|
|
2075
|
+
if (turn.finished || turn.collector.status()) {
|
|
2076
|
+
return;
|
|
2077
|
+
}
|
|
2078
|
+
console.warn("gateway clark turn abandoned", {
|
|
2079
|
+
applicationId,
|
|
2080
|
+
reason: "SABS terminated the live-edit session",
|
|
2081
|
+
});
|
|
2082
|
+
this.settleAsLiveEditTerminated(turn, "");
|
|
2083
|
+
}
|
|
2084
|
+
/**
|
|
2085
|
+
* Ends a turn whose socket has stopped answering.
|
|
2086
|
+
*
|
|
2087
|
+
* Two gates, because they answer different questions. `SocketHeartbeat`
|
|
2088
|
+
* declares the beat lost after consecutive genuine misses, having already
|
|
2089
|
+
* excused a run of unanswered beats and every silence that inbound traffic
|
|
2090
|
+
* explained - the right threshold for a diagnostic, and too eager for this: a
|
|
2091
|
+
* sandbox that misses two beats and answers the next was recovering, and its
|
|
2092
|
+
* turn has to survive that. So the loss also has to outlast
|
|
2093
|
+
* {@link DEFAULT_BEAT_LOST_GRACE_MS}, and any beat that succeeds clears it.
|
|
2094
|
+
*
|
|
2095
|
+
* What is left is a request channel that has been silent for minutes on a
|
|
2096
|
+
* socket that never closed, with a turn waiting on it - which without this
|
|
2097
|
+
* waits out the whole turn budget.
|
|
2098
|
+
*
|
|
2099
|
+
* This is the only trigger that reaches a half-open socket. An unsolicited
|
|
2100
|
+
* close is the ordinary one and never arrives for a connection that stays up
|
|
2101
|
+
* with nothing behind it, and the SABS `session: terminated` event cannot
|
|
2102
|
+
* reach a gateway with no inbound route - which is every gateway run on a
|
|
2103
|
+
* laptop, where a loopback public URL leaves SABS no way to post at all.
|
|
2104
|
+
*
|
|
2105
|
+
* The socket is deliberately left in place. Whether a connection proven
|
|
2106
|
+
* wedged should also be torn down rather than handed to the next prompt is a
|
|
2107
|
+
* separate question from whether this turn should keep waiting on it.
|
|
2108
|
+
*/
|
|
2109
|
+
abandonTurnOnBeatLost(applicationId, heartbeat) {
|
|
2110
|
+
const turn = this.turns.get(applicationId);
|
|
2111
|
+
// The same rule the disconnect and SABS paths use: settle only a turn that
|
|
2112
|
+
// has not settled itself. A finished turn's socket keeps beating through
|
|
2113
|
+
// the post-turn grace, so a sandbox reaped in that window must not
|
|
2114
|
+
// overwrite the outcome the caller already has.
|
|
2115
|
+
if (!turn || turn.finished || turn.collector.status()) {
|
|
2116
|
+
return;
|
|
2117
|
+
}
|
|
2118
|
+
console.warn("gateway clark turn abandoned", {
|
|
2119
|
+
applicationId,
|
|
2120
|
+
beatFailures: heartbeat.failures,
|
|
2121
|
+
beatLapses: heartbeat.lapses,
|
|
2122
|
+
beatsSent: heartbeat.sent,
|
|
2123
|
+
lastBeatError: heartbeat.lastError,
|
|
2124
|
+
reason: "the live-edit socket stopped answering",
|
|
2125
|
+
});
|
|
2126
|
+
this.settleAsLiveEditTerminated(turn, describeHeartbeat(heartbeat, Date.now()));
|
|
2127
|
+
}
|
|
2128
|
+
/**
|
|
2129
|
+
* Records a dead live edit as the turn's outcome and ends every wait on it.
|
|
2130
|
+
* `evidence` is appended to the reply because it is the only account of the
|
|
2131
|
+
* cause that reaches the caller.
|
|
2132
|
+
*/
|
|
2133
|
+
settleAsLiveEditTerminated(turn, evidence) {
|
|
2134
|
+
this.settleTurn(turn, {
|
|
2135
|
+
reply: LIVE_EDIT_TERMINATED_REPLY + evidence,
|
|
2136
|
+
generationStatus: "live_edit_terminated",
|
|
2137
|
+
viewStatus: turn.collector.viewableStatus(),
|
|
2138
|
+
});
|
|
2139
|
+
}
|
|
2140
|
+
/**
|
|
2141
|
+
* Decides what a close the gateway did not ask for actually means, by going
|
|
2142
|
+
* back for the live edit rather than inferring from the close.
|
|
2143
|
+
*
|
|
2144
|
+
* The reattach machinery exists for exactly this: a turn whose socket went
|
|
2145
|
+
* away is one the gateway can move onto a replacement. Abandoning here is
|
|
2146
|
+
* what used to make that unreachable, and it charged a recoverable blip the
|
|
2147
|
+
* whole turn - the caller is told to send the prompt again, and the work
|
|
2148
|
+
* already done in it is discarded.
|
|
2149
|
+
*
|
|
2150
|
+
* Each of the three outcomes is a different thing to tell the caller:
|
|
2151
|
+
*
|
|
2152
|
+
* - taken over: nothing to tell. The turn carries on where it got to.
|
|
2153
|
+
* - unreachable for the whole grace: the termination the close only hinted
|
|
2154
|
+
* at, now with the evidence to say so.
|
|
2155
|
+
* - reached but refused the turn: the sandbox is alive and building where
|
|
2156
|
+
* this gateway cannot watch it, which is `detached`. Calling that a death
|
|
2157
|
+
* would send the caller to retry a build that is still running.
|
|
2158
|
+
*
|
|
2159
|
+
* A reconnect that fails while SABS is still relaying the turn settles
|
|
2160
|
+
* nothing: the socket is one of two ways this turn reports, and the events
|
|
2161
|
+
* still arriving are the better account of whether the build is alive.
|
|
2162
|
+
*
|
|
2163
|
+
* Never throws: nothing awaits this, and every outcome it can reach belongs
|
|
2164
|
+
* on the turn rather than on the socket handler that started it.
|
|
2165
|
+
*/
|
|
2166
|
+
async recoverTurn(input) {
|
|
2167
|
+
const applicationId = input.previous.applicationId;
|
|
2168
|
+
const turn = this.turns.get(applicationId);
|
|
2169
|
+
// A turn that has moved on from this session, or already has an outcome,
|
|
2170
|
+
// has nothing riding on the socket that just closed.
|
|
2171
|
+
if (!turn || turn.finished || turn.held !== input.previous) {
|
|
2172
|
+
return;
|
|
2173
|
+
}
|
|
2174
|
+
const heartbeat = input.previous.heartbeat.account;
|
|
2175
|
+
const heardAt = turn.collector.lastEventAt();
|
|
2176
|
+
if (!this.takeReconnectAllowance(turn, heardAt)) {
|
|
2177
|
+
console.warn("gateway clark turn reconnects exhausted", {
|
|
2178
|
+
applicationId,
|
|
2179
|
+
attempts: turn.reconnects,
|
|
2180
|
+
closeCode: input.close.code,
|
|
2181
|
+
});
|
|
2182
|
+
this.abandonTurnOnDisconnect(applicationId, input.close, heartbeat);
|
|
2183
|
+
return;
|
|
2184
|
+
}
|
|
2185
|
+
this.note(turn, "Reconnecting to the Superblocks live-edit session");
|
|
2186
|
+
const outcome = await this.takeOverTurn({
|
|
2187
|
+
deadline: Date.now() +
|
|
2188
|
+
(this.options.reconnectGraceMs ?? DEFAULT_RECONNECT_GRACE_MS),
|
|
2189
|
+
previous: input.previous,
|
|
2190
|
+
turn,
|
|
2191
|
+
});
|
|
2192
|
+
if (turn.finished || outcome.kind === "stale") {
|
|
2193
|
+
return;
|
|
2194
|
+
}
|
|
2195
|
+
if (outcome.kind === "took_over") {
|
|
2196
|
+
this.note(turn, "Reconnected to the Superblocks live-edit session");
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
// TODO(ENG-5922): nothing tries to reconnect again from here. A turn kept
|
|
2200
|
+
// alive on the relay reaches its terminal state with no socket, so the
|
|
2201
|
+
// checkpoint that would commit Clark's work cannot land and the turn
|
|
2202
|
+
// completes without one.
|
|
2203
|
+
if (turn.collector.lastEventAt() > heardAt) {
|
|
2204
|
+
console.warn("gateway clark turn kept on the relay", {
|
|
2205
|
+
applicationId,
|
|
2206
|
+
closeCode: input.close.code,
|
|
2207
|
+
outcome: outcome.kind,
|
|
2208
|
+
});
|
|
2209
|
+
this.note(turn, "Lost the live-edit socket; still following the build through Superblocks");
|
|
2210
|
+
return;
|
|
2211
|
+
}
|
|
2212
|
+
if (outcome.kind === "unreachable") {
|
|
2213
|
+
this.abandonTurnOnDisconnect(applicationId, input.close, heartbeat);
|
|
2214
|
+
return;
|
|
2215
|
+
}
|
|
2216
|
+
console.warn("gateway clark turn not handed back", {
|
|
2217
|
+
applicationId,
|
|
2218
|
+
closeCode: input.close.code,
|
|
2219
|
+
});
|
|
2220
|
+
this.settleTurn(turn, {
|
|
2221
|
+
reply: turn.collector.reply(),
|
|
2222
|
+
generationStatus: "detached",
|
|
2223
|
+
viewStatus: turn.collector.viewableStatus(),
|
|
2224
|
+
...decisionFields(turn.collector.decision()),
|
|
2225
|
+
});
|
|
2226
|
+
}
|
|
2227
|
+
/**
|
|
2228
|
+
* Ends a prior turn for `applicationId` before a newer prompt claims the map
|
|
2229
|
+
* slot. Callers polling the old turn see `cancelled`; its runTurn exits via
|
|
2230
|
+
* AbortSignal and must not keep listening or arming release on the held
|
|
2231
|
+
* session.
|
|
2232
|
+
*/
|
|
2233
|
+
supersedeTurn(applicationId) {
|
|
2234
|
+
const prior = this.turns.get(applicationId);
|
|
2235
|
+
this.turns.delete(applicationId);
|
|
2236
|
+
if (!prior) {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
prior.finished ??= {
|
|
2240
|
+
reply: "This turn was replaced by a newer prompt on the same app, so its work was cancelled.",
|
|
2241
|
+
generationStatus: "cancelled",
|
|
2242
|
+
viewStatus: prior.collector.viewableStatus(),
|
|
2243
|
+
};
|
|
2244
|
+
const held = this.held.get(applicationId);
|
|
2245
|
+
held?.listeners.delete(prior.listener);
|
|
2246
|
+
prior.unsubscribe();
|
|
2247
|
+
prior.collector.dispose();
|
|
2248
|
+
prior.abort.abort();
|
|
2249
|
+
this.wake(prior);
|
|
2250
|
+
}
|
|
2251
|
+
armPostTurnRelease(held, turn) {
|
|
2252
|
+
this.clearTimers(held);
|
|
2253
|
+
const delayMs = this.options.releaseAfterTurnMs ?? DEFAULT_RELEASE_AFTER_TURN_MS;
|
|
2254
|
+
held.idleTimer = setTimeout(() => {
|
|
2255
|
+
const active = this.turns.get(held.applicationId);
|
|
2256
|
+
if (active && active !== turn && !active.finished) {
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
if (this.held.get(held.applicationId) === held) {
|
|
2260
|
+
this.drop(held);
|
|
2261
|
+
}
|
|
2262
|
+
}, delayMs);
|
|
2263
|
+
held.idleTimer.unref?.();
|
|
2264
|
+
}
|
|
2265
|
+
armKeepalive(held) {
|
|
2266
|
+
if (held.keepaliveTimer) {
|
|
2267
|
+
clearInterval(held.keepaliveTimer);
|
|
2268
|
+
held.keepaliveTimer = undefined;
|
|
2269
|
+
}
|
|
2270
|
+
if (!held.keepalive) {
|
|
2271
|
+
return;
|
|
2272
|
+
}
|
|
2273
|
+
const keepaliveMs = this.options.keepaliveMs ?? DEFAULT_KEEPALIVE_MS;
|
|
2274
|
+
held.keepaliveTimer = setInterval(() => {
|
|
2275
|
+
void held.keepalive?.().catch(() => {
|
|
2276
|
+
// Best-effort: a failed connect refresh must not tear down the held
|
|
2277
|
+
// session socket; the next MCP call will reconnect if needed.
|
|
2278
|
+
});
|
|
2279
|
+
}, keepaliveMs);
|
|
2280
|
+
held.keepaliveTimer.unref?.();
|
|
2281
|
+
}
|
|
2282
|
+
/**
|
|
2283
|
+
* Clears the turn timers only. The beat is stopped separately on purpose: it
|
|
2284
|
+
* belongs to the socket and outlives these, which the post-turn grace period
|
|
2285
|
+
* resets.
|
|
2286
|
+
*/
|
|
2287
|
+
clearTimers(held) {
|
|
2288
|
+
if (held.idleTimer) {
|
|
2289
|
+
clearTimeout(held.idleTimer);
|
|
2290
|
+
held.idleTimer = undefined;
|
|
2291
|
+
}
|
|
2292
|
+
if (held.keepaliveTimer) {
|
|
2293
|
+
clearInterval(held.keepaliveTimer);
|
|
2294
|
+
held.keepaliveTimer = undefined;
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
drop(held) {
|
|
2298
|
+
console.error("gateway held session released", {
|
|
2299
|
+
applicationId: held.applicationId,
|
|
2300
|
+
});
|
|
2301
|
+
held.dropping = true;
|
|
2302
|
+
this.clearTimers(held);
|
|
2303
|
+
held.heartbeat.stop();
|
|
2304
|
+
// Only if it is still the current session: a reattach registers its
|
|
2305
|
+
// replacement before closing this one, and unregistering by app id alone
|
|
2306
|
+
// would take the replacement out of the map with it.
|
|
2307
|
+
if (this.held.get(held.applicationId) === held) {
|
|
2308
|
+
this.held.delete(held.applicationId);
|
|
2309
|
+
}
|
|
2310
|
+
try {
|
|
2311
|
+
held.socket.close("held session released");
|
|
2312
|
+
}
|
|
2313
|
+
catch {
|
|
2314
|
+
// Socket may already be closed.
|
|
2315
|
+
}
|
|
2316
|
+
try {
|
|
2317
|
+
if (held.ws.readyState === WebSocket.OPEN ||
|
|
2318
|
+
held.ws.readyState === WebSocket.CONNECTING) {
|
|
2319
|
+
held.ws.close();
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
catch {
|
|
2323
|
+
// Ignore double-close.
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
editorContextFor(held) {
|
|
2327
|
+
if (!held?.apiKey ||
|
|
2328
|
+
!held.organizationId ||
|
|
2329
|
+
!held.sessionToken ||
|
|
2330
|
+
!this.options.server ||
|
|
2331
|
+
!held.orchestrator) {
|
|
2332
|
+
return undefined;
|
|
2333
|
+
}
|
|
2334
|
+
return {
|
|
2335
|
+
apiKey: held.apiKey,
|
|
2336
|
+
applicationId: held.applicationId,
|
|
2337
|
+
branchName: held.branchName,
|
|
2338
|
+
organizationId: held.organizationId,
|
|
2339
|
+
orchestrator: held.orchestrator,
|
|
2340
|
+
profileKey: held.profileKey ?? pickEditorProfileKey(this.options.profileKey, []),
|
|
2341
|
+
server: this.options.server,
|
|
2342
|
+
sessionToken: held.sessionToken,
|
|
2343
|
+
waitUntilFilesSynced: async () => {
|
|
2344
|
+
const collector = this.turns.get(held.applicationId)?.collector;
|
|
2345
|
+
if (!collector) {
|
|
2346
|
+
return true;
|
|
2347
|
+
}
|
|
2348
|
+
return await collector.waitUntilInSync({
|
|
2349
|
+
timeoutMs: this.options.readyWaitMs ?? DEFAULT_READY_WAIT_MS,
|
|
2350
|
+
});
|
|
2351
|
+
},
|
|
2352
|
+
};
|
|
2353
|
+
}
|
|
2354
|
+
/**
|
|
2355
|
+
* Context the sandbox needs before it will let Clark drive a browser.
|
|
2356
|
+
*
|
|
2357
|
+
* Absent when headless testing is off or the session is missing something
|
|
2358
|
+
* the app shell cannot boot without — Clark then runs the turn with its
|
|
2359
|
+
* ordinary tools rather than failing it.
|
|
2360
|
+
*/
|
|
2361
|
+
browserContextFor(held) {
|
|
2362
|
+
if (!this.options.testingEnabled) {
|
|
2363
|
+
return undefined;
|
|
2364
|
+
}
|
|
2365
|
+
return this.buildHeldBrowserContext(held);
|
|
2366
|
+
}
|
|
2367
|
+
/**
|
|
2368
|
+
* Library iframe bootstrap from a held session. Used for Gateway's own
|
|
2369
|
+
* live-canvas screenshot even when Clark's in-sandbox Playwright is off.
|
|
2370
|
+
*/
|
|
2371
|
+
buildHeldBrowserContext(held) {
|
|
2372
|
+
if (!held.orchestratorUrl) {
|
|
2373
|
+
console.error("gateway live canvas skipped: missing_orchestrator_url", {
|
|
2374
|
+
applicationId: held.applicationId,
|
|
2375
|
+
});
|
|
2376
|
+
return undefined;
|
|
2377
|
+
}
|
|
2378
|
+
if (!this.options.uiBaseUrl) {
|
|
2379
|
+
console.error("gateway live canvas skipped: missing_ui_base_url", {
|
|
2380
|
+
applicationId: held.applicationId,
|
|
2381
|
+
});
|
|
2382
|
+
return undefined;
|
|
2383
|
+
}
|
|
2384
|
+
if (!held.sessionToken) {
|
|
2385
|
+
console.error("gateway live canvas skipped: missing_session_token", {
|
|
2386
|
+
applicationId: held.applicationId,
|
|
2387
|
+
});
|
|
2388
|
+
return undefined;
|
|
2389
|
+
}
|
|
2390
|
+
return buildBrowserContext({
|
|
2391
|
+
...(held.apiKey ? { apiKey: held.apiKey } : {}),
|
|
2392
|
+
applicationId: held.applicationId,
|
|
2393
|
+
liveEditUrl: held.liveEdit.url,
|
|
2394
|
+
orchestratorUrl: held.orchestratorUrl,
|
|
2395
|
+
organizationId: held.organizationId,
|
|
2396
|
+
peerId: held.peerId,
|
|
2397
|
+
profileKey: held.profileKey,
|
|
2398
|
+
profiles: held.profiles,
|
|
2399
|
+
sessionToken: held.sessionToken,
|
|
2400
|
+
uiBaseUrl: this.options.uiBaseUrl,
|
|
2401
|
+
userId: held.userId,
|
|
2402
|
+
});
|
|
2403
|
+
}
|
|
2404
|
+
/**
|
|
2405
|
+
* Resolves a real org profile key once per held session. Sending the legacy
|
|
2406
|
+
* placeholder `default` makes the orchestrator fail every integration query
|
|
2407
|
+
* with "Invalid profile specified" / HTTP 500.
|
|
2408
|
+
*/
|
|
2409
|
+
async resolveHeldProfileKey(held) {
|
|
2410
|
+
if (held.profileKey || !held.apiKey || !held.organizationId) {
|
|
2411
|
+
return;
|
|
2412
|
+
}
|
|
2413
|
+
if (!this.options.server) {
|
|
2414
|
+
held.profileKey = pickEditorProfileKey(this.options.profileKey, []);
|
|
2415
|
+
return;
|
|
2416
|
+
}
|
|
2417
|
+
try {
|
|
2418
|
+
const profiles = await this.options.server.getOrganizationProfiles({
|
|
2419
|
+
apiKey: held.apiKey,
|
|
2420
|
+
organizationId: held.organizationId,
|
|
2421
|
+
});
|
|
2422
|
+
held.profiles = profiles;
|
|
2423
|
+
held.profileKey = pickEditorProfileKey(this.options.profileKey, profiles);
|
|
2424
|
+
console.error("gateway resolved integration profile", {
|
|
2425
|
+
applicationId: held.applicationId,
|
|
2426
|
+
organizationId: held.organizationId,
|
|
2427
|
+
profileKey: held.profileKey,
|
|
2428
|
+
preferred: this.options.profileKey,
|
|
2429
|
+
});
|
|
2430
|
+
}
|
|
2431
|
+
catch (error) {
|
|
2432
|
+
held.profileKey = pickEditorProfileKey(this.options.profileKey, []);
|
|
2433
|
+
console.warn("gateway failed to list org profiles; using fallback", {
|
|
2434
|
+
applicationId: held.applicationId,
|
|
2435
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2436
|
+
profileKey: held.profileKey,
|
|
2437
|
+
});
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
/**
|
|
2441
|
+
* Discovers the orchestrator / agent base URL the way the browser does:
|
|
2442
|
+
* Server `/users/me` agents filtered by org agentType, Active status, and
|
|
2443
|
+
* profile tags. Operators never set this URL.
|
|
2444
|
+
*/
|
|
2445
|
+
async resolveHeldOrchestrator(held) {
|
|
2446
|
+
if (held.orchestrator || held.orchestratorUrl) {
|
|
2447
|
+
return;
|
|
2448
|
+
}
|
|
2449
|
+
if (this.options.orchestrator || this.options.orchestratorUrl) {
|
|
2450
|
+
held.orchestrator = this.options.orchestrator;
|
|
2451
|
+
held.orchestratorUrl = this.options.orchestratorUrl;
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
if (!held.apiKey || !held.organizationId || !this.options.server) {
|
|
2455
|
+
return;
|
|
2456
|
+
}
|
|
2457
|
+
const profileKey = held.profileKey ?? pickEditorProfileKey(this.options.profileKey, []);
|
|
2458
|
+
try {
|
|
2459
|
+
const inventory = await this.options.server.getAgentInventory({
|
|
2460
|
+
apiKey: held.apiKey,
|
|
2461
|
+
organizationId: held.organizationId,
|
|
2462
|
+
});
|
|
2463
|
+
const orchestratorUrl = pickOrchestratorBaseUrl({
|
|
2464
|
+
agentType: inventory.agentType,
|
|
2465
|
+
agents: inventory.agents,
|
|
2466
|
+
profileKey,
|
|
2467
|
+
});
|
|
2468
|
+
held.orchestratorUrl = orchestratorUrl;
|
|
2469
|
+
held.orchestrator = new HttpOrchestratorClient(orchestratorUrl);
|
|
2470
|
+
console.error("gateway resolved orchestrator url", {
|
|
2471
|
+
applicationId: held.applicationId,
|
|
2472
|
+
organizationId: held.organizationId,
|
|
2473
|
+
orchestratorUrl,
|
|
2474
|
+
profileKey,
|
|
2475
|
+
});
|
|
2476
|
+
}
|
|
2477
|
+
catch (error) {
|
|
2478
|
+
console.warn("gateway failed to discover orchestrator url", {
|
|
2479
|
+
applicationId: held.applicationId,
|
|
2480
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2481
|
+
organizationId: held.organizationId,
|
|
2482
|
+
profileKey,
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
//# sourceMappingURL=websocket-session-peer.js.map
|