@opengeni/worker-bundle 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7113 -1652
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4337 -1052
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
package/src/sandbox-resume.ts
CHANGED
|
@@ -6,13 +6,12 @@
|
|
|
6
6
|
//
|
|
7
7
|
// 1. acquireLease (group-keyed) under the DB FOR UPDATE + cold->warming CAS —
|
|
8
8
|
// the SOLE double-spawn guard (P1.1).
|
|
9
|
-
// 2.
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
// non-desktop op or after a snapshot rollover dropped the X stack.
|
|
9
|
+
// 2. The cold->warming winner may create/restore. Every attached caller is
|
|
10
|
+
// resume-only; a missing provider box retires the exact warm epoch and
|
|
11
|
+
// re-enters admission instead of creating a rival.
|
|
12
|
+
// 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
|
|
13
|
+
// envelope onto the lease). Optional desktop work is deliberately absent:
|
|
14
|
+
// the viewer and the first actual computer-use action initialize :0 lazily.
|
|
16
15
|
// 4. the caller injects {client, session, sessionState} NON-OWNED into the run
|
|
17
16
|
// (the SDK never reaps it — the keystone), runs, then in `finally` calls the
|
|
18
17
|
// returned `release()` and drops the in-memory handle. NEVER provider-delete
|
|
@@ -27,41 +26,58 @@ import {
|
|
|
27
26
|
commitWarmingToWarm,
|
|
28
27
|
failWarmingToCold,
|
|
29
28
|
getSandboxSessionEnvelope,
|
|
29
|
+
markWarmLeaseInstanceLost,
|
|
30
|
+
persistWarmSnapshot,
|
|
30
31
|
readLease,
|
|
31
32
|
recordWarmingSandboxCreated,
|
|
32
33
|
releaseLeaseHolder,
|
|
34
|
+
touchLeaseHolder,
|
|
33
35
|
SandboxLeaseSupersededError,
|
|
34
36
|
type Database,
|
|
35
37
|
type LeaseHolderKind,
|
|
36
38
|
} from "@opengeni/db";
|
|
37
39
|
import {
|
|
38
40
|
establishSandboxSessionFromEnvelope,
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
isProviderSandboxNotFoundError,
|
|
42
|
+
SandboxResumeStateUnavailableError,
|
|
41
43
|
serializeEstablishedSandboxEnvelope,
|
|
42
|
-
|
|
43
|
-
DisplayStackUnsupportedError,
|
|
44
|
-
buildStreamUrl,
|
|
45
|
-
StreamPortUnavailableError,
|
|
46
|
-
modalSandboxAttributionEnvironment,
|
|
44
|
+
deletePriorPersistedSnapshot,
|
|
47
45
|
tagModalSandbox,
|
|
48
46
|
type EstablishedSandboxSession,
|
|
49
|
-
type ExposedPortEndpoint,
|
|
50
47
|
type RuntimeMetricsHooks,
|
|
51
48
|
} from "@opengeni/runtime";
|
|
52
|
-
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
53
|
-
|
|
54
|
-
export { DESKTOP_STREAM_PORT };
|
|
55
49
|
|
|
56
50
|
// Re-exported for callers that just want the ack-kind union.
|
|
57
51
|
export type ResumeHolderKind = LeaseHolderKind;
|
|
58
52
|
|
|
53
|
+
/**
|
|
54
|
+
* The durable identity of a turn holding a shared sandbox lease.
|
|
55
|
+
*
|
|
56
|
+
* Temporal activity ids cannot satisfy this type: they are workflow-local
|
|
57
|
+
* sequence numbers and collide across sessions that share a sandbox group.
|
|
58
|
+
*/
|
|
59
|
+
export type TurnSandboxLeaseHolderId = `turn-attempt:${string}`;
|
|
60
|
+
|
|
61
|
+
export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLeaseHolderId {
|
|
62
|
+
const normalized = attemptId.trim();
|
|
63
|
+
if (!normalized) {
|
|
64
|
+
throw new Error("Sandbox lease holder requires a turn attempt id");
|
|
65
|
+
}
|
|
66
|
+
return `turn-attempt:${normalized}`;
|
|
67
|
+
}
|
|
68
|
+
|
|
59
69
|
/** The minimal services surface resumeBoxForTurn needs. A subset of
|
|
60
70
|
* ActivityServices so a test (and the API later) can pass a lean bag. */
|
|
61
71
|
export type SandboxResumeServices = {
|
|
62
72
|
db: Database;
|
|
63
73
|
settings: Settings;
|
|
64
74
|
sandboxMetrics?: RuntimeMetricsHooks;
|
|
75
|
+
/** Called only by the observer that wins the exact warm->cold loss CAS. */
|
|
76
|
+
onSandboxLost?: (input: {
|
|
77
|
+
sandboxGroupId: string;
|
|
78
|
+
instanceId: string;
|
|
79
|
+
leaseEpoch: number;
|
|
80
|
+
}) => Promise<void>;
|
|
65
81
|
};
|
|
66
82
|
|
|
67
83
|
export type ResumeBoxIds = {
|
|
@@ -94,6 +110,14 @@ export type ResumeBoxIds = {
|
|
|
94
110
|
* selfhosted path never passes it; a legacy/null-image box never conflicts).
|
|
95
111
|
*/
|
|
96
112
|
image?: string;
|
|
113
|
+
/**
|
|
114
|
+
* RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
|
|
115
|
+
* acquireLease, which stamps it on the cold-create and conflicts on a live box
|
|
116
|
+
* set up under a DIFFERENT rig version (solo holder recreates cold on the new
|
|
117
|
+
* rig; N-holders throw SandboxRigConflictError). Omitted for a rig-less session
|
|
118
|
+
* -> rig is never stamped or enforced (shares exactly as today).
|
|
119
|
+
*/
|
|
120
|
+
rigVersionId?: string;
|
|
97
121
|
};
|
|
98
122
|
|
|
99
123
|
/** What resumeBoxForTurn returns: the live NON-OWNED session to inject, the
|
|
@@ -113,7 +137,10 @@ export type ResumedTurnSandbox = {
|
|
|
113
137
|
export class SandboxWarmingTimeoutError extends Error {
|
|
114
138
|
readonly code = "sandbox_warming_timeout";
|
|
115
139
|
|
|
116
|
-
constructor(
|
|
140
|
+
constructor(
|
|
141
|
+
public readonly backend: string,
|
|
142
|
+
public readonly timeoutMs: number,
|
|
143
|
+
) {
|
|
117
144
|
super(
|
|
118
145
|
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
|
|
119
146
|
);
|
|
@@ -121,19 +148,124 @@ export class SandboxWarmingTimeoutError extends Error {
|
|
|
121
148
|
}
|
|
122
149
|
}
|
|
123
150
|
|
|
151
|
+
/** The exact attached caller that won warm->cold after proving the provider
|
|
152
|
+
* instance gone. It is a lease supersession (the same logical turn recovers),
|
|
153
|
+
* plus the lost id needed for one durable observability event. */
|
|
154
|
+
export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
|
|
155
|
+
constructor(
|
|
156
|
+
sandboxGroupId: string,
|
|
157
|
+
leaseEpoch: number,
|
|
158
|
+
public readonly lostInstanceId: string,
|
|
159
|
+
) {
|
|
160
|
+
super(sandboxGroupId, leaseEpoch);
|
|
161
|
+
this.name = "SandboxLeaseInstanceLostError";
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
124
165
|
// Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
|
|
125
166
|
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
126
167
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
168
|
+
const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
|
|
127
169
|
|
|
128
170
|
async function sleep(ms: number): Promise<void> {
|
|
129
171
|
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
130
172
|
}
|
|
131
173
|
|
|
132
|
-
|
|
174
|
+
/**
|
|
175
|
+
* Modal may return a sandbox handle before its command router accepts the first
|
|
176
|
+
* exec. The upstream session's yieldTimeMs starts only after sandbox.exec()
|
|
177
|
+
* returns, so it cannot bound that initial RPC. Probe it before publishing the
|
|
178
|
+
* lease as warm and enforce our own wall-clock deadline.
|
|
179
|
+
*/
|
|
180
|
+
export async function waitForSandboxExecReadiness(
|
|
181
|
+
established: EstablishedSandboxSession,
|
|
182
|
+
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
183
|
+
): Promise<void> {
|
|
184
|
+
if (established.backendId !== "modal") return;
|
|
185
|
+
|
|
186
|
+
const session = established.session as {
|
|
187
|
+
exec?: (args: {
|
|
188
|
+
cmd: string;
|
|
189
|
+
yieldTimeMs?: number;
|
|
190
|
+
maxOutputTokens?: number;
|
|
191
|
+
}) => Promise<unknown>;
|
|
192
|
+
execCommand?: (args: {
|
|
193
|
+
cmd: string;
|
|
194
|
+
yieldTimeMs?: number;
|
|
195
|
+
maxOutputTokens?: number;
|
|
196
|
+
}) => Promise<unknown>;
|
|
197
|
+
};
|
|
198
|
+
const run = session.exec ?? session.execCommand;
|
|
199
|
+
if (!run) {
|
|
200
|
+
throw new Error("Modal sandbox session does not expose exec");
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
204
|
+
try {
|
|
205
|
+
await Promise.race([
|
|
206
|
+
run.call(session, {
|
|
207
|
+
cmd: "true",
|
|
208
|
+
yieldTimeMs: 1_000,
|
|
209
|
+
maxOutputTokens: 1_000,
|
|
210
|
+
}),
|
|
211
|
+
new Promise<never>((_, reject) => {
|
|
212
|
+
timer = setTimeout(
|
|
213
|
+
() => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
|
|
214
|
+
timeoutMs,
|
|
215
|
+
);
|
|
216
|
+
if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
217
|
+
}),
|
|
218
|
+
]);
|
|
219
|
+
} finally {
|
|
220
|
+
if (timer) clearTimeout(timer);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
class SnapshotTimeoutError extends Error {
|
|
225
|
+
constructor(timeoutMs: number) {
|
|
226
|
+
super(`workspace snapshot timed out after ${timeoutMs}ms`);
|
|
227
|
+
this.name = "SnapshotTimeoutError";
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export async function waitForWarmSnapshot(
|
|
232
|
+
snapshot: Promise<unknown>,
|
|
233
|
+
timeoutMs: number,
|
|
234
|
+
): Promise<boolean> {
|
|
235
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
236
|
+
try {
|
|
237
|
+
await Promise.race([
|
|
238
|
+
snapshot,
|
|
239
|
+
new Promise<never>((_, reject) => {
|
|
240
|
+
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
241
|
+
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
242
|
+
timeout.unref();
|
|
243
|
+
}
|
|
244
|
+
}),
|
|
245
|
+
]);
|
|
246
|
+
return true;
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if (error instanceof SnapshotTimeoutError) {
|
|
249
|
+
console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
return true;
|
|
253
|
+
} finally {
|
|
254
|
+
if (timeout) {
|
|
255
|
+
clearTimeout(timeout);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function terminateEstablishedSandbox(
|
|
261
|
+
established: EstablishedSandboxSession | null,
|
|
262
|
+
): Promise<boolean> {
|
|
133
263
|
if (!established) {
|
|
134
264
|
return true;
|
|
135
265
|
}
|
|
136
|
-
const client = established.client as {
|
|
266
|
+
const client = established.client as {
|
|
267
|
+
delete?: (state: unknown) => Promise<unknown>;
|
|
268
|
+
};
|
|
137
269
|
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
138
270
|
try {
|
|
139
271
|
await client.delete(established.sessionState);
|
|
@@ -175,7 +307,10 @@ function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: numbe
|
|
|
175
307
|
: error;
|
|
176
308
|
}
|
|
177
309
|
|
|
178
|
-
function recordSandboxWarmingTimeout(
|
|
310
|
+
function recordSandboxWarmingTimeout(
|
|
311
|
+
metrics: RuntimeMetricsHooks | undefined,
|
|
312
|
+
error: unknown,
|
|
313
|
+
): void {
|
|
179
314
|
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
180
315
|
return;
|
|
181
316
|
}
|
|
@@ -186,25 +321,42 @@ function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, e
|
|
|
186
321
|
}
|
|
187
322
|
}
|
|
188
323
|
|
|
189
|
-
function
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
324
|
+
function workspaceArchiveFieldsFromEnvelope(
|
|
325
|
+
envelope: Record<string, unknown> | null | undefined,
|
|
326
|
+
): Record<string, string> | null {
|
|
327
|
+
const sessionState =
|
|
328
|
+
envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
329
|
+
? (envelope.sessionState as Record<string, unknown>)
|
|
330
|
+
: null;
|
|
193
331
|
const archive = sessionState?.workspaceArchive;
|
|
194
|
-
|
|
332
|
+
if (typeof archive !== "string" || archive.length === 0) {
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
const previous = sessionState?.workspaceArchivePrev;
|
|
336
|
+
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
337
|
+
return {
|
|
338
|
+
workspaceArchive: archive,
|
|
339
|
+
...(typeof previous === "string" && previous.length > 0
|
|
340
|
+
? { workspaceArchivePrev: previous }
|
|
341
|
+
: {}),
|
|
342
|
+
...(typeof capturedAt === "string" && capturedAt.length > 0
|
|
343
|
+
? { workspaceArchiveAt: capturedAt }
|
|
344
|
+
: {}),
|
|
345
|
+
};
|
|
195
346
|
}
|
|
196
347
|
|
|
197
|
-
function
|
|
348
|
+
function preserveWorkspaceArchivesOnResumeState(
|
|
198
349
|
resumeState: Record<string, unknown> | null,
|
|
199
350
|
archiveSource: Record<string, unknown> | null,
|
|
200
351
|
): Record<string, unknown> | null {
|
|
201
|
-
const
|
|
202
|
-
if (!
|
|
352
|
+
const archiveFields = workspaceArchiveFieldsFromEnvelope(archiveSource);
|
|
353
|
+
if (!archiveFields) {
|
|
203
354
|
return resumeState;
|
|
204
355
|
}
|
|
205
|
-
const existingSessionState =
|
|
206
|
-
|
|
207
|
-
|
|
356
|
+
const existingSessionState =
|
|
357
|
+
resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
|
|
358
|
+
? (resumeState.sessionState as Record<string, unknown>)
|
|
359
|
+
: {};
|
|
208
360
|
return {
|
|
209
361
|
...(resumeState ?? {}),
|
|
210
362
|
...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
|
|
@@ -212,24 +364,134 @@ function preserveWorkspaceArchiveOnInterimResumeState(
|
|
|
212
364
|
: {}),
|
|
213
365
|
sessionState: {
|
|
214
366
|
...existingSessionState,
|
|
215
|
-
|
|
367
|
+
...archiveFields,
|
|
216
368
|
},
|
|
217
369
|
};
|
|
218
370
|
}
|
|
219
371
|
|
|
372
|
+
/**
|
|
373
|
+
* MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
|
|
374
|
+
* drain-persist only protects boxes the reaper itself kills; anything else —
|
|
375
|
+
* Modal's hard creation-time timeout catching a session busy past it, provider
|
|
376
|
+
* OOM/infra death — loses everything since the last clean drain (staging
|
|
377
|
+
* session e644e8a8, 2026-07-06: mid-turn box termination cost an unpushed
|
|
378
|
+
* branch + 2 commits). While a turn HOLDS the live box, this folds a fresh
|
|
379
|
+
* snapshot onto the lease through persistWarmSnapshot (the warm sibling of the
|
|
380
|
+
* drain seam: epoch-fenced CAS + atomic throttle re-check) and GCs the
|
|
381
|
+
* superseded snapshot image, bounding worst-case loss of ANY unclean box death
|
|
382
|
+
* to sandboxSnapshotIntervalMs.
|
|
383
|
+
*
|
|
384
|
+
* Never throws and never blocks turn progress semantics: every failure path
|
|
385
|
+
* returns false (the snapshot is protection, not a turn dependency). No-ops
|
|
386
|
+
* when the interval is 0, the backend has no persistWorkspace (selfhosted =
|
|
387
|
+
* the user's machine IS the persistence), or a snapshot newer than the
|
|
388
|
+
* interval already rides the lease.
|
|
389
|
+
*/
|
|
390
|
+
export async function maybePersistWarmWorkspaceSnapshot(
|
|
391
|
+
services: SandboxResumeServices,
|
|
392
|
+
ids: { accountId: string; workspaceId: string; sandboxGroupId: string },
|
|
393
|
+
session: unknown,
|
|
394
|
+
leaseEpoch: number,
|
|
395
|
+
): Promise<boolean> {
|
|
396
|
+
const { db, settings } = services;
|
|
397
|
+
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
398
|
+
if (intervalMs <= 0) {
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
const persistable = session as {
|
|
402
|
+
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
403
|
+
};
|
|
404
|
+
if (typeof persistable.persistWorkspace !== "function") {
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
try {
|
|
408
|
+
// Cheap throttle pre-check before the (potentially slow) capture;
|
|
409
|
+
// persistWarmSnapshot re-checks atomically under the row lock, so this is
|
|
410
|
+
// purely a cost optimization, not the correctness guard.
|
|
411
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
412
|
+
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
|
|
413
|
+
return false;
|
|
414
|
+
}
|
|
415
|
+
const sessionState =
|
|
416
|
+
lease.resumeState && typeof lease.resumeState === "object"
|
|
417
|
+
? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
|
|
418
|
+
: undefined;
|
|
419
|
+
const priorAtRaw =
|
|
420
|
+
sessionState && typeof sessionState === "object"
|
|
421
|
+
? sessionState.workspaceArchiveAt
|
|
422
|
+
: undefined;
|
|
423
|
+
const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
|
|
424
|
+
if (Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
428
|
+
// Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
|
|
429
|
+
// by capture-initiation, not land time, so a slower heartbeat capture that
|
|
430
|
+
// started earlier can never overwrite a fresher turn-end capture that landed
|
|
431
|
+
// first (the bounded-wait race Bugbot flagged).
|
|
432
|
+
const capturedAtMs = Date.now();
|
|
433
|
+
// On timeout the race settles on `undefined` while persistWorkspace() keeps
|
|
434
|
+
// running orphaned. Swallow its LATE settlement: an un-awaited rejection
|
|
435
|
+
// after the race resolved would surface as an unhandledRejection and can
|
|
436
|
+
// take down the whole worker process — which would defeat the very hang-
|
|
437
|
+
// guard this timeout exists to provide.
|
|
438
|
+
const capture = persistable.persistWorkspace();
|
|
439
|
+
capture.catch(() => undefined);
|
|
440
|
+
const bytes = await Promise.race([
|
|
441
|
+
capture,
|
|
442
|
+
new Promise<undefined>((resolve) => {
|
|
443
|
+
timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
|
|
444
|
+
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
445
|
+
timeout.unref();
|
|
446
|
+
}
|
|
447
|
+
}),
|
|
448
|
+
]).finally(() => {
|
|
449
|
+
if (timeout) {
|
|
450
|
+
clearTimeout(timeout);
|
|
451
|
+
}
|
|
452
|
+
});
|
|
453
|
+
if (!bytes || bytes.length === 0) {
|
|
454
|
+
return false;
|
|
455
|
+
}
|
|
456
|
+
const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
|
|
457
|
+
accountId: ids.accountId,
|
|
458
|
+
workspaceId: ids.workspaceId,
|
|
459
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
460
|
+
expectedEpoch: leaseEpoch,
|
|
461
|
+
workspaceArchive: Buffer.from(bytes).toString("base64"),
|
|
462
|
+
minIntervalMs: intervalMs,
|
|
463
|
+
capturedAtMs,
|
|
464
|
+
});
|
|
465
|
+
if (!wrote) {
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
// Warm snapshots retain a 2-deep restore window. Only the two-ago archive
|
|
469
|
+
// returned by persistWarmSnapshot is GC-eligible.
|
|
470
|
+
await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
|
|
471
|
+
return true;
|
|
472
|
+
} catch (error) {
|
|
473
|
+
// Protection, not a dependency: a failed snapshot must never fail (or slow
|
|
474
|
+
// down retrying) the turn. The next heartbeat/turn-end tick retries.
|
|
475
|
+
console.error("mid-session workspace snapshot failed (turn unaffected)", error);
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
220
480
|
/**
|
|
221
481
|
* Resume the one box for a single turn (or any worker-side resume op). Returns
|
|
222
482
|
* the live non-owned session + the fence epoch + a release fn. The CALLER owns
|
|
223
483
|
* the lifecycle: inject non-owned, run, then `await release()` and drop the
|
|
224
484
|
* handle in `finally`.
|
|
225
485
|
*
|
|
226
|
-
* holderId is the unique-
|
|
486
|
+
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
487
|
+
* Temporal activity id, because activity ids are only workflow-local and
|
|
488
|
+
* collide when sibling sessions share one sandbox group.
|
|
227
489
|
*/
|
|
228
490
|
export async function resumeBoxForTurn(
|
|
229
491
|
services: SandboxResumeServices,
|
|
230
492
|
ids: ResumeBoxIds,
|
|
231
|
-
kind:
|
|
232
|
-
holderId:
|
|
493
|
+
kind: "turn",
|
|
494
|
+
holderId: TurnSandboxLeaseHolderId,
|
|
233
495
|
): Promise<ResumedTurnSandbox> {
|
|
234
496
|
const { db, settings } = services;
|
|
235
497
|
const os = ids.os ?? "linux";
|
|
@@ -238,11 +500,15 @@ export async function resumeBoxForTurn(
|
|
|
238
500
|
// The release closure is created eagerly so the caller can always release in
|
|
239
501
|
// finally, even if establish/commit throws after the holder was registered.
|
|
240
502
|
let released = false;
|
|
503
|
+
let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
|
|
241
504
|
const release = async (): Promise<void> => {
|
|
242
505
|
if (released) {
|
|
243
506
|
return;
|
|
244
507
|
}
|
|
245
508
|
released = true;
|
|
509
|
+
if (holderLivenessTimer) {
|
|
510
|
+
clearInterval(holderLivenessTimer);
|
|
511
|
+
}
|
|
246
512
|
await releaseLeaseHolder(db, {
|
|
247
513
|
accountId: ids.accountId,
|
|
248
514
|
workspaceId: ids.workspaceId,
|
|
@@ -267,9 +533,37 @@ export async function resumeBoxForTurn(
|
|
|
267
533
|
// propagates to the turn activity (an actionable error); a solo image change is handled
|
|
268
534
|
// by acquireLease recreating the box cold on the new image.
|
|
269
535
|
...(ids.image ? { image: ids.image } : {}),
|
|
536
|
+
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
537
|
+
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
538
|
+
// propagates to the turn activity (actionable); a solo rig change recreates cold.
|
|
539
|
+
...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
|
|
270
540
|
leaseTtlMs,
|
|
541
|
+
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
271
542
|
});
|
|
272
543
|
|
|
544
|
+
// HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
|
|
545
|
+
// registered until release. The dead-worker turn-holder reap judges liveness
|
|
546
|
+
// by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
|
|
547
|
+
// agent-turn) only starts AFTER this function returns — while waitForWarm and
|
|
548
|
+
// establish/cold-restore can legitimately run for many minutes in here,
|
|
549
|
+
// COMPOUNDING past any fixed reap horizon. With this
|
|
550
|
+
// loop no live holder is ever silent for more than one tick, so the reap
|
|
551
|
+
// horizon is pure defense-in-depth, not a tuned guess about path lengths.
|
|
552
|
+
// Epoch-free by design (touch only refreshes our own row's timestamp);
|
|
553
|
+
// best-effort (a transient DB error must never fail the resume).
|
|
554
|
+
holderLivenessTimer = setInterval(() => {
|
|
555
|
+
void touchLeaseHolder(db, {
|
|
556
|
+
accountId: ids.accountId,
|
|
557
|
+
workspaceId: ids.workspaceId,
|
|
558
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
559
|
+
kind,
|
|
560
|
+
holderId,
|
|
561
|
+
}).catch(() => undefined);
|
|
562
|
+
}, 10_000);
|
|
563
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
564
|
+
holderLivenessTimer.unref();
|
|
565
|
+
}
|
|
566
|
+
|
|
273
567
|
// FENCED: a newer epoch exists (a later turn re-established the box). Back off;
|
|
274
568
|
// NEVER create(). Release our (just-registered) holder so we don't pin a stale
|
|
275
569
|
// lease, then surface the supersession.
|
|
@@ -279,17 +573,10 @@ export async function resumeBoxForTurn(
|
|
|
279
573
|
}
|
|
280
574
|
|
|
281
575
|
// SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
|
|
282
|
-
//
|
|
576
|
+
// then commit warm (lease_epoch++). Optional desktop setup is lazy.
|
|
283
577
|
if (acquired.role === "spawner") {
|
|
284
578
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
285
579
|
let createdEstablished: EstablishedSandboxSession | null = null;
|
|
286
|
-
if (ids.environment && ids.backend === "modal") {
|
|
287
|
-
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
288
|
-
leaseId: acquired.lease.id,
|
|
289
|
-
workspaceId: ids.workspaceId,
|
|
290
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
291
|
-
}));
|
|
292
|
-
}
|
|
293
580
|
try {
|
|
294
581
|
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
295
582
|
// Prefer the COLD lease's preserved resume_state when it carries a persisted
|
|
@@ -304,12 +591,13 @@ export async function resumeBoxForTurn(
|
|
|
304
591
|
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
305
592
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
306
593
|
sessionId: ids.sessionId,
|
|
594
|
+
recovery: "create-or-restore",
|
|
307
595
|
backendOverride: ids.backend as never,
|
|
308
596
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
309
597
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
310
598
|
onSandboxCreated: async (created) => {
|
|
311
599
|
createdEstablished = created;
|
|
312
|
-
const resumeEnvelope =
|
|
600
|
+
const resumeEnvelope = preserveWorkspaceArchivesOnResumeState(
|
|
313
601
|
(await serializeEstablishedSandboxEnvelope(created)) ?? null,
|
|
314
602
|
spawnEnvelope,
|
|
315
603
|
);
|
|
@@ -322,6 +610,9 @@ export async function resumeBoxForTurn(
|
|
|
322
610
|
resumeBackendId: created.backendId,
|
|
323
611
|
resumeState: resumeEnvelope,
|
|
324
612
|
leaseTtlMs,
|
|
613
|
+
// Keep the warming budget after create(): manifest setup and
|
|
614
|
+
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
615
|
+
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
325
616
|
});
|
|
326
617
|
if (!recorded.recorded) {
|
|
327
618
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
@@ -336,8 +627,11 @@ export async function resumeBoxForTurn(
|
|
|
336
627
|
},
|
|
337
628
|
});
|
|
338
629
|
createdEstablished = established;
|
|
339
|
-
|
|
340
|
-
|
|
630
|
+
// A sandbox handle is not sufficient evidence that Modal's command router
|
|
631
|
+
// is live. Do not publish a warm lease until one bounded no-op exec works.
|
|
632
|
+
// On timeout the catch below terminates the box and rolls warming -> cold,
|
|
633
|
+
// so the next turn cold-creates instead of hanging forever on first use.
|
|
634
|
+
await waitForSandboxExecReadiness(established);
|
|
341
635
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
342
636
|
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
343
637
|
// viewer.ts). Without this the turn committed the ORIGINAL session manifest
|
|
@@ -345,14 +639,29 @@ export async function resumeBoxForTurn(
|
|
|
345
639
|
// terminal, the desktop viewer, the reaper) cold-restored a FRESH rival box
|
|
346
640
|
// and never saw the turn's live box. Fall back to the session envelope only
|
|
347
641
|
// when the client cannot serialize live state.
|
|
348
|
-
const
|
|
642
|
+
const serializedResumeEnvelope =
|
|
643
|
+
(await serializeEstablishedSandboxEnvelope(established)) ?? envelope;
|
|
644
|
+
// A successful cold hydrate has already proved this archive usable and the
|
|
645
|
+
// replacement box now contains its files. Keep the current + fallback
|
|
646
|
+
// archive pointers on the committed live envelope until a later warm
|
|
647
|
+
// snapshot replaces them. Without this merge, serialization publishes only
|
|
648
|
+
// the new provider id; a second provider loss before the snapshot cadence
|
|
649
|
+
// fires would retire the lease with no archive and recreate an empty box.
|
|
650
|
+
// A failed hydrate falls back to a clean box with origin="created", so its
|
|
651
|
+
// unusable archive is deliberately cleared by the unmerged serialized state.
|
|
652
|
+
const resumeEnvelope =
|
|
653
|
+
established.origin === "restored"
|
|
654
|
+
? preserveWorkspaceArchivesOnResumeState(serializedResumeEnvelope, spawnEnvelope)
|
|
655
|
+
: serializedResumeEnvelope;
|
|
349
656
|
const committed = await commitWarmingToWarm(db, {
|
|
350
657
|
accountId: ids.accountId,
|
|
351
658
|
workspaceId: ids.workspaceId,
|
|
352
659
|
sandboxGroupId: ids.sandboxGroupId,
|
|
353
660
|
expectedEpoch,
|
|
354
661
|
instanceId: established.instanceId,
|
|
355
|
-
|
|
662
|
+
// Viewer attach resolves and records the tunnel URL lazily. A headless
|
|
663
|
+
// turn must not touch the optional desktop data plane.
|
|
664
|
+
dataPlaneUrl: null,
|
|
356
665
|
resumeBackendId: established.backendId,
|
|
357
666
|
resumeState: resumeEnvelope,
|
|
358
667
|
leaseTtlMs,
|
|
@@ -388,7 +697,11 @@ export async function resumeBoxForTurn(
|
|
|
388
697
|
});
|
|
389
698
|
}
|
|
390
699
|
await release();
|
|
391
|
-
const warmingError = asSandboxWarmingError(
|
|
700
|
+
const warmingError = asSandboxWarmingError(
|
|
701
|
+
error,
|
|
702
|
+
ids.backend,
|
|
703
|
+
settings.sandboxWarmingTimeoutMs,
|
|
704
|
+
);
|
|
392
705
|
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
393
706
|
throw warmingError;
|
|
394
707
|
}
|
|
@@ -397,8 +710,8 @@ export async function resumeBoxForTurn(
|
|
|
397
710
|
// ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
|
|
398
711
|
// BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
|
|
399
712
|
// first wait for the spawner to commit warm, bounded by the explicit warming
|
|
400
|
-
// budget. A cold reset means the spawner died;
|
|
401
|
-
// trying to enter the spawner create path from the attached branch.
|
|
713
|
+
// budget. A cold reset means the spawner died; recover the same turn instead
|
|
714
|
+
// of trying to enter the spawner create path from the attached branch.
|
|
402
715
|
try {
|
|
403
716
|
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
404
717
|
if (acquired.lease.liveness === "warming") {
|
|
@@ -410,28 +723,62 @@ export async function resumeBoxForTurn(
|
|
|
410
723
|
// manifest into a rival. Fall back to the session envelope only when the
|
|
411
724
|
// lease carries no resume_state (matches channel-a.ts's attached branch).
|
|
412
725
|
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
413
|
-
|
|
414
|
-
live
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
726
|
+
if (
|
|
727
|
+
!live ||
|
|
728
|
+
live.liveness !== "warm" ||
|
|
729
|
+
live.leaseEpoch !== leaseEpoch ||
|
|
730
|
+
live.instanceId === null
|
|
731
|
+
) {
|
|
732
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
|
|
733
|
+
}
|
|
734
|
+
let established: EstablishedSandboxSession;
|
|
735
|
+
try {
|
|
736
|
+
established = await establishSandboxSessionFromEnvelope(settings, live.resumeState, {
|
|
737
|
+
sessionId: ids.sessionId,
|
|
738
|
+
recovery: "resume-only",
|
|
739
|
+
backendOverride: ids.backend as never,
|
|
740
|
+
...(ids.environment ? { environment: ids.environment } : {}),
|
|
741
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
742
|
+
});
|
|
743
|
+
} catch (error) {
|
|
744
|
+
if (
|
|
745
|
+
!(error instanceof SandboxResumeStateUnavailableError) &&
|
|
746
|
+
!isProviderSandboxNotFoundError(ids.backend, error)
|
|
747
|
+
) {
|
|
748
|
+
throw error;
|
|
749
|
+
}
|
|
750
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
751
|
+
accountId: ids.accountId,
|
|
752
|
+
workspaceId: ids.workspaceId,
|
|
753
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
754
|
+
expectedEpoch: leaseEpoch,
|
|
755
|
+
expectedInstanceId: live.instanceId,
|
|
756
|
+
});
|
|
757
|
+
if (marked.status === "marked") {
|
|
758
|
+
await services.onSandboxLost?.({
|
|
759
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
760
|
+
instanceId: live.instanceId,
|
|
761
|
+
leaseEpoch: marked.lease.leaseEpoch,
|
|
762
|
+
});
|
|
763
|
+
throw new SandboxLeaseInstanceLostError(
|
|
764
|
+
ids.sandboxGroupId,
|
|
765
|
+
marked.lease.leaseEpoch,
|
|
766
|
+
live.instanceId,
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
throw new SandboxLeaseSupersededError(
|
|
770
|
+
ids.sandboxGroupId,
|
|
771
|
+
marked.lease?.leaseEpoch ?? leaseEpoch,
|
|
772
|
+
);
|
|
773
|
+
}
|
|
431
774
|
return { established, leaseEpoch, release };
|
|
432
775
|
} catch (error) {
|
|
433
776
|
await release();
|
|
434
|
-
const warmingError = asSandboxWarmingError(
|
|
777
|
+
const warmingError = asSandboxWarmingError(
|
|
778
|
+
error,
|
|
779
|
+
ids.backend,
|
|
780
|
+
settings.sandboxWarmingTimeoutMs,
|
|
781
|
+
);
|
|
435
782
|
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
436
783
|
throw warmingError;
|
|
437
784
|
}
|
|
@@ -451,13 +798,20 @@ export async function resumeBoxForTurn(
|
|
|
451
798
|
* epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
|
|
452
799
|
* identical to the cloud path.
|
|
453
800
|
*
|
|
454
|
-
* holderId is the unique-
|
|
801
|
+
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
802
|
+
* Temporal activity id, because activity ids are only workflow-local and
|
|
803
|
+
* collide when sibling sessions share one sandbox group.
|
|
455
804
|
*/
|
|
456
805
|
export async function acquireSelfhostedLeaseForTurn(
|
|
457
806
|
services: SandboxResumeServices,
|
|
458
|
-
ids: {
|
|
459
|
-
|
|
460
|
-
|
|
807
|
+
ids: {
|
|
808
|
+
accountId: string;
|
|
809
|
+
workspaceId: string;
|
|
810
|
+
sandboxGroupId: string;
|
|
811
|
+
sessionId: string;
|
|
812
|
+
},
|
|
813
|
+
kind: "turn",
|
|
814
|
+
holderId: TurnSandboxLeaseHolderId,
|
|
461
815
|
): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
|
|
462
816
|
const { db, settings } = services;
|
|
463
817
|
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
@@ -488,6 +842,7 @@ export async function acquireSelfhostedLeaseForTurn(
|
|
|
488
842
|
backend: "selfhosted",
|
|
489
843
|
os: "linux",
|
|
490
844
|
leaseTtlMs,
|
|
845
|
+
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
491
846
|
});
|
|
492
847
|
|
|
493
848
|
// FENCED: a newer epoch re-established the group concurrently. Release our
|
|
@@ -569,128 +924,3 @@ async function waitForWarm(
|
|
|
569
924
|
}
|
|
570
925
|
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
571
926
|
}
|
|
572
|
-
|
|
573
|
-
// ============================================================================
|
|
574
|
-
// Channel-B display-stack launch (P4.1 — ensureDisplayStack is now real).
|
|
575
|
-
//
|
|
576
|
-
// Idempotent + callable by ANY worker on the resumed handle (and by the API on a
|
|
577
|
-
// viewer op). When the desktop tier is OFF (sandboxDesktopEnabled=false) or the
|
|
578
|
-
// backend is headless-only, this is a NO-OP — the HEADLESS ROLLOVER branch (I5),
|
|
579
|
-
// so the headless turn path is byte-for-byte unchanged. When the tier is ON for
|
|
580
|
-
// a desktop-capable backend, it execs the canonical opengeni-desktop-up under an
|
|
581
|
-
// in-box flock (re-establishes after a rollover; safe to call N times).
|
|
582
|
-
// `exposeStreamPort` real body remains a P4.2 concern.
|
|
583
|
-
// ============================================================================
|
|
584
|
-
|
|
585
|
-
/**
|
|
586
|
-
* Ensure the desktop display stack (Xvfb -> XFCE -> x11vnc ->
|
|
587
|
-
* websockify:6080 -> noVNC) is up on the live box. Idempotent: the lock-free
|
|
588
|
-
* pre-check + the in-box flock + the up-script's per-stage PID guards make a
|
|
589
|
-
* second call a cheap no-op. NO-OP when the desktop tier is disabled or the
|
|
590
|
-
* backend cannot serve a desktop (degradation is a value: a headless-only session
|
|
591
|
-
* simply skips the stack). Delegates to the agent-loop-free leaf
|
|
592
|
-
* (@opengeni/runtime/sandbox display-stack).
|
|
593
|
-
*
|
|
594
|
-
* BEST-EFFORT — NEVER fails the turn. The display stack powers the OPTIONAL
|
|
595
|
-
* Channel-B desktop / computer-use surface; it is NOT load-bearing for the
|
|
596
|
-
* agent's work. A `DisplayStackError` (a real stage failure, OR — the regression
|
|
597
|
-
* this guards — a timeout-derived exit -1 when a viewer attach already holds /
|
|
598
|
-
* contends the up-script's flock and the turn's ensure waits ~45s and times out)
|
|
599
|
-
* is CAUGHT + logged and swallowed, so a slow/contended/failed stack degrades to
|
|
600
|
-
* Channel-A-only rather than killing the turn. The fast lock-free pre-check
|
|
601
|
-
* upstream means the already-up case resolves in milliseconds and never reaches
|
|
602
|
-
* this catch in the first place. `DisplayStackUnsupportedError` (a box that can't
|
|
603
|
-
* run commands at all) is likewise swallowed.
|
|
604
|
-
*/
|
|
605
|
-
export async function ensureDisplayStack(
|
|
606
|
-
settings: Settings,
|
|
607
|
-
established: EstablishedSandboxSession,
|
|
608
|
-
): Promise<void> {
|
|
609
|
-
// Headless rollover branch (I5): the tier is off OR the backend is
|
|
610
|
-
// headless-only -> no display stack. Behavior-preserving for the headless path.
|
|
611
|
-
// This is ALSO the "is the desktop relevant to this turn?" gate: we only touch
|
|
612
|
-
// the box when the desktop tier is enabled for a desktop-capable backend.
|
|
613
|
-
if (!settings.sandboxDesktopEnabled) {
|
|
614
|
-
return;
|
|
615
|
-
}
|
|
616
|
-
if (!desktopCapableBackend(established.backendId)) {
|
|
617
|
-
return;
|
|
618
|
-
}
|
|
619
|
-
try {
|
|
620
|
-
await ensureDisplayStackOnBox(established.session);
|
|
621
|
-
} catch (error) {
|
|
622
|
-
// The desktop is a value-add, not load-bearing for the agent's work, so NO
|
|
623
|
-
// display-stack failure may fail the turn:
|
|
624
|
-
// - DisplayStackUnsupportedError: the box genuinely can't run commands ->
|
|
625
|
-
// Channel-A-only.
|
|
626
|
-
// - DisplayStackError: a stage failure OR a contended-lock timeout (exit -1)
|
|
627
|
-
// after a viewer attach already brought the stack up / is mid-launch. The
|
|
628
|
-
// pre-check makes the already-up case fast; if we still time out or a stage
|
|
629
|
-
// really failed, degrade — don't die.
|
|
630
|
-
if (error instanceof DisplayStackUnsupportedError || error instanceof DisplayStackError) {
|
|
631
|
-
console.warn(
|
|
632
|
-
`[sandbox-resume] ensureDisplayStack degraded to Channel-A-only (turn continues): ${
|
|
633
|
-
error instanceof Error ? error.message : String(error)
|
|
634
|
-
}`,
|
|
635
|
-
);
|
|
636
|
-
return;
|
|
637
|
-
}
|
|
638
|
-
// An unexpected non-display error (e.g. the session blew up entirely) still
|
|
639
|
-
// propagates — that is not a desktop-surface degradation.
|
|
640
|
-
throw error;
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
// The structural slice of a provider session we need to resolve the tunnel.
|
|
645
|
-
type PortResolvableSession = {
|
|
646
|
-
resolveExposedPort?: (port: number) => Promise<ExposedPortEndpoint>;
|
|
647
|
-
};
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Resolve the desktop tunnel URL (resolveExposedPort(6080)) and assemble the
|
|
651
|
-
* direct-to-provider WS URL, recorded on the lease as `data_plane_url` at
|
|
652
|
-
* commit. The per-viewer scoped token is minted at viewer-attach time (the
|
|
653
|
-
* handshake), NOT here — the spawner records only the box-scoped tunnel URL so
|
|
654
|
-
* the lease carries a fresh value across a rollover. P4.2 productionizes the
|
|
655
|
-
* P4.1 stub.
|
|
656
|
-
*
|
|
657
|
-
* Returns null (degradation is a value, NEVER a throw):
|
|
658
|
-
* - the desktop tier is off (sandboxDesktopEnabled=false), or
|
|
659
|
-
* - the backend is headless-only (no desktop), or
|
|
660
|
-
* - the provider session cannot resolve the port (no resolveExposedPort), or
|
|
661
|
-
* - the tunnel lookup transiently failed.
|
|
662
|
-
* In every null case the lease's data_plane_url stays null and the next
|
|
663
|
-
* API-direct viewer op re-mints it; the turn never fails on a desktop hiccup.
|
|
664
|
-
*/
|
|
665
|
-
export async function exposeStreamPort(
|
|
666
|
-
settings: Settings,
|
|
667
|
-
established: EstablishedSandboxSession,
|
|
668
|
-
): Promise<{ url: string; expiresAt: Date } | null> {
|
|
669
|
-
// Headless rollover branch (I5): tier !== desktop -> no stream port to expose.
|
|
670
|
-
if (!settings.sandboxDesktopEnabled) {
|
|
671
|
-
return null;
|
|
672
|
-
}
|
|
673
|
-
if (!desktopCapableBackend(established.backendId)) {
|
|
674
|
-
return null;
|
|
675
|
-
}
|
|
676
|
-
const session = established.session as PortResolvableSession;
|
|
677
|
-
if (typeof session?.resolveExposedPort !== "function") {
|
|
678
|
-
return null;
|
|
679
|
-
}
|
|
680
|
-
try {
|
|
681
|
-
const endpoint = await session.resolveExposedPort(DESKTOP_STREAM_PORT);
|
|
682
|
-
const url = buildStreamUrl(endpoint);
|
|
683
|
-
// The provider tunnel URL is box-lifetime-valid (Modal raw TLS) or
|
|
684
|
-
// provider-TTL-signed (Daytona/Blaxel); the per-viewer token's TTL bounds the
|
|
685
|
-
// viewer's freshness. We record only the URL on the lease; expiresAt here is a
|
|
686
|
-
// soft hint (the box's idle horizon), not a hard token expiry.
|
|
687
|
-
return { url, expiresAt: new Date(Date.now() + settings.sandboxLeaseTtlMs) };
|
|
688
|
-
} catch (error) {
|
|
689
|
-
// Degradation is a value: a transient resolve failure leaves data_plane_url
|
|
690
|
-
// null and the next viewer op re-mints. Never fail the turn on a desktop op.
|
|
691
|
-
if (error instanceof StreamPortUnavailableError) {
|
|
692
|
-
return null;
|
|
693
|
-
}
|
|
694
|
-
return null;
|
|
695
|
-
}
|
|
696
|
-
}
|