@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
// apps/worker/src/sandbox-resume.ts — the stateless per-turn resume-by-id path.
|
|
2
|
+
//
|
|
3
|
+
// This is the turn-side half of the P1.2 ownership inversion. There is NO class,
|
|
4
|
+
// NO timers, NO per-session owner, NO Map<id, owner> — every turn is a
|
|
5
|
+
// self-contained critical section run by ANY pool worker:
|
|
6
|
+
//
|
|
7
|
+
// 1. acquireLease (group-keyed) under the DB FOR UPDATE + cold->warming CAS —
|
|
8
|
+
// the SOLE double-spawn guard (P1.1).
|
|
9
|
+
// 2. establishSandboxSessionFromEnvelope — resume the one box BY ID (warm
|
|
10
|
+
// reattach, R4-safe) or cold-restore from snapshot on a provider NotFound.
|
|
11
|
+
// 3. (spawner) ensureDisplayStack + exposeStreamPort, then commitWarmingToWarm
|
|
12
|
+
// (the lease_epoch++ fence + folds the resume envelope onto the lease). The
|
|
13
|
+
// ATTACHED/REARMED path RE-ensures the display stack too (idempotent) so a
|
|
14
|
+
// computer-use turn always finds a live :0 even on a box first warmed by a
|
|
15
|
+
// non-desktop op or after a snapshot rollover dropped the X stack.
|
|
16
|
+
// 4. the caller injects {client, session, sessionState} NON-OWNED into the run
|
|
17
|
+
// (the SDK never reaps it — the keystone), runs, then in `finally` calls the
|
|
18
|
+
// returned `release()` and drops the in-memory handle. NEVER provider-delete
|
|
19
|
+
// — the box rides the provider idle-timeout; the reaper (P1.3) stop()s it at
|
|
20
|
+
// refcount 0.
|
|
21
|
+
//
|
|
22
|
+
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
23
|
+
|
|
24
|
+
import type { Settings } from "@opengeni/config";
|
|
25
|
+
import {
|
|
26
|
+
acquireLease,
|
|
27
|
+
commitWarmingToWarm,
|
|
28
|
+
failWarmingToCold,
|
|
29
|
+
getSandboxSessionEnvelope,
|
|
30
|
+
readLease,
|
|
31
|
+
releaseLeaseHolder,
|
|
32
|
+
SandboxLeaseSupersededError,
|
|
33
|
+
type Database,
|
|
34
|
+
type LeaseHolderKind,
|
|
35
|
+
} from "@opengeni/db";
|
|
36
|
+
import {
|
|
37
|
+
establishSandboxSessionFromEnvelope,
|
|
38
|
+
ensureDisplayStack as ensureDisplayStackOnBox,
|
|
39
|
+
desktopCapableBackend,
|
|
40
|
+
serializeEstablishedSandboxEnvelope,
|
|
41
|
+
DisplayStackError,
|
|
42
|
+
DisplayStackUnsupportedError,
|
|
43
|
+
buildStreamUrl,
|
|
44
|
+
StreamPortUnavailableError,
|
|
45
|
+
type EstablishedSandboxSession,
|
|
46
|
+
type ExposedPortEndpoint,
|
|
47
|
+
} from "@opengeni/runtime";
|
|
48
|
+
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
49
|
+
|
|
50
|
+
export { DESKTOP_STREAM_PORT };
|
|
51
|
+
|
|
52
|
+
// Re-exported for callers that just want the ack-kind union.
|
|
53
|
+
export type ResumeHolderKind = LeaseHolderKind;
|
|
54
|
+
|
|
55
|
+
/** The minimal services surface resumeBoxForTurn needs. A subset of
|
|
56
|
+
* ActivityServices so a test (and the API later) can pass a lean bag. */
|
|
57
|
+
export type SandboxResumeServices = {
|
|
58
|
+
db: Database;
|
|
59
|
+
settings: Settings;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
export type ResumeBoxIds = {
|
|
63
|
+
accountId: string;
|
|
64
|
+
workspaceId: string;
|
|
65
|
+
sandboxGroupId: string;
|
|
66
|
+
/** The attributing session within the group (holders carry session_id for
|
|
67
|
+
* disclosure/attribution). For a singleton group this == sandboxGroupId. */
|
|
68
|
+
sessionId: string;
|
|
69
|
+
/** The backend the box runs on (sessions.sandbox_backend). */
|
|
70
|
+
backend: string;
|
|
71
|
+
/** The OS axis (sessions.sandbox_os); default 'linux'. */
|
|
72
|
+
os?: string;
|
|
73
|
+
/**
|
|
74
|
+
* The FULL environment the agent will declare for this run (the SAME object
|
|
75
|
+
* passed to runtime.buildAgent's `sandboxEnvironment`). The box's manifest is
|
|
76
|
+
* created with this environment so that when the SDK applies the agent's
|
|
77
|
+
* manifest to this NON-OWNED provided session, the environments match exactly
|
|
78
|
+
* and `validateNoEnvironmentDelta` finds an empty delta (otherwise it throws
|
|
79
|
+
* "Live sandbox sessions cannot change manifest environment variables" and the
|
|
80
|
+
* turn dies). Omitted → the leaf falls back to collectSandboxEnvironment(settings)
|
|
81
|
+
* (the legacy default; only the resume/spawn-without-an-agent callers rely on it).
|
|
82
|
+
*/
|
|
83
|
+
environment?: Record<string, string>;
|
|
84
|
+
/**
|
|
85
|
+
* IMAGE IS SHARED STATE (B3): the container image this run resolves (Modal image ref
|
|
86
|
+
* / docker image). Threaded to acquireLease, which stamps it on the cold-create and
|
|
87
|
+
* conflicts on a live box already running a DIFFERENT image (solo holder recreates;
|
|
88
|
+
* N-holders throw SandboxImageConflictError). Omitted -> image is not enforced (the
|
|
89
|
+
* selfhosted path never passes it; a legacy/null-image box never conflicts).
|
|
90
|
+
*/
|
|
91
|
+
image?: string;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/** What resumeBoxForTurn returns: the live NON-OWNED session to inject, the
|
|
95
|
+
* fence token (lease_epoch) it was established under, and a release function
|
|
96
|
+
* the caller invokes in `finally` (idempotent delete-my-holder-row). */
|
|
97
|
+
export type ResumedTurnSandbox = {
|
|
98
|
+
/** The live, externally-owned session — inject {client, session, sessionState}
|
|
99
|
+
* NON-OWNED into runStream's `ownedSandbox`; the SDK never reaps it. */
|
|
100
|
+
established: EstablishedSandboxSession;
|
|
101
|
+
/** The lease_epoch this turn holds; the heartbeat/fence token. */
|
|
102
|
+
leaseEpoch: number;
|
|
103
|
+
/** Idempotent release: deletes this holder row and (if refcount hits 0 with no
|
|
104
|
+
* turn holders) CASes warm->draining. NEVER stops the box. Safe to call once. */
|
|
105
|
+
release: () => Promise<void>;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
// Bounded poll while a sibling spawner is mid cold-restore. The reaper resets a
|
|
109
|
+
// dead warming row after sandboxLeaseWarmingTtlMs; we poll up to that horizon.
|
|
110
|
+
const WARMING_POLL_INTERVAL_MS = 250;
|
|
111
|
+
|
|
112
|
+
async function sleep(ms: number): Promise<void> {
|
|
113
|
+
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Resume the one box for a single turn (or any worker-side resume op). Returns
|
|
118
|
+
* the live non-owned session + the fence epoch + a release fn. The CALLER owns
|
|
119
|
+
* the lifecycle: inject non-owned, run, then `await release()` and drop the
|
|
120
|
+
* handle in `finally`.
|
|
121
|
+
*
|
|
122
|
+
* holderId is the unique-per-execution id (the Temporal activityId for a turn).
|
|
123
|
+
*/
|
|
124
|
+
export async function resumeBoxForTurn(
|
|
125
|
+
services: SandboxResumeServices,
|
|
126
|
+
ids: ResumeBoxIds,
|
|
127
|
+
kind: LeaseHolderKind,
|
|
128
|
+
holderId: string,
|
|
129
|
+
): Promise<ResumedTurnSandbox> {
|
|
130
|
+
const { db, settings } = services;
|
|
131
|
+
const os = ids.os ?? "linux";
|
|
132
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
133
|
+
|
|
134
|
+
// The release closure is created eagerly so the caller can always release in
|
|
135
|
+
// finally, even if establish/commit throws after the holder was registered.
|
|
136
|
+
let released = false;
|
|
137
|
+
const release = async (): Promise<void> => {
|
|
138
|
+
if (released) {
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
released = true;
|
|
142
|
+
await releaseLeaseHolder(db, {
|
|
143
|
+
accountId: ids.accountId,
|
|
144
|
+
workspaceId: ids.workspaceId,
|
|
145
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
146
|
+
kind,
|
|
147
|
+
holderId,
|
|
148
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
149
|
+
});
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const acquired = await acquireLease(db, {
|
|
153
|
+
accountId: ids.accountId,
|
|
154
|
+
workspaceId: ids.workspaceId,
|
|
155
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
156
|
+
kind,
|
|
157
|
+
holderId,
|
|
158
|
+
subjectId: ids.sessionId,
|
|
159
|
+
backend: ids.backend,
|
|
160
|
+
os,
|
|
161
|
+
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
162
|
+
// conflicts on a live box already running a different image. A SandboxImageConflictError
|
|
163
|
+
// propagates to the turn activity (an actionable error); a solo image change is handled
|
|
164
|
+
// by acquireLease recreating the box cold on the new image.
|
|
165
|
+
...(ids.image ? { image: ids.image } : {}),
|
|
166
|
+
leaseTtlMs,
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
// FENCED: a newer epoch exists (a later turn re-established the box). Back off;
|
|
170
|
+
// NEVER create(). Release our (just-registered) holder so we don't pin a stale
|
|
171
|
+
// lease, then surface the supersession.
|
|
172
|
+
if (acquired.role === "fenced") {
|
|
173
|
+
await release();
|
|
174
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
|
|
178
|
+
// expose the stream port, then commit warm (lease_epoch++).
|
|
179
|
+
if (acquired.role === "spawner") {
|
|
180
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
181
|
+
try {
|
|
182
|
+
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
183
|
+
// Prefer the COLD lease's preserved resume_state when it carries a persisted
|
|
184
|
+
// /workspace snapshot (confirmDrainCold keeps a minimal archive-only envelope
|
|
185
|
+
// across draining->cold for exactly this re-warm). establishSandboxSessionFromEnvelope
|
|
186
|
+
// cold-creates a fresh box and replays the archive via hydrateWorkspace, so
|
|
187
|
+
// /workspace survives box churn (sandbox-file-persistence). No archive ->
|
|
188
|
+
// the bare session envelope (a never-warmed cold start). Mirrors channel-a.ts's
|
|
189
|
+
// spawner branch: the lease's resume_state is authoritative; the session
|
|
190
|
+
// `_sandbox` envelope is the per-session fallback. Without this a turn-first
|
|
191
|
+
// re-warm after a drain->cold would ignore the archive and start an EMPTY box.
|
|
192
|
+
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
193
|
+
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
194
|
+
sessionId: ids.sessionId,
|
|
195
|
+
backendOverride: ids.backend as never,
|
|
196
|
+
...(ids.environment ? { environment: ids.environment } : {}),
|
|
197
|
+
});
|
|
198
|
+
await ensureDisplayStack(settings, established);
|
|
199
|
+
const endpoint = await exposeStreamPort(settings, established);
|
|
200
|
+
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
201
|
+
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
202
|
+
// viewer.ts). Without this the turn committed the ORIGINAL session manifest
|
|
203
|
+
// as resume_state, so every LATER op off this lease (Channel-A fs/git/
|
|
204
|
+
// terminal, the desktop viewer, the reaper) cold-restored a FRESH rival box
|
|
205
|
+
// and never saw the turn's live box. Fall back to the session envelope only
|
|
206
|
+
// when the client cannot serialize live state.
|
|
207
|
+
const resumeEnvelope = (await serializeEstablishedSandboxEnvelope(established)) ?? envelope ?? null;
|
|
208
|
+
const committed = await commitWarmingToWarm(db, {
|
|
209
|
+
accountId: ids.accountId,
|
|
210
|
+
workspaceId: ids.workspaceId,
|
|
211
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
212
|
+
expectedEpoch,
|
|
213
|
+
instanceId: established.instanceId,
|
|
214
|
+
dataPlaneUrl: endpoint?.url ?? null,
|
|
215
|
+
resumeBackendId: established.backendId,
|
|
216
|
+
resumeState: resumeEnvelope,
|
|
217
|
+
leaseTtlMs,
|
|
218
|
+
});
|
|
219
|
+
if (!committed.committed || !committed.lease) {
|
|
220
|
+
// A reaper reset our warming row (we were too slow) or a sibling
|
|
221
|
+
// re-established and bumped the epoch. Drop the handle; release our
|
|
222
|
+
// holder; surface supersession. NEVER provider-delete the box.
|
|
223
|
+
await release();
|
|
224
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
225
|
+
}
|
|
226
|
+
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (error instanceof SandboxLeaseSupersededError) {
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
// Caught spawn failure: roll the warming row back to cold so a queued turn
|
|
232
|
+
// re-acquires and re-spawns. Holders are intentionally left for the
|
|
233
|
+
// re-acquire (failWarmingToCold keeps them); then release our own holder.
|
|
234
|
+
await failWarmingToCold(db, {
|
|
235
|
+
accountId: ids.accountId,
|
|
236
|
+
workspaceId: ids.workspaceId,
|
|
237
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
238
|
+
expectedEpoch,
|
|
239
|
+
});
|
|
240
|
+
await release();
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
|
|
246
|
+
// BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
|
|
247
|
+
// first wait for the spawner to commit warm (or for the row to flip cold so we
|
|
248
|
+
// can re-acquire as spawner).
|
|
249
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
250
|
+
if (acquired.lease.liveness === "warming") {
|
|
251
|
+
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
try {
|
|
255
|
+
// Prefer the lease's resume_state (the LIVE box the spawner committed) so we
|
|
256
|
+
// re-attach to the SAME box by id, not cold-restore the original session
|
|
257
|
+
// manifest into a rival. Fall back to the session envelope only when the
|
|
258
|
+
// lease carries no resume_state (matches channel-a.ts's attached branch).
|
|
259
|
+
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
260
|
+
const envelope =
|
|
261
|
+
live?.resumeState ?? (await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId));
|
|
262
|
+
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
263
|
+
sessionId: ids.sessionId,
|
|
264
|
+
backendOverride: ids.backend as never,
|
|
265
|
+
...(ids.environment ? { environment: ids.environment } : {}),
|
|
266
|
+
});
|
|
267
|
+
// Re-ensure the desktop display stack on the ATTACHED/REARMED path too — NOT
|
|
268
|
+
// just the spawner path. A turn attaching to a warm box whose :0 was never
|
|
269
|
+
// brought up (the box was first warmed by a Channel-A op, or a snapshot
|
|
270
|
+
// rollover dropped the X stack) would otherwise drive computer-use against a
|
|
271
|
+
// dead display: scrot yields an empty PNG, the SDK builds `image_url: ''`,
|
|
272
|
+
// and the model rejects the turn with "400 Invalid input[N].output.image_url".
|
|
273
|
+
// ensureDisplayStack is idempotent + flock-guarded (cheap no-op when up) and
|
|
274
|
+
// NO-OPs when the desktop tier is off or the backend is headless-only, so the
|
|
275
|
+
// headless turn path stays byte-for-byte unchanged.
|
|
276
|
+
await ensureDisplayStack(settings, established);
|
|
277
|
+
return { established, leaseEpoch, release };
|
|
278
|
+
} catch (error) {
|
|
279
|
+
await release();
|
|
280
|
+
throw error;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Stage D machine-primary: acquire the group lease for a turn whose ACTIVE sandbox
|
|
286
|
+
* is a connected machine (selfhosted) WITHOUT establishing a provider box. The
|
|
287
|
+
* "box" is the user's physical machine reached over NATS; there is NOTHING to
|
|
288
|
+
* spawn, snapshot, expose, or serialize — so we take the lease (the SAME group-keyed
|
|
289
|
+
* refcount/idle/epoch bookkeeping resumeBoxForTurn uses) but skip every box step.
|
|
290
|
+
*
|
|
291
|
+
* The lease is keyed backend "selfhosted": the reaper's terminateProviderBox
|
|
292
|
+
* short-circuits a selfhosted lease (it is the user's machine — NEVER provider-stop),
|
|
293
|
+
* draining it to cold with no stop; and the reaper's warm-meter rate for "selfhosted"
|
|
294
|
+
* is 0 (no cloud seconds billed for a box that does not exist). Returns the lease
|
|
295
|
+
* epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
|
|
296
|
+
* identical to the cloud path.
|
|
297
|
+
*
|
|
298
|
+
* holderId is the unique-per-execution id (the Temporal activityId for a turn).
|
|
299
|
+
*/
|
|
300
|
+
export async function acquireSelfhostedLeaseForTurn(
|
|
301
|
+
services: SandboxResumeServices,
|
|
302
|
+
ids: { accountId: string; workspaceId: string; sandboxGroupId: string; sessionId: string },
|
|
303
|
+
kind: LeaseHolderKind,
|
|
304
|
+
holderId: string,
|
|
305
|
+
): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
|
|
306
|
+
const { db, settings } = services;
|
|
307
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
308
|
+
|
|
309
|
+
let released = false;
|
|
310
|
+
const release = async (): Promise<void> => {
|
|
311
|
+
if (released) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
released = true;
|
|
315
|
+
await releaseLeaseHolder(db, {
|
|
316
|
+
accountId: ids.accountId,
|
|
317
|
+
workspaceId: ids.workspaceId,
|
|
318
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
319
|
+
kind,
|
|
320
|
+
holderId,
|
|
321
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
322
|
+
});
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
const acquired = await acquireLease(db, {
|
|
326
|
+
accountId: ids.accountId,
|
|
327
|
+
workspaceId: ids.workspaceId,
|
|
328
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
329
|
+
kind,
|
|
330
|
+
holderId,
|
|
331
|
+
subjectId: ids.sessionId,
|
|
332
|
+
backend: "selfhosted",
|
|
333
|
+
os: "linux",
|
|
334
|
+
leaseTtlMs,
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
// FENCED: a newer epoch re-established the group concurrently. Release our
|
|
338
|
+
// just-registered holder + surface the supersession (the outer turn catch
|
|
339
|
+
// requeues, mirroring resumeBoxForTurn).
|
|
340
|
+
if (acquired.role === "fenced") {
|
|
341
|
+
await release();
|
|
342
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// SPAWNER: we won the cold->warming CAS. There is NO box to establish — commit
|
|
346
|
+
// warm immediately so the lease does not linger in 'warming' (no box id, no
|
|
347
|
+
// resume_state; selfhosted is re-addressed by enrollment via the active pointer,
|
|
348
|
+
// never resumed from an envelope). resume_backend_id "selfhosted" keeps the
|
|
349
|
+
// reaper's never-provider-stop short-circuit correct.
|
|
350
|
+
if (acquired.role === "spawner") {
|
|
351
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
352
|
+
const committed = await commitWarmingToWarm(db, {
|
|
353
|
+
accountId: ids.accountId,
|
|
354
|
+
workspaceId: ids.workspaceId,
|
|
355
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
356
|
+
expectedEpoch,
|
|
357
|
+
// No provider box id; stamp a clearly non-box marker for diagnostics.
|
|
358
|
+
instanceId: `selfhosted:${ids.sessionId}`,
|
|
359
|
+
resumeBackendId: "selfhosted",
|
|
360
|
+
resumeState: null,
|
|
361
|
+
leaseTtlMs,
|
|
362
|
+
});
|
|
363
|
+
if (!committed.committed || !committed.lease) {
|
|
364
|
+
// A reaper reset our warming row, or a sibling re-established + bumped the
|
|
365
|
+
// epoch. Release our holder; surface supersession (NEVER touch any box).
|
|
366
|
+
await release();
|
|
367
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
368
|
+
}
|
|
369
|
+
return { leaseEpoch: committed.lease.leaseEpoch, release };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// ATTACHED / REARMED: the lease is live. There is no box to resume — just carry
|
|
373
|
+
// the lease epoch (same-session turns are serialized, so a concurrent 'warming'
|
|
374
|
+
// spawner on this group is not expected; the machine session is independent of the
|
|
375
|
+
// lease state regardless).
|
|
376
|
+
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Poll a warming lease until the spawner commits warm. If the warming row is
|
|
381
|
+
* reset to cold (the spawner died and the reaper reset it), re-acquire — we may
|
|
382
|
+
* now win the cold->warming CAS ourselves. Bounded by the warming TTL.
|
|
383
|
+
*/
|
|
384
|
+
async function waitForWarmOrReacquire(
|
|
385
|
+
services: SandboxResumeServices,
|
|
386
|
+
ids: ResumeBoxIds,
|
|
387
|
+
kind: LeaseHolderKind,
|
|
388
|
+
holderId: string,
|
|
389
|
+
): Promise<{ liveness: string; leaseEpoch: number }> {
|
|
390
|
+
const { db, settings } = services;
|
|
391
|
+
const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
|
|
392
|
+
while (Date.now() < deadline) {
|
|
393
|
+
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
394
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
395
|
+
if (!lease) {
|
|
396
|
+
// Lease vanished (cold-reaped). Re-acquire from scratch.
|
|
397
|
+
break;
|
|
398
|
+
}
|
|
399
|
+
if (lease.liveness === "warm" || lease.liveness === "draining") {
|
|
400
|
+
return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
|
|
401
|
+
}
|
|
402
|
+
if (lease.liveness === "cold") {
|
|
403
|
+
// The spawner died; the reaper reset to cold. Re-acquire — we might win.
|
|
404
|
+
break;
|
|
405
|
+
}
|
|
406
|
+
// still warming — keep polling.
|
|
407
|
+
}
|
|
408
|
+
// Re-acquire: if we now win cold->warming we become the spawner; if the box is
|
|
409
|
+
// warm we attach. Either way resumeBoxForTurn's caller already holds a holder
|
|
410
|
+
// row (idempotent), so this re-acquire just re-reads/re-CASes.
|
|
411
|
+
const reacquired = await acquireLease(db, {
|
|
412
|
+
accountId: ids.accountId,
|
|
413
|
+
workspaceId: ids.workspaceId,
|
|
414
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
415
|
+
kind,
|
|
416
|
+
holderId,
|
|
417
|
+
subjectId: ids.sessionId,
|
|
418
|
+
backend: ids.backend,
|
|
419
|
+
os: ids.os ?? "linux",
|
|
420
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
421
|
+
});
|
|
422
|
+
if (reacquired.role === "fenced") {
|
|
423
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
424
|
+
}
|
|
425
|
+
// For 'spawner' we'd need to run the cold-restore path; to keep resumeBoxForTurn
|
|
426
|
+
// a single critical section we recurse the spawner handling by surfacing it as a
|
|
427
|
+
// re-establish from the (now-cold) envelope. The simplest correct behavior: if
|
|
428
|
+
// we re-won the CAS, establish + commit happens on the NEXT resumeBoxForTurn
|
|
429
|
+
// call (the queued turn re-dispatch); here we just return the lease snapshot so
|
|
430
|
+
// the attached path resumes by id. A cold lease has no box yet, so treat it as
|
|
431
|
+
// warming-resolved only when warm/draining.
|
|
432
|
+
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// ============================================================================
|
|
436
|
+
// Channel-B display-stack launch (P4.1 — ensureDisplayStack is now real).
|
|
437
|
+
//
|
|
438
|
+
// Idempotent + callable by ANY worker on the resumed handle (and by the API on a
|
|
439
|
+
// viewer op). When the desktop tier is OFF (sandboxDesktopEnabled=false) or the
|
|
440
|
+
// backend is headless-only, this is a NO-OP — the HEADLESS ROLLOVER branch (I5),
|
|
441
|
+
// so the headless turn path is byte-for-byte unchanged. When the tier is ON for
|
|
442
|
+
// a desktop-capable backend, it execs the canonical opengeni-desktop-up under an
|
|
443
|
+
// in-box flock (re-establishes after a rollover; safe to call N times).
|
|
444
|
+
// `exposeStreamPort` real body remains a P4.2 concern.
|
|
445
|
+
// ============================================================================
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Ensure the desktop display stack (Xvfb -> XFCE -> x11vnc ->
|
|
449
|
+
* websockify:6080 -> noVNC) is up on the live box. Idempotent: the lock-free
|
|
450
|
+
* pre-check + the in-box flock + the up-script's per-stage PID guards make a
|
|
451
|
+
* second call a cheap no-op. NO-OP when the desktop tier is disabled or the
|
|
452
|
+
* backend cannot serve a desktop (degradation is a value: a headless-only session
|
|
453
|
+
* simply skips the stack). Delegates to the agent-loop-free leaf
|
|
454
|
+
* (@opengeni/runtime/sandbox display-stack).
|
|
455
|
+
*
|
|
456
|
+
* BEST-EFFORT — NEVER fails the turn. The display stack powers the OPTIONAL
|
|
457
|
+
* Channel-B desktop / computer-use surface; it is NOT load-bearing for the
|
|
458
|
+
* agent's work. A `DisplayStackError` (a real stage failure, OR — the regression
|
|
459
|
+
* this guards — a timeout-derived exit -1 when a viewer attach already holds /
|
|
460
|
+
* contends the up-script's flock and the turn's ensure waits ~45s and times out)
|
|
461
|
+
* is CAUGHT + logged and swallowed, so a slow/contended/failed stack degrades to
|
|
462
|
+
* Channel-A-only rather than killing the turn. The fast lock-free pre-check
|
|
463
|
+
* upstream means the already-up case resolves in milliseconds and never reaches
|
|
464
|
+
* this catch in the first place. `DisplayStackUnsupportedError` (a box that can't
|
|
465
|
+
* run commands at all) is likewise swallowed.
|
|
466
|
+
*/
|
|
467
|
+
export async function ensureDisplayStack(
|
|
468
|
+
settings: Settings,
|
|
469
|
+
established: EstablishedSandboxSession,
|
|
470
|
+
): Promise<void> {
|
|
471
|
+
// Headless rollover branch (I5): the tier is off OR the backend is
|
|
472
|
+
// headless-only -> no display stack. Behavior-preserving for the headless path.
|
|
473
|
+
// This is ALSO the "is the desktop relevant to this turn?" gate: we only touch
|
|
474
|
+
// the box when the desktop tier is enabled for a desktop-capable backend.
|
|
475
|
+
if (!settings.sandboxDesktopEnabled) {
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
if (!desktopCapableBackend(established.backendId)) {
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
try {
|
|
482
|
+
await ensureDisplayStackOnBox(established.session);
|
|
483
|
+
} catch (error) {
|
|
484
|
+
// The desktop is a value-add, not load-bearing for the agent's work, so NO
|
|
485
|
+
// display-stack failure may fail the turn:
|
|
486
|
+
// - DisplayStackUnsupportedError: the box genuinely can't run commands ->
|
|
487
|
+
// Channel-A-only.
|
|
488
|
+
// - DisplayStackError: a stage failure OR a contended-lock timeout (exit -1)
|
|
489
|
+
// after a viewer attach already brought the stack up / is mid-launch. The
|
|
490
|
+
// pre-check makes the already-up case fast; if we still time out or a stage
|
|
491
|
+
// really failed, degrade — don't die.
|
|
492
|
+
if (error instanceof DisplayStackUnsupportedError || error instanceof DisplayStackError) {
|
|
493
|
+
console.warn(
|
|
494
|
+
`[sandbox-resume] ensureDisplayStack degraded to Channel-A-only (turn continues): ${
|
|
495
|
+
error instanceof Error ? error.message : String(error)
|
|
496
|
+
}`,
|
|
497
|
+
);
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
// An unexpected non-display error (e.g. the session blew up entirely) still
|
|
501
|
+
// propagates — that is not a desktop-surface degradation.
|
|
502
|
+
throw error;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
// The structural slice of a provider session we need to resolve the tunnel.
|
|
507
|
+
type PortResolvableSession = {
|
|
508
|
+
resolveExposedPort?: (port: number) => Promise<ExposedPortEndpoint>;
|
|
509
|
+
};
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Resolve the desktop tunnel URL (resolveExposedPort(6080)) and assemble the
|
|
513
|
+
* direct-to-provider WS URL, recorded on the lease as `data_plane_url` at
|
|
514
|
+
* commit. The per-viewer scoped token is minted at viewer-attach time (the
|
|
515
|
+
* handshake), NOT here — the spawner records only the box-scoped tunnel URL so
|
|
516
|
+
* the lease carries a fresh value across a rollover. P4.2 productionizes the
|
|
517
|
+
* P4.1 stub.
|
|
518
|
+
*
|
|
519
|
+
* Returns null (degradation is a value, NEVER a throw):
|
|
520
|
+
* - the desktop tier is off (sandboxDesktopEnabled=false), or
|
|
521
|
+
* - the backend is headless-only (no desktop), or
|
|
522
|
+
* - the provider session cannot resolve the port (no resolveExposedPort), or
|
|
523
|
+
* - the tunnel lookup transiently failed.
|
|
524
|
+
* In every null case the lease's data_plane_url stays null and the next
|
|
525
|
+
* API-direct viewer op re-mints it; the turn never fails on a desktop hiccup.
|
|
526
|
+
*/
|
|
527
|
+
export async function exposeStreamPort(
|
|
528
|
+
settings: Settings,
|
|
529
|
+
established: EstablishedSandboxSession,
|
|
530
|
+
): Promise<{ url: string; expiresAt: Date } | null> {
|
|
531
|
+
// Headless rollover branch (I5): tier !== desktop -> no stream port to expose.
|
|
532
|
+
if (!settings.sandboxDesktopEnabled) {
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
if (!desktopCapableBackend(established.backendId)) {
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
538
|
+
const session = established.session as PortResolvableSession;
|
|
539
|
+
if (typeof session?.resolveExposedPort !== "function") {
|
|
540
|
+
return null;
|
|
541
|
+
}
|
|
542
|
+
try {
|
|
543
|
+
const endpoint = await session.resolveExposedPort(DESKTOP_STREAM_PORT);
|
|
544
|
+
const url = buildStreamUrl(endpoint);
|
|
545
|
+
// The provider tunnel URL is box-lifetime-valid (Modal raw TLS) or
|
|
546
|
+
// provider-TTL-signed (Daytona/Blaxel); the per-viewer token's TTL bounds the
|
|
547
|
+
// viewer's freshness. We record only the URL on the lease; expiresAt here is a
|
|
548
|
+
// soft hint (the box's idle horizon), not a hard token expiry.
|
|
549
|
+
return { url, expiresAt: new Date(Date.now() + settings.sandboxLeaseTtlMs) };
|
|
550
|
+
} catch (error) {
|
|
551
|
+
// Degradation is a value: a transient resolve failure leaves data_plane_url
|
|
552
|
+
// null and the next viewer op re-mints. Never fail the turn on a desktop op.
|
|
553
|
+
if (error instanceof StreamPortUnavailableError) {
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
558
|
+
}
|