@opengeni/worker-bundle 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +644 -80
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +32 -4
- package/src/activities/sandbox-lease.ts +130 -3
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +197 -59
package/src/sandbox-resume.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
failWarmingToCold,
|
|
29
29
|
getSandboxSessionEnvelope,
|
|
30
30
|
readLease,
|
|
31
|
+
recordWarmingSandboxCreated,
|
|
31
32
|
releaseLeaseHolder,
|
|
32
33
|
SandboxLeaseSupersededError,
|
|
33
34
|
type Database,
|
|
@@ -42,8 +43,11 @@ import {
|
|
|
42
43
|
DisplayStackUnsupportedError,
|
|
43
44
|
buildStreamUrl,
|
|
44
45
|
StreamPortUnavailableError,
|
|
46
|
+
modalSandboxAttributionEnvironment,
|
|
47
|
+
tagModalSandbox,
|
|
45
48
|
type EstablishedSandboxSession,
|
|
46
49
|
type ExposedPortEndpoint,
|
|
50
|
+
type RuntimeMetricsHooks,
|
|
47
51
|
} from "@opengeni/runtime";
|
|
48
52
|
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
49
53
|
|
|
@@ -57,6 +61,7 @@ export type ResumeHolderKind = LeaseHolderKind;
|
|
|
57
61
|
export type SandboxResumeServices = {
|
|
58
62
|
db: Database;
|
|
59
63
|
settings: Settings;
|
|
64
|
+
sandboxMetrics?: RuntimeMetricsHooks;
|
|
60
65
|
};
|
|
61
66
|
|
|
62
67
|
export type ResumeBoxIds = {
|
|
@@ -105,14 +110,113 @@ export type ResumedTurnSandbox = {
|
|
|
105
110
|
release: () => Promise<void>;
|
|
106
111
|
};
|
|
107
112
|
|
|
108
|
-
|
|
109
|
-
|
|
113
|
+
export class SandboxWarmingTimeoutError extends Error {
|
|
114
|
+
readonly code = "sandbox_warming_timeout";
|
|
115
|
+
|
|
116
|
+
constructor(public readonly backend: string, public readonly timeoutMs: number) {
|
|
117
|
+
super(
|
|
118
|
+
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
|
|
119
|
+
);
|
|
120
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
|
|
125
|
+
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
110
126
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
111
127
|
|
|
112
128
|
async function sleep(ms: number): Promise<void> {
|
|
113
129
|
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
114
130
|
}
|
|
115
131
|
|
|
132
|
+
async function terminateEstablishedSandbox(established: EstablishedSandboxSession | null): Promise<boolean> {
|
|
133
|
+
if (!established) {
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
|
|
137
|
+
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
138
|
+
try {
|
|
139
|
+
await client.delete(established.sessionState);
|
|
140
|
+
return true;
|
|
141
|
+
} catch {
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const session = established.session as {
|
|
146
|
+
close?: () => Promise<unknown>;
|
|
147
|
+
terminate?: () => Promise<unknown>;
|
|
148
|
+
kill?: () => Promise<unknown>;
|
|
149
|
+
closed?: boolean;
|
|
150
|
+
};
|
|
151
|
+
try {
|
|
152
|
+
if (session.terminate) {
|
|
153
|
+
await session.terminate();
|
|
154
|
+
return true;
|
|
155
|
+
} else if (session.kill) {
|
|
156
|
+
await session.kill();
|
|
157
|
+
return true;
|
|
158
|
+
} else if (session.close) {
|
|
159
|
+
if (!session.closed) {
|
|
160
|
+
await session.close();
|
|
161
|
+
}
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
return false;
|
|
165
|
+
} catch {
|
|
166
|
+
// Best-effort cleanup. A provider-side orphan sweep is the backstop.
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
|
|
172
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
173
|
+
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
|
|
174
|
+
? new SandboxWarmingTimeoutError(backend, timeoutMs)
|
|
175
|
+
: error;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, error: unknown): void {
|
|
179
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
try {
|
|
183
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
184
|
+
} catch {
|
|
185
|
+
// Metrics emission must never affect sandbox recovery or error propagation.
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
|
|
190
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
191
|
+
? envelope.sessionState as Record<string, unknown>
|
|
192
|
+
: null;
|
|
193
|
+
const archive = sessionState?.workspaceArchive;
|
|
194
|
+
return typeof archive === "string" && archive.length > 0 ? archive : null;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function preserveWorkspaceArchiveOnInterimResumeState(
|
|
198
|
+
resumeState: Record<string, unknown> | null,
|
|
199
|
+
archiveSource: Record<string, unknown> | null,
|
|
200
|
+
): Record<string, unknown> | null {
|
|
201
|
+
const archive = workspaceArchiveFromEnvelope(archiveSource);
|
|
202
|
+
if (!archive) {
|
|
203
|
+
return resumeState;
|
|
204
|
+
}
|
|
205
|
+
const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
|
|
206
|
+
? resumeState.sessionState as Record<string, unknown>
|
|
207
|
+
: {};
|
|
208
|
+
return {
|
|
209
|
+
...(resumeState ?? {}),
|
|
210
|
+
...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
|
|
211
|
+
? { backendId: archiveSource.backendId }
|
|
212
|
+
: {}),
|
|
213
|
+
sessionState: {
|
|
214
|
+
...existingSessionState,
|
|
215
|
+
workspaceArchive: archive,
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
116
220
|
/**
|
|
117
221
|
* Resume the one box for a single turn (or any worker-side resume op). Returns
|
|
118
222
|
* the live non-owned session + the fence epoch + a release fn. The CALLER owns
|
|
@@ -178,6 +282,14 @@ export async function resumeBoxForTurn(
|
|
|
178
282
|
// expose the stream port, then commit warm (lease_epoch++).
|
|
179
283
|
if (acquired.role === "spawner") {
|
|
180
284
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
285
|
+
let createdEstablished: EstablishedSandboxSession | null = null;
|
|
286
|
+
if (ids.environment && ids.backend === "modal") {
|
|
287
|
+
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
288
|
+
leaseId: acquired.lease.id,
|
|
289
|
+
workspaceId: ids.workspaceId,
|
|
290
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
291
|
+
}));
|
|
292
|
+
}
|
|
181
293
|
try {
|
|
182
294
|
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
183
295
|
// Prefer the COLD lease's preserved resume_state when it carries a persisted
|
|
@@ -194,7 +306,36 @@ export async function resumeBoxForTurn(
|
|
|
194
306
|
sessionId: ids.sessionId,
|
|
195
307
|
backendOverride: ids.backend as never,
|
|
196
308
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
309
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
310
|
+
onSandboxCreated: async (created) => {
|
|
311
|
+
createdEstablished = created;
|
|
312
|
+
const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
|
|
313
|
+
(await serializeEstablishedSandboxEnvelope(created)) ?? null,
|
|
314
|
+
spawnEnvelope,
|
|
315
|
+
);
|
|
316
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
317
|
+
accountId: ids.accountId,
|
|
318
|
+
workspaceId: ids.workspaceId,
|
|
319
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
320
|
+
expectedEpoch,
|
|
321
|
+
instanceId: created.instanceId,
|
|
322
|
+
resumeBackendId: created.backendId,
|
|
323
|
+
resumeState: resumeEnvelope,
|
|
324
|
+
leaseTtlMs,
|
|
325
|
+
});
|
|
326
|
+
if (!recorded.recorded) {
|
|
327
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
328
|
+
}
|
|
329
|
+
if (created.backendId === "modal") {
|
|
330
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
331
|
+
leaseId: acquired.lease.id,
|
|
332
|
+
workspaceId: ids.workspaceId,
|
|
333
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
334
|
+
}).catch(() => undefined);
|
|
335
|
+
}
|
|
336
|
+
},
|
|
197
337
|
});
|
|
338
|
+
createdEstablished = established;
|
|
198
339
|
await ensureDisplayStack(settings, established);
|
|
199
340
|
const endpoint = await exposeStreamPort(settings, established);
|
|
200
341
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
@@ -219,39 +360,51 @@ export async function resumeBoxForTurn(
|
|
|
219
360
|
if (!committed.committed || !committed.lease) {
|
|
220
361
|
// A reaper reset our warming row (we were too slow) or a sibling
|
|
221
362
|
// re-established and bumped the epoch. Drop the handle; release our
|
|
222
|
-
// holder; surface supersession.
|
|
363
|
+
// holder; surface supersession. This spawner created the box, so stop it
|
|
364
|
+
// before retrying to avoid an untracked running sandbox.
|
|
365
|
+
await terminateEstablishedSandbox(established);
|
|
223
366
|
await release();
|
|
224
367
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
225
368
|
}
|
|
226
369
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
227
370
|
} catch (error) {
|
|
228
371
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
372
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
373
|
+
await release();
|
|
229
374
|
throw error;
|
|
230
375
|
}
|
|
231
|
-
|
|
232
|
-
//
|
|
233
|
-
//
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
376
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
377
|
+
// Caught spawn failure: if the just-created sandbox was actually stopped,
|
|
378
|
+
// roll the warming row back to cold so a queued turn can re-acquire and
|
|
379
|
+
// re-spawn. If termination itself failed, keep the recorded instance_id on
|
|
380
|
+
// the warming row; the lease TTL/reaper and Modal orphan sweep are the
|
|
381
|
+
// tracked backstops, and we must not erase the only provider pointer.
|
|
382
|
+
if (terminated) {
|
|
383
|
+
await failWarmingToCold(db, {
|
|
384
|
+
accountId: ids.accountId,
|
|
385
|
+
workspaceId: ids.workspaceId,
|
|
386
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
387
|
+
expectedEpoch,
|
|
388
|
+
});
|
|
389
|
+
}
|
|
240
390
|
await release();
|
|
241
|
-
|
|
391
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
392
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
393
|
+
throw warmingError;
|
|
242
394
|
}
|
|
243
395
|
}
|
|
244
396
|
|
|
245
397
|
// ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
|
|
246
398
|
// BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
|
|
247
|
-
// first wait for the spawner to commit warm
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
if (acquired.lease.liveness === "warming") {
|
|
251
|
-
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
252
|
-
}
|
|
253
|
-
|
|
399
|
+
// first wait for the spawner to commit warm, bounded by the explicit warming
|
|
400
|
+
// budget. A cold reset means the spawner died; requeue the turn instead of
|
|
401
|
+
// trying to enter the spawner create path from the attached branch.
|
|
254
402
|
try {
|
|
403
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
404
|
+
if (acquired.lease.liveness === "warming") {
|
|
405
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
406
|
+
}
|
|
407
|
+
|
|
255
408
|
// Prefer the lease's resume_state (the LIVE box the spawner committed) so we
|
|
256
409
|
// re-attach to the SAME box by id, not cold-restore the original session
|
|
257
410
|
// manifest into a rival. Fall back to the session envelope only when the
|
|
@@ -263,6 +416,7 @@ export async function resumeBoxForTurn(
|
|
|
263
416
|
sessionId: ids.sessionId,
|
|
264
417
|
backendOverride: ids.backend as never,
|
|
265
418
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
419
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
266
420
|
});
|
|
267
421
|
// Re-ensure the desktop display stack on the ATTACHED/REARMED path too — NOT
|
|
268
422
|
// just the spawner path. A turn attaching to a warm box whose :0 was never
|
|
@@ -277,7 +431,9 @@ export async function resumeBoxForTurn(
|
|
|
277
431
|
return { established, leaseEpoch, release };
|
|
278
432
|
} catch (error) {
|
|
279
433
|
await release();
|
|
280
|
-
|
|
434
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
435
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
436
|
+
throw warmingError;
|
|
281
437
|
}
|
|
282
438
|
}
|
|
283
439
|
|
|
@@ -378,58 +534,40 @@ export async function acquireSelfhostedLeaseForTurn(
|
|
|
378
534
|
|
|
379
535
|
/**
|
|
380
536
|
* Poll a warming lease until the spawner commits warm. If the warming row is
|
|
381
|
-
* reset to cold (the spawner died and the reaper reset it),
|
|
382
|
-
*
|
|
537
|
+
* reset to cold (the spawner died and the reaper reset it), surface supersession
|
|
538
|
+
* so the turn is re-dispatched and can enter the normal spawner branch from
|
|
539
|
+
* acquireLease. Bounded by OPENGENI_SANDBOX_WARMING_TIMEOUT_MS, not the lease TTL.
|
|
383
540
|
*/
|
|
384
|
-
async function
|
|
541
|
+
async function waitForWarm(
|
|
385
542
|
services: SandboxResumeServices,
|
|
386
543
|
ids: ResumeBoxIds,
|
|
387
|
-
|
|
388
|
-
holderId: string,
|
|
389
|
-
): Promise<{ liveness: string; leaseEpoch: number }> {
|
|
544
|
+
): Promise<{ leaseEpoch: number }> {
|
|
390
545
|
const { db, settings } = services;
|
|
391
|
-
const deadline = Date.now() + settings.
|
|
546
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
392
547
|
while (Date.now() < deadline) {
|
|
393
548
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
394
549
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
395
550
|
if (!lease) {
|
|
396
|
-
// Lease vanished (cold-reaped). Re-
|
|
397
|
-
|
|
551
|
+
// Lease vanished (cold-reaped). Re-dispatch from scratch.
|
|
552
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
398
553
|
}
|
|
399
|
-
if (lease.liveness === "warm"
|
|
400
|
-
return {
|
|
554
|
+
if (lease.liveness === "warm") {
|
|
555
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
556
|
+
}
|
|
557
|
+
if (lease.liveness === "draining") {
|
|
558
|
+
// The warming attempt either failed into drain or committed+released before
|
|
559
|
+
// this waiter observed it. Re-dispatch so acquireLease can re-arm or spawn
|
|
560
|
+
// through the normal path.
|
|
561
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
401
562
|
}
|
|
402
563
|
if (lease.liveness === "cold") {
|
|
403
|
-
// The spawner died;
|
|
404
|
-
|
|
564
|
+
// The spawner died; re-dispatch so the normal acquireLease path can win
|
|
565
|
+
// cold->warming and run the full spawner branch.
|
|
566
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
405
567
|
}
|
|
406
568
|
// still warming — keep polling.
|
|
407
569
|
}
|
|
408
|
-
|
|
409
|
-
// warm we attach. Either way resumeBoxForTurn's caller already holds a holder
|
|
410
|
-
// row (idempotent), so this re-acquire just re-reads/re-CASes.
|
|
411
|
-
const reacquired = await acquireLease(db, {
|
|
412
|
-
accountId: ids.accountId,
|
|
413
|
-
workspaceId: ids.workspaceId,
|
|
414
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
415
|
-
kind,
|
|
416
|
-
holderId,
|
|
417
|
-
subjectId: ids.sessionId,
|
|
418
|
-
backend: ids.backend,
|
|
419
|
-
os: ids.os ?? "linux",
|
|
420
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
421
|
-
});
|
|
422
|
-
if (reacquired.role === "fenced") {
|
|
423
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
424
|
-
}
|
|
425
|
-
// For 'spawner' we'd need to run the cold-restore path; to keep resumeBoxForTurn
|
|
426
|
-
// a single critical section we recurse the spawner handling by surfacing it as a
|
|
427
|
-
// re-establish from the (now-cold) envelope. The simplest correct behavior: if
|
|
428
|
-
// we re-won the CAS, establish + commit happens on the NEXT resumeBoxForTurn
|
|
429
|
-
// call (the queued turn re-dispatch); here we just return the lease snapshot so
|
|
430
|
-
// the attached path resumes by id. A cold lease has no box yet, so treat it as
|
|
431
|
-
// warming-resolved only when warm/draining.
|
|
432
|
-
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
570
|
+
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
433
571
|
}
|
|
434
572
|
|
|
435
573
|
// ============================================================================
|