@opengeni/worker-bundle 0.2.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,6 +28,7 @@ import {
28
28
  failWarmingToCold,
29
29
  getSandboxSessionEnvelope,
30
30
  readLease,
31
+ recordWarmingSandboxCreated,
31
32
  releaseLeaseHolder,
32
33
  SandboxLeaseSupersededError,
33
34
  type Database,
@@ -42,6 +43,8 @@ import {
42
43
  DisplayStackUnsupportedError,
43
44
  buildStreamUrl,
44
45
  StreamPortUnavailableError,
46
+ modalSandboxAttributionEnvironment,
47
+ tagModalSandbox,
45
48
  type EstablishedSandboxSession,
46
49
  type ExposedPortEndpoint,
47
50
  } from "@opengeni/runtime";
@@ -105,14 +108,102 @@ export type ResumedTurnSandbox = {
105
108
  release: () => Promise<void>;
106
109
  };
107
110
 
108
- // Bounded poll while a sibling spawner is mid cold-restore. The reaper resets a
109
- // dead warming row after sandboxLeaseWarmingTtlMs; we poll up to that horizon.
111
+ export class SandboxWarmingTimeoutError extends Error {
112
+ readonly code = "sandbox_warming_timeout";
113
+
114
+ constructor(public readonly backend: string, public readonly timeoutMs: number) {
115
+ super(
116
+ `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
117
+ );
118
+ this.name = "SandboxWarmingTimeoutError";
119
+ }
120
+ }
121
+
122
+ // Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
123
+ // user-facing and separate from the lease TTL heartbeat/reaper horizon.
110
124
  const WARMING_POLL_INTERVAL_MS = 250;
111
125
 
112
126
  async function sleep(ms: number): Promise<void> {
113
127
  await new Promise<void>((resolve) => setTimeout(resolve, ms));
114
128
  }
115
129
 
130
+ async function terminateEstablishedSandbox(established: EstablishedSandboxSession | null): Promise<boolean> {
131
+ if (!established) {
132
+ return true;
133
+ }
134
+ const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
135
+ if (typeof client.delete === "function" && established.sessionState !== undefined) {
136
+ try {
137
+ await client.delete(established.sessionState);
138
+ return true;
139
+ } catch {
140
+ return false;
141
+ }
142
+ }
143
+ const session = established.session as {
144
+ close?: () => Promise<unknown>;
145
+ terminate?: () => Promise<unknown>;
146
+ kill?: () => Promise<unknown>;
147
+ closed?: boolean;
148
+ };
149
+ try {
150
+ if (session.terminate) {
151
+ await session.terminate();
152
+ return true;
153
+ } else if (session.kill) {
154
+ await session.kill();
155
+ return true;
156
+ } else if (session.close) {
157
+ if (!session.closed) {
158
+ await session.close();
159
+ }
160
+ return true;
161
+ }
162
+ return false;
163
+ } catch {
164
+ // Best-effort cleanup. A provider-side orphan sweep is the backstop.
165
+ return false;
166
+ }
167
+ }
168
+
169
+ function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
170
+ const message = error instanceof Error ? error.message : String(error);
171
+ return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
172
+ ? new SandboxWarmingTimeoutError(backend, timeoutMs)
173
+ : error;
174
+ }
175
+
176
+ function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
177
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
178
+ ? envelope.sessionState as Record<string, unknown>
179
+ : null;
180
+ const archive = sessionState?.workspaceArchive;
181
+ return typeof archive === "string" && archive.length > 0 ? archive : null;
182
+ }
183
+
184
+ function preserveWorkspaceArchiveOnInterimResumeState(
185
+ resumeState: Record<string, unknown> | null,
186
+ archiveSource: Record<string, unknown> | null,
187
+ ): Record<string, unknown> | null {
188
+ const archive = workspaceArchiveFromEnvelope(archiveSource);
189
+ if (!archive) {
190
+ return resumeState;
191
+ }
192
+ const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
193
+ ? resumeState.sessionState as Record<string, unknown>
194
+ : {};
195
+ return {
196
+ ...(resumeState ?? {}),
197
+ ...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
198
+ ? { backendId: archiveSource.backendId }
199
+ : {}),
200
+ sessionState: {
201
+ ...existingSessionState,
202
+ workspaceArchive: archive,
203
+ },
204
+ };
205
+ }
206
+
116
207
  /**
117
208
  * Resume the one box for a single turn (or any worker-side resume op). Returns
118
209
  * the live non-owned session + the fence epoch + a release fn. The CALLER owns
@@ -178,6 +269,14 @@ export async function resumeBoxForTurn(
178
269
  // expose the stream port, then commit warm (lease_epoch++).
179
270
  if (acquired.role === "spawner") {
180
271
  const expectedEpoch = acquired.lease.leaseEpoch;
272
+ let createdEstablished: EstablishedSandboxSession | null = null;
273
+ if (ids.environment && ids.backend === "modal") {
274
+ Object.assign(ids.environment, modalSandboxAttributionEnvironment({
275
+ leaseId: acquired.lease.id,
276
+ workspaceId: ids.workspaceId,
277
+ sandboxGroupId: ids.sandboxGroupId,
278
+ }));
279
+ }
181
280
  try {
182
281
  const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
183
282
  // Prefer the COLD lease's preserved resume_state when it carries a persisted
@@ -194,7 +293,35 @@ export async function resumeBoxForTurn(
194
293
  sessionId: ids.sessionId,
195
294
  backendOverride: ids.backend as never,
196
295
  ...(ids.environment ? { environment: ids.environment } : {}),
296
+ onSandboxCreated: async (created) => {
297
+ createdEstablished = created;
298
+ const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
299
+ (await serializeEstablishedSandboxEnvelope(created)) ?? null,
300
+ spawnEnvelope,
301
+ );
302
+ const recorded = await recordWarmingSandboxCreated(db, {
303
+ accountId: ids.accountId,
304
+ workspaceId: ids.workspaceId,
305
+ sandboxGroupId: ids.sandboxGroupId,
306
+ expectedEpoch,
307
+ instanceId: created.instanceId,
308
+ resumeBackendId: created.backendId,
309
+ resumeState: resumeEnvelope,
310
+ leaseTtlMs,
311
+ });
312
+ if (!recorded.recorded) {
313
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
314
+ }
315
+ if (created.backendId === "modal") {
316
+ await tagModalSandbox(settings, created.instanceId, {
317
+ leaseId: acquired.lease.id,
318
+ workspaceId: ids.workspaceId,
319
+ sandboxGroupId: ids.sandboxGroupId,
320
+ }).catch(() => undefined);
321
+ }
322
+ },
197
323
  });
324
+ createdEstablished = established;
198
325
  await ensureDisplayStack(settings, established);
199
326
  const endpoint = await exposeStreamPort(settings, established);
200
327
  // Fold the LIVE box into a re-resumable envelope and persist it as the
@@ -219,39 +346,49 @@ export async function resumeBoxForTurn(
219
346
  if (!committed.committed || !committed.lease) {
220
347
  // A reaper reset our warming row (we were too slow) or a sibling
221
348
  // re-established and bumped the epoch. Drop the handle; release our
222
- // holder; surface supersession. NEVER provider-delete the box.
349
+ // holder; surface supersession. This spawner created the box, so stop it
350
+ // before retrying to avoid an untracked running sandbox.
351
+ await terminateEstablishedSandbox(established);
223
352
  await release();
224
353
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
225
354
  }
226
355
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
227
356
  } catch (error) {
228
357
  if (error instanceof SandboxLeaseSupersededError) {
358
+ await terminateEstablishedSandbox(createdEstablished);
359
+ await release();
229
360
  throw error;
230
361
  }
231
- // Caught spawn failure: roll the warming row back to cold so a queued turn
232
- // re-acquires and re-spawns. Holders are intentionally left for the
233
- // re-acquire (failWarmingToCold keeps them); then release our own holder.
234
- await failWarmingToCold(db, {
235
- accountId: ids.accountId,
236
- workspaceId: ids.workspaceId,
237
- sandboxGroupId: ids.sandboxGroupId,
238
- expectedEpoch,
239
- });
362
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
363
+ // Caught spawn failure: if the just-created sandbox was actually stopped,
364
+ // roll the warming row back to cold so a queued turn can re-acquire and
365
+ // re-spawn. If termination itself failed, keep the recorded instance_id on
366
+ // the warming row; the lease TTL/reaper and Modal orphan sweep are the
367
+ // tracked backstops, and we must not erase the only provider pointer.
368
+ if (terminated) {
369
+ await failWarmingToCold(db, {
370
+ accountId: ids.accountId,
371
+ workspaceId: ids.workspaceId,
372
+ sandboxGroupId: ids.sandboxGroupId,
373
+ expectedEpoch,
374
+ });
375
+ }
240
376
  await release();
241
- throw error;
377
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
242
378
  }
243
379
  }
244
380
 
245
381
  // ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
246
382
  // BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
247
- // first wait for the spawner to commit warm (or for the row to flip cold so we
248
- // can re-acquire as spawner).
249
- let leaseEpoch = acquired.lease.leaseEpoch;
250
- if (acquired.lease.liveness === "warming") {
251
- leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
252
- }
253
-
383
+ // first wait for the spawner to commit warm, bounded by the explicit warming
384
+ // budget. A cold reset means the spawner died; requeue the turn instead of
385
+ // trying to enter the spawner create path from the attached branch.
254
386
  try {
387
+ let leaseEpoch = acquired.lease.leaseEpoch;
388
+ if (acquired.lease.liveness === "warming") {
389
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
390
+ }
391
+
255
392
  // Prefer the lease's resume_state (the LIVE box the spawner committed) so we
256
393
  // re-attach to the SAME box by id, not cold-restore the original session
257
394
  // manifest into a rival. Fall back to the session envelope only when the
@@ -277,7 +414,7 @@ export async function resumeBoxForTurn(
277
414
  return { established, leaseEpoch, release };
278
415
  } catch (error) {
279
416
  await release();
280
- throw error;
417
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
281
418
  }
282
419
  }
283
420
 
@@ -378,58 +515,40 @@ export async function acquireSelfhostedLeaseForTurn(
378
515
 
379
516
  /**
380
517
  * Poll a warming lease until the spawner commits warm. If the warming row is
381
- * reset to cold (the spawner died and the reaper reset it), re-acquire — we may
382
- * now win the cold->warming CAS ourselves. Bounded by the warming TTL.
518
+ * reset to cold (the spawner died and the reaper reset it), surface supersession
519
+ * so the turn is re-dispatched and can enter the normal spawner branch from
520
+ * acquireLease. Bounded by OPENGENI_SANDBOX_WARMING_TIMEOUT_MS, not the lease TTL.
383
521
  */
384
- async function waitForWarmOrReacquire(
522
+ async function waitForWarm(
385
523
  services: SandboxResumeServices,
386
524
  ids: ResumeBoxIds,
387
- kind: LeaseHolderKind,
388
- holderId: string,
389
- ): Promise<{ liveness: string; leaseEpoch: number }> {
525
+ ): Promise<{ leaseEpoch: number }> {
390
526
  const { db, settings } = services;
391
- const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
527
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
392
528
  while (Date.now() < deadline) {
393
529
  await sleep(WARMING_POLL_INTERVAL_MS);
394
530
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
395
531
  if (!lease) {
396
- // Lease vanished (cold-reaped). Re-acquire from scratch.
397
- break;
532
+ // Lease vanished (cold-reaped). Re-dispatch from scratch.
533
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
534
+ }
535
+ if (lease.liveness === "warm") {
536
+ return { leaseEpoch: lease.leaseEpoch };
398
537
  }
399
- if (lease.liveness === "warm" || lease.liveness === "draining") {
400
- return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
538
+ if (lease.liveness === "draining") {
539
+ // The warming attempt either failed into drain or committed+released before
540
+ // this waiter observed it. Re-dispatch so acquireLease can re-arm or spawn
541
+ // through the normal path.
542
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
401
543
  }
402
544
  if (lease.liveness === "cold") {
403
- // The spawner died; the reaper reset to cold. Re-acquire — we might win.
404
- break;
545
+ // The spawner died; re-dispatch so the normal acquireLease path can win
546
+ // cold->warming and run the full spawner branch.
547
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
405
548
  }
406
549
  // still warming — keep polling.
407
550
  }
408
- // Re-acquire: if we now win cold->warming we become the spawner; if the box is
409
- // warm we attach. Either way resumeBoxForTurn's caller already holds a holder
410
- // row (idempotent), so this re-acquire just re-reads/re-CASes.
411
- const reacquired = await acquireLease(db, {
412
- accountId: ids.accountId,
413
- workspaceId: ids.workspaceId,
414
- sandboxGroupId: ids.sandboxGroupId,
415
- kind,
416
- holderId,
417
- subjectId: ids.sessionId,
418
- backend: ids.backend,
419
- os: ids.os ?? "linux",
420
- leaseTtlMs: settings.sandboxLeaseTtlMs,
421
- });
422
- if (reacquired.role === "fenced") {
423
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
424
- }
425
- // For 'spawner' we'd need to run the cold-restore path; to keep resumeBoxForTurn
426
- // a single critical section we recurse the spawner handling by surfacing it as a
427
- // re-establish from the (now-cold) envelope. The simplest correct behavior: if
428
- // we re-won the CAS, establish + commit happens on the NEXT resumeBoxForTurn
429
- // call (the queued turn re-dispatch); here we just return the lease snapshot so
430
- // the attached path resumes by id. A cold lease has no box yet, so treat it as
431
- // warming-resolved only when warm/draining.
432
- return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
551
+ throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
433
552
  }
434
553
 
435
554
  // ============================================================================