@opengeni/worker-bundle 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7127 -1654
  3. package/dist/index.js.map +1 -1
  4. package/package.json +19 -19
  5. package/src/activities/agent-turn.ts +4405 -1089
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -6,13 +6,12 @@
6
6
  //
7
7
  // 1. acquireLease (group-keyed) under the DB FOR UPDATE + cold->warming CAS —
8
8
  // the SOLE double-spawn guard (P1.1).
9
- // 2. establishSandboxSessionFromEnvelope — resume the one box BY ID (warm
10
- // reattach, R4-safe) or cold-restore from snapshot on a provider NotFound.
11
- // 3. (spawner) ensureDisplayStack + exposeStreamPort, then commitWarmingToWarm
12
- // (the lease_epoch++ fence + folds the resume envelope onto the lease). The
13
- // ATTACHED/REARMED path RE-ensures the display stack too (idempotent) so a
14
- // computer-use turn always finds a live :0 even on a box first warmed by a
15
- // non-desktop op or after a snapshot rollover dropped the X stack.
9
+ // 2. The cold->warming winner may create/restore. Every attached caller is
10
+ // resume-only; a missing provider box retires the exact warm epoch and
11
+ // re-enters admission instead of creating a rival.
12
+ // 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
13
+ // envelope onto the lease). Optional desktop work is deliberately absent:
14
+ // the viewer and the first actual computer-use action initialize :0 lazily.
16
15
  // 4. the caller injects {client, session, sessionState} NON-OWNED into the run
17
16
  // (the SDK never reaps it — the keystone), runs, then in `finally` calls the
18
17
  // returned `release()` and drops the in-memory handle. NEVER provider-delete
@@ -27,41 +26,58 @@ import {
27
26
  commitWarmingToWarm,
28
27
  failWarmingToCold,
29
28
  getSandboxSessionEnvelope,
29
+ markWarmLeaseInstanceLost,
30
+ persistWarmSnapshot,
30
31
  readLease,
31
32
  recordWarmingSandboxCreated,
32
33
  releaseLeaseHolder,
34
+ touchLeaseHolder,
33
35
  SandboxLeaseSupersededError,
34
36
  type Database,
35
37
  type LeaseHolderKind,
36
38
  } from "@opengeni/db";
37
39
  import {
38
40
  establishSandboxSessionFromEnvelope,
39
- ensureDisplayStack as ensureDisplayStackOnBox,
40
- desktopCapableBackend,
41
+ isProviderSandboxNotFoundError,
42
+ SandboxResumeStateUnavailableError,
41
43
  serializeEstablishedSandboxEnvelope,
42
- DisplayStackError,
43
- DisplayStackUnsupportedError,
44
- buildStreamUrl,
45
- StreamPortUnavailableError,
46
- modalSandboxAttributionEnvironment,
44
+ deletePriorPersistedSnapshot,
47
45
  tagModalSandbox,
48
46
  type EstablishedSandboxSession,
49
- type ExposedPortEndpoint,
50
47
  type RuntimeMetricsHooks,
51
48
  } from "@opengeni/runtime";
52
- import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
53
-
54
- export { DESKTOP_STREAM_PORT };
55
49
 
56
50
  // Re-exported for callers that just want the ack-kind union.
57
51
  export type ResumeHolderKind = LeaseHolderKind;
58
52
 
53
+ /**
54
+ * The durable identity of a turn holding a shared sandbox lease.
55
+ *
56
+ * Temporal activity ids cannot satisfy this type: they are workflow-local
57
+ * sequence numbers and collide across sessions that share a sandbox group.
58
+ */
59
+ export type TurnSandboxLeaseHolderId = `turn-attempt:${string}`;
60
+
61
+ export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLeaseHolderId {
62
+ const normalized = attemptId.trim();
63
+ if (!normalized) {
64
+ throw new Error("Sandbox lease holder requires a turn attempt id");
65
+ }
66
+ return `turn-attempt:${normalized}`;
67
+ }
68
+
59
69
  /** The minimal services surface resumeBoxForTurn needs. A subset of
60
70
  * ActivityServices so a test (and the API later) can pass a lean bag. */
61
71
  export type SandboxResumeServices = {
62
72
  db: Database;
63
73
  settings: Settings;
64
74
  sandboxMetrics?: RuntimeMetricsHooks;
75
+ /** Called only by the observer that wins the exact warm->cold loss CAS. */
76
+ onSandboxLost?: (input: {
77
+ sandboxGroupId: string;
78
+ instanceId: string;
79
+ leaseEpoch: number;
80
+ }) => Promise<void>;
65
81
  };
66
82
 
67
83
  export type ResumeBoxIds = {
@@ -94,6 +110,14 @@ export type ResumeBoxIds = {
94
110
  * selfhosted path never passes it; a legacy/null-image box never conflicts).
95
111
  */
96
112
  image?: string;
113
+ /**
114
+ * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
115
+ * acquireLease, which stamps it on the cold-create and conflicts on a live box
116
+ * set up under a DIFFERENT rig version (solo holder recreates cold on the new
117
+ * rig; N-holders throw SandboxRigConflictError). Omitted for a rig-less session
118
+ * -> rig is never stamped or enforced (shares exactly as today).
119
+ */
120
+ rigVersionId?: string;
97
121
  };
98
122
 
99
123
  /** What resumeBoxForTurn returns: the live NON-OWNED session to inject, the
@@ -113,7 +137,10 @@ export type ResumedTurnSandbox = {
113
137
  export class SandboxWarmingTimeoutError extends Error {
114
138
  readonly code = "sandbox_warming_timeout";
115
139
 
116
- constructor(public readonly backend: string, public readonly timeoutMs: number) {
140
+ constructor(
141
+ public readonly backend: string,
142
+ public readonly timeoutMs: number,
143
+ ) {
117
144
  super(
118
145
  `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
119
146
  );
@@ -121,19 +148,124 @@ export class SandboxWarmingTimeoutError extends Error {
121
148
  }
122
149
  }
123
150
 
151
+ /** The exact attached caller that won warm->cold after proving the provider
152
+ * instance gone. It is a lease supersession (the same logical turn recovers),
153
+ * plus the lost id needed for one durable observability event. */
154
+ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
155
+ constructor(
156
+ sandboxGroupId: string,
157
+ leaseEpoch: number,
158
+ public readonly lostInstanceId: string,
159
+ ) {
160
+ super(sandboxGroupId, leaseEpoch);
161
+ this.name = "SandboxLeaseInstanceLostError";
162
+ }
163
+ }
164
+
124
165
  // Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
125
166
  // user-facing and separate from the lease TTL heartbeat/reaper horizon.
126
167
  const WARMING_POLL_INTERVAL_MS = 250;
168
+ const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
127
169
 
128
170
  async function sleep(ms: number): Promise<void> {
129
171
  await new Promise<void>((resolve) => setTimeout(resolve, ms));
130
172
  }
131
173
 
132
- async function terminateEstablishedSandbox(established: EstablishedSandboxSession | null): Promise<boolean> {
174
+ /**
175
+ * Modal may return a sandbox handle before its command router accepts the first
176
+ * exec. The upstream session's yieldTimeMs starts only after sandbox.exec()
177
+ * returns, so it cannot bound that initial RPC. Probe it before publishing the
178
+ * lease as warm and enforce our own wall-clock deadline.
179
+ */
180
+ export async function waitForSandboxExecReadiness(
181
+ established: EstablishedSandboxSession,
182
+ timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
183
+ ): Promise<void> {
184
+ if (established.backendId !== "modal") return;
185
+
186
+ const session = established.session as {
187
+ exec?: (args: {
188
+ cmd: string;
189
+ yieldTimeMs?: number;
190
+ maxOutputTokens?: number;
191
+ }) => Promise<unknown>;
192
+ execCommand?: (args: {
193
+ cmd: string;
194
+ yieldTimeMs?: number;
195
+ maxOutputTokens?: number;
196
+ }) => Promise<unknown>;
197
+ };
198
+ const run = session.exec ?? session.execCommand;
199
+ if (!run) {
200
+ throw new Error("Modal sandbox session does not expose exec");
201
+ }
202
+
203
+ let timer: ReturnType<typeof setTimeout> | undefined;
204
+ try {
205
+ await Promise.race([
206
+ run.call(session, {
207
+ cmd: "true",
208
+ yieldTimeMs: 1_000,
209
+ maxOutputTokens: 1_000,
210
+ }),
211
+ new Promise<never>((_, reject) => {
212
+ timer = setTimeout(
213
+ () => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
214
+ timeoutMs,
215
+ );
216
+ if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
217
+ }),
218
+ ]);
219
+ } finally {
220
+ if (timer) clearTimeout(timer);
221
+ }
222
+ }
223
+
224
+ class SnapshotTimeoutError extends Error {
225
+ constructor(timeoutMs: number) {
226
+ super(`workspace snapshot timed out after ${timeoutMs}ms`);
227
+ this.name = "SnapshotTimeoutError";
228
+ }
229
+ }
230
+
231
+ export async function waitForWarmSnapshot(
232
+ snapshot: Promise<unknown>,
233
+ timeoutMs: number,
234
+ ): Promise<boolean> {
235
+ let timeout: ReturnType<typeof setTimeout> | undefined;
236
+ try {
237
+ await Promise.race([
238
+ snapshot,
239
+ new Promise<never>((_, reject) => {
240
+ timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
241
+ if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
242
+ timeout.unref();
243
+ }
244
+ }),
245
+ ]);
246
+ return true;
247
+ } catch (error) {
248
+ if (error instanceof SnapshotTimeoutError) {
249
+ console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
250
+ return false;
251
+ }
252
+ return true;
253
+ } finally {
254
+ if (timeout) {
255
+ clearTimeout(timeout);
256
+ }
257
+ }
258
+ }
259
+
260
+ async function terminateEstablishedSandbox(
261
+ established: EstablishedSandboxSession | null,
262
+ ): Promise<boolean> {
133
263
  if (!established) {
134
264
  return true;
135
265
  }
136
- const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
266
+ const client = established.client as {
267
+ delete?: (state: unknown) => Promise<unknown>;
268
+ };
137
269
  if (typeof client.delete === "function" && established.sessionState !== undefined) {
138
270
  try {
139
271
  await client.delete(established.sessionState);
@@ -175,7 +307,10 @@ function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: numbe
175
307
  : error;
176
308
  }
177
309
 
178
- function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, error: unknown): void {
310
+ function recordSandboxWarmingTimeout(
311
+ metrics: RuntimeMetricsHooks | undefined,
312
+ error: unknown,
313
+ ): void {
179
314
  if (!(error instanceof SandboxWarmingTimeoutError)) {
180
315
  return;
181
316
  }
@@ -186,25 +321,42 @@ function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, e
186
321
  }
187
322
  }
188
323
 
189
- function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
190
- const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
191
- ? envelope.sessionState as Record<string, unknown>
192
- : null;
324
+ function workspaceArchiveFieldsFromEnvelope(
325
+ envelope: Record<string, unknown> | null | undefined,
326
+ ): Record<string, string> | null {
327
+ const sessionState =
328
+ envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
329
+ ? (envelope.sessionState as Record<string, unknown>)
330
+ : null;
193
331
  const archive = sessionState?.workspaceArchive;
194
- return typeof archive === "string" && archive.length > 0 ? archive : null;
332
+ if (typeof archive !== "string" || archive.length === 0) {
333
+ return null;
334
+ }
335
+ const previous = sessionState?.workspaceArchivePrev;
336
+ const capturedAt = sessionState?.workspaceArchiveAt;
337
+ return {
338
+ workspaceArchive: archive,
339
+ ...(typeof previous === "string" && previous.length > 0
340
+ ? { workspaceArchivePrev: previous }
341
+ : {}),
342
+ ...(typeof capturedAt === "string" && capturedAt.length > 0
343
+ ? { workspaceArchiveAt: capturedAt }
344
+ : {}),
345
+ };
195
346
  }
196
347
 
197
- function preserveWorkspaceArchiveOnInterimResumeState(
348
+ function preserveWorkspaceArchivesOnResumeState(
198
349
  resumeState: Record<string, unknown> | null,
199
350
  archiveSource: Record<string, unknown> | null,
200
351
  ): Record<string, unknown> | null {
201
- const archive = workspaceArchiveFromEnvelope(archiveSource);
202
- if (!archive) {
352
+ const archiveFields = workspaceArchiveFieldsFromEnvelope(archiveSource);
353
+ if (!archiveFields) {
203
354
  return resumeState;
204
355
  }
205
- const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
206
- ? resumeState.sessionState as Record<string, unknown>
207
- : {};
356
+ const existingSessionState =
357
+ resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
358
+ ? (resumeState.sessionState as Record<string, unknown>)
359
+ : {};
208
360
  return {
209
361
  ...(resumeState ?? {}),
210
362
  ...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
@@ -212,24 +364,134 @@ function preserveWorkspaceArchiveOnInterimResumeState(
212
364
  : {}),
213
365
  sessionState: {
214
366
  ...existingSessionState,
215
- workspaceArchive: archive,
367
+ ...archiveFields,
216
368
  },
217
369
  };
218
370
  }
219
371
 
372
+ /**
373
+ * MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
374
+ * drain-persist only protects boxes the reaper itself kills; anything else —
375
+ * Modal's hard creation-time timeout catching a session busy past it, provider
376
+ * OOM/infra death — loses everything since the last clean drain (staging
377
+ * session e644e8a8, 2026-07-06: mid-turn box termination cost an unpushed
378
+ * branch + 2 commits). While a turn HOLDS the live box, this folds a fresh
379
+ * snapshot onto the lease through persistWarmSnapshot (the warm sibling of the
380
+ * drain seam: epoch-fenced CAS + atomic throttle re-check) and GCs the
381
+ * superseded snapshot image, bounding worst-case loss of ANY unclean box death
382
+ * to sandboxSnapshotIntervalMs.
383
+ *
384
+ * Never throws and never blocks turn progress semantics: every failure path
385
+ * returns false (the snapshot is protection, not a turn dependency). No-ops
386
+ * when the interval is 0, the backend has no persistWorkspace (selfhosted =
387
+ * the user's machine IS the persistence), or a snapshot newer than the
388
+ * interval already rides the lease.
389
+ */
390
+ export async function maybePersistWarmWorkspaceSnapshot(
391
+ services: SandboxResumeServices,
392
+ ids: { accountId: string; workspaceId: string; sandboxGroupId: string },
393
+ session: unknown,
394
+ leaseEpoch: number,
395
+ ): Promise<boolean> {
396
+ const { db, settings } = services;
397
+ const intervalMs = settings.sandboxSnapshotIntervalMs;
398
+ if (intervalMs <= 0) {
399
+ return false;
400
+ }
401
+ const persistable = session as {
402
+ persistWorkspace?: () => Promise<Uint8Array | undefined>;
403
+ };
404
+ if (typeof persistable.persistWorkspace !== "function") {
405
+ return false;
406
+ }
407
+ try {
408
+ // Cheap throttle pre-check before the (potentially slow) capture;
409
+ // persistWarmSnapshot re-checks atomically under the row lock, so this is
410
+ // purely a cost optimization, not the correctness guard.
411
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
412
+ if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
413
+ return false;
414
+ }
415
+ const sessionState =
416
+ lease.resumeState && typeof lease.resumeState === "object"
417
+ ? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
418
+ : undefined;
419
+ const priorAtRaw =
420
+ sessionState && typeof sessionState === "object"
421
+ ? sessionState.workspaceArchiveAt
422
+ : undefined;
423
+ const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
424
+ if (Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
425
+ return false;
426
+ }
427
+ let timeout: ReturnType<typeof setTimeout> | undefined;
428
+ // Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
429
+ // by capture-initiation, not land time, so a slower heartbeat capture that
430
+ // started earlier can never overwrite a fresher turn-end capture that landed
431
+ // first (the bounded-wait race Bugbot flagged).
432
+ const capturedAtMs = Date.now();
433
+ // On timeout the race settles on `undefined` while persistWorkspace() keeps
434
+ // running orphaned. Swallow its LATE settlement: an un-awaited rejection
435
+ // after the race resolved would surface as an unhandledRejection and can
436
+ // take down the whole worker process — which would defeat the very hang-
437
+ // guard this timeout exists to provide.
438
+ const capture = persistable.persistWorkspace();
439
+ capture.catch(() => undefined);
440
+ const bytes = await Promise.race([
441
+ capture,
442
+ new Promise<undefined>((resolve) => {
443
+ timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
444
+ if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
445
+ timeout.unref();
446
+ }
447
+ }),
448
+ ]).finally(() => {
449
+ if (timeout) {
450
+ clearTimeout(timeout);
451
+ }
452
+ });
453
+ if (!bytes || bytes.length === 0) {
454
+ return false;
455
+ }
456
+ const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
457
+ accountId: ids.accountId,
458
+ workspaceId: ids.workspaceId,
459
+ sandboxGroupId: ids.sandboxGroupId,
460
+ expectedEpoch: leaseEpoch,
461
+ workspaceArchive: Buffer.from(bytes).toString("base64"),
462
+ minIntervalMs: intervalMs,
463
+ capturedAtMs,
464
+ });
465
+ if (!wrote) {
466
+ return false;
467
+ }
468
+ // Warm snapshots retain a 2-deep restore window. Only the two-ago archive
469
+ // returned by persistWarmSnapshot is GC-eligible.
470
+ await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
471
+ return true;
472
+ } catch (error) {
473
+ // Protection, not a dependency: a failed snapshot must never fail (or slow
474
+ // down retrying) the turn. The next heartbeat/turn-end tick retries.
475
+ console.error("mid-session workspace snapshot failed (turn unaffected)", error);
476
+ return false;
477
+ }
478
+ }
479
+
220
480
  /**
221
481
  * Resume the one box for a single turn (or any worker-side resume op). Returns
222
482
  * the live non-owned session + the fence epoch + a release fn. The CALLER owns
223
483
  * the lifecycle: inject non-owned, run, then `await release()` and drop the
224
484
  * handle in `finally`.
225
485
  *
226
- * holderId is the unique-per-execution id (the Temporal activityId for a turn).
486
+ * holderId is the globally unique durable turn-attempt id. It must not be a
487
+ * Temporal activity id, because activity ids are only workflow-local and
488
+ * collide when sibling sessions share one sandbox group.
227
489
  */
228
490
  export async function resumeBoxForTurn(
229
491
  services: SandboxResumeServices,
230
492
  ids: ResumeBoxIds,
231
- kind: LeaseHolderKind,
232
- holderId: string,
493
+ kind: "turn",
494
+ holderId: TurnSandboxLeaseHolderId,
233
495
  ): Promise<ResumedTurnSandbox> {
234
496
  const { db, settings } = services;
235
497
  const os = ids.os ?? "linux";
@@ -238,11 +500,15 @@ export async function resumeBoxForTurn(
238
500
  // The release closure is created eagerly so the caller can always release in
239
501
  // finally, even if establish/commit throws after the holder was registered.
240
502
  let released = false;
503
+ let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
241
504
  const release = async (): Promise<void> => {
242
505
  if (released) {
243
506
  return;
244
507
  }
245
508
  released = true;
509
+ if (holderLivenessTimer) {
510
+ clearInterval(holderLivenessTimer);
511
+ }
246
512
  await releaseLeaseHolder(db, {
247
513
  accountId: ids.accountId,
248
514
  workspaceId: ids.workspaceId,
@@ -267,9 +533,37 @@ export async function resumeBoxForTurn(
267
533
  // propagates to the turn activity (an actionable error); a solo image change is handled
268
534
  // by acquireLease recreating the box cold on the new image.
269
535
  ...(ids.image ? { image: ids.image } : {}),
536
+ // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
537
+ // + conflicts on a live box under a different rig. A SandboxRigConflictError
538
+ // propagates to the turn activity (actionable); a solo rig change recreates cold.
539
+ ...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
270
540
  leaseTtlMs,
541
+ warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
271
542
  });
272
543
 
544
+ // HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
545
+ // registered until release. The dead-worker turn-holder reap judges liveness
546
+ // by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
547
+ // agent-turn) only starts AFTER this function returns — while waitForWarm and
548
+ // establish/cold-restore can legitimately run for many minutes in here,
549
+ // COMPOUNDING past any fixed reap horizon. With this
550
+ // loop no live holder is ever silent for more than one tick, so the reap
551
+ // horizon is pure defense-in-depth, not a tuned guess about path lengths.
552
+ // Epoch-free by design (touch only refreshes our own row's timestamp);
553
+ // best-effort (a transient DB error must never fail the resume).
554
+ holderLivenessTimer = setInterval(() => {
555
+ void touchLeaseHolder(db, {
556
+ accountId: ids.accountId,
557
+ workspaceId: ids.workspaceId,
558
+ sandboxGroupId: ids.sandboxGroupId,
559
+ kind,
560
+ holderId,
561
+ }).catch(() => undefined);
562
+ }, 10_000);
563
+ if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
564
+ holderLivenessTimer.unref();
565
+ }
566
+
273
567
  // FENCED: a newer epoch exists (a later turn re-established the box). Back off;
274
568
  // NEVER create(). Release our (just-registered) holder so we don't pin a stale
275
569
  // lease, then surface the supersession.
@@ -279,17 +573,10 @@ export async function resumeBoxForTurn(
279
573
  }
280
574
 
281
575
  // SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
282
- // expose the stream port, then commit warm (lease_epoch++).
576
+ // then commit warm (lease_epoch++). Optional desktop setup is lazy.
283
577
  if (acquired.role === "spawner") {
284
578
  const expectedEpoch = acquired.lease.leaseEpoch;
285
579
  let createdEstablished: EstablishedSandboxSession | null = null;
286
- if (ids.environment && ids.backend === "modal") {
287
- Object.assign(ids.environment, modalSandboxAttributionEnvironment({
288
- leaseId: acquired.lease.id,
289
- workspaceId: ids.workspaceId,
290
- sandboxGroupId: ids.sandboxGroupId,
291
- }));
292
- }
293
580
  try {
294
581
  const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
295
582
  // Prefer the COLD lease's preserved resume_state when it carries a persisted
@@ -304,12 +591,13 @@ export async function resumeBoxForTurn(
304
591
  const spawnEnvelope = acquired.lease.resumeState ?? envelope;
305
592
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
306
593
  sessionId: ids.sessionId,
594
+ recovery: "create-or-restore",
307
595
  backendOverride: ids.backend as never,
308
596
  ...(ids.environment ? { environment: ids.environment } : {}),
309
597
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
310
598
  onSandboxCreated: async (created) => {
311
599
  createdEstablished = created;
312
- const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
600
+ const resumeEnvelope = preserveWorkspaceArchivesOnResumeState(
313
601
  (await serializeEstablishedSandboxEnvelope(created)) ?? null,
314
602
  spawnEnvelope,
315
603
  );
@@ -322,6 +610,9 @@ export async function resumeBoxForTurn(
322
610
  resumeBackendId: created.backendId,
323
611
  resumeState: resumeEnvelope,
324
612
  leaseTtlMs,
613
+ // Keep the warming budget after create(): manifest setup and
614
+ // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
615
+ warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
325
616
  });
326
617
  if (!recorded.recorded) {
327
618
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
@@ -336,8 +627,11 @@ export async function resumeBoxForTurn(
336
627
  },
337
628
  });
338
629
  createdEstablished = established;
339
- await ensureDisplayStack(settings, established);
340
- const endpoint = await exposeStreamPort(settings, established);
630
+ // A sandbox handle is not sufficient evidence that Modal's command router
631
+ // is live. Do not publish a warm lease until one bounded no-op exec works.
632
+ // On timeout the catch below terminates the box and rolls warming -> cold,
633
+ // so the next turn cold-creates instead of hanging forever on first use.
634
+ await waitForSandboxExecReadiness(established);
341
635
  // Fold the LIVE box into a re-resumable envelope and persist it as the
342
636
  // lease's resume_state — exactly like the API-direct paths (channel-a.ts /
343
637
  // viewer.ts). Without this the turn committed the ORIGINAL session manifest
@@ -345,14 +639,29 @@ export async function resumeBoxForTurn(
345
639
  // terminal, the desktop viewer, the reaper) cold-restored a FRESH rival box
346
640
  // and never saw the turn's live box. Fall back to the session envelope only
347
641
  // when the client cannot serialize live state.
348
- const resumeEnvelope = (await serializeEstablishedSandboxEnvelope(established)) ?? envelope ?? null;
642
+ const serializedResumeEnvelope =
643
+ (await serializeEstablishedSandboxEnvelope(established)) ?? envelope;
644
+ // A successful cold hydrate has already proved this archive usable and the
645
+ // replacement box now contains its files. Keep the current + fallback
646
+ // archive pointers on the committed live envelope until a later warm
647
+ // snapshot replaces them. Without this merge, serialization publishes only
648
+ // the new provider id; a second provider loss before the snapshot cadence
649
+ // fires would retire the lease with no archive and recreate an empty box.
650
+ // A failed hydrate falls back to a clean box with origin="created", so its
651
+ // unusable archive is deliberately cleared by the unmerged serialized state.
652
+ const resumeEnvelope =
653
+ established.origin === "restored"
654
+ ? preserveWorkspaceArchivesOnResumeState(serializedResumeEnvelope, spawnEnvelope)
655
+ : serializedResumeEnvelope;
349
656
  const committed = await commitWarmingToWarm(db, {
350
657
  accountId: ids.accountId,
351
658
  workspaceId: ids.workspaceId,
352
659
  sandboxGroupId: ids.sandboxGroupId,
353
660
  expectedEpoch,
354
661
  instanceId: established.instanceId,
355
- dataPlaneUrl: endpoint?.url ?? null,
662
+ // Viewer attach resolves and records the tunnel URL lazily. A headless
663
+ // turn must not touch the optional desktop data plane.
664
+ dataPlaneUrl: null,
356
665
  resumeBackendId: established.backendId,
357
666
  resumeState: resumeEnvelope,
358
667
  leaseTtlMs,
@@ -388,7 +697,11 @@ export async function resumeBoxForTurn(
388
697
  });
389
698
  }
390
699
  await release();
391
- const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
700
+ const warmingError = asSandboxWarmingError(
701
+ error,
702
+ ids.backend,
703
+ settings.sandboxWarmingTimeoutMs,
704
+ );
392
705
  recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
393
706
  throw warmingError;
394
707
  }
@@ -397,8 +710,8 @@ export async function resumeBoxForTurn(
397
710
  // ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
398
711
  // BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
399
712
  // first wait for the spawner to commit warm, bounded by the explicit warming
400
- // budget. A cold reset means the spawner died; requeue the turn instead of
401
- // trying to enter the spawner create path from the attached branch.
713
+ // budget. A cold reset means the spawner died; recover the same turn instead
714
+ // of trying to enter the spawner create path from the attached branch.
402
715
  try {
403
716
  let leaseEpoch = acquired.lease.leaseEpoch;
404
717
  if (acquired.lease.liveness === "warming") {
@@ -410,28 +723,62 @@ export async function resumeBoxForTurn(
410
723
  // manifest into a rival. Fall back to the session envelope only when the
411
724
  // lease carries no resume_state (matches channel-a.ts's attached branch).
412
725
  const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
413
- const envelope =
414
- live?.resumeState ?? (await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId));
415
- const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
416
- sessionId: ids.sessionId,
417
- backendOverride: ids.backend as never,
418
- ...(ids.environment ? { environment: ids.environment } : {}),
419
- ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
420
- });
421
- // Re-ensure the desktop display stack on the ATTACHED/REARMED path too — NOT
422
- // just the spawner path. A turn attaching to a warm box whose :0 was never
423
- // brought up (the box was first warmed by a Channel-A op, or a snapshot
424
- // rollover dropped the X stack) would otherwise drive computer-use against a
425
- // dead display: scrot yields an empty PNG, the SDK builds `image_url: ''`,
426
- // and the model rejects the turn with "400 Invalid input[N].output.image_url".
427
- // ensureDisplayStack is idempotent + flock-guarded (cheap no-op when up) and
428
- // NO-OPs when the desktop tier is off or the backend is headless-only, so the
429
- // headless turn path stays byte-for-byte unchanged.
430
- await ensureDisplayStack(settings, established);
726
+ if (
727
+ !live ||
728
+ live.liveness !== "warm" ||
729
+ live.leaseEpoch !== leaseEpoch ||
730
+ live.instanceId === null
731
+ ) {
732
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
733
+ }
734
+ let established: EstablishedSandboxSession;
735
+ try {
736
+ established = await establishSandboxSessionFromEnvelope(settings, live.resumeState, {
737
+ sessionId: ids.sessionId,
738
+ recovery: "resume-only",
739
+ backendOverride: ids.backend as never,
740
+ ...(ids.environment ? { environment: ids.environment } : {}),
741
+ ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
742
+ });
743
+ } catch (error) {
744
+ if (
745
+ !(error instanceof SandboxResumeStateUnavailableError) &&
746
+ !isProviderSandboxNotFoundError(ids.backend, error)
747
+ ) {
748
+ throw error;
749
+ }
750
+ const marked = await markWarmLeaseInstanceLost(db, {
751
+ accountId: ids.accountId,
752
+ workspaceId: ids.workspaceId,
753
+ sandboxGroupId: ids.sandboxGroupId,
754
+ expectedEpoch: leaseEpoch,
755
+ expectedInstanceId: live.instanceId,
756
+ });
757
+ if (marked.status === "marked") {
758
+ await services.onSandboxLost?.({
759
+ sandboxGroupId: ids.sandboxGroupId,
760
+ instanceId: live.instanceId,
761
+ leaseEpoch: marked.lease.leaseEpoch,
762
+ });
763
+ throw new SandboxLeaseInstanceLostError(
764
+ ids.sandboxGroupId,
765
+ marked.lease.leaseEpoch,
766
+ live.instanceId,
767
+ );
768
+ }
769
+ throw new SandboxLeaseSupersededError(
770
+ ids.sandboxGroupId,
771
+ marked.lease?.leaseEpoch ?? leaseEpoch,
772
+ );
773
+ }
431
774
  return { established, leaseEpoch, release };
432
775
  } catch (error) {
433
776
  await release();
434
- const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
777
+ const warmingError = asSandboxWarmingError(
778
+ error,
779
+ ids.backend,
780
+ settings.sandboxWarmingTimeoutMs,
781
+ );
435
782
  recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
436
783
  throw warmingError;
437
784
  }
@@ -451,13 +798,20 @@ export async function resumeBoxForTurn(
451
798
  * epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
452
799
  * identical to the cloud path.
453
800
  *
454
- * holderId is the unique-per-execution id (the Temporal activityId for a turn).
801
+ * holderId is the globally unique durable turn-attempt id. It must not be a
802
+ * Temporal activity id, because activity ids are only workflow-local and
803
+ * collide when sibling sessions share one sandbox group.
455
804
  */
456
805
  export async function acquireSelfhostedLeaseForTurn(
457
806
  services: SandboxResumeServices,
458
- ids: { accountId: string; workspaceId: string; sandboxGroupId: string; sessionId: string },
459
- kind: LeaseHolderKind,
460
- holderId: string,
807
+ ids: {
808
+ accountId: string;
809
+ workspaceId: string;
810
+ sandboxGroupId: string;
811
+ sessionId: string;
812
+ },
813
+ kind: "turn",
814
+ holderId: TurnSandboxLeaseHolderId,
461
815
  ): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
462
816
  const { db, settings } = services;
463
817
  const leaseTtlMs = settings.sandboxLeaseTtlMs;
@@ -488,6 +842,7 @@ export async function acquireSelfhostedLeaseForTurn(
488
842
  backend: "selfhosted",
489
843
  os: "linux",
490
844
  leaseTtlMs,
845
+ warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
491
846
  });
492
847
 
493
848
  // FENCED: a newer epoch re-established the group concurrently. Release our
@@ -569,128 +924,3 @@ async function waitForWarm(
569
924
  }
570
925
  throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
571
926
  }
572
-
573
- // ============================================================================
574
- // Channel-B display-stack launch (P4.1 — ensureDisplayStack is now real).
575
- //
576
- // Idempotent + callable by ANY worker on the resumed handle (and by the API on a
577
- // viewer op). When the desktop tier is OFF (sandboxDesktopEnabled=false) or the
578
- // backend is headless-only, this is a NO-OP — the HEADLESS ROLLOVER branch (I5),
579
- // so the headless turn path is byte-for-byte unchanged. When the tier is ON for
580
- // a desktop-capable backend, it execs the canonical opengeni-desktop-up under an
581
- // in-box flock (re-establishes after a rollover; safe to call N times).
582
- // `exposeStreamPort` real body remains a P4.2 concern.
583
- // ============================================================================
584
-
585
- /**
586
- * Ensure the desktop display stack (Xvfb -> XFCE -> x11vnc ->
587
- * websockify:6080 -> noVNC) is up on the live box. Idempotent: the lock-free
588
- * pre-check + the in-box flock + the up-script's per-stage PID guards make a
589
- * second call a cheap no-op. NO-OP when the desktop tier is disabled or the
590
- * backend cannot serve a desktop (degradation is a value: a headless-only session
591
- * simply skips the stack). Delegates to the agent-loop-free leaf
592
- * (@opengeni/runtime/sandbox display-stack).
593
- *
594
- * BEST-EFFORT — NEVER fails the turn. The display stack powers the OPTIONAL
595
- * Channel-B desktop / computer-use surface; it is NOT load-bearing for the
596
- * agent's work. A `DisplayStackError` (a real stage failure, OR — the regression
597
- * this guards — a timeout-derived exit -1 when a viewer attach already holds /
598
- * contends the up-script's flock and the turn's ensure waits ~45s and times out)
599
- * is CAUGHT + logged and swallowed, so a slow/contended/failed stack degrades to
600
- * Channel-A-only rather than killing the turn. The fast lock-free pre-check
601
- * upstream means the already-up case resolves in milliseconds and never reaches
602
- * this catch in the first place. `DisplayStackUnsupportedError` (a box that can't
603
- * run commands at all) is likewise swallowed.
604
- */
605
- export async function ensureDisplayStack(
606
- settings: Settings,
607
- established: EstablishedSandboxSession,
608
- ): Promise<void> {
609
- // Headless rollover branch (I5): the tier is off OR the backend is
610
- // headless-only -> no display stack. Behavior-preserving for the headless path.
611
- // This is ALSO the "is the desktop relevant to this turn?" gate: we only touch
612
- // the box when the desktop tier is enabled for a desktop-capable backend.
613
- if (!settings.sandboxDesktopEnabled) {
614
- return;
615
- }
616
- if (!desktopCapableBackend(established.backendId)) {
617
- return;
618
- }
619
- try {
620
- await ensureDisplayStackOnBox(established.session);
621
- } catch (error) {
622
- // The desktop is a value-add, not load-bearing for the agent's work, so NO
623
- // display-stack failure may fail the turn:
624
- // - DisplayStackUnsupportedError: the box genuinely can't run commands ->
625
- // Channel-A-only.
626
- // - DisplayStackError: a stage failure OR a contended-lock timeout (exit -1)
627
- // after a viewer attach already brought the stack up / is mid-launch. The
628
- // pre-check makes the already-up case fast; if we still time out or a stage
629
- // really failed, degrade — don't die.
630
- if (error instanceof DisplayStackUnsupportedError || error instanceof DisplayStackError) {
631
- console.warn(
632
- `[sandbox-resume] ensureDisplayStack degraded to Channel-A-only (turn continues): ${
633
- error instanceof Error ? error.message : String(error)
634
- }`,
635
- );
636
- return;
637
- }
638
- // An unexpected non-display error (e.g. the session blew up entirely) still
639
- // propagates — that is not a desktop-surface degradation.
640
- throw error;
641
- }
642
- }
643
-
644
- // The structural slice of a provider session we need to resolve the tunnel.
645
- type PortResolvableSession = {
646
- resolveExposedPort?: (port: number) => Promise<ExposedPortEndpoint>;
647
- };
648
-
649
- /**
650
- * Resolve the desktop tunnel URL (resolveExposedPort(6080)) and assemble the
651
- * direct-to-provider WS URL, recorded on the lease as `data_plane_url` at
652
- * commit. The per-viewer scoped token is minted at viewer-attach time (the
653
- * handshake), NOT here — the spawner records only the box-scoped tunnel URL so
654
- * the lease carries a fresh value across a rollover. P4.2 productionizes the
655
- * P4.1 stub.
656
- *
657
- * Returns null (degradation is a value, NEVER a throw):
658
- * - the desktop tier is off (sandboxDesktopEnabled=false), or
659
- * - the backend is headless-only (no desktop), or
660
- * - the provider session cannot resolve the port (no resolveExposedPort), or
661
- * - the tunnel lookup transiently failed.
662
- * In every null case the lease's data_plane_url stays null and the next
663
- * API-direct viewer op re-mints it; the turn never fails on a desktop hiccup.
664
- */
665
- export async function exposeStreamPort(
666
- settings: Settings,
667
- established: EstablishedSandboxSession,
668
- ): Promise<{ url: string; expiresAt: Date } | null> {
669
- // Headless rollover branch (I5): tier !== desktop -> no stream port to expose.
670
- if (!settings.sandboxDesktopEnabled) {
671
- return null;
672
- }
673
- if (!desktopCapableBackend(established.backendId)) {
674
- return null;
675
- }
676
- const session = established.session as PortResolvableSession;
677
- if (typeof session?.resolveExposedPort !== "function") {
678
- return null;
679
- }
680
- try {
681
- const endpoint = await session.resolveExposedPort(DESKTOP_STREAM_PORT);
682
- const url = buildStreamUrl(endpoint);
683
- // The provider tunnel URL is box-lifetime-valid (Modal raw TLS) or
684
- // provider-TTL-signed (Daytona/Blaxel); the per-viewer token's TTL bounds the
685
- // viewer's freshness. We record only the URL on the lease; expiresAt here is a
686
- // soft hint (the box's idle horizon), not a hard token expiry.
687
- return { url, expiresAt: new Date(Date.now() + settings.sandboxLeaseTtlMs) };
688
- } catch (error) {
689
- // Degradation is a value: a transient resolve failure leaves data_plane_url
690
- // null and the next viewer op re-mints. Never fail the turn on a desktop op.
691
- if (error instanceof StreamPortUnavailableError) {
692
- return null;
693
- }
694
- return null;
695
- }
696
- }