@opengeni/worker-bundle 0.16.29 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +110 -19
  3. package/dist/activities/api-integrations.d.ts +21 -0
  4. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  5. package/dist/activities/capabilities.d.ts +6 -0
  6. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  7. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  8. package/dist/activities/codex-capacity.d.ts +6 -5
  9. package/dist/activities/codex-image-generation.d.ts +21 -0
  10. package/dist/activities/context-compaction.d.ts +1 -0
  11. package/dist/activities/documents.d.ts +3 -2
  12. package/dist/activities/environment.d.ts +17 -8
  13. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  14. package/dist/activities/file-upload-reaper.d.ts +7 -2
  15. package/dist/activities/gateway-image-generation.d.ts +39 -0
  16. package/dist/activities/gateway-video-generation.d.ts +53 -0
  17. package/dist/activities/generated-images.d.ts +112 -0
  18. package/dist/activities/git-credential-renewal.d.ts +1 -1
  19. package/dist/activities/goals.d.ts +2 -2
  20. package/dist/activities/google-drive-changes.d.ts +99 -0
  21. package/dist/activities/image-generation-operation.d.ts +74 -0
  22. package/dist/activities/image-generation-references.d.ts +25 -0
  23. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  24. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  25. package/dist/activities/packs.d.ts +21 -36
  26. package/dist/activities/parent-wake.d.ts +1 -1
  27. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  28. package/dist/activities/retained-screenshots.d.ts +36 -0
  29. package/dist/activities/rig-verification.d.ts +82 -5
  30. package/dist/activities/run-credential-renewal.d.ts +1 -1
  31. package/dist/activities/run-input.d.ts +3 -1
  32. package/dist/activities/sandbox-images.d.ts +14 -0
  33. package/dist/activities/sandbox-lease.d.ts +47 -9
  34. package/dist/activities/scheduled-tasks.d.ts +2 -2
  35. package/dist/activities/session-state.d.ts +7 -5
  36. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  37. package/dist/activities/streaming.d.ts +1 -1
  38. package/dist/activities/types.d.ts +87 -5
  39. package/dist/activities/video-generation-admission.d.ts +43 -0
  40. package/dist/activities/video-generation-credential.d.ts +24 -0
  41. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  42. package/dist/activities/video-media-validation.d.ts +34 -0
  43. package/dist/activities/video-output-retention.d.ts +36 -0
  44. package/dist/activities/video-reference-staging.d.ts +55 -0
  45. package/dist/activities/workflow-wake.d.ts +2 -2
  46. package/dist/activities/workspace-capture.d.ts +10 -1
  47. package/dist/activities/xai-auth.d.ts +23 -0
  48. package/dist/activities/xai-image-generation.d.ts +26 -0
  49. package/dist/activities/xai-video-generation.d.ts +18 -0
  50. package/dist/activities-control-XURE4VKN.js +7221 -0
  51. package/dist/activities-control-XURE4VKN.js.map +1 -0
  52. package/dist/activities-control.d.ts +297 -0
  53. package/dist/activities-turn-6EBXGGLS.js +16036 -0
  54. package/dist/activities-turn-6EBXGGLS.js.map +1 -0
  55. package/dist/activities-turn.d.ts +17 -0
  56. package/dist/activities.d.ts +138 -150
  57. package/dist/activity-services.d.ts +10 -0
  58. package/dist/artifact-materializer-entry.d.ts +6 -0
  59. package/dist/artifact-materializer-entry.js +2662 -0
  60. package/dist/artifact-materializer-entry.js.map +1 -0
  61. package/dist/artifact-outbox-entry.d.ts +2 -0
  62. package/dist/artifact-outbox-entry.js +681 -0
  63. package/dist/artifact-outbox-entry.js.map +1 -0
  64. package/dist/chunk-5GRJIP5H.js +3092 -0
  65. package/dist/chunk-5GRJIP5H.js.map +1 -0
  66. package/dist/chunk-GRFQTTSQ.js +448 -0
  67. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  68. package/dist/chunk-TYNPCG4U.js +906 -0
  69. package/dist/chunk-TYNPCG4U.js.map +1 -0
  70. package/dist/concurrency.d.ts +108 -5
  71. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  72. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  73. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  74. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  75. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  76. package/dist/editable-artifact-materializer.d.ts +262 -0
  77. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  78. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  79. package/dist/host-export.d.ts +1 -1
  80. package/dist/index.d.ts +189 -10
  81. package/dist/index.js +715 -17380
  82. package/dist/index.js.map +1 -1
  83. package/dist/memory-pressure-guard.d.ts +57 -0
  84. package/dist/model-checkpoint-memory-collector.d.ts +23 -0
  85. package/dist/model-context-contributions.d.ts +40 -0
  86. package/dist/observability-metrics.d.ts +53 -1
  87. package/dist/sandbox-reaper-contract.d.ts +40 -0
  88. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  89. package/dist/sandbox-resume.d.ts +34 -7
  90. package/dist/sandbox-routing.d.ts +6 -0
  91. package/dist/worker-service-lifecycle.d.ts +6 -2
  92. package/dist/workflow-bundle.js +1094 -370
  93. package/dist/workflows/activities.d.ts +8 -2
  94. package/dist/workflows/document-index.d.ts +1 -1
  95. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  96. package/dist/workflows/rig-verification.d.ts +1 -1
  97. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  98. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  99. package/dist/workflows/session.d.ts +2 -2
  100. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  101. package/dist/workflows/video-generation.d.ts +10 -0
  102. package/dist/workflows/workflow-wake.d.ts +1 -1
  103. package/dist/workflows.d.ts +10 -7
  104. package/package.json +35 -15
  105. package/src/activities/agent-run-admission.ts +76 -0
  106. package/src/activities/agent-turn.ts +2600 -340
  107. package/src/activities/api-integrations.ts +138 -0
  108. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  109. package/src/activities/capabilities.ts +10 -0
  110. package/src/activities/codemode-dispatcher.ts +335 -0
  111. package/src/activities/codemode-token-renewal.ts +64 -0
  112. package/src/activities/codex-capacity.ts +54 -7
  113. package/src/activities/codex-image-generation.ts +56 -0
  114. package/src/activities/context-compaction.ts +38 -3
  115. package/src/activities/documents.ts +14 -4
  116. package/src/activities/environment.ts +54 -45
  117. package/src/activities/expiring-material-renewal.ts +1 -1
  118. package/src/activities/file-upload-reaper.ts +34 -3
  119. package/src/activities/gateway-image-generation.ts +146 -0
  120. package/src/activities/gateway-video-generation.ts +298 -0
  121. package/src/activities/generated-images.ts +860 -0
  122. package/src/activities/goals.ts +8 -3
  123. package/src/activities/google-drive-changes.ts +454 -0
  124. package/src/activities/image-generation-operation.ts +333 -0
  125. package/src/activities/image-generation-references.ts +118 -0
  126. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  127. package/src/activities/knowledge-source-sync.ts +2840 -0
  128. package/src/activities/mcp-credentials.ts +1 -0
  129. package/src/activities/packs.ts +75 -152
  130. package/src/activities/parent-wake.ts +13 -1
  131. package/src/activities/retained-screenshot-reaper.ts +2 -2
  132. package/src/activities/retained-screenshots.ts +453 -28
  133. package/src/activities/rig-verification.ts +554 -43
  134. package/src/activities/run-input.ts +22 -6
  135. package/src/activities/sandbox-images.ts +167 -0
  136. package/src/activities/sandbox-lease.ts +1065 -483
  137. package/src/activities/scheduled-tasks.ts +104 -74
  138. package/src/activities/session-state.ts +22 -4
  139. package/src/activities/site-auth-maintenance.ts +269 -0
  140. package/src/activities/streaming.ts +42 -3
  141. package/src/activities/types.ts +104 -5
  142. package/src/activities/video-generation-admission.ts +303 -0
  143. package/src/activities/video-generation-credential.ts +112 -0
  144. package/src/activities/video-generation-reconciliation.ts +771 -0
  145. package/src/activities/video-media-validation.ts +332 -0
  146. package/src/activities/video-output-retention.ts +314 -0
  147. package/src/activities/video-reference-staging.ts +350 -0
  148. package/src/activities/workflow-wake.ts +2 -2
  149. package/src/activities/workspace-capture.ts +44 -10
  150. package/src/activities/xai-auth.ts +122 -0
  151. package/src/activities/xai-image-generation.ts +86 -0
  152. package/src/activities/xai-video-generation.ts +69 -0
  153. package/src/activities-control.ts +58 -0
  154. package/src/activities-turn.ts +40 -0
  155. package/src/activities.ts +53 -124
  156. package/src/activity-services.ts +62 -0
  157. package/src/artifact-materializer-entry.ts +35 -0
  158. package/src/artifact-outbox-entry.ts +29 -0
  159. package/src/concurrency.ts +484 -6
  160. package/src/editable-artifact-hint-broker.ts +235 -0
  161. package/src/editable-artifact-materializer-launcher.ts +515 -0
  162. package/src/editable-artifact-materializer-service.ts +787 -0
  163. package/src/editable-artifact-materializer-storage.ts +413 -0
  164. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  165. package/src/editable-artifact-materializer.ts +1289 -0
  166. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  167. package/src/editable-artifact-outbox-service.ts +438 -0
  168. package/src/index.ts +328 -72
  169. package/src/memory-pressure-guard.ts +489 -0
  170. package/src/model-checkpoint-memory-collector.ts +123 -0
  171. package/src/model-context-contributions.ts +159 -0
  172. package/src/observability-metrics.ts +297 -13
  173. package/src/sandbox-reaper-contract.ts +73 -0
  174. package/src/sandbox-reaper-timeout.ts +79 -0
  175. package/src/sandbox-resume.ts +294 -153
  176. package/src/sandbox-routing.ts +32 -24
  177. package/src/worker-service-lifecycle.ts +80 -14
  178. package/src/workflows/activities.ts +31 -0
  179. package/src/workflows/file-upload-reaper.ts +9 -1
  180. package/src/workflows/knowledge-source-sync.ts +83 -0
  181. package/src/workflows/sandbox-reaper.ts +183 -21
  182. package/src/workflows/scheduled-tasks.ts +42 -6
  183. package/src/workflows/session.ts +20 -6
  184. package/src/workflows/site-auth-maintenance.ts +14 -0
  185. package/src/workflows/video-generation.ts +36 -0
  186. package/src/workflows.ts +18 -1
  187. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  188. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -20,7 +20,11 @@
20
20
  //
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
- import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
23
+ import {
24
+ sandboxArchiveCaptureTimeoutMs,
25
+ sandboxLifecycleTransitionWaitMs,
26
+ type Settings,
27
+ } from "@opengeni/config";
24
28
  import { randomUUID } from "node:crypto";
25
29
  import {
26
30
  acquireLease,
@@ -31,6 +35,7 @@ import {
31
35
  failSandboxRematerialization,
32
36
  failWarmingToCold,
33
37
  getSandboxSessionEnvelope,
38
+ heartbeatLeaseHolder,
34
39
  markSandboxRestoreVerifying,
35
40
  markWarmLeaseInstanceLost,
36
41
  markSandboxCheckpointArtifactDeletePending,
@@ -43,22 +48,29 @@ import {
43
48
  touchLeaseHolder,
44
49
  SandboxLeaseRecoveryBlockedError,
45
50
  SandboxLeaseSupersededError,
51
+ SandboxLeaseTransitionError,
46
52
  type Database,
47
53
  type LeaseHolderKind,
48
54
  } from "@opengeni/db";
49
55
  import {
50
56
  captureVerifiedWorkspaceArchive,
51
57
  describeLegacyNativeSnapshotArchive,
58
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
59
+ SandboxExecReadinessError,
52
60
  WorkspaceArchiveIntegrityError,
53
61
  establishSandboxSessionFromEnvelope,
54
62
  isProviderSandboxNotFoundError,
55
63
  parseWorkspaceArchiveDescriptor,
64
+ providerWorkspaceCapturePolicy,
65
+ SandboxProviderContinuityUnavailableError,
56
66
  requirePersistableReplacementSandboxEnvelope,
57
67
  modalSessionMatchesCheckpointProviderBinding,
58
68
  resolveModalCheckpointProviderBindingForSession,
59
69
  serializeReplacementSandboxEnvelope,
60
70
  tagModalSandbox,
71
+ terminateUnpublishedSandboxSession,
61
72
  verifySandboxExecReadiness,
73
+ withoutSandboxProviderIdentity,
62
74
  type EstablishedSandboxSession,
63
75
  type RuntimeMetricsHooks,
64
76
  type WorkspaceArchiveDescriptor,
@@ -95,6 +107,10 @@ export function isRetryableDegradedRestore(restore: {
95
107
  export type SandboxResumeServices = {
96
108
  db: Database;
97
109
  settings: Settings;
110
+ /** Exact settings before a verified rig provider image overlaid the logical
111
+ * pack/deployment image. Fresh-create NotFound fallback must preserve an
112
+ * ID-only logical base instead of selecting the provider default. */
113
+ logicalFallbackSettings?: Settings;
98
114
  sandboxMetrics?: RuntimeMetricsHooks;
99
115
  /**
100
116
  * The logical turn-attempt lifetime, not merely the provider request lifetime.
@@ -166,25 +182,78 @@ export type ResumedTurnSandbox = {
166
182
  established: EstablishedSandboxSession;
167
183
  /** The lease_epoch this turn holds; the heartbeat/fence token. */
168
184
  leaseEpoch: number;
169
- /** Idempotent release: deletes this holder row and (if refcount hits 0 with no
170
- * turn holders) CASes warm->draining. NEVER stops the box. Safe to call once. */
171
- release: () => Promise<void>;
185
+ /** Staged idempotent release. The ordinary form immediately prevents a
186
+ * holder/timer leak. The quiesced form may be called later—even after the
187
+ * ordinary form—to repair null-outcome admissions only after every exact
188
+ * attempt-owned workspace writer has physically drained. NEVER stops the box. */
189
+ release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
172
190
  };
173
191
 
174
- export class SandboxWarmingTimeoutError extends Error {
175
- readonly code = "sandbox_warming_timeout";
192
+ export abstract class SandboxWarmingTimeoutError extends Error {
193
+ abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
176
194
 
177
- constructor(
195
+ protected constructor(
178
196
  public readonly backend: string,
179
197
  public readonly timeoutMs: number,
198
+ public readonly stage: "exec_readiness" | "sibling_warming",
199
+ public readonly sandboxGroupId: string | null,
200
+ public readonly instanceId: string | null,
180
201
  ) {
202
+ const timeoutSeconds = Math.ceil(timeoutMs / 1000);
203
+ const identity = [
204
+ sandboxGroupId ? `group ${sandboxGroupId}` : null,
205
+ instanceId ? `sandbox ${instanceId}` : null,
206
+ ]
207
+ .filter((part): part is string => part !== null)
208
+ .join(", ");
209
+ const target = identity ? ` (${identity})` : "";
181
210
  super(
182
- `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
211
+ stage === "exec_readiness"
212
+ ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.`
213
+ : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`,
183
214
  );
184
215
  this.name = "SandboxWarmingTimeoutError";
185
216
  }
186
217
  }
187
218
 
219
+ export class SandboxExecReadinessTimeoutError extends SandboxWarmingTimeoutError {
220
+ readonly code = "sandbox_exec_readiness_timeout" as const;
221
+
222
+ constructor(
223
+ backend: string,
224
+ timeoutMs: number,
225
+ identity: { sandboxGroupId?: string | null; instanceId?: string | null } = {},
226
+ ) {
227
+ super(
228
+ backend,
229
+ timeoutMs,
230
+ "exec_readiness",
231
+ identity.sandboxGroupId ?? null,
232
+ identity.instanceId ?? null,
233
+ );
234
+ this.name = "SandboxExecReadinessTimeoutError";
235
+ }
236
+ }
237
+
238
+ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutError {
239
+ readonly code = "sandbox_sibling_warming_timeout" as const;
240
+
241
+ constructor(
242
+ backend: string,
243
+ timeoutMs: number,
244
+ identity: { sandboxGroupId: string; instanceId?: string | null },
245
+ ) {
246
+ super(
247
+ backend,
248
+ timeoutMs,
249
+ "sibling_warming",
250
+ identity.sandboxGroupId,
251
+ identity.instanceId ?? null,
252
+ );
253
+ this.name = "SandboxSiblingWarmingTimeoutError";
254
+ }
255
+ }
256
+
188
257
  /** The exact attached caller that won warm->cold after proving the provider
189
258
  * instance gone. It is a lease supersession (the same logical turn recovers),
190
259
  * plus the lost id needed for one durable observability event. */
@@ -202,7 +271,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
202
271
  // Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
203
272
  // user-facing and separate from the lease TTL heartbeat/reaper horizon.
204
273
  const WARMING_POLL_INTERVAL_MS = 250;
205
- const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
206
274
 
207
275
  async function sleep(ms: number): Promise<void> {
208
276
  await new Promise<void>((resolve) => setTimeout(resolve, ms));
@@ -217,12 +285,16 @@ async function sleep(ms: number): Promise<void> {
217
285
  export async function waitForSandboxExecReadiness(
218
286
  established: EstablishedSandboxSession,
219
287
  timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
288
+ identity: { sandboxGroupId?: string | null } = {},
220
289
  ): Promise<void> {
221
290
  try {
222
291
  await verifySandboxExecReadiness(established, timeoutMs);
223
292
  } catch (error) {
224
- if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
225
- throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
293
+ if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
294
+ throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
295
+ sandboxGroupId: identity.sandboxGroupId ?? null,
296
+ instanceId: established.instanceId,
297
+ });
226
298
  }
227
299
  throw error;
228
300
  }
@@ -354,43 +426,10 @@ async function resolveModalCheckpointBindingBeforeCapture(
354
426
  async function terminateEstablishedSandbox(
355
427
  established: EstablishedSandboxSession | null,
356
428
  ): Promise<boolean> {
357
- if (!established) {
358
- return true;
359
- }
360
- const client = established.client as {
361
- delete?: (state: unknown) => Promise<unknown>;
362
- };
363
- if (typeof client.delete === "function" && established.sessionState !== undefined) {
364
- try {
365
- await client.delete(established.sessionState);
366
- return true;
367
- } catch (error) {
368
- if (isProviderSandboxNotFoundError(established.backendId, error)) {
369
- return true;
370
- }
371
- return false;
372
- }
373
- }
374
- const session = established.session as {
375
- close?: () => Promise<unknown>;
376
- terminate?: () => Promise<unknown>;
377
- kill?: () => Promise<unknown>;
378
- closed?: boolean;
379
- };
429
+ if (!established) return true;
380
430
  try {
381
- if (session.terminate) {
382
- await session.terminate();
383
- return true;
384
- } else if (session.kill) {
385
- await session.kill();
386
- return true;
387
- } else if (session.close) {
388
- if (!session.closed) {
389
- await session.close();
390
- }
391
- return true;
392
- }
393
- return false;
431
+ await terminateUnpublishedSandboxSession(established);
432
+ return true;
394
433
  } catch (error) {
395
434
  if (isProviderSandboxNotFoundError(established.backendId, error)) {
396
435
  return true;
@@ -400,13 +439,6 @@ async function terminateEstablishedSandbox(
400
439
  }
401
440
  }
402
441
 
403
- function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
404
- const message = error instanceof Error ? error.message : String(error);
405
- return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
406
- ? new SandboxWarmingTimeoutError(backend, timeoutMs)
407
- : error;
408
- }
409
-
410
442
  function recordSandboxWarmingTimeout(
411
443
  metrics: RuntimeMetricsHooks | undefined,
412
444
  error: unknown,
@@ -415,7 +447,7 @@ function recordSandboxWarmingTimeout(
415
447
  return;
416
448
  }
417
449
  try {
418
- metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
450
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
419
451
  } catch {
420
452
  // Metrics emission must never affect sandbox recovery or error propagation.
421
453
  }
@@ -463,23 +495,6 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
463
495
  );
464
496
  }
465
497
 
466
- /** A per-session fallback may still carry the dead provider identity that made
467
- * recovery necessary. It is useful only as durable archive/config input: never
468
- * let a cold rematerialization resume that stale provider before hydrating the
469
- * selected revision. */
470
- function withoutProviderIdentity(
471
- envelope: Record<string, unknown> | null,
472
- ): Record<string, unknown> | null {
473
- if (!envelope) return null;
474
- const sessionState =
475
- envelope.sessionState && typeof envelope.sessionState === "object"
476
- ? (envelope.sessionState as Record<string, unknown>)
477
- : null;
478
- if (!sessionState) return envelope;
479
- const { providerState: _providerState, ...providerIndependentState } = sessionState;
480
- return { ...envelope, sessionState: providerIndependentState };
481
- }
482
-
483
498
  /**
484
499
  * MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
485
500
  * drain-persist only protects boxes the reaper itself kills; anything else —
@@ -528,17 +543,14 @@ export async function maybePersistWarmWorkspaceSnapshot(
528
543
  if (typeof persistable.persistWorkspace !== "function") {
529
544
  return false;
530
545
  }
531
- // Modal filesystem snapshots terminate the source sandbox. They are safe at
532
- // controlled drain/rotation (force=true), but a periodic or turn-end warm
533
- // checkpoint would kill the box while its lease still points at it. The next
534
- // dock read or turn would then route to a dead provider instance. Let the
535
- // drain reaper capture this backend instead.
546
+ // Filesystem and directory snapshots create retained Images without
547
+ // terminating the source Sandbox. (Modal's termination warning applies to
548
+ // memory snapshots.) They are therefore the preferred warm-checkpoint path:
549
+ // the durable capture gate pauses OpenGeni commands while the provider reads
550
+ // the filesystem, then the same live instance continues serving the turn.
536
551
  const workspacePersistence =
537
552
  persistable.state?.workspacePersistence ??
538
553
  persistable.state?.providerState?.workspacePersistence;
539
- if (!force && workspacePersistence === "snapshot_filesystem") {
540
- return false;
541
- }
542
554
  if (signal?.aborted) {
543
555
  return false;
544
556
  }
@@ -555,7 +567,17 @@ export async function maybePersistWarmWorkspaceSnapshot(
555
567
  ) {
556
568
  return false;
557
569
  }
570
+ // A checkpoint of this exact mutation generation already protects every
571
+ // settled operation admitted so far. Capturing it again cannot improve the
572
+ // recovery point, even when the wall-clock interval has elapsed.
573
+ if (lease.archiveComplete) {
574
+ return false;
575
+ }
558
576
  const instanceId = lease.instanceId;
577
+ const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
578
+ if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
579
+ return false;
580
+ }
559
581
  const nativeModalPersistence =
560
582
  workspacePersistence === "snapshot_filesystem" ||
561
583
  workspacePersistence === "snapshot_directory";
@@ -576,6 +598,8 @@ export async function maybePersistWarmWorkspaceSnapshot(
576
598
  liveness: "warm",
577
599
  captureTimeoutMs,
578
600
  minIntervalMs: force ? 0 : intervalMs,
601
+ providerReplaySafe: capturePolicy.takeover === "same_request",
602
+ takeoverSafe: capturePolicy.takeover !== "exclusive",
579
603
  warmAttempt: {
580
604
  sessionId: ids.sessionId,
581
605
  turnId: ids.turnId,
@@ -634,7 +658,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
634
658
  const captureAndPublish = (async (): Promise<boolean> => {
635
659
  let candidate: { id: string } | null = null;
636
660
  try {
637
- const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs);
661
+ const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
662
+ requestId: claimed.claim.providerRequestId,
663
+ strategy: capturePolicy.strategy,
664
+ });
638
665
  candidate = await registerCandidate(archive);
639
666
  const { wrote } = await persistWarmSnapshot(db, {
640
667
  accountId: ids.accountId,
@@ -749,37 +776,74 @@ export async function resumeBoxForTurn(
749
776
  // It is also bound directly to the logical attempt signal: an uninterruptible
750
777
  // provider promise must never keep its private warmup timer and holder alive
751
778
  // after Temporal has abandoned the activity.
752
- let released = false;
779
+ let releaseStarted = false;
780
+ let holderRelease: Promise<void> | null = null;
781
+ let quiescedRelease: Promise<void> | null = null;
753
782
  let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
754
783
  let cancellationListener: (() => void) | undefined;
755
- const release = async (): Promise<void> => {
756
- if (released) {
757
- return;
784
+ const release = (options?: { workspaceWritersQuiesced?: boolean }): Promise<void> => {
785
+ const writersQuiesced = options?.workspaceWritersQuiesced === true;
786
+ // First invocation synchronously closes every in-memory liveness source.
787
+ // A later quiesced invocation is still allowed to enter the DB: cancellation
788
+ // commonly wins this race and drops the holder before the physical writer
789
+ // drain can prove that abandoned turn admissions are safe to settle.
790
+ if (!releaseStarted) {
791
+ releaseStarted = true;
792
+ if (cancellationListener) {
793
+ cancellationSignal?.removeEventListener("abort", cancellationListener);
794
+ cancellationListener = undefined;
795
+ }
796
+ if (holderLivenessTimer) {
797
+ clearInterval(holderLivenessTimer);
798
+ holderLivenessTimer = undefined;
799
+ }
758
800
  }
759
- released = true;
760
- if (cancellationListener) {
761
- cancellationSignal?.removeEventListener("abort", cancellationListener);
762
- cancellationListener = undefined;
801
+ const persistRelease = async (workspaceWritersQuiesced: boolean): Promise<void> => {
802
+ await releaseLeaseHolder(db, {
803
+ accountId: ids.accountId,
804
+ workspaceId: ids.workspaceId,
805
+ sandboxGroupId: ids.sandboxGroupId,
806
+ kind,
807
+ holderId,
808
+ idleGraceMs: settings.sandboxIdleGraceMs,
809
+ ...(workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}),
810
+ });
811
+ };
812
+ if (writersQuiesced) {
813
+ if (!quiescedRelease) {
814
+ // Serialize behind an eager cancellation release when one exists. Its
815
+ // failure is intentionally swallowed only as a predecessor: this exact
816
+ // idempotent proof-bearing call retries both holder deletion and
817
+ // admission settlement in one transaction.
818
+ const predecessor = holderRelease;
819
+ const attempt = (predecessor ? predecessor.catch(() => undefined) : Promise.resolve()).then(
820
+ async () => await persistRelease(true),
821
+ );
822
+ const tracked = attempt.catch((error) => {
823
+ if (quiescedRelease === tracked) quiescedRelease = null;
824
+ throw error;
825
+ });
826
+ quiescedRelease = tracked;
827
+ holderRelease = tracked;
828
+ }
829
+ return quiescedRelease;
763
830
  }
764
- if (holderLivenessTimer) {
765
- clearInterval(holderLivenessTimer);
766
- holderLivenessTimer = undefined;
831
+ if (!holderRelease) {
832
+ const attempt = persistRelease(false);
833
+ const tracked = attempt.catch((error) => {
834
+ if (holderRelease === tracked) holderRelease = null;
835
+ throw error;
836
+ });
837
+ holderRelease = tracked;
767
838
  }
768
- await releaseLeaseHolder(db, {
769
- accountId: ids.accountId,
770
- workspaceId: ids.workspaceId,
771
- sandboxGroupId: ids.sandboxGroupId,
772
- kind,
773
- holderId,
774
- idleGraceMs: settings.sandboxIdleGraceMs,
775
- });
839
+ return holderRelease;
776
840
  };
777
841
  const cancellationError = (): Error =>
778
842
  cancellationSignal?.reason instanceof Error
779
843
  ? cancellationSignal.reason
780
844
  : new Error("Sandbox resume was cancelled with its owning turn attempt");
781
845
  const throwIfReleasedOrCancelled = (): void => {
782
- if (released || cancellationSignal?.aborted) {
846
+ if (releaseStarted || cancellationSignal?.aborted) {
783
847
  throw cancellationError();
784
848
  }
785
849
  };
@@ -805,15 +869,18 @@ export async function resumeBoxForTurn(
805
869
  // durable capture-and-drain rotation as an image change.
806
870
  ...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
807
871
  leaseTtlMs,
808
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
809
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
872
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
873
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
874
+ ...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
810
875
  });
811
876
 
812
877
  if (cancellationSignal) {
813
878
  cancellationListener = () => {
814
- // release() flips `released` and clears the timer synchronously before its
815
- // first await. The detached rejection handler is intentional: the outer
816
- // turn owns diagnostics, while this listener owns leak prevention.
879
+ // release() flips `releaseStarted` and clears the timer synchronously
880
+ // before its first await. This eager form deliberately carries no writer-
881
+ // quiescence proof; the outer turn supplies that second stage after its
882
+ // physical drain. The detached rejection handler is intentional: the
883
+ // outer turn owns diagnostics, while this listener owns leak prevention.
817
884
  void release().catch(() => undefined);
818
885
  };
819
886
  cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
@@ -823,7 +890,7 @@ export async function resumeBoxForTurn(
823
890
  throw cancellationError();
824
891
  }
825
892
 
826
- // HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
893
+ // HOLDER-LIVENESS loop: refresh OUR holder every 10s from registration.
827
894
  // registered until release. The dead-worker turn-holder reap judges liveness
828
895
  // by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
829
896
  // agent-turn) only starts AFTER this function returns — while waitForWarm and
@@ -831,21 +898,46 @@ export async function resumeBoxForTurn(
831
898
  // COMPOUNDING past any fixed reap horizon. With this
832
899
  // loop no live holder is ever silent for more than one tick, so the reap
833
900
  // horizon is pure defense-in-depth, not a tuned guess about path lengths.
834
- // Epoch-free by design (touch only refreshes our own row's timestamp);
835
- // best-effort (a transient DB error must never fail the resume).
901
+ // The elected spawner also heartbeats the short rolling warming lease; this
902
+ // makes warming expiry a crash detector rather than a fixed upper bound on a
903
+ // legitimate provider create/restore. A warming waiter only touches its own
904
+ // holder so it cannot keep a dead spawner's lease alive. Once the box is warm,
905
+ // every owner uses the normal epoch-fenced lease heartbeat.
906
+ let holderLeaseHeartbeat: { expectedEpoch: number; leaseTtlMs: number } | null =
907
+ acquired.role === "spawner"
908
+ ? {
909
+ expectedEpoch: acquired.lease.leaseEpoch,
910
+ leaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
911
+ }
912
+ : (acquired.role === "attached" || acquired.role === "rearmed") &&
913
+ acquired.lease.liveness === "warm"
914
+ ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
915
+ : null;
836
916
  holderLivenessTimer = setInterval(() => {
837
- void touchLeaseHolder(db, {
838
- accountId: ids.accountId,
839
- workspaceId: ids.workspaceId,
840
- sandboxGroupId: ids.sandboxGroupId,
841
- kind,
842
- holderId,
843
- })
917
+ const heartbeat = holderLeaseHeartbeat;
918
+ const refresh = heartbeat
919
+ ? heartbeatLeaseHolder(db, {
920
+ accountId: ids.accountId,
921
+ workspaceId: ids.workspaceId,
922
+ sandboxGroupId: ids.sandboxGroupId,
923
+ kind,
924
+ holderId,
925
+ expectedEpoch: heartbeat.expectedEpoch,
926
+ leaseTtlMs: heartbeat.leaseTtlMs,
927
+ })
928
+ : touchLeaseHolder(db, {
929
+ accountId: ids.accountId,
930
+ workspaceId: ids.workspaceId,
931
+ sandboxGroupId: ids.sandboxGroupId,
932
+ kind,
933
+ holderId,
934
+ });
935
+ void refresh
844
936
  .then((touched) => {
845
937
  // A canonical turn holder is rejected once its exact attempt is no
846
938
  // longer the active writer. Stop this otherwise-unbounded provider
847
939
  // operation and idempotently drop any remaining holder state.
848
- if (!touched) {
940
+ if (!touched && heartbeat === holderLeaseHeartbeat) {
849
941
  void release().catch(() => undefined);
850
942
  }
851
943
  })
@@ -869,7 +961,21 @@ export async function resumeBoxForTurn(
869
961
  }
870
962
  if (acquired.role === "fenced") {
871
963
  await release();
872
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
964
+ if (acquired.reason !== "superseded") {
965
+ throw new SandboxLeaseTransitionError(
966
+ ids.sandboxGroupId,
967
+ acquired.lease.leaseEpoch,
968
+ acquired.reason,
969
+ acquired.lease.backend,
970
+ acquired.lease.instanceId,
971
+ acquired.lease.liveness,
972
+ );
973
+ }
974
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
975
+ backend: acquired.lease.backend,
976
+ instanceId: acquired.lease.instanceId,
977
+ liveness: acquired.lease.liveness,
978
+ });
873
979
  }
874
980
 
875
981
  // SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
@@ -898,16 +1004,19 @@ export async function resumeBoxForTurn(
898
1004
  const fallbackArchiveEnvelope =
899
1005
  acquired.lease.recovery.archive.status === "none" &&
900
1006
  workspaceArchiveFieldsFromEnvelope(envelope) !== null
901
- ? withoutProviderIdentity(envelope)
1007
+ ? withoutSandboxProviderIdentity(envelope)
902
1008
  : null;
903
1009
  let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
904
1010
  const archiveSource =
905
1011
  acquired.lease.recovery.archive.status === "none"
906
1012
  ? fallbackArchiveEnvelope
907
1013
  : acquired.lease.resumeState;
1014
+ const continuityRecovery = acquired.lease.recovery.continuity;
908
1015
  if (
909
- acquired.lease.recovery.archive.status === "available" ||
910
- workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
1016
+ (acquired.lease.recovery.archive.status === "available" &&
1017
+ acquired.lease.archiveComplete) ||
1018
+ (acquired.lease.recovery.archive.status === "none" &&
1019
+ workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
911
1020
  ) {
912
1021
  const rematerializationId = crypto.randomUUID();
913
1022
  const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -950,7 +1059,7 @@ export async function resumeBoxForTurn(
950
1059
  legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
951
1060
  legacyProviderBinding: null,
952
1061
  };
953
- } else if (acquired.lease.recovery.archive.status !== "none") {
1062
+ } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
954
1063
  throw new SandboxLeaseRecoveryBlockedError(
955
1064
  ids.sandboxGroupId,
956
1065
  expectedEpoch,
@@ -974,6 +1083,9 @@ export async function resumeBoxForTurn(
974
1083
  backendOverride: ids.backend as never,
975
1084
  ...(ids.environment ? { environment: ids.environment } : {}),
976
1085
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
1086
+ ...(services.logicalFallbackSettings
1087
+ ? { logicalFallbackSettings: services.logicalFallbackSettings }
1088
+ : {}),
977
1089
  onSandboxCreated: async (created) => {
978
1090
  createdEstablished = created;
979
1091
  throwIfReleasedOrCancelled();
@@ -1027,6 +1139,9 @@ export async function resumeBoxForTurn(
1027
1139
  sandboxGroupId: ids.sandboxGroupId,
1028
1140
  expectedEpoch,
1029
1141
  rematerializationId: rematerialization?.id ?? null,
1142
+ ...(created.providerContinuity
1143
+ ? { continuityRecovery: created.providerContinuity }
1144
+ : {}),
1030
1145
  instanceId: created.instanceId,
1031
1146
  resumeBackendId: created.backendId,
1032
1147
  resumeState: resumeEnvelope,
@@ -1041,7 +1156,7 @@ export async function resumeBoxForTurn(
1041
1156
  leaseTtlMs,
1042
1157
  // Keep the warming budget after create(): manifest setup and
1043
1158
  // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
1044
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
1159
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
1045
1160
  });
1046
1161
  if (!recorded.recorded) {
1047
1162
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
@@ -1107,7 +1222,9 @@ export async function resumeBoxForTurn(
1107
1222
  // is live. Do not publish a warm lease until one bounded no-op exec works.
1108
1223
  // On timeout the catch below terminates the box and rolls warming -> cold,
1109
1224
  // so the next turn cold-creates instead of hanging forever on first use.
1110
- await waitForSandboxExecReadiness(established);
1225
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1226
+ sandboxGroupId: ids.sandboxGroupId,
1227
+ });
1111
1228
  throwIfReleasedOrCancelled();
1112
1229
  // Fold the LIVE box into a re-resumable envelope and persist it as the
1113
1230
  // lease's resume_state — exactly like the API-direct paths (channel-a.ts /
@@ -1133,6 +1250,7 @@ export async function resumeBoxForTurn(
1133
1250
  throwIfReleasedOrCancelled();
1134
1251
  if (
1135
1252
  rematerialization &&
1253
+ !established.providerContinuity &&
1136
1254
  established.restoredArchive?.revision !== rematerialization.selectedRevision
1137
1255
  ) {
1138
1256
  throw new WorkspaceArchiveIntegrityError(
@@ -1151,14 +1269,16 @@ export async function resumeBoxForTurn(
1151
1269
  dataPlaneUrl: null,
1152
1270
  resumeBackendId: established.backendId,
1153
1271
  resumeState: resumeEnvelope,
1154
- ...(rematerialization
1155
- ? {
1156
- rematerialization: {
1157
- id: rematerialization.id,
1158
- verifiedRevision: rematerialization.selectedRevision,
1159
- },
1160
- }
1161
- : {}),
1272
+ ...(established.providerContinuity
1273
+ ? { continuityRecovery: established.providerContinuity }
1274
+ : rematerialization
1275
+ ? {
1276
+ rematerialization: {
1277
+ id: rematerialization.id,
1278
+ verifiedRevision: rematerialization.selectedRevision,
1279
+ },
1280
+ }
1281
+ : {}),
1162
1282
  leaseTtlMs,
1163
1283
  });
1164
1284
  if (!committed.committed || !committed.lease) {
@@ -1167,7 +1287,7 @@ export async function resumeBoxForTurn(
1167
1287
  // holder; surface supersession. This spawner created the box, so stop it
1168
1288
  // before retrying to avoid an untracked running sandbox.
1169
1289
  const terminated = await terminateEstablishedSandbox(established);
1170
- if (terminated && rematerialization) {
1290
+ if (terminated && rematerialization && !established.providerContinuity) {
1171
1291
  await failSandboxRematerialization(db, {
1172
1292
  accountId: ids.accountId,
1173
1293
  workspaceId: ids.workspaceId,
@@ -1177,10 +1297,21 @@ export async function resumeBoxForTurn(
1177
1297
  failureCode: committed.reason ?? "warm_commit_rejected",
1178
1298
  retryable: false,
1179
1299
  });
1300
+ } else if (terminated) {
1301
+ await failWarmingToCold(db, {
1302
+ accountId: ids.accountId,
1303
+ workspaceId: ids.workspaceId,
1304
+ sandboxGroupId: ids.sandboxGroupId,
1305
+ expectedEpoch,
1306
+ });
1180
1307
  }
1181
1308
  await release();
1182
1309
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1183
1310
  }
1311
+ holderLeaseHeartbeat = {
1312
+ expectedEpoch: committed.lease.leaseEpoch,
1313
+ leaseTtlMs,
1314
+ };
1184
1315
  throwIfReleasedOrCancelled();
1185
1316
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1186
1317
  } catch (error) {
@@ -1196,18 +1327,25 @@ export async function resumeBoxForTurn(
1196
1327
  // the warming row; the lease TTL/reaper and Modal orphan sweep are the
1197
1328
  // tracked backstops, and we must not erase the only provider pointer.
1198
1329
  if (terminated) {
1199
- if (rematerialization) {
1330
+ const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
1331
+ if (rematerialization && !createdEstablished?.providerContinuity) {
1200
1332
  await failSandboxRematerialization(db, {
1201
1333
  accountId: ids.accountId,
1202
1334
  workspaceId: ids.workspaceId,
1203
1335
  sandboxGroupId: ids.sandboxGroupId,
1204
1336
  expectedEpoch,
1205
1337
  rematerializationId: rematerialization.id,
1206
- failureCode:
1207
- error instanceof WorkspaceArchiveIntegrityError
1338
+ failureCode: continuityUnavailable
1339
+ ? "provider_continuity_unavailable"
1340
+ : error instanceof WorkspaceArchiveIntegrityError
1208
1341
  ? error.code
1209
1342
  : "sandbox_rematerialization_failed",
1210
- retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
1343
+ retryable: continuityUnavailable
1344
+ ? error.retryable
1345
+ : error instanceof WorkspaceArchiveIntegrityError
1346
+ ? error.retryable
1347
+ : true,
1348
+ ...(continuityUnavailable ? { discardContinuity: true } : {}),
1211
1349
  });
1212
1350
  } else {
1213
1351
  await failWarmingToCold(db, {
@@ -1215,17 +1353,13 @@ export async function resumeBoxForTurn(
1215
1353
  workspaceId: ids.workspaceId,
1216
1354
  sandboxGroupId: ids.sandboxGroupId,
1217
1355
  expectedEpoch,
1356
+ ...(continuityUnavailable ? { discardContinuity: true } : {}),
1218
1357
  });
1219
1358
  }
1220
1359
  }
1221
1360
  await release();
1222
- const warmingError = asSandboxWarmingError(
1223
- error,
1224
- ids.backend,
1225
- settings.sandboxWarmingTimeoutMs,
1226
- );
1227
- recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1228
- throw warmingError;
1361
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1362
+ throw error;
1229
1363
  }
1230
1364
  }
1231
1365
 
@@ -1238,6 +1372,7 @@ export async function resumeBoxForTurn(
1238
1372
  let leaseEpoch = acquired.lease.leaseEpoch;
1239
1373
  if (acquired.lease.liveness === "warming") {
1240
1374
  leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1375
+ holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
1241
1376
  }
1242
1377
  throwIfReleasedOrCancelled();
1243
1378
 
@@ -1270,7 +1405,13 @@ export async function resumeBoxForTurn(
1270
1405
  // was idle. Prove the command router before handing the session to the
1271
1406
  // agent so terminal evidence enters the atomic warm->cold recovery path
1272
1407
  // below instead of surfacing inside a model-visible tool call.
1273
- await (services.verifyAttachedSandboxReadiness ?? waitForSandboxExecReadiness)(established);
1408
+ if (services.verifyAttachedSandboxReadiness) {
1409
+ await services.verifyAttachedSandboxReadiness(established);
1410
+ } else {
1411
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1412
+ sandboxGroupId: ids.sandboxGroupId,
1413
+ });
1414
+ }
1274
1415
  throwIfReleasedOrCancelled();
1275
1416
  } catch (error) {
1276
1417
  if (!isProviderSandboxNotFoundError(ids.backend, error)) {
@@ -1304,13 +1445,8 @@ export async function resumeBoxForTurn(
1304
1445
  return { established, leaseEpoch, release };
1305
1446
  } catch (error) {
1306
1447
  await release();
1307
- const warmingError = asSandboxWarmingError(
1308
- error,
1309
- ids.backend,
1310
- settings.sandboxWarmingTimeoutMs,
1311
- );
1312
- recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1313
- throw warmingError;
1448
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1449
+ throw error;
1314
1450
  }
1315
1451
  }
1316
1452
 
@@ -1326,6 +1462,7 @@ async function waitForWarm(
1326
1462
  ): Promise<{ leaseEpoch: number }> {
1327
1463
  const { db, settings } = services;
1328
1464
  const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1465
+ let instanceId: string | null = null;
1329
1466
  while (Date.now() < deadline) {
1330
1467
  await sleep(WARMING_POLL_INTERVAL_MS);
1331
1468
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
@@ -1333,6 +1470,7 @@ async function waitForWarm(
1333
1470
  // Lease vanished (cold-reaped). Re-dispatch from scratch.
1334
1471
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1335
1472
  }
1473
+ instanceId = lease.instanceId;
1336
1474
  if (lease.liveness === "warm") {
1337
1475
  return { leaseEpoch: lease.leaseEpoch };
1338
1476
  }
@@ -1368,5 +1506,8 @@ async function waitForWarm(
1368
1506
  }
1369
1507
  // still warming — keep polling.
1370
1508
  }
1371
- throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
1509
+ throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
1510
+ sandboxGroupId: ids.sandboxGroupId,
1511
+ instanceId,
1512
+ });
1372
1513
  }