@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -20,8 +20,11 @@
20
20
  //
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
- import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
24
- import { redactSensitiveText } from "@opengeni/contracts";
23
+ import {
24
+ sandboxArchiveCaptureTimeoutMs,
25
+ sandboxLifecycleTransitionWaitMs,
26
+ type Settings,
27
+ } from "@opengeni/config";
25
28
  import { randomUUID } from "node:crypto";
26
29
  import {
27
30
  acquireLease,
@@ -32,6 +35,7 @@ import {
32
35
  failSandboxRematerialization,
33
36
  failWarmingToCold,
34
37
  getSandboxSessionEnvelope,
38
+ heartbeatLeaseHolder,
35
39
  markSandboxRestoreVerifying,
36
40
  markWarmLeaseInstanceLost,
37
41
  markSandboxCheckpointArtifactDeletePending,
@@ -44,22 +48,29 @@ import {
44
48
  touchLeaseHolder,
45
49
  SandboxLeaseRecoveryBlockedError,
46
50
  SandboxLeaseSupersededError,
51
+ SandboxLeaseTransitionError,
47
52
  type Database,
48
53
  type LeaseHolderKind,
49
54
  } from "@opengeni/db";
50
55
  import {
51
56
  captureVerifiedWorkspaceArchive,
52
57
  describeLegacyNativeSnapshotArchive,
58
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
59
+ SandboxExecReadinessError,
53
60
  WorkspaceArchiveIntegrityError,
54
61
  establishSandboxSessionFromEnvelope,
55
62
  isProviderSandboxNotFoundError,
56
63
  parseWorkspaceArchiveDescriptor,
64
+ providerWorkspaceCapturePolicy,
65
+ SandboxProviderContinuityUnavailableError,
57
66
  requirePersistableReplacementSandboxEnvelope,
58
67
  modalSessionMatchesCheckpointProviderBinding,
59
68
  resolveModalCheckpointProviderBindingForSession,
60
69
  serializeReplacementSandboxEnvelope,
61
70
  tagModalSandbox,
71
+ terminateUnpublishedSandboxSession,
62
72
  verifySandboxExecReadiness,
73
+ withoutSandboxProviderIdentity,
63
74
  type EstablishedSandboxSession,
64
75
  type RuntimeMetricsHooks,
65
76
  type WorkspaceArchiveDescriptor,
@@ -96,6 +107,10 @@ export function isRetryableDegradedRestore(restore: {
96
107
  export type SandboxResumeServices = {
97
108
  db: Database;
98
109
  settings: Settings;
110
+ /** Exact settings before a verified rig provider image overlaid the logical
111
+ * pack/deployment image. Fresh-create NotFound fallback must preserve an
112
+ * ID-only logical base instead of selecting the provider default. */
113
+ logicalFallbackSettings?: Settings;
99
114
  sandboxMetrics?: RuntimeMetricsHooks;
100
115
  /**
101
116
  * The logical turn-attempt lifetime, not merely the provider request lifetime.
@@ -167,25 +182,78 @@ export type ResumedTurnSandbox = {
167
182
  established: EstablishedSandboxSession;
168
183
  /** The lease_epoch this turn holds; the heartbeat/fence token. */
169
184
  leaseEpoch: number;
170
- /** Idempotent release: deletes this holder row and (if refcount hits 0 with no
171
- * turn holders) CASes warm->draining. NEVER stops the box. Safe to call once. */
172
- release: () => Promise<void>;
185
+ /** Staged idempotent release. The ordinary form immediately prevents a
186
+ * holder/timer leak. The quiesced form may be called later—even after the
187
+ * ordinary form—to repair null-outcome admissions only after every exact
188
+ * attempt-owned workspace writer has physically drained. NEVER stops the box. */
189
+ release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
173
190
  };
174
191
 
175
- export class SandboxWarmingTimeoutError extends Error {
176
- readonly code = "sandbox_warming_timeout";
192
+ export abstract class SandboxWarmingTimeoutError extends Error {
193
+ abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
177
194
 
178
- constructor(
195
+ protected constructor(
179
196
  public readonly backend: string,
180
197
  public readonly timeoutMs: number,
198
+ public readonly stage: "exec_readiness" | "sibling_warming",
199
+ public readonly sandboxGroupId: string | null,
200
+ public readonly instanceId: string | null,
181
201
  ) {
202
+ const timeoutSeconds = Math.ceil(timeoutMs / 1000);
203
+ const identity = [
204
+ sandboxGroupId ? `group ${sandboxGroupId}` : null,
205
+ instanceId ? `sandbox ${instanceId}` : null,
206
+ ]
207
+ .filter((part): part is string => part !== null)
208
+ .join(", ");
209
+ const target = identity ? ` (${identity})` : "";
182
210
  super(
183
- `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
211
+ stage === "exec_readiness"
212
+ ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.`
213
+ : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`,
184
214
  );
185
215
  this.name = "SandboxWarmingTimeoutError";
186
216
  }
187
217
  }
188
218
 
219
+ export class SandboxExecReadinessTimeoutError extends SandboxWarmingTimeoutError {
220
+ readonly code = "sandbox_exec_readiness_timeout" as const;
221
+
222
+ constructor(
223
+ backend: string,
224
+ timeoutMs: number,
225
+ identity: { sandboxGroupId?: string | null; instanceId?: string | null } = {},
226
+ ) {
227
+ super(
228
+ backend,
229
+ timeoutMs,
230
+ "exec_readiness",
231
+ identity.sandboxGroupId ?? null,
232
+ identity.instanceId ?? null,
233
+ );
234
+ this.name = "SandboxExecReadinessTimeoutError";
235
+ }
236
+ }
237
+
238
+ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutError {
239
+ readonly code = "sandbox_sibling_warming_timeout" as const;
240
+
241
+ constructor(
242
+ backend: string,
243
+ timeoutMs: number,
244
+ identity: { sandboxGroupId: string; instanceId?: string | null },
245
+ ) {
246
+ super(
247
+ backend,
248
+ timeoutMs,
249
+ "sibling_warming",
250
+ identity.sandboxGroupId,
251
+ identity.instanceId ?? null,
252
+ );
253
+ this.name = "SandboxSiblingWarmingTimeoutError";
254
+ }
255
+ }
256
+
189
257
  /** The exact attached caller that won warm->cold after proving the provider
190
258
  * instance gone. It is a lease supersession (the same logical turn recovers),
191
259
  * plus the lost id needed for one durable observability event. */
@@ -203,7 +271,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
203
271
  // Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
204
272
  // user-facing and separate from the lease TTL heartbeat/reaper horizon.
205
273
  const WARMING_POLL_INTERVAL_MS = 250;
206
- const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
207
274
 
208
275
  async function sleep(ms: number): Promise<void> {
209
276
  await new Promise<void>((resolve) => setTimeout(resolve, ms));
@@ -218,12 +285,16 @@ async function sleep(ms: number): Promise<void> {
218
285
  export async function waitForSandboxExecReadiness(
219
286
  established: EstablishedSandboxSession,
220
287
  timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
288
+ identity: { sandboxGroupId?: string | null } = {},
221
289
  ): Promise<void> {
222
290
  try {
223
291
  await verifySandboxExecReadiness(established, timeoutMs);
224
292
  } catch (error) {
225
- if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
226
- throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
293
+ if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
294
+ throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
295
+ sandboxGroupId: identity.sandboxGroupId ?? null,
296
+ instanceId: established.instanceId,
297
+ });
227
298
  }
228
299
  throw error;
229
300
  }
@@ -236,15 +307,35 @@ class SnapshotTimeoutError extends Error {
236
307
  }
237
308
  }
238
309
 
239
- function safeSnapshotError(error: unknown): { name: string; message: string } {
240
- const rawName = error instanceof Error ? error.name : "Error";
241
- return {
242
- name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
243
- message: redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(
244
- 0,
245
- 2_048,
246
- ),
310
+ export function safeSnapshotError(error: unknown): {
311
+ errorClass: "SnapshotOperationError";
312
+ errorCode: "snapshot_operation_failed";
313
+ status?: number;
314
+ origin: "sandbox-resume";
315
+ } {
316
+ const fields: {
317
+ errorClass: "SnapshotOperationError";
318
+ errorCode: "snapshot_operation_failed";
319
+ status?: number;
320
+ origin: "sandbox-resume";
321
+ } = {
322
+ errorClass: "SnapshotOperationError",
323
+ errorCode: "snapshot_operation_failed",
324
+ origin: "sandbox-resume",
247
325
  };
326
+ try {
327
+ const rawStatus =
328
+ error && typeof error === "object"
329
+ ? ((error as { status?: unknown; statusCode?: unknown }).status ??
330
+ (error as { statusCode?: unknown }).statusCode)
331
+ : undefined;
332
+ const status = Number(rawStatus);
333
+ if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
334
+ } catch {
335
+ // Public diagnostics are best-effort and must never replace the exact
336
+ // internal snapshot failure.
337
+ }
338
+ return fields;
248
339
  }
249
340
 
250
341
  export async function waitForWarmSnapshot(
@@ -335,43 +426,10 @@ async function resolveModalCheckpointBindingBeforeCapture(
335
426
  async function terminateEstablishedSandbox(
336
427
  established: EstablishedSandboxSession | null,
337
428
  ): Promise<boolean> {
338
- if (!established) {
339
- return true;
340
- }
341
- const client = established.client as {
342
- delete?: (state: unknown) => Promise<unknown>;
343
- };
344
- if (typeof client.delete === "function" && established.sessionState !== undefined) {
345
- try {
346
- await client.delete(established.sessionState);
347
- return true;
348
- } catch (error) {
349
- if (isProviderSandboxNotFoundError(established.backendId, error)) {
350
- return true;
351
- }
352
- return false;
353
- }
354
- }
355
- const session = established.session as {
356
- close?: () => Promise<unknown>;
357
- terminate?: () => Promise<unknown>;
358
- kill?: () => Promise<unknown>;
359
- closed?: boolean;
360
- };
429
+ if (!established) return true;
361
430
  try {
362
- if (session.terminate) {
363
- await session.terminate();
364
- return true;
365
- } else if (session.kill) {
366
- await session.kill();
367
- return true;
368
- } else if (session.close) {
369
- if (!session.closed) {
370
- await session.close();
371
- }
372
- return true;
373
- }
374
- return false;
431
+ await terminateUnpublishedSandboxSession(established);
432
+ return true;
375
433
  } catch (error) {
376
434
  if (isProviderSandboxNotFoundError(established.backendId, error)) {
377
435
  return true;
@@ -381,13 +439,6 @@ async function terminateEstablishedSandbox(
381
439
  }
382
440
  }
383
441
 
384
- function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
385
- const message = error instanceof Error ? error.message : String(error);
386
- return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
387
- ? new SandboxWarmingTimeoutError(backend, timeoutMs)
388
- : error;
389
- }
390
-
391
442
  function recordSandboxWarmingTimeout(
392
443
  metrics: RuntimeMetricsHooks | undefined,
393
444
  error: unknown,
@@ -396,7 +447,7 @@ function recordSandboxWarmingTimeout(
396
447
  return;
397
448
  }
398
449
  try {
399
- metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
450
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
400
451
  } catch {
401
452
  // Metrics emission must never affect sandbox recovery or error propagation.
402
453
  }
@@ -444,23 +495,6 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
444
495
  );
445
496
  }
446
497
 
447
- /** A per-session fallback may still carry the dead provider identity that made
448
- * recovery necessary. It is useful only as durable archive/config input: never
449
- * let a cold rematerialization resume that stale provider before hydrating the
450
- * selected revision. */
451
- function withoutProviderIdentity(
452
- envelope: Record<string, unknown> | null,
453
- ): Record<string, unknown> | null {
454
- if (!envelope) return null;
455
- const sessionState =
456
- envelope.sessionState && typeof envelope.sessionState === "object"
457
- ? (envelope.sessionState as Record<string, unknown>)
458
- : null;
459
- if (!sessionState) return envelope;
460
- const { providerState: _providerState, ...providerIndependentState } = sessionState;
461
- return { ...envelope, sessionState: providerIndependentState };
462
- }
463
-
464
498
  /**
465
499
  * MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
466
500
  * drain-persist only protects boxes the reaper itself kills; anything else —
@@ -501,11 +535,22 @@ export async function maybePersistWarmWorkspaceSnapshot(
501
535
  }
502
536
  const persistable = session as {
503
537
  persistWorkspace?: () => Promise<Uint8Array | undefined>;
504
- state?: { workspacePersistence?: unknown };
538
+ state?: {
539
+ workspacePersistence?: unknown;
540
+ providerState?: { workspacePersistence?: unknown };
541
+ };
505
542
  };
506
543
  if (typeof persistable.persistWorkspace !== "function") {
507
544
  return false;
508
545
  }
546
+ // Filesystem and directory snapshots create retained Images without
547
+ // terminating the source Sandbox. (Modal's termination warning applies to
548
+ // memory snapshots.) They are therefore the preferred warm-checkpoint path:
549
+ // the durable capture gate pauses OpenGeni commands while the provider reads
550
+ // the filesystem, then the same live instance continues serving the turn.
551
+ const workspacePersistence =
552
+ persistable.state?.workspacePersistence ??
553
+ persistable.state?.providerState?.workspacePersistence;
509
554
  if (signal?.aborted) {
510
555
  return false;
511
556
  }
@@ -522,10 +567,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
522
567
  ) {
523
568
  return false;
524
569
  }
570
+ // A checkpoint of this exact mutation generation already protects every
571
+ // settled operation admitted so far. Capturing it again cannot improve the
572
+ // recovery point, even when the wall-clock interval has elapsed.
573
+ if (lease.archiveComplete) {
574
+ return false;
575
+ }
525
576
  const instanceId = lease.instanceId;
577
+ const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
578
+ if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
579
+ return false;
580
+ }
526
581
  const nativeModalPersistence =
527
- persistable.state?.workspacePersistence === "snapshot_filesystem" ||
528
- persistable.state?.workspacePersistence === "snapshot_directory";
582
+ workspacePersistence === "snapshot_filesystem" ||
583
+ workspacePersistence === "snapshot_directory";
529
584
  const checkpointBinding = nativeModalPersistence
530
585
  ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
531
586
  : null;
@@ -543,6 +598,8 @@ export async function maybePersistWarmWorkspaceSnapshot(
543
598
  liveness: "warm",
544
599
  captureTimeoutMs,
545
600
  minIntervalMs: force ? 0 : intervalMs,
601
+ providerReplaySafe: capturePolicy.takeover === "same_request",
602
+ takeoverSafe: capturePolicy.takeover !== "exclusive",
546
603
  warmAttempt: {
547
604
  sessionId: ids.sessionId,
548
605
  turnId: ids.turnId,
@@ -601,7 +658,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
601
658
  const captureAndPublish = (async (): Promise<boolean> => {
602
659
  let candidate: { id: string } | null = null;
603
660
  try {
604
- const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs);
661
+ const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
662
+ requestId: claimed.claim.providerRequestId,
663
+ strategy: capturePolicy.strategy,
664
+ });
605
665
  candidate = await registerCandidate(archive);
606
666
  const { wrote } = await persistWarmSnapshot(db, {
607
667
  accountId: ids.accountId,
@@ -716,37 +776,74 @@ export async function resumeBoxForTurn(
716
776
  // It is also bound directly to the logical attempt signal: an uninterruptible
717
777
  // provider promise must never keep its private warmup timer and holder alive
718
778
  // after Temporal has abandoned the activity.
719
- let released = false;
779
+ let releaseStarted = false;
780
+ let holderRelease: Promise<void> | null = null;
781
+ let quiescedRelease: Promise<void> | null = null;
720
782
  let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
721
783
  let cancellationListener: (() => void) | undefined;
722
- const release = async (): Promise<void> => {
723
- if (released) {
724
- return;
784
+ const release = (options?: { workspaceWritersQuiesced?: boolean }): Promise<void> => {
785
+ const writersQuiesced = options?.workspaceWritersQuiesced === true;
786
+ // First invocation synchronously closes every in-memory liveness source.
787
+ // A later quiesced invocation is still allowed to enter the DB: cancellation
788
+ // commonly wins this race and drops the holder before the physical writer
789
+ // drain can prove that abandoned turn admissions are safe to settle.
790
+ if (!releaseStarted) {
791
+ releaseStarted = true;
792
+ if (cancellationListener) {
793
+ cancellationSignal?.removeEventListener("abort", cancellationListener);
794
+ cancellationListener = undefined;
795
+ }
796
+ if (holderLivenessTimer) {
797
+ clearInterval(holderLivenessTimer);
798
+ holderLivenessTimer = undefined;
799
+ }
725
800
  }
726
- released = true;
727
- if (cancellationListener) {
728
- cancellationSignal?.removeEventListener("abort", cancellationListener);
729
- cancellationListener = undefined;
801
+ const persistRelease = async (workspaceWritersQuiesced: boolean): Promise<void> => {
802
+ await releaseLeaseHolder(db, {
803
+ accountId: ids.accountId,
804
+ workspaceId: ids.workspaceId,
805
+ sandboxGroupId: ids.sandboxGroupId,
806
+ kind,
807
+ holderId,
808
+ idleGraceMs: settings.sandboxIdleGraceMs,
809
+ ...(workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}),
810
+ });
811
+ };
812
+ if (writersQuiesced) {
813
+ if (!quiescedRelease) {
814
+ // Serialize behind an eager cancellation release when one exists. Its
815
+ // failure is intentionally swallowed only as a predecessor: this exact
816
+ // idempotent proof-bearing call retries both holder deletion and
817
+ // admission settlement in one transaction.
818
+ const predecessor = holderRelease;
819
+ const attempt = (predecessor ? predecessor.catch(() => undefined) : Promise.resolve()).then(
820
+ async () => await persistRelease(true),
821
+ );
822
+ const tracked = attempt.catch((error) => {
823
+ if (quiescedRelease === tracked) quiescedRelease = null;
824
+ throw error;
825
+ });
826
+ quiescedRelease = tracked;
827
+ holderRelease = tracked;
828
+ }
829
+ return quiescedRelease;
730
830
  }
731
- if (holderLivenessTimer) {
732
- clearInterval(holderLivenessTimer);
733
- holderLivenessTimer = undefined;
831
+ if (!holderRelease) {
832
+ const attempt = persistRelease(false);
833
+ const tracked = attempt.catch((error) => {
834
+ if (holderRelease === tracked) holderRelease = null;
835
+ throw error;
836
+ });
837
+ holderRelease = tracked;
734
838
  }
735
- await releaseLeaseHolder(db, {
736
- accountId: ids.accountId,
737
- workspaceId: ids.workspaceId,
738
- sandboxGroupId: ids.sandboxGroupId,
739
- kind,
740
- holderId,
741
- idleGraceMs: settings.sandboxIdleGraceMs,
742
- });
839
+ return holderRelease;
743
840
  };
744
841
  const cancellationError = (): Error =>
745
842
  cancellationSignal?.reason instanceof Error
746
843
  ? cancellationSignal.reason
747
844
  : new Error("Sandbox resume was cancelled with its owning turn attempt");
748
845
  const throwIfReleasedOrCancelled = (): void => {
749
- if (released || cancellationSignal?.aborted) {
846
+ if (releaseStarted || cancellationSignal?.aborted) {
750
847
  throw cancellationError();
751
848
  }
752
849
  };
@@ -772,15 +869,18 @@ export async function resumeBoxForTurn(
772
869
  // durable capture-and-drain rotation as an image change.
773
870
  ...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
774
871
  leaseTtlMs,
775
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
776
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
872
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
873
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
874
+ ...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
777
875
  });
778
876
 
779
877
  if (cancellationSignal) {
780
878
  cancellationListener = () => {
781
- // release() flips `released` and clears the timer synchronously before its
782
- // first await. The detached rejection handler is intentional: the outer
783
- // turn owns diagnostics, while this listener owns leak prevention.
879
+ // release() flips `releaseStarted` and clears the timer synchronously
880
+ // before its first await. This eager form deliberately carries no writer-
881
+ // quiescence proof; the outer turn supplies that second stage after its
882
+ // physical drain. The detached rejection handler is intentional: the
883
+ // outer turn owns diagnostics, while this listener owns leak prevention.
784
884
  void release().catch(() => undefined);
785
885
  };
786
886
  cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
@@ -790,7 +890,7 @@ export async function resumeBoxForTurn(
790
890
  throw cancellationError();
791
891
  }
792
892
 
793
- // HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
893
+ // HOLDER-LIVENESS loop: refresh OUR holder every 10s from registration.
794
894
  // registered until release. The dead-worker turn-holder reap judges liveness
795
895
  // by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
796
896
  // agent-turn) only starts AFTER this function returns — while waitForWarm and
@@ -798,21 +898,46 @@ export async function resumeBoxForTurn(
798
898
  // COMPOUNDING past any fixed reap horizon. With this
799
899
  // loop no live holder is ever silent for more than one tick, so the reap
800
900
  // horizon is pure defense-in-depth, not a tuned guess about path lengths.
801
- // Epoch-free by design (touch only refreshes our own row's timestamp);
802
- // best-effort (a transient DB error must never fail the resume).
901
+ // The elected spawner also heartbeats the short rolling warming lease; this
902
+ // makes warming expiry a crash detector rather than a fixed upper bound on a
903
+ // legitimate provider create/restore. A warming waiter only touches its own
904
+ // holder so it cannot keep a dead spawner's lease alive. Once the box is warm,
905
+ // every owner uses the normal epoch-fenced lease heartbeat.
906
+ let holderLeaseHeartbeat: { expectedEpoch: number; leaseTtlMs: number } | null =
907
+ acquired.role === "spawner"
908
+ ? {
909
+ expectedEpoch: acquired.lease.leaseEpoch,
910
+ leaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
911
+ }
912
+ : (acquired.role === "attached" || acquired.role === "rearmed") &&
913
+ acquired.lease.liveness === "warm"
914
+ ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
915
+ : null;
803
916
  holderLivenessTimer = setInterval(() => {
804
- void touchLeaseHolder(db, {
805
- accountId: ids.accountId,
806
- workspaceId: ids.workspaceId,
807
- sandboxGroupId: ids.sandboxGroupId,
808
- kind,
809
- holderId,
810
- })
917
+ const heartbeat = holderLeaseHeartbeat;
918
+ const refresh = heartbeat
919
+ ? heartbeatLeaseHolder(db, {
920
+ accountId: ids.accountId,
921
+ workspaceId: ids.workspaceId,
922
+ sandboxGroupId: ids.sandboxGroupId,
923
+ kind,
924
+ holderId,
925
+ expectedEpoch: heartbeat.expectedEpoch,
926
+ leaseTtlMs: heartbeat.leaseTtlMs,
927
+ })
928
+ : touchLeaseHolder(db, {
929
+ accountId: ids.accountId,
930
+ workspaceId: ids.workspaceId,
931
+ sandboxGroupId: ids.sandboxGroupId,
932
+ kind,
933
+ holderId,
934
+ });
935
+ void refresh
811
936
  .then((touched) => {
812
937
  // A canonical turn holder is rejected once its exact attempt is no
813
938
  // longer the active writer. Stop this otherwise-unbounded provider
814
939
  // operation and idempotently drop any remaining holder state.
815
- if (!touched) {
940
+ if (!touched && heartbeat === holderLeaseHeartbeat) {
816
941
  void release().catch(() => undefined);
817
942
  }
818
943
  })
@@ -836,7 +961,21 @@ export async function resumeBoxForTurn(
836
961
  }
837
962
  if (acquired.role === "fenced") {
838
963
  await release();
839
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
964
+ if (acquired.reason !== "superseded") {
965
+ throw new SandboxLeaseTransitionError(
966
+ ids.sandboxGroupId,
967
+ acquired.lease.leaseEpoch,
968
+ acquired.reason,
969
+ acquired.lease.backend,
970
+ acquired.lease.instanceId,
971
+ acquired.lease.liveness,
972
+ );
973
+ }
974
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
975
+ backend: acquired.lease.backend,
976
+ instanceId: acquired.lease.instanceId,
977
+ liveness: acquired.lease.liveness,
978
+ });
840
979
  }
841
980
 
842
981
  // SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
@@ -865,16 +1004,19 @@ export async function resumeBoxForTurn(
865
1004
  const fallbackArchiveEnvelope =
866
1005
  acquired.lease.recovery.archive.status === "none" &&
867
1006
  workspaceArchiveFieldsFromEnvelope(envelope) !== null
868
- ? withoutProviderIdentity(envelope)
1007
+ ? withoutSandboxProviderIdentity(envelope)
869
1008
  : null;
870
1009
  let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
871
1010
  const archiveSource =
872
1011
  acquired.lease.recovery.archive.status === "none"
873
1012
  ? fallbackArchiveEnvelope
874
1013
  : acquired.lease.resumeState;
1014
+ const continuityRecovery = acquired.lease.recovery.continuity;
875
1015
  if (
876
- acquired.lease.recovery.archive.status === "available" ||
877
- workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
1016
+ (acquired.lease.recovery.archive.status === "available" &&
1017
+ acquired.lease.archiveComplete) ||
1018
+ (acquired.lease.recovery.archive.status === "none" &&
1019
+ workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
878
1020
  ) {
879
1021
  const rematerializationId = crypto.randomUUID();
880
1022
  const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -917,7 +1059,7 @@ export async function resumeBoxForTurn(
917
1059
  legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
918
1060
  legacyProviderBinding: null,
919
1061
  };
920
- } else if (acquired.lease.recovery.archive.status !== "none") {
1062
+ } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
921
1063
  throw new SandboxLeaseRecoveryBlockedError(
922
1064
  ids.sandboxGroupId,
923
1065
  expectedEpoch,
@@ -941,6 +1083,9 @@ export async function resumeBoxForTurn(
941
1083
  backendOverride: ids.backend as never,
942
1084
  ...(ids.environment ? { environment: ids.environment } : {}),
943
1085
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
1086
+ ...(services.logicalFallbackSettings
1087
+ ? { logicalFallbackSettings: services.logicalFallbackSettings }
1088
+ : {}),
944
1089
  onSandboxCreated: async (created) => {
945
1090
  createdEstablished = created;
946
1091
  throwIfReleasedOrCancelled();
@@ -994,6 +1139,9 @@ export async function resumeBoxForTurn(
994
1139
  sandboxGroupId: ids.sandboxGroupId,
995
1140
  expectedEpoch,
996
1141
  rematerializationId: rematerialization?.id ?? null,
1142
+ ...(created.providerContinuity
1143
+ ? { continuityRecovery: created.providerContinuity }
1144
+ : {}),
997
1145
  instanceId: created.instanceId,
998
1146
  resumeBackendId: created.backendId,
999
1147
  resumeState: resumeEnvelope,
@@ -1008,7 +1156,7 @@ export async function resumeBoxForTurn(
1008
1156
  leaseTtlMs,
1009
1157
  // Keep the warming budget after create(): manifest setup and
1010
1158
  // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
1011
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
1159
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
1012
1160
  });
1013
1161
  if (!recorded.recorded) {
1014
1162
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
@@ -1074,7 +1222,9 @@ export async function resumeBoxForTurn(
1074
1222
  // is live. Do not publish a warm lease until one bounded no-op exec works.
1075
1223
  // On timeout the catch below terminates the box and rolls warming -> cold,
1076
1224
  // so the next turn cold-creates instead of hanging forever on first use.
1077
- await waitForSandboxExecReadiness(established);
1225
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1226
+ sandboxGroupId: ids.sandboxGroupId,
1227
+ });
1078
1228
  throwIfReleasedOrCancelled();
1079
1229
  // Fold the LIVE box into a re-resumable envelope and persist it as the
1080
1230
  // lease's resume_state — exactly like the API-direct paths (channel-a.ts /
@@ -1100,6 +1250,7 @@ export async function resumeBoxForTurn(
1100
1250
  throwIfReleasedOrCancelled();
1101
1251
  if (
1102
1252
  rematerialization &&
1253
+ !established.providerContinuity &&
1103
1254
  established.restoredArchive?.revision !== rematerialization.selectedRevision
1104
1255
  ) {
1105
1256
  throw new WorkspaceArchiveIntegrityError(
@@ -1118,14 +1269,16 @@ export async function resumeBoxForTurn(
1118
1269
  dataPlaneUrl: null,
1119
1270
  resumeBackendId: established.backendId,
1120
1271
  resumeState: resumeEnvelope,
1121
- ...(rematerialization
1122
- ? {
1123
- rematerialization: {
1124
- id: rematerialization.id,
1125
- verifiedRevision: rematerialization.selectedRevision,
1126
- },
1127
- }
1128
- : {}),
1272
+ ...(established.providerContinuity
1273
+ ? { continuityRecovery: established.providerContinuity }
1274
+ : rematerialization
1275
+ ? {
1276
+ rematerialization: {
1277
+ id: rematerialization.id,
1278
+ verifiedRevision: rematerialization.selectedRevision,
1279
+ },
1280
+ }
1281
+ : {}),
1129
1282
  leaseTtlMs,
1130
1283
  });
1131
1284
  if (!committed.committed || !committed.lease) {
@@ -1134,7 +1287,7 @@ export async function resumeBoxForTurn(
1134
1287
  // holder; surface supersession. This spawner created the box, so stop it
1135
1288
  // before retrying to avoid an untracked running sandbox.
1136
1289
  const terminated = await terminateEstablishedSandbox(established);
1137
- if (terminated && rematerialization) {
1290
+ if (terminated && rematerialization && !established.providerContinuity) {
1138
1291
  await failSandboxRematerialization(db, {
1139
1292
  accountId: ids.accountId,
1140
1293
  workspaceId: ids.workspaceId,
@@ -1144,10 +1297,21 @@ export async function resumeBoxForTurn(
1144
1297
  failureCode: committed.reason ?? "warm_commit_rejected",
1145
1298
  retryable: false,
1146
1299
  });
1300
+ } else if (terminated) {
1301
+ await failWarmingToCold(db, {
1302
+ accountId: ids.accountId,
1303
+ workspaceId: ids.workspaceId,
1304
+ sandboxGroupId: ids.sandboxGroupId,
1305
+ expectedEpoch,
1306
+ });
1147
1307
  }
1148
1308
  await release();
1149
1309
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1150
1310
  }
1311
+ holderLeaseHeartbeat = {
1312
+ expectedEpoch: committed.lease.leaseEpoch,
1313
+ leaseTtlMs,
1314
+ };
1151
1315
  throwIfReleasedOrCancelled();
1152
1316
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1153
1317
  } catch (error) {
@@ -1163,18 +1327,25 @@ export async function resumeBoxForTurn(
1163
1327
  // the warming row; the lease TTL/reaper and Modal orphan sweep are the
1164
1328
  // tracked backstops, and we must not erase the only provider pointer.
1165
1329
  if (terminated) {
1166
- if (rematerialization) {
1330
+ const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
1331
+ if (rematerialization && !createdEstablished?.providerContinuity) {
1167
1332
  await failSandboxRematerialization(db, {
1168
1333
  accountId: ids.accountId,
1169
1334
  workspaceId: ids.workspaceId,
1170
1335
  sandboxGroupId: ids.sandboxGroupId,
1171
1336
  expectedEpoch,
1172
1337
  rematerializationId: rematerialization.id,
1173
- failureCode:
1174
- error instanceof WorkspaceArchiveIntegrityError
1338
+ failureCode: continuityUnavailable
1339
+ ? "provider_continuity_unavailable"
1340
+ : error instanceof WorkspaceArchiveIntegrityError
1175
1341
  ? error.code
1176
1342
  : "sandbox_rematerialization_failed",
1177
- retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
1343
+ retryable: continuityUnavailable
1344
+ ? error.retryable
1345
+ : error instanceof WorkspaceArchiveIntegrityError
1346
+ ? error.retryable
1347
+ : true,
1348
+ ...(continuityUnavailable ? { discardContinuity: true } : {}),
1178
1349
  });
1179
1350
  } else {
1180
1351
  await failWarmingToCold(db, {
@@ -1182,17 +1353,13 @@ export async function resumeBoxForTurn(
1182
1353
  workspaceId: ids.workspaceId,
1183
1354
  sandboxGroupId: ids.sandboxGroupId,
1184
1355
  expectedEpoch,
1356
+ ...(continuityUnavailable ? { discardContinuity: true } : {}),
1185
1357
  });
1186
1358
  }
1187
1359
  }
1188
1360
  await release();
1189
- const warmingError = asSandboxWarmingError(
1190
- error,
1191
- ids.backend,
1192
- settings.sandboxWarmingTimeoutMs,
1193
- );
1194
- recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1195
- throw warmingError;
1361
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1362
+ throw error;
1196
1363
  }
1197
1364
  }
1198
1365
 
@@ -1205,6 +1372,7 @@ export async function resumeBoxForTurn(
1205
1372
  let leaseEpoch = acquired.lease.leaseEpoch;
1206
1373
  if (acquired.lease.liveness === "warming") {
1207
1374
  leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1375
+ holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
1208
1376
  }
1209
1377
  throwIfReleasedOrCancelled();
1210
1378
 
@@ -1237,7 +1405,13 @@ export async function resumeBoxForTurn(
1237
1405
  // was idle. Prove the command router before handing the session to the
1238
1406
  // agent so terminal evidence enters the atomic warm->cold recovery path
1239
1407
  // below instead of surfacing inside a model-visible tool call.
1240
- await (services.verifyAttachedSandboxReadiness ?? waitForSandboxExecReadiness)(established);
1408
+ if (services.verifyAttachedSandboxReadiness) {
1409
+ await services.verifyAttachedSandboxReadiness(established);
1410
+ } else {
1411
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1412
+ sandboxGroupId: ids.sandboxGroupId,
1413
+ });
1414
+ }
1241
1415
  throwIfReleasedOrCancelled();
1242
1416
  } catch (error) {
1243
1417
  if (!isProviderSandboxNotFoundError(ids.backend, error)) {
@@ -1271,13 +1445,8 @@ export async function resumeBoxForTurn(
1271
1445
  return { established, leaseEpoch, release };
1272
1446
  } catch (error) {
1273
1447
  await release();
1274
- const warmingError = asSandboxWarmingError(
1275
- error,
1276
- ids.backend,
1277
- settings.sandboxWarmingTimeoutMs,
1278
- );
1279
- recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1280
- throw warmingError;
1448
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1449
+ throw error;
1281
1450
  }
1282
1451
  }
1283
1452
 
@@ -1293,6 +1462,7 @@ async function waitForWarm(
1293
1462
  ): Promise<{ leaseEpoch: number }> {
1294
1463
  const { db, settings } = services;
1295
1464
  const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1465
+ let instanceId: string | null = null;
1296
1466
  while (Date.now() < deadline) {
1297
1467
  await sleep(WARMING_POLL_INTERVAL_MS);
1298
1468
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
@@ -1300,6 +1470,7 @@ async function waitForWarm(
1300
1470
  // Lease vanished (cold-reaped). Re-dispatch from scratch.
1301
1471
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1302
1472
  }
1473
+ instanceId = lease.instanceId;
1303
1474
  if (lease.liveness === "warm") {
1304
1475
  return { leaseEpoch: lease.leaseEpoch };
1305
1476
  }
@@ -1335,5 +1506,8 @@ async function waitForWarm(
1335
1506
  }
1336
1507
  // still warming — keep polling.
1337
1508
  }
1338
- throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
1509
+ throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
1510
+ sandboxGroupId: ids.sandboxGroupId,
1511
+ instanceId,
1512
+ });
1339
1513
  }