@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +66 -0
  42. package/dist/activities/image-generation-references.d.ts +2 -0
  43. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  44. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  45. package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
  46. package/dist/activities/knowledge-source-sync.d.ts +33 -1
  47. package/dist/activities/mcp-credentials.d.ts +9 -3
  48. package/dist/activities/open-suffix-resume.d.ts +77 -0
  49. package/dist/activities/packs.d.ts +3 -11
  50. package/dist/activities/parent-wake.d.ts +19 -0
  51. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  52. package/dist/activities/retained-screenshots.d.ts +1 -1
  53. package/dist/activities/rig-verification.d.ts +1 -1
  54. package/dist/activities/run-input.d.ts +16 -7
  55. package/dist/activities/sandbox-images.d.ts +1 -14
  56. package/dist/activities/sandbox-lease.d.ts +21 -3
  57. package/dist/activities/scheduled-tasks.d.ts +47 -0
  58. package/dist/activities/types.d.ts +23 -2
  59. package/dist/activities/video-media-validation.d.ts +1 -1
  60. package/dist/activities/video-output-retention.d.ts +1 -2
  61. package/dist/activities/xai-auth.d.ts +3 -0
  62. package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
  63. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  64. package/dist/activities-control.d.ts +6 -2
  65. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  66. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  67. package/dist/activities.d.ts +7 -3
  68. package/dist/artifact-materializer-entry.js +41 -22
  69. package/dist/artifact-materializer-entry.js.map +1 -1
  70. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  71. package/dist/chunk-CE7TPKBS.js.map +1 -0
  72. package/dist/chunk-CNYGVRKF.js +4698 -0
  73. package/dist/chunk-CNYGVRKF.js.map +1 -0
  74. package/dist/connection-provider-use.d.ts +18 -0
  75. package/dist/editable-artifact-materializer.d.ts +4 -3
  76. package/dist/github-rest-mcp.d.ts +26 -0
  77. package/dist/index.d.ts +10 -2
  78. package/dist/index.js +58 -14
  79. package/dist/index.js.map +1 -1
  80. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  81. package/dist/model-context-contributions.d.ts +8 -4
  82. package/dist/observability-metrics.d.ts +63 -4
  83. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  84. package/dist/personal-github-git-credentials.d.ts +9 -0
  85. package/dist/pr-review-credentials.d.ts +6 -0
  86. package/dist/sandbox-archive-storage.d.ts +45 -0
  87. package/dist/sandbox-resume.d.ts +65 -6
  88. package/dist/sandbox-routing.d.ts +38 -2
  89. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  90. package/dist/workflow-bundle.js +87 -28
  91. package/dist/workflows/activities.d.ts +1 -0
  92. package/dist/workflows/automations.d.ts +6 -0
  93. package/dist/workflows/document-index.d.ts +1 -0
  94. package/dist/workflows.d.ts +1 -0
  95. package/package.json +17 -17
  96. package/src/activities/agent-turn/admission.ts +292 -0
  97. package/src/activities/agent-turn/agent-build.ts +802 -0
  98. package/src/activities/agent-turn/claim.ts +590 -0
  99. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  100. package/src/activities/agent-turn/codex.ts +94 -0
  101. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  102. package/src/activities/agent-turn/credential-leases.ts +240 -0
  103. package/src/activities/agent-turn/errors.ts +1068 -0
  104. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  105. package/src/activities/agent-turn/file-resources.ts +227 -0
  106. package/src/activities/agent-turn/finalization.ts +754 -0
  107. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  108. package/src/activities/agent-turn/governance-model.ts +465 -0
  109. package/src/activities/agent-turn/history-sink.ts +139 -0
  110. package/src/activities/agent-turn/history.ts +200 -0
  111. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  112. package/src/activities/agent-turn/model-usage.ts +823 -0
  113. package/src/activities/agent-turn/quiescence.ts +353 -0
  114. package/src/activities/agent-turn/run-credentials.ts +761 -0
  115. package/src/activities/agent-turn/run.ts +1530 -0
  116. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  117. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  118. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  119. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  120. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  121. package/src/activities/agent-turn/tool-environment.ts +729 -0
  122. package/src/activities/agent-turn/tool-policy.ts +216 -0
  123. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  124. package/src/activities/agent-turn/turn-context.ts +281 -0
  125. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  126. package/src/activities/agent-turn.ts +12 -12575
  127. package/src/activities/api-integrations.ts +3 -0
  128. package/src/activities/automations.ts +214 -0
  129. package/src/activities/codemode-dispatcher.ts +12 -1
  130. package/src/activities/common.ts +1 -5
  131. package/src/activities/connector-attachments.ts +168 -0
  132. package/src/activities/context-compaction.ts +18 -0
  133. package/src/activities/documents.ts +6 -4
  134. package/src/activities/environment.ts +151 -55
  135. package/src/activities/generated-images.ts +26 -11
  136. package/src/activities/git-credential-renewal.ts +13 -3
  137. package/src/activities/goals.ts +85 -19
  138. package/src/activities/google-drive-provider.ts +178 -0
  139. package/src/activities/google-drive-publication.ts +938 -0
  140. package/src/activities/image-generation-references.ts +11 -2
  141. package/src/activities/incident-telemetry-authority.ts +327 -0
  142. package/src/activities/incident-telemetry-preflight.ts +317 -0
  143. package/src/activities/knowledge-source-sync-driver.ts +23 -0
  144. package/src/activities/knowledge-source-sync.ts +556 -40
  145. package/src/activities/mcp-credentials.ts +191 -17
  146. package/src/activities/open-suffix-resume.ts +422 -0
  147. package/src/activities/packs.ts +12 -60
  148. package/src/activities/parent-wake.ts +64 -14
  149. package/src/activities/retained-screenshot-reaper.ts +6 -3
  150. package/src/activities/retained-screenshots.ts +15 -47
  151. package/src/activities/rig-verification.ts +1 -1
  152. package/src/activities/run-input.ts +128 -136
  153. package/src/activities/sandbox-images.ts +7 -166
  154. package/src/activities/sandbox-lease.ts +406 -15
  155. package/src/activities/scheduled-tasks.ts +2103 -411
  156. package/src/activities/session-state.ts +10 -2
  157. package/src/activities/types.ts +32 -2
  158. package/src/activities/video-media-validation.ts +3 -3
  159. package/src/activities/video-output-retention.ts +2 -57
  160. package/src/activities/xai-auth.ts +8 -0
  161. package/src/activities-control.ts +2 -0
  162. package/src/activities.ts +3 -0
  163. package/src/activity-services.ts +13 -4
  164. package/src/connection-provider-use.ts +36 -0
  165. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  166. package/src/editable-artifact-materializer.ts +29 -18
  167. package/src/github-rest-mcp.ts +713 -0
  168. package/src/http.ts +16 -6
  169. package/src/index.ts +40 -5
  170. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  171. package/src/model-context-contributions.ts +33 -10
  172. package/src/observability-metrics.ts +357 -0
  173. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  174. package/src/personal-github-git-credentials.ts +331 -0
  175. package/src/pr-review-credentials.ts +163 -0
  176. package/src/sandbox-archive-storage.ts +159 -0
  177. package/src/sandbox-resume.ts +449 -63
  178. package/src/sandbox-routing.ts +314 -16
  179. package/src/scheduled-alert-occurrence.ts +176 -0
  180. package/src/workflows/activities.ts +18 -2
  181. package/src/workflows/automations.ts +11 -0
  182. package/src/workflows/session.ts +13 -3
  183. package/src/workflows.ts +1 -0
  184. package/dist/activities-control-SMKF2OBE.js.map +0 -1
  185. package/dist/activities-turn-IDS3WEHA.js +0 -16532
  186. package/dist/activities-turn-IDS3WEHA.js.map +0 -1
  187. package/dist/chunk-N5TJIFCE.js.map +0 -1
  188. package/dist/chunk-P6Z2SLAA.js +0 -3092
  189. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -21,6 +21,7 @@
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
23
  import {
24
+ effectiveSandboxLifecycle,
24
25
  sandboxArchiveCaptureTimeoutMs,
25
26
  sandboxLifecycleTransitionWaitMs,
26
27
  type Settings,
@@ -35,7 +36,7 @@ import {
35
36
  failSandboxRematerialization,
36
37
  failWarmingToCold,
37
38
  getSandboxSessionEnvelope,
38
- heartbeatLeaseHolder,
39
+ heartbeatLeaseHolderStatus,
39
40
  markSandboxRestoreVerifying,
40
41
  markWarmLeaseInstanceLost,
41
42
  markSandboxCheckpointArtifactDeletePending,
@@ -43,6 +44,7 @@ import {
43
44
  readLease,
44
45
  registerSandboxCheckpointArtifact,
45
46
  recordWarmingSandboxCreated,
47
+ reinstateTurnLeaseHolder,
46
48
  releaseWorkspaceArchiveCapture,
47
49
  releaseLeaseHolder,
48
50
  touchLeaseHolder,
@@ -55,6 +57,7 @@ import {
55
57
  import {
56
58
  captureVerifiedWorkspaceArchive,
57
59
  describeLegacyNativeSnapshotArchive,
60
+ inlineWorkspaceArchiveForRestore,
58
61
  MODAL_EXEC_READINESS_TIMEOUT_MS,
59
62
  SandboxExecReadinessError,
60
63
  WorkspaceArchiveIntegrityError,
@@ -64,6 +67,7 @@ import {
64
67
  providerWorkspaceCapturePolicy,
65
68
  SandboxProviderContinuityUnavailableError,
66
69
  requirePersistableReplacementSandboxEnvelope,
70
+ renewSandboxProviderExpiration,
67
71
  modalSessionMatchesCheckpointProviderBinding,
68
72
  resolveModalCheckpointProviderBindingForSession,
69
73
  serializeReplacementSandboxEnvelope,
@@ -75,6 +79,15 @@ import {
75
79
  type RuntimeMetricsHooks,
76
80
  type WorkspaceArchiveDescriptor,
77
81
  } from "@opengeni/runtime";
82
+ import type { ObjectStorage } from "@opengeni/storage";
83
+ import type { Observability } from "@opengeni/observability";
84
+ import { parseWorkspaceArchiveObjectRef } from "@opengeni/contracts";
85
+ import {
86
+ collectWorkspaceArchiveObjectKeys,
87
+ deleteUnpublishedWorkspaceArchiveObject,
88
+ deleteWorkspaceArchiveObjectKeys,
89
+ putVersion1TarArchiveOrInline,
90
+ } from "./sandbox-archive-storage";
78
91
 
79
92
  // Re-exported for callers that just want the ack-kind union.
80
93
  export type ResumeHolderKind = LeaseHolderKind;
@@ -107,11 +120,18 @@ export function isRetryableDegradedRestore(restore: {
107
120
  export type SandboxResumeServices = {
108
121
  db: Database;
109
122
  settings: Settings;
123
+ objectStorage?: ObjectStorage | null;
110
124
  /** Exact settings before a verified rig provider image overlaid the logical
111
125
  * pack/deployment image. Fresh-create NotFound fallback must preserve an
112
126
  * ID-only logical base instead of selecting the provider default. */
113
127
  logicalFallbackSettings?: Settings;
114
128
  sandboxMetrics?: RuntimeMetricsHooks;
129
+ /** Structured logger for lifecycle decisions (holder release reasons). Falls
130
+ * back to the console when a lean caller passes none. */
131
+ observability?: Pick<Observability, "info" | "warn">;
132
+ /** Test seam only: cadence of the private holder-liveness loop (default 10 s).
133
+ * Production keeps the default; the dead-worker reap horizon assumes it. */
134
+ holderLivenessIntervalMs?: number;
115
135
  /**
116
136
  * The logical turn-attempt lifetime, not merely the provider request lifetime.
117
137
  * Some provider SDK calls cannot be interrupted. Aborting this signal still
@@ -140,7 +160,7 @@ export type ResumeBoxIds = {
140
160
  * disclosure/attribution). For a singleton group this == sandboxGroupId. */
141
161
  sessionId: string;
142
162
  /** The backend the box runs on (sessions.sandbox_backend). */
143
- backend: string;
163
+ backend: Settings["sandboxBackend"];
144
164
  /** The OS axis (sessions.sandbox_os); default 'linux'. */
145
165
  os?: string;
146
166
  /**
@@ -189,6 +209,44 @@ export type ResumedTurnSandbox = {
189
209
  release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
190
210
  };
191
211
 
212
+ /**
213
+ * Structural ownership of the provider boundary that failed while establishing
214
+ * a logical sandbox provision. Object errors retain their identity and receive
215
+ * an out-of-band stage; primitive thrown values use a typed wrapper. In both
216
+ * cases the source diagnostic is unchanged and the stage is never guessed from
217
+ * prose.
218
+ */
219
+ export type SandboxProvisionFailureStage = "create" | "resume" | "archive_recovery";
220
+
221
+ export class SandboxProvisionStageError extends Error {
222
+ readonly name = "SandboxProvisionStageError";
223
+
224
+ constructor(
225
+ public readonly stage: SandboxProvisionFailureStage,
226
+ public readonly source: unknown,
227
+ ) {
228
+ super(source instanceof Error ? source.message : String(source), { cause: source });
229
+ }
230
+ }
231
+
232
+ const sandboxProvisionFailureStages = new WeakMap<object, SandboxProvisionFailureStage>();
233
+
234
+ export function sandboxProvisionFailureStage(error: unknown): SandboxProvisionFailureStage | null {
235
+ if (!error || typeof error !== "object") return null;
236
+ return (
237
+ sandboxProvisionFailureStages.get(error) ??
238
+ (error instanceof SandboxProvisionStageError ? error.stage : null)
239
+ );
240
+ }
241
+
242
+ function sandboxProvisionStageError(stage: SandboxProvisionFailureStage, error: unknown): unknown {
243
+ if (error && typeof error === "object") {
244
+ sandboxProvisionFailureStages.set(error, stage);
245
+ return error;
246
+ }
247
+ return new SandboxProvisionStageError(stage, error);
248
+ }
249
+
192
250
  export abstract class SandboxWarmingTimeoutError extends Error {
193
251
  abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
194
252
 
@@ -277,10 +335,10 @@ async function sleep(ms: number): Promise<void> {
277
335
  }
278
336
 
279
337
  /**
280
- * Modal may return a sandbox handle before its command router accepts the first
281
- * exec. The upstream session's yieldTimeMs starts only after sandbox.exec()
282
- * returns, so it cannot bound that initial RPC. Probe it before publishing the
283
- * lease as warm and enforce our own wall-clock deadline.
338
+ * A remote provider may return a sandbox handle before its command router
339
+ * accepts the first exec. The upstream session's yieldTimeMs starts only after
340
+ * sandbox.exec() returns, so it cannot bound that initial RPC. Probe it before
341
+ * publishing the lease as warm and enforce our own wall-clock deadline.
284
342
  */
285
343
  export async function waitForSandboxExecReadiness(
286
344
  established: EstablishedSandboxSession,
@@ -312,25 +370,42 @@ export function safeSnapshotError(error: unknown): {
312
370
  errorCode: "snapshot_operation_failed";
313
371
  status?: number;
314
372
  origin: "sandbox-resume";
373
+ causeName?: string;
374
+ integrityCode?: string;
315
375
  } {
316
376
  const fields: {
317
377
  errorClass: "SnapshotOperationError";
318
378
  errorCode: "snapshot_operation_failed";
319
379
  status?: number;
320
380
  origin: "sandbox-resume";
381
+ causeName?: string;
382
+ integrityCode?: string;
321
383
  } = {
322
384
  errorClass: "SnapshotOperationError",
323
385
  errorCode: "snapshot_operation_failed",
324
386
  origin: "sandbox-resume",
325
387
  };
326
388
  try {
327
- const rawStatus =
328
- error && typeof error === "object"
329
- ? ((error as { status?: unknown; statusCode?: unknown }).status ??
330
- (error as { statusCode?: unknown }).statusCode)
331
- : undefined;
332
- const status = Number(rawStatus);
333
- if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
389
+ if (error && typeof error === "object") {
390
+ const candidate = error as {
391
+ name?: unknown;
392
+ code?: unknown;
393
+ status?: unknown;
394
+ statusCode?: unknown;
395
+ };
396
+ if (
397
+ typeof candidate.name === "string" &&
398
+ /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
399
+ ) {
400
+ fields.causeName = candidate.name;
401
+ }
402
+ if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
403
+ fields.integrityCode = candidate.code;
404
+ }
405
+ const rawStatus = candidate.status ?? candidate.statusCode;
406
+ const status = Number(rawStatus);
407
+ if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
408
+ }
334
409
  } catch {
335
410
  // Public diagnostics are best-effort and must never replace the exact
336
411
  // internal snapshot failure.
@@ -461,19 +536,24 @@ function workspaceArchiveFieldsFromEnvelope(
461
536
  ? (envelope.sessionState as Record<string, unknown>)
462
537
  : null;
463
538
  const archive = sessionState?.workspaceArchive;
464
- if (typeof archive !== "string" || archive.length === 0) {
539
+ const archiveRef = parseWorkspaceArchiveObjectRef(sessionState?.workspaceArchiveRef);
540
+ const hasInline = typeof archive === "string" && archive.length > 0;
541
+ if (!hasInline && !archiveRef) {
465
542
  return null;
466
543
  }
467
544
  const previous = sessionState?.workspaceArchivePrev;
545
+ const previousRef = parseWorkspaceArchiveObjectRef(sessionState?.workspaceArchivePrevRef);
468
546
  const metadata = sessionState?.workspaceArchiveMeta;
469
547
  const previousMetadata = sessionState?.workspaceArchivePrevMeta;
470
548
  const capturedAt = sessionState?.workspaceArchiveAt;
471
549
  return {
472
- workspaceArchive: archive,
550
+ ...(hasInline && !archiveRef ? { workspaceArchive: archive } : {}),
551
+ ...(archiveRef ? { workspaceArchiveRef: archiveRef } : {}),
473
552
  ...(metadata !== undefined ? { workspaceArchiveMeta: metadata } : {}),
474
- ...(typeof previous === "string" && previous.length > 0
553
+ ...(typeof previous === "string" && previous.length > 0 && !previousRef
475
554
  ? { workspaceArchivePrev: previous }
476
555
  : {}),
556
+ ...(previousRef ? { workspaceArchivePrevRef: previousRef } : {}),
477
557
  ...(previousMetadata !== undefined ? { workspaceArchivePrevMeta: previousMetadata } : {}),
478
558
  ...(typeof capturedAt === "string" && capturedAt.length > 0
479
559
  ? { workspaceArchiveAt: capturedAt }
@@ -481,6 +561,35 @@ function workspaceArchiveFieldsFromEnvelope(
481
561
  };
482
562
  }
483
563
 
564
+ async function materializeSpawnEnvelopeArchive(
565
+ envelope: unknown,
566
+ objectStorage: ObjectStorage | null | undefined,
567
+ ): Promise<unknown> {
568
+ if (!envelope || typeof envelope !== "object") return envelope;
569
+ const record = envelope as Record<string, unknown>;
570
+ const sessionState =
571
+ record.sessionState &&
572
+ typeof record.sessionState === "object" &&
573
+ !Array.isArray(record.sessionState)
574
+ ? (record.sessionState as Record<string, unknown>)
575
+ : null;
576
+ if (!sessionState || !parseWorkspaceArchiveObjectRef(sessionState.workspaceArchiveRef)) {
577
+ return envelope;
578
+ }
579
+ if (!objectStorage) {
580
+ throw new WorkspaceArchiveIntegrityError(
581
+ "archive_base64_invalid",
582
+ "workspace archive object storage is not configured",
583
+ );
584
+ }
585
+ return {
586
+ ...record,
587
+ sessionState: await inlineWorkspaceArchiveForRestore(sessionState, (key) =>
588
+ objectStorage.getObjectBytes(key),
589
+ ),
590
+ };
591
+ }
592
+
484
593
  function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined) {
485
594
  const sessionState =
486
595
  envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
@@ -495,6 +604,93 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
495
604
  );
496
605
  }
497
606
 
607
+ export type SandboxDeadlineRotationCheckpointResult = {
608
+ /** The exact established epoch still carries a requested rotation and a
609
+ * complete current-generation archive: the turn may abort orderly. */
610
+ checkpointed: boolean;
611
+ /** What the defense-in-depth holder step observed before the capture. */
612
+ holder: Awaited<ReturnType<typeof reinstateTurnLeaseHolder>>["status"];
613
+ lease: Awaited<ReturnType<typeof readLease>>;
614
+ };
615
+
616
+ /**
617
+ * The orderly provider-deadline rotation checkpoint for one running turn. The
618
+ * turn-side lease heartbeat calls this once it observes a rotation-fenced lease
619
+ * at its established epoch/instance; `checkpointed: true` licenses the
620
+ * SandboxDeadlineRotationError abort that hands the same logical turn to the
621
+ * cold successor.
622
+ *
623
+ * Defense in depth: the warm capture claim requires exactly this turn's holder.
624
+ * If that holder was lost while the attempt is still the active writer (a stale
625
+ * release, a reap race), `reinstateTurnLeaseHolder` puts it back under the
626
+ * exact epoch/instance fence before the capture, so a lost holder cannot
627
+ * silently disable the orderly abort and leave the reaper to terminate the box
628
+ * under the running turn. Never captures when the epoch/instance changed.
629
+ */
630
+ export async function persistSandboxDeadlineRotationCheckpoint(
631
+ services: SandboxResumeServices,
632
+ ids: {
633
+ accountId: string;
634
+ workspaceId: string;
635
+ sessionId: string;
636
+ turnId: string;
637
+ attemptId: string;
638
+ sandboxGroupId: string;
639
+ },
640
+ session: unknown,
641
+ expected: { leaseEpoch: number; instanceId: string },
642
+ signal?: AbortSignal,
643
+ ): Promise<SandboxDeadlineRotationCheckpointResult> {
644
+ const { db, settings } = services;
645
+ // A cancelled attempt must never briefly regain its holder: its eager
646
+ // cancellation listener may already have released it, and reinstating here
647
+ // would pin the lease behind an attempt that is being torn down.
648
+ if (signal?.aborted) {
649
+ return { checkpointed: false, holder: "attempt_fenced", lease: null };
650
+ }
651
+ const reinstated = await reinstateTurnLeaseHolder(db, {
652
+ accountId: ids.accountId,
653
+ workspaceId: ids.workspaceId,
654
+ sandboxGroupId: ids.sandboxGroupId,
655
+ sessionId: ids.sessionId,
656
+ attemptId: ids.attemptId,
657
+ expectedEpoch: expected.leaseEpoch,
658
+ expectedInstanceId: expected.instanceId,
659
+ leaseTtlMs: settings.sandboxLeaseTtlMs,
660
+ });
661
+ if (reinstated.status === "reinstated") {
662
+ const message = "sandbox rotation checkpoint reinstated the live turn holder";
663
+ const fields = {
664
+ workspaceId: ids.workspaceId,
665
+ sessionId: ids.sessionId,
666
+ sandboxGroupId: ids.sandboxGroupId,
667
+ leaseEpoch: expected.leaseEpoch,
668
+ liveness: reinstated.lease.liveness,
669
+ };
670
+ if (services.observability) services.observability.warn(message, fields);
671
+ else console.warn(message, fields);
672
+ }
673
+ if (reinstated.status !== "present" && reinstated.status !== "reinstated") {
674
+ return { checkpointed: false, holder: reinstated.status, lease: null };
675
+ }
676
+ await maybePersistWarmWorkspaceSnapshot(
677
+ services,
678
+ ids,
679
+ session,
680
+ expected.leaseEpoch,
681
+ signal,
682
+ true,
683
+ );
684
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
685
+ const checkpointed =
686
+ lease?.leaseEpoch === expected.leaseEpoch &&
687
+ lease.instanceId === expected.instanceId &&
688
+ lease.rotationRequestedAt !== null &&
689
+ lease.archiveComplete &&
690
+ signal?.aborted !== true;
691
+ return { checkpointed, holder: reinstated.status, lease };
692
+ }
693
+
498
694
  /**
499
695
  * MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
500
696
  * drain-persist only protects boxes the reaper itself kills; anything else —
@@ -535,12 +731,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
535
731
  }
536
732
  const persistable = session as {
537
733
  persistWorkspace?: () => Promise<Uint8Array | undefined>;
734
+ persistWorkspaceTar?: () => Promise<Uint8Array | undefined>;
735
+ backendId?: unknown;
538
736
  state?: {
539
737
  workspacePersistence?: unknown;
540
738
  providerState?: { workspacePersistence?: unknown };
541
739
  };
542
740
  };
543
- if (typeof persistable.persistWorkspace !== "function") {
741
+ if (
742
+ typeof persistable.persistWorkspace !== "function" &&
743
+ typeof persistable.persistWorkspaceTar !== "function"
744
+ ) {
745
+ console.error("mid-session workspace snapshot skipped (no persist primitive)", {
746
+ backendId: typeof persistable.backendId === "string" ? persistable.backendId : null,
747
+ });
544
748
  return false;
545
749
  }
546
750
  // Filesystem and directory snapshots create retained Images without
@@ -559,12 +763,21 @@ export async function maybePersistWarmWorkspaceSnapshot(
559
763
  // This work cannot pause or mutate the box, so keeping it outside the claim
560
764
  // minimizes the time that command admission must wait.
561
765
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
562
- if (
563
- !lease ||
564
- lease.leaseEpoch !== leaseEpoch ||
565
- lease.liveness !== "warm" ||
566
- lease.instanceId === null
567
- ) {
766
+ const canWarmCapture =
767
+ lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
768
+ const canForceDrainingCapture =
769
+ force === true &&
770
+ lease?.liveness === "draining" &&
771
+ lease.instanceId !== null &&
772
+ lease.leaseEpoch === leaseEpoch;
773
+ if (!lease || (!canWarmCapture && !canForceDrainingCapture)) {
774
+ console.error("mid-session workspace snapshot skipped (lease not warm)", {
775
+ sandboxGroupId: ids.sandboxGroupId,
776
+ expectedEpoch: leaseEpoch,
777
+ liveness: lease?.liveness ?? null,
778
+ leaseEpoch: lease?.leaseEpoch ?? null,
779
+ hasInstance: lease?.instanceId !== null && lease?.instanceId !== undefined,
780
+ });
568
781
  return false;
569
782
  }
570
783
  // A checkpoint of this exact mutation generation already protects every
@@ -573,9 +786,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
573
786
  if (lease.archiveComplete) {
574
787
  return false;
575
788
  }
789
+ if (lease.instanceId === null) {
790
+ return false;
791
+ }
576
792
  const instanceId = lease.instanceId;
793
+ const captureLiveness: "warm" | "draining" = canWarmCapture ? "warm" : "draining";
577
794
  const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
578
795
  if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
796
+ console.error("mid-session workspace snapshot skipped (capture policy)", {
797
+ sandboxGroupId: ids.sandboxGroupId,
798
+ backend: lease.backend,
799
+ liveInstance: capturePolicy?.liveInstance ?? null,
800
+ });
579
801
  return false;
580
802
  }
581
803
  const nativeModalPersistence =
@@ -595,19 +817,29 @@ export async function maybePersistWarmWorkspaceSnapshot(
595
817
  captureId,
596
818
  expectedEpoch: leaseEpoch,
597
819
  expectedInstanceId: instanceId,
598
- liveness: "warm",
820
+ liveness: captureLiveness,
599
821
  captureTimeoutMs,
600
822
  minIntervalMs: force ? 0 : intervalMs,
601
823
  providerReplaySafe: capturePolicy.takeover === "same_request",
602
824
  takeoverSafe: capturePolicy.takeover !== "exclusive",
603
- warmAttempt: {
604
- sessionId: ids.sessionId,
605
- turnId: ids.turnId,
606
- attemptId: ids.attemptId,
607
- holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
608
- },
825
+ ...(captureLiveness === "warm"
826
+ ? {
827
+ warmAttempt: {
828
+ sessionId: ids.sessionId,
829
+ turnId: ids.turnId,
830
+ attemptId: ids.attemptId,
831
+ holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
832
+ },
833
+ }
834
+ : {}),
609
835
  });
610
- if (claimed.status !== "claimed") return false;
836
+ if (claimed.status !== "claimed") {
837
+ console.error("mid-session workspace snapshot skipped (capture claim)", {
838
+ sandboxGroupId: ids.sandboxGroupId,
839
+ status: claimed.status,
840
+ });
841
+ return false;
842
+ }
611
843
 
612
844
  let timeout: ReturnType<typeof setTimeout> | undefined;
613
845
  // Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
@@ -657,12 +889,33 @@ export async function maybePersistWarmWorkspaceSnapshot(
657
889
  // of the exact admission gate; a late callback cannot release a successor.
658
890
  const captureAndPublish = (async (): Promise<boolean> => {
659
891
  let candidate: { id: string } | null = null;
892
+ let workspaceArchiveRef: Awaited<
893
+ ReturnType<typeof putVersion1TarArchiveOrInline>
894
+ >["workspaceArchiveRef"];
660
895
  try {
661
896
  const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
662
897
  requestId: claimed.claim.providerRequestId,
663
898
  strategy: capturePolicy.strategy,
664
899
  });
665
900
  candidate = await registerCandidate(archive);
901
+ const priorLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
902
+ const priorKeys = collectWorkspaceArchiveObjectKeys(
903
+ (priorLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
904
+ );
905
+ const published = await putVersion1TarArchiveOrInline({
906
+ backend: lease.backend,
907
+ objectStorage: services.objectStorage,
908
+ accountId: ids.accountId,
909
+ workspaceId: ids.workspaceId,
910
+ sandboxGroupId: ids.sandboxGroupId,
911
+ archive: {
912
+ bytes: archive.bytes,
913
+ descriptor: archive.descriptor,
914
+ base64: archive.base64,
915
+ },
916
+ ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
917
+ });
918
+ workspaceArchiveRef = published.workspaceArchiveRef;
666
919
  const { wrote } = await persistWarmSnapshot(db, {
667
920
  accountId: ids.accountId,
668
921
  workspaceId: ids.workspaceId,
@@ -674,17 +927,40 @@ export async function maybePersistWarmWorkspaceSnapshot(
674
927
  expectedInstanceId: instanceId,
675
928
  expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
676
929
  captureId,
677
- workspaceArchive: archive.base64,
678
930
  workspaceArchiveMeta: archive.descriptor,
931
+ ...published,
679
932
  checkpointArtifactId: candidate?.id ?? null,
680
933
  minIntervalMs: force ? 0 : intervalMs,
681
934
  capturedAtMs,
682
935
  });
936
+ if (published.workspaceArchiveRef && services.objectStorage) {
937
+ if (!wrote) {
938
+ await deleteUnpublishedWorkspaceArchiveObject(
939
+ services.objectStorage,
940
+ published.workspaceArchiveRef,
941
+ services.sandboxMetrics,
942
+ );
943
+ } else {
944
+ const afterLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
945
+ const afterKeys = collectWorkspaceArchiveObjectKeys(
946
+ (afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
947
+ );
948
+ await deleteWorkspaceArchiveObjectKeys(
949
+ services.objectStorage,
950
+ [...priorKeys].filter((key) => !afterKeys.has(key)),
951
+ ).catch(() => undefined);
952
+ }
953
+ }
683
954
  if (!wrote && candidate) {
684
955
  await abandonCandidate(candidate.id, "snapshot_publication_fenced");
685
956
  }
686
957
  return wrote;
687
958
  } catch (error) {
959
+ await deleteUnpublishedWorkspaceArchiveObject(
960
+ services.objectStorage,
961
+ workspaceArchiveRef,
962
+ services.sandboxMetrics,
963
+ );
688
964
  if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
689
965
  throw error;
690
966
  } finally {
@@ -913,36 +1189,122 @@ export async function resumeBoxForTurn(
913
1189
  acquired.lease.liveness === "warm"
914
1190
  ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
915
1191
  : null;
1192
+ let providerRenewalTarget: { backend: Settings["sandboxBackend"]; instanceId: string } | null =
1193
+ acquired.lease.liveness === "warm" && acquired.lease.instanceId
1194
+ ? {
1195
+ backend: ids.backend,
1196
+ instanceId: acquired.lease.instanceId,
1197
+ }
1198
+ : null;
1199
+ let providerRenewedAtMs = Date.now();
1200
+ let providerRenewalInFlight: Promise<void> | null = null;
1201
+ const maybeRenewProviderExpiration = async (force = false): Promise<void> => {
1202
+ const target = providerRenewalTarget;
1203
+ if (!target || providerRenewalInFlight) return;
1204
+ const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
1205
+ if (ttlSeconds === null) return;
1206
+ const intervalMs = Math.max(10_000, Math.floor((ttlSeconds * 1000) / 3));
1207
+ if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
1208
+ providerRenewalInFlight = renewSandboxProviderExpiration({
1209
+ backend: target.backend,
1210
+ settings,
1211
+ instanceId: target.instanceId,
1212
+ ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
1213
+ })
1214
+ .then(() => {
1215
+ providerRenewedAtMs = Date.now();
1216
+ })
1217
+ .finally(() => {
1218
+ providerRenewalInFlight = null;
1219
+ });
1220
+ await providerRenewalInFlight;
1221
+ };
1222
+ // Release ONLY on holder death. `heartbeatLeaseHolderStatus` separates "this
1223
+ // holder is gone / its attempt is no longer the active writer" from "the lease
1224
+ // could not be extended" (rotation requested, draining, another epoch). The
1225
+ // second is NOT a reason to release: once a provider-deadline rotation is
1226
+ // requested the lease heartbeat is fenced by design while the still-running
1227
+ // turn checkpoints, and the turn-side heartbeat owns that orderly
1228
+ // SandboxDeadlineRotationError abort. Dropping the live holder here instead
1229
+ // flips the lease to draining under the running turn, the reaper terminates
1230
+ // the box, and the checkpoint then fails `attempt_fenced` (the staging
1231
+ // ~61-minute box-age failure). Both loops therefore converge on
1232
+ // "holder alive -> never release"; this loop keeps touching the holder.
1233
+ const releaseDeadHolder = (reason: string): void => {
1234
+ const message = "sandbox resume holder released: holder no longer live";
1235
+ const fields = {
1236
+ workspaceId: ids.workspaceId,
1237
+ sessionId: ids.sessionId,
1238
+ sandboxGroupId: ids.sandboxGroupId,
1239
+ holderKind: kind,
1240
+ reason,
1241
+ };
1242
+ if (services.observability) services.observability.warn(message, fields);
1243
+ else console.warn(message, fields);
1244
+ void release().catch(() => undefined);
1245
+ };
916
1246
  holderLivenessTimer = setInterval(() => {
917
1247
  const heartbeat = holderLeaseHeartbeat;
918
- const refresh = heartbeat
919
- ? heartbeatLeaseHolder(db, {
920
- accountId: ids.accountId,
921
- workspaceId: ids.workspaceId,
922
- sandboxGroupId: ids.sandboxGroupId,
923
- kind,
924
- holderId,
925
- expectedEpoch: heartbeat.expectedEpoch,
926
- leaseTtlMs: heartbeat.leaseTtlMs,
1248
+ if (heartbeat) {
1249
+ void heartbeatLeaseHolderStatus(db, {
1250
+ accountId: ids.accountId,
1251
+ workspaceId: ids.workspaceId,
1252
+ sandboxGroupId: ids.sandboxGroupId,
1253
+ kind,
1254
+ holderId,
1255
+ expectedEpoch: heartbeat.expectedEpoch,
1256
+ leaseTtlMs: heartbeat.leaseTtlMs,
1257
+ })
1258
+ .then((status) => {
1259
+ if (heartbeat !== holderLeaseHeartbeat) return;
1260
+ if (!status.holderAlive) {
1261
+ // Reaped, released, or its exact attempt is no longer the active
1262
+ // writer. Stop this otherwise-unbounded provider operation and
1263
+ // idempotently drop any remaining holder state.
1264
+ releaseDeadHolder(status.fence);
1265
+ return;
1266
+ }
1267
+ if (status.fence === "epoch") {
1268
+ // The lease epoch never regresses: a newer epoch means the box this
1269
+ // holder was registered against is definitively superseded (lost
1270
+ // instance replaced, re-established by a later turn). The holder
1271
+ // row is stale authority for an instance that no longer exists;
1272
+ // drop it promptly instead of pinning a successor's lease.
1273
+ releaseDeadHolder(status.fence);
1274
+ return;
1275
+ }
1276
+ // The box is still serving this live holder: keep the provider TTL
1277
+ // renewed while the lease is ours (extended, or only rotation-fenced at
1278
+ // the same epoch). A liveness fence (draining at our epoch) means the
1279
+ // reaper now governs the instance; keep the holder but leave its
1280
+ // expiry alone.
1281
+ if (status.leaseExtended || status.fence === "rotation_requested") {
1282
+ void maybeRenewProviderExpiration().catch(() => undefined);
1283
+ }
927
1284
  })
928
- : touchLeaseHolder(db, {
929
- accountId: ids.accountId,
930
- workspaceId: ids.workspaceId,
931
- sandboxGroupId: ids.sandboxGroupId,
932
- kind,
933
- holderId,
934
- });
935
- void refresh
1285
+ .catch(() => undefined);
1286
+ return;
1287
+ }
1288
+ void touchLeaseHolder(db, {
1289
+ accountId: ids.accountId,
1290
+ workspaceId: ids.workspaceId,
1291
+ sandboxGroupId: ids.sandboxGroupId,
1292
+ kind,
1293
+ holderId,
1294
+ })
936
1295
  .then((touched) => {
1296
+ if (heartbeat !== holderLeaseHeartbeat) return;
937
1297
  // A canonical turn holder is rejected once its exact attempt is no
938
- // longer the active writer. Stop this otherwise-unbounded provider
939
- // operation and idempotently drop any remaining holder state.
940
- if (!touched && heartbeat === holderLeaseHeartbeat) {
941
- void release().catch(() => undefined);
1298
+ // longer the active writer (touch is attempt-fenced only; it never
1299
+ // consults lease authority, so false here always means holder death).
1300
+ if (!touched) {
1301
+ releaseDeadHolder("holder_gone");
1302
+ return;
942
1303
  }
1304
+ void maybeRenewProviderExpiration().catch(() => undefined);
943
1305
  })
944
1306
  .catch(() => undefined);
945
- }, 10_000);
1307
+ }, services.holderLivenessIntervalMs ?? 10_000);
946
1308
  if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
947
1309
  holderLivenessTimer.unref();
948
1310
  }
@@ -1077,7 +1439,19 @@ export async function resumeBoxForTurn(
1077
1439
  // `_sandbox` envelope is the per-session fallback. Without this a turn-first
1078
1440
  // re-warm after a drain->cold would ignore the archive and start an EMPTY box.
1079
1441
  const providerCreateStartedAt = new Date();
1080
- const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
1442
+ const persistArchiveSource =
1443
+ spawnEnvelope && typeof spawnEnvelope === "object"
1444
+ ? (spawnEnvelope as Record<string, unknown>)
1445
+ : null;
1446
+ const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
1447
+ persistArchiveSource,
1448
+ services.objectStorage,
1449
+ );
1450
+ const hydrateEnvelope =
1451
+ hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object"
1452
+ ? (hydrateEnvelopeRaw as Record<string, unknown>)
1453
+ : null;
1454
+ const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
1081
1455
  sessionId: ids.sessionId,
1082
1456
  recovery: "create-or-restore",
1083
1457
  backendOverride: ids.backend as never,
@@ -1088,6 +1462,11 @@ export async function resumeBoxForTurn(
1088
1462
  : {}),
1089
1463
  onSandboxCreated: async (created) => {
1090
1464
  createdEstablished = created;
1465
+ providerRenewalTarget = {
1466
+ backend: ids.backend,
1467
+ instanceId: created.instanceId,
1468
+ };
1469
+ providerRenewedAtMs = Date.now();
1091
1470
  throwIfReleasedOrCancelled();
1092
1471
  if (
1093
1472
  rematerialization &&
@@ -1130,7 +1509,7 @@ export async function resumeBoxForTurn(
1130
1509
  }
1131
1510
  }
1132
1511
  const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
1133
- await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
1512
+ await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
1134
1513
  created.backendId,
1135
1514
  );
1136
1515
  const recorded = await recordWarmingSandboxCreated(db, {
@@ -1218,13 +1597,15 @@ export async function resumeBoxForTurn(
1218
1597
  });
1219
1598
  createdEstablished = established;
1220
1599
  throwIfReleasedOrCancelled();
1221
- // A sandbox handle is not sufficient evidence that Modal's command router
1222
- // is live. Do not publish a warm lease until one bounded no-op exec works.
1600
+ // A sandbox handle is not sufficient evidence that an asynchronous
1601
+ // provider's command router is live. Do not publish a warm lease until
1602
+ // one bounded no-op exec works.
1223
1603
  // On timeout the catch below terminates the box and rolls warming -> cold,
1224
1604
  // so the next turn cold-creates instead of hanging forever on first use.
1225
1605
  await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1226
1606
  sandboxGroupId: ids.sandboxGroupId,
1227
1607
  });
1608
+ await maybeRenewProviderExpiration(true);
1228
1609
  throwIfReleasedOrCancelled();
1229
1610
  // Fold the LIVE box into a re-resumable envelope and persist it as the
1230
1611
  // lease's resume_state — exactly like the API-direct paths (channel-a.ts /
@@ -1244,7 +1625,7 @@ export async function resumeBoxForTurn(
1244
1625
  // attempts terminate the replacement and fail closed; they never publish a
1245
1626
  // clean or mixed workspace.
1246
1627
  const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
1247
- await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
1628
+ await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
1248
1629
  established.backendId,
1249
1630
  );
1250
1631
  throwIfReleasedOrCancelled();
@@ -1359,7 +1740,7 @@ export async function resumeBoxForTurn(
1359
1740
  }
1360
1741
  await release();
1361
1742
  recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1362
- throw error;
1743
+ throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
1363
1744
  }
1364
1745
  }
1365
1746
 
@@ -1401,8 +1782,8 @@ export async function resumeBoxForTurn(
1401
1782
  });
1402
1783
  throwIfReleasedOrCancelled();
1403
1784
  // A durable `warm` row is an ownership assertion, not provider liveness.
1404
- // Modal may have ended the exact box at its finite timeout while OpenGeni
1405
- // was idle. Prove the command router before handing the session to the
1785
+ // A provider may have ended the exact box while OpenGeni was idle. Prove
1786
+ // the command router before handing the session to the
1406
1787
  // agent so terminal evidence enters the atomic warm->cold recovery path
1407
1788
  // below instead of surfacing inside a model-visible tool call.
1408
1789
  if (services.verifyAttachedSandboxReadiness) {
@@ -1412,10 +1793,15 @@ export async function resumeBoxForTurn(
1412
1793
  sandboxGroupId: ids.sandboxGroupId,
1413
1794
  });
1414
1795
  }
1796
+ providerRenewalTarget = {
1797
+ backend: ids.backend,
1798
+ instanceId: established.instanceId,
1799
+ };
1800
+ await maybeRenewProviderExpiration(true);
1415
1801
  throwIfReleasedOrCancelled();
1416
1802
  } catch (error) {
1417
1803
  if (!isProviderSandboxNotFoundError(ids.backend, error)) {
1418
- throw error;
1804
+ throw sandboxProvisionStageError("resume", error);
1419
1805
  }
1420
1806
  const marked = await markWarmLeaseInstanceLost(db, {
1421
1807
  accountId: ids.accountId,