@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -1,36 +1,21 @@
1
- // apps/worker/src/activities/sandbox-lease.ts — the SOLE liveness/GC/cost-stop
2
- // driver (P1.3 / OD-3).
1
+ // apps/worker/src/activities/sandbox-lease.ts — the sole global lease inventory,
2
+ // GC and cost-stop activity set (P1.3 / OD-3).
3
3
  //
4
- // There is exactly ONE reaper activity: `reapSandboxLeases`. It is fired by the
5
- // ONE global reaper Temporal Schedule (registered in apps/worker/src/index.ts).
6
- // There is NO ownerHeartbeat, NO per-session timer, NO per-RPC workflow, NO
7
- // *ForViewer activity, NO resolveOwnerTaskQueue. Turn-holder lifecycle is bound
8
- // to Temporal *activity* liveness (the turn activity acquires/releases the turn
9
- // holder); a crashed founder's leaked turn holder becomes reapable via the lease
10
- // TTL — TTL-exemption means a *live* turn is never idle-reaped, NOT that a dead
11
- // turn's holder is immortal.
4
+ // One scheduled workflow inventories at most the DB reaper's fair 500-row batch,
5
+ // then starts one durable child per exact (workspace, group, epoch). Each child
6
+ // claims, snapshots and terminates only its own provider box. Temporal's control
7
+ // worker bounds real provider concurrency; a slow or dead provider cannot hold
8
+ // the global scan or any sibling drain. Activity heartbeats make worker loss
9
+ // visible promptly, while the DB capture id remains the authoritative teardown
10
+ // fence through provider termination and the final draining->cold commit.
12
11
  //
13
- // One pass per fire:
14
- // 1. reapStaleLeaseHoldersGlobal (the P1.1 SECURITY-DEFINER cross-workspace
15
- // sweep): TTL-reaps stale viewer holders, resets warming-death rows to cold,
16
- // recomputes refcounts + enters draining at refcount 0, and RETURNS the
17
- // drainable rows (workspace, group, instance, epoch) whose drain grace has
18
- // elapsed at refcount 0. DB-only — no provider call inside the sweep.
19
- // 2. For each drainable row: resume/attach the provider box BY ID (off the
20
- // lease's resume envelope, via createSandboxClientForBackend +
21
- // establishSandboxSessionFromEnvelope), call the provider terminate, then
22
- // confirmDrainCold (the CAS draining->cold under the epoch fence).
23
- //
24
- // IDEMPOTENT + safe to run concurrently with itself: the drain CAS is guarded on
25
- // (draining AND refcount=0 AND lease_epoch=expected). If another sweep already
26
- // drained the row, or a late re-arm flipped it warm, or a newer epoch snuck in,
27
- // confirmDrainCold returns wentCold:false and we skip — provider stop() fires
28
- // ONLY when the CAS proves the box is still the draining box we observed. We
29
- // confirm the CAS would still pass (re-read the lease) BEFORE the provider call
30
- // so a box that was re-armed mid-sweep is never terminated out from under a live
31
- // holder.
32
-
33
- import { randomUUID } from "node:crypto";
12
+ // The legacy `reapSandboxLeases` composite remains only as a direct/embedded test
13
+ // harness and executes admitted drains concurrently. There is no per-session
14
+ // timer, viewer activity, owner task queue, or provider-specific lifecycle path
15
+ // in the normal drain state machine.
16
+
17
+ import { createHash, randomUUID } from "node:crypto";
18
+ import { Context } from "@temporalio/activity";
34
19
  import {
35
20
  accrueWarmSeconds,
36
21
  adoptLegacyModalCheckpointArtifact,
@@ -58,13 +43,13 @@ import {
58
43
  pruneDeletedSandboxCheckpointArtifacts,
59
44
  registerSandboxCheckpointArtifact,
60
45
  recordRetainedProcessReconciliationProof,
61
- releaseWorkspaceArchiveCapture,
62
- replaceExpiredWorkspaceArchiveCapture,
46
+ replaceWorkspaceArchiveCaptureAfterProof,
63
47
  readLease,
64
48
  reapExpiredSessionListSnapshots,
65
49
  reapStaleLeaseHoldersGlobal,
66
50
  requestDueSandboxRotationsGlobal,
67
51
  readSandboxRotationBacklog,
52
+ workspaceArchiveCaptureDeadlineElapsed,
68
53
  retainedProcessReconciliationProof,
69
54
  retainedProcessSettlementIdentity,
70
55
  settleSandboxCheckpointArtifactGc,
@@ -76,7 +61,8 @@ import {
76
61
  type SandboxRetainedProcess,
77
62
  type LeaseSnapshot,
78
63
  } from "@opengeni/db";
79
- import { sandboxArchiveCaptureTimeoutMs, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
64
+ import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
65
+ import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
80
66
  import {
81
67
  // Normal drain teardown builds the client and resumes the envelope directly:
82
68
  // a live box gets its /workspace persisted before termination, while a gone
@@ -84,6 +70,7 @@ import {
84
70
  // establishSandboxSessionFromEnvelope in strict resume-only mode so it can
85
71
  // reuse the runtime's provider-ready proof without ever cold-restoring.
86
72
  captureVerifiedWorkspaceArchive,
73
+ assertConsistentSandboxProviderIdentity,
87
74
  createSandboxClientForBackend,
88
75
  deleteModalCheckpointSnapshot,
89
76
  deserializeSandboxSessionStateEnvelope,
@@ -92,20 +79,35 @@ import {
92
79
  isExecSessionLostBanner,
93
80
  isProviderSandboxNotFoundError,
94
81
  parseExecBannerExitCode,
82
+ prepareProviderForTeardownAfterCapture,
83
+ providerWorkspaceCapturePolicy,
95
84
  resolveModalCheckpointProviderBindingForLiveSandbox,
96
85
  resolveModalCheckpointProviderBinding,
97
86
  resolveModalCheckpointProviderBindingForSession,
87
+ resumeExactSandboxSession,
88
+ sandboxProviderContinuityForState,
98
89
  sandboxBackendForSdkBackendId,
90
+ sandboxProviderInstanceIdFromEnvelope,
99
91
  sandboxCommandExitCode,
100
92
  sandboxCommandStillRunning,
101
93
  sweepModalOrphanSandboxes,
94
+ terminateManagedSandboxSession,
102
95
  terminateModalSandboxById,
103
96
  verifySandboxExecReadiness,
104
97
  type ModalOrphanSweepTermination,
105
98
  type ModalCheckpointProviderBinding,
99
+ type ProviderWorkspaceCapturePolicy,
106
100
  type WorkspaceArchiveDescriptor,
107
- } from "@opengeni/runtime";
108
- import type { ActivityServices } from "./types";
101
+ } from "@opengeni/runtime/sandbox";
102
+ import {
103
+ SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
104
+ SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
105
+ type SandboxDrainActivityInput,
106
+ type SandboxLeaseSweepMaintenanceInput,
107
+ } from "../sandbox-reaper-contract";
108
+ import { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from "../concurrency";
109
+ import { assertSandboxDrainInputTiming, sandboxDrainTiming } from "../sandbox-reaper-timeout";
110
+ import type { ControlActivityServices as ActivityServices } from "./types";
109
111
  import { reconcilePendingParentSystemUpdates } from "./parent-wake";
110
112
  import {
111
113
  recordCreditBalanceGauges,
@@ -127,6 +129,8 @@ import {
127
129
  } from "../observability-metrics";
128
130
  import { providerIdentityFromResumeState } from "../sandbox-routing";
129
131
 
132
+ export { sandboxLeaseTelemetryKey } from "@opengeni/observability";
133
+
130
134
  export type ReapSandboxLeasesResult = {
131
135
  /** Stale viewer holders + warming-death rows the sweep touched is folded into
132
136
  * the DB pass; here we report the terminate outcomes. */
@@ -146,29 +150,15 @@ export type ReapSandboxLeasesResult = {
146
150
  modalOrphansTerminated: number;
147
151
  };
148
152
 
149
- // The structural slice of a provider SandboxClient we need to resume-by-id and
150
- // terminate a box. `delete(state)` is the provider's stop()/terminate (the
151
- // runtime SandboxClient maps it to the per-provider teardown). Narrowed so this
152
- // stays agent-loop-free.
153
- type TerminableClient = {
153
+ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
154
+
155
+ type DrainSandboxClient = {
154
156
  backendId: string;
155
157
  resume?: (state: unknown) => Promise<unknown>;
156
158
  deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
157
- delete?: (state: unknown) => Promise<unknown>;
158
- };
159
-
160
- // A live session handle may expose a kill/terminate/close itself (some providers
161
- // tear the box down from the session, not the client). We try the client.delete
162
- // first (the canonical teardown), then fall back to a session-level terminator.
163
- type TerminableSession = {
164
- kill?: () => Promise<unknown>;
165
- terminate?: () => Promise<unknown>;
166
- close?: () => Promise<unknown>;
167
- closed?: boolean;
159
+ delete?: (state: unknown) => Promise<void>;
168
160
  };
169
161
 
170
- type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
171
-
172
162
  /**
173
163
  * The CAS-write the terminate seam calls to PERSIST the captured /workspace
174
164
  * archive onto the lease under the epoch fence, BEFORE the box is terminated
@@ -203,6 +193,8 @@ export type ProviderTerminationOutcome = {
203
193
  providerMissingBeforeCapture: boolean;
204
194
  };
205
195
 
196
+ export type DrainCaptureDisposition = "capture_required" | "archive_published";
197
+
206
198
  /** The provider-terminate seam. Production wires the real resume-by-id ->
207
199
  * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
208
200
  * provider stop() (`terminateProviderBox`); a unit test injects a spy so the
@@ -215,6 +207,9 @@ export type TerminateBoxFn = (
215
207
  lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
216
208
  observability: ActivityServices["observability"],
217
209
  persistArchive: PersistArchiveFn,
210
+ providerCaptureRequestId?: string,
211
+ captureDisposition?: DrainCaptureDisposition,
212
+ capturePolicy?: ProviderWorkspaceCapturePolicy | null,
218
213
  ) => Promise<boolean | ProviderTerminationOutcome>;
219
214
 
220
215
  export type SweepModalOrphansFn = (
@@ -279,230 +274,539 @@ export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
279
274
  export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
280
275
  export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
281
276
 
277
+ export class SandboxProviderCaptureTimeoutError extends Error {
278
+ readonly name = "SandboxProviderCaptureTimeoutError";
279
+
280
+ constructor(
281
+ public readonly sandboxGroupId: string,
282
+ public readonly backend: string,
283
+ public readonly timeoutMs: number,
284
+ public readonly leaseEpoch: number,
285
+ public readonly instanceId: string,
286
+ ) {
287
+ super(
288
+ `sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} ` +
289
+ `workspace capture exceeded ${timeoutMs}ms; ` +
290
+ "the exact durable claim remains fenced while its provider response settles",
291
+ );
292
+ }
293
+ }
294
+
295
+ async function awaitProviderCaptureWithLatePublication<T>(input: {
296
+ capture: Promise<T>;
297
+ timeoutMs: number;
298
+ timeoutError: SandboxProviderCaptureTimeoutError;
299
+ publishLate: (value: T) => Promise<void>;
300
+ observeLateFailure: (error: unknown) => void;
301
+ }): Promise<T> {
302
+ let timeout: ReturnType<typeof setTimeout> | undefined;
303
+ const timeoutError = input.timeoutError;
304
+ try {
305
+ return await Promise.race([
306
+ input.capture,
307
+ new Promise<never>((_, reject) => {
308
+ timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);
309
+ }),
310
+ ]);
311
+ } catch (error) {
312
+ if (error !== timeoutError) throw error;
313
+ // Promise cancellation is not a generic SandboxSession contract. Keep the
314
+ // exact failed activity's result path alive in-process, but permit only the
315
+ // durable archive publication: never provider teardown or a cold commit from
316
+ // an activity Temporal has already rejected. A replaced claim fences this
317
+ // callback; Modal candidates then flow to artifact-ledger GC.
318
+ void input.capture.then(input.publishLate).catch(input.observeLateFailure);
319
+ throw error;
320
+ } finally {
321
+ if (timeout) clearTimeout(timeout);
322
+ }
323
+ }
324
+
325
+ function startSandboxReaperHeartbeat(details: Record<string, string | number | null>): () => void {
326
+ let context: Context;
327
+ try {
328
+ context = Context.current();
329
+ } catch {
330
+ // Direct unit/embedded harnesses intentionally run without Temporal.
331
+ return () => undefined;
332
+ }
333
+ const heartbeat = (): void => {
334
+ try {
335
+ context.heartbeat({ ...details, heartbeatAt: new Date().toISOString() });
336
+ } catch {
337
+ // Heartbeat can synchronously surface Temporal cancellation. This timer
338
+ // must never turn that into an uncaught process-level exception while an
339
+ // already-issued provider capture/teardown is settling under its durable
340
+ // DB claim. The activity result/cancellation boundary remains Temporal's;
341
+ // claim recovery remains the database's.
342
+ }
343
+ };
344
+ heartbeat();
345
+ const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
346
+ if ("unref" in timer && typeof timer.unref === "function") timer.unref();
347
+ return () => clearInterval(timer);
348
+ }
349
+
350
+ export type SandboxLeaseSweepPlan = {
351
+ drainable: ReapDrainable[];
352
+ timeoutClass: SandboxDrainActivityInput["timeoutClass"];
353
+ snapshotTimeoutMs: number;
354
+ captureTimeoutMs: number;
355
+ /** Deadline rotations admitted immediately before inventory, so zero-holder
356
+ * boxes enter the same sweep instead of paying another schedule period. */
357
+ rotationsRequested: number;
358
+ };
359
+
360
+ export type SandboxLeaseSweepMaintenanceResult = {
361
+ metered: number;
362
+ forceDrained: number;
363
+ rotationsRequested: number;
364
+ modalOrphansTerminated: number;
365
+ };
366
+
367
+ export type SandboxDrainActivityResult = { status: "terminated" | "skipped" };
368
+
369
+ type SandboxDrainCaptureAttempt = {
370
+ operationId: string;
371
+ captureId: string;
372
+ attempt: number;
373
+ /** Durable child input, frozen when the workflow was created. Never
374
+ * re-derive this from a later deployment's config/formula. */
375
+ captureTimeoutMs: number;
376
+ };
377
+
378
+ class SandboxDrainRecoveryDeferredError extends Error {
379
+ readonly name = "SandboxDrainRecoveryDeferredError";
380
+ }
381
+
382
+ /** Stable UUID per logical operation + Temporal attempt. Attempt one uses the
383
+ * operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.
384
+ * This makes the DB receipt idempotent even if an activity delivery is replayed. */
385
+ export function sandboxDrainCaptureId(operationId: string, attempt: number): string {
386
+ if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {
387
+ throw new Error("Sandbox drain operation id is invalid");
388
+ }
389
+ if (!Number.isSafeInteger(attempt) || attempt <= 0) {
390
+ throw new Error("Sandbox drain activity attempt is invalid");
391
+ }
392
+ if (attempt === 1) return operationId.toLowerCase();
393
+ const bytes = createHash("sha256")
394
+ .update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`)
395
+ .digest()
396
+ .subarray(0, 16);
397
+ bytes[6] = (bytes[6]! & 0x0f) | 0x50;
398
+ bytes[8] = (bytes[8]! & 0x3f) | 0x80;
399
+ const hex = bytes.toString("hex");
400
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
401
+ }
402
+
403
+ function sandboxDrainActivityAttempt(): number {
404
+ try {
405
+ const attempt = Context.current().info.attempt;
406
+ return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
407
+ } catch {
408
+ return 1;
409
+ }
410
+ }
411
+
282
412
  export function createSandboxLeaseActivities(
283
413
  services: () => Promise<ActivityServices>,
284
414
  options: SandboxLeaseActivityOptions = {},
285
415
  ) {
286
- const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
416
+ const terminateBox: TerminateBoxFn =
417
+ options.terminateBox ??
418
+ (async (
419
+ settings,
420
+ lease,
421
+ observability,
422
+ persistArchive,
423
+ providerCaptureRequestId,
424
+ captureDisposition,
425
+ capturePolicy,
426
+ ) =>
427
+ await terminateProviderBox(
428
+ settings,
429
+ lease,
430
+ observability,
431
+ persistArchive,
432
+ createSandboxClientForBackend,
433
+ terminateModalSandboxById,
434
+ providerCaptureRequestId,
435
+ captureDisposition,
436
+ capturePolicy,
437
+ ));
287
438
  const sweepModalOrphans: SweepModalOrphansFn =
288
439
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
289
440
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
290
441
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
291
- /**
292
- * The one global reaper sweep. Idempotent; concurrency-safe with itself.
293
- * The global Schedule always runs. A defensive mutation gate here ensures
294
- * that an ownership-off deployment still refreshes read-only metrics and
295
- * repairs the system-update outbox but can never terminate a box.
296
- */
297
- async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
298
- const service = await services();
299
- const { db, settings, observability } = service;
300
- const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch((error) => {
301
- observability.warn("system-update outbox reconciliation failed", {
302
- error: error instanceof Error ? error.message : String(error),
303
- });
304
- return { claimed: 0, delivered: 0, failed: 1 };
305
- });
306
- const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
307
- (error) => {
308
- observability.warn("session-list snapshot reconciliation failed", {
442
+ async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
443
+ const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
444
+ try {
445
+ const { db, settings, observability } = await services();
446
+ const timing = sandboxDrainTiming(settings);
447
+ if (!settings.sandboxOwnershipEnabled) {
448
+ return {
449
+ drainable: [],
450
+ timeoutClass: timing.timeoutClass,
451
+ snapshotTimeoutMs: timing.snapshotTimeoutMs,
452
+ captureTimeoutMs: timing.captureTimeoutMs,
453
+ rotationsRequested: 0,
454
+ };
455
+ }
456
+
457
+ // Admit finite-lifetime rotations immediately before inventory. The DB
458
+ // function is bounded and provider-I/O-free; a due zero-holder box becomes
459
+ // draining there and is therefore dispatched in THIS sweep, eliminating
460
+ // the old extra schedule-period availability gap. Failure is isolated: the
461
+ // inventory must still dispatch every row that was already drainable.
462
+ const rotationsRequested = await requestDueSandboxRotationsGlobal(
463
+ db,
464
+ settings.sandboxRotationLeadMs,
465
+ settings.sandboxRotationBatchSize,
466
+ ).catch((error) => {
467
+ observability.warn("sandbox reaper: provider-deadline rotation request failed", {
309
468
  error: error instanceof Error ? error.message : String(error),
310
469
  });
311
470
  return 0;
312
- },
313
- );
314
- if (expiredSessionListSnapshots > 0) {
315
- observability.info("expired session-list snapshots reaped", {
316
- deleted: expiredSessionListSnapshots,
317
471
  });
318
- }
319
- // Inventory is useful even while provider ownership is disabled, and the
320
- // same globally scheduled activity remains its single projection owner.
321
- // Refresh before the mutation gate so dashboards show truthful zero/history
322
- // state and absence alerts do not misclassify an intentional rollout flag.
323
- await refreshQueueLeaseAndCreditGauges(db, observability);
324
- if (!settings.sandboxOwnershipEnabled) {
325
- if (parentUpdates.claimed > 0) {
326
- observability.info("system-update outbox reconciled", parentUpdates);
472
+ recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
473
+
474
+ // Keep the remaining dispatch-critical path to one bounded DB inventory.
475
+ // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
476
+ // run only after every drainable box has its own durable child.
477
+ const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
478
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
479
+ // Dead-worker turn holders: a live holder is touched every 10s from the
480
+ // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
481
+ // the whole warmup — waitForWarm/establish/display-stack — and the turn
482
+ // heartbeat covers the run), so NO live path is ever silent for more than
483
+ // one tick. The ordinary warm lease TTL is therefore already a generous
484
+ // dead-worker horizon. Canonical holders whose turn is durably closed are
485
+ // removed immediately by the DB function; no path-duration guess belongs
486
+ // in this lifecycle contract.
487
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
488
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
489
+ idleGraceMs: settings.sandboxIdleGraceMs,
490
+ });
491
+ const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
492
+ if (drainableInventory.length > drainable.length) {
493
+ observability.info("sandbox reaper: drain inventory continues next sweep", {
494
+ admitted: drainable.length,
495
+ deferred: drainableInventory.length - drainable.length,
496
+ });
327
497
  }
328
498
  return {
329
- examined: 0,
330
- terminated: 0,
331
- skipped: 0,
332
- metered: 0,
333
- forceDrained: 0,
334
- modalOrphansTerminated: 0,
499
+ drainable,
500
+ timeoutClass: timing.timeoutClass,
501
+ snapshotTimeoutMs: timing.snapshotTimeoutMs,
502
+ captureTimeoutMs: timing.captureTimeoutMs,
503
+ rotationsRequested,
335
504
  };
505
+ } finally {
506
+ stopHeartbeat();
336
507
  }
508
+ }
337
509
 
338
- const rotationsRequested = await requestDueSandboxRotationsGlobal(
339
- db,
340
- settings.sandboxRotationLeadMs,
341
- settings.sandboxRotationBatchSize,
342
- ).catch((error) => {
343
- observability.warn("sandbox reaper: provider-deadline rotation request failed", {
344
- error: error instanceof Error ? error.message : String(error),
345
- });
346
- return 0;
510
+ async function drainSandboxLease(
511
+ input: SandboxDrainActivityInput,
512
+ ): Promise<SandboxDrainActivityResult> {
513
+ const stopHeartbeat = startSandboxReaperHeartbeat({
514
+ phase: "drain",
515
+ workspaceId: input.target.workspaceId,
516
+ sandboxGroupId: input.target.sandboxGroupId,
517
+ leaseEpoch: input.target.leaseEpoch,
518
+ instanceId: input.target.instanceId,
347
519
  });
348
- recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
349
-
350
- // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
351
- // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
352
- // them). GROUP+epoch+tick idempotent → a shared box is one stream; an
353
- // overlapping/re-fired sweep cannot double-charge. Best-effort per row.
354
- const metered = await accrueWarmTick(db, settings, observability);
355
-
356
- // (0b) Per-workspace warm-cap + force-drain (P2.1) — under the usage lock, a
357
- // workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
358
- // boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
359
- // draining rows are caught by the same sweep's terminate below.
360
- const forceDrainWorkspaceIds = new Set<string>();
361
- if (
362
- settings.billingMode === "stripe" ||
363
- settings.usageLimitsMode === "managed" ||
364
- settings.sandboxMaxWarmSecondsPerWorkspace > 0
365
- ) {
366
- for (const workspaceId of metered.workspaceIds) {
367
- forceDrainWorkspaceIds.add(workspaceId);
368
- }
369
- }
370
520
  try {
371
- for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
372
- forceDrainWorkspaceIds.add(workspaceId);
373
- }
374
- } catch (error) {
375
- observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
376
- error: error instanceof Error ? error.message : String(error),
377
- });
378
- }
379
- const forceDrained = await forceDrainOverLimitWorkspaces(
380
- db,
381
- settings,
382
- forceDrainWorkspaceIds,
383
- observability,
384
- );
385
-
386
- // (0c) Terminal-owner retained-process reconciliation. Owner state selects
387
- // only a bounded inspection batch; exact provider exit/loss proof is
388
- // checkpointed before canonical settlement. Ambiguous, running, timed-out,
389
- // unsupported, and transient provider states preserve every durable row.
390
- await reconcileTerminalRetainedProcesses(
391
- db,
392
- settings,
393
- observability,
394
- probeRetainedProcess,
395
- options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
396
- );
397
-
398
- // (1) The DB-only cross-workspace sweep. Returns the drainable rows.
399
- const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
400
- viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
401
- // Dead-worker turn holders: a live holder is touched every 10s from the
402
- // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
403
- // the whole warmup — waitForWarm/establish/display-stack — and the turn
404
- // heartbeat covers the run), so NO live path is ever silent for more than
405
- // one tick. The horizon is deliberately generous defense-in-depth (not a
406
- // tuned guess about path lengths): a killed worker's frozen holder —
407
- // which would otherwise pin refcount >= 1 FOREVER, so the lease never
408
- // drains and the box dies at the provider hard-timeout UNPERSISTED —
409
- // clears within ~12 minutes.
410
- turnHolderTtlMs: settings.sandboxWarmingTimeoutMs + settings.sandboxLeaseTtlMs,
411
- idleGraceMs: settings.sandboxIdleGraceMs,
412
- });
413
-
414
- let terminated = 0;
415
- let skipped = 0;
416
- let modalOrphansTerminated = 0;
417
-
418
- // (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
419
- // are isolated: one box's provider error must not abort the whole sweep (the
420
- // next sweep retries it; the provider idle-timeout is the backstop).
421
- for (const row of drainable) {
521
+ const { db, settings, observability } = await services();
522
+ if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
523
+ assertSandboxDrainInputTiming(input);
524
+ const drainSettings =
525
+ settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
526
+ ? settings
527
+ : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
528
+ const row = input.target;
529
+ const activityAttempt = sandboxDrainActivityAttempt();
530
+ const captureAttempt: SandboxDrainCaptureAttempt = {
531
+ operationId: input.operationId,
532
+ captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),
533
+ attempt: activityAttempt,
534
+ captureTimeoutMs: input.captureTimeoutMs,
535
+ };
422
536
  try {
423
537
  const drainedCold = await terminateDrainableBox(
424
538
  db,
425
- settings,
539
+ drainSettings,
426
540
  row,
427
541
  observability,
428
542
  terminateBox,
429
543
  probeDrainableProvider,
544
+ captureAttempt,
430
545
  );
431
- if (drainedCold) {
432
- terminated += 1;
433
- } else {
434
- skipped += 1;
435
- }
546
+ return { status: drainedCold ? "terminated" : "skipped" };
436
547
  } catch (error) {
437
- skipped += 1;
438
- observability.warn("sandbox reaper: terminate failed for drainable lease", {
548
+ const lease = await readLease(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
549
+ const details = {
550
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
439
551
  workspaceId: row.workspaceId,
440
552
  sandboxGroupId: row.sandboxGroupId,
441
553
  leaseEpoch: row.leaseEpoch,
554
+ backend: lease?.backend ?? "unknown",
555
+ instanceId: lease?.instanceId ?? row.instanceId,
556
+ captureId: lease?.archiveCapture?.id ?? null,
557
+ captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,
558
+ captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,
559
+ timeoutClass: input.timeoutClass,
442
560
  error: error instanceof Error ? error.message : String(error),
443
- });
561
+ };
562
+ if (error instanceof SandboxDrainRecoveryDeferredError) {
563
+ observability.info("sandbox reaper: per-sandbox recovery deferred safely", details);
564
+ } else {
565
+ observability.warn("sandbox reaper: per-sandbox drain failed", details);
566
+ }
567
+ throw error;
444
568
  }
569
+ } finally {
570
+ stopHeartbeat();
445
571
  }
572
+ }
446
573
 
574
+ async function maintainSandboxLeaseSweep(
575
+ input: SandboxLeaseSweepMaintenanceInput,
576
+ ): Promise<SandboxLeaseSweepMaintenanceResult> {
577
+ const stopHeartbeat = startSandboxReaperHeartbeat({
578
+ phase: "maintain",
579
+ examined: input.examined,
580
+ started: input.started,
581
+ });
447
582
  try {
448
- modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
449
- recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
450
- } catch (error) {
451
- observability.warn("sandbox reaper: Modal orphan sweep failed", {
452
- error: error instanceof Error ? error.message : String(error),
453
- });
454
- }
583
+ const service = await services();
584
+ const { db, settings, observability } = service;
585
+ const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(
586
+ (error) => {
587
+ observability.warn("system-update outbox reconciliation failed", {
588
+ error: error instanceof Error ? error.message : String(error),
589
+ });
590
+ return { claimed: 0, delivered: 0, failed: 1 };
591
+ },
592
+ );
593
+ const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
594
+ (error) => {
595
+ observability.warn("session-list snapshot reconciliation failed", {
596
+ error: error instanceof Error ? error.message : String(error),
597
+ });
598
+ return 0;
599
+ },
600
+ );
601
+ if (expiredSessionListSnapshots > 0) {
602
+ observability.info("expired session-list snapshots reaped", {
603
+ deleted: expiredSessionListSnapshots,
604
+ });
605
+ }
606
+ if (parentUpdates.claimed > 0) {
607
+ observability.info("system-update outbox reconciled", parentUpdates);
608
+ }
455
609
 
456
- const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
457
- (error) => {
458
- observability.warn("sandbox reaper: checkpoint artifact GC failed", {
610
+ let metered = 0;
611
+ let forceDrained = 0;
612
+ const rotationsRequested = input.rotationsRequested;
613
+ let modalOrphansTerminated = 0;
614
+ if (!settings.sandboxOwnershipEnabled) {
615
+ // Inventory remains useful while provider ownership is intentionally
616
+ // disabled, and this scheduled activity is its single projection owner.
617
+ await refreshQueueLeaseAndCreditGauges(db, observability);
618
+ return {
619
+ metered,
620
+ forceDrained,
621
+ rotationsRequested,
622
+ modalOrphansTerminated,
623
+ };
624
+ }
625
+
626
+ // Warm metering and cost-stop run after current drain children launch.
627
+ // Freshly force-drained billing rows are admission-fenced immediately and
628
+ // receive their own provider child on the next bounded schedule tick.
629
+ const meterResult = await accrueWarmTick(db, settings, observability);
630
+ metered = meterResult.accrued;
631
+ const forceDrainWorkspaceIds = new Set<string>();
632
+ if (
633
+ settings.billingMode === "stripe" ||
634
+ settings.usageLimitsMode === "managed" ||
635
+ settings.sandboxMaxWarmSecondsPerWorkspace > 0
636
+ ) {
637
+ for (const workspaceId of meterResult.workspaceIds) {
638
+ forceDrainWorkspaceIds.add(workspaceId);
639
+ }
640
+ }
641
+ try {
642
+ for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
643
+ forceDrainWorkspaceIds.add(workspaceId);
644
+ }
645
+ } catch (error) {
646
+ observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
459
647
  error: error instanceof Error ? error.message : String(error),
460
648
  });
461
- return { claimed: 0, deleted: 0, failed: 0 };
462
- },
463
- );
649
+ }
650
+ forceDrained = await forceDrainOverLimitWorkspaces(
651
+ db,
652
+ settings,
653
+ forceDrainWorkspaceIds,
654
+ observability,
655
+ );
464
656
 
465
- await refreshQueueLeaseAndCreditGauges(db, observability);
657
+ // Terminal-owner process reconciliation is ancillary to lease dispatch.
658
+ // Exact provider proof is durably checkpointed before settlement; every
659
+ // ambiguous outcome remains claimed/deferred for another sweep.
660
+ await reconcileTerminalRetainedProcesses(
661
+ db,
662
+ settings,
663
+ observability,
664
+ probeRetainedProcess,
665
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
666
+ );
466
667
 
467
- if (
468
- drainable.length > 0 ||
469
- metered.accrued > 0 ||
470
- forceDrained > 0 ||
471
- modalOrphansTerminated > 0 ||
472
- rotationsRequested > 0 ||
473
- checkpointGc.claimed > 0
474
- ) {
475
- observability.info("sandbox reaper swept", {
476
- drainable: drainable.length,
477
- terminated,
478
- skipped,
479
- metered: metered.accrued,
668
+ try {
669
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
670
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
671
+ } catch (error) {
672
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
673
+ error: error instanceof Error ? error.message : String(error),
674
+ });
675
+ }
676
+
677
+ const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
678
+ (error) => {
679
+ observability.warn("sandbox reaper: checkpoint artifact GC failed", {
680
+ error: error instanceof Error ? error.message : String(error),
681
+ });
682
+ return { claimed: 0, deleted: 0, failed: 0 };
683
+ },
684
+ );
685
+
686
+ await refreshQueueLeaseAndCreditGauges(db, observability);
687
+
688
+ if (
689
+ input.examined > 0 ||
690
+ metered > 0 ||
691
+ forceDrained > 0 ||
692
+ modalOrphansTerminated > 0 ||
693
+ rotationsRequested > 0 ||
694
+ checkpointGc.claimed > 0
695
+ ) {
696
+ observability.info("sandbox reaper dispatched", {
697
+ drainable: input.examined,
698
+ childrenStarted: input.started,
699
+ childrenAlreadyRunning: input.alreadyRunning,
700
+ childrenStartFailed: input.startFailed,
701
+ metered,
702
+ forceDrained,
703
+ modalOrphansTerminated,
704
+ rotationsRequested,
705
+ checkpointArtifactsClaimed: checkpointGc.claimed,
706
+ checkpointArtifactsDeleted: checkpointGc.deleted,
707
+ checkpointArtifactsFailed: checkpointGc.failed,
708
+ });
709
+ }
710
+
711
+ return {
712
+ metered,
480
713
  forceDrained,
481
- modalOrphansTerminated,
482
714
  rotationsRequested,
483
- checkpointArtifactsClaimed: checkpointGc.claimed,
484
- checkpointArtifactsDeleted: checkpointGc.deleted,
485
- checkpointArtifactsFailed: checkpointGc.failed,
715
+ modalOrphansTerminated,
716
+ };
717
+ } finally {
718
+ stopHeartbeat();
719
+ }
720
+ }
721
+
722
+ /** Direct/embedded compatibility harness. Production uses the split Temporal
723
+ * workflow above. The production activity is also the rolling-deploy bridge:
724
+ * its workflow command stays byte-for-byte compatible while this activity
725
+ * starts V2 through the injected client edge. Old workers may still execute
726
+ * the bounded composite during rollout; exact DB claims keep both paths safe. */
727
+ async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
728
+ const service = await services();
729
+ if (service.startSandboxReaperWorkflow) {
730
+ const dispatch = await service.startSandboxReaperWorkflow();
731
+ service.observability.info("sandbox reaper routed to versioned workflow", {
732
+ dispatch,
486
733
  });
734
+ return {
735
+ examined: 0,
736
+ terminated: 0,
737
+ skipped: 0,
738
+ metered: 0,
739
+ forceDrained: 0,
740
+ modalOrphansTerminated: 0,
741
+ };
487
742
  }
488
743
 
744
+ const plan = await prepareSandboxLeaseSweep();
745
+ const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);
746
+ const outcomes = await Promise.allSettled(
747
+ admitted.map((target) =>
748
+ drainSandboxLease({
749
+ target,
750
+ timeoutClass: plan.timeoutClass,
751
+ snapshotTimeoutMs: plan.snapshotTimeoutMs,
752
+ captureTimeoutMs: plan.captureTimeoutMs,
753
+ operationId: randomUUID(),
754
+ }),
755
+ ),
756
+ );
757
+ const terminated = outcomes.filter(
758
+ (outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated",
759
+ ).length;
760
+ const skipped = outcomes.length - terminated;
761
+ const maintenance = await maintainSandboxLeaseSweep({
762
+ examined: admitted.length,
763
+ started: admitted.length,
764
+ alreadyRunning: 0,
765
+ startFailed: 0,
766
+ rotationsRequested: plan.rotationsRequested,
767
+ });
489
768
  return {
490
- examined: drainable.length,
769
+ examined: admitted.length,
491
770
  terminated,
492
771
  skipped,
493
- metered: metered.accrued,
494
- forceDrained,
495
- modalOrphansTerminated,
772
+ metered: maintenance.metered,
773
+ forceDrained: maintenance.forceDrained,
774
+ modalOrphansTerminated: maintenance.modalOrphansTerminated,
496
775
  };
497
776
  }
498
777
 
499
- return { reapSandboxLeases };
778
+ return {
779
+ prepareSandboxLeaseSweep,
780
+ drainSandboxLease,
781
+ maintainSandboxLeaseSweep,
782
+ reapSandboxLeases,
783
+ };
500
784
  }
501
785
 
502
786
  const CHECKPOINT_GC_LIMIT = 50;
503
787
  const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
504
788
  const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
505
789
  const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
790
+ const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
791
+
792
+ async function forEachWithConcurrency<T>(
793
+ values: readonly T[],
794
+ concurrency: number,
795
+ visit: (value: T, index: number) => Promise<void>,
796
+ ): Promise<void> {
797
+ if (values.length === 0) return;
798
+ const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
799
+ let nextIndex = 0;
800
+ await Promise.all(
801
+ Array.from({ length: workerCount }, async () => {
802
+ while (nextIndex < values.length) {
803
+ const index = nextIndex;
804
+ nextIndex += 1;
805
+ await visit(values[index]!, index);
806
+ }
807
+ }),
808
+ );
809
+ }
506
810
 
507
811
  async function gcSandboxCheckpointArtifacts(
508
812
  db: ActivityServices["db"],
@@ -533,7 +837,7 @@ async function gcSandboxCheckpointArtifacts(
533
837
  recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
534
838
  let deleted = 0;
535
839
  let failed = 0;
536
- for (const claim of claims) {
840
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
537
841
  try {
538
842
  if (claim.providerBackend !== "modal") {
539
843
  throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
@@ -571,7 +875,7 @@ async function gcSandboxCheckpointArtifacts(
571
875
  error: error instanceof Error ? error.message : String(error),
572
876
  });
573
877
  }
574
- }
878
+ });
575
879
  const pruned = await pruneDeletedSandboxCheckpointArtifacts(
576
880
  db,
577
881
  CHECKPOINT_TOMBSTONE_RETENTION_MS,
@@ -593,7 +897,7 @@ async function adoptLegacyModalCheckpointReceipts(
593
897
  Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>
594
898
  >();
595
899
  let adopted = 0;
596
- for (const slot of slots) {
900
+ await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {
597
901
  try {
598
902
  let binding = bindings.get(slot.instanceId);
599
903
  if (!binding) {
@@ -626,7 +930,7 @@ async function adoptLegacyModalCheckpointReceipts(
626
930
  error: error instanceof Error ? error.message : String(error),
627
931
  });
628
932
  }
629
- }
933
+ });
630
934
  return adopted;
631
935
  }
632
936
 
@@ -674,7 +978,10 @@ async function reconcileTerminalRetainedProcesses(
674
978
  inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),
675
979
  );
676
980
  if (lifecycle.status === "not_found") {
677
- observation = { status: "deferred", reason: "provider_binding_missing" };
981
+ observation = {
982
+ status: "deferred",
983
+ reason: "provider_binding_missing",
984
+ };
678
985
  } else {
679
986
  process = await bindRetainedProcessProviderIdentity(db, {
680
987
  accountId: process.accountId,
@@ -997,6 +1304,7 @@ export async function probeRetainedProcessAtProvider(
997
1304
  resumedState = await deserializeSandboxSessionStateEnvelope(
998
1305
  client as never,
999
1306
  envelopeSessionState,
1307
+ process.providerInstanceId,
1000
1308
  );
1001
1309
  } catch {
1002
1310
  return { status: "deferred", reason: "provider_error" };
@@ -1007,9 +1315,15 @@ export async function probeRetainedProcessAtProvider(
1007
1315
 
1008
1316
  let session: RetainedProcessProbeSession;
1009
1317
  try {
1010
- session = (await withRetainedProcessProbeTimeout(
1011
- client.resume(resumedState),
1012
- )) as RetainedProcessProbeSession;
1318
+ const resumed = await withRetainedProcessProbeTimeout(
1319
+ resumeExactSandboxSession(
1320
+ client,
1321
+ process.providerBackend,
1322
+ resumedState,
1323
+ process.providerInstanceId,
1324
+ ),
1325
+ );
1326
+ session = resumed.session as RetainedProcessProbeSession;
1013
1327
  } catch (error) {
1014
1328
  if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
1015
1329
  return { status: "deferred", reason: "provider_timeout" };
@@ -1190,6 +1504,8 @@ async function accrueWarmTick(
1190
1504
  }
1191
1505
  for (const lease of leases) {
1192
1506
  workspaceIds.add(lease.workspaceId);
1507
+ }
1508
+ await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
1193
1509
  try {
1194
1510
  const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
1195
1511
  const result = await accrueWarmSeconds(db, {
@@ -1211,7 +1527,7 @@ async function accrueWarmTick(
1211
1527
  error: error instanceof Error ? error.message : String(error),
1212
1528
  });
1213
1529
  }
1214
- }
1530
+ });
1215
1531
  return { accrued, workspaceIds };
1216
1532
  }
1217
1533
 
@@ -1219,64 +1535,65 @@ async function refreshQueueLeaseAndCreditGauges(
1219
1535
  db: ActivityServices["db"],
1220
1536
  observability: ActivityServices["observability"],
1221
1537
  ): Promise<void> {
1222
- try {
1223
- recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
1224
- } catch (error) {
1225
- observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
1226
- error: error instanceof Error ? error.message : String(error),
1227
- });
1228
- }
1229
- await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
1230
- recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
1231
- });
1232
- await refreshSandboxInventoryGauge(
1233
- observability,
1234
- "checkpoint_artifacts",
1235
- "checkpoint-artifact",
1236
- async () => {
1237
- recordSandboxCheckpointArtifactGauges(
1238
- observability,
1239
- await countSandboxCheckpointArtifactsByState(db),
1240
- );
1241
- },
1242
- );
1243
- await refreshSandboxInventoryGauge(
1244
- observability,
1245
- "rotation_backlog",
1246
- "rotation-backlog",
1247
- async () => {
1248
- recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1249
- },
1250
- );
1251
- await refreshSandboxInventoryGauge(
1252
- observability,
1253
- "retained_processes",
1254
- "retained-process",
1255
- async () => {
1256
- recordRetainedProcessInventoryGauges(
1257
- observability,
1258
- await countActiveRetainedProcessesByOwnerState(db),
1259
- );
1260
- },
1261
- );
1262
- await refreshSandboxInventoryGauge(
1263
- observability,
1264
- "expired_drains",
1265
- "expired-draining",
1266
- async () => {
1538
+ await Promise.all([
1539
+ (async () => {
1540
+ try {
1541
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
1542
+ } catch (error) {
1543
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
1544
+ error: error instanceof Error ? error.message : String(error),
1545
+ });
1546
+ }
1547
+ })(),
1548
+ refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
1549
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
1550
+ }),
1551
+ refreshSandboxInventoryGauge(
1552
+ observability,
1553
+ "checkpoint_artifacts",
1554
+ "checkpoint-artifact",
1555
+ async () => {
1556
+ recordSandboxCheckpointArtifactGauges(
1557
+ observability,
1558
+ await countSandboxCheckpointArtifactsByState(db),
1559
+ );
1560
+ },
1561
+ ),
1562
+ refreshSandboxInventoryGauge(
1563
+ observability,
1564
+ "rotation_backlog",
1565
+ "rotation-backlog",
1566
+ async () => {
1567
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1568
+ },
1569
+ ),
1570
+ refreshSandboxInventoryGauge(
1571
+ observability,
1572
+ "retained_processes",
1573
+ "retained-process",
1574
+ async () => {
1575
+ recordRetainedProcessInventoryGauges(
1576
+ observability,
1577
+ await countActiveRetainedProcessesByOwnerState(db),
1578
+ );
1579
+ },
1580
+ ),
1581
+ refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
1267
1582
  recordExpiredDrainingSandboxLeaseGauges(
1268
1583
  observability,
1269
1584
  await countExpiredDrainingSandboxLeases(db),
1270
1585
  );
1271
- },
1272
- );
1273
- try {
1274
- recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1275
- } catch (error) {
1276
- observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1277
- error: error instanceof Error ? error.message : String(error),
1278
- });
1279
- }
1586
+ }),
1587
+ (async () => {
1588
+ try {
1589
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1590
+ } catch (error) {
1591
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1592
+ error: error instanceof Error ? error.message : String(error),
1593
+ });
1594
+ }
1595
+ })(),
1596
+ ]);
1280
1597
  }
1281
1598
 
1282
1599
  async function refreshSandboxInventoryGauge(
@@ -1313,34 +1630,38 @@ async function forceDrainOverLimitWorkspaces(
1313
1630
  settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
1314
1631
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
1315
1632
  let forceDrained = 0;
1316
- for (const workspaceId of workspaceIds) {
1317
- try {
1318
- const { accountId } = await rlsContextForWorkspace(db, workspaceId);
1319
- const balance = enforceBalance
1320
- ? await getBillingBalance(db, accountId)
1321
- : ({ balanceMicros: 1 } as { balanceMicros: number });
1322
- const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
1323
- workspaceId,
1324
- balanceMicros: balance.balanceMicros,
1325
- enforceBalance,
1326
- maxWarmSecondsPerWorkspace: cap,
1327
- idleGraceMs: settings.sandboxIdleGraceMs,
1328
- });
1329
- if (result.overLimit && result.drained.length > 0) {
1330
- forceDrained += result.drained.length;
1331
- observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
1633
+ await forEachWithConcurrency(
1634
+ [...workspaceIds],
1635
+ SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
1636
+ async (workspaceId) => {
1637
+ try {
1638
+ const { accountId } = await rlsContextForWorkspace(db, workspaceId);
1639
+ const balance = enforceBalance
1640
+ ? await getBillingBalance(db, accountId)
1641
+ : ({ balanceMicros: 1 } as { balanceMicros: number });
1642
+ const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
1332
1643
  workspaceId,
1333
- reason: result.reason,
1334
- drained: result.drained.length,
1644
+ balanceMicros: balance.balanceMicros,
1645
+ enforceBalance,
1646
+ maxWarmSecondsPerWorkspace: cap,
1647
+ idleGraceMs: settings.sandboxIdleGraceMs,
1648
+ });
1649
+ if (result.overLimit && result.drained.length > 0) {
1650
+ forceDrained += result.drained.length;
1651
+ observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
1652
+ workspaceId,
1653
+ reason: result.reason,
1654
+ drained: result.drained.length,
1655
+ });
1656
+ }
1657
+ } catch (error) {
1658
+ observability.warn("sandbox reaper: force-drain check failed for workspace", {
1659
+ workspaceId,
1660
+ error: error instanceof Error ? error.message : String(error),
1335
1661
  });
1336
1662
  }
1337
- } catch (error) {
1338
- observability.warn("sandbox reaper: force-drain check failed for workspace", {
1339
- workspaceId,
1340
- error: error instanceof Error ? error.message : String(error),
1341
- });
1342
- }
1343
- }
1663
+ },
1664
+ );
1344
1665
  return forceDrained;
1345
1666
  }
1346
1667
 
@@ -1351,6 +1672,7 @@ export function modalOrphanTerminationStillEligible(
1351
1672
  if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
1352
1673
  return false;
1353
1674
  }
1675
+
1354
1676
  const leaseId = candidate.tags.opengeni_lease_id;
1355
1677
  const workspaceId = candidate.tags.opengeni_workspace_id;
1356
1678
  const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
@@ -1418,9 +1740,10 @@ async function sweepModalOrphansForConfiguredBackend(
1418
1740
  * A capture claim whose owning worker disappeared must not fence a workspace
1419
1741
  * forever. The drain reaper reaches this seam only after every holder is gone.
1420
1742
  * It resumes the exact attributed provider without replacement and proves that
1421
- * a no-op command can execute. A paused snapshot cannot pass that proof; a
1422
- * missing provider is returned as typed loss; every ambiguous error preserves
1423
- * the old claim for the next sweep.
1743
+ * a no-op command can execute. This distinguishes typed provider loss from a
1744
+ * still-addressable instance; it does not prove that an old capture RPC has
1745
+ * stopped. The caller applies the provider's durable takeover policy and the
1746
+ * claim deadline. Ambiguity preserves the old claim and therefore the files.
1424
1747
  */
1425
1748
  async function probeDrainableProviderReadiness(
1426
1749
  settings: ActivityServices["settings"],
@@ -1466,7 +1789,11 @@ async function probeDrainableProviderReadiness(
1466
1789
  if (!run) throw new Error("Expired workspace capture provider has no readiness command");
1467
1790
  let timer: ReturnType<typeof setTimeout> | undefined;
1468
1791
  const result = await Promise.race([
1469
- run.call(session, { cmd: "true", yieldTimeMs: 1_000, maxOutputTokens: 1_000 }),
1792
+ run.call(session, {
1793
+ cmd: "true",
1794
+ yieldTimeMs: 1_000,
1795
+ maxOutputTokens: 1_000,
1796
+ }),
1470
1797
  new Promise<never>((_, reject) => {
1471
1798
  timer = setTimeout(
1472
1799
  () => reject(new Error("Expired workspace capture readiness probe timed out")),
@@ -1492,16 +1819,15 @@ async function probeDrainableProviderReadiness(
1492
1819
  * and was stopped), false when a concurrent sweep / re-arm / newer epoch means
1493
1820
  * we must NOT stop it (provider terminate is skipped).
1494
1821
  *
1495
- * The ordering is deliberately CAS-gated on BOTH ends:
1496
- * - BEFORE provider terminate: re-read the lease and assert it is STILL
1497
- * draining at refcount 0 at the SAME epoch we observed. A re-arm (a viewer
1498
- * or turn arrived during the grace window) flips it back to warm and bumps
1499
- * no epoch but changes liveness/refcount — we skip and never touch the box.
1500
- * - AFTER provider terminate: confirmDrainCold's CAS (draining AND refcount=0
1501
- * AND lease_epoch=expected) is the authoritative commit; if it returns false
1502
- * a late re-arm raced us between our re-read and the stop — but the box is
1503
- * already torn down, so we let the next acquire cold-restore it (NEVER a
1504
- * double-spawn: the lease is the singleton, the box is just gone).
1822
+ * The ordering is deliberately receipt-gated on BOTH ends:
1823
+ * - BEFORE provider I/O: re-read the exact draining epoch and acquire/retain
1824
+ * its durable capture/teardown claim. Before that claim an arrival may
1825
+ * re-arm for availability; afterward every arrival waits and cannot race
1826
+ * provider termination.
1827
+ * - AFTER provider terminate: confirmDrainCold commits only the exact current
1828
+ * capture id together with draining->cold. A miss means a recovery attempt
1829
+ * or epoch already replaced this callback; it says nothing about provider
1830
+ * liveness, so only that durable successor may reconcile and commit cold.
1505
1831
  */
1506
1832
  async function terminateDrainableBox(
1507
1833
  db: ActivityServices["db"],
@@ -1510,6 +1836,7 @@ async function terminateDrainableBox(
1510
1836
  observability: ActivityServices["observability"],
1511
1837
  terminateBox: TerminateBoxFn,
1512
1838
  probeDrainableProvider: DrainableProviderProbeFn,
1839
+ attempt: SandboxDrainCaptureAttempt,
1513
1840
  ): Promise<boolean> {
1514
1841
  // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
1515
1842
  // returns no account_id; the workspace->account map is the bootstrap read).
@@ -1532,60 +1859,186 @@ async function terminateDrainableBox(
1532
1859
  return false;
1533
1860
  }
1534
1861
 
1535
- const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
1862
+ const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
1863
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
1864
+ const managedProvider = backend !== "none" && backend !== "selfhosted";
1865
+ const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);
1866
+ if (managedProvider && !capturePolicy) {
1867
+ throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
1868
+ }
1869
+ if (capturePolicy?.liveInstance === "replaced") {
1870
+ throw new Error(
1871
+ `sandbox backend ${backend} workspace capture would replace the fenced live instance`,
1872
+ );
1873
+ }
1874
+
1875
+ const captureTimeoutMs = attempt.captureTimeoutMs;
1536
1876
  let captureClaim:
1537
1877
  | NonNullable<
1538
1878
  Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
1539
1879
  >["claim"]
1540
1880
  | null = null;
1541
1881
  let providerMissingBeforeCapture = false;
1542
- if (lease.instanceId) {
1882
+ let captureDisposition: DrainCaptureDisposition = "capture_required";
1883
+ if (lease.instanceId && managedProvider) {
1543
1884
  if (lease.archiveCapture) {
1544
- if (lease.archiveCapture.deadlineAt.getTime() > Date.now()) {
1545
- // A live owner still holds the provider pause gate. It will release the
1546
- // exact claim on settlement; this sweep must not compete.
1547
- return false;
1548
- }
1549
- // A deadline is never evidence that a non-cancellable provider capture
1550
- // stopped. Without an exact resume envelope there is no safe way to prove
1551
- // the attributed provider is command-ready, so preserve the gate.
1552
- if (!lease.resumeState) return false;
1553
- const providerState = await probeDrainableProvider(settings, lease);
1554
- if (providerState === "missing") {
1555
- providerMissingBeforeCapture = true;
1885
+ const priorCapture = lease.archiveCapture;
1886
+ const instanceId = lease.instanceId;
1887
+ if (priorCapture.publishedAt) {
1888
+ // Publication is an irreversible durable phase transition. The archive
1889
+ // already represents this exact fenced generation, so a crash/retry
1890
+ // resumes teardown immediately: no deadline wait, provider probe, claim
1891
+ // replacement, or duplicate snapshot request.
1556
1892
  captureClaim = {
1557
- ...lease.archiveCapture,
1893
+ ...priorCapture,
1558
1894
  leaseId: lease.id,
1559
1895
  leaseEpoch: lease.leaseEpoch,
1560
- instanceId: lease.instanceId,
1896
+ instanceId,
1561
1897
  archiveGeneration: lease.archiveGeneration,
1562
1898
  archiveComplete: lease.archiveComplete,
1563
1899
  };
1900
+ captureDisposition = "archive_published";
1564
1901
  } else {
1565
- const replacement = await replaceExpiredWorkspaceArchiveCapture(db, {
1902
+ if (
1903
+ priorCapture.operationId === attempt.operationId &&
1904
+ priorCapture.attempt >= attempt.attempt
1905
+ ) {
1906
+ // Never let an accepted duplicate/stale delivery complete the durable
1907
+ // child while its exact claim still fences the lease. Throwing asks
1908
+ // Temporal for a strictly newer attempt; an actually stale task token is
1909
+ // rejected by Temporal and cannot affect the workflow either way.
1910
+ throw new SandboxDrainRecoveryDeferredError(
1911
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`,
1912
+ );
1913
+ }
1914
+ const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {
1566
1915
  accountId,
1567
1916
  workspaceId: row.workspaceId,
1568
1917
  sandboxGroupId: row.sandboxGroupId,
1569
- priorCaptureId: lease.archiveCapture.id,
1570
- captureId: randomUUID(),
1918
+ captureId: priorCapture.id,
1571
1919
  expectedEpoch: row.leaseEpoch,
1572
- expectedInstanceId: lease.instanceId,
1573
- captureTimeoutMs,
1920
+ expectedInstanceId: instanceId,
1574
1921
  });
1575
- if (!replacement) return false;
1576
- captureClaim = replacement;
1922
+ const unpublishedWarmingProvider =
1923
+ backend === "modal" &&
1924
+ !sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) &&
1925
+ lease.recovery.provider.status === "creating" &&
1926
+ lease.recovery.provider.instanceId === instanceId &&
1927
+ lease.recovery.workspace.status === "not_ready";
1928
+ // Temporal knows that the prior activity attempt is no longer accepted,
1929
+ // but it cannot generically cancel provider I/O. Immediate replacement is
1930
+ // legal only when BOTH the durable claim and the current adapter declare
1931
+ // takeover safety: either the same idempotent request, or an independently
1932
+ // repeatable read-only capture. A readiness probe alone never proves an
1933
+ // unrelated exclusive snapshot RPC has settled.
1934
+ const replace = async () =>
1935
+ await replaceWorkspaceArchiveCaptureAfterProof(db, {
1936
+ accountId,
1937
+ workspaceId: row.workspaceId,
1938
+ sandboxGroupId: row.sandboxGroupId,
1939
+ priorCaptureId: priorCapture.id,
1940
+ captureId: attempt.captureId,
1941
+ operationId: attempt.operationId,
1942
+ attempt: attempt.attempt,
1943
+ expectedEpoch: row.leaseEpoch,
1944
+ expectedInstanceId: instanceId,
1945
+ captureTimeoutMs,
1946
+ });
1947
+
1948
+ // If no durable complete archive exists, replacement is safe only when
1949
+ // this exact provider can be recaptured. A typed missing provider before
1950
+ // the old deadline may mean the dead worker still holds the sole archive
1951
+ // bytes in memory, so preserve its callback until that bounded window.
1952
+ const currentTakeoverSafe =
1953
+ capturePolicy !== null && capturePolicy.takeover !== "exclusive";
1954
+ if (!lease.resumeState) {
1955
+ if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {
1956
+ const replacement = await replace();
1957
+ if (!replacement) {
1958
+ throw new SandboxDrainRecoveryDeferredError(
1959
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`,
1960
+ );
1961
+ }
1962
+ captureClaim = replacement;
1963
+ } else {
1964
+ if (!deadlineExpired) {
1965
+ throw new SandboxDrainRecoveryDeferredError(
1966
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,
1967
+ );
1968
+ }
1969
+ if (!unpublishedWarmingProvider) {
1970
+ throw new SandboxDrainRecoveryDeferredError(
1971
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`,
1972
+ );
1973
+ }
1974
+ // Legacy/unrelated owner on a never-published Modal warming box. Its
1975
+ // deadline elapsed and no user-visible workspace ever existed. Fence
1976
+ // the old callback before the by-id termination below.
1977
+ const replacement = await replace();
1978
+ if (!replacement) {
1979
+ throw new SandboxDrainRecoveryDeferredError(
1980
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`,
1981
+ );
1982
+ }
1983
+ captureClaim = replacement;
1984
+ }
1985
+ } else if (priorCapture.takeoverSafe && currentTakeoverSafe) {
1986
+ // The stable lineage reuses the exact external idempotency key where
1987
+ // supported. Parallel-read adapters may instead recapture independently;
1988
+ // both cases preserve the addressed instance and cannot overlap writes.
1989
+ const replacement = await replace();
1990
+ if (!replacement) {
1991
+ throw new SandboxDrainRecoveryDeferredError(
1992
+ `sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`,
1993
+ );
1994
+ }
1995
+ captureClaim = replacement;
1996
+ } else {
1997
+ if (!deadlineExpired) {
1998
+ throw new SandboxDrainRecoveryDeferredError(
1999
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,
2000
+ );
2001
+ }
2002
+ const providerState = await probeDrainableProvider(settings, lease);
2003
+ if (providerState === "missing") {
2004
+ // Take over on the DB clock before committing loss. This fences an
2005
+ // old teardown callback. Its provider-request lineage remains
2006
+ // durable so a verified late archive can still repair the cold row.
2007
+ const replacement = await replace();
2008
+ if (!replacement) {
2009
+ throw new SandboxDrainRecoveryDeferredError(
2010
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`,
2011
+ );
2012
+ }
2013
+ providerMissingBeforeCapture = true;
2014
+ captureClaim = replacement;
2015
+ } else {
2016
+ // An exclusive provider operation can outlive both its Temporal
2017
+ // activity and its diagnostic deadline. A live command channel says
2018
+ // nothing about whether that operation is still reading/pausing the
2019
+ // workspace. Keep the durable fence; an operator/provider-specific
2020
+ // receipt is required to resolve this deliberately fail-closed case.
2021
+ throw new SandboxDrainRecoveryDeferredError(
2022
+ `sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`,
2023
+ );
2024
+ }
2025
+ }
1577
2026
  }
1578
2027
  } else {
1579
2028
  const claimed = await claimWorkspaceArchiveCapture(db, {
1580
2029
  accountId,
1581
2030
  workspaceId: row.workspaceId,
1582
2031
  sandboxGroupId: row.sandboxGroupId,
1583
- captureId: randomUUID(),
2032
+ captureId: attempt.captureId,
2033
+ operationId: attempt.operationId,
2034
+ attempt: attempt.attempt,
1584
2035
  expectedEpoch: row.leaseEpoch,
1585
2036
  expectedInstanceId: lease.instanceId,
1586
2037
  liveness: "draining",
1587
2038
  captureTimeoutMs,
1588
2039
  minIntervalMs: 0,
2040
+ providerReplaySafe: capturePolicy?.takeover === "same_request",
2041
+ takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive",
1589
2042
  });
1590
2043
  if (claimed.status === "claimed") {
1591
2044
  captureClaim = claimed.claim;
@@ -1599,6 +2052,13 @@ async function terminateDrainableBox(
1599
2052
  const providerState = await probeDrainableProvider(settings, lease);
1600
2053
  if (providerState !== "missing") return false;
1601
2054
  providerMissingBeforeCapture = true;
2055
+ } else if (claimed.status === "capture_in_progress") {
2056
+ // A legacy activity or a racing delivery installed the durable claim.
2057
+ // Its owner may still complete; this child remains the repair owner if
2058
+ // it does not. Completing here would recreate a schedule-sized gap.
2059
+ throw new SandboxDrainRecoveryDeferredError(
2060
+ `sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`,
2061
+ );
1602
2062
  } else {
1603
2063
  return false;
1604
2064
  }
@@ -1607,14 +2067,17 @@ async function terminateDrainableBox(
1607
2067
 
1608
2068
  // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
1609
2069
  // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
1610
- // Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
1611
- // expected): a re-arm or newer epoch that snuck in writes ZERO rows → wrote:false
1612
- // → the seam leaves the box RUNNING and we skip the cold-commit below.
2070
+ // Same exact lease/capture guard as confirmDrainCold. A stale capture owner or
2071
+ // newer epoch writes ZERO rows → wrote:false → the seam leaves the box RUNNING
2072
+ // and skips the cold-commit below.
1613
2073
  // `persisted` tracks whether a real archive landed on the lease this drain —
1614
2074
  // the durable sandbox.box.terminated event below carries it, so a "terminated
1615
2075
  // with NOTHING persisted" (box already dead at drain) is visible in the DB.
1616
- let persisted = false;
1617
- let archiveRevision: string | null = null;
2076
+ let persisted = captureDisposition === "archive_published";
2077
+ let archiveRevision: string | null =
2078
+ captureDisposition === "archive_published"
2079
+ ? (lease.recovery.archive.current?.revision ?? null)
2080
+ : null;
1618
2081
  const persistArchive: PersistArchiveFn = async (
1619
2082
  archiveBase64: string | null,
1620
2083
  archiveMetadata?: WorkspaceArchiveDescriptor,
@@ -1658,14 +2121,19 @@ async function terminateDrainableBox(
1658
2121
  accountId,
1659
2122
  workspaceId: row.workspaceId,
1660
2123
  sandboxGroupId: row.sandboxGroupId,
2124
+ expectedLeaseId: lease.id,
1661
2125
  expectedEpoch: row.leaseEpoch,
1662
2126
  expectedInstanceId: lease.instanceId,
1663
2127
  expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
1664
2128
  captureId: captureClaim.id,
2129
+ providerRequestId: captureClaim.providerRequestId,
1665
2130
  };
1666
2131
  let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;
1667
2132
  if (archiveBase64 === null) {
1668
- result = await persistDrainSnapshot(db, { ...baseInput, workspaceArchive: null });
2133
+ result = await persistDrainSnapshot(db, {
2134
+ ...baseInput,
2135
+ workspaceArchive: null,
2136
+ });
1669
2137
  } else {
1670
2138
  if (!archiveMetadata) {
1671
2139
  throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
@@ -1701,83 +2169,103 @@ async function terminateDrainableBox(
1701
2169
  // lease draining for a later sweep (NEVER terminate a box whose files we
1702
2170
  // could not capture). A persist CAS miss means the box was re-armed and left
1703
2171
  // running, so the cold commit is skipped.
1704
- try {
1705
- const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
1706
- ? { terminated: true, providerMissingBeforeCapture: true }
1707
- : await terminateBox(settings, lease, observability, persistArchive);
1708
- const terminated = typeof termination === "boolean" ? termination : termination.terminated;
1709
- if (!terminated) {
1710
- return false;
1711
- }
2172
+ const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
2173
+ ? { terminated: true, providerMissingBeforeCapture: true }
2174
+ : await terminateBox(
2175
+ settings,
2176
+ lease,
2177
+ observability,
2178
+ persistArchive,
2179
+ captureClaim?.providerRequestId ?? attempt.operationId,
2180
+ captureDisposition,
2181
+ capturePolicy,
2182
+ );
2183
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
2184
+ if (!terminated) {
2185
+ return false;
2186
+ }
1712
2187
 
1713
- const providerMissing =
1714
- providerMissingBeforeCapture ||
1715
- (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
1716
- // The authoritative commit: CAS draining->cold under the epoch fence. If a
1717
- // late re-arm or newer epoch raced in after our re-read, wentCold:false and we
1718
- // report a skip (the box is down; a fresh acquire cold-restores it).
1719
- const { wentCold } = await confirmDrainCold(db, {
1720
- accountId,
1721
- workspaceId: row.workspaceId,
1722
- sandboxGroupId: row.sandboxGroupId,
1723
- expectedEpoch: row.leaseEpoch,
1724
- providerMissingBeforeCapture: providerMissing,
1725
- });
1726
- if (wentCold) {
1727
- // Durable termination record (sandbox-file-persistence observability): who
1728
- // ended this box and whether its /workspace was captured first, appended to
1729
- // every session sharing the group's box. Best-effort: attribution must
1730
- // never affect the drain outcome.
1731
- try {
1732
- await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
1733
- type: "sandbox.box.terminated",
1734
- payload: {
1735
- actor: "reaper",
1736
- persisted,
1737
- archiveRevision,
1738
- instanceId: lease.instanceId,
1739
- providerMissingBeforeCapture: providerMissing,
1740
- },
1741
- });
1742
- } catch (eventError) {
1743
- observability.warn(
1744
- "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
1745
- {
1746
- sandboxGroupId: row.sandboxGroupId,
1747
- error: eventError instanceof Error ? eventError.message : String(eventError),
1748
- },
1749
- );
1750
- }
1751
- }
1752
- return wentCold;
1753
- } finally {
1754
- if (captureClaim && lease.instanceId) {
1755
- await releaseWorkspaceArchiveCapture(db, {
1756
- accountId,
1757
- workspaceId: row.workspaceId,
1758
- sandboxGroupId: row.sandboxGroupId,
1759
- captureId: captureClaim.id,
1760
- expectedEpoch: row.leaseEpoch,
1761
- expectedInstanceId: lease.instanceId,
1762
- }).catch((error) => {
1763
- observability.warn("sandbox reaper: archive capture gate release failed", {
2188
+ const providerMissing =
2189
+ providerMissingBeforeCapture ||
2190
+ (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
2191
+ // The authoritative commit clears the durable capture/teardown claim together
2192
+ // with draining->cold. Until this succeeds, arrivals remain fenced by that
2193
+ // exact claim; a timestamp or a failed provider call can never reopen a box
2194
+ // while termination may still be in flight.
2195
+ const { wentCold } = await confirmDrainCold(db, {
2196
+ accountId,
2197
+ workspaceId: row.workspaceId,
2198
+ sandboxGroupId: row.sandboxGroupId,
2199
+ expectedEpoch: row.leaseEpoch,
2200
+ ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
2201
+ providerMissingBeforeCapture: providerMissing,
2202
+ });
2203
+ if (wentCold) {
2204
+ // Durable termination record (sandbox-file-persistence observability): who
2205
+ // ended this box and whether its /workspace was captured first, appended to
2206
+ // every session sharing the group's box. Best-effort: attribution must
2207
+ // never affect the drain outcome.
2208
+ try {
2209
+ await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
2210
+ type: "sandbox.box.terminated",
2211
+ payload: {
2212
+ actor: "reaper",
2213
+ persisted,
2214
+ archiveRevision,
2215
+ instanceId: lease.instanceId,
2216
+ providerMissingBeforeCapture: providerMissing,
2217
+ },
2218
+ });
2219
+ } catch (eventError) {
2220
+ observability.warn(
2221
+ "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
2222
+ {
1764
2223
  sandboxGroupId: row.sandboxGroupId,
1765
2224
  leaseEpoch: row.leaseEpoch,
1766
- error: error instanceof Error ? error.message : String(error),
1767
- });
1768
- });
2225
+ backend: lease.backend,
2226
+ instanceId: lease.instanceId,
2227
+ error: eventError instanceof Error ? eventError.message : String(eventError),
2228
+ },
2229
+ );
1769
2230
  }
1770
2231
  }
2232
+ return wentCold;
1771
2233
  }
1772
2234
 
1773
2235
  // The persist-capable slice of a live provider session. persistWorkspace()
1774
2236
  // snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
1775
2237
  // archive) and returns the archive bytes. The session also carries the Modal SDK
1776
2238
  // client used to bind the snapshot receipt to the exact provider namespace.
1777
- type PersistableSession = TerminableSession & {
1778
- persistWorkspace?: () => Promise<Uint8Array | undefined>;
2239
+ type PersistableSession = {
2240
+ state?: { workspacePersistence?: unknown };
2241
+ persistWorkspace?: (options?: { requestId: string }) => Promise<Uint8Array | undefined>;
2242
+ runPreStopHooks?: () => Promise<void>;
2243
+ preStop?: (options?: { reason?: string }) => Promise<void>;
2244
+ stop?: (options?: { reason?: string }) => Promise<void>;
2245
+ shutdown?: (options?: { reason?: string }) => Promise<void>;
2246
+ delete?: (options?: { reason?: string }) => Promise<void>;
2247
+ close?: () => Promise<void>;
1779
2248
  };
1780
2249
 
2250
+ function providerDrainLogIdentity(
2251
+ lease: LeaseSnapshot,
2252
+ backend: string,
2253
+ ): {
2254
+ leaseId: string;
2255
+ sandboxGroupId: string;
2256
+ leaseEpoch: number;
2257
+ backend: string;
2258
+ instanceId: string | null;
2259
+ } {
2260
+ return {
2261
+ leaseId: lease.id,
2262
+ sandboxGroupId: lease.sandboxGroupId,
2263
+ leaseEpoch: lease.leaseEpoch,
2264
+ backend,
2265
+ instanceId: lease.instanceId,
2266
+ };
2267
+ }
2268
+
1781
2269
  /**
1782
2270
  * Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
1783
2271
  * lease under the epoch fence, hand the superseded snapshot to durable GC, THEN
@@ -1789,7 +2277,7 @@ type PersistableSession = TerminableSession & {
1789
2277
  * - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark
1790
2278
  * the displaced artifact delete-pending;
1791
2279
  * - the provider-bound global GC deletes that object asynchronously;
1792
- * - terminate the live handle (client.delete / session kill|terminate|close).
2280
+ * - terminate the live handle through the standard Agents SDK lifecycle.
1793
2281
  *
1794
2282
  * Returns true when the box was terminated (or was already gone), false when the
1795
2283
  * persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.
@@ -1803,8 +2291,10 @@ type PersistableSession = TerminableSession & {
1803
2291
  * BEFORE any terminate, so the box is NOT torn down with un-captured files;
1804
2292
  * the next sweep retries, and the provider idle-timeout is the ultimate
1805
2293
  * backstop. We never terminate a box whose snapshot we could not capture.
1806
- * For a draining lease with no envelope (a warming-death row that committed no
1807
- * box, or a 'none'-backed group) there is no live box — a no-op (return true).
2294
+ * A provider-backed row with instance_id but no envelope is never guessed away:
2295
+ * Modal has an explicit by-id rescue adapter above; every other backend stays
2296
+ * draining until its adapter can prove termination. A truly instance-less or
2297
+ * `none`-backed row is a no-op.
1808
2298
  */
1809
2299
  export async function terminateProviderBox(
1810
2300
  settings: ActivityServices["settings"],
@@ -1812,9 +2302,14 @@ export async function terminateProviderBox(
1812
2302
  observability: ActivityServices["observability"],
1813
2303
  persistArchive: PersistArchiveFn,
1814
2304
  createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
2305
+ terminateModalById: typeof terminateModalSandboxById = terminateModalSandboxById,
2306
+ providerCaptureRequestId?: string,
2307
+ captureDisposition: DrainCaptureDisposition = "capture_required",
2308
+ claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
1815
2309
  ): Promise<ProviderTerminationOutcome> {
1816
2310
  const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
1817
2311
  const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
2312
+ const logIdentity = providerDrainLogIdentity(lease, backend);
1818
2313
  // 'none' / no backend -> nothing to terminate.
1819
2314
  if (!backend || backend === "none") {
1820
2315
  return { terminated: true, providerMissingBeforeCapture: false };
@@ -1837,40 +2332,73 @@ export async function terminateProviderBox(
1837
2332
  observability.info(
1838
2333
  "sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)",
1839
2334
  {
1840
- sandboxGroupId: lease.sandboxGroupId,
1841
- backend,
2335
+ ...logIdentity,
1842
2336
  },
1843
2337
  );
1844
2338
  return { terminated: true, providerMissingBeforeCapture: false };
1845
2339
  }
1846
2340
 
2341
+ const durableCapturePolicy =
2342
+ claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);
2343
+ if (!durableCapturePolicy) {
2344
+ throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
2345
+ }
2346
+ if (durableCapturePolicy.liveInstance !== "preserved") {
2347
+ throw new Error(
2348
+ `sandbox backend ${backend} workspace capture would replace the fenced live instance`,
2349
+ );
2350
+ }
2351
+
2352
+ const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
2353
+ // `instance_id` is the lease's authoritative provider address. A recovery
2354
+ // envelope without it is archive/config state, never permission to target a
2355
+ // provider inferred only from stale JSON. Conversely, a provider identity in
2356
+ // the envelope with no authoritative lease identity is inconsistent state:
2357
+ // fail closed instead of silently leaving a live provider behind while the
2358
+ // caller commits the lease cold.
2359
+ if (!lease.instanceId) {
2360
+ if (persistedInstanceId) {
2361
+ throw new Error(
2362
+ `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,
2363
+ );
2364
+ }
2365
+ return { terminated: true, providerMissingBeforeCapture: false };
2366
+ }
2367
+ if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {
2368
+ throw new Error(
2369
+ `sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`,
2370
+ );
2371
+ }
2372
+
1847
2373
  // A warming-death row can have a provider instance id recorded immediately
1848
2374
  // after create() but no resumable envelope yet. For Modal, instance_id is
1849
2375
  // enough to terminate directly; CAS-check first so a re-arm during the sweep
1850
2376
  // leaves the box running.
1851
- if (backend === "modal" && !lease.resumeState && lease.instanceId) {
2377
+ const isUnpublishedWarmingProvider =
2378
+ backend === "modal" &&
2379
+ !persistedInstanceId &&
2380
+ lease.recovery.provider.status === "creating" &&
2381
+ lease.recovery.provider.instanceId === lease.instanceId &&
2382
+ lease.recovery.workspace.status === "not_ready";
2383
+ if (isUnpublishedWarmingProvider) {
1852
2384
  const { wrote } = await persistArchive(null);
1853
2385
  if (!wrote) {
1854
2386
  observability.info(
1855
2387
  "sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING",
1856
2388
  {
1857
- sandboxGroupId: lease.sandboxGroupId,
1858
- backend,
1859
- instanceId: lease.instanceId,
2389
+ ...logIdentity,
1860
2390
  },
1861
2391
  );
1862
2392
  return { terminated: false, providerMissingBeforeCapture: false };
1863
2393
  }
1864
2394
  try {
1865
- await terminateModalSandboxById(settings, lease.instanceId);
2395
+ await terminateModalById(settings, lease.instanceId);
1866
2396
  } catch (error) {
1867
2397
  if (!isProviderSandboxNotFoundError("modal", error)) {
1868
2398
  throw error;
1869
2399
  }
1870
2400
  observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
1871
- sandboxGroupId: lease.sandboxGroupId,
1872
- backend,
1873
- instanceId: lease.instanceId,
2401
+ ...logIdentity,
1874
2402
  });
1875
2403
  }
1876
2404
  // This branch is a warming-death box recorded before any resumable envelope
@@ -1880,12 +2408,18 @@ export async function terminateProviderBox(
1880
2408
  }
1881
2409
 
1882
2410
  // resume_state is the folded group box-envelope (the provider sessionState the
1883
- // box was last persisted as). No envelope -> no live box to stop.
1884
- if (!lease.resumeState) {
1885
- return { terminated: true, providerMissingBeforeCapture: false };
2411
+ // box was last persisted as). An attributed provider with no envelope may
2412
+ // still be live. Only Modal exposes a verified by-id rescue adapter above;
2413
+ // never cold another backend merely because generic resume is impossible.
2414
+ if (!persistedInstanceId || !lease.resumeState) {
2415
+ throw new Error(
2416
+ `sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`,
2417
+ );
1886
2418
  }
1887
2419
 
1888
- const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
2420
+ const client = createClientForBackend(backend as never, settings) as
2421
+ | DrainSandboxClient
2422
+ | undefined;
1889
2423
  if (!client) {
1890
2424
  // 'none' backend resolved to no client.
1891
2425
  return { terminated: true, providerMissingBeforeCapture: false };
@@ -1894,7 +2428,7 @@ export async function terminateProviderBox(
1894
2428
  // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
1895
2429
  // already gone; success.
1896
2430
  let session: PersistableSession | undefined;
1897
- let sessionState: unknown;
2431
+ let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;
1898
2432
  try {
1899
2433
  if (!client.resume || !client.deserializeSessionState) {
1900
2434
  // A cloud backend that cannot prove provider state is not safely
@@ -1918,25 +2452,48 @@ export async function terminateProviderBox(
1918
2452
  const resumedState = await deserializeSandboxSessionStateEnvelope(
1919
2453
  client as never,
1920
2454
  envelopeSessionState,
2455
+ lease.instanceId,
1921
2456
  );
1922
2457
  if (resumedState === undefined) {
1923
2458
  throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
1924
2459
  }
1925
- session = (await client.resume(resumedState)) as PersistableSession;
1926
- sessionState = resumedState;
2460
+ const continuity = sandboxProviderContinuityForState(backend, resumedState, lease.instanceId);
2461
+ const resumed = await resumeExactSandboxSession(
2462
+ client,
2463
+ backend,
2464
+ resumedState,
2465
+ lease.instanceId,
2466
+ continuity ? { continuity } : undefined,
2467
+ );
2468
+ session = resumed.session as PersistableSession;
2469
+ sessionState = resumed.sessionState;
1927
2470
  } catch (error) {
1928
2471
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
1929
2472
  observability.info("sandbox reaper: drainable box already gone before workspace capture", {
1930
- sandboxGroupId: lease.sandboxGroupId,
1931
- backend,
2473
+ ...logIdentity,
1932
2474
  });
1933
- return { terminated: true, providerMissingBeforeCapture: true };
2475
+ return {
2476
+ terminated: true,
2477
+ providerMissingBeforeCapture: captureDisposition !== "archive_published",
2478
+ };
1934
2479
  }
1935
2480
  // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
1936
2481
  // draining for the next sweep) — never cold a box we could not prove is gone.
1937
2482
  throw error;
1938
2483
  }
1939
2484
 
2485
+ const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);
2486
+ if (
2487
+ !liveCapturePolicy ||
2488
+ liveCapturePolicy.takeover !== durableCapturePolicy.takeover ||
2489
+ liveCapturePolicy.strategy !== durableCapturePolicy.strategy ||
2490
+ liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance
2491
+ ) {
2492
+ throw new Error(
2493
+ `sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`,
2494
+ );
2495
+ }
2496
+
1940
2497
  // PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to
1941
2498
  // snapshot must re-throw BEFORE any terminate so files are never lost — the next
1942
2499
  // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
@@ -1946,6 +2503,7 @@ export async function terminateProviderBox(
1946
2503
  session as { state?: { workspacePersistence?: unknown } } | undefined
1947
2504
  )?.state?.workspacePersistence;
1948
2505
  const checkpointBinding =
2506
+ captureDisposition === "capture_required" &&
1949
2507
  backend === "modal" &&
1950
2508
  (sessionWorkspacePersistence === "snapshot_filesystem" ||
1951
2509
  sessionWorkspacePersistence === "snapshot_directory")
@@ -1954,22 +2512,63 @@ export async function terminateProviderBox(
1954
2512
  )
1955
2513
  : null;
1956
2514
  try {
1957
- verifiedArchive = session?.persistWorkspace
1958
- ? await captureVerifiedWorkspaceArchive(session)
1959
- : undefined;
2515
+ if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
2516
+ const capture = captureVerifiedWorkspaceArchive(session, Date.now(), {
2517
+ requestId: providerCaptureRequestId ?? randomUUID(),
2518
+ strategy: liveCapturePolicy.strategy,
2519
+ });
2520
+ verifiedArchive = await awaitProviderCaptureWithLatePublication({
2521
+ capture,
2522
+ timeoutMs: settings.sandboxSnapshotTimeoutMs,
2523
+ timeoutError: new SandboxProviderCaptureTimeoutError(
2524
+ lease.sandboxGroupId,
2525
+ backend,
2526
+ settings.sandboxSnapshotTimeoutMs,
2527
+ lease.leaseEpoch,
2528
+ lease.instanceId,
2529
+ ),
2530
+ publishLate: async (archive) => {
2531
+ try {
2532
+ const result = await persistArchive(
2533
+ archive.base64,
2534
+ archive.descriptor,
2535
+ session,
2536
+ checkpointBinding,
2537
+ );
2538
+ observability.info(
2539
+ result.wrote
2540
+ ? "sandbox reaper: late workspace capture published under durable claim"
2541
+ : "sandbox reaper: late workspace capture fenced by successor",
2542
+ {
2543
+ ...logIdentity,
2544
+ },
2545
+ );
2546
+ } catch (error) {
2547
+ observability.warn("sandbox reaper: late workspace capture publication failed", {
2548
+ ...logIdentity,
2549
+ error: error instanceof Error ? error.message : String(error),
2550
+ });
2551
+ }
2552
+ },
2553
+ observeLateFailure: (error) => {
2554
+ observability.warn("sandbox reaper: timed-out provider capture later failed", {
2555
+ ...logIdentity,
2556
+ error: error instanceof Error ? error.message : String(error),
2557
+ });
2558
+ },
2559
+ });
2560
+ }
1960
2561
  } catch (error) {
1961
2562
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
1962
2563
  observability.info("sandbox reaper: box gone during workspace capture", {
1963
- sandboxGroupId: lease.sandboxGroupId,
1964
- backend,
2564
+ ...logIdentity,
1965
2565
  });
1966
2566
  return { terminated: true, providerMissingBeforeCapture: true };
1967
2567
  }
1968
2568
  observability.warn(
1969
2569
  "sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
1970
2570
  {
1971
- sandboxGroupId: lease.sandboxGroupId,
1972
- backend,
2571
+ ...logIdentity,
1973
2572
  error: error instanceof Error ? error.message : String(error),
1974
2573
  },
1975
2574
  );
@@ -1977,19 +2576,18 @@ export async function terminateProviderBox(
1977
2576
  throw error;
1978
2577
  }
1979
2578
 
1980
- // Fold the captured archive onto the lease under the epoch fence, then GC the
1981
- // superseded snapshot. A CAS miss (wrote:false) means the lease was re-armed
1982
- // mid-drain: the box is wanted again — DO NOT terminate it (return false; the
1983
- // caller skips confirmDrainCold).
2579
+ // Fold the captured archive onto the lease under the exact capture fence, then
2580
+ // GC the superseded snapshot. A CAS miss means this callback no longer owns
2581
+ // teardown — DO NOT terminate (the caller skips confirmDrainCold).
1984
2582
  //
1985
- // Re-arm guard for no-archive path: even when persistWorkspace returned no bytes
1986
- // (a backend with no persistWorkspace, or an empty result), we MUST still CAS-
1987
- // check before delete(). The snapshot window (resume → persistWorkspace) can be
1988
- // long; a late acquireLease re-arm (draining→warm, same epoch) can land in it,
1989
- // so without this check we would delete a box the lease now treats as live. The
1990
- // null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
1991
- // refcount/epoch guard, no write. wrote:false → abort the terminate.
1992
- if (verifiedArchive) {
2583
+ // Ownership guard for the no-archive path: even when persistWorkspace returns
2584
+ // no bytes, require the exact capture receipt before delete(). The null-archive
2585
+ // path performs that CAS without changing archive bytes; wrote:false aborts.
2586
+ if (captureDisposition === "archive_published") {
2587
+ observability.info("sandbox reaper: resuming teardown from durable archive publication", {
2588
+ ...logIdentity,
2589
+ });
2590
+ } else if (verifiedArchive) {
1993
2591
  const { wrote } = await persistArchive(
1994
2592
  verifiedArchive.base64,
1995
2593
  verifiedArchive.descriptor,
@@ -2000,8 +2598,7 @@ export async function terminateProviderBox(
2000
2598
  observability.info(
2001
2599
  "sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
2002
2600
  {
2003
- sandboxGroupId: lease.sandboxGroupId,
2004
- backend,
2601
+ ...logIdentity,
2005
2602
  },
2006
2603
  );
2007
2604
  return { terminated: false, providerMissingBeforeCapture: false };
@@ -2015,33 +2612,19 @@ export async function terminateProviderBox(
2015
2612
  );
2016
2613
  }
2017
2614
 
2018
- // Provider terminate. Prefer the client.delete(state) teardown (the canonical
2019
- // provider stop()); fall back to a session-level kill/terminate/close. A
2020
- // terminate that fails because the box is already gone is success.
2615
+ // Provider terminate through the declared Agents SDK lifecycle. A terminate
2616
+ // that fails because the box is already gone is success.
2021
2617
  try {
2022
- if (client.delete && sessionState !== undefined) {
2023
- await client.delete(sessionState);
2024
- return { terminated: true, providerMissingBeforeCapture: false };
2025
- }
2026
- if (session?.kill) {
2027
- await session.kill();
2028
- return { terminated: true, providerMissingBeforeCapture: false };
2029
- }
2030
- if (session?.terminate) {
2031
- await session.terminate();
2032
- return { terminated: true, providerMissingBeforeCapture: false };
2033
- }
2034
- if (session?.close) {
2035
- if (!session.closed) {
2036
- await session.close();
2037
- }
2038
- return { terminated: true, providerMissingBeforeCapture: false };
2618
+ if (!session || !sessionState) {
2619
+ throw new Error(`sandbox backend ${backend} returned no terminable session state`);
2039
2620
  }
2621
+ prepareProviderForTeardownAfterCapture(backend, session);
2622
+ await terminateManagedSandboxSession(client, sessionState, session);
2623
+ return { terminated: true, providerMissingBeforeCapture: false };
2040
2624
  } catch (error) {
2041
2625
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
2042
2626
  observability.info("sandbox reaper: provider already gone during terminate", {
2043
- sandboxGroupId: lease.sandboxGroupId,
2044
- backend,
2627
+ ...logIdentity,
2045
2628
  });
2046
2629
  // Capture was already durably folded above, so this is not a
2047
2630
  // missing-before-capture outcome.
@@ -2049,5 +2632,4 @@ export async function terminateProviderBox(
2049
2632
  }
2050
2633
  throw error;
2051
2634
  }
2052
- throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
2053
2635
  }