@opengeni/worker-bundle 0.12.7 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/activities/agent-turn.d.ts +778 -0
  2. package/dist/activities/capabilities.d.ts +18 -0
  3. package/dist/activities/codex-auth.d.ts +1 -0
  4. package/dist/activities/codex-capacity.d.ts +31 -0
  5. package/dist/activities/codex-fleet-shadow.d.ts +74 -0
  6. package/dist/activities/codex-rotation.d.ts +261 -0
  7. package/dist/activities/common.d.ts +17 -0
  8. package/dist/activities/context-compaction.d.ts +81 -0
  9. package/dist/activities/documents.d.ts +39 -0
  10. package/dist/activities/environment.d.ts +97 -0
  11. package/dist/activities/expiring-material-renewal.d.ts +38 -0
  12. package/dist/activities/file-upload-reaper.d.ts +25 -0
  13. package/dist/activities/git-credential-renewal.d.ts +41 -0
  14. package/dist/activities/goals.d.ts +16 -0
  15. package/dist/activities/mcp-credentials.d.ts +14 -0
  16. package/dist/activities/packs.d.ts +77 -0
  17. package/dist/activities/parent-wake.d.ts +54 -0
  18. package/dist/activities/recording.d.ts +90 -0
  19. package/dist/activities/redaction.d.ts +1 -0
  20. package/dist/activities/rig-verification.d.ts +131 -0
  21. package/dist/activities/run-credential-renewal.d.ts +28 -0
  22. package/dist/activities/run-credentials.d.ts +36 -0
  23. package/dist/activities/run-input.d.ts +106 -0
  24. package/dist/activities/sandbox-lease.d.ts +128 -0
  25. package/dist/activities/scheduled-tasks.d.ts +4 -0
  26. package/dist/activities/session-state.d.ts +39 -0
  27. package/dist/activities/streaming.d.ts +24 -0
  28. package/dist/activities/toolspace-token-renewal.d.ts +23 -0
  29. package/dist/activities/turn-attempt-fenced.d.ts +11 -0
  30. package/dist/activities/types.d.ts +240 -0
  31. package/dist/activities/workflow-wake.d.ts +15 -0
  32. package/dist/activities/workspace-capture.d.ts +105 -0
  33. package/dist/activities.d.ts +312 -0
  34. package/dist/concurrency.d.ts +19 -0
  35. package/dist/host-export-pump.d.ts +66 -0
  36. package/dist/host-export.d.ts +6 -67
  37. package/dist/http.d.ts +32 -0
  38. package/dist/index.d.ts +27 -388
  39. package/dist/index.js +2730 -1211
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +200 -0
  42. package/dist/op-journal.d.ts +10 -0
  43. package/dist/sandbox-resume.d.ts +152 -0
  44. package/dist/sandbox-routing.d.ts +159 -0
  45. package/dist/worker-service-lifecycle.d.ts +20 -0
  46. package/dist/workflow-bundle.js +88 -20
  47. package/dist/workflows/activities.d.ts +27 -0
  48. package/dist/workflows/document-index.d.ts +41 -0
  49. package/dist/workflows/file-upload-reaper.d.ts +2 -0
  50. package/dist/workflows/rig-verification.d.ts +3 -0
  51. package/dist/workflows/sandbox-reaper.d.ts +1 -0
  52. package/dist/workflows/scheduled-tasks.d.ts +17 -0
  53. package/dist/workflows/session.d.ts +58 -0
  54. package/dist/workflows/workflow-wake.d.ts +3 -0
  55. package/dist/workflows.d.ts +7 -0
  56. package/package.json +13 -13
  57. package/src/activities/agent-turn.ts +874 -134
  58. package/src/activities/context-compaction.ts +265 -76
  59. package/src/activities/environment.ts +1 -0
  60. package/src/activities/goals.ts +14 -0
  61. package/src/activities/rig-verification.ts +2 -0
  62. package/src/activities/run-input.ts +18 -4
  63. package/src/activities/sandbox-lease.ts +787 -133
  64. package/src/activities/session-state.ts +56 -0
  65. package/src/activities/types.ts +21 -0
  66. package/src/activities/workspace-capture.ts +95 -4
  67. package/src/activities.ts +3 -0
  68. package/src/observability-metrics.ts +94 -0
  69. package/src/sandbox-resume.ts +383 -90
  70. package/src/sandbox-routing.ts +32 -10
  71. package/src/workflows/activities.ts +1 -0
  72. package/src/workflows/session.ts +70 -0
@@ -30,12 +30,18 @@
30
30
  // so a box that was re-armed mid-sweep is never terminated out from under a live
31
31
  // holder.
32
32
 
33
+ import { randomUUID } from "node:crypto";
33
34
  import {
34
35
  accrueWarmSeconds,
36
+ adoptLegacyModalCheckpointArtifact,
35
37
  confirmDrainCold,
36
38
  appendSessionEventToSandboxGroup,
39
+ bindRetainedProcessProviderIdentity,
40
+ claimWorkspaceArchiveCapture,
41
+ claimSandboxCheckpointArtifactsForGc,
37
42
  claimTerminalRetainedProcesses,
38
43
  countActiveRetainedProcessesByOwnerState,
44
+ countSandboxCheckpointArtifactsByState,
39
45
  countExpiredDrainingSandboxLeases,
40
46
  countQueuedTurns,
41
47
  countSandboxLeasesByLiveness,
@@ -43,16 +49,25 @@ import {
43
49
  forceDrainOverLimitViewerOnlyBoxes,
44
50
  getBillingBalance,
45
51
  listCreditBalancesByAccount,
52
+ listLegacyModalCheckpointSlots,
46
53
  listLiveModalSandboxLeaseAttributions,
47
54
  listMeterableWarmLeases,
55
+ listSandboxViewerForceDrainWorkspaceIds,
56
+ markSandboxCheckpointArtifactDeletePending,
48
57
  persistDrainSnapshot,
58
+ pruneDeletedSandboxCheckpointArtifacts,
59
+ registerSandboxCheckpointArtifact,
49
60
  recordRetainedProcessReconciliationProof,
50
- readWorkspaceArchiveCapturePreflight,
61
+ releaseWorkspaceArchiveCapture,
62
+ replaceExpiredWorkspaceArchiveCapture,
51
63
  readLease,
52
64
  reapExpiredSessionListSnapshots,
53
65
  reapStaleLeaseHoldersGlobal,
66
+ requestDueSandboxRotationsGlobal,
67
+ readSandboxRotationBacklog,
54
68
  retainedProcessReconciliationProof,
55
69
  retainedProcessSettlementIdentity,
70
+ settleSandboxCheckpointArtifactGc,
56
71
  rlsContextForWorkspace,
57
72
  settleRetainedProcess,
58
73
  type MeterableWarmLease,
@@ -61,24 +76,32 @@ import {
61
76
  type SandboxRetainedProcess,
62
77
  type LeaseSnapshot,
63
78
  } from "@opengeni/db";
64
- import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
79
+ import { sandboxArchiveCaptureTimeoutMs, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
65
80
  import {
66
- // The reaper attaches to the box by id to terminate it. It does NOT use
67
- // establishSandboxSessionFromEnvelope (which cold-RESTORES via create() on a
68
- // NotFound) — restoring a box just to immediately kill it is wrong. Instead it
69
- // builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
70
- // its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
71
- // (NotFound on resume) is already down, a clean no-op.
81
+ // Normal drain teardown builds the client and resumes the envelope directly:
82
+ // a live box gets its /workspace persisted before termination, while a gone
83
+ // box is typed as provider loss. Stale capture reconciliation separately uses
84
+ // establishSandboxSessionFromEnvelope in strict resume-only mode so it can
85
+ // reuse the runtime's provider-ready proof without ever cold-restoring.
72
86
  captureVerifiedWorkspaceArchive,
73
87
  createSandboxClientForBackend,
74
- deletePriorPersistedSnapshot,
88
+ deleteModalCheckpointSnapshot,
75
89
  deserializeSandboxSessionStateEnvelope,
90
+ establishSandboxSessionFromEnvelope,
91
+ inspectModalSandboxLifecycle,
76
92
  isExecSessionLostBanner,
77
93
  isProviderSandboxNotFoundError,
78
94
  parseExecBannerExitCode,
95
+ resolveModalCheckpointProviderBindingForLiveSandbox,
96
+ resolveModalCheckpointProviderBinding,
97
+ resolveModalCheckpointProviderBindingForSession,
98
+ sandboxCommandExitCode,
99
+ sandboxCommandStillRunning,
79
100
  sweepModalOrphanSandboxes,
80
101
  terminateModalSandboxById,
102
+ verifySandboxExecReadiness,
81
103
  type ModalOrphanSweepTermination,
104
+ type ModalCheckpointProviderBinding,
82
105
  type WorkspaceArchiveDescriptor,
83
106
  } from "@opengeni/runtime";
84
107
  import type { ActivityServices } from "./types";
@@ -89,9 +112,13 @@ import {
89
112
  recordExpiredDrainingSandboxLeaseGauges,
90
113
  recordRetainedProcessInventoryGauges,
91
114
  recordRetainedProcessReconciliation,
115
+ recordSandboxCheckpointArtifactGauges,
116
+ recordSandboxCheckpointArtifactOutcome,
117
+ recordSandboxDeadlineRotationsRequested,
92
118
  type RetainedProcessReconciliationOutcome,
93
119
  recordSandboxLeaseGauges,
94
120
  recordSandboxOrphansTerminated,
121
+ recordSandboxRotationBacklogGauges,
95
122
  recordTurnsQueuedGauge,
96
123
  } from "../observability-metrics";
97
124
  import { providerIdentityFromResumeState } from "../sandbox-routing";
@@ -143,21 +170,23 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
143
170
  * archive onto the lease under the epoch fence, BEFORE the box is terminated
144
171
  * (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over
145
172
  * db/accountId/row and delegates to persistDrainSnapshot. Returns:
146
- * - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
147
- * is wanted again; the seam MUST NOT terminate it.
148
- * - priorArchiveForGc -> the one archive made unreachable by rotation.
173
+ * `wrote:false` means the CAS missed (re-armed / newer epoch / vanished). The
174
+ * box is wanted again, so the seam MUST NOT terminate it. Provider-native
175
+ * deletion is owned independently by artifact-ledger GC.
149
176
  *
150
177
  * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
151
178
  * backends with no persistWorkspace — ensures a re-arm during the snapshot window
152
- * aborts the terminate before client.delete()). The GC candidate is always
153
- * null in this case.
179
+ * aborts the terminate before client.delete()).
154
180
  */
155
181
  export type PersistArchiveFn = (
156
182
  archiveBase64: string | null,
157
183
  archiveMetadata?: WorkspaceArchiveDescriptor,
184
+ providerSession?: unknown,
185
+ providerBinding?: Awaited<
186
+ ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
187
+ > | null,
158
188
  ) => Promise<{
159
189
  wrote: boolean;
160
- priorArchiveForGc: string | null;
161
190
  archiveRevision?: string | null;
162
191
  }>;
163
192
 
@@ -200,10 +229,21 @@ export type SandboxLeaseActivityOptions = {
200
229
  /** Override only the read-only provider process probe. The canonical DB
201
230
  * settlement remains real in tests and production. */
202
231
  probeRetainedProcess?: RetainedProcessProbeFn;
232
+ /** Observe a historical Modal box after its lease row has advanced to a
233
+ * successor identity. Lifecycle-only; a live box remains ambiguous. */
234
+ inspectHistoricalModalSandbox?: HistoricalModalSandboxLifecycleProbeFn;
235
+ /** Override the read-only exact-instance readiness probe used before the
236
+ * reaper settles blockers for a provider that has definitively vanished. */
237
+ probeDrainableProvider?: DrainableProviderProbeFn;
203
238
  };
204
239
 
205
240
  export type RetainedProcessProbeResult =
206
241
  | { status: "proved"; proof: RetainedProcessProviderProof }
242
+ | {
243
+ status: "binding";
244
+ providerBindingKey: string;
245
+ providerBinding: ModalCheckpointProviderBinding;
246
+ }
207
247
  | {
208
248
  status: "deferred";
209
249
  reason:
@@ -213,7 +253,9 @@ export type RetainedProcessProbeResult =
213
253
  | "provider_running"
214
254
  | "provider_unknown"
215
255
  | "provider_timeout"
216
- | "provider_error";
256
+ | "provider_error"
257
+ | "provider_binding_missing"
258
+ | "provider_binding_mismatch";
217
259
  };
218
260
 
219
261
  export type RetainedProcessProbeFn = (
@@ -222,6 +264,13 @@ export type RetainedProcessProbeFn = (
222
264
  process: SandboxRetainedProcess,
223
265
  ) => Promise<RetainedProcessProbeResult>;
224
266
 
267
+ export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
268
+
269
+ export type DrainableProviderProbeFn = (
270
+ settings: ActivityServices["settings"],
271
+ lease: LeaseSnapshot,
272
+ ) => Promise<"ready" | "missing">;
273
+
225
274
  export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
226
275
  export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
227
276
  export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
@@ -234,6 +283,7 @@ export function createSandboxLeaseActivities(
234
283
  const sweepModalOrphans: SweepModalOrphansFn =
235
284
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
236
285
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
286
+ const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
237
287
  /**
238
288
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
239
289
  * Gated by the caller (the Schedule is only registered when
@@ -277,6 +327,18 @@ export function createSandboxLeaseActivities(
277
327
  }
278
328
  await refreshQueueLeaseAndCreditGauges(db, observability);
279
329
 
330
+ const rotationsRequested = await requestDueSandboxRotationsGlobal(
331
+ db,
332
+ settings.sandboxRotationLeadMs,
333
+ settings.sandboxRotationBatchSize,
334
+ ).catch((error) => {
335
+ observability.warn("sandbox reaper: provider-deadline rotation request failed", {
336
+ error: error instanceof Error ? error.message : String(error),
337
+ });
338
+ return 0;
339
+ });
340
+ recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
341
+
280
342
  // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
281
343
  // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
282
344
  // them). GROUP+epoch+tick idempotent → a shared box is one stream; an
@@ -287,10 +349,29 @@ export function createSandboxLeaseActivities(
287
349
  // workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
288
350
  // boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
289
351
  // draining rows are caught by the same sweep's terminate below.
352
+ const forceDrainWorkspaceIds = new Set<string>();
353
+ if (
354
+ settings.billingMode === "stripe" ||
355
+ settings.usageLimitsMode === "managed" ||
356
+ settings.sandboxMaxWarmSecondsPerWorkspace > 0
357
+ ) {
358
+ for (const workspaceId of metered.workspaceIds) {
359
+ forceDrainWorkspaceIds.add(workspaceId);
360
+ }
361
+ }
362
+ try {
363
+ for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
364
+ forceDrainWorkspaceIds.add(workspaceId);
365
+ }
366
+ } catch (error) {
367
+ observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
368
+ error: error instanceof Error ? error.message : String(error),
369
+ });
370
+ }
290
371
  const forceDrained = await forceDrainOverLimitWorkspaces(
291
372
  db,
292
373
  settings,
293
- metered.workspaceIds,
374
+ forceDrainWorkspaceIds,
294
375
  observability,
295
376
  );
296
377
 
@@ -298,7 +379,13 @@ export function createSandboxLeaseActivities(
298
379
  // only a bounded inspection batch; exact provider exit/loss proof is
299
380
  // checkpointed before canonical settlement. Ambiguous, running, timed-out,
300
381
  // unsupported, and transient provider states preserve every durable row.
301
- await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
382
+ await reconcileTerminalRetainedProcesses(
383
+ db,
384
+ settings,
385
+ observability,
386
+ probeRetainedProcess,
387
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
388
+ );
302
389
 
303
390
  // (1) The DB-only cross-workspace sweep. Returns the drainable rows.
304
391
  const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
@@ -331,6 +418,7 @@ export function createSandboxLeaseActivities(
331
418
  row,
332
419
  observability,
333
420
  terminateBox,
421
+ probeDrainableProvider,
334
422
  );
335
423
  if (drainedCold) {
336
424
  terminated += 1;
@@ -357,13 +445,24 @@ export function createSandboxLeaseActivities(
357
445
  });
358
446
  }
359
447
 
448
+ const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
449
+ (error) => {
450
+ observability.warn("sandbox reaper: checkpoint artifact GC failed", {
451
+ error: error instanceof Error ? error.message : String(error),
452
+ });
453
+ return { claimed: 0, deleted: 0, failed: 0 };
454
+ },
455
+ );
456
+
360
457
  await refreshQueueLeaseAndCreditGauges(db, observability);
361
458
 
362
459
  if (
363
460
  drainable.length > 0 ||
364
461
  metered.accrued > 0 ||
365
462
  forceDrained > 0 ||
366
- modalOrphansTerminated > 0
463
+ modalOrphansTerminated > 0 ||
464
+ rotationsRequested > 0 ||
465
+ checkpointGc.claimed > 0
367
466
  ) {
368
467
  observability.info("sandbox reaper swept", {
369
468
  drainable: drainable.length,
@@ -372,6 +471,10 @@ export function createSandboxLeaseActivities(
372
471
  metered: metered.accrued,
373
472
  forceDrained,
374
473
  modalOrphansTerminated,
474
+ rotationsRequested,
475
+ checkpointArtifactsClaimed: checkpointGc.claimed,
476
+ checkpointArtifactsDeleted: checkpointGc.deleted,
477
+ checkpointArtifactsFailed: checkpointGc.failed,
375
478
  });
376
479
  }
377
480
 
@@ -388,11 +491,143 @@ export function createSandboxLeaseActivities(
388
491
  return { reapSandboxLeases };
389
492
  }
390
493
 
494
+ const CHECKPOINT_GC_LIMIT = 50;
495
+ const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
496
+ const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
497
+ const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
498
+
499
+ async function gcSandboxCheckpointArtifacts(
500
+ db: ActivityServices["db"],
501
+ settings: ActivityServices["settings"],
502
+ observability: ActivityServices["observability"],
503
+ ): Promise<{ claimed: number; deleted: number; failed: number }> {
504
+ // Explicit OPENGENI_MODAL_TOKEN_* settings are optional: the SDK also supports
505
+ // an operator-provisioned Modal profile. Durable artifacts must not become
506
+ // immortal merely because credentials arrive through that supported path.
507
+ const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(
508
+ (error) => {
509
+ observability.warn("sandbox reaper: legacy Modal checkpoint discovery failed", {
510
+ error: error instanceof Error ? error.message : String(error),
511
+ });
512
+ return 0;
513
+ },
514
+ );
515
+ if (adopted > 0) {
516
+ recordSandboxCheckpointArtifactOutcome(observability, "legacy_adopted", adopted);
517
+ observability.info("sandbox reaper: adopted legacy Modal checkpoint receipts", { adopted });
518
+ }
519
+ const claimId = crypto.randomUUID();
520
+ const claims = await claimSandboxCheckpointArtifactsForGc(db, {
521
+ claimId,
522
+ limit: CHECKPOINT_GC_LIMIT,
523
+ claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS,
524
+ });
525
+ recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
526
+ let deleted = 0;
527
+ let failed = 0;
528
+ for (const claim of claims) {
529
+ try {
530
+ if (claim.providerBackend !== "modal") {
531
+ throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
532
+ }
533
+ await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);
534
+ const settled = await settleSandboxCheckpointArtifactGc(db, {
535
+ artifactId: claim.id,
536
+ claimId,
537
+ deleted: true,
538
+ retryAfterMs: 1_000,
539
+ });
540
+ if (settled) {
541
+ deleted += 1;
542
+ recordSandboxCheckpointArtifactOutcome(observability, "deleted");
543
+ }
544
+ } catch (error) {
545
+ failed += 1;
546
+ recordSandboxCheckpointArtifactOutcome(observability, "delete_failed");
547
+ const retryAfterMs = Math.min(
548
+ 6 * 60 * 60_000,
549
+ 5_000 * 2 ** Math.min(claim.deleteAttempts, 17),
550
+ );
551
+ await settleSandboxCheckpointArtifactGc(db, {
552
+ artifactId: claim.id,
553
+ claimId,
554
+ deleted: false,
555
+ error: error instanceof Error ? error.message : String(error),
556
+ retryAfterMs,
557
+ }).catch(() => false);
558
+ observability.warn("sandbox reaper: checkpoint artifact delete failed", {
559
+ artifactId: claim.id,
560
+ providerBackend: claim.providerBackend,
561
+ objectKind: claim.objectKind,
562
+ attempt: claim.deleteAttempts,
563
+ error: error instanceof Error ? error.message : String(error),
564
+ });
565
+ }
566
+ }
567
+ const pruned = await pruneDeletedSandboxCheckpointArtifacts(
568
+ db,
569
+ CHECKPOINT_TOMBSTONE_RETENTION_MS,
570
+ CHECKPOINT_TOMBSTONE_PRUNE_LIMIT,
571
+ );
572
+ recordSandboxCheckpointArtifactOutcome(observability, "tombstone_pruned", pruned);
573
+ return { claimed: claims.length, deleted, failed };
574
+ }
575
+
576
+ async function adoptLegacyModalCheckpointReceipts(
577
+ db: ActivityServices["db"],
578
+ settings: ActivityServices["settings"],
579
+ observability: ActivityServices["observability"],
580
+ ): Promise<number> {
581
+ const slots = await listLegacyModalCheckpointSlots(db, 25);
582
+ if (slots.length === 0) return 0;
583
+ const bindings = new Map<
584
+ string,
585
+ Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>
586
+ >();
587
+ let adopted = 0;
588
+ for (const slot of slots) {
589
+ try {
590
+ let binding = bindings.get(slot.instanceId);
591
+ if (!binding) {
592
+ binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);
593
+ bindings.set(slot.instanceId, binding);
594
+ }
595
+ const resolvedBinding = await binding;
596
+ if (
597
+ await adoptLegacyModalCheckpointArtifact(db, {
598
+ accountId: slot.accountId,
599
+ workspaceId: slot.workspaceId,
600
+ sandboxGroupId: slot.sandboxGroupId,
601
+ leaseId: slot.leaseId,
602
+ leaseEpoch: slot.leaseEpoch,
603
+ workspaceGeneration: slot.workspaceGeneration,
604
+ slot: slot.slot,
605
+ archiveBase64: slot.archiveBase64,
606
+ descriptor: slot.descriptor,
607
+ providerBindingKey: resolvedBinding.key,
608
+ providerBinding: resolvedBinding.binding,
609
+ })
610
+ ) {
611
+ adopted += 1;
612
+ }
613
+ } catch (error) {
614
+ observability.warn("sandbox reaper: legacy Modal checkpoint adoption failed", {
615
+ workspaceId: slot.workspaceId,
616
+ sandboxGroupId: slot.sandboxGroupId,
617
+ slot: slot.slot,
618
+ error: error instanceof Error ? error.message : String(error),
619
+ });
620
+ }
621
+ }
622
+ return adopted;
623
+ }
624
+
391
625
  async function reconcileTerminalRetainedProcesses(
392
626
  db: ActivityServices["db"],
393
627
  settings: ActivityServices["settings"],
394
628
  observability: ActivityServices["observability"],
395
629
  probe: RetainedProcessProbeFn,
630
+ inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
396
631
  ): Promise<void> {
397
632
  const claimId = crypto.randomUUID();
398
633
  let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
@@ -415,39 +650,143 @@ async function reconcileTerminalRetainedProcesses(
415
650
  const expected = retainedProcessSettlementIdentity(process);
416
651
  let proof = retainedProcessReconciliationProof(process);
417
652
  if (!proof) {
418
- const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
419
- () => null,
420
- );
653
+ let observation: RetainedProcessProbeResult | null = null;
654
+
655
+ // Legacy rows created before provider bindings existed are adoptable only
656
+ // when the exact historical sandbox resolves in the configured Modal
657
+ // namespace. NotFound is ambiguous here: it can mean either deletion or
658
+ // credential rotation, so it never creates ownership or loss proof.
421
659
  if (
422
- !lease ||
423
- lease.id !== process.leaseId ||
424
- lease.leaseEpoch !== process.leaseEpoch ||
425
- lease.backend !== process.providerBackend ||
426
- lease.instanceId !== process.providerInstanceId ||
427
- process.routeTargetId !== null
660
+ process.providerBackend === "modal" &&
661
+ process.routeTargetId === null &&
662
+ !process.providerBindingKey
428
663
  ) {
429
- await deferRetainedProcessClaim(
430
- db,
431
- settings,
432
- observability,
433
- process,
434
- expected,
435
- claim.claimId,
436
- "identity_mismatch",
437
- );
438
- continue;
664
+ try {
665
+ const lifecycle = await withRetainedProcessProbeTimeout(
666
+ inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),
667
+ );
668
+ if (lifecycle.status === "not_found") {
669
+ observation = { status: "deferred", reason: "provider_binding_missing" };
670
+ } else {
671
+ process = await bindRetainedProcessProviderIdentity(db, {
672
+ accountId: process.accountId,
673
+ workspaceId: process.workspaceId,
674
+ sessionId: process.sessionId,
675
+ processId: process.id,
676
+ expected,
677
+ claimId: claim.claimId,
678
+ providerBindingKey: lifecycle.providerBindingKey,
679
+ providerBinding: lifecycle.providerBinding,
680
+ });
681
+ recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
682
+ if (lifecycle.status === "terminated") {
683
+ observation = {
684
+ status: "proved",
685
+ proof: {
686
+ outcome: "lost",
687
+ exitCode: null,
688
+ reason: "provider_instance_terminated",
689
+ },
690
+ };
691
+ }
692
+ }
693
+ } catch (error) {
694
+ observation = {
695
+ status: "deferred",
696
+ reason:
697
+ error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
698
+ };
699
+ }
439
700
  }
440
701
 
441
- let observation: RetainedProcessProbeResult;
442
- try {
443
- observation = await probe(settings, lease, process);
444
- } catch (error) {
445
- observability.warn("sandbox reaper: retained-process provider probe failed", {
446
- processId: process.id,
447
- providerBackend: process.providerBackend,
448
- error: error instanceof Error ? error.message : String(error),
449
- });
450
- observation = { status: "deferred", reason: "provider_error" };
702
+ if (observation === null) {
703
+ const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
704
+ () => null,
705
+ );
706
+ const currentLeaseIdentityMatches =
707
+ Boolean(lease) &&
708
+ lease!.id === process.leaseId &&
709
+ lease!.leaseEpoch === process.leaseEpoch &&
710
+ lease!.backend === process.providerBackend &&
711
+ lease!.instanceId === process.providerInstanceId &&
712
+ process.routeTargetId === null;
713
+ if (!currentLeaseIdentityMatches) {
714
+ // The lease may already be cold or belong to a successor. Its current
715
+ // resume envelope cannot interrogate the historical process, but a
716
+ // bound process can safely inspect the exact old Modal sandbox id.
717
+ if (process.providerBackend !== "modal" || process.routeTargetId !== null) {
718
+ observation = { status: "deferred", reason: "identity_mismatch" };
719
+ } else {
720
+ try {
721
+ const lifecycle = await withRetainedProcessProbeTimeout(
722
+ inspectHistoricalModalSandbox(
723
+ settings,
724
+ process.providerInstanceId,
725
+ process.providerBindingKey,
726
+ ),
727
+ );
728
+ observation =
729
+ lifecycle.status === "not_found"
730
+ ? {
731
+ status: "proved",
732
+ proof: {
733
+ outcome: "lost",
734
+ exitCode: null,
735
+ reason: "provider_instance_not_found",
736
+ },
737
+ }
738
+ : lifecycle.status === "terminated"
739
+ ? {
740
+ status: "proved",
741
+ proof: {
742
+ outcome: "lost",
743
+ exitCode: null,
744
+ reason: "provider_instance_terminated",
745
+ },
746
+ }
747
+ : { status: "deferred", reason: "identity_mismatch" };
748
+ } catch (error) {
749
+ observation = {
750
+ status: "deferred",
751
+ reason:
752
+ error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
753
+ };
754
+ }
755
+ }
756
+ } else {
757
+ try {
758
+ observation = await probe(settings, lease!, process);
759
+ } catch (error) {
760
+ observability.warn("sandbox reaper: retained-process provider probe failed", {
761
+ processId: process.id,
762
+ providerBackend: process.providerBackend,
763
+ error: error instanceof Error ? error.message : String(error),
764
+ });
765
+ observation = { status: "deferred", reason: "provider_error" };
766
+ }
767
+ }
768
+ }
769
+ if (observation.status === "binding") {
770
+ try {
771
+ process = await bindRetainedProcessProviderIdentity(db, {
772
+ accountId: process.accountId,
773
+ workspaceId: process.workspaceId,
774
+ sessionId: process.sessionId,
775
+ processId: process.id,
776
+ expected,
777
+ claimId: claim.claimId,
778
+ providerBindingKey: observation.providerBindingKey,
779
+ providerBinding: observation.providerBinding,
780
+ });
781
+ recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
782
+ observation = { status: "deferred", reason: "provider_running" };
783
+ } catch (error) {
784
+ observability.warn("sandbox reaper: retained-process binding adoption failed", {
785
+ processId: process.id,
786
+ error: error instanceof Error ? error.message : String(error),
787
+ });
788
+ observation = { status: "deferred", reason: "provider_error" };
789
+ }
451
790
  }
452
791
  if (observation.status === "deferred") {
453
792
  await deferRetainedProcessClaim(
@@ -613,6 +952,25 @@ export async function probeRetainedProcessAtProvider(
613
952
  return { status: "deferred", reason: "identity_mismatch" };
614
953
  }
615
954
 
955
+ let configuredModalBinding: Awaited<
956
+ ReturnType<typeof resolveModalCheckpointProviderBinding>
957
+ > | null = null;
958
+ if (process.providerBackend === "modal" && process.providerBindingKey) {
959
+ try {
960
+ configuredModalBinding = await withRetainedProcessProbeTimeout(
961
+ resolveModalCheckpointProviderBinding(settings),
962
+ );
963
+ } catch (error) {
964
+ return {
965
+ status: "deferred",
966
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
967
+ };
968
+ }
969
+ if (configuredModalBinding.key !== process.providerBindingKey) {
970
+ return { status: "deferred", reason: "provider_binding_mismatch" };
971
+ }
972
+ }
973
+
616
974
  let client: RetainedProcessProbeClient | undefined;
617
975
  try {
618
976
  client = createSandboxClientForBackend(process.providerBackend as never, settings) as
@@ -649,6 +1007,9 @@ export async function probeRetainedProcessAtProvider(
649
1007
  return { status: "deferred", reason: "provider_timeout" };
650
1008
  }
651
1009
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
1010
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
1011
+ return { status: "deferred", reason: "provider_binding_missing" };
1012
+ }
652
1013
  return {
653
1014
  status: "proved",
654
1015
  proof: {
@@ -663,6 +1024,32 @@ export async function probeRetainedProcessAtProvider(
663
1024
  if (typeof session.writeStdin !== "function") {
664
1025
  return { status: "deferred", reason: "backend_unsupported" };
665
1026
  }
1027
+ if (process.providerBackend === "modal") {
1028
+ let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;
1029
+ try {
1030
+ liveBinding = await withRetainedProcessProbeTimeout(
1031
+ resolveModalCheckpointProviderBindingForSession(settings, session),
1032
+ );
1033
+ } catch (error) {
1034
+ return {
1035
+ status: "deferred",
1036
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
1037
+ };
1038
+ }
1039
+ if (!process.providerBindingKey) {
1040
+ return {
1041
+ status: "binding",
1042
+ providerBindingKey: liveBinding.key,
1043
+ providerBinding: liveBinding.binding,
1044
+ };
1045
+ }
1046
+ if (
1047
+ liveBinding.key !== process.providerBindingKey ||
1048
+ configuredModalBinding?.key !== process.providerBindingKey
1049
+ ) {
1050
+ return { status: "deferred", reason: "provider_binding_mismatch" };
1051
+ }
1052
+ }
666
1053
 
667
1054
  let result: unknown;
668
1055
  try {
@@ -679,6 +1066,9 @@ export async function probeRetainedProcessAtProvider(
679
1066
  return { status: "deferred", reason: "provider_timeout" };
680
1067
  }
681
1068
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
1069
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
1070
+ return { status: "deferred", reason: "provider_binding_missing" };
1071
+ }
682
1072
  return {
683
1073
  status: "proved",
684
1074
  proof: {
@@ -690,7 +1080,46 @@ export async function probeRetainedProcessAtProvider(
690
1080
  }
691
1081
  return { status: "deferred", reason: "provider_error" };
692
1082
  }
693
- return classifyRetainedProcessPollResult(result, process.providerSessionId);
1083
+ const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
1084
+ if (
1085
+ observation.status === "deferred" &&
1086
+ observation.reason === "provider_running" &&
1087
+ lease.rotationRequestedAt !== null
1088
+ ) {
1089
+ // A background PTY cannot outlive the finite provider box. Interrupt only
1090
+ // this exact durable provider session after rotation admission is fenced;
1091
+ // settlement still requires an exit/loss banner on this or a later sweep.
1092
+ try {
1093
+ const interrupted = await withRetainedProcessProbeTimeout(
1094
+ session.writeStdin({
1095
+ sessionId: process.providerSessionId,
1096
+ chars: "\u0003",
1097
+ yieldTimeMs: 1_000,
1098
+ maxOutputTokens: 2_000,
1099
+ }),
1100
+ );
1101
+ return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
1102
+ } catch (error) {
1103
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
1104
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
1105
+ return { status: "deferred", reason: "provider_binding_missing" };
1106
+ }
1107
+ return {
1108
+ status: "proved",
1109
+ proof: {
1110
+ outcome: "lost",
1111
+ exitCode: null,
1112
+ reason: "provider_instance_not_found",
1113
+ },
1114
+ };
1115
+ }
1116
+ return {
1117
+ status: "deferred",
1118
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
1119
+ };
1120
+ }
1121
+ }
1122
+ return observation;
694
1123
  }
695
1124
 
696
1125
  /** Classify only exact SDK control banners. Arbitrary output, malformed text,
@@ -796,6 +1225,23 @@ async function refreshQueueLeaseAndCreditGauges(
796
1225
  error: error instanceof Error ? error.message : String(error),
797
1226
  });
798
1227
  }
1228
+ try {
1229
+ recordSandboxCheckpointArtifactGauges(
1230
+ observability,
1231
+ await countSandboxCheckpointArtifactsByState(db),
1232
+ );
1233
+ } catch (error) {
1234
+ observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
1235
+ error: error instanceof Error ? error.message : String(error),
1236
+ });
1237
+ }
1238
+ try {
1239
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1240
+ } catch (error) {
1241
+ observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
1242
+ error: error instanceof Error ? error.message : String(error),
1243
+ });
1244
+ }
799
1245
  try {
800
1246
  recordRetainedProcessInventoryGauges(
801
1247
  observability,
@@ -841,10 +1287,6 @@ async function forceDrainOverLimitWorkspaces(
841
1287
  const enforceBalance =
842
1288
  settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
843
1289
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
844
- // Nothing to enforce → skip the whole pass (no lock churn).
845
- if (!enforceBalance && cap <= 0) {
846
- return 0;
847
- }
848
1290
  let forceDrained = 0;
849
1291
  for (const workspaceId of workspaceIds) {
850
1292
  try {
@@ -947,6 +1389,77 @@ async function sweepModalOrphansForConfiguredBackend(
947
1389
  return result.terminated.length;
948
1390
  }
949
1391
 
1392
+ /**
1393
+ * A capture claim whose owning worker disappeared must not fence a workspace
1394
+ * forever. The drain reaper reaches this seam only after every holder is gone.
1395
+ * It resumes the exact attributed provider without replacement and proves that
1396
+ * a no-op command can execute. A paused snapshot cannot pass that proof; a
1397
+ * missing provider is returned as typed loss; every ambiguous error preserves
1398
+ * the old claim for the next sweep.
1399
+ */
1400
+ async function probeDrainableProviderReadiness(
1401
+ settings: ActivityServices["settings"],
1402
+ lease: LeaseSnapshot,
1403
+ ): Promise<"ready" | "missing"> {
1404
+ if (!lease.instanceId || !lease.resumeState) {
1405
+ throw new Error("Expired workspace capture has no resumable provider identity");
1406
+ }
1407
+ const backend = (lease.resumeBackendId ?? lease.backend) as string;
1408
+ let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;
1409
+ try {
1410
+ established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
1411
+ sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,
1412
+ recovery: "resume-only",
1413
+ backendOverride: backend as never,
1414
+ });
1415
+ } catch (error) {
1416
+ if (isProviderSandboxNotFoundError(backend, error)) return "missing";
1417
+ throw error;
1418
+ }
1419
+ if (established.instanceId !== lease.instanceId) {
1420
+ throw new Error("Expired workspace capture resumed a different provider instance");
1421
+ }
1422
+ try {
1423
+ if (established.backendId === "modal") {
1424
+ await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);
1425
+ return "ready";
1426
+ }
1427
+ const session = established.session as {
1428
+ exec?: (args: {
1429
+ cmd: string;
1430
+ yieldTimeMs?: number;
1431
+ maxOutputTokens?: number;
1432
+ }) => Promise<unknown>;
1433
+ execCommand?: (args: {
1434
+ cmd: string;
1435
+ yieldTimeMs?: number;
1436
+ maxOutputTokens?: number;
1437
+ }) => Promise<unknown>;
1438
+ };
1439
+ const run = session.exec ?? session.execCommand;
1440
+ if (!run) throw new Error("Expired workspace capture provider has no readiness command");
1441
+ let timer: ReturnType<typeof setTimeout> | undefined;
1442
+ const result = await Promise.race([
1443
+ run.call(session, { cmd: "true", yieldTimeMs: 1_000, maxOutputTokens: 1_000 }),
1444
+ new Promise<never>((_, reject) => {
1445
+ timer = setTimeout(
1446
+ () => reject(new Error("Expired workspace capture readiness probe timed out")),
1447
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1448
+ );
1449
+ }),
1450
+ ]).finally(() => {
1451
+ if (timer) clearTimeout(timer);
1452
+ });
1453
+ if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {
1454
+ throw new Error("Expired workspace capture provider is not command-ready");
1455
+ }
1456
+ return "ready";
1457
+ } catch (error) {
1458
+ if (isProviderSandboxNotFoundError(backend, error)) return "missing";
1459
+ throw error;
1460
+ }
1461
+ }
1462
+
950
1463
  /**
951
1464
  * Terminate one drainable box by id, then CAS its lease draining->cold under the
952
1465
  * epoch fence. Returns true when the lease went cold (the box is ours to stop
@@ -970,6 +1483,7 @@ async function terminateDrainableBox(
970
1483
  row: ReapDrainable,
971
1484
  observability: ActivityServices["observability"],
972
1485
  terminateBox: TerminateBoxFn,
1486
+ probeDrainableProvider: DrainableProviderProbeFn,
973
1487
  ): Promise<boolean> {
974
1488
  // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
975
1489
  // returns no account_id; the workspace->account map is the bootstrap read).
@@ -992,16 +1506,78 @@ async function terminateDrainableBox(
992
1506
  return false;
993
1507
  }
994
1508
 
995
- const capturePreflight = lease.instanceId
996
- ? await readWorkspaceArchiveCapturePreflight(db, {
1509
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
1510
+ let captureClaim:
1511
+ | NonNullable<
1512
+ Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
1513
+ >["claim"]
1514
+ | null = null;
1515
+ let providerMissingBeforeCapture = false;
1516
+ if (lease.instanceId) {
1517
+ if (lease.archiveCapture) {
1518
+ if (lease.archiveCapture.deadlineAt.getTime() > Date.now()) {
1519
+ // A live owner still holds the provider pause gate. It will release the
1520
+ // exact claim on settlement; this sweep must not compete.
1521
+ return false;
1522
+ }
1523
+ // A deadline is never evidence that a non-cancellable provider capture
1524
+ // stopped. Without an exact resume envelope there is no safe way to prove
1525
+ // the attributed provider is command-ready, so preserve the gate.
1526
+ if (!lease.resumeState) return false;
1527
+ const providerState = await probeDrainableProvider(settings, lease);
1528
+ if (providerState === "missing") {
1529
+ providerMissingBeforeCapture = true;
1530
+ captureClaim = {
1531
+ ...lease.archiveCapture,
1532
+ leaseId: lease.id,
1533
+ leaseEpoch: lease.leaseEpoch,
1534
+ instanceId: lease.instanceId,
1535
+ archiveGeneration: lease.archiveGeneration,
1536
+ archiveComplete: lease.archiveComplete,
1537
+ };
1538
+ } else {
1539
+ const replacement = await replaceExpiredWorkspaceArchiveCapture(db, {
1540
+ accountId,
1541
+ workspaceId: row.workspaceId,
1542
+ sandboxGroupId: row.sandboxGroupId,
1543
+ priorCaptureId: lease.archiveCapture.id,
1544
+ captureId: randomUUID(),
1545
+ expectedEpoch: row.leaseEpoch,
1546
+ expectedInstanceId: lease.instanceId,
1547
+ captureTimeoutMs,
1548
+ });
1549
+ if (!replacement) return false;
1550
+ captureClaim = replacement;
1551
+ }
1552
+ } else {
1553
+ const claimed = await claimWorkspaceArchiveCapture(db, {
997
1554
  accountId,
998
1555
  workspaceId: row.workspaceId,
999
1556
  sandboxGroupId: row.sandboxGroupId,
1557
+ captureId: randomUUID(),
1000
1558
  expectedEpoch: row.leaseEpoch,
1001
1559
  expectedInstanceId: lease.instanceId,
1002
1560
  liveness: "draining",
1003
- })
1004
- : null;
1561
+ captureTimeoutMs,
1562
+ minIntervalMs: 0,
1563
+ });
1564
+ if (claimed.status === "claimed") {
1565
+ captureClaim = claimed.claim;
1566
+ } else if (claimed.status === "mutation_in_progress") {
1567
+ // An admission is normally authoritative proof that a provider
1568
+ // operation may still be running. It cannot, however, pin a draining
1569
+ // lease forever after the exact provider has disappeared. Probe the
1570
+ // attributed instance without replacement. Only typed NotFound permits
1571
+ // the cold commit to reject the exact stale admissions; a live or
1572
+ // ambiguous provider remains fenced for a later sweep.
1573
+ const providerState = await probeDrainableProvider(settings, lease);
1574
+ if (providerState !== "missing") return false;
1575
+ providerMissingBeforeCapture = true;
1576
+ } else {
1577
+ return false;
1578
+ }
1579
+ }
1580
+ }
1005
1581
 
1006
1582
  // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
1007
1583
  // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
@@ -1016,23 +1592,73 @@ async function terminateDrainableBox(
1016
1592
  const persistArchive: PersistArchiveFn = async (
1017
1593
  archiveBase64: string | null,
1018
1594
  archiveMetadata?: WorkspaceArchiveDescriptor,
1595
+ _providerSession?: unknown,
1596
+ providerBinding?: Awaited<
1597
+ ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
1598
+ > | null,
1019
1599
  ) => {
1020
- // A live persistable provider must have passed an exact instance/generation
1021
- // preflight before capture. Without it we cannot prove that these bytes came
1022
- // from the lease row we are about to fold, so fail closed and leave it live.
1023
- if (!lease.instanceId || !capturePreflight) {
1024
- return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
1600
+ // A live persistable provider must own the exact durable capture claim.
1601
+ // Without it we cannot prove that these bytes came from a provider that was
1602
+ // admission-fenced for the full pause/capture interval.
1603
+ if (!lease.instanceId || !captureClaim) {
1604
+ return { wrote: false, archiveRevision: null };
1025
1605
  }
1026
- const result = await persistDrainSnapshot(db, {
1606
+ let checkpointArtifactId: string | null = null;
1607
+ if (
1608
+ archiveBase64 &&
1609
+ archiveMetadata?.version === 2 &&
1610
+ (archiveMetadata.provider === "modal_snapshot_filesystem" ||
1611
+ archiveMetadata.provider === "modal_snapshot_directory")
1612
+ ) {
1613
+ if (!providerBinding) {
1614
+ throw new Error("Modal native snapshot has no exact session provider identity");
1615
+ }
1616
+ const candidate = await registerSandboxCheckpointArtifact(db, {
1617
+ accountId,
1618
+ workspaceId: row.workspaceId,
1619
+ sandboxGroupId: row.sandboxGroupId,
1620
+ sourceLeaseId: lease.id,
1621
+ sourceLeaseEpoch: row.leaseEpoch,
1622
+ sourceInstanceId: lease.instanceId,
1623
+ sourceWorkspaceGeneration: captureClaim.workspaceGeneration,
1624
+ providerBindingKey: providerBinding.key,
1625
+ providerBinding: providerBinding.binding,
1626
+ workspaceArchive: archiveBase64,
1627
+ workspaceArchiveMeta: archiveMetadata,
1628
+ });
1629
+ checkpointArtifactId = candidate.id;
1630
+ }
1631
+ const baseInput = {
1027
1632
  accountId,
1028
1633
  workspaceId: row.workspaceId,
1029
1634
  sandboxGroupId: row.sandboxGroupId,
1030
1635
  expectedEpoch: row.leaseEpoch,
1031
1636
  expectedInstanceId: lease.instanceId,
1032
- expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
1033
- workspaceArchive: archiveBase64,
1034
- ...(archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}),
1035
- });
1637
+ expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
1638
+ captureId: captureClaim.id,
1639
+ };
1640
+ let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;
1641
+ if (archiveBase64 === null) {
1642
+ result = await persistDrainSnapshot(db, { ...baseInput, workspaceArchive: null });
1643
+ } else {
1644
+ if (!archiveMetadata) {
1645
+ throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
1646
+ }
1647
+ result = await persistDrainSnapshot(db, {
1648
+ ...baseInput,
1649
+ workspaceArchive: archiveBase64,
1650
+ workspaceArchiveMeta: archiveMetadata,
1651
+ ...(checkpointArtifactId ? { checkpointArtifactId } : {}),
1652
+ });
1653
+ }
1654
+ if (!result.wrote && checkpointArtifactId) {
1655
+ await markSandboxCheckpointArtifactDeletePending(db, {
1656
+ accountId,
1657
+ workspaceId: row.workspaceId,
1658
+ artifactId: checkpointArtifactId,
1659
+ reason: "drain_snapshot_publication_fenced",
1660
+ }).catch(() => undefined);
1661
+ }
1036
1662
  if (result.wrote && archiveBase64 !== null) {
1037
1663
  persisted = true;
1038
1664
  archiveRevision = result.archiveRevision;
@@ -1040,78 +1666,103 @@ async function terminateDrainableBox(
1040
1666
  return result;
1041
1667
  };
1042
1668
 
1043
- // Resume-by-id -> verified capture -> persist-onto-lease -> GC prior snapshot
1044
- // -> provider terminate. Definitive NotFound before capture is a typed loss,
1045
- // not ordinary teardown success: confirmDrainCold preserves any prior archive
1046
- // or marks recovery unrecoverable when none exists. A genuine capture/resume
1047
- // failure leaves the lease draining for a later sweep (NEVER terminate a box
1048
- // whose files we could not capture). A persist CAS miss means the box was
1049
- // re-armed and left running, so the cold commit is skipped.
1050
- const termination = await terminateBox(settings, lease, observability, persistArchive);
1051
- const terminated = typeof termination === "boolean" ? termination : termination.terminated;
1052
- if (!terminated) {
1053
- return false;
1054
- }
1669
+ // Resume-by-id -> verified capture -> persist-onto-lease -> provider
1670
+ // terminate. Persisting atomically hands any displaced provider snapshot to
1671
+ // the provider-bound artifact ledger; its global GC worker owns deletion.
1672
+ // Definitive NotFound before capture is a typed loss, not ordinary teardown
1673
+ // success: confirmDrainCold preserves any prior archive or marks recovery
1674
+ // unrecoverable when none exists. A genuine capture/resume failure leaves the
1675
+ // lease draining for a later sweep (NEVER terminate a box whose files we
1676
+ // could not capture). A persist CAS miss means the box was re-armed and left
1677
+ // running, so the cold commit is skipped.
1678
+ try {
1679
+ const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
1680
+ ? { terminated: true, providerMissingBeforeCapture: true }
1681
+ : await terminateBox(settings, lease, observability, persistArchive);
1682
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
1683
+ if (!terminated) {
1684
+ return false;
1685
+ }
1055
1686
 
1056
- // The authoritative commit: CAS draining->cold under the epoch fence. If a
1057
- // late re-arm or newer epoch raced in after our re-read, wentCold:false and we
1058
- // report a skip (the box is down; a fresh acquire cold-restores it).
1059
- const { wentCold } = await confirmDrainCold(db, {
1060
- accountId,
1061
- workspaceId: row.workspaceId,
1062
- sandboxGroupId: row.sandboxGroupId,
1063
- expectedEpoch: row.leaseEpoch,
1064
- providerMissingBeforeCapture:
1065
- typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
1066
- });
1067
- if (wentCold) {
1068
- // Durable termination record (sandbox-file-persistence observability): who
1069
- // ended this box and whether its /workspace was captured first, appended to
1070
- // every session sharing the group's box. Best-effort: attribution must
1071
- // never affect the drain outcome.
1072
- try {
1073
- await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
1074
- type: "sandbox.box.terminated",
1075
- payload: {
1076
- actor: "reaper",
1077
- persisted,
1078
- archiveRevision,
1079
- instanceId: lease.instanceId,
1080
- providerMissingBeforeCapture:
1081
- typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
1082
- },
1083
- });
1084
- } catch (eventError) {
1085
- observability.warn(
1086
- "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
1087
- {
1687
+ const providerMissing =
1688
+ providerMissingBeforeCapture ||
1689
+ (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
1690
+ // The authoritative commit: CAS draining->cold under the epoch fence. If a
1691
+ // late re-arm or newer epoch raced in after our re-read, wentCold:false and we
1692
+ // report a skip (the box is down; a fresh acquire cold-restores it).
1693
+ const { wentCold } = await confirmDrainCold(db, {
1694
+ accountId,
1695
+ workspaceId: row.workspaceId,
1696
+ sandboxGroupId: row.sandboxGroupId,
1697
+ expectedEpoch: row.leaseEpoch,
1698
+ providerMissingBeforeCapture: providerMissing,
1699
+ });
1700
+ if (wentCold) {
1701
+ // Durable termination record (sandbox-file-persistence observability): who
1702
+ // ended this box and whether its /workspace was captured first, appended to
1703
+ // every session sharing the group's box. Best-effort: attribution must
1704
+ // never affect the drain outcome.
1705
+ try {
1706
+ await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
1707
+ type: "sandbox.box.terminated",
1708
+ payload: {
1709
+ actor: "reaper",
1710
+ persisted,
1711
+ archiveRevision,
1712
+ instanceId: lease.instanceId,
1713
+ providerMissingBeforeCapture: providerMissing,
1714
+ },
1715
+ });
1716
+ } catch (eventError) {
1717
+ observability.warn(
1718
+ "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
1719
+ {
1720
+ sandboxGroupId: row.sandboxGroupId,
1721
+ error: eventError instanceof Error ? eventError.message : String(eventError),
1722
+ },
1723
+ );
1724
+ }
1725
+ }
1726
+ return wentCold;
1727
+ } finally {
1728
+ if (captureClaim && lease.instanceId) {
1729
+ await releaseWorkspaceArchiveCapture(db, {
1730
+ accountId,
1731
+ workspaceId: row.workspaceId,
1732
+ sandboxGroupId: row.sandboxGroupId,
1733
+ captureId: captureClaim.id,
1734
+ expectedEpoch: row.leaseEpoch,
1735
+ expectedInstanceId: lease.instanceId,
1736
+ }).catch((error) => {
1737
+ observability.warn("sandbox reaper: archive capture gate release failed", {
1088
1738
  sandboxGroupId: row.sandboxGroupId,
1089
- error: eventError instanceof Error ? eventError.message : String(eventError),
1090
- },
1091
- );
1739
+ leaseEpoch: row.leaseEpoch,
1740
+ error: error instanceof Error ? error.message : String(error),
1741
+ });
1742
+ });
1092
1743
  }
1093
1744
  }
1094
- return wentCold;
1095
1745
  }
1096
1746
 
1097
1747
  // The persist-capable slice of a live provider session. persistWorkspace()
1098
1748
  // snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
1099
- // archive) and returns the archive bytes. modal carries the Modal SDK client used
1100
- // for snapshot-image GC (deletePriorPersistedSnapshot reads session.modal).
1749
+ // archive) and returns the archive bytes. The session also carries the Modal SDK
1750
+ // client used to bind the snapshot receipt to the exact provider namespace.
1101
1751
  type PersistableSession = TerminableSession & {
1102
1752
  persistWorkspace?: () => Promise<Uint8Array | undefined>;
1103
1753
  };
1104
1754
 
1105
1755
  /**
1106
1756
  * Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
1107
- * lease under the epoch fence, GC the superseded snapshot, THEN terminate. Resumes
1108
- * DIRECTLY (no cold-restore fallback) so a gone box is a clean no-op, never a
1109
- * wasteful create-then-kill:
1757
+ * lease under the epoch fence, hand the superseded snapshot to durable GC, THEN
1758
+ * terminate. Resumes DIRECTLY (no cold-restore fallback) so a gone box is a clean
1759
+ * no-op, never a wasteful create-then-kill:
1110
1760
  * - build the client for the lease's backend;
1111
1761
  * - deserialize + resume the envelope (warm reattach by id, R4-safe);
1112
1762
  * - session.persistWorkspace() -> capture the /workspace archive;
1113
- * - persistArchive(base64) -> CAS-fold it onto the lease (epoch-fenced);
1114
- * - GC the PRIOR snapshot (keep-latest-per-lease);
1763
+ * - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark
1764
+ * the displaced artifact delete-pending;
1765
+ * - the provider-bound global GC deletes that object asynchronously;
1115
1766
  * - terminate the live handle (client.delete / session kill|terminate|close).
1116
1767
  *
1117
1768
  * Returns true when the box was terminated (or was already gone), false when the
@@ -1264,6 +1915,17 @@ export async function terminateProviderBox(
1264
1915
  // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
1265
1916
  // box raced gone between resume and persist) is success: nothing to persist.
1266
1917
  let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
1918
+ const sessionWorkspacePersistence = (
1919
+ session as { state?: { workspacePersistence?: unknown } } | undefined
1920
+ )?.state?.workspacePersistence;
1921
+ const checkpointBinding =
1922
+ backend === "modal" &&
1923
+ (sessionWorkspacePersistence === "snapshot_filesystem" ||
1924
+ sessionWorkspacePersistence === "snapshot_directory")
1925
+ ? await withRetainedProcessProbeTimeout(
1926
+ resolveModalCheckpointProviderBindingForSession(settings, session),
1927
+ )
1928
+ : null;
1267
1929
  try {
1268
1930
  verifiedArchive = session?.persistWorkspace
1269
1931
  ? await captureVerifiedWorkspaceArchive(session)
@@ -1301,9 +1963,11 @@ export async function terminateProviderBox(
1301
1963
  // null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
1302
1964
  // refcount/epoch guard, no write. wrote:false → abort the terminate.
1303
1965
  if (verifiedArchive) {
1304
- const { wrote, priorArchiveForGc } = await persistArchive(
1966
+ const { wrote } = await persistArchive(
1305
1967
  verifiedArchive.base64,
1306
1968
  verifiedArchive.descriptor,
1969
+ session,
1970
+ checkpointBinding,
1307
1971
  );
1308
1972
  if (!wrote) {
1309
1973
  observability.info(
@@ -1315,16 +1979,6 @@ export async function terminateProviderBox(
1315
1979
  );
1316
1980
  return { terminated: false, providerMissingBeforeCapture: false };
1317
1981
  }
1318
- // Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
1319
- // while the session (and its Modal client) is still live. Never throws.
1320
- const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
1321
- if (deleted) {
1322
- observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
1323
- sandboxGroupId: lease.sandboxGroupId,
1324
- backend,
1325
- snapshotId: deleted,
1326
- });
1327
- }
1328
1982
  } else {
1329
1983
  // A resumable cloud box that produces no verified archive cannot be safely
1330
1984
  // deleted: it may contain work newer than the lease's last durable revision.