@opengeni/worker-bundle 0.12.7 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2730 -1211
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +874 -134
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
|
@@ -30,12 +30,18 @@
|
|
|
30
30
|
// so a box that was re-armed mid-sweep is never terminated out from under a live
|
|
31
31
|
// holder.
|
|
32
32
|
|
|
33
|
+
import { randomUUID } from "node:crypto";
|
|
33
34
|
import {
|
|
34
35
|
accrueWarmSeconds,
|
|
36
|
+
adoptLegacyModalCheckpointArtifact,
|
|
35
37
|
confirmDrainCold,
|
|
36
38
|
appendSessionEventToSandboxGroup,
|
|
39
|
+
bindRetainedProcessProviderIdentity,
|
|
40
|
+
claimWorkspaceArchiveCapture,
|
|
41
|
+
claimSandboxCheckpointArtifactsForGc,
|
|
37
42
|
claimTerminalRetainedProcesses,
|
|
38
43
|
countActiveRetainedProcessesByOwnerState,
|
|
44
|
+
countSandboxCheckpointArtifactsByState,
|
|
39
45
|
countExpiredDrainingSandboxLeases,
|
|
40
46
|
countQueuedTurns,
|
|
41
47
|
countSandboxLeasesByLiveness,
|
|
@@ -43,16 +49,25 @@ import {
|
|
|
43
49
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
44
50
|
getBillingBalance,
|
|
45
51
|
listCreditBalancesByAccount,
|
|
52
|
+
listLegacyModalCheckpointSlots,
|
|
46
53
|
listLiveModalSandboxLeaseAttributions,
|
|
47
54
|
listMeterableWarmLeases,
|
|
55
|
+
listSandboxViewerForceDrainWorkspaceIds,
|
|
56
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
48
57
|
persistDrainSnapshot,
|
|
58
|
+
pruneDeletedSandboxCheckpointArtifacts,
|
|
59
|
+
registerSandboxCheckpointArtifact,
|
|
49
60
|
recordRetainedProcessReconciliationProof,
|
|
50
|
-
|
|
61
|
+
releaseWorkspaceArchiveCapture,
|
|
62
|
+
replaceExpiredWorkspaceArchiveCapture,
|
|
51
63
|
readLease,
|
|
52
64
|
reapExpiredSessionListSnapshots,
|
|
53
65
|
reapStaleLeaseHoldersGlobal,
|
|
66
|
+
requestDueSandboxRotationsGlobal,
|
|
67
|
+
readSandboxRotationBacklog,
|
|
54
68
|
retainedProcessReconciliationProof,
|
|
55
69
|
retainedProcessSettlementIdentity,
|
|
70
|
+
settleSandboxCheckpointArtifactGc,
|
|
56
71
|
rlsContextForWorkspace,
|
|
57
72
|
settleRetainedProcess,
|
|
58
73
|
type MeterableWarmLease,
|
|
@@ -61,24 +76,32 @@ import {
|
|
|
61
76
|
type SandboxRetainedProcess,
|
|
62
77
|
type LeaseSnapshot,
|
|
63
78
|
} from "@opengeni/db";
|
|
64
|
-
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
79
|
+
import { sandboxArchiveCaptureTimeoutMs, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
65
80
|
import {
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
//
|
|
71
|
-
// (NotFound on resume) is already down, a clean no-op.
|
|
81
|
+
// Normal drain teardown builds the client and resumes the envelope directly:
|
|
82
|
+
// a live box gets its /workspace persisted before termination, while a gone
|
|
83
|
+
// box is typed as provider loss. Stale capture reconciliation separately uses
|
|
84
|
+
// establishSandboxSessionFromEnvelope in strict resume-only mode so it can
|
|
85
|
+
// reuse the runtime's provider-ready proof without ever cold-restoring.
|
|
72
86
|
captureVerifiedWorkspaceArchive,
|
|
73
87
|
createSandboxClientForBackend,
|
|
74
|
-
|
|
88
|
+
deleteModalCheckpointSnapshot,
|
|
75
89
|
deserializeSandboxSessionStateEnvelope,
|
|
90
|
+
establishSandboxSessionFromEnvelope,
|
|
91
|
+
inspectModalSandboxLifecycle,
|
|
76
92
|
isExecSessionLostBanner,
|
|
77
93
|
isProviderSandboxNotFoundError,
|
|
78
94
|
parseExecBannerExitCode,
|
|
95
|
+
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
96
|
+
resolveModalCheckpointProviderBinding,
|
|
97
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
98
|
+
sandboxCommandExitCode,
|
|
99
|
+
sandboxCommandStillRunning,
|
|
79
100
|
sweepModalOrphanSandboxes,
|
|
80
101
|
terminateModalSandboxById,
|
|
102
|
+
verifySandboxExecReadiness,
|
|
81
103
|
type ModalOrphanSweepTermination,
|
|
104
|
+
type ModalCheckpointProviderBinding,
|
|
82
105
|
type WorkspaceArchiveDescriptor,
|
|
83
106
|
} from "@opengeni/runtime";
|
|
84
107
|
import type { ActivityServices } from "./types";
|
|
@@ -89,9 +112,13 @@ import {
|
|
|
89
112
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
90
113
|
recordRetainedProcessInventoryGauges,
|
|
91
114
|
recordRetainedProcessReconciliation,
|
|
115
|
+
recordSandboxCheckpointArtifactGauges,
|
|
116
|
+
recordSandboxCheckpointArtifactOutcome,
|
|
117
|
+
recordSandboxDeadlineRotationsRequested,
|
|
92
118
|
type RetainedProcessReconciliationOutcome,
|
|
93
119
|
recordSandboxLeaseGauges,
|
|
94
120
|
recordSandboxOrphansTerminated,
|
|
121
|
+
recordSandboxRotationBacklogGauges,
|
|
95
122
|
recordTurnsQueuedGauge,
|
|
96
123
|
} from "../observability-metrics";
|
|
97
124
|
import { providerIdentityFromResumeState } from "../sandbox-routing";
|
|
@@ -143,21 +170,23 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
|
143
170
|
* archive onto the lease under the epoch fence, BEFORE the box is terminated
|
|
144
171
|
* (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over
|
|
145
172
|
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
173
|
+
* `wrote:false` means the CAS missed (re-armed / newer epoch / vanished). The
|
|
174
|
+
* box is wanted again, so the seam MUST NOT terminate it. Provider-native
|
|
175
|
+
* deletion is owned independently by artifact-ledger GC.
|
|
149
176
|
*
|
|
150
177
|
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
151
178
|
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
152
|
-
* aborts the terminate before client.delete()).
|
|
153
|
-
* null in this case.
|
|
179
|
+
* aborts the terminate before client.delete()).
|
|
154
180
|
*/
|
|
155
181
|
export type PersistArchiveFn = (
|
|
156
182
|
archiveBase64: string | null,
|
|
157
183
|
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
184
|
+
providerSession?: unknown,
|
|
185
|
+
providerBinding?: Awaited<
|
|
186
|
+
ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
|
|
187
|
+
> | null,
|
|
158
188
|
) => Promise<{
|
|
159
189
|
wrote: boolean;
|
|
160
|
-
priorArchiveForGc: string | null;
|
|
161
190
|
archiveRevision?: string | null;
|
|
162
191
|
}>;
|
|
163
192
|
|
|
@@ -200,10 +229,21 @@ export type SandboxLeaseActivityOptions = {
|
|
|
200
229
|
/** Override only the read-only provider process probe. The canonical DB
|
|
201
230
|
* settlement remains real in tests and production. */
|
|
202
231
|
probeRetainedProcess?: RetainedProcessProbeFn;
|
|
232
|
+
/** Observe a historical Modal box after its lease row has advanced to a
|
|
233
|
+
* successor identity. Lifecycle-only; a live box remains ambiguous. */
|
|
234
|
+
inspectHistoricalModalSandbox?: HistoricalModalSandboxLifecycleProbeFn;
|
|
235
|
+
/** Override the read-only exact-instance readiness probe used before the
|
|
236
|
+
* reaper settles blockers for a provider that has definitively vanished. */
|
|
237
|
+
probeDrainableProvider?: DrainableProviderProbeFn;
|
|
203
238
|
};
|
|
204
239
|
|
|
205
240
|
export type RetainedProcessProbeResult =
|
|
206
241
|
| { status: "proved"; proof: RetainedProcessProviderProof }
|
|
242
|
+
| {
|
|
243
|
+
status: "binding";
|
|
244
|
+
providerBindingKey: string;
|
|
245
|
+
providerBinding: ModalCheckpointProviderBinding;
|
|
246
|
+
}
|
|
207
247
|
| {
|
|
208
248
|
status: "deferred";
|
|
209
249
|
reason:
|
|
@@ -213,7 +253,9 @@ export type RetainedProcessProbeResult =
|
|
|
213
253
|
| "provider_running"
|
|
214
254
|
| "provider_unknown"
|
|
215
255
|
| "provider_timeout"
|
|
216
|
-
| "provider_error"
|
|
256
|
+
| "provider_error"
|
|
257
|
+
| "provider_binding_missing"
|
|
258
|
+
| "provider_binding_mismatch";
|
|
217
259
|
};
|
|
218
260
|
|
|
219
261
|
export type RetainedProcessProbeFn = (
|
|
@@ -222,6 +264,13 @@ export type RetainedProcessProbeFn = (
|
|
|
222
264
|
process: SandboxRetainedProcess,
|
|
223
265
|
) => Promise<RetainedProcessProbeResult>;
|
|
224
266
|
|
|
267
|
+
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
268
|
+
|
|
269
|
+
export type DrainableProviderProbeFn = (
|
|
270
|
+
settings: ActivityServices["settings"],
|
|
271
|
+
lease: LeaseSnapshot,
|
|
272
|
+
) => Promise<"ready" | "missing">;
|
|
273
|
+
|
|
225
274
|
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
226
275
|
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
227
276
|
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
@@ -234,6 +283,7 @@ export function createSandboxLeaseActivities(
|
|
|
234
283
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
235
284
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
236
285
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
286
|
+
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
237
287
|
/**
|
|
238
288
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
239
289
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -277,6 +327,18 @@ export function createSandboxLeaseActivities(
|
|
|
277
327
|
}
|
|
278
328
|
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
279
329
|
|
|
330
|
+
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
331
|
+
db,
|
|
332
|
+
settings.sandboxRotationLeadMs,
|
|
333
|
+
settings.sandboxRotationBatchSize,
|
|
334
|
+
).catch((error) => {
|
|
335
|
+
observability.warn("sandbox reaper: provider-deadline rotation request failed", {
|
|
336
|
+
error: error instanceof Error ? error.message : String(error),
|
|
337
|
+
});
|
|
338
|
+
return 0;
|
|
339
|
+
});
|
|
340
|
+
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
341
|
+
|
|
280
342
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
281
343
|
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
282
344
|
// them). GROUP+epoch+tick idempotent → a shared box is one stream; an
|
|
@@ -287,10 +349,29 @@ export function createSandboxLeaseActivities(
|
|
|
287
349
|
// workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
|
|
288
350
|
// boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
|
|
289
351
|
// draining rows are caught by the same sweep's terminate below.
|
|
352
|
+
const forceDrainWorkspaceIds = new Set<string>();
|
|
353
|
+
if (
|
|
354
|
+
settings.billingMode === "stripe" ||
|
|
355
|
+
settings.usageLimitsMode === "managed" ||
|
|
356
|
+
settings.sandboxMaxWarmSecondsPerWorkspace > 0
|
|
357
|
+
) {
|
|
358
|
+
for (const workspaceId of metered.workspaceIds) {
|
|
359
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
try {
|
|
363
|
+
for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
|
|
364
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
365
|
+
}
|
|
366
|
+
} catch (error) {
|
|
367
|
+
observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
|
|
368
|
+
error: error instanceof Error ? error.message : String(error),
|
|
369
|
+
});
|
|
370
|
+
}
|
|
290
371
|
const forceDrained = await forceDrainOverLimitWorkspaces(
|
|
291
372
|
db,
|
|
292
373
|
settings,
|
|
293
|
-
|
|
374
|
+
forceDrainWorkspaceIds,
|
|
294
375
|
observability,
|
|
295
376
|
);
|
|
296
377
|
|
|
@@ -298,7 +379,13 @@ export function createSandboxLeaseActivities(
|
|
|
298
379
|
// only a bounded inspection batch; exact provider exit/loss proof is
|
|
299
380
|
// checkpointed before canonical settlement. Ambiguous, running, timed-out,
|
|
300
381
|
// unsupported, and transient provider states preserve every durable row.
|
|
301
|
-
await reconcileTerminalRetainedProcesses(
|
|
382
|
+
await reconcileTerminalRetainedProcesses(
|
|
383
|
+
db,
|
|
384
|
+
settings,
|
|
385
|
+
observability,
|
|
386
|
+
probeRetainedProcess,
|
|
387
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
388
|
+
);
|
|
302
389
|
|
|
303
390
|
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
304
391
|
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -331,6 +418,7 @@ export function createSandboxLeaseActivities(
|
|
|
331
418
|
row,
|
|
332
419
|
observability,
|
|
333
420
|
terminateBox,
|
|
421
|
+
probeDrainableProvider,
|
|
334
422
|
);
|
|
335
423
|
if (drainedCold) {
|
|
336
424
|
terminated += 1;
|
|
@@ -357,13 +445,24 @@ export function createSandboxLeaseActivities(
|
|
|
357
445
|
});
|
|
358
446
|
}
|
|
359
447
|
|
|
448
|
+
const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
|
|
449
|
+
(error) => {
|
|
450
|
+
observability.warn("sandbox reaper: checkpoint artifact GC failed", {
|
|
451
|
+
error: error instanceof Error ? error.message : String(error),
|
|
452
|
+
});
|
|
453
|
+
return { claimed: 0, deleted: 0, failed: 0 };
|
|
454
|
+
},
|
|
455
|
+
);
|
|
456
|
+
|
|
360
457
|
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
361
458
|
|
|
362
459
|
if (
|
|
363
460
|
drainable.length > 0 ||
|
|
364
461
|
metered.accrued > 0 ||
|
|
365
462
|
forceDrained > 0 ||
|
|
366
|
-
modalOrphansTerminated > 0
|
|
463
|
+
modalOrphansTerminated > 0 ||
|
|
464
|
+
rotationsRequested > 0 ||
|
|
465
|
+
checkpointGc.claimed > 0
|
|
367
466
|
) {
|
|
368
467
|
observability.info("sandbox reaper swept", {
|
|
369
468
|
drainable: drainable.length,
|
|
@@ -372,6 +471,10 @@ export function createSandboxLeaseActivities(
|
|
|
372
471
|
metered: metered.accrued,
|
|
373
472
|
forceDrained,
|
|
374
473
|
modalOrphansTerminated,
|
|
474
|
+
rotationsRequested,
|
|
475
|
+
checkpointArtifactsClaimed: checkpointGc.claimed,
|
|
476
|
+
checkpointArtifactsDeleted: checkpointGc.deleted,
|
|
477
|
+
checkpointArtifactsFailed: checkpointGc.failed,
|
|
375
478
|
});
|
|
376
479
|
}
|
|
377
480
|
|
|
@@ -388,11 +491,143 @@ export function createSandboxLeaseActivities(
|
|
|
388
491
|
return { reapSandboxLeases };
|
|
389
492
|
}
|
|
390
493
|
|
|
494
|
+
const CHECKPOINT_GC_LIMIT = 50;
|
|
495
|
+
const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
|
|
496
|
+
const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
497
|
+
const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
498
|
+
|
|
499
|
+
async function gcSandboxCheckpointArtifacts(
|
|
500
|
+
db: ActivityServices["db"],
|
|
501
|
+
settings: ActivityServices["settings"],
|
|
502
|
+
observability: ActivityServices["observability"],
|
|
503
|
+
): Promise<{ claimed: number; deleted: number; failed: number }> {
|
|
504
|
+
// Explicit OPENGENI_MODAL_TOKEN_* settings are optional: the SDK also supports
|
|
505
|
+
// an operator-provisioned Modal profile. Durable artifacts must not become
|
|
506
|
+
// immortal merely because credentials arrive through that supported path.
|
|
507
|
+
const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(
|
|
508
|
+
(error) => {
|
|
509
|
+
observability.warn("sandbox reaper: legacy Modal checkpoint discovery failed", {
|
|
510
|
+
error: error instanceof Error ? error.message : String(error),
|
|
511
|
+
});
|
|
512
|
+
return 0;
|
|
513
|
+
},
|
|
514
|
+
);
|
|
515
|
+
if (adopted > 0) {
|
|
516
|
+
recordSandboxCheckpointArtifactOutcome(observability, "legacy_adopted", adopted);
|
|
517
|
+
observability.info("sandbox reaper: adopted legacy Modal checkpoint receipts", { adopted });
|
|
518
|
+
}
|
|
519
|
+
const claimId = crypto.randomUUID();
|
|
520
|
+
const claims = await claimSandboxCheckpointArtifactsForGc(db, {
|
|
521
|
+
claimId,
|
|
522
|
+
limit: CHECKPOINT_GC_LIMIT,
|
|
523
|
+
claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS,
|
|
524
|
+
});
|
|
525
|
+
recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
|
|
526
|
+
let deleted = 0;
|
|
527
|
+
let failed = 0;
|
|
528
|
+
for (const claim of claims) {
|
|
529
|
+
try {
|
|
530
|
+
if (claim.providerBackend !== "modal") {
|
|
531
|
+
throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
|
|
532
|
+
}
|
|
533
|
+
await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);
|
|
534
|
+
const settled = await settleSandboxCheckpointArtifactGc(db, {
|
|
535
|
+
artifactId: claim.id,
|
|
536
|
+
claimId,
|
|
537
|
+
deleted: true,
|
|
538
|
+
retryAfterMs: 1_000,
|
|
539
|
+
});
|
|
540
|
+
if (settled) {
|
|
541
|
+
deleted += 1;
|
|
542
|
+
recordSandboxCheckpointArtifactOutcome(observability, "deleted");
|
|
543
|
+
}
|
|
544
|
+
} catch (error) {
|
|
545
|
+
failed += 1;
|
|
546
|
+
recordSandboxCheckpointArtifactOutcome(observability, "delete_failed");
|
|
547
|
+
const retryAfterMs = Math.min(
|
|
548
|
+
6 * 60 * 60_000,
|
|
549
|
+
5_000 * 2 ** Math.min(claim.deleteAttempts, 17),
|
|
550
|
+
);
|
|
551
|
+
await settleSandboxCheckpointArtifactGc(db, {
|
|
552
|
+
artifactId: claim.id,
|
|
553
|
+
claimId,
|
|
554
|
+
deleted: false,
|
|
555
|
+
error: error instanceof Error ? error.message : String(error),
|
|
556
|
+
retryAfterMs,
|
|
557
|
+
}).catch(() => false);
|
|
558
|
+
observability.warn("sandbox reaper: checkpoint artifact delete failed", {
|
|
559
|
+
artifactId: claim.id,
|
|
560
|
+
providerBackend: claim.providerBackend,
|
|
561
|
+
objectKind: claim.objectKind,
|
|
562
|
+
attempt: claim.deleteAttempts,
|
|
563
|
+
error: error instanceof Error ? error.message : String(error),
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
const pruned = await pruneDeletedSandboxCheckpointArtifacts(
|
|
568
|
+
db,
|
|
569
|
+
CHECKPOINT_TOMBSTONE_RETENTION_MS,
|
|
570
|
+
CHECKPOINT_TOMBSTONE_PRUNE_LIMIT,
|
|
571
|
+
);
|
|
572
|
+
recordSandboxCheckpointArtifactOutcome(observability, "tombstone_pruned", pruned);
|
|
573
|
+
return { claimed: claims.length, deleted, failed };
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
async function adoptLegacyModalCheckpointReceipts(
|
|
577
|
+
db: ActivityServices["db"],
|
|
578
|
+
settings: ActivityServices["settings"],
|
|
579
|
+
observability: ActivityServices["observability"],
|
|
580
|
+
): Promise<number> {
|
|
581
|
+
const slots = await listLegacyModalCheckpointSlots(db, 25);
|
|
582
|
+
if (slots.length === 0) return 0;
|
|
583
|
+
const bindings = new Map<
|
|
584
|
+
string,
|
|
585
|
+
Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>
|
|
586
|
+
>();
|
|
587
|
+
let adopted = 0;
|
|
588
|
+
for (const slot of slots) {
|
|
589
|
+
try {
|
|
590
|
+
let binding = bindings.get(slot.instanceId);
|
|
591
|
+
if (!binding) {
|
|
592
|
+
binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);
|
|
593
|
+
bindings.set(slot.instanceId, binding);
|
|
594
|
+
}
|
|
595
|
+
const resolvedBinding = await binding;
|
|
596
|
+
if (
|
|
597
|
+
await adoptLegacyModalCheckpointArtifact(db, {
|
|
598
|
+
accountId: slot.accountId,
|
|
599
|
+
workspaceId: slot.workspaceId,
|
|
600
|
+
sandboxGroupId: slot.sandboxGroupId,
|
|
601
|
+
leaseId: slot.leaseId,
|
|
602
|
+
leaseEpoch: slot.leaseEpoch,
|
|
603
|
+
workspaceGeneration: slot.workspaceGeneration,
|
|
604
|
+
slot: slot.slot,
|
|
605
|
+
archiveBase64: slot.archiveBase64,
|
|
606
|
+
descriptor: slot.descriptor,
|
|
607
|
+
providerBindingKey: resolvedBinding.key,
|
|
608
|
+
providerBinding: resolvedBinding.binding,
|
|
609
|
+
})
|
|
610
|
+
) {
|
|
611
|
+
adopted += 1;
|
|
612
|
+
}
|
|
613
|
+
} catch (error) {
|
|
614
|
+
observability.warn("sandbox reaper: legacy Modal checkpoint adoption failed", {
|
|
615
|
+
workspaceId: slot.workspaceId,
|
|
616
|
+
sandboxGroupId: slot.sandboxGroupId,
|
|
617
|
+
slot: slot.slot,
|
|
618
|
+
error: error instanceof Error ? error.message : String(error),
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return adopted;
|
|
623
|
+
}
|
|
624
|
+
|
|
391
625
|
async function reconcileTerminalRetainedProcesses(
|
|
392
626
|
db: ActivityServices["db"],
|
|
393
627
|
settings: ActivityServices["settings"],
|
|
394
628
|
observability: ActivityServices["observability"],
|
|
395
629
|
probe: RetainedProcessProbeFn,
|
|
630
|
+
inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
|
|
396
631
|
): Promise<void> {
|
|
397
632
|
const claimId = crypto.randomUUID();
|
|
398
633
|
let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
|
|
@@ -415,39 +650,143 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
415
650
|
const expected = retainedProcessSettlementIdentity(process);
|
|
416
651
|
let proof = retainedProcessReconciliationProof(process);
|
|
417
652
|
if (!proof) {
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
653
|
+
let observation: RetainedProcessProbeResult | null = null;
|
|
654
|
+
|
|
655
|
+
// Legacy rows created before provider bindings existed are adoptable only
|
|
656
|
+
// when the exact historical sandbox resolves in the configured Modal
|
|
657
|
+
// namespace. NotFound is ambiguous here: it can mean either deletion or
|
|
658
|
+
// credential rotation, so it never creates ownership or loss proof.
|
|
421
659
|
if (
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
lease.backend !== process.providerBackend ||
|
|
426
|
-
lease.instanceId !== process.providerInstanceId ||
|
|
427
|
-
process.routeTargetId !== null
|
|
660
|
+
process.providerBackend === "modal" &&
|
|
661
|
+
process.routeTargetId === null &&
|
|
662
|
+
!process.providerBindingKey
|
|
428
663
|
) {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
664
|
+
try {
|
|
665
|
+
const lifecycle = await withRetainedProcessProbeTimeout(
|
|
666
|
+
inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),
|
|
667
|
+
);
|
|
668
|
+
if (lifecycle.status === "not_found") {
|
|
669
|
+
observation = { status: "deferred", reason: "provider_binding_missing" };
|
|
670
|
+
} else {
|
|
671
|
+
process = await bindRetainedProcessProviderIdentity(db, {
|
|
672
|
+
accountId: process.accountId,
|
|
673
|
+
workspaceId: process.workspaceId,
|
|
674
|
+
sessionId: process.sessionId,
|
|
675
|
+
processId: process.id,
|
|
676
|
+
expected,
|
|
677
|
+
claimId: claim.claimId,
|
|
678
|
+
providerBindingKey: lifecycle.providerBindingKey,
|
|
679
|
+
providerBinding: lifecycle.providerBinding,
|
|
680
|
+
});
|
|
681
|
+
recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
|
|
682
|
+
if (lifecycle.status === "terminated") {
|
|
683
|
+
observation = {
|
|
684
|
+
status: "proved",
|
|
685
|
+
proof: {
|
|
686
|
+
outcome: "lost",
|
|
687
|
+
exitCode: null,
|
|
688
|
+
reason: "provider_instance_terminated",
|
|
689
|
+
},
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
} catch (error) {
|
|
694
|
+
observation = {
|
|
695
|
+
status: "deferred",
|
|
696
|
+
reason:
|
|
697
|
+
error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
|
|
698
|
+
};
|
|
699
|
+
}
|
|
439
700
|
}
|
|
440
701
|
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
702
|
+
if (observation === null) {
|
|
703
|
+
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
704
|
+
() => null,
|
|
705
|
+
);
|
|
706
|
+
const currentLeaseIdentityMatches =
|
|
707
|
+
Boolean(lease) &&
|
|
708
|
+
lease!.id === process.leaseId &&
|
|
709
|
+
lease!.leaseEpoch === process.leaseEpoch &&
|
|
710
|
+
lease!.backend === process.providerBackend &&
|
|
711
|
+
lease!.instanceId === process.providerInstanceId &&
|
|
712
|
+
process.routeTargetId === null;
|
|
713
|
+
if (!currentLeaseIdentityMatches) {
|
|
714
|
+
// The lease may already be cold or belong to a successor. Its current
|
|
715
|
+
// resume envelope cannot interrogate the historical process, but a
|
|
716
|
+
// bound process can safely inspect the exact old Modal sandbox id.
|
|
717
|
+
if (process.providerBackend !== "modal" || process.routeTargetId !== null) {
|
|
718
|
+
observation = { status: "deferred", reason: "identity_mismatch" };
|
|
719
|
+
} else {
|
|
720
|
+
try {
|
|
721
|
+
const lifecycle = await withRetainedProcessProbeTimeout(
|
|
722
|
+
inspectHistoricalModalSandbox(
|
|
723
|
+
settings,
|
|
724
|
+
process.providerInstanceId,
|
|
725
|
+
process.providerBindingKey,
|
|
726
|
+
),
|
|
727
|
+
);
|
|
728
|
+
observation =
|
|
729
|
+
lifecycle.status === "not_found"
|
|
730
|
+
? {
|
|
731
|
+
status: "proved",
|
|
732
|
+
proof: {
|
|
733
|
+
outcome: "lost",
|
|
734
|
+
exitCode: null,
|
|
735
|
+
reason: "provider_instance_not_found",
|
|
736
|
+
},
|
|
737
|
+
}
|
|
738
|
+
: lifecycle.status === "terminated"
|
|
739
|
+
? {
|
|
740
|
+
status: "proved",
|
|
741
|
+
proof: {
|
|
742
|
+
outcome: "lost",
|
|
743
|
+
exitCode: null,
|
|
744
|
+
reason: "provider_instance_terminated",
|
|
745
|
+
},
|
|
746
|
+
}
|
|
747
|
+
: { status: "deferred", reason: "identity_mismatch" };
|
|
748
|
+
} catch (error) {
|
|
749
|
+
observation = {
|
|
750
|
+
status: "deferred",
|
|
751
|
+
reason:
|
|
752
|
+
error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
|
|
753
|
+
};
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
} else {
|
|
757
|
+
try {
|
|
758
|
+
observation = await probe(settings, lease!, process);
|
|
759
|
+
} catch (error) {
|
|
760
|
+
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
761
|
+
processId: process.id,
|
|
762
|
+
providerBackend: process.providerBackend,
|
|
763
|
+
error: error instanceof Error ? error.message : String(error),
|
|
764
|
+
});
|
|
765
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
if (observation.status === "binding") {
|
|
770
|
+
try {
|
|
771
|
+
process = await bindRetainedProcessProviderIdentity(db, {
|
|
772
|
+
accountId: process.accountId,
|
|
773
|
+
workspaceId: process.workspaceId,
|
|
774
|
+
sessionId: process.sessionId,
|
|
775
|
+
processId: process.id,
|
|
776
|
+
expected,
|
|
777
|
+
claimId: claim.claimId,
|
|
778
|
+
providerBindingKey: observation.providerBindingKey,
|
|
779
|
+
providerBinding: observation.providerBinding,
|
|
780
|
+
});
|
|
781
|
+
recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
|
|
782
|
+
observation = { status: "deferred", reason: "provider_running" };
|
|
783
|
+
} catch (error) {
|
|
784
|
+
observability.warn("sandbox reaper: retained-process binding adoption failed", {
|
|
785
|
+
processId: process.id,
|
|
786
|
+
error: error instanceof Error ? error.message : String(error),
|
|
787
|
+
});
|
|
788
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
789
|
+
}
|
|
451
790
|
}
|
|
452
791
|
if (observation.status === "deferred") {
|
|
453
792
|
await deferRetainedProcessClaim(
|
|
@@ -613,6 +952,25 @@ export async function probeRetainedProcessAtProvider(
|
|
|
613
952
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
614
953
|
}
|
|
615
954
|
|
|
955
|
+
let configuredModalBinding: Awaited<
|
|
956
|
+
ReturnType<typeof resolveModalCheckpointProviderBinding>
|
|
957
|
+
> | null = null;
|
|
958
|
+
if (process.providerBackend === "modal" && process.providerBindingKey) {
|
|
959
|
+
try {
|
|
960
|
+
configuredModalBinding = await withRetainedProcessProbeTimeout(
|
|
961
|
+
resolveModalCheckpointProviderBinding(settings),
|
|
962
|
+
);
|
|
963
|
+
} catch (error) {
|
|
964
|
+
return {
|
|
965
|
+
status: "deferred",
|
|
966
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
if (configuredModalBinding.key !== process.providerBindingKey) {
|
|
970
|
+
return { status: "deferred", reason: "provider_binding_mismatch" };
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
|
|
616
974
|
let client: RetainedProcessProbeClient | undefined;
|
|
617
975
|
try {
|
|
618
976
|
client = createSandboxClientForBackend(process.providerBackend as never, settings) as
|
|
@@ -649,6 +1007,9 @@ export async function probeRetainedProcessAtProvider(
|
|
|
649
1007
|
return { status: "deferred", reason: "provider_timeout" };
|
|
650
1008
|
}
|
|
651
1009
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
1010
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
1011
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
1012
|
+
}
|
|
652
1013
|
return {
|
|
653
1014
|
status: "proved",
|
|
654
1015
|
proof: {
|
|
@@ -663,6 +1024,32 @@ export async function probeRetainedProcessAtProvider(
|
|
|
663
1024
|
if (typeof session.writeStdin !== "function") {
|
|
664
1025
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
665
1026
|
}
|
|
1027
|
+
if (process.providerBackend === "modal") {
|
|
1028
|
+
let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;
|
|
1029
|
+
try {
|
|
1030
|
+
liveBinding = await withRetainedProcessProbeTimeout(
|
|
1031
|
+
resolveModalCheckpointProviderBindingForSession(settings, session),
|
|
1032
|
+
);
|
|
1033
|
+
} catch (error) {
|
|
1034
|
+
return {
|
|
1035
|
+
status: "deferred",
|
|
1036
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
if (!process.providerBindingKey) {
|
|
1040
|
+
return {
|
|
1041
|
+
status: "binding",
|
|
1042
|
+
providerBindingKey: liveBinding.key,
|
|
1043
|
+
providerBinding: liveBinding.binding,
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
if (
|
|
1047
|
+
liveBinding.key !== process.providerBindingKey ||
|
|
1048
|
+
configuredModalBinding?.key !== process.providerBindingKey
|
|
1049
|
+
) {
|
|
1050
|
+
return { status: "deferred", reason: "provider_binding_mismatch" };
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
666
1053
|
|
|
667
1054
|
let result: unknown;
|
|
668
1055
|
try {
|
|
@@ -679,6 +1066,9 @@ export async function probeRetainedProcessAtProvider(
|
|
|
679
1066
|
return { status: "deferred", reason: "provider_timeout" };
|
|
680
1067
|
}
|
|
681
1068
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
1069
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
1070
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
1071
|
+
}
|
|
682
1072
|
return {
|
|
683
1073
|
status: "proved",
|
|
684
1074
|
proof: {
|
|
@@ -690,7 +1080,46 @@ export async function probeRetainedProcessAtProvider(
|
|
|
690
1080
|
}
|
|
691
1081
|
return { status: "deferred", reason: "provider_error" };
|
|
692
1082
|
}
|
|
693
|
-
|
|
1083
|
+
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
1084
|
+
if (
|
|
1085
|
+
observation.status === "deferred" &&
|
|
1086
|
+
observation.reason === "provider_running" &&
|
|
1087
|
+
lease.rotationRequestedAt !== null
|
|
1088
|
+
) {
|
|
1089
|
+
// A background PTY cannot outlive the finite provider box. Interrupt only
|
|
1090
|
+
// this exact durable provider session after rotation admission is fenced;
|
|
1091
|
+
// settlement still requires an exit/loss banner on this or a later sweep.
|
|
1092
|
+
try {
|
|
1093
|
+
const interrupted = await withRetainedProcessProbeTimeout(
|
|
1094
|
+
session.writeStdin({
|
|
1095
|
+
sessionId: process.providerSessionId,
|
|
1096
|
+
chars: "\u0003",
|
|
1097
|
+
yieldTimeMs: 1_000,
|
|
1098
|
+
maxOutputTokens: 2_000,
|
|
1099
|
+
}),
|
|
1100
|
+
);
|
|
1101
|
+
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
|
|
1102
|
+
} catch (error) {
|
|
1103
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
1104
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
1105
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
1106
|
+
}
|
|
1107
|
+
return {
|
|
1108
|
+
status: "proved",
|
|
1109
|
+
proof: {
|
|
1110
|
+
outcome: "lost",
|
|
1111
|
+
exitCode: null,
|
|
1112
|
+
reason: "provider_instance_not_found",
|
|
1113
|
+
},
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
return {
|
|
1117
|
+
status: "deferred",
|
|
1118
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error",
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
return observation;
|
|
694
1123
|
}
|
|
695
1124
|
|
|
696
1125
|
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
@@ -796,6 +1225,23 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
796
1225
|
error: error instanceof Error ? error.message : String(error),
|
|
797
1226
|
});
|
|
798
1227
|
}
|
|
1228
|
+
try {
|
|
1229
|
+
recordSandboxCheckpointArtifactGauges(
|
|
1230
|
+
observability,
|
|
1231
|
+
await countSandboxCheckpointArtifactsByState(db),
|
|
1232
|
+
);
|
|
1233
|
+
} catch (error) {
|
|
1234
|
+
observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
|
|
1235
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
try {
|
|
1239
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
1240
|
+
} catch (error) {
|
|
1241
|
+
observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
|
|
1242
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
799
1245
|
try {
|
|
800
1246
|
recordRetainedProcessInventoryGauges(
|
|
801
1247
|
observability,
|
|
@@ -841,10 +1287,6 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
841
1287
|
const enforceBalance =
|
|
842
1288
|
settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
843
1289
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
844
|
-
// Nothing to enforce → skip the whole pass (no lock churn).
|
|
845
|
-
if (!enforceBalance && cap <= 0) {
|
|
846
|
-
return 0;
|
|
847
|
-
}
|
|
848
1290
|
let forceDrained = 0;
|
|
849
1291
|
for (const workspaceId of workspaceIds) {
|
|
850
1292
|
try {
|
|
@@ -947,6 +1389,77 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
947
1389
|
return result.terminated.length;
|
|
948
1390
|
}
|
|
949
1391
|
|
|
1392
|
+
/**
|
|
1393
|
+
* A capture claim whose owning worker disappeared must not fence a workspace
|
|
1394
|
+
* forever. The drain reaper reaches this seam only after every holder is gone.
|
|
1395
|
+
* It resumes the exact attributed provider without replacement and proves that
|
|
1396
|
+
* a no-op command can execute. A paused snapshot cannot pass that proof; a
|
|
1397
|
+
* missing provider is returned as typed loss; every ambiguous error preserves
|
|
1398
|
+
* the old claim for the next sweep.
|
|
1399
|
+
*/
|
|
1400
|
+
async function probeDrainableProviderReadiness(
|
|
1401
|
+
settings: ActivityServices["settings"],
|
|
1402
|
+
lease: LeaseSnapshot,
|
|
1403
|
+
): Promise<"ready" | "missing"> {
|
|
1404
|
+
if (!lease.instanceId || !lease.resumeState) {
|
|
1405
|
+
throw new Error("Expired workspace capture has no resumable provider identity");
|
|
1406
|
+
}
|
|
1407
|
+
const backend = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1408
|
+
let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;
|
|
1409
|
+
try {
|
|
1410
|
+
established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
|
|
1411
|
+
sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,
|
|
1412
|
+
recovery: "resume-only",
|
|
1413
|
+
backendOverride: backend as never,
|
|
1414
|
+
});
|
|
1415
|
+
} catch (error) {
|
|
1416
|
+
if (isProviderSandboxNotFoundError(backend, error)) return "missing";
|
|
1417
|
+
throw error;
|
|
1418
|
+
}
|
|
1419
|
+
if (established.instanceId !== lease.instanceId) {
|
|
1420
|
+
throw new Error("Expired workspace capture resumed a different provider instance");
|
|
1421
|
+
}
|
|
1422
|
+
try {
|
|
1423
|
+
if (established.backendId === "modal") {
|
|
1424
|
+
await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);
|
|
1425
|
+
return "ready";
|
|
1426
|
+
}
|
|
1427
|
+
const session = established.session as {
|
|
1428
|
+
exec?: (args: {
|
|
1429
|
+
cmd: string;
|
|
1430
|
+
yieldTimeMs?: number;
|
|
1431
|
+
maxOutputTokens?: number;
|
|
1432
|
+
}) => Promise<unknown>;
|
|
1433
|
+
execCommand?: (args: {
|
|
1434
|
+
cmd: string;
|
|
1435
|
+
yieldTimeMs?: number;
|
|
1436
|
+
maxOutputTokens?: number;
|
|
1437
|
+
}) => Promise<unknown>;
|
|
1438
|
+
};
|
|
1439
|
+
const run = session.exec ?? session.execCommand;
|
|
1440
|
+
if (!run) throw new Error("Expired workspace capture provider has no readiness command");
|
|
1441
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1442
|
+
const result = await Promise.race([
|
|
1443
|
+
run.call(session, { cmd: "true", yieldTimeMs: 1_000, maxOutputTokens: 1_000 }),
|
|
1444
|
+
new Promise<never>((_, reject) => {
|
|
1445
|
+
timer = setTimeout(
|
|
1446
|
+
() => reject(new Error("Expired workspace capture readiness probe timed out")),
|
|
1447
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1448
|
+
);
|
|
1449
|
+
}),
|
|
1450
|
+
]).finally(() => {
|
|
1451
|
+
if (timer) clearTimeout(timer);
|
|
1452
|
+
});
|
|
1453
|
+
if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {
|
|
1454
|
+
throw new Error("Expired workspace capture provider is not command-ready");
|
|
1455
|
+
}
|
|
1456
|
+
return "ready";
|
|
1457
|
+
} catch (error) {
|
|
1458
|
+
if (isProviderSandboxNotFoundError(backend, error)) return "missing";
|
|
1459
|
+
throw error;
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
|
|
950
1463
|
/**
|
|
951
1464
|
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
952
1465
|
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
@@ -970,6 +1483,7 @@ async function terminateDrainableBox(
|
|
|
970
1483
|
row: ReapDrainable,
|
|
971
1484
|
observability: ActivityServices["observability"],
|
|
972
1485
|
terminateBox: TerminateBoxFn,
|
|
1486
|
+
probeDrainableProvider: DrainableProviderProbeFn,
|
|
973
1487
|
): Promise<boolean> {
|
|
974
1488
|
// Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
|
|
975
1489
|
// returns no account_id; the workspace->account map is the bootstrap read).
|
|
@@ -992,16 +1506,78 @@ async function terminateDrainableBox(
|
|
|
992
1506
|
return false;
|
|
993
1507
|
}
|
|
994
1508
|
|
|
995
|
-
const
|
|
996
|
-
|
|
1509
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
1510
|
+
let captureClaim:
|
|
1511
|
+
| NonNullable<
|
|
1512
|
+
Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
|
|
1513
|
+
>["claim"]
|
|
1514
|
+
| null = null;
|
|
1515
|
+
let providerMissingBeforeCapture = false;
|
|
1516
|
+
if (lease.instanceId) {
|
|
1517
|
+
if (lease.archiveCapture) {
|
|
1518
|
+
if (lease.archiveCapture.deadlineAt.getTime() > Date.now()) {
|
|
1519
|
+
// A live owner still holds the provider pause gate. It will release the
|
|
1520
|
+
// exact claim on settlement; this sweep must not compete.
|
|
1521
|
+
return false;
|
|
1522
|
+
}
|
|
1523
|
+
// A deadline is never evidence that a non-cancellable provider capture
|
|
1524
|
+
// stopped. Without an exact resume envelope there is no safe way to prove
|
|
1525
|
+
// the attributed provider is command-ready, so preserve the gate.
|
|
1526
|
+
if (!lease.resumeState) return false;
|
|
1527
|
+
const providerState = await probeDrainableProvider(settings, lease);
|
|
1528
|
+
if (providerState === "missing") {
|
|
1529
|
+
providerMissingBeforeCapture = true;
|
|
1530
|
+
captureClaim = {
|
|
1531
|
+
...lease.archiveCapture,
|
|
1532
|
+
leaseId: lease.id,
|
|
1533
|
+
leaseEpoch: lease.leaseEpoch,
|
|
1534
|
+
instanceId: lease.instanceId,
|
|
1535
|
+
archiveGeneration: lease.archiveGeneration,
|
|
1536
|
+
archiveComplete: lease.archiveComplete,
|
|
1537
|
+
};
|
|
1538
|
+
} else {
|
|
1539
|
+
const replacement = await replaceExpiredWorkspaceArchiveCapture(db, {
|
|
1540
|
+
accountId,
|
|
1541
|
+
workspaceId: row.workspaceId,
|
|
1542
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1543
|
+
priorCaptureId: lease.archiveCapture.id,
|
|
1544
|
+
captureId: randomUUID(),
|
|
1545
|
+
expectedEpoch: row.leaseEpoch,
|
|
1546
|
+
expectedInstanceId: lease.instanceId,
|
|
1547
|
+
captureTimeoutMs,
|
|
1548
|
+
});
|
|
1549
|
+
if (!replacement) return false;
|
|
1550
|
+
captureClaim = replacement;
|
|
1551
|
+
}
|
|
1552
|
+
} else {
|
|
1553
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
997
1554
|
accountId,
|
|
998
1555
|
workspaceId: row.workspaceId,
|
|
999
1556
|
sandboxGroupId: row.sandboxGroupId,
|
|
1557
|
+
captureId: randomUUID(),
|
|
1000
1558
|
expectedEpoch: row.leaseEpoch,
|
|
1001
1559
|
expectedInstanceId: lease.instanceId,
|
|
1002
1560
|
liveness: "draining",
|
|
1003
|
-
|
|
1004
|
-
|
|
1561
|
+
captureTimeoutMs,
|
|
1562
|
+
minIntervalMs: 0,
|
|
1563
|
+
});
|
|
1564
|
+
if (claimed.status === "claimed") {
|
|
1565
|
+
captureClaim = claimed.claim;
|
|
1566
|
+
} else if (claimed.status === "mutation_in_progress") {
|
|
1567
|
+
// An admission is normally authoritative proof that a provider
|
|
1568
|
+
// operation may still be running. It cannot, however, pin a draining
|
|
1569
|
+
// lease forever after the exact provider has disappeared. Probe the
|
|
1570
|
+
// attributed instance without replacement. Only typed NotFound permits
|
|
1571
|
+
// the cold commit to reject the exact stale admissions; a live or
|
|
1572
|
+
// ambiguous provider remains fenced for a later sweep.
|
|
1573
|
+
const providerState = await probeDrainableProvider(settings, lease);
|
|
1574
|
+
if (providerState !== "missing") return false;
|
|
1575
|
+
providerMissingBeforeCapture = true;
|
|
1576
|
+
} else {
|
|
1577
|
+
return false;
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1005
1581
|
|
|
1006
1582
|
// The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
|
|
1007
1583
|
// snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
|
|
@@ -1016,23 +1592,73 @@ async function terminateDrainableBox(
|
|
|
1016
1592
|
const persistArchive: PersistArchiveFn = async (
|
|
1017
1593
|
archiveBase64: string | null,
|
|
1018
1594
|
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
1595
|
+
_providerSession?: unknown,
|
|
1596
|
+
providerBinding?: Awaited<
|
|
1597
|
+
ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
|
|
1598
|
+
> | null,
|
|
1019
1599
|
) => {
|
|
1020
|
-
// A live persistable provider must
|
|
1021
|
-
//
|
|
1022
|
-
//
|
|
1023
|
-
if (!lease.instanceId || !
|
|
1024
|
-
return { wrote: false,
|
|
1600
|
+
// A live persistable provider must own the exact durable capture claim.
|
|
1601
|
+
// Without it we cannot prove that these bytes came from a provider that was
|
|
1602
|
+
// admission-fenced for the full pause/capture interval.
|
|
1603
|
+
if (!lease.instanceId || !captureClaim) {
|
|
1604
|
+
return { wrote: false, archiveRevision: null };
|
|
1025
1605
|
}
|
|
1026
|
-
|
|
1606
|
+
let checkpointArtifactId: string | null = null;
|
|
1607
|
+
if (
|
|
1608
|
+
archiveBase64 &&
|
|
1609
|
+
archiveMetadata?.version === 2 &&
|
|
1610
|
+
(archiveMetadata.provider === "modal_snapshot_filesystem" ||
|
|
1611
|
+
archiveMetadata.provider === "modal_snapshot_directory")
|
|
1612
|
+
) {
|
|
1613
|
+
if (!providerBinding) {
|
|
1614
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
1615
|
+
}
|
|
1616
|
+
const candidate = await registerSandboxCheckpointArtifact(db, {
|
|
1617
|
+
accountId,
|
|
1618
|
+
workspaceId: row.workspaceId,
|
|
1619
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1620
|
+
sourceLeaseId: lease.id,
|
|
1621
|
+
sourceLeaseEpoch: row.leaseEpoch,
|
|
1622
|
+
sourceInstanceId: lease.instanceId,
|
|
1623
|
+
sourceWorkspaceGeneration: captureClaim.workspaceGeneration,
|
|
1624
|
+
providerBindingKey: providerBinding.key,
|
|
1625
|
+
providerBinding: providerBinding.binding,
|
|
1626
|
+
workspaceArchive: archiveBase64,
|
|
1627
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
1628
|
+
});
|
|
1629
|
+
checkpointArtifactId = candidate.id;
|
|
1630
|
+
}
|
|
1631
|
+
const baseInput = {
|
|
1027
1632
|
accountId,
|
|
1028
1633
|
workspaceId: row.workspaceId,
|
|
1029
1634
|
sandboxGroupId: row.sandboxGroupId,
|
|
1030
1635
|
expectedEpoch: row.leaseEpoch,
|
|
1031
1636
|
expectedInstanceId: lease.instanceId,
|
|
1032
|
-
expectedWorkspaceGeneration:
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1637
|
+
expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
|
|
1638
|
+
captureId: captureClaim.id,
|
|
1639
|
+
};
|
|
1640
|
+
let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;
|
|
1641
|
+
if (archiveBase64 === null) {
|
|
1642
|
+
result = await persistDrainSnapshot(db, { ...baseInput, workspaceArchive: null });
|
|
1643
|
+
} else {
|
|
1644
|
+
if (!archiveMetadata) {
|
|
1645
|
+
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
1646
|
+
}
|
|
1647
|
+
result = await persistDrainSnapshot(db, {
|
|
1648
|
+
...baseInput,
|
|
1649
|
+
workspaceArchive: archiveBase64,
|
|
1650
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
1651
|
+
...(checkpointArtifactId ? { checkpointArtifactId } : {}),
|
|
1652
|
+
});
|
|
1653
|
+
}
|
|
1654
|
+
if (!result.wrote && checkpointArtifactId) {
|
|
1655
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
1656
|
+
accountId,
|
|
1657
|
+
workspaceId: row.workspaceId,
|
|
1658
|
+
artifactId: checkpointArtifactId,
|
|
1659
|
+
reason: "drain_snapshot_publication_fenced",
|
|
1660
|
+
}).catch(() => undefined);
|
|
1661
|
+
}
|
|
1036
1662
|
if (result.wrote && archiveBase64 !== null) {
|
|
1037
1663
|
persisted = true;
|
|
1038
1664
|
archiveRevision = result.archiveRevision;
|
|
@@ -1040,78 +1666,103 @@ async function terminateDrainableBox(
|
|
|
1040
1666
|
return result;
|
|
1041
1667
|
};
|
|
1042
1668
|
|
|
1043
|
-
// Resume-by-id -> verified capture -> persist-onto-lease ->
|
|
1044
|
-
//
|
|
1045
|
-
//
|
|
1046
|
-
//
|
|
1047
|
-
//
|
|
1048
|
-
//
|
|
1049
|
-
//
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1669
|
+
// Resume-by-id -> verified capture -> persist-onto-lease -> provider
|
|
1670
|
+
// terminate. Persisting atomically hands any displaced provider snapshot to
|
|
1671
|
+
// the provider-bound artifact ledger; its global GC worker owns deletion.
|
|
1672
|
+
// Definitive NotFound before capture is a typed loss, not ordinary teardown
|
|
1673
|
+
// success: confirmDrainCold preserves any prior archive or marks recovery
|
|
1674
|
+
// unrecoverable when none exists. A genuine capture/resume failure leaves the
|
|
1675
|
+
// lease draining for a later sweep (NEVER terminate a box whose files we
|
|
1676
|
+
// could not capture). A persist CAS miss means the box was re-armed and left
|
|
1677
|
+
// running, so the cold commit is skipped.
|
|
1678
|
+
try {
|
|
1679
|
+
const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
|
|
1680
|
+
? { terminated: true, providerMissingBeforeCapture: true }
|
|
1681
|
+
: await terminateBox(settings, lease, observability, persistArchive);
|
|
1682
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
1683
|
+
if (!terminated) {
|
|
1684
|
+
return false;
|
|
1685
|
+
}
|
|
1055
1686
|
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1687
|
+
const providerMissing =
|
|
1688
|
+
providerMissingBeforeCapture ||
|
|
1689
|
+
(typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
1690
|
+
// The authoritative commit: CAS draining->cold under the epoch fence. If a
|
|
1691
|
+
// late re-arm or newer epoch raced in after our re-read, wentCold:false and we
|
|
1692
|
+
// report a skip (the box is down; a fresh acquire cold-restores it).
|
|
1693
|
+
const { wentCold } = await confirmDrainCold(db, {
|
|
1694
|
+
accountId,
|
|
1695
|
+
workspaceId: row.workspaceId,
|
|
1696
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1697
|
+
expectedEpoch: row.leaseEpoch,
|
|
1698
|
+
providerMissingBeforeCapture: providerMissing,
|
|
1699
|
+
});
|
|
1700
|
+
if (wentCold) {
|
|
1701
|
+
// Durable termination record (sandbox-file-persistence observability): who
|
|
1702
|
+
// ended this box and whether its /workspace was captured first, appended to
|
|
1703
|
+
// every session sharing the group's box. Best-effort: attribution must
|
|
1704
|
+
// never affect the drain outcome.
|
|
1705
|
+
try {
|
|
1706
|
+
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
1707
|
+
type: "sandbox.box.terminated",
|
|
1708
|
+
payload: {
|
|
1709
|
+
actor: "reaper",
|
|
1710
|
+
persisted,
|
|
1711
|
+
archiveRevision,
|
|
1712
|
+
instanceId: lease.instanceId,
|
|
1713
|
+
providerMissingBeforeCapture: providerMissing,
|
|
1714
|
+
},
|
|
1715
|
+
});
|
|
1716
|
+
} catch (eventError) {
|
|
1717
|
+
observability.warn(
|
|
1718
|
+
"sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
|
|
1719
|
+
{
|
|
1720
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1721
|
+
error: eventError instanceof Error ? eventError.message : String(eventError),
|
|
1722
|
+
},
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
return wentCold;
|
|
1727
|
+
} finally {
|
|
1728
|
+
if (captureClaim && lease.instanceId) {
|
|
1729
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
1730
|
+
accountId,
|
|
1731
|
+
workspaceId: row.workspaceId,
|
|
1732
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1733
|
+
captureId: captureClaim.id,
|
|
1734
|
+
expectedEpoch: row.leaseEpoch,
|
|
1735
|
+
expectedInstanceId: lease.instanceId,
|
|
1736
|
+
}).catch((error) => {
|
|
1737
|
+
observability.warn("sandbox reaper: archive capture gate release failed", {
|
|
1088
1738
|
sandboxGroupId: row.sandboxGroupId,
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1739
|
+
leaseEpoch: row.leaseEpoch,
|
|
1740
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1741
|
+
});
|
|
1742
|
+
});
|
|
1092
1743
|
}
|
|
1093
1744
|
}
|
|
1094
|
-
return wentCold;
|
|
1095
1745
|
}
|
|
1096
1746
|
|
|
1097
1747
|
// The persist-capable slice of a live provider session. persistWorkspace()
|
|
1098
1748
|
// snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
|
|
1099
|
-
// archive) and returns the archive bytes.
|
|
1100
|
-
//
|
|
1749
|
+
// archive) and returns the archive bytes. The session also carries the Modal SDK
|
|
1750
|
+
// client used to bind the snapshot receipt to the exact provider namespace.
|
|
1101
1751
|
type PersistableSession = TerminableSession & {
|
|
1102
1752
|
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
1103
1753
|
};
|
|
1104
1754
|
|
|
1105
1755
|
/**
|
|
1106
1756
|
* Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
|
|
1107
|
-
* lease under the epoch fence,
|
|
1108
|
-
* DIRECTLY (no cold-restore fallback) so a gone box is a clean
|
|
1109
|
-
* wasteful create-then-kill:
|
|
1757
|
+
* lease under the epoch fence, hand the superseded snapshot to durable GC, THEN
|
|
1758
|
+
* terminate. Resumes DIRECTLY (no cold-restore fallback) so a gone box is a clean
|
|
1759
|
+
* no-op, never a wasteful create-then-kill:
|
|
1110
1760
|
* - build the client for the lease's backend;
|
|
1111
1761
|
* - deserialize + resume the envelope (warm reattach by id, R4-safe);
|
|
1112
1762
|
* - session.persistWorkspace() -> capture the /workspace archive;
|
|
1113
|
-
* - persistArchive(base64) -> CAS-fold it onto the lease
|
|
1114
|
-
*
|
|
1763
|
+
* - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark
|
|
1764
|
+
* the displaced artifact delete-pending;
|
|
1765
|
+
* - the provider-bound global GC deletes that object asynchronously;
|
|
1115
1766
|
* - terminate the live handle (client.delete / session kill|terminate|close).
|
|
1116
1767
|
*
|
|
1117
1768
|
* Returns true when the box was terminated (or was already gone), false when the
|
|
@@ -1264,6 +1915,17 @@ export async function terminateProviderBox(
|
|
|
1264
1915
|
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
1265
1916
|
// box raced gone between resume and persist) is success: nothing to persist.
|
|
1266
1917
|
let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
|
|
1918
|
+
const sessionWorkspacePersistence = (
|
|
1919
|
+
session as { state?: { workspacePersistence?: unknown } } | undefined
|
|
1920
|
+
)?.state?.workspacePersistence;
|
|
1921
|
+
const checkpointBinding =
|
|
1922
|
+
backend === "modal" &&
|
|
1923
|
+
(sessionWorkspacePersistence === "snapshot_filesystem" ||
|
|
1924
|
+
sessionWorkspacePersistence === "snapshot_directory")
|
|
1925
|
+
? await withRetainedProcessProbeTimeout(
|
|
1926
|
+
resolveModalCheckpointProviderBindingForSession(settings, session),
|
|
1927
|
+
)
|
|
1928
|
+
: null;
|
|
1267
1929
|
try {
|
|
1268
1930
|
verifiedArchive = session?.persistWorkspace
|
|
1269
1931
|
? await captureVerifiedWorkspaceArchive(session)
|
|
@@ -1301,9 +1963,11 @@ export async function terminateProviderBox(
|
|
|
1301
1963
|
// null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
|
|
1302
1964
|
// refcount/epoch guard, no write. wrote:false → abort the terminate.
|
|
1303
1965
|
if (verifiedArchive) {
|
|
1304
|
-
const { wrote
|
|
1966
|
+
const { wrote } = await persistArchive(
|
|
1305
1967
|
verifiedArchive.base64,
|
|
1306
1968
|
verifiedArchive.descriptor,
|
|
1969
|
+
session,
|
|
1970
|
+
checkpointBinding,
|
|
1307
1971
|
);
|
|
1308
1972
|
if (!wrote) {
|
|
1309
1973
|
observability.info(
|
|
@@ -1315,16 +1979,6 @@ export async function terminateProviderBox(
|
|
|
1315
1979
|
);
|
|
1316
1980
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
1317
1981
|
}
|
|
1318
|
-
// Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
|
|
1319
|
-
// while the session (and its Modal client) is still live. Never throws.
|
|
1320
|
-
const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
|
|
1321
|
-
if (deleted) {
|
|
1322
|
-
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
1323
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
1324
|
-
backend,
|
|
1325
|
-
snapshotId: deleted,
|
|
1326
|
-
});
|
|
1327
|
-
}
|
|
1328
1982
|
} else {
|
|
1329
1983
|
// A resumable cloud box that produces no verified archive cannot be safely
|
|
1330
1984
|
// deleted: it may contain work newer than the lease's last durable revision.
|