@opengeni/worker-bundle 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,612 @@
1
+ // apps/worker/src/activities/sandbox-lease.ts — the SOLE liveness/GC/cost-stop
2
+ // driver (P1.3 / OD-3).
3
+ //
4
+ // There is exactly ONE reaper activity: `reapSandboxLeases`. It is fired by the
5
+ // ONE global reaper Temporal Schedule (registered in apps/worker/src/index.ts).
6
+ // There is NO ownerHeartbeat, NO per-session timer, NO per-RPC workflow, NO
7
+ // *ForViewer activity, NO resolveOwnerTaskQueue. Turn-holder lifecycle is bound
8
+ // to Temporal *activity* liveness (the turn activity acquires/releases the turn
9
+ // holder); a crashed founder's leaked turn holder becomes reapable via the lease
10
+ // TTL — TTL-exemption means a *live* turn is never idle-reaped, NOT that a dead
11
+ // turn's holder is immortal.
12
+ //
13
+ // One pass per fire:
14
+ // 1. reapStaleLeaseHoldersGlobal (the P1.1 SECURITY-DEFINER cross-workspace
15
+ // sweep): TTL-reaps stale viewer holders, resets warming-death rows to cold,
16
+ // recomputes refcounts + enters draining at refcount 0, and RETURNS the
17
+ // drainable rows (workspace, group, instance, epoch) whose drain grace has
18
+ // elapsed at refcount 0. DB-only — no provider call inside the sweep.
19
+ // 2. For each drainable row: resume/attach the provider box BY ID (off the
20
+ // lease's resume envelope, via createSandboxClientForBackend +
21
+ // establishSandboxSessionFromEnvelope), call the provider terminate, then
22
+ // confirmDrainCold (the CAS draining->cold under the epoch fence).
23
+ //
24
+ // IDEMPOTENT + safe to run concurrently with itself: the drain CAS is guarded on
25
+ // (draining AND refcount=0 AND lease_epoch=expected). If another sweep already
26
+ // drained the row, or a late re-arm flipped it warm, or a newer epoch snuck in,
27
+ // confirmDrainCold returns wentCold:false and we skip — provider stop() fires
28
+ // ONLY when the CAS proves the box is still the draining box we observed. We
29
+ // confirm the CAS would still pass (re-read the lease) BEFORE the provider call
30
+ // so a box that was re-armed mid-sweep is never terminated out from under a live
31
+ // holder.
32
+
33
+ import {
34
+ accrueWarmSeconds,
35
+ confirmDrainCold,
36
+ forceDrainOverLimitViewerOnlyBoxes,
37
+ getBillingBalance,
38
+ listMeterableWarmLeases,
39
+ persistDrainSnapshot,
40
+ readLease,
41
+ reapStaleLeaseHoldersGlobal,
42
+ rlsContextForWorkspace,
43
+ type MeterableWarmLease,
44
+ type ReapDrainable,
45
+ } from "@opengeni/db";
46
+ import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
47
+ import {
48
+ // The reaper attaches to the box by id to terminate it. It does NOT use
49
+ // establishSandboxSessionFromEnvelope (which cold-RESTORES via create() on a
50
+ // NotFound) — restoring a box just to immediately kill it is wrong. Instead it
51
+ // builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
52
+ // its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
53
+ // (NotFound on resume) is already down, a clean no-op.
54
+ createSandboxClientForBackend,
55
+ deletePriorPersistedSnapshot,
56
+ deserializeSandboxSessionStateEnvelope,
57
+ isProviderSandboxNotFoundError,
58
+ } from "@opengeni/runtime";
59
+ import type { ActivityServices } from "./types";
60
+
61
+ export type ReapSandboxLeasesResult = {
62
+ /** Stale viewer holders + warming-death rows the sweep touched is folded into
63
+ * the DB pass; here we report the terminate outcomes. */
64
+ examined: number;
65
+ /** Boxes whose provider terminate fired AND whose lease went cold. */
66
+ terminated: number;
67
+ /** Drainable rows skipped because the CAS no longer held (re-armed / newer
68
+ * epoch / already drained by a concurrent sweep) — provider stop() did NOT
69
+ * fire. */
70
+ skipped: number;
71
+ /** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
72
+ metered: number;
73
+ /** Viewer-only boxes force-drained because their workspace is over a limit
74
+ * (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
75
+ forceDrained: number;
76
+ };
77
+
78
+ // The structural slice of a provider SandboxClient we need to resume-by-id and
79
+ // terminate a box. `delete(state)` is the provider's stop()/terminate (the
80
+ // runtime SandboxClient maps it to the per-provider teardown). Narrowed so this
81
+ // stays agent-loop-free.
82
+ type TerminableClient = {
83
+ backendId: string;
84
+ resume?: (state: unknown) => Promise<unknown>;
85
+ deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
86
+ delete?: (state: unknown) => Promise<unknown>;
87
+ };
88
+
89
+ // A live session handle may expose a kill/terminate/close itself (some providers
90
+ // tear the box down from the session, not the client). We try the client.delete
91
+ // first (the canonical teardown), then fall back to a session-level terminator.
92
+ type TerminableSession = {
93
+ kill?: () => Promise<unknown>;
94
+ terminate?: () => Promise<unknown>;
95
+ close?: () => Promise<unknown>;
96
+ closed?: boolean;
97
+ };
98
+
99
+ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
100
+
101
+ /**
102
+ * The CAS-write the terminate seam calls to PERSIST the captured /workspace
103
+ * archive onto the lease under the epoch fence, BEFORE the box is terminated
104
+ * (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over
105
+ * db/accountId/row and delegates to persistDrainSnapshot. Returns:
106
+ * - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
107
+ * is wanted again; the seam MUST NOT terminate it.
108
+ * - priorArchive -> the superseded archive (if any) to GC the prior snapshot.
109
+ *
110
+ * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
111
+ * backends with no persistWorkspace — ensures a re-arm during the snapshot window
112
+ * aborts the terminate before client.delete()). priorArchive is always null in
113
+ * this case.
114
+ */
115
+ export type PersistArchiveFn = (
116
+ archiveBase64: string | null,
117
+ ) => Promise<{ wrote: boolean; priorArchive: string | null }>;
118
+
119
+ /** The provider-terminate seam. Production wires the real resume-by-id ->
120
+ * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
121
+ * provider stop() (`terminateProviderBox`); a unit test injects a spy so the
122
+ * drain/CAS logic is exercised against a real DB without a live provider box.
123
+ * Returns true when the box was terminated (or already gone), false when persist
124
+ * found the lease re-armed and the box was deliberately LEFT RUNNING. */
125
+ export type TerminateBoxFn = (
126
+ settings: ActivityServices["settings"],
127
+ lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
128
+ observability: ActivityServices["observability"],
129
+ persistArchive: PersistArchiveFn,
130
+ ) => Promise<boolean>;
131
+
132
+ export type SandboxLeaseActivityOptions = {
133
+ /** Override the provider terminate (tests spy this; defaults to the real
134
+ * resume-by-id + provider stop()). */
135
+ terminateBox?: TerminateBoxFn;
136
+ };
137
+
138
+ export function createSandboxLeaseActivities(
139
+ services: () => Promise<ActivityServices>,
140
+ options: SandboxLeaseActivityOptions = {},
141
+ ) {
142
+ const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
143
+ /**
144
+ * The one global reaper sweep. Idempotent; concurrency-safe with itself.
145
+ * Gated by the caller (the Schedule is only registered when
146
+ * sandboxOwnershipEnabled); a defensive no-op here too so a manual trigger
147
+ * with the flag off can never terminate a box.
148
+ */
149
+ async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
150
+ const { db, settings, observability } = await services();
151
+ if (!settings.sandboxOwnershipEnabled) {
152
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
153
+ }
154
+
155
+ // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
156
+ // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
157
+ // them). GROUP+epoch+tick idempotent → a shared box is one stream; an
158
+ // overlapping/re-fired sweep cannot double-charge. Best-effort per row.
159
+ const metered = await accrueWarmTick(db, settings, observability);
160
+
161
+ // (0b) Per-workspace warm-cap + force-drain (P2.1) — under the usage lock, a
162
+ // workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
163
+ // boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
164
+ // draining rows are caught by the same sweep's terminate below.
165
+ const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
166
+
167
+ // (1) The DB-only cross-workspace sweep. Returns the drainable rows.
168
+ const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
169
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
170
+ idleGraceMs: settings.sandboxIdleGraceMs,
171
+ });
172
+
173
+ let terminated = 0;
174
+ let skipped = 0;
175
+
176
+ // (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
177
+ // are isolated: one box's provider error must not abort the whole sweep (the
178
+ // next sweep retries it; the provider idle-timeout is the backstop).
179
+ for (const row of drainable) {
180
+ try {
181
+ const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
182
+ if (drainedCold) {
183
+ terminated += 1;
184
+ } else {
185
+ skipped += 1;
186
+ }
187
+ } catch (error) {
188
+ skipped += 1;
189
+ observability.warn("sandbox reaper: terminate failed for drainable lease", {
190
+ workspaceId: row.workspaceId,
191
+ sandboxGroupId: row.sandboxGroupId,
192
+ leaseEpoch: row.leaseEpoch,
193
+ error: error instanceof Error ? error.message : String(error),
194
+ });
195
+ }
196
+ }
197
+
198
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
199
+ observability.info("sandbox reaper swept", {
200
+ drainable: drainable.length,
201
+ terminated,
202
+ skipped,
203
+ metered: metered.accrued,
204
+ forceDrained,
205
+ });
206
+ }
207
+
208
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
209
+ }
210
+
211
+ return { reapSandboxLeases };
212
+ }
213
+
214
+ /**
215
+ * The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM
216
+ * viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those
217
+ * meter on the turn heartbeat). Returns the count accrued + the distinct
218
+ * workspaces touched (so the force-drain pass only checks workspaces that have a
219
+ * live warm box, not the whole fleet). Per-row best-effort: one row's metering
220
+ * error must not abort the sweep.
221
+ */
222
+ async function accrueWarmTick(
223
+ db: ActivityServices["db"],
224
+ settings: ActivityServices["settings"],
225
+ observability: ActivityServices["observability"],
226
+ ): Promise<{ accrued: number; workspaceIds: Set<string> }> {
227
+ const workspaceIds = new Set<string>();
228
+ let accrued = 0;
229
+ let leases: MeterableWarmLease[] = [];
230
+ try {
231
+ leases = await listMeterableWarmLeases(db);
232
+ } catch (error) {
233
+ observability.warn("sandbox reaper: warm-lease read failed", {
234
+ error: error instanceof Error ? error.message : String(error),
235
+ });
236
+ return { accrued, workspaceIds };
237
+ }
238
+ for (const lease of leases) {
239
+ workspaceIds.add(lease.workspaceId);
240
+ try {
241
+ const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
242
+ const result = await accrueWarmSeconds(db, {
243
+ accountId: lease.accountId,
244
+ workspaceId: lease.workspaceId,
245
+ sandboxGroupId: lease.sandboxGroupId,
246
+ expectedEpoch: lease.leaseEpoch,
247
+ warmRateMicrosPerSecond: rate,
248
+ subjectId: lease.sandboxGroupId,
249
+ });
250
+ if (result.accrued) {
251
+ accrued += 1;
252
+ }
253
+ } catch (error) {
254
+ observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
255
+ workspaceId: lease.workspaceId,
256
+ sandboxGroupId: lease.sandboxGroupId,
257
+ error: error instanceof Error ? error.message : String(error),
258
+ });
259
+ }
260
+ }
261
+ return { accrued, workspaceIds };
262
+ }
263
+
264
+ /**
265
+ * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
266
+ * live warm box, under the usage lock: if it is at 0 balance (when a billing /
267
+ * managed mode is on) or over its warm cap, force-drain its VIEWER-ONLY boxes
268
+ * (guarded turn_holders=0 — a paying turn is never killed). Returns the count of
269
+ * viewer-only boxes force-drained. Per-workspace best-effort.
270
+ */
271
+ async function forceDrainOverLimitWorkspaces(
272
+ db: ActivityServices["db"],
273
+ settings: ActivityServices["settings"],
274
+ workspaceIds: Set<string>,
275
+ observability: ActivityServices["observability"],
276
+ ): Promise<number> {
277
+ const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
278
+ const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
279
+ // Nothing to enforce → skip the whole pass (no lock churn).
280
+ if (!enforceBalance && cap <= 0) {
281
+ return 0;
282
+ }
283
+ let forceDrained = 0;
284
+ for (const workspaceId of workspaceIds) {
285
+ try {
286
+ const { accountId } = await rlsContextForWorkspace(db, workspaceId);
287
+ const balance = enforceBalance ? await getBillingBalance(db, accountId) : { balanceMicros: 1 } as { balanceMicros: number };
288
+ const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
289
+ workspaceId,
290
+ balanceMicros: balance.balanceMicros,
291
+ enforceBalance,
292
+ maxWarmSecondsPerWorkspace: cap,
293
+ idleGraceMs: settings.sandboxIdleGraceMs,
294
+ });
295
+ if (result.overLimit && result.drained.length > 0) {
296
+ forceDrained += result.drained.length;
297
+ observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
298
+ workspaceId,
299
+ reason: result.reason,
300
+ drained: result.drained.length,
301
+ });
302
+ }
303
+ } catch (error) {
304
+ observability.warn("sandbox reaper: force-drain check failed for workspace", {
305
+ workspaceId,
306
+ error: error instanceof Error ? error.message : String(error),
307
+ });
308
+ }
309
+ }
310
+ return forceDrained;
311
+ }
312
+
313
+ /**
314
+ * Terminate one drainable box by id, then CAS its lease draining->cold under the
315
+ * epoch fence. Returns true when the lease went cold (the box is ours to stop
316
+ * and was stopped), false when a concurrent sweep / re-arm / newer epoch means
317
+ * we must NOT stop it (provider terminate is skipped).
318
+ *
319
+ * The ordering is deliberately CAS-gated on BOTH ends:
320
+ * - BEFORE provider terminate: re-read the lease and assert it is STILL
321
+ * draining at refcount 0 at the SAME epoch we observed. A re-arm (a viewer
322
+ * or turn arrived during the grace window) flips it back to warm and bumps
323
+ * no epoch but changes liveness/refcount — we skip and never touch the box.
324
+ * - AFTER provider terminate: confirmDrainCold's CAS (draining AND refcount=0
325
+ * AND lease_epoch=expected) is the authoritative commit; if it returns false
326
+ * a late re-arm raced us between our re-read and the stop — but the box is
327
+ * already torn down, so we let the next acquire cold-restore it (NEVER a
328
+ * double-spawn: the lease is the singleton, the box is just gone).
329
+ */
330
+ async function terminateDrainableBox(
331
+ db: ActivityServices["db"],
332
+ settings: ActivityServices["settings"],
333
+ row: ReapDrainable,
334
+ observability: ActivityServices["observability"],
335
+ terminateBox: TerminateBoxFn,
336
+ ): Promise<boolean> {
337
+ // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
338
+ // returns no account_id; the workspace->account map is the bootstrap read).
339
+ const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
340
+
341
+ // Re-read the lease for the resume envelope AND the pre-terminate CAS guard.
342
+ const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
343
+ if (!lease) {
344
+ // Lease vanished (cold-reaped by a concurrent sweep). Nothing to stop.
345
+ return false;
346
+ }
347
+ if (
348
+ lease.liveness !== "draining"
349
+ || lease.refcount !== 0
350
+ || lease.leaseEpoch !== row.leaseEpoch
351
+ ) {
352
+ // Re-armed (warm again) / a newer epoch / already drained by a concurrent
353
+ // sweep. Skip — provider stop() must fire ONLY past the drain grace at
354
+ // refcount=0 on the box we observed, never while a turn or viewer holds it.
355
+ return false;
356
+ }
357
+
358
+ // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
359
+ // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
360
+ // Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
361
+ // expected): a re-arm or newer epoch that snuck in writes ZERO rows → wrote:false
362
+ // → the seam leaves the box RUNNING and we skip the cold-commit below.
363
+ const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) =>
364
+ await persistDrainSnapshot(db, {
365
+ accountId,
366
+ workspaceId: row.workspaceId,
367
+ sandboxGroupId: row.sandboxGroupId,
368
+ expectedEpoch: row.leaseEpoch,
369
+ workspaceArchive: archiveBase64,
370
+ });
371
+
372
+ // Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
373
+ // provider terminate. A box that is already gone (NotFound) is success (the goal
374
+ // is "box is not running"). A genuine PERSIST failure (could not snapshot) or a
375
+ // resume failure on a box that IS alive re-throws -> the lease stays draining for
376
+ // the next sweep (NEVER terminate a box whose files we could not capture). A
377
+ // persist CAS-miss (re-armed) returns false -> the box was left running; skip the
378
+ // cold-commit so the re-armed box is not cold-leaked.
379
+ const terminated = await terminateBox(settings, lease, observability, persistArchive);
380
+ if (!terminated) {
381
+ return false;
382
+ }
383
+
384
+ // The authoritative commit: CAS draining->cold under the epoch fence. If a
385
+ // late re-arm or newer epoch raced in after our re-read, wentCold:false and we
386
+ // report a skip (the box is down; a fresh acquire cold-restores it).
387
+ const { wentCold } = await confirmDrainCold(db, {
388
+ accountId,
389
+ workspaceId: row.workspaceId,
390
+ sandboxGroupId: row.sandboxGroupId,
391
+ expectedEpoch: row.leaseEpoch,
392
+ });
393
+ return wentCold;
394
+ }
395
+
396
+ // The persist-capable slice of a live provider session. persistWorkspace()
397
+ // snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
398
+ // archive) and returns the archive bytes. modal carries the Modal SDK client used
399
+ // for snapshot-image GC (deletePriorPersistedSnapshot reads session.modal).
400
+ type PersistableSession = TerminableSession & {
401
+ persistWorkspace?: () => Promise<Uint8Array | undefined>;
402
+ };
403
+
404
+ /**
405
+ * Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
406
+ * lease under the epoch fence, GC the superseded snapshot, THEN terminate. Resumes
407
+ * DIRECTLY (no cold-restore fallback) so a gone box is a clean no-op, never a
408
+ * wasteful create-then-kill:
409
+ * - build the client for the lease's backend;
410
+ * - deserialize + resume the envelope (warm reattach by id, R4-safe);
411
+ * - session.persistWorkspace() -> capture the /workspace archive;
412
+ * - persistArchive(base64) -> CAS-fold it onto the lease (epoch-fenced);
413
+ * - GC the PRIOR snapshot (keep-latest-per-lease);
414
+ * - terminate the live handle (client.delete / session kill|terminate|close).
415
+ *
416
+ * Returns true when the box was terminated (or was already gone), false when the
417
+ * persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.
418
+ *
419
+ * Failure discipline (NEVER lose files, NEVER leak a box):
420
+ * - resume NotFound -> the box is already down; SUCCESS (return true), nothing
421
+ * to persist, the caller colds the lease.
422
+ * - a transient/auth/network resume failure on a box that may be ALIVE -> throw
423
+ * (the caller skips; the lease stays draining for the next sweep).
424
+ * - a persistWorkspace failure (snapshot timeout / provider error) -> throw
425
+ * BEFORE any terminate, so the box is NOT torn down with un-captured files;
426
+ * the next sweep retries, and the provider idle-timeout is the ultimate
427
+ * backstop. We never terminate a box whose snapshot we could not capture.
428
+ * For a draining lease with no envelope (a warming-death row that committed no
429
+ * box, or a 'none'-backed group) there is no live box — a no-op (return true).
430
+ */
431
+ export async function terminateProviderBox(
432
+ settings: ActivityServices["settings"],
433
+ lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
434
+ observability: ActivityServices["observability"],
435
+ persistArchive: PersistArchiveFn,
436
+ createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
437
+ ): Promise<boolean> {
438
+ const backend = (lease.resumeBackendId ?? lease.backend) as string;
439
+ // 'none' / no backend -> nothing to terminate.
440
+ if (!backend || backend === "none") {
441
+ return true;
442
+ }
443
+
444
+ // CRITICAL SAFETY (bring-your-own-compute, dossier §19/§21): NEVER provider-stop
445
+ // a selfhosted box. The "box" is a user's PHYSICAL machine — you cannot kill it,
446
+ // and a delete()/kill() reaching the agent would be catastrophic. The reaper
447
+ // DRAINS the lease to cold only (refcount→0 → draining → this returns true so
448
+ // the caller's confirmDrainCold flips draining→cold) but issues NO provider stop:
449
+ // no client is built, no resume, no persistWorkspace (the machine IS the
450
+ // persistence — nothing to snapshot), no delete/kill. The session simply detaches;
451
+ // the machine stays up under the agent's own process lifetime (§23.0). Checked on
452
+ // BOTH the lease backend and the resume envelope's backend so neither path leaks.
453
+ if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
454
+ observability.info("sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)", {
455
+ sandboxGroupId: lease.sandboxGroupId,
456
+ backend,
457
+ });
458
+ return true;
459
+ }
460
+
461
+ // resume_state is the folded group box-envelope (the provider sessionState the
462
+ // box was last persisted as). No envelope -> no live box to stop.
463
+ if (!lease.resumeState) {
464
+ return true;
465
+ }
466
+
467
+ const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
468
+ if (!client) {
469
+ // 'none' backend resolved to no client.
470
+ return true;
471
+ }
472
+
473
+ // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
474
+ // already gone; success.
475
+ let session: PersistableSession | undefined;
476
+ let sessionState: unknown;
477
+ try {
478
+ if (!client.resume || !client.deserializeSessionState) {
479
+ // A backend that cannot resume cannot be attached-to for terminate; the
480
+ // provider idle-timeout is the backstop. No-op.
481
+ return true;
482
+ }
483
+ // resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
484
+ // providerState: { sandboxId, ... }, manifest, ... } }` (the shape
485
+ // serializeEstablishedSandboxEnvelope folds onto the lease). The provider
486
+ // payload deserializeSandboxSessionStateEnvelope re-hydrates is the INNER
487
+ // `sessionState` — pass the WHOLE envelope and it reads `state.providerState`
488
+ // at the top level, finds nothing (providerState is nested under
489
+ // `sessionState`), drops `sandboxId`, and `client.resume(state)` throws
490
+ // "requires a persisted sandboxId". This is exactly what the working
491
+ // resume-by-id paths (establishSandboxSessionFromEnvelope) avoid: they unwrap
492
+ // `envelope.sessionState` first. Mirror that. (`?? lease.resumeState` keeps a
493
+ // legacy flat envelope — providerState at the top — resumable too.)
494
+ const envelopeSessionState =
495
+ (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
496
+ const resumedState = await deserializeSandboxSessionStateEnvelope(client as never, envelopeSessionState);
497
+ if (resumedState === undefined) {
498
+ return true;
499
+ }
500
+ session = (await client.resume(resumedState)) as PersistableSession;
501
+ sessionState = resumedState;
502
+ } catch (error) {
503
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
504
+ observability.info("sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold", {
505
+ sandboxGroupId: lease.sandboxGroupId,
506
+ backend,
507
+ });
508
+ return true;
509
+ }
510
+ // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
511
+ // draining for the next sweep) — never cold a box we could not prove is gone.
512
+ throw error;
513
+ }
514
+
515
+ // PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to
516
+ // snapshot must re-throw BEFORE any terminate so files are never lost — the next
517
+ // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
518
+ // box raced gone between resume and persist) is success: nothing to persist.
519
+ let archiveBytes: Uint8Array | undefined;
520
+ try {
521
+ archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
522
+ } catch (error) {
523
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
524
+ observability.info("sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold", {
525
+ sandboxGroupId: lease.sandboxGroupId,
526
+ backend,
527
+ });
528
+ return true;
529
+ }
530
+ observability.warn("sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)", {
531
+ sandboxGroupId: lease.sandboxGroupId,
532
+ backend,
533
+ error: error instanceof Error ? error.message : String(error),
534
+ });
535
+ // NEVER terminate a box whose snapshot we could not capture.
536
+ throw error;
537
+ }
538
+
539
+ // Fold the captured archive onto the lease under the epoch fence, then GC the
540
+ // superseded snapshot. A CAS miss (wrote:false) means the lease was re-armed
541
+ // mid-drain: the box is wanted again — DO NOT terminate it (return false; the
542
+ // caller skips confirmDrainCold).
543
+ //
544
+ // Re-arm guard for no-archive path: even when persistWorkspace returned no bytes
545
+ // (a backend with no persistWorkspace, or an empty result), we MUST still CAS-
546
+ // check before delete(). The snapshot window (resume → persistWorkspace) can be
547
+ // long; a late acquireLease re-arm (draining→warm, same epoch) can land in it,
548
+ // so without this check we would delete a box the lease now treats as live. The
549
+ // null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
550
+ // refcount/epoch guard, no write. wrote:false → abort the terminate.
551
+ if (archiveBytes && archiveBytes.length > 0) {
552
+ const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
553
+ const { wrote, priorArchive } = await persistArchive(archiveBase64);
554
+ if (!wrote) {
555
+ observability.info("sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)", {
556
+ sandboxGroupId: lease.sandboxGroupId,
557
+ backend,
558
+ });
559
+ return false;
560
+ }
561
+ // Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
562
+ // while the session (and its Modal client) is still live. Never throws.
563
+ const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
564
+ if (deleted) {
565
+ observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
566
+ sandboxGroupId: lease.sandboxGroupId,
567
+ backend,
568
+ snapshotId: deleted,
569
+ });
570
+ }
571
+ } else {
572
+ // No archive produced (backend has no persistWorkspace or returned empty).
573
+ // CAS-check via persistArchive(null): if the lease was re-armed during the
574
+ // snapshot window, wrote:false → leave the box RUNNING (abort terminate).
575
+ const { wrote } = await persistArchive(null);
576
+ if (!wrote) {
577
+ observability.info("sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)", {
578
+ sandboxGroupId: lease.sandboxGroupId,
579
+ backend,
580
+ });
581
+ return false;
582
+ }
583
+ }
584
+
585
+ // Provider terminate. Prefer the client.delete(state) teardown (the canonical
586
+ // provider stop()); fall back to a session-level kill/terminate/close. A
587
+ // terminate that fails because the box is already gone is success.
588
+ try {
589
+ if (client.delete && sessionState !== undefined) {
590
+ await client.delete(sessionState);
591
+ return true;
592
+ }
593
+ if (session?.kill) {
594
+ await session.kill();
595
+ return true;
596
+ }
597
+ if (session?.terminate) {
598
+ await session.terminate();
599
+ return true;
600
+ }
601
+ if (session?.close && !session.closed) {
602
+ await session.close();
603
+ }
604
+ } catch (error) {
605
+ observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
606
+ sandboxGroupId: lease.sandboxGroupId,
607
+ backend,
608
+ error: error instanceof Error ? error.message : String(error),
609
+ });
610
+ }
611
+ return true;
612
+ }