@opengeni/worker-bundle 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7113 -1652
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4337 -1052
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
import {
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
|
+
appendSessionEventToSandboxGroup,
|
|
36
37
|
countQueuedTurns,
|
|
37
38
|
countSandboxLeasesByLiveness,
|
|
38
39
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
listMeterableWarmLeases,
|
|
43
44
|
persistDrainSnapshot,
|
|
44
45
|
readLease,
|
|
46
|
+
reapExpiredSessionListSnapshots,
|
|
45
47
|
reapStaleLeaseHoldersGlobal,
|
|
46
48
|
rlsContextForWorkspace,
|
|
47
49
|
type MeterableWarmLease,
|
|
@@ -63,6 +65,7 @@ import {
|
|
|
63
65
|
terminateModalSandboxById,
|
|
64
66
|
} from "@opengeni/runtime";
|
|
65
67
|
import type { ActivityServices } from "./types";
|
|
68
|
+
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
66
69
|
import {
|
|
67
70
|
recordCreditBalanceGauges,
|
|
68
71
|
recordCreditMicros,
|
|
@@ -120,16 +123,16 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
|
120
123
|
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
121
124
|
* - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
|
|
122
125
|
* is wanted again; the seam MUST NOT terminate it.
|
|
123
|
-
* - priorArchive ->
|
|
126
|
+
* - priorArchive / priorArchivePrev -> superseded archives to GC at drain.
|
|
124
127
|
*
|
|
125
128
|
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
126
129
|
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
127
|
-
* aborts the terminate before client.delete()). priorArchive
|
|
128
|
-
* this case.
|
|
130
|
+
* aborts the terminate before client.delete()). priorArchive values are always
|
|
131
|
+
* null in this case.
|
|
129
132
|
*/
|
|
130
133
|
export type PersistArchiveFn = (
|
|
131
134
|
archiveBase64: string | null,
|
|
132
|
-
) => Promise<{ wrote: boolean; priorArchive: string | null }>;
|
|
135
|
+
) => Promise<{ wrote: boolean; priorArchive: string | null; priorArchivePrev: string | null }>;
|
|
133
136
|
|
|
134
137
|
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
135
138
|
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
@@ -164,7 +167,8 @@ export function createSandboxLeaseActivities(
|
|
|
164
167
|
options: SandboxLeaseActivityOptions = {},
|
|
165
168
|
) {
|
|
166
169
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
167
|
-
const sweepModalOrphans: SweepModalOrphansFn =
|
|
170
|
+
const sweepModalOrphans: SweepModalOrphansFn =
|
|
171
|
+
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
168
172
|
/**
|
|
169
173
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
170
174
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -172,9 +176,39 @@ export function createSandboxLeaseActivities(
|
|
|
172
176
|
* with the flag off can never terminate a box.
|
|
173
177
|
*/
|
|
174
178
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
175
|
-
const
|
|
179
|
+
const service = await services();
|
|
180
|
+
const { db, settings, observability } = service;
|
|
181
|
+
const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch((error) => {
|
|
182
|
+
observability.warn("system-update outbox reconciliation failed", {
|
|
183
|
+
error: error instanceof Error ? error.message : String(error),
|
|
184
|
+
});
|
|
185
|
+
return { claimed: 0, delivered: 0, failed: 1 };
|
|
186
|
+
});
|
|
187
|
+
const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
|
|
188
|
+
(error) => {
|
|
189
|
+
observability.warn("session-list snapshot reconciliation failed", {
|
|
190
|
+
error: error instanceof Error ? error.message : String(error),
|
|
191
|
+
});
|
|
192
|
+
return 0;
|
|
193
|
+
},
|
|
194
|
+
);
|
|
195
|
+
if (expiredSessionListSnapshots > 0) {
|
|
196
|
+
observability.info("expired session-list snapshots reaped", {
|
|
197
|
+
deleted: expiredSessionListSnapshots,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
176
200
|
if (!settings.sandboxOwnershipEnabled) {
|
|
177
|
-
|
|
201
|
+
if (parentUpdates.claimed > 0) {
|
|
202
|
+
observability.info("system-update outbox reconciled", parentUpdates);
|
|
203
|
+
}
|
|
204
|
+
return {
|
|
205
|
+
examined: 0,
|
|
206
|
+
terminated: 0,
|
|
207
|
+
skipped: 0,
|
|
208
|
+
metered: 0,
|
|
209
|
+
forceDrained: 0,
|
|
210
|
+
modalOrphansTerminated: 0,
|
|
211
|
+
};
|
|
178
212
|
}
|
|
179
213
|
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
180
214
|
|
|
@@ -188,11 +222,26 @@ export function createSandboxLeaseActivities(
|
|
|
188
222
|
// workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
|
|
189
223
|
// boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
|
|
190
224
|
// draining rows are caught by the same sweep's terminate below.
|
|
191
|
-
const forceDrained = await forceDrainOverLimitWorkspaces(
|
|
225
|
+
const forceDrained = await forceDrainOverLimitWorkspaces(
|
|
226
|
+
db,
|
|
227
|
+
settings,
|
|
228
|
+
metered.workspaceIds,
|
|
229
|
+
observability,
|
|
230
|
+
);
|
|
192
231
|
|
|
193
232
|
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
194
233
|
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
195
234
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
235
|
+
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
236
|
+
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
237
|
+
// the whole warmup — waitForWarm/establish/display-stack — and the turn
|
|
238
|
+
// heartbeat covers the run), so NO live path is ever silent for more than
|
|
239
|
+
// one tick. The horizon is deliberately generous defense-in-depth (not a
|
|
240
|
+
// tuned guess about path lengths): a killed worker's frozen holder —
|
|
241
|
+
// which would otherwise pin refcount >= 1 FOREVER, so the lease never
|
|
242
|
+
// drains and the box dies at the provider hard-timeout UNPERSISTED —
|
|
243
|
+
// clears within ~12 minutes.
|
|
244
|
+
turnHolderTtlMs: settings.sandboxWarmingTimeoutMs + settings.sandboxLeaseTtlMs,
|
|
196
245
|
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
197
246
|
});
|
|
198
247
|
|
|
@@ -205,7 +254,13 @@ export function createSandboxLeaseActivities(
|
|
|
205
254
|
// next sweep retries it; the provider idle-timeout is the backstop).
|
|
206
255
|
for (const row of drainable) {
|
|
207
256
|
try {
|
|
208
|
-
const drainedCold = await terminateDrainableBox(
|
|
257
|
+
const drainedCold = await terminateDrainableBox(
|
|
258
|
+
db,
|
|
259
|
+
settings,
|
|
260
|
+
row,
|
|
261
|
+
observability,
|
|
262
|
+
terminateBox,
|
|
263
|
+
);
|
|
209
264
|
if (drainedCold) {
|
|
210
265
|
terminated += 1;
|
|
211
266
|
} else {
|
|
@@ -233,7 +288,12 @@ export function createSandboxLeaseActivities(
|
|
|
233
288
|
|
|
234
289
|
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
235
290
|
|
|
236
|
-
if (
|
|
291
|
+
if (
|
|
292
|
+
drainable.length > 0 ||
|
|
293
|
+
metered.accrued > 0 ||
|
|
294
|
+
forceDrained > 0 ||
|
|
295
|
+
modalOrphansTerminated > 0
|
|
296
|
+
) {
|
|
237
297
|
observability.info("sandbox reaper swept", {
|
|
238
298
|
drainable: drainable.length,
|
|
239
299
|
terminated,
|
|
@@ -244,7 +304,14 @@ export function createSandboxLeaseActivities(
|
|
|
244
304
|
});
|
|
245
305
|
}
|
|
246
306
|
|
|
247
|
-
return {
|
|
307
|
+
return {
|
|
308
|
+
examined: drainable.length,
|
|
309
|
+
terminated,
|
|
310
|
+
skipped,
|
|
311
|
+
metered: metered.accrued,
|
|
312
|
+
forceDrained,
|
|
313
|
+
modalOrphansTerminated,
|
|
314
|
+
};
|
|
248
315
|
}
|
|
249
316
|
|
|
250
317
|
return { reapSandboxLeases };
|
|
@@ -341,7 +408,8 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
341
408
|
workspaceIds: Set<string>,
|
|
342
409
|
observability: ActivityServices["observability"],
|
|
343
410
|
): Promise<number> {
|
|
344
|
-
const enforceBalance =
|
|
411
|
+
const enforceBalance =
|
|
412
|
+
settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
345
413
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
346
414
|
// Nothing to enforce → skip the whole pass (no lock churn).
|
|
347
415
|
if (!enforceBalance && cap <= 0) {
|
|
@@ -351,7 +419,9 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
351
419
|
for (const workspaceId of workspaceIds) {
|
|
352
420
|
try {
|
|
353
421
|
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
354
|
-
const balance = enforceBalance
|
|
422
|
+
const balance = enforceBalance
|
|
423
|
+
? await getBillingBalance(db, accountId)
|
|
424
|
+
: ({ balanceMicros: 1 } as { balanceMicros: number });
|
|
355
425
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
356
426
|
workspaceId,
|
|
357
427
|
balanceMicros: balance.balanceMicros,
|
|
@@ -452,9 +522,9 @@ async function terminateDrainableBox(
|
|
|
452
522
|
return false;
|
|
453
523
|
}
|
|
454
524
|
if (
|
|
455
|
-
lease.liveness !== "draining"
|
|
456
|
-
|
|
457
|
-
|
|
525
|
+
lease.liveness !== "draining" ||
|
|
526
|
+
lease.refcount !== 0 ||
|
|
527
|
+
lease.leaseEpoch !== row.leaseEpoch
|
|
458
528
|
) {
|
|
459
529
|
// Re-armed (warm again) / a newer epoch / already drained by a concurrent
|
|
460
530
|
// sweep. Skip — provider stop() must fire ONLY past the drain grace at
|
|
@@ -467,14 +537,23 @@ async function terminateDrainableBox(
|
|
|
467
537
|
// Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
|
|
468
538
|
// expected): a re-arm or newer epoch that snuck in writes ZERO rows → wrote:false
|
|
469
539
|
// → the seam leaves the box RUNNING and we skip the cold-commit below.
|
|
470
|
-
|
|
471
|
-
|
|
540
|
+
// `persisted` tracks whether a real archive landed on the lease this drain —
|
|
541
|
+
// the durable sandbox.box.terminated event below carries it, so a "terminated
|
|
542
|
+
// with NOTHING persisted" (box already dead at drain) is visible in the DB.
|
|
543
|
+
let persisted = false;
|
|
544
|
+
const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) => {
|
|
545
|
+
const result = await persistDrainSnapshot(db, {
|
|
472
546
|
accountId,
|
|
473
547
|
workspaceId: row.workspaceId,
|
|
474
548
|
sandboxGroupId: row.sandboxGroupId,
|
|
475
549
|
expectedEpoch: row.leaseEpoch,
|
|
476
550
|
workspaceArchive: archiveBase64,
|
|
477
551
|
});
|
|
552
|
+
if (result.wrote && archiveBase64 !== null) {
|
|
553
|
+
persisted = true;
|
|
554
|
+
}
|
|
555
|
+
return result;
|
|
556
|
+
};
|
|
478
557
|
|
|
479
558
|
// Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
|
|
480
559
|
// provider terminate. A box that is already gone (NotFound) is success (the goal
|
|
@@ -497,6 +576,26 @@ async function terminateDrainableBox(
|
|
|
497
576
|
sandboxGroupId: row.sandboxGroupId,
|
|
498
577
|
expectedEpoch: row.leaseEpoch,
|
|
499
578
|
});
|
|
579
|
+
if (wentCold) {
|
|
580
|
+
// Durable termination record (sandbox-file-persistence observability): who
|
|
581
|
+
// ended this box and whether its /workspace was captured first, appended to
|
|
582
|
+
// every session sharing the group's box. Best-effort: attribution must
|
|
583
|
+
// never affect the drain outcome.
|
|
584
|
+
try {
|
|
585
|
+
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
586
|
+
type: "sandbox.box.terminated",
|
|
587
|
+
payload: { actor: "reaper", persisted, instanceId: lease.instanceId },
|
|
588
|
+
});
|
|
589
|
+
} catch (eventError) {
|
|
590
|
+
observability.warn(
|
|
591
|
+
"sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
|
|
592
|
+
{
|
|
593
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
594
|
+
error: eventError instanceof Error ? eventError.message : String(eventError),
|
|
595
|
+
},
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
}
|
|
500
599
|
return wentCold;
|
|
501
600
|
}
|
|
502
601
|
|
|
@@ -557,11 +656,18 @@ export async function terminateProviderBox(
|
|
|
557
656
|
// persistence — nothing to snapshot), no delete/kill. The session simply detaches;
|
|
558
657
|
// the machine stays up under the agent's own process lifetime (§23.0). Checked on
|
|
559
658
|
// BOTH the lease backend and the resume envelope's backend so neither path leaks.
|
|
560
|
-
if (
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
659
|
+
if (
|
|
660
|
+
backend === "selfhosted" ||
|
|
661
|
+
lease.backend === "selfhosted" ||
|
|
662
|
+
lease.resumeBackendId === "selfhosted"
|
|
663
|
+
) {
|
|
664
|
+
observability.info(
|
|
665
|
+
"sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)",
|
|
666
|
+
{
|
|
667
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
668
|
+
backend,
|
|
669
|
+
},
|
|
670
|
+
);
|
|
565
671
|
return true;
|
|
566
672
|
}
|
|
567
673
|
|
|
@@ -572,11 +678,14 @@ export async function terminateProviderBox(
|
|
|
572
678
|
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
573
679
|
const { wrote } = await persistArchive(null);
|
|
574
680
|
if (!wrote) {
|
|
575
|
-
observability.info(
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
681
|
+
observability.info(
|
|
682
|
+
"sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING",
|
|
683
|
+
{
|
|
684
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
685
|
+
backend,
|
|
686
|
+
instanceId: lease.instanceId,
|
|
687
|
+
},
|
|
688
|
+
);
|
|
580
689
|
return false;
|
|
581
690
|
}
|
|
582
691
|
try {
|
|
@@ -629,7 +738,10 @@ export async function terminateProviderBox(
|
|
|
629
738
|
// legacy flat envelope — providerState at the top — resumable too.)
|
|
630
739
|
const envelopeSessionState =
|
|
631
740
|
(lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
|
|
632
|
-
const resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
741
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
742
|
+
client as never,
|
|
743
|
+
envelopeSessionState,
|
|
744
|
+
);
|
|
633
745
|
if (resumedState === undefined) {
|
|
634
746
|
return true;
|
|
635
747
|
}
|
|
@@ -637,10 +749,13 @@ export async function terminateProviderBox(
|
|
|
637
749
|
sessionState = resumedState;
|
|
638
750
|
} catch (error) {
|
|
639
751
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
640
|
-
observability.info(
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
752
|
+
observability.info(
|
|
753
|
+
"sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold",
|
|
754
|
+
{
|
|
755
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
756
|
+
backend,
|
|
757
|
+
},
|
|
758
|
+
);
|
|
644
759
|
return true;
|
|
645
760
|
}
|
|
646
761
|
// Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
|
|
@@ -657,17 +772,23 @@ export async function terminateProviderBox(
|
|
|
657
772
|
archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
|
|
658
773
|
} catch (error) {
|
|
659
774
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
660
|
-
observability.info(
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
775
|
+
observability.info(
|
|
776
|
+
"sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold",
|
|
777
|
+
{
|
|
778
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
779
|
+
backend,
|
|
780
|
+
},
|
|
781
|
+
);
|
|
664
782
|
return true;
|
|
665
783
|
}
|
|
666
|
-
observability.warn(
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
784
|
+
observability.warn(
|
|
785
|
+
"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
|
|
786
|
+
{
|
|
787
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
788
|
+
backend,
|
|
789
|
+
error: error instanceof Error ? error.message : String(error),
|
|
790
|
+
},
|
|
791
|
+
);
|
|
671
792
|
// NEVER terminate a box whose snapshot we could not capture.
|
|
672
793
|
throw error;
|
|
673
794
|
}
|
|
@@ -686,12 +807,15 @@ export async function terminateProviderBox(
|
|
|
686
807
|
// refcount/epoch guard, no write. wrote:false → abort the terminate.
|
|
687
808
|
if (archiveBytes && archiveBytes.length > 0) {
|
|
688
809
|
const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
|
|
689
|
-
const { wrote, priorArchive } = await persistArchive(archiveBase64);
|
|
810
|
+
const { wrote, priorArchive, priorArchivePrev } = await persistArchive(archiveBase64);
|
|
690
811
|
if (!wrote) {
|
|
691
|
-
observability.info(
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
812
|
+
observability.info(
|
|
813
|
+
"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
|
|
814
|
+
{
|
|
815
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
816
|
+
backend,
|
|
817
|
+
},
|
|
818
|
+
);
|
|
695
819
|
return false;
|
|
696
820
|
}
|
|
697
821
|
// Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
|
|
@@ -704,16 +828,27 @@ export async function terminateProviderBox(
|
|
|
704
828
|
snapshotId: deleted,
|
|
705
829
|
});
|
|
706
830
|
}
|
|
831
|
+
const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
|
|
832
|
+
if (deletedPrev) {
|
|
833
|
+
observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
|
|
834
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
835
|
+
backend,
|
|
836
|
+
snapshotId: deletedPrev,
|
|
837
|
+
});
|
|
838
|
+
}
|
|
707
839
|
} else {
|
|
708
840
|
// No archive produced (backend has no persistWorkspace or returned empty).
|
|
709
841
|
// CAS-check via persistArchive(null): if the lease was re-armed during the
|
|
710
842
|
// snapshot window, wrote:false → leave the box RUNNING (abort terminate).
|
|
711
843
|
const { wrote } = await persistArchive(null);
|
|
712
844
|
if (!wrote) {
|
|
713
|
-
observability.info(
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
845
|
+
observability.info(
|
|
846
|
+
"sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
|
|
847
|
+
{
|
|
848
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
849
|
+
backend,
|
|
850
|
+
},
|
|
851
|
+
);
|
|
717
852
|
return false;
|
|
718
853
|
}
|
|
719
854
|
}
|