@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -1,36 +1,21 @@
|
|
|
1
|
-
// apps/worker/src/activities/sandbox-lease.ts — the
|
|
2
|
-
//
|
|
1
|
+
// apps/worker/src/activities/sandbox-lease.ts — the sole global lease inventory,
|
|
2
|
+
// GC and cost-stop activity set (P1.3 / OD-3).
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
// turn's holder is immortal.
|
|
4
|
+
// One scheduled workflow inventories at most the DB reaper's fair 500-row batch,
|
|
5
|
+
// then starts one durable child per exact (workspace, group, epoch). Each child
|
|
6
|
+
// claims, snapshots and terminates only its own provider box. Temporal's control
|
|
7
|
+
// worker bounds real provider concurrency; a slow or dead provider cannot hold
|
|
8
|
+
// the global scan or any sibling drain. Activity heartbeats make worker loss
|
|
9
|
+
// visible promptly, while the DB capture id remains the authoritative teardown
|
|
10
|
+
// fence through provider termination and the final draining->cold commit.
|
|
12
11
|
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
// lease's resume envelope, via createSandboxClientForBackend +
|
|
21
|
-
// establishSandboxSessionFromEnvelope), call the provider terminate, then
|
|
22
|
-
// confirmDrainCold (the CAS draining->cold under the epoch fence).
|
|
23
|
-
//
|
|
24
|
-
// IDEMPOTENT + safe to run concurrently with itself: the drain CAS is guarded on
|
|
25
|
-
// (draining AND refcount=0 AND lease_epoch=expected). If another sweep already
|
|
26
|
-
// drained the row, or a late re-arm flipped it warm, or a newer epoch snuck in,
|
|
27
|
-
// confirmDrainCold returns wentCold:false and we skip — provider stop() fires
|
|
28
|
-
// ONLY when the CAS proves the box is still the draining box we observed. We
|
|
29
|
-
// confirm the CAS would still pass (re-read the lease) BEFORE the provider call
|
|
30
|
-
// so a box that was re-armed mid-sweep is never terminated out from under a live
|
|
31
|
-
// holder.
|
|
32
|
-
|
|
33
|
-
import { randomUUID } from "node:crypto";
|
|
12
|
+
// The legacy `reapSandboxLeases` composite remains only as a direct/embedded test
|
|
13
|
+
// harness and executes admitted drains concurrently. There is no per-session
|
|
14
|
+
// timer, viewer activity, owner task queue, or provider-specific lifecycle path
|
|
15
|
+
// in the normal drain state machine.
|
|
16
|
+
|
|
17
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
18
|
+
import { Context } from "@temporalio/activity";
|
|
34
19
|
import {
|
|
35
20
|
accrueWarmSeconds,
|
|
36
21
|
adoptLegacyModalCheckpointArtifact,
|
|
@@ -58,13 +43,13 @@ import {
|
|
|
58
43
|
pruneDeletedSandboxCheckpointArtifacts,
|
|
59
44
|
registerSandboxCheckpointArtifact,
|
|
60
45
|
recordRetainedProcessReconciliationProof,
|
|
61
|
-
|
|
62
|
-
replaceExpiredWorkspaceArchiveCapture,
|
|
46
|
+
replaceWorkspaceArchiveCaptureAfterProof,
|
|
63
47
|
readLease,
|
|
64
48
|
reapExpiredSessionListSnapshots,
|
|
65
49
|
reapStaleLeaseHoldersGlobal,
|
|
66
50
|
requestDueSandboxRotationsGlobal,
|
|
67
51
|
readSandboxRotationBacklog,
|
|
52
|
+
workspaceArchiveCaptureDeadlineElapsed,
|
|
68
53
|
retainedProcessReconciliationProof,
|
|
69
54
|
retainedProcessSettlementIdentity,
|
|
70
55
|
settleSandboxCheckpointArtifactGc,
|
|
@@ -76,7 +61,8 @@ import {
|
|
|
76
61
|
type SandboxRetainedProcess,
|
|
77
62
|
type LeaseSnapshot,
|
|
78
63
|
} from "@opengeni/db";
|
|
79
|
-
import {
|
|
64
|
+
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
65
|
+
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
80
66
|
import {
|
|
81
67
|
// Normal drain teardown builds the client and resumes the envelope directly:
|
|
82
68
|
// a live box gets its /workspace persisted before termination, while a gone
|
|
@@ -84,6 +70,7 @@ import {
|
|
|
84
70
|
// establishSandboxSessionFromEnvelope in strict resume-only mode so it can
|
|
85
71
|
// reuse the runtime's provider-ready proof without ever cold-restoring.
|
|
86
72
|
captureVerifiedWorkspaceArchive,
|
|
73
|
+
assertConsistentSandboxProviderIdentity,
|
|
87
74
|
createSandboxClientForBackend,
|
|
88
75
|
deleteModalCheckpointSnapshot,
|
|
89
76
|
deserializeSandboxSessionStateEnvelope,
|
|
@@ -92,20 +79,35 @@ import {
|
|
|
92
79
|
isExecSessionLostBanner,
|
|
93
80
|
isProviderSandboxNotFoundError,
|
|
94
81
|
parseExecBannerExitCode,
|
|
82
|
+
prepareProviderForTeardownAfterCapture,
|
|
83
|
+
providerWorkspaceCapturePolicy,
|
|
95
84
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
96
85
|
resolveModalCheckpointProviderBinding,
|
|
97
86
|
resolveModalCheckpointProviderBindingForSession,
|
|
87
|
+
resumeExactSandboxSession,
|
|
88
|
+
sandboxProviderContinuityForState,
|
|
98
89
|
sandboxBackendForSdkBackendId,
|
|
90
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
99
91
|
sandboxCommandExitCode,
|
|
100
92
|
sandboxCommandStillRunning,
|
|
101
93
|
sweepModalOrphanSandboxes,
|
|
94
|
+
terminateManagedSandboxSession,
|
|
102
95
|
terminateModalSandboxById,
|
|
103
96
|
verifySandboxExecReadiness,
|
|
104
97
|
type ModalOrphanSweepTermination,
|
|
105
98
|
type ModalCheckpointProviderBinding,
|
|
99
|
+
type ProviderWorkspaceCapturePolicy,
|
|
106
100
|
type WorkspaceArchiveDescriptor,
|
|
107
|
-
} from "@opengeni/runtime";
|
|
108
|
-
import
|
|
101
|
+
} from "@opengeni/runtime/sandbox";
|
|
102
|
+
import {
|
|
103
|
+
SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
|
|
104
|
+
SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
|
|
105
|
+
type SandboxDrainActivityInput,
|
|
106
|
+
type SandboxLeaseSweepMaintenanceInput,
|
|
107
|
+
} from "../sandbox-reaper-contract";
|
|
108
|
+
import { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from "../concurrency";
|
|
109
|
+
import { assertSandboxDrainInputTiming, sandboxDrainTiming } from "../sandbox-reaper-timeout";
|
|
110
|
+
import type { ControlActivityServices as ActivityServices } from "./types";
|
|
109
111
|
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
110
112
|
import {
|
|
111
113
|
recordCreditBalanceGauges,
|
|
@@ -127,6 +129,8 @@ import {
|
|
|
127
129
|
} from "../observability-metrics";
|
|
128
130
|
import { providerIdentityFromResumeState } from "../sandbox-routing";
|
|
129
131
|
|
|
132
|
+
export { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
133
|
+
|
|
130
134
|
export type ReapSandboxLeasesResult = {
|
|
131
135
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
132
136
|
* the DB pass; here we report the terminate outcomes. */
|
|
@@ -146,29 +150,15 @@ export type ReapSandboxLeasesResult = {
|
|
|
146
150
|
modalOrphansTerminated: number;
|
|
147
151
|
};
|
|
148
152
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
// stays agent-loop-free.
|
|
153
|
-
type TerminableClient = {
|
|
153
|
+
type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
154
|
+
|
|
155
|
+
type DrainSandboxClient = {
|
|
154
156
|
backendId: string;
|
|
155
157
|
resume?: (state: unknown) => Promise<unknown>;
|
|
156
158
|
deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
|
|
157
|
-
delete?: (state: unknown) => Promise<
|
|
158
|
-
};
|
|
159
|
-
|
|
160
|
-
// A live session handle may expose a kill/terminate/close itself (some providers
|
|
161
|
-
// tear the box down from the session, not the client). We try the client.delete
|
|
162
|
-
// first (the canonical teardown), then fall back to a session-level terminator.
|
|
163
|
-
type TerminableSession = {
|
|
164
|
-
kill?: () => Promise<unknown>;
|
|
165
|
-
terminate?: () => Promise<unknown>;
|
|
166
|
-
close?: () => Promise<unknown>;
|
|
167
|
-
closed?: boolean;
|
|
159
|
+
delete?: (state: unknown) => Promise<void>;
|
|
168
160
|
};
|
|
169
161
|
|
|
170
|
-
type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
171
|
-
|
|
172
162
|
/**
|
|
173
163
|
* The CAS-write the terminate seam calls to PERSIST the captured /workspace
|
|
174
164
|
* archive onto the lease under the epoch fence, BEFORE the box is terminated
|
|
@@ -203,6 +193,8 @@ export type ProviderTerminationOutcome = {
|
|
|
203
193
|
providerMissingBeforeCapture: boolean;
|
|
204
194
|
};
|
|
205
195
|
|
|
196
|
+
export type DrainCaptureDisposition = "capture_required" | "archive_published";
|
|
197
|
+
|
|
206
198
|
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
207
199
|
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
208
200
|
* provider stop() (`terminateProviderBox`); a unit test injects a spy so the
|
|
@@ -215,6 +207,9 @@ export type TerminateBoxFn = (
|
|
|
215
207
|
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
216
208
|
observability: ActivityServices["observability"],
|
|
217
209
|
persistArchive: PersistArchiveFn,
|
|
210
|
+
providerCaptureRequestId?: string,
|
|
211
|
+
captureDisposition?: DrainCaptureDisposition,
|
|
212
|
+
capturePolicy?: ProviderWorkspaceCapturePolicy | null,
|
|
218
213
|
) => Promise<boolean | ProviderTerminationOutcome>;
|
|
219
214
|
|
|
220
215
|
export type SweepModalOrphansFn = (
|
|
@@ -279,230 +274,539 @@ export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
|
279
274
|
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
280
275
|
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
281
276
|
|
|
277
|
+
export class SandboxProviderCaptureTimeoutError extends Error {
|
|
278
|
+
readonly name = "SandboxProviderCaptureTimeoutError";
|
|
279
|
+
|
|
280
|
+
constructor(
|
|
281
|
+
public readonly sandboxGroupId: string,
|
|
282
|
+
public readonly backend: string,
|
|
283
|
+
public readonly timeoutMs: number,
|
|
284
|
+
public readonly leaseEpoch: number,
|
|
285
|
+
public readonly instanceId: string,
|
|
286
|
+
) {
|
|
287
|
+
super(
|
|
288
|
+
`sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} ` +
|
|
289
|
+
`workspace capture exceeded ${timeoutMs}ms; ` +
|
|
290
|
+
"the exact durable claim remains fenced while its provider response settles",
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
async function awaitProviderCaptureWithLatePublication<T>(input: {
|
|
296
|
+
capture: Promise<T>;
|
|
297
|
+
timeoutMs: number;
|
|
298
|
+
timeoutError: SandboxProviderCaptureTimeoutError;
|
|
299
|
+
publishLate: (value: T) => Promise<void>;
|
|
300
|
+
observeLateFailure: (error: unknown) => void;
|
|
301
|
+
}): Promise<T> {
|
|
302
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
303
|
+
const timeoutError = input.timeoutError;
|
|
304
|
+
try {
|
|
305
|
+
return await Promise.race([
|
|
306
|
+
input.capture,
|
|
307
|
+
new Promise<never>((_, reject) => {
|
|
308
|
+
timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);
|
|
309
|
+
}),
|
|
310
|
+
]);
|
|
311
|
+
} catch (error) {
|
|
312
|
+
if (error !== timeoutError) throw error;
|
|
313
|
+
// Promise cancellation is not a generic SandboxSession contract. Keep the
|
|
314
|
+
// exact failed activity's result path alive in-process, but permit only the
|
|
315
|
+
// durable archive publication: never provider teardown or a cold commit from
|
|
316
|
+
// an activity Temporal has already rejected. A replaced claim fences this
|
|
317
|
+
// callback; Modal candidates then flow to artifact-ledger GC.
|
|
318
|
+
void input.capture.then(input.publishLate).catch(input.observeLateFailure);
|
|
319
|
+
throw error;
|
|
320
|
+
} finally {
|
|
321
|
+
if (timeout) clearTimeout(timeout);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function startSandboxReaperHeartbeat(details: Record<string, string | number | null>): () => void {
|
|
326
|
+
let context: Context;
|
|
327
|
+
try {
|
|
328
|
+
context = Context.current();
|
|
329
|
+
} catch {
|
|
330
|
+
// Direct unit/embedded harnesses intentionally run without Temporal.
|
|
331
|
+
return () => undefined;
|
|
332
|
+
}
|
|
333
|
+
const heartbeat = (): void => {
|
|
334
|
+
try {
|
|
335
|
+
context.heartbeat({ ...details, heartbeatAt: new Date().toISOString() });
|
|
336
|
+
} catch {
|
|
337
|
+
// Heartbeat can synchronously surface Temporal cancellation. This timer
|
|
338
|
+
// must never turn that into an uncaught process-level exception while an
|
|
339
|
+
// already-issued provider capture/teardown is settling under its durable
|
|
340
|
+
// DB claim. The activity result/cancellation boundary remains Temporal's;
|
|
341
|
+
// claim recovery remains the database's.
|
|
342
|
+
}
|
|
343
|
+
};
|
|
344
|
+
heartbeat();
|
|
345
|
+
const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
|
|
346
|
+
if ("unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
347
|
+
return () => clearInterval(timer);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export type SandboxLeaseSweepPlan = {
|
|
351
|
+
drainable: ReapDrainable[];
|
|
352
|
+
timeoutClass: SandboxDrainActivityInput["timeoutClass"];
|
|
353
|
+
snapshotTimeoutMs: number;
|
|
354
|
+
captureTimeoutMs: number;
|
|
355
|
+
/** Deadline rotations admitted immediately before inventory, so zero-holder
|
|
356
|
+
* boxes enter the same sweep instead of paying another schedule period. */
|
|
357
|
+
rotationsRequested: number;
|
|
358
|
+
};
|
|
359
|
+
|
|
360
|
+
export type SandboxLeaseSweepMaintenanceResult = {
|
|
361
|
+
metered: number;
|
|
362
|
+
forceDrained: number;
|
|
363
|
+
rotationsRequested: number;
|
|
364
|
+
modalOrphansTerminated: number;
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
export type SandboxDrainActivityResult = { status: "terminated" | "skipped" };
|
|
368
|
+
|
|
369
|
+
type SandboxDrainCaptureAttempt = {
|
|
370
|
+
operationId: string;
|
|
371
|
+
captureId: string;
|
|
372
|
+
attempt: number;
|
|
373
|
+
/** Durable child input, frozen when the workflow was created. Never
|
|
374
|
+
* re-derive this from a later deployment's config/formula. */
|
|
375
|
+
captureTimeoutMs: number;
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
class SandboxDrainRecoveryDeferredError extends Error {
|
|
379
|
+
readonly name = "SandboxDrainRecoveryDeferredError";
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Stable UUID per logical operation + Temporal attempt. Attempt one uses the
|
|
383
|
+
* operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.
|
|
384
|
+
* This makes the DB receipt idempotent even if an activity delivery is replayed. */
|
|
385
|
+
export function sandboxDrainCaptureId(operationId: string, attempt: number): string {
|
|
386
|
+
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {
|
|
387
|
+
throw new Error("Sandbox drain operation id is invalid");
|
|
388
|
+
}
|
|
389
|
+
if (!Number.isSafeInteger(attempt) || attempt <= 0) {
|
|
390
|
+
throw new Error("Sandbox drain activity attempt is invalid");
|
|
391
|
+
}
|
|
392
|
+
if (attempt === 1) return operationId.toLowerCase();
|
|
393
|
+
const bytes = createHash("sha256")
|
|
394
|
+
.update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`)
|
|
395
|
+
.digest()
|
|
396
|
+
.subarray(0, 16);
|
|
397
|
+
bytes[6] = (bytes[6]! & 0x0f) | 0x50;
|
|
398
|
+
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
|
399
|
+
const hex = bytes.toString("hex");
|
|
400
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function sandboxDrainActivityAttempt(): number {
|
|
404
|
+
try {
|
|
405
|
+
const attempt = Context.current().info.attempt;
|
|
406
|
+
return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
|
|
407
|
+
} catch {
|
|
408
|
+
return 1;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
282
412
|
export function createSandboxLeaseActivities(
|
|
283
413
|
services: () => Promise<ActivityServices>,
|
|
284
414
|
options: SandboxLeaseActivityOptions = {},
|
|
285
415
|
) {
|
|
286
|
-
const terminateBox: TerminateBoxFn =
|
|
416
|
+
const terminateBox: TerminateBoxFn =
|
|
417
|
+
options.terminateBox ??
|
|
418
|
+
(async (
|
|
419
|
+
settings,
|
|
420
|
+
lease,
|
|
421
|
+
observability,
|
|
422
|
+
persistArchive,
|
|
423
|
+
providerCaptureRequestId,
|
|
424
|
+
captureDisposition,
|
|
425
|
+
capturePolicy,
|
|
426
|
+
) =>
|
|
427
|
+
await terminateProviderBox(
|
|
428
|
+
settings,
|
|
429
|
+
lease,
|
|
430
|
+
observability,
|
|
431
|
+
persistArchive,
|
|
432
|
+
createSandboxClientForBackend,
|
|
433
|
+
terminateModalSandboxById,
|
|
434
|
+
providerCaptureRequestId,
|
|
435
|
+
captureDisposition,
|
|
436
|
+
capturePolicy,
|
|
437
|
+
));
|
|
287
438
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
288
439
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
289
440
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
290
441
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
442
|
+
async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
|
|
443
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
|
|
444
|
+
try {
|
|
445
|
+
const { db, settings, observability } = await services();
|
|
446
|
+
const timing = sandboxDrainTiming(settings);
|
|
447
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
448
|
+
return {
|
|
449
|
+
drainable: [],
|
|
450
|
+
timeoutClass: timing.timeoutClass,
|
|
451
|
+
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
452
|
+
captureTimeoutMs: timing.captureTimeoutMs,
|
|
453
|
+
rotationsRequested: 0,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Admit finite-lifetime rotations immediately before inventory. The DB
|
|
458
|
+
// function is bounded and provider-I/O-free; a due zero-holder box becomes
|
|
459
|
+
// draining there and is therefore dispatched in THIS sweep, eliminating
|
|
460
|
+
// the old extra schedule-period availability gap. Failure is isolated: the
|
|
461
|
+
// inventory must still dispatch every row that was already drainable.
|
|
462
|
+
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
463
|
+
db,
|
|
464
|
+
settings.sandboxRotationLeadMs,
|
|
465
|
+
settings.sandboxRotationBatchSize,
|
|
466
|
+
).catch((error) => {
|
|
467
|
+
observability.warn("sandbox reaper: provider-deadline rotation request failed", {
|
|
309
468
|
error: error instanceof Error ? error.message : String(error),
|
|
310
469
|
});
|
|
311
470
|
return 0;
|
|
312
|
-
},
|
|
313
|
-
);
|
|
314
|
-
if (expiredSessionListSnapshots > 0) {
|
|
315
|
-
observability.info("expired session-list snapshots reaped", {
|
|
316
|
-
deleted: expiredSessionListSnapshots,
|
|
317
471
|
});
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
472
|
+
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
473
|
+
|
|
474
|
+
// Keep the remaining dispatch-critical path to one bounded DB inventory.
|
|
475
|
+
// Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
|
|
476
|
+
// run only after every drainable box has its own durable child.
|
|
477
|
+
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
478
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
479
|
+
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
480
|
+
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
481
|
+
// the whole warmup — waitForWarm/establish/display-stack — and the turn
|
|
482
|
+
// heartbeat covers the run), so NO live path is ever silent for more than
|
|
483
|
+
// one tick. The ordinary warm lease TTL is therefore already a generous
|
|
484
|
+
// dead-worker horizon. Canonical holders whose turn is durably closed are
|
|
485
|
+
// removed immediately by the DB function; no path-duration guess belongs
|
|
486
|
+
// in this lifecycle contract.
|
|
487
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
488
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
489
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
490
|
+
});
|
|
491
|
+
const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
492
|
+
if (drainableInventory.length > drainable.length) {
|
|
493
|
+
observability.info("sandbox reaper: drain inventory continues next sweep", {
|
|
494
|
+
admitted: drainable.length,
|
|
495
|
+
deferred: drainableInventory.length - drainable.length,
|
|
496
|
+
});
|
|
327
497
|
}
|
|
328
498
|
return {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
modalOrphansTerminated: 0,
|
|
499
|
+
drainable,
|
|
500
|
+
timeoutClass: timing.timeoutClass,
|
|
501
|
+
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
502
|
+
captureTimeoutMs: timing.captureTimeoutMs,
|
|
503
|
+
rotationsRequested,
|
|
335
504
|
};
|
|
505
|
+
} finally {
|
|
506
|
+
stopHeartbeat();
|
|
336
507
|
}
|
|
508
|
+
}
|
|
337
509
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
510
|
+
async function drainSandboxLease(
|
|
511
|
+
input: SandboxDrainActivityInput,
|
|
512
|
+
): Promise<SandboxDrainActivityResult> {
|
|
513
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
514
|
+
phase: "drain",
|
|
515
|
+
workspaceId: input.target.workspaceId,
|
|
516
|
+
sandboxGroupId: input.target.sandboxGroupId,
|
|
517
|
+
leaseEpoch: input.target.leaseEpoch,
|
|
518
|
+
instanceId: input.target.instanceId,
|
|
347
519
|
});
|
|
348
|
-
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
349
|
-
|
|
350
|
-
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
351
|
-
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
352
|
-
// them). GROUP+epoch+tick idempotent → a shared box is one stream; an
|
|
353
|
-
// overlapping/re-fired sweep cannot double-charge. Best-effort per row.
|
|
354
|
-
const metered = await accrueWarmTick(db, settings, observability);
|
|
355
|
-
|
|
356
|
-
// (0b) Per-workspace warm-cap + force-drain (P2.1) — under the usage lock, a
|
|
357
|
-
// workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
|
|
358
|
-
// boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
|
|
359
|
-
// draining rows are caught by the same sweep's terminate below.
|
|
360
|
-
const forceDrainWorkspaceIds = new Set<string>();
|
|
361
|
-
if (
|
|
362
|
-
settings.billingMode === "stripe" ||
|
|
363
|
-
settings.usageLimitsMode === "managed" ||
|
|
364
|
-
settings.sandboxMaxWarmSecondsPerWorkspace > 0
|
|
365
|
-
) {
|
|
366
|
-
for (const workspaceId of metered.workspaceIds) {
|
|
367
|
-
forceDrainWorkspaceIds.add(workspaceId);
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
520
|
try {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
// (0c) Terminal-owner retained-process reconciliation. Owner state selects
|
|
387
|
-
// only a bounded inspection batch; exact provider exit/loss proof is
|
|
388
|
-
// checkpointed before canonical settlement. Ambiguous, running, timed-out,
|
|
389
|
-
// unsupported, and transient provider states preserve every durable row.
|
|
390
|
-
await reconcileTerminalRetainedProcesses(
|
|
391
|
-
db,
|
|
392
|
-
settings,
|
|
393
|
-
observability,
|
|
394
|
-
probeRetainedProcess,
|
|
395
|
-
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
396
|
-
);
|
|
397
|
-
|
|
398
|
-
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
399
|
-
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
400
|
-
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
401
|
-
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
402
|
-
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
403
|
-
// the whole warmup — waitForWarm/establish/display-stack — and the turn
|
|
404
|
-
// heartbeat covers the run), so NO live path is ever silent for more than
|
|
405
|
-
// one tick. The horizon is deliberately generous defense-in-depth (not a
|
|
406
|
-
// tuned guess about path lengths): a killed worker's frozen holder —
|
|
407
|
-
// which would otherwise pin refcount >= 1 FOREVER, so the lease never
|
|
408
|
-
// drains and the box dies at the provider hard-timeout UNPERSISTED —
|
|
409
|
-
// clears within ~12 minutes.
|
|
410
|
-
turnHolderTtlMs: settings.sandboxWarmingTimeoutMs + settings.sandboxLeaseTtlMs,
|
|
411
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
let terminated = 0;
|
|
415
|
-
let skipped = 0;
|
|
416
|
-
let modalOrphansTerminated = 0;
|
|
417
|
-
|
|
418
|
-
// (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
|
|
419
|
-
// are isolated: one box's provider error must not abort the whole sweep (the
|
|
420
|
-
// next sweep retries it; the provider idle-timeout is the backstop).
|
|
421
|
-
for (const row of drainable) {
|
|
521
|
+
const { db, settings, observability } = await services();
|
|
522
|
+
if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
|
|
523
|
+
assertSandboxDrainInputTiming(input);
|
|
524
|
+
const drainSettings =
|
|
525
|
+
settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
|
|
526
|
+
? settings
|
|
527
|
+
: { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
528
|
+
const row = input.target;
|
|
529
|
+
const activityAttempt = sandboxDrainActivityAttempt();
|
|
530
|
+
const captureAttempt: SandboxDrainCaptureAttempt = {
|
|
531
|
+
operationId: input.operationId,
|
|
532
|
+
captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),
|
|
533
|
+
attempt: activityAttempt,
|
|
534
|
+
captureTimeoutMs: input.captureTimeoutMs,
|
|
535
|
+
};
|
|
422
536
|
try {
|
|
423
537
|
const drainedCold = await terminateDrainableBox(
|
|
424
538
|
db,
|
|
425
|
-
|
|
539
|
+
drainSettings,
|
|
426
540
|
row,
|
|
427
541
|
observability,
|
|
428
542
|
terminateBox,
|
|
429
543
|
probeDrainableProvider,
|
|
544
|
+
captureAttempt,
|
|
430
545
|
);
|
|
431
|
-
|
|
432
|
-
terminated += 1;
|
|
433
|
-
} else {
|
|
434
|
-
skipped += 1;
|
|
435
|
-
}
|
|
546
|
+
return { status: drainedCold ? "terminated" : "skipped" };
|
|
436
547
|
} catch (error) {
|
|
437
|
-
|
|
438
|
-
|
|
548
|
+
const lease = await readLease(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
|
|
549
|
+
const details = {
|
|
550
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
|
|
439
551
|
workspaceId: row.workspaceId,
|
|
440
552
|
sandboxGroupId: row.sandboxGroupId,
|
|
441
553
|
leaseEpoch: row.leaseEpoch,
|
|
554
|
+
backend: lease?.backend ?? "unknown",
|
|
555
|
+
instanceId: lease?.instanceId ?? row.instanceId,
|
|
556
|
+
captureId: lease?.archiveCapture?.id ?? null,
|
|
557
|
+
captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,
|
|
558
|
+
captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,
|
|
559
|
+
timeoutClass: input.timeoutClass,
|
|
442
560
|
error: error instanceof Error ? error.message : String(error),
|
|
443
|
-
}
|
|
561
|
+
};
|
|
562
|
+
if (error instanceof SandboxDrainRecoveryDeferredError) {
|
|
563
|
+
observability.info("sandbox reaper: per-sandbox recovery deferred safely", details);
|
|
564
|
+
} else {
|
|
565
|
+
observability.warn("sandbox reaper: per-sandbox drain failed", details);
|
|
566
|
+
}
|
|
567
|
+
throw error;
|
|
444
568
|
}
|
|
569
|
+
} finally {
|
|
570
|
+
stopHeartbeat();
|
|
445
571
|
}
|
|
572
|
+
}
|
|
446
573
|
|
|
574
|
+
async function maintainSandboxLeaseSweep(
|
|
575
|
+
input: SandboxLeaseSweepMaintenanceInput,
|
|
576
|
+
): Promise<SandboxLeaseSweepMaintenanceResult> {
|
|
577
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
578
|
+
phase: "maintain",
|
|
579
|
+
examined: input.examined,
|
|
580
|
+
started: input.started,
|
|
581
|
+
});
|
|
447
582
|
try {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
583
|
+
const service = await services();
|
|
584
|
+
const { db, settings, observability } = service;
|
|
585
|
+
const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(
|
|
586
|
+
(error) => {
|
|
587
|
+
observability.warn("system-update outbox reconciliation failed", {
|
|
588
|
+
error: error instanceof Error ? error.message : String(error),
|
|
589
|
+
});
|
|
590
|
+
return { claimed: 0, delivered: 0, failed: 1 };
|
|
591
|
+
},
|
|
592
|
+
);
|
|
593
|
+
const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
|
|
594
|
+
(error) => {
|
|
595
|
+
observability.warn("session-list snapshot reconciliation failed", {
|
|
596
|
+
error: error instanceof Error ? error.message : String(error),
|
|
597
|
+
});
|
|
598
|
+
return 0;
|
|
599
|
+
},
|
|
600
|
+
);
|
|
601
|
+
if (expiredSessionListSnapshots > 0) {
|
|
602
|
+
observability.info("expired session-list snapshots reaped", {
|
|
603
|
+
deleted: expiredSessionListSnapshots,
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
if (parentUpdates.claimed > 0) {
|
|
607
|
+
observability.info("system-update outbox reconciled", parentUpdates);
|
|
608
|
+
}
|
|
455
609
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
610
|
+
let metered = 0;
|
|
611
|
+
let forceDrained = 0;
|
|
612
|
+
const rotationsRequested = input.rotationsRequested;
|
|
613
|
+
let modalOrphansTerminated = 0;
|
|
614
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
615
|
+
// Inventory remains useful while provider ownership is intentionally
|
|
616
|
+
// disabled, and this scheduled activity is its single projection owner.
|
|
617
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
618
|
+
return {
|
|
619
|
+
metered,
|
|
620
|
+
forceDrained,
|
|
621
|
+
rotationsRequested,
|
|
622
|
+
modalOrphansTerminated,
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Warm metering and cost-stop run after current drain children launch.
|
|
627
|
+
// Freshly force-drained billing rows are admission-fenced immediately and
|
|
628
|
+
// receive their own provider child on the next bounded schedule tick.
|
|
629
|
+
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
630
|
+
metered = meterResult.accrued;
|
|
631
|
+
const forceDrainWorkspaceIds = new Set<string>();
|
|
632
|
+
if (
|
|
633
|
+
settings.billingMode === "stripe" ||
|
|
634
|
+
settings.usageLimitsMode === "managed" ||
|
|
635
|
+
settings.sandboxMaxWarmSecondsPerWorkspace > 0
|
|
636
|
+
) {
|
|
637
|
+
for (const workspaceId of meterResult.workspaceIds) {
|
|
638
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
try {
|
|
642
|
+
for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
|
|
643
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
644
|
+
}
|
|
645
|
+
} catch (error) {
|
|
646
|
+
observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
|
|
459
647
|
error: error instanceof Error ? error.message : String(error),
|
|
460
648
|
});
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
649
|
+
}
|
|
650
|
+
forceDrained = await forceDrainOverLimitWorkspaces(
|
|
651
|
+
db,
|
|
652
|
+
settings,
|
|
653
|
+
forceDrainWorkspaceIds,
|
|
654
|
+
observability,
|
|
655
|
+
);
|
|
464
656
|
|
|
465
|
-
|
|
657
|
+
// Terminal-owner process reconciliation is ancillary to lease dispatch.
|
|
658
|
+
// Exact provider proof is durably checkpointed before settlement; every
|
|
659
|
+
// ambiguous outcome remains claimed/deferred for another sweep.
|
|
660
|
+
await reconcileTerminalRetainedProcesses(
|
|
661
|
+
db,
|
|
662
|
+
settings,
|
|
663
|
+
observability,
|
|
664
|
+
probeRetainedProcess,
|
|
665
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
666
|
+
);
|
|
466
667
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
668
|
+
try {
|
|
669
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
670
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
671
|
+
} catch (error) {
|
|
672
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
673
|
+
error: error instanceof Error ? error.message : String(error),
|
|
674
|
+
});
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
|
|
678
|
+
(error) => {
|
|
679
|
+
observability.warn("sandbox reaper: checkpoint artifact GC failed", {
|
|
680
|
+
error: error instanceof Error ? error.message : String(error),
|
|
681
|
+
});
|
|
682
|
+
return { claimed: 0, deleted: 0, failed: 0 };
|
|
683
|
+
},
|
|
684
|
+
);
|
|
685
|
+
|
|
686
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
687
|
+
|
|
688
|
+
if (
|
|
689
|
+
input.examined > 0 ||
|
|
690
|
+
metered > 0 ||
|
|
691
|
+
forceDrained > 0 ||
|
|
692
|
+
modalOrphansTerminated > 0 ||
|
|
693
|
+
rotationsRequested > 0 ||
|
|
694
|
+
checkpointGc.claimed > 0
|
|
695
|
+
) {
|
|
696
|
+
observability.info("sandbox reaper dispatched", {
|
|
697
|
+
drainable: input.examined,
|
|
698
|
+
childrenStarted: input.started,
|
|
699
|
+
childrenAlreadyRunning: input.alreadyRunning,
|
|
700
|
+
childrenStartFailed: input.startFailed,
|
|
701
|
+
metered,
|
|
702
|
+
forceDrained,
|
|
703
|
+
modalOrphansTerminated,
|
|
704
|
+
rotationsRequested,
|
|
705
|
+
checkpointArtifactsClaimed: checkpointGc.claimed,
|
|
706
|
+
checkpointArtifactsDeleted: checkpointGc.deleted,
|
|
707
|
+
checkpointArtifactsFailed: checkpointGc.failed,
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
return {
|
|
712
|
+
metered,
|
|
480
713
|
forceDrained,
|
|
481
|
-
modalOrphansTerminated,
|
|
482
714
|
rotationsRequested,
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
715
|
+
modalOrphansTerminated,
|
|
716
|
+
};
|
|
717
|
+
} finally {
|
|
718
|
+
stopHeartbeat();
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** Direct/embedded compatibility harness. Production uses the split Temporal
|
|
723
|
+
* workflow above. The production activity is also the rolling-deploy bridge:
|
|
724
|
+
* its workflow command stays byte-for-byte compatible while this activity
|
|
725
|
+
* starts V2 through the injected client edge. Old workers may still execute
|
|
726
|
+
* the bounded composite during rollout; exact DB claims keep both paths safe. */
|
|
727
|
+
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
728
|
+
const service = await services();
|
|
729
|
+
if (service.startSandboxReaperWorkflow) {
|
|
730
|
+
const dispatch = await service.startSandboxReaperWorkflow();
|
|
731
|
+
service.observability.info("sandbox reaper routed to versioned workflow", {
|
|
732
|
+
dispatch,
|
|
486
733
|
});
|
|
734
|
+
return {
|
|
735
|
+
examined: 0,
|
|
736
|
+
terminated: 0,
|
|
737
|
+
skipped: 0,
|
|
738
|
+
metered: 0,
|
|
739
|
+
forceDrained: 0,
|
|
740
|
+
modalOrphansTerminated: 0,
|
|
741
|
+
};
|
|
487
742
|
}
|
|
488
743
|
|
|
744
|
+
const plan = await prepareSandboxLeaseSweep();
|
|
745
|
+
const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);
|
|
746
|
+
const outcomes = await Promise.allSettled(
|
|
747
|
+
admitted.map((target) =>
|
|
748
|
+
drainSandboxLease({
|
|
749
|
+
target,
|
|
750
|
+
timeoutClass: plan.timeoutClass,
|
|
751
|
+
snapshotTimeoutMs: plan.snapshotTimeoutMs,
|
|
752
|
+
captureTimeoutMs: plan.captureTimeoutMs,
|
|
753
|
+
operationId: randomUUID(),
|
|
754
|
+
}),
|
|
755
|
+
),
|
|
756
|
+
);
|
|
757
|
+
const terminated = outcomes.filter(
|
|
758
|
+
(outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated",
|
|
759
|
+
).length;
|
|
760
|
+
const skipped = outcomes.length - terminated;
|
|
761
|
+
const maintenance = await maintainSandboxLeaseSweep({
|
|
762
|
+
examined: admitted.length,
|
|
763
|
+
started: admitted.length,
|
|
764
|
+
alreadyRunning: 0,
|
|
765
|
+
startFailed: 0,
|
|
766
|
+
rotationsRequested: plan.rotationsRequested,
|
|
767
|
+
});
|
|
489
768
|
return {
|
|
490
|
-
examined:
|
|
769
|
+
examined: admitted.length,
|
|
491
770
|
terminated,
|
|
492
771
|
skipped,
|
|
493
|
-
metered: metered
|
|
494
|
-
forceDrained,
|
|
495
|
-
modalOrphansTerminated,
|
|
772
|
+
metered: maintenance.metered,
|
|
773
|
+
forceDrained: maintenance.forceDrained,
|
|
774
|
+
modalOrphansTerminated: maintenance.modalOrphansTerminated,
|
|
496
775
|
};
|
|
497
776
|
}
|
|
498
777
|
|
|
499
|
-
return {
|
|
778
|
+
return {
|
|
779
|
+
prepareSandboxLeaseSweep,
|
|
780
|
+
drainSandboxLease,
|
|
781
|
+
maintainSandboxLeaseSweep,
|
|
782
|
+
reapSandboxLeases,
|
|
783
|
+
};
|
|
500
784
|
}
|
|
501
785
|
|
|
502
786
|
const CHECKPOINT_GC_LIMIT = 50;
|
|
503
787
|
const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
|
|
504
788
|
const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
505
789
|
const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
790
|
+
const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
791
|
+
|
|
792
|
+
async function forEachWithConcurrency<T>(
|
|
793
|
+
values: readonly T[],
|
|
794
|
+
concurrency: number,
|
|
795
|
+
visit: (value: T, index: number) => Promise<void>,
|
|
796
|
+
): Promise<void> {
|
|
797
|
+
if (values.length === 0) return;
|
|
798
|
+
const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
|
|
799
|
+
let nextIndex = 0;
|
|
800
|
+
await Promise.all(
|
|
801
|
+
Array.from({ length: workerCount }, async () => {
|
|
802
|
+
while (nextIndex < values.length) {
|
|
803
|
+
const index = nextIndex;
|
|
804
|
+
nextIndex += 1;
|
|
805
|
+
await visit(values[index]!, index);
|
|
806
|
+
}
|
|
807
|
+
}),
|
|
808
|
+
);
|
|
809
|
+
}
|
|
506
810
|
|
|
507
811
|
async function gcSandboxCheckpointArtifacts(
|
|
508
812
|
db: ActivityServices["db"],
|
|
@@ -533,7 +837,7 @@ async function gcSandboxCheckpointArtifacts(
|
|
|
533
837
|
recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
|
|
534
838
|
let deleted = 0;
|
|
535
839
|
let failed = 0;
|
|
536
|
-
|
|
840
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
537
841
|
try {
|
|
538
842
|
if (claim.providerBackend !== "modal") {
|
|
539
843
|
throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
|
|
@@ -571,7 +875,7 @@ async function gcSandboxCheckpointArtifacts(
|
|
|
571
875
|
error: error instanceof Error ? error.message : String(error),
|
|
572
876
|
});
|
|
573
877
|
}
|
|
574
|
-
}
|
|
878
|
+
});
|
|
575
879
|
const pruned = await pruneDeletedSandboxCheckpointArtifacts(
|
|
576
880
|
db,
|
|
577
881
|
CHECKPOINT_TOMBSTONE_RETENTION_MS,
|
|
@@ -593,7 +897,7 @@ async function adoptLegacyModalCheckpointReceipts(
|
|
|
593
897
|
Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>
|
|
594
898
|
>();
|
|
595
899
|
let adopted = 0;
|
|
596
|
-
|
|
900
|
+
await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {
|
|
597
901
|
try {
|
|
598
902
|
let binding = bindings.get(slot.instanceId);
|
|
599
903
|
if (!binding) {
|
|
@@ -626,7 +930,7 @@ async function adoptLegacyModalCheckpointReceipts(
|
|
|
626
930
|
error: error instanceof Error ? error.message : String(error),
|
|
627
931
|
});
|
|
628
932
|
}
|
|
629
|
-
}
|
|
933
|
+
});
|
|
630
934
|
return adopted;
|
|
631
935
|
}
|
|
632
936
|
|
|
@@ -674,7 +978,10 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
674
978
|
inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),
|
|
675
979
|
);
|
|
676
980
|
if (lifecycle.status === "not_found") {
|
|
677
|
-
observation = {
|
|
981
|
+
observation = {
|
|
982
|
+
status: "deferred",
|
|
983
|
+
reason: "provider_binding_missing",
|
|
984
|
+
};
|
|
678
985
|
} else {
|
|
679
986
|
process = await bindRetainedProcessProviderIdentity(db, {
|
|
680
987
|
accountId: process.accountId,
|
|
@@ -997,6 +1304,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
997
1304
|
resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
998
1305
|
client as never,
|
|
999
1306
|
envelopeSessionState,
|
|
1307
|
+
process.providerInstanceId,
|
|
1000
1308
|
);
|
|
1001
1309
|
} catch {
|
|
1002
1310
|
return { status: "deferred", reason: "provider_error" };
|
|
@@ -1007,9 +1315,15 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1007
1315
|
|
|
1008
1316
|
let session: RetainedProcessProbeSession;
|
|
1009
1317
|
try {
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1318
|
+
const resumed = await withRetainedProcessProbeTimeout(
|
|
1319
|
+
resumeExactSandboxSession(
|
|
1320
|
+
client,
|
|
1321
|
+
process.providerBackend,
|
|
1322
|
+
resumedState,
|
|
1323
|
+
process.providerInstanceId,
|
|
1324
|
+
),
|
|
1325
|
+
);
|
|
1326
|
+
session = resumed.session as RetainedProcessProbeSession;
|
|
1013
1327
|
} catch (error) {
|
|
1014
1328
|
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
1015
1329
|
return { status: "deferred", reason: "provider_timeout" };
|
|
@@ -1190,6 +1504,8 @@ async function accrueWarmTick(
|
|
|
1190
1504
|
}
|
|
1191
1505
|
for (const lease of leases) {
|
|
1192
1506
|
workspaceIds.add(lease.workspaceId);
|
|
1507
|
+
}
|
|
1508
|
+
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
1193
1509
|
try {
|
|
1194
1510
|
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
1195
1511
|
const result = await accrueWarmSeconds(db, {
|
|
@@ -1211,7 +1527,7 @@ async function accrueWarmTick(
|
|
|
1211
1527
|
error: error instanceof Error ? error.message : String(error),
|
|
1212
1528
|
});
|
|
1213
1529
|
}
|
|
1214
|
-
}
|
|
1530
|
+
});
|
|
1215
1531
|
return { accrued, workspaceIds };
|
|
1216
1532
|
}
|
|
1217
1533
|
|
|
@@ -1219,64 +1535,65 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
1219
1535
|
db: ActivityServices["db"],
|
|
1220
1536
|
observability: ActivityServices["observability"],
|
|
1221
1537
|
): Promise<void> {
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
"expired-draining",
|
|
1266
|
-
async () => {
|
|
1538
|
+
await Promise.all([
|
|
1539
|
+
(async () => {
|
|
1540
|
+
try {
|
|
1541
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
1542
|
+
} catch (error) {
|
|
1543
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
1544
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
})(),
|
|
1548
|
+
refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
1549
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
1550
|
+
}),
|
|
1551
|
+
refreshSandboxInventoryGauge(
|
|
1552
|
+
observability,
|
|
1553
|
+
"checkpoint_artifacts",
|
|
1554
|
+
"checkpoint-artifact",
|
|
1555
|
+
async () => {
|
|
1556
|
+
recordSandboxCheckpointArtifactGauges(
|
|
1557
|
+
observability,
|
|
1558
|
+
await countSandboxCheckpointArtifactsByState(db),
|
|
1559
|
+
);
|
|
1560
|
+
},
|
|
1561
|
+
),
|
|
1562
|
+
refreshSandboxInventoryGauge(
|
|
1563
|
+
observability,
|
|
1564
|
+
"rotation_backlog",
|
|
1565
|
+
"rotation-backlog",
|
|
1566
|
+
async () => {
|
|
1567
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
1568
|
+
},
|
|
1569
|
+
),
|
|
1570
|
+
refreshSandboxInventoryGauge(
|
|
1571
|
+
observability,
|
|
1572
|
+
"retained_processes",
|
|
1573
|
+
"retained-process",
|
|
1574
|
+
async () => {
|
|
1575
|
+
recordRetainedProcessInventoryGauges(
|
|
1576
|
+
observability,
|
|
1577
|
+
await countActiveRetainedProcessesByOwnerState(db),
|
|
1578
|
+
);
|
|
1579
|
+
},
|
|
1580
|
+
),
|
|
1581
|
+
refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
|
|
1267
1582
|
recordExpiredDrainingSandboxLeaseGauges(
|
|
1268
1583
|
observability,
|
|
1269
1584
|
await countExpiredDrainingSandboxLeases(db),
|
|
1270
1585
|
);
|
|
1271
|
-
},
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1586
|
+
}),
|
|
1587
|
+
(async () => {
|
|
1588
|
+
try {
|
|
1589
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
1590
|
+
} catch (error) {
|
|
1591
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
1592
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1593
|
+
});
|
|
1594
|
+
}
|
|
1595
|
+
})(),
|
|
1596
|
+
]);
|
|
1280
1597
|
}
|
|
1281
1598
|
|
|
1282
1599
|
async function refreshSandboxInventoryGauge(
|
|
@@ -1313,34 +1630,38 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
1313
1630
|
settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
1314
1631
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
1315
1632
|
let forceDrained = 0;
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
maxWarmSecondsPerWorkspace: cap,
|
|
1327
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1328
|
-
});
|
|
1329
|
-
if (result.overLimit && result.drained.length > 0) {
|
|
1330
|
-
forceDrained += result.drained.length;
|
|
1331
|
-
observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
|
|
1633
|
+
await forEachWithConcurrency(
|
|
1634
|
+
[...workspaceIds],
|
|
1635
|
+
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
1636
|
+
async (workspaceId) => {
|
|
1637
|
+
try {
|
|
1638
|
+
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
1639
|
+
const balance = enforceBalance
|
|
1640
|
+
? await getBillingBalance(db, accountId)
|
|
1641
|
+
: ({ balanceMicros: 1 } as { balanceMicros: number });
|
|
1642
|
+
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
1332
1643
|
workspaceId,
|
|
1333
|
-
|
|
1334
|
-
|
|
1644
|
+
balanceMicros: balance.balanceMicros,
|
|
1645
|
+
enforceBalance,
|
|
1646
|
+
maxWarmSecondsPerWorkspace: cap,
|
|
1647
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1648
|
+
});
|
|
1649
|
+
if (result.overLimit && result.drained.length > 0) {
|
|
1650
|
+
forceDrained += result.drained.length;
|
|
1651
|
+
observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
|
|
1652
|
+
workspaceId,
|
|
1653
|
+
reason: result.reason,
|
|
1654
|
+
drained: result.drained.length,
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
} catch (error) {
|
|
1658
|
+
observability.warn("sandbox reaper: force-drain check failed for workspace", {
|
|
1659
|
+
workspaceId,
|
|
1660
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1335
1661
|
});
|
|
1336
1662
|
}
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
workspaceId,
|
|
1340
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1341
|
-
});
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1663
|
+
},
|
|
1664
|
+
);
|
|
1344
1665
|
return forceDrained;
|
|
1345
1666
|
}
|
|
1346
1667
|
|
|
@@ -1351,6 +1672,7 @@ export function modalOrphanTerminationStillEligible(
|
|
|
1351
1672
|
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
1352
1673
|
return false;
|
|
1353
1674
|
}
|
|
1675
|
+
|
|
1354
1676
|
const leaseId = candidate.tags.opengeni_lease_id;
|
|
1355
1677
|
const workspaceId = candidate.tags.opengeni_workspace_id;
|
|
1356
1678
|
const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
|
|
@@ -1418,9 +1740,10 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
1418
1740
|
* A capture claim whose owning worker disappeared must not fence a workspace
|
|
1419
1741
|
* forever. The drain reaper reaches this seam only after every holder is gone.
|
|
1420
1742
|
* It resumes the exact attributed provider without replacement and proves that
|
|
1421
|
-
* a no-op command can execute.
|
|
1422
|
-
*
|
|
1423
|
-
*
|
|
1743
|
+
* a no-op command can execute. This distinguishes typed provider loss from a
|
|
1744
|
+
* still-addressable instance; it does not prove that an old capture RPC has
|
|
1745
|
+
* stopped. The caller applies the provider's durable takeover policy and the
|
|
1746
|
+
* claim deadline. Ambiguity preserves the old claim and therefore the files.
|
|
1424
1747
|
*/
|
|
1425
1748
|
async function probeDrainableProviderReadiness(
|
|
1426
1749
|
settings: ActivityServices["settings"],
|
|
@@ -1466,7 +1789,11 @@ async function probeDrainableProviderReadiness(
|
|
|
1466
1789
|
if (!run) throw new Error("Expired workspace capture provider has no readiness command");
|
|
1467
1790
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1468
1791
|
const result = await Promise.race([
|
|
1469
|
-
run.call(session, {
|
|
1792
|
+
run.call(session, {
|
|
1793
|
+
cmd: "true",
|
|
1794
|
+
yieldTimeMs: 1_000,
|
|
1795
|
+
maxOutputTokens: 1_000,
|
|
1796
|
+
}),
|
|
1470
1797
|
new Promise<never>((_, reject) => {
|
|
1471
1798
|
timer = setTimeout(
|
|
1472
1799
|
() => reject(new Error("Expired workspace capture readiness probe timed out")),
|
|
@@ -1492,16 +1819,15 @@ async function probeDrainableProviderReadiness(
|
|
|
1492
1819
|
* and was stopped), false when a concurrent sweep / re-arm / newer epoch means
|
|
1493
1820
|
* we must NOT stop it (provider terminate is skipped).
|
|
1494
1821
|
*
|
|
1495
|
-
* The ordering is deliberately
|
|
1496
|
-
* - BEFORE provider
|
|
1497
|
-
*
|
|
1498
|
-
*
|
|
1499
|
-
*
|
|
1500
|
-
* - AFTER provider terminate: confirmDrainCold
|
|
1501
|
-
*
|
|
1502
|
-
*
|
|
1503
|
-
*
|
|
1504
|
-
* double-spawn: the lease is the singleton, the box is just gone).
|
|
1822
|
+
* The ordering is deliberately receipt-gated on BOTH ends:
|
|
1823
|
+
* - BEFORE provider I/O: re-read the exact draining epoch and acquire/retain
|
|
1824
|
+
* its durable capture/teardown claim. Before that claim an arrival may
|
|
1825
|
+
* re-arm for availability; afterward every arrival waits and cannot race
|
|
1826
|
+
* provider termination.
|
|
1827
|
+
* - AFTER provider terminate: confirmDrainCold commits only the exact current
|
|
1828
|
+
* capture id together with draining->cold. A miss means a recovery attempt
|
|
1829
|
+
* or epoch already replaced this callback; it says nothing about provider
|
|
1830
|
+
* liveness, so only that durable successor may reconcile and commit cold.
|
|
1505
1831
|
*/
|
|
1506
1832
|
async function terminateDrainableBox(
|
|
1507
1833
|
db: ActivityServices["db"],
|
|
@@ -1510,6 +1836,7 @@ async function terminateDrainableBox(
|
|
|
1510
1836
|
observability: ActivityServices["observability"],
|
|
1511
1837
|
terminateBox: TerminateBoxFn,
|
|
1512
1838
|
probeDrainableProvider: DrainableProviderProbeFn,
|
|
1839
|
+
attempt: SandboxDrainCaptureAttempt,
|
|
1513
1840
|
): Promise<boolean> {
|
|
1514
1841
|
// Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
|
|
1515
1842
|
// returns no account_id; the workspace->account map is the bootstrap read).
|
|
@@ -1532,60 +1859,186 @@ async function terminateDrainableBox(
|
|
|
1532
1859
|
return false;
|
|
1533
1860
|
}
|
|
1534
1861
|
|
|
1535
|
-
const
|
|
1862
|
+
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1863
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
1864
|
+
const managedProvider = backend !== "none" && backend !== "selfhosted";
|
|
1865
|
+
const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);
|
|
1866
|
+
if (managedProvider && !capturePolicy) {
|
|
1867
|
+
throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
|
|
1868
|
+
}
|
|
1869
|
+
if (capturePolicy?.liveInstance === "replaced") {
|
|
1870
|
+
throw new Error(
|
|
1871
|
+
`sandbox backend ${backend} workspace capture would replace the fenced live instance`,
|
|
1872
|
+
);
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
const captureTimeoutMs = attempt.captureTimeoutMs;
|
|
1536
1876
|
let captureClaim:
|
|
1537
1877
|
| NonNullable<
|
|
1538
1878
|
Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
|
|
1539
1879
|
>["claim"]
|
|
1540
1880
|
| null = null;
|
|
1541
1881
|
let providerMissingBeforeCapture = false;
|
|
1542
|
-
|
|
1882
|
+
let captureDisposition: DrainCaptureDisposition = "capture_required";
|
|
1883
|
+
if (lease.instanceId && managedProvider) {
|
|
1543
1884
|
if (lease.archiveCapture) {
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
// the attributed provider is command-ready, so preserve the gate.
|
|
1552
|
-
if (!lease.resumeState) return false;
|
|
1553
|
-
const providerState = await probeDrainableProvider(settings, lease);
|
|
1554
|
-
if (providerState === "missing") {
|
|
1555
|
-
providerMissingBeforeCapture = true;
|
|
1885
|
+
const priorCapture = lease.archiveCapture;
|
|
1886
|
+
const instanceId = lease.instanceId;
|
|
1887
|
+
if (priorCapture.publishedAt) {
|
|
1888
|
+
// Publication is an irreversible durable phase transition. The archive
|
|
1889
|
+
// already represents this exact fenced generation, so a crash/retry
|
|
1890
|
+
// resumes teardown immediately: no deadline wait, provider probe, claim
|
|
1891
|
+
// replacement, or duplicate snapshot request.
|
|
1556
1892
|
captureClaim = {
|
|
1557
|
-
...
|
|
1893
|
+
...priorCapture,
|
|
1558
1894
|
leaseId: lease.id,
|
|
1559
1895
|
leaseEpoch: lease.leaseEpoch,
|
|
1560
|
-
instanceId
|
|
1896
|
+
instanceId,
|
|
1561
1897
|
archiveGeneration: lease.archiveGeneration,
|
|
1562
1898
|
archiveComplete: lease.archiveComplete,
|
|
1563
1899
|
};
|
|
1900
|
+
captureDisposition = "archive_published";
|
|
1564
1901
|
} else {
|
|
1565
|
-
|
|
1902
|
+
if (
|
|
1903
|
+
priorCapture.operationId === attempt.operationId &&
|
|
1904
|
+
priorCapture.attempt >= attempt.attempt
|
|
1905
|
+
) {
|
|
1906
|
+
// Never let an accepted duplicate/stale delivery complete the durable
|
|
1907
|
+
// child while its exact claim still fences the lease. Throwing asks
|
|
1908
|
+
// Temporal for a strictly newer attempt; an actually stale task token is
|
|
1909
|
+
// rejected by Temporal and cannot affect the workflow either way.
|
|
1910
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1911
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`,
|
|
1912
|
+
);
|
|
1913
|
+
}
|
|
1914
|
+
const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {
|
|
1566
1915
|
accountId,
|
|
1567
1916
|
workspaceId: row.workspaceId,
|
|
1568
1917
|
sandboxGroupId: row.sandboxGroupId,
|
|
1569
|
-
|
|
1570
|
-
captureId: randomUUID(),
|
|
1918
|
+
captureId: priorCapture.id,
|
|
1571
1919
|
expectedEpoch: row.leaseEpoch,
|
|
1572
|
-
expectedInstanceId:
|
|
1573
|
-
captureTimeoutMs,
|
|
1920
|
+
expectedInstanceId: instanceId,
|
|
1574
1921
|
});
|
|
1575
|
-
|
|
1576
|
-
|
|
1922
|
+
const unpublishedWarmingProvider =
|
|
1923
|
+
backend === "modal" &&
|
|
1924
|
+
!sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) &&
|
|
1925
|
+
lease.recovery.provider.status === "creating" &&
|
|
1926
|
+
lease.recovery.provider.instanceId === instanceId &&
|
|
1927
|
+
lease.recovery.workspace.status === "not_ready";
|
|
1928
|
+
// Temporal knows that the prior activity attempt is no longer accepted,
|
|
1929
|
+
// but it cannot generically cancel provider I/O. Immediate replacement is
|
|
1930
|
+
// legal only when BOTH the durable claim and the current adapter declare
|
|
1931
|
+
// takeover safety: either the same idempotent request, or an independently
|
|
1932
|
+
// repeatable read-only capture. A readiness probe alone never proves an
|
|
1933
|
+
// unrelated exclusive snapshot RPC has settled.
|
|
1934
|
+
const replace = async () =>
|
|
1935
|
+
await replaceWorkspaceArchiveCaptureAfterProof(db, {
|
|
1936
|
+
accountId,
|
|
1937
|
+
workspaceId: row.workspaceId,
|
|
1938
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
1939
|
+
priorCaptureId: priorCapture.id,
|
|
1940
|
+
captureId: attempt.captureId,
|
|
1941
|
+
operationId: attempt.operationId,
|
|
1942
|
+
attempt: attempt.attempt,
|
|
1943
|
+
expectedEpoch: row.leaseEpoch,
|
|
1944
|
+
expectedInstanceId: instanceId,
|
|
1945
|
+
captureTimeoutMs,
|
|
1946
|
+
});
|
|
1947
|
+
|
|
1948
|
+
// If no durable complete archive exists, replacement is safe only when
|
|
1949
|
+
// this exact provider can be recaptured. A typed missing provider before
|
|
1950
|
+
// the old deadline may mean the dead worker still holds the sole archive
|
|
1951
|
+
// bytes in memory, so preserve its callback until that bounded window.
|
|
1952
|
+
const currentTakeoverSafe =
|
|
1953
|
+
capturePolicy !== null && capturePolicy.takeover !== "exclusive";
|
|
1954
|
+
if (!lease.resumeState) {
|
|
1955
|
+
if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {
|
|
1956
|
+
const replacement = await replace();
|
|
1957
|
+
if (!replacement) {
|
|
1958
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1959
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`,
|
|
1960
|
+
);
|
|
1961
|
+
}
|
|
1962
|
+
captureClaim = replacement;
|
|
1963
|
+
} else {
|
|
1964
|
+
if (!deadlineExpired) {
|
|
1965
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1966
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,
|
|
1967
|
+
);
|
|
1968
|
+
}
|
|
1969
|
+
if (!unpublishedWarmingProvider) {
|
|
1970
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1971
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`,
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
// Legacy/unrelated owner on a never-published Modal warming box. Its
|
|
1975
|
+
// deadline elapsed and no user-visible workspace ever existed. Fence
|
|
1976
|
+
// the old callback before the by-id termination below.
|
|
1977
|
+
const replacement = await replace();
|
|
1978
|
+
if (!replacement) {
|
|
1979
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1980
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`,
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
captureClaim = replacement;
|
|
1984
|
+
}
|
|
1985
|
+
} else if (priorCapture.takeoverSafe && currentTakeoverSafe) {
|
|
1986
|
+
// The stable lineage reuses the exact external idempotency key where
|
|
1987
|
+
// supported. Parallel-read adapters may instead recapture independently;
|
|
1988
|
+
// both cases preserve the addressed instance and cannot overlap writes.
|
|
1989
|
+
const replacement = await replace();
|
|
1990
|
+
if (!replacement) {
|
|
1991
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1992
|
+
`sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`,
|
|
1993
|
+
);
|
|
1994
|
+
}
|
|
1995
|
+
captureClaim = replacement;
|
|
1996
|
+
} else {
|
|
1997
|
+
if (!deadlineExpired) {
|
|
1998
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
1999
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,
|
|
2000
|
+
);
|
|
2001
|
+
}
|
|
2002
|
+
const providerState = await probeDrainableProvider(settings, lease);
|
|
2003
|
+
if (providerState === "missing") {
|
|
2004
|
+
// Take over on the DB clock before committing loss. This fences an
|
|
2005
|
+
// old teardown callback. Its provider-request lineage remains
|
|
2006
|
+
// durable so a verified late archive can still repair the cold row.
|
|
2007
|
+
const replacement = await replace();
|
|
2008
|
+
if (!replacement) {
|
|
2009
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
2010
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`,
|
|
2011
|
+
);
|
|
2012
|
+
}
|
|
2013
|
+
providerMissingBeforeCapture = true;
|
|
2014
|
+
captureClaim = replacement;
|
|
2015
|
+
} else {
|
|
2016
|
+
// An exclusive provider operation can outlive both its Temporal
|
|
2017
|
+
// activity and its diagnostic deadline. A live command channel says
|
|
2018
|
+
// nothing about whether that operation is still reading/pausing the
|
|
2019
|
+
// workspace. Keep the durable fence; an operator/provider-specific
|
|
2020
|
+
// receipt is required to resolve this deliberately fail-closed case.
|
|
2021
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
2022
|
+
`sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`,
|
|
2023
|
+
);
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
1577
2026
|
}
|
|
1578
2027
|
} else {
|
|
1579
2028
|
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
1580
2029
|
accountId,
|
|
1581
2030
|
workspaceId: row.workspaceId,
|
|
1582
2031
|
sandboxGroupId: row.sandboxGroupId,
|
|
1583
|
-
captureId:
|
|
2032
|
+
captureId: attempt.captureId,
|
|
2033
|
+
operationId: attempt.operationId,
|
|
2034
|
+
attempt: attempt.attempt,
|
|
1584
2035
|
expectedEpoch: row.leaseEpoch,
|
|
1585
2036
|
expectedInstanceId: lease.instanceId,
|
|
1586
2037
|
liveness: "draining",
|
|
1587
2038
|
captureTimeoutMs,
|
|
1588
2039
|
minIntervalMs: 0,
|
|
2040
|
+
providerReplaySafe: capturePolicy?.takeover === "same_request",
|
|
2041
|
+
takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive",
|
|
1589
2042
|
});
|
|
1590
2043
|
if (claimed.status === "claimed") {
|
|
1591
2044
|
captureClaim = claimed.claim;
|
|
@@ -1599,6 +2052,13 @@ async function terminateDrainableBox(
|
|
|
1599
2052
|
const providerState = await probeDrainableProvider(settings, lease);
|
|
1600
2053
|
if (providerState !== "missing") return false;
|
|
1601
2054
|
providerMissingBeforeCapture = true;
|
|
2055
|
+
} else if (claimed.status === "capture_in_progress") {
|
|
2056
|
+
// A legacy activity or a racing delivery installed the durable claim.
|
|
2057
|
+
// Its owner may still complete; this child remains the repair owner if
|
|
2058
|
+
// it does not. Completing here would recreate a schedule-sized gap.
|
|
2059
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
2060
|
+
`sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`,
|
|
2061
|
+
);
|
|
1602
2062
|
} else {
|
|
1603
2063
|
return false;
|
|
1604
2064
|
}
|
|
@@ -1607,14 +2067,17 @@ async function terminateDrainableBox(
|
|
|
1607
2067
|
|
|
1608
2068
|
// The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
|
|
1609
2069
|
// snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
|
|
1610
|
-
// Same guard as confirmDrainCold
|
|
1611
|
-
//
|
|
1612
|
-
//
|
|
2070
|
+
// Same exact lease/capture guard as confirmDrainCold. A stale capture owner or
|
|
2071
|
+
// newer epoch writes ZERO rows → wrote:false → the seam leaves the box RUNNING
|
|
2072
|
+
// and skips the cold-commit below.
|
|
1613
2073
|
// `persisted` tracks whether a real archive landed on the lease this drain —
|
|
1614
2074
|
// the durable sandbox.box.terminated event below carries it, so a "terminated
|
|
1615
2075
|
// with NOTHING persisted" (box already dead at drain) is visible in the DB.
|
|
1616
|
-
let persisted =
|
|
1617
|
-
let archiveRevision: string | null =
|
|
2076
|
+
let persisted = captureDisposition === "archive_published";
|
|
2077
|
+
let archiveRevision: string | null =
|
|
2078
|
+
captureDisposition === "archive_published"
|
|
2079
|
+
? (lease.recovery.archive.current?.revision ?? null)
|
|
2080
|
+
: null;
|
|
1618
2081
|
const persistArchive: PersistArchiveFn = async (
|
|
1619
2082
|
archiveBase64: string | null,
|
|
1620
2083
|
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
@@ -1658,14 +2121,19 @@ async function terminateDrainableBox(
|
|
|
1658
2121
|
accountId,
|
|
1659
2122
|
workspaceId: row.workspaceId,
|
|
1660
2123
|
sandboxGroupId: row.sandboxGroupId,
|
|
2124
|
+
expectedLeaseId: lease.id,
|
|
1661
2125
|
expectedEpoch: row.leaseEpoch,
|
|
1662
2126
|
expectedInstanceId: lease.instanceId,
|
|
1663
2127
|
expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
|
|
1664
2128
|
captureId: captureClaim.id,
|
|
2129
|
+
providerRequestId: captureClaim.providerRequestId,
|
|
1665
2130
|
};
|
|
1666
2131
|
let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;
|
|
1667
2132
|
if (archiveBase64 === null) {
|
|
1668
|
-
result = await persistDrainSnapshot(db, {
|
|
2133
|
+
result = await persistDrainSnapshot(db, {
|
|
2134
|
+
...baseInput,
|
|
2135
|
+
workspaceArchive: null,
|
|
2136
|
+
});
|
|
1669
2137
|
} else {
|
|
1670
2138
|
if (!archiveMetadata) {
|
|
1671
2139
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
@@ -1701,83 +2169,103 @@ async function terminateDrainableBox(
|
|
|
1701
2169
|
// lease draining for a later sweep (NEVER terminate a box whose files we
|
|
1702
2170
|
// could not capture). A persist CAS miss means the box was re-armed and left
|
|
1703
2171
|
// running, so the cold commit is skipped.
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
2172
|
+
const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
|
|
2173
|
+
? { terminated: true, providerMissingBeforeCapture: true }
|
|
2174
|
+
: await terminateBox(
|
|
2175
|
+
settings,
|
|
2176
|
+
lease,
|
|
2177
|
+
observability,
|
|
2178
|
+
persistArchive,
|
|
2179
|
+
captureClaim?.providerRequestId ?? attempt.operationId,
|
|
2180
|
+
captureDisposition,
|
|
2181
|
+
capturePolicy,
|
|
2182
|
+
);
|
|
2183
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
2184
|
+
if (!terminated) {
|
|
2185
|
+
return false;
|
|
2186
|
+
}
|
|
1712
2187
|
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
})
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
},
|
|
1749
|
-
);
|
|
1750
|
-
}
|
|
1751
|
-
}
|
|
1752
|
-
return wentCold;
|
|
1753
|
-
} finally {
|
|
1754
|
-
if (captureClaim && lease.instanceId) {
|
|
1755
|
-
await releaseWorkspaceArchiveCapture(db, {
|
|
1756
|
-
accountId,
|
|
1757
|
-
workspaceId: row.workspaceId,
|
|
1758
|
-
sandboxGroupId: row.sandboxGroupId,
|
|
1759
|
-
captureId: captureClaim.id,
|
|
1760
|
-
expectedEpoch: row.leaseEpoch,
|
|
1761
|
-
expectedInstanceId: lease.instanceId,
|
|
1762
|
-
}).catch((error) => {
|
|
1763
|
-
observability.warn("sandbox reaper: archive capture gate release failed", {
|
|
2188
|
+
const providerMissing =
|
|
2189
|
+
providerMissingBeforeCapture ||
|
|
2190
|
+
(typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
2191
|
+
// The authoritative commit clears the durable capture/teardown claim together
|
|
2192
|
+
// with draining->cold. Until this succeeds, arrivals remain fenced by that
|
|
2193
|
+
// exact claim; a timestamp or a failed provider call can never reopen a box
|
|
2194
|
+
// while termination may still be in flight.
|
|
2195
|
+
const { wentCold } = await confirmDrainCold(db, {
|
|
2196
|
+
accountId,
|
|
2197
|
+
workspaceId: row.workspaceId,
|
|
2198
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
2199
|
+
expectedEpoch: row.leaseEpoch,
|
|
2200
|
+
...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
|
|
2201
|
+
providerMissingBeforeCapture: providerMissing,
|
|
2202
|
+
});
|
|
2203
|
+
if (wentCold) {
|
|
2204
|
+
// Durable termination record (sandbox-file-persistence observability): who
|
|
2205
|
+
// ended this box and whether its /workspace was captured first, appended to
|
|
2206
|
+
// every session sharing the group's box. Best-effort: attribution must
|
|
2207
|
+
// never affect the drain outcome.
|
|
2208
|
+
try {
|
|
2209
|
+
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
2210
|
+
type: "sandbox.box.terminated",
|
|
2211
|
+
payload: {
|
|
2212
|
+
actor: "reaper",
|
|
2213
|
+
persisted,
|
|
2214
|
+
archiveRevision,
|
|
2215
|
+
instanceId: lease.instanceId,
|
|
2216
|
+
providerMissingBeforeCapture: providerMissing,
|
|
2217
|
+
},
|
|
2218
|
+
});
|
|
2219
|
+
} catch (eventError) {
|
|
2220
|
+
observability.warn(
|
|
2221
|
+
"sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
|
|
2222
|
+
{
|
|
1764
2223
|
sandboxGroupId: row.sandboxGroupId,
|
|
1765
2224
|
leaseEpoch: row.leaseEpoch,
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
2225
|
+
backend: lease.backend,
|
|
2226
|
+
instanceId: lease.instanceId,
|
|
2227
|
+
error: eventError instanceof Error ? eventError.message : String(eventError),
|
|
2228
|
+
},
|
|
2229
|
+
);
|
|
1769
2230
|
}
|
|
1770
2231
|
}
|
|
2232
|
+
return wentCold;
|
|
1771
2233
|
}
|
|
1772
2234
|
|
|
1773
2235
|
// The persist-capable slice of a live provider session. persistWorkspace()
|
|
1774
2236
|
// snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
|
|
1775
2237
|
// archive) and returns the archive bytes. The session also carries the Modal SDK
|
|
1776
2238
|
// client used to bind the snapshot receipt to the exact provider namespace.
|
|
1777
|
-
type PersistableSession =
|
|
1778
|
-
|
|
2239
|
+
type PersistableSession = {
|
|
2240
|
+
state?: { workspacePersistence?: unknown };
|
|
2241
|
+
persistWorkspace?: (options?: { requestId: string }) => Promise<Uint8Array | undefined>;
|
|
2242
|
+
runPreStopHooks?: () => Promise<void>;
|
|
2243
|
+
preStop?: (options?: { reason?: string }) => Promise<void>;
|
|
2244
|
+
stop?: (options?: { reason?: string }) => Promise<void>;
|
|
2245
|
+
shutdown?: (options?: { reason?: string }) => Promise<void>;
|
|
2246
|
+
delete?: (options?: { reason?: string }) => Promise<void>;
|
|
2247
|
+
close?: () => Promise<void>;
|
|
1779
2248
|
};
|
|
1780
2249
|
|
|
2250
|
+
function providerDrainLogIdentity(
|
|
2251
|
+
lease: LeaseSnapshot,
|
|
2252
|
+
backend: string,
|
|
2253
|
+
): {
|
|
2254
|
+
leaseId: string;
|
|
2255
|
+
sandboxGroupId: string;
|
|
2256
|
+
leaseEpoch: number;
|
|
2257
|
+
backend: string;
|
|
2258
|
+
instanceId: string | null;
|
|
2259
|
+
} {
|
|
2260
|
+
return {
|
|
2261
|
+
leaseId: lease.id,
|
|
2262
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
2263
|
+
leaseEpoch: lease.leaseEpoch,
|
|
2264
|
+
backend,
|
|
2265
|
+
instanceId: lease.instanceId,
|
|
2266
|
+
};
|
|
2267
|
+
}
|
|
2268
|
+
|
|
1781
2269
|
/**
|
|
1782
2270
|
* Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
|
|
1783
2271
|
* lease under the epoch fence, hand the superseded snapshot to durable GC, THEN
|
|
@@ -1789,7 +2277,7 @@ type PersistableSession = TerminableSession & {
|
|
|
1789
2277
|
* - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark
|
|
1790
2278
|
* the displaced artifact delete-pending;
|
|
1791
2279
|
* - the provider-bound global GC deletes that object asynchronously;
|
|
1792
|
-
* - terminate the live handle
|
|
2280
|
+
* - terminate the live handle through the standard Agents SDK lifecycle.
|
|
1793
2281
|
*
|
|
1794
2282
|
* Returns true when the box was terminated (or was already gone), false when the
|
|
1795
2283
|
* persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.
|
|
@@ -1803,8 +2291,10 @@ type PersistableSession = TerminableSession & {
|
|
|
1803
2291
|
* BEFORE any terminate, so the box is NOT torn down with un-captured files;
|
|
1804
2292
|
* the next sweep retries, and the provider idle-timeout is the ultimate
|
|
1805
2293
|
* backstop. We never terminate a box whose snapshot we could not capture.
|
|
1806
|
-
*
|
|
1807
|
-
*
|
|
2294
|
+
* A provider-backed row with instance_id but no envelope is never guessed away:
|
|
2295
|
+
* Modal has an explicit by-id rescue adapter above; every other backend stays
|
|
2296
|
+
* draining until its adapter can prove termination. A truly instance-less or
|
|
2297
|
+
* `none`-backed row is a no-op.
|
|
1808
2298
|
*/
|
|
1809
2299
|
export async function terminateProviderBox(
|
|
1810
2300
|
settings: ActivityServices["settings"],
|
|
@@ -1812,9 +2302,14 @@ export async function terminateProviderBox(
|
|
|
1812
2302
|
observability: ActivityServices["observability"],
|
|
1813
2303
|
persistArchive: PersistArchiveFn,
|
|
1814
2304
|
createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
|
|
2305
|
+
terminateModalById: typeof terminateModalSandboxById = terminateModalSandboxById,
|
|
2306
|
+
providerCaptureRequestId?: string,
|
|
2307
|
+
captureDisposition: DrainCaptureDisposition = "capture_required",
|
|
2308
|
+
claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
|
|
1815
2309
|
): Promise<ProviderTerminationOutcome> {
|
|
1816
2310
|
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1817
2311
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
2312
|
+
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
1818
2313
|
// 'none' / no backend -> nothing to terminate.
|
|
1819
2314
|
if (!backend || backend === "none") {
|
|
1820
2315
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
@@ -1837,40 +2332,73 @@ export async function terminateProviderBox(
|
|
|
1837
2332
|
observability.info(
|
|
1838
2333
|
"sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)",
|
|
1839
2334
|
{
|
|
1840
|
-
|
|
1841
|
-
backend,
|
|
2335
|
+
...logIdentity,
|
|
1842
2336
|
},
|
|
1843
2337
|
);
|
|
1844
2338
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
1845
2339
|
}
|
|
1846
2340
|
|
|
2341
|
+
const durableCapturePolicy =
|
|
2342
|
+
claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);
|
|
2343
|
+
if (!durableCapturePolicy) {
|
|
2344
|
+
throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
|
|
2345
|
+
}
|
|
2346
|
+
if (durableCapturePolicy.liveInstance !== "preserved") {
|
|
2347
|
+
throw new Error(
|
|
2348
|
+
`sandbox backend ${backend} workspace capture would replace the fenced live instance`,
|
|
2349
|
+
);
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
|
|
2353
|
+
// `instance_id` is the lease's authoritative provider address. A recovery
|
|
2354
|
+
// envelope without it is archive/config state, never permission to target a
|
|
2355
|
+
// provider inferred only from stale JSON. Conversely, a provider identity in
|
|
2356
|
+
// the envelope with no authoritative lease identity is inconsistent state:
|
|
2357
|
+
// fail closed instead of silently leaving a live provider behind while the
|
|
2358
|
+
// caller commits the lease cold.
|
|
2359
|
+
if (!lease.instanceId) {
|
|
2360
|
+
if (persistedInstanceId) {
|
|
2361
|
+
throw new Error(
|
|
2362
|
+
`sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,
|
|
2363
|
+
);
|
|
2364
|
+
}
|
|
2365
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2366
|
+
}
|
|
2367
|
+
if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {
|
|
2368
|
+
throw new Error(
|
|
2369
|
+
`sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`,
|
|
2370
|
+
);
|
|
2371
|
+
}
|
|
2372
|
+
|
|
1847
2373
|
// A warming-death row can have a provider instance id recorded immediately
|
|
1848
2374
|
// after create() but no resumable envelope yet. For Modal, instance_id is
|
|
1849
2375
|
// enough to terminate directly; CAS-check first so a re-arm during the sweep
|
|
1850
2376
|
// leaves the box running.
|
|
1851
|
-
|
|
2377
|
+
const isUnpublishedWarmingProvider =
|
|
2378
|
+
backend === "modal" &&
|
|
2379
|
+
!persistedInstanceId &&
|
|
2380
|
+
lease.recovery.provider.status === "creating" &&
|
|
2381
|
+
lease.recovery.provider.instanceId === lease.instanceId &&
|
|
2382
|
+
lease.recovery.workspace.status === "not_ready";
|
|
2383
|
+
if (isUnpublishedWarmingProvider) {
|
|
1852
2384
|
const { wrote } = await persistArchive(null);
|
|
1853
2385
|
if (!wrote) {
|
|
1854
2386
|
observability.info(
|
|
1855
2387
|
"sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING",
|
|
1856
2388
|
{
|
|
1857
|
-
|
|
1858
|
-
backend,
|
|
1859
|
-
instanceId: lease.instanceId,
|
|
2389
|
+
...logIdentity,
|
|
1860
2390
|
},
|
|
1861
2391
|
);
|
|
1862
2392
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
1863
2393
|
}
|
|
1864
2394
|
try {
|
|
1865
|
-
await
|
|
2395
|
+
await terminateModalById(settings, lease.instanceId);
|
|
1866
2396
|
} catch (error) {
|
|
1867
2397
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
1868
2398
|
throw error;
|
|
1869
2399
|
}
|
|
1870
2400
|
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
1871
|
-
|
|
1872
|
-
backend,
|
|
1873
|
-
instanceId: lease.instanceId,
|
|
2401
|
+
...logIdentity,
|
|
1874
2402
|
});
|
|
1875
2403
|
}
|
|
1876
2404
|
// This branch is a warming-death box recorded before any resumable envelope
|
|
@@ -1880,12 +2408,18 @@ export async function terminateProviderBox(
|
|
|
1880
2408
|
}
|
|
1881
2409
|
|
|
1882
2410
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
1883
|
-
// box was last persisted as).
|
|
1884
|
-
|
|
1885
|
-
|
|
2411
|
+
// box was last persisted as). An attributed provider with no envelope may
|
|
2412
|
+
// still be live. Only Modal exposes a verified by-id rescue adapter above;
|
|
2413
|
+
// never cold another backend merely because generic resume is impossible.
|
|
2414
|
+
if (!persistedInstanceId || !lease.resumeState) {
|
|
2415
|
+
throw new Error(
|
|
2416
|
+
`sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`,
|
|
2417
|
+
);
|
|
1886
2418
|
}
|
|
1887
2419
|
|
|
1888
|
-
const client = createClientForBackend(backend as never, settings) as
|
|
2420
|
+
const client = createClientForBackend(backend as never, settings) as
|
|
2421
|
+
| DrainSandboxClient
|
|
2422
|
+
| undefined;
|
|
1889
2423
|
if (!client) {
|
|
1890
2424
|
// 'none' backend resolved to no client.
|
|
1891
2425
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
@@ -1894,7 +2428,7 @@ export async function terminateProviderBox(
|
|
|
1894
2428
|
// Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
|
|
1895
2429
|
// already gone; success.
|
|
1896
2430
|
let session: PersistableSession | undefined;
|
|
1897
|
-
let sessionState:
|
|
2431
|
+
let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;
|
|
1898
2432
|
try {
|
|
1899
2433
|
if (!client.resume || !client.deserializeSessionState) {
|
|
1900
2434
|
// A cloud backend that cannot prove provider state is not safely
|
|
@@ -1918,25 +2452,48 @@ export async function terminateProviderBox(
|
|
|
1918
2452
|
const resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
1919
2453
|
client as never,
|
|
1920
2454
|
envelopeSessionState,
|
|
2455
|
+
lease.instanceId,
|
|
1921
2456
|
);
|
|
1922
2457
|
if (resumedState === undefined) {
|
|
1923
2458
|
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
1924
2459
|
}
|
|
1925
|
-
|
|
1926
|
-
|
|
2460
|
+
const continuity = sandboxProviderContinuityForState(backend, resumedState, lease.instanceId);
|
|
2461
|
+
const resumed = await resumeExactSandboxSession(
|
|
2462
|
+
client,
|
|
2463
|
+
backend,
|
|
2464
|
+
resumedState,
|
|
2465
|
+
lease.instanceId,
|
|
2466
|
+
continuity ? { continuity } : undefined,
|
|
2467
|
+
);
|
|
2468
|
+
session = resumed.session as PersistableSession;
|
|
2469
|
+
sessionState = resumed.sessionState;
|
|
1927
2470
|
} catch (error) {
|
|
1928
2471
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
1929
2472
|
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
1930
|
-
|
|
1931
|
-
backend,
|
|
2473
|
+
...logIdentity,
|
|
1932
2474
|
});
|
|
1933
|
-
return {
|
|
2475
|
+
return {
|
|
2476
|
+
terminated: true,
|
|
2477
|
+
providerMissingBeforeCapture: captureDisposition !== "archive_published",
|
|
2478
|
+
};
|
|
1934
2479
|
}
|
|
1935
2480
|
// Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
|
|
1936
2481
|
// draining for the next sweep) — never cold a box we could not prove is gone.
|
|
1937
2482
|
throw error;
|
|
1938
2483
|
}
|
|
1939
2484
|
|
|
2485
|
+
const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);
|
|
2486
|
+
if (
|
|
2487
|
+
!liveCapturePolicy ||
|
|
2488
|
+
liveCapturePolicy.takeover !== durableCapturePolicy.takeover ||
|
|
2489
|
+
liveCapturePolicy.strategy !== durableCapturePolicy.strategy ||
|
|
2490
|
+
liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance
|
|
2491
|
+
) {
|
|
2492
|
+
throw new Error(
|
|
2493
|
+
`sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`,
|
|
2494
|
+
);
|
|
2495
|
+
}
|
|
2496
|
+
|
|
1940
2497
|
// PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to
|
|
1941
2498
|
// snapshot must re-throw BEFORE any terminate so files are never lost — the next
|
|
1942
2499
|
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
@@ -1946,6 +2503,7 @@ export async function terminateProviderBox(
|
|
|
1946
2503
|
session as { state?: { workspacePersistence?: unknown } } | undefined
|
|
1947
2504
|
)?.state?.workspacePersistence;
|
|
1948
2505
|
const checkpointBinding =
|
|
2506
|
+
captureDisposition === "capture_required" &&
|
|
1949
2507
|
backend === "modal" &&
|
|
1950
2508
|
(sessionWorkspacePersistence === "snapshot_filesystem" ||
|
|
1951
2509
|
sessionWorkspacePersistence === "snapshot_directory")
|
|
@@ -1954,22 +2512,63 @@ export async function terminateProviderBox(
|
|
|
1954
2512
|
)
|
|
1955
2513
|
: null;
|
|
1956
2514
|
try {
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
2515
|
+
if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
|
|
2516
|
+
const capture = captureVerifiedWorkspaceArchive(session, Date.now(), {
|
|
2517
|
+
requestId: providerCaptureRequestId ?? randomUUID(),
|
|
2518
|
+
strategy: liveCapturePolicy.strategy,
|
|
2519
|
+
});
|
|
2520
|
+
verifiedArchive = await awaitProviderCaptureWithLatePublication({
|
|
2521
|
+
capture,
|
|
2522
|
+
timeoutMs: settings.sandboxSnapshotTimeoutMs,
|
|
2523
|
+
timeoutError: new SandboxProviderCaptureTimeoutError(
|
|
2524
|
+
lease.sandboxGroupId,
|
|
2525
|
+
backend,
|
|
2526
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
2527
|
+
lease.leaseEpoch,
|
|
2528
|
+
lease.instanceId,
|
|
2529
|
+
),
|
|
2530
|
+
publishLate: async (archive) => {
|
|
2531
|
+
try {
|
|
2532
|
+
const result = await persistArchive(
|
|
2533
|
+
archive.base64,
|
|
2534
|
+
archive.descriptor,
|
|
2535
|
+
session,
|
|
2536
|
+
checkpointBinding,
|
|
2537
|
+
);
|
|
2538
|
+
observability.info(
|
|
2539
|
+
result.wrote
|
|
2540
|
+
? "sandbox reaper: late workspace capture published under durable claim"
|
|
2541
|
+
: "sandbox reaper: late workspace capture fenced by successor",
|
|
2542
|
+
{
|
|
2543
|
+
...logIdentity,
|
|
2544
|
+
},
|
|
2545
|
+
);
|
|
2546
|
+
} catch (error) {
|
|
2547
|
+
observability.warn("sandbox reaper: late workspace capture publication failed", {
|
|
2548
|
+
...logIdentity,
|
|
2549
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2550
|
+
});
|
|
2551
|
+
}
|
|
2552
|
+
},
|
|
2553
|
+
observeLateFailure: (error) => {
|
|
2554
|
+
observability.warn("sandbox reaper: timed-out provider capture later failed", {
|
|
2555
|
+
...logIdentity,
|
|
2556
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2557
|
+
});
|
|
2558
|
+
},
|
|
2559
|
+
});
|
|
2560
|
+
}
|
|
1960
2561
|
} catch (error) {
|
|
1961
2562
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
1962
2563
|
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
1963
|
-
|
|
1964
|
-
backend,
|
|
2564
|
+
...logIdentity,
|
|
1965
2565
|
});
|
|
1966
2566
|
return { terminated: true, providerMissingBeforeCapture: true };
|
|
1967
2567
|
}
|
|
1968
2568
|
observability.warn(
|
|
1969
2569
|
"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
|
|
1970
2570
|
{
|
|
1971
|
-
|
|
1972
|
-
backend,
|
|
2571
|
+
...logIdentity,
|
|
1973
2572
|
error: error instanceof Error ? error.message : String(error),
|
|
1974
2573
|
},
|
|
1975
2574
|
);
|
|
@@ -1977,19 +2576,18 @@ export async function terminateProviderBox(
|
|
|
1977
2576
|
throw error;
|
|
1978
2577
|
}
|
|
1979
2578
|
|
|
1980
|
-
// Fold the captured archive onto the lease under the
|
|
1981
|
-
// superseded snapshot. A CAS miss
|
|
1982
|
-
//
|
|
1983
|
-
// caller skips confirmDrainCold).
|
|
2579
|
+
// Fold the captured archive onto the lease under the exact capture fence, then
|
|
2580
|
+
// GC the superseded snapshot. A CAS miss means this callback no longer owns
|
|
2581
|
+
// teardown — DO NOT terminate (the caller skips confirmDrainCold).
|
|
1984
2582
|
//
|
|
1985
|
-
//
|
|
1986
|
-
//
|
|
1987
|
-
//
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
if (verifiedArchive) {
|
|
2583
|
+
// Ownership guard for the no-archive path: even when persistWorkspace returns
|
|
2584
|
+
// no bytes, require the exact capture receipt before delete(). The null-archive
|
|
2585
|
+
// path performs that CAS without changing archive bytes; wrote:false aborts.
|
|
2586
|
+
if (captureDisposition === "archive_published") {
|
|
2587
|
+
observability.info("sandbox reaper: resuming teardown from durable archive publication", {
|
|
2588
|
+
...logIdentity,
|
|
2589
|
+
});
|
|
2590
|
+
} else if (verifiedArchive) {
|
|
1993
2591
|
const { wrote } = await persistArchive(
|
|
1994
2592
|
verifiedArchive.base64,
|
|
1995
2593
|
verifiedArchive.descriptor,
|
|
@@ -2000,8 +2598,7 @@ export async function terminateProviderBox(
|
|
|
2000
2598
|
observability.info(
|
|
2001
2599
|
"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
|
|
2002
2600
|
{
|
|
2003
|
-
|
|
2004
|
-
backend,
|
|
2601
|
+
...logIdentity,
|
|
2005
2602
|
},
|
|
2006
2603
|
);
|
|
2007
2604
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
@@ -2015,33 +2612,19 @@ export async function terminateProviderBox(
|
|
|
2015
2612
|
);
|
|
2016
2613
|
}
|
|
2017
2614
|
|
|
2018
|
-
// Provider terminate
|
|
2019
|
-
//
|
|
2020
|
-
// terminate that fails because the box is already gone is success.
|
|
2615
|
+
// Provider terminate through the declared Agents SDK lifecycle. A terminate
|
|
2616
|
+
// that fails because the box is already gone is success.
|
|
2021
2617
|
try {
|
|
2022
|
-
if (
|
|
2023
|
-
|
|
2024
|
-
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2025
|
-
}
|
|
2026
|
-
if (session?.kill) {
|
|
2027
|
-
await session.kill();
|
|
2028
|
-
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2029
|
-
}
|
|
2030
|
-
if (session?.terminate) {
|
|
2031
|
-
await session.terminate();
|
|
2032
|
-
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2033
|
-
}
|
|
2034
|
-
if (session?.close) {
|
|
2035
|
-
if (!session.closed) {
|
|
2036
|
-
await session.close();
|
|
2037
|
-
}
|
|
2038
|
-
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2618
|
+
if (!session || !sessionState) {
|
|
2619
|
+
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
2039
2620
|
}
|
|
2621
|
+
prepareProviderForTeardownAfterCapture(backend, session);
|
|
2622
|
+
await terminateManagedSandboxSession(client, sessionState, session);
|
|
2623
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
2040
2624
|
} catch (error) {
|
|
2041
2625
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
2042
2626
|
observability.info("sandbox reaper: provider already gone during terminate", {
|
|
2043
|
-
|
|
2044
|
-
backend,
|
|
2627
|
+
...logIdentity,
|
|
2045
2628
|
});
|
|
2046
2629
|
// Capture was already durably folded above, so this is not a
|
|
2047
2630
|
// missing-before-capture outcome.
|
|
@@ -2049,5 +2632,4 @@ export async function terminateProviderBox(
|
|
|
2049
2632
|
}
|
|
2050
2633
|
throw error;
|
|
2051
2634
|
}
|
|
2052
|
-
throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
|
|
2053
2635
|
}
|