@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
package/src/sandbox-resume.ts
CHANGED
|
@@ -20,8 +20,11 @@
|
|
|
20
20
|
//
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
|
-
import {
|
|
24
|
-
|
|
23
|
+
import {
|
|
24
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
25
|
+
sandboxLifecycleTransitionWaitMs,
|
|
26
|
+
type Settings,
|
|
27
|
+
} from "@opengeni/config";
|
|
25
28
|
import { randomUUID } from "node:crypto";
|
|
26
29
|
import {
|
|
27
30
|
acquireLease,
|
|
@@ -32,6 +35,7 @@ import {
|
|
|
32
35
|
failSandboxRematerialization,
|
|
33
36
|
failWarmingToCold,
|
|
34
37
|
getSandboxSessionEnvelope,
|
|
38
|
+
heartbeatLeaseHolder,
|
|
35
39
|
markSandboxRestoreVerifying,
|
|
36
40
|
markWarmLeaseInstanceLost,
|
|
37
41
|
markSandboxCheckpointArtifactDeletePending,
|
|
@@ -44,22 +48,29 @@ import {
|
|
|
44
48
|
touchLeaseHolder,
|
|
45
49
|
SandboxLeaseRecoveryBlockedError,
|
|
46
50
|
SandboxLeaseSupersededError,
|
|
51
|
+
SandboxLeaseTransitionError,
|
|
47
52
|
type Database,
|
|
48
53
|
type LeaseHolderKind,
|
|
49
54
|
} from "@opengeni/db";
|
|
50
55
|
import {
|
|
51
56
|
captureVerifiedWorkspaceArchive,
|
|
52
57
|
describeLegacyNativeSnapshotArchive,
|
|
58
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
59
|
+
SandboxExecReadinessError,
|
|
53
60
|
WorkspaceArchiveIntegrityError,
|
|
54
61
|
establishSandboxSessionFromEnvelope,
|
|
55
62
|
isProviderSandboxNotFoundError,
|
|
56
63
|
parseWorkspaceArchiveDescriptor,
|
|
64
|
+
providerWorkspaceCapturePolicy,
|
|
65
|
+
SandboxProviderContinuityUnavailableError,
|
|
57
66
|
requirePersistableReplacementSandboxEnvelope,
|
|
58
67
|
modalSessionMatchesCheckpointProviderBinding,
|
|
59
68
|
resolveModalCheckpointProviderBindingForSession,
|
|
60
69
|
serializeReplacementSandboxEnvelope,
|
|
61
70
|
tagModalSandbox,
|
|
71
|
+
terminateUnpublishedSandboxSession,
|
|
62
72
|
verifySandboxExecReadiness,
|
|
73
|
+
withoutSandboxProviderIdentity,
|
|
63
74
|
type EstablishedSandboxSession,
|
|
64
75
|
type RuntimeMetricsHooks,
|
|
65
76
|
type WorkspaceArchiveDescriptor,
|
|
@@ -96,6 +107,10 @@ export function isRetryableDegradedRestore(restore: {
|
|
|
96
107
|
export type SandboxResumeServices = {
|
|
97
108
|
db: Database;
|
|
98
109
|
settings: Settings;
|
|
110
|
+
/** Exact settings before a verified rig provider image overlaid the logical
|
|
111
|
+
* pack/deployment image. Fresh-create NotFound fallback must preserve an
|
|
112
|
+
* ID-only logical base instead of selecting the provider default. */
|
|
113
|
+
logicalFallbackSettings?: Settings;
|
|
99
114
|
sandboxMetrics?: RuntimeMetricsHooks;
|
|
100
115
|
/**
|
|
101
116
|
* The logical turn-attempt lifetime, not merely the provider request lifetime.
|
|
@@ -167,25 +182,78 @@ export type ResumedTurnSandbox = {
|
|
|
167
182
|
established: EstablishedSandboxSession;
|
|
168
183
|
/** The lease_epoch this turn holds; the heartbeat/fence token. */
|
|
169
184
|
leaseEpoch: number;
|
|
170
|
-
/**
|
|
171
|
-
*
|
|
172
|
-
|
|
185
|
+
/** Staged idempotent release. The ordinary form immediately prevents a
|
|
186
|
+
* holder/timer leak. The quiesced form may be called later—even after the
|
|
187
|
+
* ordinary form—to repair null-outcome admissions only after every exact
|
|
188
|
+
* attempt-owned workspace writer has physically drained. NEVER stops the box. */
|
|
189
|
+
release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
|
|
173
190
|
};
|
|
174
191
|
|
|
175
|
-
export class SandboxWarmingTimeoutError extends Error {
|
|
176
|
-
readonly code
|
|
192
|
+
export abstract class SandboxWarmingTimeoutError extends Error {
|
|
193
|
+
abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
|
|
177
194
|
|
|
178
|
-
constructor(
|
|
195
|
+
protected constructor(
|
|
179
196
|
public readonly backend: string,
|
|
180
197
|
public readonly timeoutMs: number,
|
|
198
|
+
public readonly stage: "exec_readiness" | "sibling_warming",
|
|
199
|
+
public readonly sandboxGroupId: string | null,
|
|
200
|
+
public readonly instanceId: string | null,
|
|
181
201
|
) {
|
|
202
|
+
const timeoutSeconds = Math.ceil(timeoutMs / 1000);
|
|
203
|
+
const identity = [
|
|
204
|
+
sandboxGroupId ? `group ${sandboxGroupId}` : null,
|
|
205
|
+
instanceId ? `sandbox ${instanceId}` : null,
|
|
206
|
+
]
|
|
207
|
+
.filter((part): part is string => part !== null)
|
|
208
|
+
.join(", ");
|
|
209
|
+
const target = identity ? ` (${identity})` : "";
|
|
182
210
|
super(
|
|
183
|
-
|
|
211
|
+
stage === "exec_readiness"
|
|
212
|
+
? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.`
|
|
213
|
+
: `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`,
|
|
184
214
|
);
|
|
185
215
|
this.name = "SandboxWarmingTimeoutError";
|
|
186
216
|
}
|
|
187
217
|
}
|
|
188
218
|
|
|
219
|
+
export class SandboxExecReadinessTimeoutError extends SandboxWarmingTimeoutError {
|
|
220
|
+
readonly code = "sandbox_exec_readiness_timeout" as const;
|
|
221
|
+
|
|
222
|
+
constructor(
|
|
223
|
+
backend: string,
|
|
224
|
+
timeoutMs: number,
|
|
225
|
+
identity: { sandboxGroupId?: string | null; instanceId?: string | null } = {},
|
|
226
|
+
) {
|
|
227
|
+
super(
|
|
228
|
+
backend,
|
|
229
|
+
timeoutMs,
|
|
230
|
+
"exec_readiness",
|
|
231
|
+
identity.sandboxGroupId ?? null,
|
|
232
|
+
identity.instanceId ?? null,
|
|
233
|
+
);
|
|
234
|
+
this.name = "SandboxExecReadinessTimeoutError";
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutError {
|
|
239
|
+
readonly code = "sandbox_sibling_warming_timeout" as const;
|
|
240
|
+
|
|
241
|
+
constructor(
|
|
242
|
+
backend: string,
|
|
243
|
+
timeoutMs: number,
|
|
244
|
+
identity: { sandboxGroupId: string; instanceId?: string | null },
|
|
245
|
+
) {
|
|
246
|
+
super(
|
|
247
|
+
backend,
|
|
248
|
+
timeoutMs,
|
|
249
|
+
"sibling_warming",
|
|
250
|
+
identity.sandboxGroupId,
|
|
251
|
+
identity.instanceId ?? null,
|
|
252
|
+
);
|
|
253
|
+
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
189
257
|
/** The exact attached caller that won warm->cold after proving the provider
|
|
190
258
|
* instance gone. It is a lease supersession (the same logical turn recovers),
|
|
191
259
|
* plus the lost id needed for one durable observability event. */
|
|
@@ -203,7 +271,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
|
|
|
203
271
|
// Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
|
|
204
272
|
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
205
273
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
206
|
-
const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
|
|
207
274
|
|
|
208
275
|
async function sleep(ms: number): Promise<void> {
|
|
209
276
|
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
@@ -218,12 +285,16 @@ async function sleep(ms: number): Promise<void> {
|
|
|
218
285
|
export async function waitForSandboxExecReadiness(
|
|
219
286
|
established: EstablishedSandboxSession,
|
|
220
287
|
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
288
|
+
identity: { sandboxGroupId?: string | null } = {},
|
|
221
289
|
): Promise<void> {
|
|
222
290
|
try {
|
|
223
291
|
await verifySandboxExecReadiness(established, timeoutMs);
|
|
224
292
|
} catch (error) {
|
|
225
|
-
if (
|
|
226
|
-
throw new
|
|
293
|
+
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
294
|
+
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
295
|
+
sandboxGroupId: identity.sandboxGroupId ?? null,
|
|
296
|
+
instanceId: established.instanceId,
|
|
297
|
+
});
|
|
227
298
|
}
|
|
228
299
|
throw error;
|
|
229
300
|
}
|
|
@@ -236,15 +307,35 @@ class SnapshotTimeoutError extends Error {
|
|
|
236
307
|
}
|
|
237
308
|
}
|
|
238
309
|
|
|
239
|
-
function safeSnapshotError(error: unknown): {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
310
|
+
export function safeSnapshotError(error: unknown): {
|
|
311
|
+
errorClass: "SnapshotOperationError";
|
|
312
|
+
errorCode: "snapshot_operation_failed";
|
|
313
|
+
status?: number;
|
|
314
|
+
origin: "sandbox-resume";
|
|
315
|
+
} {
|
|
316
|
+
const fields: {
|
|
317
|
+
errorClass: "SnapshotOperationError";
|
|
318
|
+
errorCode: "snapshot_operation_failed";
|
|
319
|
+
status?: number;
|
|
320
|
+
origin: "sandbox-resume";
|
|
321
|
+
} = {
|
|
322
|
+
errorClass: "SnapshotOperationError",
|
|
323
|
+
errorCode: "snapshot_operation_failed",
|
|
324
|
+
origin: "sandbox-resume",
|
|
247
325
|
};
|
|
326
|
+
try {
|
|
327
|
+
const rawStatus =
|
|
328
|
+
error && typeof error === "object"
|
|
329
|
+
? ((error as { status?: unknown; statusCode?: unknown }).status ??
|
|
330
|
+
(error as { statusCode?: unknown }).statusCode)
|
|
331
|
+
: undefined;
|
|
332
|
+
const status = Number(rawStatus);
|
|
333
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
334
|
+
} catch {
|
|
335
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
336
|
+
// internal snapshot failure.
|
|
337
|
+
}
|
|
338
|
+
return fields;
|
|
248
339
|
}
|
|
249
340
|
|
|
250
341
|
export async function waitForWarmSnapshot(
|
|
@@ -335,43 +426,10 @@ async function resolveModalCheckpointBindingBeforeCapture(
|
|
|
335
426
|
async function terminateEstablishedSandbox(
|
|
336
427
|
established: EstablishedSandboxSession | null,
|
|
337
428
|
): Promise<boolean> {
|
|
338
|
-
if (!established)
|
|
339
|
-
return true;
|
|
340
|
-
}
|
|
341
|
-
const client = established.client as {
|
|
342
|
-
delete?: (state: unknown) => Promise<unknown>;
|
|
343
|
-
};
|
|
344
|
-
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
345
|
-
try {
|
|
346
|
-
await client.delete(established.sessionState);
|
|
347
|
-
return true;
|
|
348
|
-
} catch (error) {
|
|
349
|
-
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
350
|
-
return true;
|
|
351
|
-
}
|
|
352
|
-
return false;
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
const session = established.session as {
|
|
356
|
-
close?: () => Promise<unknown>;
|
|
357
|
-
terminate?: () => Promise<unknown>;
|
|
358
|
-
kill?: () => Promise<unknown>;
|
|
359
|
-
closed?: boolean;
|
|
360
|
-
};
|
|
429
|
+
if (!established) return true;
|
|
361
430
|
try {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
return true;
|
|
365
|
-
} else if (session.kill) {
|
|
366
|
-
await session.kill();
|
|
367
|
-
return true;
|
|
368
|
-
} else if (session.close) {
|
|
369
|
-
if (!session.closed) {
|
|
370
|
-
await session.close();
|
|
371
|
-
}
|
|
372
|
-
return true;
|
|
373
|
-
}
|
|
374
|
-
return false;
|
|
431
|
+
await terminateUnpublishedSandboxSession(established);
|
|
432
|
+
return true;
|
|
375
433
|
} catch (error) {
|
|
376
434
|
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
377
435
|
return true;
|
|
@@ -381,13 +439,6 @@ async function terminateEstablishedSandbox(
|
|
|
381
439
|
}
|
|
382
440
|
}
|
|
383
441
|
|
|
384
|
-
function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
|
|
385
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
386
|
-
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
|
|
387
|
-
? new SandboxWarmingTimeoutError(backend, timeoutMs)
|
|
388
|
-
: error;
|
|
389
|
-
}
|
|
390
|
-
|
|
391
442
|
function recordSandboxWarmingTimeout(
|
|
392
443
|
metrics: RuntimeMetricsHooks | undefined,
|
|
393
444
|
error: unknown,
|
|
@@ -396,7 +447,7 @@ function recordSandboxWarmingTimeout(
|
|
|
396
447
|
return;
|
|
397
448
|
}
|
|
398
449
|
try {
|
|
399
|
-
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
450
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
|
|
400
451
|
} catch {
|
|
401
452
|
// Metrics emission must never affect sandbox recovery or error propagation.
|
|
402
453
|
}
|
|
@@ -444,23 +495,6 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
|
|
|
444
495
|
);
|
|
445
496
|
}
|
|
446
497
|
|
|
447
|
-
/** A per-session fallback may still carry the dead provider identity that made
|
|
448
|
-
* recovery necessary. It is useful only as durable archive/config input: never
|
|
449
|
-
* let a cold rematerialization resume that stale provider before hydrating the
|
|
450
|
-
* selected revision. */
|
|
451
|
-
function withoutProviderIdentity(
|
|
452
|
-
envelope: Record<string, unknown> | null,
|
|
453
|
-
): Record<string, unknown> | null {
|
|
454
|
-
if (!envelope) return null;
|
|
455
|
-
const sessionState =
|
|
456
|
-
envelope.sessionState && typeof envelope.sessionState === "object"
|
|
457
|
-
? (envelope.sessionState as Record<string, unknown>)
|
|
458
|
-
: null;
|
|
459
|
-
if (!sessionState) return envelope;
|
|
460
|
-
const { providerState: _providerState, ...providerIndependentState } = sessionState;
|
|
461
|
-
return { ...envelope, sessionState: providerIndependentState };
|
|
462
|
-
}
|
|
463
|
-
|
|
464
498
|
/**
|
|
465
499
|
* MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
|
|
466
500
|
* drain-persist only protects boxes the reaper itself kills; anything else —
|
|
@@ -501,11 +535,22 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
501
535
|
}
|
|
502
536
|
const persistable = session as {
|
|
503
537
|
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
504
|
-
state?: {
|
|
538
|
+
state?: {
|
|
539
|
+
workspacePersistence?: unknown;
|
|
540
|
+
providerState?: { workspacePersistence?: unknown };
|
|
541
|
+
};
|
|
505
542
|
};
|
|
506
543
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
507
544
|
return false;
|
|
508
545
|
}
|
|
546
|
+
// Filesystem and directory snapshots create retained Images without
|
|
547
|
+
// terminating the source Sandbox. (Modal's termination warning applies to
|
|
548
|
+
// memory snapshots.) They are therefore the preferred warm-checkpoint path:
|
|
549
|
+
// the durable capture gate pauses OpenGeni commands while the provider reads
|
|
550
|
+
// the filesystem, then the same live instance continues serving the turn.
|
|
551
|
+
const workspacePersistence =
|
|
552
|
+
persistable.state?.workspacePersistence ??
|
|
553
|
+
persistable.state?.providerState?.workspacePersistence;
|
|
509
554
|
if (signal?.aborted) {
|
|
510
555
|
return false;
|
|
511
556
|
}
|
|
@@ -522,10 +567,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
522
567
|
) {
|
|
523
568
|
return false;
|
|
524
569
|
}
|
|
570
|
+
// A checkpoint of this exact mutation generation already protects every
|
|
571
|
+
// settled operation admitted so far. Capturing it again cannot improve the
|
|
572
|
+
// recovery point, even when the wall-clock interval has elapsed.
|
|
573
|
+
if (lease.archiveComplete) {
|
|
574
|
+
return false;
|
|
575
|
+
}
|
|
525
576
|
const instanceId = lease.instanceId;
|
|
577
|
+
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
578
|
+
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
526
581
|
const nativeModalPersistence =
|
|
527
|
-
|
|
528
|
-
|
|
582
|
+
workspacePersistence === "snapshot_filesystem" ||
|
|
583
|
+
workspacePersistence === "snapshot_directory";
|
|
529
584
|
const checkpointBinding = nativeModalPersistence
|
|
530
585
|
? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
|
|
531
586
|
: null;
|
|
@@ -543,6 +598,8 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
543
598
|
liveness: "warm",
|
|
544
599
|
captureTimeoutMs,
|
|
545
600
|
minIntervalMs: force ? 0 : intervalMs,
|
|
601
|
+
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
602
|
+
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
546
603
|
warmAttempt: {
|
|
547
604
|
sessionId: ids.sessionId,
|
|
548
605
|
turnId: ids.turnId,
|
|
@@ -601,7 +658,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
601
658
|
const captureAndPublish = (async (): Promise<boolean> => {
|
|
602
659
|
let candidate: { id: string } | null = null;
|
|
603
660
|
try {
|
|
604
|
-
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs
|
|
661
|
+
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
|
|
662
|
+
requestId: claimed.claim.providerRequestId,
|
|
663
|
+
strategy: capturePolicy.strategy,
|
|
664
|
+
});
|
|
605
665
|
candidate = await registerCandidate(archive);
|
|
606
666
|
const { wrote } = await persistWarmSnapshot(db, {
|
|
607
667
|
accountId: ids.accountId,
|
|
@@ -716,37 +776,74 @@ export async function resumeBoxForTurn(
|
|
|
716
776
|
// It is also bound directly to the logical attempt signal: an uninterruptible
|
|
717
777
|
// provider promise must never keep its private warmup timer and holder alive
|
|
718
778
|
// after Temporal has abandoned the activity.
|
|
719
|
-
let
|
|
779
|
+
let releaseStarted = false;
|
|
780
|
+
let holderRelease: Promise<void> | null = null;
|
|
781
|
+
let quiescedRelease: Promise<void> | null = null;
|
|
720
782
|
let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
|
|
721
783
|
let cancellationListener: (() => void) | undefined;
|
|
722
|
-
const release =
|
|
723
|
-
|
|
724
|
-
|
|
784
|
+
const release = (options?: { workspaceWritersQuiesced?: boolean }): Promise<void> => {
|
|
785
|
+
const writersQuiesced = options?.workspaceWritersQuiesced === true;
|
|
786
|
+
// First invocation synchronously closes every in-memory liveness source.
|
|
787
|
+
// A later quiesced invocation is still allowed to enter the DB: cancellation
|
|
788
|
+
// commonly wins this race and drops the holder before the physical writer
|
|
789
|
+
// drain can prove that abandoned turn admissions are safe to settle.
|
|
790
|
+
if (!releaseStarted) {
|
|
791
|
+
releaseStarted = true;
|
|
792
|
+
if (cancellationListener) {
|
|
793
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
794
|
+
cancellationListener = undefined;
|
|
795
|
+
}
|
|
796
|
+
if (holderLivenessTimer) {
|
|
797
|
+
clearInterval(holderLivenessTimer);
|
|
798
|
+
holderLivenessTimer = undefined;
|
|
799
|
+
}
|
|
725
800
|
}
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
801
|
+
const persistRelease = async (workspaceWritersQuiesced: boolean): Promise<void> => {
|
|
802
|
+
await releaseLeaseHolder(db, {
|
|
803
|
+
accountId: ids.accountId,
|
|
804
|
+
workspaceId: ids.workspaceId,
|
|
805
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
806
|
+
kind,
|
|
807
|
+
holderId,
|
|
808
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
809
|
+
...(workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}),
|
|
810
|
+
});
|
|
811
|
+
};
|
|
812
|
+
if (writersQuiesced) {
|
|
813
|
+
if (!quiescedRelease) {
|
|
814
|
+
// Serialize behind an eager cancellation release when one exists. Its
|
|
815
|
+
// failure is intentionally swallowed only as a predecessor: this exact
|
|
816
|
+
// idempotent proof-bearing call retries both holder deletion and
|
|
817
|
+
// admission settlement in one transaction.
|
|
818
|
+
const predecessor = holderRelease;
|
|
819
|
+
const attempt = (predecessor ? predecessor.catch(() => undefined) : Promise.resolve()).then(
|
|
820
|
+
async () => await persistRelease(true),
|
|
821
|
+
);
|
|
822
|
+
const tracked = attempt.catch((error) => {
|
|
823
|
+
if (quiescedRelease === tracked) quiescedRelease = null;
|
|
824
|
+
throw error;
|
|
825
|
+
});
|
|
826
|
+
quiescedRelease = tracked;
|
|
827
|
+
holderRelease = tracked;
|
|
828
|
+
}
|
|
829
|
+
return quiescedRelease;
|
|
730
830
|
}
|
|
731
|
-
if (
|
|
732
|
-
|
|
733
|
-
|
|
831
|
+
if (!holderRelease) {
|
|
832
|
+
const attempt = persistRelease(false);
|
|
833
|
+
const tracked = attempt.catch((error) => {
|
|
834
|
+
if (holderRelease === tracked) holderRelease = null;
|
|
835
|
+
throw error;
|
|
836
|
+
});
|
|
837
|
+
holderRelease = tracked;
|
|
734
838
|
}
|
|
735
|
-
|
|
736
|
-
accountId: ids.accountId,
|
|
737
|
-
workspaceId: ids.workspaceId,
|
|
738
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
739
|
-
kind,
|
|
740
|
-
holderId,
|
|
741
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
742
|
-
});
|
|
839
|
+
return holderRelease;
|
|
743
840
|
};
|
|
744
841
|
const cancellationError = (): Error =>
|
|
745
842
|
cancellationSignal?.reason instanceof Error
|
|
746
843
|
? cancellationSignal.reason
|
|
747
844
|
: new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
748
845
|
const throwIfReleasedOrCancelled = (): void => {
|
|
749
|
-
if (
|
|
846
|
+
if (releaseStarted || cancellationSignal?.aborted) {
|
|
750
847
|
throw cancellationError();
|
|
751
848
|
}
|
|
752
849
|
};
|
|
@@ -772,15 +869,18 @@ export async function resumeBoxForTurn(
|
|
|
772
869
|
// durable capture-and-drain rotation as an image change.
|
|
773
870
|
...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
|
|
774
871
|
leaseTtlMs,
|
|
775
|
-
warmingLeaseTtlMs: settings.
|
|
776
|
-
captureWaitMs:
|
|
872
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
873
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
874
|
+
...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
|
|
777
875
|
});
|
|
778
876
|
|
|
779
877
|
if (cancellationSignal) {
|
|
780
878
|
cancellationListener = () => {
|
|
781
|
-
// release() flips `
|
|
782
|
-
// first await.
|
|
783
|
-
//
|
|
879
|
+
// release() flips `releaseStarted` and clears the timer synchronously
|
|
880
|
+
// before its first await. This eager form deliberately carries no writer-
|
|
881
|
+
// quiescence proof; the outer turn supplies that second stage after its
|
|
882
|
+
// physical drain. The detached rejection handler is intentional: the
|
|
883
|
+
// outer turn owns diagnostics, while this listener owns leak prevention.
|
|
784
884
|
void release().catch(() => undefined);
|
|
785
885
|
};
|
|
786
886
|
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
@@ -790,7 +890,7 @@ export async function resumeBoxForTurn(
|
|
|
790
890
|
throw cancellationError();
|
|
791
891
|
}
|
|
792
892
|
|
|
793
|
-
// HOLDER-LIVENESS loop:
|
|
893
|
+
// HOLDER-LIVENESS loop: refresh OUR holder every 10s from registration.
|
|
794
894
|
// registered until release. The dead-worker turn-holder reap judges liveness
|
|
795
895
|
// by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
|
|
796
896
|
// agent-turn) only starts AFTER this function returns — while waitForWarm and
|
|
@@ -798,21 +898,46 @@ export async function resumeBoxForTurn(
|
|
|
798
898
|
// COMPOUNDING past any fixed reap horizon. With this
|
|
799
899
|
// loop no live holder is ever silent for more than one tick, so the reap
|
|
800
900
|
// horizon is pure defense-in-depth, not a tuned guess about path lengths.
|
|
801
|
-
//
|
|
802
|
-
//
|
|
901
|
+
// The elected spawner also heartbeats the short rolling warming lease; this
|
|
902
|
+
// makes warming expiry a crash detector rather than a fixed upper bound on a
|
|
903
|
+
// legitimate provider create/restore. A warming waiter only touches its own
|
|
904
|
+
// holder so it cannot keep a dead spawner's lease alive. Once the box is warm,
|
|
905
|
+
// every owner uses the normal epoch-fenced lease heartbeat.
|
|
906
|
+
let holderLeaseHeartbeat: { expectedEpoch: number; leaseTtlMs: number } | null =
|
|
907
|
+
acquired.role === "spawner"
|
|
908
|
+
? {
|
|
909
|
+
expectedEpoch: acquired.lease.leaseEpoch,
|
|
910
|
+
leaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
911
|
+
}
|
|
912
|
+
: (acquired.role === "attached" || acquired.role === "rearmed") &&
|
|
913
|
+
acquired.lease.liveness === "warm"
|
|
914
|
+
? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
|
|
915
|
+
: null;
|
|
803
916
|
holderLivenessTimer = setInterval(() => {
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
917
|
+
const heartbeat = holderLeaseHeartbeat;
|
|
918
|
+
const refresh = heartbeat
|
|
919
|
+
? heartbeatLeaseHolder(db, {
|
|
920
|
+
accountId: ids.accountId,
|
|
921
|
+
workspaceId: ids.workspaceId,
|
|
922
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
923
|
+
kind,
|
|
924
|
+
holderId,
|
|
925
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
926
|
+
leaseTtlMs: heartbeat.leaseTtlMs,
|
|
927
|
+
})
|
|
928
|
+
: touchLeaseHolder(db, {
|
|
929
|
+
accountId: ids.accountId,
|
|
930
|
+
workspaceId: ids.workspaceId,
|
|
931
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
932
|
+
kind,
|
|
933
|
+
holderId,
|
|
934
|
+
});
|
|
935
|
+
void refresh
|
|
811
936
|
.then((touched) => {
|
|
812
937
|
// A canonical turn holder is rejected once its exact attempt is no
|
|
813
938
|
// longer the active writer. Stop this otherwise-unbounded provider
|
|
814
939
|
// operation and idempotently drop any remaining holder state.
|
|
815
|
-
if (!touched) {
|
|
940
|
+
if (!touched && heartbeat === holderLeaseHeartbeat) {
|
|
816
941
|
void release().catch(() => undefined);
|
|
817
942
|
}
|
|
818
943
|
})
|
|
@@ -836,7 +961,21 @@ export async function resumeBoxForTurn(
|
|
|
836
961
|
}
|
|
837
962
|
if (acquired.role === "fenced") {
|
|
838
963
|
await release();
|
|
839
|
-
|
|
964
|
+
if (acquired.reason !== "superseded") {
|
|
965
|
+
throw new SandboxLeaseTransitionError(
|
|
966
|
+
ids.sandboxGroupId,
|
|
967
|
+
acquired.lease.leaseEpoch,
|
|
968
|
+
acquired.reason,
|
|
969
|
+
acquired.lease.backend,
|
|
970
|
+
acquired.lease.instanceId,
|
|
971
|
+
acquired.lease.liveness,
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
|
|
975
|
+
backend: acquired.lease.backend,
|
|
976
|
+
instanceId: acquired.lease.instanceId,
|
|
977
|
+
liveness: acquired.lease.liveness,
|
|
978
|
+
});
|
|
840
979
|
}
|
|
841
980
|
|
|
842
981
|
// SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
|
|
@@ -865,16 +1004,19 @@ export async function resumeBoxForTurn(
|
|
|
865
1004
|
const fallbackArchiveEnvelope =
|
|
866
1005
|
acquired.lease.recovery.archive.status === "none" &&
|
|
867
1006
|
workspaceArchiveFieldsFromEnvelope(envelope) !== null
|
|
868
|
-
?
|
|
1007
|
+
? withoutSandboxProviderIdentity(envelope)
|
|
869
1008
|
: null;
|
|
870
1009
|
let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
871
1010
|
const archiveSource =
|
|
872
1011
|
acquired.lease.recovery.archive.status === "none"
|
|
873
1012
|
? fallbackArchiveEnvelope
|
|
874
1013
|
: acquired.lease.resumeState;
|
|
1014
|
+
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
875
1015
|
if (
|
|
876
|
-
acquired.lease.recovery.archive.status === "available"
|
|
877
|
-
|
|
1016
|
+
(acquired.lease.recovery.archive.status === "available" &&
|
|
1017
|
+
acquired.lease.archiveComplete) ||
|
|
1018
|
+
(acquired.lease.recovery.archive.status === "none" &&
|
|
1019
|
+
workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
|
|
878
1020
|
) {
|
|
879
1021
|
const rematerializationId = crypto.randomUUID();
|
|
880
1022
|
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
@@ -917,7 +1059,7 @@ export async function resumeBoxForTurn(
|
|
|
917
1059
|
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
918
1060
|
legacyProviderBinding: null,
|
|
919
1061
|
};
|
|
920
|
-
} else if (acquired.lease.recovery.archive.status !== "none") {
|
|
1062
|
+
} else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
921
1063
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
922
1064
|
ids.sandboxGroupId,
|
|
923
1065
|
expectedEpoch,
|
|
@@ -941,6 +1083,9 @@ export async function resumeBoxForTurn(
|
|
|
941
1083
|
backendOverride: ids.backend as never,
|
|
942
1084
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
943
1085
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
1086
|
+
...(services.logicalFallbackSettings
|
|
1087
|
+
? { logicalFallbackSettings: services.logicalFallbackSettings }
|
|
1088
|
+
: {}),
|
|
944
1089
|
onSandboxCreated: async (created) => {
|
|
945
1090
|
createdEstablished = created;
|
|
946
1091
|
throwIfReleasedOrCancelled();
|
|
@@ -994,6 +1139,9 @@ export async function resumeBoxForTurn(
|
|
|
994
1139
|
sandboxGroupId: ids.sandboxGroupId,
|
|
995
1140
|
expectedEpoch,
|
|
996
1141
|
rematerializationId: rematerialization?.id ?? null,
|
|
1142
|
+
...(created.providerContinuity
|
|
1143
|
+
? { continuityRecovery: created.providerContinuity }
|
|
1144
|
+
: {}),
|
|
997
1145
|
instanceId: created.instanceId,
|
|
998
1146
|
resumeBackendId: created.backendId,
|
|
999
1147
|
resumeState: resumeEnvelope,
|
|
@@ -1008,7 +1156,7 @@ export async function resumeBoxForTurn(
|
|
|
1008
1156
|
leaseTtlMs,
|
|
1009
1157
|
// Keep the warming budget after create(): manifest setup and
|
|
1010
1158
|
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
1011
|
-
warmingLeaseTtlMs: settings.
|
|
1159
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
1012
1160
|
});
|
|
1013
1161
|
if (!recorded.recorded) {
|
|
1014
1162
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
@@ -1074,7 +1222,9 @@ export async function resumeBoxForTurn(
|
|
|
1074
1222
|
// is live. Do not publish a warm lease until one bounded no-op exec works.
|
|
1075
1223
|
// On timeout the catch below terminates the box and rolls warming -> cold,
|
|
1076
1224
|
// so the next turn cold-creates instead of hanging forever on first use.
|
|
1077
|
-
await waitForSandboxExecReadiness(established
|
|
1225
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
1226
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1227
|
+
});
|
|
1078
1228
|
throwIfReleasedOrCancelled();
|
|
1079
1229
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
1080
1230
|
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
@@ -1100,6 +1250,7 @@ export async function resumeBoxForTurn(
|
|
|
1100
1250
|
throwIfReleasedOrCancelled();
|
|
1101
1251
|
if (
|
|
1102
1252
|
rematerialization &&
|
|
1253
|
+
!established.providerContinuity &&
|
|
1103
1254
|
established.restoredArchive?.revision !== rematerialization.selectedRevision
|
|
1104
1255
|
) {
|
|
1105
1256
|
throw new WorkspaceArchiveIntegrityError(
|
|
@@ -1118,14 +1269,16 @@ export async function resumeBoxForTurn(
|
|
|
1118
1269
|
dataPlaneUrl: null,
|
|
1119
1270
|
resumeBackendId: established.backendId,
|
|
1120
1271
|
resumeState: resumeEnvelope,
|
|
1121
|
-
...(
|
|
1122
|
-
? {
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1272
|
+
...(established.providerContinuity
|
|
1273
|
+
? { continuityRecovery: established.providerContinuity }
|
|
1274
|
+
: rematerialization
|
|
1275
|
+
? {
|
|
1276
|
+
rematerialization: {
|
|
1277
|
+
id: rematerialization.id,
|
|
1278
|
+
verifiedRevision: rematerialization.selectedRevision,
|
|
1279
|
+
},
|
|
1280
|
+
}
|
|
1281
|
+
: {}),
|
|
1129
1282
|
leaseTtlMs,
|
|
1130
1283
|
});
|
|
1131
1284
|
if (!committed.committed || !committed.lease) {
|
|
@@ -1134,7 +1287,7 @@ export async function resumeBoxForTurn(
|
|
|
1134
1287
|
// holder; surface supersession. This spawner created the box, so stop it
|
|
1135
1288
|
// before retrying to avoid an untracked running sandbox.
|
|
1136
1289
|
const terminated = await terminateEstablishedSandbox(established);
|
|
1137
|
-
if (terminated && rematerialization) {
|
|
1290
|
+
if (terminated && rematerialization && !established.providerContinuity) {
|
|
1138
1291
|
await failSandboxRematerialization(db, {
|
|
1139
1292
|
accountId: ids.accountId,
|
|
1140
1293
|
workspaceId: ids.workspaceId,
|
|
@@ -1144,10 +1297,21 @@ export async function resumeBoxForTurn(
|
|
|
1144
1297
|
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
1145
1298
|
retryable: false,
|
|
1146
1299
|
});
|
|
1300
|
+
} else if (terminated) {
|
|
1301
|
+
await failWarmingToCold(db, {
|
|
1302
|
+
accountId: ids.accountId,
|
|
1303
|
+
workspaceId: ids.workspaceId,
|
|
1304
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1305
|
+
expectedEpoch,
|
|
1306
|
+
});
|
|
1147
1307
|
}
|
|
1148
1308
|
await release();
|
|
1149
1309
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1150
1310
|
}
|
|
1311
|
+
holderLeaseHeartbeat = {
|
|
1312
|
+
expectedEpoch: committed.lease.leaseEpoch,
|
|
1313
|
+
leaseTtlMs,
|
|
1314
|
+
};
|
|
1151
1315
|
throwIfReleasedOrCancelled();
|
|
1152
1316
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1153
1317
|
} catch (error) {
|
|
@@ -1163,18 +1327,25 @@ export async function resumeBoxForTurn(
|
|
|
1163
1327
|
// the warming row; the lease TTL/reaper and Modal orphan sweep are the
|
|
1164
1328
|
// tracked backstops, and we must not erase the only provider pointer.
|
|
1165
1329
|
if (terminated) {
|
|
1166
|
-
|
|
1330
|
+
const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
|
|
1331
|
+
if (rematerialization && !createdEstablished?.providerContinuity) {
|
|
1167
1332
|
await failSandboxRematerialization(db, {
|
|
1168
1333
|
accountId: ids.accountId,
|
|
1169
1334
|
workspaceId: ids.workspaceId,
|
|
1170
1335
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1171
1336
|
expectedEpoch,
|
|
1172
1337
|
rematerializationId: rematerialization.id,
|
|
1173
|
-
failureCode:
|
|
1174
|
-
|
|
1338
|
+
failureCode: continuityUnavailable
|
|
1339
|
+
? "provider_continuity_unavailable"
|
|
1340
|
+
: error instanceof WorkspaceArchiveIntegrityError
|
|
1175
1341
|
? error.code
|
|
1176
1342
|
: "sandbox_rematerialization_failed",
|
|
1177
|
-
retryable:
|
|
1343
|
+
retryable: continuityUnavailable
|
|
1344
|
+
? error.retryable
|
|
1345
|
+
: error instanceof WorkspaceArchiveIntegrityError
|
|
1346
|
+
? error.retryable
|
|
1347
|
+
: true,
|
|
1348
|
+
...(continuityUnavailable ? { discardContinuity: true } : {}),
|
|
1178
1349
|
});
|
|
1179
1350
|
} else {
|
|
1180
1351
|
await failWarmingToCold(db, {
|
|
@@ -1182,17 +1353,13 @@ export async function resumeBoxForTurn(
|
|
|
1182
1353
|
workspaceId: ids.workspaceId,
|
|
1183
1354
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1184
1355
|
expectedEpoch,
|
|
1356
|
+
...(continuityUnavailable ? { discardContinuity: true } : {}),
|
|
1185
1357
|
});
|
|
1186
1358
|
}
|
|
1187
1359
|
}
|
|
1188
1360
|
await release();
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
ids.backend,
|
|
1192
|
-
settings.sandboxWarmingTimeoutMs,
|
|
1193
|
-
);
|
|
1194
|
-
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1195
|
-
throw warmingError;
|
|
1361
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1362
|
+
throw error;
|
|
1196
1363
|
}
|
|
1197
1364
|
}
|
|
1198
1365
|
|
|
@@ -1205,6 +1372,7 @@ export async function resumeBoxForTurn(
|
|
|
1205
1372
|
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1206
1373
|
if (acquired.lease.liveness === "warming") {
|
|
1207
1374
|
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1375
|
+
holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
|
|
1208
1376
|
}
|
|
1209
1377
|
throwIfReleasedOrCancelled();
|
|
1210
1378
|
|
|
@@ -1237,7 +1405,13 @@ export async function resumeBoxForTurn(
|
|
|
1237
1405
|
// was idle. Prove the command router before handing the session to the
|
|
1238
1406
|
// agent so terminal evidence enters the atomic warm->cold recovery path
|
|
1239
1407
|
// below instead of surfacing inside a model-visible tool call.
|
|
1240
|
-
|
|
1408
|
+
if (services.verifyAttachedSandboxReadiness) {
|
|
1409
|
+
await services.verifyAttachedSandboxReadiness(established);
|
|
1410
|
+
} else {
|
|
1411
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
1412
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1241
1415
|
throwIfReleasedOrCancelled();
|
|
1242
1416
|
} catch (error) {
|
|
1243
1417
|
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
@@ -1271,13 +1445,8 @@ export async function resumeBoxForTurn(
|
|
|
1271
1445
|
return { established, leaseEpoch, release };
|
|
1272
1446
|
} catch (error) {
|
|
1273
1447
|
await release();
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
ids.backend,
|
|
1277
|
-
settings.sandboxWarmingTimeoutMs,
|
|
1278
|
-
);
|
|
1279
|
-
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1280
|
-
throw warmingError;
|
|
1448
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1449
|
+
throw error;
|
|
1281
1450
|
}
|
|
1282
1451
|
}
|
|
1283
1452
|
|
|
@@ -1293,6 +1462,7 @@ async function waitForWarm(
|
|
|
1293
1462
|
): Promise<{ leaseEpoch: number }> {
|
|
1294
1463
|
const { db, settings } = services;
|
|
1295
1464
|
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1465
|
+
let instanceId: string | null = null;
|
|
1296
1466
|
while (Date.now() < deadline) {
|
|
1297
1467
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1298
1468
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
@@ -1300,6 +1470,7 @@ async function waitForWarm(
|
|
|
1300
1470
|
// Lease vanished (cold-reaped). Re-dispatch from scratch.
|
|
1301
1471
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1302
1472
|
}
|
|
1473
|
+
instanceId = lease.instanceId;
|
|
1303
1474
|
if (lease.liveness === "warm") {
|
|
1304
1475
|
return { leaseEpoch: lease.leaseEpoch };
|
|
1305
1476
|
}
|
|
@@ -1335,5 +1506,8 @@ async function waitForWarm(
|
|
|
1335
1506
|
}
|
|
1336
1507
|
// still warming — keep polling.
|
|
1337
1508
|
}
|
|
1338
|
-
throw new
|
|
1509
|
+
throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
|
|
1510
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1511
|
+
instanceId,
|
|
1512
|
+
});
|
|
1339
1513
|
}
|