@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/sandbox-resume.ts
CHANGED
|
@@ -20,7 +20,11 @@
|
|
|
20
20
|
//
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
25
|
+
sandboxLifecycleTransitionWaitMs,
|
|
26
|
+
type Settings,
|
|
27
|
+
} from "@opengeni/config";
|
|
24
28
|
import { randomUUID } from "node:crypto";
|
|
25
29
|
import {
|
|
26
30
|
acquireLease,
|
|
@@ -31,6 +35,7 @@ import {
|
|
|
31
35
|
failSandboxRematerialization,
|
|
32
36
|
failWarmingToCold,
|
|
33
37
|
getSandboxSessionEnvelope,
|
|
38
|
+
heartbeatLeaseHolder,
|
|
34
39
|
markSandboxRestoreVerifying,
|
|
35
40
|
markWarmLeaseInstanceLost,
|
|
36
41
|
markSandboxCheckpointArtifactDeletePending,
|
|
@@ -43,22 +48,29 @@ import {
|
|
|
43
48
|
touchLeaseHolder,
|
|
44
49
|
SandboxLeaseRecoveryBlockedError,
|
|
45
50
|
SandboxLeaseSupersededError,
|
|
51
|
+
SandboxLeaseTransitionError,
|
|
46
52
|
type Database,
|
|
47
53
|
type LeaseHolderKind,
|
|
48
54
|
} from "@opengeni/db";
|
|
49
55
|
import {
|
|
50
56
|
captureVerifiedWorkspaceArchive,
|
|
51
57
|
describeLegacyNativeSnapshotArchive,
|
|
58
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
59
|
+
SandboxExecReadinessError,
|
|
52
60
|
WorkspaceArchiveIntegrityError,
|
|
53
61
|
establishSandboxSessionFromEnvelope,
|
|
54
62
|
isProviderSandboxNotFoundError,
|
|
55
63
|
parseWorkspaceArchiveDescriptor,
|
|
64
|
+
providerWorkspaceCapturePolicy,
|
|
65
|
+
SandboxProviderContinuityUnavailableError,
|
|
56
66
|
requirePersistableReplacementSandboxEnvelope,
|
|
57
67
|
modalSessionMatchesCheckpointProviderBinding,
|
|
58
68
|
resolveModalCheckpointProviderBindingForSession,
|
|
59
69
|
serializeReplacementSandboxEnvelope,
|
|
60
70
|
tagModalSandbox,
|
|
71
|
+
terminateUnpublishedSandboxSession,
|
|
61
72
|
verifySandboxExecReadiness,
|
|
73
|
+
withoutSandboxProviderIdentity,
|
|
62
74
|
type EstablishedSandboxSession,
|
|
63
75
|
type RuntimeMetricsHooks,
|
|
64
76
|
type WorkspaceArchiveDescriptor,
|
|
@@ -95,6 +107,10 @@ export function isRetryableDegradedRestore(restore: {
|
|
|
95
107
|
export type SandboxResumeServices = {
|
|
96
108
|
db: Database;
|
|
97
109
|
settings: Settings;
|
|
110
|
+
/** Exact settings before a verified rig provider image overlaid the logical
|
|
111
|
+
* pack/deployment image. Fresh-create NotFound fallback must preserve an
|
|
112
|
+
* ID-only logical base instead of selecting the provider default. */
|
|
113
|
+
logicalFallbackSettings?: Settings;
|
|
98
114
|
sandboxMetrics?: RuntimeMetricsHooks;
|
|
99
115
|
/**
|
|
100
116
|
* The logical turn-attempt lifetime, not merely the provider request lifetime.
|
|
@@ -166,25 +182,78 @@ export type ResumedTurnSandbox = {
|
|
|
166
182
|
established: EstablishedSandboxSession;
|
|
167
183
|
/** The lease_epoch this turn holds; the heartbeat/fence token. */
|
|
168
184
|
leaseEpoch: number;
|
|
169
|
-
/**
|
|
170
|
-
*
|
|
171
|
-
|
|
185
|
+
/** Staged idempotent release. The ordinary form immediately prevents a
|
|
186
|
+
* holder/timer leak. The quiesced form may be called later—even after the
|
|
187
|
+
* ordinary form—to repair null-outcome admissions only after every exact
|
|
188
|
+
* attempt-owned workspace writer has physically drained. NEVER stops the box. */
|
|
189
|
+
release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
|
|
172
190
|
};
|
|
173
191
|
|
|
174
|
-
export class SandboxWarmingTimeoutError extends Error {
|
|
175
|
-
readonly code
|
|
192
|
+
export abstract class SandboxWarmingTimeoutError extends Error {
|
|
193
|
+
abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
|
|
176
194
|
|
|
177
|
-
constructor(
|
|
195
|
+
protected constructor(
|
|
178
196
|
public readonly backend: string,
|
|
179
197
|
public readonly timeoutMs: number,
|
|
198
|
+
public readonly stage: "exec_readiness" | "sibling_warming",
|
|
199
|
+
public readonly sandboxGroupId: string | null,
|
|
200
|
+
public readonly instanceId: string | null,
|
|
180
201
|
) {
|
|
202
|
+
const timeoutSeconds = Math.ceil(timeoutMs / 1000);
|
|
203
|
+
const identity = [
|
|
204
|
+
sandboxGroupId ? `group ${sandboxGroupId}` : null,
|
|
205
|
+
instanceId ? `sandbox ${instanceId}` : null,
|
|
206
|
+
]
|
|
207
|
+
.filter((part): part is string => part !== null)
|
|
208
|
+
.join(", ");
|
|
209
|
+
const target = identity ? ` (${identity})` : "";
|
|
181
210
|
super(
|
|
182
|
-
|
|
211
|
+
stage === "exec_readiness"
|
|
212
|
+
? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.`
|
|
213
|
+
: `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`,
|
|
183
214
|
);
|
|
184
215
|
this.name = "SandboxWarmingTimeoutError";
|
|
185
216
|
}
|
|
186
217
|
}
|
|
187
218
|
|
|
219
|
+
export class SandboxExecReadinessTimeoutError extends SandboxWarmingTimeoutError {
|
|
220
|
+
readonly code = "sandbox_exec_readiness_timeout" as const;
|
|
221
|
+
|
|
222
|
+
constructor(
|
|
223
|
+
backend: string,
|
|
224
|
+
timeoutMs: number,
|
|
225
|
+
identity: { sandboxGroupId?: string | null; instanceId?: string | null } = {},
|
|
226
|
+
) {
|
|
227
|
+
super(
|
|
228
|
+
backend,
|
|
229
|
+
timeoutMs,
|
|
230
|
+
"exec_readiness",
|
|
231
|
+
identity.sandboxGroupId ?? null,
|
|
232
|
+
identity.instanceId ?? null,
|
|
233
|
+
);
|
|
234
|
+
this.name = "SandboxExecReadinessTimeoutError";
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutError {
|
|
239
|
+
readonly code = "sandbox_sibling_warming_timeout" as const;
|
|
240
|
+
|
|
241
|
+
constructor(
|
|
242
|
+
backend: string,
|
|
243
|
+
timeoutMs: number,
|
|
244
|
+
identity: { sandboxGroupId: string; instanceId?: string | null },
|
|
245
|
+
) {
|
|
246
|
+
super(
|
|
247
|
+
backend,
|
|
248
|
+
timeoutMs,
|
|
249
|
+
"sibling_warming",
|
|
250
|
+
identity.sandboxGroupId,
|
|
251
|
+
identity.instanceId ?? null,
|
|
252
|
+
);
|
|
253
|
+
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
188
257
|
/** The exact attached caller that won warm->cold after proving the provider
|
|
189
258
|
* instance gone. It is a lease supersession (the same logical turn recovers),
|
|
190
259
|
* plus the lost id needed for one durable observability event. */
|
|
@@ -202,7 +271,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
|
|
|
202
271
|
// Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
|
|
203
272
|
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
204
273
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
205
|
-
const MODAL_EXEC_READINESS_TIMEOUT_MS = 15_000;
|
|
206
274
|
|
|
207
275
|
async function sleep(ms: number): Promise<void> {
|
|
208
276
|
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
@@ -217,12 +285,16 @@ async function sleep(ms: number): Promise<void> {
|
|
|
217
285
|
export async function waitForSandboxExecReadiness(
|
|
218
286
|
established: EstablishedSandboxSession,
|
|
219
287
|
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
288
|
+
identity: { sandboxGroupId?: string | null } = {},
|
|
220
289
|
): Promise<void> {
|
|
221
290
|
try {
|
|
222
291
|
await verifySandboxExecReadiness(established, timeoutMs);
|
|
223
292
|
} catch (error) {
|
|
224
|
-
if (
|
|
225
|
-
throw new
|
|
293
|
+
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
294
|
+
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
295
|
+
sandboxGroupId: identity.sandboxGroupId ?? null,
|
|
296
|
+
instanceId: established.instanceId,
|
|
297
|
+
});
|
|
226
298
|
}
|
|
227
299
|
throw error;
|
|
228
300
|
}
|
|
@@ -354,43 +426,10 @@ async function resolveModalCheckpointBindingBeforeCapture(
|
|
|
354
426
|
async function terminateEstablishedSandbox(
|
|
355
427
|
established: EstablishedSandboxSession | null,
|
|
356
428
|
): Promise<boolean> {
|
|
357
|
-
if (!established)
|
|
358
|
-
return true;
|
|
359
|
-
}
|
|
360
|
-
const client = established.client as {
|
|
361
|
-
delete?: (state: unknown) => Promise<unknown>;
|
|
362
|
-
};
|
|
363
|
-
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
364
|
-
try {
|
|
365
|
-
await client.delete(established.sessionState);
|
|
366
|
-
return true;
|
|
367
|
-
} catch (error) {
|
|
368
|
-
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
369
|
-
return true;
|
|
370
|
-
}
|
|
371
|
-
return false;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
const session = established.session as {
|
|
375
|
-
close?: () => Promise<unknown>;
|
|
376
|
-
terminate?: () => Promise<unknown>;
|
|
377
|
-
kill?: () => Promise<unknown>;
|
|
378
|
-
closed?: boolean;
|
|
379
|
-
};
|
|
429
|
+
if (!established) return true;
|
|
380
430
|
try {
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
return true;
|
|
384
|
-
} else if (session.kill) {
|
|
385
|
-
await session.kill();
|
|
386
|
-
return true;
|
|
387
|
-
} else if (session.close) {
|
|
388
|
-
if (!session.closed) {
|
|
389
|
-
await session.close();
|
|
390
|
-
}
|
|
391
|
-
return true;
|
|
392
|
-
}
|
|
393
|
-
return false;
|
|
431
|
+
await terminateUnpublishedSandboxSession(established);
|
|
432
|
+
return true;
|
|
394
433
|
} catch (error) {
|
|
395
434
|
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
396
435
|
return true;
|
|
@@ -400,13 +439,6 @@ async function terminateEstablishedSandbox(
|
|
|
400
439
|
}
|
|
401
440
|
}
|
|
402
441
|
|
|
403
|
-
function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
|
|
404
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
405
|
-
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
|
|
406
|
-
? new SandboxWarmingTimeoutError(backend, timeoutMs)
|
|
407
|
-
: error;
|
|
408
|
-
}
|
|
409
|
-
|
|
410
442
|
function recordSandboxWarmingTimeout(
|
|
411
443
|
metrics: RuntimeMetricsHooks | undefined,
|
|
412
444
|
error: unknown,
|
|
@@ -415,7 +447,7 @@ function recordSandboxWarmingTimeout(
|
|
|
415
447
|
return;
|
|
416
448
|
}
|
|
417
449
|
try {
|
|
418
|
-
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
450
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
|
|
419
451
|
} catch {
|
|
420
452
|
// Metrics emission must never affect sandbox recovery or error propagation.
|
|
421
453
|
}
|
|
@@ -463,23 +495,6 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
|
|
|
463
495
|
);
|
|
464
496
|
}
|
|
465
497
|
|
|
466
|
-
/** A per-session fallback may still carry the dead provider identity that made
|
|
467
|
-
* recovery necessary. It is useful only as durable archive/config input: never
|
|
468
|
-
* let a cold rematerialization resume that stale provider before hydrating the
|
|
469
|
-
* selected revision. */
|
|
470
|
-
function withoutProviderIdentity(
|
|
471
|
-
envelope: Record<string, unknown> | null,
|
|
472
|
-
): Record<string, unknown> | null {
|
|
473
|
-
if (!envelope) return null;
|
|
474
|
-
const sessionState =
|
|
475
|
-
envelope.sessionState && typeof envelope.sessionState === "object"
|
|
476
|
-
? (envelope.sessionState as Record<string, unknown>)
|
|
477
|
-
: null;
|
|
478
|
-
if (!sessionState) return envelope;
|
|
479
|
-
const { providerState: _providerState, ...providerIndependentState } = sessionState;
|
|
480
|
-
return { ...envelope, sessionState: providerIndependentState };
|
|
481
|
-
}
|
|
482
|
-
|
|
483
498
|
/**
|
|
484
499
|
* MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
|
|
485
500
|
* drain-persist only protects boxes the reaper itself kills; anything else —
|
|
@@ -528,17 +543,14 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
528
543
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
529
544
|
return false;
|
|
530
545
|
}
|
|
531
|
-
//
|
|
532
|
-
//
|
|
533
|
-
//
|
|
534
|
-
//
|
|
535
|
-
//
|
|
546
|
+
// Filesystem and directory snapshots create retained Images without
|
|
547
|
+
// terminating the source Sandbox. (Modal's termination warning applies to
|
|
548
|
+
// memory snapshots.) They are therefore the preferred warm-checkpoint path:
|
|
549
|
+
// the durable capture gate pauses OpenGeni commands while the provider reads
|
|
550
|
+
// the filesystem, then the same live instance continues serving the turn.
|
|
536
551
|
const workspacePersistence =
|
|
537
552
|
persistable.state?.workspacePersistence ??
|
|
538
553
|
persistable.state?.providerState?.workspacePersistence;
|
|
539
|
-
if (!force && workspacePersistence === "snapshot_filesystem") {
|
|
540
|
-
return false;
|
|
541
|
-
}
|
|
542
554
|
if (signal?.aborted) {
|
|
543
555
|
return false;
|
|
544
556
|
}
|
|
@@ -555,7 +567,17 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
555
567
|
) {
|
|
556
568
|
return false;
|
|
557
569
|
}
|
|
570
|
+
// A checkpoint of this exact mutation generation already protects every
|
|
571
|
+
// settled operation admitted so far. Capturing it again cannot improve the
|
|
572
|
+
// recovery point, even when the wall-clock interval has elapsed.
|
|
573
|
+
if (lease.archiveComplete) {
|
|
574
|
+
return false;
|
|
575
|
+
}
|
|
558
576
|
const instanceId = lease.instanceId;
|
|
577
|
+
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
578
|
+
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
559
581
|
const nativeModalPersistence =
|
|
560
582
|
workspacePersistence === "snapshot_filesystem" ||
|
|
561
583
|
workspacePersistence === "snapshot_directory";
|
|
@@ -576,6 +598,8 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
576
598
|
liveness: "warm",
|
|
577
599
|
captureTimeoutMs,
|
|
578
600
|
minIntervalMs: force ? 0 : intervalMs,
|
|
601
|
+
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
602
|
+
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
579
603
|
warmAttempt: {
|
|
580
604
|
sessionId: ids.sessionId,
|
|
581
605
|
turnId: ids.turnId,
|
|
@@ -634,7 +658,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
634
658
|
const captureAndPublish = (async (): Promise<boolean> => {
|
|
635
659
|
let candidate: { id: string } | null = null;
|
|
636
660
|
try {
|
|
637
|
-
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs
|
|
661
|
+
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
|
|
662
|
+
requestId: claimed.claim.providerRequestId,
|
|
663
|
+
strategy: capturePolicy.strategy,
|
|
664
|
+
});
|
|
638
665
|
candidate = await registerCandidate(archive);
|
|
639
666
|
const { wrote } = await persistWarmSnapshot(db, {
|
|
640
667
|
accountId: ids.accountId,
|
|
@@ -749,37 +776,74 @@ export async function resumeBoxForTurn(
|
|
|
749
776
|
// It is also bound directly to the logical attempt signal: an uninterruptible
|
|
750
777
|
// provider promise must never keep its private warmup timer and holder alive
|
|
751
778
|
// after Temporal has abandoned the activity.
|
|
752
|
-
let
|
|
779
|
+
let releaseStarted = false;
|
|
780
|
+
let holderRelease: Promise<void> | null = null;
|
|
781
|
+
let quiescedRelease: Promise<void> | null = null;
|
|
753
782
|
let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
|
|
754
783
|
let cancellationListener: (() => void) | undefined;
|
|
755
|
-
const release =
|
|
756
|
-
|
|
757
|
-
|
|
784
|
+
const release = (options?: { workspaceWritersQuiesced?: boolean }): Promise<void> => {
|
|
785
|
+
const writersQuiesced = options?.workspaceWritersQuiesced === true;
|
|
786
|
+
// First invocation synchronously closes every in-memory liveness source.
|
|
787
|
+
// A later quiesced invocation is still allowed to enter the DB: cancellation
|
|
788
|
+
// commonly wins this race and drops the holder before the physical writer
|
|
789
|
+
// drain can prove that abandoned turn admissions are safe to settle.
|
|
790
|
+
if (!releaseStarted) {
|
|
791
|
+
releaseStarted = true;
|
|
792
|
+
if (cancellationListener) {
|
|
793
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
794
|
+
cancellationListener = undefined;
|
|
795
|
+
}
|
|
796
|
+
if (holderLivenessTimer) {
|
|
797
|
+
clearInterval(holderLivenessTimer);
|
|
798
|
+
holderLivenessTimer = undefined;
|
|
799
|
+
}
|
|
758
800
|
}
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
801
|
+
const persistRelease = async (workspaceWritersQuiesced: boolean): Promise<void> => {
|
|
802
|
+
await releaseLeaseHolder(db, {
|
|
803
|
+
accountId: ids.accountId,
|
|
804
|
+
workspaceId: ids.workspaceId,
|
|
805
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
806
|
+
kind,
|
|
807
|
+
holderId,
|
|
808
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
809
|
+
...(workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}),
|
|
810
|
+
});
|
|
811
|
+
};
|
|
812
|
+
if (writersQuiesced) {
|
|
813
|
+
if (!quiescedRelease) {
|
|
814
|
+
// Serialize behind an eager cancellation release when one exists. Its
|
|
815
|
+
// failure is intentionally swallowed only as a predecessor: this exact
|
|
816
|
+
// idempotent proof-bearing call retries both holder deletion and
|
|
817
|
+
// admission settlement in one transaction.
|
|
818
|
+
const predecessor = holderRelease;
|
|
819
|
+
const attempt = (predecessor ? predecessor.catch(() => undefined) : Promise.resolve()).then(
|
|
820
|
+
async () => await persistRelease(true),
|
|
821
|
+
);
|
|
822
|
+
const tracked = attempt.catch((error) => {
|
|
823
|
+
if (quiescedRelease === tracked) quiescedRelease = null;
|
|
824
|
+
throw error;
|
|
825
|
+
});
|
|
826
|
+
quiescedRelease = tracked;
|
|
827
|
+
holderRelease = tracked;
|
|
828
|
+
}
|
|
829
|
+
return quiescedRelease;
|
|
763
830
|
}
|
|
764
|
-
if (
|
|
765
|
-
|
|
766
|
-
|
|
831
|
+
if (!holderRelease) {
|
|
832
|
+
const attempt = persistRelease(false);
|
|
833
|
+
const tracked = attempt.catch((error) => {
|
|
834
|
+
if (holderRelease === tracked) holderRelease = null;
|
|
835
|
+
throw error;
|
|
836
|
+
});
|
|
837
|
+
holderRelease = tracked;
|
|
767
838
|
}
|
|
768
|
-
|
|
769
|
-
accountId: ids.accountId,
|
|
770
|
-
workspaceId: ids.workspaceId,
|
|
771
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
772
|
-
kind,
|
|
773
|
-
holderId,
|
|
774
|
-
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
775
|
-
});
|
|
839
|
+
return holderRelease;
|
|
776
840
|
};
|
|
777
841
|
const cancellationError = (): Error =>
|
|
778
842
|
cancellationSignal?.reason instanceof Error
|
|
779
843
|
? cancellationSignal.reason
|
|
780
844
|
: new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
781
845
|
const throwIfReleasedOrCancelled = (): void => {
|
|
782
|
-
if (
|
|
846
|
+
if (releaseStarted || cancellationSignal?.aborted) {
|
|
783
847
|
throw cancellationError();
|
|
784
848
|
}
|
|
785
849
|
};
|
|
@@ -805,15 +869,18 @@ export async function resumeBoxForTurn(
|
|
|
805
869
|
// durable capture-and-drain rotation as an image change.
|
|
806
870
|
...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
|
|
807
871
|
leaseTtlMs,
|
|
808
|
-
warmingLeaseTtlMs: settings.
|
|
809
|
-
captureWaitMs:
|
|
872
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
873
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
874
|
+
...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
|
|
810
875
|
});
|
|
811
876
|
|
|
812
877
|
if (cancellationSignal) {
|
|
813
878
|
cancellationListener = () => {
|
|
814
|
-
// release() flips `
|
|
815
|
-
// first await.
|
|
816
|
-
//
|
|
879
|
+
// release() flips `releaseStarted` and clears the timer synchronously
|
|
880
|
+
// before its first await. This eager form deliberately carries no writer-
|
|
881
|
+
// quiescence proof; the outer turn supplies that second stage after its
|
|
882
|
+
// physical drain. The detached rejection handler is intentional: the
|
|
883
|
+
// outer turn owns diagnostics, while this listener owns leak prevention.
|
|
817
884
|
void release().catch(() => undefined);
|
|
818
885
|
};
|
|
819
886
|
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
@@ -823,7 +890,7 @@ export async function resumeBoxForTurn(
|
|
|
823
890
|
throw cancellationError();
|
|
824
891
|
}
|
|
825
892
|
|
|
826
|
-
// HOLDER-LIVENESS loop:
|
|
893
|
+
// HOLDER-LIVENESS loop: refresh OUR holder every 10s from registration.
|
|
827
894
|
// registered until release. The dead-worker turn-holder reap judges liveness
|
|
828
895
|
// by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
|
|
829
896
|
// agent-turn) only starts AFTER this function returns — while waitForWarm and
|
|
@@ -831,21 +898,46 @@ export async function resumeBoxForTurn(
|
|
|
831
898
|
// COMPOUNDING past any fixed reap horizon. With this
|
|
832
899
|
// loop no live holder is ever silent for more than one tick, so the reap
|
|
833
900
|
// horizon is pure defense-in-depth, not a tuned guess about path lengths.
|
|
834
|
-
//
|
|
835
|
-
//
|
|
901
|
+
// The elected spawner also heartbeats the short rolling warming lease; this
|
|
902
|
+
// makes warming expiry a crash detector rather than a fixed upper bound on a
|
|
903
|
+
// legitimate provider create/restore. A warming waiter only touches its own
|
|
904
|
+
// holder so it cannot keep a dead spawner's lease alive. Once the box is warm,
|
|
905
|
+
// every owner uses the normal epoch-fenced lease heartbeat.
|
|
906
|
+
let holderLeaseHeartbeat: { expectedEpoch: number; leaseTtlMs: number } | null =
|
|
907
|
+
acquired.role === "spawner"
|
|
908
|
+
? {
|
|
909
|
+
expectedEpoch: acquired.lease.leaseEpoch,
|
|
910
|
+
leaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
911
|
+
}
|
|
912
|
+
: (acquired.role === "attached" || acquired.role === "rearmed") &&
|
|
913
|
+
acquired.lease.liveness === "warm"
|
|
914
|
+
? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
|
|
915
|
+
: null;
|
|
836
916
|
holderLivenessTimer = setInterval(() => {
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
917
|
+
const heartbeat = holderLeaseHeartbeat;
|
|
918
|
+
const refresh = heartbeat
|
|
919
|
+
? heartbeatLeaseHolder(db, {
|
|
920
|
+
accountId: ids.accountId,
|
|
921
|
+
workspaceId: ids.workspaceId,
|
|
922
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
923
|
+
kind,
|
|
924
|
+
holderId,
|
|
925
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
926
|
+
leaseTtlMs: heartbeat.leaseTtlMs,
|
|
927
|
+
})
|
|
928
|
+
: touchLeaseHolder(db, {
|
|
929
|
+
accountId: ids.accountId,
|
|
930
|
+
workspaceId: ids.workspaceId,
|
|
931
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
932
|
+
kind,
|
|
933
|
+
holderId,
|
|
934
|
+
});
|
|
935
|
+
void refresh
|
|
844
936
|
.then((touched) => {
|
|
845
937
|
// A canonical turn holder is rejected once its exact attempt is no
|
|
846
938
|
// longer the active writer. Stop this otherwise-unbounded provider
|
|
847
939
|
// operation and idempotently drop any remaining holder state.
|
|
848
|
-
if (!touched) {
|
|
940
|
+
if (!touched && heartbeat === holderLeaseHeartbeat) {
|
|
849
941
|
void release().catch(() => undefined);
|
|
850
942
|
}
|
|
851
943
|
})
|
|
@@ -869,7 +961,21 @@ export async function resumeBoxForTurn(
|
|
|
869
961
|
}
|
|
870
962
|
if (acquired.role === "fenced") {
|
|
871
963
|
await release();
|
|
872
|
-
|
|
964
|
+
if (acquired.reason !== "superseded") {
|
|
965
|
+
throw new SandboxLeaseTransitionError(
|
|
966
|
+
ids.sandboxGroupId,
|
|
967
|
+
acquired.lease.leaseEpoch,
|
|
968
|
+
acquired.reason,
|
|
969
|
+
acquired.lease.backend,
|
|
970
|
+
acquired.lease.instanceId,
|
|
971
|
+
acquired.lease.liveness,
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
|
|
975
|
+
backend: acquired.lease.backend,
|
|
976
|
+
instanceId: acquired.lease.instanceId,
|
|
977
|
+
liveness: acquired.lease.liveness,
|
|
978
|
+
});
|
|
873
979
|
}
|
|
874
980
|
|
|
875
981
|
// SPAWNER: we won the cold->warming CAS. Establish (cold-restore/create),
|
|
@@ -898,16 +1004,19 @@ export async function resumeBoxForTurn(
|
|
|
898
1004
|
const fallbackArchiveEnvelope =
|
|
899
1005
|
acquired.lease.recovery.archive.status === "none" &&
|
|
900
1006
|
workspaceArchiveFieldsFromEnvelope(envelope) !== null
|
|
901
|
-
?
|
|
1007
|
+
? withoutSandboxProviderIdentity(envelope)
|
|
902
1008
|
: null;
|
|
903
1009
|
let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
904
1010
|
const archiveSource =
|
|
905
1011
|
acquired.lease.recovery.archive.status === "none"
|
|
906
1012
|
? fallbackArchiveEnvelope
|
|
907
1013
|
: acquired.lease.resumeState;
|
|
1014
|
+
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
908
1015
|
if (
|
|
909
|
-
acquired.lease.recovery.archive.status === "available"
|
|
910
|
-
|
|
1016
|
+
(acquired.lease.recovery.archive.status === "available" &&
|
|
1017
|
+
acquired.lease.archiveComplete) ||
|
|
1018
|
+
(acquired.lease.recovery.archive.status === "none" &&
|
|
1019
|
+
workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
|
|
911
1020
|
) {
|
|
912
1021
|
const rematerializationId = crypto.randomUUID();
|
|
913
1022
|
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
@@ -950,7 +1059,7 @@ export async function resumeBoxForTurn(
|
|
|
950
1059
|
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
951
1060
|
legacyProviderBinding: null,
|
|
952
1061
|
};
|
|
953
|
-
} else if (acquired.lease.recovery.archive.status !== "none") {
|
|
1062
|
+
} else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
954
1063
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
955
1064
|
ids.sandboxGroupId,
|
|
956
1065
|
expectedEpoch,
|
|
@@ -974,6 +1083,9 @@ export async function resumeBoxForTurn(
|
|
|
974
1083
|
backendOverride: ids.backend as never,
|
|
975
1084
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
976
1085
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
1086
|
+
...(services.logicalFallbackSettings
|
|
1087
|
+
? { logicalFallbackSettings: services.logicalFallbackSettings }
|
|
1088
|
+
: {}),
|
|
977
1089
|
onSandboxCreated: async (created) => {
|
|
978
1090
|
createdEstablished = created;
|
|
979
1091
|
throwIfReleasedOrCancelled();
|
|
@@ -1027,6 +1139,9 @@ export async function resumeBoxForTurn(
|
|
|
1027
1139
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1028
1140
|
expectedEpoch,
|
|
1029
1141
|
rematerializationId: rematerialization?.id ?? null,
|
|
1142
|
+
...(created.providerContinuity
|
|
1143
|
+
? { continuityRecovery: created.providerContinuity }
|
|
1144
|
+
: {}),
|
|
1030
1145
|
instanceId: created.instanceId,
|
|
1031
1146
|
resumeBackendId: created.backendId,
|
|
1032
1147
|
resumeState: resumeEnvelope,
|
|
@@ -1041,7 +1156,7 @@ export async function resumeBoxForTurn(
|
|
|
1041
1156
|
leaseTtlMs,
|
|
1042
1157
|
// Keep the warming budget after create(): manifest setup and
|
|
1043
1158
|
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
1044
|
-
warmingLeaseTtlMs: settings.
|
|
1159
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
1045
1160
|
});
|
|
1046
1161
|
if (!recorded.recorded) {
|
|
1047
1162
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
@@ -1107,7 +1222,9 @@ export async function resumeBoxForTurn(
|
|
|
1107
1222
|
// is live. Do not publish a warm lease until one bounded no-op exec works.
|
|
1108
1223
|
// On timeout the catch below terminates the box and rolls warming -> cold,
|
|
1109
1224
|
// so the next turn cold-creates instead of hanging forever on first use.
|
|
1110
|
-
await waitForSandboxExecReadiness(established
|
|
1225
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
1226
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1227
|
+
});
|
|
1111
1228
|
throwIfReleasedOrCancelled();
|
|
1112
1229
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
1113
1230
|
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
@@ -1133,6 +1250,7 @@ export async function resumeBoxForTurn(
|
|
|
1133
1250
|
throwIfReleasedOrCancelled();
|
|
1134
1251
|
if (
|
|
1135
1252
|
rematerialization &&
|
|
1253
|
+
!established.providerContinuity &&
|
|
1136
1254
|
established.restoredArchive?.revision !== rematerialization.selectedRevision
|
|
1137
1255
|
) {
|
|
1138
1256
|
throw new WorkspaceArchiveIntegrityError(
|
|
@@ -1151,14 +1269,16 @@ export async function resumeBoxForTurn(
|
|
|
1151
1269
|
dataPlaneUrl: null,
|
|
1152
1270
|
resumeBackendId: established.backendId,
|
|
1153
1271
|
resumeState: resumeEnvelope,
|
|
1154
|
-
...(
|
|
1155
|
-
? {
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1272
|
+
...(established.providerContinuity
|
|
1273
|
+
? { continuityRecovery: established.providerContinuity }
|
|
1274
|
+
: rematerialization
|
|
1275
|
+
? {
|
|
1276
|
+
rematerialization: {
|
|
1277
|
+
id: rematerialization.id,
|
|
1278
|
+
verifiedRevision: rematerialization.selectedRevision,
|
|
1279
|
+
},
|
|
1280
|
+
}
|
|
1281
|
+
: {}),
|
|
1162
1282
|
leaseTtlMs,
|
|
1163
1283
|
});
|
|
1164
1284
|
if (!committed.committed || !committed.lease) {
|
|
@@ -1167,7 +1287,7 @@ export async function resumeBoxForTurn(
|
|
|
1167
1287
|
// holder; surface supersession. This spawner created the box, so stop it
|
|
1168
1288
|
// before retrying to avoid an untracked running sandbox.
|
|
1169
1289
|
const terminated = await terminateEstablishedSandbox(established);
|
|
1170
|
-
if (terminated && rematerialization) {
|
|
1290
|
+
if (terminated && rematerialization && !established.providerContinuity) {
|
|
1171
1291
|
await failSandboxRematerialization(db, {
|
|
1172
1292
|
accountId: ids.accountId,
|
|
1173
1293
|
workspaceId: ids.workspaceId,
|
|
@@ -1177,10 +1297,21 @@ export async function resumeBoxForTurn(
|
|
|
1177
1297
|
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
1178
1298
|
retryable: false,
|
|
1179
1299
|
});
|
|
1300
|
+
} else if (terminated) {
|
|
1301
|
+
await failWarmingToCold(db, {
|
|
1302
|
+
accountId: ids.accountId,
|
|
1303
|
+
workspaceId: ids.workspaceId,
|
|
1304
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1305
|
+
expectedEpoch,
|
|
1306
|
+
});
|
|
1180
1307
|
}
|
|
1181
1308
|
await release();
|
|
1182
1309
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1183
1310
|
}
|
|
1311
|
+
holderLeaseHeartbeat = {
|
|
1312
|
+
expectedEpoch: committed.lease.leaseEpoch,
|
|
1313
|
+
leaseTtlMs,
|
|
1314
|
+
};
|
|
1184
1315
|
throwIfReleasedOrCancelled();
|
|
1185
1316
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1186
1317
|
} catch (error) {
|
|
@@ -1196,18 +1327,25 @@ export async function resumeBoxForTurn(
|
|
|
1196
1327
|
// the warming row; the lease TTL/reaper and Modal orphan sweep are the
|
|
1197
1328
|
// tracked backstops, and we must not erase the only provider pointer.
|
|
1198
1329
|
if (terminated) {
|
|
1199
|
-
|
|
1330
|
+
const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
|
|
1331
|
+
if (rematerialization && !createdEstablished?.providerContinuity) {
|
|
1200
1332
|
await failSandboxRematerialization(db, {
|
|
1201
1333
|
accountId: ids.accountId,
|
|
1202
1334
|
workspaceId: ids.workspaceId,
|
|
1203
1335
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1204
1336
|
expectedEpoch,
|
|
1205
1337
|
rematerializationId: rematerialization.id,
|
|
1206
|
-
failureCode:
|
|
1207
|
-
|
|
1338
|
+
failureCode: continuityUnavailable
|
|
1339
|
+
? "provider_continuity_unavailable"
|
|
1340
|
+
: error instanceof WorkspaceArchiveIntegrityError
|
|
1208
1341
|
? error.code
|
|
1209
1342
|
: "sandbox_rematerialization_failed",
|
|
1210
|
-
retryable:
|
|
1343
|
+
retryable: continuityUnavailable
|
|
1344
|
+
? error.retryable
|
|
1345
|
+
: error instanceof WorkspaceArchiveIntegrityError
|
|
1346
|
+
? error.retryable
|
|
1347
|
+
: true,
|
|
1348
|
+
...(continuityUnavailable ? { discardContinuity: true } : {}),
|
|
1211
1349
|
});
|
|
1212
1350
|
} else {
|
|
1213
1351
|
await failWarmingToCold(db, {
|
|
@@ -1215,17 +1353,13 @@ export async function resumeBoxForTurn(
|
|
|
1215
1353
|
workspaceId: ids.workspaceId,
|
|
1216
1354
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1217
1355
|
expectedEpoch,
|
|
1356
|
+
...(continuityUnavailable ? { discardContinuity: true } : {}),
|
|
1218
1357
|
});
|
|
1219
1358
|
}
|
|
1220
1359
|
}
|
|
1221
1360
|
await release();
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
ids.backend,
|
|
1225
|
-
settings.sandboxWarmingTimeoutMs,
|
|
1226
|
-
);
|
|
1227
|
-
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1228
|
-
throw warmingError;
|
|
1361
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1362
|
+
throw error;
|
|
1229
1363
|
}
|
|
1230
1364
|
}
|
|
1231
1365
|
|
|
@@ -1238,6 +1372,7 @@ export async function resumeBoxForTurn(
|
|
|
1238
1372
|
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1239
1373
|
if (acquired.lease.liveness === "warming") {
|
|
1240
1374
|
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1375
|
+
holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
|
|
1241
1376
|
}
|
|
1242
1377
|
throwIfReleasedOrCancelled();
|
|
1243
1378
|
|
|
@@ -1270,7 +1405,13 @@ export async function resumeBoxForTurn(
|
|
|
1270
1405
|
// was idle. Prove the command router before handing the session to the
|
|
1271
1406
|
// agent so terminal evidence enters the atomic warm->cold recovery path
|
|
1272
1407
|
// below instead of surfacing inside a model-visible tool call.
|
|
1273
|
-
|
|
1408
|
+
if (services.verifyAttachedSandboxReadiness) {
|
|
1409
|
+
await services.verifyAttachedSandboxReadiness(established);
|
|
1410
|
+
} else {
|
|
1411
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
1412
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1274
1415
|
throwIfReleasedOrCancelled();
|
|
1275
1416
|
} catch (error) {
|
|
1276
1417
|
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
@@ -1304,13 +1445,8 @@ export async function resumeBoxForTurn(
|
|
|
1304
1445
|
return { established, leaseEpoch, release };
|
|
1305
1446
|
} catch (error) {
|
|
1306
1447
|
await release();
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
ids.backend,
|
|
1310
|
-
settings.sandboxWarmingTimeoutMs,
|
|
1311
|
-
);
|
|
1312
|
-
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1313
|
-
throw warmingError;
|
|
1448
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1449
|
+
throw error;
|
|
1314
1450
|
}
|
|
1315
1451
|
}
|
|
1316
1452
|
|
|
@@ -1326,6 +1462,7 @@ async function waitForWarm(
|
|
|
1326
1462
|
): Promise<{ leaseEpoch: number }> {
|
|
1327
1463
|
const { db, settings } = services;
|
|
1328
1464
|
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1465
|
+
let instanceId: string | null = null;
|
|
1329
1466
|
while (Date.now() < deadline) {
|
|
1330
1467
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1331
1468
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
@@ -1333,6 +1470,7 @@ async function waitForWarm(
|
|
|
1333
1470
|
// Lease vanished (cold-reaped). Re-dispatch from scratch.
|
|
1334
1471
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1335
1472
|
}
|
|
1473
|
+
instanceId = lease.instanceId;
|
|
1336
1474
|
if (lease.liveness === "warm") {
|
|
1337
1475
|
return { leaseEpoch: lease.leaseEpoch };
|
|
1338
1476
|
}
|
|
@@ -1368,5 +1506,8 @@ async function waitForWarm(
|
|
|
1368
1506
|
}
|
|
1369
1507
|
// still warming — keep polling.
|
|
1370
1508
|
}
|
|
1371
|
-
throw new
|
|
1509
|
+
throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
|
|
1510
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1511
|
+
instanceId,
|
|
1512
|
+
});
|
|
1372
1513
|
}
|