@opengeni/worker-bundle 0.12.7 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2730 -1211
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +874 -134
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
package/src/sandbox-resume.ts
CHANGED
|
@@ -20,21 +20,26 @@
|
|
|
20
20
|
//
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
|
-
import type
|
|
23
|
+
import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
|
|
24
24
|
import { redactSensitiveText } from "@opengeni/contracts";
|
|
25
|
+
import { randomUUID } from "node:crypto";
|
|
25
26
|
import {
|
|
26
27
|
acquireLease,
|
|
28
|
+
adoptLegacyModalCheckpointArtifact,
|
|
27
29
|
beginSandboxRematerialization,
|
|
30
|
+
claimWorkspaceArchiveCapture,
|
|
28
31
|
commitWarmingToWarm,
|
|
29
32
|
failSandboxRematerialization,
|
|
30
33
|
failWarmingToCold,
|
|
31
34
|
getSandboxSessionEnvelope,
|
|
32
35
|
markSandboxRestoreVerifying,
|
|
33
36
|
markWarmLeaseInstanceLost,
|
|
37
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
34
38
|
persistWarmSnapshot,
|
|
35
|
-
readWorkspaceArchiveCapturePreflight,
|
|
36
39
|
readLease,
|
|
40
|
+
registerSandboxCheckpointArtifact,
|
|
37
41
|
recordWarmingSandboxCreated,
|
|
42
|
+
releaseWorkspaceArchiveCapture,
|
|
38
43
|
releaseLeaseHolder,
|
|
39
44
|
touchLeaseHolder,
|
|
40
45
|
SandboxLeaseRecoveryBlockedError,
|
|
@@ -44,12 +49,15 @@ import {
|
|
|
44
49
|
} from "@opengeni/db";
|
|
45
50
|
import {
|
|
46
51
|
captureVerifiedWorkspaceArchive,
|
|
52
|
+
describeLegacyNativeSnapshotArchive,
|
|
47
53
|
WorkspaceArchiveIntegrityError,
|
|
48
54
|
establishSandboxSessionFromEnvelope,
|
|
49
55
|
isProviderSandboxNotFoundError,
|
|
56
|
+
parseWorkspaceArchiveDescriptor,
|
|
50
57
|
requirePersistableReplacementSandboxEnvelope,
|
|
58
|
+
modalSessionMatchesCheckpointProviderBinding,
|
|
59
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
51
60
|
serializeReplacementSandboxEnvelope,
|
|
52
|
-
deletePriorPersistedSnapshot,
|
|
53
61
|
tagModalSandbox,
|
|
54
62
|
verifySandboxExecReadiness,
|
|
55
63
|
type EstablishedSandboxSession,
|
|
@@ -89,6 +97,18 @@ export type SandboxResumeServices = {
|
|
|
89
97
|
db: Database;
|
|
90
98
|
settings: Settings;
|
|
91
99
|
sandboxMetrics?: RuntimeMetricsHooks;
|
|
100
|
+
/**
|
|
101
|
+
* The logical turn-attempt lifetime, not merely the provider request lifetime.
|
|
102
|
+
* Some provider SDK calls cannot be interrupted. Aborting this signal still
|
|
103
|
+
* releases the durable holder immediately and fences any late provider result.
|
|
104
|
+
*/
|
|
105
|
+
cancellationSignal?: AbortSignal;
|
|
106
|
+
/** Test seam for the attached/resumed path. Production uses the one runtime
|
|
107
|
+
* resume primitive; callers must not use this to create a replacement box. */
|
|
108
|
+
establishAttachedSandbox?: typeof establishSandboxSessionFromEnvelope;
|
|
109
|
+
/** Test seam for the bounded command-readiness proof performed before an
|
|
110
|
+
* attached provider box is handed to the agent. */
|
|
111
|
+
verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
|
|
92
112
|
/** Called only by the observer that wins the exact warm->cold loss CAS. */
|
|
93
113
|
onSandboxLost?: (input: {
|
|
94
114
|
sandboxGroupId: string;
|
|
@@ -122,17 +142,18 @@ export type ResumeBoxIds = {
|
|
|
122
142
|
/**
|
|
123
143
|
* IMAGE IS SHARED STATE (B3): the container image this run resolves (Modal image ref
|
|
124
144
|
* / docker image). Threaded to acquireLease, which stamps it on the cold-create and
|
|
125
|
-
* conflicts on a live box already running a DIFFERENT image (solo holder
|
|
126
|
-
* N-holders throw SandboxImageConflictError). Omitted ->
|
|
127
|
-
* selfhosted path never passes it; a legacy/null-image box
|
|
145
|
+
* conflicts on a live box already running a DIFFERENT image (a solo holder requests
|
|
146
|
+
* capture-and-drain rotation; N-holders throw SandboxImageConflictError). Omitted ->
|
|
147
|
+
* image is not enforced (the selfhosted path never passes it; a legacy/null-image box
|
|
148
|
+
* never conflicts).
|
|
128
149
|
*/
|
|
129
150
|
image?: string;
|
|
130
151
|
/**
|
|
131
152
|
* RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
|
|
132
153
|
* acquireLease, which stamps it on the cold-create and conflicts on a live box
|
|
133
|
-
* set up under a DIFFERENT rig version (solo holder
|
|
134
|
-
*
|
|
135
|
-
* -> rig is never stamped or enforced (shares exactly as today).
|
|
154
|
+
* set up under a DIFFERENT rig version (a solo holder requests capture-and-drain
|
|
155
|
+
* rotation; N-holders throw SandboxRigConflictError). Omitted for a rig-less
|
|
156
|
+
* session -> rig is never stamped or enforced (shares exactly as today).
|
|
136
157
|
*/
|
|
137
158
|
rigVersionId?: string;
|
|
138
159
|
};
|
|
@@ -273,6 +294,44 @@ export async function waitForWarmSnapshot(
|
|
|
273
294
|
}
|
|
274
295
|
}
|
|
275
296
|
|
|
297
|
+
async function resolveModalCheckpointBindingBeforeCapture(
|
|
298
|
+
settings: SandboxResumeServices["settings"],
|
|
299
|
+
session: unknown,
|
|
300
|
+
signal?: AbortSignal,
|
|
301
|
+
): Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>> {
|
|
302
|
+
if (signal?.aborted) {
|
|
303
|
+
throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
|
|
304
|
+
}
|
|
305
|
+
const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
|
|
306
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
307
|
+
let cancelListener: (() => void) | undefined;
|
|
308
|
+
try {
|
|
309
|
+
return await Promise.race([
|
|
310
|
+
pending,
|
|
311
|
+
...(signal
|
|
312
|
+
? [
|
|
313
|
+
new Promise<never>((_resolve, reject) => {
|
|
314
|
+
cancelListener = () =>
|
|
315
|
+
reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
|
|
316
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
317
|
+
if (signal.aborted) cancelListener();
|
|
318
|
+
}),
|
|
319
|
+
]
|
|
320
|
+
: []),
|
|
321
|
+
new Promise<never>((_resolve, reject) => {
|
|
322
|
+
timeout = setTimeout(
|
|
323
|
+
() => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
324
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
325
|
+
);
|
|
326
|
+
timeout.unref?.();
|
|
327
|
+
}),
|
|
328
|
+
]);
|
|
329
|
+
} finally {
|
|
330
|
+
if (timeout) clearTimeout(timeout);
|
|
331
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
276
335
|
async function terminateEstablishedSandbox(
|
|
277
336
|
established: EstablishedSandboxSession | null,
|
|
278
337
|
): Promise<boolean> {
|
|
@@ -371,6 +430,20 @@ function workspaceArchiveFieldsFromEnvelope(
|
|
|
371
430
|
};
|
|
372
431
|
}
|
|
373
432
|
|
|
433
|
+
function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined) {
|
|
434
|
+
const sessionState =
|
|
435
|
+
envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
436
|
+
? (envelope.sessionState as Record<string, unknown>)
|
|
437
|
+
: null;
|
|
438
|
+
if (!sessionState) return null;
|
|
439
|
+
const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
440
|
+
if (existing?.version === 2) return null;
|
|
441
|
+
return describeLegacyNativeSnapshotArchive(
|
|
442
|
+
sessionState.workspaceArchive,
|
|
443
|
+
existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now(),
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
|
|
374
447
|
/** A per-session fallback may still carry the dead provider identity that made
|
|
375
448
|
* recovery necessary. It is useful only as durable archive/config input: never
|
|
376
449
|
* let a cold rematerialization resume that stale provider before hydrating the
|
|
@@ -419,14 +492,16 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
419
492
|
session: unknown,
|
|
420
493
|
leaseEpoch: number,
|
|
421
494
|
signal?: AbortSignal,
|
|
495
|
+
force = false,
|
|
422
496
|
): Promise<boolean> {
|
|
423
497
|
const { db, settings } = services;
|
|
424
498
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
425
|
-
if (intervalMs <= 0) {
|
|
499
|
+
if (intervalMs <= 0 && !force) {
|
|
426
500
|
return false;
|
|
427
501
|
}
|
|
428
502
|
const persistable = session as {
|
|
429
503
|
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
504
|
+
state?: { workspacePersistence?: unknown };
|
|
430
505
|
};
|
|
431
506
|
if (typeof persistable.persistWorkspace !== "function") {
|
|
432
507
|
return false;
|
|
@@ -435,9 +510,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
435
510
|
return false;
|
|
436
511
|
}
|
|
437
512
|
try {
|
|
438
|
-
//
|
|
439
|
-
//
|
|
440
|
-
//
|
|
513
|
+
// Resolve the provider namespace before taking the durable capture gate.
|
|
514
|
+
// This work cannot pause or mutate the box, so keeping it outside the claim
|
|
515
|
+
// minimizes the time that command admission must wait.
|
|
441
516
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
442
517
|
if (
|
|
443
518
|
!lease ||
|
|
@@ -447,99 +522,163 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
447
522
|
) {
|
|
448
523
|
return false;
|
|
449
524
|
}
|
|
450
|
-
const
|
|
525
|
+
const instanceId = lease.instanceId;
|
|
526
|
+
const nativeModalPersistence =
|
|
527
|
+
persistable.state?.workspacePersistence === "snapshot_filesystem" ||
|
|
528
|
+
persistable.state?.workspacePersistence === "snapshot_directory";
|
|
529
|
+
const checkpointBinding = nativeModalPersistence
|
|
530
|
+
? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
|
|
531
|
+
: null;
|
|
532
|
+
if (signal?.aborted) return false;
|
|
533
|
+
|
|
534
|
+
const captureId = randomUUID();
|
|
535
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
536
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
451
537
|
accountId: ids.accountId,
|
|
452
538
|
workspaceId: ids.workspaceId,
|
|
453
539
|
sandboxGroupId: ids.sandboxGroupId,
|
|
540
|
+
captureId,
|
|
454
541
|
expectedEpoch: leaseEpoch,
|
|
455
|
-
expectedInstanceId:
|
|
542
|
+
expectedInstanceId: instanceId,
|
|
456
543
|
liveness: "warm",
|
|
544
|
+
captureTimeoutMs,
|
|
545
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
546
|
+
warmAttempt: {
|
|
547
|
+
sessionId: ids.sessionId,
|
|
548
|
+
turnId: ids.turnId,
|
|
549
|
+
attemptId: ids.attemptId,
|
|
550
|
+
holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
|
|
551
|
+
},
|
|
457
552
|
});
|
|
458
|
-
if (
|
|
459
|
-
|
|
460
|
-
return false;
|
|
461
|
-
}
|
|
462
|
-
const sessionState =
|
|
463
|
-
lease.resumeState && typeof lease.resumeState === "object"
|
|
464
|
-
? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
|
|
465
|
-
: undefined;
|
|
466
|
-
const priorAtRaw =
|
|
467
|
-
sessionState && typeof sessionState === "object"
|
|
468
|
-
? sessionState.workspaceArchiveAt
|
|
469
|
-
: undefined;
|
|
470
|
-
const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
|
|
471
|
-
if (
|
|
472
|
-
preflight.archiveComplete &&
|
|
473
|
-
Number.isFinite(priorAtMs) &&
|
|
474
|
-
Date.now() - priorAtMs < intervalMs
|
|
475
|
-
) {
|
|
476
|
-
return false;
|
|
477
|
-
}
|
|
553
|
+
if (claimed.status !== "claimed") return false;
|
|
554
|
+
|
|
478
555
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
479
556
|
// Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
|
|
480
557
|
// by capture-initiation, not land time, so a slower heartbeat capture that
|
|
481
558
|
// started earlier can never overwrite a fresher turn-end capture that landed
|
|
482
559
|
// first (the bounded-wait race Bugbot flagged).
|
|
483
560
|
const capturedAtMs = Date.now();
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
561
|
+
const registerCandidate = async (
|
|
562
|
+
archive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>>,
|
|
563
|
+
): Promise<{ id: string } | null> => {
|
|
564
|
+
if (
|
|
565
|
+
archive.descriptor.version !== 2 ||
|
|
566
|
+
(archive.descriptor.provider !== "modal_snapshot_filesystem" &&
|
|
567
|
+
archive.descriptor.provider !== "modal_snapshot_directory")
|
|
568
|
+
) {
|
|
569
|
+
return null;
|
|
570
|
+
}
|
|
571
|
+
if (!checkpointBinding) {
|
|
572
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
573
|
+
}
|
|
574
|
+
return await registerSandboxCheckpointArtifact(db, {
|
|
575
|
+
accountId: ids.accountId,
|
|
576
|
+
workspaceId: ids.workspaceId,
|
|
577
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
578
|
+
sourceLeaseId: claimed.claim.leaseId,
|
|
579
|
+
sourceLeaseEpoch: leaseEpoch,
|
|
580
|
+
sourceInstanceId: instanceId,
|
|
581
|
+
sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
582
|
+
providerBindingKey: checkpointBinding.key,
|
|
583
|
+
providerBinding: checkpointBinding.binding,
|
|
584
|
+
workspaceArchive: archive.base64,
|
|
585
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
586
|
+
});
|
|
587
|
+
};
|
|
588
|
+
|
|
589
|
+
const abandonCandidate = async (artifactId: string, reason: string): Promise<void> => {
|
|
590
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
591
|
+
accountId: ids.accountId,
|
|
592
|
+
workspaceId: ids.workspaceId,
|
|
593
|
+
artifactId,
|
|
594
|
+
reason,
|
|
595
|
+
}).catch(() => undefined);
|
|
596
|
+
};
|
|
597
|
+
// The SDK capture itself is not cancellable. Keep one owned continuation
|
|
598
|
+
// alive through provider settlement even if this caller's bounded wait or
|
|
599
|
+
// turn signal resolves first. Its finally block is the only normal release
|
|
600
|
+
// of the exact admission gate; a late callback cannot release a successor.
|
|
601
|
+
const captureAndPublish = (async (): Promise<boolean> => {
|
|
602
|
+
let candidate: { id: string } | null = null;
|
|
603
|
+
try {
|
|
604
|
+
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs);
|
|
605
|
+
candidate = await registerCandidate(archive);
|
|
606
|
+
const { wrote } = await persistWarmSnapshot(db, {
|
|
607
|
+
accountId: ids.accountId,
|
|
608
|
+
workspaceId: ids.workspaceId,
|
|
609
|
+
sessionId: ids.sessionId,
|
|
610
|
+
turnId: ids.turnId,
|
|
611
|
+
attemptId: ids.attemptId,
|
|
612
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
613
|
+
expectedEpoch: leaseEpoch,
|
|
614
|
+
expectedInstanceId: instanceId,
|
|
615
|
+
expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
616
|
+
captureId,
|
|
617
|
+
workspaceArchive: archive.base64,
|
|
618
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
619
|
+
checkpointArtifactId: candidate?.id ?? null,
|
|
620
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
621
|
+
capturedAtMs,
|
|
622
|
+
});
|
|
623
|
+
if (!wrote && candidate) {
|
|
624
|
+
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
625
|
+
}
|
|
626
|
+
return wrote;
|
|
627
|
+
} catch (error) {
|
|
628
|
+
if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
629
|
+
throw error;
|
|
630
|
+
} finally {
|
|
631
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
632
|
+
accountId: ids.accountId,
|
|
633
|
+
workspaceId: ids.workspaceId,
|
|
634
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
635
|
+
captureId,
|
|
636
|
+
expectedEpoch: leaseEpoch,
|
|
637
|
+
expectedInstanceId: instanceId,
|
|
638
|
+
}).catch((error) => {
|
|
639
|
+
console.error(
|
|
640
|
+
"mid-session workspace capture gate release failed",
|
|
641
|
+
safeSnapshotError(error),
|
|
642
|
+
);
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
})();
|
|
646
|
+
const settled = captureAndPublish.then(
|
|
647
|
+
(persisted) => ({ kind: "settled" as const, persisted }),
|
|
648
|
+
(error) => {
|
|
649
|
+
console.error(
|
|
650
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
651
|
+
safeSnapshotError(error),
|
|
652
|
+
);
|
|
653
|
+
return { kind: "settled" as const, persisted: false };
|
|
654
|
+
},
|
|
655
|
+
);
|
|
491
656
|
let cancelListener: (() => void) | undefined;
|
|
492
|
-
const
|
|
493
|
-
|
|
657
|
+
const outcome = await Promise.race([
|
|
658
|
+
settled,
|
|
494
659
|
...(signal
|
|
495
660
|
? [
|
|
496
|
-
new Promise<
|
|
497
|
-
cancelListener = () => resolve(
|
|
661
|
+
new Promise<{ kind: "cancelled" }>((resolve) => {
|
|
662
|
+
cancelListener = () => resolve({ kind: "cancelled" });
|
|
498
663
|
signal.addEventListener("abort", cancelListener, { once: true });
|
|
499
664
|
if (signal.aborted) cancelListener();
|
|
500
665
|
}),
|
|
501
666
|
]
|
|
502
667
|
: []),
|
|
503
|
-
new Promise<
|
|
504
|
-
timeout = setTimeout(
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
668
|
+
new Promise<{ kind: "timed_out" }>((resolve) => {
|
|
669
|
+
timeout = setTimeout(
|
|
670
|
+
() => resolve({ kind: "timed_out" }),
|
|
671
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
672
|
+
);
|
|
508
673
|
}),
|
|
509
674
|
]).finally(() => {
|
|
510
|
-
if (timeout)
|
|
511
|
-
clearTimeout(timeout);
|
|
512
|
-
}
|
|
675
|
+
if (timeout) clearTimeout(timeout);
|
|
513
676
|
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
514
677
|
});
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
return false;
|
|
520
|
-
}
|
|
521
|
-
const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
|
|
522
|
-
accountId: ids.accountId,
|
|
523
|
-
workspaceId: ids.workspaceId,
|
|
524
|
-
sessionId: ids.sessionId,
|
|
525
|
-
turnId: ids.turnId,
|
|
526
|
-
attemptId: ids.attemptId,
|
|
527
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
528
|
-
expectedEpoch: leaseEpoch,
|
|
529
|
-
expectedInstanceId: lease.instanceId,
|
|
530
|
-
expectedWorkspaceGeneration: preflight.workspaceGeneration,
|
|
531
|
-
workspaceArchive: archive.base64,
|
|
532
|
-
workspaceArchiveMeta: archive.descriptor,
|
|
533
|
-
minIntervalMs: intervalMs,
|
|
534
|
-
capturedAtMs,
|
|
535
|
-
});
|
|
536
|
-
if (!wrote) {
|
|
537
|
-
return false;
|
|
538
|
-
}
|
|
539
|
-
// Warm snapshots retain a 2-deep restore window. Only the two-ago archive
|
|
540
|
-
// returned by persistWarmSnapshot is GC-eligible.
|
|
541
|
-
await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
|
|
542
|
-
return true;
|
|
678
|
+
// A timed-out/cancelled local waiter deliberately leaves the already-owned
|
|
679
|
+
// provider continuation running; `settled` has both rejection handling and
|
|
680
|
+
// exact claim cleanup, so it cannot leak an unhandled promise or artifact.
|
|
681
|
+
return outcome.kind === "settled" ? outcome.persisted : false;
|
|
543
682
|
} catch (error) {
|
|
544
683
|
// Protection, not a dependency: a failed snapshot must never fail (or slow
|
|
545
684
|
// down retrying) the turn. The next heartbeat/turn-end tick retries.
|
|
@@ -570,18 +709,28 @@ export async function resumeBoxForTurn(
|
|
|
570
709
|
const { db, settings } = services;
|
|
571
710
|
const os = ids.os ?? "linux";
|
|
572
711
|
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
712
|
+
const cancellationSignal = services.cancellationSignal;
|
|
573
713
|
|
|
574
714
|
// The release closure is created eagerly so the caller can always release in
|
|
575
715
|
// finally, even if establish/commit throws after the holder was registered.
|
|
716
|
+
// It is also bound directly to the logical attempt signal: an uninterruptible
|
|
717
|
+
// provider promise must never keep its private warmup timer and holder alive
|
|
718
|
+
// after Temporal has abandoned the activity.
|
|
576
719
|
let released = false;
|
|
577
720
|
let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
|
|
721
|
+
let cancellationListener: (() => void) | undefined;
|
|
578
722
|
const release = async (): Promise<void> => {
|
|
579
723
|
if (released) {
|
|
580
724
|
return;
|
|
581
725
|
}
|
|
582
726
|
released = true;
|
|
727
|
+
if (cancellationListener) {
|
|
728
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
729
|
+
cancellationListener = undefined;
|
|
730
|
+
}
|
|
583
731
|
if (holderLivenessTimer) {
|
|
584
732
|
clearInterval(holderLivenessTimer);
|
|
733
|
+
holderLivenessTimer = undefined;
|
|
585
734
|
}
|
|
586
735
|
await releaseLeaseHolder(db, {
|
|
587
736
|
accountId: ids.accountId,
|
|
@@ -592,6 +741,15 @@ export async function resumeBoxForTurn(
|
|
|
592
741
|
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
593
742
|
});
|
|
594
743
|
};
|
|
744
|
+
const cancellationError = (): Error =>
|
|
745
|
+
cancellationSignal?.reason instanceof Error
|
|
746
|
+
? cancellationSignal.reason
|
|
747
|
+
: new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
748
|
+
const throwIfReleasedOrCancelled = (): void => {
|
|
749
|
+
if (released || cancellationSignal?.aborted) {
|
|
750
|
+
throw cancellationError();
|
|
751
|
+
}
|
|
752
|
+
};
|
|
595
753
|
|
|
596
754
|
const acquired = await acquireLease(db, {
|
|
597
755
|
accountId: ids.accountId,
|
|
@@ -603,18 +761,35 @@ export async function resumeBoxForTurn(
|
|
|
603
761
|
backend: ids.backend,
|
|
604
762
|
os,
|
|
605
763
|
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
606
|
-
// conflicts on a live box already running a different image. A
|
|
607
|
-
// propagates
|
|
608
|
-
//
|
|
764
|
+
// conflicts on a live box already running a different image. A
|
|
765
|
+
// SandboxImageConflictError propagates while another holder is active; a
|
|
766
|
+
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
767
|
+
// after the cold successor can safely stamp the new image.
|
|
609
768
|
...(ids.image ? { image: ids.image } : {}),
|
|
610
769
|
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
611
770
|
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
612
|
-
// propagates
|
|
771
|
+
// propagates while another holder is active; a solo change uses the same
|
|
772
|
+
// durable capture-and-drain rotation as an image change.
|
|
613
773
|
...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
|
|
614
774
|
leaseTtlMs,
|
|
615
775
|
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
|
|
776
|
+
captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
|
|
616
777
|
});
|
|
617
778
|
|
|
779
|
+
if (cancellationSignal) {
|
|
780
|
+
cancellationListener = () => {
|
|
781
|
+
// release() flips `released` and clears the timer synchronously before its
|
|
782
|
+
// first await. The detached rejection handler is intentional: the outer
|
|
783
|
+
// turn owns diagnostics, while this listener owns leak prevention.
|
|
784
|
+
void release().catch(() => undefined);
|
|
785
|
+
};
|
|
786
|
+
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
787
|
+
}
|
|
788
|
+
if (cancellationSignal?.aborted) {
|
|
789
|
+
await release();
|
|
790
|
+
throw cancellationError();
|
|
791
|
+
}
|
|
792
|
+
|
|
618
793
|
// HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
|
|
619
794
|
// registered until release. The dead-worker turn-holder reap judges liveness
|
|
620
795
|
// by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
|
|
@@ -632,7 +807,16 @@ export async function resumeBoxForTurn(
|
|
|
632
807
|
sandboxGroupId: ids.sandboxGroupId,
|
|
633
808
|
kind,
|
|
634
809
|
holderId,
|
|
635
|
-
})
|
|
810
|
+
})
|
|
811
|
+
.then((touched) => {
|
|
812
|
+
// A canonical turn holder is rejected once its exact attempt is no
|
|
813
|
+
// longer the active writer. Stop this otherwise-unbounded provider
|
|
814
|
+
// operation and idempotently drop any remaining holder state.
|
|
815
|
+
if (!touched) {
|
|
816
|
+
void release().catch(() => undefined);
|
|
817
|
+
}
|
|
818
|
+
})
|
|
819
|
+
.catch(() => undefined);
|
|
636
820
|
}, 10_000);
|
|
637
821
|
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
638
822
|
holderLivenessTimer.unref();
|
|
@@ -663,6 +847,12 @@ export async function resumeBoxForTurn(
|
|
|
663
847
|
let rematerialization: {
|
|
664
848
|
id: string;
|
|
665
849
|
selectedRevision: string;
|
|
850
|
+
workspaceGeneration: number;
|
|
851
|
+
providerBindingKey: string | null;
|
|
852
|
+
legacyCheckpoint: NonNullable<ReturnType<typeof legacyNativeArchiveFromEnvelope>> | null;
|
|
853
|
+
legacyProviderBinding: Awaited<
|
|
854
|
+
ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
|
|
855
|
+
> | null;
|
|
666
856
|
} | null = null;
|
|
667
857
|
try {
|
|
668
858
|
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
@@ -677,7 +867,7 @@ export async function resumeBoxForTurn(
|
|
|
677
867
|
workspaceArchiveFieldsFromEnvelope(envelope) !== null
|
|
678
868
|
? withoutProviderIdentity(envelope)
|
|
679
869
|
: null;
|
|
680
|
-
|
|
870
|
+
let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
681
871
|
const archiveSource =
|
|
682
872
|
acquired.lease.recovery.archive.status === "none"
|
|
683
873
|
? fallbackArchiveEnvelope
|
|
@@ -687,6 +877,7 @@ export async function resumeBoxForTurn(
|
|
|
687
877
|
workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
|
|
688
878
|
) {
|
|
689
879
|
const rematerializationId = crypto.randomUUID();
|
|
880
|
+
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
690
881
|
const begun = await beginSandboxRematerialization(db, {
|
|
691
882
|
accountId: ids.accountId,
|
|
692
883
|
workspaceId: ids.workspaceId,
|
|
@@ -694,6 +885,7 @@ export async function resumeBoxForTurn(
|
|
|
694
885
|
expectedEpoch,
|
|
695
886
|
rematerializationId,
|
|
696
887
|
archiveSource,
|
|
888
|
+
legacyNativeArchive,
|
|
697
889
|
});
|
|
698
890
|
if (begun.status !== "started") {
|
|
699
891
|
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
@@ -709,6 +901,7 @@ export async function resumeBoxForTurn(
|
|
|
709
901
|
begun.lease?.recovery ?? acquired.lease.recovery,
|
|
710
902
|
);
|
|
711
903
|
}
|
|
904
|
+
spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
|
|
712
905
|
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
713
906
|
if (!selectedRevision) {
|
|
714
907
|
throw new WorkspaceArchiveIntegrityError(
|
|
@@ -716,7 +909,14 @@ export async function resumeBoxForTurn(
|
|
|
716
909
|
"sandbox rematerialization selected no durable archive revision",
|
|
717
910
|
);
|
|
718
911
|
}
|
|
719
|
-
rematerialization = {
|
|
912
|
+
rematerialization = {
|
|
913
|
+
id: rematerializationId,
|
|
914
|
+
selectedRevision,
|
|
915
|
+
workspaceGeneration: begun.lease.workspaceGeneration,
|
|
916
|
+
providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
|
|
917
|
+
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
918
|
+
legacyProviderBinding: null,
|
|
919
|
+
};
|
|
720
920
|
} else if (acquired.lease.recovery.archive.status !== "none") {
|
|
721
921
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
722
922
|
ids.sandboxGroupId,
|
|
@@ -734,6 +934,7 @@ export async function resumeBoxForTurn(
|
|
|
734
934
|
// spawner branch: the lease's resume_state is authoritative; the session
|
|
735
935
|
// `_sandbox` envelope is the per-session fallback. Without this a turn-first
|
|
736
936
|
// re-warm after a drain->cold would ignore the archive and start an EMPTY box.
|
|
937
|
+
const providerCreateStartedAt = new Date();
|
|
737
938
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
738
939
|
sessionId: ids.sessionId,
|
|
739
940
|
recovery: "create-or-restore",
|
|
@@ -742,6 +943,47 @@ export async function resumeBoxForTurn(
|
|
|
742
943
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
743
944
|
onSandboxCreated: async (created) => {
|
|
744
945
|
createdEstablished = created;
|
|
946
|
+
throwIfReleasedOrCancelled();
|
|
947
|
+
if (
|
|
948
|
+
rematerialization &&
|
|
949
|
+
(rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)
|
|
950
|
+
) {
|
|
951
|
+
if (created.backendId !== "modal") {
|
|
952
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
953
|
+
"native_snapshot_reference_invalid",
|
|
954
|
+
"Modal checkpoint restore resolved a non-Modal sandbox backend",
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
if (rematerialization.providerBindingKey) {
|
|
958
|
+
if (
|
|
959
|
+
!(await modalSessionMatchesCheckpointProviderBinding(
|
|
960
|
+
settings,
|
|
961
|
+
created.session,
|
|
962
|
+
rematerialization.providerBindingKey,
|
|
963
|
+
))
|
|
964
|
+
) {
|
|
965
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
966
|
+
"native_snapshot_reference_invalid",
|
|
967
|
+
"Modal checkpoint restore refused because the authenticated provider workspace changed",
|
|
968
|
+
);
|
|
969
|
+
}
|
|
970
|
+
} else {
|
|
971
|
+
const resolved = await resolveModalCheckpointProviderBindingForSession(
|
|
972
|
+
settings,
|
|
973
|
+
created.session,
|
|
974
|
+
);
|
|
975
|
+
if (
|
|
976
|
+
rematerialization.legacyProviderBinding &&
|
|
977
|
+
rematerialization.legacyProviderBinding.key !== resolved.key
|
|
978
|
+
) {
|
|
979
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
980
|
+
"native_snapshot_reference_invalid",
|
|
981
|
+
"Legacy Modal checkpoint restore crossed authenticated provider workspaces",
|
|
982
|
+
);
|
|
983
|
+
}
|
|
984
|
+
rematerialization.legacyProviderBinding = resolved;
|
|
985
|
+
}
|
|
986
|
+
}
|
|
745
987
|
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
746
988
|
await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
|
|
747
989
|
created.backendId,
|
|
@@ -755,6 +997,14 @@ export async function resumeBoxForTurn(
|
|
|
755
997
|
instanceId: created.instanceId,
|
|
756
998
|
resumeBackendId: created.backendId,
|
|
757
999
|
resumeState: resumeEnvelope,
|
|
1000
|
+
...(created.backendId === "modal"
|
|
1001
|
+
? {
|
|
1002
|
+
providerCreatedAt: providerCreateStartedAt,
|
|
1003
|
+
providerDeadlineAt: new Date(
|
|
1004
|
+
providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1000,
|
|
1005
|
+
),
|
|
1006
|
+
}
|
|
1007
|
+
: {}),
|
|
758
1008
|
leaseTtlMs,
|
|
759
1009
|
// Keep the warming budget after create(): manifest setup and
|
|
760
1010
|
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
@@ -770,6 +1020,7 @@ export async function resumeBoxForTurn(
|
|
|
770
1020
|
sandboxGroupId: ids.sandboxGroupId,
|
|
771
1021
|
}).catch(() => undefined);
|
|
772
1022
|
}
|
|
1023
|
+
throwIfReleasedOrCancelled();
|
|
773
1024
|
},
|
|
774
1025
|
onWorkspaceRestoreVerifying: async (descriptor: WorkspaceArchiveDescriptor) => {
|
|
775
1026
|
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
@@ -778,6 +1029,33 @@ export async function resumeBoxForTurn(
|
|
|
778
1029
|
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`,
|
|
779
1030
|
);
|
|
780
1031
|
}
|
|
1032
|
+
if (rematerialization.legacyCheckpoint) {
|
|
1033
|
+
const binding = rematerialization.legacyProviderBinding;
|
|
1034
|
+
if (!binding) {
|
|
1035
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1036
|
+
"native_snapshot_reference_invalid",
|
|
1037
|
+
"Legacy Modal checkpoint restore produced no authenticated provider identity",
|
|
1038
|
+
);
|
|
1039
|
+
}
|
|
1040
|
+
const adopted = await adoptLegacyModalCheckpointArtifact(db, {
|
|
1041
|
+
accountId: ids.accountId,
|
|
1042
|
+
workspaceId: ids.workspaceId,
|
|
1043
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1044
|
+
leaseId: acquired.lease.id,
|
|
1045
|
+
leaseEpoch: expectedEpoch,
|
|
1046
|
+
workspaceGeneration: rematerialization.workspaceGeneration,
|
|
1047
|
+
slot: "current",
|
|
1048
|
+
archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
|
|
1049
|
+
descriptor: rematerialization.legacyCheckpoint.descriptor,
|
|
1050
|
+
providerBindingKey: binding.key,
|
|
1051
|
+
providerBinding: binding.binding,
|
|
1052
|
+
rematerializationId: rematerialization.id,
|
|
1053
|
+
});
|
|
1054
|
+
if (!adopted) {
|
|
1055
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1056
|
+
}
|
|
1057
|
+
rematerialization.providerBindingKey = binding.key;
|
|
1058
|
+
}
|
|
781
1059
|
const verifying = await markSandboxRestoreVerifying(db, {
|
|
782
1060
|
accountId: ids.accountId,
|
|
783
1061
|
workspaceId: ids.workspaceId,
|
|
@@ -791,11 +1069,13 @@ export async function resumeBoxForTurn(
|
|
|
791
1069
|
},
|
|
792
1070
|
});
|
|
793
1071
|
createdEstablished = established;
|
|
1072
|
+
throwIfReleasedOrCancelled();
|
|
794
1073
|
// A sandbox handle is not sufficient evidence that Modal's command router
|
|
795
1074
|
// is live. Do not publish a warm lease until one bounded no-op exec works.
|
|
796
1075
|
// On timeout the catch below terminates the box and rolls warming -> cold,
|
|
797
1076
|
// so the next turn cold-creates instead of hanging forever on first use.
|
|
798
1077
|
await waitForSandboxExecReadiness(established);
|
|
1078
|
+
throwIfReleasedOrCancelled();
|
|
799
1079
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
800
1080
|
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
801
1081
|
// viewer.ts). Without this the turn committed the ORIGINAL session manifest
|
|
@@ -817,6 +1097,7 @@ export async function resumeBoxForTurn(
|
|
|
817
1097
|
await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
|
|
818
1098
|
established.backendId,
|
|
819
1099
|
);
|
|
1100
|
+
throwIfReleasedOrCancelled();
|
|
820
1101
|
if (
|
|
821
1102
|
rematerialization &&
|
|
822
1103
|
established.restoredArchive?.revision !== rematerialization.selectedRevision
|
|
@@ -867,6 +1148,7 @@ export async function resumeBoxForTurn(
|
|
|
867
1148
|
await release();
|
|
868
1149
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
869
1150
|
}
|
|
1151
|
+
throwIfReleasedOrCancelled();
|
|
870
1152
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
871
1153
|
} catch (error) {
|
|
872
1154
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
@@ -924,6 +1206,7 @@ export async function resumeBoxForTurn(
|
|
|
924
1206
|
if (acquired.lease.liveness === "warming") {
|
|
925
1207
|
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
926
1208
|
}
|
|
1209
|
+
throwIfReleasedOrCancelled();
|
|
927
1210
|
|
|
928
1211
|
// Prefer the lease's resume_state (the LIVE box the spawner committed) so we
|
|
929
1212
|
// re-attach to the SAME box by id, not cold-restore the original session
|
|
@@ -940,13 +1223,22 @@ export async function resumeBoxForTurn(
|
|
|
940
1223
|
}
|
|
941
1224
|
let established: EstablishedSandboxSession;
|
|
942
1225
|
try {
|
|
943
|
-
|
|
1226
|
+
const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
|
|
1227
|
+
established = await establish(settings, live.resumeState, {
|
|
944
1228
|
sessionId: ids.sessionId,
|
|
945
1229
|
recovery: "resume-only",
|
|
946
1230
|
backendOverride: ids.backend as never,
|
|
947
1231
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
948
1232
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
949
1233
|
});
|
|
1234
|
+
throwIfReleasedOrCancelled();
|
|
1235
|
+
// A durable `warm` row is an ownership assertion, not provider liveness.
|
|
1236
|
+
// Modal may have ended the exact box at its finite timeout while OpenGeni
|
|
1237
|
+
// was idle. Prove the command router before handing the session to the
|
|
1238
|
+
// agent so terminal evidence enters the atomic warm->cold recovery path
|
|
1239
|
+
// below instead of surfacing inside a model-visible tool call.
|
|
1240
|
+
await (services.verifyAttachedSandboxReadiness ?? waitForSandboxExecReadiness)(established);
|
|
1241
|
+
throwIfReleasedOrCancelled();
|
|
950
1242
|
} catch (error) {
|
|
951
1243
|
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
952
1244
|
throw error;
|
|
@@ -975,6 +1267,7 @@ export async function resumeBoxForTurn(
|
|
|
975
1267
|
marked.lease?.leaseEpoch ?? leaseEpoch,
|
|
976
1268
|
);
|
|
977
1269
|
}
|
|
1270
|
+
throwIfReleasedOrCancelled();
|
|
978
1271
|
return { established, leaseEpoch, release };
|
|
979
1272
|
} catch (error) {
|
|
980
1273
|
await release();
|