@opengeni/worker-bundle 0.8.3 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2735 -905
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +7 -4
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1123 -462
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/environment.ts +142 -11
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +162 -84
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
- package/src/workflows/rig-verification.ts +3 -1
package/src/sandbox-resume.ts
CHANGED
|
@@ -23,28 +23,37 @@
|
|
|
23
23
|
import type { Settings } from "@opengeni/config";
|
|
24
24
|
import {
|
|
25
25
|
acquireLease,
|
|
26
|
+
beginSandboxRematerialization,
|
|
26
27
|
commitWarmingToWarm,
|
|
28
|
+
failSandboxRematerialization,
|
|
27
29
|
failWarmingToCold,
|
|
28
30
|
getSandboxSessionEnvelope,
|
|
31
|
+
markSandboxRestoreVerifying,
|
|
29
32
|
markWarmLeaseInstanceLost,
|
|
30
33
|
persistWarmSnapshot,
|
|
34
|
+
readWorkspaceArchiveCapturePreflight,
|
|
31
35
|
readLease,
|
|
32
36
|
recordWarmingSandboxCreated,
|
|
33
37
|
releaseLeaseHolder,
|
|
34
38
|
touchLeaseHolder,
|
|
39
|
+
SandboxLeaseRecoveryBlockedError,
|
|
35
40
|
SandboxLeaseSupersededError,
|
|
36
41
|
type Database,
|
|
37
42
|
type LeaseHolderKind,
|
|
38
43
|
} from "@opengeni/db";
|
|
39
44
|
import {
|
|
45
|
+
captureVerifiedWorkspaceArchive,
|
|
46
|
+
WorkspaceArchiveIntegrityError,
|
|
40
47
|
establishSandboxSessionFromEnvelope,
|
|
41
48
|
isProviderSandboxNotFoundError,
|
|
42
|
-
|
|
43
|
-
|
|
49
|
+
requirePersistableReplacementSandboxEnvelope,
|
|
50
|
+
serializeReplacementSandboxEnvelope,
|
|
44
51
|
deletePriorPersistedSnapshot,
|
|
45
52
|
tagModalSandbox,
|
|
53
|
+
verifySandboxExecReadiness,
|
|
46
54
|
type EstablishedSandboxSession,
|
|
47
55
|
type RuntimeMetricsHooks,
|
|
56
|
+
type WorkspaceArchiveDescriptor,
|
|
48
57
|
} from "@opengeni/runtime";
|
|
49
58
|
|
|
50
59
|
// Re-exported for callers that just want the ack-kind union.
|
|
@@ -181,43 +190,13 @@ export async function waitForSandboxExecReadiness(
|
|
|
181
190
|
established: EstablishedSandboxSession,
|
|
182
191
|
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
183
192
|
): Promise<void> {
|
|
184
|
-
if (established.backendId !== "modal") return;
|
|
185
|
-
|
|
186
|
-
const session = established.session as {
|
|
187
|
-
exec?: (args: {
|
|
188
|
-
cmd: string;
|
|
189
|
-
yieldTimeMs?: number;
|
|
190
|
-
maxOutputTokens?: number;
|
|
191
|
-
}) => Promise<unknown>;
|
|
192
|
-
execCommand?: (args: {
|
|
193
|
-
cmd: string;
|
|
194
|
-
yieldTimeMs?: number;
|
|
195
|
-
maxOutputTokens?: number;
|
|
196
|
-
}) => Promise<unknown>;
|
|
197
|
-
};
|
|
198
|
-
const run = session.exec ?? session.execCommand;
|
|
199
|
-
if (!run) {
|
|
200
|
-
throw new Error("Modal sandbox session does not expose exec");
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
204
193
|
try {
|
|
205
|
-
await
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
new Promise<never>((_, reject) => {
|
|
212
|
-
timer = setTimeout(
|
|
213
|
-
() => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
|
|
214
|
-
timeoutMs,
|
|
215
|
-
);
|
|
216
|
-
if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
217
|
-
}),
|
|
218
|
-
]);
|
|
219
|
-
} finally {
|
|
220
|
-
if (timer) clearTimeout(timer);
|
|
194
|
+
await verifySandboxExecReadiness(established, timeoutMs);
|
|
195
|
+
} catch (error) {
|
|
196
|
+
if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
|
|
197
|
+
throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
|
|
198
|
+
}
|
|
199
|
+
throw error;
|
|
221
200
|
}
|
|
222
201
|
}
|
|
223
202
|
|
|
@@ -285,7 +264,10 @@ async function terminateEstablishedSandbox(
|
|
|
285
264
|
try {
|
|
286
265
|
await client.delete(established.sessionState);
|
|
287
266
|
return true;
|
|
288
|
-
} catch {
|
|
267
|
+
} catch (error) {
|
|
268
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
289
271
|
return false;
|
|
290
272
|
}
|
|
291
273
|
}
|
|
@@ -309,7 +291,10 @@ async function terminateEstablishedSandbox(
|
|
|
309
291
|
return true;
|
|
310
292
|
}
|
|
311
293
|
return false;
|
|
312
|
-
} catch {
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
313
298
|
// Best-effort cleanup. A provider-side orphan sweep is the backstop.
|
|
314
299
|
return false;
|
|
315
300
|
}
|
|
@@ -338,7 +323,7 @@ function recordSandboxWarmingTimeout(
|
|
|
338
323
|
|
|
339
324
|
function workspaceArchiveFieldsFromEnvelope(
|
|
340
325
|
envelope: Record<string, unknown> | null | undefined,
|
|
341
|
-
): Record<string,
|
|
326
|
+
): Record<string, unknown> | null {
|
|
342
327
|
const sessionState =
|
|
343
328
|
envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
344
329
|
? (envelope.sessionState as Record<string, unknown>)
|
|
@@ -348,40 +333,37 @@ function workspaceArchiveFieldsFromEnvelope(
|
|
|
348
333
|
return null;
|
|
349
334
|
}
|
|
350
335
|
const previous = sessionState?.workspaceArchivePrev;
|
|
336
|
+
const metadata = sessionState?.workspaceArchiveMeta;
|
|
337
|
+
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
351
338
|
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
352
339
|
return {
|
|
353
340
|
workspaceArchive: archive,
|
|
341
|
+
...(metadata !== undefined ? { workspaceArchiveMeta: metadata } : {}),
|
|
354
342
|
...(typeof previous === "string" && previous.length > 0
|
|
355
343
|
? { workspaceArchivePrev: previous }
|
|
356
344
|
: {}),
|
|
345
|
+
...(previousMetadata !== undefined ? { workspaceArchivePrevMeta: previousMetadata } : {}),
|
|
357
346
|
...(typeof capturedAt === "string" && capturedAt.length > 0
|
|
358
347
|
? { workspaceArchiveAt: capturedAt }
|
|
359
348
|
: {}),
|
|
360
349
|
};
|
|
361
350
|
}
|
|
362
351
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
352
|
+
/** A per-session fallback may still carry the dead provider identity that made
|
|
353
|
+
* recovery necessary. It is useful only as durable archive/config input: never
|
|
354
|
+
* let a cold rematerialization resume that stale provider before hydrating the
|
|
355
|
+
* selected revision. */
|
|
356
|
+
function withoutProviderIdentity(
|
|
357
|
+
envelope: Record<string, unknown> | null,
|
|
366
358
|
): Record<string, unknown> | null {
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
return {
|
|
376
|
-
...(resumeState ?? {}),
|
|
377
|
-
...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
|
|
378
|
-
? { backendId: archiveSource.backendId }
|
|
379
|
-
: {}),
|
|
380
|
-
sessionState: {
|
|
381
|
-
...existingSessionState,
|
|
382
|
-
...archiveFields,
|
|
383
|
-
},
|
|
384
|
-
};
|
|
359
|
+
if (!envelope) return null;
|
|
360
|
+
const sessionState =
|
|
361
|
+
envelope.sessionState && typeof envelope.sessionState === "object"
|
|
362
|
+
? (envelope.sessionState as Record<string, unknown>)
|
|
363
|
+
: null;
|
|
364
|
+
if (!sessionState) return envelope;
|
|
365
|
+
const { providerState: _providerState, ...providerIndependentState } = sessionState;
|
|
366
|
+
return { ...envelope, sessionState: providerIndependentState };
|
|
385
367
|
}
|
|
386
368
|
|
|
387
369
|
/**
|
|
@@ -435,9 +417,23 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
435
417
|
// persistWarmSnapshot re-checks atomically under the row lock, so this is
|
|
436
418
|
// purely a cost optimization, not the correctness guard.
|
|
437
419
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
438
|
-
if (
|
|
420
|
+
if (
|
|
421
|
+
!lease ||
|
|
422
|
+
lease.leaseEpoch !== leaseEpoch ||
|
|
423
|
+
lease.liveness !== "warm" ||
|
|
424
|
+
lease.instanceId === null
|
|
425
|
+
) {
|
|
439
426
|
return false;
|
|
440
427
|
}
|
|
428
|
+
const preflight = await readWorkspaceArchiveCapturePreflight(db, {
|
|
429
|
+
accountId: ids.accountId,
|
|
430
|
+
workspaceId: ids.workspaceId,
|
|
431
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
432
|
+
expectedEpoch: leaseEpoch,
|
|
433
|
+
expectedInstanceId: lease.instanceId,
|
|
434
|
+
liveness: "warm",
|
|
435
|
+
});
|
|
436
|
+
if (!preflight) return false;
|
|
441
437
|
if (signal?.aborted) {
|
|
442
438
|
return false;
|
|
443
439
|
}
|
|
@@ -450,7 +446,11 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
450
446
|
? sessionState.workspaceArchiveAt
|
|
451
447
|
: undefined;
|
|
452
448
|
const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
|
|
453
|
-
if (
|
|
449
|
+
if (
|
|
450
|
+
preflight.archiveComplete &&
|
|
451
|
+
Number.isFinite(priorAtMs) &&
|
|
452
|
+
Date.now() - priorAtMs < intervalMs
|
|
453
|
+
) {
|
|
454
454
|
return false;
|
|
455
455
|
}
|
|
456
456
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -464,10 +464,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
464
464
|
// after the race resolved would surface as an unhandledRejection and can
|
|
465
465
|
// take down the whole worker process — which would defeat the very hang-
|
|
466
466
|
// guard this timeout exists to provide.
|
|
467
|
-
const capture =
|
|
467
|
+
const capture = captureVerifiedWorkspaceArchive(session, capturedAtMs);
|
|
468
468
|
capture.catch(() => undefined);
|
|
469
469
|
let cancelListener: (() => void) | undefined;
|
|
470
|
-
const
|
|
470
|
+
const archive = await Promise.race([
|
|
471
471
|
capture,
|
|
472
472
|
...(signal
|
|
473
473
|
? [
|
|
@@ -490,7 +490,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
490
490
|
}
|
|
491
491
|
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
492
492
|
});
|
|
493
|
-
if (!
|
|
493
|
+
if (!archive) {
|
|
494
494
|
return false;
|
|
495
495
|
}
|
|
496
496
|
if (signal?.aborted) {
|
|
@@ -504,7 +504,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
504
504
|
attemptId: ids.attemptId,
|
|
505
505
|
sandboxGroupId: ids.sandboxGroupId,
|
|
506
506
|
expectedEpoch: leaseEpoch,
|
|
507
|
-
|
|
507
|
+
expectedInstanceId: lease.instanceId,
|
|
508
|
+
expectedWorkspaceGeneration: preflight.workspaceGeneration,
|
|
509
|
+
workspaceArchive: archive.base64,
|
|
510
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
508
511
|
minIntervalMs: intervalMs,
|
|
509
512
|
capturedAtMs,
|
|
510
513
|
});
|
|
@@ -613,6 +616,15 @@ export async function resumeBoxForTurn(
|
|
|
613
616
|
// FENCED: a newer epoch exists (a later turn re-established the box). Back off;
|
|
614
617
|
// NEVER create(). Release our (just-registered) holder so we don't pin a stale
|
|
615
618
|
// lease, then surface the supersession.
|
|
619
|
+
if (acquired.role === "blocked") {
|
|
620
|
+
await release();
|
|
621
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
622
|
+
ids.sandboxGroupId,
|
|
623
|
+
acquired.lease.leaseEpoch,
|
|
624
|
+
acquired.code,
|
|
625
|
+
acquired.lease.recovery,
|
|
626
|
+
);
|
|
627
|
+
}
|
|
616
628
|
if (acquired.role === "fenced") {
|
|
617
629
|
await release();
|
|
618
630
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
@@ -623,8 +635,71 @@ export async function resumeBoxForTurn(
|
|
|
623
635
|
if (acquired.role === "spawner") {
|
|
624
636
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
625
637
|
let createdEstablished: EstablishedSandboxSession | null = null;
|
|
638
|
+
let rematerialization: {
|
|
639
|
+
id: string;
|
|
640
|
+
selectedRevision: string;
|
|
641
|
+
} | null = null;
|
|
626
642
|
try {
|
|
627
643
|
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
644
|
+
// The lease is authoritative. A legacy per-session fallback archive may
|
|
645
|
+
// only be used after beginSandboxRematerialization imports its archive
|
|
646
|
+
// fields under the warming-row lock and records one selected revision.
|
|
647
|
+
// Select the fallback only when the lease has no archive truth at all. A
|
|
648
|
+
// lease-carried unverified/invalid archive must fail closed rather than
|
|
649
|
+
// silently substituting another revision.
|
|
650
|
+
const fallbackArchiveEnvelope =
|
|
651
|
+
acquired.lease.recovery.archive.status === "none" &&
|
|
652
|
+
workspaceArchiveFieldsFromEnvelope(envelope) !== null
|
|
653
|
+
? withoutProviderIdentity(envelope)
|
|
654
|
+
: null;
|
|
655
|
+
const spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
656
|
+
const archiveSource =
|
|
657
|
+
acquired.lease.recovery.archive.status === "none"
|
|
658
|
+
? fallbackArchiveEnvelope
|
|
659
|
+
: acquired.lease.resumeState;
|
|
660
|
+
if (
|
|
661
|
+
acquired.lease.recovery.archive.status === "available" ||
|
|
662
|
+
workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
|
|
663
|
+
) {
|
|
664
|
+
const rematerializationId = crypto.randomUUID();
|
|
665
|
+
const begun = await beginSandboxRematerialization(db, {
|
|
666
|
+
accountId: ids.accountId,
|
|
667
|
+
workspaceId: ids.workspaceId,
|
|
668
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
669
|
+
expectedEpoch,
|
|
670
|
+
rematerializationId,
|
|
671
|
+
archiveSource,
|
|
672
|
+
});
|
|
673
|
+
if (begun.status !== "started") {
|
|
674
|
+
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
675
|
+
throw new SandboxLeaseSupersededError(
|
|
676
|
+
ids.sandboxGroupId,
|
|
677
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
681
|
+
ids.sandboxGroupId,
|
|
682
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
683
|
+
begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
|
|
684
|
+
begun.lease?.recovery ?? acquired.lease.recovery,
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
688
|
+
if (!selectedRevision) {
|
|
689
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
690
|
+
"archive_metadata_invalid",
|
|
691
|
+
"sandbox rematerialization selected no durable archive revision",
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
rematerialization = { id: rematerializationId, selectedRevision };
|
|
695
|
+
} else if (acquired.lease.recovery.archive.status !== "none") {
|
|
696
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
697
|
+
ids.sandboxGroupId,
|
|
698
|
+
expectedEpoch,
|
|
699
|
+
"restore_degraded",
|
|
700
|
+
acquired.lease.recovery,
|
|
701
|
+
);
|
|
702
|
+
}
|
|
628
703
|
// Prefer the COLD lease's preserved resume_state when it carries a persisted
|
|
629
704
|
// /workspace snapshot (confirmDrainCold keeps a minimal archive-only envelope
|
|
630
705
|
// across draining->cold for exactly this re-warm). establishSandboxSessionFromEnvelope
|
|
@@ -634,7 +709,6 @@ export async function resumeBoxForTurn(
|
|
|
634
709
|
// spawner branch: the lease's resume_state is authoritative; the session
|
|
635
710
|
// `_sandbox` envelope is the per-session fallback. Without this a turn-first
|
|
636
711
|
// re-warm after a drain->cold would ignore the archive and start an EMPTY box.
|
|
637
|
-
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
638
712
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
639
713
|
sessionId: ids.sessionId,
|
|
640
714
|
recovery: "create-or-restore",
|
|
@@ -643,15 +717,16 @@ export async function resumeBoxForTurn(
|
|
|
643
717
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
644
718
|
onSandboxCreated: async (created) => {
|
|
645
719
|
createdEstablished = created;
|
|
646
|
-
const resumeEnvelope =
|
|
647
|
-
|
|
648
|
-
|
|
720
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
721
|
+
await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
|
|
722
|
+
created.backendId,
|
|
649
723
|
);
|
|
650
724
|
const recorded = await recordWarmingSandboxCreated(db, {
|
|
651
725
|
accountId: ids.accountId,
|
|
652
726
|
workspaceId: ids.workspaceId,
|
|
653
727
|
sandboxGroupId: ids.sandboxGroupId,
|
|
654
728
|
expectedEpoch,
|
|
729
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
655
730
|
instanceId: created.instanceId,
|
|
656
731
|
resumeBackendId: created.backendId,
|
|
657
732
|
resumeState: resumeEnvelope,
|
|
@@ -671,6 +746,24 @@ export async function resumeBoxForTurn(
|
|
|
671
746
|
}).catch(() => undefined);
|
|
672
747
|
}
|
|
673
748
|
},
|
|
749
|
+
onWorkspaceRestoreVerifying: async (descriptor: WorkspaceArchiveDescriptor) => {
|
|
750
|
+
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
751
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
752
|
+
"archive_metadata_invalid",
|
|
753
|
+
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`,
|
|
754
|
+
);
|
|
755
|
+
}
|
|
756
|
+
const verifying = await markSandboxRestoreVerifying(db, {
|
|
757
|
+
accountId: ids.accountId,
|
|
758
|
+
workspaceId: ids.workspaceId,
|
|
759
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
760
|
+
expectedEpoch,
|
|
761
|
+
rematerializationId: rematerialization.id,
|
|
762
|
+
});
|
|
763
|
+
if (!verifying.wrote) {
|
|
764
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
765
|
+
}
|
|
766
|
+
},
|
|
674
767
|
});
|
|
675
768
|
createdEstablished = established;
|
|
676
769
|
// A sandbox handle is not sufficient evidence that Modal's command router
|
|
@@ -683,22 +776,31 @@ export async function resumeBoxForTurn(
|
|
|
683
776
|
// viewer.ts). Without this the turn committed the ORIGINAL session manifest
|
|
684
777
|
// as resume_state, so every LATER op off this lease (Channel-A fs/git/
|
|
685
778
|
// terminal, the desktop viewer, the reaper) cold-restored a FRESH rival box
|
|
686
|
-
// and never saw the turn's live box.
|
|
687
|
-
//
|
|
688
|
-
|
|
689
|
-
|
|
779
|
+
// and never saw the turn's live box. Historical state may contribute only
|
|
780
|
+
// durable archive pointers: if live serialization fails, publishing its
|
|
781
|
+
// dead provider identity would pair the replacement instance with the box
|
|
782
|
+
// that initiated recovery.
|
|
690
783
|
// A successful cold hydrate has already proved this archive usable and the
|
|
691
784
|
// replacement box now contains its files. Keep the current + fallback
|
|
692
785
|
// archive pointers on the committed live envelope until a later warm
|
|
693
786
|
// snapshot replaces them. Without this merge, serialization publishes only
|
|
694
787
|
// the new provider id; a second provider loss before the snapshot cadence
|
|
695
|
-
// fires would
|
|
696
|
-
//
|
|
697
|
-
//
|
|
698
|
-
const resumeEnvelope =
|
|
699
|
-
established
|
|
700
|
-
|
|
701
|
-
|
|
788
|
+
// fires would otherwise make truthful recovery impossible. Failed hydrate
|
|
789
|
+
// attempts terminate the replacement and fail closed; they never publish a
|
|
790
|
+
// clean or mixed workspace.
|
|
791
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
792
|
+
await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
|
|
793
|
+
established.backendId,
|
|
794
|
+
);
|
|
795
|
+
if (
|
|
796
|
+
rematerialization &&
|
|
797
|
+
established.restoredArchive?.revision !== rematerialization.selectedRevision
|
|
798
|
+
) {
|
|
799
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
800
|
+
"workspace_fingerprint_mismatch",
|
|
801
|
+
"sandbox restore completed without the exact selected durable archive revision",
|
|
802
|
+
);
|
|
803
|
+
}
|
|
702
804
|
const committed = await commitWarmingToWarm(db, {
|
|
703
805
|
accountId: ids.accountId,
|
|
704
806
|
workspaceId: ids.workspaceId,
|
|
@@ -710,6 +812,14 @@ export async function resumeBoxForTurn(
|
|
|
710
812
|
dataPlaneUrl: null,
|
|
711
813
|
resumeBackendId: established.backendId,
|
|
712
814
|
resumeState: resumeEnvelope,
|
|
815
|
+
...(rematerialization
|
|
816
|
+
? {
|
|
817
|
+
rematerialization: {
|
|
818
|
+
id: rematerialization.id,
|
|
819
|
+
verifiedRevision: rematerialization.selectedRevision,
|
|
820
|
+
},
|
|
821
|
+
}
|
|
822
|
+
: {}),
|
|
713
823
|
leaseTtlMs,
|
|
714
824
|
});
|
|
715
825
|
if (!committed.committed || !committed.lease) {
|
|
@@ -717,7 +827,18 @@ export async function resumeBoxForTurn(
|
|
|
717
827
|
// re-established and bumped the epoch. Drop the handle; release our
|
|
718
828
|
// holder; surface supersession. This spawner created the box, so stop it
|
|
719
829
|
// before retrying to avoid an untracked running sandbox.
|
|
720
|
-
await terminateEstablishedSandbox(established);
|
|
830
|
+
const terminated = await terminateEstablishedSandbox(established);
|
|
831
|
+
if (terminated && rematerialization) {
|
|
832
|
+
await failSandboxRematerialization(db, {
|
|
833
|
+
accountId: ids.accountId,
|
|
834
|
+
workspaceId: ids.workspaceId,
|
|
835
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
836
|
+
expectedEpoch,
|
|
837
|
+
rematerializationId: rematerialization.id,
|
|
838
|
+
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
839
|
+
retryable: false,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
721
842
|
await release();
|
|
722
843
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
723
844
|
}
|
|
@@ -735,12 +856,27 @@ export async function resumeBoxForTurn(
|
|
|
735
856
|
// the warming row; the lease TTL/reaper and Modal orphan sweep are the
|
|
736
857
|
// tracked backstops, and we must not erase the only provider pointer.
|
|
737
858
|
if (terminated) {
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
859
|
+
if (rematerialization) {
|
|
860
|
+
await failSandboxRematerialization(db, {
|
|
861
|
+
accountId: ids.accountId,
|
|
862
|
+
workspaceId: ids.workspaceId,
|
|
863
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
864
|
+
expectedEpoch,
|
|
865
|
+
rematerializationId: rematerialization.id,
|
|
866
|
+
failureCode:
|
|
867
|
+
error instanceof WorkspaceArchiveIntegrityError
|
|
868
|
+
? error.code
|
|
869
|
+
: "sandbox_rematerialization_failed",
|
|
870
|
+
retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
|
|
871
|
+
});
|
|
872
|
+
} else {
|
|
873
|
+
await failWarmingToCold(db, {
|
|
874
|
+
accountId: ids.accountId,
|
|
875
|
+
workspaceId: ids.workspaceId,
|
|
876
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
877
|
+
expectedEpoch,
|
|
878
|
+
});
|
|
879
|
+
}
|
|
744
880
|
}
|
|
745
881
|
await release();
|
|
746
882
|
const warmingError = asSandboxWarmingError(
|
|
@@ -787,10 +923,7 @@ export async function resumeBoxForTurn(
|
|
|
787
923
|
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
788
924
|
});
|
|
789
925
|
} catch (error) {
|
|
790
|
-
if (
|
|
791
|
-
!(error instanceof SandboxResumeStateUnavailableError) &&
|
|
792
|
-
!isProviderSandboxNotFoundError(ids.backend, error)
|
|
793
|
-
) {
|
|
926
|
+
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
794
927
|
throw error;
|
|
795
928
|
}
|
|
796
929
|
const marked = await markWarmLeaseInstanceLost(db, {
|
|
@@ -894,6 +1027,15 @@ export async function acquireSelfhostedLeaseForTurn(
|
|
|
894
1027
|
// FENCED: a newer epoch re-established the group concurrently. Release our
|
|
895
1028
|
// just-registered holder + surface the supersession (the outer turn catch
|
|
896
1029
|
// requeues, mirroring resumeBoxForTurn).
|
|
1030
|
+
if (acquired.role === "blocked") {
|
|
1031
|
+
await release();
|
|
1032
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1033
|
+
ids.sandboxGroupId,
|
|
1034
|
+
acquired.lease.leaseEpoch,
|
|
1035
|
+
acquired.code,
|
|
1036
|
+
acquired.lease.recovery,
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
897
1039
|
if (acquired.role === "fenced") {
|
|
898
1040
|
await release();
|
|
899
1041
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
@@ -962,6 +1104,19 @@ async function waitForWarm(
|
|
|
962
1104
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
963
1105
|
}
|
|
964
1106
|
if (lease.liveness === "cold") {
|
|
1107
|
+
if (
|
|
1108
|
+
lease.recovery.restore.status === "degraded" ||
|
|
1109
|
+
lease.recovery.restore.status === "unrecoverable"
|
|
1110
|
+
) {
|
|
1111
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1112
|
+
ids.sandboxGroupId,
|
|
1113
|
+
lease.leaseEpoch,
|
|
1114
|
+
lease.recovery.restore.status === "degraded"
|
|
1115
|
+
? "restore_degraded"
|
|
1116
|
+
: "restore_unrecoverable",
|
|
1117
|
+
lease.recovery,
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
965
1120
|
// The spawner died; re-dispatch so the normal acquireLease path can win
|
|
966
1121
|
// cold->warming and run the full spawner branch.
|
|
967
1122
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|