@opengeni/worker-bundle 0.8.3 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,28 +23,37 @@
23
23
  import type { Settings } from "@opengeni/config";
24
24
  import {
25
25
  acquireLease,
26
+ beginSandboxRematerialization,
26
27
  commitWarmingToWarm,
28
+ failSandboxRematerialization,
27
29
  failWarmingToCold,
28
30
  getSandboxSessionEnvelope,
31
+ markSandboxRestoreVerifying,
29
32
  markWarmLeaseInstanceLost,
30
33
  persistWarmSnapshot,
34
+ readWorkspaceArchiveCapturePreflight,
31
35
  readLease,
32
36
  recordWarmingSandboxCreated,
33
37
  releaseLeaseHolder,
34
38
  touchLeaseHolder,
39
+ SandboxLeaseRecoveryBlockedError,
35
40
  SandboxLeaseSupersededError,
36
41
  type Database,
37
42
  type LeaseHolderKind,
38
43
  } from "@opengeni/db";
39
44
  import {
45
+ captureVerifiedWorkspaceArchive,
46
+ WorkspaceArchiveIntegrityError,
40
47
  establishSandboxSessionFromEnvelope,
41
48
  isProviderSandboxNotFoundError,
42
- SandboxResumeStateUnavailableError,
43
- serializeEstablishedSandboxEnvelope,
49
+ requirePersistableReplacementSandboxEnvelope,
50
+ serializeReplacementSandboxEnvelope,
44
51
  deletePriorPersistedSnapshot,
45
52
  tagModalSandbox,
53
+ verifySandboxExecReadiness,
46
54
  type EstablishedSandboxSession,
47
55
  type RuntimeMetricsHooks,
56
+ type WorkspaceArchiveDescriptor,
48
57
  } from "@opengeni/runtime";
49
58
 
50
59
  // Re-exported for callers that just want the ack-kind union.
@@ -181,43 +190,13 @@ export async function waitForSandboxExecReadiness(
181
190
  established: EstablishedSandboxSession,
182
191
  timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
183
192
  ): Promise<void> {
184
- if (established.backendId !== "modal") return;
185
-
186
- const session = established.session as {
187
- exec?: (args: {
188
- cmd: string;
189
- yieldTimeMs?: number;
190
- maxOutputTokens?: number;
191
- }) => Promise<unknown>;
192
- execCommand?: (args: {
193
- cmd: string;
194
- yieldTimeMs?: number;
195
- maxOutputTokens?: number;
196
- }) => Promise<unknown>;
197
- };
198
- const run = session.exec ?? session.execCommand;
199
- if (!run) {
200
- throw new Error("Modal sandbox session does not expose exec");
201
- }
202
-
203
- let timer: ReturnType<typeof setTimeout> | undefined;
204
193
  try {
205
- await Promise.race([
206
- run.call(session, {
207
- cmd: "true",
208
- yieldTimeMs: 1_000,
209
- maxOutputTokens: 1_000,
210
- }),
211
- new Promise<never>((_, reject) => {
212
- timer = setTimeout(
213
- () => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
214
- timeoutMs,
215
- );
216
- if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
217
- }),
218
- ]);
219
- } finally {
220
- if (timer) clearTimeout(timer);
194
+ await verifySandboxExecReadiness(established, timeoutMs);
195
+ } catch (error) {
196
+ if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
197
+ throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
198
+ }
199
+ throw error;
221
200
  }
222
201
  }
223
202
 
@@ -285,7 +264,10 @@ async function terminateEstablishedSandbox(
285
264
  try {
286
265
  await client.delete(established.sessionState);
287
266
  return true;
288
- } catch {
267
+ } catch (error) {
268
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
269
+ return true;
270
+ }
289
271
  return false;
290
272
  }
291
273
  }
@@ -309,7 +291,10 @@ async function terminateEstablishedSandbox(
309
291
  return true;
310
292
  }
311
293
  return false;
312
- } catch {
294
+ } catch (error) {
295
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
296
+ return true;
297
+ }
313
298
  // Best-effort cleanup. A provider-side orphan sweep is the backstop.
314
299
  return false;
315
300
  }
@@ -338,7 +323,7 @@ function recordSandboxWarmingTimeout(
338
323
 
339
324
  function workspaceArchiveFieldsFromEnvelope(
340
325
  envelope: Record<string, unknown> | null | undefined,
341
- ): Record<string, string> | null {
326
+ ): Record<string, unknown> | null {
342
327
  const sessionState =
343
328
  envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
344
329
  ? (envelope.sessionState as Record<string, unknown>)
@@ -348,40 +333,37 @@ function workspaceArchiveFieldsFromEnvelope(
348
333
  return null;
349
334
  }
350
335
  const previous = sessionState?.workspaceArchivePrev;
336
+ const metadata = sessionState?.workspaceArchiveMeta;
337
+ const previousMetadata = sessionState?.workspaceArchivePrevMeta;
351
338
  const capturedAt = sessionState?.workspaceArchiveAt;
352
339
  return {
353
340
  workspaceArchive: archive,
341
+ ...(metadata !== undefined ? { workspaceArchiveMeta: metadata } : {}),
354
342
  ...(typeof previous === "string" && previous.length > 0
355
343
  ? { workspaceArchivePrev: previous }
356
344
  : {}),
345
+ ...(previousMetadata !== undefined ? { workspaceArchivePrevMeta: previousMetadata } : {}),
357
346
  ...(typeof capturedAt === "string" && capturedAt.length > 0
358
347
  ? { workspaceArchiveAt: capturedAt }
359
348
  : {}),
360
349
  };
361
350
  }
362
351
 
363
- function preserveWorkspaceArchivesOnResumeState(
364
- resumeState: Record<string, unknown> | null,
365
- archiveSource: Record<string, unknown> | null,
352
+ /** A per-session fallback may still carry the dead provider identity that made
353
+ * recovery necessary. It is useful only as durable archive/config input: never
354
+ * let a cold rematerialization resume that stale provider before hydrating the
355
+ * selected revision. */
356
+ function withoutProviderIdentity(
357
+ envelope: Record<string, unknown> | null,
366
358
  ): Record<string, unknown> | null {
367
- const archiveFields = workspaceArchiveFieldsFromEnvelope(archiveSource);
368
- if (!archiveFields) {
369
- return resumeState;
370
- }
371
- const existingSessionState =
372
- resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
373
- ? (resumeState.sessionState as Record<string, unknown>)
374
- : {};
375
- return {
376
- ...(resumeState ?? {}),
377
- ...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
378
- ? { backendId: archiveSource.backendId }
379
- : {}),
380
- sessionState: {
381
- ...existingSessionState,
382
- ...archiveFields,
383
- },
384
- };
359
+ if (!envelope) return null;
360
+ const sessionState =
361
+ envelope.sessionState && typeof envelope.sessionState === "object"
362
+ ? (envelope.sessionState as Record<string, unknown>)
363
+ : null;
364
+ if (!sessionState) return envelope;
365
+ const { providerState: _providerState, ...providerIndependentState } = sessionState;
366
+ return { ...envelope, sessionState: providerIndependentState };
385
367
  }
386
368
 
387
369
  /**
@@ -435,9 +417,23 @@ export async function maybePersistWarmWorkspaceSnapshot(
435
417
  // persistWarmSnapshot re-checks atomically under the row lock, so this is
436
418
  // purely a cost optimization, not the correctness guard.
437
419
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
438
- if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
420
+ if (
421
+ !lease ||
422
+ lease.leaseEpoch !== leaseEpoch ||
423
+ lease.liveness !== "warm" ||
424
+ lease.instanceId === null
425
+ ) {
439
426
  return false;
440
427
  }
428
+ const preflight = await readWorkspaceArchiveCapturePreflight(db, {
429
+ accountId: ids.accountId,
430
+ workspaceId: ids.workspaceId,
431
+ sandboxGroupId: ids.sandboxGroupId,
432
+ expectedEpoch: leaseEpoch,
433
+ expectedInstanceId: lease.instanceId,
434
+ liveness: "warm",
435
+ });
436
+ if (!preflight) return false;
441
437
  if (signal?.aborted) {
442
438
  return false;
443
439
  }
@@ -450,7 +446,11 @@ export async function maybePersistWarmWorkspaceSnapshot(
450
446
  ? sessionState.workspaceArchiveAt
451
447
  : undefined;
452
448
  const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
453
- if (Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
449
+ if (
450
+ preflight.archiveComplete &&
451
+ Number.isFinite(priorAtMs) &&
452
+ Date.now() - priorAtMs < intervalMs
453
+ ) {
454
454
  return false;
455
455
  }
456
456
  let timeout: ReturnType<typeof setTimeout> | undefined;
@@ -464,10 +464,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
464
464
  // after the race resolved would surface as an unhandledRejection and can
465
465
  // take down the whole worker process — which would defeat the very hang-
466
466
  // guard this timeout exists to provide.
467
- const capture = persistable.persistWorkspace();
467
+ const capture = captureVerifiedWorkspaceArchive(session, capturedAtMs);
468
468
  capture.catch(() => undefined);
469
469
  let cancelListener: (() => void) | undefined;
470
- const bytes = await Promise.race([
470
+ const archive = await Promise.race([
471
471
  capture,
472
472
  ...(signal
473
473
  ? [
@@ -490,7 +490,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
490
490
  }
491
491
  if (cancelListener) signal?.removeEventListener("abort", cancelListener);
492
492
  });
493
- if (!bytes || bytes.length === 0) {
493
+ if (!archive) {
494
494
  return false;
495
495
  }
496
496
  if (signal?.aborted) {
@@ -504,7 +504,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
504
504
  attemptId: ids.attemptId,
505
505
  sandboxGroupId: ids.sandboxGroupId,
506
506
  expectedEpoch: leaseEpoch,
507
- workspaceArchive: Buffer.from(bytes).toString("base64"),
507
+ expectedInstanceId: lease.instanceId,
508
+ expectedWorkspaceGeneration: preflight.workspaceGeneration,
509
+ workspaceArchive: archive.base64,
510
+ workspaceArchiveMeta: archive.descriptor,
508
511
  minIntervalMs: intervalMs,
509
512
  capturedAtMs,
510
513
  });
@@ -613,6 +616,15 @@ export async function resumeBoxForTurn(
613
616
  // FENCED: a newer epoch exists (a later turn re-established the box). Back off;
614
617
  // NEVER create(). Release our (just-registered) holder so we don't pin a stale
615
618
  // lease, then surface the supersession.
619
+ if (acquired.role === "blocked") {
620
+ await release();
621
+ throw new SandboxLeaseRecoveryBlockedError(
622
+ ids.sandboxGroupId,
623
+ acquired.lease.leaseEpoch,
624
+ acquired.code,
625
+ acquired.lease.recovery,
626
+ );
627
+ }
616
628
  if (acquired.role === "fenced") {
617
629
  await release();
618
630
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
@@ -623,8 +635,71 @@ export async function resumeBoxForTurn(
623
635
  if (acquired.role === "spawner") {
624
636
  const expectedEpoch = acquired.lease.leaseEpoch;
625
637
  let createdEstablished: EstablishedSandboxSession | null = null;
638
+ let rematerialization: {
639
+ id: string;
640
+ selectedRevision: string;
641
+ } | null = null;
626
642
  try {
627
643
  const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
644
+ // The lease is authoritative. A legacy per-session fallback archive may
645
+ // only be used after beginSandboxRematerialization imports its archive
646
+ // fields under the warming-row lock and records one selected revision.
647
+ // Select the fallback only when the lease has no archive truth at all. A
648
+ // lease-carried unverified/invalid archive must fail closed rather than
649
+ // silently substituting another revision.
650
+ const fallbackArchiveEnvelope =
651
+ acquired.lease.recovery.archive.status === "none" &&
652
+ workspaceArchiveFieldsFromEnvelope(envelope) !== null
653
+ ? withoutProviderIdentity(envelope)
654
+ : null;
655
+ const spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
656
+ const archiveSource =
657
+ acquired.lease.recovery.archive.status === "none"
658
+ ? fallbackArchiveEnvelope
659
+ : acquired.lease.resumeState;
660
+ if (
661
+ acquired.lease.recovery.archive.status === "available" ||
662
+ workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
663
+ ) {
664
+ const rematerializationId = crypto.randomUUID();
665
+ const begun = await beginSandboxRematerialization(db, {
666
+ accountId: ids.accountId,
667
+ workspaceId: ids.workspaceId,
668
+ sandboxGroupId: ids.sandboxGroupId,
669
+ expectedEpoch,
670
+ rematerializationId,
671
+ archiveSource,
672
+ });
673
+ if (begun.status !== "started") {
674
+ if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
675
+ throw new SandboxLeaseSupersededError(
676
+ ids.sandboxGroupId,
677
+ begun.lease?.leaseEpoch ?? expectedEpoch,
678
+ );
679
+ }
680
+ throw new SandboxLeaseRecoveryBlockedError(
681
+ ids.sandboxGroupId,
682
+ begun.lease?.leaseEpoch ?? expectedEpoch,
683
+ begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
684
+ begun.lease?.recovery ?? acquired.lease.recovery,
685
+ );
686
+ }
687
+ const selectedRevision = begun.lease.recovery.restore.selectedRevision;
688
+ if (!selectedRevision) {
689
+ throw new WorkspaceArchiveIntegrityError(
690
+ "archive_metadata_invalid",
691
+ "sandbox rematerialization selected no durable archive revision",
692
+ );
693
+ }
694
+ rematerialization = { id: rematerializationId, selectedRevision };
695
+ } else if (acquired.lease.recovery.archive.status !== "none") {
696
+ throw new SandboxLeaseRecoveryBlockedError(
697
+ ids.sandboxGroupId,
698
+ expectedEpoch,
699
+ "restore_degraded",
700
+ acquired.lease.recovery,
701
+ );
702
+ }
628
703
  // Prefer the COLD lease's preserved resume_state when it carries a persisted
629
704
  // /workspace snapshot (confirmDrainCold keeps a minimal archive-only envelope
630
705
  // across draining->cold for exactly this re-warm). establishSandboxSessionFromEnvelope
@@ -634,7 +709,6 @@ export async function resumeBoxForTurn(
634
709
  // spawner branch: the lease's resume_state is authoritative; the session
635
710
  // `_sandbox` envelope is the per-session fallback. Without this a turn-first
636
711
  // re-warm after a drain->cold would ignore the archive and start an EMPTY box.
637
- const spawnEnvelope = acquired.lease.resumeState ?? envelope;
638
712
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
639
713
  sessionId: ids.sessionId,
640
714
  recovery: "create-or-restore",
@@ -643,15 +717,16 @@ export async function resumeBoxForTurn(
643
717
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
644
718
  onSandboxCreated: async (created) => {
645
719
  createdEstablished = created;
646
- const resumeEnvelope = preserveWorkspaceArchivesOnResumeState(
647
- (await serializeEstablishedSandboxEnvelope(created)) ?? null,
648
- spawnEnvelope,
720
+ const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
721
+ await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
722
+ created.backendId,
649
723
  );
650
724
  const recorded = await recordWarmingSandboxCreated(db, {
651
725
  accountId: ids.accountId,
652
726
  workspaceId: ids.workspaceId,
653
727
  sandboxGroupId: ids.sandboxGroupId,
654
728
  expectedEpoch,
729
+ rematerializationId: rematerialization?.id ?? null,
655
730
  instanceId: created.instanceId,
656
731
  resumeBackendId: created.backendId,
657
732
  resumeState: resumeEnvelope,
@@ -671,6 +746,24 @@ export async function resumeBoxForTurn(
671
746
  }).catch(() => undefined);
672
747
  }
673
748
  },
749
+ onWorkspaceRestoreVerifying: async (descriptor: WorkspaceArchiveDescriptor) => {
750
+ if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
751
+ throw new WorkspaceArchiveIntegrityError(
752
+ "archive_metadata_invalid",
753
+ `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`,
754
+ );
755
+ }
756
+ const verifying = await markSandboxRestoreVerifying(db, {
757
+ accountId: ids.accountId,
758
+ workspaceId: ids.workspaceId,
759
+ sandboxGroupId: ids.sandboxGroupId,
760
+ expectedEpoch,
761
+ rematerializationId: rematerialization.id,
762
+ });
763
+ if (!verifying.wrote) {
764
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
765
+ }
766
+ },
674
767
  });
675
768
  createdEstablished = established;
676
769
  // A sandbox handle is not sufficient evidence that Modal's command router
@@ -683,22 +776,31 @@ export async function resumeBoxForTurn(
683
776
  // viewer.ts). Without this the turn committed the ORIGINAL session manifest
684
777
  // as resume_state, so every LATER op off this lease (Channel-A fs/git/
685
778
  // terminal, the desktop viewer, the reaper) cold-restored a FRESH rival box
686
- // and never saw the turn's live box. Fall back to the session envelope only
687
- // when the client cannot serialize live state.
688
- const serializedResumeEnvelope =
689
- (await serializeEstablishedSandboxEnvelope(established)) ?? envelope;
779
+ // and never saw the turn's live box. Historical state may contribute only
780
+ // durable archive pointers: if live serialization fails, publishing its
781
+ // dead provider identity would pair the replacement instance with the box
782
+ // that initiated recovery.
690
783
  // A successful cold hydrate has already proved this archive usable and the
691
784
  // replacement box now contains its files. Keep the current + fallback
692
785
  // archive pointers on the committed live envelope until a later warm
693
786
  // snapshot replaces them. Without this merge, serialization publishes only
694
787
  // the new provider id; a second provider loss before the snapshot cadence
695
- // fires would retire the lease with no archive and recreate an empty box.
696
- // A failed hydrate falls back to a clean box with origin="created", so its
697
- // unusable archive is deliberately cleared by the unmerged serialized state.
698
- const resumeEnvelope =
699
- established.origin === "restored"
700
- ? preserveWorkspaceArchivesOnResumeState(serializedResumeEnvelope, spawnEnvelope)
701
- : serializedResumeEnvelope;
788
+ // fires would otherwise make truthful recovery impossible. Failed hydrate
789
+ // attempts terminate the replacement and fail closed; they never publish a
790
+ // clean or mixed workspace.
791
+ const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
792
+ await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
793
+ established.backendId,
794
+ );
795
+ if (
796
+ rematerialization &&
797
+ established.restoredArchive?.revision !== rematerialization.selectedRevision
798
+ ) {
799
+ throw new WorkspaceArchiveIntegrityError(
800
+ "workspace_fingerprint_mismatch",
801
+ "sandbox restore completed without the exact selected durable archive revision",
802
+ );
803
+ }
702
804
  const committed = await commitWarmingToWarm(db, {
703
805
  accountId: ids.accountId,
704
806
  workspaceId: ids.workspaceId,
@@ -710,6 +812,14 @@ export async function resumeBoxForTurn(
710
812
  dataPlaneUrl: null,
711
813
  resumeBackendId: established.backendId,
712
814
  resumeState: resumeEnvelope,
815
+ ...(rematerialization
816
+ ? {
817
+ rematerialization: {
818
+ id: rematerialization.id,
819
+ verifiedRevision: rematerialization.selectedRevision,
820
+ },
821
+ }
822
+ : {}),
713
823
  leaseTtlMs,
714
824
  });
715
825
  if (!committed.committed || !committed.lease) {
@@ -717,7 +827,18 @@ export async function resumeBoxForTurn(
717
827
  // re-established and bumped the epoch. Drop the handle; release our
718
828
  // holder; surface supersession. This spawner created the box, so stop it
719
829
  // before retrying to avoid an untracked running sandbox.
720
- await terminateEstablishedSandbox(established);
830
+ const terminated = await terminateEstablishedSandbox(established);
831
+ if (terminated && rematerialization) {
832
+ await failSandboxRematerialization(db, {
833
+ accountId: ids.accountId,
834
+ workspaceId: ids.workspaceId,
835
+ sandboxGroupId: ids.sandboxGroupId,
836
+ expectedEpoch,
837
+ rematerializationId: rematerialization.id,
838
+ failureCode: committed.reason ?? "warm_commit_rejected",
839
+ retryable: false,
840
+ });
841
+ }
721
842
  await release();
722
843
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
723
844
  }
@@ -735,12 +856,27 @@ export async function resumeBoxForTurn(
735
856
  // the warming row; the lease TTL/reaper and Modal orphan sweep are the
736
857
  // tracked backstops, and we must not erase the only provider pointer.
737
858
  if (terminated) {
738
- await failWarmingToCold(db, {
739
- accountId: ids.accountId,
740
- workspaceId: ids.workspaceId,
741
- sandboxGroupId: ids.sandboxGroupId,
742
- expectedEpoch,
743
- });
859
+ if (rematerialization) {
860
+ await failSandboxRematerialization(db, {
861
+ accountId: ids.accountId,
862
+ workspaceId: ids.workspaceId,
863
+ sandboxGroupId: ids.sandboxGroupId,
864
+ expectedEpoch,
865
+ rematerializationId: rematerialization.id,
866
+ failureCode:
867
+ error instanceof WorkspaceArchiveIntegrityError
868
+ ? error.code
869
+ : "sandbox_rematerialization_failed",
870
+ retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
871
+ });
872
+ } else {
873
+ await failWarmingToCold(db, {
874
+ accountId: ids.accountId,
875
+ workspaceId: ids.workspaceId,
876
+ sandboxGroupId: ids.sandboxGroupId,
877
+ expectedEpoch,
878
+ });
879
+ }
744
880
  }
745
881
  await release();
746
882
  const warmingError = asSandboxWarmingError(
@@ -787,10 +923,7 @@ export async function resumeBoxForTurn(
787
923
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
788
924
  });
789
925
  } catch (error) {
790
- if (
791
- !(error instanceof SandboxResumeStateUnavailableError) &&
792
- !isProviderSandboxNotFoundError(ids.backend, error)
793
- ) {
926
+ if (!isProviderSandboxNotFoundError(ids.backend, error)) {
794
927
  throw error;
795
928
  }
796
929
  const marked = await markWarmLeaseInstanceLost(db, {
@@ -894,6 +1027,15 @@ export async function acquireSelfhostedLeaseForTurn(
894
1027
  // FENCED: a newer epoch re-established the group concurrently. Release our
895
1028
  // just-registered holder + surface the supersession (the outer turn catch
896
1029
  // requeues, mirroring resumeBoxForTurn).
1030
+ if (acquired.role === "blocked") {
1031
+ await release();
1032
+ throw new SandboxLeaseRecoveryBlockedError(
1033
+ ids.sandboxGroupId,
1034
+ acquired.lease.leaseEpoch,
1035
+ acquired.code,
1036
+ acquired.lease.recovery,
1037
+ );
1038
+ }
897
1039
  if (acquired.role === "fenced") {
898
1040
  await release();
899
1041
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
@@ -962,6 +1104,19 @@ async function waitForWarm(
962
1104
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
963
1105
  }
964
1106
  if (lease.liveness === "cold") {
1107
+ if (
1108
+ lease.recovery.restore.status === "degraded" ||
1109
+ lease.recovery.restore.status === "unrecoverable"
1110
+ ) {
1111
+ throw new SandboxLeaseRecoveryBlockedError(
1112
+ ids.sandboxGroupId,
1113
+ lease.leaseEpoch,
1114
+ lease.recovery.restore.status === "degraded"
1115
+ ? "restore_degraded"
1116
+ : "restore_unrecoverable",
1117
+ lease.recovery,
1118
+ );
1119
+ }
965
1120
  // The spawner died; re-dispatch so the normal acquireLease path can win
966
1121
  // cold->warming and run the full spawner branch.
967
1122
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);