@opengeni/worker-bundle 0.12.10 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/activities/agent-turn.d.ts +778 -0
  2. package/dist/activities/capabilities.d.ts +18 -0
  3. package/dist/activities/codex-auth.d.ts +1 -0
  4. package/dist/activities/codex-capacity.d.ts +31 -0
  5. package/dist/activities/codex-fleet-shadow.d.ts +74 -0
  6. package/dist/activities/codex-rotation.d.ts +261 -0
  7. package/dist/activities/common.d.ts +17 -0
  8. package/dist/activities/context-compaction.d.ts +81 -0
  9. package/dist/activities/documents.d.ts +39 -0
  10. package/dist/activities/environment.d.ts +97 -0
  11. package/dist/activities/expiring-material-renewal.d.ts +38 -0
  12. package/dist/activities/file-upload-reaper.d.ts +25 -0
  13. package/dist/activities/git-credential-renewal.d.ts +41 -0
  14. package/dist/activities/goals.d.ts +16 -0
  15. package/dist/activities/mcp-credentials.d.ts +14 -0
  16. package/dist/activities/packs.d.ts +77 -0
  17. package/dist/activities/parent-wake.d.ts +54 -0
  18. package/dist/activities/recording.d.ts +90 -0
  19. package/dist/activities/redaction.d.ts +1 -0
  20. package/dist/activities/rig-verification.d.ts +131 -0
  21. package/dist/activities/run-credential-renewal.d.ts +28 -0
  22. package/dist/activities/run-credentials.d.ts +36 -0
  23. package/dist/activities/run-input.d.ts +106 -0
  24. package/dist/activities/sandbox-lease.d.ts +128 -0
  25. package/dist/activities/scheduled-tasks.d.ts +4 -0
  26. package/dist/activities/session-state.d.ts +39 -0
  27. package/dist/activities/streaming.d.ts +24 -0
  28. package/dist/activities/toolspace-token-renewal.d.ts +23 -0
  29. package/dist/activities/turn-attempt-fenced.d.ts +11 -0
  30. package/dist/activities/types.d.ts +240 -0
  31. package/dist/activities/workflow-wake.d.ts +15 -0
  32. package/dist/activities/workspace-capture.d.ts +105 -0
  33. package/dist/activities.d.ts +312 -0
  34. package/dist/concurrency.d.ts +19 -0
  35. package/dist/host-export-pump.d.ts +66 -0
  36. package/dist/host-export.d.ts +6 -67
  37. package/dist/http.d.ts +32 -0
  38. package/dist/index.d.ts +27 -388
  39. package/dist/index.js +2729 -1208
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +200 -0
  42. package/dist/op-journal.d.ts +10 -0
  43. package/dist/sandbox-resume.d.ts +152 -0
  44. package/dist/sandbox-routing.d.ts +159 -0
  45. package/dist/worker-service-lifecycle.d.ts +20 -0
  46. package/dist/workflow-bundle.js +88 -20
  47. package/dist/workflows/activities.d.ts +27 -0
  48. package/dist/workflows/document-index.d.ts +41 -0
  49. package/dist/workflows/file-upload-reaper.d.ts +2 -0
  50. package/dist/workflows/rig-verification.d.ts +3 -0
  51. package/dist/workflows/sandbox-reaper.d.ts +1 -0
  52. package/dist/workflows/scheduled-tasks.d.ts +17 -0
  53. package/dist/workflows/session.d.ts +58 -0
  54. package/dist/workflows/workflow-wake.d.ts +3 -0
  55. package/dist/workflows.d.ts +7 -0
  56. package/package.json +13 -13
  57. package/src/activities/agent-turn.ts +873 -131
  58. package/src/activities/context-compaction.ts +265 -76
  59. package/src/activities/environment.ts +1 -0
  60. package/src/activities/goals.ts +14 -0
  61. package/src/activities/rig-verification.ts +2 -0
  62. package/src/activities/run-input.ts +18 -4
  63. package/src/activities/sandbox-lease.ts +787 -133
  64. package/src/activities/session-state.ts +56 -0
  65. package/src/activities/types.ts +21 -0
  66. package/src/activities/workspace-capture.ts +95 -4
  67. package/src/activities.ts +3 -0
  68. package/src/observability-metrics.ts +94 -0
  69. package/src/sandbox-resume.ts +383 -90
  70. package/src/sandbox-routing.ts +32 -10
  71. package/src/workflows/activities.ts +1 -0
  72. package/src/workflows/session.ts +70 -0
@@ -20,21 +20,26 @@
20
20
  //
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
- import type { Settings } from "@opengeni/config";
23
+ import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
24
24
  import { redactSensitiveText } from "@opengeni/contracts";
25
+ import { randomUUID } from "node:crypto";
25
26
  import {
26
27
  acquireLease,
28
+ adoptLegacyModalCheckpointArtifact,
27
29
  beginSandboxRematerialization,
30
+ claimWorkspaceArchiveCapture,
28
31
  commitWarmingToWarm,
29
32
  failSandboxRematerialization,
30
33
  failWarmingToCold,
31
34
  getSandboxSessionEnvelope,
32
35
  markSandboxRestoreVerifying,
33
36
  markWarmLeaseInstanceLost,
37
+ markSandboxCheckpointArtifactDeletePending,
34
38
  persistWarmSnapshot,
35
- readWorkspaceArchiveCapturePreflight,
36
39
  readLease,
40
+ registerSandboxCheckpointArtifact,
37
41
  recordWarmingSandboxCreated,
42
+ releaseWorkspaceArchiveCapture,
38
43
  releaseLeaseHolder,
39
44
  touchLeaseHolder,
40
45
  SandboxLeaseRecoveryBlockedError,
@@ -44,12 +49,15 @@ import {
44
49
  } from "@opengeni/db";
45
50
  import {
46
51
  captureVerifiedWorkspaceArchive,
52
+ describeLegacyNativeSnapshotArchive,
47
53
  WorkspaceArchiveIntegrityError,
48
54
  establishSandboxSessionFromEnvelope,
49
55
  isProviderSandboxNotFoundError,
56
+ parseWorkspaceArchiveDescriptor,
50
57
  requirePersistableReplacementSandboxEnvelope,
58
+ modalSessionMatchesCheckpointProviderBinding,
59
+ resolveModalCheckpointProviderBindingForSession,
51
60
  serializeReplacementSandboxEnvelope,
52
- deletePriorPersistedSnapshot,
53
61
  tagModalSandbox,
54
62
  verifySandboxExecReadiness,
55
63
  type EstablishedSandboxSession,
@@ -89,6 +97,18 @@ export type SandboxResumeServices = {
89
97
  db: Database;
90
98
  settings: Settings;
91
99
  sandboxMetrics?: RuntimeMetricsHooks;
100
+ /**
101
+ * The logical turn-attempt lifetime, not merely the provider request lifetime.
102
+ * Some provider SDK calls cannot be interrupted. Aborting this signal still
103
+ * releases the durable holder immediately and fences any late provider result.
104
+ */
105
+ cancellationSignal?: AbortSignal;
106
+ /** Test seam for the attached/resumed path. Production uses the one runtime
107
+ * resume primitive; callers must not use this to create a replacement box. */
108
+ establishAttachedSandbox?: typeof establishSandboxSessionFromEnvelope;
109
+ /** Test seam for the bounded command-readiness proof performed before an
110
+ * attached provider box is handed to the agent. */
111
+ verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
92
112
  /** Called only by the observer that wins the exact warm->cold loss CAS. */
93
113
  onSandboxLost?: (input: {
94
114
  sandboxGroupId: string;
@@ -122,17 +142,18 @@ export type ResumeBoxIds = {
122
142
  /**
123
143
  * IMAGE IS SHARED STATE (B3): the container image this run resolves (Modal image ref
124
144
  * / docker image). Threaded to acquireLease, which stamps it on the cold-create and
125
- * conflicts on a live box already running a DIFFERENT image (solo holder recreates;
126
- * N-holders throw SandboxImageConflictError). Omitted -> image is not enforced (the
127
- * selfhosted path never passes it; a legacy/null-image box never conflicts).
145
+ * conflicts on a live box already running a DIFFERENT image (a solo holder requests
146
+ * capture-and-drain rotation; N-holders throw SandboxImageConflictError). Omitted ->
147
+ * image is not enforced (the selfhosted path never passes it; a legacy/null-image box
148
+ * never conflicts).
128
149
  */
129
150
  image?: string;
130
151
  /**
131
152
  * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
132
153
  * acquireLease, which stamps it on the cold-create and conflicts on a live box
133
- * set up under a DIFFERENT rig version (solo holder recreates cold on the new
134
- * rig; N-holders throw SandboxRigConflictError). Omitted for a rig-less session
135
- * -> rig is never stamped or enforced (shares exactly as today).
154
+ * set up under a DIFFERENT rig version (a solo holder requests capture-and-drain
155
+ * rotation; N-holders throw SandboxRigConflictError). Omitted for a rig-less
156
+ * session -> rig is never stamped or enforced (shares exactly as today).
136
157
  */
137
158
  rigVersionId?: string;
138
159
  };
@@ -273,6 +294,44 @@ export async function waitForWarmSnapshot(
273
294
  }
274
295
  }
275
296
 
297
+ async function resolveModalCheckpointBindingBeforeCapture(
298
+ settings: SandboxResumeServices["settings"],
299
+ session: unknown,
300
+ signal?: AbortSignal,
301
+ ): Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>> {
302
+ if (signal?.aborted) {
303
+ throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
304
+ }
305
+ const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
306
+ let timeout: ReturnType<typeof setTimeout> | undefined;
307
+ let cancelListener: (() => void) | undefined;
308
+ try {
309
+ return await Promise.race([
310
+ pending,
311
+ ...(signal
312
+ ? [
313
+ new Promise<never>((_resolve, reject) => {
314
+ cancelListener = () =>
315
+ reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
316
+ signal.addEventListener("abort", cancelListener, { once: true });
317
+ if (signal.aborted) cancelListener();
318
+ }),
319
+ ]
320
+ : []),
321
+ new Promise<never>((_resolve, reject) => {
322
+ timeout = setTimeout(
323
+ () => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
324
+ settings.sandboxSnapshotTimeoutMs,
325
+ );
326
+ timeout.unref?.();
327
+ }),
328
+ ]);
329
+ } finally {
330
+ if (timeout) clearTimeout(timeout);
331
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
332
+ }
333
+ }
334
+
276
335
  async function terminateEstablishedSandbox(
277
336
  established: EstablishedSandboxSession | null,
278
337
  ): Promise<boolean> {
@@ -371,6 +430,20 @@ function workspaceArchiveFieldsFromEnvelope(
371
430
  };
372
431
  }
373
432
 
433
+ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined) {
434
+ const sessionState =
435
+ envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
436
+ ? (envelope.sessionState as Record<string, unknown>)
437
+ : null;
438
+ if (!sessionState) return null;
439
+ const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
440
+ if (existing?.version === 2) return null;
441
+ return describeLegacyNativeSnapshotArchive(
442
+ sessionState.workspaceArchive,
443
+ existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now(),
444
+ );
445
+ }
446
+
374
447
  /** A per-session fallback may still carry the dead provider identity that made
375
448
  * recovery necessary. It is useful only as durable archive/config input: never
376
449
  * let a cold rematerialization resume that stale provider before hydrating the
@@ -419,14 +492,16 @@ export async function maybePersistWarmWorkspaceSnapshot(
419
492
  session: unknown,
420
493
  leaseEpoch: number,
421
494
  signal?: AbortSignal,
495
+ force = false,
422
496
  ): Promise<boolean> {
423
497
  const { db, settings } = services;
424
498
  const intervalMs = settings.sandboxSnapshotIntervalMs;
425
- if (intervalMs <= 0) {
499
+ if (intervalMs <= 0 && !force) {
426
500
  return false;
427
501
  }
428
502
  const persistable = session as {
429
503
  persistWorkspace?: () => Promise<Uint8Array | undefined>;
504
+ state?: { workspacePersistence?: unknown };
430
505
  };
431
506
  if (typeof persistable.persistWorkspace !== "function") {
432
507
  return false;
@@ -435,9 +510,9 @@ export async function maybePersistWarmWorkspaceSnapshot(
435
510
  return false;
436
511
  }
437
512
  try {
438
- // Cheap throttle pre-check before the (potentially slow) capture;
439
- // persistWarmSnapshot re-checks atomically under the row lock, so this is
440
- // purely a cost optimization, not the correctness guard.
513
+ // Resolve the provider namespace before taking the durable capture gate.
514
+ // This work cannot pause or mutate the box, so keeping it outside the claim
515
+ // minimizes the time that command admission must wait.
441
516
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
442
517
  if (
443
518
  !lease ||
@@ -447,99 +522,163 @@ export async function maybePersistWarmWorkspaceSnapshot(
447
522
  ) {
448
523
  return false;
449
524
  }
450
- const preflight = await readWorkspaceArchiveCapturePreflight(db, {
525
+ const instanceId = lease.instanceId;
526
+ const nativeModalPersistence =
527
+ persistable.state?.workspacePersistence === "snapshot_filesystem" ||
528
+ persistable.state?.workspacePersistence === "snapshot_directory";
529
+ const checkpointBinding = nativeModalPersistence
530
+ ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
531
+ : null;
532
+ if (signal?.aborted) return false;
533
+
534
+ const captureId = randomUUID();
535
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
536
+ const claimed = await claimWorkspaceArchiveCapture(db, {
451
537
  accountId: ids.accountId,
452
538
  workspaceId: ids.workspaceId,
453
539
  sandboxGroupId: ids.sandboxGroupId,
540
+ captureId,
454
541
  expectedEpoch: leaseEpoch,
455
- expectedInstanceId: lease.instanceId,
542
+ expectedInstanceId: instanceId,
456
543
  liveness: "warm",
544
+ captureTimeoutMs,
545
+ minIntervalMs: force ? 0 : intervalMs,
546
+ warmAttempt: {
547
+ sessionId: ids.sessionId,
548
+ turnId: ids.turnId,
549
+ attemptId: ids.attemptId,
550
+ holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
551
+ },
457
552
  });
458
- if (!preflight) return false;
459
- if (signal?.aborted) {
460
- return false;
461
- }
462
- const sessionState =
463
- lease.resumeState && typeof lease.resumeState === "object"
464
- ? (lease.resumeState as { sessionState?: Record<string, unknown> }).sessionState
465
- : undefined;
466
- const priorAtRaw =
467
- sessionState && typeof sessionState === "object"
468
- ? sessionState.workspaceArchiveAt
469
- : undefined;
470
- const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
471
- if (
472
- preflight.archiveComplete &&
473
- Number.isFinite(priorAtMs) &&
474
- Date.now() - priorAtMs < intervalMs
475
- ) {
476
- return false;
477
- }
553
+ if (claimed.status !== "claimed") return false;
554
+
478
555
  let timeout: ReturnType<typeof setTimeout> | undefined;
479
556
  // Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
480
557
  // by capture-initiation, not land time, so a slower heartbeat capture that
481
558
  // started earlier can never overwrite a fresher turn-end capture that landed
482
559
  // first (the bounded-wait race Bugbot flagged).
483
560
  const capturedAtMs = Date.now();
484
- // On timeout the race settles on `undefined` while persistWorkspace() keeps
485
- // running orphaned. Swallow its LATE settlement: an un-awaited rejection
486
- // after the race resolved would surface as an unhandledRejection and can
487
- // take down the whole worker process — which would defeat the very hang-
488
- // guard this timeout exists to provide.
489
- const capture = captureVerifiedWorkspaceArchive(session, capturedAtMs);
490
- capture.catch(() => undefined);
561
+ const registerCandidate = async (
562
+ archive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>>,
563
+ ): Promise<{ id: string } | null> => {
564
+ if (
565
+ archive.descriptor.version !== 2 ||
566
+ (archive.descriptor.provider !== "modal_snapshot_filesystem" &&
567
+ archive.descriptor.provider !== "modal_snapshot_directory")
568
+ ) {
569
+ return null;
570
+ }
571
+ if (!checkpointBinding) {
572
+ throw new Error("Modal native snapshot has no exact session provider identity");
573
+ }
574
+ return await registerSandboxCheckpointArtifact(db, {
575
+ accountId: ids.accountId,
576
+ workspaceId: ids.workspaceId,
577
+ sandboxGroupId: ids.sandboxGroupId,
578
+ sourceLeaseId: claimed.claim.leaseId,
579
+ sourceLeaseEpoch: leaseEpoch,
580
+ sourceInstanceId: instanceId,
581
+ sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
582
+ providerBindingKey: checkpointBinding.key,
583
+ providerBinding: checkpointBinding.binding,
584
+ workspaceArchive: archive.base64,
585
+ workspaceArchiveMeta: archive.descriptor,
586
+ });
587
+ };
588
+
589
+ const abandonCandidate = async (artifactId: string, reason: string): Promise<void> => {
590
+ await markSandboxCheckpointArtifactDeletePending(db, {
591
+ accountId: ids.accountId,
592
+ workspaceId: ids.workspaceId,
593
+ artifactId,
594
+ reason,
595
+ }).catch(() => undefined);
596
+ };
597
+ // The SDK capture itself is not cancellable. Keep one owned continuation
598
+ // alive through provider settlement even if this caller's bounded wait or
599
+ // turn signal resolves first. Its finally block is the only normal release
600
+ // of the exact admission gate; a late callback cannot release a successor.
601
+ const captureAndPublish = (async (): Promise<boolean> => {
602
+ let candidate: { id: string } | null = null;
603
+ try {
604
+ const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs);
605
+ candidate = await registerCandidate(archive);
606
+ const { wrote } = await persistWarmSnapshot(db, {
607
+ accountId: ids.accountId,
608
+ workspaceId: ids.workspaceId,
609
+ sessionId: ids.sessionId,
610
+ turnId: ids.turnId,
611
+ attemptId: ids.attemptId,
612
+ sandboxGroupId: ids.sandboxGroupId,
613
+ expectedEpoch: leaseEpoch,
614
+ expectedInstanceId: instanceId,
615
+ expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
616
+ captureId,
617
+ workspaceArchive: archive.base64,
618
+ workspaceArchiveMeta: archive.descriptor,
619
+ checkpointArtifactId: candidate?.id ?? null,
620
+ minIntervalMs: force ? 0 : intervalMs,
621
+ capturedAtMs,
622
+ });
623
+ if (!wrote && candidate) {
624
+ await abandonCandidate(candidate.id, "snapshot_publication_fenced");
625
+ }
626
+ return wrote;
627
+ } catch (error) {
628
+ if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
629
+ throw error;
630
+ } finally {
631
+ await releaseWorkspaceArchiveCapture(db, {
632
+ accountId: ids.accountId,
633
+ workspaceId: ids.workspaceId,
634
+ sandboxGroupId: ids.sandboxGroupId,
635
+ captureId,
636
+ expectedEpoch: leaseEpoch,
637
+ expectedInstanceId: instanceId,
638
+ }).catch((error) => {
639
+ console.error(
640
+ "mid-session workspace capture gate release failed",
641
+ safeSnapshotError(error),
642
+ );
643
+ });
644
+ }
645
+ })();
646
+ const settled = captureAndPublish.then(
647
+ (persisted) => ({ kind: "settled" as const, persisted }),
648
+ (error) => {
649
+ console.error(
650
+ "mid-session workspace snapshot failed (turn unaffected)",
651
+ safeSnapshotError(error),
652
+ );
653
+ return { kind: "settled" as const, persisted: false };
654
+ },
655
+ );
491
656
  let cancelListener: (() => void) | undefined;
492
- const archive = await Promise.race([
493
- capture,
657
+ const outcome = await Promise.race([
658
+ settled,
494
659
  ...(signal
495
660
  ? [
496
- new Promise<undefined>((resolve) => {
497
- cancelListener = () => resolve(undefined);
661
+ new Promise<{ kind: "cancelled" }>((resolve) => {
662
+ cancelListener = () => resolve({ kind: "cancelled" });
498
663
  signal.addEventListener("abort", cancelListener, { once: true });
499
664
  if (signal.aborted) cancelListener();
500
665
  }),
501
666
  ]
502
667
  : []),
503
- new Promise<undefined>((resolve) => {
504
- timeout = setTimeout(() => resolve(undefined), settings.sandboxSnapshotTimeoutMs);
505
- if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
506
- timeout.unref();
507
- }
668
+ new Promise<{ kind: "timed_out" }>((resolve) => {
669
+ timeout = setTimeout(
670
+ () => resolve({ kind: "timed_out" }),
671
+ settings.sandboxSnapshotTimeoutMs,
672
+ );
508
673
  }),
509
674
  ]).finally(() => {
510
- if (timeout) {
511
- clearTimeout(timeout);
512
- }
675
+ if (timeout) clearTimeout(timeout);
513
676
  if (cancelListener) signal?.removeEventListener("abort", cancelListener);
514
677
  });
515
- if (!archive) {
516
- return false;
517
- }
518
- if (signal?.aborted) {
519
- return false;
520
- }
521
- const { wrote, priorArchiveForGc } = await persistWarmSnapshot(db, {
522
- accountId: ids.accountId,
523
- workspaceId: ids.workspaceId,
524
- sessionId: ids.sessionId,
525
- turnId: ids.turnId,
526
- attemptId: ids.attemptId,
527
- sandboxGroupId: ids.sandboxGroupId,
528
- expectedEpoch: leaseEpoch,
529
- expectedInstanceId: lease.instanceId,
530
- expectedWorkspaceGeneration: preflight.workspaceGeneration,
531
- workspaceArchive: archive.base64,
532
- workspaceArchiveMeta: archive.descriptor,
533
- minIntervalMs: intervalMs,
534
- capturedAtMs,
535
- });
536
- if (!wrote) {
537
- return false;
538
- }
539
- // Warm snapshots retain a 2-deep restore window. Only the two-ago archive
540
- // returned by persistWarmSnapshot is GC-eligible.
541
- await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
542
- return true;
678
+ // A timed-out/cancelled local waiter deliberately leaves the already-owned
679
+ // provider continuation running; `settled` has both rejection handling and
680
+ // exact claim cleanup, so it cannot leak an unhandled promise or artifact.
681
+ return outcome.kind === "settled" ? outcome.persisted : false;
543
682
  } catch (error) {
544
683
  // Protection, not a dependency: a failed snapshot must never fail (or slow
545
684
  // down retrying) the turn. The next heartbeat/turn-end tick retries.
@@ -570,18 +709,28 @@ export async function resumeBoxForTurn(
570
709
  const { db, settings } = services;
571
710
  const os = ids.os ?? "linux";
572
711
  const leaseTtlMs = settings.sandboxLeaseTtlMs;
712
+ const cancellationSignal = services.cancellationSignal;
573
713
 
574
714
  // The release closure is created eagerly so the caller can always release in
575
715
  // finally, even if establish/commit throws after the holder was registered.
716
+ // It is also bound directly to the logical attempt signal: an uninterruptible
717
+ // provider promise must never keep its private warmup timer and holder alive
718
+ // after Temporal has abandoned the activity.
576
719
  let released = false;
577
720
  let holderLivenessTimer: ReturnType<typeof setInterval> | undefined;
721
+ let cancellationListener: (() => void) | undefined;
578
722
  const release = async (): Promise<void> => {
579
723
  if (released) {
580
724
  return;
581
725
  }
582
726
  released = true;
727
+ if (cancellationListener) {
728
+ cancellationSignal?.removeEventListener("abort", cancellationListener);
729
+ cancellationListener = undefined;
730
+ }
583
731
  if (holderLivenessTimer) {
584
732
  clearInterval(holderLivenessTimer);
733
+ holderLivenessTimer = undefined;
585
734
  }
586
735
  await releaseLeaseHolder(db, {
587
736
  accountId: ids.accountId,
@@ -592,6 +741,15 @@ export async function resumeBoxForTurn(
592
741
  idleGraceMs: settings.sandboxIdleGraceMs,
593
742
  });
594
743
  };
744
+ const cancellationError = (): Error =>
745
+ cancellationSignal?.reason instanceof Error
746
+ ? cancellationSignal.reason
747
+ : new Error("Sandbox resume was cancelled with its owning turn attempt");
748
+ const throwIfReleasedOrCancelled = (): void => {
749
+ if (released || cancellationSignal?.aborted) {
750
+ throw cancellationError();
751
+ }
752
+ };
595
753
 
596
754
  const acquired = await acquireLease(db, {
597
755
  accountId: ids.accountId,
@@ -603,18 +761,35 @@ export async function resumeBoxForTurn(
603
761
  backend: ids.backend,
604
762
  os,
605
763
  // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
606
- // conflicts on a live box already running a different image. A SandboxImageConflictError
607
- // propagates to the turn activity (an actionable error); a solo image change is handled
608
- // by acquireLease recreating the box cold on the new image.
764
+ // conflicts on a live box already running a different image. A
765
+ // SandboxImageConflictError propagates while another holder is active; a
766
+ // solo change requests a capture-and-drain rotation and this attempt retries
767
+ // after the cold successor can safely stamp the new image.
609
768
  ...(ids.image ? { image: ids.image } : {}),
610
769
  // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
611
770
  // + conflicts on a live box under a different rig. A SandboxRigConflictError
612
- // propagates to the turn activity (actionable); a solo rig change recreates cold.
771
+ // propagates while another holder is active; a solo change uses the same
772
+ // durable capture-and-drain rotation as an image change.
613
773
  ...(ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {}),
614
774
  leaseTtlMs,
615
775
  warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
776
+ captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
616
777
  });
617
778
 
779
+ if (cancellationSignal) {
780
+ cancellationListener = () => {
781
+ // release() flips `released` and clears the timer synchronously before its
782
+ // first await. The detached rejection handler is intentional: the outer
783
+ // turn owns diagnostics, while this listener owns leak prevention.
784
+ void release().catch(() => undefined);
785
+ };
786
+ cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
787
+ }
788
+ if (cancellationSignal?.aborted) {
789
+ await release();
790
+ throw cancellationError();
791
+ }
792
+
618
793
  // HOLDER-LIVENESS loop: touch OUR holder row every 10s from the moment it is
619
794
  // registered until release. The dead-worker turn-holder reap judges liveness
620
795
  // by last_heartbeat_at, and the full turn heartbeat (heartbeatLeaseHolder in
@@ -632,7 +807,16 @@ export async function resumeBoxForTurn(
632
807
  sandboxGroupId: ids.sandboxGroupId,
633
808
  kind,
634
809
  holderId,
635
- }).catch(() => undefined);
810
+ })
811
+ .then((touched) => {
812
+ // A canonical turn holder is rejected once its exact attempt is no
813
+ // longer the active writer. Stop this otherwise-unbounded provider
814
+ // operation and idempotently drop any remaining holder state.
815
+ if (!touched) {
816
+ void release().catch(() => undefined);
817
+ }
818
+ })
819
+ .catch(() => undefined);
636
820
  }, 10_000);
637
821
  if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
638
822
  holderLivenessTimer.unref();
@@ -663,6 +847,12 @@ export async function resumeBoxForTurn(
663
847
  let rematerialization: {
664
848
  id: string;
665
849
  selectedRevision: string;
850
+ workspaceGeneration: number;
851
+ providerBindingKey: string | null;
852
+ legacyCheckpoint: NonNullable<ReturnType<typeof legacyNativeArchiveFromEnvelope>> | null;
853
+ legacyProviderBinding: Awaited<
854
+ ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
855
+ > | null;
666
856
  } | null = null;
667
857
  try {
668
858
  const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
@@ -677,7 +867,7 @@ export async function resumeBoxForTurn(
677
867
  workspaceArchiveFieldsFromEnvelope(envelope) !== null
678
868
  ? withoutProviderIdentity(envelope)
679
869
  : null;
680
- const spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
870
+ let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
681
871
  const archiveSource =
682
872
  acquired.lease.recovery.archive.status === "none"
683
873
  ? fallbackArchiveEnvelope
@@ -687,6 +877,7 @@ export async function resumeBoxForTurn(
687
877
  workspaceArchiveFieldsFromEnvelope(archiveSource) !== null
688
878
  ) {
689
879
  const rematerializationId = crypto.randomUUID();
880
+ const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
690
881
  const begun = await beginSandboxRematerialization(db, {
691
882
  accountId: ids.accountId,
692
883
  workspaceId: ids.workspaceId,
@@ -694,6 +885,7 @@ export async function resumeBoxForTurn(
694
885
  expectedEpoch,
695
886
  rematerializationId,
696
887
  archiveSource,
888
+ legacyNativeArchive,
697
889
  });
698
890
  if (begun.status !== "started") {
699
891
  if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
@@ -709,6 +901,7 @@ export async function resumeBoxForTurn(
709
901
  begun.lease?.recovery ?? acquired.lease.recovery,
710
902
  );
711
903
  }
904
+ spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
712
905
  const selectedRevision = begun.lease.recovery.restore.selectedRevision;
713
906
  if (!selectedRevision) {
714
907
  throw new WorkspaceArchiveIntegrityError(
@@ -716,7 +909,14 @@ export async function resumeBoxForTurn(
716
909
  "sandbox rematerialization selected no durable archive revision",
717
910
  );
718
911
  }
719
- rematerialization = { id: rematerializationId, selectedRevision };
912
+ rematerialization = {
913
+ id: rematerializationId,
914
+ selectedRevision,
915
+ workspaceGeneration: begun.lease.workspaceGeneration,
916
+ providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
917
+ legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
918
+ legacyProviderBinding: null,
919
+ };
720
920
  } else if (acquired.lease.recovery.archive.status !== "none") {
721
921
  throw new SandboxLeaseRecoveryBlockedError(
722
922
  ids.sandboxGroupId,
@@ -734,6 +934,7 @@ export async function resumeBoxForTurn(
734
934
  // spawner branch: the lease's resume_state is authoritative; the session
735
935
  // `_sandbox` envelope is the per-session fallback. Without this a turn-first
736
936
  // re-warm after a drain->cold would ignore the archive and start an EMPTY box.
937
+ const providerCreateStartedAt = new Date();
737
938
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
738
939
  sessionId: ids.sessionId,
739
940
  recovery: "create-or-restore",
@@ -742,6 +943,47 @@ export async function resumeBoxForTurn(
742
943
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
743
944
  onSandboxCreated: async (created) => {
744
945
  createdEstablished = created;
946
+ throwIfReleasedOrCancelled();
947
+ if (
948
+ rematerialization &&
949
+ (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)
950
+ ) {
951
+ if (created.backendId !== "modal") {
952
+ throw new WorkspaceArchiveIntegrityError(
953
+ "native_snapshot_reference_invalid",
954
+ "Modal checkpoint restore resolved a non-Modal sandbox backend",
955
+ );
956
+ }
957
+ if (rematerialization.providerBindingKey) {
958
+ if (
959
+ !(await modalSessionMatchesCheckpointProviderBinding(
960
+ settings,
961
+ created.session,
962
+ rematerialization.providerBindingKey,
963
+ ))
964
+ ) {
965
+ throw new WorkspaceArchiveIntegrityError(
966
+ "native_snapshot_reference_invalid",
967
+ "Modal checkpoint restore refused because the authenticated provider workspace changed",
968
+ );
969
+ }
970
+ } else {
971
+ const resolved = await resolveModalCheckpointProviderBindingForSession(
972
+ settings,
973
+ created.session,
974
+ );
975
+ if (
976
+ rematerialization.legacyProviderBinding &&
977
+ rematerialization.legacyProviderBinding.key !== resolved.key
978
+ ) {
979
+ throw new WorkspaceArchiveIntegrityError(
980
+ "native_snapshot_reference_invalid",
981
+ "Legacy Modal checkpoint restore crossed authenticated provider workspaces",
982
+ );
983
+ }
984
+ rematerialization.legacyProviderBinding = resolved;
985
+ }
986
+ }
745
987
  const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
746
988
  await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
747
989
  created.backendId,
@@ -755,6 +997,14 @@ export async function resumeBoxForTurn(
755
997
  instanceId: created.instanceId,
756
998
  resumeBackendId: created.backendId,
757
999
  resumeState: resumeEnvelope,
1000
+ ...(created.backendId === "modal"
1001
+ ? {
1002
+ providerCreatedAt: providerCreateStartedAt,
1003
+ providerDeadlineAt: new Date(
1004
+ providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1000,
1005
+ ),
1006
+ }
1007
+ : {}),
758
1008
  leaseTtlMs,
759
1009
  // Keep the warming budget after create(): manifest setup and
760
1010
  // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
@@ -770,6 +1020,7 @@ export async function resumeBoxForTurn(
770
1020
  sandboxGroupId: ids.sandboxGroupId,
771
1021
  }).catch(() => undefined);
772
1022
  }
1023
+ throwIfReleasedOrCancelled();
773
1024
  },
774
1025
  onWorkspaceRestoreVerifying: async (descriptor: WorkspaceArchiveDescriptor) => {
775
1026
  if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
@@ -778,6 +1029,33 @@ export async function resumeBoxForTurn(
778
1029
  `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`,
779
1030
  );
780
1031
  }
1032
+ if (rematerialization.legacyCheckpoint) {
1033
+ const binding = rematerialization.legacyProviderBinding;
1034
+ if (!binding) {
1035
+ throw new WorkspaceArchiveIntegrityError(
1036
+ "native_snapshot_reference_invalid",
1037
+ "Legacy Modal checkpoint restore produced no authenticated provider identity",
1038
+ );
1039
+ }
1040
+ const adopted = await adoptLegacyModalCheckpointArtifact(db, {
1041
+ accountId: ids.accountId,
1042
+ workspaceId: ids.workspaceId,
1043
+ sandboxGroupId: ids.sandboxGroupId,
1044
+ leaseId: acquired.lease.id,
1045
+ leaseEpoch: expectedEpoch,
1046
+ workspaceGeneration: rematerialization.workspaceGeneration,
1047
+ slot: "current",
1048
+ archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
1049
+ descriptor: rematerialization.legacyCheckpoint.descriptor,
1050
+ providerBindingKey: binding.key,
1051
+ providerBinding: binding.binding,
1052
+ rematerializationId: rematerialization.id,
1053
+ });
1054
+ if (!adopted) {
1055
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1056
+ }
1057
+ rematerialization.providerBindingKey = binding.key;
1058
+ }
781
1059
  const verifying = await markSandboxRestoreVerifying(db, {
782
1060
  accountId: ids.accountId,
783
1061
  workspaceId: ids.workspaceId,
@@ -791,11 +1069,13 @@ export async function resumeBoxForTurn(
791
1069
  },
792
1070
  });
793
1071
  createdEstablished = established;
1072
+ throwIfReleasedOrCancelled();
794
1073
  // A sandbox handle is not sufficient evidence that Modal's command router
795
1074
  // is live. Do not publish a warm lease until one bounded no-op exec works.
796
1075
  // On timeout the catch below terminates the box and rolls warming -> cold,
797
1076
  // so the next turn cold-creates instead of hanging forever on first use.
798
1077
  await waitForSandboxExecReadiness(established);
1078
+ throwIfReleasedOrCancelled();
799
1079
  // Fold the LIVE box into a re-resumable envelope and persist it as the
800
1080
  // lease's resume_state — exactly like the API-direct paths (channel-a.ts /
801
1081
  // viewer.ts). Without this the turn committed the ORIGINAL session manifest
@@ -817,6 +1097,7 @@ export async function resumeBoxForTurn(
817
1097
  await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
818
1098
  established.backendId,
819
1099
  );
1100
+ throwIfReleasedOrCancelled();
820
1101
  if (
821
1102
  rematerialization &&
822
1103
  established.restoredArchive?.revision !== rematerialization.selectedRevision
@@ -867,6 +1148,7 @@ export async function resumeBoxForTurn(
867
1148
  await release();
868
1149
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
869
1150
  }
1151
+ throwIfReleasedOrCancelled();
870
1152
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
871
1153
  } catch (error) {
872
1154
  if (error instanceof SandboxLeaseSupersededError) {
@@ -924,6 +1206,7 @@ export async function resumeBoxForTurn(
924
1206
  if (acquired.lease.liveness === "warming") {
925
1207
  leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
926
1208
  }
1209
+ throwIfReleasedOrCancelled();
927
1210
 
928
1211
  // Prefer the lease's resume_state (the LIVE box the spawner committed) so we
929
1212
  // re-attach to the SAME box by id, not cold-restore the original session
@@ -940,13 +1223,22 @@ export async function resumeBoxForTurn(
940
1223
  }
941
1224
  let established: EstablishedSandboxSession;
942
1225
  try {
943
- established = await establishSandboxSessionFromEnvelope(settings, live.resumeState, {
1226
+ const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
1227
+ established = await establish(settings, live.resumeState, {
944
1228
  sessionId: ids.sessionId,
945
1229
  recovery: "resume-only",
946
1230
  backendOverride: ids.backend as never,
947
1231
  ...(ids.environment ? { environment: ids.environment } : {}),
948
1232
  ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
949
1233
  });
1234
+ throwIfReleasedOrCancelled();
1235
+ // A durable `warm` row is an ownership assertion, not provider liveness.
1236
+ // Modal may have ended the exact box at its finite timeout while OpenGeni
1237
+ // was idle. Prove the command router before handing the session to the
1238
+ // agent so terminal evidence enters the atomic warm->cold recovery path
1239
+ // below instead of surfacing inside a model-visible tool call.
1240
+ await (services.verifyAttachedSandboxReadiness ?? waitForSandboxExecReadiness)(established);
1241
+ throwIfReleasedOrCancelled();
950
1242
  } catch (error) {
951
1243
  if (!isProviderSandboxNotFoundError(ids.backend, error)) {
952
1244
  throw error;
@@ -975,6 +1267,7 @@ export async function resumeBoxForTurn(
975
1267
  marked.lease?.leaseEpoch ?? leaseEpoch,
976
1268
  );
977
1269
  }
1270
+ throwIfReleasedOrCancelled();
978
1271
  return { established, leaseEpoch, release };
979
1272
  } catch (error) {
980
1273
  await release();