@opengeni/worker-bundle 0.8.3 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,6 +42,7 @@ import {
42
42
  listLiveModalSandboxLeaseAttributions,
43
43
  listMeterableWarmLeases,
44
44
  persistDrainSnapshot,
45
+ readWorkspaceArchiveCapturePreflight,
45
46
  readLease,
46
47
  reapExpiredSessionListSnapshots,
47
48
  reapStaleLeaseHoldersGlobal,
@@ -57,12 +58,15 @@ import {
57
58
  // builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
58
59
  // its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
59
60
  // (NotFound on resume) is already down, a clean no-op.
61
+ captureVerifiedWorkspaceArchive,
60
62
  createSandboxClientForBackend,
61
63
  deletePriorPersistedSnapshot,
62
64
  deserializeSandboxSessionStateEnvelope,
63
65
  isProviderSandboxNotFoundError,
64
66
  sweepModalOrphanSandboxes,
65
67
  terminateModalSandboxById,
68
+ type ModalOrphanSweepTermination,
69
+ type WorkspaceArchiveDescriptor,
66
70
  } from "@opengeni/runtime";
67
71
  import type { ActivityServices } from "./types";
68
72
  import { reconcilePendingParentSystemUpdates } from "./parent-wake";
@@ -123,29 +127,44 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
123
127
  * db/accountId/row and delegates to persistDrainSnapshot. Returns:
124
128
  * - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
125
129
  * is wanted again; the seam MUST NOT terminate it.
126
- * - priorArchive / priorArchivePrev -> superseded archives to GC at drain.
130
+ * - priorArchiveForGc -> the one archive made unreachable by rotation.
127
131
  *
128
132
  * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
129
133
  * backends with no persistWorkspace — ensures a re-arm during the snapshot window
130
- * aborts the terminate before client.delete()). priorArchive values are always
134
+ * aborts the terminate before client.delete()). The GC candidate is always
131
135
  * null in this case.
132
136
  */
133
137
  export type PersistArchiveFn = (
134
138
  archiveBase64: string | null,
135
- ) => Promise<{ wrote: boolean; priorArchive: string | null; priorArchivePrev: string | null }>;
139
+ archiveMetadata?: WorkspaceArchiveDescriptor,
140
+ ) => Promise<{
141
+ wrote: boolean;
142
+ priorArchiveForGc: string | null;
143
+ archiveRevision?: string | null;
144
+ }>;
145
+
146
+ export type ProviderTerminationOutcome = {
147
+ /** false means an epoch/refcount CAS proved the box was re-armed, so it was
148
+ * deliberately left running and the cold commit must not execute. */
149
+ terminated: boolean;
150
+ /** True only when a definitive provider NotFound proved the cloud box had
151
+ * already vanished before this drain could capture its workspace. */
152
+ providerMissingBeforeCapture: boolean;
153
+ };
136
154
 
137
155
  /** The provider-terminate seam. Production wires the real resume-by-id ->
138
156
  * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
139
157
  * provider stop() (`terminateProviderBox`); a unit test injects a spy so the
140
158
  * drain/CAS logic is exercised against a real DB without a live provider box.
141
- * Returns true when the box was terminated (or already gone), false when persist
142
- * found the lease re-armed and the box was deliberately LEFT RUNNING. */
159
+ * Test seams may return the legacy boolean shorthand. Production returns the
160
+ * typed outcome so the cold commit can distinguish a clean stop from definitive
161
+ * provider disappearance before capture. */
143
162
  export type TerminateBoxFn = (
144
163
  settings: ActivityServices["settings"],
145
164
  lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
146
165
  observability: ActivityServices["observability"],
147
166
  persistArchive: PersistArchiveFn,
148
- ) => Promise<boolean>;
167
+ ) => Promise<boolean | ProviderTerminationOutcome>;
149
168
 
150
169
  export type SweepModalOrphansFn = (
151
170
  settings: ActivityServices["settings"],
@@ -447,6 +466,31 @@ async function forceDrainOverLimitWorkspaces(
447
466
  return forceDrained;
448
467
  }
449
468
 
469
+ export function modalOrphanTerminationStillEligible(
470
+ latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
471
+ candidate: ModalOrphanSweepTermination,
472
+ ): boolean {
473
+ if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
474
+ return false;
475
+ }
476
+ const leaseId = candidate.tags.opengeni_lease_id;
477
+ const workspaceId = candidate.tags.opengeni_workspace_id;
478
+ const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
479
+ if (!leaseId || !workspaceId || !sandboxGroupId) {
480
+ return true;
481
+ }
482
+ const activeAttribution = latest.find(
483
+ (lease) =>
484
+ lease.leaseId === leaseId &&
485
+ lease.workspaceId === workspaceId &&
486
+ lease.sandboxGroupId === sandboxGroupId,
487
+ );
488
+ return Boolean(
489
+ !activeAttribution ||
490
+ (activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),
491
+ );
492
+ }
493
+
450
494
  async function sweepModalOrphansForConfiguredBackend(
451
495
  settings: ActivityServices["settings"],
452
496
  db: ActivityServices["db"],
@@ -468,7 +512,12 @@ async function sweepModalOrphansForConfiguredBackend(
468
512
  return 0;
469
513
  }
470
514
 
471
- const result = await sweepModalOrphanSandboxes(settings, liveLeases);
515
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
516
+ revalidateTermination: async (candidate) => {
517
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
518
+ return modalOrphanTerminationStillEligible(latest, candidate);
519
+ },
520
+ });
472
521
  for (const terminated of result.terminated) {
473
522
  observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
474
523
  sandboxId: terminated.sandboxId,
@@ -532,6 +581,17 @@ async function terminateDrainableBox(
532
581
  return false;
533
582
  }
534
583
 
584
+ const capturePreflight = lease.instanceId
585
+ ? await readWorkspaceArchiveCapturePreflight(db, {
586
+ accountId,
587
+ workspaceId: row.workspaceId,
588
+ sandboxGroupId: row.sandboxGroupId,
589
+ expectedEpoch: row.leaseEpoch,
590
+ expectedInstanceId: lease.instanceId,
591
+ liveness: "draining",
592
+ })
593
+ : null;
594
+
535
595
  // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
536
596
  // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
537
597
  // Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
@@ -541,28 +601,43 @@ async function terminateDrainableBox(
541
601
  // the durable sandbox.box.terminated event below carries it, so a "terminated
542
602
  // with NOTHING persisted" (box already dead at drain) is visible in the DB.
543
603
  let persisted = false;
544
- const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) => {
604
+ let archiveRevision: string | null = null;
605
+ const persistArchive: PersistArchiveFn = async (
606
+ archiveBase64: string | null,
607
+ archiveMetadata?: WorkspaceArchiveDescriptor,
608
+ ) => {
609
+ // A live persistable provider must have passed an exact instance/generation
610
+ // preflight before capture. Without it we cannot prove that these bytes came
611
+ // from the lease row we are about to fold, so fail closed and leave it live.
612
+ if (!lease.instanceId || !capturePreflight) {
613
+ return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
614
+ }
545
615
  const result = await persistDrainSnapshot(db, {
546
616
  accountId,
547
617
  workspaceId: row.workspaceId,
548
618
  sandboxGroupId: row.sandboxGroupId,
549
619
  expectedEpoch: row.leaseEpoch,
620
+ expectedInstanceId: lease.instanceId,
621
+ expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
550
622
  workspaceArchive: archiveBase64,
623
+ ...(archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}),
551
624
  });
552
625
  if (result.wrote && archiveBase64 !== null) {
553
626
  persisted = true;
627
+ archiveRevision = result.archiveRevision;
554
628
  }
555
629
  return result;
556
630
  };
557
631
 
558
- // Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
559
- // provider terminate. A box that is already gone (NotFound) is success (the goal
560
- // is "box is not running"). A genuine PERSIST failure (could not snapshot) or a
561
- // resume failure on a box that IS alive re-throws -> the lease stays draining for
562
- // the next sweep (NEVER terminate a box whose files we could not capture). A
563
- // persist CAS-miss (re-armed) returns false -> the box was left running; skip the
564
- // cold-commit so the re-armed box is not cold-leaked.
565
- const terminated = await terminateBox(settings, lease, observability, persistArchive);
632
+ // Resume-by-id -> verified capture -> persist-onto-lease -> GC prior snapshot
633
+ // -> provider terminate. Definitive NotFound before capture is a typed loss,
634
+ // not ordinary teardown success: confirmDrainCold preserves any prior archive
635
+ // or marks recovery unrecoverable when none exists. A genuine capture/resume
636
+ // failure leaves the lease draining for a later sweep (NEVER terminate a box
637
+ // whose files we could not capture). A persist CAS miss means the box was
638
+ // re-armed and left running, so the cold commit is skipped.
639
+ const termination = await terminateBox(settings, lease, observability, persistArchive);
640
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
566
641
  if (!terminated) {
567
642
  return false;
568
643
  }
@@ -575,6 +650,8 @@ async function terminateDrainableBox(
575
650
  workspaceId: row.workspaceId,
576
651
  sandboxGroupId: row.sandboxGroupId,
577
652
  expectedEpoch: row.leaseEpoch,
653
+ providerMissingBeforeCapture:
654
+ typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
578
655
  });
579
656
  if (wentCold) {
580
657
  // Durable termination record (sandbox-file-persistence observability): who
@@ -584,7 +661,14 @@ async function terminateDrainableBox(
584
661
  try {
585
662
  await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
586
663
  type: "sandbox.box.terminated",
587
- payload: { actor: "reaper", persisted, instanceId: lease.instanceId },
664
+ payload: {
665
+ actor: "reaper",
666
+ persisted,
667
+ archiveRevision,
668
+ instanceId: lease.instanceId,
669
+ providerMissingBeforeCapture:
670
+ typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
671
+ },
588
672
  });
589
673
  } catch (eventError) {
590
674
  observability.warn(
@@ -640,11 +724,11 @@ export async function terminateProviderBox(
640
724
  observability: ActivityServices["observability"],
641
725
  persistArchive: PersistArchiveFn,
642
726
  createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
643
- ): Promise<boolean> {
727
+ ): Promise<ProviderTerminationOutcome> {
644
728
  const backend = (lease.resumeBackendId ?? lease.backend) as string;
645
729
  // 'none' / no backend -> nothing to terminate.
646
730
  if (!backend || backend === "none") {
647
- return true;
731
+ return { terminated: true, providerMissingBeforeCapture: false };
648
732
  }
649
733
 
650
734
  // CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
@@ -668,7 +752,7 @@ export async function terminateProviderBox(
668
752
  backend,
669
753
  },
670
754
  );
671
- return true;
755
+ return { terminated: true, providerMissingBeforeCapture: false };
672
756
  }
673
757
 
674
758
  // A warming-death row can have a provider instance id recorded immediately
@@ -686,7 +770,7 @@ export async function terminateProviderBox(
686
770
  instanceId: lease.instanceId,
687
771
  },
688
772
  );
689
- return false;
773
+ return { terminated: false, providerMissingBeforeCapture: false };
690
774
  }
691
775
  try {
692
776
  await terminateModalSandboxById(settings, lease.instanceId);
@@ -700,19 +784,22 @@ export async function terminateProviderBox(
700
784
  instanceId: lease.instanceId,
701
785
  });
702
786
  }
703
- return true;
787
+ // This branch is a warming-death box recorded before any resumable envelope
788
+ // was published. It was never a ready workspace, so a missing instance does
789
+ // not imply loss of a previously exposed workspace revision.
790
+ return { terminated: true, providerMissingBeforeCapture: false };
704
791
  }
705
792
 
706
793
  // resume_state is the folded group box-envelope (the provider sessionState the
707
794
  // box was last persisted as). No envelope -> no live box to stop.
708
795
  if (!lease.resumeState) {
709
- return true;
796
+ return { terminated: true, providerMissingBeforeCapture: false };
710
797
  }
711
798
 
712
799
  const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
713
800
  if (!client) {
714
801
  // 'none' backend resolved to no client.
715
- return true;
802
+ return { terminated: true, providerMissingBeforeCapture: false };
716
803
  }
717
804
 
718
805
  // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
@@ -721,9 +808,10 @@ export async function terminateProviderBox(
721
808
  let sessionState: unknown;
722
809
  try {
723
810
  if (!client.resume || !client.deserializeSessionState) {
724
- // A backend that cannot resume cannot be attached-to for terminate; the
725
- // provider idle-timeout is the backstop. No-op.
726
- return true;
811
+ // A cloud backend that cannot prove provider state is not safely
812
+ // terminable: treating this as success would cold the lease while the
813
+ // provider may still be live. Leave it draining for a later retry.
814
+ throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
727
815
  }
728
816
  // resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
729
817
  // providerState: { sandboxId, ... }, manifest, ... } }` (the shape
@@ -743,20 +831,17 @@ export async function terminateProviderBox(
743
831
  envelopeSessionState,
744
832
  );
745
833
  if (resumedState === undefined) {
746
- return true;
834
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
747
835
  }
748
836
  session = (await client.resume(resumedState)) as PersistableSession;
749
837
  sessionState = resumedState;
750
838
  } catch (error) {
751
839
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
752
- observability.info(
753
- "sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold",
754
- {
755
- sandboxGroupId: lease.sandboxGroupId,
756
- backend,
757
- },
758
- );
759
- return true;
840
+ observability.info("sandbox reaper: drainable box already gone before workspace capture", {
841
+ sandboxGroupId: lease.sandboxGroupId,
842
+ backend,
843
+ });
844
+ return { terminated: true, providerMissingBeforeCapture: true };
760
845
  }
761
846
  // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
762
847
  // draining for the next sweep) — never cold a box we could not prove is gone.
@@ -767,19 +852,18 @@ export async function terminateProviderBox(
767
852
  // snapshot must re-throw BEFORE any terminate so files are never lost — the next
768
853
  // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
769
854
  // box raced gone between resume and persist) is success: nothing to persist.
770
- let archiveBytes: Uint8Array | undefined;
855
+ let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
771
856
  try {
772
- archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
857
+ verifiedArchive = session?.persistWorkspace
858
+ ? await captureVerifiedWorkspaceArchive(session)
859
+ : undefined;
773
860
  } catch (error) {
774
861
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
775
- observability.info(
776
- "sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold",
777
- {
778
- sandboxGroupId: lease.sandboxGroupId,
779
- backend,
780
- },
781
- );
782
- return true;
862
+ observability.info("sandbox reaper: box gone during workspace capture", {
863
+ sandboxGroupId: lease.sandboxGroupId,
864
+ backend,
865
+ });
866
+ return { terminated: true, providerMissingBeforeCapture: true };
783
867
  }
784
868
  observability.warn(
785
869
  "sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
@@ -805,9 +889,11 @@ export async function terminateProviderBox(
805
889
  // so without this check we would delete a box the lease now treats as live. The
806
890
  // null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
807
891
  // refcount/epoch guard, no write. wrote:false → abort the terminate.
808
- if (archiveBytes && archiveBytes.length > 0) {
809
- const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
810
- const { wrote, priorArchive, priorArchivePrev } = await persistArchive(archiveBase64);
892
+ if (verifiedArchive) {
893
+ const { wrote, priorArchiveForGc } = await persistArchive(
894
+ verifiedArchive.base64,
895
+ verifiedArchive.descriptor,
896
+ );
811
897
  if (!wrote) {
812
898
  observability.info(
813
899
  "sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
@@ -816,11 +902,11 @@ export async function terminateProviderBox(
816
902
  backend,
817
903
  },
818
904
  );
819
- return false;
905
+ return { terminated: false, providerMissingBeforeCapture: false };
820
906
  }
821
907
  // Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
822
908
  // while the session (and its Modal client) is still live. Never throws.
823
- const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
909
+ const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
824
910
  if (deleted) {
825
911
  observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
826
912
  sandboxGroupId: lease.sandboxGroupId,
@@ -828,29 +914,13 @@ export async function terminateProviderBox(
828
914
  snapshotId: deleted,
829
915
  });
830
916
  }
831
- const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
832
- if (deletedPrev) {
833
- observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
834
- sandboxGroupId: lease.sandboxGroupId,
835
- backend,
836
- snapshotId: deletedPrev,
837
- });
838
- }
839
917
  } else {
840
- // No archive produced (backend has no persistWorkspace or returned empty).
841
- // CAS-check via persistArchive(null): if the lease was re-armed during the
842
- // snapshot window, wrote:false → leave the box RUNNING (abort terminate).
843
- const { wrote } = await persistArchive(null);
844
- if (!wrote) {
845
- observability.info(
846
- "sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
847
- {
848
- sandboxGroupId: lease.sandboxGroupId,
849
- backend,
850
- },
851
- );
852
- return false;
853
- }
918
+ // A resumable cloud box that produces no verified archive cannot be safely
919
+ // deleted: it may contain work newer than the lease's last durable revision.
920
+ // Keep the lease draining and retry instead of knowingly discarding bytes.
921
+ throw new Error(
922
+ `sandbox backend ${backend} produced no verified workspace archive during drain`,
923
+ );
854
924
  }
855
925
 
856
926
  // Provider terminate. Prefer the client.delete(state) teardown (the canonical
@@ -859,25 +929,33 @@ export async function terminateProviderBox(
859
929
  try {
860
930
  if (client.delete && sessionState !== undefined) {
861
931
  await client.delete(sessionState);
862
- return true;
932
+ return { terminated: true, providerMissingBeforeCapture: false };
863
933
  }
864
934
  if (session?.kill) {
865
935
  await session.kill();
866
- return true;
936
+ return { terminated: true, providerMissingBeforeCapture: false };
867
937
  }
868
938
  if (session?.terminate) {
869
939
  await session.terminate();
870
- return true;
940
+ return { terminated: true, providerMissingBeforeCapture: false };
871
941
  }
872
- if (session?.close && !session.closed) {
873
- await session.close();
942
+ if (session?.close) {
943
+ if (!session.closed) {
944
+ await session.close();
945
+ }
946
+ return { terminated: true, providerMissingBeforeCapture: false };
874
947
  }
875
948
  } catch (error) {
876
- observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
877
- sandboxGroupId: lease.sandboxGroupId,
878
- backend,
879
- error: error instanceof Error ? error.message : String(error),
880
- });
949
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
950
+ observability.info("sandbox reaper: provider already gone during terminate", {
951
+ sandboxGroupId: lease.sandboxGroupId,
952
+ backend,
953
+ });
954
+ // Capture was already durably folded above, so this is not a
955
+ // missing-before-capture outcome.
956
+ return { terminated: true, providerMissingBeforeCapture: false };
957
+ }
958
+ throw error;
881
959
  }
882
- return true;
960
+ throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
883
961
  }
@@ -15,6 +15,7 @@ import {
15
15
  requireSession,
16
16
  setTemporalWorkflowId,
17
17
  settleScheduledTaskRunInTransaction,
18
+ SessionSpawnDeniedDbError,
18
19
  sumUsageQuantity,
19
20
  updateScheduledTask,
20
21
  upsertSessionGoal,
@@ -158,34 +159,48 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
158
159
  frozenRigId = rig.id;
159
160
  frozenRigVersionId = rig.activeVersion.id;
160
161
  }
161
- const session = await createSession(db, {
162
- accountId: task.accountId,
163
- workspaceId: task.workspaceId,
164
- initialMessage: task.agentConfig.prompt,
165
- resources: task.agentConfig.resources,
166
- tools: taskTools,
167
- metadata: {
168
- ...task.agentConfig.metadata,
162
+ let session: Awaited<ReturnType<typeof createSession>>;
163
+ try {
164
+ session = await createSession(db, {
165
+ accountId: task.accountId,
166
+ workspaceId: task.workspaceId,
167
+ initialMessage: task.agentConfig.prompt,
168
+ resources: task.agentConfig.resources,
169
+ tools: taskTools,
170
+ metadata: {
171
+ ...task.agentConfig.metadata,
172
+ model,
173
+ reasoningEffort,
174
+ scheduledTaskId: task.id,
175
+ scheduledTaskRunId: run.id,
176
+ },
177
+ createdBy: {
178
+ kind: "service",
179
+ subjectId: "scheduler",
180
+ label: "OpenGeni scheduler",
181
+ },
182
+ createdByContext: {
183
+ scheduledTaskId: task.id,
184
+ scheduledTaskRunId: run.id,
185
+ },
169
186
  model,
170
- reasoningEffort,
171
- scheduledTaskId: task.id,
172
- scheduledTaskRunId: run.id,
173
- },
174
- createdBy: {
175
- kind: "service",
176
- subjectId: "scheduler",
177
- label: "OpenGeni scheduler",
178
- },
179
- createdByContext: {
180
- scheduledTaskId: task.id,
181
- scheduledTaskRunId: run.id,
182
- },
183
- model,
184
- sandboxBackend,
185
- variableSetId: task.variableSetId ?? null,
186
- rigId: frozenRigId,
187
- rigVersionId: frozenRigVersionId,
188
- });
187
+ sandboxBackend,
188
+ variableSetId: task.variableSetId ?? null,
189
+ rigId: frozenRigId,
190
+ rigVersionId: frozenRigVersionId,
191
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
192
+ // The durable agent config was privilege-checked when the task
193
+ // was created/updated. Preserve that explicit policy if a broader
194
+ // workspace/deployment limit is narrowed before a later fire.
195
+ allowNestedAgentDepthIncrease: true,
196
+ subjectId: `scheduled_task:${task.id}`,
197
+ });
198
+ } catch (error) {
199
+ if (error instanceof SessionSpawnDeniedDbError) {
200
+ throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
201
+ }
202
+ throw error;
203
+ }
189
204
  const goal = goalSpec
190
205
  ? await createSessionGoal(db, {
191
206
  accountId: task.accountId,