@opengeni/worker-bundle 0.10.8 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,6 +42,7 @@ import {
42
42
  listLiveModalSandboxLeaseAttributions,
43
43
  listMeterableWarmLeases,
44
44
  persistDrainSnapshot,
45
+ readWorkspaceArchiveCapturePreflight,
45
46
  readLease,
46
47
  reapExpiredSessionListSnapshots,
47
48
  reapStaleLeaseHoldersGlobal,
@@ -57,6 +58,7 @@ import {
57
58
  // builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
58
59
  // its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
59
60
  // (NotFound on resume) is already down, a clean no-op.
61
+ captureVerifiedWorkspaceArchive,
60
62
  createSandboxClientForBackend,
61
63
  deletePriorPersistedSnapshot,
62
64
  deserializeSandboxSessionStateEnvelope,
@@ -64,6 +66,7 @@ import {
64
66
  sweepModalOrphanSandboxes,
65
67
  terminateModalSandboxById,
66
68
  type ModalOrphanSweepTermination,
69
+ type WorkspaceArchiveDescriptor,
67
70
  } from "@opengeni/runtime";
68
71
  import type { ActivityServices } from "./types";
69
72
  import { reconcilePendingParentSystemUpdates } from "./parent-wake";
@@ -124,29 +127,44 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
124
127
  * db/accountId/row and delegates to persistDrainSnapshot. Returns:
125
128
  * - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
126
129
  * is wanted again; the seam MUST NOT terminate it.
127
- * - priorArchive / priorArchivePrev -> superseded archives to GC at drain.
130
+ * - priorArchiveForGc -> the one archive made unreachable by rotation.
128
131
  *
129
132
  * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
130
133
  * backends with no persistWorkspace — ensures a re-arm during the snapshot window
131
- * aborts the terminate before client.delete()). priorArchive values are always
134
+ * aborts the terminate before client.delete()). The GC candidate is always
132
135
  * null in this case.
133
136
  */
134
137
  export type PersistArchiveFn = (
135
138
  archiveBase64: string | null,
136
- ) => Promise<{ wrote: boolean; priorArchive: string | null; priorArchivePrev: string | null }>;
139
+ archiveMetadata?: WorkspaceArchiveDescriptor,
140
+ ) => Promise<{
141
+ wrote: boolean;
142
+ priorArchiveForGc: string | null;
143
+ archiveRevision?: string | null;
144
+ }>;
145
+
146
+ export type ProviderTerminationOutcome = {
147
+ /** false means an epoch/refcount CAS proved the box was re-armed, so it was
148
+ * deliberately left running and the cold commit must not execute. */
149
+ terminated: boolean;
150
+ /** True only when a definitive provider NotFound proved the cloud box had
151
+ * already vanished before this drain could capture its workspace. */
152
+ providerMissingBeforeCapture: boolean;
153
+ };
137
154
 
138
155
  /** The provider-terminate seam. Production wires the real resume-by-id ->
139
156
  * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
140
157
  * provider stop() (`terminateProviderBox`); a unit test injects a spy so the
141
158
  * drain/CAS logic is exercised against a real DB without a live provider box.
142
- * Returns true when the box was terminated (or already gone), false when persist
143
- * found the lease re-armed and the box was deliberately LEFT RUNNING. */
159
+ * Test seams may return the legacy boolean shorthand. Production returns the
160
+ * typed outcome so the cold commit can distinguish a clean stop from definitive
161
+ * provider disappearance before capture. */
144
162
  export type TerminateBoxFn = (
145
163
  settings: ActivityServices["settings"],
146
164
  lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
147
165
  observability: ActivityServices["observability"],
148
166
  persistArchive: PersistArchiveFn,
149
- ) => Promise<boolean>;
167
+ ) => Promise<boolean | ProviderTerminationOutcome>;
150
168
 
151
169
  export type SweepModalOrphansFn = (
152
170
  settings: ActivityServices["settings"],
@@ -563,6 +581,17 @@ async function terminateDrainableBox(
563
581
  return false;
564
582
  }
565
583
 
584
+ const capturePreflight = lease.instanceId
585
+ ? await readWorkspaceArchiveCapturePreflight(db, {
586
+ accountId,
587
+ workspaceId: row.workspaceId,
588
+ sandboxGroupId: row.sandboxGroupId,
589
+ expectedEpoch: row.leaseEpoch,
590
+ expectedInstanceId: lease.instanceId,
591
+ liveness: "draining",
592
+ })
593
+ : null;
594
+
566
595
  // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
567
596
  // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
568
597
  // Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
@@ -572,28 +601,43 @@ async function terminateDrainableBox(
572
601
  // the durable sandbox.box.terminated event below carries it, so a "terminated
573
602
  // with NOTHING persisted" (box already dead at drain) is visible in the DB.
574
603
  let persisted = false;
575
- const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) => {
604
+ let archiveRevision: string | null = null;
605
+ const persistArchive: PersistArchiveFn = async (
606
+ archiveBase64: string | null,
607
+ archiveMetadata?: WorkspaceArchiveDescriptor,
608
+ ) => {
609
+ // A live persistable provider must have passed an exact instance/generation
610
+ // preflight before capture. Without it we cannot prove that these bytes came
611
+ // from the lease row we are about to fold, so fail closed and leave it live.
612
+ if (!lease.instanceId || !capturePreflight) {
613
+ return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
614
+ }
576
615
  const result = await persistDrainSnapshot(db, {
577
616
  accountId,
578
617
  workspaceId: row.workspaceId,
579
618
  sandboxGroupId: row.sandboxGroupId,
580
619
  expectedEpoch: row.leaseEpoch,
620
+ expectedInstanceId: lease.instanceId,
621
+ expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
581
622
  workspaceArchive: archiveBase64,
623
+ ...(archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}),
582
624
  });
583
625
  if (result.wrote && archiveBase64 !== null) {
584
626
  persisted = true;
627
+ archiveRevision = result.archiveRevision;
585
628
  }
586
629
  return result;
587
630
  };
588
631
 
589
- // Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
590
- // provider terminate. A box that is already gone (NotFound) is success (the goal
591
- // is "box is not running"). A genuine PERSIST failure (could not snapshot) or a
592
- // resume failure on a box that IS alive re-throws -> the lease stays draining for
593
- // the next sweep (NEVER terminate a box whose files we could not capture). A
594
- // persist CAS-miss (re-armed) returns false -> the box was left running; skip the
595
- // cold-commit so the re-armed box is not cold-leaked.
596
- const terminated = await terminateBox(settings, lease, observability, persistArchive);
632
+ // Resume-by-id -> verified capture -> persist-onto-lease -> GC prior snapshot
633
+ // -> provider terminate. Definitive NotFound before capture is a typed loss,
634
+ // not ordinary teardown success: confirmDrainCold preserves any prior archive
635
+ // or marks recovery unrecoverable when none exists. A genuine capture/resume
636
+ // failure leaves the lease draining for a later sweep (NEVER terminate a box
637
+ // whose files we could not capture). A persist CAS miss means the box was
638
+ // re-armed and left running, so the cold commit is skipped.
639
+ const termination = await terminateBox(settings, lease, observability, persistArchive);
640
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
597
641
  if (!terminated) {
598
642
  return false;
599
643
  }
@@ -606,6 +650,8 @@ async function terminateDrainableBox(
606
650
  workspaceId: row.workspaceId,
607
651
  sandboxGroupId: row.sandboxGroupId,
608
652
  expectedEpoch: row.leaseEpoch,
653
+ providerMissingBeforeCapture:
654
+ typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
609
655
  });
610
656
  if (wentCold) {
611
657
  // Durable termination record (sandbox-file-persistence observability): who
@@ -615,7 +661,14 @@ async function terminateDrainableBox(
615
661
  try {
616
662
  await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
617
663
  type: "sandbox.box.terminated",
618
- payload: { actor: "reaper", persisted, instanceId: lease.instanceId },
664
+ payload: {
665
+ actor: "reaper",
666
+ persisted,
667
+ archiveRevision,
668
+ instanceId: lease.instanceId,
669
+ providerMissingBeforeCapture:
670
+ typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
671
+ },
619
672
  });
620
673
  } catch (eventError) {
621
674
  observability.warn(
@@ -671,11 +724,11 @@ export async function terminateProviderBox(
671
724
  observability: ActivityServices["observability"],
672
725
  persistArchive: PersistArchiveFn,
673
726
  createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
674
- ): Promise<boolean> {
727
+ ): Promise<ProviderTerminationOutcome> {
675
728
  const backend = (lease.resumeBackendId ?? lease.backend) as string;
676
729
  // 'none' / no backend -> nothing to terminate.
677
730
  if (!backend || backend === "none") {
678
- return true;
731
+ return { terminated: true, providerMissingBeforeCapture: false };
679
732
  }
680
733
 
681
734
  // CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
@@ -699,7 +752,7 @@ export async function terminateProviderBox(
699
752
  backend,
700
753
  },
701
754
  );
702
- return true;
755
+ return { terminated: true, providerMissingBeforeCapture: false };
703
756
  }
704
757
 
705
758
  // A warming-death row can have a provider instance id recorded immediately
@@ -717,7 +770,7 @@ export async function terminateProviderBox(
717
770
  instanceId: lease.instanceId,
718
771
  },
719
772
  );
720
- return false;
773
+ return { terminated: false, providerMissingBeforeCapture: false };
721
774
  }
722
775
  try {
723
776
  await terminateModalSandboxById(settings, lease.instanceId);
@@ -731,19 +784,22 @@ export async function terminateProviderBox(
731
784
  instanceId: lease.instanceId,
732
785
  });
733
786
  }
734
- return true;
787
+ // This branch is a warming-death box recorded before any resumable envelope
788
+ // was published. It was never a ready workspace, so a missing instance does
789
+ // not imply loss of a previously exposed workspace revision.
790
+ return { terminated: true, providerMissingBeforeCapture: false };
735
791
  }
736
792
 
737
793
  // resume_state is the folded group box-envelope (the provider sessionState the
738
794
  // box was last persisted as). No envelope -> no live box to stop.
739
795
  if (!lease.resumeState) {
740
- return true;
796
+ return { terminated: true, providerMissingBeforeCapture: false };
741
797
  }
742
798
 
743
799
  const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
744
800
  if (!client) {
745
801
  // 'none' backend resolved to no client.
746
- return true;
802
+ return { terminated: true, providerMissingBeforeCapture: false };
747
803
  }
748
804
 
749
805
  // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
@@ -752,9 +808,10 @@ export async function terminateProviderBox(
752
808
  let sessionState: unknown;
753
809
  try {
754
810
  if (!client.resume || !client.deserializeSessionState) {
755
- // A backend that cannot resume cannot be attached-to for terminate; the
756
- // provider idle-timeout is the backstop. No-op.
757
- return true;
811
+ // A cloud backend that cannot prove provider state is not safely
812
+ // terminable: treating this as success would cold the lease while the
813
+ // provider may still be live. Leave it draining for a later retry.
814
+ throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
758
815
  }
759
816
  // resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
760
817
  // providerState: { sandboxId, ... }, manifest, ... } }` (the shape
@@ -774,20 +831,17 @@ export async function terminateProviderBox(
774
831
  envelopeSessionState,
775
832
  );
776
833
  if (resumedState === undefined) {
777
- return true;
834
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
778
835
  }
779
836
  session = (await client.resume(resumedState)) as PersistableSession;
780
837
  sessionState = resumedState;
781
838
  } catch (error) {
782
839
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
783
- observability.info(
784
- "sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold",
785
- {
786
- sandboxGroupId: lease.sandboxGroupId,
787
- backend,
788
- },
789
- );
790
- return true;
840
+ observability.info("sandbox reaper: drainable box already gone before workspace capture", {
841
+ sandboxGroupId: lease.sandboxGroupId,
842
+ backend,
843
+ });
844
+ return { terminated: true, providerMissingBeforeCapture: true };
791
845
  }
792
846
  // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
793
847
  // draining for the next sweep) — never cold a box we could not prove is gone.
@@ -798,19 +852,18 @@ export async function terminateProviderBox(
798
852
  // snapshot must re-throw BEFORE any terminate so files are never lost — the next
799
853
  // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
800
854
  // box raced gone between resume and persist) is success: nothing to persist.
801
- let archiveBytes: Uint8Array | undefined;
855
+ let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
802
856
  try {
803
- archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
857
+ verifiedArchive = session?.persistWorkspace
858
+ ? await captureVerifiedWorkspaceArchive(session)
859
+ : undefined;
804
860
  } catch (error) {
805
861
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
806
- observability.info(
807
- "sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold",
808
- {
809
- sandboxGroupId: lease.sandboxGroupId,
810
- backend,
811
- },
812
- );
813
- return true;
862
+ observability.info("sandbox reaper: box gone during workspace capture", {
863
+ sandboxGroupId: lease.sandboxGroupId,
864
+ backend,
865
+ });
866
+ return { terminated: true, providerMissingBeforeCapture: true };
814
867
  }
815
868
  observability.warn(
816
869
  "sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
@@ -836,9 +889,11 @@ export async function terminateProviderBox(
836
889
  // so without this check we would delete a box the lease now treats as live. The
837
890
  // null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
838
891
  // refcount/epoch guard, no write. wrote:false → abort the terminate.
839
- if (archiveBytes && archiveBytes.length > 0) {
840
- const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
841
- const { wrote, priorArchive, priorArchivePrev } = await persistArchive(archiveBase64);
892
+ if (verifiedArchive) {
893
+ const { wrote, priorArchiveForGc } = await persistArchive(
894
+ verifiedArchive.base64,
895
+ verifiedArchive.descriptor,
896
+ );
842
897
  if (!wrote) {
843
898
  observability.info(
844
899
  "sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
@@ -847,11 +902,11 @@ export async function terminateProviderBox(
847
902
  backend,
848
903
  },
849
904
  );
850
- return false;
905
+ return { terminated: false, providerMissingBeforeCapture: false };
851
906
  }
852
907
  // Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
853
908
  // while the session (and its Modal client) is still live. Never throws.
854
- const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
909
+ const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
855
910
  if (deleted) {
856
911
  observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
857
912
  sandboxGroupId: lease.sandboxGroupId,
@@ -859,29 +914,13 @@ export async function terminateProviderBox(
859
914
  snapshotId: deleted,
860
915
  });
861
916
  }
862
- const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
863
- if (deletedPrev) {
864
- observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
865
- sandboxGroupId: lease.sandboxGroupId,
866
- backend,
867
- snapshotId: deletedPrev,
868
- });
869
- }
870
917
  } else {
871
- // No archive produced (backend has no persistWorkspace or returned empty).
872
- // CAS-check via persistArchive(null): if the lease was re-armed during the
873
- // snapshot window, wrote:false → leave the box RUNNING (abort terminate).
874
- const { wrote } = await persistArchive(null);
875
- if (!wrote) {
876
- observability.info(
877
- "sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
878
- {
879
- sandboxGroupId: lease.sandboxGroupId,
880
- backend,
881
- },
882
- );
883
- return false;
884
- }
918
+ // A resumable cloud box that produces no verified archive cannot be safely
919
+ // deleted: it may contain work newer than the lease's last durable revision.
920
+ // Keep the lease draining and retry instead of knowingly discarding bytes.
921
+ throw new Error(
922
+ `sandbox backend ${backend} produced no verified workspace archive during drain`,
923
+ );
885
924
  }
886
925
 
887
926
  // Provider terminate. Prefer the client.delete(state) teardown (the canonical
@@ -890,25 +929,33 @@ export async function terminateProviderBox(
890
929
  try {
891
930
  if (client.delete && sessionState !== undefined) {
892
931
  await client.delete(sessionState);
893
- return true;
932
+ return { terminated: true, providerMissingBeforeCapture: false };
894
933
  }
895
934
  if (session?.kill) {
896
935
  await session.kill();
897
- return true;
936
+ return { terminated: true, providerMissingBeforeCapture: false };
898
937
  }
899
938
  if (session?.terminate) {
900
939
  await session.terminate();
901
- return true;
940
+ return { terminated: true, providerMissingBeforeCapture: false };
902
941
  }
903
- if (session?.close && !session.closed) {
904
- await session.close();
942
+ if (session?.close) {
943
+ if (!session.closed) {
944
+ await session.close();
945
+ }
946
+ return { terminated: true, providerMissingBeforeCapture: false };
905
947
  }
906
948
  } catch (error) {
907
- observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
908
- sandboxGroupId: lease.sandboxGroupId,
909
- backend,
910
- error: error instanceof Error ? error.message : String(error),
911
- });
949
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
950
+ observability.info("sandbox reaper: provider already gone during terminate", {
951
+ sandboxGroupId: lease.sandboxGroupId,
952
+ backend,
953
+ });
954
+ // Capture was already durably folded above, so this is not a
955
+ // missing-before-capture outcome.
956
+ return { terminated: true, providerMissingBeforeCapture: false };
957
+ }
958
+ throw error;
912
959
  }
913
- return true;
960
+ throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
914
961
  }
@@ -15,6 +15,7 @@ import {
15
15
  requireSession,
16
16
  setTemporalWorkflowId,
17
17
  settleScheduledTaskRunInTransaction,
18
+ SessionSpawnDeniedDbError,
18
19
  sumUsageQuantity,
19
20
  updateScheduledTask,
20
21
  upsertSessionGoal,
@@ -158,34 +159,48 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
158
159
  frozenRigId = rig.id;
159
160
  frozenRigVersionId = rig.activeVersion.id;
160
161
  }
161
- const session = await createSession(db, {
162
- accountId: task.accountId,
163
- workspaceId: task.workspaceId,
164
- initialMessage: task.agentConfig.prompt,
165
- resources: task.agentConfig.resources,
166
- tools: taskTools,
167
- metadata: {
168
- ...task.agentConfig.metadata,
162
+ let session: Awaited<ReturnType<typeof createSession>>;
163
+ try {
164
+ session = await createSession(db, {
165
+ accountId: task.accountId,
166
+ workspaceId: task.workspaceId,
167
+ initialMessage: task.agentConfig.prompt,
168
+ resources: task.agentConfig.resources,
169
+ tools: taskTools,
170
+ metadata: {
171
+ ...task.agentConfig.metadata,
172
+ model,
173
+ reasoningEffort,
174
+ scheduledTaskId: task.id,
175
+ scheduledTaskRunId: run.id,
176
+ },
177
+ createdBy: {
178
+ kind: "service",
179
+ subjectId: "scheduler",
180
+ label: "OpenGeni scheduler",
181
+ },
182
+ createdByContext: {
183
+ scheduledTaskId: task.id,
184
+ scheduledTaskRunId: run.id,
185
+ },
169
186
  model,
170
- reasoningEffort,
171
- scheduledTaskId: task.id,
172
- scheduledTaskRunId: run.id,
173
- },
174
- createdBy: {
175
- kind: "service",
176
- subjectId: "scheduler",
177
- label: "OpenGeni scheduler",
178
- },
179
- createdByContext: {
180
- scheduledTaskId: task.id,
181
- scheduledTaskRunId: run.id,
182
- },
183
- model,
184
- sandboxBackend,
185
- variableSetId: task.variableSetId ?? null,
186
- rigId: frozenRigId,
187
- rigVersionId: frozenRigVersionId,
188
- });
187
+ sandboxBackend,
188
+ variableSetId: task.variableSetId ?? null,
189
+ rigId: frozenRigId,
190
+ rigVersionId: frozenRigVersionId,
191
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
192
+ // The durable agent config was privilege-checked when the task
193
+ // was created/updated. Preserve that explicit policy if a broader
194
+ // workspace/deployment limit is narrowed before a later fire.
195
+ allowNestedAgentDepthIncrease: true,
196
+ subjectId: `scheduled_task:${task.id}`,
197
+ });
198
+ } catch (error) {
199
+ if (error instanceof SessionSpawnDeniedDbError) {
200
+ throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
201
+ }
202
+ throw error;
203
+ }
189
204
  const goal = goalSpec
190
205
  ? await createSessionGoal(db, {
191
206
  accountId: task.accountId,