@opengeni/worker-bundle 0.8.3 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2735 -905
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +7 -4
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1123 -462
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/environment.ts +142 -11
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +162 -84
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
- package/src/workflows/rig-verification.ts +3 -1
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
listLiveModalSandboxLeaseAttributions,
|
|
43
43
|
listMeterableWarmLeases,
|
|
44
44
|
persistDrainSnapshot,
|
|
45
|
+
readWorkspaceArchiveCapturePreflight,
|
|
45
46
|
readLease,
|
|
46
47
|
reapExpiredSessionListSnapshots,
|
|
47
48
|
reapStaleLeaseHoldersGlobal,
|
|
@@ -57,12 +58,15 @@ import {
|
|
|
57
58
|
// builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
|
|
58
59
|
// its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
|
|
59
60
|
// (NotFound on resume) is already down, a clean no-op.
|
|
61
|
+
captureVerifiedWorkspaceArchive,
|
|
60
62
|
createSandboxClientForBackend,
|
|
61
63
|
deletePriorPersistedSnapshot,
|
|
62
64
|
deserializeSandboxSessionStateEnvelope,
|
|
63
65
|
isProviderSandboxNotFoundError,
|
|
64
66
|
sweepModalOrphanSandboxes,
|
|
65
67
|
terminateModalSandboxById,
|
|
68
|
+
type ModalOrphanSweepTermination,
|
|
69
|
+
type WorkspaceArchiveDescriptor,
|
|
66
70
|
} from "@opengeni/runtime";
|
|
67
71
|
import type { ActivityServices } from "./types";
|
|
68
72
|
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
@@ -123,29 +127,44 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
|
123
127
|
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
124
128
|
* - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
|
|
125
129
|
* is wanted again; the seam MUST NOT terminate it.
|
|
126
|
-
* -
|
|
130
|
+
* - priorArchiveForGc -> the one archive made unreachable by rotation.
|
|
127
131
|
*
|
|
128
132
|
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
129
133
|
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
130
|
-
* aborts the terminate before client.delete()).
|
|
134
|
+
* aborts the terminate before client.delete()). The GC candidate is always
|
|
131
135
|
* null in this case.
|
|
132
136
|
*/
|
|
133
137
|
export type PersistArchiveFn = (
|
|
134
138
|
archiveBase64: string | null,
|
|
135
|
-
|
|
139
|
+
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
140
|
+
) => Promise<{
|
|
141
|
+
wrote: boolean;
|
|
142
|
+
priorArchiveForGc: string | null;
|
|
143
|
+
archiveRevision?: string | null;
|
|
144
|
+
}>;
|
|
145
|
+
|
|
146
|
+
export type ProviderTerminationOutcome = {
|
|
147
|
+
/** false means an epoch/refcount CAS proved the box was re-armed, so it was
|
|
148
|
+
* deliberately left running and the cold commit must not execute. */
|
|
149
|
+
terminated: boolean;
|
|
150
|
+
/** True only when a definitive provider NotFound proved the cloud box had
|
|
151
|
+
* already vanished before this drain could capture its workspace. */
|
|
152
|
+
providerMissingBeforeCapture: boolean;
|
|
153
|
+
};
|
|
136
154
|
|
|
137
155
|
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
138
156
|
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
139
157
|
* provider stop() (`terminateProviderBox`); a unit test injects a spy so the
|
|
140
158
|
* drain/CAS logic is exercised against a real DB without a live provider box.
|
|
141
|
-
*
|
|
142
|
-
*
|
|
159
|
+
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
160
|
+
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
161
|
+
* provider disappearance before capture. */
|
|
143
162
|
export type TerminateBoxFn = (
|
|
144
163
|
settings: ActivityServices["settings"],
|
|
145
164
|
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
146
165
|
observability: ActivityServices["observability"],
|
|
147
166
|
persistArchive: PersistArchiveFn,
|
|
148
|
-
) => Promise<boolean>;
|
|
167
|
+
) => Promise<boolean | ProviderTerminationOutcome>;
|
|
149
168
|
|
|
150
169
|
export type SweepModalOrphansFn = (
|
|
151
170
|
settings: ActivityServices["settings"],
|
|
@@ -447,6 +466,31 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
447
466
|
return forceDrained;
|
|
448
467
|
}
|
|
449
468
|
|
|
469
|
+
export function modalOrphanTerminationStillEligible(
|
|
470
|
+
latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
|
|
471
|
+
candidate: ModalOrphanSweepTermination,
|
|
472
|
+
): boolean {
|
|
473
|
+
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
474
|
+
return false;
|
|
475
|
+
}
|
|
476
|
+
const leaseId = candidate.tags.opengeni_lease_id;
|
|
477
|
+
const workspaceId = candidate.tags.opengeni_workspace_id;
|
|
478
|
+
const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
|
|
479
|
+
if (!leaseId || !workspaceId || !sandboxGroupId) {
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
const activeAttribution = latest.find(
|
|
483
|
+
(lease) =>
|
|
484
|
+
lease.leaseId === leaseId &&
|
|
485
|
+
lease.workspaceId === workspaceId &&
|
|
486
|
+
lease.sandboxGroupId === sandboxGroupId,
|
|
487
|
+
);
|
|
488
|
+
return Boolean(
|
|
489
|
+
!activeAttribution ||
|
|
490
|
+
(activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
|
|
450
494
|
async function sweepModalOrphansForConfiguredBackend(
|
|
451
495
|
settings: ActivityServices["settings"],
|
|
452
496
|
db: ActivityServices["db"],
|
|
@@ -468,7 +512,12 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
468
512
|
return 0;
|
|
469
513
|
}
|
|
470
514
|
|
|
471
|
-
const result = await sweepModalOrphanSandboxes(settings, liveLeases
|
|
515
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
|
|
516
|
+
revalidateTermination: async (candidate) => {
|
|
517
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
518
|
+
return modalOrphanTerminationStillEligible(latest, candidate);
|
|
519
|
+
},
|
|
520
|
+
});
|
|
472
521
|
for (const terminated of result.terminated) {
|
|
473
522
|
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
474
523
|
sandboxId: terminated.sandboxId,
|
|
@@ -532,6 +581,17 @@ async function terminateDrainableBox(
|
|
|
532
581
|
return false;
|
|
533
582
|
}
|
|
534
583
|
|
|
584
|
+
const capturePreflight = lease.instanceId
|
|
585
|
+
? await readWorkspaceArchiveCapturePreflight(db, {
|
|
586
|
+
accountId,
|
|
587
|
+
workspaceId: row.workspaceId,
|
|
588
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
589
|
+
expectedEpoch: row.leaseEpoch,
|
|
590
|
+
expectedInstanceId: lease.instanceId,
|
|
591
|
+
liveness: "draining",
|
|
592
|
+
})
|
|
593
|
+
: null;
|
|
594
|
+
|
|
535
595
|
// The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
|
|
536
596
|
// snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
|
|
537
597
|
// Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
|
|
@@ -541,28 +601,43 @@ async function terminateDrainableBox(
|
|
|
541
601
|
// the durable sandbox.box.terminated event below carries it, so a "terminated
|
|
542
602
|
// with NOTHING persisted" (box already dead at drain) is visible in the DB.
|
|
543
603
|
let persisted = false;
|
|
544
|
-
|
|
604
|
+
let archiveRevision: string | null = null;
|
|
605
|
+
const persistArchive: PersistArchiveFn = async (
|
|
606
|
+
archiveBase64: string | null,
|
|
607
|
+
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
608
|
+
) => {
|
|
609
|
+
// A live persistable provider must have passed an exact instance/generation
|
|
610
|
+
// preflight before capture. Without it we cannot prove that these bytes came
|
|
611
|
+
// from the lease row we are about to fold, so fail closed and leave it live.
|
|
612
|
+
if (!lease.instanceId || !capturePreflight) {
|
|
613
|
+
return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
|
|
614
|
+
}
|
|
545
615
|
const result = await persistDrainSnapshot(db, {
|
|
546
616
|
accountId,
|
|
547
617
|
workspaceId: row.workspaceId,
|
|
548
618
|
sandboxGroupId: row.sandboxGroupId,
|
|
549
619
|
expectedEpoch: row.leaseEpoch,
|
|
620
|
+
expectedInstanceId: lease.instanceId,
|
|
621
|
+
expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
|
|
550
622
|
workspaceArchive: archiveBase64,
|
|
623
|
+
...(archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}),
|
|
551
624
|
});
|
|
552
625
|
if (result.wrote && archiveBase64 !== null) {
|
|
553
626
|
persisted = true;
|
|
627
|
+
archiveRevision = result.archiveRevision;
|
|
554
628
|
}
|
|
555
629
|
return result;
|
|
556
630
|
};
|
|
557
631
|
|
|
558
|
-
// Resume-by-id ->
|
|
559
|
-
// provider terminate.
|
|
560
|
-
//
|
|
561
|
-
//
|
|
562
|
-
// the
|
|
563
|
-
//
|
|
564
|
-
//
|
|
565
|
-
const
|
|
632
|
+
// Resume-by-id -> verified capture -> persist-onto-lease -> GC prior snapshot
|
|
633
|
+
// -> provider terminate. Definitive NotFound before capture is a typed loss,
|
|
634
|
+
// not ordinary teardown success: confirmDrainCold preserves any prior archive
|
|
635
|
+
// or marks recovery unrecoverable when none exists. A genuine capture/resume
|
|
636
|
+
// failure leaves the lease draining for a later sweep (NEVER terminate a box
|
|
637
|
+
// whose files we could not capture). A persist CAS miss means the box was
|
|
638
|
+
// re-armed and left running, so the cold commit is skipped.
|
|
639
|
+
const termination = await terminateBox(settings, lease, observability, persistArchive);
|
|
640
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
566
641
|
if (!terminated) {
|
|
567
642
|
return false;
|
|
568
643
|
}
|
|
@@ -575,6 +650,8 @@ async function terminateDrainableBox(
|
|
|
575
650
|
workspaceId: row.workspaceId,
|
|
576
651
|
sandboxGroupId: row.sandboxGroupId,
|
|
577
652
|
expectedEpoch: row.leaseEpoch,
|
|
653
|
+
providerMissingBeforeCapture:
|
|
654
|
+
typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
|
|
578
655
|
});
|
|
579
656
|
if (wentCold) {
|
|
580
657
|
// Durable termination record (sandbox-file-persistence observability): who
|
|
@@ -584,7 +661,14 @@ async function terminateDrainableBox(
|
|
|
584
661
|
try {
|
|
585
662
|
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
586
663
|
type: "sandbox.box.terminated",
|
|
587
|
-
payload: {
|
|
664
|
+
payload: {
|
|
665
|
+
actor: "reaper",
|
|
666
|
+
persisted,
|
|
667
|
+
archiveRevision,
|
|
668
|
+
instanceId: lease.instanceId,
|
|
669
|
+
providerMissingBeforeCapture:
|
|
670
|
+
typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
|
|
671
|
+
},
|
|
588
672
|
});
|
|
589
673
|
} catch (eventError) {
|
|
590
674
|
observability.warn(
|
|
@@ -640,11 +724,11 @@ export async function terminateProviderBox(
|
|
|
640
724
|
observability: ActivityServices["observability"],
|
|
641
725
|
persistArchive: PersistArchiveFn,
|
|
642
726
|
createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
|
|
643
|
-
): Promise<
|
|
727
|
+
): Promise<ProviderTerminationOutcome> {
|
|
644
728
|
const backend = (lease.resumeBackendId ?? lease.backend) as string;
|
|
645
729
|
// 'none' / no backend -> nothing to terminate.
|
|
646
730
|
if (!backend || backend === "none") {
|
|
647
|
-
return true;
|
|
731
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
648
732
|
}
|
|
649
733
|
|
|
650
734
|
// CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
|
|
@@ -668,7 +752,7 @@ export async function terminateProviderBox(
|
|
|
668
752
|
backend,
|
|
669
753
|
},
|
|
670
754
|
);
|
|
671
|
-
return true;
|
|
755
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
672
756
|
}
|
|
673
757
|
|
|
674
758
|
// A warming-death row can have a provider instance id recorded immediately
|
|
@@ -686,7 +770,7 @@ export async function terminateProviderBox(
|
|
|
686
770
|
instanceId: lease.instanceId,
|
|
687
771
|
},
|
|
688
772
|
);
|
|
689
|
-
return false;
|
|
773
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
690
774
|
}
|
|
691
775
|
try {
|
|
692
776
|
await terminateModalSandboxById(settings, lease.instanceId);
|
|
@@ -700,19 +784,22 @@ export async function terminateProviderBox(
|
|
|
700
784
|
instanceId: lease.instanceId,
|
|
701
785
|
});
|
|
702
786
|
}
|
|
703
|
-
|
|
787
|
+
// This branch is a warming-death box recorded before any resumable envelope
|
|
788
|
+
// was published. It was never a ready workspace, so a missing instance does
|
|
789
|
+
// not imply loss of a previously exposed workspace revision.
|
|
790
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
704
791
|
}
|
|
705
792
|
|
|
706
793
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
707
794
|
// box was last persisted as). No envelope -> no live box to stop.
|
|
708
795
|
if (!lease.resumeState) {
|
|
709
|
-
return true;
|
|
796
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
710
797
|
}
|
|
711
798
|
|
|
712
799
|
const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
|
|
713
800
|
if (!client) {
|
|
714
801
|
// 'none' backend resolved to no client.
|
|
715
|
-
return true;
|
|
802
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
716
803
|
}
|
|
717
804
|
|
|
718
805
|
// Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
|
|
@@ -721,9 +808,10 @@ export async function terminateProviderBox(
|
|
|
721
808
|
let sessionState: unknown;
|
|
722
809
|
try {
|
|
723
810
|
if (!client.resume || !client.deserializeSessionState) {
|
|
724
|
-
// A backend that cannot
|
|
725
|
-
//
|
|
726
|
-
|
|
811
|
+
// A cloud backend that cannot prove provider state is not safely
|
|
812
|
+
// terminable: treating this as success would cold the lease while the
|
|
813
|
+
// provider may still be live. Leave it draining for a later retry.
|
|
814
|
+
throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
|
|
727
815
|
}
|
|
728
816
|
// resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
|
|
729
817
|
// providerState: { sandboxId, ... }, manifest, ... } }` (the shape
|
|
@@ -743,20 +831,17 @@ export async function terminateProviderBox(
|
|
|
743
831
|
envelopeSessionState,
|
|
744
832
|
);
|
|
745
833
|
if (resumedState === undefined) {
|
|
746
|
-
|
|
834
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
747
835
|
}
|
|
748
836
|
session = (await client.resume(resumedState)) as PersistableSession;
|
|
749
837
|
sessionState = resumedState;
|
|
750
838
|
} catch (error) {
|
|
751
839
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
752
|
-
observability.info(
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
},
|
|
758
|
-
);
|
|
759
|
-
return true;
|
|
840
|
+
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
841
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
842
|
+
backend,
|
|
843
|
+
});
|
|
844
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
760
845
|
}
|
|
761
846
|
// Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
|
|
762
847
|
// draining for the next sweep) — never cold a box we could not prove is gone.
|
|
@@ -767,19 +852,18 @@ export async function terminateProviderBox(
|
|
|
767
852
|
// snapshot must re-throw BEFORE any terminate so files are never lost — the next
|
|
768
853
|
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
769
854
|
// box raced gone between resume and persist) is success: nothing to persist.
|
|
770
|
-
let
|
|
855
|
+
let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
|
|
771
856
|
try {
|
|
772
|
-
|
|
857
|
+
verifiedArchive = session?.persistWorkspace
|
|
858
|
+
? await captureVerifiedWorkspaceArchive(session)
|
|
859
|
+
: undefined;
|
|
773
860
|
} catch (error) {
|
|
774
861
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
775
|
-
observability.info(
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
},
|
|
781
|
-
);
|
|
782
|
-
return true;
|
|
862
|
+
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
863
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
864
|
+
backend,
|
|
865
|
+
});
|
|
866
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
783
867
|
}
|
|
784
868
|
observability.warn(
|
|
785
869
|
"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
|
|
@@ -805,9 +889,11 @@ export async function terminateProviderBox(
|
|
|
805
889
|
// so without this check we would delete a box the lease now treats as live. The
|
|
806
890
|
// null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
|
|
807
891
|
// refcount/epoch guard, no write. wrote:false → abort the terminate.
|
|
808
|
-
if (
|
|
809
|
-
const
|
|
810
|
-
|
|
892
|
+
if (verifiedArchive) {
|
|
893
|
+
const { wrote, priorArchiveForGc } = await persistArchive(
|
|
894
|
+
verifiedArchive.base64,
|
|
895
|
+
verifiedArchive.descriptor,
|
|
896
|
+
);
|
|
811
897
|
if (!wrote) {
|
|
812
898
|
observability.info(
|
|
813
899
|
"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
|
|
@@ -816,11 +902,11 @@ export async function terminateProviderBox(
|
|
|
816
902
|
backend,
|
|
817
903
|
},
|
|
818
904
|
);
|
|
819
|
-
return false;
|
|
905
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
820
906
|
}
|
|
821
907
|
// Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
|
|
822
908
|
// while the session (and its Modal client) is still live. Never throws.
|
|
823
|
-
const deleted = await deletePriorPersistedSnapshot(session,
|
|
909
|
+
const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
|
|
824
910
|
if (deleted) {
|
|
825
911
|
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
826
912
|
sandboxGroupId: lease.sandboxGroupId,
|
|
@@ -828,29 +914,13 @@ export async function terminateProviderBox(
|
|
|
828
914
|
snapshotId: deleted,
|
|
829
915
|
});
|
|
830
916
|
}
|
|
831
|
-
const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
|
|
832
|
-
if (deletedPrev) {
|
|
833
|
-
observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
|
|
834
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
835
|
-
backend,
|
|
836
|
-
snapshotId: deletedPrev,
|
|
837
|
-
});
|
|
838
|
-
}
|
|
839
917
|
} else {
|
|
840
|
-
//
|
|
841
|
-
//
|
|
842
|
-
//
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
"sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
|
|
847
|
-
{
|
|
848
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
849
|
-
backend,
|
|
850
|
-
},
|
|
851
|
-
);
|
|
852
|
-
return false;
|
|
853
|
-
}
|
|
918
|
+
// A resumable cloud box that produces no verified archive cannot be safely
|
|
919
|
+
// deleted: it may contain work newer than the lease's last durable revision.
|
|
920
|
+
// Keep the lease draining and retry instead of knowingly discarding bytes.
|
|
921
|
+
throw new Error(
|
|
922
|
+
`sandbox backend ${backend} produced no verified workspace archive during drain`,
|
|
923
|
+
);
|
|
854
924
|
}
|
|
855
925
|
|
|
856
926
|
// Provider terminate. Prefer the client.delete(state) teardown (the canonical
|
|
@@ -859,25 +929,33 @@ export async function terminateProviderBox(
|
|
|
859
929
|
try {
|
|
860
930
|
if (client.delete && sessionState !== undefined) {
|
|
861
931
|
await client.delete(sessionState);
|
|
862
|
-
return true;
|
|
932
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
863
933
|
}
|
|
864
934
|
if (session?.kill) {
|
|
865
935
|
await session.kill();
|
|
866
|
-
return true;
|
|
936
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
867
937
|
}
|
|
868
938
|
if (session?.terminate) {
|
|
869
939
|
await session.terminate();
|
|
870
|
-
return true;
|
|
940
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
871
941
|
}
|
|
872
|
-
if (session?.close
|
|
873
|
-
|
|
942
|
+
if (session?.close) {
|
|
943
|
+
if (!session.closed) {
|
|
944
|
+
await session.close();
|
|
945
|
+
}
|
|
946
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
874
947
|
}
|
|
875
948
|
} catch (error) {
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
949
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
950
|
+
observability.info("sandbox reaper: provider already gone during terminate", {
|
|
951
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
952
|
+
backend,
|
|
953
|
+
});
|
|
954
|
+
// Capture was already durably folded above, so this is not a
|
|
955
|
+
// missing-before-capture outcome.
|
|
956
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
957
|
+
}
|
|
958
|
+
throw error;
|
|
881
959
|
}
|
|
882
|
-
|
|
960
|
+
throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
|
|
883
961
|
}
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
requireSession,
|
|
16
16
|
setTemporalWorkflowId,
|
|
17
17
|
settleScheduledTaskRunInTransaction,
|
|
18
|
+
SessionSpawnDeniedDbError,
|
|
18
19
|
sumUsageQuantity,
|
|
19
20
|
updateScheduledTask,
|
|
20
21
|
upsertSessionGoal,
|
|
@@ -158,34 +159,48 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
158
159
|
frozenRigId = rig.id;
|
|
159
160
|
frozenRigVersionId = rig.activeVersion.id;
|
|
160
161
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
162
|
+
let session: Awaited<ReturnType<typeof createSession>>;
|
|
163
|
+
try {
|
|
164
|
+
session = await createSession(db, {
|
|
165
|
+
accountId: task.accountId,
|
|
166
|
+
workspaceId: task.workspaceId,
|
|
167
|
+
initialMessage: task.agentConfig.prompt,
|
|
168
|
+
resources: task.agentConfig.resources,
|
|
169
|
+
tools: taskTools,
|
|
170
|
+
metadata: {
|
|
171
|
+
...task.agentConfig.metadata,
|
|
172
|
+
model,
|
|
173
|
+
reasoningEffort,
|
|
174
|
+
scheduledTaskId: task.id,
|
|
175
|
+
scheduledTaskRunId: run.id,
|
|
176
|
+
},
|
|
177
|
+
createdBy: {
|
|
178
|
+
kind: "service",
|
|
179
|
+
subjectId: "scheduler",
|
|
180
|
+
label: "OpenGeni scheduler",
|
|
181
|
+
},
|
|
182
|
+
createdByContext: {
|
|
183
|
+
scheduledTaskId: task.id,
|
|
184
|
+
scheduledTaskRunId: run.id,
|
|
185
|
+
},
|
|
169
186
|
model,
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
rigVersionId: frozenRigVersionId,
|
|
188
|
-
});
|
|
187
|
+
sandboxBackend,
|
|
188
|
+
variableSetId: task.variableSetId ?? null,
|
|
189
|
+
rigId: frozenRigId,
|
|
190
|
+
rigVersionId: frozenRigVersionId,
|
|
191
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
192
|
+
// The durable agent config was privilege-checked when the task
|
|
193
|
+
// was created/updated. Preserve that explicit policy if a broader
|
|
194
|
+
// workspace/deployment limit is narrowed before a later fire.
|
|
195
|
+
allowNestedAgentDepthIncrease: true,
|
|
196
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
197
|
+
});
|
|
198
|
+
} catch (error) {
|
|
199
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
200
|
+
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
|
|
201
|
+
}
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
189
204
|
const goal = goalSpec
|
|
190
205
|
? await createSessionGoal(db, {
|
|
191
206
|
accountId: task.accountId,
|