@opengeni/worker-bundle 0.10.8 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1484 -481
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +5 -4
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +484 -169
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +130 -83
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
listLiveModalSandboxLeaseAttributions,
|
|
43
43
|
listMeterableWarmLeases,
|
|
44
44
|
persistDrainSnapshot,
|
|
45
|
+
readWorkspaceArchiveCapturePreflight,
|
|
45
46
|
readLease,
|
|
46
47
|
reapExpiredSessionListSnapshots,
|
|
47
48
|
reapStaleLeaseHoldersGlobal,
|
|
@@ -57,6 +58,7 @@ import {
|
|
|
57
58
|
// builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
|
|
58
59
|
// its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
|
|
59
60
|
// (NotFound on resume) is already down, a clean no-op.
|
|
61
|
+
captureVerifiedWorkspaceArchive,
|
|
60
62
|
createSandboxClientForBackend,
|
|
61
63
|
deletePriorPersistedSnapshot,
|
|
62
64
|
deserializeSandboxSessionStateEnvelope,
|
|
@@ -64,6 +66,7 @@ import {
|
|
|
64
66
|
sweepModalOrphanSandboxes,
|
|
65
67
|
terminateModalSandboxById,
|
|
66
68
|
type ModalOrphanSweepTermination,
|
|
69
|
+
type WorkspaceArchiveDescriptor,
|
|
67
70
|
} from "@opengeni/runtime";
|
|
68
71
|
import type { ActivityServices } from "./types";
|
|
69
72
|
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
@@ -124,29 +127,44 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
|
124
127
|
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
125
128
|
* - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
|
|
126
129
|
* is wanted again; the seam MUST NOT terminate it.
|
|
127
|
-
* -
|
|
130
|
+
* - priorArchiveForGc -> the one archive made unreachable by rotation.
|
|
128
131
|
*
|
|
129
132
|
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
130
133
|
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
131
|
-
* aborts the terminate before client.delete()).
|
|
134
|
+
* aborts the terminate before client.delete()). The GC candidate is always
|
|
132
135
|
* null in this case.
|
|
133
136
|
*/
|
|
134
137
|
export type PersistArchiveFn = (
|
|
135
138
|
archiveBase64: string | null,
|
|
136
|
-
|
|
139
|
+
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
140
|
+
) => Promise<{
|
|
141
|
+
wrote: boolean;
|
|
142
|
+
priorArchiveForGc: string | null;
|
|
143
|
+
archiveRevision?: string | null;
|
|
144
|
+
}>;
|
|
145
|
+
|
|
146
|
+
export type ProviderTerminationOutcome = {
|
|
147
|
+
/** false means an epoch/refcount CAS proved the box was re-armed, so it was
|
|
148
|
+
* deliberately left running and the cold commit must not execute. */
|
|
149
|
+
terminated: boolean;
|
|
150
|
+
/** True only when a definitive provider NotFound proved the cloud box had
|
|
151
|
+
* already vanished before this drain could capture its workspace. */
|
|
152
|
+
providerMissingBeforeCapture: boolean;
|
|
153
|
+
};
|
|
137
154
|
|
|
138
155
|
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
139
156
|
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
140
157
|
* provider stop() (`terminateProviderBox`); a unit test injects a spy so the
|
|
141
158
|
* drain/CAS logic is exercised against a real DB without a live provider box.
|
|
142
|
-
*
|
|
143
|
-
*
|
|
159
|
+
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
160
|
+
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
161
|
+
* provider disappearance before capture. */
|
|
144
162
|
export type TerminateBoxFn = (
|
|
145
163
|
settings: ActivityServices["settings"],
|
|
146
164
|
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
147
165
|
observability: ActivityServices["observability"],
|
|
148
166
|
persistArchive: PersistArchiveFn,
|
|
149
|
-
) => Promise<boolean>;
|
|
167
|
+
) => Promise<boolean | ProviderTerminationOutcome>;
|
|
150
168
|
|
|
151
169
|
export type SweepModalOrphansFn = (
|
|
152
170
|
settings: ActivityServices["settings"],
|
|
@@ -563,6 +581,17 @@ async function terminateDrainableBox(
|
|
|
563
581
|
return false;
|
|
564
582
|
}
|
|
565
583
|
|
|
584
|
+
const capturePreflight = lease.instanceId
|
|
585
|
+
? await readWorkspaceArchiveCapturePreflight(db, {
|
|
586
|
+
accountId,
|
|
587
|
+
workspaceId: row.workspaceId,
|
|
588
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
589
|
+
expectedEpoch: row.leaseEpoch,
|
|
590
|
+
expectedInstanceId: lease.instanceId,
|
|
591
|
+
liveness: "draining",
|
|
592
|
+
})
|
|
593
|
+
: null;
|
|
594
|
+
|
|
566
595
|
// The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
|
|
567
596
|
// snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
|
|
568
597
|
// Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
|
|
@@ -572,28 +601,43 @@ async function terminateDrainableBox(
|
|
|
572
601
|
// the durable sandbox.box.terminated event below carries it, so a "terminated
|
|
573
602
|
// with NOTHING persisted" (box already dead at drain) is visible in the DB.
|
|
574
603
|
let persisted = false;
|
|
575
|
-
|
|
604
|
+
let archiveRevision: string | null = null;
|
|
605
|
+
const persistArchive: PersistArchiveFn = async (
|
|
606
|
+
archiveBase64: string | null,
|
|
607
|
+
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
608
|
+
) => {
|
|
609
|
+
// A live persistable provider must have passed an exact instance/generation
|
|
610
|
+
// preflight before capture. Without it we cannot prove that these bytes came
|
|
611
|
+
// from the lease row we are about to fold, so fail closed and leave it live.
|
|
612
|
+
if (!lease.instanceId || !capturePreflight) {
|
|
613
|
+
return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
|
|
614
|
+
}
|
|
576
615
|
const result = await persistDrainSnapshot(db, {
|
|
577
616
|
accountId,
|
|
578
617
|
workspaceId: row.workspaceId,
|
|
579
618
|
sandboxGroupId: row.sandboxGroupId,
|
|
580
619
|
expectedEpoch: row.leaseEpoch,
|
|
620
|
+
expectedInstanceId: lease.instanceId,
|
|
621
|
+
expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
|
|
581
622
|
workspaceArchive: archiveBase64,
|
|
623
|
+
...(archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}),
|
|
582
624
|
});
|
|
583
625
|
if (result.wrote && archiveBase64 !== null) {
|
|
584
626
|
persisted = true;
|
|
627
|
+
archiveRevision = result.archiveRevision;
|
|
585
628
|
}
|
|
586
629
|
return result;
|
|
587
630
|
};
|
|
588
631
|
|
|
589
|
-
// Resume-by-id ->
|
|
590
|
-
// provider terminate.
|
|
591
|
-
//
|
|
592
|
-
//
|
|
593
|
-
// the
|
|
594
|
-
//
|
|
595
|
-
//
|
|
596
|
-
const
|
|
632
|
+
// Resume-by-id -> verified capture -> persist-onto-lease -> GC prior snapshot
|
|
633
|
+
// -> provider terminate. Definitive NotFound before capture is a typed loss,
|
|
634
|
+
// not ordinary teardown success: confirmDrainCold preserves any prior archive
|
|
635
|
+
// or marks recovery unrecoverable when none exists. A genuine capture/resume
|
|
636
|
+
// failure leaves the lease draining for a later sweep (NEVER terminate a box
|
|
637
|
+
// whose files we could not capture). A persist CAS miss means the box was
|
|
638
|
+
// re-armed and left running, so the cold commit is skipped.
|
|
639
|
+
const termination = await terminateBox(settings, lease, observability, persistArchive);
|
|
640
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
597
641
|
if (!terminated) {
|
|
598
642
|
return false;
|
|
599
643
|
}
|
|
@@ -606,6 +650,8 @@ async function terminateDrainableBox(
|
|
|
606
650
|
workspaceId: row.workspaceId,
|
|
607
651
|
sandboxGroupId: row.sandboxGroupId,
|
|
608
652
|
expectedEpoch: row.leaseEpoch,
|
|
653
|
+
providerMissingBeforeCapture:
|
|
654
|
+
typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
|
|
609
655
|
});
|
|
610
656
|
if (wentCold) {
|
|
611
657
|
// Durable termination record (sandbox-file-persistence observability): who
|
|
@@ -615,7 +661,14 @@ async function terminateDrainableBox(
|
|
|
615
661
|
try {
|
|
616
662
|
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
617
663
|
type: "sandbox.box.terminated",
|
|
618
|
-
payload: {
|
|
664
|
+
payload: {
|
|
665
|
+
actor: "reaper",
|
|
666
|
+
persisted,
|
|
667
|
+
archiveRevision,
|
|
668
|
+
instanceId: lease.instanceId,
|
|
669
|
+
providerMissingBeforeCapture:
|
|
670
|
+
typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture,
|
|
671
|
+
},
|
|
619
672
|
});
|
|
620
673
|
} catch (eventError) {
|
|
621
674
|
observability.warn(
|
|
@@ -671,11 +724,11 @@ export async function terminateProviderBox(
|
|
|
671
724
|
observability: ActivityServices["observability"],
|
|
672
725
|
persistArchive: PersistArchiveFn,
|
|
673
726
|
createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
|
|
674
|
-
): Promise<
|
|
727
|
+
): Promise<ProviderTerminationOutcome> {
|
|
675
728
|
const backend = (lease.resumeBackendId ?? lease.backend) as string;
|
|
676
729
|
// 'none' / no backend -> nothing to terminate.
|
|
677
730
|
if (!backend || backend === "none") {
|
|
678
|
-
return true;
|
|
731
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
679
732
|
}
|
|
680
733
|
|
|
681
734
|
// CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop
|
|
@@ -699,7 +752,7 @@ export async function terminateProviderBox(
|
|
|
699
752
|
backend,
|
|
700
753
|
},
|
|
701
754
|
);
|
|
702
|
-
return true;
|
|
755
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
703
756
|
}
|
|
704
757
|
|
|
705
758
|
// A warming-death row can have a provider instance id recorded immediately
|
|
@@ -717,7 +770,7 @@ export async function terminateProviderBox(
|
|
|
717
770
|
instanceId: lease.instanceId,
|
|
718
771
|
},
|
|
719
772
|
);
|
|
720
|
-
return false;
|
|
773
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
721
774
|
}
|
|
722
775
|
try {
|
|
723
776
|
await terminateModalSandboxById(settings, lease.instanceId);
|
|
@@ -731,19 +784,22 @@ export async function terminateProviderBox(
|
|
|
731
784
|
instanceId: lease.instanceId,
|
|
732
785
|
});
|
|
733
786
|
}
|
|
734
|
-
|
|
787
|
+
// This branch is a warming-death box recorded before any resumable envelope
|
|
788
|
+
// was published. It was never a ready workspace, so a missing instance does
|
|
789
|
+
// not imply loss of a previously exposed workspace revision.
|
|
790
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
735
791
|
}
|
|
736
792
|
|
|
737
793
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
738
794
|
// box was last persisted as). No envelope -> no live box to stop.
|
|
739
795
|
if (!lease.resumeState) {
|
|
740
|
-
return true;
|
|
796
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
741
797
|
}
|
|
742
798
|
|
|
743
799
|
const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
|
|
744
800
|
if (!client) {
|
|
745
801
|
// 'none' backend resolved to no client.
|
|
746
|
-
return true;
|
|
802
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
747
803
|
}
|
|
748
804
|
|
|
749
805
|
// Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
|
|
@@ -752,9 +808,10 @@ export async function terminateProviderBox(
|
|
|
752
808
|
let sessionState: unknown;
|
|
753
809
|
try {
|
|
754
810
|
if (!client.resume || !client.deserializeSessionState) {
|
|
755
|
-
// A backend that cannot
|
|
756
|
-
//
|
|
757
|
-
|
|
811
|
+
// A cloud backend that cannot prove provider state is not safely
|
|
812
|
+
// terminable: treating this as success would cold the lease while the
|
|
813
|
+
// provider may still be live. Leave it draining for a later retry.
|
|
814
|
+
throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
|
|
758
815
|
}
|
|
759
816
|
// resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
|
|
760
817
|
// providerState: { sandboxId, ... }, manifest, ... } }` (the shape
|
|
@@ -774,20 +831,17 @@ export async function terminateProviderBox(
|
|
|
774
831
|
envelopeSessionState,
|
|
775
832
|
);
|
|
776
833
|
if (resumedState === undefined) {
|
|
777
|
-
|
|
834
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
778
835
|
}
|
|
779
836
|
session = (await client.resume(resumedState)) as PersistableSession;
|
|
780
837
|
sessionState = resumedState;
|
|
781
838
|
} catch (error) {
|
|
782
839
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
783
|
-
observability.info(
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
},
|
|
789
|
-
);
|
|
790
|
-
return true;
|
|
840
|
+
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
841
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
842
|
+
backend,
|
|
843
|
+
});
|
|
844
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
791
845
|
}
|
|
792
846
|
// Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
|
|
793
847
|
// draining for the next sweep) — never cold a box we could not prove is gone.
|
|
@@ -798,19 +852,18 @@ export async function terminateProviderBox(
|
|
|
798
852
|
// snapshot must re-throw BEFORE any terminate so files are never lost — the next
|
|
799
853
|
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
800
854
|
// box raced gone between resume and persist) is success: nothing to persist.
|
|
801
|
-
let
|
|
855
|
+
let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
|
|
802
856
|
try {
|
|
803
|
-
|
|
857
|
+
verifiedArchive = session?.persistWorkspace
|
|
858
|
+
? await captureVerifiedWorkspaceArchive(session)
|
|
859
|
+
: undefined;
|
|
804
860
|
} catch (error) {
|
|
805
861
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
806
|
-
observability.info(
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
},
|
|
812
|
-
);
|
|
813
|
-
return true;
|
|
862
|
+
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
863
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
864
|
+
backend,
|
|
865
|
+
});
|
|
866
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
814
867
|
}
|
|
815
868
|
observability.warn(
|
|
816
869
|
"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
|
|
@@ -836,9 +889,11 @@ export async function terminateProviderBox(
|
|
|
836
889
|
// so without this check we would delete a box the lease now treats as live. The
|
|
837
890
|
// null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
|
|
838
891
|
// refcount/epoch guard, no write. wrote:false → abort the terminate.
|
|
839
|
-
if (
|
|
840
|
-
const
|
|
841
|
-
|
|
892
|
+
if (verifiedArchive) {
|
|
893
|
+
const { wrote, priorArchiveForGc } = await persistArchive(
|
|
894
|
+
verifiedArchive.base64,
|
|
895
|
+
verifiedArchive.descriptor,
|
|
896
|
+
);
|
|
842
897
|
if (!wrote) {
|
|
843
898
|
observability.info(
|
|
844
899
|
"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
|
|
@@ -847,11 +902,11 @@ export async function terminateProviderBox(
|
|
|
847
902
|
backend,
|
|
848
903
|
},
|
|
849
904
|
);
|
|
850
|
-
return false;
|
|
905
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
851
906
|
}
|
|
852
907
|
// Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
|
|
853
908
|
// while the session (and its Modal client) is still live. Never throws.
|
|
854
|
-
const deleted = await deletePriorPersistedSnapshot(session,
|
|
909
|
+
const deleted = await deletePriorPersistedSnapshot(session, priorArchiveForGc);
|
|
855
910
|
if (deleted) {
|
|
856
911
|
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
857
912
|
sandboxGroupId: lease.sandboxGroupId,
|
|
@@ -859,29 +914,13 @@ export async function terminateProviderBox(
|
|
|
859
914
|
snapshotId: deleted,
|
|
860
915
|
});
|
|
861
916
|
}
|
|
862
|
-
const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
|
|
863
|
-
if (deletedPrev) {
|
|
864
|
-
observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
|
|
865
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
866
|
-
backend,
|
|
867
|
-
snapshotId: deletedPrev,
|
|
868
|
-
});
|
|
869
|
-
}
|
|
870
917
|
} else {
|
|
871
|
-
//
|
|
872
|
-
//
|
|
873
|
-
//
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
"sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
|
|
878
|
-
{
|
|
879
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
880
|
-
backend,
|
|
881
|
-
},
|
|
882
|
-
);
|
|
883
|
-
return false;
|
|
884
|
-
}
|
|
918
|
+
// A resumable cloud box that produces no verified archive cannot be safely
|
|
919
|
+
// deleted: it may contain work newer than the lease's last durable revision.
|
|
920
|
+
// Keep the lease draining and retry instead of knowingly discarding bytes.
|
|
921
|
+
throw new Error(
|
|
922
|
+
`sandbox backend ${backend} produced no verified workspace archive during drain`,
|
|
923
|
+
);
|
|
885
924
|
}
|
|
886
925
|
|
|
887
926
|
// Provider terminate. Prefer the client.delete(state) teardown (the canonical
|
|
@@ -890,25 +929,33 @@ export async function terminateProviderBox(
|
|
|
890
929
|
try {
|
|
891
930
|
if (client.delete && sessionState !== undefined) {
|
|
892
931
|
await client.delete(sessionState);
|
|
893
|
-
return true;
|
|
932
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
894
933
|
}
|
|
895
934
|
if (session?.kill) {
|
|
896
935
|
await session.kill();
|
|
897
|
-
return true;
|
|
936
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
898
937
|
}
|
|
899
938
|
if (session?.terminate) {
|
|
900
939
|
await session.terminate();
|
|
901
|
-
return true;
|
|
940
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
902
941
|
}
|
|
903
|
-
if (session?.close
|
|
904
|
-
|
|
942
|
+
if (session?.close) {
|
|
943
|
+
if (!session.closed) {
|
|
944
|
+
await session.close();
|
|
945
|
+
}
|
|
946
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
905
947
|
}
|
|
906
948
|
} catch (error) {
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
949
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
950
|
+
observability.info("sandbox reaper: provider already gone during terminate", {
|
|
951
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
952
|
+
backend,
|
|
953
|
+
});
|
|
954
|
+
// Capture was already durably folded above, so this is not a
|
|
955
|
+
// missing-before-capture outcome.
|
|
956
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
957
|
+
}
|
|
958
|
+
throw error;
|
|
912
959
|
}
|
|
913
|
-
|
|
960
|
+
throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
|
|
914
961
|
}
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
requireSession,
|
|
16
16
|
setTemporalWorkflowId,
|
|
17
17
|
settleScheduledTaskRunInTransaction,
|
|
18
|
+
SessionSpawnDeniedDbError,
|
|
18
19
|
sumUsageQuantity,
|
|
19
20
|
updateScheduledTask,
|
|
20
21
|
upsertSessionGoal,
|
|
@@ -158,34 +159,48 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
|
|
|
158
159
|
frozenRigId = rig.id;
|
|
159
160
|
frozenRigVersionId = rig.activeVersion.id;
|
|
160
161
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
162
|
+
let session: Awaited<ReturnType<typeof createSession>>;
|
|
163
|
+
try {
|
|
164
|
+
session = await createSession(db, {
|
|
165
|
+
accountId: task.accountId,
|
|
166
|
+
workspaceId: task.workspaceId,
|
|
167
|
+
initialMessage: task.agentConfig.prompt,
|
|
168
|
+
resources: task.agentConfig.resources,
|
|
169
|
+
tools: taskTools,
|
|
170
|
+
metadata: {
|
|
171
|
+
...task.agentConfig.metadata,
|
|
172
|
+
model,
|
|
173
|
+
reasoningEffort,
|
|
174
|
+
scheduledTaskId: task.id,
|
|
175
|
+
scheduledTaskRunId: run.id,
|
|
176
|
+
},
|
|
177
|
+
createdBy: {
|
|
178
|
+
kind: "service",
|
|
179
|
+
subjectId: "scheduler",
|
|
180
|
+
label: "OpenGeni scheduler",
|
|
181
|
+
},
|
|
182
|
+
createdByContext: {
|
|
183
|
+
scheduledTaskId: task.id,
|
|
184
|
+
scheduledTaskRunId: run.id,
|
|
185
|
+
},
|
|
169
186
|
model,
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
rigVersionId: frozenRigVersionId,
|
|
188
|
-
});
|
|
187
|
+
sandboxBackend,
|
|
188
|
+
variableSetId: task.variableSetId ?? null,
|
|
189
|
+
rigId: frozenRigId,
|
|
190
|
+
rigVersionId: frozenRigVersionId,
|
|
191
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
192
|
+
// The durable agent config was privilege-checked when the task
|
|
193
|
+
// was created/updated. Preserve that explicit policy if a broader
|
|
194
|
+
// workspace/deployment limit is narrowed before a later fire.
|
|
195
|
+
allowNestedAgentDepthIncrease: true,
|
|
196
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
197
|
+
});
|
|
198
|
+
} catch (error) {
|
|
199
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
200
|
+
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
|
|
201
|
+
}
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
189
204
|
const goal = goalSpec
|
|
190
205
|
? await createSessionGoal(db, {
|
|
191
206
|
accountId: task.accountId,
|