@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +8 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +11 -6
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
- package/dist/activities/agent-turn/turn-context.d.ts +13 -2
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +12 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +19 -1
- package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +4 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +35 -8
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +96 -10
- package/dist/workflows/activities.d.ts +6 -1
- package/dist/workflows/automations.d.ts +5 -0
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +59 -6
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +32 -17
- package/src/activities/agent-turn/run.ts +98 -9
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +281 -136
- package/src/activities/agent-turn/tool-environment.ts +132 -39
- package/src/activities/agent-turn/tool-policy.ts +71 -12
- package/src/activities/agent-turn/turn-context.ts +21 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +116 -39
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/session-state.ts +17 -0
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +22 -8
- package/src/activities.ts +2 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -83
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +158 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +16 -0
- package/src/workflows/automations.ts +21 -2
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -141
package/src/sandbox-resume.ts
CHANGED
|
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
|
|
|
709
709
|
* the user's machine IS the persistence), or a snapshot newer than the
|
|
710
710
|
* interval already rides the lease.
|
|
711
711
|
*/
|
|
712
|
-
export
|
|
712
|
+
export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
|
|
713
|
+
/** Settles only after any provider capture continuation has published or
|
|
714
|
+
* failed and released its exact workspace-capture admission. */
|
|
715
|
+
readonly settled: Promise<void>;
|
|
716
|
+
};
|
|
717
|
+
|
|
718
|
+
export function maybePersistWarmWorkspaceSnapshot(
|
|
713
719
|
services: SandboxResumeServices,
|
|
714
720
|
ids: {
|
|
715
721
|
accountId: string;
|
|
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
723
729
|
leaseEpoch: number,
|
|
724
730
|
signal?: AbortSignal,
|
|
725
731
|
force = false,
|
|
732
|
+
): WarmWorkspaceSnapshotPromise {
|
|
733
|
+
let continuationRegistered = false;
|
|
734
|
+
let resolvePhysicalSettlement: () => void = () => undefined;
|
|
735
|
+
const physicalSettlement = new Promise<void>((resolve) => {
|
|
736
|
+
resolvePhysicalSettlement = resolve;
|
|
737
|
+
});
|
|
738
|
+
const result = persistWarmWorkspaceSnapshot(
|
|
739
|
+
services,
|
|
740
|
+
ids,
|
|
741
|
+
session,
|
|
742
|
+
leaseEpoch,
|
|
743
|
+
signal,
|
|
744
|
+
force,
|
|
745
|
+
(continuation) => {
|
|
746
|
+
continuationRegistered = true;
|
|
747
|
+
void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
|
|
748
|
+
},
|
|
749
|
+
);
|
|
750
|
+
void result.then(
|
|
751
|
+
() => {
|
|
752
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
753
|
+
},
|
|
754
|
+
() => {
|
|
755
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
756
|
+
},
|
|
757
|
+
);
|
|
758
|
+
return Object.assign(result, { settled: physicalSettlement });
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function persistWarmWorkspaceSnapshot(
|
|
762
|
+
services: SandboxResumeServices,
|
|
763
|
+
ids: {
|
|
764
|
+
accountId: string;
|
|
765
|
+
workspaceId: string;
|
|
766
|
+
sessionId: string;
|
|
767
|
+
turnId: string;
|
|
768
|
+
attemptId: string;
|
|
769
|
+
sandboxGroupId: string;
|
|
770
|
+
},
|
|
771
|
+
session: unknown,
|
|
772
|
+
leaseEpoch: number,
|
|
773
|
+
signal: AbortSignal | undefined,
|
|
774
|
+
force: boolean,
|
|
775
|
+
registerOwnedContinuation: (continuation: Promise<void>) => void,
|
|
726
776
|
): Promise<boolean> {
|
|
727
777
|
const { db, settings } = services;
|
|
728
778
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
989
1039
|
return { kind: "settled" as const, persisted: false };
|
|
990
1040
|
},
|
|
991
1041
|
);
|
|
1042
|
+
registerOwnedContinuation(settled.then(() => undefined));
|
|
992
1043
|
let cancelListener: (() => void) | undefined;
|
|
993
1044
|
const outcome = await Promise.race([
|
|
994
1045
|
settled,
|
|
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
|
|
|
1223
1274
|
// holder is gone / its attempt is no longer the active writer" from "the lease
|
|
1224
1275
|
// could not be extended" (rotation requested, draining, another epoch). The
|
|
1225
1276
|
// second is NOT a reason to release: once a provider-deadline rotation is
|
|
1226
|
-
// requested the
|
|
1227
|
-
//
|
|
1228
|
-
//
|
|
1229
|
-
//
|
|
1230
|
-
//
|
|
1231
|
-
//
|
|
1232
|
-
// "holder alive -> never release"; this loop keeps touching the holder.
|
|
1277
|
+
// requested the turn-side heartbeat immediately preempts the attempt, whose
|
|
1278
|
+
// finalizer retains this holder until every attempt-owned writer is physically
|
|
1279
|
+
// drained. Dropping it here would let the reaper capture or terminate while a
|
|
1280
|
+
// provider operation may still be mutating /workspace. Both loops therefore
|
|
1281
|
+
// converge on "holder alive -> never release"; finalization performs the
|
|
1282
|
+
// proof-bearing release.
|
|
1233
1283
|
const releaseDeadHolder = (reason: string): void => {
|
|
1234
1284
|
const message = "sandbox resume holder released: holder no longer live";
|
|
1235
1285
|
const fields = {
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -17,11 +17,13 @@
|
|
|
17
17
|
import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
adoptConnectedMachineSessionBackgroundCommand,
|
|
20
|
+
adoptManagedSessionBackgroundCommand,
|
|
20
21
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
21
22
|
advanceWorkspaceGeneration,
|
|
22
23
|
getRetainedProcess,
|
|
23
24
|
retainWorkspaceMutationProcess,
|
|
24
25
|
retainedProcessSettlementIdentity,
|
|
26
|
+
settleConnectedMachineSessionBackgroundCommand,
|
|
25
27
|
settleRetainedProcess,
|
|
26
28
|
verifyRetainedProcessMutationSettlement,
|
|
27
29
|
verifyWorkspaceMutationSettlement,
|
|
@@ -36,10 +38,6 @@ import {
|
|
|
36
38
|
type EnrollmentRecord,
|
|
37
39
|
type SandboxWorkspaceMutationAdmission,
|
|
38
40
|
} from "@opengeni/db";
|
|
39
|
-
import {
|
|
40
|
-
settleConnectedMachineSessionBackgroundCommand,
|
|
41
|
-
settleSessionBackgroundCommandForRetainedProcess,
|
|
42
|
-
} from "@opengeni/db/session-background-commands";
|
|
43
41
|
import type { EventBus } from "@opengeni/events";
|
|
44
42
|
import {
|
|
45
43
|
buildSelfhostedBackendSession,
|
|
@@ -515,10 +513,6 @@ function afterPersistableHomeMutation(
|
|
|
515
513
|
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
516
514
|
operation: op,
|
|
517
515
|
providerBinding: boundAdmission.providerBinding ?? null,
|
|
518
|
-
backgroundCommand: {
|
|
519
|
-
commandId: retainedProcess.id,
|
|
520
|
-
command: op,
|
|
521
|
-
},
|
|
522
516
|
owner: {
|
|
523
517
|
kind: "turn",
|
|
524
518
|
turnId: fence.turnId,
|
|
@@ -686,14 +680,50 @@ function settleRetainedProcessForTurn(
|
|
|
686
680
|
if (settlement.process.state === "active") {
|
|
687
681
|
throw new Error("Retained-process settlement returned an active durable process");
|
|
688
682
|
}
|
|
689
|
-
|
|
683
|
+
if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
|
|
684
|
+
await services.bus
|
|
685
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents)
|
|
686
|
+
.catch(() => undefined);
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
function adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
692
|
+
services: RoutingWiringServices,
|
|
693
|
+
ids: RoutingWiringIds,
|
|
694
|
+
):
|
|
695
|
+
| ((input: { backend: ResolvedActiveBackend; process: RoutingRetainedProcess }) => Promise<void>)
|
|
696
|
+
| undefined {
|
|
697
|
+
const fence = ids.workspaceMutationFence;
|
|
698
|
+
if (!fence) return undefined;
|
|
699
|
+
return async ({ backend, process }) => {
|
|
700
|
+
const durable = await getRetainedProcess(services.db, {
|
|
701
|
+
workspaceId: ids.workspaceId,
|
|
702
|
+
sessionId: ids.sessionId,
|
|
703
|
+
processId: process.id,
|
|
704
|
+
});
|
|
705
|
+
if (
|
|
706
|
+
!durable ||
|
|
707
|
+
durable.providerSessionId !== process.providerSessionId ||
|
|
708
|
+
durable.providerBackend !== backend.kind ||
|
|
709
|
+
durable.providerInstanceId !== backend.providerInstanceId ||
|
|
710
|
+
durable.leaseEpoch !== backend.leaseEpoch ||
|
|
711
|
+
durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
|
|
712
|
+
durable.routeTargetId !== backend.sandboxId ||
|
|
713
|
+
durable.routeEpoch !== backend.activeEpoch
|
|
714
|
+
) {
|
|
715
|
+
throw new Error("Background command adoption lost its exact retained-process identity");
|
|
716
|
+
}
|
|
717
|
+
await adoptManagedSessionBackgroundCommand(services.db, {
|
|
690
718
|
accountId: fence.accountId,
|
|
691
719
|
workspaceId: ids.workspaceId,
|
|
692
720
|
sessionId: ids.sessionId,
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
721
|
+
turnId: fence.turnId,
|
|
722
|
+
executionGeneration: fence.executionGeneration,
|
|
723
|
+
attemptId: fence.attemptId,
|
|
724
|
+
processId: process.id,
|
|
725
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
726
|
+
command: "execCommand",
|
|
697
727
|
});
|
|
698
728
|
};
|
|
699
729
|
}
|
|
@@ -736,6 +766,10 @@ export function wrapTurnBoxWithRouting(
|
|
|
736
766
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
737
767
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
738
768
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
769
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
770
|
+
services,
|
|
771
|
+
ids,
|
|
772
|
+
);
|
|
739
773
|
const resolver = makeActiveBackendResolver({
|
|
740
774
|
workspaceId: ids.workspaceId,
|
|
741
775
|
defaultBackend: established.session as RoutableBackendSession,
|
|
@@ -794,7 +828,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
794
828
|
return { commandId: adopted.id };
|
|
795
829
|
},
|
|
796
830
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
797
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
831
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
798
832
|
accountId: backgroundCommandAccountId,
|
|
799
833
|
workspaceId: ids.workspaceId,
|
|
800
834
|
sessionId: ids.sessionId,
|
|
@@ -807,6 +841,11 @@ export function wrapTurnBoxWithRouting(
|
|
|
807
841
|
exitCode: command.exitCode,
|
|
808
842
|
reason: command.reason,
|
|
809
843
|
});
|
|
844
|
+
if (settlement && bus) {
|
|
845
|
+
await bus
|
|
846
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.events)
|
|
847
|
+
.catch(() => undefined);
|
|
848
|
+
}
|
|
810
849
|
},
|
|
811
850
|
}
|
|
812
851
|
: {}),
|
|
@@ -892,6 +931,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
892
931
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
893
932
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
894
933
|
...(settleProcess ? { settleProcess } : {}),
|
|
934
|
+
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
895
935
|
...(ids.homeLease
|
|
896
936
|
? {
|
|
897
937
|
onDefaultBackendError: async ({
|
|
@@ -970,6 +1010,10 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
970
1010
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
971
1011
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
972
1012
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
1013
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
1014
|
+
services,
|
|
1015
|
+
ids,
|
|
1016
|
+
);
|
|
973
1017
|
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
974
1018
|
const defaultSupportsPty = defaultBackend
|
|
975
1019
|
? selectBackend(defaultBackend).capabilities.Terminal.pty
|
|
@@ -1038,7 +1082,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1038
1082
|
return { commandId: adopted.id };
|
|
1039
1083
|
},
|
|
1040
1084
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
1041
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1085
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1042
1086
|
accountId: backgroundCommandAccountId,
|
|
1043
1087
|
workspaceId: ids.workspaceId,
|
|
1044
1088
|
sessionId: ids.sessionId,
|
|
@@ -1051,6 +1095,11 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1051
1095
|
exitCode: command.exitCode,
|
|
1052
1096
|
reason: command.reason,
|
|
1053
1097
|
});
|
|
1098
|
+
if (settlement && bus) {
|
|
1099
|
+
await bus
|
|
1100
|
+
.publish(ids.workspaceId, ids.sessionId, settlement.events)
|
|
1101
|
+
.catch(() => undefined);
|
|
1102
|
+
}
|
|
1054
1103
|
},
|
|
1055
1104
|
}
|
|
1056
1105
|
: {}),
|
|
@@ -1105,6 +1154,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1105
1154
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
1106
1155
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
1107
1156
|
...(settleProcess ? { settleProcess } : {}),
|
|
1157
|
+
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
1108
1158
|
...(args.homeLeaseIdentity
|
|
1109
1159
|
? {
|
|
1110
1160
|
onDefaultBackendError: async ({
|
|
@@ -1383,7 +1433,7 @@ export async function establishSelfhostedTurnSession(
|
|
|
1383
1433
|
}
|
|
1384
1434
|
: {}),
|
|
1385
1435
|
settleBackgroundCommand: async (command) => {
|
|
1386
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1436
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1387
1437
|
accountId: args.accountId,
|
|
1388
1438
|
workspaceId: args.workspaceId,
|
|
1389
1439
|
sessionId: args.sessionId,
|
|
@@ -1396,6 +1446,11 @@ export async function establishSelfhostedTurnSession(
|
|
|
1396
1446
|
exitCode: command.exitCode,
|
|
1397
1447
|
reason: command.reason,
|
|
1398
1448
|
});
|
|
1449
|
+
if (settlement && bus) {
|
|
1450
|
+
await bus
|
|
1451
|
+
.publish(args.workspaceId, args.sessionId, settlement.events)
|
|
1452
|
+
.catch(() => undefined);
|
|
1453
|
+
}
|
|
1399
1454
|
},
|
|
1400
1455
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
1401
1456
|
...(onOp !== undefined ? { onOp } : {}),
|
|
@@ -20,6 +20,7 @@ type WorkflowControlActivities = Pick<
|
|
|
20
20
|
| "recoverDispatch"
|
|
21
21
|
| "recoverEscapedMcpTimeout"
|
|
22
22
|
| "settleSessionInterruptions"
|
|
23
|
+
| "settleSessionInputWait"
|
|
23
24
|
>;
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -68,6 +69,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
|
|
|
68
69
|
},
|
|
69
70
|
);
|
|
70
71
|
|
|
72
|
+
/** Dispatch has a bounded retry window, but an accepted automation event must
|
|
73
|
+
* not remain durably `dispatching` after that window closes. This compensating
|
|
74
|
+
* database write is idempotent and retries without a cap until the run reaches
|
|
75
|
+
* a terminal state (or is observed as already dispatched). */
|
|
76
|
+
export const automationFailureActivity = proxyActivities<
|
|
77
|
+
Pick<typeof activities, "settleAutomationRunFailure">
|
|
78
|
+
>({
|
|
79
|
+
startToCloseTimeout: "2 minutes",
|
|
80
|
+
retry: {
|
|
81
|
+
initialInterval: "1 second",
|
|
82
|
+
backoffCoefficient: 2,
|
|
83
|
+
maximumInterval: "30 seconds",
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
|
|
71
87
|
/** Goal continuation evaluates a durable Postgres obligation at an idle
|
|
72
88
|
* boundary. A transient failure gets a short retry window, then records an
|
|
73
89
|
* explicit delayed outbox wake instead of relying on an unrelated mutation or
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { automationActivity } from "./activities";
|
|
1
|
+
import { automationActivity, automationFailureActivity } from "./activities";
|
|
2
2
|
|
|
3
3
|
export type AutomationRunWorkflowInput = {
|
|
4
4
|
accountId: string;
|
|
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
|
|
|
6
6
|
runId: string;
|
|
7
7
|
};
|
|
8
8
|
|
|
9
|
+
export type AutomationRunWorkflowActivities = {
|
|
10
|
+
dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
|
|
11
|
+
settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export async function runAutomationRunWorkflow(
|
|
15
|
+
input: AutomationRunWorkflowInput,
|
|
16
|
+
activities: AutomationRunWorkflowActivities,
|
|
17
|
+
): Promise<void> {
|
|
18
|
+
try {
|
|
19
|
+
await activities.dispatchAutomationRun(input);
|
|
20
|
+
} catch {
|
|
21
|
+
await activities.settleAutomationRunFailure(input);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
9
25
|
export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
|
|
10
|
-
await
|
|
26
|
+
await runAutomationRunWorkflow(input, {
|
|
27
|
+
dispatchAutomationRun: automationActivity.dispatchAutomationRun,
|
|
28
|
+
settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
|
|
29
|
+
});
|
|
11
30
|
}
|
package/src/workflows/session.ts
CHANGED
|
@@ -95,6 +95,14 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
|
|
|
95
95
|
return currentWakeups !== entryWakeups;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
/** A typed activity cancellation is recoverable unless its exact-attempt
|
|
99
|
+
* redispatch budget was atomically exhausted. */
|
|
100
|
+
export function cancelledAttemptRecoveryMayContinue(
|
|
101
|
+
action: activities.RecoverDispatchResult["action"],
|
|
102
|
+
): boolean {
|
|
103
|
+
return action !== "exceeded";
|
|
104
|
+
}
|
|
105
|
+
|
|
98
106
|
/**
|
|
99
107
|
* Bound repeated failures that happen before an attempt row exists. The
|
|
100
108
|
* activity mirrors this deterministic delay into the durable wake outbox, so
|
|
@@ -342,6 +350,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
342
350
|
const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
|
|
343
351
|
const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
|
|
344
352
|
const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
|
|
353
|
+
// PR #2208 changed a typed-cancelled result from a plain re-peek into a
|
|
354
|
+
// recoverDispatch activity. Version that new command so histories which
|
|
355
|
+
// already recorded the legacy re-peek remain deterministic on replay.
|
|
356
|
+
const cancelledAttemptRecovery = patched("session-cancelled-attempt-recovery-v1");
|
|
345
357
|
const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
|
|
346
358
|
let approvalWakeups = 0;
|
|
347
359
|
let interruptionWakeups = 0;
|
|
@@ -644,6 +656,48 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
644
656
|
}
|
|
645
657
|
continue;
|
|
646
658
|
}
|
|
659
|
+
if (peek.kind === "input-wait") {
|
|
660
|
+
const settlement = await activity.settleSessionInputWait({
|
|
661
|
+
accountId: input.accountId,
|
|
662
|
+
workspaceId: input.workspaceId,
|
|
663
|
+
sessionId: input.sessionId,
|
|
664
|
+
waitTurnId: peek.waitTurnId,
|
|
665
|
+
disposition: peek.disposition,
|
|
666
|
+
});
|
|
667
|
+
if (settlement.action !== "held") continue;
|
|
668
|
+
|
|
669
|
+
// The durable outbox owns the long deadline. Keep this workflow run open
|
|
670
|
+
// only for the same bounded close-race window used by ordinary idle; any
|
|
671
|
+
// signal is a hint to re-peek PostgreSQL truth.
|
|
672
|
+
const seenWakeups = wakeups;
|
|
673
|
+
const seenApprovalWakeups = approvalWakeups;
|
|
674
|
+
const seenInterruptionWakeups = interruptionWakeups;
|
|
675
|
+
const woke = await condition(
|
|
676
|
+
() =>
|
|
677
|
+
interruptionWakeups !== seenInterruptionWakeups ||
|
|
678
|
+
wakeups !== seenWakeups ||
|
|
679
|
+
approvalWakeups !== seenApprovalWakeups,
|
|
680
|
+
"5s",
|
|
681
|
+
);
|
|
682
|
+
if (woke) continue;
|
|
683
|
+
const finalPeek = await activity.peekSessionWork({
|
|
684
|
+
workspaceId: input.workspaceId,
|
|
685
|
+
sessionId: input.sessionId,
|
|
686
|
+
});
|
|
687
|
+
if (
|
|
688
|
+
finalPeek.kind !== "input-wait" ||
|
|
689
|
+
finalPeek.disposition !== "held" ||
|
|
690
|
+
finalPeek.waitTurnId !== peek.waitTurnId
|
|
691
|
+
) {
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
await activity.markSessionIdle({
|
|
695
|
+
workspaceId: input.workspaceId,
|
|
696
|
+
sessionId: input.sessionId,
|
|
697
|
+
});
|
|
698
|
+
if (signalVersion !== closeSignalVersion) continue;
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
647
701
|
if (peek.kind === "idle") {
|
|
648
702
|
let continuation: activities.MaybeContinueGoalResult = { action: "none" };
|
|
649
703
|
try {
|
|
@@ -663,11 +717,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
663
717
|
});
|
|
664
718
|
}
|
|
665
719
|
if (continuation.action === "continue" || continuation.action === "queue") continue;
|
|
666
|
-
// `none`, `paused`,
|
|
667
|
-
//
|
|
668
|
-
//
|
|
669
|
-
//
|
|
670
|
-
// restarts the workflow. No Temporal timer is used for pacing.
|
|
720
|
+
// `none`, `paused`, and `deferred` all close this run below. A deferred
|
|
721
|
+
// idle-backoff goal keeps its durable obligation armed; the delayed
|
|
722
|
+
// wake-outbox row at the pacing deadline or any producer signal restarts
|
|
723
|
+
// the workflow. No Temporal timer is used for pacing.
|
|
671
724
|
const seenWakeups = wakeups;
|
|
672
725
|
const seenApprovalWakeups = approvalWakeups;
|
|
673
726
|
const seenInterruptionWakeups = interruptionWakeups;
|
|
@@ -1052,8 +1105,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
|
|
|
1052
1105
|
return true;
|
|
1053
1106
|
}
|
|
1054
1107
|
|
|
1055
|
-
if (outcome.result.status === "failed"
|
|
1056
|
-
return
|
|
1108
|
+
if (outcome.result.status === "failed") {
|
|
1109
|
+
return false;
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
if (outcome.result.status === "cancelled") {
|
|
1113
|
+
// Histories created before session-cancelled-attempt-recovery-v1 must
|
|
1114
|
+
// preserve the old command sequence. Their durable state is still safe:
|
|
1115
|
+
// operator recovery or a later fresh workflow run can close a stranded
|
|
1116
|
+
// owner, while new histories use the bounded exact-attempt transaction.
|
|
1117
|
+
if (!cancelledAttemptRecovery) return true;
|
|
1118
|
+
// A typed cancellation is normally observed through the control-signal
|
|
1119
|
+
// branch above, after Pause/Steer has durably fenced the attempt. A
|
|
1120
|
+
// worker can nevertheless return the same shape after a shutdown or a
|
|
1121
|
+
// stale settlement race without closing its attempt row. Blindly
|
|
1122
|
+
// re-peeking then retries forever against `turn.status = running` and
|
|
1123
|
+
// strands every later prompt. Reconcile the exact attempt through the
|
|
1124
|
+
// same bounded, generation-fenced redispatch transaction used after an
|
|
1125
|
+
// activity heartbeat loss. If the attempt actually settled meanwhile,
|
|
1126
|
+
// the transaction is a stale no-op and the next peek observes truth.
|
|
1127
|
+
const recovery = await activity.recoverDispatch({
|
|
1128
|
+
accountId,
|
|
1129
|
+
workspaceId,
|
|
1130
|
+
sessionId,
|
|
1131
|
+
attemptId: outcome.result.attemptId,
|
|
1132
|
+
timeoutType: "HEARTBEAT",
|
|
1133
|
+
});
|
|
1134
|
+
return cancelledAttemptRecoveryMayContinue(recovery.action);
|
|
1057
1135
|
}
|
|
1058
1136
|
|
|
1059
1137
|
if (outcome.result.capacityWait) {
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { type IntegrationInvocationAuthority } from "@opengeni/capabilities";
|
|
2
|
-
import type { Settings } from "@opengeni/config";
|
|
3
|
-
import type { ToolAuthNeededPayload } from "@opengeni/contracts";
|
|
4
|
-
import type { ApiIntegrationRuntime, ResolveConnectionCredentialInput, ResolveConnectionCredentialResult } from "@opengeni/db";
|
|
5
|
-
import type { LocalMcpServerRegistration } from "@opengeni/runtime";
|
|
6
|
-
import type { FetchLike } from "@opengeni/network";
|
|
7
|
-
export type BuildApiIntegrationServersInput = {
|
|
8
|
-
settings: Settings;
|
|
9
|
-
integrations: readonly ApiIntegrationRuntime[];
|
|
10
|
-
authority: Omit<IntegrationInvocationAuthority, "connectionRef">;
|
|
11
|
-
resolveCredential: (input: ResolveConnectionCredentialInput) => Promise<ResolveConnectionCredentialResult>;
|
|
12
|
-
onAuthNeeded?: (payload: ToolAuthNeededPayload) => Promise<void> | void;
|
|
13
|
-
fetchImpl?: FetchLike;
|
|
14
|
-
};
|
|
15
|
-
/**
|
|
16
|
-
* Compile active persisted API facets into ordinary in-process MCP servers for
|
|
17
|
-
* one exact attempt. Connection resolution remains request-time, destination-
|
|
18
|
-
* bound, personal-delegation-fenced, and refreshable through the same worker
|
|
19
|
-
* resolver used by remote MCP servers.
|
|
20
|
-
*/
|
|
21
|
-
export declare function buildApiIntegrationServersForTurn(input: BuildApiIntegrationServersInput): LocalMcpServerRegistration[];
|