@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
package/src/sandbox-routing.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// target lookup), and the selfhosted ControlRpc built over the events bus's NATS
|
|
15
15
|
// request/reply connection.
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
verifyRetainedProcessMutationSettlement,
|
|
26
26
|
verifyWorkspaceMutationSettlement,
|
|
27
27
|
getSandbox,
|
|
28
|
+
getEnrollment,
|
|
28
29
|
markWarmLeaseInstanceLost,
|
|
29
30
|
readLease,
|
|
30
31
|
readActiveSandbox,
|
|
@@ -44,7 +45,9 @@ import {
|
|
|
44
45
|
RoutingBackendRecoveryRequiredError,
|
|
45
46
|
RoutingSandboxSession,
|
|
46
47
|
resolveModalCheckpointProviderBindingForSession,
|
|
48
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
47
49
|
sandboxBackendForSdkBackendId,
|
|
50
|
+
selectBackend,
|
|
48
51
|
verifySandboxExecReadiness,
|
|
49
52
|
type ControlRpc,
|
|
50
53
|
type EstablishedSandboxSession,
|
|
@@ -99,6 +102,15 @@ export type RoutingWiringServices = {
|
|
|
99
102
|
established: EstablishedSandboxSession;
|
|
100
103
|
leaseEpoch: number;
|
|
101
104
|
}) => void;
|
|
105
|
+
/** Cancel bounded capture waits with the owning turn activity. */
|
|
106
|
+
waitSignal?: AbortSignal;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
/** Worker-turn routing may final-ack streamed command output, so it must always
|
|
110
|
+
* carry the turn's durable frontier journal. Non-turn helpers may still omit it
|
|
111
|
+
* through the broader service shape above. */
|
|
112
|
+
type TurnRoutingWiringServices = RoutingWiringServices & {
|
|
113
|
+
opJournal: OpStreamJournal;
|
|
102
114
|
};
|
|
103
115
|
|
|
104
116
|
export type RoutingWiringIds = {
|
|
@@ -172,8 +184,8 @@ export function relayConfigFromSettings(settings: Settings): SelfhostedRelayConf
|
|
|
172
184
|
|
|
173
185
|
/** The selfhosted CONTROL vs EXEC op deadlines for a turn, from settings. Control
|
|
174
186
|
* ops (ping/fs/desktop/pty) stay on the short timeout so machine liveness is never
|
|
175
|
-
* masked by a slow op; exec
|
|
176
|
-
*
|
|
187
|
+
* masked by a slow op; exec has its own setting and defaults to no duration wall.
|
|
188
|
+
* Threaded into every turn-path session build + resolver. */
|
|
177
189
|
export function selfhostedTimeoutsFromSettings(settings: Settings): {
|
|
178
190
|
timeoutMs: number;
|
|
179
191
|
execTimeoutMs: number;
|
|
@@ -254,27 +266,7 @@ function homeRouteRecoveryError(
|
|
|
254
266
|
export function providerIdentityFromResumeState(
|
|
255
267
|
resumeState: Record<string, unknown>,
|
|
256
268
|
): string | null {
|
|
257
|
-
|
|
258
|
-
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
259
|
-
? (resumeState.sessionState as Record<string, unknown>)
|
|
260
|
-
: null;
|
|
261
|
-
const providerState =
|
|
262
|
-
sessionState?.providerState && typeof sessionState.providerState === "object"
|
|
263
|
-
? (sessionState.providerState as Record<string, unknown>)
|
|
264
|
-
: null;
|
|
265
|
-
for (const field of [
|
|
266
|
-
"sandboxId",
|
|
267
|
-
"instanceId",
|
|
268
|
-
"id",
|
|
269
|
-
"hostId",
|
|
270
|
-
"containerId",
|
|
271
|
-
"workspaceRootPath",
|
|
272
|
-
"agentId",
|
|
273
|
-
]) {
|
|
274
|
-
const value = providerState?.[field];
|
|
275
|
-
if (typeof value === "string" && value.length > 0) return value;
|
|
276
|
-
}
|
|
277
|
-
return null;
|
|
269
|
+
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
278
270
|
}
|
|
279
271
|
|
|
280
272
|
/**
|
|
@@ -440,7 +432,8 @@ function beforePersistableHomeMutation(
|
|
|
440
432
|
expectedEpoch: backend.leaseEpoch,
|
|
441
433
|
expectedInstanceId: backend.providerInstanceId,
|
|
442
434
|
operation: op,
|
|
443
|
-
captureWaitMs:
|
|
435
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
436
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
444
437
|
});
|
|
445
438
|
return { admission, providerBinding };
|
|
446
439
|
};
|
|
@@ -575,7 +568,8 @@ function beforeRetainedProcessMutation(
|
|
|
575
568
|
sessionId: ids.sessionId,
|
|
576
569
|
processId: process.id,
|
|
577
570
|
operation: op,
|
|
578
|
-
captureWaitMs:
|
|
571
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
572
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
579
573
|
});
|
|
580
574
|
}
|
|
581
575
|
|
|
@@ -686,7 +680,7 @@ function controlRpcFactory(bus: EventBus | undefined): () => ControlRpc {
|
|
|
686
680
|
* swap's active_epoch.
|
|
687
681
|
*/
|
|
688
682
|
export function wrapTurnBoxWithRouting(
|
|
689
|
-
services:
|
|
683
|
+
services: TurnRoutingWiringServices,
|
|
690
684
|
ids: RoutingWiringIds,
|
|
691
685
|
established: EstablishedSandboxSession,
|
|
692
686
|
): EstablishedSandboxSession {
|
|
@@ -712,6 +706,11 @@ export function wrapTurnBoxWithRouting(
|
|
|
712
706
|
: null;
|
|
713
707
|
},
|
|
714
708
|
controlRpcFactory: controlRpcFactory(bus),
|
|
709
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
710
|
+
if (!sandbox.enrollmentId) return undefined;
|
|
711
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
712
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
713
|
+
},
|
|
715
714
|
relay: relayConfigFromSettings(settings),
|
|
716
715
|
// A selfhosted swap target runs real commands too, so give it the same split
|
|
717
716
|
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
@@ -846,7 +845,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
846
845
|
}
|
|
847
846
|
|
|
848
847
|
export function wrapLazyTurnBoxWithRouting(
|
|
849
|
-
services:
|
|
848
|
+
services: TurnRoutingWiringServices,
|
|
850
849
|
ids: RoutingWiringIds,
|
|
851
850
|
args: {
|
|
852
851
|
client: EstablishedSandboxSession["client"];
|
|
@@ -871,8 +870,17 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
871
870
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
872
871
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
873
872
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
873
|
+
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
874
|
+
const defaultSupportsPty = defaultBackend
|
|
875
|
+
? selectBackend(defaultBackend).capabilities.Terminal.pty
|
|
876
|
+
: false;
|
|
874
877
|
const syntheticSession: RoutableBackendSession = {
|
|
875
878
|
state: { manifest: args.agentDefaultManifest },
|
|
879
|
+
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
880
|
+
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
881
|
+
// write_stdin on its first turn operation; the async cancellation transport
|
|
882
|
+
// below still resolves the exact live route before executing a command.
|
|
883
|
+
supportsPty: () => defaultSupportsPty,
|
|
876
884
|
};
|
|
877
885
|
const routedResolver = makeActiveBackendResolver({
|
|
878
886
|
workspaceId: ids.workspaceId,
|
|
@@ -890,6 +898,11 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
890
898
|
: null;
|
|
891
899
|
},
|
|
892
900
|
controlRpcFactory: controlRpcFactory(bus),
|
|
901
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
902
|
+
if (!sandbox.enrollmentId) return undefined;
|
|
903
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
904
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
905
|
+
},
|
|
893
906
|
relay: relayConfigFromSettings(settings),
|
|
894
907
|
...selfhostedResolverTimeouts(settings),
|
|
895
908
|
...(onOp !== undefined ? { selfhostedOnOp: onOp } : {}),
|
|
@@ -1013,8 +1026,6 @@ export type SelfhostedTurnSessionArgs = {
|
|
|
1013
1026
|
workingDir: string | null;
|
|
1014
1027
|
};
|
|
1015
1028
|
|
|
1016
|
-
type LegacySelfhostedTurnSessionArgs = Omit<SelfhostedTurnSessionArgs, "opStream">;
|
|
1017
|
-
|
|
1018
1029
|
/**
|
|
1019
1030
|
* Stage D machine-primary establish: bind the live SelfhostedSession for a turn
|
|
1020
1031
|
* whose ACTIVE sandbox is a connected machine — WITHOUT establishing or leasing a
|
|
@@ -1034,11 +1045,10 @@ type LegacySelfhostedTurnSessionArgs = Omit<SelfhostedTurnSessionArgs, "opStream
|
|
|
1034
1045
|
* The op-stream injection for a machine-primary turn: present iff the machine
|
|
1035
1046
|
* advertised `Capabilities.op_stream` in its latest Hello AND the server flag
|
|
1036
1047
|
* is on AND a bus exists to carry frames. The transport rides the SAME managed
|
|
1037
|
-
* NATS connection as the control rpc (the bus's op-stream accessor)
|
|
1038
|
-
* without the accessor
|
|
1039
|
-
*
|
|
1040
|
-
*
|
|
1041
|
-
* resolver, and legacy is always correct.
|
|
1048
|
+
* NATS connection as the control rpc (the bus's op-stream accessor). A bus
|
|
1049
|
+
* without the accessor yields no stream; an unbounded exec then fails before
|
|
1050
|
+
* starting instead of silently degrading to request/reply. Swap targets resolve
|
|
1051
|
+
* the same enrollment capability through the injected backend-resolver seam.
|
|
1042
1052
|
*/
|
|
1043
1053
|
function opStreamDepsFor(
|
|
1044
1054
|
services: RoutingWiringServices,
|
|
@@ -1056,11 +1066,11 @@ function opStreamDepsFor(
|
|
|
1056
1066
|
|
|
1057
1067
|
export async function establishSelfhostedTurnSession(
|
|
1058
1068
|
services: RoutingWiringServices,
|
|
1059
|
-
args: SelfhostedTurnSessionArgs
|
|
1069
|
+
args: SelfhostedTurnSessionArgs,
|
|
1060
1070
|
): Promise<EstablishedSandboxSession> {
|
|
1061
1071
|
const { settings, bus, onOp } = services;
|
|
1062
1072
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
1063
|
-
const opStream = opStreamDepsFor(services,
|
|
1073
|
+
const opStream = opStreamDepsFor(services, args.opStream);
|
|
1064
1074
|
const { client, session } = await buildSelfhostedBackendSession({
|
|
1065
1075
|
workspaceId: args.workspaceId,
|
|
1066
1076
|
agentId: args.agentId,
|
|
@@ -1076,8 +1086,8 @@ export async function establishSelfhostedTurnSession(
|
|
|
1076
1086
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
1077
1087
|
...(onOp !== undefined ? { onOp } : {}),
|
|
1078
1088
|
// The streaming exec transport — present iff the machine advertised the
|
|
1079
|
-
// capability AND the server flag is on
|
|
1080
|
-
// legacy
|
|
1089
|
+
// capability AND the server flag is on. It is required when execTimeoutMs=0;
|
|
1090
|
+
// legacy request/reply remains available only for explicitly bounded exec.
|
|
1081
1091
|
...(opStream !== undefined ? { opStream } : {}),
|
|
1082
1092
|
});
|
|
1083
1093
|
return {
|
|
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
|
|
|
20
20
|
shutdown(): void;
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
+
/** One process may poll several disjoint Temporal queues, but it still has one
|
|
24
|
+
* host lifecycle. Start and stop every poller as a unit; a failed poller drains
|
|
25
|
+
* its siblings and preserves the original failure. */
|
|
26
|
+
export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
|
|
27
|
+
if (targets.length === 0) {
|
|
28
|
+
throw new Error("a worker service requires at least one Temporal poller");
|
|
29
|
+
}
|
|
30
|
+
let runPromise: Promise<void> | undefined;
|
|
31
|
+
const shutdown = (): void => {
|
|
32
|
+
let firstError: unknown;
|
|
33
|
+
for (const target of targets) {
|
|
34
|
+
try {
|
|
35
|
+
target.shutdown();
|
|
36
|
+
} catch (error) {
|
|
37
|
+
firstError ??= error;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (firstError !== undefined) throw firstError;
|
|
41
|
+
};
|
|
42
|
+
return {
|
|
43
|
+
run: () => {
|
|
44
|
+
runPromise ??= (async () => {
|
|
45
|
+
const runs = targets.map((target) => {
|
|
46
|
+
try {
|
|
47
|
+
return target.run();
|
|
48
|
+
} catch (error) {
|
|
49
|
+
return Promise.reject(error);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
try {
|
|
53
|
+
await Promise.all(runs);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
try {
|
|
56
|
+
shutdown();
|
|
57
|
+
} catch {
|
|
58
|
+
// Preserve the poller failure. Every sibling still received its
|
|
59
|
+
// shutdown request even when one request itself failed.
|
|
60
|
+
}
|
|
61
|
+
await Promise.allSettled(runs);
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
})();
|
|
65
|
+
return runPromise;
|
|
66
|
+
},
|
|
67
|
+
shutdown,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
23
71
|
export type WorkerServiceLifecycle = {
|
|
24
72
|
state(): WorkerLifecycleState;
|
|
25
73
|
run(): Promise<void>;
|
|
26
|
-
drain(reason?: string):
|
|
74
|
+
drain(reason?: string): boolean;
|
|
27
75
|
close(): Promise<void>;
|
|
28
76
|
};
|
|
29
77
|
|
|
@@ -72,25 +120,39 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
72
120
|
})();
|
|
73
121
|
return runPromise;
|
|
74
122
|
},
|
|
75
|
-
drain: (
|
|
123
|
+
drain: (_reason = "host request") => {
|
|
76
124
|
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
77
|
-
return;
|
|
125
|
+
return true;
|
|
78
126
|
}
|
|
127
|
+
const previousState = state;
|
|
128
|
+
safeLifecycleLog(() =>
|
|
129
|
+
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
130
|
+
role: input.role,
|
|
131
|
+
errorClass: "WorkerLifecycleOperation",
|
|
132
|
+
errorCode: "worker_draining",
|
|
133
|
+
origin: "worker-lifecycle",
|
|
134
|
+
}),
|
|
135
|
+
);
|
|
79
136
|
state = "draining";
|
|
80
|
-
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
81
|
-
role: input.role,
|
|
82
|
-
reason,
|
|
83
|
-
});
|
|
84
137
|
try {
|
|
85
138
|
input.worker.shutdown();
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
139
|
+
return true;
|
|
140
|
+
} catch {
|
|
141
|
+
state = previousState;
|
|
142
|
+
safeLifecycleLog(() =>
|
|
143
|
+
input.observability.warn("worker shutdown request failed", {
|
|
144
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
145
|
+
errorCode: "worker_shutdown_request_failed",
|
|
146
|
+
origin: "worker-lifecycle",
|
|
147
|
+
}),
|
|
148
|
+
);
|
|
149
|
+
return false;
|
|
90
150
|
}
|
|
91
151
|
},
|
|
92
152
|
close: async () => {
|
|
93
|
-
lifecycle.drain("service close")
|
|
153
|
+
if (!lifecycle.drain("service close")) {
|
|
154
|
+
throw new Error("worker shutdown request failed");
|
|
155
|
+
}
|
|
94
156
|
if (runPromise) {
|
|
95
157
|
await runPromise.catch(() => undefined);
|
|
96
158
|
} else {
|
|
@@ -102,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
102
164
|
|
|
103
165
|
return lifecycle;
|
|
104
166
|
}
|
|
167
|
+
|
|
168
|
+
function safeLifecycleLog(log: () => void): void {
|
|
169
|
+
try {
|
|
170
|
+
log();
|
|
171
|
+
} catch {
|
|
172
|
+
// Telemetry cannot change worker lifecycle state or shutdown delivery.
|
|
173
|
+
}
|
|
174
|
+
}
|
|
@@ -91,11 +91,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
|
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
|
|
95
|
+
return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
|
|
96
|
+
taskQueue: turnTaskQueue(baseTaskQueue),
|
|
97
|
+
startToCloseTimeout: "20 minutes",
|
|
98
|
+
retry: {
|
|
99
|
+
initialInterval: "1 second",
|
|
100
|
+
backoffCoefficient: 2,
|
|
101
|
+
maximumInterval: "30 seconds",
|
|
102
|
+
maximumAttempts: 3,
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
94
107
|
export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
|
|
95
108
|
startToCloseTimeout: "30 minutes",
|
|
96
109
|
retry: { maximumAttempts: 1 },
|
|
97
110
|
});
|
|
98
111
|
|
|
112
|
+
/** Connector inventory/download/index batches are provider-I/O activities. The
|
|
113
|
+
* implementation heartbeats between bounded items and owns its resumable
|
|
114
|
+
* checkpoint; Temporal retries therefore repeat only idempotent source/object
|
|
115
|
+
* operations and never invoke the agent runtime. */
|
|
116
|
+
export const knowledgeSourceSyncActivity = proxyActivities<
|
|
117
|
+
Pick<typeof activities, "runKnowledgeSourceSyncBatch">
|
|
118
|
+
>({
|
|
119
|
+
startToCloseTimeout: "1 hour",
|
|
120
|
+
heartbeatTimeout: "2 minutes",
|
|
121
|
+
retry: {
|
|
122
|
+
initialInterval: "2 seconds",
|
|
123
|
+
backoffCoefficient: 2,
|
|
124
|
+
maximumInterval: "1 minute",
|
|
125
|
+
maximumAttempts: 5,
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
|
|
99
129
|
export function workflowFailureMessage(error: unknown): string {
|
|
100
130
|
if (error instanceof Error) {
|
|
101
131
|
return error.message;
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { proxyActivities } from "@temporalio/workflow";
|
|
2
2
|
import type * as activities from "../activities";
|
|
3
3
|
|
|
4
|
-
const reaperActivity = proxyActivities<
|
|
4
|
+
const reaperActivity = proxyActivities<
|
|
5
|
+
Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots"> &
|
|
6
|
+
Pick<typeof activities, "recoverVideoGenerationWorkflows">
|
|
7
|
+
>({
|
|
5
8
|
startToCloseTimeout: "5 minutes",
|
|
6
9
|
retry: { maximumAttempts: 1 },
|
|
7
10
|
});
|
|
@@ -9,4 +12,6 @@ const reaperActivity = proxyActivities<Pick<typeof activities, "reapExpiredFileU
|
|
|
9
12
|
/** One bounded provider/DB sweep; the Temporal Schedule owns the cadence. */
|
|
10
13
|
export async function fileUploadReaperWorkflow(): Promise<void> {
|
|
11
14
|
await reaperActivity.reapExpiredFileUploads();
|
|
15
|
+
await reaperActivity.maintainRetainedScreenshots();
|
|
16
|
+
await reaperActivity.recoverVideoGenerationWorkflows();
|
|
12
17
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
|
|
2
|
+
import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
|
|
3
|
+
import { knowledgeSourceSyncActivity } from "./activities";
|
|
4
|
+
|
|
5
|
+
export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
|
|
6
|
+
|
|
7
|
+
export const knowledgeSourceSyncWake =
|
|
8
|
+
defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
|
|
9
|
+
|
|
10
|
+
type KnowledgeSourceSyncWorkflowState = {
|
|
11
|
+
pending: KnowledgeSourceSyncWorkflowInput | null;
|
|
12
|
+
batchesThisRun: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
|
|
16
|
+
|
|
17
|
+
/** One stable workflow ID serializes all fires for a source. Signals retain at
|
|
18
|
+
* most one newest pending wake; Postgres lease/checkpoint rows remain the
|
|
19
|
+
* authoritative overlap and resumability fence across workflow generations. */
|
|
20
|
+
export async function knowledgeSourceSyncWorkflow(
|
|
21
|
+
input: KnowledgeSourceSyncWorkflowInput,
|
|
22
|
+
carried?: KnowledgeSourceSyncWorkflowState,
|
|
23
|
+
): Promise<void> {
|
|
24
|
+
let current = input;
|
|
25
|
+
let pending = carried?.pending ?? null;
|
|
26
|
+
let batchesThisRun = carried?.batchesThisRun ?? 0;
|
|
27
|
+
|
|
28
|
+
setHandler(knowledgeSourceSyncWake, (next) => {
|
|
29
|
+
if (
|
|
30
|
+
next.accountId !== input.accountId ||
|
|
31
|
+
next.workspaceId !== input.workspaceId ||
|
|
32
|
+
next.sourceId !== input.sourceId
|
|
33
|
+
) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
pending = next;
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
while (true) {
|
|
40
|
+
const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
|
|
41
|
+
batchesThisRun += 1;
|
|
42
|
+
|
|
43
|
+
if (result.action === "continue") {
|
|
44
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
45
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
46
|
+
pending,
|
|
47
|
+
batchesThisRun: 0,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (
|
|
54
|
+
(result.action === "complete" || result.action === "failed") &&
|
|
55
|
+
result.bufferedScheduledTaskRunId
|
|
56
|
+
) {
|
|
57
|
+
current = {
|
|
58
|
+
...current,
|
|
59
|
+
scheduledTaskRunId: result.bufferedScheduledTaskRunId,
|
|
60
|
+
};
|
|
61
|
+
pending = null;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (pending) {
|
|
66
|
+
current = pending;
|
|
67
|
+
pending = null;
|
|
68
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
69
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
70
|
+
pending: null,
|
|
71
|
+
batchesThisRun: 0,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
|
|
82
|
+
return `knowledge-source-sync-${sourceId}`;
|
|
83
|
+
}
|