@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/sandbox-routing.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// target lookup), and the selfhosted ControlRpc built over the events bus's NATS
|
|
15
15
|
// request/reply connection.
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
@@ -45,7 +45,9 @@ import {
|
|
|
45
45
|
RoutingBackendRecoveryRequiredError,
|
|
46
46
|
RoutingSandboxSession,
|
|
47
47
|
resolveModalCheckpointProviderBindingForSession,
|
|
48
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
48
49
|
sandboxBackendForSdkBackendId,
|
|
50
|
+
selectBackend,
|
|
49
51
|
verifySandboxExecReadiness,
|
|
50
52
|
type ControlRpc,
|
|
51
53
|
type EstablishedSandboxSession,
|
|
@@ -100,6 +102,8 @@ export type RoutingWiringServices = {
|
|
|
100
102
|
established: EstablishedSandboxSession;
|
|
101
103
|
leaseEpoch: number;
|
|
102
104
|
}) => void;
|
|
105
|
+
/** Cancel bounded capture waits with the owning turn activity. */
|
|
106
|
+
waitSignal?: AbortSignal;
|
|
103
107
|
};
|
|
104
108
|
|
|
105
109
|
/** Worker-turn routing may final-ack streamed command output, so it must always
|
|
@@ -137,6 +141,8 @@ export type RoutingWiringIds = {
|
|
|
137
141
|
* manifest environment variables." Optional → the resolver defaults to `{}`.
|
|
138
142
|
*/
|
|
139
143
|
environment?: Record<string, string>;
|
|
144
|
+
/** Attempt-local values projected only onto Connected Machine child execs. */
|
|
145
|
+
transientExecEnvironment?: () => Readonly<Record<string, string>>;
|
|
140
146
|
/**
|
|
141
147
|
* Stage D machine-primary: PIN the already-established turn SelfhostedSession
|
|
142
148
|
* (the `established` arg's session) for THIS machine pointer `(sandboxId, epoch)`
|
|
@@ -262,27 +268,7 @@ function homeRouteRecoveryError(
|
|
|
262
268
|
export function providerIdentityFromResumeState(
|
|
263
269
|
resumeState: Record<string, unknown>,
|
|
264
270
|
): string | null {
|
|
265
|
-
|
|
266
|
-
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
267
|
-
? (resumeState.sessionState as Record<string, unknown>)
|
|
268
|
-
: null;
|
|
269
|
-
const providerState =
|
|
270
|
-
sessionState?.providerState && typeof sessionState.providerState === "object"
|
|
271
|
-
? (sessionState.providerState as Record<string, unknown>)
|
|
272
|
-
: null;
|
|
273
|
-
for (const field of [
|
|
274
|
-
"sandboxId",
|
|
275
|
-
"instanceId",
|
|
276
|
-
"id",
|
|
277
|
-
"hostId",
|
|
278
|
-
"containerId",
|
|
279
|
-
"workspaceRootPath",
|
|
280
|
-
"agentId",
|
|
281
|
-
]) {
|
|
282
|
-
const value = providerState?.[field];
|
|
283
|
-
if (typeof value === "string" && value.length > 0) return value;
|
|
284
|
-
}
|
|
285
|
-
return null;
|
|
271
|
+
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
286
272
|
}
|
|
287
273
|
|
|
288
274
|
/**
|
|
@@ -448,7 +434,8 @@ function beforePersistableHomeMutation(
|
|
|
448
434
|
expectedEpoch: backend.leaseEpoch,
|
|
449
435
|
expectedInstanceId: backend.providerInstanceId,
|
|
450
436
|
operation: op,
|
|
451
|
-
captureWaitMs:
|
|
437
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
438
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
452
439
|
});
|
|
453
440
|
return { admission, providerBinding };
|
|
454
441
|
};
|
|
@@ -583,7 +570,8 @@ function beforeRetainedProcessMutation(
|
|
|
583
570
|
sessionId: ids.sessionId,
|
|
584
571
|
processId: process.id,
|
|
585
572
|
operation: op,
|
|
586
|
-
captureWaitMs:
|
|
573
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
574
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
587
575
|
});
|
|
588
576
|
}
|
|
589
577
|
|
|
@@ -735,6 +723,9 @@ export function wrapTurnBoxWithRouting(
|
|
|
735
723
|
// environment variables" throw when the turn pins to a vm). Mirrors the group
|
|
736
724
|
// box, which is created WITH this same environment (resumeBoxForTurn).
|
|
737
725
|
...(ids.environment !== undefined ? { environment: ids.environment } : {}),
|
|
726
|
+
...(ids.transientExecEnvironment !== undefined
|
|
727
|
+
? { transientExecEnvironment: ids.transientExecEnvironment }
|
|
728
|
+
: {}),
|
|
738
729
|
// Stage D machine-primary: pin THIS established SelfhostedSession for the machine
|
|
739
730
|
// pointer so the resolver returns the SAME instance (no two-instance manifest
|
|
740
731
|
// divergence). `established.session` is the SelfhostedSession the establish branch
|
|
@@ -884,8 +875,17 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
884
875
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
885
876
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
886
877
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
878
|
+
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
879
|
+
const defaultSupportsPty = defaultBackend
|
|
880
|
+
? selectBackend(defaultBackend).capabilities.Terminal.pty
|
|
881
|
+
: false;
|
|
887
882
|
const syntheticSession: RoutableBackendSession = {
|
|
888
883
|
state: { manifest: args.agentDefaultManifest },
|
|
884
|
+
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
885
|
+
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
886
|
+
// write_stdin on its first turn operation; the async cancellation transport
|
|
887
|
+
// below still resolves the exact live route before executing a command.
|
|
888
|
+
supportsPty: () => defaultSupportsPty,
|
|
889
889
|
};
|
|
890
890
|
const routedResolver = makeActiveBackendResolver({
|
|
891
891
|
workspaceId: ids.workspaceId,
|
|
@@ -912,6 +912,9 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
912
912
|
...selfhostedResolverTimeouts(settings),
|
|
913
913
|
...(onOp !== undefined ? { selfhostedOnOp: onOp } : {}),
|
|
914
914
|
...(ids.environment !== undefined ? { environment: ids.environment } : {}),
|
|
915
|
+
...(ids.transientExecEnvironment !== undefined
|
|
916
|
+
? { transientExecEnvironment: ids.transientExecEnvironment }
|
|
917
|
+
: {}),
|
|
915
918
|
});
|
|
916
919
|
|
|
917
920
|
const proxy = new RoutingSandboxSession({
|
|
@@ -1027,6 +1030,8 @@ export type SelfhostedTurnSessionArgs = {
|
|
|
1027
1030
|
* the manifest), threaded so the SDK's per-turn provided-session env delta is
|
|
1028
1031
|
* empty. */
|
|
1029
1032
|
environment: Record<string, string>;
|
|
1033
|
+
/** Attempt-local values projected only onto this machine's child execs. */
|
|
1034
|
+
transientExecEnvironment?: () => Readonly<Record<string, string>>;
|
|
1030
1035
|
/** The session working directory (per-session pointer). Null ⇒ workspace_root. */
|
|
1031
1036
|
workingDir: string | null;
|
|
1032
1037
|
};
|
|
@@ -1083,6 +1088,9 @@ export async function establishSelfhostedTurnSession(
|
|
|
1083
1088
|
controlRpcFactory: controlRpcFactory(bus),
|
|
1084
1089
|
epoch: args.epoch,
|
|
1085
1090
|
environment: args.environment,
|
|
1091
|
+
...(args.transientExecEnvironment !== undefined
|
|
1092
|
+
? { transientExecEnvironment: args.transientExecEnvironment }
|
|
1093
|
+
: {}),
|
|
1086
1094
|
workingDir: args.workingDir,
|
|
1087
1095
|
// Give this turn's exec ops the long deadline (control ops stay short) so a real
|
|
1088
1096
|
// command is not killed at the control wall.
|
|
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
|
|
|
20
20
|
shutdown(): void;
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
+
/** One process may poll several disjoint Temporal queues, but it still has one
|
|
24
|
+
* host lifecycle. Start and stop every poller as a unit; a failed poller drains
|
|
25
|
+
* its siblings and preserves the original failure. */
|
|
26
|
+
export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
|
|
27
|
+
if (targets.length === 0) {
|
|
28
|
+
throw new Error("a worker service requires at least one Temporal poller");
|
|
29
|
+
}
|
|
30
|
+
let runPromise: Promise<void> | undefined;
|
|
31
|
+
const shutdown = (): void => {
|
|
32
|
+
let firstError: unknown;
|
|
33
|
+
for (const target of targets) {
|
|
34
|
+
try {
|
|
35
|
+
target.shutdown();
|
|
36
|
+
} catch (error) {
|
|
37
|
+
firstError ??= error;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (firstError !== undefined) throw firstError;
|
|
41
|
+
};
|
|
42
|
+
return {
|
|
43
|
+
run: () => {
|
|
44
|
+
runPromise ??= (async () => {
|
|
45
|
+
const runs = targets.map((target) => {
|
|
46
|
+
try {
|
|
47
|
+
return target.run();
|
|
48
|
+
} catch (error) {
|
|
49
|
+
return Promise.reject(error);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
try {
|
|
53
|
+
await Promise.all(runs);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
try {
|
|
56
|
+
shutdown();
|
|
57
|
+
} catch {
|
|
58
|
+
// Preserve the poller failure. Every sibling still received its
|
|
59
|
+
// shutdown request even when one request itself failed.
|
|
60
|
+
}
|
|
61
|
+
await Promise.allSettled(runs);
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
})();
|
|
65
|
+
return runPromise;
|
|
66
|
+
},
|
|
67
|
+
shutdown,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
23
71
|
export type WorkerServiceLifecycle = {
|
|
24
72
|
state(): WorkerLifecycleState;
|
|
25
73
|
run(): Promise<void>;
|
|
26
|
-
drain(reason?: string):
|
|
74
|
+
drain(reason?: string): boolean;
|
|
27
75
|
close(): Promise<void>;
|
|
28
76
|
};
|
|
29
77
|
|
|
@@ -74,27 +122,37 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
74
122
|
},
|
|
75
123
|
drain: (_reason = "host request") => {
|
|
76
124
|
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
77
|
-
return;
|
|
125
|
+
return true;
|
|
78
126
|
}
|
|
127
|
+
const previousState = state;
|
|
128
|
+
safeLifecycleLog(() =>
|
|
129
|
+
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
130
|
+
role: input.role,
|
|
131
|
+
errorClass: "WorkerLifecycleOperation",
|
|
132
|
+
errorCode: "worker_draining",
|
|
133
|
+
origin: "worker-lifecycle",
|
|
134
|
+
}),
|
|
135
|
+
);
|
|
79
136
|
state = "draining";
|
|
80
|
-
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
81
|
-
role: input.role,
|
|
82
|
-
errorClass: "WorkerLifecycleOperation",
|
|
83
|
-
errorCode: "worker_draining",
|
|
84
|
-
origin: "worker-lifecycle",
|
|
85
|
-
});
|
|
86
137
|
try {
|
|
87
138
|
input.worker.shutdown();
|
|
139
|
+
return true;
|
|
88
140
|
} catch {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
141
|
+
state = previousState;
|
|
142
|
+
safeLifecycleLog(() =>
|
|
143
|
+
input.observability.warn("worker shutdown request failed", {
|
|
144
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
145
|
+
errorCode: "worker_shutdown_request_failed",
|
|
146
|
+
origin: "worker-lifecycle",
|
|
147
|
+
}),
|
|
148
|
+
);
|
|
149
|
+
return false;
|
|
94
150
|
}
|
|
95
151
|
},
|
|
96
152
|
close: async () => {
|
|
97
|
-
lifecycle.drain("service close")
|
|
153
|
+
if (!lifecycle.drain("service close")) {
|
|
154
|
+
throw new Error("worker shutdown request failed");
|
|
155
|
+
}
|
|
98
156
|
if (runPromise) {
|
|
99
157
|
await runPromise.catch(() => undefined);
|
|
100
158
|
} else {
|
|
@@ -106,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
106
164
|
|
|
107
165
|
return lifecycle;
|
|
108
166
|
}
|
|
167
|
+
|
|
168
|
+
function safeLifecycleLog(log: () => void): void {
|
|
169
|
+
try {
|
|
170
|
+
log();
|
|
171
|
+
} catch {
|
|
172
|
+
// Telemetry cannot change worker lifecycle state or shutdown delivery.
|
|
173
|
+
}
|
|
174
|
+
}
|
|
@@ -5,6 +5,7 @@ type WorkflowControlActivities = Pick<
|
|
|
5
5
|
typeof activities,
|
|
6
6
|
| "enqueueGoalRetryWake"
|
|
7
7
|
| "expireSessionHumanInput"
|
|
8
|
+
| "expireSessionInteractionIntervention"
|
|
8
9
|
| "failSessionAttempt"
|
|
9
10
|
| "getCodexCapacityWait"
|
|
10
11
|
| "markSessionIdle"
|
|
@@ -91,11 +92,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
|
|
|
91
92
|
});
|
|
92
93
|
}
|
|
93
94
|
|
|
95
|
+
export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
|
|
96
|
+
return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
|
|
97
|
+
taskQueue: turnTaskQueue(baseTaskQueue),
|
|
98
|
+
startToCloseTimeout: "20 minutes",
|
|
99
|
+
retry: {
|
|
100
|
+
initialInterval: "1 second",
|
|
101
|
+
backoffCoefficient: 2,
|
|
102
|
+
maximumInterval: "30 seconds",
|
|
103
|
+
maximumAttempts: 3,
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
|
|
94
108
|
export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
|
|
95
109
|
startToCloseTimeout: "30 minutes",
|
|
96
110
|
retry: { maximumAttempts: 1 },
|
|
97
111
|
});
|
|
98
112
|
|
|
113
|
+
/** Connector inventory/download/index batches are provider-I/O activities. The
|
|
114
|
+
* implementation heartbeats between bounded items and owns its resumable
|
|
115
|
+
* checkpoint; Temporal retries therefore repeat only idempotent source/object
|
|
116
|
+
* operations and never invoke the agent runtime. */
|
|
117
|
+
export const knowledgeSourceSyncActivity = proxyActivities<
|
|
118
|
+
Pick<typeof activities, "runKnowledgeSourceSyncBatch">
|
|
119
|
+
>({
|
|
120
|
+
startToCloseTimeout: "1 hour",
|
|
121
|
+
heartbeatTimeout: "2 minutes",
|
|
122
|
+
retry: {
|
|
123
|
+
initialInterval: "2 seconds",
|
|
124
|
+
backoffCoefficient: 2,
|
|
125
|
+
maximumInterval: "1 minute",
|
|
126
|
+
maximumAttempts: 5,
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
|
|
99
130
|
export function workflowFailureMessage(error: unknown): string {
|
|
100
131
|
if (error instanceof Error) {
|
|
101
132
|
return error.message;
|
|
@@ -2,7 +2,13 @@ import { proxyActivities } from "@temporalio/workflow";
|
|
|
2
2
|
import type * as activities from "../activities";
|
|
3
3
|
|
|
4
4
|
const reaperActivity = proxyActivities<
|
|
5
|
-
Pick<
|
|
5
|
+
Pick<
|
|
6
|
+
typeof activities,
|
|
7
|
+
| "reapExpiredFileUploads"
|
|
8
|
+
| "maintainRetainedScreenshots"
|
|
9
|
+
| "maintainBrowserStateArtifacts"
|
|
10
|
+
| "recoverVideoGenerationWorkflows"
|
|
11
|
+
>
|
|
6
12
|
>({
|
|
7
13
|
startToCloseTimeout: "5 minutes",
|
|
8
14
|
retry: { maximumAttempts: 1 },
|
|
@@ -12,4 +18,6 @@ const reaperActivity = proxyActivities<
|
|
|
12
18
|
export async function fileUploadReaperWorkflow(): Promise<void> {
|
|
13
19
|
await reaperActivity.reapExpiredFileUploads();
|
|
14
20
|
await reaperActivity.maintainRetainedScreenshots();
|
|
21
|
+
await reaperActivity.maintainBrowserStateArtifacts();
|
|
22
|
+
await reaperActivity.recoverVideoGenerationWorkflows();
|
|
15
23
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
|
|
2
|
+
import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
|
|
3
|
+
import { knowledgeSourceSyncActivity } from "./activities";
|
|
4
|
+
|
|
5
|
+
export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
|
|
6
|
+
|
|
7
|
+
export const knowledgeSourceSyncWake =
|
|
8
|
+
defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
|
|
9
|
+
|
|
10
|
+
type KnowledgeSourceSyncWorkflowState = {
|
|
11
|
+
pending: KnowledgeSourceSyncWorkflowInput | null;
|
|
12
|
+
batchesThisRun: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
|
|
16
|
+
|
|
17
|
+
/** One stable workflow ID serializes all fires for a source. Signals retain at
|
|
18
|
+
* most one newest pending wake; Postgres lease/checkpoint rows remain the
|
|
19
|
+
* authoritative overlap and resumability fence across workflow generations. */
|
|
20
|
+
export async function knowledgeSourceSyncWorkflow(
|
|
21
|
+
input: KnowledgeSourceSyncWorkflowInput,
|
|
22
|
+
carried?: KnowledgeSourceSyncWorkflowState,
|
|
23
|
+
): Promise<void> {
|
|
24
|
+
let current = input;
|
|
25
|
+
let pending = carried?.pending ?? null;
|
|
26
|
+
let batchesThisRun = carried?.batchesThisRun ?? 0;
|
|
27
|
+
|
|
28
|
+
setHandler(knowledgeSourceSyncWake, (next) => {
|
|
29
|
+
if (
|
|
30
|
+
next.accountId !== input.accountId ||
|
|
31
|
+
next.workspaceId !== input.workspaceId ||
|
|
32
|
+
next.sourceId !== input.sourceId
|
|
33
|
+
) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
pending = next;
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
while (true) {
|
|
40
|
+
const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
|
|
41
|
+
batchesThisRun += 1;
|
|
42
|
+
|
|
43
|
+
if (result.action === "continue") {
|
|
44
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
45
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
46
|
+
pending,
|
|
47
|
+
batchesThisRun: 0,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (
|
|
54
|
+
(result.action === "complete" || result.action === "failed") &&
|
|
55
|
+
result.bufferedScheduledTaskRunId
|
|
56
|
+
) {
|
|
57
|
+
current = {
|
|
58
|
+
...current,
|
|
59
|
+
scheduledTaskRunId: result.bufferedScheduledTaskRunId,
|
|
60
|
+
};
|
|
61
|
+
pending = null;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (pending) {
|
|
66
|
+
current = pending;
|
|
67
|
+
pending = null;
|
|
68
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
69
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
70
|
+
pending: null,
|
|
71
|
+
batchesThisRun: 0,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
|
|
82
|
+
return `knowledge-source-sync-${sourceId}`;
|
|
83
|
+
}
|
|
@@ -1,28 +1,190 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
import { proxyActivities } from "@temporalio/workflow";
|
|
1
|
+
// The global schedule performs only bounded inventory/maintenance work and
|
|
2
|
+
// starts one durable child per exact (workspace, group, epoch). Provider I/O is
|
|
3
|
+
// therefore parallel at the Temporal worker's activity-concurrency limit; one
|
|
4
|
+
// slow snapshot can delay only its own box, never the global reaper or a sibling.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
ParentClosePolicy,
|
|
8
|
+
WorkflowIdReusePolicy,
|
|
9
|
+
log,
|
|
10
|
+
proxyActivities,
|
|
11
|
+
startChild,
|
|
12
|
+
uuid4,
|
|
13
|
+
workflowInfo,
|
|
14
|
+
} from "@temporalio/workflow";
|
|
16
15
|
import type * as activities from "../activities";
|
|
16
|
+
import {
|
|
17
|
+
SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
|
|
18
|
+
SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS,
|
|
19
|
+
SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
20
|
+
SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
|
|
21
|
+
SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
22
|
+
sandboxDrainWorkflowId,
|
|
23
|
+
sandboxLifecycleTaskQueue,
|
|
24
|
+
type SandboxDrainActivityInput,
|
|
25
|
+
type SandboxDrainWorkflowInput,
|
|
26
|
+
type SandboxLeaseSweepMaintenanceInput,
|
|
27
|
+
} from "../sandbox-reaper-contract";
|
|
28
|
+
|
|
29
|
+
function lifecycleTaskQueue(): string {
|
|
30
|
+
return sandboxLifecycleTaskQueue(workflowInfo().taskQueue);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function sweepActivities() {
|
|
34
|
+
return proxyActivities<Pick<typeof activities, "prepareSandboxLeaseSweep">>({
|
|
35
|
+
taskQueue: lifecycleTaskQueue(),
|
|
36
|
+
startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
37
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
38
|
+
// Inventory is the admission edge for every per-box child. A worker crash
|
|
39
|
+
// must not silently consume the only scheduled sweep.
|
|
40
|
+
retry: {
|
|
41
|
+
initialInterval: "1 second",
|
|
42
|
+
backoffCoefficient: 2,
|
|
43
|
+
maximumInterval: "30 seconds",
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function maintenanceActivity() {
|
|
49
|
+
return proxyActivities<Pick<typeof activities, "maintainSandboxLeaseSweep">>({
|
|
50
|
+
taskQueue: lifecycleTaskQueue(),
|
|
51
|
+
startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
52
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
53
|
+
// Maintenance is ancillary and internally exact/idempotent. Never let one
|
|
54
|
+
// pathological pass keep the fixed-id V2 workflow open forever and thereby
|
|
55
|
+
// suppress all later drain inventories. The next Schedule tick retries it.
|
|
56
|
+
retry: { maximumAttempts: 1 },
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function drainActivity(startToCloseTimeout: number) {
|
|
61
|
+
return proxyActivities<Pick<typeof activities, "drainSandboxLease">>({
|
|
62
|
+
taskQueue: lifecycleTaskQueue(),
|
|
63
|
+
startToCloseTimeout,
|
|
64
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
65
|
+
// Each retry is a new exact DB capture attempt under one logical operation.
|
|
66
|
+
// Heartbeat timeout moves a crashed worker to a healthy poller promptly;
|
|
67
|
+
// exact-attempt CAS fencing makes late callbacks harmless.
|
|
68
|
+
retry: {
|
|
69
|
+
initialInterval: "1 second",
|
|
70
|
+
backoffCoefficient: 2,
|
|
71
|
+
maximumInterval: "30 seconds",
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
}
|
|
17
75
|
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
|
|
76
|
+
// This proxy and workflow deliberately preserve the exact pre-refactor
|
|
77
|
+
// ScheduleActivity command. A patch marker would let a new binary replay old
|
|
78
|
+
// histories, but an old binary could not replay a history first executed by a
|
|
79
|
+
// new worker during a rolling deployment. Routing happens inside the activity,
|
|
80
|
+
// where changing implementation is safe and mixed old/new workers remain valid.
|
|
81
|
+
const legacyReaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
|
|
82
|
+
startToCloseTimeout: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
|
|
23
83
|
retry: { maximumAttempts: 1 },
|
|
24
84
|
});
|
|
25
85
|
|
|
86
|
+
export async function sandboxDrainWorkflow(input: SandboxDrainWorkflowInput): Promise<void> {
|
|
87
|
+
const activityInput: SandboxDrainActivityInput = {
|
|
88
|
+
...input,
|
|
89
|
+
operationId: uuid4(),
|
|
90
|
+
};
|
|
91
|
+
if (input.timeoutClass === "fast") {
|
|
92
|
+
await drainActivity(SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS).drainSandboxLease(activityInput);
|
|
93
|
+
} else {
|
|
94
|
+
await drainActivity(SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS).drainSandboxLease(
|
|
95
|
+
activityInput,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Ancillary billing/reconciliation/GC is durable but never allowed to hold the
|
|
101
|
+
* fixed-ID inventory workflow open behind a provider backlog. One singleton
|
|
102
|
+
* maintenance child may queue on the lifecycle workers while every schedule
|
|
103
|
+
* tick remains free to inventory and dispatch newly-drainable boxes. */
|
|
104
|
+
export async function sandboxReaperMaintenanceWorkflow(
|
|
105
|
+
input: SandboxLeaseSweepMaintenanceInput,
|
|
106
|
+
): Promise<void> {
|
|
107
|
+
try {
|
|
108
|
+
await maintenanceActivity().maintainSandboxLeaseSweep(input);
|
|
109
|
+
} catch (error) {
|
|
110
|
+
// The next Schedule tick starts another singleton after this run closes.
|
|
111
|
+
// Drain children were already made durable before this workflow was started.
|
|
112
|
+
log.warn("sandbox reaper maintenance deferred to a later sweep", {
|
|
113
|
+
error: error instanceof Error ? error.message : String(error),
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function alreadyRunningChild(error: unknown): boolean {
|
|
119
|
+
let current: unknown = error;
|
|
120
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
121
|
+
const record = current as { name?: unknown; cause?: unknown };
|
|
122
|
+
if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
|
|
123
|
+
current = record.cause;
|
|
124
|
+
}
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
|
|
26
128
|
export async function sandboxReaperWorkflow(): Promise<void> {
|
|
27
|
-
await
|
|
129
|
+
await legacyReaperActivity.reapSandboxLeases();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export async function sandboxReaperWorkflowV2(): Promise<void> {
|
|
133
|
+
const sweep = sweepActivities();
|
|
134
|
+
const plan = await sweep.prepareSandboxLeaseSweep();
|
|
135
|
+
const starts = await Promise.all(
|
|
136
|
+
plan.drainable.map(async (target) => {
|
|
137
|
+
try {
|
|
138
|
+
await startChild(sandboxDrainWorkflow, {
|
|
139
|
+
workflowId: sandboxDrainWorkflowId(target),
|
|
140
|
+
taskQueue: lifecycleTaskQueue(),
|
|
141
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
142
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
143
|
+
args: [
|
|
144
|
+
{
|
|
145
|
+
target,
|
|
146
|
+
timeoutClass: plan.timeoutClass,
|
|
147
|
+
snapshotTimeoutMs: plan.snapshotTimeoutMs,
|
|
148
|
+
captureTimeoutMs: plan.captureTimeoutMs,
|
|
149
|
+
},
|
|
150
|
+
],
|
|
151
|
+
});
|
|
152
|
+
return "started" as const;
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (alreadyRunningChild(error)) return "already_running" as const;
|
|
155
|
+
log.error("sandbox reaper child start failed", {
|
|
156
|
+
workspaceId: target.workspaceId,
|
|
157
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
158
|
+
leaseEpoch: target.leaseEpoch,
|
|
159
|
+
instanceId: target.instanceId,
|
|
160
|
+
error: error instanceof Error ? error.message : String(error),
|
|
161
|
+
});
|
|
162
|
+
return "failed" as const;
|
|
163
|
+
}
|
|
164
|
+
}),
|
|
165
|
+
);
|
|
166
|
+
try {
|
|
167
|
+
await startChild(sandboxReaperMaintenanceWorkflow, {
|
|
168
|
+
workflowId: SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
|
|
169
|
+
taskQueue: lifecycleTaskQueue(),
|
|
170
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
171
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
172
|
+
args: [
|
|
173
|
+
{
|
|
174
|
+
examined: plan.drainable.length,
|
|
175
|
+
started: starts.filter((outcome) => outcome === "started").length,
|
|
176
|
+
alreadyRunning: starts.filter((outcome) => outcome === "already_running").length,
|
|
177
|
+
startFailed: starts.filter((outcome) => outcome === "failed").length,
|
|
178
|
+
rotationsRequested: plan.rotationsRequested,
|
|
179
|
+
},
|
|
180
|
+
],
|
|
181
|
+
});
|
|
182
|
+
} catch (error) {
|
|
183
|
+
if (alreadyRunningChild(error)) return;
|
|
184
|
+
// Every drain child is already durable and this inventory workflow still
|
|
185
|
+
// completes, so the next Schedule tick can retry maintenance independently.
|
|
186
|
+
log.warn("sandbox reaper maintenance child start deferred to next sweep", {
|
|
187
|
+
error: error instanceof Error ? error.message : String(error),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
28
190
|
}
|