@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/index.ts
CHANGED
|
@@ -26,20 +26,20 @@ import {
|
|
|
26
26
|
ScheduleAlreadyRunning,
|
|
27
27
|
ScheduleOverlapPolicy,
|
|
28
28
|
Client as TemporalClient,
|
|
29
|
+
WorkflowExecutionAlreadyStartedError,
|
|
29
30
|
} from "@temporalio/client";
|
|
30
31
|
import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
|
|
31
32
|
import { existsSync } from "node:fs";
|
|
32
33
|
import { fileURLToPath } from "node:url";
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
|
|
36
|
-
createTurnActivities,
|
|
37
|
-
type ActivityDependencies,
|
|
38
|
-
} from "./activities";
|
|
34
|
+
import type { createControlActivities } from "./activities-control";
|
|
35
|
+
import type { createTurnActivities } from "./activities-turn";
|
|
36
|
+
import type { ActivityDependencies } from "./activities/types";
|
|
39
37
|
import type {
|
|
40
38
|
InspectSessionAttemptActivity,
|
|
41
39
|
SignalCodexCapacityWorkflow,
|
|
42
40
|
SignalSessionAttemptQuiesced,
|
|
41
|
+
StartSandboxReaperWorkflow,
|
|
42
|
+
StartVideoGenerationWorkflow,
|
|
43
43
|
WakeSessionWorkflowSignal,
|
|
44
44
|
} from "./activities/types";
|
|
45
45
|
import { turnTaskQueue } from "./workflows/activities";
|
|
@@ -50,23 +50,42 @@ import {
|
|
|
50
50
|
temporalReadyCheck,
|
|
51
51
|
type WorkerLifecycleState,
|
|
52
52
|
} from "./http";
|
|
53
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
normalizeTurnTaskQueueStats,
|
|
55
|
+
observabilityEventLogger,
|
|
56
|
+
startTurnCapacityMonitor,
|
|
57
|
+
type TurnTaskQueueStats,
|
|
58
|
+
} from "./observability-metrics";
|
|
54
59
|
import {
|
|
55
60
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
56
61
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
57
62
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
63
|
+
TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
58
64
|
} from "@opengeni/core";
|
|
59
65
|
import {
|
|
66
|
+
CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
60
67
|
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
61
68
|
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
69
|
+
createTurnWorkerConcurrencyPlan,
|
|
62
70
|
turnWorkerConcurrencyLogFields,
|
|
63
|
-
turnWorkerConcurrencyOptions,
|
|
64
71
|
} from "./concurrency";
|
|
65
72
|
import {
|
|
73
|
+
combineWorkerRunTargets,
|
|
66
74
|
constructWithOwnedConnection,
|
|
67
75
|
createWorkerServiceLifecycle,
|
|
68
76
|
type WorkerServiceLifecycle,
|
|
77
|
+
type WorkerRunTarget,
|
|
69
78
|
} from "./worker-service-lifecycle";
|
|
79
|
+
import {
|
|
80
|
+
createTurnWorkerMemoryPressureGuard,
|
|
81
|
+
turnWorkerMemoryPressureGuardEnabled,
|
|
82
|
+
type TurnWorkerMemoryPressureGuard,
|
|
83
|
+
} from "./memory-pressure-guard";
|
|
84
|
+
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
85
|
+
import {
|
|
86
|
+
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
87
|
+
sandboxLifecycleTaskQueue,
|
|
88
|
+
} from "./sandbox-reaper-contract";
|
|
70
89
|
|
|
71
90
|
export {
|
|
72
91
|
createHostExportPump,
|
|
@@ -135,6 +154,18 @@ export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
|
|
|
135
154
|
return isGrpcServiceError(error) && error.code === 5;
|
|
136
155
|
}
|
|
137
156
|
|
|
157
|
+
/** Load exactly one role's activity graph. Exported for embedded hosts and the
|
|
158
|
+
* process-RSS conformance benchmark; construction remains side-effect free
|
|
159
|
+
* until an activity first resolves its injected services. */
|
|
160
|
+
export async function createDefaultWorkerActivities(
|
|
161
|
+
role: OpenGeniWorkerRole,
|
|
162
|
+
dependencies: ActivityDependencies = {},
|
|
163
|
+
) {
|
|
164
|
+
return role === "control"
|
|
165
|
+
? (await import("./activities-control")).createControlActivities(dependencies)
|
|
166
|
+
: (await import("./activities-turn")).createTurnActivities(dependencies);
|
|
167
|
+
}
|
|
168
|
+
|
|
138
169
|
type WorkerWorkflowDefinition =
|
|
139
170
|
| { workflowBundle: WorkflowBundleOption }
|
|
140
171
|
| { workflowsPath: string };
|
|
@@ -161,11 +192,24 @@ export function resolveOpenGeniWorkflowDefinition(
|
|
|
161
192
|
return { workflowBundle: { codePath } };
|
|
162
193
|
}
|
|
163
194
|
|
|
195
|
+
export function turnCancellationHeartbeatThrottleOptions() {
|
|
196
|
+
return {
|
|
197
|
+
maxHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
198
|
+
defaultHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
199
|
+
} as const;
|
|
200
|
+
}
|
|
201
|
+
|
|
164
202
|
export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
165
|
-
worker:
|
|
203
|
+
worker: WorkerRunTarget;
|
|
166
204
|
connection: NativeConnection;
|
|
167
205
|
}> {
|
|
168
206
|
const settings = options.settings ?? getSettings();
|
|
207
|
+
if (options.role === "control") {
|
|
208
|
+
// The exported lower-level factory is itself poll-capable. Validate here so
|
|
209
|
+
// embedded hosts cannot bypass the reaper budget contract by omitting the
|
|
210
|
+
// higher-level service and its Schedule-registration lifecycle.
|
|
211
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
212
|
+
}
|
|
169
213
|
const observability =
|
|
170
214
|
options.activityDependencies?.observability ??
|
|
171
215
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
@@ -178,6 +222,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
178
222
|
// Otherwise this is a no-op unless the registry secret + logical ref are both
|
|
179
223
|
// set. Memoized in the provider, so it runs once per process.
|
|
180
224
|
if (options.role === "turn") {
|
|
225
|
+
const { ensureModalRegistryImage } = await import("@opengeni/runtime/sandbox");
|
|
181
226
|
await retryStartupDependency(
|
|
182
227
|
"Modal private-registry image",
|
|
183
228
|
() => ensureModalRegistryImage(settings),
|
|
@@ -198,50 +243,87 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
198
243
|
},
|
|
199
244
|
),
|
|
200
245
|
async (connection) => {
|
|
246
|
+
const activityDependencies = {
|
|
247
|
+
...options.activityDependencies,
|
|
248
|
+
settings,
|
|
249
|
+
observability,
|
|
250
|
+
};
|
|
201
251
|
const activities =
|
|
202
252
|
options.activities ??
|
|
203
|
-
(options.role
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
observability
|
|
207
|
-
|
|
208
|
-
const
|
|
253
|
+
(await createDefaultWorkerActivities(options.role, activityDependencies));
|
|
254
|
+
const turnConcurrency =
|
|
255
|
+
options.role === "turn"
|
|
256
|
+
? createTurnWorkerConcurrencyPlan(settings, { observability })
|
|
257
|
+
: null;
|
|
258
|
+
const workflowDefinition =
|
|
259
|
+
options.role === "control"
|
|
260
|
+
? options.workflowBundle
|
|
261
|
+
? { workflowBundle: options.workflowBundle }
|
|
262
|
+
: resolveOpenGeniWorkflowDefinition()
|
|
263
|
+
: {};
|
|
264
|
+
const sharedWorkerOptions = {
|
|
209
265
|
connection,
|
|
210
266
|
namespace: settings.temporalNamespace,
|
|
267
|
+
activities,
|
|
268
|
+
// Cancellation is delivered through an activity heartbeat. Keep the SDK
|
|
269
|
+
// throttle aligned with runAgentTurn's local heartbeat cadence so the
|
|
270
|
+
// four-second control contract still has time for physical writer drain
|
|
271
|
+
// and exact receipt-gated replacement admission.
|
|
272
|
+
...turnCancellationHeartbeatThrottleOptions(),
|
|
273
|
+
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
274
|
+
// after shutdown() stops polling, in-flight activities get this long to
|
|
275
|
+
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN.
|
|
276
|
+
shutdownGraceTime: "5s",
|
|
277
|
+
// Hard ceiling inside the pod's termination grace period.
|
|
278
|
+
shutdownForceTime: "100s",
|
|
279
|
+
} as const;
|
|
280
|
+
const worker = await Worker.create({
|
|
281
|
+
...sharedWorkerOptions,
|
|
211
282
|
taskQueue:
|
|
212
283
|
options.role === "control"
|
|
213
284
|
? settings.temporalTaskQueue
|
|
214
285
|
: turnTaskQueue(settings.temporalTaskQueue),
|
|
215
286
|
...(options.role === "control"
|
|
216
287
|
? {
|
|
217
|
-
...
|
|
218
|
-
|
|
219
|
-
|
|
288
|
+
...workflowDefinition,
|
|
289
|
+
reuseV8Context: true,
|
|
290
|
+
workflowThreadPoolSize: 1,
|
|
291
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
220
292
|
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
221
293
|
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
222
294
|
}
|
|
223
|
-
:
|
|
224
|
-
activities,
|
|
225
|
-
// Cancellation is delivered through an activity heartbeat. The SDK would
|
|
226
|
-
// otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
|
|
227
|
-
// making Pause/Steer take roughly a minute even though runAgentTurn emits a
|
|
228
|
-
// heartbeat every ten seconds. Keep delivery bounded independently of the
|
|
229
|
-
// heartbeat timeout and local timer cadence.
|
|
230
|
-
maxHeartbeatThrottleInterval: "5s",
|
|
231
|
-
defaultHeartbeatThrottleInterval: "5s",
|
|
232
|
-
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
233
|
-
// after shutdown() stops polling, in-flight activities get this long to
|
|
234
|
-
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
|
|
235
|
-
// which triggers agent-turn's same-turn recovery checkpoint instead of a
|
|
236
|
-
// heartbeat-timeout worker_death. Short on purpose: a long grace here
|
|
237
|
-
// only delays the checkpoint window long turns actually need.
|
|
238
|
-
shutdownGraceTime: "5s",
|
|
239
|
-
// Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
|
|
240
|
-
// wedged checkpoint force-stops here, on our terms, rather than riding
|
|
241
|
-
// into the kubelet's SIGKILL mid-DB-write.
|
|
242
|
-
shutdownForceTime: "100s",
|
|
295
|
+
: turnConcurrency!.options),
|
|
243
296
|
});
|
|
244
|
-
|
|
297
|
+
if (options.role !== "control") {
|
|
298
|
+
turnConcurrency?.admission?.finalizeStartupBaseline();
|
|
299
|
+
return { worker, connection };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
try {
|
|
303
|
+
const sandboxLifecycleWorker = await Worker.create({
|
|
304
|
+
...sharedWorkerOptions,
|
|
305
|
+
...workflowDefinition,
|
|
306
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
307
|
+
reuseV8Context: true,
|
|
308
|
+
workflowThreadPoolSize: 1,
|
|
309
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
310
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
311
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
312
|
+
});
|
|
313
|
+
return {
|
|
314
|
+
worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
|
|
315
|
+
connection,
|
|
316
|
+
};
|
|
317
|
+
} catch (error) {
|
|
318
|
+
try {
|
|
319
|
+
worker.shutdown();
|
|
320
|
+
} catch {
|
|
321
|
+
// Preserve the lifecycle worker construction failure. The base poller
|
|
322
|
+
// still received its shutdown request; a secondary synchronous
|
|
323
|
+
// shutdown error must not replace the startup root cause.
|
|
324
|
+
}
|
|
325
|
+
throw error;
|
|
326
|
+
}
|
|
245
327
|
},
|
|
246
328
|
(connection) => connection.close(),
|
|
247
329
|
);
|
|
@@ -260,6 +342,9 @@ export async function createWorkerWorkflowSignaler(
|
|
|
260
342
|
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
|
|
261
343
|
inspectSessionAttemptActivity: InspectSessionAttemptActivity;
|
|
262
344
|
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
345
|
+
getTurnTaskQueueStats: () => Promise<TurnTaskQueueStats>;
|
|
346
|
+
startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
|
|
347
|
+
startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
|
|
263
348
|
check: () => Promise<void>;
|
|
264
349
|
close: () => Promise<void>;
|
|
265
350
|
}> {
|
|
@@ -354,6 +439,61 @@ export async function createWorkerWorkflowSignaler(
|
|
|
354
439
|
// another producer may have advanced it with a Pause/Steer that requires
|
|
355
440
|
// sessionControl. The global dispatcher owns that acknowledgement.
|
|
356
441
|
},
|
|
442
|
+
getTurnTaskQueueStats: async () => {
|
|
443
|
+
const response = await connection.workflowService.describeTaskQueue({
|
|
444
|
+
namespace: settings.temporalNamespace,
|
|
445
|
+
taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
|
|
446
|
+
// temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
|
|
447
|
+
// the supported DEFAULT mode; `stats.approximateBacklogCount` is the
|
|
448
|
+
// server-documented scaling signal.
|
|
449
|
+
taskQueueType: 2,
|
|
450
|
+
reportStats: true,
|
|
451
|
+
});
|
|
452
|
+
return normalizeTurnTaskQueueStats(response.stats);
|
|
453
|
+
},
|
|
454
|
+
startSandboxReaperWorkflow: async () => {
|
|
455
|
+
try {
|
|
456
|
+
await temporal.workflow.start("sandboxReaperWorkflowV2", {
|
|
457
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
458
|
+
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
459
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
460
|
+
args: [],
|
|
461
|
+
});
|
|
462
|
+
return "started";
|
|
463
|
+
} catch (error) {
|
|
464
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) {
|
|
465
|
+
return "already_running";
|
|
466
|
+
}
|
|
467
|
+
throw error;
|
|
468
|
+
}
|
|
469
|
+
},
|
|
470
|
+
startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
|
|
471
|
+
try {
|
|
472
|
+
await temporal.workflow.start("videoGenerationWorkflow", {
|
|
473
|
+
taskQueue: settings.temporalTaskQueue,
|
|
474
|
+
workflowId: `video-generation:${operationId}`,
|
|
475
|
+
// Reject a concurrent or already-completed run, but allow the repair
|
|
476
|
+
// sweep to restart this exact operation if the workflow itself failed.
|
|
477
|
+
// Every provider/storage transition remains CAS/idempotency fenced in
|
|
478
|
+
// Postgres, so recovery never creates a second logical operation.
|
|
479
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE_FAILED_ONLY",
|
|
480
|
+
args: [
|
|
481
|
+
{
|
|
482
|
+
accountId,
|
|
483
|
+
workspaceId,
|
|
484
|
+
operationId,
|
|
485
|
+
baseTaskQueue: settings.temporalTaskQueue,
|
|
486
|
+
},
|
|
487
|
+
],
|
|
488
|
+
});
|
|
489
|
+
return "started";
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) {
|
|
492
|
+
return "already_running";
|
|
493
|
+
}
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
},
|
|
357
497
|
check: async () => {
|
|
358
498
|
await connection.workflowService.getSystemInfo({});
|
|
359
499
|
},
|
|
@@ -369,13 +509,14 @@ export async function createWorkerWorkflowSignaler(
|
|
|
369
509
|
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
370
510
|
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
371
511
|
*
|
|
372
|
-
* The Schedule fires sandboxReaperWorkflow on the
|
|
512
|
+
* The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
|
|
373
513
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
374
514
|
* `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
|
|
375
515
|
* validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
|
|
376
516
|
* sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
|
|
377
|
-
* worker pool collides on ScheduleAlreadyRunning and
|
|
378
|
-
*
|
|
517
|
+
* worker pool collides on ScheduleAlreadyRunning and reconciles the same
|
|
518
|
+
* definition. New workers route onto the versioned lifecycle queue; old workers
|
|
519
|
+
* can still run the legacy action during rollout or a full binary rollback.
|
|
379
520
|
*
|
|
380
521
|
* Returns a `close()` for the dedicated client connection (separate from the
|
|
381
522
|
* worker's NativeConnection — the Schedule client is a @temporalio/client).
|
|
@@ -384,29 +525,32 @@ export async function registerSandboxReaperSchedule(
|
|
|
384
525
|
settings: Settings,
|
|
385
526
|
observability: Observability,
|
|
386
527
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
528
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
387
529
|
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
388
530
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
531
|
+
const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
|
|
532
|
+
const action = {
|
|
533
|
+
type: "startWorkflow" as const,
|
|
534
|
+
workflowType: "sandboxReaperWorkflow",
|
|
535
|
+
taskQueue: settings.temporalTaskQueue,
|
|
536
|
+
args: [] as [],
|
|
537
|
+
};
|
|
538
|
+
const policies = {
|
|
539
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
540
|
+
catchupWindow: "1m",
|
|
541
|
+
pauseOnFailure: false,
|
|
542
|
+
} as const;
|
|
389
543
|
try {
|
|
390
544
|
await temporal.schedule.create({
|
|
391
545
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
workflowType: "sandboxReaperWorkflow",
|
|
401
|
-
taskQueue: settings.temporalTaskQueue,
|
|
402
|
-
args: [],
|
|
403
|
-
},
|
|
404
|
-
policies: {
|
|
405
|
-
// A slow sweep must never overlap itself; the next fire is skipped.
|
|
406
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
407
|
-
catchupWindow: "1m",
|
|
408
|
-
pauseOnFailure: false,
|
|
409
|
-
},
|
|
546
|
+
// @every-style interval: fire once per reaper period. The boot invariant
|
|
547
|
+
// guarantees reaperPeriod < viewerHolderTTL and
|
|
548
|
+
// reaperPeriod + idleGrace < providerLifetime.
|
|
549
|
+
spec,
|
|
550
|
+
action,
|
|
551
|
+
// The base workflow is a short router on new binaries; legacy binaries
|
|
552
|
+
// still execute the pre-refactor composite under this same overlap fence.
|
|
553
|
+
policies,
|
|
410
554
|
});
|
|
411
555
|
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
412
556
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
@@ -420,12 +564,24 @@ export async function registerSandboxReaperSchedule(
|
|
|
420
564
|
};
|
|
421
565
|
} catch (error) {
|
|
422
566
|
if (error instanceof ScheduleAlreadyRunning) {
|
|
423
|
-
//
|
|
424
|
-
//
|
|
425
|
-
//
|
|
426
|
-
|
|
427
|
-
|
|
567
|
+
// Never move this action off the base queue: doing so would leave a full
|
|
568
|
+
// rollback with no worker polling the only reaper Schedule. The workflow
|
|
569
|
+
// implementation itself owns the forward-compatible queue handoff.
|
|
570
|
+
await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
|
|
571
|
+
spec,
|
|
572
|
+
action,
|
|
573
|
+
policies,
|
|
574
|
+
state: {
|
|
575
|
+
paused: previous.state.paused,
|
|
576
|
+
...(previous.state.note ? { note: previous.state.note } : {}),
|
|
577
|
+
...(previous.state.remainingActions !== undefined
|
|
578
|
+
? { remainingActions: previous.state.remainingActions }
|
|
579
|
+
: {}),
|
|
580
|
+
},
|
|
581
|
+
}));
|
|
582
|
+
observability.info("Reconciled the global sandbox-lease reaper Schedule", {
|
|
428
583
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
584
|
+
temporalTaskQueue: action.taskQueue,
|
|
429
585
|
});
|
|
430
586
|
return {
|
|
431
587
|
registered: false,
|
|
@@ -548,7 +704,7 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
|
|
|
548
704
|
|
|
549
705
|
export type OpenGeniWorkerService = {
|
|
550
706
|
readonly role: OpenGeniWorkerRole;
|
|
551
|
-
readonly worker:
|
|
707
|
+
readonly worker: WorkerRunTarget;
|
|
552
708
|
readonly connection: NativeConnection;
|
|
553
709
|
state(): WorkerLifecycleState;
|
|
554
710
|
run(): Promise<void>;
|
|
@@ -582,15 +738,20 @@ export async function createOpenGeniWorkerService(
|
|
|
582
738
|
let lifecycle: WorkerServiceLifecycle | undefined;
|
|
583
739
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
584
740
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
741
|
+
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
585
742
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
586
743
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
744
|
+
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
587
745
|
|
|
588
746
|
try {
|
|
589
747
|
const needsSignaler =
|
|
748
|
+
options.role === "turn" ||
|
|
590
749
|
!options.activityDependencies.wakeSessionWorkflow ||
|
|
591
750
|
!options.activityDependencies.signalSessionAttemptQuiesced ||
|
|
592
751
|
!options.activityDependencies.inspectSessionAttemptActivity ||
|
|
593
|
-
!options.activityDependencies.signalCodexCapacityWorkflow
|
|
752
|
+
!options.activityDependencies.signalCodexCapacityWorkflow ||
|
|
753
|
+
!options.activityDependencies.startSandboxReaperWorkflow ||
|
|
754
|
+
!options.activityDependencies.startVideoGenerationWorkflow;
|
|
594
755
|
if (needsSignaler) {
|
|
595
756
|
signaler = await retryStartupDependency(
|
|
596
757
|
"Temporal client",
|
|
@@ -609,11 +770,19 @@ export async function createOpenGeniWorkerService(
|
|
|
609
770
|
const signalCodexCapacityWorkflow =
|
|
610
771
|
options.activityDependencies.signalCodexCapacityWorkflow ??
|
|
611
772
|
signaler?.signalCodexCapacityWorkflow;
|
|
773
|
+
const startSandboxReaperWorkflow =
|
|
774
|
+
options.activityDependencies.startSandboxReaperWorkflow ??
|
|
775
|
+
signaler?.startSandboxReaperWorkflow;
|
|
776
|
+
const startVideoGenerationWorkflow =
|
|
777
|
+
options.activityDependencies.startVideoGenerationWorkflow ??
|
|
778
|
+
signaler?.startVideoGenerationWorkflow;
|
|
612
779
|
if (
|
|
613
780
|
!wakeSessionWorkflow ||
|
|
614
781
|
!signalSessionAttemptQuiesced ||
|
|
615
782
|
!inspectSessionAttemptActivity ||
|
|
616
|
-
!signalCodexCapacityWorkflow
|
|
783
|
+
!signalCodexCapacityWorkflow ||
|
|
784
|
+
!startSandboxReaperWorkflow ||
|
|
785
|
+
!startVideoGenerationWorkflow
|
|
617
786
|
) {
|
|
618
787
|
throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
|
|
619
788
|
}
|
|
@@ -630,9 +799,21 @@ export async function createOpenGeniWorkerService(
|
|
|
630
799
|
signalSessionAttemptQuiesced,
|
|
631
800
|
inspectSessionAttemptActivity,
|
|
632
801
|
signalCodexCapacityWorkflow,
|
|
802
|
+
startSandboxReaperWorkflow,
|
|
803
|
+
startVideoGenerationWorkflow,
|
|
633
804
|
},
|
|
634
805
|
});
|
|
635
806
|
|
|
807
|
+
if (options.role === "turn") {
|
|
808
|
+
if (!signaler) {
|
|
809
|
+
throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
|
|
810
|
+
}
|
|
811
|
+
turnCapacityMonitor = startTurnCapacityMonitor({
|
|
812
|
+
observability,
|
|
813
|
+
read: signaler.getTurnTaskQueueStats,
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
|
|
636
817
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
637
818
|
schedules.push(
|
|
638
819
|
await retryStartupDependency(
|
|
@@ -673,6 +854,7 @@ export async function createOpenGeniWorkerService(
|
|
|
673
854
|
} catch (error) {
|
|
674
855
|
httpServer?.stop(true);
|
|
675
856
|
await Promise.allSettled([
|
|
857
|
+
turnCapacityMonitor?.close(),
|
|
676
858
|
workerBundle?.connection.close(),
|
|
677
859
|
signaler?.close(),
|
|
678
860
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -691,8 +873,10 @@ export async function createOpenGeniWorkerService(
|
|
|
691
873
|
worker: activeWorkerBundle.worker,
|
|
692
874
|
observability,
|
|
693
875
|
closeOwnedResources: async () => {
|
|
876
|
+
memoryPressureGuard?.close();
|
|
694
877
|
httpServer?.stop(true);
|
|
695
878
|
await Promise.allSettled([
|
|
879
|
+
turnCapacityMonitor?.close(),
|
|
696
880
|
activeWorkerBundle.connection.close(),
|
|
697
881
|
activeSignaler?.close(),
|
|
698
882
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -715,19 +899,40 @@ export async function createOpenGeniWorkerService(
|
|
|
715
899
|
}),
|
|
716
900
|
maxConcurrentWorkflowTaskExecutions:
|
|
717
901
|
options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
902
|
+
maxCachedWorkflows: options.role === "control" ? CONTROL_WORKER_MAX_CACHED_WORKFLOWS : 0,
|
|
718
903
|
httpPort: options.http === false ? null : settings.workerHttpPort,
|
|
719
904
|
});
|
|
720
905
|
},
|
|
721
906
|
});
|
|
907
|
+
const activeLifecycle = lifecycle;
|
|
908
|
+
|
|
909
|
+
if (turnWorkerMemoryPressureGuardEnabled(options.role, settings)) {
|
|
910
|
+
try {
|
|
911
|
+
memoryPressureGuard = createTurnWorkerMemoryPressureGuard({
|
|
912
|
+
settings,
|
|
913
|
+
observability,
|
|
914
|
+
drain: () => {
|
|
915
|
+
if (!activeLifecycle.drain("memory pressure guard")) {
|
|
916
|
+
throw new Error("worker shutdown request failed");
|
|
917
|
+
}
|
|
918
|
+
},
|
|
919
|
+
});
|
|
920
|
+
} catch (error) {
|
|
921
|
+
await activeLifecycle.close();
|
|
922
|
+
throw error;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
722
925
|
|
|
723
926
|
return {
|
|
724
927
|
role: options.role,
|
|
725
928
|
worker: activeWorkerBundle.worker,
|
|
726
929
|
connection: activeWorkerBundle.connection,
|
|
727
|
-
state:
|
|
728
|
-
run:
|
|
729
|
-
drain:
|
|
730
|
-
|
|
930
|
+
state: activeLifecycle.state,
|
|
931
|
+
run: activeLifecycle.run,
|
|
932
|
+
drain: (reason) => {
|
|
933
|
+
activeLifecycle.drain(reason);
|
|
934
|
+
},
|
|
935
|
+
close: activeLifecycle.close,
|
|
731
936
|
};
|
|
732
937
|
}
|
|
733
938
|
|