@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/index.ts
CHANGED
|
@@ -26,20 +26,20 @@ import {
|
|
|
26
26
|
ScheduleAlreadyRunning,
|
|
27
27
|
ScheduleOverlapPolicy,
|
|
28
28
|
Client as TemporalClient,
|
|
29
|
+
WorkflowExecutionAlreadyStartedError,
|
|
29
30
|
} from "@temporalio/client";
|
|
30
31
|
import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
|
|
31
32
|
import { existsSync } from "node:fs";
|
|
32
33
|
import { fileURLToPath } from "node:url";
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
|
|
36
|
-
createTurnActivities,
|
|
37
|
-
type ActivityDependencies,
|
|
38
|
-
} from "./activities";
|
|
34
|
+
import type { createControlActivities } from "./activities-control";
|
|
35
|
+
import type { createTurnActivities } from "./activities-turn";
|
|
36
|
+
import type { ActivityDependencies } from "./activities/types";
|
|
39
37
|
import type {
|
|
40
38
|
InspectSessionAttemptActivity,
|
|
41
39
|
SignalCodexCapacityWorkflow,
|
|
42
40
|
SignalSessionAttemptQuiesced,
|
|
41
|
+
StartSandboxReaperWorkflow,
|
|
42
|
+
StartVideoGenerationWorkflow,
|
|
43
43
|
WakeSessionWorkflowSignal,
|
|
44
44
|
} from "./activities/types";
|
|
45
45
|
import { turnTaskQueue } from "./workflows/activities";
|
|
@@ -50,23 +50,42 @@ import {
|
|
|
50
50
|
temporalReadyCheck,
|
|
51
51
|
type WorkerLifecycleState,
|
|
52
52
|
} from "./http";
|
|
53
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
normalizeTurnTaskQueueStats,
|
|
55
|
+
observabilityEventLogger,
|
|
56
|
+
startTurnCapacityMonitor,
|
|
57
|
+
type TurnTaskQueueStats,
|
|
58
|
+
} from "./observability-metrics";
|
|
54
59
|
import {
|
|
55
60
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
56
61
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
57
62
|
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
63
|
+
TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
58
64
|
} from "@opengeni/core";
|
|
59
65
|
import {
|
|
66
|
+
CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
60
67
|
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
61
68
|
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
69
|
+
createTurnWorkerConcurrencyPlan,
|
|
62
70
|
turnWorkerConcurrencyLogFields,
|
|
63
|
-
turnWorkerConcurrencyOptions,
|
|
64
71
|
} from "./concurrency";
|
|
65
72
|
import {
|
|
73
|
+
combineWorkerRunTargets,
|
|
66
74
|
constructWithOwnedConnection,
|
|
67
75
|
createWorkerServiceLifecycle,
|
|
68
76
|
type WorkerServiceLifecycle,
|
|
77
|
+
type WorkerRunTarget,
|
|
69
78
|
} from "./worker-service-lifecycle";
|
|
79
|
+
import {
|
|
80
|
+
createTurnWorkerMemoryPressureGuard,
|
|
81
|
+
turnWorkerMemoryPressureGuardEnabled,
|
|
82
|
+
type TurnWorkerMemoryPressureGuard,
|
|
83
|
+
} from "./memory-pressure-guard";
|
|
84
|
+
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
85
|
+
import {
|
|
86
|
+
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
87
|
+
sandboxLifecycleTaskQueue,
|
|
88
|
+
} from "./sandbox-reaper-contract";
|
|
70
89
|
|
|
71
90
|
export {
|
|
72
91
|
createHostExportPump,
|
|
@@ -89,6 +108,8 @@ export {
|
|
|
89
108
|
const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
90
109
|
export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
91
110
|
export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
|
|
111
|
+
export const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
|
|
112
|
+
export const SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1_000;
|
|
92
113
|
export type OpenGeniWorkerRole = "control" | "turn";
|
|
93
114
|
|
|
94
115
|
export type WorkerOptions = {
|
|
@@ -135,6 +156,18 @@ export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
|
|
|
135
156
|
return isGrpcServiceError(error) && error.code === 5;
|
|
136
157
|
}
|
|
137
158
|
|
|
159
|
+
/** Load exactly one role's activity graph. Exported for embedded hosts and the
|
|
160
|
+
* process-RSS conformance benchmark; construction remains side-effect free
|
|
161
|
+
* until an activity first resolves its injected services. */
|
|
162
|
+
export async function createDefaultWorkerActivities(
|
|
163
|
+
role: OpenGeniWorkerRole,
|
|
164
|
+
dependencies: ActivityDependencies = {},
|
|
165
|
+
) {
|
|
166
|
+
return role === "control"
|
|
167
|
+
? (await import("./activities-control")).createControlActivities(dependencies)
|
|
168
|
+
: (await import("./activities-turn")).createTurnActivities(dependencies);
|
|
169
|
+
}
|
|
170
|
+
|
|
138
171
|
type WorkerWorkflowDefinition =
|
|
139
172
|
| { workflowBundle: WorkflowBundleOption }
|
|
140
173
|
| { workflowsPath: string };
|
|
@@ -161,11 +194,24 @@ export function resolveOpenGeniWorkflowDefinition(
|
|
|
161
194
|
return { workflowBundle: { codePath } };
|
|
162
195
|
}
|
|
163
196
|
|
|
197
|
+
export function turnCancellationHeartbeatThrottleOptions() {
|
|
198
|
+
return {
|
|
199
|
+
maxHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
200
|
+
defaultHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
201
|
+
} as const;
|
|
202
|
+
}
|
|
203
|
+
|
|
164
204
|
export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
165
|
-
worker:
|
|
205
|
+
worker: WorkerRunTarget;
|
|
166
206
|
connection: NativeConnection;
|
|
167
207
|
}> {
|
|
168
208
|
const settings = options.settings ?? getSettings();
|
|
209
|
+
if (options.role === "control") {
|
|
210
|
+
// The exported lower-level factory is itself poll-capable. Validate here so
|
|
211
|
+
// embedded hosts cannot bypass the reaper budget contract by omitting the
|
|
212
|
+
// higher-level service and its Schedule-registration lifecycle.
|
|
213
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
214
|
+
}
|
|
169
215
|
const observability =
|
|
170
216
|
options.activityDependencies?.observability ??
|
|
171
217
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
@@ -178,6 +224,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
178
224
|
// Otherwise this is a no-op unless the registry secret + logical ref are both
|
|
179
225
|
// set. Memoized in the provider, so it runs once per process.
|
|
180
226
|
if (options.role === "turn") {
|
|
227
|
+
const { ensureModalRegistryImage } = await import("@opengeni/runtime/sandbox");
|
|
181
228
|
await retryStartupDependency(
|
|
182
229
|
"Modal private-registry image",
|
|
183
230
|
() => ensureModalRegistryImage(settings),
|
|
@@ -198,50 +245,87 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
198
245
|
},
|
|
199
246
|
),
|
|
200
247
|
async (connection) => {
|
|
248
|
+
const activityDependencies = {
|
|
249
|
+
...options.activityDependencies,
|
|
250
|
+
settings,
|
|
251
|
+
observability,
|
|
252
|
+
};
|
|
201
253
|
const activities =
|
|
202
254
|
options.activities ??
|
|
203
|
-
(options.role
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
observability
|
|
207
|
-
|
|
208
|
-
const
|
|
255
|
+
(await createDefaultWorkerActivities(options.role, activityDependencies));
|
|
256
|
+
const turnConcurrency =
|
|
257
|
+
options.role === "turn"
|
|
258
|
+
? createTurnWorkerConcurrencyPlan(settings, { observability })
|
|
259
|
+
: null;
|
|
260
|
+
const workflowDefinition =
|
|
261
|
+
options.role === "control"
|
|
262
|
+
? options.workflowBundle
|
|
263
|
+
? { workflowBundle: options.workflowBundle }
|
|
264
|
+
: resolveOpenGeniWorkflowDefinition()
|
|
265
|
+
: {};
|
|
266
|
+
const sharedWorkerOptions = {
|
|
209
267
|
connection,
|
|
210
268
|
namespace: settings.temporalNamespace,
|
|
269
|
+
activities,
|
|
270
|
+
// Cancellation is delivered through an activity heartbeat. Keep the SDK
|
|
271
|
+
// throttle aligned with runAgentTurn's local heartbeat cadence so the
|
|
272
|
+
// four-second control contract still has time for physical writer drain
|
|
273
|
+
// and exact receipt-gated replacement admission.
|
|
274
|
+
...turnCancellationHeartbeatThrottleOptions(),
|
|
275
|
+
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
276
|
+
// after shutdown() stops polling, in-flight activities get this long to
|
|
277
|
+
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN.
|
|
278
|
+
shutdownGraceTime: "5s",
|
|
279
|
+
// Hard ceiling inside the pod's termination grace period.
|
|
280
|
+
shutdownForceTime: "100s",
|
|
281
|
+
} as const;
|
|
282
|
+
const worker = await Worker.create({
|
|
283
|
+
...sharedWorkerOptions,
|
|
211
284
|
taskQueue:
|
|
212
285
|
options.role === "control"
|
|
213
286
|
? settings.temporalTaskQueue
|
|
214
287
|
: turnTaskQueue(settings.temporalTaskQueue),
|
|
215
288
|
...(options.role === "control"
|
|
216
289
|
? {
|
|
217
|
-
...
|
|
218
|
-
|
|
219
|
-
|
|
290
|
+
...workflowDefinition,
|
|
291
|
+
reuseV8Context: true,
|
|
292
|
+
workflowThreadPoolSize: 1,
|
|
293
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
220
294
|
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
221
295
|
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
222
296
|
}
|
|
223
|
-
:
|
|
224
|
-
activities,
|
|
225
|
-
// Cancellation is delivered through an activity heartbeat. The SDK would
|
|
226
|
-
// otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
|
|
227
|
-
// making Pause/Steer take roughly a minute even though runAgentTurn emits a
|
|
228
|
-
// heartbeat every ten seconds. Keep delivery bounded independently of the
|
|
229
|
-
// heartbeat timeout and local timer cadence.
|
|
230
|
-
maxHeartbeatThrottleInterval: "5s",
|
|
231
|
-
defaultHeartbeatThrottleInterval: "5s",
|
|
232
|
-
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
233
|
-
// after shutdown() stops polling, in-flight activities get this long to
|
|
234
|
-
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
|
|
235
|
-
// which triggers agent-turn's same-turn recovery checkpoint instead of a
|
|
236
|
-
// heartbeat-timeout worker_death. Short on purpose: a long grace here
|
|
237
|
-
// only delays the checkpoint window long turns actually need.
|
|
238
|
-
shutdownGraceTime: "5s",
|
|
239
|
-
// Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
|
|
240
|
-
// wedged checkpoint force-stops here, on our terms, rather than riding
|
|
241
|
-
// into the kubelet's SIGKILL mid-DB-write.
|
|
242
|
-
shutdownForceTime: "100s",
|
|
297
|
+
: turnConcurrency!.options),
|
|
243
298
|
});
|
|
244
|
-
|
|
299
|
+
if (options.role !== "control") {
|
|
300
|
+
turnConcurrency?.admission?.finalizeStartupBaseline();
|
|
301
|
+
return { worker, connection };
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
try {
|
|
305
|
+
const sandboxLifecycleWorker = await Worker.create({
|
|
306
|
+
...sharedWorkerOptions,
|
|
307
|
+
...workflowDefinition,
|
|
308
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
309
|
+
reuseV8Context: true,
|
|
310
|
+
workflowThreadPoolSize: 1,
|
|
311
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
312
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
313
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
314
|
+
});
|
|
315
|
+
return {
|
|
316
|
+
worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
|
|
317
|
+
connection,
|
|
318
|
+
};
|
|
319
|
+
} catch (error) {
|
|
320
|
+
try {
|
|
321
|
+
worker.shutdown();
|
|
322
|
+
} catch {
|
|
323
|
+
// Preserve the lifecycle worker construction failure. The base poller
|
|
324
|
+
// still received its shutdown request; a secondary synchronous
|
|
325
|
+
// shutdown error must not replace the startup root cause.
|
|
326
|
+
}
|
|
327
|
+
throw error;
|
|
328
|
+
}
|
|
245
329
|
},
|
|
246
330
|
(connection) => connection.close(),
|
|
247
331
|
);
|
|
@@ -260,6 +344,9 @@ export async function createWorkerWorkflowSignaler(
|
|
|
260
344
|
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
|
|
261
345
|
inspectSessionAttemptActivity: InspectSessionAttemptActivity;
|
|
262
346
|
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
347
|
+
getTurnTaskQueueStats: () => Promise<TurnTaskQueueStats>;
|
|
348
|
+
startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
|
|
349
|
+
startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
|
|
263
350
|
check: () => Promise<void>;
|
|
264
351
|
close: () => Promise<void>;
|
|
265
352
|
}> {
|
|
@@ -354,6 +441,61 @@ export async function createWorkerWorkflowSignaler(
|
|
|
354
441
|
// another producer may have advanced it with a Pause/Steer that requires
|
|
355
442
|
// sessionControl. The global dispatcher owns that acknowledgement.
|
|
356
443
|
},
|
|
444
|
+
getTurnTaskQueueStats: async () => {
|
|
445
|
+
const response = await connection.workflowService.describeTaskQueue({
|
|
446
|
+
namespace: settings.temporalNamespace,
|
|
447
|
+
taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
|
|
448
|
+
// temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
|
|
449
|
+
// the supported DEFAULT mode; `stats.approximateBacklogCount` is the
|
|
450
|
+
// server-documented scaling signal.
|
|
451
|
+
taskQueueType: 2,
|
|
452
|
+
reportStats: true,
|
|
453
|
+
});
|
|
454
|
+
return normalizeTurnTaskQueueStats(response.stats);
|
|
455
|
+
},
|
|
456
|
+
startSandboxReaperWorkflow: async () => {
|
|
457
|
+
try {
|
|
458
|
+
await temporal.workflow.start("sandboxReaperWorkflowV2", {
|
|
459
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
460
|
+
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
461
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
462
|
+
args: [],
|
|
463
|
+
});
|
|
464
|
+
return "started";
|
|
465
|
+
} catch (error) {
|
|
466
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) {
|
|
467
|
+
return "already_running";
|
|
468
|
+
}
|
|
469
|
+
throw error;
|
|
470
|
+
}
|
|
471
|
+
},
|
|
472
|
+
startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
|
|
473
|
+
try {
|
|
474
|
+
await temporal.workflow.start("videoGenerationWorkflow", {
|
|
475
|
+
taskQueue: settings.temporalTaskQueue,
|
|
476
|
+
workflowId: `video-generation:${operationId}`,
|
|
477
|
+
// Reject a concurrent or already-completed run, but allow the repair
|
|
478
|
+
// sweep to restart this exact operation if the workflow itself failed.
|
|
479
|
+
// Every provider/storage transition remains CAS/idempotency fenced in
|
|
480
|
+
// Postgres, so recovery never creates a second logical operation.
|
|
481
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE_FAILED_ONLY",
|
|
482
|
+
args: [
|
|
483
|
+
{
|
|
484
|
+
accountId,
|
|
485
|
+
workspaceId,
|
|
486
|
+
operationId,
|
|
487
|
+
baseTaskQueue: settings.temporalTaskQueue,
|
|
488
|
+
},
|
|
489
|
+
],
|
|
490
|
+
});
|
|
491
|
+
return "started";
|
|
492
|
+
} catch (error) {
|
|
493
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) {
|
|
494
|
+
return "already_running";
|
|
495
|
+
}
|
|
496
|
+
throw error;
|
|
497
|
+
}
|
|
498
|
+
},
|
|
357
499
|
check: async () => {
|
|
358
500
|
await connection.workflowService.getSystemInfo({});
|
|
359
501
|
},
|
|
@@ -369,13 +511,14 @@ export async function createWorkerWorkflowSignaler(
|
|
|
369
511
|
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
370
512
|
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
371
513
|
*
|
|
372
|
-
* The Schedule fires sandboxReaperWorkflow on the
|
|
514
|
+
* The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
|
|
373
515
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
374
516
|
* `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
|
|
375
517
|
* validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
|
|
376
518
|
* sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
|
|
377
|
-
* worker pool collides on ScheduleAlreadyRunning and
|
|
378
|
-
*
|
|
519
|
+
* worker pool collides on ScheduleAlreadyRunning and reconciles the same
|
|
520
|
+
* definition. New workers route onto the versioned lifecycle queue; old workers
|
|
521
|
+
* can still run the legacy action during rollout or a full binary rollback.
|
|
379
522
|
*
|
|
380
523
|
* Returns a `close()` for the dedicated client connection (separate from the
|
|
381
524
|
* worker's NativeConnection — the Schedule client is a @temporalio/client).
|
|
@@ -384,29 +527,32 @@ export async function registerSandboxReaperSchedule(
|
|
|
384
527
|
settings: Settings,
|
|
385
528
|
observability: Observability,
|
|
386
529
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
530
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
387
531
|
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
388
532
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
533
|
+
const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
|
|
534
|
+
const action = {
|
|
535
|
+
type: "startWorkflow" as const,
|
|
536
|
+
workflowType: "sandboxReaperWorkflow",
|
|
537
|
+
taskQueue: settings.temporalTaskQueue,
|
|
538
|
+
args: [] as [],
|
|
539
|
+
};
|
|
540
|
+
const policies = {
|
|
541
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
542
|
+
catchupWindow: "1m",
|
|
543
|
+
pauseOnFailure: false,
|
|
544
|
+
} as const;
|
|
389
545
|
try {
|
|
390
546
|
await temporal.schedule.create({
|
|
391
547
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
workflowType: "sandboxReaperWorkflow",
|
|
401
|
-
taskQueue: settings.temporalTaskQueue,
|
|
402
|
-
args: [],
|
|
403
|
-
},
|
|
404
|
-
policies: {
|
|
405
|
-
// A slow sweep must never overlap itself; the next fire is skipped.
|
|
406
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
407
|
-
catchupWindow: "1m",
|
|
408
|
-
pauseOnFailure: false,
|
|
409
|
-
},
|
|
548
|
+
// @every-style interval: fire once per reaper period. The boot invariant
|
|
549
|
+
// guarantees reaperPeriod < viewerHolderTTL and
|
|
550
|
+
// reaperPeriod + idleGrace < providerLifetime.
|
|
551
|
+
spec,
|
|
552
|
+
action,
|
|
553
|
+
// The base workflow is a short router on new binaries; legacy binaries
|
|
554
|
+
// still execute the pre-refactor composite under this same overlap fence.
|
|
555
|
+
policies,
|
|
410
556
|
});
|
|
411
557
|
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
412
558
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
@@ -420,12 +566,24 @@ export async function registerSandboxReaperSchedule(
|
|
|
420
566
|
};
|
|
421
567
|
} catch (error) {
|
|
422
568
|
if (error instanceof ScheduleAlreadyRunning) {
|
|
423
|
-
//
|
|
424
|
-
//
|
|
425
|
-
//
|
|
426
|
-
|
|
427
|
-
|
|
569
|
+
// Never move this action off the base queue: doing so would leave a full
|
|
570
|
+
// rollback with no worker polling the only reaper Schedule. The workflow
|
|
571
|
+
// implementation itself owns the forward-compatible queue handoff.
|
|
572
|
+
await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
|
|
573
|
+
spec,
|
|
574
|
+
action,
|
|
575
|
+
policies,
|
|
576
|
+
state: {
|
|
577
|
+
paused: previous.state.paused,
|
|
578
|
+
...(previous.state.note ? { note: previous.state.note } : {}),
|
|
579
|
+
...(previous.state.remainingActions !== undefined
|
|
580
|
+
? { remainingActions: previous.state.remainingActions }
|
|
581
|
+
: {}),
|
|
582
|
+
},
|
|
583
|
+
}));
|
|
584
|
+
observability.info("Reconciled the global sandbox-lease reaper Schedule", {
|
|
428
585
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
586
|
+
temporalTaskQueue: action.taskQueue,
|
|
429
587
|
});
|
|
430
588
|
return {
|
|
431
589
|
registered: false,
|
|
@@ -484,6 +642,48 @@ export async function registerFileUploadReaperSchedule(
|
|
|
484
642
|
}
|
|
485
643
|
}
|
|
486
644
|
|
|
645
|
+
/** Register the deployment-wide maintained-auth dispatcher. Its one-minute
|
|
646
|
+
* cadence matches the minimum public policy interval; DB claims own overlap,
|
|
647
|
+
* crash recovery, and exact session idempotency. */
|
|
648
|
+
export async function registerSiteAuthMaintenanceSchedule(
|
|
649
|
+
settings: Settings,
|
|
650
|
+
observability: Observability,
|
|
651
|
+
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
652
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
653
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
654
|
+
try {
|
|
655
|
+
await temporal.schedule.create({
|
|
656
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
657
|
+
spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
|
|
658
|
+
action: {
|
|
659
|
+
type: "startWorkflow",
|
|
660
|
+
workflowType: "siteAuthMaintenanceWorkflow",
|
|
661
|
+
taskQueue: settings.temporalTaskQueue,
|
|
662
|
+
args: [],
|
|
663
|
+
},
|
|
664
|
+
policies: {
|
|
665
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
666
|
+
catchupWindow: "1m",
|
|
667
|
+
pauseOnFailure: false,
|
|
668
|
+
},
|
|
669
|
+
});
|
|
670
|
+
observability.info("Registered the global site-auth maintenance Schedule", {
|
|
671
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
672
|
+
maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
673
|
+
});
|
|
674
|
+
return { registered: true, close: async () => connection.close() };
|
|
675
|
+
} catch (error) {
|
|
676
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
677
|
+
observability.info("Global site-auth maintenance Schedule already registered", {
|
|
678
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
679
|
+
});
|
|
680
|
+
return { registered: false, close: async () => connection.close() };
|
|
681
|
+
}
|
|
682
|
+
await connection.close().catch(() => undefined);
|
|
683
|
+
throw error;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
487
687
|
/**
|
|
488
688
|
* Register the one repair cadence for committed workflow-wake revisions. The
|
|
489
689
|
* activity only reads the transactional outbox and sends revision-scoped
|
|
@@ -548,7 +748,7 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
|
|
|
548
748
|
|
|
549
749
|
export type OpenGeniWorkerService = {
|
|
550
750
|
readonly role: OpenGeniWorkerRole;
|
|
551
|
-
readonly worker:
|
|
751
|
+
readonly worker: WorkerRunTarget;
|
|
552
752
|
readonly connection: NativeConnection;
|
|
553
753
|
state(): WorkerLifecycleState;
|
|
554
754
|
run(): Promise<void>;
|
|
@@ -582,15 +782,20 @@ export async function createOpenGeniWorkerService(
|
|
|
582
782
|
let lifecycle: WorkerServiceLifecycle | undefined;
|
|
583
783
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
584
784
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
785
|
+
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
585
786
|
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
586
787
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
788
|
+
let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
|
|
587
789
|
|
|
588
790
|
try {
|
|
589
791
|
const needsSignaler =
|
|
792
|
+
options.role === "turn" ||
|
|
590
793
|
!options.activityDependencies.wakeSessionWorkflow ||
|
|
591
794
|
!options.activityDependencies.signalSessionAttemptQuiesced ||
|
|
592
795
|
!options.activityDependencies.inspectSessionAttemptActivity ||
|
|
593
|
-
!options.activityDependencies.signalCodexCapacityWorkflow
|
|
796
|
+
!options.activityDependencies.signalCodexCapacityWorkflow ||
|
|
797
|
+
!options.activityDependencies.startSandboxReaperWorkflow ||
|
|
798
|
+
!options.activityDependencies.startVideoGenerationWorkflow;
|
|
594
799
|
if (needsSignaler) {
|
|
595
800
|
signaler = await retryStartupDependency(
|
|
596
801
|
"Temporal client",
|
|
@@ -609,11 +814,19 @@ export async function createOpenGeniWorkerService(
|
|
|
609
814
|
const signalCodexCapacityWorkflow =
|
|
610
815
|
options.activityDependencies.signalCodexCapacityWorkflow ??
|
|
611
816
|
signaler?.signalCodexCapacityWorkflow;
|
|
817
|
+
const startSandboxReaperWorkflow =
|
|
818
|
+
options.activityDependencies.startSandboxReaperWorkflow ??
|
|
819
|
+
signaler?.startSandboxReaperWorkflow;
|
|
820
|
+
const startVideoGenerationWorkflow =
|
|
821
|
+
options.activityDependencies.startVideoGenerationWorkflow ??
|
|
822
|
+
signaler?.startVideoGenerationWorkflow;
|
|
612
823
|
if (
|
|
613
824
|
!wakeSessionWorkflow ||
|
|
614
825
|
!signalSessionAttemptQuiesced ||
|
|
615
826
|
!inspectSessionAttemptActivity ||
|
|
616
|
-
!signalCodexCapacityWorkflow
|
|
827
|
+
!signalCodexCapacityWorkflow ||
|
|
828
|
+
!startSandboxReaperWorkflow ||
|
|
829
|
+
!startVideoGenerationWorkflow
|
|
617
830
|
) {
|
|
618
831
|
throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
|
|
619
832
|
}
|
|
@@ -630,9 +843,21 @@ export async function createOpenGeniWorkerService(
|
|
|
630
843
|
signalSessionAttemptQuiesced,
|
|
631
844
|
inspectSessionAttemptActivity,
|
|
632
845
|
signalCodexCapacityWorkflow,
|
|
846
|
+
startSandboxReaperWorkflow,
|
|
847
|
+
startVideoGenerationWorkflow,
|
|
633
848
|
},
|
|
634
849
|
});
|
|
635
850
|
|
|
851
|
+
if (options.role === "turn") {
|
|
852
|
+
if (!signaler) {
|
|
853
|
+
throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
|
|
854
|
+
}
|
|
855
|
+
turnCapacityMonitor = startTurnCapacityMonitor({
|
|
856
|
+
observability,
|
|
857
|
+
read: signaler.getTurnTaskQueueStats,
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
|
|
636
861
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
637
862
|
schedules.push(
|
|
638
863
|
await retryStartupDependency(
|
|
@@ -648,6 +873,13 @@ export async function createOpenGeniWorkerService(
|
|
|
648
873
|
{ ...retryOptions, onRetry },
|
|
649
874
|
),
|
|
650
875
|
);
|
|
876
|
+
schedules.push(
|
|
877
|
+
await retryStartupDependency(
|
|
878
|
+
"Temporal schedule (site auth maintenance)",
|
|
879
|
+
() => registerSiteAuthMaintenanceSchedule(settings, observability),
|
|
880
|
+
{ ...retryOptions, onRetry },
|
|
881
|
+
),
|
|
882
|
+
);
|
|
651
883
|
schedules.push(
|
|
652
884
|
await retryStartupDependency(
|
|
653
885
|
"Temporal schedule (session-workflow wake dispatcher)",
|
|
@@ -673,6 +905,7 @@ export async function createOpenGeniWorkerService(
|
|
|
673
905
|
} catch (error) {
|
|
674
906
|
httpServer?.stop(true);
|
|
675
907
|
await Promise.allSettled([
|
|
908
|
+
turnCapacityMonitor?.close(),
|
|
676
909
|
workerBundle?.connection.close(),
|
|
677
910
|
signaler?.close(),
|
|
678
911
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -691,8 +924,10 @@ export async function createOpenGeniWorkerService(
|
|
|
691
924
|
worker: activeWorkerBundle.worker,
|
|
692
925
|
observability,
|
|
693
926
|
closeOwnedResources: async () => {
|
|
927
|
+
memoryPressureGuard?.close();
|
|
694
928
|
httpServer?.stop(true);
|
|
695
929
|
await Promise.allSettled([
|
|
930
|
+
turnCapacityMonitor?.close(),
|
|
696
931
|
activeWorkerBundle.connection.close(),
|
|
697
932
|
activeSignaler?.close(),
|
|
698
933
|
...schedules.map((schedule) => schedule.close()),
|
|
@@ -715,19 +950,40 @@ export async function createOpenGeniWorkerService(
|
|
|
715
950
|
}),
|
|
716
951
|
maxConcurrentWorkflowTaskExecutions:
|
|
717
952
|
options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
953
|
+
maxCachedWorkflows: options.role === "control" ? CONTROL_WORKER_MAX_CACHED_WORKFLOWS : 0,
|
|
718
954
|
httpPort: options.http === false ? null : settings.workerHttpPort,
|
|
719
955
|
});
|
|
720
956
|
},
|
|
721
957
|
});
|
|
958
|
+
const activeLifecycle = lifecycle;
|
|
959
|
+
|
|
960
|
+
if (turnWorkerMemoryPressureGuardEnabled(options.role, settings)) {
|
|
961
|
+
try {
|
|
962
|
+
memoryPressureGuard = createTurnWorkerMemoryPressureGuard({
|
|
963
|
+
settings,
|
|
964
|
+
observability,
|
|
965
|
+
drain: () => {
|
|
966
|
+
if (!activeLifecycle.drain("memory pressure guard")) {
|
|
967
|
+
throw new Error("worker shutdown request failed");
|
|
968
|
+
}
|
|
969
|
+
},
|
|
970
|
+
});
|
|
971
|
+
} catch (error) {
|
|
972
|
+
await activeLifecycle.close();
|
|
973
|
+
throw error;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
722
976
|
|
|
723
977
|
return {
|
|
724
978
|
role: options.role,
|
|
725
979
|
worker: activeWorkerBundle.worker,
|
|
726
980
|
connection: activeWorkerBundle.connection,
|
|
727
|
-
state:
|
|
728
|
-
run:
|
|
729
|
-
drain:
|
|
730
|
-
|
|
981
|
+
state: activeLifecycle.state,
|
|
982
|
+
run: activeLifecycle.run,
|
|
983
|
+
drain: (reason) => {
|
|
984
|
+
activeLifecycle.drain(reason);
|
|
985
|
+
},
|
|
986
|
+
close: activeLifecycle.close,
|
|
731
987
|
};
|
|
732
988
|
}
|
|
733
989
|
|