@opengeni/worker-bundle 0.7.5 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2812 -813
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2045 -545
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
package/src/index.ts
CHANGED
|
@@ -4,10 +4,11 @@ import {
|
|
|
4
4
|
resolveNatsControlPlaneAuth,
|
|
5
5
|
retryStartupDependency,
|
|
6
6
|
startupRetryOptions,
|
|
7
|
+
temporalConnectionOptions,
|
|
7
8
|
type Settings,
|
|
8
9
|
} from "@opengeni/config";
|
|
9
10
|
import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
|
|
10
|
-
import { createNatsEventBus } from "@opengeni/events";
|
|
11
|
+
import { createNatsEventBus, type EventBus } from "@opengeni/events";
|
|
11
12
|
import {
|
|
12
13
|
createObservability,
|
|
13
14
|
logStartupDependencyRetry,
|
|
@@ -19,16 +20,28 @@ import {
|
|
|
19
20
|
ScheduleOverlapPolicy,
|
|
20
21
|
Client as TemporalClient,
|
|
21
22
|
} from "@temporalio/client";
|
|
22
|
-
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
23
|
+
import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
|
|
24
|
+
import { existsSync } from "node:fs";
|
|
25
|
+
import { fileURLToPath } from "node:url";
|
|
23
26
|
import { ensureModalRegistryImage } from "@opengeni/runtime";
|
|
24
27
|
import {
|
|
25
28
|
createControlActivities,
|
|
26
29
|
createTurnActivities,
|
|
27
30
|
type ActivityDependencies,
|
|
28
31
|
} from "./activities";
|
|
29
|
-
import type {
|
|
32
|
+
import type {
|
|
33
|
+
SignalCodexCapacityWorkflow,
|
|
34
|
+
SignalSessionAttemptQuiesced,
|
|
35
|
+
WakeSessionWorkflowSignal,
|
|
36
|
+
} from "./activities/types";
|
|
30
37
|
import { turnTaskQueue } from "./workflows/activities";
|
|
31
|
-
import {
|
|
38
|
+
import {
|
|
39
|
+
dbReadyCheck,
|
|
40
|
+
natsReadyCheck,
|
|
41
|
+
startWorkerHttpServer,
|
|
42
|
+
temporalReadyCheck,
|
|
43
|
+
type WorkerLifecycleState,
|
|
44
|
+
} from "./http";
|
|
32
45
|
import { observabilityEventLogger } from "./observability-metrics";
|
|
33
46
|
import {
|
|
34
47
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
@@ -40,6 +53,24 @@ import {
|
|
|
40
53
|
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
41
54
|
TURN_WORKER_MAX_CONCURRENT_TURNS,
|
|
42
55
|
} from "./concurrency";
|
|
56
|
+
import {
|
|
57
|
+
constructWithOwnedConnection,
|
|
58
|
+
createWorkerServiceLifecycle,
|
|
59
|
+
type WorkerServiceLifecycle,
|
|
60
|
+
} from "./worker-service-lifecycle";
|
|
61
|
+
|
|
62
|
+
export {
|
|
63
|
+
createHostExportPump,
|
|
64
|
+
type HostExportDrainResult,
|
|
65
|
+
type HostExportPump,
|
|
66
|
+
type HostExportPumpOptions,
|
|
67
|
+
type HostEventExport,
|
|
68
|
+
type HostEventExportBatch,
|
|
69
|
+
type HostEventSink,
|
|
70
|
+
type HostUsageExport,
|
|
71
|
+
type HostUsageExportBatch,
|
|
72
|
+
type HostUsageSink,
|
|
73
|
+
} from "./host-export";
|
|
43
74
|
|
|
44
75
|
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
45
76
|
// create() is idempotent across every worker in the pool: the first worker to
|
|
@@ -56,13 +87,36 @@ export type WorkerOptions = {
|
|
|
56
87
|
settings?: Settings;
|
|
57
88
|
activities?: ReturnType<typeof createControlActivities> | ReturnType<typeof createTurnActivities>;
|
|
58
89
|
activityDependencies?: ActivityDependencies;
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
// src/workflows.ts to a host-owned path and point the worker at it here.
|
|
62
|
-
// Unset (standalone) keeps today's in-package path byte-for-byte.
|
|
63
|
-
workflowsPath?: string;
|
|
90
|
+
/** Override the release-coherent workflow artifact. Most hosts should omit this. */
|
|
91
|
+
workflowBundle?: WorkflowBundleOption;
|
|
64
92
|
};
|
|
65
93
|
|
|
94
|
+
type WorkerWorkflowDefinition =
|
|
95
|
+
| { workflowBundle: WorkflowBundleOption }
|
|
96
|
+
| { workflowsPath: string };
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Resolve the deterministic workflow graph without asking installed hosts to
|
|
100
|
+
* transpile or relocate package TypeScript. Monorepo source execution retains
|
|
101
|
+
* the source path for the local development loop; published dist execution
|
|
102
|
+
* fails closed unless the build-generated sibling artifact is present.
|
|
103
|
+
*/
|
|
104
|
+
export function resolveOpenGeniWorkflowDefinition(
|
|
105
|
+
moduleUrl: string = import.meta.url,
|
|
106
|
+
): WorkerWorkflowDefinition {
|
|
107
|
+
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
108
|
+
if (modulePath.endsWith("/src/index.ts")) {
|
|
109
|
+
return { workflowsPath: fileURLToPath(new URL("./workflows.ts", moduleUrl)) };
|
|
110
|
+
}
|
|
111
|
+
const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
|
|
112
|
+
if (!existsSync(codePath)) {
|
|
113
|
+
throw new Error(
|
|
114
|
+
`OpenGeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
return { workflowBundle: { codePath } };
|
|
118
|
+
}
|
|
119
|
+
|
|
66
120
|
export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
67
121
|
worker: Worker;
|
|
68
122
|
connection: NativeConnection;
|
|
@@ -71,6 +125,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
71
125
|
const observability =
|
|
72
126
|
options.activityDependencies?.observability ??
|
|
73
127
|
createObservability(settings, { component: `worker-${options.role}` });
|
|
128
|
+
if (options.role === "turn" && options.workflowBundle) {
|
|
129
|
+
throw new Error("workflowBundle is valid only for the control worker role");
|
|
130
|
+
}
|
|
74
131
|
// Pre-resolve a PRIVATE-registry sandbox image before any turn creates a box.
|
|
75
132
|
// No-op unless OPENGENI_MODAL_IMAGE_REGISTRY_SECRET + OPENGENI_MODAL_IMAGE_REF are
|
|
76
133
|
// both set (so non-modal / public-image deployments are byte-unchanged and never
|
|
@@ -85,53 +142,67 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
85
142
|
},
|
|
86
143
|
);
|
|
87
144
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
145
|
+
return constructWithOwnedConnection(
|
|
146
|
+
() =>
|
|
147
|
+
retryStartupDependency(
|
|
148
|
+
"Temporal",
|
|
149
|
+
() => NativeConnection.connect(temporalConnectionOptions(settings)),
|
|
150
|
+
{
|
|
151
|
+
...startupRetryOptions(settings),
|
|
152
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
153
|
+
},
|
|
154
|
+
),
|
|
155
|
+
async (connection) => {
|
|
156
|
+
const activities =
|
|
157
|
+
options.activities ??
|
|
158
|
+
(options.role === "control" ? createControlActivities : createTurnActivities)({
|
|
159
|
+
...options.activityDependencies,
|
|
160
|
+
settings,
|
|
161
|
+
observability,
|
|
162
|
+
});
|
|
163
|
+
const worker = await Worker.create({
|
|
164
|
+
connection,
|
|
165
|
+
namespace: settings.temporalNamespace,
|
|
166
|
+
taskQueue:
|
|
167
|
+
options.role === "control"
|
|
168
|
+
? settings.temporalTaskQueue
|
|
169
|
+
: turnTaskQueue(settings.temporalTaskQueue),
|
|
170
|
+
...(options.role === "control"
|
|
171
|
+
? {
|
|
172
|
+
...(options.workflowBundle
|
|
173
|
+
? { workflowBundle: options.workflowBundle }
|
|
174
|
+
: resolveOpenGeniWorkflowDefinition()),
|
|
175
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
176
|
+
}
|
|
177
|
+
: {}),
|
|
178
|
+
activities,
|
|
179
|
+
maxConcurrentActivityTaskExecutions:
|
|
180
|
+
options.role === "turn"
|
|
181
|
+
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
182
|
+
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
183
|
+
// Cancellation is delivered through an activity heartbeat. The SDK would
|
|
184
|
+
// otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
|
|
185
|
+
// making Pause/Steer take roughly a minute even though runAgentTurn emits a
|
|
186
|
+
// heartbeat every ten seconds. Keep delivery bounded independently of the
|
|
187
|
+
// heartbeat timeout and local timer cadence.
|
|
188
|
+
maxHeartbeatThrottleInterval: "5s",
|
|
189
|
+
defaultHeartbeatThrottleInterval: "5s",
|
|
190
|
+
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
191
|
+
// after shutdown() stops polling, in-flight activities get this long to
|
|
192
|
+
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
|
|
193
|
+
// which triggers agent-turn's same-turn recovery checkpoint instead of a
|
|
194
|
+
// heartbeat-timeout worker_death. Short on purpose: a long grace here
|
|
195
|
+
// only delays the checkpoint window long turns actually need.
|
|
196
|
+
shutdownGraceTime: "5s",
|
|
197
|
+
// Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
|
|
198
|
+
// wedged checkpoint force-stops here, on our terms, rather than riding
|
|
199
|
+
// into the kubelet's SIGKILL mid-DB-write.
|
|
200
|
+
shutdownForceTime: "100s",
|
|
201
|
+
});
|
|
202
|
+
return { worker, connection };
|
|
94
203
|
},
|
|
204
|
+
(connection) => connection.close(),
|
|
95
205
|
);
|
|
96
|
-
const activities =
|
|
97
|
-
options.activities ??
|
|
98
|
-
(options.role === "control" ? createControlActivities : createTurnActivities)({
|
|
99
|
-
...options.activityDependencies,
|
|
100
|
-
settings,
|
|
101
|
-
observability,
|
|
102
|
-
});
|
|
103
|
-
const worker = await Worker.create({
|
|
104
|
-
connection,
|
|
105
|
-
namespace: settings.temporalNamespace,
|
|
106
|
-
taskQueue:
|
|
107
|
-
options.role === "control"
|
|
108
|
-
? settings.temporalTaskQueue
|
|
109
|
-
: turnTaskQueue(settings.temporalTaskQueue),
|
|
110
|
-
...(options.role === "control"
|
|
111
|
-
? {
|
|
112
|
-
workflowsPath:
|
|
113
|
-
options.workflowsPath ?? new URL("../src/workflows.ts", import.meta.url).pathname,
|
|
114
|
-
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
115
|
-
}
|
|
116
|
-
: {}),
|
|
117
|
-
activities,
|
|
118
|
-
maxConcurrentActivityTaskExecutions:
|
|
119
|
-
options.role === "turn"
|
|
120
|
-
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
121
|
-
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
122
|
-
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
123
|
-
// after shutdown() stops polling, in-flight activities get this long to
|
|
124
|
-
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
|
|
125
|
-
// which triggers agent-turn's same-turn recovery checkpoint instead of a
|
|
126
|
-
// heartbeat-timeout worker_death. Short on purpose: a long grace here
|
|
127
|
-
// only delays the checkpoint window long turns actually need.
|
|
128
|
-
shutdownGraceTime: "5s",
|
|
129
|
-
// Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
|
|
130
|
-
// wedged checkpoint force-stops here, on our terms, rather than riding
|
|
131
|
-
// into the kubelet's SIGKILL mid-DB-write.
|
|
132
|
-
shutdownForceTime: "100s",
|
|
133
|
-
});
|
|
134
|
-
return { worker, connection };
|
|
135
206
|
}
|
|
136
207
|
|
|
137
208
|
// A signalWithStart capability so a worker activity can wake a PARENT
|
|
@@ -144,11 +215,12 @@ export async function createWorkerWorkflowSignaler(
|
|
|
144
215
|
db: Database,
|
|
145
216
|
): Promise<{
|
|
146
217
|
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
218
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
|
|
147
219
|
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
148
220
|
check: () => Promise<void>;
|
|
149
221
|
close: () => Promise<void>;
|
|
150
222
|
}> {
|
|
151
|
-
const connection = await Connection.connect(
|
|
223
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
152
224
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
153
225
|
return {
|
|
154
226
|
wakeSessionWorkflow: async ({
|
|
@@ -185,6 +257,25 @@ export async function createWorkerWorkflowSignaler(
|
|
|
185
257
|
wakeRevision,
|
|
186
258
|
});
|
|
187
259
|
},
|
|
260
|
+
signalSessionAttemptQuiesced: async (proof) => {
|
|
261
|
+
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
262
|
+
taskQueue: settings.temporalTaskQueue,
|
|
263
|
+
workflowId: proof.workflowId,
|
|
264
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
265
|
+
args: [
|
|
266
|
+
{
|
|
267
|
+
accountId: proof.accountId,
|
|
268
|
+
workspaceId: proof.workspaceId,
|
|
269
|
+
sessionId: proof.sessionId,
|
|
270
|
+
},
|
|
271
|
+
],
|
|
272
|
+
signal: "sessionAttemptQuiesced",
|
|
273
|
+
signalArgs: [proof],
|
|
274
|
+
});
|
|
275
|
+
// No wake-outbox row exists yet: the direct receipt transaction failed.
|
|
276
|
+
// The signalled workflow's DB-only control activity owns committing the
|
|
277
|
+
// receipt and its exact wake revision atomically.
|
|
278
|
+
},
|
|
188
279
|
signalCodexCapacityWorkflow: async ({
|
|
189
280
|
accountId,
|
|
190
281
|
workspaceId,
|
|
@@ -234,7 +325,7 @@ export async function registerSandboxReaperSchedule(
|
|
|
234
325
|
settings: Settings,
|
|
235
326
|
observability: Observability,
|
|
236
327
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
237
|
-
const connection = await Connection.connect(
|
|
328
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
238
329
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
239
330
|
try {
|
|
240
331
|
await temporal.schedule.create({
|
|
@@ -299,7 +390,7 @@ export async function registerFileUploadReaperSchedule(
|
|
|
299
390
|
settings: Settings,
|
|
300
391
|
observability: Observability,
|
|
301
392
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
302
|
-
const connection = await Connection.connect(
|
|
393
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
303
394
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
304
395
|
try {
|
|
305
396
|
await temporal.schedule.create({
|
|
@@ -343,7 +434,7 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
343
434
|
settings: Settings,
|
|
344
435
|
observability: Observability,
|
|
345
436
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
346
|
-
const connection = await Connection.connect(
|
|
437
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
347
438
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
348
439
|
try {
|
|
349
440
|
await temporal.schedule.create({
|
|
@@ -378,6 +469,215 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
378
469
|
}
|
|
379
470
|
}
|
|
380
471
|
|
|
472
|
+
export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
|
|
473
|
+
activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
|
|
474
|
+
/**
|
|
475
|
+
* `role-default` registers OpenGeni's internal maintenance schedules on a
|
|
476
|
+
* control worker only. These are engine maintenance schedules, not a host's
|
|
477
|
+
* product-level scheduled-agent jobs. Use `none` when another control worker
|
|
478
|
+
* in the same deployment owns them.
|
|
479
|
+
*/
|
|
480
|
+
internalSchedules?: "role-default" | "none";
|
|
481
|
+
/** Set false only when the host exposes equivalent lifecycle endpoints itself. */
|
|
482
|
+
http?: false | { readinessTimeoutMs?: number };
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
export type OpenGeniWorkerService = {
|
|
486
|
+
readonly role: OpenGeniWorkerRole;
|
|
487
|
+
readonly worker: Worker;
|
|
488
|
+
readonly connection: NativeConnection;
|
|
489
|
+
state(): WorkerLifecycleState;
|
|
490
|
+
run(): Promise<void>;
|
|
491
|
+
drain(reason?: string): void;
|
|
492
|
+
close(): Promise<void>;
|
|
493
|
+
};
|
|
494
|
+
|
|
495
|
+
export function workerOwnsInternalSchedules(
|
|
496
|
+
role: OpenGeniWorkerRole,
|
|
497
|
+
policy: OpenGeniWorkerServiceOptions["internalSchedules"] = "role-default",
|
|
498
|
+
): boolean {
|
|
499
|
+
return role === "control" && policy !== "none";
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Construct one role-specific worker process around the lower-level Temporal
|
|
504
|
+
* worker factory. The service owns every Temporal client and HTTP listener it
|
|
505
|
+
* creates. The embedding host retains ownership of its injected DB and EventBus
|
|
506
|
+
* handles and closes those only after this service has drained.
|
|
507
|
+
*/
|
|
508
|
+
export async function createOpenGeniWorkerService(
|
|
509
|
+
options: OpenGeniWorkerServiceOptions,
|
|
510
|
+
): Promise<OpenGeniWorkerService> {
|
|
511
|
+
const settings = options.settings ?? getSettings();
|
|
512
|
+
const observability =
|
|
513
|
+
options.activityDependencies.observability ??
|
|
514
|
+
createObservability(settings, { component: `worker-${options.role}` });
|
|
515
|
+
const retryOptions = startupRetryOptions(settings);
|
|
516
|
+
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
517
|
+
logStartupDependencyRetry(observability, event);
|
|
518
|
+
let lifecycle: WorkerServiceLifecycle | undefined;
|
|
519
|
+
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
520
|
+
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
521
|
+
const schedules: Array<{ close: () => Promise<void> }> = [];
|
|
522
|
+
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
523
|
+
|
|
524
|
+
try {
|
|
525
|
+
const needsSignaler =
|
|
526
|
+
!options.activityDependencies.wakeSessionWorkflow ||
|
|
527
|
+
!options.activityDependencies.signalSessionAttemptQuiesced ||
|
|
528
|
+
!options.activityDependencies.signalCodexCapacityWorkflow;
|
|
529
|
+
if (needsSignaler) {
|
|
530
|
+
signaler = await retryStartupDependency(
|
|
531
|
+
"Temporal client",
|
|
532
|
+
() => createWorkerWorkflowSignaler(settings, options.activityDependencies.db),
|
|
533
|
+
{ ...retryOptions, onRetry },
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
const wakeSessionWorkflow =
|
|
537
|
+
options.activityDependencies.wakeSessionWorkflow ?? signaler?.wakeSessionWorkflow;
|
|
538
|
+
const signalSessionAttemptQuiesced =
|
|
539
|
+
options.activityDependencies.signalSessionAttemptQuiesced ??
|
|
540
|
+
signaler?.signalSessionAttemptQuiesced;
|
|
541
|
+
const signalCodexCapacityWorkflow =
|
|
542
|
+
options.activityDependencies.signalCodexCapacityWorkflow ??
|
|
543
|
+
signaler?.signalCodexCapacityWorkflow;
|
|
544
|
+
if (!wakeSessionWorkflow || !signalSessionAttemptQuiesced || !signalCodexCapacityWorkflow) {
|
|
545
|
+
throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
|
|
546
|
+
}
|
|
547
|
+
workerBundle = await createOpenGeniWorker({
|
|
548
|
+
role: options.role,
|
|
549
|
+
settings,
|
|
550
|
+
...(options.activities ? { activities: options.activities } : {}),
|
|
551
|
+
...(options.workflowBundle ? { workflowBundle: options.workflowBundle } : {}),
|
|
552
|
+
activityDependencies: {
|
|
553
|
+
...options.activityDependencies,
|
|
554
|
+
settings,
|
|
555
|
+
observability,
|
|
556
|
+
wakeSessionWorkflow,
|
|
557
|
+
signalSessionAttemptQuiesced,
|
|
558
|
+
signalCodexCapacityWorkflow,
|
|
559
|
+
},
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
563
|
+
schedules.push(
|
|
564
|
+
await retryStartupDependency(
|
|
565
|
+
"Temporal schedule (sandbox reaper)",
|
|
566
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
567
|
+
{ ...retryOptions, onRetry },
|
|
568
|
+
),
|
|
569
|
+
);
|
|
570
|
+
schedules.push(
|
|
571
|
+
await retryStartupDependency(
|
|
572
|
+
"Temporal schedule (file upload reaper)",
|
|
573
|
+
() => registerFileUploadReaperSchedule(settings, observability),
|
|
574
|
+
{ ...retryOptions, onRetry },
|
|
575
|
+
),
|
|
576
|
+
);
|
|
577
|
+
schedules.push(
|
|
578
|
+
await retryStartupDependency(
|
|
579
|
+
"Temporal schedule (session-workflow wake dispatcher)",
|
|
580
|
+
() => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
|
|
581
|
+
{ ...retryOptions, onRetry },
|
|
582
|
+
),
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
if (options.http !== false) {
|
|
587
|
+
httpServer = startWorkerHttpServer({
|
|
588
|
+
settings,
|
|
589
|
+
observability,
|
|
590
|
+
checks: {
|
|
591
|
+
db: dbReadyCheck(options.activityDependencies.db),
|
|
592
|
+
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
593
|
+
temporal: temporalReadyCheck(workerBundle.connection),
|
|
594
|
+
},
|
|
595
|
+
...(options.http?.readinessTimeoutMs ? { timeoutMs: options.http.readinessTimeoutMs } : {}),
|
|
596
|
+
lifecycle: { role: options.role, state: () => lifecycle?.state() ?? "starting" },
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
} catch (error) {
|
|
600
|
+
httpServer?.stop(true);
|
|
601
|
+
await Promise.allSettled([
|
|
602
|
+
workerBundle?.connection.close(),
|
|
603
|
+
signaler?.close(),
|
|
604
|
+
...schedules.map((schedule) => schedule.close()),
|
|
605
|
+
]);
|
|
606
|
+
throw error;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
const activeWorkerBundle = workerBundle;
|
|
610
|
+
const activeSignaler = signaler;
|
|
611
|
+
if (!activeWorkerBundle) {
|
|
612
|
+
throw new Error("OpenGeni worker service initialization did not complete");
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
lifecycle = createWorkerServiceLifecycle({
|
|
616
|
+
role: options.role,
|
|
617
|
+
worker: activeWorkerBundle.worker,
|
|
618
|
+
observability,
|
|
619
|
+
closeOwnedResources: async () => {
|
|
620
|
+
httpServer?.stop(true);
|
|
621
|
+
await Promise.allSettled([
|
|
622
|
+
activeWorkerBundle.connection.close(),
|
|
623
|
+
activeSignaler?.close(),
|
|
624
|
+
...schedules.map((schedule) => schedule.close()),
|
|
625
|
+
]);
|
|
626
|
+
},
|
|
627
|
+
onReady: () => {
|
|
628
|
+
observability.info("OpenGeni worker listening", {
|
|
629
|
+
role: options.role,
|
|
630
|
+
temporalTaskQueue:
|
|
631
|
+
options.role === "control"
|
|
632
|
+
? settings.temporalTaskQueue
|
|
633
|
+
: turnTaskQueue(settings.temporalTaskQueue),
|
|
634
|
+
maxConcurrentActivityTaskExecutions:
|
|
635
|
+
options.role === "turn"
|
|
636
|
+
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
637
|
+
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
638
|
+
maxConcurrentWorkflowTaskExecutions:
|
|
639
|
+
options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
640
|
+
httpPort: options.http === false ? null : settings.workerHttpPort,
|
|
641
|
+
});
|
|
642
|
+
},
|
|
643
|
+
});
|
|
644
|
+
|
|
645
|
+
return {
|
|
646
|
+
role: options.role,
|
|
647
|
+
worker: activeWorkerBundle.worker,
|
|
648
|
+
connection: activeWorkerBundle.connection,
|
|
649
|
+
state: lifecycle.state,
|
|
650
|
+
run: lifecycle.run,
|
|
651
|
+
drain: lifecycle.drain,
|
|
652
|
+
close: lifecycle.close,
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
export type RunOpenGeniWorkerOptions = OpenGeniWorkerServiceOptions & {
|
|
657
|
+
/** Defaults to SIGTERM and SIGINT. Pass false when the host owns process signals. */
|
|
658
|
+
shutdownSignals?: false | ReadonlyArray<"SIGTERM" | "SIGINT">;
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
/** Start, drain, and close one embedded worker process. */
|
|
662
|
+
export async function runOpenGeniWorker(options: RunOpenGeniWorkerOptions): Promise<void> {
|
|
663
|
+
const service = await createOpenGeniWorkerService(options);
|
|
664
|
+
const signals =
|
|
665
|
+
options.shutdownSignals === false ? [] : (options.shutdownSignals ?? ["SIGTERM", "SIGINT"]);
|
|
666
|
+
const handlers = signals.map((signal) => {
|
|
667
|
+
const handler = () => service.drain(signal);
|
|
668
|
+
process.on(signal, handler);
|
|
669
|
+
return { signal, handler };
|
|
670
|
+
});
|
|
671
|
+
try {
|
|
672
|
+
await service.run();
|
|
673
|
+
} finally {
|
|
674
|
+
for (const { signal, handler } of handlers) {
|
|
675
|
+
process.off(signal, handler);
|
|
676
|
+
}
|
|
677
|
+
await service.close();
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
381
681
|
export async function startWorker() {
|
|
382
682
|
const role = process.env.OPENGENI_WORKER_ROLE;
|
|
383
683
|
if (role !== "control" && role !== "turn") {
|
|
@@ -395,16 +695,6 @@ export async function startWorker() {
|
|
|
395
695
|
});
|
|
396
696
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
397
697
|
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
398
|
-
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
399
|
-
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
400
|
-
let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
|
|
401
|
-
let fileUploadReaperSchedule:
|
|
402
|
-
| Awaited<ReturnType<typeof registerFileUploadReaperSchedule>>
|
|
403
|
-
| undefined;
|
|
404
|
-
let workflowWakeDispatcherSchedule:
|
|
405
|
-
| Awaited<ReturnType<typeof registerSessionWorkflowWakeDispatcherSchedule>>
|
|
406
|
-
| undefined;
|
|
407
|
-
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
408
698
|
try {
|
|
409
699
|
bus = await retryStartupDependency(
|
|
410
700
|
"NATS",
|
|
@@ -418,103 +708,17 @@ export async function startWorker() {
|
|
|
418
708
|
),
|
|
419
709
|
{ ...retryOptions, onRetry },
|
|
420
710
|
);
|
|
421
|
-
|
|
422
|
-
"Temporal client",
|
|
423
|
-
() => createWorkerWorkflowSignaler(settings, dbClient.db),
|
|
424
|
-
{ ...retryOptions, onRetry },
|
|
425
|
-
);
|
|
426
|
-
workerBundle = await createOpenGeniWorker({
|
|
711
|
+
await runOpenGeniWorker({
|
|
427
712
|
role,
|
|
428
713
|
settings,
|
|
429
714
|
activityDependencies: {
|
|
430
715
|
observability,
|
|
431
|
-
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
432
|
-
signalCodexCapacityWorkflow: signaler.signalCodexCapacityWorkflow,
|
|
433
716
|
db: dbClient.db,
|
|
434
717
|
bus,
|
|
435
718
|
},
|
|
436
719
|
});
|
|
437
|
-
httpServer = startWorkerHttpServer({
|
|
438
|
-
settings,
|
|
439
|
-
observability,
|
|
440
|
-
checks: {
|
|
441
|
-
db: dbReadyCheck(dbClient.db),
|
|
442
|
-
nats: natsReadyCheck(bus),
|
|
443
|
-
temporal: signaler.check,
|
|
444
|
-
},
|
|
445
|
-
});
|
|
446
|
-
// Control workers alone own global schedules. Turn workers poll only the
|
|
447
|
-
// bounded inference activity queue and can never duplicate schedule setup.
|
|
448
|
-
if (role === "control") {
|
|
449
|
-
reaperSchedule = await retryStartupDependency(
|
|
450
|
-
"Temporal schedule (sandbox reaper)",
|
|
451
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
452
|
-
{ ...retryOptions, onRetry },
|
|
453
|
-
);
|
|
454
|
-
fileUploadReaperSchedule = await retryStartupDependency(
|
|
455
|
-
"Temporal schedule (file upload reaper)",
|
|
456
|
-
() => registerFileUploadReaperSchedule(settings, observability),
|
|
457
|
-
{ ...retryOptions, onRetry },
|
|
458
|
-
);
|
|
459
|
-
workflowWakeDispatcherSchedule = await retryStartupDependency(
|
|
460
|
-
"Temporal schedule (session-workflow wake dispatcher)",
|
|
461
|
-
() => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
|
|
462
|
-
{ ...retryOptions, onRetry },
|
|
463
|
-
);
|
|
464
|
-
}
|
|
465
|
-
observability.info("OpenGeni worker listening", {
|
|
466
|
-
role,
|
|
467
|
-
temporalTaskQueue:
|
|
468
|
-
role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
|
|
469
|
-
maxConcurrentActivityTaskExecutions:
|
|
470
|
-
role === "turn"
|
|
471
|
-
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
472
|
-
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
473
|
-
maxConcurrentWorkflowTaskExecutions:
|
|
474
|
-
role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
475
|
-
httpPort: settings.workerHttpPort,
|
|
476
|
-
});
|
|
477
|
-
// GRACEFUL DEPLOY SHUTDOWN — the missing link that made every deploy a
|
|
478
|
-
// worker_death. The worker is the container's MAIN process (PID 1), and a
|
|
479
|
-
// PID-1 process with no explicit handler IGNORES SIGTERM: the pod sat
|
|
480
|
-
// through the entire terminationGracePeriodSeconds doing nothing, then the
|
|
481
|
-
// kubelet SIGKILLed it — so agent-turn's worker-shutdown checkpoint path
|
|
482
|
-
// (same-turn recovery + run-state save) never once ran in production
|
|
483
|
-
// (forensics 2026-07-06: every observed recovery was
|
|
484
|
-
// reason=worker_death). This handler starts the drain the moment k8s asks:
|
|
485
|
-
// shutdown() stops polling, then (per shutdownGraceTime/ForceTime above)
|
|
486
|
-
// cancels in-flight activities with WORKER_SHUTDOWN so long turns
|
|
487
|
-
// checkpoint INSIDE the grace window. run() resolves once
|
|
488
|
-
// drained and the finally below closes connections cleanly.
|
|
489
|
-
let shutdownRequested = false;
|
|
490
|
-
const requestShutdown = (signal: string) => {
|
|
491
|
-
if (shutdownRequested) {
|
|
492
|
-
return;
|
|
493
|
-
}
|
|
494
|
-
shutdownRequested = true;
|
|
495
|
-
observability.info("OpenGeni worker draining (graceful shutdown)", { signal });
|
|
496
|
-
try {
|
|
497
|
-
workerBundle!.worker.shutdown();
|
|
498
|
-
} catch (error) {
|
|
499
|
-
observability.warn("worker shutdown request failed", {
|
|
500
|
-
error: error instanceof Error ? error.message : String(error),
|
|
501
|
-
});
|
|
502
|
-
}
|
|
503
|
-
};
|
|
504
|
-
process.on("SIGTERM", () => requestShutdown("SIGTERM"));
|
|
505
|
-
process.on("SIGINT", () => requestShutdown("SIGINT"));
|
|
506
|
-
await workerBundle.worker.run();
|
|
507
720
|
} finally {
|
|
508
|
-
|
|
509
|
-
await Promise.allSettled([
|
|
510
|
-
workerBundle?.connection.close(),
|
|
511
|
-
signaler?.close(),
|
|
512
|
-
reaperSchedule?.close(),
|
|
513
|
-
fileUploadReaperSchedule?.close(),
|
|
514
|
-
workflowWakeDispatcherSchedule?.close(),
|
|
515
|
-
bus?.close(),
|
|
516
|
-
dbClient.close(),
|
|
517
|
-
]);
|
|
721
|
+
await Promise.allSettled([bus?.close(), dbClient.close()]);
|
|
518
722
|
}
|
|
519
723
|
}
|
|
520
724
|
|