@opengeni/worker-bundle 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +184 -68
- package/dist/index.js +7099 -1643
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4332 -1068
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +14 -2
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +216 -42
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +165 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +106 -36
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflows/activities.ts +65 -8
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -192
- package/src/workflows/workflow-wake.ts +13 -0
- package/src/workflows.ts +9 -7
package/src/index.ts
CHANGED
|
@@ -1,14 +1,45 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
dbSearchPath,
|
|
3
|
+
getSettings,
|
|
4
|
+
resolveNatsControlPlaneAuth,
|
|
5
|
+
retryStartupDependency,
|
|
6
|
+
startupRetryOptions,
|
|
7
|
+
type Settings,
|
|
8
|
+
} from "@opengeni/config";
|
|
9
|
+
import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
|
|
3
10
|
import { createNatsEventBus } from "@opengeni/events";
|
|
4
|
-
import {
|
|
5
|
-
|
|
11
|
+
import {
|
|
12
|
+
createObservability,
|
|
13
|
+
logStartupDependencyRetry,
|
|
14
|
+
type Observability,
|
|
15
|
+
} from "@opengeni/observability";
|
|
16
|
+
import {
|
|
17
|
+
Connection,
|
|
18
|
+
ScheduleAlreadyRunning,
|
|
19
|
+
ScheduleOverlapPolicy,
|
|
20
|
+
Client as TemporalClient,
|
|
21
|
+
} from "@temporalio/client";
|
|
6
22
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
7
23
|
import { ensureModalRegistryImage } from "@opengeni/runtime";
|
|
8
|
-
import {
|
|
9
|
-
|
|
24
|
+
import {
|
|
25
|
+
createControlActivities,
|
|
26
|
+
createTurnActivities,
|
|
27
|
+
type ActivityDependencies,
|
|
28
|
+
} from "./activities";
|
|
29
|
+
import type { SignalCodexCapacityWorkflow, WakeSessionWorkflowSignal } from "./activities/types";
|
|
30
|
+
import { turnTaskQueue } from "./workflows/activities";
|
|
10
31
|
import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
|
|
11
32
|
import { observabilityEventLogger } from "./observability-metrics";
|
|
33
|
+
import {
|
|
34
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
35
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
36
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
37
|
+
} from "@opengeni/core";
|
|
38
|
+
import {
|
|
39
|
+
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
40
|
+
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
41
|
+
TURN_WORKER_MAX_CONCURRENT_TURNS,
|
|
42
|
+
} from "./concurrency";
|
|
12
43
|
|
|
13
44
|
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
14
45
|
// create() is idempotent across every worker in the pool: the first worker to
|
|
@@ -16,10 +47,14 @@ import { observabilityEventLogger } from "./observability-metrics";
|
|
|
16
47
|
// no-op — so the Schedule is registered EXACTLY ONCE per deployment regardless
|
|
17
48
|
// of replica count.
|
|
18
49
|
const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
50
|
+
export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
51
|
+
export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
|
|
52
|
+
export type OpenGeniWorkerRole = "control" | "turn";
|
|
19
53
|
|
|
20
54
|
export type WorkerOptions = {
|
|
55
|
+
role: OpenGeniWorkerRole;
|
|
21
56
|
settings?: Settings;
|
|
22
|
-
activities?: ReturnType<typeof
|
|
57
|
+
activities?: ReturnType<typeof createControlActivities> | ReturnType<typeof createTurnActivities>;
|
|
23
58
|
activityDependencies?: ActivityDependencies;
|
|
24
59
|
// Embedded hosts install @opengeni/worker-bundle under node_modules, where
|
|
25
60
|
// Temporal's workflow webpack refuses to transpile TS. They relocate
|
|
@@ -28,24 +63,28 @@ export type WorkerOptions = {
|
|
|
28
63
|
workflowsPath?: string;
|
|
29
64
|
};
|
|
30
65
|
|
|
31
|
-
export async function createOpenGeniWorker(options: WorkerOptions
|
|
66
|
+
export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
32
67
|
worker: Worker;
|
|
33
68
|
connection: NativeConnection;
|
|
34
69
|
}> {
|
|
35
70
|
const settings = options.settings ?? getSettings();
|
|
36
|
-
const observability =
|
|
71
|
+
const observability =
|
|
72
|
+
options.activityDependencies?.observability ??
|
|
73
|
+
createObservability(settings, { component: `worker-${options.role}` });
|
|
37
74
|
// Pre-resolve a PRIVATE-registry sandbox image before any turn creates a box.
|
|
38
75
|
// No-op unless OPENGENI_MODAL_IMAGE_REGISTRY_SECRET + OPENGENI_MODAL_IMAGE_REF are
|
|
39
76
|
// both set (so non-modal / public-image deployments are byte-unchanged and never
|
|
40
77
|
// load the modal SDK here). Memoized in the provider, so this runs once per process.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
78
|
+
if (options.role === "turn") {
|
|
79
|
+
await retryStartupDependency(
|
|
80
|
+
"Modal private-registry image",
|
|
81
|
+
() => ensureModalRegistryImage(settings),
|
|
82
|
+
{
|
|
83
|
+
...startupRetryOptions(settings),
|
|
84
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
85
|
+
},
|
|
86
|
+
);
|
|
87
|
+
}
|
|
49
88
|
const connection = await retryStartupDependency(
|
|
50
89
|
"Temporal",
|
|
51
90
|
() => NativeConnection.connect({ address: settings.temporalHost }),
|
|
@@ -54,17 +93,43 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
54
93
|
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
55
94
|
},
|
|
56
95
|
);
|
|
57
|
-
const activities =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
96
|
+
const activities =
|
|
97
|
+
options.activities ??
|
|
98
|
+
(options.role === "control" ? createControlActivities : createTurnActivities)({
|
|
99
|
+
...options.activityDependencies,
|
|
100
|
+
settings,
|
|
101
|
+
observability,
|
|
102
|
+
});
|
|
62
103
|
const worker = await Worker.create({
|
|
63
104
|
connection,
|
|
64
105
|
namespace: settings.temporalNamespace,
|
|
65
|
-
taskQueue:
|
|
66
|
-
|
|
106
|
+
taskQueue:
|
|
107
|
+
options.role === "control"
|
|
108
|
+
? settings.temporalTaskQueue
|
|
109
|
+
: turnTaskQueue(settings.temporalTaskQueue),
|
|
110
|
+
...(options.role === "control"
|
|
111
|
+
? {
|
|
112
|
+
workflowsPath:
|
|
113
|
+
options.workflowsPath ?? new URL("../src/workflows.ts", import.meta.url).pathname,
|
|
114
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
115
|
+
}
|
|
116
|
+
: {}),
|
|
67
117
|
activities,
|
|
118
|
+
maxConcurrentActivityTaskExecutions:
|
|
119
|
+
options.role === "turn"
|
|
120
|
+
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
121
|
+
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
122
|
+
// GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
|
|
123
|
+
// after shutdown() stops polling, in-flight activities get this long to
|
|
124
|
+
// finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
|
|
125
|
+
// which triggers agent-turn's same-turn recovery checkpoint instead of a
|
|
126
|
+
// heartbeat-timeout worker_death. Short on purpose: a long grace here
|
|
127
|
+
// only delays the checkpoint window long turns actually need.
|
|
128
|
+
shutdownGraceTime: "5s",
|
|
129
|
+
// Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
|
|
130
|
+
// wedged checkpoint force-stops here, on our terms, rather than riding
|
|
131
|
+
// into the kubelet's SIGKILL mid-DB-write.
|
|
132
|
+
shutdownForceTime: "100s",
|
|
68
133
|
});
|
|
69
134
|
return { worker, connection };
|
|
70
135
|
}
|
|
@@ -74,22 +139,70 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
74
139
|
// idled and let its run finish, so a plain signal would not start one).
|
|
75
140
|
// Separate from the worker's NativeConnection: the @temporalio/client
|
|
76
141
|
// Connection is what exposes workflow.signalWithStart.
|
|
77
|
-
export async function createWorkerWorkflowSignaler(
|
|
142
|
+
export async function createWorkerWorkflowSignaler(
|
|
143
|
+
settings: Settings,
|
|
144
|
+
db: Database,
|
|
145
|
+
): Promise<{
|
|
78
146
|
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
147
|
+
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
79
148
|
check: () => Promise<void>;
|
|
80
149
|
close: () => Promise<void>;
|
|
81
150
|
}> {
|
|
82
151
|
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
83
152
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
84
153
|
return {
|
|
85
|
-
wakeSessionWorkflow: async ({
|
|
154
|
+
wakeSessionWorkflow: async ({
|
|
155
|
+
accountId,
|
|
156
|
+
workspaceId,
|
|
157
|
+
sessionId,
|
|
158
|
+
workflowId,
|
|
159
|
+
wakeRevision,
|
|
160
|
+
interruptionRequested,
|
|
161
|
+
}) => {
|
|
162
|
+
if (interruptionRequested) {
|
|
163
|
+
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
164
|
+
taskQueue: settings.temporalTaskQueue,
|
|
165
|
+
workflowId,
|
|
166
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
167
|
+
args: [{ accountId, workspaceId, sessionId }],
|
|
168
|
+
signal: "sessionControl",
|
|
169
|
+
signalArgs: [],
|
|
170
|
+
});
|
|
171
|
+
} else {
|
|
172
|
+
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
173
|
+
taskQueue: settings.temporalTaskQueue,
|
|
174
|
+
workflowId,
|
|
175
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
176
|
+
args: [{ accountId, workspaceId, sessionId }],
|
|
177
|
+
signal: "queueChanged",
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
await markSessionWorkflowWakeDelivered(db, {
|
|
181
|
+
accountId,
|
|
182
|
+
workspaceId,
|
|
183
|
+
sessionId,
|
|
184
|
+
temporalWorkflowId: workflowId,
|
|
185
|
+
wakeRevision,
|
|
186
|
+
});
|
|
187
|
+
},
|
|
188
|
+
signalCodexCapacityWorkflow: async ({
|
|
189
|
+
accountId,
|
|
190
|
+
workspaceId,
|
|
191
|
+
sessionId,
|
|
192
|
+
workflowId,
|
|
193
|
+
wakeRevision,
|
|
194
|
+
}) => {
|
|
86
195
|
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
87
196
|
taskQueue: settings.temporalTaskQueue,
|
|
88
197
|
workflowId,
|
|
89
198
|
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
90
199
|
args: [{ accountId, workspaceId, sessionId }],
|
|
91
|
-
signal: "
|
|
200
|
+
signal: "codexCapacityChanged",
|
|
201
|
+
signalArgs: [wakeRevision],
|
|
92
202
|
});
|
|
203
|
+
// A typed capacity signal cannot acknowledge the generic outbox row:
|
|
204
|
+
// another producer may have advanced it with a Pause/Steer that requires
|
|
205
|
+
// sessionControl. The global dispatcher owns that acknowledgement.
|
|
93
206
|
},
|
|
94
207
|
check: async () => {
|
|
95
208
|
await connection.workflowService.getSystemInfo({});
|
|
@@ -102,9 +215,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
|
|
|
102
215
|
|
|
103
216
|
/**
|
|
104
217
|
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
105
|
-
* driver — P1.3 / OD-3)
|
|
106
|
-
*
|
|
107
|
-
* terminates
|
|
218
|
+
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
219
|
+
* sandbox ownership off the activity performs only bounded DB outbox repair;
|
|
220
|
+
* it never reads/terminates sandbox leases.
|
|
108
221
|
*
|
|
109
222
|
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
110
223
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
@@ -121,9 +234,6 @@ export async function registerSandboxReaperSchedule(
|
|
|
121
234
|
settings: Settings,
|
|
122
235
|
observability: Observability,
|
|
123
236
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
124
|
-
if (!settings.sandboxOwnershipEnabled) {
|
|
125
|
-
return { registered: false, close: async () => {} };
|
|
126
|
-
}
|
|
127
237
|
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
128
238
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
129
239
|
try {
|
|
@@ -152,7 +262,12 @@ export async function registerSandboxReaperSchedule(
|
|
|
152
262
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
153
263
|
reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs,
|
|
154
264
|
});
|
|
155
|
-
return {
|
|
265
|
+
return {
|
|
266
|
+
registered: true,
|
|
267
|
+
close: async () => {
|
|
268
|
+
await connection.close();
|
|
269
|
+
},
|
|
270
|
+
};
|
|
156
271
|
} catch (error) {
|
|
157
272
|
if (error instanceof ScheduleAlreadyRunning) {
|
|
158
273
|
// Another worker in the pool already created it. The Schedule exists
|
|
@@ -162,7 +277,101 @@ export async function registerSandboxReaperSchedule(
|
|
|
162
277
|
observability.info("Global sandbox-lease reaper Schedule already registered", {
|
|
163
278
|
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
164
279
|
});
|
|
165
|
-
return {
|
|
280
|
+
return {
|
|
281
|
+
registered: false,
|
|
282
|
+
close: async () => {
|
|
283
|
+
await connection.close();
|
|
284
|
+
},
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
await connection.close().catch(() => undefined);
|
|
288
|
+
throw error;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Register the one provider-neutral expired direct-upload cleanup Schedule.
|
|
294
|
+
* Unlike sandbox GC this is always registered: file uploads can be enabled in
|
|
295
|
+
* deployments where sandbox ownership is disabled. The activity is a cheap
|
|
296
|
+
* no-op when object storage is not configured.
|
|
297
|
+
*/
|
|
298
|
+
export async function registerFileUploadReaperSchedule(
|
|
299
|
+
settings: Settings,
|
|
300
|
+
observability: Observability,
|
|
301
|
+
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
302
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
303
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
304
|
+
try {
|
|
305
|
+
await temporal.schedule.create({
|
|
306
|
+
scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
307
|
+
spec: { intervals: [{ every: FILE_UPLOAD_REAPER_PERIOD_MS }] },
|
|
308
|
+
action: {
|
|
309
|
+
type: "startWorkflow",
|
|
310
|
+
workflowType: "fileUploadReaperWorkflow",
|
|
311
|
+
taskQueue: settings.temporalTaskQueue,
|
|
312
|
+
args: [],
|
|
313
|
+
},
|
|
314
|
+
policies: {
|
|
315
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
316
|
+
catchupWindow: "1m",
|
|
317
|
+
pauseOnFailure: false,
|
|
318
|
+
},
|
|
319
|
+
});
|
|
320
|
+
observability.info("Registered the global file-upload reaper Schedule", {
|
|
321
|
+
scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
322
|
+
reaperPeriodMs: FILE_UPLOAD_REAPER_PERIOD_MS,
|
|
323
|
+
});
|
|
324
|
+
return { registered: true, close: async () => connection.close() };
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
327
|
+
observability.info("Global file-upload reaper Schedule already registered", {
|
|
328
|
+
scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
329
|
+
});
|
|
330
|
+
return { registered: false, close: async () => connection.close() };
|
|
331
|
+
}
|
|
332
|
+
await connection.close().catch(() => undefined);
|
|
333
|
+
throw error;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Register the one repair cadence for committed workflow-wake revisions. The
|
|
339
|
+
* activity only reads the transactional outbox and sends revision-scoped
|
|
340
|
+
* signals; it is independent of sandbox ownership and child-agent features.
|
|
341
|
+
*/
|
|
342
|
+
export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
343
|
+
settings: Settings,
|
|
344
|
+
observability: Observability,
|
|
345
|
+
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
346
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
347
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
348
|
+
try {
|
|
349
|
+
await temporal.schedule.create({
|
|
350
|
+
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
351
|
+
spec: { intervals: [{ every: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS }] },
|
|
352
|
+
action: {
|
|
353
|
+
type: "startWorkflow",
|
|
354
|
+
workflowType: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
355
|
+
taskQueue: settings.temporalTaskQueue,
|
|
356
|
+
args: [],
|
|
357
|
+
},
|
|
358
|
+
policies: {
|
|
359
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
360
|
+
catchupWindow: "1m",
|
|
361
|
+
pauseOnFailure: false,
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
observability.info("Registered the session-workflow wake dispatcher Schedule", {
|
|
365
|
+
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
366
|
+
periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
367
|
+
});
|
|
368
|
+
return { registered: true, close: async () => connection.close() };
|
|
369
|
+
} catch (error) {
|
|
370
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
371
|
+
observability.info("Session-workflow wake dispatcher Schedule already registered", {
|
|
372
|
+
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
373
|
+
});
|
|
374
|
+
return { registered: false, close: async () => connection.close() };
|
|
166
375
|
}
|
|
167
376
|
await connection.close().catch(() => undefined);
|
|
168
377
|
throw error;
|
|
@@ -170,10 +379,15 @@ export async function registerSandboxReaperSchedule(
|
|
|
170
379
|
}
|
|
171
380
|
|
|
172
381
|
export async function startWorker() {
|
|
382
|
+
const role = process.env.OPENGENI_WORKER_ROLE;
|
|
383
|
+
if (role !== "control" && role !== "turn") {
|
|
384
|
+
throw new Error("OPENGENI_WORKER_ROLE must be explicitly set to 'control' or 'turn'");
|
|
385
|
+
}
|
|
173
386
|
const settings = getSettings();
|
|
174
|
-
const observability = createObservability(settings, { component:
|
|
387
|
+
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
175
388
|
const retryOptions = startupRetryOptions(settings);
|
|
176
|
-
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
389
|
+
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
390
|
+
logStartupDependencyRetry(observability, event);
|
|
177
391
|
const searchPath = dbSearchPath(settings);
|
|
178
392
|
const dbClient = createDb(settings.databaseUrl, {
|
|
179
393
|
...(searchPath ? { searchPath } : {}),
|
|
@@ -184,6 +398,12 @@ export async function startWorker() {
|
|
|
184
398
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
185
399
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
186
400
|
let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
|
|
401
|
+
let fileUploadReaperSchedule:
|
|
402
|
+
| Awaited<ReturnType<typeof registerFileUploadReaperSchedule>>
|
|
403
|
+
| undefined;
|
|
404
|
+
let workflowWakeDispatcherSchedule:
|
|
405
|
+
| Awaited<ReturnType<typeof registerSessionWorkflowWakeDispatcherSchedule>>
|
|
406
|
+
| undefined;
|
|
187
407
|
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
188
408
|
try {
|
|
189
409
|
bus = await retryStartupDependency(
|
|
@@ -191,21 +411,25 @@ export async function startWorker() {
|
|
|
191
411
|
() =>
|
|
192
412
|
createNatsEventBus(
|
|
193
413
|
settings.natsUrl,
|
|
194
|
-
controlPlaneAuth
|
|
414
|
+
controlPlaneAuth
|
|
415
|
+
? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
|
|
416
|
+
: undefined,
|
|
195
417
|
{ logger: observabilityEventLogger(observability) },
|
|
196
418
|
),
|
|
197
419
|
{ ...retryOptions, onRetry },
|
|
198
420
|
);
|
|
199
421
|
signaler = await retryStartupDependency(
|
|
200
422
|
"Temporal client",
|
|
201
|
-
() => createWorkerWorkflowSignaler(settings),
|
|
423
|
+
() => createWorkerWorkflowSignaler(settings, dbClient.db),
|
|
202
424
|
{ ...retryOptions, onRetry },
|
|
203
425
|
);
|
|
204
426
|
workerBundle = await createOpenGeniWorker({
|
|
427
|
+
role,
|
|
205
428
|
settings,
|
|
206
429
|
activityDependencies: {
|
|
207
430
|
observability,
|
|
208
431
|
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
432
|
+
signalCodexCapacityWorkflow: signaler.signalCodexCapacityWorkflow,
|
|
209
433
|
db: dbClient.db,
|
|
210
434
|
bus,
|
|
211
435
|
},
|
|
@@ -219,19 +443,66 @@ export async function startWorker() {
|
|
|
219
443
|
temporal: signaler.check,
|
|
220
444
|
},
|
|
221
445
|
});
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
446
|
+
// Control workers alone own global schedules. Turn workers poll only the
|
|
447
|
+
// bounded inference activity queue and can never duplicate schedule setup.
|
|
448
|
+
if (role === "control") {
|
|
449
|
+
reaperSchedule = await retryStartupDependency(
|
|
450
|
+
"Temporal schedule (sandbox reaper)",
|
|
451
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
452
|
+
{ ...retryOptions, onRetry },
|
|
453
|
+
);
|
|
454
|
+
fileUploadReaperSchedule = await retryStartupDependency(
|
|
455
|
+
"Temporal schedule (file upload reaper)",
|
|
456
|
+
() => registerFileUploadReaperSchedule(settings, observability),
|
|
457
|
+
{ ...retryOptions, onRetry },
|
|
458
|
+
);
|
|
459
|
+
workflowWakeDispatcherSchedule = await retryStartupDependency(
|
|
460
|
+
"Temporal schedule (session-workflow wake dispatcher)",
|
|
461
|
+
() => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
|
|
462
|
+
{ ...retryOptions, onRetry },
|
|
463
|
+
);
|
|
464
|
+
}
|
|
231
465
|
observability.info("OpenGeni worker listening", {
|
|
232
|
-
|
|
466
|
+
role,
|
|
467
|
+
temporalTaskQueue:
|
|
468
|
+
role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
|
|
469
|
+
maxConcurrentActivityTaskExecutions:
|
|
470
|
+
role === "turn"
|
|
471
|
+
? TURN_WORKER_MAX_CONCURRENT_TURNS
|
|
472
|
+
: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
473
|
+
maxConcurrentWorkflowTaskExecutions:
|
|
474
|
+
role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
|
|
233
475
|
httpPort: settings.workerHttpPort,
|
|
234
476
|
});
|
|
477
|
+
// GRACEFUL DEPLOY SHUTDOWN — the missing link that made every deploy a
|
|
478
|
+
// worker_death. The worker is the container's MAIN process (PID 1), and a
|
|
479
|
+
// PID-1 process with no explicit handler IGNORES SIGTERM: the pod sat
|
|
480
|
+
// through the entire terminationGracePeriodSeconds doing nothing, then the
|
|
481
|
+
// kubelet SIGKILLed it — so agent-turn's worker-shutdown checkpoint path
|
|
482
|
+
// (same-turn recovery + run-state save) never once ran in production
|
|
483
|
+
// (forensics 2026-07-06: every observed recovery was
|
|
484
|
+
// reason=worker_death). This handler starts the drain the moment k8s asks:
|
|
485
|
+
// shutdown() stops polling, then (per shutdownGraceTime/ForceTime above)
|
|
486
|
+
// cancels in-flight activities with WORKER_SHUTDOWN so long turns
|
|
487
|
+
// checkpoint INSIDE the grace window. run() resolves once
|
|
488
|
+
// drained and the finally below closes connections cleanly.
|
|
489
|
+
let shutdownRequested = false;
|
|
490
|
+
const requestShutdown = (signal: string) => {
|
|
491
|
+
if (shutdownRequested) {
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
shutdownRequested = true;
|
|
495
|
+
observability.info("OpenGeni worker draining (graceful shutdown)", { signal });
|
|
496
|
+
try {
|
|
497
|
+
workerBundle!.worker.shutdown();
|
|
498
|
+
} catch (error) {
|
|
499
|
+
observability.warn("worker shutdown request failed", {
|
|
500
|
+
error: error instanceof Error ? error.message : String(error),
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
};
|
|
504
|
+
process.on("SIGTERM", () => requestShutdown("SIGTERM"));
|
|
505
|
+
process.on("SIGINT", () => requestShutdown("SIGINT"));
|
|
235
506
|
await workerBundle.worker.run();
|
|
236
507
|
} finally {
|
|
237
508
|
httpServer?.stop(true);
|
|
@@ -239,6 +510,8 @@ export async function startWorker() {
|
|
|
239
510
|
workerBundle?.connection.close(),
|
|
240
511
|
signaler?.close(),
|
|
241
512
|
reaperSchedule?.close(),
|
|
513
|
+
fileUploadReaperSchedule?.close(),
|
|
514
|
+
workflowWakeDispatcherSchedule?.close(),
|
|
242
515
|
bus?.close(),
|
|
243
516
|
dbClient.close(),
|
|
244
517
|
]);
|