@opengeni/worker-bundle 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7127 -1654
  3. package/dist/index.js.map +1 -1
  4. package/package.json +19 -19
  5. package/src/activities/agent-turn.ts +4405 -1089
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
package/src/index.ts CHANGED
@@ -1,14 +1,45 @@
1
- import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
2
- import { createDb } from "@opengeni/db";
1
+ import {
2
+ dbSearchPath,
3
+ getSettings,
4
+ resolveNatsControlPlaneAuth,
5
+ retryStartupDependency,
6
+ startupRetryOptions,
7
+ type Settings,
8
+ } from "@opengeni/config";
9
+ import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
3
10
  import { createNatsEventBus } from "@opengeni/events";
4
- import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
5
- import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
11
+ import {
12
+ createObservability,
13
+ logStartupDependencyRetry,
14
+ type Observability,
15
+ } from "@opengeni/observability";
16
+ import {
17
+ Connection,
18
+ ScheduleAlreadyRunning,
19
+ ScheduleOverlapPolicy,
20
+ Client as TemporalClient,
21
+ } from "@temporalio/client";
6
22
  import { NativeConnection, Worker } from "@temporalio/worker";
7
23
  import { ensureModalRegistryImage } from "@opengeni/runtime";
8
- import { createActivities, type ActivityDependencies } from "./activities";
9
- import type { WakeSessionWorkflowSignal } from "./activities/types";
24
+ import {
25
+ createControlActivities,
26
+ createTurnActivities,
27
+ type ActivityDependencies,
28
+ } from "./activities";
29
+ import type { SignalCodexCapacityWorkflow, WakeSessionWorkflowSignal } from "./activities/types";
30
+ import { turnTaskQueue } from "./workflows/activities";
10
31
  import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
11
32
  import { observabilityEventLogger } from "./observability-metrics";
33
+ import {
34
+ SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
35
+ SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
36
+ SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
37
+ } from "./workflow-wake-contract";
38
+ import {
39
+ CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
40
+ CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
41
+ TURN_WORKER_MAX_CONCURRENT_TURNS,
42
+ } from "./concurrency";
12
43
 
13
44
  // The deterministic id of the ONE global reaper Schedule. A single id means
14
45
  // create() is idempotent across every worker in the pool: the first worker to
@@ -16,10 +47,14 @@ import { observabilityEventLogger } from "./observability-metrics";
16
47
  // no-op — so the Schedule is registered EXACTLY ONCE per deployment regardless
17
48
  // of replica count.
18
49
  const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
50
+ export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
51
+ export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
52
+ export type OpenGeniWorkerRole = "control" | "turn";
19
53
 
20
54
  export type WorkerOptions = {
55
+ role: OpenGeniWorkerRole;
21
56
  settings?: Settings;
22
- activities?: ReturnType<typeof createActivities>;
57
+ activities?: ReturnType<typeof createControlActivities> | ReturnType<typeof createTurnActivities>;
23
58
  activityDependencies?: ActivityDependencies;
24
59
  // Embedded hosts install @opengeni/worker-bundle under node_modules, where
25
60
  // Temporal's workflow webpack refuses to transpile TS. They relocate
@@ -28,24 +63,28 @@ export type WorkerOptions = {
28
63
  workflowsPath?: string;
29
64
  };
30
65
 
31
- export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise<{
66
+ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
32
67
  worker: Worker;
33
68
  connection: NativeConnection;
34
69
  }> {
35
70
  const settings = options.settings ?? getSettings();
36
- const observability = options.activityDependencies?.observability ?? createObservability(settings, { component: "worker" });
71
+ const observability =
72
+ options.activityDependencies?.observability ??
73
+ createObservability(settings, { component: `worker-${options.role}` });
37
74
  // Pre-resolve a PRIVATE-registry sandbox image before any turn creates a box.
38
75
  // No-op unless OPENGENI_MODAL_IMAGE_REGISTRY_SECRET + OPENGENI_MODAL_IMAGE_REF are
39
76
  // both set (so non-modal / public-image deployments are byte-unchanged and never
40
77
  // load the modal SDK here). Memoized in the provider, so this runs once per process.
41
- await retryStartupDependency(
42
- "Modal private-registry image",
43
- () => ensureModalRegistryImage(settings),
44
- {
45
- ...startupRetryOptions(settings),
46
- onRetry: (event) => logStartupDependencyRetry(observability, event),
47
- },
48
- );
78
+ if (options.role === "turn") {
79
+ await retryStartupDependency(
80
+ "Modal private-registry image",
81
+ () => ensureModalRegistryImage(settings),
82
+ {
83
+ ...startupRetryOptions(settings),
84
+ onRetry: (event) => logStartupDependencyRetry(observability, event),
85
+ },
86
+ );
87
+ }
49
88
  const connection = await retryStartupDependency(
50
89
  "Temporal",
51
90
  () => NativeConnection.connect({ address: settings.temporalHost }),
@@ -54,17 +93,43 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
54
93
  onRetry: (event) => logStartupDependencyRetry(observability, event),
55
94
  },
56
95
  );
57
- const activities = options.activities ?? createActivities({
58
- ...options.activityDependencies,
59
- settings,
60
- observability,
61
- });
96
+ const activities =
97
+ options.activities ??
98
+ (options.role === "control" ? createControlActivities : createTurnActivities)({
99
+ ...options.activityDependencies,
100
+ settings,
101
+ observability,
102
+ });
62
103
  const worker = await Worker.create({
63
104
  connection,
64
105
  namespace: settings.temporalNamespace,
65
- taskQueue: settings.temporalTaskQueue,
66
- workflowsPath: options.workflowsPath ?? new URL("../src/workflows.ts", import.meta.url).pathname,
106
+ taskQueue:
107
+ options.role === "control"
108
+ ? settings.temporalTaskQueue
109
+ : turnTaskQueue(settings.temporalTaskQueue),
110
+ ...(options.role === "control"
111
+ ? {
112
+ workflowsPath:
113
+ options.workflowsPath ?? new URL("../src/workflows.ts", import.meta.url).pathname,
114
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
115
+ }
116
+ : {}),
67
117
  activities,
118
+ maxConcurrentActivityTaskExecutions:
119
+ options.role === "turn"
120
+ ? TURN_WORKER_MAX_CONCURRENT_TURNS
121
+ : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
122
+ // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
123
+ // after shutdown() stops polling, in-flight activities get this long to
124
+ // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
125
+ // which triggers agent-turn's same-turn recovery checkpoint instead of a
126
+ // heartbeat-timeout worker_death. Short on purpose: a long grace here
127
+ // only delays the checkpoint window long turns actually need.
128
+ shutdownGraceTime: "5s",
129
+ // Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
130
+ // wedged checkpoint force-stops here, on our terms, rather than riding
131
+ // into the kubelet's SIGKILL mid-DB-write.
132
+ shutdownForceTime: "100s",
68
133
  });
69
134
  return { worker, connection };
70
135
  }
@@ -74,22 +139,70 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
74
139
  // idled and let its run finish, so a plain signal would not start one).
75
140
  // Separate from the worker's NativeConnection: the @temporalio/client
76
141
  // Connection is what exposes workflow.signalWithStart.
77
- export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
142
+ export async function createWorkerWorkflowSignaler(
143
+ settings: Settings,
144
+ db: Database,
145
+ ): Promise<{
78
146
  wakeSessionWorkflow: WakeSessionWorkflowSignal;
147
+ signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
79
148
  check: () => Promise<void>;
80
149
  close: () => Promise<void>;
81
150
  }> {
82
151
  const connection = await Connection.connect({ address: settings.temporalHost });
83
152
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
84
153
  return {
85
- wakeSessionWorkflow: async ({ accountId, workspaceId, sessionId, workflowId }) => {
154
+ wakeSessionWorkflow: async ({
155
+ accountId,
156
+ workspaceId,
157
+ sessionId,
158
+ workflowId,
159
+ wakeRevision,
160
+ controlEventId,
161
+ }) => {
162
+ if (controlEventId) {
163
+ await temporal.workflow.signalWithStart("sessionWorkflow", {
164
+ taskQueue: settings.temporalTaskQueue,
165
+ workflowId,
166
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
167
+ args: [{ accountId, workspaceId, sessionId }],
168
+ signal: "sessionControl",
169
+ signalArgs: [controlEventId],
170
+ });
171
+ } else {
172
+ await temporal.workflow.signalWithStart("sessionWorkflow", {
173
+ taskQueue: settings.temporalTaskQueue,
174
+ workflowId,
175
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
176
+ args: [{ accountId, workspaceId, sessionId }],
177
+ signal: "queueChanged",
178
+ });
179
+ }
180
+ await markSessionWorkflowWakeDelivered(db, {
181
+ accountId,
182
+ workspaceId,
183
+ sessionId,
184
+ temporalWorkflowId: workflowId,
185
+ wakeRevision,
186
+ });
187
+ },
188
+ signalCodexCapacityWorkflow: async ({
189
+ accountId,
190
+ workspaceId,
191
+ sessionId,
192
+ workflowId,
193
+ wakeRevision,
194
+ }) => {
86
195
  await temporal.workflow.signalWithStart("sessionWorkflow", {
87
196
  taskQueue: settings.temporalTaskQueue,
88
197
  workflowId,
89
198
  workflowIdReusePolicy: "ALLOW_DUPLICATE",
90
199
  args: [{ accountId, workspaceId, sessionId }],
91
- signal: "queueChanged",
200
+ signal: "codexCapacityChanged",
201
+ signalArgs: [wakeRevision],
92
202
  });
203
+ // A typed capacity signal cannot acknowledge the generic outbox row:
204
+ // another producer may have advanced it with a Pause/Steer that requires
205
+ // sessionControl. The global dispatcher owns that acknowledgement.
93
206
  },
94
207
  check: async () => {
95
208
  await connection.workflowService.getSystemInfo({});
@@ -102,9 +215,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
102
215
 
103
216
  /**
104
217
  * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
105
- * driver — P1.3 / OD-3). Gated on sandboxOwnershipEnabled: with the flag off the
106
- * Schedule is never created, so the lease feature is fully dark (no sweeps, no
107
- * terminates).
218
+ * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
219
+ * sandbox ownership off the activity performs only bounded DB outbox repair;
220
+ * it never reads/terminates sandbox leases.
108
221
  *
109
222
  * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
110
223
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
@@ -121,9 +234,6 @@ export async function registerSandboxReaperSchedule(
121
234
  settings: Settings,
122
235
  observability: Observability,
123
236
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
124
- if (!settings.sandboxOwnershipEnabled) {
125
- return { registered: false, close: async () => {} };
126
- }
127
237
  const connection = await Connection.connect({ address: settings.temporalHost });
128
238
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
129
239
  try {
@@ -152,7 +262,12 @@ export async function registerSandboxReaperSchedule(
152
262
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
153
263
  reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs,
154
264
  });
155
- return { registered: true, close: async () => { await connection.close(); } };
265
+ return {
266
+ registered: true,
267
+ close: async () => {
268
+ await connection.close();
269
+ },
270
+ };
156
271
  } catch (error) {
157
272
  if (error instanceof ScheduleAlreadyRunning) {
158
273
  // Another worker in the pool already created it. The Schedule exists
@@ -162,7 +277,101 @@ export async function registerSandboxReaperSchedule(
162
277
  observability.info("Global sandbox-lease reaper Schedule already registered", {
163
278
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
164
279
  });
165
- return { registered: false, close: async () => { await connection.close(); } };
280
+ return {
281
+ registered: false,
282
+ close: async () => {
283
+ await connection.close();
284
+ },
285
+ };
286
+ }
287
+ await connection.close().catch(() => undefined);
288
+ throw error;
289
+ }
290
+ }
291
+
292
+ /**
293
+ * Register the one provider-neutral expired direct-upload cleanup Schedule.
294
+ * Unlike sandbox GC this is always registered: file uploads can be enabled in
295
+ * deployments where sandbox ownership is disabled. The activity is a cheap
296
+ * no-op when object storage is not configured.
297
+ */
298
+ export async function registerFileUploadReaperSchedule(
299
+ settings: Settings,
300
+ observability: Observability,
301
+ ): Promise<{ registered: boolean; close: () => Promise<void> }> {
302
+ const connection = await Connection.connect({ address: settings.temporalHost });
303
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
304
+ try {
305
+ await temporal.schedule.create({
306
+ scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
307
+ spec: { intervals: [{ every: FILE_UPLOAD_REAPER_PERIOD_MS }] },
308
+ action: {
309
+ type: "startWorkflow",
310
+ workflowType: "fileUploadReaperWorkflow",
311
+ taskQueue: settings.temporalTaskQueue,
312
+ args: [],
313
+ },
314
+ policies: {
315
+ overlap: ScheduleOverlapPolicy.SKIP,
316
+ catchupWindow: "1m",
317
+ pauseOnFailure: false,
318
+ },
319
+ });
320
+ observability.info("Registered the global file-upload reaper Schedule", {
321
+ scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
322
+ reaperPeriodMs: FILE_UPLOAD_REAPER_PERIOD_MS,
323
+ });
324
+ return { registered: true, close: async () => connection.close() };
325
+ } catch (error) {
326
+ if (error instanceof ScheduleAlreadyRunning) {
327
+ observability.info("Global file-upload reaper Schedule already registered", {
328
+ scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
329
+ });
330
+ return { registered: false, close: async () => connection.close() };
331
+ }
332
+ await connection.close().catch(() => undefined);
333
+ throw error;
334
+ }
335
+ }
336
+
337
+ /**
338
+ * Register the one repair cadence for committed workflow-wake revisions. The
339
+ * activity only reads the transactional outbox and sends revision-scoped
340
+ * signals; it is independent of sandbox ownership and child-agent features.
341
+ */
342
+ export async function registerSessionWorkflowWakeDispatcherSchedule(
343
+ settings: Settings,
344
+ observability: Observability,
345
+ ): Promise<{ registered: boolean; close: () => Promise<void> }> {
346
+ const connection = await Connection.connect({ address: settings.temporalHost });
347
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
348
+ try {
349
+ await temporal.schedule.create({
350
+ scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
351
+ spec: { intervals: [{ every: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS }] },
352
+ action: {
353
+ type: "startWorkflow",
354
+ workflowType: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
355
+ taskQueue: settings.temporalTaskQueue,
356
+ args: [],
357
+ },
358
+ policies: {
359
+ overlap: ScheduleOverlapPolicy.SKIP,
360
+ catchupWindow: "1m",
361
+ pauseOnFailure: false,
362
+ },
363
+ });
364
+ observability.info("Registered the session-workflow wake dispatcher Schedule", {
365
+ scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
366
+ periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
367
+ });
368
+ return { registered: true, close: async () => connection.close() };
369
+ } catch (error) {
370
+ if (error instanceof ScheduleAlreadyRunning) {
371
+ observability.info("Session-workflow wake dispatcher Schedule already registered", {
372
+ scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
373
+ });
374
+ return { registered: false, close: async () => connection.close() };
166
375
  }
167
376
  await connection.close().catch(() => undefined);
168
377
  throw error;
@@ -170,10 +379,15 @@ export async function registerSandboxReaperSchedule(
170
379
  }
171
380
 
172
381
  export async function startWorker() {
382
+ const role = process.env.OPENGENI_WORKER_ROLE;
383
+ if (role !== "control" && role !== "turn") {
384
+ throw new Error("OPENGENI_WORKER_ROLE must be explicitly set to 'control' or 'turn'");
385
+ }
173
386
  const settings = getSettings();
174
- const observability = createObservability(settings, { component: "worker" });
387
+ const observability = createObservability(settings, { component: `worker-${role}` });
175
388
  const retryOptions = startupRetryOptions(settings);
176
- const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
389
+ const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
390
+ logStartupDependencyRetry(observability, event);
177
391
  const searchPath = dbSearchPath(settings);
178
392
  const dbClient = createDb(settings.databaseUrl, {
179
393
  ...(searchPath ? { searchPath } : {}),
@@ -184,6 +398,12 @@ export async function startWorker() {
184
398
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
185
399
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
186
400
  let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
401
+ let fileUploadReaperSchedule:
402
+ | Awaited<ReturnType<typeof registerFileUploadReaperSchedule>>
403
+ | undefined;
404
+ let workflowWakeDispatcherSchedule:
405
+ | Awaited<ReturnType<typeof registerSessionWorkflowWakeDispatcherSchedule>>
406
+ | undefined;
187
407
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
188
408
  try {
189
409
  bus = await retryStartupDependency(
@@ -191,21 +411,25 @@ export async function startWorker() {
191
411
  () =>
192
412
  createNatsEventBus(
193
413
  settings.natsUrl,
194
- controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
414
+ controlPlaneAuth
415
+ ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
416
+ : undefined,
195
417
  { logger: observabilityEventLogger(observability) },
196
418
  ),
197
419
  { ...retryOptions, onRetry },
198
420
  );
199
421
  signaler = await retryStartupDependency(
200
422
  "Temporal client",
201
- () => createWorkerWorkflowSignaler(settings),
423
+ () => createWorkerWorkflowSignaler(settings, dbClient.db),
202
424
  { ...retryOptions, onRetry },
203
425
  );
204
426
  workerBundle = await createOpenGeniWorker({
427
+ role,
205
428
  settings,
206
429
  activityDependencies: {
207
430
  observability,
208
431
  wakeSessionWorkflow: signaler.wakeSessionWorkflow,
432
+ signalCodexCapacityWorkflow: signaler.signalCodexCapacityWorkflow,
209
433
  db: dbClient.db,
210
434
  bus,
211
435
  },
@@ -219,19 +443,66 @@ export async function startWorker() {
219
443
  temporal: signaler.check,
220
444
  },
221
445
  });
222
- // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
223
- // is false). Idempotent across the pool: only the first worker creates it. The
224
- // static global-queue Worker.create above is UNCHANGED — there is no
225
- // per-session worker factory.
226
- reaperSchedule = await retryStartupDependency(
227
- "Temporal schedule (sandbox reaper)",
228
- () => registerSandboxReaperSchedule(settings, observability),
229
- { ...retryOptions, onRetry },
230
- );
446
+ // Control workers alone own global schedules. Turn workers poll only the
447
+ // bounded inference activity queue and can never duplicate schedule setup.
448
+ if (role === "control") {
449
+ reaperSchedule = await retryStartupDependency(
450
+ "Temporal schedule (sandbox reaper)",
451
+ () => registerSandboxReaperSchedule(settings, observability),
452
+ { ...retryOptions, onRetry },
453
+ );
454
+ fileUploadReaperSchedule = await retryStartupDependency(
455
+ "Temporal schedule (file upload reaper)",
456
+ () => registerFileUploadReaperSchedule(settings, observability),
457
+ { ...retryOptions, onRetry },
458
+ );
459
+ workflowWakeDispatcherSchedule = await retryStartupDependency(
460
+ "Temporal schedule (session-workflow wake dispatcher)",
461
+ () => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
462
+ { ...retryOptions, onRetry },
463
+ );
464
+ }
231
465
  observability.info("OpenGeni worker listening", {
232
- temporalTaskQueue: settings.temporalTaskQueue,
466
+ role,
467
+ temporalTaskQueue:
468
+ role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
469
+ maxConcurrentActivityTaskExecutions:
470
+ role === "turn"
471
+ ? TURN_WORKER_MAX_CONCURRENT_TURNS
472
+ : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
473
+ maxConcurrentWorkflowTaskExecutions:
474
+ role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
233
475
  httpPort: settings.workerHttpPort,
234
476
  });
477
+ // GRACEFUL DEPLOY SHUTDOWN — the missing link that made every deploy a
478
+ // worker_death. The worker is the container's MAIN process (PID 1), and a
479
+ // PID-1 process with no explicit handler IGNORES SIGTERM: the pod sat
480
+ // through the entire terminationGracePeriodSeconds doing nothing, then the
481
+ // kubelet SIGKILLed it — so agent-turn's worker-shutdown checkpoint path
482
+ // (same-turn recovery + run-state save) never once ran in production
483
+ // (forensics 2026-07-06: every observed recovery was
484
+ // reason=worker_death). This handler starts the drain the moment k8s asks:
485
+ // shutdown() stops polling, then (per shutdownGraceTime/ForceTime above)
486
+ // cancels in-flight activities with WORKER_SHUTDOWN so long turns
487
+ // checkpoint INSIDE the grace window. run() resolves once
488
+ // drained and the finally below closes connections cleanly.
489
+ let shutdownRequested = false;
490
+ const requestShutdown = (signal: string) => {
491
+ if (shutdownRequested) {
492
+ return;
493
+ }
494
+ shutdownRequested = true;
495
+ observability.info("OpenGeni worker draining (graceful shutdown)", { signal });
496
+ try {
497
+ workerBundle!.worker.shutdown();
498
+ } catch (error) {
499
+ observability.warn("worker shutdown request failed", {
500
+ error: error instanceof Error ? error.message : String(error),
501
+ });
502
+ }
503
+ };
504
+ process.on("SIGTERM", () => requestShutdown("SIGTERM"));
505
+ process.on("SIGINT", () => requestShutdown("SIGINT"));
235
506
  await workerBundle.worker.run();
236
507
  } finally {
237
508
  httpServer?.stop(true);
@@ -239,6 +510,8 @@ export async function startWorker() {
239
510
  workerBundle?.connection.close(),
240
511
  signaler?.close(),
241
512
  reaperSchedule?.close(),
513
+ fileUploadReaperSchedule?.close(),
514
+ workflowWakeDispatcherSchedule?.close(),
242
515
  bus?.close(),
243
516
  dbClient.close(),
244
517
  ]);