@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
package/src/index.ts CHANGED
@@ -4,10 +4,11 @@ import {
4
4
  resolveNatsControlPlaneAuth,
5
5
  retryStartupDependency,
6
6
  startupRetryOptions,
7
+ temporalConnectionOptions,
7
8
  type Settings,
8
9
  } from "@opengeni/config";
9
10
  import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
10
- import { createNatsEventBus } from "@opengeni/events";
11
+ import { createNatsEventBus, type EventBus } from "@opengeni/events";
11
12
  import {
12
13
  createObservability,
13
14
  logStartupDependencyRetry,
@@ -19,16 +20,28 @@ import {
19
20
  ScheduleOverlapPolicy,
20
21
  Client as TemporalClient,
21
22
  } from "@temporalio/client";
22
- import { NativeConnection, Worker } from "@temporalio/worker";
23
+ import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
24
+ import { existsSync } from "node:fs";
25
+ import { fileURLToPath } from "node:url";
23
26
  import { ensureModalRegistryImage } from "@opengeni/runtime";
24
27
  import {
25
28
  createControlActivities,
26
29
  createTurnActivities,
27
30
  type ActivityDependencies,
28
31
  } from "./activities";
29
- import type { SignalCodexCapacityWorkflow, WakeSessionWorkflowSignal } from "./activities/types";
32
+ import type {
33
+ SignalCodexCapacityWorkflow,
34
+ SignalSessionAttemptQuiesced,
35
+ WakeSessionWorkflowSignal,
36
+ } from "./activities/types";
30
37
  import { turnTaskQueue } from "./workflows/activities";
31
- import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
38
+ import {
39
+ dbReadyCheck,
40
+ natsReadyCheck,
41
+ startWorkerHttpServer,
42
+ temporalReadyCheck,
43
+ type WorkerLifecycleState,
44
+ } from "./http";
32
45
  import { observabilityEventLogger } from "./observability-metrics";
33
46
  import {
34
47
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
@@ -40,6 +53,24 @@ import {
40
53
  CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
41
54
  TURN_WORKER_MAX_CONCURRENT_TURNS,
42
55
  } from "./concurrency";
56
+ import {
57
+ constructWithOwnedConnection,
58
+ createWorkerServiceLifecycle,
59
+ type WorkerServiceLifecycle,
60
+ } from "./worker-service-lifecycle";
61
+
62
+ export {
63
+ createHostExportPump,
64
+ type HostExportDrainResult,
65
+ type HostExportPump,
66
+ type HostExportPumpOptions,
67
+ type HostEventExport,
68
+ type HostEventExportBatch,
69
+ type HostEventSink,
70
+ type HostUsageExport,
71
+ type HostUsageExportBatch,
72
+ type HostUsageSink,
73
+ } from "./host-export";
43
74
 
44
75
  // The deterministic id of the ONE global reaper Schedule. A single id means
45
76
  // create() is idempotent across every worker in the pool: the first worker to
@@ -56,13 +87,36 @@ export type WorkerOptions = {
56
87
  settings?: Settings;
57
88
  activities?: ReturnType<typeof createControlActivities> | ReturnType<typeof createTurnActivities>;
58
89
  activityDependencies?: ActivityDependencies;
59
- // Embedded hosts install @opengeni/worker-bundle under node_modules, where
60
- // Temporal's workflow webpack refuses to transpile TS. They relocate
61
- // src/workflows.ts to a host-owned path and point the worker at it here.
62
- // Unset (standalone) keeps today's in-package path byte-for-byte.
63
- workflowsPath?: string;
90
+ /** Override the release-coherent workflow artifact. Most hosts should omit this. */
91
+ workflowBundle?: WorkflowBundleOption;
64
92
  };
65
93
 
94
+ type WorkerWorkflowDefinition =
95
+ | { workflowBundle: WorkflowBundleOption }
96
+ | { workflowsPath: string };
97
+
98
+ /**
99
+ * Resolve the deterministic workflow graph without asking installed hosts to
100
+ * transpile or relocate package TypeScript. Monorepo source execution retains
101
+ * the source path for the local development loop; published dist execution
102
+ * fails closed unless the build-generated sibling artifact is present.
103
+ */
104
+ export function resolveOpenGeniWorkflowDefinition(
105
+ moduleUrl: string = import.meta.url,
106
+ ): WorkerWorkflowDefinition {
107
+ const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
108
+ if (modulePath.endsWith("/src/index.ts")) {
109
+ return { workflowsPath: fileURLToPath(new URL("./workflows.ts", moduleUrl)) };
110
+ }
111
+ const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
112
+ if (!existsSync(codePath)) {
113
+ throw new Error(
114
+ `OpenGeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`,
115
+ );
116
+ }
117
+ return { workflowBundle: { codePath } };
118
+ }
119
+
66
120
  export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
67
121
  worker: Worker;
68
122
  connection: NativeConnection;
@@ -71,6 +125,9 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
71
125
  const observability =
72
126
  options.activityDependencies?.observability ??
73
127
  createObservability(settings, { component: `worker-${options.role}` });
128
+ if (options.role === "turn" && options.workflowBundle) {
129
+ throw new Error("workflowBundle is valid only for the control worker role");
130
+ }
74
131
  // Pre-resolve a PRIVATE-registry sandbox image before any turn creates a box.
75
132
  // No-op unless OPENGENI_MODAL_IMAGE_REGISTRY_SECRET + OPENGENI_MODAL_IMAGE_REF are
76
133
  // both set (so non-modal / public-image deployments are byte-unchanged and never
@@ -85,53 +142,67 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
85
142
  },
86
143
  );
87
144
  }
88
- const connection = await retryStartupDependency(
89
- "Temporal",
90
- () => NativeConnection.connect({ address: settings.temporalHost }),
91
- {
92
- ...startupRetryOptions(settings),
93
- onRetry: (event) => logStartupDependencyRetry(observability, event),
145
+ return constructWithOwnedConnection(
146
+ () =>
147
+ retryStartupDependency(
148
+ "Temporal",
149
+ () => NativeConnection.connect(temporalConnectionOptions(settings)),
150
+ {
151
+ ...startupRetryOptions(settings),
152
+ onRetry: (event) => logStartupDependencyRetry(observability, event),
153
+ },
154
+ ),
155
+ async (connection) => {
156
+ const activities =
157
+ options.activities ??
158
+ (options.role === "control" ? createControlActivities : createTurnActivities)({
159
+ ...options.activityDependencies,
160
+ settings,
161
+ observability,
162
+ });
163
+ const worker = await Worker.create({
164
+ connection,
165
+ namespace: settings.temporalNamespace,
166
+ taskQueue:
167
+ options.role === "control"
168
+ ? settings.temporalTaskQueue
169
+ : turnTaskQueue(settings.temporalTaskQueue),
170
+ ...(options.role === "control"
171
+ ? {
172
+ ...(options.workflowBundle
173
+ ? { workflowBundle: options.workflowBundle }
174
+ : resolveOpenGeniWorkflowDefinition()),
175
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
176
+ }
177
+ : {}),
178
+ activities,
179
+ maxConcurrentActivityTaskExecutions:
180
+ options.role === "turn"
181
+ ? TURN_WORKER_MAX_CONCURRENT_TURNS
182
+ : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
183
+ // Cancellation is delivered through an activity heartbeat. The SDK would
184
+ // otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
185
+ // making Pause/Steer take roughly a minute even though runAgentTurn emits a
186
+ // heartbeat every ten seconds. Keep delivery bounded independently of the
187
+ // heartbeat timeout and local timer cadence.
188
+ maxHeartbeatThrottleInterval: "5s",
189
+ defaultHeartbeatThrottleInterval: "5s",
190
+ // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
191
+ // after shutdown() stops polling, in-flight activities get this long to
192
+ // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
193
+ // which triggers agent-turn's same-turn recovery checkpoint instead of a
194
+ // heartbeat-timeout worker_death. Short on purpose: a long grace here
195
+ // only delays the checkpoint window long turns actually need.
196
+ shutdownGraceTime: "5s",
197
+ // Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
198
+ // wedged checkpoint force-stops here, on our terms, rather than riding
199
+ // into the kubelet's SIGKILL mid-DB-write.
200
+ shutdownForceTime: "100s",
201
+ });
202
+ return { worker, connection };
94
203
  },
204
+ (connection) => connection.close(),
95
205
  );
96
- const activities =
97
- options.activities ??
98
- (options.role === "control" ? createControlActivities : createTurnActivities)({
99
- ...options.activityDependencies,
100
- settings,
101
- observability,
102
- });
103
- const worker = await Worker.create({
104
- connection,
105
- namespace: settings.temporalNamespace,
106
- taskQueue:
107
- options.role === "control"
108
- ? settings.temporalTaskQueue
109
- : turnTaskQueue(settings.temporalTaskQueue),
110
- ...(options.role === "control"
111
- ? {
112
- workflowsPath:
113
- options.workflowsPath ?? new URL("../src/workflows.ts", import.meta.url).pathname,
114
- maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
115
- }
116
- : {}),
117
- activities,
118
- maxConcurrentActivityTaskExecutions:
119
- options.role === "turn"
120
- ? TURN_WORKER_MAX_CONCURRENT_TURNS
121
- : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
122
- // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
123
- // after shutdown() stops polling, in-flight activities get this long to
124
- // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
125
- // which triggers agent-turn's same-turn recovery checkpoint instead of a
126
- // heartbeat-timeout worker_death. Short on purpose: a long grace here
127
- // only delays the checkpoint window long turns actually need.
128
- shutdownGraceTime: "5s",
129
- // Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
130
- // wedged checkpoint force-stops here, on our terms, rather than riding
131
- // into the kubelet's SIGKILL mid-DB-write.
132
- shutdownForceTime: "100s",
133
- });
134
- return { worker, connection };
135
206
  }
136
207
 
137
208
  // A signalWithStart capability so a worker activity can wake a PARENT
@@ -144,11 +215,12 @@ export async function createWorkerWorkflowSignaler(
144
215
  db: Database,
145
216
  ): Promise<{
146
217
  wakeSessionWorkflow: WakeSessionWorkflowSignal;
218
+ signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
147
219
  signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
148
220
  check: () => Promise<void>;
149
221
  close: () => Promise<void>;
150
222
  }> {
151
- const connection = await Connection.connect({ address: settings.temporalHost });
223
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
152
224
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
153
225
  return {
154
226
  wakeSessionWorkflow: async ({
@@ -185,6 +257,25 @@ export async function createWorkerWorkflowSignaler(
185
257
  wakeRevision,
186
258
  });
187
259
  },
260
+ signalSessionAttemptQuiesced: async (proof) => {
261
+ await temporal.workflow.signalWithStart("sessionWorkflow", {
262
+ taskQueue: settings.temporalTaskQueue,
263
+ workflowId: proof.workflowId,
264
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
265
+ args: [
266
+ {
267
+ accountId: proof.accountId,
268
+ workspaceId: proof.workspaceId,
269
+ sessionId: proof.sessionId,
270
+ },
271
+ ],
272
+ signal: "sessionAttemptQuiesced",
273
+ signalArgs: [proof],
274
+ });
275
+ // No wake-outbox row exists yet: the direct receipt transaction failed.
276
+ // The signalled workflow's DB-only control activity owns committing the
277
+ // receipt and its exact wake revision atomically.
278
+ },
188
279
  signalCodexCapacityWorkflow: async ({
189
280
  accountId,
190
281
  workspaceId,
@@ -234,7 +325,7 @@ export async function registerSandboxReaperSchedule(
234
325
  settings: Settings,
235
326
  observability: Observability,
236
327
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
237
- const connection = await Connection.connect({ address: settings.temporalHost });
328
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
238
329
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
239
330
  try {
240
331
  await temporal.schedule.create({
@@ -299,7 +390,7 @@ export async function registerFileUploadReaperSchedule(
299
390
  settings: Settings,
300
391
  observability: Observability,
301
392
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
302
- const connection = await Connection.connect({ address: settings.temporalHost });
393
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
303
394
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
304
395
  try {
305
396
  await temporal.schedule.create({
@@ -343,7 +434,7 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
343
434
  settings: Settings,
344
435
  observability: Observability,
345
436
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
346
- const connection = await Connection.connect({ address: settings.temporalHost });
437
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
347
438
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
348
439
  try {
349
440
  await temporal.schedule.create({
@@ -378,6 +469,215 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
378
469
  }
379
470
  }
380
471
 
472
+ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
473
+ activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
474
+ /**
475
+ * `role-default` registers OpenGeni's internal maintenance schedules on a
476
+ * control worker only. These are engine maintenance schedules, not a host's
477
+ * product-level scheduled-agent jobs. Use `none` when another control worker
478
+ * in the same deployment owns them.
479
+ */
480
+ internalSchedules?: "role-default" | "none";
481
+ /** Set false only when the host exposes equivalent lifecycle endpoints itself. */
482
+ http?: false | { readinessTimeoutMs?: number };
483
+ };
484
+
485
+ export type OpenGeniWorkerService = {
486
+ readonly role: OpenGeniWorkerRole;
487
+ readonly worker: Worker;
488
+ readonly connection: NativeConnection;
489
+ state(): WorkerLifecycleState;
490
+ run(): Promise<void>;
491
+ drain(reason?: string): void;
492
+ close(): Promise<void>;
493
+ };
494
+
495
+ export function workerOwnsInternalSchedules(
496
+ role: OpenGeniWorkerRole,
497
+ policy: OpenGeniWorkerServiceOptions["internalSchedules"] = "role-default",
498
+ ): boolean {
499
+ return role === "control" && policy !== "none";
500
+ }
501
+
502
+ /**
503
+ * Construct one role-specific worker process around the lower-level Temporal
504
+ * worker factory. The service owns every Temporal client and HTTP listener it
505
+ * creates. The embedding host retains ownership of its injected DB and EventBus
506
+ * handles and closes those only after this service has drained.
507
+ */
508
+ export async function createOpenGeniWorkerService(
509
+ options: OpenGeniWorkerServiceOptions,
510
+ ): Promise<OpenGeniWorkerService> {
511
+ const settings = options.settings ?? getSettings();
512
+ const observability =
513
+ options.activityDependencies.observability ??
514
+ createObservability(settings, { component: `worker-${options.role}` });
515
+ const retryOptions = startupRetryOptions(settings);
516
+ const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
517
+ logStartupDependencyRetry(observability, event);
518
+ let lifecycle: WorkerServiceLifecycle | undefined;
519
+ let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
520
+ let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
521
+ const schedules: Array<{ close: () => Promise<void> }> = [];
522
+ let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
523
+
524
+ try {
525
+ const needsSignaler =
526
+ !options.activityDependencies.wakeSessionWorkflow ||
527
+ !options.activityDependencies.signalSessionAttemptQuiesced ||
528
+ !options.activityDependencies.signalCodexCapacityWorkflow;
529
+ if (needsSignaler) {
530
+ signaler = await retryStartupDependency(
531
+ "Temporal client",
532
+ () => createWorkerWorkflowSignaler(settings, options.activityDependencies.db),
533
+ { ...retryOptions, onRetry },
534
+ );
535
+ }
536
+ const wakeSessionWorkflow =
537
+ options.activityDependencies.wakeSessionWorkflow ?? signaler?.wakeSessionWorkflow;
538
+ const signalSessionAttemptQuiesced =
539
+ options.activityDependencies.signalSessionAttemptQuiesced ??
540
+ signaler?.signalSessionAttemptQuiesced;
541
+ const signalCodexCapacityWorkflow =
542
+ options.activityDependencies.signalCodexCapacityWorkflow ??
543
+ signaler?.signalCodexCapacityWorkflow;
544
+ if (!wakeSessionWorkflow || !signalSessionAttemptQuiesced || !signalCodexCapacityWorkflow) {
545
+ throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
546
+ }
547
+ workerBundle = await createOpenGeniWorker({
548
+ role: options.role,
549
+ settings,
550
+ ...(options.activities ? { activities: options.activities } : {}),
551
+ ...(options.workflowBundle ? { workflowBundle: options.workflowBundle } : {}),
552
+ activityDependencies: {
553
+ ...options.activityDependencies,
554
+ settings,
555
+ observability,
556
+ wakeSessionWorkflow,
557
+ signalSessionAttemptQuiesced,
558
+ signalCodexCapacityWorkflow,
559
+ },
560
+ });
561
+
562
+ if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
563
+ schedules.push(
564
+ await retryStartupDependency(
565
+ "Temporal schedule (sandbox reaper)",
566
+ () => registerSandboxReaperSchedule(settings, observability),
567
+ { ...retryOptions, onRetry },
568
+ ),
569
+ );
570
+ schedules.push(
571
+ await retryStartupDependency(
572
+ "Temporal schedule (file upload reaper)",
573
+ () => registerFileUploadReaperSchedule(settings, observability),
574
+ { ...retryOptions, onRetry },
575
+ ),
576
+ );
577
+ schedules.push(
578
+ await retryStartupDependency(
579
+ "Temporal schedule (session-workflow wake dispatcher)",
580
+ () => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
581
+ { ...retryOptions, onRetry },
582
+ ),
583
+ );
584
+ }
585
+
586
+ if (options.http !== false) {
587
+ httpServer = startWorkerHttpServer({
588
+ settings,
589
+ observability,
590
+ checks: {
591
+ db: dbReadyCheck(options.activityDependencies.db),
592
+ nats: natsReadyCheck(options.activityDependencies.bus),
593
+ temporal: temporalReadyCheck(workerBundle.connection),
594
+ },
595
+ ...(options.http?.readinessTimeoutMs ? { timeoutMs: options.http.readinessTimeoutMs } : {}),
596
+ lifecycle: { role: options.role, state: () => lifecycle?.state() ?? "starting" },
597
+ });
598
+ }
599
+ } catch (error) {
600
+ httpServer?.stop(true);
601
+ await Promise.allSettled([
602
+ workerBundle?.connection.close(),
603
+ signaler?.close(),
604
+ ...schedules.map((schedule) => schedule.close()),
605
+ ]);
606
+ throw error;
607
+ }
608
+
609
+ const activeWorkerBundle = workerBundle;
610
+ const activeSignaler = signaler;
611
+ if (!activeWorkerBundle) {
612
+ throw new Error("OpenGeni worker service initialization did not complete");
613
+ }
614
+
615
+ lifecycle = createWorkerServiceLifecycle({
616
+ role: options.role,
617
+ worker: activeWorkerBundle.worker,
618
+ observability,
619
+ closeOwnedResources: async () => {
620
+ httpServer?.stop(true);
621
+ await Promise.allSettled([
622
+ activeWorkerBundle.connection.close(),
623
+ activeSignaler?.close(),
624
+ ...schedules.map((schedule) => schedule.close()),
625
+ ]);
626
+ },
627
+ onReady: () => {
628
+ observability.info("OpenGeni worker listening", {
629
+ role: options.role,
630
+ temporalTaskQueue:
631
+ options.role === "control"
632
+ ? settings.temporalTaskQueue
633
+ : turnTaskQueue(settings.temporalTaskQueue),
634
+ maxConcurrentActivityTaskExecutions:
635
+ options.role === "turn"
636
+ ? TURN_WORKER_MAX_CONCURRENT_TURNS
637
+ : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
638
+ maxConcurrentWorkflowTaskExecutions:
639
+ options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
640
+ httpPort: options.http === false ? null : settings.workerHttpPort,
641
+ });
642
+ },
643
+ });
644
+
645
+ return {
646
+ role: options.role,
647
+ worker: activeWorkerBundle.worker,
648
+ connection: activeWorkerBundle.connection,
649
+ state: lifecycle.state,
650
+ run: lifecycle.run,
651
+ drain: lifecycle.drain,
652
+ close: lifecycle.close,
653
+ };
654
+ }
655
+
656
+ export type RunOpenGeniWorkerOptions = OpenGeniWorkerServiceOptions & {
657
+ /** Defaults to SIGTERM and SIGINT. Pass false when the host owns process signals. */
658
+ shutdownSignals?: false | ReadonlyArray<"SIGTERM" | "SIGINT">;
659
+ };
660
+
661
+ /** Start, drain, and close one embedded worker process. */
662
+ export async function runOpenGeniWorker(options: RunOpenGeniWorkerOptions): Promise<void> {
663
+ const service = await createOpenGeniWorkerService(options);
664
+ const signals =
665
+ options.shutdownSignals === false ? [] : (options.shutdownSignals ?? ["SIGTERM", "SIGINT"]);
666
+ const handlers = signals.map((signal) => {
667
+ const handler = () => service.drain(signal);
668
+ process.on(signal, handler);
669
+ return { signal, handler };
670
+ });
671
+ try {
672
+ await service.run();
673
+ } finally {
674
+ for (const { signal, handler } of handlers) {
675
+ process.off(signal, handler);
676
+ }
677
+ await service.close();
678
+ }
679
+ }
680
+
381
681
  export async function startWorker() {
382
682
  const role = process.env.OPENGENI_WORKER_ROLE;
383
683
  if (role !== "control" && role !== "turn") {
@@ -395,16 +695,6 @@ export async function startWorker() {
395
695
  });
396
696
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
397
697
  let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
398
- let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
399
- let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
400
- let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
401
- let fileUploadReaperSchedule:
402
- | Awaited<ReturnType<typeof registerFileUploadReaperSchedule>>
403
- | undefined;
404
- let workflowWakeDispatcherSchedule:
405
- | Awaited<ReturnType<typeof registerSessionWorkflowWakeDispatcherSchedule>>
406
- | undefined;
407
- let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
408
698
  try {
409
699
  bus = await retryStartupDependency(
410
700
  "NATS",
@@ -418,103 +708,17 @@ export async function startWorker() {
418
708
  ),
419
709
  { ...retryOptions, onRetry },
420
710
  );
421
- signaler = await retryStartupDependency(
422
- "Temporal client",
423
- () => createWorkerWorkflowSignaler(settings, dbClient.db),
424
- { ...retryOptions, onRetry },
425
- );
426
- workerBundle = await createOpenGeniWorker({
711
+ await runOpenGeniWorker({
427
712
  role,
428
713
  settings,
429
714
  activityDependencies: {
430
715
  observability,
431
- wakeSessionWorkflow: signaler.wakeSessionWorkflow,
432
- signalCodexCapacityWorkflow: signaler.signalCodexCapacityWorkflow,
433
716
  db: dbClient.db,
434
717
  bus,
435
718
  },
436
719
  });
437
- httpServer = startWorkerHttpServer({
438
- settings,
439
- observability,
440
- checks: {
441
- db: dbReadyCheck(dbClient.db),
442
- nats: natsReadyCheck(bus),
443
- temporal: signaler.check,
444
- },
445
- });
446
- // Control workers alone own global schedules. Turn workers poll only the
447
- // bounded inference activity queue and can never duplicate schedule setup.
448
- if (role === "control") {
449
- reaperSchedule = await retryStartupDependency(
450
- "Temporal schedule (sandbox reaper)",
451
- () => registerSandboxReaperSchedule(settings, observability),
452
- { ...retryOptions, onRetry },
453
- );
454
- fileUploadReaperSchedule = await retryStartupDependency(
455
- "Temporal schedule (file upload reaper)",
456
- () => registerFileUploadReaperSchedule(settings, observability),
457
- { ...retryOptions, onRetry },
458
- );
459
- workflowWakeDispatcherSchedule = await retryStartupDependency(
460
- "Temporal schedule (session-workflow wake dispatcher)",
461
- () => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
462
- { ...retryOptions, onRetry },
463
- );
464
- }
465
- observability.info("OpenGeni worker listening", {
466
- role,
467
- temporalTaskQueue:
468
- role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
469
- maxConcurrentActivityTaskExecutions:
470
- role === "turn"
471
- ? TURN_WORKER_MAX_CONCURRENT_TURNS
472
- : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
473
- maxConcurrentWorkflowTaskExecutions:
474
- role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
475
- httpPort: settings.workerHttpPort,
476
- });
477
- // GRACEFUL DEPLOY SHUTDOWN — the missing link that made every deploy a
478
- // worker_death. The worker is the container's MAIN process (PID 1), and a
479
- // PID-1 process with no explicit handler IGNORES SIGTERM: the pod sat
480
- // through the entire terminationGracePeriodSeconds doing nothing, then the
481
- // kubelet SIGKILLed it — so agent-turn's worker-shutdown checkpoint path
482
- // (same-turn recovery + run-state save) never once ran in production
483
- // (forensics 2026-07-06: every observed recovery was
484
- // reason=worker_death). This handler starts the drain the moment k8s asks:
485
- // shutdown() stops polling, then (per shutdownGraceTime/ForceTime above)
486
- // cancels in-flight activities with WORKER_SHUTDOWN so long turns
487
- // checkpoint INSIDE the grace window. run() resolves once
488
- // drained and the finally below closes connections cleanly.
489
- let shutdownRequested = false;
490
- const requestShutdown = (signal: string) => {
491
- if (shutdownRequested) {
492
- return;
493
- }
494
- shutdownRequested = true;
495
- observability.info("OpenGeni worker draining (graceful shutdown)", { signal });
496
- try {
497
- workerBundle!.worker.shutdown();
498
- } catch (error) {
499
- observability.warn("worker shutdown request failed", {
500
- error: error instanceof Error ? error.message : String(error),
501
- });
502
- }
503
- };
504
- process.on("SIGTERM", () => requestShutdown("SIGTERM"));
505
- process.on("SIGINT", () => requestShutdown("SIGINT"));
506
- await workerBundle.worker.run();
507
720
  } finally {
508
- httpServer?.stop(true);
509
- await Promise.allSettled([
510
- workerBundle?.connection.close(),
511
- signaler?.close(),
512
- reaperSchedule?.close(),
513
- fileUploadReaperSchedule?.close(),
514
- workflowWakeDispatcherSchedule?.close(),
515
- bus?.close(),
516
- dbClient.close(),
517
- ]);
721
+ await Promise.allSettled([bus?.close(), dbClient.close()]);
518
722
  }
519
723
  }
520
724