@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -166,7 +166,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
166
166
  maxBytes: batchMaxBytes,
167
167
  };
168
168
  const settleFailure = async (error: unknown): Promise<HostExportDrainResult> => {
169
- const message = safeErrorMessage(error);
169
+ const message = exactHostExportErrorMessage(error);
170
170
  let failures = 0;
171
171
  try {
172
172
  failures = await failHostExportBatch(options.db, {
@@ -179,17 +179,15 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
179
179
  } catch (settlementError) {
180
180
  options.observability?.warn("host export failure settlement was stale", {
181
181
  kind,
182
- consumerId,
183
- error: safeErrorMessage(settlementError),
182
+ ...hostExportPublicErrorFields(settlementError, "host_export_failure_settlement_stale"),
184
183
  });
185
184
  }
186
185
  const blocked = failures >= maxFailures;
187
186
  options.observability?.warn("host export batch failed", {
188
187
  kind,
189
- consumerId,
190
188
  failures,
191
189
  blocked,
192
- error: message,
190
+ ...hostExportPublicErrorFields(error, "host_export_batch_delivery_failed"),
193
191
  });
194
192
  const result: HostExportDrainResult = {
195
193
  kind,
@@ -222,8 +220,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
222
220
  // a duplicate settlement attempt for the now-stale lease.
223
221
  options.observability?.error("host export retention prune failed", {
224
222
  kind,
225
- consumerId,
226
- error: safeErrorMessage(error),
223
+ ...hostExportPublicErrorFields(error, "host_export_retention_prune_failed"),
227
224
  });
228
225
  }
229
226
  const result: HostExportDrainResult = {
@@ -344,7 +341,7 @@ export function createHostExportPump(options: HostExportPumpOptions): HostExport
344
341
  }
345
342
  } catch (error) {
346
343
  options.observability?.error("host export pump iteration failed", {
347
- error: safeErrorMessage(error),
344
+ ...hostExportPublicErrorFields(error, "host_export_pump_iteration_failed"),
348
345
  });
349
346
  }
350
347
  if (running) await waitForPoll();
@@ -410,7 +407,27 @@ function validateInstanceId(value: string): string {
410
407
  return value;
411
408
  }
412
409
 
413
- function safeErrorMessage(error: unknown): string {
414
- const message = error instanceof Error ? error.message : String(error);
415
- return message.trim().slice(0, 500) || "host sink failed";
410
+ function exactHostExportErrorMessage(error: unknown): string {
411
+ return error instanceof Error ? error.message : String(error);
412
+ }
413
+
414
+ type HostExportPublicFailureCode =
415
+ | "host_export_failure_settlement_stale"
416
+ | "host_export_batch_delivery_failed"
417
+ | "host_export_retention_prune_failed"
418
+ | "host_export_pump_iteration_failed";
419
+
420
+ function hostExportPublicErrorFields(
421
+ _error: unknown,
422
+ errorCode: HostExportPublicFailureCode,
423
+ ): {
424
+ errorClass: "HostExportOperationError";
425
+ errorCode: HostExportPublicFailureCode;
426
+ origin: "host-export";
427
+ } {
428
+ return {
429
+ errorClass: "HostExportOperationError",
430
+ errorCode,
431
+ origin: "host-export",
432
+ };
416
433
  }
package/src/http.ts CHANGED
@@ -72,7 +72,7 @@ export function createWorkerHttpHandler(
72
72
  checks: Object.fromEntries(
73
73
  (Object.keys(checks) as ReadinessCheckName[]).map((name) => [
74
74
  name,
75
- { ok: false, error: `worker is ${state}` },
75
+ { ok: false, error: "worker_not_ready" },
76
76
  ]),
77
77
  ) as ReadinessResult["checks"],
78
78
  }
@@ -101,11 +101,8 @@ export async function runReadinessChecks(
101
101
  try {
102
102
  await withTimeout(Promise.resolve().then(check), timeoutMs);
103
103
  return [name, { ok: true }] as const;
104
- } catch (error) {
105
- return [
106
- name,
107
- { ok: false, error: error instanceof Error ? error.message : String(error) },
108
- ] as const;
104
+ } catch {
105
+ return [name, { ok: false, error: "dependency_unavailable" }] as const;
109
106
  }
110
107
  },
111
108
  ),
package/src/index.ts CHANGED
@@ -26,20 +26,20 @@ import {
26
26
  ScheduleAlreadyRunning,
27
27
  ScheduleOverlapPolicy,
28
28
  Client as TemporalClient,
29
+ WorkflowExecutionAlreadyStartedError,
29
30
  } from "@temporalio/client";
30
31
  import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
31
32
  import { existsSync } from "node:fs";
32
33
  import { fileURLToPath } from "node:url";
33
- import { ensureModalRegistryImage } from "@opengeni/runtime";
34
- import {
35
- createControlActivities,
36
- createTurnActivities,
37
- type ActivityDependencies,
38
- } from "./activities";
34
+ import type { createControlActivities } from "./activities-control";
35
+ import type { createTurnActivities } from "./activities-turn";
36
+ import type { ActivityDependencies } from "./activities/types";
39
37
  import type {
40
38
  InspectSessionAttemptActivity,
41
39
  SignalCodexCapacityWorkflow,
42
40
  SignalSessionAttemptQuiesced,
41
+ StartSandboxReaperWorkflow,
42
+ StartVideoGenerationWorkflow,
43
43
  WakeSessionWorkflowSignal,
44
44
  } from "./activities/types";
45
45
  import { turnTaskQueue } from "./workflows/activities";
@@ -50,23 +50,42 @@ import {
50
50
  temporalReadyCheck,
51
51
  type WorkerLifecycleState,
52
52
  } from "./http";
53
- import { observabilityEventLogger } from "./observability-metrics";
53
+ import {
54
+ normalizeTurnTaskQueueStats,
55
+ observabilityEventLogger,
56
+ startTurnCapacityMonitor,
57
+ type TurnTaskQueueStats,
58
+ } from "./observability-metrics";
54
59
  import {
55
60
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
56
61
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
57
62
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
63
+ TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
58
64
  } from "@opengeni/core";
59
65
  import {
66
+ CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
60
67
  CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
61
68
  CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
69
+ createTurnWorkerConcurrencyPlan,
62
70
  turnWorkerConcurrencyLogFields,
63
- turnWorkerConcurrencyOptions,
64
71
  } from "./concurrency";
65
72
  import {
73
+ combineWorkerRunTargets,
66
74
  constructWithOwnedConnection,
67
75
  createWorkerServiceLifecycle,
68
76
  type WorkerServiceLifecycle,
77
+ type WorkerRunTarget,
69
78
  } from "./worker-service-lifecycle";
79
+ import {
80
+ createTurnWorkerMemoryPressureGuard,
81
+ turnWorkerMemoryPressureGuardEnabled,
82
+ type TurnWorkerMemoryPressureGuard,
83
+ } from "./memory-pressure-guard";
84
+ import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
85
+ import {
86
+ SANDBOX_REAPER_V2_WORKFLOW_ID,
87
+ sandboxLifecycleTaskQueue,
88
+ } from "./sandbox-reaper-contract";
70
89
 
71
90
  export {
72
91
  createHostExportPump,
@@ -135,6 +154,18 @@ export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
135
154
  return isGrpcServiceError(error) && error.code === 5;
136
155
  }
137
156
 
157
+ /** Load exactly one role's activity graph. Exported for embedded hosts and the
158
+ * process-RSS conformance benchmark; construction remains side-effect free
159
+ * until an activity first resolves its injected services. */
160
+ export async function createDefaultWorkerActivities(
161
+ role: OpenGeniWorkerRole,
162
+ dependencies: ActivityDependencies = {},
163
+ ) {
164
+ return role === "control"
165
+ ? (await import("./activities-control")).createControlActivities(dependencies)
166
+ : (await import("./activities-turn")).createTurnActivities(dependencies);
167
+ }
168
+
138
169
  type WorkerWorkflowDefinition =
139
170
  | { workflowBundle: WorkflowBundleOption }
140
171
  | { workflowsPath: string };
@@ -161,11 +192,24 @@ export function resolveOpenGeniWorkflowDefinition(
161
192
  return { workflowBundle: { codePath } };
162
193
  }
163
194
 
195
+ export function turnCancellationHeartbeatThrottleOptions() {
196
+ return {
197
+ maxHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
198
+ defaultHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
199
+ } as const;
200
+ }
201
+
164
202
  export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
165
- worker: Worker;
203
+ worker: WorkerRunTarget;
166
204
  connection: NativeConnection;
167
205
  }> {
168
206
  const settings = options.settings ?? getSettings();
207
+ if (options.role === "control") {
208
+ // The exported lower-level factory is itself poll-capable. Validate here so
209
+ // embedded hosts cannot bypass the reaper budget contract by omitting the
210
+ // higher-level service and its Schedule-registration lifecycle.
211
+ assertSandboxReaperActivityTimeout(settings);
212
+ }
169
213
  const observability =
170
214
  options.activityDependencies?.observability ??
171
215
  createObservability(settings, { component: `worker-${options.role}` });
@@ -178,6 +222,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
178
222
  // Otherwise this is a no-op unless the registry secret + logical ref are both
179
223
  // set. Memoized in the provider, so it runs once per process.
180
224
  if (options.role === "turn") {
225
+ const { ensureModalRegistryImage } = await import("@opengeni/runtime/sandbox");
181
226
  await retryStartupDependency(
182
227
  "Modal private-registry image",
183
228
  () => ensureModalRegistryImage(settings),
@@ -198,50 +243,87 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
198
243
  },
199
244
  ),
200
245
  async (connection) => {
246
+ const activityDependencies = {
247
+ ...options.activityDependencies,
248
+ settings,
249
+ observability,
250
+ };
201
251
  const activities =
202
252
  options.activities ??
203
- (options.role === "control" ? createControlActivities : createTurnActivities)({
204
- ...options.activityDependencies,
205
- settings,
206
- observability,
207
- });
208
- const worker = await Worker.create({
253
+ (await createDefaultWorkerActivities(options.role, activityDependencies));
254
+ const turnConcurrency =
255
+ options.role === "turn"
256
+ ? createTurnWorkerConcurrencyPlan(settings, { observability })
257
+ : null;
258
+ const workflowDefinition =
259
+ options.role === "control"
260
+ ? options.workflowBundle
261
+ ? { workflowBundle: options.workflowBundle }
262
+ : resolveOpenGeniWorkflowDefinition()
263
+ : {};
264
+ const sharedWorkerOptions = {
209
265
  connection,
210
266
  namespace: settings.temporalNamespace,
267
+ activities,
268
+ // Cancellation is delivered through an activity heartbeat. Keep the SDK
269
+ // throttle aligned with runAgentTurn's local heartbeat cadence so the
270
+ // four-second control contract still has time for physical writer drain
271
+ // and exact receipt-gated replacement admission.
272
+ ...turnCancellationHeartbeatThrottleOptions(),
273
+ // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
274
+ // after shutdown() stops polling, in-flight activities get this long to
275
+ // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN.
276
+ shutdownGraceTime: "5s",
277
+ // Hard ceiling inside the pod's termination grace period.
278
+ shutdownForceTime: "100s",
279
+ } as const;
280
+ const worker = await Worker.create({
281
+ ...sharedWorkerOptions,
211
282
  taskQueue:
212
283
  options.role === "control"
213
284
  ? settings.temporalTaskQueue
214
285
  : turnTaskQueue(settings.temporalTaskQueue),
215
286
  ...(options.role === "control"
216
287
  ? {
217
- ...(options.workflowBundle
218
- ? { workflowBundle: options.workflowBundle }
219
- : resolveOpenGeniWorkflowDefinition()),
288
+ ...workflowDefinition,
289
+ reuseV8Context: true,
290
+ workflowThreadPoolSize: 1,
291
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
220
292
  maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
221
293
  maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
222
294
  }
223
- : turnWorkerConcurrencyOptions(settings)),
224
- activities,
225
- // Cancellation is delivered through an activity heartbeat. The SDK would
226
- // otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
227
- // making Pause/Steer take roughly a minute even though runAgentTurn emits a
228
- // heartbeat every ten seconds. Keep delivery bounded independently of the
229
- // heartbeat timeout and local timer cadence.
230
- maxHeartbeatThrottleInterval: "5s",
231
- defaultHeartbeatThrottleInterval: "5s",
232
- // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
233
- // after shutdown() stops polling, in-flight activities get this long to
234
- // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
235
- // which triggers agent-turn's same-turn recovery checkpoint instead of a
236
- // heartbeat-timeout worker_death. Short on purpose: a long grace here
237
- // only delays the checkpoint window long turns actually need.
238
- shutdownGraceTime: "5s",
239
- // Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
240
- // wedged checkpoint force-stops here, on our terms, rather than riding
241
- // into the kubelet's SIGKILL mid-DB-write.
242
- shutdownForceTime: "100s",
295
+ : turnConcurrency!.options),
243
296
  });
244
- return { worker, connection };
297
+ if (options.role !== "control") {
298
+ turnConcurrency?.admission?.finalizeStartupBaseline();
299
+ return { worker, connection };
300
+ }
301
+
302
+ try {
303
+ const sandboxLifecycleWorker = await Worker.create({
304
+ ...sharedWorkerOptions,
305
+ ...workflowDefinition,
306
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
307
+ reuseV8Context: true,
308
+ workflowThreadPoolSize: 1,
309
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
310
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
311
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
312
+ });
313
+ return {
314
+ worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
315
+ connection,
316
+ };
317
+ } catch (error) {
318
+ try {
319
+ worker.shutdown();
320
+ } catch {
321
+ // Preserve the lifecycle worker construction failure. The base poller
322
+ // still received its shutdown request; a secondary synchronous
323
+ // shutdown error must not replace the startup root cause.
324
+ }
325
+ throw error;
326
+ }
245
327
  },
246
328
  (connection) => connection.close(),
247
329
  );
@@ -260,6 +342,9 @@ export async function createWorkerWorkflowSignaler(
260
342
  signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
261
343
  inspectSessionAttemptActivity: InspectSessionAttemptActivity;
262
344
  signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
345
+ getTurnTaskQueueStats: () => Promise<TurnTaskQueueStats>;
346
+ startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
347
+ startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
263
348
  check: () => Promise<void>;
264
349
  close: () => Promise<void>;
265
350
  }> {
@@ -354,6 +439,61 @@ export async function createWorkerWorkflowSignaler(
354
439
  // another producer may have advanced it with a Pause/Steer that requires
355
440
  // sessionControl. The global dispatcher owns that acknowledgement.
356
441
  },
442
+ getTurnTaskQueueStats: async () => {
443
+ const response = await connection.workflowService.describeTaskQueue({
444
+ namespace: settings.temporalNamespace,
445
+ taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
446
+ // temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
447
+ // the supported DEFAULT mode; `stats.approximateBacklogCount` is the
448
+ // server-documented scaling signal.
449
+ taskQueueType: 2,
450
+ reportStats: true,
451
+ });
452
+ return normalizeTurnTaskQueueStats(response.stats);
453
+ },
454
+ startSandboxReaperWorkflow: async () => {
455
+ try {
456
+ await temporal.workflow.start("sandboxReaperWorkflowV2", {
457
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
458
+ workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
459
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
460
+ args: [],
461
+ });
462
+ return "started";
463
+ } catch (error) {
464
+ if (error instanceof WorkflowExecutionAlreadyStartedError) {
465
+ return "already_running";
466
+ }
467
+ throw error;
468
+ }
469
+ },
470
+ startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
471
+ try {
472
+ await temporal.workflow.start("videoGenerationWorkflow", {
473
+ taskQueue: settings.temporalTaskQueue,
474
+ workflowId: `video-generation:${operationId}`,
475
+ // Reject a concurrent or already-completed run, but allow the repair
476
+ // sweep to restart this exact operation if the workflow itself failed.
477
+ // Every provider/storage transition remains CAS/idempotency fenced in
478
+ // Postgres, so recovery never creates a second logical operation.
479
+ workflowIdReusePolicy: "ALLOW_DUPLICATE_FAILED_ONLY",
480
+ args: [
481
+ {
482
+ accountId,
483
+ workspaceId,
484
+ operationId,
485
+ baseTaskQueue: settings.temporalTaskQueue,
486
+ },
487
+ ],
488
+ });
489
+ return "started";
490
+ } catch (error) {
491
+ if (error instanceof WorkflowExecutionAlreadyStartedError) {
492
+ return "already_running";
493
+ }
494
+ throw error;
495
+ }
496
+ },
357
497
  check: async () => {
358
498
  await connection.workflowService.getSystemInfo({});
359
499
  },
@@ -369,13 +509,14 @@ export async function createWorkerWorkflowSignaler(
369
509
  * sandbox ownership off the activity performs bounded DB outbox repair and
370
510
  * read-only observability projections; it never mutates or terminates sandbox leases.
371
511
  *
372
- * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
512
+ * The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
373
513
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
374
514
  * `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
375
515
  * validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
376
516
  * sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
377
- * worker pool collides on ScheduleAlreadyRunning and no-ops, so the Schedule is
378
- * registered exactly once per deployment.
517
+ * worker pool collides on ScheduleAlreadyRunning and reconciles the same
518
+ * definition. New workers route onto the versioned lifecycle queue; old workers
519
+ * can still run the legacy action during rollout or a full binary rollback.
379
520
  *
380
521
  * Returns a `close()` for the dedicated client connection (separate from the
381
522
  * worker's NativeConnection — the Schedule client is a @temporalio/client).
@@ -384,29 +525,32 @@ export async function registerSandboxReaperSchedule(
384
525
  settings: Settings,
385
526
  observability: Observability,
386
527
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
528
+ assertSandboxReaperActivityTimeout(settings);
387
529
  const connection = await Connection.connect(temporalConnectionOptions(settings));
388
530
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
531
+ const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
532
+ const action = {
533
+ type: "startWorkflow" as const,
534
+ workflowType: "sandboxReaperWorkflow",
535
+ taskQueue: settings.temporalTaskQueue,
536
+ args: [] as [],
537
+ };
538
+ const policies = {
539
+ overlap: ScheduleOverlapPolicy.SKIP,
540
+ catchupWindow: "1m",
541
+ pauseOnFailure: false,
542
+ } as const;
389
543
  try {
390
544
  await temporal.schedule.create({
391
545
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
392
- spec: {
393
- // @every-style interval: fire once per reaper period. The boot invariant
394
- // (config) guarantees reaperPeriod < viewerHolderTTL and
395
- // reaperPeriod + idleGrace < providerLifetime.
396
- intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }],
397
- },
398
- action: {
399
- type: "startWorkflow",
400
- workflowType: "sandboxReaperWorkflow",
401
- taskQueue: settings.temporalTaskQueue,
402
- args: [],
403
- },
404
- policies: {
405
- // A slow sweep must never overlap itself; the next fire is skipped.
406
- overlap: ScheduleOverlapPolicy.SKIP,
407
- catchupWindow: "1m",
408
- pauseOnFailure: false,
409
- },
546
+ // @every-style interval: fire once per reaper period. The boot invariant
547
+ // guarantees reaperPeriod < viewerHolderTTL and
548
+ // reaperPeriod + idleGrace < providerLifetime.
549
+ spec,
550
+ action,
551
+ // The base workflow is a short router on new binaries; legacy binaries
552
+ // still execute the pre-refactor composite under this same overlap fence.
553
+ policies,
410
554
  });
411
555
  observability.info("Registered the global sandbox-lease reaper Schedule", {
412
556
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
@@ -420,12 +564,24 @@ export async function registerSandboxReaperSchedule(
420
564
  };
421
565
  } catch (error) {
422
566
  if (error instanceof ScheduleAlreadyRunning) {
423
- // Another worker in the pool already created it. The Schedule exists
424
- // exactly once — this is the expected no-op on every replica after the
425
- // first. (We do NOT update the spec here: a redeploy with a changed cadence
426
- // is an operational concern handled by deleting+recreating the Schedule.)
427
- observability.info("Global sandbox-lease reaper Schedule already registered", {
567
+ // Never move this action off the base queue: doing so would leave a full
568
+ // rollback with no worker polling the only reaper Schedule. The workflow
569
+ // implementation itself owns the forward-compatible queue handoff.
570
+ await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
571
+ spec,
572
+ action,
573
+ policies,
574
+ state: {
575
+ paused: previous.state.paused,
576
+ ...(previous.state.note ? { note: previous.state.note } : {}),
577
+ ...(previous.state.remainingActions !== undefined
578
+ ? { remainingActions: previous.state.remainingActions }
579
+ : {}),
580
+ },
581
+ }));
582
+ observability.info("Reconciled the global sandbox-lease reaper Schedule", {
428
583
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
584
+ temporalTaskQueue: action.taskQueue,
429
585
  });
430
586
  return {
431
587
  registered: false,
@@ -548,7 +704,7 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
548
704
 
549
705
  export type OpenGeniWorkerService = {
550
706
  readonly role: OpenGeniWorkerRole;
551
- readonly worker: Worker;
707
+ readonly worker: WorkerRunTarget;
552
708
  readonly connection: NativeConnection;
553
709
  state(): WorkerLifecycleState;
554
710
  run(): Promise<void>;
@@ -582,15 +738,20 @@ export async function createOpenGeniWorkerService(
582
738
  let lifecycle: WorkerServiceLifecycle | undefined;
583
739
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
584
740
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
741
+ let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
585
742
  const schedules: Array<{ close: () => Promise<void> }> = [];
586
743
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
744
+ let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
587
745
 
588
746
  try {
589
747
  const needsSignaler =
748
+ options.role === "turn" ||
590
749
  !options.activityDependencies.wakeSessionWorkflow ||
591
750
  !options.activityDependencies.signalSessionAttemptQuiesced ||
592
751
  !options.activityDependencies.inspectSessionAttemptActivity ||
593
- !options.activityDependencies.signalCodexCapacityWorkflow;
752
+ !options.activityDependencies.signalCodexCapacityWorkflow ||
753
+ !options.activityDependencies.startSandboxReaperWorkflow ||
754
+ !options.activityDependencies.startVideoGenerationWorkflow;
594
755
  if (needsSignaler) {
595
756
  signaler = await retryStartupDependency(
596
757
  "Temporal client",
@@ -609,11 +770,19 @@ export async function createOpenGeniWorkerService(
609
770
  const signalCodexCapacityWorkflow =
610
771
  options.activityDependencies.signalCodexCapacityWorkflow ??
611
772
  signaler?.signalCodexCapacityWorkflow;
773
+ const startSandboxReaperWorkflow =
774
+ options.activityDependencies.startSandboxReaperWorkflow ??
775
+ signaler?.startSandboxReaperWorkflow;
776
+ const startVideoGenerationWorkflow =
777
+ options.activityDependencies.startVideoGenerationWorkflow ??
778
+ signaler?.startVideoGenerationWorkflow;
612
779
  if (
613
780
  !wakeSessionWorkflow ||
614
781
  !signalSessionAttemptQuiesced ||
615
782
  !inspectSessionAttemptActivity ||
616
- !signalCodexCapacityWorkflow
783
+ !signalCodexCapacityWorkflow ||
784
+ !startSandboxReaperWorkflow ||
785
+ !startVideoGenerationWorkflow
617
786
  ) {
618
787
  throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
619
788
  }
@@ -630,9 +799,21 @@ export async function createOpenGeniWorkerService(
630
799
  signalSessionAttemptQuiesced,
631
800
  inspectSessionAttemptActivity,
632
801
  signalCodexCapacityWorkflow,
802
+ startSandboxReaperWorkflow,
803
+ startVideoGenerationWorkflow,
633
804
  },
634
805
  });
635
806
 
807
+ if (options.role === "turn") {
808
+ if (!signaler) {
809
+ throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
810
+ }
811
+ turnCapacityMonitor = startTurnCapacityMonitor({
812
+ observability,
813
+ read: signaler.getTurnTaskQueueStats,
814
+ });
815
+ }
816
+
636
817
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
637
818
  schedules.push(
638
819
  await retryStartupDependency(
@@ -673,6 +854,7 @@ export async function createOpenGeniWorkerService(
673
854
  } catch (error) {
674
855
  httpServer?.stop(true);
675
856
  await Promise.allSettled([
857
+ turnCapacityMonitor?.close(),
676
858
  workerBundle?.connection.close(),
677
859
  signaler?.close(),
678
860
  ...schedules.map((schedule) => schedule.close()),
@@ -691,8 +873,10 @@ export async function createOpenGeniWorkerService(
691
873
  worker: activeWorkerBundle.worker,
692
874
  observability,
693
875
  closeOwnedResources: async () => {
876
+ memoryPressureGuard?.close();
694
877
  httpServer?.stop(true);
695
878
  await Promise.allSettled([
879
+ turnCapacityMonitor?.close(),
696
880
  activeWorkerBundle.connection.close(),
697
881
  activeSignaler?.close(),
698
882
  ...schedules.map((schedule) => schedule.close()),
@@ -715,19 +899,40 @@ export async function createOpenGeniWorkerService(
715
899
  }),
716
900
  maxConcurrentWorkflowTaskExecutions:
717
901
  options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
902
+ maxCachedWorkflows: options.role === "control" ? CONTROL_WORKER_MAX_CACHED_WORKFLOWS : 0,
718
903
  httpPort: options.http === false ? null : settings.workerHttpPort,
719
904
  });
720
905
  },
721
906
  });
907
+ const activeLifecycle = lifecycle;
908
+
909
+ if (turnWorkerMemoryPressureGuardEnabled(options.role, settings)) {
910
+ try {
911
+ memoryPressureGuard = createTurnWorkerMemoryPressureGuard({
912
+ settings,
913
+ observability,
914
+ drain: () => {
915
+ if (!activeLifecycle.drain("memory pressure guard")) {
916
+ throw new Error("worker shutdown request failed");
917
+ }
918
+ },
919
+ });
920
+ } catch (error) {
921
+ await activeLifecycle.close();
922
+ throw error;
923
+ }
924
+ }
722
925
 
723
926
  return {
724
927
  role: options.role,
725
928
  worker: activeWorkerBundle.worker,
726
929
  connection: activeWorkerBundle.connection,
727
- state: lifecycle.state,
728
- run: lifecycle.run,
729
- drain: lifecycle.drain,
730
- close: lifecycle.close,
930
+ state: activeLifecycle.state,
931
+ run: activeLifecycle.run,
932
+ drain: (reason) => {
933
+ activeLifecycle.drain(reason);
934
+ },
935
+ close: activeLifecycle.close,
731
936
  };
732
937
  }
733
938