@opengeni/worker-bundle 0.16.29 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +110 -19
  3. package/dist/activities/api-integrations.d.ts +21 -0
  4. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  5. package/dist/activities/capabilities.d.ts +6 -0
  6. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  7. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  8. package/dist/activities/codex-capacity.d.ts +6 -5
  9. package/dist/activities/codex-image-generation.d.ts +21 -0
  10. package/dist/activities/context-compaction.d.ts +1 -0
  11. package/dist/activities/documents.d.ts +3 -2
  12. package/dist/activities/environment.d.ts +17 -8
  13. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  14. package/dist/activities/file-upload-reaper.d.ts +7 -2
  15. package/dist/activities/gateway-image-generation.d.ts +39 -0
  16. package/dist/activities/gateway-video-generation.d.ts +53 -0
  17. package/dist/activities/generated-images.d.ts +112 -0
  18. package/dist/activities/git-credential-renewal.d.ts +1 -1
  19. package/dist/activities/goals.d.ts +2 -2
  20. package/dist/activities/google-drive-changes.d.ts +99 -0
  21. package/dist/activities/image-generation-operation.d.ts +74 -0
  22. package/dist/activities/image-generation-references.d.ts +25 -0
  23. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  24. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  25. package/dist/activities/packs.d.ts +21 -36
  26. package/dist/activities/parent-wake.d.ts +1 -1
  27. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  28. package/dist/activities/retained-screenshots.d.ts +36 -0
  29. package/dist/activities/rig-verification.d.ts +82 -5
  30. package/dist/activities/run-credential-renewal.d.ts +1 -1
  31. package/dist/activities/run-input.d.ts +3 -1
  32. package/dist/activities/sandbox-images.d.ts +14 -0
  33. package/dist/activities/sandbox-lease.d.ts +47 -9
  34. package/dist/activities/scheduled-tasks.d.ts +2 -2
  35. package/dist/activities/session-state.d.ts +7 -5
  36. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  37. package/dist/activities/streaming.d.ts +1 -1
  38. package/dist/activities/types.d.ts +87 -5
  39. package/dist/activities/video-generation-admission.d.ts +43 -0
  40. package/dist/activities/video-generation-credential.d.ts +24 -0
  41. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  42. package/dist/activities/video-media-validation.d.ts +34 -0
  43. package/dist/activities/video-output-retention.d.ts +36 -0
  44. package/dist/activities/video-reference-staging.d.ts +55 -0
  45. package/dist/activities/workflow-wake.d.ts +2 -2
  46. package/dist/activities/workspace-capture.d.ts +10 -1
  47. package/dist/activities/xai-auth.d.ts +23 -0
  48. package/dist/activities/xai-image-generation.d.ts +26 -0
  49. package/dist/activities/xai-video-generation.d.ts +18 -0
  50. package/dist/activities-control-XURE4VKN.js +7221 -0
  51. package/dist/activities-control-XURE4VKN.js.map +1 -0
  52. package/dist/activities-control.d.ts +297 -0
  53. package/dist/activities-turn-6EBXGGLS.js +16036 -0
  54. package/dist/activities-turn-6EBXGGLS.js.map +1 -0
  55. package/dist/activities-turn.d.ts +17 -0
  56. package/dist/activities.d.ts +138 -150
  57. package/dist/activity-services.d.ts +10 -0
  58. package/dist/artifact-materializer-entry.d.ts +6 -0
  59. package/dist/artifact-materializer-entry.js +2662 -0
  60. package/dist/artifact-materializer-entry.js.map +1 -0
  61. package/dist/artifact-outbox-entry.d.ts +2 -0
  62. package/dist/artifact-outbox-entry.js +681 -0
  63. package/dist/artifact-outbox-entry.js.map +1 -0
  64. package/dist/chunk-5GRJIP5H.js +3092 -0
  65. package/dist/chunk-5GRJIP5H.js.map +1 -0
  66. package/dist/chunk-GRFQTTSQ.js +448 -0
  67. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  68. package/dist/chunk-TYNPCG4U.js +906 -0
  69. package/dist/chunk-TYNPCG4U.js.map +1 -0
  70. package/dist/concurrency.d.ts +108 -5
  71. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  72. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  73. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  74. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  75. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  76. package/dist/editable-artifact-materializer.d.ts +262 -0
  77. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  78. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  79. package/dist/host-export.d.ts +1 -1
  80. package/dist/index.d.ts +189 -10
  81. package/dist/index.js +715 -17380
  82. package/dist/index.js.map +1 -1
  83. package/dist/memory-pressure-guard.d.ts +57 -0
  84. package/dist/model-checkpoint-memory-collector.d.ts +23 -0
  85. package/dist/model-context-contributions.d.ts +40 -0
  86. package/dist/observability-metrics.d.ts +53 -1
  87. package/dist/sandbox-reaper-contract.d.ts +40 -0
  88. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  89. package/dist/sandbox-resume.d.ts +34 -7
  90. package/dist/sandbox-routing.d.ts +6 -0
  91. package/dist/worker-service-lifecycle.d.ts +6 -2
  92. package/dist/workflow-bundle.js +1094 -370
  93. package/dist/workflows/activities.d.ts +8 -2
  94. package/dist/workflows/document-index.d.ts +1 -1
  95. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  96. package/dist/workflows/rig-verification.d.ts +1 -1
  97. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  98. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  99. package/dist/workflows/session.d.ts +2 -2
  100. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  101. package/dist/workflows/video-generation.d.ts +10 -0
  102. package/dist/workflows/workflow-wake.d.ts +1 -1
  103. package/dist/workflows.d.ts +10 -7
  104. package/package.json +35 -15
  105. package/src/activities/agent-run-admission.ts +76 -0
  106. package/src/activities/agent-turn.ts +2600 -340
  107. package/src/activities/api-integrations.ts +138 -0
  108. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  109. package/src/activities/capabilities.ts +10 -0
  110. package/src/activities/codemode-dispatcher.ts +335 -0
  111. package/src/activities/codemode-token-renewal.ts +64 -0
  112. package/src/activities/codex-capacity.ts +54 -7
  113. package/src/activities/codex-image-generation.ts +56 -0
  114. package/src/activities/context-compaction.ts +38 -3
  115. package/src/activities/documents.ts +14 -4
  116. package/src/activities/environment.ts +54 -45
  117. package/src/activities/expiring-material-renewal.ts +1 -1
  118. package/src/activities/file-upload-reaper.ts +34 -3
  119. package/src/activities/gateway-image-generation.ts +146 -0
  120. package/src/activities/gateway-video-generation.ts +298 -0
  121. package/src/activities/generated-images.ts +860 -0
  122. package/src/activities/goals.ts +8 -3
  123. package/src/activities/google-drive-changes.ts +454 -0
  124. package/src/activities/image-generation-operation.ts +333 -0
  125. package/src/activities/image-generation-references.ts +118 -0
  126. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  127. package/src/activities/knowledge-source-sync.ts +2840 -0
  128. package/src/activities/mcp-credentials.ts +1 -0
  129. package/src/activities/packs.ts +75 -152
  130. package/src/activities/parent-wake.ts +13 -1
  131. package/src/activities/retained-screenshot-reaper.ts +2 -2
  132. package/src/activities/retained-screenshots.ts +453 -28
  133. package/src/activities/rig-verification.ts +554 -43
  134. package/src/activities/run-input.ts +22 -6
  135. package/src/activities/sandbox-images.ts +167 -0
  136. package/src/activities/sandbox-lease.ts +1065 -483
  137. package/src/activities/scheduled-tasks.ts +104 -74
  138. package/src/activities/session-state.ts +22 -4
  139. package/src/activities/site-auth-maintenance.ts +269 -0
  140. package/src/activities/streaming.ts +42 -3
  141. package/src/activities/types.ts +104 -5
  142. package/src/activities/video-generation-admission.ts +303 -0
  143. package/src/activities/video-generation-credential.ts +112 -0
  144. package/src/activities/video-generation-reconciliation.ts +771 -0
  145. package/src/activities/video-media-validation.ts +332 -0
  146. package/src/activities/video-output-retention.ts +314 -0
  147. package/src/activities/video-reference-staging.ts +350 -0
  148. package/src/activities/workflow-wake.ts +2 -2
  149. package/src/activities/workspace-capture.ts +44 -10
  150. package/src/activities/xai-auth.ts +122 -0
  151. package/src/activities/xai-image-generation.ts +86 -0
  152. package/src/activities/xai-video-generation.ts +69 -0
  153. package/src/activities-control.ts +58 -0
  154. package/src/activities-turn.ts +40 -0
  155. package/src/activities.ts +53 -124
  156. package/src/activity-services.ts +62 -0
  157. package/src/artifact-materializer-entry.ts +35 -0
  158. package/src/artifact-outbox-entry.ts +29 -0
  159. package/src/concurrency.ts +484 -6
  160. package/src/editable-artifact-hint-broker.ts +235 -0
  161. package/src/editable-artifact-materializer-launcher.ts +515 -0
  162. package/src/editable-artifact-materializer-service.ts +787 -0
  163. package/src/editable-artifact-materializer-storage.ts +413 -0
  164. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  165. package/src/editable-artifact-materializer.ts +1289 -0
  166. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  167. package/src/editable-artifact-outbox-service.ts +438 -0
  168. package/src/index.ts +328 -72
  169. package/src/memory-pressure-guard.ts +489 -0
  170. package/src/model-checkpoint-memory-collector.ts +123 -0
  171. package/src/model-context-contributions.ts +159 -0
  172. package/src/observability-metrics.ts +297 -13
  173. package/src/sandbox-reaper-contract.ts +73 -0
  174. package/src/sandbox-reaper-timeout.ts +79 -0
  175. package/src/sandbox-resume.ts +294 -153
  176. package/src/sandbox-routing.ts +32 -24
  177. package/src/worker-service-lifecycle.ts +80 -14
  178. package/src/workflows/activities.ts +31 -0
  179. package/src/workflows/file-upload-reaper.ts +9 -1
  180. package/src/workflows/knowledge-source-sync.ts +83 -0
  181. package/src/workflows/sandbox-reaper.ts +183 -21
  182. package/src/workflows/scheduled-tasks.ts +42 -6
  183. package/src/workflows/session.ts +20 -6
  184. package/src/workflows/site-auth-maintenance.ts +14 -0
  185. package/src/workflows/video-generation.ts +36 -0
  186. package/src/workflows.ts +18 -1
  187. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  188. package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/index.ts CHANGED
@@ -26,20 +26,20 @@ import {
26
26
  ScheduleAlreadyRunning,
27
27
  ScheduleOverlapPolicy,
28
28
  Client as TemporalClient,
29
+ WorkflowExecutionAlreadyStartedError,
29
30
  } from "@temporalio/client";
30
31
  import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
31
32
  import { existsSync } from "node:fs";
32
33
  import { fileURLToPath } from "node:url";
33
- import { ensureModalRegistryImage } from "@opengeni/runtime";
34
- import {
35
- createControlActivities,
36
- createTurnActivities,
37
- type ActivityDependencies,
38
- } from "./activities";
34
+ import type { createControlActivities } from "./activities-control";
35
+ import type { createTurnActivities } from "./activities-turn";
36
+ import type { ActivityDependencies } from "./activities/types";
39
37
  import type {
40
38
  InspectSessionAttemptActivity,
41
39
  SignalCodexCapacityWorkflow,
42
40
  SignalSessionAttemptQuiesced,
41
+ StartSandboxReaperWorkflow,
42
+ StartVideoGenerationWorkflow,
43
43
  WakeSessionWorkflowSignal,
44
44
  } from "./activities/types";
45
45
  import { turnTaskQueue } from "./workflows/activities";
@@ -50,23 +50,42 @@ import {
50
50
  temporalReadyCheck,
51
51
  type WorkerLifecycleState,
52
52
  } from "./http";
53
- import { observabilityEventLogger } from "./observability-metrics";
53
+ import {
54
+ normalizeTurnTaskQueueStats,
55
+ observabilityEventLogger,
56
+ startTurnCapacityMonitor,
57
+ type TurnTaskQueueStats,
58
+ } from "./observability-metrics";
54
59
  import {
55
60
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
56
61
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
57
62
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
63
+ TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
58
64
  } from "@opengeni/core";
59
65
  import {
66
+ CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
60
67
  CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
61
68
  CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
69
+ createTurnWorkerConcurrencyPlan,
62
70
  turnWorkerConcurrencyLogFields,
63
- turnWorkerConcurrencyOptions,
64
71
  } from "./concurrency";
65
72
  import {
73
+ combineWorkerRunTargets,
66
74
  constructWithOwnedConnection,
67
75
  createWorkerServiceLifecycle,
68
76
  type WorkerServiceLifecycle,
77
+ type WorkerRunTarget,
69
78
  } from "./worker-service-lifecycle";
79
+ import {
80
+ createTurnWorkerMemoryPressureGuard,
81
+ turnWorkerMemoryPressureGuardEnabled,
82
+ type TurnWorkerMemoryPressureGuard,
83
+ } from "./memory-pressure-guard";
84
+ import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
85
+ import {
86
+ SANDBOX_REAPER_V2_WORKFLOW_ID,
87
+ sandboxLifecycleTaskQueue,
88
+ } from "./sandbox-reaper-contract";
70
89
 
71
90
  export {
72
91
  createHostExportPump,
@@ -89,6 +108,8 @@ export {
89
108
  const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
90
109
  export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
91
110
  export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
111
+ export const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
112
+ export const SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1_000;
92
113
  export type OpenGeniWorkerRole = "control" | "turn";
93
114
 
94
115
  export type WorkerOptions = {
@@ -135,6 +156,18 @@ export function temporalWorkflowExecutionNotFound(error: unknown): boolean {
135
156
  return isGrpcServiceError(error) && error.code === 5;
136
157
  }
137
158
 
159
+ /** Load exactly one role's activity graph. Exported for embedded hosts and the
160
+ * process-RSS conformance benchmark; construction remains side-effect free
161
+ * until an activity first resolves its injected services. */
162
+ export async function createDefaultWorkerActivities(
163
+ role: OpenGeniWorkerRole,
164
+ dependencies: ActivityDependencies = {},
165
+ ) {
166
+ return role === "control"
167
+ ? (await import("./activities-control")).createControlActivities(dependencies)
168
+ : (await import("./activities-turn")).createTurnActivities(dependencies);
169
+ }
170
+
138
171
  type WorkerWorkflowDefinition =
139
172
  | { workflowBundle: WorkflowBundleOption }
140
173
  | { workflowsPath: string };
@@ -161,11 +194,24 @@ export function resolveOpenGeniWorkflowDefinition(
161
194
  return { workflowBundle: { codePath } };
162
195
  }
163
196
 
197
+ export function turnCancellationHeartbeatThrottleOptions() {
198
+ return {
199
+ maxHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
200
+ defaultHeartbeatThrottleInterval: TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
201
+ } as const;
202
+ }
203
+
164
204
  export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
165
- worker: Worker;
205
+ worker: WorkerRunTarget;
166
206
  connection: NativeConnection;
167
207
  }> {
168
208
  const settings = options.settings ?? getSettings();
209
+ if (options.role === "control") {
210
+ // The exported lower-level factory is itself poll-capable. Validate here so
211
+ // embedded hosts cannot bypass the reaper budget contract by omitting the
212
+ // higher-level service and its Schedule-registration lifecycle.
213
+ assertSandboxReaperActivityTimeout(settings);
214
+ }
169
215
  const observability =
170
216
  options.activityDependencies?.observability ??
171
217
  createObservability(settings, { component: `worker-${options.role}` });
@@ -178,6 +224,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
178
224
  // Otherwise this is a no-op unless the registry secret + logical ref are both
179
225
  // set. Memoized in the provider, so it runs once per process.
180
226
  if (options.role === "turn") {
227
+ const { ensureModalRegistryImage } = await import("@opengeni/runtime/sandbox");
181
228
  await retryStartupDependency(
182
229
  "Modal private-registry image",
183
230
  () => ensureModalRegistryImage(settings),
@@ -198,50 +245,87 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
198
245
  },
199
246
  ),
200
247
  async (connection) => {
248
+ const activityDependencies = {
249
+ ...options.activityDependencies,
250
+ settings,
251
+ observability,
252
+ };
201
253
  const activities =
202
254
  options.activities ??
203
- (options.role === "control" ? createControlActivities : createTurnActivities)({
204
- ...options.activityDependencies,
205
- settings,
206
- observability,
207
- });
208
- const worker = await Worker.create({
255
+ (await createDefaultWorkerActivities(options.role, activityDependencies));
256
+ const turnConcurrency =
257
+ options.role === "turn"
258
+ ? createTurnWorkerConcurrencyPlan(settings, { observability })
259
+ : null;
260
+ const workflowDefinition =
261
+ options.role === "control"
262
+ ? options.workflowBundle
263
+ ? { workflowBundle: options.workflowBundle }
264
+ : resolveOpenGeniWorkflowDefinition()
265
+ : {};
266
+ const sharedWorkerOptions = {
209
267
  connection,
210
268
  namespace: settings.temporalNamespace,
269
+ activities,
270
+ // Cancellation is delivered through an activity heartbeat. Keep the SDK
271
+ // throttle aligned with runAgentTurn's local heartbeat cadence so the
272
+ // four-second control contract still has time for physical writer drain
273
+ // and exact receipt-gated replacement admission.
274
+ ...turnCancellationHeartbeatThrottleOptions(),
275
+ // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
276
+ // after shutdown() stops polling, in-flight activities get this long to
277
+ // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN.
278
+ shutdownGraceTime: "5s",
279
+ // Hard ceiling inside the pod's termination grace period.
280
+ shutdownForceTime: "100s",
281
+ } as const;
282
+ const worker = await Worker.create({
283
+ ...sharedWorkerOptions,
211
284
  taskQueue:
212
285
  options.role === "control"
213
286
  ? settings.temporalTaskQueue
214
287
  : turnTaskQueue(settings.temporalTaskQueue),
215
288
  ...(options.role === "control"
216
289
  ? {
217
- ...(options.workflowBundle
218
- ? { workflowBundle: options.workflowBundle }
219
- : resolveOpenGeniWorkflowDefinition()),
290
+ ...workflowDefinition,
291
+ reuseV8Context: true,
292
+ workflowThreadPoolSize: 1,
293
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
220
294
  maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
221
295
  maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
222
296
  }
223
- : turnWorkerConcurrencyOptions(settings)),
224
- activities,
225
- // Cancellation is delivered through an activity heartbeat. The SDK would
226
- // otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
227
- // making Pause/Steer take roughly a minute even though runAgentTurn emits a
228
- // heartbeat every ten seconds. Keep delivery bounded independently of the
229
- // heartbeat timeout and local timer cadence.
230
- maxHeartbeatThrottleInterval: "5s",
231
- defaultHeartbeatThrottleInterval: "5s",
232
- // GRACEFUL DEPLOY SHUTDOWN (with the SIGTERM handler in startWorker):
233
- // after shutdown() stops polling, in-flight activities get this long to
234
- // finish naturally; the rest are then CANCELLED with WORKER_SHUTDOWN —
235
- // which triggers agent-turn's same-turn recovery checkpoint instead of a
236
- // heartbeat-timeout worker_death. Short on purpose: a long grace here
237
- // only delays the checkpoint window long turns actually need.
238
- shutdownGraceTime: "5s",
239
- // Hard ceiling INSIDE the pod's terminationGracePeriodSeconds (120s): a
240
- // wedged checkpoint force-stops here, on our terms, rather than riding
241
- // into the kubelet's SIGKILL mid-DB-write.
242
- shutdownForceTime: "100s",
297
+ : turnConcurrency!.options),
243
298
  });
244
- return { worker, connection };
299
+ if (options.role !== "control") {
300
+ turnConcurrency?.admission?.finalizeStartupBaseline();
301
+ return { worker, connection };
302
+ }
303
+
304
+ try {
305
+ const sandboxLifecycleWorker = await Worker.create({
306
+ ...sharedWorkerOptions,
307
+ ...workflowDefinition,
308
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
309
+ reuseV8Context: true,
310
+ workflowThreadPoolSize: 1,
311
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
312
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
313
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
314
+ });
315
+ return {
316
+ worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
317
+ connection,
318
+ };
319
+ } catch (error) {
320
+ try {
321
+ worker.shutdown();
322
+ } catch {
323
+ // Preserve the lifecycle worker construction failure. The base poller
324
+ // still received its shutdown request; a secondary synchronous
325
+ // shutdown error must not replace the startup root cause.
326
+ }
327
+ throw error;
328
+ }
245
329
  },
246
330
  (connection) => connection.close(),
247
331
  );
@@ -260,6 +344,9 @@ export async function createWorkerWorkflowSignaler(
260
344
  signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
261
345
  inspectSessionAttemptActivity: InspectSessionAttemptActivity;
262
346
  signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
347
+ getTurnTaskQueueStats: () => Promise<TurnTaskQueueStats>;
348
+ startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
349
+ startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
263
350
  check: () => Promise<void>;
264
351
  close: () => Promise<void>;
265
352
  }> {
@@ -354,6 +441,61 @@ export async function createWorkerWorkflowSignaler(
354
441
  // another producer may have advanced it with a Pause/Steer that requires
355
442
  // sessionControl. The global dispatcher owns that acknowledgement.
356
443
  },
444
+ getTurnTaskQueueStats: async () => {
445
+ const response = await connection.workflowService.describeTaskQueue({
446
+ namespace: settings.temporalNamespace,
447
+ taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
448
+ // temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
449
+ // the supported DEFAULT mode; `stats.approximateBacklogCount` is the
450
+ // server-documented scaling signal.
451
+ taskQueueType: 2,
452
+ reportStats: true,
453
+ });
454
+ return normalizeTurnTaskQueueStats(response.stats);
455
+ },
456
+ startSandboxReaperWorkflow: async () => {
457
+ try {
458
+ await temporal.workflow.start("sandboxReaperWorkflowV2", {
459
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
460
+ workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
461
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
462
+ args: [],
463
+ });
464
+ return "started";
465
+ } catch (error) {
466
+ if (error instanceof WorkflowExecutionAlreadyStartedError) {
467
+ return "already_running";
468
+ }
469
+ throw error;
470
+ }
471
+ },
472
+ startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
473
+ try {
474
+ await temporal.workflow.start("videoGenerationWorkflow", {
475
+ taskQueue: settings.temporalTaskQueue,
476
+ workflowId: `video-generation:${operationId}`,
477
+ // Reject a concurrent or already-completed run, but allow the repair
478
+ // sweep to restart this exact operation if the workflow itself failed.
479
+ // Every provider/storage transition remains CAS/idempotency fenced in
480
+ // Postgres, so recovery never creates a second logical operation.
481
+ workflowIdReusePolicy: "ALLOW_DUPLICATE_FAILED_ONLY",
482
+ args: [
483
+ {
484
+ accountId,
485
+ workspaceId,
486
+ operationId,
487
+ baseTaskQueue: settings.temporalTaskQueue,
488
+ },
489
+ ],
490
+ });
491
+ return "started";
492
+ } catch (error) {
493
+ if (error instanceof WorkflowExecutionAlreadyStartedError) {
494
+ return "already_running";
495
+ }
496
+ throw error;
497
+ }
498
+ },
357
499
  check: async () => {
358
500
  await connection.workflowService.getSystemInfo({});
359
501
  },
@@ -369,13 +511,14 @@ export async function createWorkerWorkflowSignaler(
369
511
  * sandbox ownership off the activity performs bounded DB outbox repair and
370
512
  * read-only observability projections; it never mutates or terminates sandbox leases.
371
513
  *
372
- * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
514
+ * The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
373
515
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
374
516
  * `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
375
517
  * validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
376
518
  * sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
377
- * worker pool collides on ScheduleAlreadyRunning and no-ops, so the Schedule is
378
- * registered exactly once per deployment.
519
+ * worker pool collides on ScheduleAlreadyRunning and reconciles the same
520
+ * definition. New workers route onto the versioned lifecycle queue; old workers
521
+ * can still run the legacy action during rollout or a full binary rollback.
379
522
  *
380
523
  * Returns a `close()` for the dedicated client connection (separate from the
381
524
  * worker's NativeConnection — the Schedule client is a @temporalio/client).
@@ -384,29 +527,32 @@ export async function registerSandboxReaperSchedule(
384
527
  settings: Settings,
385
528
  observability: Observability,
386
529
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
530
+ assertSandboxReaperActivityTimeout(settings);
387
531
  const connection = await Connection.connect(temporalConnectionOptions(settings));
388
532
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
533
+ const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
534
+ const action = {
535
+ type: "startWorkflow" as const,
536
+ workflowType: "sandboxReaperWorkflow",
537
+ taskQueue: settings.temporalTaskQueue,
538
+ args: [] as [],
539
+ };
540
+ const policies = {
541
+ overlap: ScheduleOverlapPolicy.SKIP,
542
+ catchupWindow: "1m",
543
+ pauseOnFailure: false,
544
+ } as const;
389
545
  try {
390
546
  await temporal.schedule.create({
391
547
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
392
- spec: {
393
- // @every-style interval: fire once per reaper period. The boot invariant
394
- // (config) guarantees reaperPeriod < viewerHolderTTL and
395
- // reaperPeriod + idleGrace < providerLifetime.
396
- intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }],
397
- },
398
- action: {
399
- type: "startWorkflow",
400
- workflowType: "sandboxReaperWorkflow",
401
- taskQueue: settings.temporalTaskQueue,
402
- args: [],
403
- },
404
- policies: {
405
- // A slow sweep must never overlap itself; the next fire is skipped.
406
- overlap: ScheduleOverlapPolicy.SKIP,
407
- catchupWindow: "1m",
408
- pauseOnFailure: false,
409
- },
548
+ // @every-style interval: fire once per reaper period. The boot invariant
549
+ // guarantees reaperPeriod < viewerHolderTTL and
550
+ // reaperPeriod + idleGrace < providerLifetime.
551
+ spec,
552
+ action,
553
+ // The base workflow is a short router on new binaries; legacy binaries
554
+ // still execute the pre-refactor composite under this same overlap fence.
555
+ policies,
410
556
  });
411
557
  observability.info("Registered the global sandbox-lease reaper Schedule", {
412
558
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
@@ -420,12 +566,24 @@ export async function registerSandboxReaperSchedule(
420
566
  };
421
567
  } catch (error) {
422
568
  if (error instanceof ScheduleAlreadyRunning) {
423
- // Another worker in the pool already created it. The Schedule exists
424
- // exactly once — this is the expected no-op on every replica after the
425
- // first. (We do NOT update the spec here: a redeploy with a changed cadence
426
- // is an operational concern handled by deleting+recreating the Schedule.)
427
- observability.info("Global sandbox-lease reaper Schedule already registered", {
569
+ // Never move this action off the base queue: doing so would leave a full
570
+ // rollback with no worker polling the only reaper Schedule. The workflow
571
+ // implementation itself owns the forward-compatible queue handoff.
572
+ await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
573
+ spec,
574
+ action,
575
+ policies,
576
+ state: {
577
+ paused: previous.state.paused,
578
+ ...(previous.state.note ? { note: previous.state.note } : {}),
579
+ ...(previous.state.remainingActions !== undefined
580
+ ? { remainingActions: previous.state.remainingActions }
581
+ : {}),
582
+ },
583
+ }));
584
+ observability.info("Reconciled the global sandbox-lease reaper Schedule", {
428
585
  scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
586
+ temporalTaskQueue: action.taskQueue,
429
587
  });
430
588
  return {
431
589
  registered: false,
@@ -484,6 +642,48 @@ export async function registerFileUploadReaperSchedule(
484
642
  }
485
643
  }
486
644
 
645
+ /** Register the deployment-wide maintained-auth dispatcher. Its one-minute
646
+ * cadence matches the minimum public policy interval; DB claims own overlap,
647
+ * crash recovery, and exact session idempotency. */
648
+ export async function registerSiteAuthMaintenanceSchedule(
649
+ settings: Settings,
650
+ observability: Observability,
651
+ ): Promise<{ registered: boolean; close: () => Promise<void> }> {
652
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
653
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
654
+ try {
655
+ await temporal.schedule.create({
656
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
657
+ spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
658
+ action: {
659
+ type: "startWorkflow",
660
+ workflowType: "siteAuthMaintenanceWorkflow",
661
+ taskQueue: settings.temporalTaskQueue,
662
+ args: [],
663
+ },
664
+ policies: {
665
+ overlap: ScheduleOverlapPolicy.SKIP,
666
+ catchupWindow: "1m",
667
+ pauseOnFailure: false,
668
+ },
669
+ });
670
+ observability.info("Registered the global site-auth maintenance Schedule", {
671
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
672
+ maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
673
+ });
674
+ return { registered: true, close: async () => connection.close() };
675
+ } catch (error) {
676
+ if (error instanceof ScheduleAlreadyRunning) {
677
+ observability.info("Global site-auth maintenance Schedule already registered", {
678
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
679
+ });
680
+ return { registered: false, close: async () => connection.close() };
681
+ }
682
+ await connection.close().catch(() => undefined);
683
+ throw error;
684
+ }
685
+ }
686
+
487
687
  /**
488
688
  * Register the one repair cadence for committed workflow-wake revisions. The
489
689
  * activity only reads the transactional outbox and sends revision-scoped
@@ -548,7 +748,7 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
548
748
 
549
749
  export type OpenGeniWorkerService = {
550
750
  readonly role: OpenGeniWorkerRole;
551
- readonly worker: Worker;
751
+ readonly worker: WorkerRunTarget;
552
752
  readonly connection: NativeConnection;
553
753
  state(): WorkerLifecycleState;
554
754
  run(): Promise<void>;
@@ -582,15 +782,20 @@ export async function createOpenGeniWorkerService(
582
782
  let lifecycle: WorkerServiceLifecycle | undefined;
583
783
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
584
784
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
785
+ let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
585
786
  const schedules: Array<{ close: () => Promise<void> }> = [];
586
787
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
788
+ let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
587
789
 
588
790
  try {
589
791
  const needsSignaler =
792
+ options.role === "turn" ||
590
793
  !options.activityDependencies.wakeSessionWorkflow ||
591
794
  !options.activityDependencies.signalSessionAttemptQuiesced ||
592
795
  !options.activityDependencies.inspectSessionAttemptActivity ||
593
- !options.activityDependencies.signalCodexCapacityWorkflow;
796
+ !options.activityDependencies.signalCodexCapacityWorkflow ||
797
+ !options.activityDependencies.startSandboxReaperWorkflow ||
798
+ !options.activityDependencies.startVideoGenerationWorkflow;
594
799
  if (needsSignaler) {
595
800
  signaler = await retryStartupDependency(
596
801
  "Temporal client",
@@ -609,11 +814,19 @@ export async function createOpenGeniWorkerService(
609
814
  const signalCodexCapacityWorkflow =
610
815
  options.activityDependencies.signalCodexCapacityWorkflow ??
611
816
  signaler?.signalCodexCapacityWorkflow;
817
+ const startSandboxReaperWorkflow =
818
+ options.activityDependencies.startSandboxReaperWorkflow ??
819
+ signaler?.startSandboxReaperWorkflow;
820
+ const startVideoGenerationWorkflow =
821
+ options.activityDependencies.startVideoGenerationWorkflow ??
822
+ signaler?.startVideoGenerationWorkflow;
612
823
  if (
613
824
  !wakeSessionWorkflow ||
614
825
  !signalSessionAttemptQuiesced ||
615
826
  !inspectSessionAttemptActivity ||
616
- !signalCodexCapacityWorkflow
827
+ !signalCodexCapacityWorkflow ||
828
+ !startSandboxReaperWorkflow ||
829
+ !startVideoGenerationWorkflow
617
830
  ) {
618
831
  throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
619
832
  }
@@ -630,9 +843,21 @@ export async function createOpenGeniWorkerService(
630
843
  signalSessionAttemptQuiesced,
631
844
  inspectSessionAttemptActivity,
632
845
  signalCodexCapacityWorkflow,
846
+ startSandboxReaperWorkflow,
847
+ startVideoGenerationWorkflow,
633
848
  },
634
849
  });
635
850
 
851
+ if (options.role === "turn") {
852
+ if (!signaler) {
853
+ throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
854
+ }
855
+ turnCapacityMonitor = startTurnCapacityMonitor({
856
+ observability,
857
+ read: signaler.getTurnTaskQueueStats,
858
+ });
859
+ }
860
+
636
861
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
637
862
  schedules.push(
638
863
  await retryStartupDependency(
@@ -648,6 +873,13 @@ export async function createOpenGeniWorkerService(
648
873
  { ...retryOptions, onRetry },
649
874
  ),
650
875
  );
876
+ schedules.push(
877
+ await retryStartupDependency(
878
+ "Temporal schedule (site auth maintenance)",
879
+ () => registerSiteAuthMaintenanceSchedule(settings, observability),
880
+ { ...retryOptions, onRetry },
881
+ ),
882
+ );
651
883
  schedules.push(
652
884
  await retryStartupDependency(
653
885
  "Temporal schedule (session-workflow wake dispatcher)",
@@ -673,6 +905,7 @@ export async function createOpenGeniWorkerService(
673
905
  } catch (error) {
674
906
  httpServer?.stop(true);
675
907
  await Promise.allSettled([
908
+ turnCapacityMonitor?.close(),
676
909
  workerBundle?.connection.close(),
677
910
  signaler?.close(),
678
911
  ...schedules.map((schedule) => schedule.close()),
@@ -691,8 +924,10 @@ export async function createOpenGeniWorkerService(
691
924
  worker: activeWorkerBundle.worker,
692
925
  observability,
693
926
  closeOwnedResources: async () => {
927
+ memoryPressureGuard?.close();
694
928
  httpServer?.stop(true);
695
929
  await Promise.allSettled([
930
+ turnCapacityMonitor?.close(),
696
931
  activeWorkerBundle.connection.close(),
697
932
  activeSignaler?.close(),
698
933
  ...schedules.map((schedule) => schedule.close()),
@@ -715,19 +950,40 @@ export async function createOpenGeniWorkerService(
715
950
  }),
716
951
  maxConcurrentWorkflowTaskExecutions:
717
952
  options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
953
+ maxCachedWorkflows: options.role === "control" ? CONTROL_WORKER_MAX_CACHED_WORKFLOWS : 0,
718
954
  httpPort: options.http === false ? null : settings.workerHttpPort,
719
955
  });
720
956
  },
721
957
  });
958
+ const activeLifecycle = lifecycle;
959
+
960
+ if (turnWorkerMemoryPressureGuardEnabled(options.role, settings)) {
961
+ try {
962
+ memoryPressureGuard = createTurnWorkerMemoryPressureGuard({
963
+ settings,
964
+ observability,
965
+ drain: () => {
966
+ if (!activeLifecycle.drain("memory pressure guard")) {
967
+ throw new Error("worker shutdown request failed");
968
+ }
969
+ },
970
+ });
971
+ } catch (error) {
972
+ await activeLifecycle.close();
973
+ throw error;
974
+ }
975
+ }
722
976
 
723
977
  return {
724
978
  role: options.role,
725
979
  worker: activeWorkerBundle.worker,
726
980
  connection: activeWorkerBundle.connection,
727
- state: lifecycle.state,
728
- run: lifecycle.run,
729
- drain: lifecycle.drain,
730
- close: lifecycle.close,
981
+ state: activeLifecycle.state,
982
+ run: activeLifecycle.run,
983
+ drain: (reason) => {
984
+ activeLifecycle.drain(reason);
985
+ },
986
+ close: activeLifecycle.close,
731
987
  };
732
988
  }
733
989