@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
package/dist/index.js CHANGED
@@ -1,25 +1,30 @@
1
1
  import {
2
2
  createHostExportPump
3
- } from "./chunk-RAMNYSAW.js";
3
+ } from "./chunk-4ER3YXOT.js";
4
4
  import {
5
5
  CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
6
6
  CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
7
7
  CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
8
+ SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
8
9
  SANDBOX_REAPER_V2_WORKFLOW_ID,
9
10
  assertSandboxReaperActivityTimeout,
10
11
  createTurnWorkerConcurrencyPlan,
11
12
  sandboxLifecycleTaskQueue,
12
13
  turnWorkerConcurrencyLogFields
13
- } from "./chunk-GDCLDKSH.js";
14
+ } from "./chunk-24GP3BX5.js";
14
15
  import {
16
+ createTelemetryClock,
15
17
  initializeContextCompactionMetrics,
16
18
  initializeWorkerOutcomeMetrics,
17
19
  normalizeTurnTaskQueueStats,
18
- observabilityEventLogger,
20
+ observabilityEventBusOptions,
19
21
  startContextCompactionPendingMonitor,
20
22
  startSessionRecoveryMonitor,
21
- startTurnCapacityMonitor
22
- } from "./chunk-I7HKKMUJ.js";
23
+ startTurnCapacityMonitor,
24
+ telemetryInterval,
25
+ telemetryScheduler,
26
+ turnTaskQueueMetricLabels
27
+ } from "./chunk-5NNS4327.js";
23
28
  import "./chunk-PZ5AY32C.js";
24
29
 
25
30
  // src/index.ts
@@ -29,15 +34,15 @@ import {
29
34
  resolveNatsControlPlaneAuth,
30
35
  retryStartupDependency,
31
36
  startupRetryOptions,
32
- temporalConnectionOptions
37
+ temporalConnectionOptions as temporalConnectionOptions2
33
38
  } from "@opengeni/config";
34
39
  import {
35
40
  assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
36
41
  isRetryableRuntimeDatabaseStartupError,
37
- countSessionRecoveryBacklog,
38
42
  createDb,
39
43
  getContextCompactionPendingSummary,
40
- markSessionWorkflowWakeDelivered
44
+ markSessionWorkflowWakeDelivered,
45
+ summarizeSessionRecoveryBacklog
41
46
  } from "@opengeni/db";
42
47
  import {
43
48
  createNatsEventBus,
@@ -48,19 +53,25 @@ import {
48
53
  logStartupDependencyRetry
49
54
  } from "@opengeni/observability";
50
55
  import {
51
- Connection,
56
+ Connection as Connection3,
52
57
  isGrpcServiceError,
53
58
  ScheduleAlreadyRunning,
54
59
  ScheduleOverlapPolicy,
55
- Client as TemporalClient,
56
- WorkflowExecutionAlreadyStartedError
60
+ Client as TemporalClient2,
61
+ WorkflowExecutionAlreadyStartedError as WorkflowExecutionAlreadyStartedError2
57
62
  } from "@temporalio/client";
58
63
  import { NativeConnection, Worker } from "@temporalio/worker";
59
64
  import { existsSync } from "fs";
65
+ import { prepareObjectStorageProvider } from "@opengeni/storage";
60
66
  import { fileURLToPath } from "url";
61
67
 
62
68
  // src/workflows/activities.ts
63
- import { ActivityCancellationType, proxyActivities } from "@temporalio/workflow";
69
+ import {
70
+ ActivityCancellationType,
71
+ ActivityFailure,
72
+ ApplicationFailure,
73
+ proxyActivities
74
+ } from "@temporalio/workflow";
64
75
 
65
76
  // src/knowledge-source-sync-activity-policy.ts
66
77
  var KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 12e4;
@@ -130,6 +141,251 @@ var knowledgeSourceSyncActivity = proxyActivities({
130
141
  }
131
142
  });
132
143
 
144
+ // src/turn-task-queue-reader.ts
145
+ import { Connection } from "@temporalio/client";
146
+ function createTurnTaskQueueStatsClient(connectionOptions, input = {}) {
147
+ const scheduler = input.scheduler ?? telemetryScheduler;
148
+ const closeTimeoutMs = telemetryInterval(
149
+ input.closeTimeoutMs ?? 1e3,
150
+ "queue client closeTimeoutMs"
151
+ );
152
+ let connection;
153
+ let closed = false;
154
+ let closing;
155
+ const active = /* @__PURE__ */ new Set();
156
+ return {
157
+ read: async ({ signal, deadline }, identity) => {
158
+ signal.throwIfAborted();
159
+ if (closed) throw new Error("turn task-queue diagnostic client is closed");
160
+ const controller = new AbortController();
161
+ const abort = () => controller.abort(signal.reason);
162
+ signal.addEventListener("abort", abort, { once: true });
163
+ active.add(controller);
164
+ try {
165
+ connection ??= Connection.lazy({ ...connectionOptions, interceptors: [] });
166
+ const stats = await createTurnTaskQueueStatsReader(
167
+ connection,
168
+ identity
169
+ )({
170
+ signal: controller.signal,
171
+ deadline
172
+ });
173
+ controller.signal.throwIfAborted();
174
+ return stats;
175
+ } finally {
176
+ active.delete(controller);
177
+ signal.removeEventListener("abort", abort);
178
+ }
179
+ },
180
+ close: () => {
181
+ if (closing) return closing;
182
+ closed = true;
183
+ for (const controller of active) controller.abort();
184
+ const owned = connection;
185
+ if (!owned) return closing = Promise.resolve();
186
+ closing = new Promise((resolve) => {
187
+ const cancelTimeout = scheduler.timeout(resolve, closeTimeoutMs);
188
+ const settled = () => {
189
+ cancelTimeout();
190
+ resolve();
191
+ };
192
+ try {
193
+ void owned.close().then(settled, settled);
194
+ } catch {
195
+ settled();
196
+ }
197
+ });
198
+ return closing;
199
+ }
200
+ };
201
+ }
202
+ function createTurnTaskQueueStatsReader(connection, identity) {
203
+ return async ({ signal, deadline }) => {
204
+ signal.throwIfAborted();
205
+ return connection.withAbortSignal(
206
+ signal,
207
+ () => connection.withDeadline(deadline, async () => {
208
+ signal.throwIfAborted();
209
+ const response = await connection.workflowService.describeTaskQueue({
210
+ namespace: identity.temporalNamespace,
211
+ taskQueue: { name: identity.taskQueue },
212
+ // TASK_QUEUE_TYPE_ACTIVITY, supported DEFAULT mode. This is the shared
213
+ // turn-worker activity queue (also video/retries), not a turn count.
214
+ taskQueueType: 2,
215
+ reportStats: true
216
+ });
217
+ return normalizeTurnTaskQueueStats(response.stats);
218
+ })
219
+ );
220
+ };
221
+ }
222
+
223
+ // src/temporal-runtime-logger.ts
224
+ import { createTemporalSdkLogger } from "@opengeni/observability";
225
+ import { makeTelemetryFilterString, Runtime } from "@temporalio/worker";
226
+ function temporalRuntimeLoggingOptions(observability) {
227
+ return {
228
+ logger: createTemporalSdkLogger(observability),
229
+ telemetryOptions: {
230
+ logging: {
231
+ // The SDK's default core filter, now forwarded instead of printed.
232
+ filter: makeTelemetryFilterString({ core: "WARN", other: "ERROR" }),
233
+ forward: {}
234
+ }
235
+ }
236
+ };
237
+ }
238
+ function installTemporalRuntimeLogging(observability, runtime = Runtime) {
239
+ if (runtime._instance !== void 0) return false;
240
+ runtime.install(temporalRuntimeLoggingOptions(observability));
241
+ return true;
242
+ }
243
+
244
+ // src/turn-worker-activity-telemetry.ts
245
+ var TURN_WORKER_ACTIVITY_METRICS = {
246
+ inflight: "opengeni_turn_worker_activities_inflight",
247
+ timestamp: "opengeni_turn_worker_activities_last_observed_timestamp_seconds",
248
+ valid: "opengeni_turn_worker_activities_last_read_success"
249
+ };
250
+ function closeWorkerHttpAfterActivityExecution(telemetry, close) {
251
+ telemetry?.closeIfNeverRun();
252
+ if (!telemetry || telemetry.isSettled()) {
253
+ try {
254
+ void Promise.resolve(close()).catch(() => void 0);
255
+ } catch {
256
+ }
257
+ } else {
258
+ void telemetry.settled.then(close).catch(() => void 0);
259
+ }
260
+ }
261
+ function withTurnWorkerActivityTelemetry(sdkWorker, input) {
262
+ const scheduler = input.scheduler ?? telemetryScheduler;
263
+ const intervalMs = telemetryInterval(input.intervalMs ?? 15e3, "activity telemetry intervalMs");
264
+ const clock = createTelemetryClock(input.now ?? Date.now);
265
+ const labels = { ...turnTaskQueueMetricLabels(input.identity), worker_pod_uid: input.podUid };
266
+ let runStarted = false;
267
+ let runSettled = false;
268
+ let runPromise;
269
+ let disposed = false;
270
+ let queued = false;
271
+ let subscription;
272
+ let stopTimer;
273
+ let resolveSettled;
274
+ const settled = new Promise((resolve) => {
275
+ resolveSettled = resolve;
276
+ });
277
+ const set = (name, help, value) => input.observability.setGauge({ name, help, labels, value });
278
+ const invalidate = () => {
279
+ if (disposed) return;
280
+ try {
281
+ set(
282
+ TURN_WORKER_ACTIVITY_METRICS.valid,
283
+ "Whether the latest SDK activity occupancy observation succeeded.",
284
+ 0
285
+ );
286
+ } catch {
287
+ }
288
+ };
289
+ const dispose = () => {
290
+ if (disposed) return;
291
+ disposed = true;
292
+ stopTimer?.();
293
+ try {
294
+ subscription?.unsubscribe();
295
+ } catch {
296
+ }
297
+ resolveSettled();
298
+ };
299
+ const sample = () => {
300
+ if (disposed) return;
301
+ let idle = false;
302
+ try {
303
+ const status = sdkWorker.getStatus();
304
+ const count = status.numInFlightNonLocalActivities;
305
+ const local = status.numInFlightLocalActivities;
306
+ const total = status.numInFlightActivities;
307
+ if (![count, local, total].every((value) => Number.isSafeInteger(value) && value >= 0) || count + local !== total) {
308
+ throw new Error("invalid SDK executing activity state");
309
+ }
310
+ idle = total === 0;
311
+ const observedAt = clock.read();
312
+ if (observedAt === null) throw new Error("invalid activity observation clock");
313
+ set(
314
+ TURN_WORKER_ACTIVITY_METRICS.valid,
315
+ "Whether the latest SDK activity occupancy observation succeeded.",
316
+ 0
317
+ );
318
+ set(
319
+ TURN_WORKER_ACTIVITY_METRICS.inflight,
320
+ "SDK executing non-local activities on this turn worker, including preclaim, video and cancellation-unresolved execution.",
321
+ count
322
+ );
323
+ set(
324
+ TURN_WORKER_ACTIVITY_METRICS.timestamp,
325
+ "Unix timestamp of the latest successful SDK activity occupancy observation.",
326
+ observedAt / 1e3
327
+ );
328
+ set(
329
+ TURN_WORKER_ACTIVITY_METRICS.valid,
330
+ "Whether the latest SDK activity occupancy observation succeeded.",
331
+ 1
332
+ );
333
+ } catch {
334
+ invalidate();
335
+ }
336
+ if (runSettled && idle) dispose();
337
+ };
338
+ const scheduleSample = () => {
339
+ if (disposed || queued) return;
340
+ queued = true;
341
+ scheduler.defer(() => {
342
+ queued = false;
343
+ sample();
344
+ });
345
+ };
346
+ try {
347
+ subscription = sdkWorker.numInFlightActivities$.subscribe({
348
+ next: scheduleSample,
349
+ error: invalidate
350
+ });
351
+ } catch {
352
+ invalidate();
353
+ }
354
+ stopTimer = scheduler.interval(sample, intervalMs);
355
+ sample();
356
+ const activityTelemetry = {
357
+ settled,
358
+ isSettled: () => disposed,
359
+ closeIfNeverRun: () => {
360
+ if (!runStarted) {
361
+ runSettled = true;
362
+ sample();
363
+ }
364
+ }
365
+ };
366
+ return {
367
+ activityTelemetry,
368
+ worker: {
369
+ shutdown: () => sdkWorker.shutdown(),
370
+ // Drain never unregisters observation.
371
+ run: () => {
372
+ if (!runPromise && runSettled)
373
+ return Promise.reject(new Error("cannot run a closed turn worker"));
374
+ runPromise ??= (async () => {
375
+ runStarted = true;
376
+ try {
377
+ await sdkWorker.run();
378
+ } finally {
379
+ runSettled = true;
380
+ sample();
381
+ }
382
+ })();
383
+ return runPromise;
384
+ }
385
+ }
386
+ };
387
+ }
388
+
133
389
  // src/http.ts
134
390
  import {
135
391
  assertRuntimeDatabasePosture,
@@ -210,16 +466,17 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
210
466
  checks: result
211
467
  };
212
468
  }
213
- function dbReadyCheck(db, posture) {
469
+ function dbReadyCheck(db, posture, afterDatabaseReady) {
214
470
  let inFlight;
215
471
  return () => {
216
472
  if (inFlight) return inFlight;
217
473
  inFlight = (async () => {
218
474
  if (posture) {
219
475
  await assertRuntimeDatabasePosture(db, posture);
220
- return;
476
+ } else {
477
+ await db.execute(dbSql`select 1`);
221
478
  }
222
- await db.execute(dbSql`select 1`);
479
+ await afterDatabaseReady?.();
223
480
  })().finally(() => {
224
481
  inFlight = void 0;
225
482
  });
@@ -264,11 +521,362 @@ async function withTimeout(promise, timeoutMs) {
264
521
  // src/index.ts
265
522
  import {
266
523
  resolveCatalogSettings,
524
+ TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS
525
+ } from "@opengeni/core";
526
+
527
+ // src/internal-maintenance.ts
528
+ import { temporalConnectionOptions } from "@opengeni/config";
529
+ import {
530
+ SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
267
531
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
268
532
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
269
- SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
270
- TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS
533
+ SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
534
+ SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE
271
535
  } from "@opengeni/core";
536
+ import {
537
+ automaticSessionTitleFanoutDue,
538
+ browserDeadlineCheckpointDue,
539
+ knowledgeIndexDue,
540
+ sandboxIdleCheckpointDue,
541
+ sandboxLeaseReaperDue,
542
+ sandboxMaintenanceDue,
543
+ sessionWorkflowWakesDue,
544
+ siteAuthMaintenanceDue,
545
+ workspacePauseTimerDue
546
+ } from "@opengeni/db";
547
+ import {
548
+ Connection as Connection2,
549
+ ScheduleNotFoundError,
550
+ Client as TemporalClient,
551
+ WorkflowExecutionAlreadyStartedError
552
+ } from "@temporalio/client";
553
+
554
+ // src/maintenance-cadence.ts
555
+ var TICK_OUTCOMES = [
556
+ "idle",
557
+ "started",
558
+ "already_running",
559
+ "probe_failed",
560
+ "start_failed",
561
+ "busy"
562
+ ];
563
+ function startMaintenanceCadences(input) {
564
+ const scheduler = input.scheduler ?? telemetryScheduler;
565
+ const now = input.now ?? Date.now;
566
+ const jobs = /* @__PURE__ */ new Map();
567
+ for (const cadence of input.cadences) {
568
+ if (jobs.has(cadence.job)) throw new Error(`duplicate maintenance cadence ${cadence.job}`);
569
+ telemetryInterval(cadence.periodMs, `${cadence.job} maintenance period`);
570
+ jobs.set(cadence.job, cadence);
571
+ for (const outcome of TICK_OUTCOMES) {
572
+ input.observability.incrementCounter({
573
+ name: "opengeni_maintenance_cadence_ticks_total",
574
+ help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
575
+ labels: { job: cadence.job, outcome },
576
+ amount: 0
577
+ });
578
+ }
579
+ }
580
+ const inFlight = /* @__PURE__ */ new Map();
581
+ const cancels = [];
582
+ let stopped = false;
583
+ const record = (job, outcome) => {
584
+ input.observability.incrementCounter({
585
+ name: "opengeni_maintenance_cadence_ticks_total",
586
+ help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
587
+ labels: { job, outcome }
588
+ });
589
+ if (outcome !== "busy" && outcome !== "start_failed") {
590
+ input.observability.setGauge({
591
+ name: "opengeni_maintenance_cadence_last_success_timestamp_seconds",
592
+ help: "Unix time of the latest maintenance cadence tick that checked for work and, when needed, started its workflow.",
593
+ labels: { job },
594
+ value: now() / 1e3
595
+ });
596
+ }
597
+ return outcome;
598
+ };
599
+ const run = async (cadence) => {
600
+ let probeFailed = false;
601
+ try {
602
+ if (!await cadence.hasWork()) return record(cadence.job, "idle");
603
+ } catch (error) {
604
+ probeFailed = true;
605
+ input.observability.warn("maintenance cadence work check failed; starting the job anyway", {
606
+ job: cadence.job,
607
+ error: error instanceof Error ? error.message : String(error)
608
+ });
609
+ }
610
+ try {
611
+ const started = await cadence.start();
612
+ return record(cadence.job, probeFailed ? "probe_failed" : started);
613
+ } catch (error) {
614
+ input.observability.warn("maintenance cadence could not start its workflow", {
615
+ job: cadence.job,
616
+ error: error instanceof Error ? error.message : String(error)
617
+ });
618
+ return record(cadence.job, "start_failed");
619
+ }
620
+ };
621
+ const tick = async (job) => {
622
+ const cadence = jobs.get(job);
623
+ if (!cadence) throw new Error(`unknown maintenance cadence ${job}`);
624
+ if (stopped) return "busy";
625
+ if (inFlight.has(job)) return record(job, "busy");
626
+ const running = run(cadence).finally(() => inFlight.delete(job));
627
+ inFlight.set(job, running);
628
+ return await running;
629
+ };
630
+ if (input.autoStart !== false) {
631
+ for (const cadence of jobs.values()) {
632
+ const fire = () => {
633
+ void tick(cadence.job);
634
+ };
635
+ scheduler.defer(fire);
636
+ cancels.push(scheduler.interval(fire, cadence.periodMs));
637
+ }
638
+ }
639
+ return {
640
+ tick,
641
+ close: async () => {
642
+ stopped = true;
643
+ for (const cancel of cancels.splice(0)) cancel();
644
+ await Promise.allSettled([...inFlight.values()]);
645
+ }
646
+ };
647
+ }
648
+
649
+ // src/browser-deadline-checkpoint-contract.ts
650
+ var BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID = "opengeni-browser-deadline-checkpoints-v1";
651
+ var QUEUE_SUFFIX = "-browser-checkpoint-v1";
652
+ function browserDeadlineCheckpointTaskQueue(base) {
653
+ return base.endsWith(QUEUE_SUFFIX) ? base : `${base}${QUEUE_SUFFIX}`;
654
+ }
655
+
656
+ // src/internal-maintenance.ts
657
+ var KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
658
+ var KNOWLEDGE_INDEXING_WORKFLOW_ID = "opengeni-knowledge-indexing";
659
+ var KNOWLEDGE_INDEXING_PERIOD_MS = 15e3;
660
+ var SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
661
+ var SITE_AUTH_MAINTENANCE_WORKFLOW_ID = "opengeni-site-auth-maintenance";
662
+ var SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1e3;
663
+ var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
664
+ var INTERNAL_MAINTENANCE_JOBS = [
665
+ "session-workflow-wake-dispatcher",
666
+ "knowledge-indexing",
667
+ "site-auth-maintenance",
668
+ "sandbox-lease-reaper",
669
+ "sandbox-idle-checkpoint",
670
+ "browser-deadline-checkpoint"
671
+ ];
672
+ var LEGACY_SCHEDULE_IDS = {
673
+ "session-workflow-wake-dispatcher": SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
674
+ "knowledge-indexing": KNOWLEDGE_INDEXING_SCHEDULE_ID,
675
+ "site-auth-maintenance": SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
676
+ // The old reaper Schedule fired a router workflow that started the drain
677
+ // inventory, both checkpoint sweeps and the maintenance child on every tick.
678
+ "sandbox-lease-reaper": SANDBOX_REAPER_SCHEDULE_ID
679
+ };
680
+ async function startMaintenanceWorkflowOnce(workflow, input) {
681
+ try {
682
+ await workflow.start(input.workflowType, {
683
+ taskQueue: input.taskQueue,
684
+ workflowId: input.workflowId,
685
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
686
+ args: []
687
+ });
688
+ return "started";
689
+ } catch (error) {
690
+ if (error instanceof WorkflowExecutionAlreadyStartedError) return "already_running";
691
+ throw error;
692
+ }
693
+ }
694
+ async function deleteLegacyMaintenanceSchedules(schedule, jobs, observability) {
695
+ for (const job of jobs) {
696
+ const scheduleId = LEGACY_SCHEDULE_IDS[job];
697
+ if (!scheduleId) continue;
698
+ try {
699
+ await schedule.getHandle(scheduleId).delete();
700
+ observability.info(
701
+ "Deleted a per-tick maintenance Schedule replaced by a work-checked cadence",
702
+ {
703
+ job,
704
+ scheduleId
705
+ }
706
+ );
707
+ } catch (error) {
708
+ if (!(error instanceof ScheduleNotFoundError)) throw error;
709
+ }
710
+ }
711
+ }
712
+ async function internalMaintenanceCadences(input) {
713
+ const { settings, observability, db, workflow } = input;
714
+ const jobs = new Set(input.jobs ?? INTERNAL_MAINTENANCE_JOBS);
715
+ const taskQueue = settings.temporalTaskQueue;
716
+ const cadences = [];
717
+ if (jobs.has("session-workflow-wake-dispatcher")) {
718
+ const { repairMissedSessionWorkflowWakes } = await import("./parent-wake-PCLH7A3Y.js");
719
+ cadences.push({
720
+ job: "session-workflow-wake-dispatcher",
721
+ periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
722
+ hasWork: async () => {
723
+ await repairMissedSessionWorkflowWakes({ db, observability });
724
+ return await sessionWorkflowWakesDue(db) || await workspacePauseTimerDue(db) || await automaticSessionTitleFanoutDue(db);
725
+ },
726
+ start: async () => {
727
+ await workflow.signalWithStart(SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE, {
728
+ taskQueue,
729
+ workflowId: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
730
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
731
+ args: [],
732
+ signal: SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL
733
+ });
734
+ return "started";
735
+ }
736
+ });
737
+ }
738
+ if (jobs.has("knowledge-indexing")) {
739
+ let embedding;
740
+ let unavailableReported = false;
741
+ const resolveEmbedding = async () => {
742
+ const services = input.documentServices ?? (await import("@opengeni/documents")).createDocumentServices(settings);
743
+ return {
744
+ model: services.embedder.model,
745
+ dimensions: services.embedder.dimensions,
746
+ available: services.embedder.available !== false
747
+ };
748
+ };
749
+ cadences.push({
750
+ job: "knowledge-indexing",
751
+ periodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
752
+ hasWork: async () => {
753
+ embedding ??= resolveEmbedding().catch((error) => {
754
+ embedding = void 0;
755
+ throw error;
756
+ });
757
+ const current = await embedding;
758
+ if (!current.available) {
759
+ if (!unavailableReported) {
760
+ unavailableReported = true;
761
+ observability.warn(
762
+ "Knowledge indexing is idle: no document embedding credential is configured",
763
+ { job: "knowledge-indexing", model: current.model }
764
+ );
765
+ }
766
+ return false;
767
+ }
768
+ return await knowledgeIndexDue(db, current);
769
+ },
770
+ start: async () => await startMaintenanceWorkflowOnce(workflow, {
771
+ workflowType: "knowledgeIndexingWorkflow",
772
+ workflowId: KNOWLEDGE_INDEXING_WORKFLOW_ID,
773
+ taskQueue
774
+ })
775
+ });
776
+ }
777
+ if (jobs.has("site-auth-maintenance")) {
778
+ const { SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS } = await import("./site-auth-maintenance-WK3MR2PV.js");
779
+ cadences.push({
780
+ job: "site-auth-maintenance",
781
+ periodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
782
+ hasWork: async () => await siteAuthMaintenanceDue(db, {
783
+ claimTimeoutMs: SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS
784
+ }),
785
+ start: async () => await startMaintenanceWorkflowOnce(workflow, {
786
+ workflowType: "siteAuthMaintenanceWorkflow",
787
+ workflowId: SITE_AUTH_MAINTENANCE_WORKFLOW_ID,
788
+ taskQueue
789
+ })
790
+ });
791
+ }
792
+ const sandboxJobs = ["sandbox-lease-reaper", "sandbox-idle-checkpoint", "browser-deadline-checkpoint"].filter((job) => jobs.has(job));
793
+ if (sandboxJobs.length > 0) {
794
+ assertSandboxReaperActivityTimeout(settings);
795
+ const periodMs = settings.sandboxLeaseReaperPeriodMs;
796
+ const lifecycleQueue = sandboxLifecycleTaskQueue(taskQueue);
797
+ const sandbox = await import("./sandbox-lease-BJYU72N3.js");
798
+ if (jobs.has("sandbox-lease-reaper")) {
799
+ const { SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS } = sandbox;
800
+ cadences.push({
801
+ job: "sandbox-lease-reaper",
802
+ periodMs,
803
+ // The drain inventory and its maintenance pass act only on warm or held
804
+ // leases, due reconciliation rows, or (with Modal) boxes still running in
805
+ // the provider app. With none of those, the tick only refreshes the
806
+ // pass's read-only gauges in this process.
807
+ hasWork: async () => {
808
+ if (await sandboxLeaseReaperDue(db) || await sandboxMaintenanceDue(db, {
809
+ checkpointArtifactClaimTtlMs: SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS
810
+ }) || await sandbox.modalOrphanSweepMayHaveWork({ settings, observability })) {
811
+ return true;
812
+ }
813
+ await sandbox.refreshIdleSandboxReaperProjections({ db, settings, observability });
814
+ return false;
815
+ },
816
+ start: async () => await startMaintenanceWorkflowOnce(workflow, {
817
+ workflowType: "sandboxReaperWorkflowV2",
818
+ workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
819
+ taskQueue: lifecycleQueue
820
+ })
821
+ });
822
+ }
823
+ if (jobs.has("sandbox-idle-checkpoint")) {
824
+ cadences.push({
825
+ job: "sandbox-idle-checkpoint",
826
+ periodMs,
827
+ // The sweep's inventory is empty when ownership or snapshots are off.
828
+ hasWork: async () => settings.sandboxOwnershipEnabled && settings.sandboxSnapshotIntervalMs > 0 && await sandboxIdleCheckpointDue(db, { intervalMs: settings.sandboxSnapshotIntervalMs }),
829
+ start: async () => await startMaintenanceWorkflowOnce(workflow, {
830
+ workflowType: "sandboxIdleCheckpointSweepWorkflow",
831
+ workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
832
+ taskQueue: lifecycleQueue
833
+ })
834
+ });
835
+ }
836
+ if (jobs.has("browser-deadline-checkpoint")) {
837
+ cadences.push({
838
+ job: "browser-deadline-checkpoint",
839
+ periodMs,
840
+ hasWork: async () => await browserDeadlineCheckpointDue(db),
841
+ start: async () => await startMaintenanceWorkflowOnce(workflow, {
842
+ workflowType: "browserDeadlineCheckpointSweepWorkflow",
843
+ workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
844
+ taskQueue: browserDeadlineCheckpointTaskQueue(taskQueue)
845
+ })
846
+ });
847
+ }
848
+ }
849
+ return cadences;
850
+ }
851
+ async function startInternalMaintenanceCadences(input) {
852
+ const connection = await Connection2.connect(temporalConnectionOptions(input.settings));
853
+ try {
854
+ const temporal = new TemporalClient({
855
+ connection,
856
+ namespace: input.settings.temporalNamespace
857
+ });
858
+ const jobs = input.jobs ?? INTERNAL_MAINTENANCE_JOBS;
859
+ await deleteLegacyMaintenanceSchedules(temporal.schedule, jobs, input.observability);
860
+ const cadences = startMaintenanceCadences({
861
+ cadences: await internalMaintenanceCadences({ ...input, jobs, workflow: temporal.workflow }),
862
+ observability: input.observability,
863
+ ...input.scheduler ? { scheduler: input.scheduler } : {}
864
+ });
865
+ input.observability.info("Started work-checked maintenance cadences", {
866
+ jobs: jobs.join(",")
867
+ });
868
+ return {
869
+ cadences,
870
+ close: async () => {
871
+ await cadences.close();
872
+ await connection.close();
873
+ }
874
+ };
875
+ } catch (error) {
876
+ await connection.close().catch(() => void 0);
877
+ throw error;
878
+ }
879
+ }
272
880
 
273
881
  // src/worker-service-lifecycle.ts
274
882
  async function constructWithOwnedConnection(connect, construct, close) {
@@ -366,7 +974,7 @@ function createWorkerServiceLifecycle(input) {
366
974
  }
367
975
  const previousState = state;
368
976
  safeLifecycleLog(
369
- () => input.observability.info("OpenGeni worker draining (graceful shutdown)", {
977
+ () => input.observability.info("Opengeni worker draining (graceful shutdown)", {
370
978
  role: input.role,
371
979
  errorClass: "WorkerLifecycleOperation",
372
980
  errorCode: "worker_draining",
@@ -409,6 +1017,47 @@ function safeLifecycleLog(log) {
409
1017
  } catch {
410
1018
  }
411
1019
  }
1020
+ function createWorkerCleanupContainment(input) {
1021
+ let requested = false;
1022
+ let finished = false;
1023
+ let timer;
1024
+ return {
1025
+ request() {
1026
+ if (requested || finished) return;
1027
+ requested = true;
1028
+ if (input.terminate) {
1029
+ timer = setTimeout(() => {
1030
+ safeLifecycleLog(
1031
+ () => input.observability.error("worker cleanup drain exhausted; terminating host", {
1032
+ errorClass: "WorkerLifecycleOperationError",
1033
+ errorCode: "worker_cleanup_drain_exhausted",
1034
+ origin: "worker-lifecycle"
1035
+ })
1036
+ );
1037
+ input.terminate?.();
1038
+ }, input.timeoutMs ?? 1e5);
1039
+ timer.unref?.();
1040
+ }
1041
+ let accepted = false;
1042
+ try {
1043
+ accepted = input.drain();
1044
+ } catch {
1045
+ }
1046
+ if (!accepted)
1047
+ safeLifecycleLog(
1048
+ () => input.observability.error("worker cleanup drain request failed", {
1049
+ errorClass: "WorkerLifecycleOperationError",
1050
+ errorCode: "worker_shutdown_request_failed",
1051
+ origin: "worker-lifecycle"
1052
+ })
1053
+ );
1054
+ },
1055
+ finished() {
1056
+ finished = true;
1057
+ clearTimeout(timer);
1058
+ }
1059
+ };
1060
+ }
412
1061
 
413
1062
  // src/memory-pressure-guard.ts
414
1063
  import { readFileSync } from "fs";
@@ -764,13 +1413,10 @@ function installWorkerUnhandledRejectionBoundary(runtimeProcess = process, repor
764
1413
  }
765
1414
 
766
1415
  // src/index.ts
767
- var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
768
- var KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
769
- var KNOWLEDGE_INDEXING_PERIOD_MS = 15e3;
770
1416
  var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
771
1417
  var FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1e3;
772
- var SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
773
- var SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1e3;
1418
+ var SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID = "opengeni-session-storage-maintenance";
1419
+ var SESSION_STORAGE_MAINTENANCE_PERIOD_MS = 5 * 60 * 1e3;
774
1420
  function temporalTimeMs(value) {
775
1421
  if (value?.seconds === void 0 || value.seconds === null) return null;
776
1422
  return Number(String(value.seconds)) * 1e3 + Math.floor((value.nanos ?? 0) / 1e6);
@@ -785,7 +1431,10 @@ function temporalWorkflowExecutionNotFound(error) {
785
1431
  return isGrpcServiceError(error) && error.code === 5;
786
1432
  }
787
1433
  async function createDefaultWorkerActivities(role, dependencies = {}) {
788
- return role === "control" ? (await import("./activities-control-QA6OLRQY.js")).createControlActivities(dependencies) : (await import("./activities-turn-YVVKCGAK.js")).createTurnActivities(dependencies);
1434
+ if (dependencies.objectStorage == null) {
1435
+ prepareObjectStorageProvider(dependencies.settings ?? getSettings());
1436
+ }
1437
+ return role === "control" ? (await import("./activities-control-5IIVRXLA.js")).createControlActivities(dependencies) : (await import("./activities-turn-FLYJOM54.js")).createTurnActivities(dependencies);
789
1438
  }
790
1439
  function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
791
1440
  const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
@@ -795,7 +1444,7 @@ function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
795
1444
  const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
796
1445
  if (!existsSync(codePath)) {
797
1446
  throw new Error(
798
- `OpenGeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`
1447
+ `Opengeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`
799
1448
  );
800
1449
  }
801
1450
  return { workflowBundle: { codePath } };
@@ -833,17 +1482,25 @@ async function createOpenGeniWorker(options) {
833
1482
  return constructWithOwnedConnection(
834
1483
  () => retryStartupDependency(
835
1484
  "Temporal",
836
- () => NativeConnection.connect(temporalConnectionOptions(settings)),
1485
+ () => NativeConnection.connect(temporalConnectionOptions2(settings)),
837
1486
  {
838
1487
  ...startupRetryOptions(settings),
839
1488
  onRetry: (event) => logStartupDependencyRetry(observability, event)
840
1489
  }
841
1490
  ),
842
1491
  async (connection) => {
1492
+ let turnWorker;
1493
+ let cleanupDrainRequested = false;
843
1494
  const activityDependencies = {
844
1495
  ...options.activityDependencies,
845
1496
  settings,
846
- observability
1497
+ observability,
1498
+ requestWorkerDrain: options.activityDependencies?.requestWorkerDrain ?? (() => {
1499
+ if (cleanupDrainRequested) return;
1500
+ if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
1501
+ turnWorker.shutdown();
1502
+ cleanupDrainRequested = true;
1503
+ })
847
1504
  };
848
1505
  const activities = options.activities ?? await createDefaultWorkerActivities(options.role, activityDependencies);
849
1506
  const turnConcurrency = options.role === "turn" ? createTurnWorkerConcurrencyPlan(settings, { observability }) : null;
@@ -877,28 +1534,52 @@ async function createOpenGeniWorker(options) {
877
1534
  } : turnConcurrency.options
878
1535
  });
879
1536
  if (options.role !== "control") {
1537
+ turnWorker = worker;
880
1538
  turnConcurrency?.admission?.finalizeStartupBaseline();
881
- return { worker, connection };
1539
+ const turnQueueIdentity = {
1540
+ temporalNamespace: worker.options.namespace,
1541
+ taskQueue: worker.options.taskQueue
1542
+ };
1543
+ return {
1544
+ ...withTurnWorkerActivityTelemetry(worker, {
1545
+ observability,
1546
+ identity: turnQueueIdentity,
1547
+ // No invented identity when running outside Kubernetes. Scaler
1548
+ // consumers must reject empty/missing UID coverage.
1549
+ podUid: process.env.OPENGENI_POD_UID ?? ""
1550
+ }),
1551
+ connection,
1552
+ turnQueueIdentity
1553
+ };
882
1554
  }
1555
+ const ownedWorkers = [worker];
883
1556
  try {
884
- const sandboxLifecycleWorker = await Worker.create({
885
- ...sharedWorkerOptions,
886
- ...workflowDefinition,
887
- taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
888
- reuseV8Context: true,
889
- workflowThreadPoolSize: 1,
890
- maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
891
- maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
892
- maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
893
- });
1557
+ for (const taskQueue of [
1558
+ sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
1559
+ browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue)
1560
+ ]) {
1561
+ const lifecycleWorker = await Worker.create({
1562
+ ...sharedWorkerOptions,
1563
+ ...workflowDefinition,
1564
+ taskQueue,
1565
+ reuseV8Context: true,
1566
+ workflowThreadPoolSize: 1,
1567
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
1568
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
1569
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
1570
+ });
1571
+ ownedWorkers.push(lifecycleWorker);
1572
+ }
894
1573
  return {
895
- worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
1574
+ worker: combineWorkerRunTargets(ownedWorkers),
896
1575
  connection
897
1576
  };
898
1577
  } catch (error) {
899
- try {
900
- worker.shutdown();
901
- } catch {
1578
+ for (const ownedWorker of ownedWorkers) {
1579
+ try {
1580
+ ownedWorker.shutdown();
1581
+ } catch {
1582
+ }
902
1583
  }
903
1584
  throw error;
904
1585
  }
@@ -907,8 +1588,10 @@ async function createOpenGeniWorker(options) {
907
1588
  );
908
1589
  }
909
1590
  async function createWorkerWorkflowSignaler(settings, db) {
910
- const connection = await Connection.connect(temporalConnectionOptions(settings));
911
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1591
+ const connectionOptions = temporalConnectionOptions2(settings);
1592
+ const connection = await Connection3.connect(connectionOptions);
1593
+ const statsClient = createTurnTaskQueueStatsClient(connectionOptions);
1594
+ const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
912
1595
  return {
913
1596
  wakeSessionWorkflow: async ({
914
1597
  accountId,
@@ -994,33 +1677,38 @@ async function createWorkerWorkflowSignaler(settings, db) {
994
1677
  signalArgs: [wakeRevision]
995
1678
  });
996
1679
  },
997
- getTurnTaskQueueStats: async () => {
998
- const response = await connection.workflowService.describeTaskQueue({
999
- namespace: settings.temporalNamespace,
1000
- taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
1001
- // temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
1002
- // the supported DEFAULT mode; `stats.approximateBacklogCount` is the
1003
- // server-documented scaling signal.
1004
- taskQueueType: 2,
1005
- reportStats: true
1006
- });
1007
- return normalizeTurnTaskQueueStats(response.stats);
1008
- },
1680
+ getTurnTaskQueueStats: (readOptions = { signal: new AbortController().signal, deadline: Date.now() + 5e3 }, identity = {
1681
+ temporalNamespace: settings.temporalNamespace,
1682
+ taskQueue: turnTaskQueue(settings.temporalTaskQueue)
1683
+ }) => statsClient.read(readOptions, identity),
1009
1684
  startSandboxReaperWorkflow: async () => {
1010
- try {
1011
- await temporal.workflow.start("sandboxReaperWorkflowV2", {
1685
+ const starts = await Promise.allSettled([
1686
+ temporal.workflow.start("sandboxReaperWorkflowV2", {
1012
1687
  taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
1013
1688
  workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
1014
1689
  workflowIdReusePolicy: "ALLOW_DUPLICATE",
1015
1690
  args: []
1016
- });
1017
- return "started";
1018
- } catch (error) {
1019
- if (error instanceof WorkflowExecutionAlreadyStartedError) {
1020
- return "already_running";
1691
+ }),
1692
+ temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
1693
+ taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
1694
+ workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
1695
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
1696
+ args: []
1697
+ }),
1698
+ // Its own workflow type, so the drain workflow's history is unchanged.
1699
+ temporal.workflow.start("sandboxIdleCheckpointSweepWorkflow", {
1700
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
1701
+ workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
1702
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
1703
+ args: []
1704
+ })
1705
+ ]);
1706
+ for (const start of starts) {
1707
+ if (start.status === "rejected" && !(start.reason instanceof WorkflowExecutionAlreadyStartedError2)) {
1708
+ throw start.reason;
1021
1709
  }
1022
- throw error;
1023
1710
  }
1711
+ return starts[0].status === "fulfilled" ? "started" : "already_running";
1024
1712
  },
1025
1713
  startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
1026
1714
  try {
@@ -1043,7 +1731,7 @@ async function createWorkerWorkflowSignaler(settings, db) {
1043
1731
  });
1044
1732
  return "started";
1045
1733
  } catch (error) {
1046
- if (error instanceof WorkflowExecutionAlreadyStartedError) {
1734
+ if (error instanceof WorkflowExecutionAlreadyStartedError2) {
1047
1735
  return "already_running";
1048
1736
  }
1049
1737
  throw error;
@@ -1053,113 +1741,13 @@ async function createWorkerWorkflowSignaler(settings, db) {
1053
1741
  await connection.workflowService.getSystemInfo({});
1054
1742
  },
1055
1743
  close: async () => {
1056
- await connection.close();
1744
+ await Promise.all([statsClient.close(), connection.close()]);
1057
1745
  }
1058
1746
  };
1059
1747
  }
1060
- async function registerSandboxReaperSchedule(settings, observability) {
1061
- assertSandboxReaperActivityTimeout(settings);
1062
- const connection = await Connection.connect(temporalConnectionOptions(settings));
1063
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1064
- const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
1065
- const action = {
1066
- type: "startWorkflow",
1067
- workflowType: "sandboxReaperWorkflow",
1068
- taskQueue: settings.temporalTaskQueue,
1069
- args: []
1070
- };
1071
- const policies = {
1072
- overlap: ScheduleOverlapPolicy.SKIP,
1073
- catchupWindow: "1m",
1074
- pauseOnFailure: false
1075
- };
1076
- try {
1077
- await temporal.schedule.create({
1078
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
1079
- // @every-style interval: fire once per reaper period. The boot invariant
1080
- // guarantees reaperPeriod < viewerHolderTTL and
1081
- // reaperPeriod + idleGrace < providerLifetime.
1082
- spec,
1083
- action,
1084
- // The base workflow is a short router on new binaries; legacy binaries
1085
- // still execute the pre-refactor composite under this same overlap fence.
1086
- policies
1087
- });
1088
- observability.info("Registered the global sandbox-lease reaper Schedule", {
1089
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
1090
- reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs
1091
- });
1092
- return {
1093
- registered: true,
1094
- close: async () => {
1095
- await connection.close();
1096
- }
1097
- };
1098
- } catch (error) {
1099
- if (error instanceof ScheduleAlreadyRunning) {
1100
- await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
1101
- spec,
1102
- action,
1103
- policies,
1104
- state: {
1105
- paused: previous.state.paused,
1106
- ...previous.state.note ? { note: previous.state.note } : {},
1107
- ...previous.state.remainingActions !== void 0 ? { remainingActions: previous.state.remainingActions } : {}
1108
- }
1109
- }));
1110
- observability.info("Reconciled the global sandbox-lease reaper Schedule", {
1111
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
1112
- temporalTaskQueue: action.taskQueue
1113
- });
1114
- return {
1115
- registered: false,
1116
- close: async () => {
1117
- await connection.close();
1118
- }
1119
- };
1120
- }
1121
- await connection.close().catch(() => void 0);
1122
- throw error;
1123
- }
1124
- }
1125
- async function registerKnowledgeIndexingSchedule(settings, observability) {
1126
- const connection = await Connection.connect(temporalConnectionOptions(settings));
1127
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1128
- try {
1129
- await temporal.schedule.create({
1130
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
1131
- spec: { intervals: [{ every: KNOWLEDGE_INDEXING_PERIOD_MS }] },
1132
- action: {
1133
- type: "startWorkflow",
1134
- workflowType: "knowledgeIndexingWorkflow",
1135
- taskQueue: settings.temporalTaskQueue,
1136
- args: []
1137
- },
1138
- policies: {
1139
- overlap: ScheduleOverlapPolicy.SKIP,
1140
- catchupWindow: "1m",
1141
- pauseOnFailure: false
1142
- }
1143
- });
1144
- observability.info("Registered the global Knowledge indexer Schedule", {
1145
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
1146
- indexingPeriodMs: KNOWLEDGE_INDEXING_PERIOD_MS
1147
- });
1148
- return { registered: true, close: async () => connection.close() };
1149
- } catch (error) {
1150
- if (error instanceof ScheduleAlreadyRunning) {
1151
- observability.info("Global Knowledge indexer Schedule already registered", {
1152
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID
1153
- });
1154
- return { registered: false, close: async () => connection.close() };
1155
- }
1156
- await connection.close().catch(() => void 0);
1157
- throw error;
1158
- }
1159
- }
1160
1748
  async function registerFileUploadReaperSchedule(settings, observability) {
1161
- const connection = await Connection.connect(temporalConnectionOptions(settings));
1162
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1749
+ const connection = await Connection3.connect(temporalConnectionOptions2(settings));
1750
+ const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
1163
1751
  try {
1164
1752
  await temporal.schedule.create({
1165
1753
  scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
@@ -1192,16 +1780,16 @@ async function registerFileUploadReaperSchedule(settings, observability) {
1192
1780
  throw error;
1193
1781
  }
1194
1782
  }
1195
- async function registerSiteAuthMaintenanceSchedule(settings, observability) {
1196
- const connection = await Connection.connect(temporalConnectionOptions(settings));
1197
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1783
+ async function registerSessionStorageMaintenanceSchedule(settings, observability) {
1784
+ const connection = await Connection3.connect(temporalConnectionOptions2(settings));
1785
+ const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
1198
1786
  try {
1199
1787
  await temporal.schedule.create({
1200
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1201
- spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
1788
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
1789
+ spec: { intervals: [{ every: SESSION_STORAGE_MAINTENANCE_PERIOD_MS }] },
1202
1790
  action: {
1203
1791
  type: "startWorkflow",
1204
- workflowType: "siteAuthMaintenanceWorkflow",
1792
+ workflowType: "sessionStorageMaintenanceWorkflow",
1205
1793
  taskQueue: settings.temporalTaskQueue,
1206
1794
  args: []
1207
1795
  },
@@ -1211,50 +1799,15 @@ async function registerSiteAuthMaintenanceSchedule(settings, observability) {
1211
1799
  pauseOnFailure: false
1212
1800
  }
1213
1801
  });
1214
- observability.info("Registered the global site-auth maintenance Schedule", {
1215
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1216
- maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS
1802
+ observability.info("Registered the session storage maintenance Schedule", {
1803
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
1804
+ periodMs: SESSION_STORAGE_MAINTENANCE_PERIOD_MS
1217
1805
  });
1218
1806
  return { registered: true, close: async () => connection.close() };
1219
1807
  } catch (error) {
1220
1808
  if (error instanceof ScheduleAlreadyRunning) {
1221
- observability.info("Global site-auth maintenance Schedule already registered", {
1222
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID
1223
- });
1224
- return { registered: false, close: async () => connection.close() };
1225
- }
1226
- await connection.close().catch(() => void 0);
1227
- throw error;
1228
- }
1229
- }
1230
- async function registerSessionWorkflowWakeDispatcherSchedule(settings, observability) {
1231
- const connection = await Connection.connect(temporalConnectionOptions(settings));
1232
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1233
- try {
1234
- await temporal.schedule.create({
1235
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
1236
- spec: { intervals: [{ every: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS }] },
1237
- action: {
1238
- type: "startWorkflow",
1239
- workflowType: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
1240
- taskQueue: settings.temporalTaskQueue,
1241
- args: []
1242
- },
1243
- policies: {
1244
- overlap: ScheduleOverlapPolicy.SKIP,
1245
- catchupWindow: "1m",
1246
- pauseOnFailure: false
1247
- }
1248
- });
1249
- observability.info("Registered the session-workflow wake dispatcher Schedule", {
1250
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
1251
- periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS
1252
- });
1253
- return { registered: true, close: async () => connection.close() };
1254
- } catch (error) {
1255
- if (error instanceof ScheduleAlreadyRunning) {
1256
- observability.info("Session-workflow wake dispatcher Schedule already registered", {
1257
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID
1809
+ observability.info("Session storage maintenance Schedule already registered", {
1810
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID
1258
1811
  });
1259
1812
  return { registered: false, close: async () => connection.close() };
1260
1813
  }
@@ -1272,6 +1825,7 @@ async function createOpenGeniWorkerService(options) {
1272
1825
  const retryOptions = startupRetryOptions(settings);
1273
1826
  const onRetry = (event) => logStartupDependencyRetry(observability, event);
1274
1827
  let lifecycle;
1828
+ let cleanupContainment;
1275
1829
  let signaler;
1276
1830
  let workerBundle;
1277
1831
  let turnCapacityMonitor;
@@ -1286,7 +1840,7 @@ async function createOpenGeniWorkerService(options) {
1286
1840
  () => resolveCatalogSettings(options.activityDependencies.db, settings),
1287
1841
  { ...retryOptions, onRetry }
1288
1842
  );
1289
- observability.info("OpenGeni model catalog resolved", {
1843
+ observability.info("Opengeni model catalog resolved", {
1290
1844
  role: options.role,
1291
1845
  catalogSource: resolvedCatalog.source,
1292
1846
  catalogVersion: resolvedCatalog.version
@@ -1306,7 +1860,7 @@ async function createOpenGeniWorkerService(options) {
1306
1860
  const startSandboxReaperWorkflow = options.activityDependencies.startSandboxReaperWorkflow ?? signaler?.startSandboxReaperWorkflow;
1307
1861
  const startVideoGenerationWorkflow = options.activityDependencies.startVideoGenerationWorkflow ?? signaler?.startVideoGenerationWorkflow;
1308
1862
  if (!wakeSessionWorkflow || !signalSessionAttemptQuiesced || !inspectSessionAttemptActivity || !signalCodexCapacityWorkflow || !startSandboxReaperWorkflow || !startVideoGenerationWorkflow) {
1309
- throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
1863
+ throw new Error("Opengeni worker lifecycle could not resolve its workflow signalers");
1310
1864
  }
1311
1865
  workerBundle = await createOpenGeniWorker({
1312
1866
  role: options.role,
@@ -1322,21 +1876,30 @@ async function createOpenGeniWorkerService(options) {
1322
1876
  inspectSessionAttemptActivity,
1323
1877
  signalCodexCapacityWorkflow,
1324
1878
  startSandboxReaperWorkflow,
1325
- startVideoGenerationWorkflow
1879
+ startVideoGenerationWorkflow,
1880
+ requestWorkerDrain: () => {
1881
+ if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
1882
+ cleanupContainment.request();
1883
+ }
1326
1884
  }
1327
1885
  });
1328
1886
  if (options.role === "turn") {
1329
1887
  if (!signaler) {
1330
1888
  throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
1331
1889
  }
1890
+ const statsClient = signaler;
1891
+ const identity = workerBundle.turnQueueIdentity;
1892
+ if (!identity)
1893
+ throw new Error("Turn worker did not expose its actual Temporal queue identity");
1332
1894
  turnCapacityMonitor = startTurnCapacityMonitor({
1333
1895
  observability,
1334
- read: signaler.getTurnTaskQueueStats
1896
+ read: (readOptions) => statsClient.getTurnTaskQueueStats(readOptions, identity),
1897
+ identity
1335
1898
  });
1336
1899
  } else {
1337
1900
  sessionRecoveryMonitor = startSessionRecoveryMonitor({
1338
1901
  observability,
1339
- read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db)
1902
+ read: async () => await summarizeSessionRecoveryBacklog(options.activityDependencies.db)
1340
1903
  });
1341
1904
  contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
1342
1905
  observability,
@@ -1344,20 +1907,6 @@ async function createOpenGeniWorkerService(options) {
1344
1907
  });
1345
1908
  }
1346
1909
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
1347
- schedules.push(
1348
- await retryStartupDependency(
1349
- "Temporal schedule (sandbox reaper)",
1350
- () => registerSandboxReaperSchedule(settings, observability),
1351
- { ...retryOptions, onRetry }
1352
- )
1353
- );
1354
- schedules.push(
1355
- await retryStartupDependency(
1356
- "Temporal schedule (Knowledge indexing)",
1357
- () => registerKnowledgeIndexingSchedule(settings, observability),
1358
- { ...retryOptions, onRetry }
1359
- )
1360
- );
1361
1910
  schedules.push(
1362
1911
  await retryStartupDependency(
1363
1912
  "Temporal schedule (file upload reaper)",
@@ -1367,32 +1916,34 @@ async function createOpenGeniWorkerService(options) {
1367
1916
  );
1368
1917
  schedules.push(
1369
1918
  await retryStartupDependency(
1370
- "Temporal schedule (site auth maintenance)",
1371
- () => registerSiteAuthMaintenanceSchedule(settings, observability),
1919
+ "Temporal schedule (session storage maintenance)",
1920
+ () => registerSessionStorageMaintenanceSchedule(settings, observability),
1372
1921
  { ...retryOptions, onRetry }
1373
1922
  )
1374
1923
  );
1375
1924
  schedules.push(
1376
1925
  await retryStartupDependency(
1377
- "Temporal schedule (session-workflow wake dispatcher)",
1378
- () => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
1926
+ "work-checked maintenance cadences",
1927
+ () => startInternalMaintenanceCadences({
1928
+ settings,
1929
+ observability,
1930
+ db: options.activityDependencies.db,
1931
+ ...options.activityDependencies.documentServices ? { documentServices: options.activityDependencies.documentServices } : {}
1932
+ }),
1379
1933
  { ...retryOptions, onRetry }
1380
1934
  )
1381
1935
  );
1382
1936
  }
1383
1937
  if (options.http !== false) {
1384
- const databaseReady = dbReadyCheck(
1385
- options.http?.readinessDb ?? options.activityDependencies.db,
1386
- options.databasePosture
1387
- );
1938
+ const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
1939
+ const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
1940
+ await resolveCatalogSettings(readinessDb, settings);
1941
+ });
1388
1942
  httpServer = startWorkerHttpServer({
1389
1943
  settings,
1390
1944
  observability,
1391
1945
  checks: {
1392
- db: async () => {
1393
- await databaseReady();
1394
- await resolveCatalogSettings(options.activityDependencies.db, settings);
1395
- },
1946
+ db: databaseReady,
1396
1947
  nats: natsReadyCheck(options.activityDependencies.bus),
1397
1948
  temporal: temporalReadyCheck(workerBundle.connection)
1398
1949
  },
@@ -1401,6 +1952,7 @@ async function createOpenGeniWorkerService(options) {
1401
1952
  });
1402
1953
  }
1403
1954
  } catch (error) {
1955
+ workerBundle?.activityTelemetry?.closeIfNeverRun();
1404
1956
  httpServer?.stop(true);
1405
1957
  await Promise.allSettled([
1406
1958
  turnCapacityMonitor?.close(),
@@ -1415,15 +1967,30 @@ async function createOpenGeniWorkerService(options) {
1415
1967
  const activeWorkerBundle = workerBundle;
1416
1968
  const activeSignaler = signaler;
1417
1969
  if (!activeWorkerBundle) {
1418
- throw new Error("OpenGeni worker service initialization did not complete");
1970
+ throw new Error("Opengeni worker service initialization did not complete");
1419
1971
  }
1972
+ cleanupContainment = createWorkerCleanupContainment({
1973
+ drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
1974
+ ...options.terminateWorker ? { terminate: options.terminateWorker } : {},
1975
+ observability
1976
+ });
1977
+ const activeCleanupContainment = cleanupContainment;
1420
1978
  lifecycle = createWorkerServiceLifecycle({
1421
1979
  role: options.role,
1422
- worker: activeWorkerBundle.worker,
1980
+ worker: {
1981
+ shutdown: () => activeWorkerBundle.worker.shutdown(),
1982
+ run: async () => {
1983
+ await activeWorkerBundle.worker.run();
1984
+ activeCleanupContainment.finished();
1985
+ }
1986
+ },
1423
1987
  observability,
1424
1988
  closeOwnedResources: async () => {
1425
1989
  memoryPressureGuard?.close();
1426
- httpServer?.stop(true);
1990
+ closeWorkerHttpAfterActivityExecution(
1991
+ activeWorkerBundle.activityTelemetry,
1992
+ () => httpServer?.stop(true)
1993
+ );
1427
1994
  await Promise.allSettled([
1428
1995
  turnCapacityMonitor?.close(),
1429
1996
  sessionRecoveryMonitor?.close(),
@@ -1434,7 +2001,7 @@ async function createOpenGeniWorkerService(options) {
1434
2001
  ]);
1435
2002
  },
1436
2003
  onReady: () => {
1437
- observability.info("OpenGeni worker listening", {
2004
+ observability.info("Opengeni worker listening", {
1438
2005
  role: options.role,
1439
2006
  temporalTaskQueue: options.role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
1440
2007
  ...options.role === "turn" ? turnWorkerConcurrencyLogFields(settings) : {
@@ -1480,6 +2047,16 @@ async function createOpenGeniWorkerService(options) {
1480
2047
  }
1481
2048
  async function runOpenGeniWorker(options) {
1482
2049
  const service = await createOpenGeniWorkerService(options);
2050
+ void import("@opengeni/runtime/sandbox").then(
2051
+ ({ sweepOrphanedHostArchiveTemporaryDirectoriesOnce }) => sweepOrphanedHostArchiveTemporaryDirectoriesOnce()
2052
+ ).then((removed) => {
2053
+ if (removed.length > 0) {
2054
+ options.activityDependencies.observability?.info(
2055
+ "worker removed orphaned workspace archive spools",
2056
+ { count: removed.length }
2057
+ );
2058
+ }
2059
+ }).catch(() => void 0);
1483
2060
  const signals = options.shutdownSignals === false ? [] : options.shutdownSignals ?? ["SIGTERM", "SIGINT"];
1484
2061
  const handlers = signals.map((signal) => {
1485
2062
  const handler = () => service.drain(signal);
@@ -1502,6 +2079,7 @@ async function startWorker() {
1502
2079
  }
1503
2080
  const settings = getSettings();
1504
2081
  const observability = createObservability(settings, { component: `worker-${role}` });
2082
+ installTemporalRuntimeLogging(observability);
1505
2083
  const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
1506
2084
  const retryOptions = startupRetryOptions(settings);
1507
2085
  const onRetry = (event) => logStartupDependencyRetry(observability, event);
@@ -1518,8 +2096,7 @@ async function startWorker() {
1518
2096
  const databasePosture = {
1519
2097
  rlsStrategy: settings.rlsStrategy,
1520
2098
  expectedRole: settings.runtimeDatabaseRole,
1521
- targetSchema: settings.dbSchema.trim() || "public",
1522
- organizationTenancyCanonicalActivationEnabled: settings.organizationTenancyCanonicalActivationEnabled
2099
+ targetSchema: settings.dbSchema.trim() || "public"
1523
2100
  };
1524
2101
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
1525
2102
  let bus;
@@ -1534,12 +2111,13 @@ async function startWorker() {
1534
2111
  () => createNatsEventBus(
1535
2112
  settings.natsUrl,
1536
2113
  controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
1537
- { logger: observabilityEventLogger(observability) }
2114
+ observabilityEventBusOptions(observability)
1538
2115
  ),
1539
2116
  { ...retryOptions, onRetry }
1540
2117
  );
1541
2118
  await runOpenGeniWorker({
1542
2119
  role,
2120
+ terminateWorker: () => process.exit(1),
1543
2121
  settings,
1544
2122
  databasePosture,
1545
2123
  http: { readinessDb: readinessDbClient.db },
@@ -1570,8 +2148,11 @@ if (import.meta.main) {
1570
2148
  export {
1571
2149
  FILE_UPLOAD_REAPER_PERIOD_MS,
1572
2150
  FILE_UPLOAD_REAPER_SCHEDULE_ID,
2151
+ INTERNAL_MAINTENANCE_JOBS,
1573
2152
  KNOWLEDGE_INDEXING_PERIOD_MS,
1574
2153
  KNOWLEDGE_INDEXING_SCHEDULE_ID,
2154
+ SESSION_STORAGE_MAINTENANCE_PERIOD_MS,
2155
+ SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
1575
2156
  SITE_AUTH_MAINTENANCE_PERIOD_MS,
1576
2157
  SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1577
2158
  createDefaultWorkerActivities,
@@ -1580,12 +2161,10 @@ export {
1580
2161
  createOpenGeniWorkerService,
1581
2162
  createWorkerWorkflowSignaler,
1582
2163
  registerFileUploadReaperSchedule,
1583
- registerKnowledgeIndexingSchedule,
1584
- registerSandboxReaperSchedule,
1585
- registerSessionWorkflowWakeDispatcherSchedule,
1586
- registerSiteAuthMaintenanceSchedule,
2164
+ registerSessionStorageMaintenanceSchedule,
1587
2165
  resolveOpenGeniWorkflowDefinition,
1588
2166
  runOpenGeniWorker,
2167
+ startInternalMaintenanceCadences,
1589
2168
  startWorker,
1590
2169
  temporalActivityLeaseSettled,
1591
2170
  temporalWorkflowExecutionNotFound,