@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
package/src/index.ts CHANGED
@@ -10,10 +10,10 @@ import {
10
10
  import {
11
11
  assertRuntimeDatabasePosture,
12
12
  isRetryableRuntimeDatabaseStartupError,
13
- countSessionRecoveryBacklog,
14
13
  createDb,
15
14
  getContextCompactionPendingSummary,
16
15
  markSessionWorkflowWakeDelivered,
16
+ summarizeSessionRecoveryBacklog,
17
17
  type Database,
18
18
  type RuntimeDatabasePostureOptions,
19
19
  } from "@opengeni/db";
@@ -37,6 +37,7 @@ import {
37
37
  } from "@temporalio/client";
38
38
  import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
39
39
  import { existsSync } from "node:fs";
40
+ import { prepareObjectStorageProvider } from "@opengeni/storage";
40
41
  import { fileURLToPath } from "node:url";
41
42
  import type { createControlActivities } from "./activities-control";
42
43
  import type { createTurnActivities } from "./activities-turn";
@@ -50,6 +51,13 @@ import type {
50
51
  WakeSessionWorkflowSignal,
51
52
  } from "./activities/types";
52
53
  import { turnTaskQueue } from "./workflows/activities";
54
+ import { createTurnTaskQueueStatsClient } from "./turn-task-queue-reader";
55
+ import { installTemporalRuntimeLogging } from "./temporal-runtime-logger";
56
+ import {
57
+ closeWorkerHttpAfterActivityExecution,
58
+ withTurnWorkerActivityTelemetry,
59
+ type TurnWorkerActivityTelemetry,
60
+ } from "./turn-worker-activity-telemetry";
53
61
  import {
54
62
  dbReadyCheck,
55
63
  natsReadyCheck,
@@ -60,20 +68,19 @@ import {
60
68
  import {
61
69
  initializeContextCompactionMetrics,
62
70
  initializeWorkerOutcomeMetrics,
63
- normalizeTurnTaskQueueStats,
64
- observabilityEventLogger,
71
+ observabilityEventBusOptions,
65
72
  startContextCompactionPendingMonitor,
66
73
  startSessionRecoveryMonitor,
67
74
  startTurnCapacityMonitor,
68
75
  type TurnTaskQueueStats,
76
+ type TurnTaskQueueReadOptions,
77
+ type TurnTaskQueueIdentity,
69
78
  } from "./observability-metrics";
70
79
  import {
71
80
  resolveCatalogSettings,
72
- SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
73
- SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
74
- SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
75
81
  TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
76
82
  } from "@opengeni/core";
83
+ import { startInternalMaintenanceCadences } from "./internal-maintenance";
77
84
  import {
78
85
  CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
79
86
  CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
@@ -85,6 +92,7 @@ import {
85
92
  combineWorkerRunTargets,
86
93
  constructWithOwnedConnection,
87
94
  createWorkerServiceLifecycle,
95
+ createWorkerCleanupContainment,
88
96
  type WorkerServiceLifecycle,
89
97
  type WorkerRunTarget,
90
98
  } from "./worker-service-lifecycle";
@@ -96,9 +104,14 @@ import {
96
104
  import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
97
105
  import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
98
106
  import {
107
+ SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
99
108
  SANDBOX_REAPER_V2_WORKFLOW_ID,
100
109
  sandboxLifecycleTaskQueue,
101
110
  } from "./sandbox-reaper-contract";
111
+ import {
112
+ BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
113
+ browserDeadlineCheckpointTaskQueue,
114
+ } from "./browser-deadline-checkpoint-contract";
102
115
 
103
116
  export {
104
117
  createHostExportPump,
@@ -108,23 +121,27 @@ export {
108
121
  type HostEventExport,
109
122
  type HostEventExportBatch,
110
123
  type HostEventSink,
124
+ type HostLifecycleFactExport,
125
+ type HostLifecycleFactExportBatch,
126
+ type HostLifecycleFactSink,
111
127
  type HostUsageExport,
112
128
  type HostUsageExportBatch,
113
129
  type HostUsageSink,
114
130
  } from "./host-export";
131
+ export {
132
+ INTERNAL_MAINTENANCE_JOBS,
133
+ KNOWLEDGE_INDEXING_PERIOD_MS,
134
+ KNOWLEDGE_INDEXING_SCHEDULE_ID,
135
+ SITE_AUTH_MAINTENANCE_PERIOD_MS,
136
+ SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
137
+ startInternalMaintenanceCadences,
138
+ type InternalMaintenanceJob,
139
+ } from "./internal-maintenance";
115
140
 
116
- // The deterministic id of the ONE global reaper Schedule. A single id means
117
- // create() is idempotent across every worker in the pool: the first worker to
118
- // boot creates it, all others collide on this id (ScheduleAlreadyRunning) and
119
- // no-op — so the Schedule is registered EXACTLY ONCE per deployment regardless
120
- // of replica count.
121
- const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
122
- export const KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
123
- export const KNOWLEDGE_INDEXING_PERIOD_MS = 15_000;
124
141
  export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
125
142
  export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
126
- export const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
127
- export const SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1_000;
143
+ export const SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID = "opengeni-session-storage-maintenance";
144
+ export const SESSION_STORAGE_MAINTENANCE_PERIOD_MS = 5 * 60 * 1_000;
128
145
  export type OpenGeniWorkerRole = "control" | "turn";
129
146
 
130
147
  export type WorkerOptions = {
@@ -178,6 +195,9 @@ export async function createDefaultWorkerActivities(
178
195
  role: OpenGeniWorkerRole,
179
196
  dependencies: ActivityDependencies = {},
180
197
  ) {
198
+ if (dependencies.objectStorage == null) {
199
+ prepareObjectStorageProvider(dependencies.settings ?? getSettings());
200
+ }
181
201
  return role === "control"
182
202
  ? (await import("./activities-control")).createControlActivities(dependencies)
183
203
  : (await import("./activities-turn")).createTurnActivities(dependencies);
@@ -203,7 +223,7 @@ export function resolveOpenGeniWorkflowDefinition(
203
223
  const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
204
224
  if (!existsSync(codePath)) {
205
225
  throw new Error(
206
- `OpenGeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`,
226
+ `Opengeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`,
207
227
  );
208
228
  }
209
229
  return { workflowBundle: { codePath } };
@@ -219,6 +239,8 @@ export function turnCancellationHeartbeatThrottleOptions() {
219
239
  export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
220
240
  worker: WorkerRunTarget;
221
241
  connection: NativeConnection;
242
+ activityTelemetry?: TurnWorkerActivityTelemetry;
243
+ turnQueueIdentity?: TurnTaskQueueIdentity;
222
244
  }> {
223
245
  const settings = options.settings ?? getSettings();
224
246
  if (options.role === "control") {
@@ -264,10 +286,20 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
264
286
  },
265
287
  ),
266
288
  async (connection) => {
289
+ let turnWorker: Worker | undefined;
290
+ let cleanupDrainRequested = false;
267
291
  const activityDependencies = {
268
292
  ...options.activityDependencies,
269
293
  settings,
270
294
  observability,
295
+ requestWorkerDrain:
296
+ options.activityDependencies?.requestWorkerDrain ??
297
+ (() => {
298
+ if (cleanupDrainRequested) return;
299
+ if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
300
+ turnWorker.shutdown();
301
+ cleanupDrainRequested = true;
302
+ }),
271
303
  };
272
304
  const activities =
273
305
  options.activities ??
@@ -316,32 +348,58 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
316
348
  : turnConcurrency!.options),
317
349
  });
318
350
  if (options.role !== "control") {
351
+ turnWorker = worker;
319
352
  turnConcurrency?.admission?.finalizeStartupBaseline();
320
- return { worker, connection };
353
+ const turnQueueIdentity = {
354
+ temporalNamespace: worker.options.namespace,
355
+ taskQueue: worker.options.taskQueue,
356
+ };
357
+ return {
358
+ ...withTurnWorkerActivityTelemetry(worker, {
359
+ observability,
360
+ identity: turnQueueIdentity,
361
+ // No invented identity when running outside Kubernetes. Scaler
362
+ // consumers must reject empty/missing UID coverage.
363
+ podUid: process.env.OPENGENI_POD_UID ?? "",
364
+ }),
365
+ connection,
366
+ turnQueueIdentity,
367
+ };
321
368
  }
322
369
 
370
+ const ownedWorkers = [worker];
323
371
  try {
324
- const sandboxLifecycleWorker = await Worker.create({
325
- ...sharedWorkerOptions,
326
- ...workflowDefinition,
327
- taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
328
- reuseV8Context: true,
329
- workflowThreadPoolSize: 1,
330
- maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
331
- maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
332
- maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
333
- });
372
+ // The new activity type has its own queue. Retain the V1 poller so old
373
+ // drain histories finish; old binaries never receive checkpoint work.
374
+ for (const taskQueue of [
375
+ sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
376
+ browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
377
+ ]) {
378
+ const lifecycleWorker = await Worker.create({
379
+ ...sharedWorkerOptions,
380
+ ...workflowDefinition,
381
+ taskQueue,
382
+ reuseV8Context: true,
383
+ workflowThreadPoolSize: 1,
384
+ maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
385
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
386
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
387
+ });
388
+ ownedWorkers.push(lifecycleWorker);
389
+ }
334
390
  return {
335
- worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
391
+ worker: combineWorkerRunTargets(ownedWorkers),
336
392
  connection,
337
393
  };
338
394
  } catch (error) {
339
- try {
340
- worker.shutdown();
341
- } catch {
342
- // Preserve the lifecycle worker construction failure. The base poller
343
- // still received its shutdown request; a secondary synchronous
344
- // shutdown error must not replace the startup root cause.
395
+ for (const ownedWorker of ownedWorkers) {
396
+ try {
397
+ ownedWorker.shutdown();
398
+ } catch {
399
+ // Preserve the lifecycle worker construction failure. The base poller
400
+ // still received its shutdown request; a secondary synchronous
401
+ // shutdown error must not replace the startup root cause.
402
+ }
345
403
  }
346
404
  throw error;
347
405
  }
@@ -363,13 +421,18 @@ export async function createWorkerWorkflowSignaler(
363
421
  signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
364
422
  inspectSessionAttemptActivity: InspectSessionAttemptActivity;
365
423
  signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
366
- getTurnTaskQueueStats: () => Promise<TurnTaskQueueStats>;
424
+ getTurnTaskQueueStats: (
425
+ options?: TurnTaskQueueReadOptions,
426
+ identity?: TurnTaskQueueIdentity,
427
+ ) => Promise<TurnTaskQueueStats>;
367
428
  startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
368
429
  startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
369
430
  check: () => Promise<void>;
370
431
  close: () => Promise<void>;
371
432
  }> {
372
- const connection = await Connection.connect(temporalConnectionOptions(settings));
433
+ const connectionOptions = temporalConnectionOptions(settings);
434
+ const connection = await Connection.connect(connectionOptions);
435
+ const statsClient = createTurnTaskQueueStatsClient(connectionOptions);
373
436
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
374
437
  return {
375
438
  wakeSessionWorkflow: async ({
@@ -462,33 +525,48 @@ export async function createWorkerWorkflowSignaler(
462
525
  // another producer may have advanced it with a Pause/Steer that requires
463
526
  // sessionControl. The global dispatcher owns that acknowledgement.
464
527
  },
465
- getTurnTaskQueueStats: async () => {
466
- const response = await connection.workflowService.describeTaskQueue({
467
- namespace: settings.temporalNamespace,
468
- taskQueue: { name: turnTaskQueue(settings.temporalTaskQueue) },
469
- // temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
470
- // the supported DEFAULT mode; `stats.approximateBacklogCount` is the
471
- // server-documented scaling signal.
472
- taskQueueType: 2,
473
- reportStats: true,
474
- });
475
- return normalizeTurnTaskQueueStats(response.stats);
476
- },
528
+ getTurnTaskQueueStats: (
529
+ readOptions = { signal: new AbortController().signal, deadline: Date.now() + 5_000 },
530
+ identity = {
531
+ temporalNamespace: settings.temporalNamespace,
532
+ taskQueue: turnTaskQueue(settings.temporalTaskQueue),
533
+ },
534
+ ) => statsClient.read(readOptions, identity),
477
535
  startSandboxReaperWorkflow: async () => {
478
- try {
479
- await temporal.workflow.start("sandboxReaperWorkflowV2", {
536
+ // Same backend tick, no additional schedule. A bounded independent
537
+ // inventory cannot prevent the existing per-box drain from starting.
538
+ const starts = await Promise.allSettled([
539
+ temporal.workflow.start("sandboxReaperWorkflowV2", {
480
540
  taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
481
541
  workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
482
542
  workflowIdReusePolicy: "ALLOW_DUPLICATE",
483
543
  args: [],
484
- });
485
- return "started";
486
- } catch (error) {
487
- if (error instanceof WorkflowExecutionAlreadyStartedError) {
488
- return "already_running";
544
+ }),
545
+ temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
546
+ taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
547
+ workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
548
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
549
+ args: [],
550
+ }),
551
+ // Its own workflow type, so the drain workflow's history is unchanged.
552
+ temporal.workflow.start("sandboxIdleCheckpointSweepWorkflow", {
553
+ taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
554
+ workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
555
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
556
+ args: [],
557
+ }),
558
+ ]);
559
+ for (const start of starts) {
560
+ if (
561
+ start.status === "rejected" &&
562
+ !(start.reason instanceof WorkflowExecutionAlreadyStartedError)
563
+ ) {
564
+ // The other start already settled. Retry the router without losing
565
+ // ordinary draining or silently omitting the checkpoint sweep.
566
+ throw start.reason;
489
567
  }
490
- throw error;
491
568
  }
569
+ return starts[0].status === "fulfilled" ? "started" : "already_running";
492
570
  },
493
571
  startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
494
572
  try {
@@ -521,143 +599,11 @@ export async function createWorkerWorkflowSignaler(
521
599
  await connection.workflowService.getSystemInfo({});
522
600
  },
523
601
  close: async () => {
524
- await connection.close();
602
+ await Promise.all([statsClient.close(), connection.close()]);
525
603
  },
526
604
  };
527
605
  }
528
606
 
529
- /**
530
- * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
531
- * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
532
- * sandbox ownership off the activity performs bounded DB outbox repair and
533
- * read-only observability projections; it never mutates or terminates sandbox leases.
534
- *
535
- * The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
536
- * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
537
- * `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
538
- * validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
539
- * sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
540
- * worker pool collides on ScheduleAlreadyRunning and reconciles the same
541
- * definition. New workers route onto the versioned lifecycle queue; old workers
542
- * can still run the legacy action during rollout or a full binary rollback.
543
- *
544
- * Returns a `close()` for the dedicated client connection (separate from the
545
- * worker's NativeConnection — the Schedule client is a @temporalio/client).
546
- */
547
- export async function registerSandboxReaperSchedule(
548
- settings: Settings,
549
- observability: Observability,
550
- ): Promise<{ registered: boolean; close: () => Promise<void> }> {
551
- assertSandboxReaperActivityTimeout(settings);
552
- const connection = await Connection.connect(temporalConnectionOptions(settings));
553
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
554
- const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
555
- const action = {
556
- type: "startWorkflow" as const,
557
- workflowType: "sandboxReaperWorkflow",
558
- taskQueue: settings.temporalTaskQueue,
559
- args: [] as [],
560
- };
561
- const policies = {
562
- overlap: ScheduleOverlapPolicy.SKIP,
563
- catchupWindow: "1m",
564
- pauseOnFailure: false,
565
- } as const;
566
- try {
567
- await temporal.schedule.create({
568
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
569
- // @every-style interval: fire once per reaper period. The boot invariant
570
- // guarantees reaperPeriod < viewerHolderTTL and
571
- // reaperPeriod + idleGrace < providerLifetime.
572
- spec,
573
- action,
574
- // The base workflow is a short router on new binaries; legacy binaries
575
- // still execute the pre-refactor composite under this same overlap fence.
576
- policies,
577
- });
578
- observability.info("Registered the global sandbox-lease reaper Schedule", {
579
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
580
- reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs,
581
- });
582
- return {
583
- registered: true,
584
- close: async () => {
585
- await connection.close();
586
- },
587
- };
588
- } catch (error) {
589
- if (error instanceof ScheduleAlreadyRunning) {
590
- // Never move this action off the base queue: doing so would leave a full
591
- // rollback with no worker polling the only reaper Schedule. The workflow
592
- // implementation itself owns the forward-compatible queue handoff.
593
- await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
594
- spec,
595
- action,
596
- policies,
597
- state: {
598
- paused: previous.state.paused,
599
- ...(previous.state.note ? { note: previous.state.note } : {}),
600
- ...(previous.state.remainingActions !== undefined
601
- ? { remainingActions: previous.state.remainingActions }
602
- : {}),
603
- },
604
- }));
605
- observability.info("Reconciled the global sandbox-lease reaper Schedule", {
606
- scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
607
- temporalTaskQueue: action.taskQueue,
608
- });
609
- return {
610
- registered: false,
611
- close: async () => {
612
- await connection.close();
613
- },
614
- };
615
- }
616
- await connection.close().catch(() => undefined);
617
- throw error;
618
- }
619
- }
620
-
621
- /** Register the deployment-wide worker for rebuildable Knowledge search projections. */
622
- export async function registerKnowledgeIndexingSchedule(
623
- settings: Settings,
624
- observability: Observability,
625
- ): Promise<{ registered: boolean; close: () => Promise<void> }> {
626
- const connection = await Connection.connect(temporalConnectionOptions(settings));
627
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
628
- try {
629
- await temporal.schedule.create({
630
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
631
- spec: { intervals: [{ every: KNOWLEDGE_INDEXING_PERIOD_MS }] },
632
- action: {
633
- type: "startWorkflow",
634
- workflowType: "knowledgeIndexingWorkflow",
635
- taskQueue: settings.temporalTaskQueue,
636
- args: [],
637
- },
638
- policies: {
639
- overlap: ScheduleOverlapPolicy.SKIP,
640
- catchupWindow: "1m",
641
- pauseOnFailure: false,
642
- },
643
- });
644
- observability.info("Registered the global Knowledge indexer Schedule", {
645
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
646
- indexingPeriodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
647
- });
648
- return { registered: true, close: async () => connection.close() };
649
- } catch (error) {
650
- if (error instanceof ScheduleAlreadyRunning) {
651
- observability.info("Global Knowledge indexer Schedule already registered", {
652
- scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
653
- });
654
- return { registered: false, close: async () => connection.close() };
655
- }
656
- await connection.close().catch(() => undefined);
657
- throw error;
658
- }
659
- }
660
-
661
607
  /**
662
608
  * Register the one provider-neutral expired direct-upload cleanup Schedule.
663
609
  * Unlike sandbox GC this is always registered: file uploads can be enabled in
@@ -703,54 +649,12 @@ export async function registerFileUploadReaperSchedule(
703
649
  }
704
650
  }
705
651
 
706
- /** Register the deployment-wide maintained-auth dispatcher. Its one-minute
707
- * cadence matches the minimum public policy interval; DB claims own overlap,
708
- * crash recovery, and exact session idempotency. */
709
- export async function registerSiteAuthMaintenanceSchedule(
710
- settings: Settings,
711
- observability: Observability,
712
- ): Promise<{ registered: boolean; close: () => Promise<void> }> {
713
- const connection = await Connection.connect(temporalConnectionOptions(settings));
714
- const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
715
- try {
716
- await temporal.schedule.create({
717
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
718
- spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
719
- action: {
720
- type: "startWorkflow",
721
- workflowType: "siteAuthMaintenanceWorkflow",
722
- taskQueue: settings.temporalTaskQueue,
723
- args: [],
724
- },
725
- policies: {
726
- overlap: ScheduleOverlapPolicy.SKIP,
727
- catchupWindow: "1m",
728
- pauseOnFailure: false,
729
- },
730
- });
731
- observability.info("Registered the global site-auth maintenance Schedule", {
732
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
733
- maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
734
- });
735
- return { registered: true, close: async () => connection.close() };
736
- } catch (error) {
737
- if (error instanceof ScheduleAlreadyRunning) {
738
- observability.info("Global site-auth maintenance Schedule already registered", {
739
- scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
740
- });
741
- return { registered: false, close: async () => connection.close() };
742
- }
743
- await connection.close().catch(() => undefined);
744
- throw error;
745
- }
746
- }
747
-
748
652
  /**
749
- * Register the one repair cadence for committed workflow-wake revisions. The
750
- * activity only reads the transactional outbox and sends revision-scoped
751
- * signals; it is independent of sandbox ownership and child-agent features.
653
+ * Register the deployment-wide session storage maintenance Schedule. Always
654
+ * registered: legacy content compaction is lossless. Overlap is skipped so one
655
+ * pass never races another.
752
656
  */
753
- export async function registerSessionWorkflowWakeDispatcherSchedule(
657
+ export async function registerSessionStorageMaintenanceSchedule(
754
658
  settings: Settings,
755
659
  observability: Observability,
756
660
  ): Promise<{ registered: boolean; close: () => Promise<void> }> {
@@ -758,11 +662,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
758
662
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
759
663
  try {
760
664
  await temporal.schedule.create({
761
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
762
- spec: { intervals: [{ every: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS }] },
665
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
666
+ spec: { intervals: [{ every: SESSION_STORAGE_MAINTENANCE_PERIOD_MS }] },
763
667
  action: {
764
668
  type: "startWorkflow",
765
- workflowType: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
669
+ workflowType: "sessionStorageMaintenanceWorkflow",
766
670
  taskQueue: settings.temporalTaskQueue,
767
671
  args: [],
768
672
  },
@@ -772,15 +676,15 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
772
676
  pauseOnFailure: false,
773
677
  },
774
678
  });
775
- observability.info("Registered the session-workflow wake dispatcher Schedule", {
776
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
777
- periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
679
+ observability.info("Registered the session storage maintenance Schedule", {
680
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
681
+ periodMs: SESSION_STORAGE_MAINTENANCE_PERIOD_MS,
778
682
  });
779
683
  return { registered: true, close: async () => connection.close() };
780
684
  } catch (error) {
781
685
  if (error instanceof ScheduleAlreadyRunning) {
782
- observability.info("Session-workflow wake dispatcher Schedule already registered", {
783
- scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
686
+ observability.info("Session storage maintenance Schedule already registered", {
687
+ scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
784
688
  });
785
689
  return { registered: false, close: async () => connection.close() };
786
690
  }
@@ -791,13 +695,16 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
791
695
 
792
696
  export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
793
697
  activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
698
+ /** Host-owned final containment after a stalled cleanup has drained peers.
699
+ * The standalone process exits; embedded hosts own their termination policy. */
700
+ terminateWorker?: () => void;
794
701
  /**
795
702
  * Exact catalog posture required by the standalone runtime. Embedded hosts
796
703
  * may omit this when they own an equivalent database isolation contract.
797
704
  */
798
705
  databasePosture?: RuntimeDatabasePostureOptions;
799
706
  /**
800
- * `role-default` registers OpenGeni's internal maintenance schedules on a
707
+ * `role-default` registers Opengeni's internal maintenance schedules on a
801
708
  * control worker only. These are engine maintenance schedules, not a host's
802
709
  * product-level scheduled-agent jobs. Use `none` when another control worker
803
710
  * in the same deployment owns them.
@@ -847,6 +754,7 @@ export async function createOpenGeniWorkerService(
847
754
  const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
848
755
  logStartupDependencyRetry(observability, event);
849
756
  let lifecycle: WorkerServiceLifecycle | undefined;
757
+ let cleanupContainment: ReturnType<typeof createWorkerCleanupContainment> | undefined;
850
758
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
851
759
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
852
760
  let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
@@ -864,7 +772,7 @@ export async function createOpenGeniWorkerService(
864
772
  () => resolveCatalogSettings(options.activityDependencies.db, settings),
865
773
  { ...retryOptions, onRetry },
866
774
  );
867
- observability.info("OpenGeni model catalog resolved", {
775
+ observability.info("Opengeni model catalog resolved", {
868
776
  role: options.role,
869
777
  catalogSource: resolvedCatalog.source,
870
778
  catalogVersion: resolvedCatalog.version,
@@ -909,7 +817,7 @@ export async function createOpenGeniWorkerService(
909
817
  !startSandboxReaperWorkflow ||
910
818
  !startVideoGenerationWorkflow
911
819
  ) {
912
- throw new Error("OpenGeni worker lifecycle could not resolve its workflow signalers");
820
+ throw new Error("Opengeni worker lifecycle could not resolve its workflow signalers");
913
821
  }
914
822
  workerBundle = await createOpenGeniWorker({
915
823
  role: options.role,
@@ -926,6 +834,10 @@ export async function createOpenGeniWorkerService(
926
834
  signalCodexCapacityWorkflow,
927
835
  startSandboxReaperWorkflow,
928
836
  startVideoGenerationWorkflow,
837
+ requestWorkerDrain: () => {
838
+ if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
839
+ cleanupContainment.request();
840
+ },
929
841
  },
930
842
  });
931
843
 
@@ -933,14 +845,19 @@ export async function createOpenGeniWorkerService(
933
845
  if (!signaler) {
934
846
  throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
935
847
  }
848
+ const statsClient = signaler;
849
+ const identity = workerBundle.turnQueueIdentity;
850
+ if (!identity)
851
+ throw new Error("Turn worker did not expose its actual Temporal queue identity");
936
852
  turnCapacityMonitor = startTurnCapacityMonitor({
937
853
  observability,
938
- read: signaler.getTurnTaskQueueStats,
854
+ read: (readOptions) => statsClient.getTurnTaskQueueStats(readOptions, identity),
855
+ identity,
939
856
  });
940
857
  } else {
941
858
  sessionRecoveryMonitor = startSessionRecoveryMonitor({
942
859
  observability,
943
- read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
860
+ read: async () => await summarizeSessionRecoveryBacklog(options.activityDependencies.db),
944
861
  });
945
862
  contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
946
863
  observability,
@@ -949,20 +866,6 @@ export async function createOpenGeniWorkerService(
949
866
  }
950
867
 
951
868
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
952
- schedules.push(
953
- await retryStartupDependency(
954
- "Temporal schedule (sandbox reaper)",
955
- () => registerSandboxReaperSchedule(settings, observability),
956
- { ...retryOptions, onRetry },
957
- ),
958
- );
959
- schedules.push(
960
- await retryStartupDependency(
961
- "Temporal schedule (Knowledge indexing)",
962
- () => registerKnowledgeIndexingSchedule(settings, observability),
963
- { ...retryOptions, onRetry },
964
- ),
965
- );
966
869
  schedules.push(
967
870
  await retryStartupDependency(
968
871
  "Temporal schedule (file upload reaper)",
@@ -972,33 +875,38 @@ export async function createOpenGeniWorkerService(
972
875
  );
973
876
  schedules.push(
974
877
  await retryStartupDependency(
975
- "Temporal schedule (site auth maintenance)",
976
- () => registerSiteAuthMaintenanceSchedule(settings, observability),
878
+ "Temporal schedule (session storage maintenance)",
879
+ () => registerSessionStorageMaintenanceSchedule(settings, observability),
977
880
  { ...retryOptions, onRetry },
978
881
  ),
979
882
  );
980
883
  schedules.push(
981
884
  await retryStartupDependency(
982
- "Temporal schedule (session-workflow wake dispatcher)",
983
- () => registerSessionWorkflowWakeDispatcherSchedule(settings, observability),
885
+ "work-checked maintenance cadences",
886
+ () =>
887
+ startInternalMaintenanceCadences({
888
+ settings,
889
+ observability,
890
+ db: options.activityDependencies.db,
891
+ ...(options.activityDependencies.documentServices
892
+ ? { documentServices: options.activityDependencies.documentServices }
893
+ : {}),
894
+ }),
984
895
  { ...retryOptions, onRetry },
985
896
  ),
986
897
  );
987
898
  }
988
899
 
989
900
  if (options.http !== false) {
990
- const databaseReady = dbReadyCheck(
991
- options.http?.readinessDb ?? options.activityDependencies.db,
992
- options.databasePosture,
993
- );
901
+ const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
902
+ const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
903
+ await resolveCatalogSettings(readinessDb, settings);
904
+ });
994
905
  httpServer = startWorkerHttpServer({
995
906
  settings,
996
907
  observability,
997
908
  checks: {
998
- db: async () => {
999
- await databaseReady();
1000
- await resolveCatalogSettings(options.activityDependencies.db, settings);
1001
- },
909
+ db: databaseReady,
1002
910
  nats: natsReadyCheck(options.activityDependencies.bus),
1003
911
  temporal: temporalReadyCheck(workerBundle.connection),
1004
912
  },
@@ -1007,6 +915,7 @@ export async function createOpenGeniWorkerService(
1007
915
  });
1008
916
  }
1009
917
  } catch (error) {
918
+ workerBundle?.activityTelemetry?.closeIfNeverRun();
1010
919
  httpServer?.stop(true);
1011
920
  await Promise.allSettled([
1012
921
  turnCapacityMonitor?.close(),
@@ -1022,16 +931,30 @@ export async function createOpenGeniWorkerService(
1022
931
  const activeWorkerBundle = workerBundle;
1023
932
  const activeSignaler = signaler;
1024
933
  if (!activeWorkerBundle) {
1025
- throw new Error("OpenGeni worker service initialization did not complete");
934
+ throw new Error("Opengeni worker service initialization did not complete");
1026
935
  }
1027
936
 
937
+ cleanupContainment = createWorkerCleanupContainment({
938
+ drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
939
+ ...(options.terminateWorker ? { terminate: options.terminateWorker } : {}),
940
+ observability,
941
+ });
942
+ const activeCleanupContainment = cleanupContainment;
1028
943
  lifecycle = createWorkerServiceLifecycle({
1029
944
  role: options.role,
1030
- worker: activeWorkerBundle.worker,
945
+ worker: {
946
+ shutdown: () => activeWorkerBundle.worker.shutdown(),
947
+ run: async () => {
948
+ await activeWorkerBundle.worker.run();
949
+ activeCleanupContainment.finished();
950
+ },
951
+ },
1031
952
  observability,
1032
953
  closeOwnedResources: async () => {
1033
954
  memoryPressureGuard?.close();
1034
- httpServer?.stop(true);
955
+ closeWorkerHttpAfterActivityExecution(activeWorkerBundle.activityTelemetry, () =>
956
+ httpServer?.stop(true),
957
+ );
1035
958
  await Promise.allSettled([
1036
959
  turnCapacityMonitor?.close(),
1037
960
  sessionRecoveryMonitor?.close(),
@@ -1042,7 +965,7 @@ export async function createOpenGeniWorkerService(
1042
965
  ]);
1043
966
  },
1044
967
  onReady: () => {
1045
- observability.info("OpenGeni worker listening", {
968
+ observability.info("Opengeni worker listening", {
1046
969
  role: options.role,
1047
970
  temporalTaskQueue:
1048
971
  options.role === "control"
@@ -1103,6 +1026,22 @@ export type RunOpenGeniWorkerOptions = OpenGeniWorkerServiceOptions & {
1103
1026
  /** Start, drain, and close one embedded worker process. */
1104
1027
  export async function runOpenGeniWorker(options: RunOpenGeniWorkerOptions): Promise<void> {
1105
1028
  const service = await createOpenGeniWorkerService(options);
1029
+ // Reclaim multi-gigabyte workspace archive spools left by a previous worker
1030
+ // process that stopped mid-capture/upload. Only provably dead owners' spools
1031
+ // are removed, so peers sharing this TMPDIR are unaffected.
1032
+ void import("@opengeni/runtime/sandbox")
1033
+ .then(({ sweepOrphanedHostArchiveTemporaryDirectoriesOnce }) =>
1034
+ sweepOrphanedHostArchiveTemporaryDirectoriesOnce(),
1035
+ )
1036
+ .then((removed) => {
1037
+ if (removed.length > 0) {
1038
+ options.activityDependencies.observability?.info(
1039
+ "worker removed orphaned workspace archive spools",
1040
+ { count: removed.length },
1041
+ );
1042
+ }
1043
+ })
1044
+ .catch(() => undefined);
1106
1045
  const signals =
1107
1046
  options.shutdownSignals === false ? [] : (options.shutdownSignals ?? ["SIGTERM", "SIGINT"]);
1108
1047
  const handlers = signals.map((signal) => {
@@ -1127,6 +1066,8 @@ export async function startWorker() {
1127
1066
  }
1128
1067
  const settings = getSettings();
1129
1068
  const observability = createObservability(settings, { component: `worker-${role}` });
1069
+ // Before any NativeConnection: SDK and core logs become structured lines.
1070
+ installTemporalRuntimeLogging(observability);
1130
1071
  // The Agents SDK also listens for unhandled rejections and exits the process.
1131
1072
  // Keep detached SDK/transport promises observational here: durable turn
1132
1073
  // activity boundaries own failures, while process exit causes cross-session
@@ -1154,8 +1095,6 @@ export async function startWorker() {
1154
1095
  rlsStrategy: settings.rlsStrategy,
1155
1096
  expectedRole: settings.runtimeDatabaseRole,
1156
1097
  targetSchema: settings.dbSchema.trim() || "public",
1157
- organizationTenancyCanonicalActivationEnabled:
1158
- settings.organizationTenancyCanonicalActivationEnabled,
1159
1098
  } as const;
1160
1099
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
1161
1100
  let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
@@ -1173,12 +1112,13 @@ export async function startWorker() {
1173
1112
  controlPlaneAuth
1174
1113
  ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
1175
1114
  : undefined,
1176
- { logger: observabilityEventLogger(observability) },
1115
+ observabilityEventBusOptions(observability),
1177
1116
  ),
1178
1117
  { ...retryOptions, onRetry },
1179
1118
  );
1180
1119
  await runOpenGeniWorker({
1181
1120
  role,
1121
+ terminateWorker: () => process.exit(1),
1182
1122
  settings,
1183
1123
  databasePosture,
1184
1124
  http: { readinessDb: readinessDbClient.db },