@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -0,0 +1,347 @@
1
+ import { temporalConnectionOptions, type Settings } from "@opengeni/config";
2
+ import {
3
+ SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
4
+ SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
5
+ SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
6
+ SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
7
+ SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
8
+ } from "@opengeni/core";
9
+ import {
10
+ automaticSessionTitleFanoutDue,
11
+ browserDeadlineCheckpointDue,
12
+ knowledgeIndexDue,
13
+ sandboxIdleCheckpointDue,
14
+ sandboxLeaseReaperDue,
15
+ sandboxMaintenanceDue,
16
+ sessionWorkflowWakesDue,
17
+ siteAuthMaintenanceDue,
18
+ workspacePauseTimerDue,
19
+ type Database,
20
+ } from "@opengeni/db";
21
+ import type { DocumentServices } from "@opengeni/documents";
22
+ import type { Observability } from "@opengeni/observability";
23
+ import {
24
+ Connection,
25
+ ScheduleNotFoundError,
26
+ Client as TemporalClient,
27
+ WorkflowExecutionAlreadyStartedError,
28
+ } from "@temporalio/client";
29
+ import {
30
+ startMaintenanceCadences,
31
+ type MaintenanceCadence,
32
+ type MaintenanceCadences,
33
+ type MaintenanceStartResult,
34
+ } from "./maintenance-cadence";
35
+ import {
36
+ BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
37
+ browserDeadlineCheckpointTaskQueue,
38
+ } from "./browser-deadline-checkpoint-contract";
39
+ import {
40
+ SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
41
+ SANDBOX_REAPER_V2_WORKFLOW_ID,
42
+ sandboxLifecycleTaskQueue,
43
+ } from "./sandbox-reaper-contract";
44
+ import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
45
+ import type { TelemetryScheduler } from "./telemetry-scheduler";
46
+
47
+ export const KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
48
+ export const KNOWLEDGE_INDEXING_WORKFLOW_ID = "opengeni-knowledge-indexing";
49
+ export const KNOWLEDGE_INDEXING_PERIOD_MS = 15_000;
50
+ export const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
51
+ export const SITE_AUTH_MAINTENANCE_WORKFLOW_ID = "opengeni-site-auth-maintenance";
52
+ /** Matches the minimum public site-auth health policy interval. */
53
+ export const SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1_000;
54
+ /** The Schedule earlier releases used to drive the sandbox lease reaper. */
55
+ export const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
56
+
57
+ export const INTERNAL_MAINTENANCE_JOBS = [
58
+ "session-workflow-wake-dispatcher",
59
+ "knowledge-indexing",
60
+ "site-auth-maintenance",
61
+ "sandbox-lease-reaper",
62
+ "sandbox-idle-checkpoint",
63
+ "browser-deadline-checkpoint",
64
+ ] as const;
65
+ export type InternalMaintenanceJob = (typeof INTERNAL_MAINTENANCE_JOBS)[number];
66
+
67
+ /** Schedules that earlier releases created for these jobs. A paused Schedule
68
+ * still runs its scheduler workflow on every tick, so they are deleted. */
69
+ const LEGACY_SCHEDULE_IDS: Partial<Record<InternalMaintenanceJob, string>> = {
70
+ "session-workflow-wake-dispatcher": SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
71
+ "knowledge-indexing": KNOWLEDGE_INDEXING_SCHEDULE_ID,
72
+ "site-auth-maintenance": SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
73
+ // The old reaper Schedule fired a router workflow that started the drain
74
+ // inventory, both checkpoint sweeps and the maintenance child on every tick.
75
+ "sandbox-lease-reaper": SANDBOX_REAPER_SCHEDULE_ID,
76
+ };
77
+
78
+ type WorkflowStarter = Pick<TemporalClient["workflow"], "start" | "signalWithStart">;
79
+ type ScheduleDeleter = { getHandle(id: string): { delete(): Promise<void> } };
80
+
81
+ /** Start a fixed-id maintenance workflow unless one is already running; the
82
+ * running sweep's own claims cover whatever this tick saw. */
83
+ export async function startMaintenanceWorkflowOnce(
84
+ workflow: WorkflowStarter,
85
+ input: { workflowType: string; workflowId: string; taskQueue: string },
86
+ ): Promise<MaintenanceStartResult> {
87
+ try {
88
+ await workflow.start(input.workflowType, {
89
+ taskQueue: input.taskQueue,
90
+ workflowId: input.workflowId,
91
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
92
+ args: [],
93
+ });
94
+ return "started";
95
+ } catch (error) {
96
+ if (error instanceof WorkflowExecutionAlreadyStartedError) return "already_running";
97
+ throw error;
98
+ }
99
+ }
100
+
101
+ /** Remove the per-tick Schedules this release replaces. Absent is success. */
102
+ export async function deleteLegacyMaintenanceSchedules(
103
+ schedule: ScheduleDeleter,
104
+ jobs: readonly InternalMaintenanceJob[],
105
+ observability: Observability,
106
+ ): Promise<void> {
107
+ for (const job of jobs) {
108
+ const scheduleId = LEGACY_SCHEDULE_IDS[job];
109
+ if (!scheduleId) continue;
110
+ try {
111
+ await schedule.getHandle(scheduleId).delete();
112
+ observability.info(
113
+ "Deleted a per-tick maintenance Schedule replaced by a work-checked cadence",
114
+ {
115
+ job,
116
+ scheduleId,
117
+ },
118
+ );
119
+ } catch (error) {
120
+ if (!(error instanceof ScheduleNotFoundError)) throw error;
121
+ }
122
+ }
123
+ }
124
+
125
+ /**
126
+ * Build the work-checked cadences for the deployment-wide dispatchers. Each
127
+ * tick runs one cheap read-only check and touches Temporal only when the check
128
+ * may find work, so an idle deployment writes no workflow history at all.
129
+ */
130
+ export async function internalMaintenanceCadences(input: {
131
+ settings: Settings;
132
+ observability: Observability;
133
+ db: Database;
134
+ workflow: WorkflowStarter;
135
+ documentServices?: DocumentServices;
136
+ jobs?: readonly InternalMaintenanceJob[];
137
+ }): Promise<MaintenanceCadence[]> {
138
+ const { settings, observability, db, workflow } = input;
139
+ const jobs = new Set(input.jobs ?? INTERNAL_MAINTENANCE_JOBS);
140
+ const taskQueue = settings.temporalTaskQueue;
141
+ const cadences: MaintenanceCadence[] = [];
142
+ if (jobs.has("session-workflow-wake-dispatcher")) {
143
+ const { repairMissedSessionWorkflowWakes } = await import("./activities/parent-wake");
144
+ cadences.push({
145
+ job: "session-workflow-wake-dispatcher",
146
+ periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
147
+ hasWork: async () => {
148
+ // Registration is the dispatcher's own idempotent repair: it writes
149
+ // only when a producer committed work but never recorded its wake.
150
+ await repairMissedSessionWorkflowWakes({ db, observability });
151
+ return (
152
+ (await sessionWorkflowWakesDue(db)) ||
153
+ (await workspacePauseTimerDue(db)) ||
154
+ (await automaticSessionTitleFanoutDue(db))
155
+ );
156
+ },
157
+ start: async () => {
158
+ await workflow.signalWithStart(SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE, {
159
+ taskQueue,
160
+ workflowId: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
161
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
162
+ args: [],
163
+ signal: SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
164
+ });
165
+ return "started";
166
+ },
167
+ });
168
+ }
169
+ if (jobs.has("knowledge-indexing")) {
170
+ let embedding: Promise<{ model: string; dimensions: number; available: boolean }> | undefined;
171
+ let unavailableReported = false;
172
+ const resolveEmbedding = async () => {
173
+ const services =
174
+ input.documentServices ??
175
+ (await import("@opengeni/documents")).createDocumentServices(settings);
176
+ return {
177
+ model: services.embedder.model,
178
+ dimensions: services.embedder.dimensions,
179
+ available: services.embedder.available !== false,
180
+ };
181
+ };
182
+ cadences.push({
183
+ job: "knowledge-indexing",
184
+ periodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
185
+ hasWork: async () => {
186
+ embedding ??= resolveEmbedding().catch((error) => {
187
+ embedding = undefined;
188
+ throw error;
189
+ });
190
+ const current = await embedding;
191
+ // Without an embedding credential every batch fails before reaching the
192
+ // provider, so queued jobs are not work this process can do. They stay
193
+ // queued and index once the deployment configures a credential.
194
+ if (!current.available) {
195
+ if (!unavailableReported) {
196
+ unavailableReported = true;
197
+ observability.warn(
198
+ "Knowledge indexing is idle: no document embedding credential is configured",
199
+ { job: "knowledge-indexing", model: current.model },
200
+ );
201
+ }
202
+ return false;
203
+ }
204
+ return await knowledgeIndexDue(db, current);
205
+ },
206
+ start: async () =>
207
+ await startMaintenanceWorkflowOnce(workflow, {
208
+ workflowType: "knowledgeIndexingWorkflow",
209
+ workflowId: KNOWLEDGE_INDEXING_WORKFLOW_ID,
210
+ taskQueue,
211
+ }),
212
+ });
213
+ }
214
+ if (jobs.has("site-auth-maintenance")) {
215
+ const { SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS } =
216
+ await import("./activities/site-auth-maintenance");
217
+ cadences.push({
218
+ job: "site-auth-maintenance",
219
+ periodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
220
+ hasWork: async () =>
221
+ await siteAuthMaintenanceDue(db, {
222
+ claimTimeoutMs: SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS,
223
+ }),
224
+ start: async () =>
225
+ await startMaintenanceWorkflowOnce(workflow, {
226
+ workflowType: "siteAuthMaintenanceWorkflow",
227
+ workflowId: SITE_AUTH_MAINTENANCE_WORKFLOW_ID,
228
+ taskQueue,
229
+ }),
230
+ });
231
+ }
232
+ const sandboxJobs = (
233
+ ["sandbox-lease-reaper", "sandbox-idle-checkpoint", "browser-deadline-checkpoint"] as const
234
+ ).filter((job) => jobs.has(job));
235
+ if (sandboxJobs.length > 0) {
236
+ // The same period the old reaper Schedule used, so the boot invariants
237
+ // (reaper period below the viewer holder TTL, and with the idle grace
238
+ // below the provider lifetime) hold for every sandbox cadence.
239
+ assertSandboxReaperActivityTimeout(settings);
240
+ const periodMs = settings.sandboxLeaseReaperPeriodMs;
241
+ const lifecycleQueue = sandboxLifecycleTaskQueue(taskQueue);
242
+ const sandbox = await import("./activities/sandbox-lease");
243
+ if (jobs.has("sandbox-lease-reaper")) {
244
+ const { SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS } = sandbox;
245
+ cadences.push({
246
+ job: "sandbox-lease-reaper",
247
+ periodMs,
248
+ // The drain inventory and its maintenance pass act only on warm or held
249
+ // leases, due reconciliation rows, or (with Modal) boxes still running in
250
+ // the provider app. With none of those, the tick only refreshes the
251
+ // pass's read-only gauges in this process.
252
+ hasWork: async () => {
253
+ if (
254
+ (await sandboxLeaseReaperDue(db)) ||
255
+ (await sandboxMaintenanceDue(db, {
256
+ checkpointArtifactClaimTtlMs: SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS,
257
+ })) ||
258
+ (await sandbox.modalOrphanSweepMayHaveWork({ settings, observability }))
259
+ ) {
260
+ return true;
261
+ }
262
+ await sandbox.refreshIdleSandboxReaperProjections({ db, settings, observability });
263
+ return false;
264
+ },
265
+ start: async () =>
266
+ await startMaintenanceWorkflowOnce(workflow, {
267
+ workflowType: "sandboxReaperWorkflowV2",
268
+ workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
269
+ taskQueue: lifecycleQueue,
270
+ }),
271
+ });
272
+ }
273
+ if (jobs.has("sandbox-idle-checkpoint")) {
274
+ cadences.push({
275
+ job: "sandbox-idle-checkpoint",
276
+ periodMs,
277
+ // The sweep's inventory is empty when ownership or snapshots are off.
278
+ hasWork: async () =>
279
+ settings.sandboxOwnershipEnabled &&
280
+ settings.sandboxSnapshotIntervalMs > 0 &&
281
+ (await sandboxIdleCheckpointDue(db, { intervalMs: settings.sandboxSnapshotIntervalMs })),
282
+ start: async () =>
283
+ await startMaintenanceWorkflowOnce(workflow, {
284
+ workflowType: "sandboxIdleCheckpointSweepWorkflow",
285
+ workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
286
+ taskQueue: lifecycleQueue,
287
+ }),
288
+ });
289
+ }
290
+ if (jobs.has("browser-deadline-checkpoint")) {
291
+ cadences.push({
292
+ job: "browser-deadline-checkpoint",
293
+ periodMs,
294
+ hasWork: async () => await browserDeadlineCheckpointDue(db),
295
+ start: async () =>
296
+ await startMaintenanceWorkflowOnce(workflow, {
297
+ workflowType: "browserDeadlineCheckpointSweepWorkflow",
298
+ workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
299
+ taskQueue: browserDeadlineCheckpointTaskQueue(taskQueue),
300
+ }),
301
+ });
302
+ }
303
+ }
304
+ return cadences;
305
+ }
306
+
307
+ /**
308
+ * Run the deployment-wide maintenance dispatchers from this control worker.
309
+ * Several control replicas may each run the cadence: an idle tick is only a
310
+ * read, and a busy one converges on the same fixed workflow id.
311
+ */
312
+ export async function startInternalMaintenanceCadences(input: {
313
+ settings: Settings;
314
+ observability: Observability;
315
+ db: Database;
316
+ documentServices?: DocumentServices;
317
+ jobs?: readonly InternalMaintenanceJob[];
318
+ scheduler?: TelemetryScheduler;
319
+ }): Promise<{ cadences: MaintenanceCadences; close: () => Promise<void> }> {
320
+ const connection = await Connection.connect(temporalConnectionOptions(input.settings));
321
+ try {
322
+ const temporal = new TemporalClient({
323
+ connection,
324
+ namespace: input.settings.temporalNamespace,
325
+ });
326
+ const jobs = input.jobs ?? INTERNAL_MAINTENANCE_JOBS;
327
+ await deleteLegacyMaintenanceSchedules(temporal.schedule, jobs, input.observability);
328
+ const cadences = startMaintenanceCadences({
329
+ cadences: await internalMaintenanceCadences({ ...input, jobs, workflow: temporal.workflow }),
330
+ observability: input.observability,
331
+ ...(input.scheduler ? { scheduler: input.scheduler } : {}),
332
+ });
333
+ input.observability.info("Started work-checked maintenance cadences", {
334
+ jobs: jobs.join(","),
335
+ });
336
+ return {
337
+ cadences,
338
+ close: async () => {
339
+ await cadences.close();
340
+ await connection.close();
341
+ },
342
+ };
343
+ } catch (error) {
344
+ await connection.close().catch(() => undefined);
345
+ throw error;
346
+ }
347
+ }
@@ -0,0 +1,156 @@
1
+ import type { Observability } from "@opengeni/observability";
2
+ import {
3
+ telemetryInterval,
4
+ telemetryScheduler,
5
+ type TelemetryScheduler,
6
+ } from "./telemetry-scheduler";
7
+
8
+ export type MaintenanceStartResult = "started" | "already_running";
9
+
10
+ export type MaintenanceTickOutcome =
11
+ /** The work check proved there is nothing to do; no workflow was touched. */
12
+ | "idle"
13
+ | "started"
14
+ | "already_running"
15
+ /** The check failed; the workflow was started anyway (fail open). */
16
+ | "probe_failed"
17
+ | "start_failed"
18
+ /** This process's previous tick for the job has not finished. */
19
+ | "busy";
20
+
21
+ const TICK_OUTCOMES: readonly MaintenanceTickOutcome[] = [
22
+ "idle",
23
+ "started",
24
+ "already_running",
25
+ "probe_failed",
26
+ "start_failed",
27
+ "busy",
28
+ ];
29
+
30
+ /**
31
+ * One deployment-wide maintenance job driven by a worker-process cadence
32
+ * instead of a Temporal Schedule. A Schedule starts a workflow execution on
33
+ * every tick, so an idle deployment pays a history and visibility write per
34
+ * tick for no work. A cadence instead asks a cheap read-only question first and
35
+ * starts the job's workflow only when the answer may be yes.
36
+ */
37
+ export type MaintenanceCadence = {
38
+ /** Stable, low-cardinality metric label. */
39
+ job: string;
40
+ periodMs: number;
41
+ /**
42
+ * A superset of the job's claim predicate: `false` must prove that a run
43
+ * would find nothing. It may perform the job's own idempotent repair
44
+ * registration, but never claims or delivers work.
45
+ */
46
+ hasWork: () => Promise<boolean>;
47
+ /** Start or nudge the job's single dispatcher workflow. */
48
+ start: () => Promise<MaintenanceStartResult>;
49
+ };
50
+
51
+ export type MaintenanceCadences = {
52
+ /** Run one tick for a job now; used by tests and by the cadence itself. */
53
+ tick: (job: string) => Promise<MaintenanceTickOutcome>;
54
+ close: () => Promise<void>;
55
+ };
56
+
57
+ export function startMaintenanceCadences(input: {
58
+ cadences: readonly MaintenanceCadence[];
59
+ observability: Observability;
60
+ scheduler?: TelemetryScheduler;
61
+ now?: () => number;
62
+ /** Tests drive ticks explicitly. */
63
+ autoStart?: boolean;
64
+ }): MaintenanceCadences {
65
+ const scheduler = input.scheduler ?? telemetryScheduler;
66
+ const now = input.now ?? Date.now;
67
+ const jobs = new Map<string, MaintenanceCadence>();
68
+ for (const cadence of input.cadences) {
69
+ if (jobs.has(cadence.job)) throw new Error(`duplicate maintenance cadence ${cadence.job}`);
70
+ telemetryInterval(cadence.periodMs, `${cadence.job} maintenance period`);
71
+ jobs.set(cadence.job, cadence);
72
+ for (const outcome of TICK_OUTCOMES) {
73
+ input.observability.incrementCounter({
74
+ name: "opengeni_maintenance_cadence_ticks_total",
75
+ help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
76
+ labels: { job: cadence.job, outcome },
77
+ amount: 0,
78
+ });
79
+ }
80
+ }
81
+ const inFlight = new Map<string, Promise<MaintenanceTickOutcome>>();
82
+ const cancels: Array<() => void> = [];
83
+ let stopped = false;
84
+
85
+ const record = (job: string, outcome: MaintenanceTickOutcome) => {
86
+ input.observability.incrementCounter({
87
+ name: "opengeni_maintenance_cadence_ticks_total",
88
+ help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
89
+ labels: { job, outcome },
90
+ });
91
+ if (outcome !== "busy" && outcome !== "start_failed") {
92
+ input.observability.setGauge({
93
+ name: "opengeni_maintenance_cadence_last_success_timestamp_seconds",
94
+ help: "Unix time of the latest maintenance cadence tick that checked for work and, when needed, started its workflow.",
95
+ labels: { job },
96
+ value: now() / 1_000,
97
+ });
98
+ }
99
+ return outcome;
100
+ };
101
+
102
+ const run = async (cadence: MaintenanceCadence): Promise<MaintenanceTickOutcome> => {
103
+ let probeFailed = false;
104
+ try {
105
+ if (!(await cadence.hasWork())) return record(cadence.job, "idle");
106
+ } catch (error) {
107
+ // Fail open: a broken check must never silently stop maintenance. The
108
+ // workflow's own claim remains the authority on what work exists.
109
+ probeFailed = true;
110
+ input.observability.warn("maintenance cadence work check failed; starting the job anyway", {
111
+ job: cadence.job,
112
+ error: error instanceof Error ? error.message : String(error),
113
+ });
114
+ }
115
+ try {
116
+ const started = await cadence.start();
117
+ return record(cadence.job, probeFailed ? "probe_failed" : started);
118
+ } catch (error) {
119
+ input.observability.warn("maintenance cadence could not start its workflow", {
120
+ job: cadence.job,
121
+ error: error instanceof Error ? error.message : String(error),
122
+ });
123
+ return record(cadence.job, "start_failed");
124
+ }
125
+ };
126
+
127
+ const tick = async (job: string): Promise<MaintenanceTickOutcome> => {
128
+ const cadence = jobs.get(job);
129
+ if (!cadence) throw new Error(`unknown maintenance cadence ${job}`);
130
+ if (stopped) return "busy";
131
+ if (inFlight.has(job)) return record(job, "busy");
132
+ const running = run(cadence).finally(() => inFlight.delete(job));
133
+ inFlight.set(job, running);
134
+ return await running;
135
+ };
136
+
137
+ if (input.autoStart !== false) {
138
+ for (const cadence of jobs.values()) {
139
+ const fire = () => {
140
+ void tick(cadence.job);
141
+ };
142
+ // Start immediately so a restart never delays due work by a full period.
143
+ scheduler.defer(fire);
144
+ cancels.push(scheduler.interval(fire, cadence.periodMs));
145
+ }
146
+ }
147
+
148
+ return {
149
+ tick,
150
+ close: async () => {
151
+ stopped = true;
152
+ for (const cancel of cancels.splice(0)) cancel();
153
+ await Promise.allSettled([...inFlight.values()]);
154
+ },
155
+ };
156
+ }