@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -1,35 +1,40 @@
1
+ import { recordSessionUsageLimitEvent } from "@opengeni/core";
1
2
  import {
2
3
  requestSessionTurnRecovery,
3
- getSessionGoal,
4
- armXaiCapacityWait,
5
- reconcileXaiCapacityWait,
6
- listCodexAccountStatuses,
7
- quarantineCodexCredentialForLease,
8
4
  recordUsageEvent,
9
5
  getActiveSessionHistoryItemsPaged,
10
- settleCodexCredentialLeaseLoss,
11
- settleCodexCredentialFailover,
12
6
  readLease,
13
7
  SandboxLeaseSupersededError,
14
8
  isSessionEventPersistenceError,
15
- type CodexLeaseAccountStatus,
9
+ SANDBOX_SETUP_RECOVERY_LIMIT,
10
+ SubscriptionCoreCodexAccessLostError,
11
+ SubscriptionCoreCodexSourceDisconnectedError,
12
+ SubscriptionCoreCodexRequestOutcomeUnknownError,
13
+ isSubscriptionCoreCodexSourceDisconnected,
14
+ SubscriptionCoreCodexLeaseLostError,
15
+ readSessionTurnToolProgressRecoveryState,
16
16
  } from "@opengeni/db";
17
17
  import { publishDurableSessionEvents } from "@opengeni/events";
18
- import { maxTurnsExceededRunState } from "@opengeni/runtime";
19
- import { CancelledFailure } from "@temporalio/activity";
20
18
  import {
21
- authoritativeCodexCapacityResetAt,
22
- classifyCodexPin,
23
- selectCodexCredentialLeaseForTurn,
24
- type CodexRotationStrategy,
25
- } from "../codex-rotation";
26
- import type { Settings } from "@opengeni/config";
19
+ maxTurnsExceededRunState,
20
+ isModalTaskExecStartPreDispatchUnavailableError,
21
+ isModalCommandStartOutcomeUnknownError,
22
+ isProviderCommandObservationUnavailableError,
23
+ isRoutingMutationOutcomeUnknownError,
24
+ toolCallCommandNotStartedCallIds,
25
+ } from "@opengeni/runtime";
26
+ import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
27
+
28
+ import { TurnExecutionPolicyDefinitionMismatchError, type Settings } from "@opengeni/config";
27
29
  import {
30
+ CodexReloginRequired,
31
+ CodexStreamingTerminalError,
28
32
  classifyCodexEncryptedArtifactRejection,
33
+ classifyCodexEntitlementRejection,
29
34
  classifyCodexUsageLimitError,
30
35
  isCodexTransportError,
31
- type CodexUsageHeaderSnapshot,
32
36
  } from "@opengeni/codex";
37
+
33
38
  import { TurnAttemptFencedError } from "../turn-attempt-fenced";
34
39
  import { deliverFailedChildTurnToParent } from "../parent-wake";
35
40
  import type {
@@ -38,13 +43,32 @@ import type {
38
43
  RunAgentTurnResult,
39
44
  } from "../types";
40
45
  import { CodexCredentialLeaseLostError, createTurnCredentialLeases } from "./credential-leases";
46
+ import {
47
+ SubscriptionCoreCodexTurnError,
48
+ subscriptionCoreAccountRefusedFailure,
49
+ subscriptionCoreLeaseBusyChain,
50
+ subscriptionCoreLeaseBusyDelayMs,
51
+ subscriptionCoreLeaseBusyExhaustedFailure,
52
+ } from "./codex-core-errors";
53
+ import { recordCoreCodexRefusal } from "./codex-core-settlement";
54
+ import { failOverCoreCodexTurn } from "./codex-core-failover";
41
55
  import { createTurnHistorySink } from "./history-sink";
42
56
 
43
- import { BudgetExhaustedError } from "./admission";
57
+ import { BudgetExhaustedError, sessionUsageLimitStopEvents } from "./admission";
58
+ import {
59
+ SESSION_USAGE_CUSTODY_PENDING,
60
+ SESSION_USAGE_CUSTODY_RECHECK_MS,
61
+ sessionUsageCustodyPending,
62
+ } from "./session-usage-custody";
44
63
  import {
64
+ automaticRecoveryUnavailableFailure,
45
65
  providerRecoveryExhaustedFailure,
66
+ withModelRoutePresentation,
46
67
  postClaimDatabaseRecoveryFailure,
47
68
  providerRecoveryResult,
69
+ providerRecoveryCode,
70
+ providerRecoveryLimit,
71
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
48
72
  providerRetryAfterMs,
49
73
  escapedMcpTimeoutRecoveryFailure,
50
74
  preClaimAdmissionFailure,
@@ -52,9 +76,8 @@ import {
52
76
  sandboxLifecycleTransitionDiagnostic,
53
77
  sandboxRouteTransitionCode,
54
78
  safeErrorDiagnostic,
55
- classifyXaiCredentialFailure,
56
79
  agentRunFailurePayload,
57
- codexCredentialCooldownUntil,
80
+ agentRunRecoveryFailurePayload,
58
81
  classifyCodexCredentialFailure,
59
82
  codexUsageLimitFailurePayload,
60
83
  } from "./errors";
@@ -68,12 +91,22 @@ import {
68
91
  import type {
69
92
  AttemptIdentityState,
70
93
  BillingState,
94
+ CodexSubscriptionCoreTurn,
71
95
  EventingState,
72
96
  ProviderTurnState,
73
97
  TurnControlState,
74
98
  } from "./turn-context";
75
- import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
76
- import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
99
+ import type { createCoreCodexRequests } from "./codex-core-requests";
100
+ import {
101
+ XAI_CORE_REFUSAL,
102
+ xaiCoreLanesRecoverAfterCheckpoint,
103
+ xaiCoreRequestOutcomeUnknown,
104
+ } from "./xai-core";
105
+ import { settleCoreSubscriptionRefusal } from "./subscription-core-settlement";
106
+ import { claudeCoreRefusal, claudeCoreRequestOutcomeUnknown } from "./claude-core";
107
+ import { coreLanesRecoverAfterCheckpoint } from "./subscription-core-requests";
108
+
109
+ import { providerRecoveryCause, recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
77
110
 
78
111
  export type TurnFailureDeps = {
79
112
  error: unknown;
@@ -83,6 +116,8 @@ export type TurnFailureDeps = {
83
116
  bus: ActivityServices["bus"];
84
117
  observability: ActivityServices["observability"];
85
118
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
119
+ /** Typed capacity-wake signals of core waiters (a refusal's joined observations may wake them). */
120
+ signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"];
86
121
  cancellationSignal: AbortSignal | undefined;
87
122
  sandboxRotationController: AbortController;
88
123
  noteCancellationRequested: () => void;
@@ -107,124 +142,41 @@ export type TurnFailureDeps = {
107
142
 
108
143
  export type CodexDefinitiveFailureDisposition = "failover" | "wait" | "terminal";
109
144
 
110
- type CodexCapacityWaitFailurePayload = {
111
- error: string;
112
- code: string;
113
- detail?: string;
114
- retryable: false;
115
- };
116
-
117
- /**
118
- * Pure policy for a definitive serving-credential refusal. A policy-constrained
119
- * account and an all-unavailable pool wait for the same selected capacity to
120
- * recover; only a truly empty/non-allocatable pool makes an auth/forbidden
121
- * failure terminal. A different eligible account under rotation-on policy may
122
- * recover the same durable turn immediately.
123
- */
124
- export function codexDefinitiveFailureDisposition(input: {
125
- failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
126
- rotationEnabled: boolean;
127
- pinDisposition: "manual" | "sharded" | "clearStale" | "unpinned";
128
- decisionKind: "active" | "allCapped" | "none";
129
- decisionCredentialId: string | null;
130
- servingCredentialId: string;
131
- }): CodexDefinitiveFailureDisposition {
132
- const alternateAvailable =
133
- input.rotationEnabled &&
134
- input.pinDisposition !== "manual" &&
135
- input.decisionKind === "active" &&
136
- input.decisionCredentialId !== null &&
137
- input.decisionCredentialId !== input.servingCredentialId;
138
- if (alternateAvailable) return "failover";
139
- if (
140
- input.failureKind === "quota" ||
141
- input.failureKind === "rate_limit" ||
142
- input.decisionKind === "allCapped" ||
143
- !input.rotationEnabled ||
144
- input.pinDisposition === "manual"
145
- ) {
146
- return "wait";
147
- }
148
- return "terminal";
149
- }
150
-
151
- /**
152
- * Bound one turn to the alternate credentials that policy actually permits.
153
- * The effective account list is already workspace/organization scoped;
154
- * allocator-disabled rows must not enlarge the retry budget. The database
155
- * requires a positive bound even though one-account paths never fail over.
156
- */
157
- export function codexCredentialFailoverLimit(
158
- accounts: ReadonlyArray<{ id: string; allocatorEnabled: boolean }>,
159
- servingCredentialId: string,
160
- ): number {
161
- const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled).length;
162
- const servingIsAllocatable = accounts.some(
163
- (account) => account.id === servingCredentialId && account.allocatorEnabled,
164
- );
165
- return Math.max(1, allocatableAccounts - (servingIsAllocatable ? 1 : 0));
166
- }
167
-
168
- /** Build the durable waiter payload without collapsing quota refusals into 403. */
169
- export function codexCapacityWaitFailurePayload(input: {
170
- failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
171
- usageLimit: { resetsInSeconds: number | null } | null;
172
- cooldownSeconds: number | null;
173
- detail: string;
174
- allAccounts: boolean;
175
- }): CodexCapacityWaitFailurePayload {
176
- if (input.failureKind === "quota") {
177
- return codexUsageLimitFailurePayload(
178
- input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
179
- input.detail,
180
- input.allAccounts ? { allAccounts: true } : undefined,
181
- );
182
- }
183
- if (input.failureKind === "rate_limit") {
184
- return {
185
- error: "The serving Codex subscription is temporarily rate limited.",
186
- code: "codex_account_rate_limited",
187
- detail: input.detail,
188
- retryable: false,
189
- };
190
- }
191
- if (input.failureKind === "auth") {
192
- return {
193
- error: "The serving Codex account requires reconnection.",
194
- code: "codex_relogin_required",
195
- detail: "the same accepted turn is waiting for the selected account to recover",
196
- retryable: false,
197
- };
198
- }
199
- return {
200
- error: "The serving Codex account is not authorized for this request.",
201
- code: "codex_account_forbidden",
202
- detail: "the same accepted turn is waiting for the selected account to recover",
203
- retryable: false,
204
- };
205
- }
206
-
207
- function codexLeaseAccountsForSelection(
208
- accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>,
209
- ): CodexLeaseAccountStatus[] {
210
- return accounts.map((account) => ({
211
- ...account,
212
- activeLeaseCount: 0,
213
- selectionCount: 0,
214
- lastSelectedAt: null,
215
- }));
216
- }
217
-
218
- function acceptedCodexPolicySnapshot(
219
- providerTurn: ProviderTurnState,
220
- ): CodexCredentialPolicySnapshotV1 {
221
- if (!providerTurn.codexPolicySnapshot) {
222
- throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
145
+ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
146
+ try {
147
+ return await settleTurnFailureInAttempt(deps);
148
+ } catch (error) {
149
+ if (
150
+ deps.control.activityStatus === "recovering" &&
151
+ error instanceof ApplicationFailure &&
152
+ error.type === "OpenGeniPostClaimDatabaseRecovery"
153
+ )
154
+ throw error;
155
+ // Connectivity can disappear while settling an unrelated run error too.
156
+ // Do not overwrite a possibly committed settlement; the control lane
157
+ // re-reads exact ownership and becomes a stale no-op if it already closed.
158
+ if (deps.attempt.turnId && deps.attempt.triggerEventId) {
159
+ const recovery = postClaimDatabaseRecoveryFailure({
160
+ // A failed rollback/terminal write must not erase no-replay evidence
161
+ // from the failure we were settling (notably unknown tool effects).
162
+ error: new AggregateError([error, deps.error], "Turn failure settlement failed"),
163
+ turnId: deps.attempt.turnId,
164
+ triggerEventId: deps.attempt.triggerEventId,
165
+ executionGeneration: deps.attempt.executionGeneration,
166
+ requireDatabaseProvenance: true,
167
+ });
168
+ if (recovery) {
169
+ deps.control.activityStatus = "recovering";
170
+ deps.control.turnMetricOutcome = "recovering";
171
+ deps.control.activityError = error;
172
+ throw recovery;
173
+ }
174
+ }
175
+ throw error;
223
176
  }
224
- return providerTurn.codexPolicySnapshot;
225
177
  }
226
178
 
227
- export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
179
+ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
228
180
  if (deps.settings.environment === "local" && deps.error instanceof Error) {
229
181
  // Keep local startup failures diagnosable without logging error messages,
230
182
  // absolute host paths, prompts, credentials, or provider response bodies.
@@ -302,6 +254,69 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
302
254
  // back to the workflow-claimed turn when the local lookup had not
303
255
  // finished yet.
304
256
  const recoveryTurnId = attempt.turnId;
257
+ // Unlike proven pre-dispatch failure, a genuine SDK Start/Wait uncertainty
258
+ // cannot reconstruct setup on a replacement attempt. The exact command and
259
+ // writer remain retained by sandbox-runtime; the logical turn is parked as
260
+ // recovering with a durable no-replay marker, not failed or completed.
261
+ const observationUnavailable = isProviderCommandObservationUnavailableError(error);
262
+ if (
263
+ (isModalCommandStartOutcomeUnknownError(error) || observationUnavailable) &&
264
+ recoveryTurnId &&
265
+ attempt.triggerEventId &&
266
+ attempt.executionGeneration > 0
267
+ ) {
268
+ let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
269
+ try {
270
+ if (eventing.turnStartedPublished) {
271
+ await flushRuntimeBatcher();
272
+ await historySink.reconcileConversationTruth({ requireDurable: true });
273
+ }
274
+ recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
275
+ sessionId: input.sessionId,
276
+ turnId: recoveryTurnId,
277
+ triggerEventId: attempt.triggerEventId,
278
+ attemptId: input.attemptId,
279
+ reason: observationUnavailable
280
+ ? "sandbox_command_observation_unavailable"
281
+ : "sandbox_command_start_outcome_unknown",
282
+ sandboxSetupOutcomeUnknown: true,
283
+ detail: {
284
+ code: observationUnavailable
285
+ ? "sandbox_command_observation_unavailable"
286
+ : "sandbox_command_start_outcome_unknown",
287
+ retryable: false,
288
+ setupOutcome: "unknown",
289
+ replay: "blocked",
290
+ providerRecoveryCount: attempt.providerRecoveryCount,
291
+ },
292
+ });
293
+ } catch (checkpointError) {
294
+ const databaseRecovery = postClaimDatabaseRecoveryFailure({
295
+ error: checkpointError,
296
+ turnId: recoveryTurnId,
297
+ triggerEventId: attempt.triggerEventId,
298
+ executionGeneration: attempt.executionGeneration,
299
+ sandboxSetupOutcomeUnknown: true,
300
+ });
301
+ if (!databaseRecovery) throw checkpointError;
302
+ control.activityStatus = "recovering";
303
+ control.turnMetricOutcome = "recovering";
304
+ control.activityError = error;
305
+ throw databaseRecovery;
306
+ }
307
+ if (recovery.action === "stale") {
308
+ acknowledgeLostAttemptOwnership();
309
+ control.activityStatus = "cancelled";
310
+ control.turnMetricOutcome = "cancelled";
311
+ return claimedResult({ status: "cancelled" });
312
+ }
313
+ acknowledgeRecoveryQuiescence();
314
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
315
+ control.activityStatus = "recovering";
316
+ control.turnMetricOutcome = "recovering";
317
+ control.activityError = error;
318
+ return claimedResult({ status: "recovering", deferredUntilWake: true });
319
+ }
305
320
  // A true epoch supersession and a provider lifecycle transition are both
306
321
  // recoverable control-plane states, never session failures. A rotation
307
322
  // persists an exact group/epoch wait marker so the workflow parks before
@@ -380,14 +395,76 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
380
395
  throw recoveryError;
381
396
  }
382
397
  }
383
- // A managed-home session can start directly on a Connected Machine without
384
- // creating or leasing its cloud home. If sandbox_attach/sandbox_swap then
385
- // clears the durable pointer to home, the commit is valid but this exact
386
- // attempt cannot serve a later home operation. Preserve the completed attach
398
+ // A route change can require a different home, filesystem root, or native
399
+ // capability set than this attempt established. Preserve the completed attach
387
400
  // and every preceding model/tool receipt, close only the unresolved suffix,
388
401
  // and continue the SAME logical turn in a fresh attempt. That next attempt
389
- // starts from the now-null pointer and establishes home normally.
402
+ // starts from the committed pointer and establishes its route normally.
390
403
  const routeTransitionCode = sandboxRouteTransitionCode(error);
404
+ // An earlier single model call in this task still owes its usage receipt
405
+ // (a fenced custody marker, or an open one past the overdue window). This
406
+ // is a wait for the owner to fill it or for maintenance to disclose it as
407
+ // unpriced, never a turn failure: keep the turn and its durable history,
408
+ // park it as recovering, and let the replacement attempt re-check
409
+ // admission (database reads only) with the workflow's bounded backoff.
410
+ // The blocked request was never sent, so nothing is replayed.
411
+ if (
412
+ sessionUsageCustodyPending(error) &&
413
+ recoveryTurnId &&
414
+ attempt.triggerEventId &&
415
+ eventing.publish &&
416
+ eventing.turnStartedPublished
417
+ ) {
418
+ recordSessionUsageLimitEvent(observability, "between_calls", "unavailable");
419
+ let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
420
+ try {
421
+ await flushRuntimeBatcher();
422
+ await historySink.reconcileConversationTruth({ requireDurable: true });
423
+ recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
424
+ sessionId: input.sessionId,
425
+ turnId: recoveryTurnId,
426
+ triggerEventId: attempt.triggerEventId,
427
+ attemptId: input.attemptId,
428
+ reason: SESSION_USAGE_CUSTODY_PENDING,
429
+ detail: {
430
+ code: SESSION_USAGE_CUSTODY_PENDING,
431
+ retryable: true,
432
+ continueDelayMs: SESSION_USAGE_CUSTODY_RECHECK_MS,
433
+ },
434
+ });
435
+ } catch (recoveryError) {
436
+ const postClaimRecovery = postClaimDatabaseRecoveryFailure({
437
+ error: recoveryError,
438
+ turnId: recoveryTurnId,
439
+ triggerEventId: attempt.triggerEventId,
440
+ executionGeneration: attempt.executionGeneration,
441
+ });
442
+ if (postClaimRecovery) {
443
+ control.activityStatus = "recovering";
444
+ control.turnMetricOutcome = "recovering";
445
+ control.activityError = error;
446
+ throw postClaimRecovery;
447
+ }
448
+ throw recoveryError;
449
+ }
450
+ if (recovery.action === "stale") {
451
+ acknowledgeLostAttemptOwnership();
452
+ control.activityStatus = "cancelled";
453
+ control.turnMetricOutcome = "cancelled";
454
+ return claimedResult({ status: "cancelled" });
455
+ }
456
+ if (recovery.action === "recovering") {
457
+ acknowledgeRecoveryQuiescence();
458
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
459
+ control.activityStatus = "recovering";
460
+ control.turnMetricOutcome = "recovering";
461
+ control.activityError = error;
462
+ return claimedResult({
463
+ status: "recovering",
464
+ continueDelayMs: SESSION_USAGE_CUSTODY_RECHECK_MS,
465
+ });
466
+ }
467
+ }
391
468
  if (routeTransitionCode && recoveryTurnId && eventing.publish && eventing.turnStartedPublished) {
392
469
  try {
393
470
  await flushRuntimeBatcher();
@@ -410,7 +487,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
410
487
  return claimedResult({ status: "cancelled" });
411
488
  }
412
489
  if (recovery.action !== "recovering") {
413
- throw new Error("Home sandbox route transition could not recover the current turn");
490
+ throw new Error("Sandbox route transition could not recover the current turn");
414
491
  }
415
492
  acknowledgeRecoveryQuiescence();
416
493
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
@@ -544,6 +621,27 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
544
621
  control.turnMetricOutcome = "cancelled";
545
622
  throw cancellationFailure;
546
623
  }
624
+ if (attempt.turnId && attempt.triggerEventId && attempt.executionGeneration > 0) {
625
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
626
+ error,
627
+ turnId: attempt.turnId,
628
+ triggerEventId: attempt.triggerEventId,
629
+ executionGeneration: attempt.executionGeneration,
630
+ requireDatabaseProvenance: eventing.turnStartedPublished || attempt.modelRequestStarted,
631
+ });
632
+ if (recoveryFailure) {
633
+ // Stop this non-retryable activity without terminal logical settlement.
634
+ // Do not retry a delta/tool-ledger mutation with an unknown commit, nor
635
+ // claim that DB-down cleanup proved writer exit. Finally still drains
636
+ // the physical writers; the existing DB-only workflow lane closes this
637
+ // exact owner after connectivity returns and admission waits for its
638
+ // ordinary durable quiescence/writer fences.
639
+ control.activityStatus = "recovering";
640
+ control.turnMetricOutcome = "recovering";
641
+ control.activityError = error;
642
+ throw recoveryFailure;
643
+ }
644
+ }
547
645
  // The SDK's per-segment turn cap is a pacing valve, not a failure: end
548
646
  // the turn gracefully and idle the session so an active goal continues
549
647
  // via a synthesized continuation turn (or a user message resumes work).
@@ -592,112 +690,132 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
592
690
  control.activityStatus = "idle";
593
691
  return claimedResult({ status: "idle" });
594
692
  }
595
- const settleLostCodexAttempt = async (
596
- lostTurnId: string,
597
- holderId: string,
598
- generation: number,
599
- historyCheckpointDurable = false,
600
- ): Promise<RunAgentTurnResult> => {
601
- let checkpointDurable = historyCheckpointDurable;
602
- try {
603
- if (!historyCheckpointDurable) {
604
- await flushRuntimeBatcher();
605
- await historySink.reconcileConversationTruth({ requireDurable: true });
606
- }
607
- checkpointDurable = true;
608
- } catch {
609
- observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
610
- errorClass: "CodexCheckpointOperationError",
611
- errorCode: "codex_lease_loss_checkpoint_failed",
612
- origin: "worker",
613
- });
614
- }
615
-
616
- const settlement = await settleCodexCredentialLeaseLoss(db, {
617
- accountId: input.accountId,
618
- workspaceId: input.workspaceId,
619
- sessionId: input.sessionId,
620
- turnId: lostTurnId,
621
- attemptId: input.attemptId,
622
- holderId,
623
- generation,
624
- expectedRedispatches: attempt.redispatchesAtDispatch,
625
- checkpointDurable,
626
- recoveryPayload: {
627
- triggerEventId: attempt.triggerEventId!,
628
- reason: "codex_lease_lost",
629
- credentialId: providerTurn.effectiveCodexCredentialId,
630
- },
631
- failedPayload: {
632
- error:
633
- "The Codex credential lease was lost and the latest conversation checkpoint could not be persisted. Automatic replay was refused.",
634
- code: "codex_lease_checkpoint_failed",
635
- retryable: false,
636
- },
637
- });
638
- leases.codex.held = false;
639
- observability.incrementCounter({
640
- name: "opengeni_codex_lease_loss_settlements_total",
641
- help: "Fenced Codex lease-loss settlements by outcome.",
642
- labels: {
643
- workspace_key: codexWorkspaceKey,
644
- outcome: settlement.action,
645
- },
646
- });
647
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, settlement.events);
648
- control.activityError = error;
649
- if (settlement.action === "failed") {
650
- control.activityStatus = "failed";
651
- control.turnMetricOutcome = "failed";
652
- await deliverFailedChildTurnToParent(
653
- { db, bus, settings, observability, wakeSessionWorkflow },
654
- input.workspaceId,
655
- input.sessionId,
656
- lostTurnId,
657
- );
658
- return claimedResult({ status: "failed" });
659
- }
660
- control.activityStatus = "recovering";
661
- control.turnMetricOutcome = "recovering";
662
- return claimedResult({ status: "recovering" });
663
- };
664
-
665
693
  // A missing/expired/superseded lease is an execution-ownership failure,
666
694
  // not a provider failure. Settle it before credential quarantine or the
667
695
  // generic terminal path: the DB transaction marks a still-current turn
668
696
  // recoverable, but a successor attempt or worker recovery makes this activity
669
697
  // stale and unable to clobber the shared turn/session.
670
- if (
671
- (leases.codex.lost || error instanceof CodexCredentialLeaseLostError) &&
672
- billingState.isCodexTurn &&
673
- eventing.publish &&
674
- attempt.turnId &&
675
- eventing.turnStartedPublished &&
676
- leases.codex.holderId &&
677
- leases.codex.generation !== null
678
- ) {
679
- return await settleLostCodexAttempt(
680
- attempt.turnId,
681
- leases.codex.holderId,
682
- leases.codex.generation,
683
- );
698
+ // Codex refusal, quota attribution and failover belong exclusively to the
699
+ // exact core placement and its generation fence.
700
+ const coreCodex = billingState.isCodexTurn ? (providerTurn.codexSubscriptionCore ?? null) : null;
701
+ // A SuperGrok turn on the shared core: unknown request outcomes are never
702
+ // replayed, and refusals settle through the shared core (X1b) instead of
703
+ // the legacy SuperGrok wait below.
704
+ const coreXai = billingState.isXaiTurn ? (providerTurn.xaiSubscriptionCore ?? null) : null;
705
+ // A Claude turn runs only on the shared core (the legacy Claude runtime is
706
+ // deleted): its refusals, unknown outcomes and lease loss settle there.
707
+ const coreClaude = billingState.isClaudeTurn
708
+ ? (providerTurn.claudeSubscriptionCore ?? null)
709
+ : null;
710
+ // The transport may settle an ambiguous request before the SDK throws a plain
711
+ // HTTP error. That custody evidence must survive classification.
712
+ //
713
+ // When the ambiguity is this attempt's own interrupted request (a stream
714
+ // that dropped or stalled before its terminal event, or a gateway timeout),
715
+ // the same turn recovers from its durable checkpoint within the bounded
716
+ // provider-recovery budget. Codex requests are stateless (`store: false`)
717
+ // and function tools only run after a complete response, so a replay can
718
+ // at most repeat provider-hosted work and spend more quota. A refusal by
719
+ // the durable admission fence, an unsettled request, or a deterministic
720
+ // provider terminal stays a terminal unknown-outcome failure, as does an
721
+ // interruption combined with a lost lease or a disconnected source.
722
+ //
723
+ // The parallel session-title request never fences replay: it is a plain,
724
+ // tool-less request whose text only ever becomes the session title, and it
725
+ // is cancelled and joined before settlement. Its custody row keeps its
726
+ // outcome; the replacement attempt's new generation is admitted past it.
727
+ const coreUnknownRecoveryFailure =
728
+ coreCodex &&
729
+ !leases.codex.lost &&
730
+ !hasErrorInCauseChain(error, CodexCredentialLeaseLostError) &&
731
+ !hasErrorInCauseChain(error, SubscriptionCoreCodexLeaseLostError) &&
732
+ !hasErrorInCauseChain(error, SubscriptionCoreCodexSourceDisconnectedError) &&
733
+ !hasErrorInCauseChain(error, SubscriptionCoreCodexAccessLostError)
734
+ ? ownInterruptedCodexRequestRecovery(error, coreCodex, billingState.isCodexTurn)
735
+ : null;
736
+ const coreRequestOutcomeUnknown =
737
+ !coreUnknownRecoveryFailure &&
738
+ (hasErrorInCauseChain(error, SubscriptionCoreCodexRequestOutcomeUnknownError) ||
739
+ coreCodex?.requests?.hasUnknownOutcome())
740
+ ? new SubscriptionCoreCodexRequestOutcomeUnknownError()
741
+ : coreXai
742
+ ? xaiCoreRequestOutcomeUnknown(error, coreXai)
743
+ : coreClaude
744
+ ? claudeCoreRequestOutcomeUnknown(error, coreClaude)
745
+ : null;
746
+ const coreCodexLeaseLost =
747
+ coreCodex !== null &&
748
+ !coreRequestOutcomeUnknown &&
749
+ (leases.codex.lost ||
750
+ error instanceof CodexCredentialLeaseLostError ||
751
+ error instanceof SubscriptionCoreCodexLeaseLostError);
752
+ let coreCodexLeaseRecoverable = false;
753
+ let coreLeaseCheckpointFailed = false;
754
+ if (coreCodexLeaseLost && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
755
+ try {
756
+ // A completed response still owns a pending reservation until its
757
+ // history commits. Test replay safety after that rendezvous, not before.
758
+ await flushRuntimeBatcher();
759
+ await historySink.reconcileConversationTruth({ requireDurable: true });
760
+ await coreCodex.requests?.checkpoint();
761
+ coreCodexLeaseRecoverable = !coreCodex.requests || coreCodex.requests.canRecover();
762
+ } catch {
763
+ coreLeaseCheckpointFailed = true;
764
+ observability.incrementCounter({
765
+ name: "opengeni_codex_failover_checkpoints_total",
766
+ help: "Durable Codex failover checkpoint attempts by outcome.",
767
+ labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
768
+ });
769
+ }
684
770
  }
685
- if (
686
- leases.xai.lost &&
771
+ // A core SuperGrok turn recovers a lost lease only once every request it
772
+ // dispatched is settled and its committed responses are checkpointed.
773
+ const xaiLeaseRecoverable =
687
774
  billingState.isXaiTurn &&
688
- eventing.publish &&
689
- attempt.turnId &&
690
- eventing.turnStartedPublished
691
- ) {
692
- await flushRuntimeBatcher();
693
- await historySink.reconcileConversationTruth({ requireDurable: true });
775
+ leases.xai.lost &&
776
+ coreXai !== null &&
777
+ !coreRequestOutcomeUnknown &&
778
+ eventing.turnStartedPublished &&
779
+ (await xaiCoreLanesRecoverAfterCheckpoint(coreXai, async () => {
780
+ await flushRuntimeBatcher();
781
+ await historySink.reconcileConversationTruth({ requireDurable: true });
782
+ }));
783
+ // The same for a core Claude turn.
784
+ const claudeLeaseRecoverable =
785
+ coreClaude !== null &&
786
+ leases.claude.lost &&
787
+ !coreRequestOutcomeUnknown &&
788
+ eventing.turnStartedPublished &&
789
+ (await coreLanesRecoverAfterCheckpoint(coreClaude, async () => {
790
+ await flushRuntimeBatcher();
791
+ await historySink.reconcileConversationTruth({ requireDurable: true });
792
+ }));
793
+ const scopedLeaseLost = claudeLeaseRecoverable
794
+ ? "claude"
795
+ : xaiLeaseRecoverable
796
+ ? "xai"
797
+ : coreCodexLeaseRecoverable
798
+ ? "codex"
799
+ : null;
800
+ if (scopedLeaseLost && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
801
+ if (scopedLeaseLost !== "codex") {
802
+ await flushRuntimeBatcher();
803
+ await historySink.reconcileConversationTruth({ requireDurable: true });
804
+ }
694
805
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
695
806
  sessionId: input.sessionId,
696
807
  turnId: attempt.turnId,
697
808
  triggerEventId: attempt.triggerEventId!,
698
809
  attemptId: input.attemptId,
699
- reason: "xai_lease_lost",
700
- detail: { provider: "supergrok-subscription" },
810
+ reason: scopedLeaseLost + "_lease_lost",
811
+ detail: {
812
+ provider:
813
+ scopedLeaseLost === "claude"
814
+ ? "claude-subscription"
815
+ : scopedLeaseLost === "codex"
816
+ ? "codex-subscription"
817
+ : "supergrok-subscription",
818
+ },
701
819
  });
702
820
  if (recovery.action === "stale") {
703
821
  acknowledgeLostAttemptOwnership();
@@ -718,631 +836,220 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
718
836
  // 5xx does not classify here and therefore cannot consume another
719
837
  // subscription or duplicate a side effect.
720
838
  const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
721
- const codexCredentialFailure =
722
- billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
723
- ? classifyCodexCredentialFailure(error)
839
+ // A definitive account refusal on a core turn becomes a typed terminal
840
+ // failure that idles the session (settled below).
841
+ let coreAccountRefusal: SubscriptionCoreCodexTurnError | null = null;
842
+ if (coreCodex && !coreRequestOutcomeUnknown && !coreLeaseCheckpointFailed) {
843
+ // Core turns record the refusal against the leased connection (failure
844
+ // receipt, plus the quota, health or model-cooldown state that keeps
845
+ // placement away from it) and re-place the same turn within the per-turn
846
+ // failover bound (M3 PR 2a). Without a recorded refusal or a durable
847
+ // checkpoint they settle through the typed paths below instead.
848
+ const coreFailure = classifyCodexCredentialFailure(error);
849
+ // Only explicit plan evidence is an entitlement refusal; an unexplained
850
+ // 400 keeps the generic typed "request rejected" copy.
851
+ const planEntitlement =
852
+ !coreFailure && classifyCodexEntitlementRejection(error)?.evidence === "plan_entitlement";
853
+ const refusal =
854
+ coreFailure ??
855
+ (planEntitlement ? { kind: "plan_entitlement" as const, cooldownSeconds: null } : null);
856
+ // Only a pre-dispatch lifecycle fence or a definite refusal can enter
857
+ // graceful recovery. A concurrent disconnect never upgrades a timeout,
858
+ // partial stream, missing response, or failed checkpoint into replay proof.
859
+ const disconnected =
860
+ hasErrorInCauseChain(error, SubscriptionCoreCodexSourceDisconnectedError) ||
861
+ (!!refusal &&
862
+ (await isSubscriptionCoreCodexSourceDisconnected(
863
+ db,
864
+ coreCodex.identity,
865
+ coreCodex.connectionId,
866
+ )));
867
+ if (disconnected) {
868
+ const settled = await failOverCoreCodexTurn(deps, coreCodex, "source_disconnected");
869
+ if (settled) return settled;
870
+ }
871
+ if (refusal && !disconnected) {
872
+ const recorded = await recordCoreCodexRefusal({
873
+ db,
874
+ core: coreCodex,
875
+ lease: leases.codex,
876
+ failure: refusal,
877
+ credentialVersion: providerTurn.effectiveCodexCredentialVersion,
878
+ modelId: providerTurn.codexProductModelId ?? null,
879
+ }).catch(() => ({ receipt: false, health: false }));
880
+ if (recorded.receipt && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
881
+ const settled = await failOverCoreCodexTurn(deps, coreCodex, refusal);
882
+ if (settled) return settled;
883
+ }
884
+ }
885
+ coreAccountRefusal =
886
+ disconnected || hasErrorInCauseChain(error, SubscriptionCoreCodexAccessLostError)
887
+ ? subscriptionCoreAccountRefusedFailure("access_lost")
888
+ : refusal?.kind === "auth" || hasErrorInCauseChain(error, CodexReloginRequired)
889
+ ? subscriptionCoreAccountRefusedFailure("sign_in")
890
+ : refusal?.kind === "forbidden"
891
+ ? subscriptionCoreAccountRefusedFailure("forbidden")
892
+ : refusal?.kind === "plan_entitlement"
893
+ ? subscriptionCoreAccountRefusedFailure("entitlement")
894
+ : null;
895
+ }
896
+ // A definitive SuperGrok account refusal on a core turn: recorded against
897
+ // the leased connection, then the same turn re-places within the shared
898
+ // bound, or fails with typed copy that idles the session (settled below).
899
+ let coreXaiRefusal: SubscriptionCoreCodexTurnError | null = null;
900
+ if (coreXai && !coreRequestOutcomeUnknown && !leases.xai.lost) {
901
+ const settled = await settleCoreSubscriptionRefusal(deps, coreXai, XAI_CORE_REFUSAL);
902
+ if (settled.result) return settled.result;
903
+ coreXaiRefusal = settled.failure;
904
+ }
905
+ // The same for a Claude core turn: a sign-in refusal (after the transport's
906
+ // one forced refresh) or a model rate limit re-places the same turn; 529
907
+ // stays in the overload lane below and 403 fails the turn, neither moving
908
+ // it to another account. The bound counts the core's turn-failure
909
+ // receipts (the legacy `claudeAuthRecovery` marker is never read here).
910
+ let coreClaudeRefusal: SubscriptionCoreCodexTurnError | null = null;
911
+ if (coreClaude && !coreRequestOutcomeUnknown && !leases.claude.lost) {
912
+ const settled = await settleCoreSubscriptionRefusal(
913
+ deps,
914
+ coreClaude,
915
+ claudeCoreRefusal(coreClaude.productModelId),
916
+ );
917
+ if (settled.result) return settled.result;
918
+ coreClaudeRefusal = settled.failure;
919
+ }
920
+
921
+ // The leased credential path above normally quarantines quota state and
922
+ // either recovers the same turn or arms a durable capacity wait. This narrow
923
+ // fallback covers failures before a credential lease existed, or a failed
924
+ // durable checkpoint where replay would be unsafe. Keep the session usable,
925
+ // but never synthesize another turn or walk an unfenced legacy pointer.
926
+ // A lease-busy chain that outlived its bound becomes a terminal failure.
927
+ const leaseBusyChain =
928
+ error instanceof SubscriptionCoreCodexTurnError &&
929
+ error.payload.code === "subscription_lease_busy" &&
930
+ error.payload.retryable
931
+ ? subscriptionCoreLeaseBusyChain(
932
+ attempt.subscriptionLeaseBusy,
933
+ attempt.executionGeneration,
934
+ Date.now(),
935
+ )
724
936
  : null;
937
+ // An older attempt of this turn still holds its core lease. Retry right
938
+ // after that lease can expire, outside the provider recovery budget: this
939
+ // is ownership handover, not provider backpressure. The chain is bounded
940
+ // above (SUBSCRIPTION_CORE_LEASE_BUSY_MAX_MS).
941
+ let leaseBusyNotRecoverable = false;
725
942
  if (
726
- codexCredentialFailure &&
727
- providerTurn.effectiveCodexCredentialId &&
943
+ leaseBusyChain &&
944
+ !leaseBusyChain.exhausted &&
945
+ error instanceof SubscriptionCoreCodexTurnError &&
728
946
  eventing.publish &&
729
947
  attempt.turnId &&
730
948
  eventing.turnStartedPublished
731
949
  ) {
732
- observability.incrementCounter({
733
- name: "opengeni_codex_credential_failures_total",
734
- help: "Definitive Codex credential failures classified for safe failover.",
735
- labels: {
736
- workspace_key: codexWorkspaceKey,
737
- kind: codexCredentialFailure.kind,
738
- outcome: "classified",
739
- },
740
- });
741
- const failoverStartedAt = performance.now();
742
- let checkpointDurable = false;
950
+ const continueDelayMs = subscriptionCoreLeaseBusyDelayMs(
951
+ error.payload.resetsAt,
952
+ Date.now(),
953
+ Math.random(),
954
+ );
955
+ let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
743
956
  try {
744
957
  await flushRuntimeBatcher();
745
958
  await historySink.reconcileConversationTruth({ requireDurable: true });
746
- checkpointDurable = true;
747
- } catch {
748
- observability.incrementCounter({
749
- name: "opengeni_codex_failover_checkpoints_total",
750
- help: "Durable Codex failover checkpoint attempts by outcome.",
751
- labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
752
- });
753
- observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
754
- errorClass: "CodexCheckpointOperationError",
755
- errorCode: "codex_failover_checkpoint_failed",
756
- origin: "worker",
959
+ recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
960
+ sessionId: input.sessionId,
961
+ turnId: attempt.turnId,
962
+ triggerEventId: attempt.triggerEventId!,
963
+ attemptId: input.attemptId,
964
+ reason: "subscription_lease_busy",
965
+ subscriptionLeaseBusy: {
966
+ startedAt: new Date(leaseBusyChain.startedAt).toISOString(),
967
+ executionGeneration: leaseBusyChain.executionGeneration,
968
+ },
969
+ detail: { ...error.payload, continueDelayMs },
757
970
  });
758
- }
759
-
760
- if (checkpointDurable) {
761
- observability.incrementCounter({
762
- name: "opengeni_codex_failover_checkpoints_total",
763
- help: "Durable Codex failover checkpoint attempts by outcome.",
764
- labels: { workspace_key: codexWorkspaceKey, outcome: "completed" },
971
+ } catch (recoveryError) {
972
+ // Keep the turn recoverable across a transient database failure, as
973
+ // the provider recovery path does.
974
+ const postClaimRecovery = postClaimDatabaseRecoveryFailure({
975
+ error: recoveryError,
976
+ turnId: attempt.turnId,
977
+ triggerEventId: attempt.triggerEventId!,
978
+ executionGeneration: attempt.executionGeneration,
765
979
  });
766
- const now = new Date();
767
- const before = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId).catch(
768
- () => [],
769
- );
770
- const servingCached = before.find(
771
- (account) => account.id === providerTurn.effectiveCodexCredentialId,
772
- );
773
- const usageSnapshot = providerTurn.latestCodexUsage as CodexUsageHeaderSnapshot | null;
774
- const serving = servingCached
775
- ? {
776
- ...servingCached,
777
- ...(usageSnapshot
778
- ? {
779
- primaryUsedPercent: usageSnapshot.primaryUsedPercent,
780
- primaryResetAt: usageSnapshot.primaryResetAt,
781
- secondaryUsedPercent: usageSnapshot.secondaryUsedPercent,
782
- secondaryResetAt: usageSnapshot.secondaryResetAt,
783
- }
784
- : {}),
785
- }
786
- : null;
787
- const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
788
- const quarantineResult =
789
- leases.codex.holderId &&
790
- leases.codex.generation !== null &&
791
- providerTurn.effectiveCodexCredentialVersion !== null
792
- ? await quarantineCodexCredentialForLease(db, {
793
- accountId: input.accountId,
794
- workspaceId: input.workspaceId,
795
- sessionId: input.sessionId,
796
- turnId: attempt.turnId,
797
- attemptId: input.attemptId,
798
- executionGeneration: attempt.executionGeneration,
799
- workflowId: input.workflowId,
800
- workflowRunId: input.workflowRunId,
801
- dispatchId: attempt.dispatchId,
802
- expectedRedispatches: attempt.redispatchesAtDispatch,
803
- credentialId: providerTurn.effectiveCodexCredentialId,
804
- credentialVersion: providerTurn.effectiveCodexCredentialVersion,
805
- holderId: leases.codex.holderId,
806
- generation: leases.codex.generation,
807
- maxFailovers: providerTurn.codexCredentialFailoverLimit,
808
- quarantine:
809
- codexCredentialFailure.kind === "auth"
810
- ? {
811
- kind: "status",
812
- status: "needs_relogin",
813
- lastError: "model request remained unauthorized after refresh",
814
- }
815
- : codexCredentialFailure.kind === "forbidden"
816
- ? {
817
- kind: "status",
818
- status: "error",
819
- lastError: "model request was forbidden for this credential",
820
- }
821
- : {
822
- kind: "cooldown",
823
- until: cooldownUntil!,
824
- cooldownKind: codexCredentialFailure.kind,
825
- },
826
- })
827
- : null;
828
- if (
829
- quarantineResult?.action === "credential_changed" &&
830
- leases.codex.holderId &&
831
- leases.codex.generation !== null
832
- ) {
833
- const recovery = await settleCodexCredentialLeaseLoss(db, {
834
- accountId: input.accountId,
835
- workspaceId: input.workspaceId,
836
- sessionId: input.sessionId,
837
- turnId: attempt.turnId,
838
- attemptId: input.attemptId,
839
- holderId: leases.codex.holderId,
840
- generation: leases.codex.generation,
841
- expectedRedispatches: attempt.redispatchesAtDispatch,
842
- checkpointDurable: true,
843
- recoveryPayload: {
844
- triggerEventId: attempt.triggerEventId!,
845
- reason: "codex_credential_version_changed",
846
- },
847
- failedPayload: {},
848
- });
849
- if (recovery.action === "recovering") {
850
- leases.codex.held = false;
851
- await publishDurableSessionEvents(
852
- bus,
853
- input.workspaceId,
854
- input.sessionId,
855
- recovery.events,
856
- );
857
- control.activityStatus = "recovering";
858
- control.turnMetricOutcome = "recovering";
859
- return claimedResult({ status: "recovering" });
860
- }
861
- acknowledgeLostAttemptOwnership();
862
- control.activityStatus = "cancelled";
863
- control.turnMetricOutcome = "cancelled";
864
- return claimedResult({ status: "cancelled" });
865
- }
866
- const statePersisted = quarantineResult?.action === "recorded";
867
- if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
868
- leases.codex.lost = true;
869
- return await settleLostCodexAttempt(
870
- attempt.turnId,
871
- leases.codex.holderId,
872
- leases.codex.generation,
873
- true,
874
- );
875
- }
876
- if (
877
- quarantineResult?.action === "recorded" &&
878
- quarantineResult.exhausted &&
879
- leases.codex.holderId &&
880
- leases.codex.generation !== null
881
- ) {
882
- const settlement = await settleCodexCredentialFailover(db, {
883
- accountId: input.accountId,
884
- workspaceId: input.workspaceId,
885
- sessionId: input.sessionId,
886
- turnId: attempt.turnId,
887
- attemptId: input.attemptId,
888
- holderId: leases.codex.holderId,
889
- generation: leases.codex.generation,
890
- expectedRedispatches: attempt.redispatchesAtDispatch,
891
- maxFailovers: quarantineResult.maxFailovers,
892
- recoveryPayload: {
893
- triggerEventId: attempt.triggerEventId!,
894
- reason: "codex_credential_failover",
895
- credentialId: providerTurn.effectiveCodexCredentialId,
896
- failureKind: codexCredentialFailure.kind,
897
- },
898
- failedPayload: {
899
- error:
900
- "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
901
- code: "codex_credential_failover_exhausted",
902
- retryable: false,
903
- recovery: "user_message",
904
- failoverCount: quarantineResult.failoverCount,
905
- maxFailovers: quarantineResult.maxFailovers,
906
- },
907
- });
908
- if (settlement.action === "limit_exceeded") {
909
- leases.codex.held = false;
910
- await publishDurableSessionEvents(
911
- bus,
912
- input.workspaceId,
913
- input.sessionId,
914
- settlement.events,
915
- );
916
- control.activityError = error;
917
- control.activityStatus = "idle";
918
- control.turnMetricOutcome = "failed";
919
- await deliverFailedChildTurnToParent(
920
- { db, bus, settings, observability, wakeSessionWorkflow },
921
- input.workspaceId,
922
- input.sessionId,
923
- attempt.turnId,
924
- );
925
- return claimedResult({ status: "idle" });
926
- }
927
- if (settlement.action === "stale") {
928
- acknowledgeLostAttemptOwnership();
929
- control.activityStatus = "cancelled";
930
- control.turnMetricOutcome = "cancelled";
931
- return claimedResult({ status: "cancelled" });
932
- }
933
- throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
934
- }
935
- let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
936
- try {
937
- accounts = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId);
938
- } catch (metadataError) {
939
- // Current account health/cooldown metadata is still required after
940
- // quarantine. Operational database failures re-enter the existing
941
- // exact-attempt recovery lane; no policy choice is made from partial
942
- // account state.
943
- const recoveryFailure = postClaimDatabaseRecoveryFailure({
944
- error: metadataError,
945
- turnId: attempt.turnId,
946
- triggerEventId: attempt.triggerEventId!,
947
- executionGeneration: attempt.executionGeneration,
948
- });
949
- if (recoveryFailure) {
950
- control.activityStatus = "recovering";
951
- control.turnMetricOutcome = "recovering";
952
- control.activityError = metadataError;
953
- throw recoveryFailure;
954
- }
955
- throw metadataError;
956
- }
957
- const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
958
- const selected = selectCodexCredentialLeaseForTurn({
959
- context: {
960
- accounts: codexLeaseAccountsForSelection(accounts),
961
- activeCredentialId: acceptedPolicy.activeCredentialId,
962
- rotationEnabled: acceptedPolicy.rotationEnabled,
963
- rotationStrategy: acceptedPolicy.rotationStrategy,
964
- existingCredentialId: null,
965
- failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
966
- policyScope: null,
967
- unavailableDiagnostics: [],
968
- },
969
- sessionId: input.sessionId,
970
- sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
971
- sessionPinSource: acceptedPolicy.pinSource,
972
- sessionLastCredentialId: acceptedPolicy.lastCredentialId,
973
- now,
974
- });
975
- const decisionKind =
976
- selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
977
- const pinDisposition = classifyCodexPin({
978
- pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
979
- pinSource: acceptedPolicy.pinSource,
980
- strategy: acceptedPolicy.rotationStrategy as CodexRotationStrategy,
981
- rotationEnabled: acceptedPolicy.rotationEnabled,
982
- });
983
- const failureDisposition = codexDefinitiveFailureDisposition({
984
- failureKind: codexCredentialFailure.kind,
985
- rotationEnabled: acceptedPolicy.rotationEnabled,
986
- pinDisposition,
987
- decisionKind,
988
- decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
989
- servingCredentialId: providerTurn.effectiveCodexCredentialId,
990
- });
991
- const maxFailovers = providerTurn.codexCredentialFailoverLimit;
992
-
993
- if (
994
- statePersisted &&
995
- failureDisposition === "failover" &&
996
- leases.codex.holderId &&
997
- leases.codex.generation !== null
998
- ) {
999
- const settlement = await settleCodexCredentialFailover(db, {
1000
- accountId: input.accountId,
1001
- workspaceId: input.workspaceId,
1002
- sessionId: input.sessionId,
1003
- turnId: attempt.turnId,
1004
- attemptId: input.attemptId,
1005
- holderId: leases.codex.holderId,
1006
- generation: leases.codex.generation,
1007
- expectedRedispatches: attempt.redispatchesAtDispatch,
1008
- maxFailovers,
1009
- recoveryPayload: {
1010
- triggerEventId: attempt.triggerEventId!,
1011
- reason: "codex_credential_failover",
1012
- credentialId: providerTurn.effectiveCodexCredentialId,
1013
- failureKind: codexCredentialFailure.kind,
1014
- ...(cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}),
1015
- },
1016
- failedPayload: {
1017
- error:
1018
- "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
1019
- code: "codex_credential_failover_exhausted",
1020
- retryable: false,
1021
- recovery: "user_message",
1022
- failoverCount: quarantineResult.failoverCount,
1023
- maxFailovers: quarantineResult.maxFailovers,
1024
- },
1025
- });
1026
- observability.incrementCounter({
1027
- name: "opengeni_codex_failover_settlements_total",
1028
- help: "Atomic Codex failover settlements by outcome.",
1029
- labels: {
1030
- workspace_key: codexWorkspaceKey,
1031
- outcome: settlement.action,
1032
- },
1033
- });
1034
- if (settlement.action === "recovering") {
1035
- leases.codex.held = false;
1036
- await publishDurableSessionEvents(
1037
- bus,
1038
- input.workspaceId,
1039
- input.sessionId,
1040
- settlement.events,
1041
- );
1042
- observability.observeHistogram({
1043
- name: "opengeni_codex_failover_recovery_seconds",
1044
- help: "Time from credential refusal to durable same-turn recovery.",
1045
- labels: {
1046
- workspace_key: codexWorkspaceKey,
1047
- kind: codexCredentialFailure.kind,
1048
- },
1049
- value: Math.max(0, (performance.now() - failoverStartedAt) / 1000),
1050
- });
1051
- control.activityStatus = "recovering";
1052
- control.turnMetricOutcome = "recovering";
1053
- return claimedResult({ status: "recovering" });
1054
- }
1055
- if (settlement.action === "stale") {
1056
- // One transaction proves both exact-holder recovery (including a
1057
- // just-expired or reaped lease row) and successor/control-gate
1058
- // rejection. Cross the hard tool fence so a control-gate loss can
1059
- // write its quiescence receipt; a successor-only loss is a no-op.
1060
- acknowledgeLostAttemptOwnership();
1061
- control.activityStatus = "cancelled";
1062
- control.turnMetricOutcome = "cancelled";
1063
- return claimedResult({ status: "cancelled" });
1064
- }
1065
- if (settlement.action === "limit_exceeded") {
1066
- leases.codex.held = false;
1067
- await publishDurableSessionEvents(
1068
- bus,
1069
- input.workspaceId,
1070
- input.sessionId,
1071
- settlement.events,
1072
- );
1073
- control.activityError = error;
1074
- control.activityStatus = "idle";
1075
- control.turnMetricOutcome = "failed";
1076
- await deliverFailedChildTurnToParent(
1077
- { db, bus, settings, observability, wakeSessionWorkflow },
1078
- input.workspaceId,
1079
- input.sessionId,
1080
- attempt.turnId,
1081
- );
1082
- return claimedResult({ status: "idle" });
1083
- }
1084
- }
1085
-
1086
- if (
1087
- statePersisted &&
1088
- failureDisposition === "wait" &&
1089
- leases.codex.holderId &&
1090
- leases.codex.generation !== null
1091
- ) {
1092
- let goal: Awaited<ReturnType<typeof getSessionGoal>>;
1093
- try {
1094
- goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
1095
- } catch (metadataError) {
1096
- const recoveryFailure = postClaimDatabaseRecoveryFailure({
1097
- error: metadataError,
1098
- turnId: attempt.turnId,
1099
- triggerEventId: attempt.triggerEventId!,
1100
- executionGeneration: attempt.executionGeneration,
1101
- });
1102
- if (recoveryFailure) {
1103
- control.activityStatus = "recovering";
1104
- control.turnMetricOutcome = "recovering";
1105
- control.activityError = metadataError;
1106
- throw recoveryFailure;
1107
- }
1108
- throw metadataError;
1109
- }
1110
- const activeGoal = goal?.status === "active" ? goal : null;
1111
- const exactProviderReset =
1112
- codexCredentialFailure.cooldownSeconds !== null &&
1113
- Number.isFinite(codexCredentialFailure.cooldownSeconds) &&
1114
- codexCredentialFailure.cooldownSeconds > 0;
1115
- const policyCredentialId =
1116
- pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId
1117
- ? acceptedPolicy.pinnedCredentialId
1118
- : !acceptedPolicy.rotationEnabled
1119
- ? acceptedPolicy.activeCredentialId
1120
- : null;
1121
- const capacityAccounts = policyCredentialId
1122
- ? accounts.filter((account) => account.id === policyCredentialId)
1123
- : accounts;
1124
- const authoritativeResetAt = exactProviderReset
1125
- ? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
1126
- : null;
1127
- const allAccounts =
1128
- acceptedPolicy.rotationEnabled &&
1129
- pinDisposition !== "manual" &&
1130
- decisionKind === "allCapped";
1131
- const failurePayload = codexCapacityWaitFailurePayload({
1132
- failureKind: codexCredentialFailure.kind,
1133
- usageLimit,
1134
- cooldownSeconds: codexCredentialFailure.cooldownSeconds,
1135
- detail:
1136
- codexCredentialFailure.kind === "quota"
1137
- ? error instanceof Error
1138
- ? error.message
1139
- : String(error)
1140
- : "the same accepted turn is waiting for eligible credential capacity",
1141
- allAccounts,
1142
- });
1143
- const evaluated = await armAndReconcileCodexCapacityWait(
1144
- { db, bus },
1145
- {
1146
- accountId: input.accountId,
1147
- workspaceId: input.workspaceId,
1148
- sessionId: input.sessionId,
1149
- turnId: attempt.turnId,
1150
- attemptId: input.attemptId,
1151
- workflowId: input.workflowId,
1152
- goalId: activeGoal?.id ?? null,
1153
- goalVersion: activeGoal?.version ?? null,
1154
- earliestResetAt: authoritativeResetAt,
1155
- resetKind: authoritativeResetAt
1156
- ? "authoritative"
1157
- : codexCredentialFailure.kind === "auth" ||
1158
- codexCredentialFailure.kind === "forbidden"
1159
- ? "mutation_only"
1160
- : "bounded_refresh",
1161
- failurePayload,
1162
- leaseFence: {
1163
- holderId: leases.codex.holderId,
1164
- generation: leases.codex.generation,
1165
- },
1166
- expectedRedispatches: attempt.redispatchesAtDispatch,
1167
- },
1168
- { onArmed: () => (leases.codex.held = false) },
1169
- );
1170
- control.activityError = error;
1171
- if (evaluated.action === "stopped") {
1172
- control.activityStatus = evaluated.sessionStatus === "queued" ? "idle" : "failed";
1173
- control.turnMetricOutcome = "failed";
1174
- return claimedResult({ status: control.activityStatus });
1175
- }
1176
- if (evaluated.action === "resumed") {
1177
- control.activityStatus = "recovering";
1178
- control.turnMetricOutcome = "recovering";
1179
- return claimedResult({ status: "recovering" });
1180
- }
1181
- if (evaluated.action === "waiting") {
1182
- control.activityError = error;
1183
- control.activityStatus = "waiting_capacity";
1184
- control.turnMetricOutcome = "recovering";
1185
- return claimedResult({
1186
- status: "waiting_capacity",
1187
- capacityWait: {
1188
- waiterId: evaluated.waiter.id,
1189
- generation: evaluated.waiter.generation,
1190
- nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
1191
- wakeRevision: evaluated.waiter.wakeRevision,
1192
- },
1193
- });
1194
- }
1195
- acknowledgeLostAttemptOwnership();
1196
- control.activityStatus = "cancelled";
1197
- control.turnMetricOutcome = "cancelled";
1198
- return claimedResult({ status: "cancelled" });
980
+ if (postClaimRecovery) {
981
+ control.activityStatus = "recovering";
982
+ control.turnMetricOutcome = "recovering";
983
+ control.activityError = error;
984
+ throw postClaimRecovery;
1199
985
  }
986
+ throw recoveryError;
987
+ }
988
+ if (recovery.action === "stale") {
989
+ acknowledgeLostAttemptOwnership();
990
+ control.activityStatus = "cancelled";
991
+ control.turnMetricOutcome = "cancelled";
992
+ return claimedResult({ status: "cancelled" });
1200
993
  }
994
+ if (recovery.action === "recovering") {
995
+ acknowledgeRecoveryQuiescence();
996
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
997
+ control.turnMetricOutcome = "recovering";
998
+ control.activityStatus = "recovering";
999
+ control.activityError = error;
1000
+ return claimedResult({ status: "recovering", continueDelayMs });
1001
+ }
1002
+ leaseBusyNotRecoverable = true;
1201
1003
  }
1202
- const xaiCredentialFailure =
1203
- billingState.isXaiTurn && providerTurn.effectiveXaiCredentialId
1204
- ? classifyXaiCredentialFailure(error)
1205
- : null;
1004
+ // A shared-core placement outcome (no capacity, a waiting explicit choice,
1005
+ // a paused cutover, or a consumer not yet on the core) or a definitive
1006
+ // refusal from the leased account fails this turn with its typed copy and
1007
+ // keeps the session usable for the next message.
1008
+ const coreTurnFailure =
1009
+ leaseBusyChain?.exhausted || leaseBusyNotRecoverable
1010
+ ? // The chain exists only for a lease-busy core error: keep its provider.
1011
+ subscriptionCoreLeaseBusyExhaustedFailure(
1012
+ error instanceof SubscriptionCoreCodexTurnError ? error.displayName : undefined,
1013
+ )
1014
+ : error instanceof SubscriptionCoreCodexTurnError
1015
+ ? error
1016
+ : (coreAccountRefusal ?? coreXaiRefusal ?? coreClaudeRefusal);
1206
1017
  if (
1207
- xaiCredentialFailure &&
1208
- providerTurn.effectiveXaiCredentialId &&
1209
- providerTurn.xaiAuthoritySnapshot &&
1210
- leases.xai.subjectId &&
1211
- leases.xai.holderId &&
1212
- leases.xai.generation !== null &&
1018
+ coreTurnFailure &&
1019
+ !coreTurnFailure.payload.retryable &&
1213
1020
  eventing.publish &&
1214
1021
  attempt.turnId &&
1215
1022
  eventing.turnStartedPublished
1216
1023
  ) {
1217
1024
  await flushRuntimeBatcher();
1218
- await historySink.reconcileConversationTruth({ requireDurable: true });
1219
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
1220
- const activeGoal = goal?.status === "active" ? goal : null;
1221
- const now = new Date();
1222
- const cooldownUntil =
1223
- xaiCredentialFailure.kind === "rate_limit"
1224
- ? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
1225
- : null;
1226
- const failurePayload = {
1227
- error:
1228
- xaiCredentialFailure.kind === "auth"
1229
- ? "The serving SuperGrok account requires reconnection"
1230
- : xaiCredentialFailure.kind === "forbidden"
1231
- ? "The serving SuperGrok account is not authorized for this request"
1232
- : "The serving SuperGrok account is temporarily rate limited",
1233
- code:
1234
- xaiCredentialFailure.kind === "auth"
1235
- ? "xai_relogin_required"
1236
- : xaiCredentialFailure.kind === "forbidden"
1237
- ? "xai_account_forbidden"
1238
- : "xai_account_rate_limited",
1239
- detail: "the same accepted turn is waiting for another eligible account",
1240
- };
1241
- const armed = await armXaiCapacityWait(db, {
1242
- accountId: input.accountId,
1243
- workspaceId: input.workspaceId,
1244
- subjectId: leases.xai.subjectId,
1245
- sessionId: input.sessionId,
1246
- turnId: attempt.turnId,
1247
- attemptId: input.attemptId,
1248
- workflowId: input.workflowId,
1249
- authoritySnapshot: providerTurn.xaiAuthoritySnapshot,
1250
- goalId: activeGoal?.id ?? null,
1251
- goalVersion: activeGoal?.version ?? null,
1252
- earliestResetAt: cooldownUntil,
1253
- failurePayload,
1254
- leaseFence: {
1255
- holderId: leases.xai.holderId,
1256
- generation: leases.xai.generation,
1257
- },
1258
- credentialQuarantine:
1259
- xaiCredentialFailure.kind === "auth"
1260
- ? {
1261
- kind: "status",
1262
- status: "needs_relogin",
1263
- lastError: "model request remained unauthorized after refresh",
1264
- }
1265
- : xaiCredentialFailure.kind === "forbidden"
1266
- ? {
1267
- kind: "status",
1268
- status: "error",
1269
- lastError: "model request was forbidden for this credential",
1270
- }
1271
- : { kind: "cooldown", until: cooldownUntil! },
1272
- now,
1273
- });
1274
- if (armed.action === "waiting") {
1275
- leases.xai.held = false;
1276
- providerTurn.xaiCredentialQuarantined = true;
1277
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
1278
- const evaluated = await reconcileXaiCapacityWait(db, {
1279
- accountId: input.accountId,
1280
- workspaceId: input.workspaceId,
1281
- sessionId: input.sessionId,
1282
- waiterId: armed.waiter.id,
1283
- generation: armed.waiter.generation,
1284
- now,
1285
- });
1286
- if (evaluated.events.length > 0) {
1287
- await publishDurableSessionEvents(
1288
- bus,
1289
- input.workspaceId,
1290
- input.sessionId,
1291
- evaluated.events,
1292
- );
1293
- }
1294
- control.activityError = error;
1295
- if (evaluated.action === "resumed") {
1296
- control.activityStatus = "recovering";
1297
- control.turnMetricOutcome = "recovering";
1298
- return claimedResult({ status: "recovering" });
1299
- }
1300
- if (evaluated.action === "waiting") {
1301
- control.activityStatus = "waiting_capacity";
1302
- control.turnMetricOutcome = "recovering";
1303
- return claimedResult({
1304
- status: "waiting_capacity",
1305
- capacityWait: {
1306
- provider: "xai",
1307
- waiterId: evaluated.waiter.id,
1308
- generation: evaluated.waiter.generation,
1309
- nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
1310
- wakeRevision: evaluated.waiter.wakeRevision,
1025
+ await historySink.reconcileConversationTruth();
1026
+ if (
1027
+ !(await eventing.settle!({
1028
+ events: [
1029
+ {
1030
+ type: "turn.failed",
1031
+ payload: { ...coreTurnFailure.payload, recovery: "user_message" },
1311
1032
  },
1312
- });
1313
- }
1314
- acknowledgeLostAttemptOwnership();
1315
- control.activityStatus = "cancelled";
1316
- control.turnMetricOutcome = "cancelled";
1317
- return claimedResult({ status: "cancelled" });
1318
- }
1319
-
1320
- const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
1321
- sessionId: input.sessionId,
1322
- turnId: attempt.turnId,
1323
- triggerEventId: attempt.triggerEventId!,
1324
- attemptId: input.attemptId,
1325
- reason: "xai_credential_recheck",
1326
- detail: failurePayload,
1327
- });
1328
- if (recovery.action === "stale") {
1329
- acknowledgeLostAttemptOwnership();
1330
- control.activityStatus = "cancelled";
1331
- control.turnMetricOutcome = "cancelled";
1033
+ { type: "session.status.changed", payload: { status: "idle" } },
1034
+ ],
1035
+ turnStatus: "failed",
1036
+ sessionStatus: "idle",
1037
+ activeTurnId: null,
1038
+ }))
1039
+ ) {
1332
1040
  return claimedResult({ status: "cancelled" });
1333
1041
  }
1334
- acknowledgeRecoveryQuiescence();
1335
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
1336
- control.activityStatus = "recovering";
1337
- control.turnMetricOutcome = "recovering";
1042
+ control.turnMetricOutcome = "failed";
1043
+ control.activityStatus = "idle";
1338
1044
  control.activityError = error;
1339
- return claimedResult({ status: "recovering" });
1045
+ await deliverFailedChildTurnToParent(
1046
+ { db, bus, settings, observability, wakeSessionWorkflow },
1047
+ input.workspaceId,
1048
+ input.sessionId,
1049
+ attempt.turnId,
1050
+ );
1051
+ return claimedResult({ status: "idle" });
1340
1052
  }
1341
- // The leased credential path above normally quarantines quota state and
1342
- // either recovers the same turn or arms a durable capacity wait. This narrow
1343
- // fallback covers failures before a credential lease existed, or a failed
1344
- // durable checkpoint where replay would be unsafe. Keep the session usable,
1345
- // but never synthesize another turn or walk an unfenced legacy pointer.
1346
1053
  if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
1347
1054
  await flushRuntimeBatcher();
1348
1055
  await historySink.reconcileConversationTruth();
@@ -1387,23 +1094,48 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1387
1094
  ) {
1388
1095
  await flushRuntimeBatcher();
1389
1096
  await historySink.reconcileConversationTruth();
1097
+ if (error.sessionUsageLimit) {
1098
+ recordSessionUsageLimitEvent(observability, "between_calls", "reached");
1099
+ }
1390
1100
  if (
1391
- !(await eventing.settle!({
1392
- events: [
1393
- {
1394
- type: "turn.completed",
1395
- payload: {
1396
- output: "",
1397
- segmentLimit: "budget_exhausted",
1398
- detail: error.message,
1101
+ !(await eventing.settle!(
1102
+ error.sessionUsageLimit
1103
+ ? {
1104
+ // the task's usage limit was reached between model
1105
+ // calls. Already admitted usage is recorded; history is kept.
1106
+ events: sessionUsageLimitStopEvents(error.sessionUsageLimit),
1107
+ turnStatus: "completed",
1108
+ sessionStatus: "idle",
1109
+ activeTurnId: null,
1110
+ allowanceGoalPause: {
1111
+ rationale: error.sessionUsageLimit.message,
1112
+ reason: "session_usage_limit",
1113
+ },
1114
+ }
1115
+ : {
1116
+ events: [
1117
+ ...(error.allowance
1118
+ ? [{ type: "usage.exhausted" as const, payload: error.allowance }]
1119
+ : []),
1120
+ {
1121
+ type: "turn.completed",
1122
+ payload: {
1123
+ output: "",
1124
+ segmentLimit: "budget_exhausted",
1125
+ detail: error.message,
1126
+ ...(error.allowance ?? {}),
1127
+ },
1128
+ },
1129
+ { type: "session.status.changed", payload: { status: "idle" } },
1130
+ ],
1131
+ turnStatus: "completed",
1132
+ sessionStatus: "idle",
1133
+ activeTurnId: null,
1134
+ ...(error.allowance
1135
+ ? { allowanceGoalPause: { rationale: error.allowance.message } }
1136
+ : {}),
1399
1137
  },
1400
- },
1401
- { type: "session.status.changed", payload: { status: "idle" } },
1402
- ],
1403
- turnStatus: "completed",
1404
- sessionStatus: "idle",
1405
- activeTurnId: null,
1406
- }))
1138
+ ))
1407
1139
  ) {
1408
1140
  return claimedResult({ status: "cancelled" });
1409
1141
  }
@@ -1435,7 +1167,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1435
1167
  // fall through to the terminal path rather than resend an equivalent
1436
1168
  // request forever.
1437
1169
  const encryptedArtifactRejection =
1438
- billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
1170
+ !coreRequestOutcomeUnknown &&
1171
+ !coreLeaseCheckpointFailed &&
1172
+ billingState.isCodexTurn &&
1173
+ providerTurn.effectiveCodexCredentialId
1439
1174
  ? classifyCodexEncryptedArtifactRejection(error)
1440
1175
  : null;
1441
1176
  if (
@@ -1499,31 +1234,180 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1499
1234
  // truth, recover this SAME accepted turn, then let the workflow re-claim
1500
1235
  // it after a pacing delay. This is independent of goal state and never
1501
1236
  // relies on a synthetic continuation prompt.
1502
- let failure = agentRunFailurePayload(error, {
1503
- isCodexTurn: billingState.isCodexTurn,
1504
- }) as ReturnType<typeof agentRunFailurePayload>;
1505
- if (failure.retryable && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
1506
- const nextProviderRecoveryCount = attempt.providerRecoveryCount + 1;
1237
+ // A rolling-deployment definition mismatch is a separate configuration
1238
+ // class: only the exact typed setup error can use this checkpoint before
1239
+ // eventing exists. No generic setup/credential failure gains retry authority.
1240
+ const earlyDefinitionMismatch =
1241
+ error instanceof TurnExecutionPolicyDefinitionMismatchError &&
1242
+ !attempt.modelRequestStarted &&
1243
+ !eventing.turnStartedPublished &&
1244
+ !!attempt.turnId &&
1245
+ !!attempt.triggerEventId &&
1246
+ attempt.executionGeneration > 0;
1247
+ const earlyCommandStartUnavailable =
1248
+ isModalTaskExecStartPreDispatchUnavailableError(error) &&
1249
+ !attempt.modelRequestStarted &&
1250
+ !eventing.turnStartedPublished &&
1251
+ !!attempt.turnId &&
1252
+ !!attempt.triggerEventId &&
1253
+ attempt.executionGeneration > 0;
1254
+ const earlyRecoverableSetup = earlyDefinitionMismatch || earlyCommandStartUnavailable;
1255
+ // Call-scoped proof that an in-flight exec_command never dispatched its
1256
+ // command. Those calls settle as not started (replay safe), never unknown,
1257
+ // whether this turn recovers or fails.
1258
+ const notStartedToolCallIds = isRoutingMutationOutcomeUnknownError(error)
1259
+ ? []
1260
+ : toolCallCommandNotStartedCallIds(error);
1261
+ // Custody decides whether replay is safe, not what caused the failure. Keep
1262
+ // the existing diagnostic in the permission-controlled terminal event;
1263
+ // never copy it into logs or use its retry classification for admission.
1264
+ let unknownOutcomeDetail: string | undefined;
1265
+ if (coreRequestOutcomeUnknown) {
1266
+ try {
1267
+ const sourceFailure = agentRunFailurePayload(error, {
1268
+ isCodexTurn: billingState.isCodexTurn,
1269
+ });
1270
+ unknownOutcomeDetail = sourceFailure.detail ?? sourceFailure.error;
1271
+ } catch {
1272
+ // Diagnostic extraction cannot prevent durable failure settlement.
1273
+ }
1274
+ }
1275
+ let failure = withModelRoutePresentation(
1276
+ (coreRequestOutcomeUnknown
1277
+ ? {
1278
+ error: coreRequestOutcomeUnknown.message,
1279
+ code: coreRequestOutcomeUnknown.code,
1280
+ retryable: false,
1281
+ ...(unknownOutcomeDetail && unknownOutcomeDetail !== coreRequestOutcomeUnknown.message
1282
+ ? { detail: unknownOutcomeDetail }
1283
+ : {}),
1284
+ }
1285
+ : earlyDefinitionMismatch
1286
+ ? { error: error.message, code: error.code, retryable: true }
1287
+ : (coreUnknownRecoveryFailure ??
1288
+ agentRunFailurePayload(error, {
1289
+ isCodexTurn: billingState.isCodexTurn,
1290
+ }))) as ReturnType<typeof agentRunFailurePayload>,
1291
+ attempt.modelRoutePresentation,
1292
+ );
1293
+ const coreLaneRecoverable = (lane: ReturnType<typeof createCoreCodexRequests> | undefined) =>
1294
+ !lane || lane.canRecover() || (!!coreUnknownRecoveryFailure && lane.unknownIsOwnAndSettled());
1295
+ // A core lane holds the answered request that asked for a tool until the
1296
+ // next model call checkpoints it, so a retryable failure thrown while a
1297
+ // tool runs (a command that never started, an MCP transport failure) or a
1298
+ // paused search's refused continuation finds an answered, unsettled
1299
+ // request. Commit the history durably and checkpoint first, as the
1300
+ // lease-lost and failover paths do, so the failure can recover. Nothing is
1301
+ // replayed: a lane without every response stays unrecoverable. This holds
1302
+ // for every core provider: a lane's answered request is complete regardless
1303
+ // of who serves it.
1304
+ const placedCore = coreClaude ?? coreCodex ?? coreXai;
1305
+ if (
1306
+ placedCore &&
1307
+ attempt.turnId &&
1308
+ !coreRequestOutcomeUnknown &&
1309
+ !coreLeaseCheckpointFailed &&
1310
+ failure.retryable &&
1311
+ eventing.publish &&
1312
+ eventing.turnStartedPublished &&
1313
+ !coreLaneRecoverable(placedCore.requests)
1314
+ ) {
1315
+ await coreLanesRecoverAfterCheckpoint(placedCore, async () => {
1316
+ await flushRuntimeBatcher();
1317
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1318
+ });
1319
+ }
1320
+ if (
1321
+ attempt.turnId &&
1322
+ !coreRequestOutcomeUnknown &&
1323
+ !coreLeaseCheckpointFailed &&
1324
+ coreLaneRecoverable(coreCodex?.requests) &&
1325
+ coreLaneRecoverable(coreXai?.requests) &&
1326
+ coreLaneRecoverable(coreClaude?.requests) &&
1327
+ (earlyRecoverableSetup ||
1328
+ (failure.retryable && eventing.publish && eventing.turnStartedPublished))
1329
+ ) {
1330
+ const recoveryCode = providerRecoveryCode(error, failure);
1331
+ // A recovery that interrupts a running tool is followed by a successful
1332
+ // model response before the tool runs again, which resets the in-memory
1333
+ // budget. Count those recoveries since the turn's last tool progress so
1334
+ // the same bounded backoff and limit still apply.
1335
+ const toolProgress =
1336
+ earlyRecoverableSetup || recoveryCode === PROVIDER_OVERLOAD_RECOVERY_CODE
1337
+ ? null
1338
+ : await readSessionTurnToolProgressRecoveryState(db, input.workspaceId, {
1339
+ sessionId: input.sessionId,
1340
+ turnId: attempt.turnId,
1341
+ }).catch(() => null);
1342
+ const nextProviderRecoveryCount =
1343
+ (toolProgress?.toolCallInFlight
1344
+ ? Math.max(attempt.providerRecoveryCount, toolProgress.toolInterruptedRecoveries)
1345
+ : attempt.providerRecoveryCount) + 1;
1507
1346
  const recoveryResult = providerRecoveryResult({
1508
- failureCode: failure.code,
1347
+ failureCode: recoveryCode,
1509
1348
  attemptNumber: nextProviderRecoveryCount,
1349
+ recoveryStartedAt:
1350
+ attempt.providerRecoveryCount > 0 ? attempt.providerRecoveryStartedAt : undefined,
1510
1351
  retryAfterMs: providerRetryAfterMs(error),
1352
+ jitterSample: Math.random(),
1511
1353
  });
1354
+ const setupRecoveryExhausted =
1355
+ earlyCommandStartUnavailable &&
1356
+ recoveryResult.status === "exhausted" &&
1357
+ attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
1512
1358
  try {
1359
+ if (setupRecoveryExhausted) {
1360
+ // This is positive pre-dispatch proof, not an ambiguous command. Park
1361
+ // the SAME accepted turn without resetting or advancing its budget.
1362
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
1363
+ sessionId: input.sessionId,
1364
+ turnId: attempt.turnId,
1365
+ triggerEventId: attempt.triggerEventId!,
1366
+ attemptId: input.attemptId,
1367
+ reason: "sandbox_command_start_recovery_exhausted",
1368
+ sandboxSetupRecoveryExhausted: true,
1369
+ detail: {
1370
+ code: failure.code,
1371
+ error:
1372
+ "Automatic sandbox setup recovery exhausted after five retries; the accepted turn remains parked without starting a command.",
1373
+ retryable: false,
1374
+ setupOutcome: "not_started",
1375
+ replay: "blocked",
1376
+ recoveryExhausted: true,
1377
+ providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
1378
+ },
1379
+ });
1380
+ if (recovery.action === "stale") {
1381
+ acknowledgeLostAttemptOwnership();
1382
+ control.activityStatus = "cancelled";
1383
+ control.turnMetricOutcome = "cancelled";
1384
+ return claimedResult({ status: "cancelled" });
1385
+ }
1386
+ acknowledgeRecoveryQuiescence();
1387
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
1388
+ control.turnMetricOutcome = "recovering";
1389
+ control.activityStatus = "recovering";
1390
+ control.activityError = error;
1391
+ return claimedResult({ status: "recovering" });
1392
+ }
1513
1393
  if (recoveryResult.status === "recovering") {
1514
- await flushRuntimeBatcher();
1515
- await historySink.reconcileConversationTruth({ requireDurable: true });
1394
+ if (!earlyRecoverableSetup) {
1395
+ await flushRuntimeBatcher();
1396
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1397
+ }
1516
1398
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
1517
1399
  sessionId: input.sessionId,
1518
1400
  turnId: attempt.turnId,
1519
1401
  triggerEventId: attempt.triggerEventId!,
1520
1402
  attemptId: input.attemptId,
1521
- reason: failure.code ?? "provider_unavailable",
1403
+ reason: recoveryCode ?? "provider_unavailable",
1522
1404
  providerRecoveryCount: nextProviderRecoveryCount,
1405
+ ...(notStartedToolCallIds.length > 0 ? { notStartedToolCallIds } : {}),
1523
1406
  detail: {
1524
- ...failure,
1407
+ ...agentRunRecoveryFailurePayload(error, failure),
1525
1408
  continueDelayMs: recoveryResult.continueDelayMs,
1526
1409
  providerRecoveryCount: nextProviderRecoveryCount,
1410
+ maxProviderRecoveryCount: providerRecoveryLimit(recoveryCode),
1527
1411
  },
1528
1412
  });
1529
1413
  if (recovery.action === "stale") {
@@ -1537,9 +1421,45 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1537
1421
  control.turnMetricOutcome = "recovering";
1538
1422
  control.activityStatus = "recovering";
1539
1423
  control.activityError = error;
1424
+ const recoveryCause = providerRecoveryCause(failure.code);
1425
+ if (recoveryCause) {
1426
+ recordProviderRecoveryOutcome(observability, {
1427
+ route: attempt.modelMetricRoute,
1428
+ cause: recoveryCause,
1429
+ outcome: "scheduled",
1430
+ delayMs: recoveryResult.continueDelayMs,
1431
+ });
1432
+ }
1540
1433
  return claimedResult(recoveryResult);
1541
1434
  }
1542
1435
  failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
1436
+ const recoveryCause = providerRecoveryCause(failure.code);
1437
+ if (recoveryCause) {
1438
+ recordProviderRecoveryOutcome(observability, {
1439
+ route: attempt.modelMetricRoute,
1440
+ cause: recoveryCause,
1441
+ outcome: "exhausted",
1442
+ ...(attempt.providerRecoveryObservation
1443
+ ? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt }
1444
+ : {}),
1445
+ });
1446
+ }
1447
+ if (earlyRecoverableSetup) {
1448
+ // Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
1449
+ // through Temporal into exact-attempt workflow failure settlement.
1450
+ control.activityStatus = "failed";
1451
+ control.turnMetricOutcome = "failed";
1452
+ control.activityError = error;
1453
+ throw ApplicationFailure.create({
1454
+ message: earlyDefinitionMismatch
1455
+ ? `${error.message}. Automatic same-turn configuration recovery exhausted after ${recoveryResult.providerRecoveryCount} retries.`
1456
+ : failure.error,
1457
+ type: earlyDefinitionMismatch
1458
+ ? "TurnExecutionPolicyDefinitionMismatchError"
1459
+ : "SandboxCommandStartUnavailableError",
1460
+ nonRetryable: true,
1461
+ });
1462
+ }
1543
1463
  } catch (recoveryError) {
1544
1464
  const escaped =
1545
1465
  recoveryResult.status === "recovering"
@@ -1569,11 +1489,22 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1569
1489
  triggerEventId: attempt.triggerEventId!,
1570
1490
  executionGeneration: attempt.executionGeneration,
1571
1491
  providerRecovery: {
1572
- failureCode: failure.code ?? "provider_unavailable",
1492
+ failureCode: recoveryCode ?? "provider_unavailable",
1573
1493
  providerRecoveryCount: nextProviderRecoveryCount,
1494
+ ...(recoveryCode === PROVIDER_OVERLOAD_RECOVERY_CODE
1495
+ ? { continueDelayMs: recoveryResult.continueDelayMs }
1496
+ : {}),
1574
1497
  },
1575
1498
  })
1576
- : null;
1499
+ : setupRecoveryExhausted
1500
+ ? postClaimDatabaseRecoveryFailure({
1501
+ error: recoveryError,
1502
+ turnId: attempt.turnId,
1503
+ triggerEventId: attempt.triggerEventId!,
1504
+ executionGeneration: attempt.executionGeneration,
1505
+ sandboxSetupRecoveryExhausted: true,
1506
+ })
1507
+ : null;
1577
1508
  if (postClaimRecovery) {
1578
1509
  control.activityStatus = "recovering";
1579
1510
  control.turnMetricOutcome = "recovering";
@@ -1583,26 +1514,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1583
1514
  throw recoveryError;
1584
1515
  }
1585
1516
  }
1586
- if (
1587
- attempt.turnId &&
1588
- attempt.triggerEventId &&
1589
- attempt.executionGeneration > 0 &&
1590
- !eventing.turnStartedPublished &&
1591
- !attempt.modelRequestStarted
1592
- ) {
1593
- const recoveryFailure = postClaimDatabaseRecoveryFailure({
1594
- error,
1595
- turnId: attempt.turnId,
1596
- triggerEventId: attempt.triggerEventId,
1597
- executionGeneration: attempt.executionGeneration,
1598
- });
1599
- if (recoveryFailure) {
1600
- control.activityStatus = "recovering";
1601
- control.turnMetricOutcome = "recovering";
1602
- control.activityError = error;
1603
- throw recoveryFailure;
1604
- }
1605
- }
1606
1517
  control.activityStatus = "failed";
1607
1518
  control.activityError = error;
1608
1519
  if (!attempt.turnId) {
@@ -1611,13 +1522,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1611
1522
  if (!eventing.publish || !eventing.turnStartedPublished) {
1612
1523
  throw error;
1613
1524
  }
1525
+ // Automatic recovery was not authorized for this attempt. A terminal record
1526
+ // never promises a retry that will not happen.
1527
+ if (failure.retryable === true) failure = automaticRecoveryUnavailableFailure(failure);
1614
1528
  // A partial/malformed stream may have emitted assistant/tool items (and
1615
1529
  // external side effects) before its terminal error. Persist every item the
1616
1530
  // SDK state observed before marking the turn failed so a later user revive
1617
1531
  // never replays work from an incomplete history. This does not retry or
1618
1532
  // rotate the ambiguous request.
1619
1533
  await flushRuntimeBatcher();
1620
- await historySink.reconcileConversationTruth();
1534
+ if (coreRequestOutcomeUnknown || coreUnknownRecoveryFailure) {
1535
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1536
+ } else {
1537
+ await historySink.reconcileConversationTruth();
1538
+ }
1621
1539
  if (
1622
1540
  !(await eventing.settle!({
1623
1541
  events: [
@@ -1627,6 +1545,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1627
1545
  turnStatus: "failed",
1628
1546
  sessionStatus: "failed",
1629
1547
  activeTurnId: null,
1548
+ ...(notStartedToolCallIds.length > 0 ? { notStartedToolCallIds } : {}),
1630
1549
  }))
1631
1550
  ) {
1632
1551
  return claimedResult({ status: "cancelled" });
@@ -1647,3 +1566,77 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1647
1566
  );
1648
1567
  return claimedResult({ status: "failed" });
1649
1568
  }
1569
+
1570
+ /** An error of the given class, directly or anywhere in the cause chain. */
1571
+ function hasErrorInCauseChain(
1572
+ error: unknown,
1573
+ type: abstract new (...args: never[]) => Error,
1574
+ ): boolean {
1575
+ let current = error;
1576
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1577
+ if (current instanceof type) return true;
1578
+ current = (current as { cause?: unknown }).cause;
1579
+ }
1580
+ return false;
1581
+ }
1582
+
1583
+ function findInCauseChain<T extends Error>(
1584
+ error: unknown,
1585
+ type: abstract new (...args: never[]) => T,
1586
+ ): T | null {
1587
+ let current = error;
1588
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1589
+ if (current instanceof type) return current;
1590
+ current = (current as { cause?: unknown }).cause;
1591
+ }
1592
+ return null;
1593
+ }
1594
+
1595
+ const INTERRUPTED_CODEX_RESPONSE_MESSAGE = "The Codex response was interrupted before it finished";
1596
+
1597
+ /**
1598
+ * Classify a Codex failure whose only replay obstacle is this attempt's own
1599
+ * interrupted request. Returns the retryable failure the bounded provider
1600
+ * recovery should use, or null when the unknown outcome must stay terminal.
1601
+ */
1602
+ export function ownInterruptedCodexRequestRecovery(
1603
+ error: unknown,
1604
+ core: Pick<CodexSubscriptionCoreTurn, "requests">,
1605
+ isCodexTurn: boolean,
1606
+ ): ReturnType<typeof agentRunFailurePayload> | null {
1607
+ const lane = core.requests;
1608
+ if (!lane?.hasUnknownOutcome()) return null;
1609
+ if (lane.admissionRefused()) return null;
1610
+ if (!lane.unknownIsOwnAndSettled()) return null;
1611
+ // A complete provider terminal that is incomplete (for example the output
1612
+ // token limit) would end the same way again.
1613
+ if (findInCauseChain(error, CodexStreamingTerminalError)?.code === "response_incomplete") {
1614
+ return null;
1615
+ }
1616
+ // After an interrupted request the tracker refuses the SDK's own retry with
1617
+ // the native unknown-outcome error; the SDK retried, so the cause was
1618
+ // transient.
1619
+ if (hasErrorInCauseChain(error, SubscriptionCoreCodexRequestOutcomeUnknownError)) {
1620
+ return {
1621
+ error: INTERRUPTED_CODEX_RESPONSE_MESSAGE,
1622
+ code: "provider_unavailable",
1623
+ retryable: true,
1624
+ };
1625
+ }
1626
+ let failure: ReturnType<typeof agentRunFailurePayload>;
1627
+ try {
1628
+ failure = agentRunFailurePayload(error, { isCodexTurn });
1629
+ } catch {
1630
+ return null;
1631
+ }
1632
+ // A stream that ended without a terminal event, or stalled after it began,
1633
+ // is a broken connection rather than a provider answer.
1634
+ if (failure.code === "invalid_sse_terminal" || failure.code === "codex_response_timeout") {
1635
+ return {
1636
+ error: INTERRUPTED_CODEX_RESPONSE_MESSAGE,
1637
+ code: "provider_unavailable",
1638
+ retryable: true,
1639
+ };
1640
+ }
1641
+ return failure.retryable === true ? failure : null;
1642
+ }