@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38686408778001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +172 -28
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +3 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +30 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-NTXZHYET.js} +1552 -2876
  123. package/dist/activities-control-NTXZHYET.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-RKRI5GY4.js} +9238 -4392
  126. package/dist/activities-turn-RKRI5GY4.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +104 -35
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/{chunk-M3NVMF2V.js → chunk-7BIHFUGB.js} +2293 -3956
  140. package/dist/chunk-7BIHFUGB.js.map +1 -0
  141. package/dist/chunk-MVESOYFC.js +3746 -0
  142. package/dist/chunk-MVESOYFC.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1134 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +26 -4
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +94 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +599 -110
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +162 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +116 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +758 -141
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +79 -38
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -1,14 +1,22 @@
1
+ import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
2
+ import { DrizzleQueryError } from "drizzle-orm";
3
+ import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
4
+ import { heldSessionUsageReceiptsForTurn } from "./session-usage-custody";
1
5
  import {
2
6
  ActiveSessionHistoryLimitExceededError,
3
7
  ApprovalRunStateLimitExceededError,
4
8
  nestedPostgresSqlState,
9
+ databaseFailureCode,
10
+ isRetryablePersistenceSqlState,
5
11
  safeDatabaseErrorFacts,
6
12
  isRetryableDatabaseTransportFailure,
7
13
  isSessionEventPersistenceError,
14
+ DatabaseTransactionError,
8
15
  SandboxLeaseTransitionError,
9
16
  } from "@opengeni/db";
10
17
  import {
11
18
  ActiveBackendUnresolvableError,
19
+ AnthropicProviderRejection,
12
20
  CompactionProviderResponseError,
13
21
  EmptyCompactionSummaryError,
14
22
  compactionProviderRejection,
@@ -16,29 +24,61 @@ import {
16
24
  type CompactionProviderRejection,
17
25
  isMcpRequestTimeoutError,
18
26
  isMcpTransportConnectivityError,
19
- isModalTaskExecStartDnsResolutionError,
27
+ isModalTaskExecStartPreDispatchUnavailableError,
28
+ isModalCommandStartOutcomeUnknownError,
29
+ isProviderCommandObservationUnavailableError,
30
+ ProviderCommandInputOutcomeUnknownError,
31
+ ProviderCommandStartOutcomeUnknownError,
32
+ isRoutingMutationOutcomeUnknownError,
33
+ isRoutingMutationOutputRejectedError,
20
34
  RoutingWorkspaceRootChangedError,
35
+ RoutingBackendRecoveryRequiredError,
36
+ ResponsesStreamingTerminalError,
37
+ classifyModelStreamIdleTimeoutError,
21
38
  SandboxMaterializationVerificationError,
22
39
  materializationVerificationDiagnostic,
23
40
  type MaterializationVerificationDiagnostic,
41
+ PROVIDER_QUOTA_EXHAUSTED_CODE,
42
+ type ProviderQuotaExhaustion,
43
+ type ProviderQuotaScope,
44
+ classifyProviderQuotaError,
45
+ providerQuotaExhaustedMessage,
24
46
  SelfhostedWorkspaceRootChangedError,
25
47
  UNKNOWN_MODEL_FINISH_REASON_CODE,
48
+ AnthropicRequestError,
49
+ findAnthropicRequestSizeError,
50
+ AnthropicSizeRecoveryExhaustedError,
51
+ type AnthropicRequestSize,
26
52
  } from "@opengeni/runtime";
27
53
  import {
28
54
  mcpTransportRequestFailureDiagnostic,
29
55
  type McpTransportRequestFailureDiagnostic,
30
56
  } from "@opengeni/runtime/mcp-network";
31
57
  import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
32
- import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
58
+ import type { ProviderErrorOutcome } from "@opengeni/subscriptions";
59
+ import { CODEX_USAGE_EXHAUSTED_PCT } from "./codex-usage-policy";
33
60
  import { RetainedAttachmentTransportLimitError } from "../run-input";
34
61
  import type { CodexAccountStatus } from "@opengeni/db";
35
62
  import {
63
+ CODEX_USAGE_LIMIT_ERROR_TYPE,
36
64
  CodexReloginRequired,
37
65
  classifyCodexEncryptedArtifactRejection,
66
+ classifyCodexEntitlementRejection,
38
67
  classifyCodexResponseTimeoutError,
39
68
  classifyCodexUsageLimitError,
40
69
  isCodexTransportError,
41
70
  } from "@opengeni/codex";
71
+ import {
72
+ CodexPlanEntitlementError,
73
+ codexPlanEntitlementFailurePayload,
74
+ codexRequestRejectedFailurePayload,
75
+ } from "./codex-plan-entitlement";
76
+ import { SubscriptionCoreCodexTurnError } from "./codex-core-errors";
77
+ import {
78
+ CodexIncludedUsageExhaustedError,
79
+ CodexIncludedUsageUnknownError,
80
+ findCodexCreditPolicyError,
81
+ } from "./codex-credit-policy";
42
82
  import {
43
83
  classifyXaiSubscriptionStreamingTerminalError,
44
84
  classifyXaiSubscriptionStreamIdleTimeoutError,
@@ -64,12 +104,45 @@ import {
64
104
  MandatoryHistoryPersistenceError,
65
105
  type MandatoryHistoryPersistenceStage,
66
106
  } from "./quiescence";
107
+ import {
108
+ MODEL_PROVIDER_RECOVERY_CODES,
109
+ providerRecoveryExhaustedMessage,
110
+ providerRecoverySubject,
111
+ type ProviderCondition,
112
+ } from "./provider-recovery-copy";
113
+ import {
114
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
115
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
116
+ PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
117
+ validProviderOverloadRecoveryDelay,
118
+ } from "./provider-recovery-policy";
119
+ export {
120
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
121
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
122
+ PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
123
+ } from "./provider-recovery-policy";
67
124
 
68
125
  // Retryable provider connectivity/5xx failures start quickly and back off to
69
126
  // this ceiling. Explicit rate limits retain the minute-granular fallback.
70
127
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
71
128
  export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
72
129
  export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
130
+ export function providerRecoveryLimit(failureCode: unknown): number {
131
+ return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE
132
+ ? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES
133
+ : MAX_AUTOMATIC_PROVIDER_RECOVERIES;
134
+ }
135
+ /**
136
+ * Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
137
+ * answer a per-minute token limit with a `retry-after` of about a second, which
138
+ * alone would spend every automatic recovery before the window resets.
139
+ */
140
+ export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
141
+ /** Positive-only spread: never shorten the provider's minimum delay. */
142
+ export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
143
+ const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
144
+ return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
145
+ }
73
146
  export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
74
147
 
75
148
  export class PostCompactionContinuationEmptyError extends Error {
@@ -85,25 +158,47 @@ export type ProviderRecoveryResult =
85
158
  | {
86
159
  status: "recovering";
87
160
  continueDelayMs: number;
161
+ maxProviderRecoveryCount?: number;
88
162
  }
89
163
  | {
90
164
  status: "exhausted";
91
165
  providerRecoveryCount: number;
92
166
  maxProviderRecoveryCount: number;
167
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
93
168
  };
94
169
 
95
170
  export function providerRecoveryResult(input: {
96
171
  failureCode: string | undefined;
97
172
  attemptNumber: number;
98
173
  retryAfterMs?: number | null;
174
+ jitterSample?: number;
175
+ /** Durable first-failure clock, never a new clock on each replacement attempt. */
176
+ recoveryStartedAt?: number | undefined;
177
+ now?: number;
99
178
  }): ProviderRecoveryResult {
100
- if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
179
+ const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
180
+ const limit = providerRecoveryLimit(input.failureCode);
181
+ const exhausted = (
182
+ reason: "deadline" | "retry_limit" | "invalid_clock",
183
+ ): Extract<ProviderRecoveryResult, { status: "exhausted" }> => ({
184
+ status: "exhausted",
185
+ providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
186
+ maxProviderRecoveryCount: limit,
187
+ providerRecoveryExhaustedReason: reason,
188
+ });
189
+ if (input.attemptNumber > limit) {
190
+ if (overload) return exhausted("retry_limit");
101
191
  return {
102
192
  status: "exhausted",
103
193
  providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
104
194
  maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
105
195
  };
106
196
  }
197
+ const now = input.now ?? Date.now();
198
+ const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
199
+ if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
200
+ return exhausted("invalid_clock");
201
+ }
107
202
  const providerDelay =
108
203
  input.retryAfterMs !== null &&
109
204
  input.retryAfterMs !== undefined &&
@@ -113,12 +208,22 @@ export function providerRecoveryResult(input: {
113
208
  : null;
114
209
  const continueDelayMs =
115
210
  input.failureCode === "provider_rate_limited"
116
- ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
117
- : input.failureCode === "provider_unavailable" ||
211
+ ? Math.max(
212
+ providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
213
+ PROVIDER_RATE_LIMIT_BACKOFF_MS[
214
+ Math.min(
215
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
216
+ PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1,
217
+ )
218
+ ]!,
219
+ )
220
+ : overload ||
221
+ input.failureCode === "provider_unavailable" ||
118
222
  input.failureCode === "upstream_connectivity_unavailable" ||
119
223
  input.failureCode === "sandbox_command_start_unavailable" ||
120
224
  input.failureCode === "mcp_transport_timeout" ||
121
225
  input.failureCode === "mcp_transport_unavailable" ||
226
+ input.failureCode === "turn_execution_policy_definition_mismatch" ||
122
227
  input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
123
228
  ? Math.max(
124
229
  providerDelay ?? 0,
@@ -130,12 +235,79 @@ export function providerRecoveryResult(input: {
130
235
  ]!,
131
236
  )
132
237
  : PROVIDER_BACKPRESSURE_DELAY_MS;
238
+ const delayMs =
239
+ continueDelayMs +
240
+ (overload ||
241
+ input.failureCode === "provider_rate_limited" ||
242
+ input.failureCode === "provider_unavailable" ||
243
+ input.failureCode === "upstream_connectivity_unavailable"
244
+ ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
245
+ : 0);
246
+ // Do not shorten Retry-After, or schedule a replacement outside the window.
247
+ if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
248
+ return exhausted("deadline");
249
+ }
133
250
  return {
134
251
  status: "recovering",
135
- continueDelayMs,
252
+ continueDelayMs: delayMs,
253
+ ...(overload ? { maxProviderRecoveryCount: limit } : {}),
136
254
  };
137
255
  }
138
256
 
257
+ /** A timer/queue delay cannot authorize a provider call after the recovery window. */
258
+ export class ProviderOverloadRecoveryExpiredError extends Error {
259
+ readonly failure;
260
+
261
+ constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock") {
262
+ super("Confirmed provider overload recovery window expired before dispatch");
263
+ this.name = "ProviderOverloadRecoveryExpiredError";
264
+ this.failure = providerRecoveryExhaustedFailure(
265
+ {
266
+ error: this.message,
267
+ code: "provider_unavailable",
268
+ providerCondition: "overloaded" as const,
269
+ },
270
+ {
271
+ status: "exhausted",
272
+ // The queued replacement has not dispatched; count only prior retries.
273
+ providerRecoveryCount: Math.min(
274
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
275
+ Math.max(0, count - 1),
276
+ ),
277
+ maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
278
+ providerRecoveryExhaustedReason: reason,
279
+ },
280
+ );
281
+ }
282
+ }
283
+
284
+ export function assertProviderOverloadRecoveryActive(input: {
285
+ failureCode: unknown;
286
+ providerRecoveryCount: number;
287
+ recoveryStartedAt?: number | undefined;
288
+ now?: number;
289
+ }): void {
290
+ if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
291
+ return;
292
+ }
293
+ if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
294
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
295
+ }
296
+ const now = input.now ?? Date.now();
297
+ const startedAt = input.recoveryStartedAt;
298
+ if (
299
+ startedAt === undefined ||
300
+ !Number.isFinite(startedAt) ||
301
+ !Number.isFinite(now) ||
302
+ startedAt > now
303
+ ) {
304
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
305
+ }
306
+ if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
307
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
308
+ }
309
+ }
310
+
139
311
  export function providerRecoveryExhaustedFailure<
140
312
  T extends Record<string, unknown> & { error: string },
141
313
  >(
@@ -147,18 +319,89 @@ export function providerRecoveryExhaustedFailure<
147
319
  providerRecoveryCount: number;
148
320
  maxProviderRecoveryCount: number;
149
321
  lastRetryableError: string;
322
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
150
323
  } {
151
324
  return {
152
325
  ...failure,
153
- error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
326
+ error: providerRecoveryExhaustedMessage({
327
+ code: typeof failure.code === "string" ? failure.code : null,
328
+ providerCondition: isProviderCondition(failure.providerCondition)
329
+ ? failure.providerCondition
330
+ : null,
331
+ modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
332
+ providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
333
+ providerRecoveryCount: recovery.providerRecoveryCount,
334
+ }),
154
335
  retryable: false,
155
336
  recoveryExhausted: true,
156
337
  providerRecoveryCount: recovery.providerRecoveryCount,
157
338
  maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
339
+ ...(recovery.providerRecoveryExhaustedReason
340
+ ? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason }
341
+ : {}),
158
342
  lastRetryableError: failure.error,
159
343
  };
160
344
  }
161
345
 
346
+ function isProviderCondition(value: unknown): value is ProviderCondition {
347
+ return (
348
+ value === "overloaded" ||
349
+ value === "unavailable" ||
350
+ value === "unresponsive" ||
351
+ value === "rate_limited"
352
+ );
353
+ }
354
+
355
+ /**
356
+ * A retryable failure whose automatic same-turn recovery was not authorized
357
+ * (for example, request custody could not be checkpointed). The classifier's
358
+ * "will retry" sentence would be false on a terminal record, so name the
359
+ * condition and the person's options instead.
360
+ */
361
+ export function automaticRecoveryUnavailableFailure<
362
+ T extends Record<string, unknown> & { error: string },
363
+ >(
364
+ failure: T,
365
+ ): T & {
366
+ retryable: false;
367
+ automaticRecovery: "unavailable";
368
+ lastRetryableError: string;
369
+ } {
370
+ const subject = providerRecoverySubject({
371
+ code: typeof failure.code === "string" ? failure.code : null,
372
+ providerCondition: isProviderCondition(failure.providerCondition)
373
+ ? failure.providerCondition
374
+ : null,
375
+ modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
376
+ providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
377
+ });
378
+ return {
379
+ ...failure,
380
+ error: `${subject}. Opengeni couldn't retry this turn automatically. Try again, or send a message to continue.`,
381
+ retryable: false,
382
+ automaticRecovery: "unavailable",
383
+ lastRetryableError: failure.error,
384
+ };
385
+ }
386
+
387
+ /**
388
+ * Name the exact accepted model route on model-provider recovery evidence so a
389
+ * person learns which model is affected. Display labels only: the route's
390
+ * credentials, base URL and upstream ids never enter the event.
391
+ */
392
+ export function withModelRoutePresentation<T extends { code?: string | undefined }>(
393
+ failure: T,
394
+ route: { model: string; modelLabel: string; providerLabel: string } | undefined,
395
+ ): T & { model?: string; modelLabel?: string; providerLabel?: string } {
396
+ if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
397
+ return {
398
+ ...failure,
399
+ model: route.model,
400
+ modelLabel: route.modelLabel,
401
+ providerLabel: route.providerLabel,
402
+ };
403
+ }
404
+
162
405
  export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
163
406
  const value = metadata.providerRecoveryCount;
164
407
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
@@ -194,6 +437,13 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
194
437
  value.error && typeof value.error === "object"
195
438
  ? (value.error as Record<string, unknown>)
196
439
  : null;
440
+ const milliseconds = Number(
441
+ headerValue(value.headers, "retry-after-ms") ??
442
+ headerValue(value.responseHeaders, "retry-after-ms") ??
443
+ headerValue(body?.headers, "retry-after-ms") ??
444
+ undefined,
445
+ );
446
+ if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
197
447
  const directSeconds = Number(
198
448
  value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
199
449
  );
@@ -317,20 +567,199 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
317
567
  });
318
568
  }
319
569
 
570
+ /** Complete structured-cause graph or no recovery authority. Budget distinct
571
+ * objects (including array containers), not queue positions/duplicate refs.
572
+ * The separate link ceiling bounds huge duplicate arrays without truncating
573
+ * them into permission. Cycles are harmless; overflow/unreadable edges hold. */
574
+ function structuredRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
575
+ const graph = new Map<object, Set<object>>();
576
+ const queue: object[] = [];
577
+ let links = 0;
578
+ const add = (value: unknown): boolean => {
579
+ if (!value || typeof value !== "object" || graph.has(value)) return true;
580
+ if (graph.size >= 64) return false;
581
+ graph.set(value, new Set());
582
+ queue.push(value);
583
+ return true;
584
+ };
585
+ if (!add(error)) return null;
586
+ try {
587
+ for (const current of queue) {
588
+ const record = current as Record<string, unknown>;
589
+ const children = Array.isArray(current)
590
+ ? current
591
+ : ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
592
+ for (const child of children) {
593
+ if (++links > 4096 || !add(child)) return null;
594
+ if (child && typeof child === "object") graph.get(current)!.add(child);
595
+ }
596
+ }
597
+ } catch {
598
+ return null;
599
+ }
600
+ return graph;
601
+ }
602
+
320
603
  function retryableDatabaseFailureCode(
321
604
  error: unknown,
605
+ requireDatabaseProvenance = false,
322
606
  ): PostClaimDatabaseRecoveryDetail["code"] | null {
323
- const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
324
- const sqlState = persistenceFailure?.details.sqlState ?? null;
325
- if (sqlState === null && isRetryableDatabaseTransportFailure(error)) {
326
- return "db_failure";
607
+ try {
608
+ const graph = structuredRecoveryCauseGraph(error);
609
+ if (!graph) return null;
610
+ const transports = new Set<object>();
611
+ const boundaries = new Set<object>();
612
+ const ownDatabaseNodes = new Set<object>();
613
+ const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
614
+ const ownChildren = (node: object): object[] =>
615
+ node instanceof DatabaseTransactionError
616
+ ? // Callback failures retained beside a failed rollback supply vetoes,
617
+ // never driver provenance for a provider's connection-looking error.
618
+ [...graph.get(node)!].filter((child) => child === node.cause)
619
+ : [...graph.get(node)!];
620
+ // Ask the canonical transport predicate about ONLY this node's facts. Its
621
+ // recursive search must not pair a DB sibling with an unrelated provider.
622
+ for (const node of graph.keys()) {
623
+ if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
624
+ return null;
625
+ const record = node as Record<string, unknown>;
626
+ if (
627
+ requireDatabaseProvenance
628
+ ? isRunningTurnDatabaseTransportFailure(record)
629
+ : isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno })
630
+ )
631
+ transports.add(node);
632
+ const sqlState = isSessionEventPersistenceError(node)
633
+ ? node.details.sqlState
634
+ : record.name === "PostgresError" && typeof record.code === "string"
635
+ ? record.code
636
+ : null;
637
+ if (
638
+ requireDatabaseProvenance
639
+ ? isRunningTurnDatabaseConnectionSqlState(sqlState)
640
+ : isDatabaseConnectionSqlState(sqlState)
641
+ )
642
+ transports.add(node);
643
+ if (
644
+ node instanceof DrizzleQueryError ||
645
+ node instanceof DatabaseTransactionError ||
646
+ isSessionEventPersistenceError(node)
647
+ ) {
648
+ // Only actual errors raised at our ORM/typed persistence boundary own
649
+ // their driver subtree. A PostgresError name, SDK wrapper or provider
650
+ // socket by itself is never own-client provenance for a running turn.
651
+ const queue: object[] = [node];
652
+ for (const source of queue) {
653
+ if (ownDatabaseNodes.has(source)) continue;
654
+ ownDatabaseNodes.add(source);
655
+ queue.push(...ownChildren(source));
656
+ }
657
+ }
658
+ }
659
+ const hasOwnTransport = (boundary: object): boolean => {
660
+ const seen = new Set<object>();
661
+ const queue = [boundary];
662
+ for (const node of queue) {
663
+ if (seen.has(node)) continue;
664
+ seen.add(node);
665
+ if (transports.has(node)) return true;
666
+ queue.push(...ownChildren(node));
667
+ }
668
+ return false;
669
+ };
670
+ // Inspect ALL trusted DB evidence before returning a permit. A permanent,
671
+ // auth or uncertain SQLSTATE vetoes every sibling regardless of traversal
672
+ // order; a deeper transport/reset cannot override it either.
673
+ for (const node of graph.keys()) {
674
+ const record = node as Record<string, unknown>;
675
+ if (node instanceof DrizzleQueryError || node instanceof DatabaseTransactionError) {
676
+ boundaries.add(node);
677
+ if (hasOwnTransport(node)) codes.add("db_failure");
678
+ }
679
+ const typed = isSessionEventPersistenceError(node);
680
+ if (!typed && record.name !== "PostgresError") continue;
681
+ boundaries.add(node);
682
+ const sqlState = typed
683
+ ? node.details.sqlState
684
+ : typeof record.code === "string"
685
+ ? record.code
686
+ : null;
687
+ const connectionOutage = requireDatabaseProvenance
688
+ ? isRunningTurnDatabaseConnectionSqlState(sqlState)
689
+ : isDatabaseConnectionSqlState(sqlState);
690
+ // PostgreSQL aborts a deadlock or serialization victim's whole
691
+ // transaction: unlike a lost connection, that write certainly did not
692
+ // commit, so it is at least as safe for exact-attempt recovery.
693
+ const rolledBack = isRetryablePersistenceSqlState(sqlState);
694
+ const code = typed
695
+ ? retryablePersistenceFailureCode(sqlState)
696
+ : connectionOutage || rolledBack
697
+ ? databaseFailureCode(sqlState)
698
+ : null;
699
+ if (
700
+ !code ||
701
+ (requireDatabaseProvenance &&
702
+ !connectionOutage &&
703
+ !rolledBack &&
704
+ !(sqlState === null && hasOwnTransport(node)))
705
+ )
706
+ return null;
707
+ if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
708
+ }
709
+ // Preserve the legacy pre-execution transport-only allowance, but never
710
+ // borrow it across an explicit DB boundary with no eligible own evidence.
711
+ if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
712
+ codes.add("db_failure");
713
+ // Stable classification for multiple positive DB siblings too.
714
+ for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"] as const)
715
+ if (codes.has(code)) return code;
716
+ return null;
717
+ } catch {
718
+ // Unreadable structured facts are no more authority than unreadable edges.
719
+ return null;
327
720
  }
721
+ }
722
+
723
+ const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
724
+ // postgres.js reports these for a physically lost connection, including
725
+ // transaction cleanup after its socket has already closed. Own-client
726
+ // provenance and the unknown-outcome veto remain mandatory above.
727
+ "CONNECTION_CLOSED",
728
+ "CONNECTION_DESTROYED",
729
+ "CONNECTION_ENDED",
730
+ "ECONNREFUSED",
731
+ "ECONNRESET",
732
+ "CONNECT_TIMEOUT",
733
+ ]);
734
+
735
+ function isRunningTurnDatabaseTransportFailure(record: Record<string, unknown>): boolean {
736
+ return [record.code, record.errno].some(
737
+ (code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code),
738
+ );
739
+ }
740
+
741
+ function isRunningTurnDatabaseConnectionSqlState(sqlState: string | null): boolean {
742
+ return (
743
+ sqlState !== null &&
744
+ (/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState))
745
+ );
746
+ }
747
+
748
+ function isDatabaseConnectionSqlState(sqlState: string | null): boolean {
749
+ return (
750
+ sqlState !== null &&
751
+ (sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState))
752
+ );
753
+ }
754
+
755
+ function retryablePersistenceFailureCode(
756
+ sqlState: string | null,
757
+ ): PostClaimDatabaseRecoveryDetail["code"] | null {
328
758
  if (
329
- !persistenceFailure ||
330
759
  !(
331
760
  sqlState === null ||
332
761
  sqlState.startsWith("08") ||
333
- sqlState.startsWith("40") ||
762
+ isRetryablePersistenceSqlState(sqlState) ||
334
763
  sqlState.startsWith("53") ||
335
764
  sqlState === "55P03" ||
336
765
  sqlState === "57014" ||
@@ -342,32 +771,59 @@ function retryableDatabaseFailureCode(
342
771
  ) {
343
772
  return null;
344
773
  }
345
- return persistenceFailure.details.code;
774
+ return databaseFailureCode(sqlState);
346
775
  }
347
776
 
348
777
  /**
349
- * Carry one exact claimed-but-not-started attempt into the workflow's DB-only
350
- * recovery lane. Permanent database/state failures remain terminal; only the
351
- * same operational outage classes that are safe before claim are admitted.
778
+ * True when an error is a structured database failure that the exact-attempt
779
+ * database recovery path owns. Callers that translate other failures into
780
+ * user-visible states must rethrow these unchanged.
781
+ */
782
+ export function isPostClaimDatabaseRecoveryCandidate(error: unknown): boolean {
783
+ return retryableDatabaseFailureCode(error, true) !== null;
784
+ }
785
+
786
+ /**
787
+ * Carry one exact claimed attempt into the workflow's DB-only
788
+ * recovery lane. Permanent database/state failures remain terminal. The
789
+ * running-turn lane additionally requires a closed own-client outage class;
790
+ * existing pre-execution recovery classifications are unchanged.
352
791
  */
353
792
  export function postClaimDatabaseRecoveryFailure(input: {
354
793
  error: unknown;
355
794
  turnId: string;
356
795
  triggerEventId: string;
357
796
  executionGeneration: number;
797
+ /** Require an actual own-client boundary and the closed running-turn allowlist. */
798
+ requireDatabaseProvenance?: boolean;
799
+ sandboxSetupOutcomeUnknown?: true;
800
+ sandboxSetupRecoveryExhausted?: true;
358
801
  providerRecovery?: {
359
802
  failureCode: string;
360
803
  providerRecoveryCount: number;
804
+ continueDelayMs?: number;
361
805
  };
362
806
  }): ApplicationFailure | null {
363
- const code = retryableDatabaseFailureCode(input.error);
807
+ const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
364
808
  if (!code || input.executionGeneration < 1) return null;
809
+ if (
810
+ (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
811
+ ((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
812
+ input.providerRecovery)
813
+ ) {
814
+ return null;
815
+ }
365
816
  if (
366
817
  input.providerRecovery &&
367
818
  (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
368
819
  input.providerRecovery.providerRecoveryCount <= 0 ||
369
- input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
370
- !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))
820
+ input.providerRecovery.providerRecoveryCount >
821
+ providerRecoveryLimit(input.providerRecovery.failureCode) ||
822
+ !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) ||
823
+ !validProviderOverloadRecoveryDelay(
824
+ input.providerRecovery.failureCode,
825
+ input.providerRecovery.continueDelayMs,
826
+ ))
371
827
  ) {
372
828
  return null;
373
829
  }
@@ -376,13 +832,21 @@ export function postClaimDatabaseRecoveryFailure(input: {
376
832
  triggerEventId: input.triggerEventId,
377
833
  executionGeneration: input.executionGeneration,
378
834
  code,
835
+ ...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
836
+ ...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
379
837
  ...(input.providerRecovery
380
838
  ? {
381
839
  providerFailureCode: input.providerRecovery.failureCode,
382
840
  providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
841
+ ...(input.providerRecovery.continueDelayMs !== undefined
842
+ ? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs }
843
+ : {}),
383
844
  }
384
845
  : {}),
385
846
  };
847
+ // Known usage whose receipt write failed travels with the exact attempt.
848
+ const sessionUsageReceipts = heldSessionUsageReceiptsForTurn(input.turnId);
849
+ if (sessionUsageReceipts.length > 0) detail.sessionUsageReceipts = sessionUsageReceipts;
386
850
  return ApplicationFailure.create({
387
851
  message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
388
852
  type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
@@ -462,8 +926,25 @@ export function sandboxLifecycleTransitionDiagnostic(
462
926
  return null;
463
927
  }
464
928
 
929
+ export function modelPreparationFailureEventPayload(error: unknown, durationMs: number) {
930
+ const transition = sandboxLifecycleTransitionDiagnostic(error);
931
+ return {
932
+ phase: "model_preparation",
933
+ durationMs: Math.max(0, Math.round(durationMs)),
934
+ expectedTransition: transition !== null,
935
+ ...(transition
936
+ ? {
937
+ failureCategory: "drain_capture_wait",
938
+ failureStage: "lifecycle_wait",
939
+ failureCode: transition.reason,
940
+ retryable: true,
941
+ }
942
+ : {}),
943
+ };
944
+ }
945
+
465
946
  /**
466
- * Recognize the one active-route transition that cannot finish inside its
947
+ * Recognize active-route transitions that cannot finish inside their
467
948
  * originating attempt. A Modal-home session may start on a Connected Machine
468
949
  * without creating or leasing its managed home box. When an explicit attach
469
950
  * clears the active pointer back to home, the pointer commit is authoritative,
@@ -474,9 +955,56 @@ export function sandboxLifecycleTransitionDiagnostic(
474
955
  * only those structural error links with a strict bound; never classify from
475
956
  * message text, which could originate in model or tool content.
476
957
  */
958
+ /**
959
+ * Only the home resolver's positive pre-dispatch signal permits a fresh attempt.
960
+ * The same error class can also escape AFTER a provider mutation, so neither its
961
+ * retryable property nor message alone establishes replay safety. Inspect the
962
+ * complete bounded SDK cause graph and let uncertain peer outcomes veto this
963
+ * narrow recovery; unreadable or overflowing graphs fail closed.
964
+ */
965
+ function isPreDispatchHomeBackendRecoveryRequired(error: unknown): boolean {
966
+ const graph = structuredRecoveryCauseGraph(error);
967
+ if (!graph) return false;
968
+ let found = false;
969
+ try {
970
+ for (const node of graph.keys()) {
971
+ if (
972
+ isRoutingMutationOutcomeUnknownError(node) ||
973
+ isRoutingMutationOutputRejectedError(node) ||
974
+ isRawProviderCommandOutcomeUnknown(node) ||
975
+ isModalCommandStartOutcomeUnknownError(node) ||
976
+ isProviderCommandObservationUnavailableError(node)
977
+ )
978
+ return false;
979
+ if (node instanceof RoutingBackendRecoveryRequiredError) {
980
+ if (
981
+ node.op !== "resolve_home_backend" ||
982
+ !Number.isSafeInteger(node.leaseEpoch) ||
983
+ node.leaseEpoch < 0 ||
984
+ !node.retryable ||
985
+ (node.recovery !== "pending" && node.recovery !== "superseded")
986
+ )
987
+ return false;
988
+ found = true;
989
+ }
990
+ }
991
+ } catch {
992
+ return false;
993
+ }
994
+ return found;
995
+ }
996
+
477
997
  export function sandboxRouteTransitionCode(
478
998
  error: unknown,
479
- ): "home_unavailable_this_turn" | "workspace_root_changed_this_turn" | null {
999
+ ):
1000
+ | "home_unavailable_this_turn"
1001
+ | "workspace_root_changed_this_turn"
1002
+ | "native_capabilities_changed_this_attempt"
1003
+ | "home_backend_recovery_pending"
1004
+ | null {
1005
+ if (isPreDispatchHomeBackendRecoveryRequired(error)) {
1006
+ return "home_backend_recovery_pending";
1007
+ }
480
1008
  const pending: unknown[] = [error];
481
1009
  const seen = new WeakSet<object>();
482
1010
  let inspected = 0;
@@ -489,6 +1017,13 @@ export function sandboxRouteTransitionCode(
489
1017
 
490
1018
  try {
491
1019
  const record = current as Record<string, unknown>;
1020
+ if (
1021
+ (current instanceof SandboxCapabilitiesChangedError ||
1022
+ record.name === "SandboxCapabilitiesChangedError") &&
1023
+ record.code === "native_capabilities_changed_this_attempt"
1024
+ ) {
1025
+ return "native_capabilities_changed_this_attempt";
1026
+ }
492
1027
  if (
493
1028
  (current instanceof ActiveBackendUnresolvableError ||
494
1029
  record.name === "ActiveBackendUnresolvableError") &&
@@ -612,6 +1147,14 @@ export function compactionFailureReasonFromError(error: unknown): string {
612
1147
  `the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
613
1148
  );
614
1149
  }
1150
+ // An exhausted provider quota is not retried (see agentRunFailurePayload),
1151
+ // so name the refusal plainly instead of the raw diagnostic envelope.
1152
+ const quota = classifyProviderQuotaExhaustionError(error);
1153
+ if (quota) {
1154
+ return compactionFailureReason(
1155
+ `${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`,
1156
+ );
1157
+ }
615
1158
  if (
616
1159
  error instanceof CompactionProviderResponseError ||
617
1160
  error instanceof EmptyCompactionSummaryError
@@ -638,8 +1181,12 @@ export function compactionFailureTurnEventPayload(
638
1181
  recovery: "user_message";
639
1182
  compacted: false;
640
1183
  providerRejection?: CompactionProviderRejection;
1184
+ quotaScope?: ProviderQuotaScope;
641
1185
  } {
642
1186
  const rejection = compactionProviderRejection(error);
1187
+ // The same closed marker as a `provider_quota_exhausted` turn failure, so
1188
+ // clients can name the exhausted limit and offer another model here too.
1189
+ const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
643
1190
  return {
644
1191
  error: overrides.error ?? compactionFailureReasonFromError(error),
645
1192
  code: "context_compaction_failed",
@@ -647,6 +1194,7 @@ export function compactionFailureTurnEventPayload(
647
1194
  recovery: "user_message",
648
1195
  compacted: false,
649
1196
  ...(rejection ? { providerRejection: rejection } : {}),
1197
+ ...(quota ? { quotaScope: quota.scope } : {}),
650
1198
  };
651
1199
  }
652
1200
 
@@ -658,18 +1206,38 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
658
1206
 
659
1207
  export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
660
1208
  if (!(error instanceof CompactionProviderResponseError)) return false;
1209
+ // The worker's own included-usage refusal (usage exhausted or unverifiable)
1210
+ // happens before dispatch, so the checkpoint request was never sent. It
1211
+ // settles exactly like the same refusal on an ordinary request: recorded
1212
+ // against the account, then the same turn re-places or waits for capacity.
1213
+ if (findCodexCreditPolicyError(error)) return true;
661
1214
  if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
1215
+ // A Claude subscription refusal (rate limit or reconnect) on the checkpoint
1216
+ // request recovers exactly like one on an ordinary request: failure
1217
+ // settlement rotates to another account or waits for capacity on the same
1218
+ // logical turn. Its wording or retry hint never makes it a terminal quota.
1219
+ const claudeRefusal = isClaudeSubscriptionTransportError(error)
1220
+ ? classifyClaudeProviderError(error)
1221
+ : null;
1222
+ if (claudeRefusal?.kind === "unauthorized" || claudeRefusal?.kind === "rate_limited") return true;
662
1223
  // Codex may reject an opaque artifact it minted itself on the compaction
663
1224
  // request exactly as it can on an ordinary request. Failure settlement
664
1225
  // invalidates only the exact participating artifacts and recovers the same
665
1226
  // logical turn; when nothing can be invalidated it fails closed there.
666
1227
  if (classifyCodexEncryptedArtifactRejection(error)) return true;
1228
+ // A ChatGPT plan that no longer includes the model refuses the compaction
1229
+ // request exactly as it refuses an ordinary one (often with an empty 400).
1230
+ // Failure settlement re-checks the plan and, when it proves the loss,
1231
+ // excludes that account for the model and fails the same turn over; an
1232
+ // unexplained rejection still fails there with typed copy.
1233
+ if (classifyCodexEntitlementRejection(error)) return true;
667
1234
  return agentRunFailurePayload(error).retryable === true;
668
1235
  }
669
1236
 
670
1237
  export function classifyContextWindowOverflowError(
671
1238
  error: unknown,
672
1239
  ): { message: string; code?: string; detail?: string } | null {
1240
+ if (isProviderSafetyRefusal(error)) return null;
673
1241
  const fields = collectErrorStrings(error);
674
1242
  const matched = fields.find(
675
1243
  (value) =>
@@ -736,6 +1304,9 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
736
1304
  }
737
1305
  seen.add(value);
738
1306
  const out: string[] = [];
1307
+ // This diagnostic is already provider-owned and byte-bounded. Do not widen
1308
+ // generic detail traversal to arbitrary application payloads.
1309
+ if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
739
1310
  const record = value as Record<string, unknown>;
740
1311
  for (const key of ["message", "code", "type", "name", "param"]) {
741
1312
  const field = record[key];
@@ -792,15 +1363,47 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
792
1363
  */
793
1364
  export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
794
1365
  "unable to connect. is the computer able to access the url?";
1366
+ /** Bun's exact fetch text when an upstream drops an open connection. */
1367
+ export const STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE =
1368
+ "the socket connection was closed unexpectedly. for more information, pass `verbose: true` in the second argument to fetch()";
1369
+ const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES = new Set([
1370
+ STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE,
1371
+ STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE,
1372
+ ]);
795
1373
 
796
1374
  export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
797
- return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
1375
+ return STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES.has(message.trim().toLowerCase());
1376
+ }
1377
+
1378
+ const TRANSIENT_TRANSPORT_CODE = /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i;
1379
+
1380
+ /**
1381
+ * A network fault code anywhere on the Error.cause chain. SDK and runner
1382
+ * wrappers often keep the transport error only as `cause`, so the outermost
1383
+ * error's own `code` is not enough.
1384
+ */
1385
+ function transientTransportCode(error: unknown): string | undefined {
1386
+ let current = error;
1387
+ const seen = new Set<unknown>();
1388
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1389
+ if (seen.has(current)) break;
1390
+ seen.add(current);
1391
+ const value = current as { code?: unknown; cause?: unknown };
1392
+ if (typeof value.code === "string" && TRANSIENT_TRANSPORT_CODE.test(value.code)) {
1393
+ return value.code;
1394
+ }
1395
+ current = value.cause;
1396
+ }
1397
+ return undefined;
798
1398
  }
799
1399
 
800
1400
  function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
1401
+ if (error instanceof ResponsesStreamingTerminalError) {
1402
+ return error.category === "safety" ? error.detail : undefined;
1403
+ }
801
1404
  return collectErrorStrings(error).find(
802
1405
  (value) =>
803
- /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
1406
+ /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
804
1407
  value,
805
1408
  ) || /\bthis request was blocked by our safety systems\b/i.test(value),
806
1409
  );
@@ -810,16 +1413,48 @@ function isProviderSafetyRefusal(error: unknown): boolean {
810
1413
  return providerSafetyRefusalDiagnostic(error) !== undefined;
811
1414
  }
812
1415
 
1416
+ /** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
1417
+ function providerHttpStatus(error: unknown): number | undefined {
1418
+ let current = error;
1419
+ const seen = new Set<unknown>();
1420
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1421
+ if (seen.has(current)) break;
1422
+ seen.add(current);
1423
+ const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
1424
+ const status = Number(value.status ?? value.statusCode);
1425
+ if (Number.isInteger(status) && status >= 100 && status < 600) return status;
1426
+ current = value.cause;
1427
+ }
1428
+ return undefined;
1429
+ }
1430
+
1431
+ /**
1432
+ * A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
1433
+ * "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
1434
+ * Explicit cancellation (`AbortError`) is deliberately not matched.
1435
+ */
1436
+ export function isTransportTimeoutError(error: unknown): boolean {
1437
+ let current: unknown = error;
1438
+ const seen = new Set<unknown>();
1439
+ for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
1440
+ if (seen.has(current)) break;
1441
+ seen.add(current);
1442
+ const value = current as { name?: unknown; message?: unknown; cause?: unknown };
1443
+ if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
1444
+ return true;
1445
+ }
1446
+ current = value.cause;
1447
+ }
1448
+ return false;
1449
+ }
1450
+
813
1451
  export function isTransientProviderError(error: unknown): boolean {
1452
+ if (error instanceof ResponsesStreamingTerminalError) {
1453
+ return error.category === "unavailable";
1454
+ }
814
1455
  // A semantic refusal can arrive inside a 5xx transport envelope.
815
1456
  if (isProviderSafetyRefusal(error)) return false;
816
- const status =
817
- typeof error === "object" && error !== null
818
- ? Number(
819
- (error as { status?: unknown; statusCode?: unknown }).status ??
820
- (error as { statusCode?: unknown }).statusCode,
821
- )
822
- : undefined;
1457
+ const status = providerHttpStatus(error);
823
1458
  // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
824
1459
  // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
825
1460
  // a request fault that must NOT auto-retry — even if its body happens to read like
@@ -828,13 +1463,7 @@ export function isTransientProviderError(error: unknown): boolean {
828
1463
  if (status !== undefined && Number.isFinite(status)) {
829
1464
  return status >= 500 && status < 600;
830
1465
  }
831
- const code =
832
- typeof error === "object" && error !== null && "code" in error
833
- ? String((error as { code?: unknown }).code)
834
- : undefined;
835
- if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
836
- return true;
837
- }
1466
+ if (transientTransportCode(error)) return true;
838
1467
  const message = error instanceof Error ? error.message : String(error);
839
1468
  if (isExactStatuslessUpstreamConnectivityMessage(message)) {
840
1469
  return true;
@@ -844,23 +1473,58 @@ export function isTransientProviderError(error: unknown): boolean {
844
1473
  );
845
1474
  }
846
1475
 
847
- export type XaiCredentialFailure = {
848
- kind: "auth" | "forbidden" | "rate_limit";
849
- cooldownMs: number | null;
850
- };
1476
+ /** An explicit `usage_limit_reached` type or code string anywhere on the error chain. */
1477
+ function hasCodexUsageLimitType(error: unknown): boolean {
1478
+ return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
1479
+ }
851
1480
 
852
1481
  /**
853
- * Only definitive SuperGrok account refusals may move the same logical turn to
854
- * another credential. A marked HTTP 401/403/429, or an HTTP 200 SSE terminal
855
- * that is a rate-limit/capacity diagnostic, proves inference was refused
856
- * without an accepted model response; refresh relogin is equally definitive.
1482
+ * Recognize an exhausted API-key provider quota (a daily or monthly allowance,
1483
+ * a free-tier day cap, or an account out of credits) as distinct from an
1484
+ * ordinary per-minute rate limit. Retrying within the bounded same-turn budget
1485
+ * cannot succeed, so the turn fails promptly instead. Subscription transports
1486
+ * own their quota semantics through credential rotation and durable capacity
1487
+ * waits, so a Codex, SuperGrok or Claude subscription transport error never
1488
+ * classifies here.
857
1489
  */
858
- export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
1490
+ export function classifyProviderQuotaExhaustionError(
1491
+ error: unknown,
1492
+ ): ProviderQuotaExhaustion | null {
1493
+ if (
1494
+ isCodexTransportError(error) ||
1495
+ isXaiSubscriptionTransportError(error) ||
1496
+ isClaudeSubscriptionTransportError(error)
1497
+ )
1498
+ return null;
1499
+ // The same reader the OpenAI SDK retry veto uses, so the two never disagree.
1500
+ return classifyProviderQuotaError(error);
1501
+ }
1502
+
1503
+ /** A refusal raised by a Claude subscription (OAuth) transport, at any wrapper depth. */
1504
+ export function isClaudeSubscriptionTransportError(error: unknown): boolean {
1505
+ let current: unknown = error;
1506
+ for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
1507
+ if (current instanceof AnthropicRequestError && current.subscription) return true;
1508
+ current = current.cause;
1509
+ }
1510
+ return false;
1511
+ }
1512
+
1513
+ /**
1514
+ * SuperGrok's classification of a failed model request as a shared provider
1515
+ * outcome (the adapter's `classifyError`, SuperGrok track X1b): a refresh
1516
+ * relogin or a marked 401 is `unauthorized`, a marked 403 `forbidden`, and a
1517
+ * marked 429 or HTTP 200 SSE capacity terminal `rate_limited` with the
1518
+ * provider's retry time (null when it gave none: the adapter's fallback
1519
+ * applies). Anything else, a provider safety refusal included, is null: not
1520
+ * a refusal of the account.
1521
+ */
1522
+ export function classifyXaiProviderError(error: unknown): ProviderErrorOutcome | null {
859
1523
  if (isProviderSafetyRefusal(error)) return null;
860
1524
  let relogin: unknown = error;
861
1525
  for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
862
1526
  if (relogin instanceof XaiSubscriptionReloginRequired) {
863
- return { kind: "auth", cooldownMs: null };
1527
+ return { kind: "unauthorized" };
864
1528
  }
865
1529
  relogin = (relogin as Record<string, unknown>).cause;
866
1530
  }
@@ -875,10 +1539,10 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
875
1539
  const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
876
1540
  const code = String(value.code ?? body?.code ?? "").toLowerCase();
877
1541
  if (status === 401 || code === "unauthorized" || code === "invalid_token") {
878
- return { kind: "auth", cooldownMs: null };
1542
+ return { kind: "unauthorized" };
879
1543
  }
880
1544
  if (status === 403) {
881
- return { kind: "forbidden", cooldownMs: null };
1545
+ return { kind: "forbidden" };
882
1546
  }
883
1547
  const message = String(value.message ?? body?.message ?? "");
884
1548
  if (
@@ -888,10 +1552,7 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
888
1552
  status: Number.isInteger(status) ? status : null,
889
1553
  })
890
1554
  ) {
891
- return {
892
- kind: "rate_limit",
893
- cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
894
- };
1555
+ return { kind: "rate_limited", retryAfterMs: providerRetryAfterMs(error) };
895
1556
  }
896
1557
  current = value.cause;
897
1558
  }
@@ -918,15 +1579,167 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
918
1579
  return null;
919
1580
  }
920
1581
 
1582
+ function isRawDatabaseQueryError(error: unknown): boolean {
1583
+ return (
1584
+ error instanceof Error &&
1585
+ "query" in error &&
1586
+ typeof error.query === "string" &&
1587
+ "params" in error &&
1588
+ Array.isArray(error.params)
1589
+ );
1590
+ }
1591
+
1592
+ function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
1593
+ if (error instanceof AnthropicRequestError) return error;
1594
+ return error instanceof Error && error.cause instanceof AnthropicRequestError
1595
+ ? error.cause
1596
+ : undefined;
1597
+ }
1598
+
1599
+ function isRawProviderCommandOutcomeUnknown(error: unknown): boolean {
1600
+ try {
1601
+ return (
1602
+ error instanceof ProviderCommandInputOutcomeUnknownError ||
1603
+ error instanceof ProviderCommandStartOutcomeUnknownError
1604
+ );
1605
+ } catch {
1606
+ return false;
1607
+ }
1608
+ }
1609
+
921
1610
  export function agentRunFailurePayload(
922
1611
  error: unknown,
923
1612
  options: { isCodexTurn?: boolean } = {},
924
1613
  ): ReturnType<typeof baseAgentRunFailurePayload> {
1614
+ return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
1615
+ }
1616
+
1617
+ /**
1618
+ * Record the closed provider condition on a retryable model-provider failure so
1619
+ * clients can say "overloaded" rather than a generic outage. Presentation
1620
+ * evidence only: retry authority and pacing are decided by `code`.
1621
+ */
1622
+ function withProviderCondition(
1623
+ error: unknown,
1624
+ failure: ReturnType<typeof baseAgentRunFailurePayload>,
1625
+ ): ReturnType<typeof baseAgentRunFailurePayload> {
1626
+ if (failure.code === "provider_rate_limited") {
1627
+ return { ...failure, providerCondition: "rate_limited" };
1628
+ }
1629
+ if (failure.code !== "provider_unavailable") return failure;
1630
+ if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
1631
+ return {
1632
+ ...failure,
1633
+ providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable",
1634
+ };
1635
+ }
1636
+
1637
+ /** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
1638
+ export function isProviderOverloadError(
1639
+ error: unknown,
1640
+ failure: { error?: string; detail?: string } = {},
1641
+ ): boolean {
1642
+ if (providerHttpStatus(error) === 529) return true;
1643
+ const anthropic = anthropicRequestDiagnostic(error);
1644
+ return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
1645
+ (value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value),
1646
+ );
1647
+ }
1648
+
1649
+ /** Retry authority must not depend on the looser presentation-only overload label. */
1650
+ export function providerRecoveryCode(
1651
+ error: unknown,
1652
+ failure: { code?: string; retryable?: boolean },
1653
+ ): string | undefined {
1654
+ if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
1655
+ const status = providerHttpStatus(error);
1656
+ let current = error;
1657
+ const seen = new Set<unknown>();
1658
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1659
+ if (seen.has(current)) break;
1660
+ seen.add(current);
1661
+ if (
1662
+ current instanceof AnthropicRequestError &&
1663
+ status !== undefined &&
1664
+ status >= 500 &&
1665
+ status < 600 &&
1666
+ (status === 529 || current.errorType === "overloaded_error")
1667
+ ) {
1668
+ return PROVIDER_OVERLOAD_RECOVERY_CODE;
1669
+ }
1670
+ current = (current as { cause?: unknown }).cause;
1671
+ }
1672
+ return failure.code;
1673
+ }
1674
+
1675
+ function classifyAgentRunFailurePayload(
1676
+ error: unknown,
1677
+ options: { isCodexTurn?: boolean } = {},
1678
+ ): ReturnType<typeof baseAgentRunFailurePayload> {
1679
+ const sizeError = findAnthropicRequestSizeError(error);
1680
+ if (sizeError)
1681
+ return {
1682
+ error: sizeError.message,
1683
+ code: sizeError.code,
1684
+ retryable: false,
1685
+ requestSize: sizeError.requestSize,
1686
+ ...(sizeError.request_id ? { requestId: sizeError.request_id } : {}),
1687
+ };
1688
+ const graph = structuredRecoveryCauseGraph(error);
1689
+ const nodes = graph ? [...graph.keys()] : [];
1690
+ const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
1691
+ if (outputRejected) {
1692
+ // A known receipt applies only to its own mutation. Uncertain peers retain
1693
+ // their established terminal classification, regardless of SDK graph order.
1694
+ const unknown = nodes.find(
1695
+ (node) =>
1696
+ isRoutingMutationOutcomeUnknownError(node) ||
1697
+ isModalCommandStartOutcomeUnknownError(node) ||
1698
+ isProviderCommandObservationUnavailableError(node) ||
1699
+ isRawProviderCommandOutcomeUnknown(node),
1700
+ );
1701
+ if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
1702
+ return {
1703
+ error: outputRejected.message,
1704
+ code: outputRejected.code,
1705
+ retryable: false,
1706
+ };
1707
+ }
1708
+ const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
1709
+ if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
925
1710
  const failure = baseAgentRunFailurePayload(error, options);
926
1711
  const diagnostic = materializationVerificationDiagnostic(error);
1712
+ const anthropic = anthropicRequestDiagnostic(error);
1713
+ if (anthropic) {
1714
+ const authenticationRejected =
1715
+ anthropic.status === 401 &&
1716
+ (error === anthropic ||
1717
+ (error instanceof Error && (error as Error & { status?: unknown }).status === 401));
1718
+ return {
1719
+ ...failure,
1720
+ code:
1721
+ failure.code ??
1722
+ (authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
1723
+ retryable: failure.retryable ?? false,
1724
+ ...(anthropic.detail ? { detail: anthropic.detail } : {}),
1725
+ ...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
1726
+ };
1727
+ }
927
1728
  return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
928
1729
  }
929
1730
 
1731
+ /** Keep Anthropic provider text on terminal failures, never recovery events. */
1732
+ export function agentRunRecoveryFailurePayload(
1733
+ error: unknown,
1734
+ failure: ReturnType<typeof agentRunFailurePayload>,
1735
+ ): ReturnType<typeof agentRunFailurePayload> {
1736
+ if (!anthropicRequestDiagnostic(error)) return failure;
1737
+ // Project a copy: retry exhaustion still needs the terminal diagnostic.
1738
+ const recovery = { ...failure };
1739
+ delete recovery.detail;
1740
+ return recovery;
1741
+ }
1742
+
930
1743
  function baseAgentRunFailurePayload(
931
1744
  error: unknown,
932
1745
  options: { isCodexTurn?: boolean } = {},
@@ -950,6 +1763,22 @@ function baseAgentRunFailurePayload(
950
1763
  historyPersistenceStage?: MandatoryHistoryPersistenceStage;
951
1764
  mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
952
1765
  materializationDiagnostic?: MaterializationVerificationDiagnostic;
1766
+ requestSize?: AnthropicRequestSize;
1767
+ quotaScope?: ProviderQuotaScope;
1768
+ /** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
1769
+ planType?: string | null;
1770
+ /** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
1771
+ model?: string | null;
1772
+ /** Display label of the turn's model on model-provider recovery evidence. */
1773
+ modelLabel?: string;
1774
+ /** Display label of the serving provider on model-provider recovery evidence. */
1775
+ providerLabel?: string;
1776
+ /** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
1777
+ providerCondition?: ProviderCondition;
1778
+ recoveryExhausted?: boolean;
1779
+ providerRecoveryCount?: number;
1780
+ maxProviderRecoveryCount?: number;
1781
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
953
1782
  } {
954
1783
  if (error instanceof SandboxMaterializationVerificationError) {
955
1784
  return {
@@ -963,7 +1792,11 @@ function baseAgentRunFailurePayload(
963
1792
  return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
964
1793
  }
965
1794
  if (error instanceof MandatoryHistoryPersistenceError) {
966
- const underlying = isSessionEventPersistenceError(error.cause)
1795
+ const databaseFailure =
1796
+ isSessionEventPersistenceError(error.cause) ||
1797
+ findPostgresDriverError(error.cause) !== null ||
1798
+ isRawDatabaseQueryError(error.cause);
1799
+ const underlying = databaseFailure
967
1800
  ? agentRunFailurePayload(error.cause, options)
968
1801
  : {
969
1802
  error: error.cause instanceof Error ? error.cause.message : String(error.cause),
@@ -973,6 +1806,32 @@ function baseAgentRunFailurePayload(
973
1806
  historyPersistenceStage: error.stage,
974
1807
  };
975
1808
  }
1809
+ // Raw ORM wrappers can contain the full SQL and its parameters. Classify a
1810
+ // real driver before provider message heuristics; the original cause stays
1811
+ // available to internal diagnostics. This payload grants no replay authority.
1812
+ const postgresDriverError = findPostgresDriverError(error);
1813
+ const rawOrmFailure = isRawDatabaseQueryError(error);
1814
+ if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
1815
+ const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
1816
+ const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
1817
+ return {
1818
+ error: "Opengeni encountered a database error.",
1819
+ code: databaseFailureCode(sqlState),
1820
+ sqlState,
1821
+ ...(Object.keys(database).length > 0 ? { database } : {}),
1822
+ };
1823
+ }
1824
+ if (error instanceof AnthropicSizeRecoveryExhaustedError) {
1825
+ return { error: error.message, code: error.code, retryable: false };
1826
+ }
1827
+ if (error instanceof AnthropicProviderRejection) {
1828
+ return {
1829
+ error: error.message,
1830
+ code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
1831
+ retryable: false,
1832
+ ...(error.request_id ? { requestId: error.request_id } : {}),
1833
+ };
1834
+ }
976
1835
  const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
977
1836
  if (safetyRefusalDiagnostic !== undefined) {
978
1837
  return {
@@ -983,14 +1842,43 @@ function baseAgentRunFailurePayload(
983
1842
  detail: safetyRefusalDiagnostic,
984
1843
  };
985
1844
  }
1845
+ if (error instanceof ResponsesStreamingTerminalError) {
1846
+ const quota =
1847
+ error.category === "unknown" || error.category === "rate_limit"
1848
+ ? classifyProviderQuotaExhaustionError({
1849
+ code: error.code,
1850
+ error: { code: error.code, type: error.type, message: error.detail },
1851
+ retryAfterSeconds: error.retryAfterSeconds,
1852
+ })
1853
+ : null;
1854
+ if (quota) {
1855
+ return {
1856
+ error: providerQuotaExhaustedMessage(quota.scope),
1857
+ code: PROVIDER_QUOTA_EXHAUSTED_CODE,
1858
+ retryable: false,
1859
+ quotaScope: quota.scope,
1860
+ detail: error.detail,
1861
+ };
1862
+ }
1863
+ return {
1864
+ error:
1865
+ error.category === "rate_limit"
1866
+ ? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort."
1867
+ : error.category === "unavailable"
1868
+ ? "The model provider is temporarily unavailable. The same turn will retry after a short delay."
1869
+ : "The model provider rejected the response. Automatic retries stopped.",
1870
+ code:
1871
+ error.category === "rate_limit"
1872
+ ? "provider_rate_limited"
1873
+ : error.category === "unavailable"
1874
+ ? "provider_unavailable"
1875
+ : "provider_request_rejected",
1876
+ retryable: error.category === "rate_limit" || error.category === "unavailable",
1877
+ detail: error.detail,
1878
+ };
1879
+ }
986
1880
  const message = error instanceof Error ? error.message : String(error);
987
- const status =
988
- typeof error === "object" && error !== null
989
- ? Number(
990
- (error as { status?: unknown; statusCode?: unknown }).status ??
991
- (error as { statusCode?: unknown }).statusCode,
992
- )
993
- : undefined;
1881
+ const status = providerHttpStatus(error);
994
1882
  const code =
995
1883
  typeof error === "object" && error !== null && "code" in error
996
1884
  ? String((error as { code?: unknown }).code)
@@ -1021,10 +1909,29 @@ function baseAgentRunFailurePayload(
1021
1909
  retryable: true,
1022
1910
  };
1023
1911
  }
1024
- if (isModalTaskExecStartDnsResolutionError(error)) {
1912
+ if (isProviderCommandObservationUnavailableError(error)) {
1913
+ return {
1914
+ error:
1915
+ "A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
1916
+ code: "sandbox_command_observation_unavailable",
1917
+ retryable: false,
1918
+ };
1919
+ }
1920
+ if (isModalCommandStartOutcomeUnknownError(error)) {
1921
+ return {
1922
+ error:
1923
+ "A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
1924
+ code: "sandbox_command_start_outcome_unknown",
1925
+ retryable: false,
1926
+ };
1927
+ }
1928
+ if (
1929
+ !isRoutingMutationOutcomeUnknownError(error) &&
1930
+ isModalTaskExecStartPreDispatchUnavailableError(error)
1931
+ ) {
1025
1932
  return {
1026
1933
  error:
1027
- "The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
1934
+ "The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
1028
1935
  code: "sandbox_command_start_unavailable",
1029
1936
  retryable: true,
1030
1937
  };
@@ -1089,6 +1996,33 @@ function baseAgentRunFailurePayload(
1089
1996
  ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
1090
1997
  };
1091
1998
  }
1999
+ // Codex plan entitlement: the settlement path normally re-checks the plan
2000
+ // and records a precise payload. These branches keep any other path from
2001
+ // surfacing the SDK's raw "400 status code (no body)" text.
2002
+ if (error instanceof CodexPlanEntitlementError) {
2003
+ return error.payload;
2004
+ }
2005
+ // Shared subscription core placement outcomes carry their own typed copy.
2006
+ if (error instanceof SubscriptionCoreCodexTurnError) {
2007
+ return error.payload;
2008
+ }
2009
+ const entitlementRejection = classifyCodexEntitlementRejection(error);
2010
+ if (entitlementRejection) {
2011
+ return entitlementRejection.evidence === "plan_entitlement"
2012
+ ? codexPlanEntitlementFailurePayload({
2013
+ accountLabel: null,
2014
+ planType: null,
2015
+ planChanged: false,
2016
+ modelId: null,
2017
+ rejection: entitlementRejection,
2018
+ })
2019
+ : codexRequestRejectedFailurePayload({
2020
+ accountLabel: null,
2021
+ planType: null,
2022
+ rejection: entitlementRejection,
2023
+ planChecked: false,
2024
+ });
2025
+ }
1092
2026
  // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
1093
2027
  // must NOT be reported as a generic, retryable rate-limit (which would loop a
1094
2028
  // goal against a capped backend). Surface a precise, actionable message with
@@ -1099,8 +2033,15 @@ function baseAgentRunFailurePayload(
1099
2033
  // `usage_limit_reached` shape must still outrank generic 429 retryability.
1100
2034
  // Credential quarantine/failover remains separately provenance-gated by
1101
2035
  // `isCodexTransportError`; this branch only chooses the truthful user payload.
2036
+ // The looser "429 ... usage limit" wording counts only on a Codex transport
2037
+ // error: an API-key provider's 429 that says "usage limit" is provider quota
2038
+ // evidence, not a ChatGPT/Codex subscription cap.
2039
+ const creditPolicy = findCodexCreditPolicyError(error);
2040
+ if (creditPolicy) {
2041
+ return { error: creditPolicy.message, code: creditPolicy.code, retryable: false };
2042
+ }
1102
2043
  const usageLimit = classifyCodexUsageLimitError(error);
1103
- if (usageLimit) {
2044
+ if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
1104
2045
  return codexUsageLimitFailurePayload(usageLimit, message);
1105
2046
  }
1106
2047
  const codexTimeout = classifyCodexResponseTimeoutError(error, {
@@ -1144,6 +2085,39 @@ function baseAgentRunFailurePayload(
1144
2085
  ...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
1145
2086
  };
1146
2087
  }
2088
+ // A generic model stream that went silent mid-response (typed idle bound) or
2089
+ // a transport timeout ("The operation timed out." DOMException TimeoutError
2090
+ // from the fetch layer) is a dead upstream connection, not a request fault:
2091
+ // checkpoint durable truth and recover the same turn within the finite
2092
+ // provider recovery budget. Codex/SuperGrok typed timeouts are classified
2093
+ // above under their own transport policy.
2094
+ const streamIdle = classifyModelStreamIdleTimeoutError(error);
2095
+ if (streamIdle) {
2096
+ const seconds =
2097
+ streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1_000) : null;
2098
+ const window = seconds !== null ? ` for ${seconds}s` : "";
2099
+ return {
2100
+ error:
2101
+ streamIdle.kind === "progress"
2102
+ ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.`
2103
+ : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
2104
+ code: "provider_unavailable",
2105
+ retryable: true,
2106
+ timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
2107
+ responseObserved: true,
2108
+ detail: streamIdle.message,
2109
+ };
2110
+ }
2111
+ if (status === undefined && isTransportTimeoutError(error)) {
2112
+ return {
2113
+ error:
2114
+ "The model provider connection timed out. The same turn will retry after a short delay.",
2115
+ code: "provider_unavailable",
2116
+ retryable: true,
2117
+ timeoutClass: "transport",
2118
+ detail: message,
2119
+ };
2120
+ }
1147
2121
  if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
1148
2122
  return {
1149
2123
  error:
@@ -1152,10 +2126,33 @@ function baseAgentRunFailurePayload(
1152
2126
  retryable: true,
1153
2127
  };
1154
2128
  }
2129
+ // An exhausted quota also arrives as HTTP 429 (or 402), but no retry within
2130
+ // the finite same-turn budget can succeed. Fail the turn promptly with a
2131
+ // distinct code so the client can offer another model; ordinary short rate
2132
+ // limits fall through to the retryable branch below.
2133
+ if (status === 402 && code === "anthropic_billing_error") {
2134
+ return {
2135
+ error: "Claude could not bill this request. Check the account's billing and payment details.",
2136
+ code: "provider_billing_error",
2137
+ retryable: false,
2138
+ ...(message ? { detail: message } : {}),
2139
+ };
2140
+ }
2141
+ const quota = classifyProviderQuotaExhaustionError(error);
2142
+ if (quota) {
2143
+ return {
2144
+ error: providerQuotaExhaustedMessage(quota.scope),
2145
+ code: PROVIDER_QUOTA_EXHAUSTED_CODE,
2146
+ retryable: false,
2147
+ quotaScope: quota.scope,
2148
+ ...(message ? { detail: message } : {}),
2149
+ };
2150
+ }
1155
2151
  if (
1156
2152
  status === 429 ||
1157
- code === "rate_limit_exceeded" ||
1158
- /(?:too many requests|rate.?limit|\b429\b)/i.test(message)
2153
+ ((status === undefined || !Number.isFinite(status)) &&
2154
+ (code === "rate_limit_exceeded" ||
2155
+ /(?:too many requests|rate.?limit|\b429\b)/i.test(message)))
1159
2156
  ) {
1160
2157
  return {
1161
2158
  error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
@@ -1172,31 +2169,24 @@ function baseAgentRunFailurePayload(
1172
2169
  if (isExactStatuslessUpstreamConnectivityMessage(message)) {
1173
2170
  return {
1174
2171
  error:
1175
- "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
2172
+ "Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
1176
2173
  code: "upstream_connectivity_unavailable",
1177
2174
  retryable: true,
1178
2175
  };
1179
2176
  }
1180
2177
  return { error: message, code: "provider_unavailable", retryable: true };
1181
2178
  }
1182
- const postgresDriverError = findPostgresDriverError(error);
1183
- if (postgresDriverError) {
1184
- const database = safeDatabaseErrorFacts(postgresDriverError);
1185
- const sqlState = nestedPostgresSqlState(postgresDriverError);
1186
- if (sqlState !== null || Object.keys(database).length > 0) {
1187
- return {
1188
- error: message,
1189
- sqlState,
1190
- ...(Object.keys(database).length > 0 ? { database } : {}),
1191
- };
1192
- }
1193
- }
1194
2179
  return { error: message };
1195
2180
  }
1196
2181
 
1197
2182
  export type CodexCredentialFailure = {
1198
- kind: "auth" | "forbidden" | "rate_limit" | "quota";
2183
+ /**
2184
+ * `plan_entitlement` is produced only after a plan re-check proves the
2185
+ * serving account's current plan does not include the requested model.
2186
+ */
2187
+ kind: "auth" | "forbidden" | "rate_limit" | "quota" | "plan_entitlement";
1199
2188
  cooldownSeconds: number | null;
2189
+ origin?: "included_usage_policy" | "usage_verification_policy";
1200
2190
  };
1201
2191
 
1202
2192
  export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
@@ -1216,7 +2206,11 @@ export function codexCredentialCooldownUntil(
1216
2206
  > | null,
1217
2207
  now: Date,
1218
2208
  ): Date | null {
1219
- if (failure.kind === "auth" || failure.kind === "forbidden") {
2209
+ if (
2210
+ failure.kind === "auth" ||
2211
+ failure.kind === "forbidden" ||
2212
+ failure.kind === "plan_entitlement"
2213
+ ) {
1220
2214
  return null;
1221
2215
  }
1222
2216
  const providerReset =
@@ -1257,6 +2251,19 @@ export function codexCredentialCooldownUntil(
1257
2251
  * progress and therefore MUST NOT walk the credential pool automatically.
1258
2252
  */
1259
2253
  export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
2254
+ const creditPolicy = findCodexCreditPolicyError(error);
2255
+ if (creditPolicy instanceof CodexIncludedUsageUnknownError) {
2256
+ // Verified pre-dispatch refusal. A short hard hold cannot authorize credit
2257
+ // fallback and does not claim that the provider exhausted its allowance.
2258
+ return { kind: "rate_limit", cooldownSeconds: 60, origin: "usage_verification_policy" };
2259
+ }
2260
+ if (creditPolicy instanceof CodexIncludedUsageExhaustedError) {
2261
+ return {
2262
+ kind: "quota",
2263
+ cooldownSeconds: creditPolicy.resetsInSeconds,
2264
+ origin: "included_usage_policy",
2265
+ };
2266
+ }
1260
2267
  // A request safety refusal is not evidence that another account should run it.
1261
2268
  if (isProviderSafetyRefusal(error)) return null;
1262
2269
  // A permanent OAuth refresh failure is definitive and the shared resolver has
@@ -1276,6 +2283,12 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
1276
2283
  if (!isCodexTransportError(error)) {
1277
2284
  return null;
1278
2285
  }
2286
+ // Plan/model entitlement evidence (an explicit plan refusal, or an empty
2287
+ // HTTP 400) is not account health or quota. The worker re-checks the plan
2288
+ // first; only a proven entitlement loss walks the pool, as `plan_entitlement`.
2289
+ if (classifyCodexEntitlementRejection(error)) {
2290
+ return null;
2291
+ }
1279
2292
  const usageLimit = classifyCodexUsageLimitError(error);
1280
2293
  if (usageLimit) {
1281
2294
  return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
@@ -1380,3 +2393,42 @@ export function codexUsageLimitFailurePayload(
1380
2393
  // open indefinitely for a goal-bearing session; cap the continuation hold so the
1381
2394
  // goal re-evaluates at most this far out (it will re-pause if still capped).
1382
2395
  export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
2396
+
2397
+ /**
2398
+ * Claude's classification of a failed model request as a shared provider
2399
+ * outcome (the adapter's `classifyError`, Claude track C1b), for a request
2400
+ * of the Claude subscription transport only: a reconnect-required refresh or
2401
+ * HTTP 401 is `unauthorized` (the transport already ran the turn's one
2402
+ * forced refresh); 429 is `rate_limited` for the turn's model `modelId`, with
2403
+ * the provider's retry time (null: the adapter's fallback applies); 529 or
2404
+ * `overloaded_error` is `overloaded`, which never moves the turn to another
2405
+ * account; 403 is `fatal`, which never rotates or quarantines the account.
2406
+ * Anything else, a provider safety refusal included, is null: not a refusal
2407
+ * of the account.
2408
+ */
2409
+ export function classifyClaudeProviderError(
2410
+ error: unknown,
2411
+ modelId?: string,
2412
+ ): ProviderErrorOutcome | null {
2413
+ if (isProviderSafetyRefusal(error)) return null;
2414
+ let current: unknown = error;
2415
+ for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
2416
+ if (current instanceof ClaudeSubscriptionReconnectRequired) return { kind: "unauthorized" };
2417
+ if (current instanceof AnthropicRequestError) {
2418
+ if (!current.subscription) return null;
2419
+ if (current.status === 401) return { kind: "unauthorized" };
2420
+ if (current.status === 429)
2421
+ return {
2422
+ kind: "rate_limited",
2423
+ retryAfterMs: providerRetryAfterMs(current),
2424
+ ...(modelId === undefined ? {} : { modelId }),
2425
+ };
2426
+ if (current.status === 529 || current.errorType === "overloaded_error")
2427
+ return { kind: "overloaded" };
2428
+ if (current.status === 403) return { kind: "fatal" };
2429
+ return null;
2430
+ }
2431
+ current = current.cause;
2432
+ }
2433
+ return null;
2434
+ }