@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -1,14 +1,22 @@
1
+ import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
2
+ import { DrizzleQueryError } from "drizzle-orm";
3
+ import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
4
+ import { heldSessionUsageReceiptsForTurn } from "./session-usage-custody";
1
5
  import {
2
6
  ActiveSessionHistoryLimitExceededError,
3
7
  ApprovalRunStateLimitExceededError,
4
8
  nestedPostgresSqlState,
9
+ databaseFailureCode,
10
+ isRetryablePersistenceSqlState,
5
11
  safeDatabaseErrorFacts,
6
12
  isRetryableDatabaseTransportFailure,
7
13
  isSessionEventPersistenceError,
14
+ DatabaseTransactionError,
8
15
  SandboxLeaseTransitionError,
9
16
  } from "@opengeni/db";
10
17
  import {
11
18
  ActiveBackendUnresolvableError,
19
+ AnthropicProviderRejection,
12
20
  CompactionProviderResponseError,
13
21
  EmptyCompactionSummaryError,
14
22
  compactionProviderRejection,
@@ -16,29 +24,61 @@ import {
16
24
  type CompactionProviderRejection,
17
25
  isMcpRequestTimeoutError,
18
26
  isMcpTransportConnectivityError,
19
- isModalTaskExecStartDnsResolutionError,
27
+ isModalTaskExecStartPreDispatchUnavailableError,
28
+ isModalCommandStartOutcomeUnknownError,
29
+ isProviderCommandObservationUnavailableError,
30
+ ProviderCommandInputOutcomeUnknownError,
31
+ ProviderCommandStartOutcomeUnknownError,
32
+ isRoutingMutationOutcomeUnknownError,
33
+ isRoutingMutationOutputRejectedError,
20
34
  RoutingWorkspaceRootChangedError,
35
+ RoutingBackendRecoveryRequiredError,
36
+ ResponsesStreamingTerminalError,
37
+ classifyModelStreamIdleTimeoutError,
21
38
  SandboxMaterializationVerificationError,
22
39
  materializationVerificationDiagnostic,
23
40
  type MaterializationVerificationDiagnostic,
41
+ PROVIDER_QUOTA_EXHAUSTED_CODE,
42
+ type ProviderQuotaExhaustion,
43
+ type ProviderQuotaScope,
44
+ classifyProviderQuotaError,
45
+ providerQuotaExhaustedMessage,
24
46
  SelfhostedWorkspaceRootChangedError,
25
47
  UNKNOWN_MODEL_FINISH_REASON_CODE,
48
+ AnthropicRequestError,
49
+ findAnthropicRequestSizeError,
50
+ AnthropicSizeRecoveryExhaustedError,
51
+ type AnthropicRequestSize,
26
52
  } from "@opengeni/runtime";
27
53
  import {
28
54
  mcpTransportRequestFailureDiagnostic,
29
55
  type McpTransportRequestFailureDiagnostic,
30
56
  } from "@opengeni/runtime/mcp-network";
31
57
  import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
32
- import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
58
+ import type { ProviderErrorOutcome } from "@opengeni/subscriptions";
59
+ import { CODEX_USAGE_EXHAUSTED_PCT } from "./codex-usage-policy";
33
60
  import { RetainedAttachmentTransportLimitError } from "../run-input";
34
61
  import type { CodexAccountStatus } from "@opengeni/db";
35
62
  import {
63
+ CODEX_USAGE_LIMIT_ERROR_TYPE,
36
64
  CodexReloginRequired,
37
65
  classifyCodexEncryptedArtifactRejection,
66
+ classifyCodexEntitlementRejection,
38
67
  classifyCodexResponseTimeoutError,
39
68
  classifyCodexUsageLimitError,
40
69
  isCodexTransportError,
41
70
  } from "@opengeni/codex";
71
+ import {
72
+ CodexPlanEntitlementError,
73
+ codexPlanEntitlementFailurePayload,
74
+ codexRequestRejectedFailurePayload,
75
+ } from "./codex-plan-entitlement";
76
+ import { SubscriptionCoreCodexTurnError } from "./codex-core-errors";
77
+ import {
78
+ CodexIncludedUsageExhaustedError,
79
+ CodexIncludedUsageUnknownError,
80
+ findCodexCreditPolicyError,
81
+ } from "./codex-credit-policy";
42
82
  import {
43
83
  classifyXaiSubscriptionStreamingTerminalError,
44
84
  classifyXaiSubscriptionStreamIdleTimeoutError,
@@ -64,12 +104,45 @@ import {
64
104
  MandatoryHistoryPersistenceError,
65
105
  type MandatoryHistoryPersistenceStage,
66
106
  } from "./quiescence";
107
+ import {
108
+ MODEL_PROVIDER_RECOVERY_CODES,
109
+ providerRecoveryExhaustedMessage,
110
+ providerRecoverySubject,
111
+ type ProviderCondition,
112
+ } from "./provider-recovery-copy";
113
+ import {
114
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
115
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
116
+ PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
117
+ validProviderOverloadRecoveryDelay,
118
+ } from "./provider-recovery-policy";
119
+ export {
120
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
121
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
122
+ PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
123
+ } from "./provider-recovery-policy";
67
124
 
68
125
  // Retryable provider connectivity/5xx failures start quickly and back off to
69
126
  // this ceiling. Explicit rate limits retain the minute-granular fallback.
70
127
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
71
128
  export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
72
129
  export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
130
+ export function providerRecoveryLimit(failureCode: unknown): number {
131
+ return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE
132
+ ? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES
133
+ : MAX_AUTOMATIC_PROVIDER_RECOVERIES;
134
+ }
135
+ /**
136
+ * Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
137
+ * answer a per-minute token limit with a `retry-after` of about a second, which
138
+ * alone would spend every automatic recovery before the window resets.
139
+ */
140
+ export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
141
+ /** Positive-only spread: never shorten the provider's minimum delay. */
142
+ export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
143
+ const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
144
+ return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
145
+ }
73
146
  export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
74
147
 
75
148
  export class PostCompactionContinuationEmptyError extends Error {
@@ -85,25 +158,47 @@ export type ProviderRecoveryResult =
85
158
  | {
86
159
  status: "recovering";
87
160
  continueDelayMs: number;
161
+ maxProviderRecoveryCount?: number;
88
162
  }
89
163
  | {
90
164
  status: "exhausted";
91
165
  providerRecoveryCount: number;
92
166
  maxProviderRecoveryCount: number;
167
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
93
168
  };
94
169
 
95
170
  export function providerRecoveryResult(input: {
96
171
  failureCode: string | undefined;
97
172
  attemptNumber: number;
98
173
  retryAfterMs?: number | null;
174
+ jitterSample?: number;
175
+ /** Durable first-failure clock, never a new clock on each replacement attempt. */
176
+ recoveryStartedAt?: number | undefined;
177
+ now?: number;
99
178
  }): ProviderRecoveryResult {
100
- if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
179
+ const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
180
+ const limit = providerRecoveryLimit(input.failureCode);
181
+ const exhausted = (
182
+ reason: "deadline" | "retry_limit" | "invalid_clock",
183
+ ): Extract<ProviderRecoveryResult, { status: "exhausted" }> => ({
184
+ status: "exhausted",
185
+ providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
186
+ maxProviderRecoveryCount: limit,
187
+ providerRecoveryExhaustedReason: reason,
188
+ });
189
+ if (input.attemptNumber > limit) {
190
+ if (overload) return exhausted("retry_limit");
101
191
  return {
102
192
  status: "exhausted",
103
193
  providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
104
194
  maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
105
195
  };
106
196
  }
197
+ const now = input.now ?? Date.now();
198
+ const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
199
+ if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
200
+ return exhausted("invalid_clock");
201
+ }
107
202
  const providerDelay =
108
203
  input.retryAfterMs !== null &&
109
204
  input.retryAfterMs !== undefined &&
@@ -113,12 +208,22 @@ export function providerRecoveryResult(input: {
113
208
  : null;
114
209
  const continueDelayMs =
115
210
  input.failureCode === "provider_rate_limited"
116
- ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
117
- : input.failureCode === "provider_unavailable" ||
211
+ ? Math.max(
212
+ providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
213
+ PROVIDER_RATE_LIMIT_BACKOFF_MS[
214
+ Math.min(
215
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
216
+ PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1,
217
+ )
218
+ ]!,
219
+ )
220
+ : overload ||
221
+ input.failureCode === "provider_unavailable" ||
118
222
  input.failureCode === "upstream_connectivity_unavailable" ||
119
223
  input.failureCode === "sandbox_command_start_unavailable" ||
120
224
  input.failureCode === "mcp_transport_timeout" ||
121
225
  input.failureCode === "mcp_transport_unavailable" ||
226
+ input.failureCode === "turn_execution_policy_definition_mismatch" ||
122
227
  input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
123
228
  ? Math.max(
124
229
  providerDelay ?? 0,
@@ -130,12 +235,79 @@ export function providerRecoveryResult(input: {
130
235
  ]!,
131
236
  )
132
237
  : PROVIDER_BACKPRESSURE_DELAY_MS;
238
+ const delayMs =
239
+ continueDelayMs +
240
+ (overload ||
241
+ input.failureCode === "provider_rate_limited" ||
242
+ input.failureCode === "provider_unavailable" ||
243
+ input.failureCode === "upstream_connectivity_unavailable"
244
+ ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
245
+ : 0);
246
+ // Do not shorten Retry-After, or schedule a replacement outside the window.
247
+ if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
248
+ return exhausted("deadline");
249
+ }
133
250
  return {
134
251
  status: "recovering",
135
- continueDelayMs,
252
+ continueDelayMs: delayMs,
253
+ ...(overload ? { maxProviderRecoveryCount: limit } : {}),
136
254
  };
137
255
  }
138
256
 
257
+ /** A timer/queue delay cannot authorize a provider call after the recovery window. */
258
+ export class ProviderOverloadRecoveryExpiredError extends Error {
259
+ readonly failure;
260
+
261
+ constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock") {
262
+ super("Confirmed provider overload recovery window expired before dispatch");
263
+ this.name = "ProviderOverloadRecoveryExpiredError";
264
+ this.failure = providerRecoveryExhaustedFailure(
265
+ {
266
+ error: this.message,
267
+ code: "provider_unavailable",
268
+ providerCondition: "overloaded" as const,
269
+ },
270
+ {
271
+ status: "exhausted",
272
+ // The queued replacement has not dispatched; count only prior retries.
273
+ providerRecoveryCount: Math.min(
274
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
275
+ Math.max(0, count - 1),
276
+ ),
277
+ maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
278
+ providerRecoveryExhaustedReason: reason,
279
+ },
280
+ );
281
+ }
282
+ }
283
+
284
+ export function assertProviderOverloadRecoveryActive(input: {
285
+ failureCode: unknown;
286
+ providerRecoveryCount: number;
287
+ recoveryStartedAt?: number | undefined;
288
+ now?: number;
289
+ }): void {
290
+ if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
291
+ return;
292
+ }
293
+ if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
294
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
295
+ }
296
+ const now = input.now ?? Date.now();
297
+ const startedAt = input.recoveryStartedAt;
298
+ if (
299
+ startedAt === undefined ||
300
+ !Number.isFinite(startedAt) ||
301
+ !Number.isFinite(now) ||
302
+ startedAt > now
303
+ ) {
304
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
305
+ }
306
+ if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
307
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
308
+ }
309
+ }
310
+
139
311
  export function providerRecoveryExhaustedFailure<
140
312
  T extends Record<string, unknown> & { error: string },
141
313
  >(
@@ -147,18 +319,89 @@ export function providerRecoveryExhaustedFailure<
147
319
  providerRecoveryCount: number;
148
320
  maxProviderRecoveryCount: number;
149
321
  lastRetryableError: string;
322
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
150
323
  } {
151
324
  return {
152
325
  ...failure,
153
- error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
326
+ error: providerRecoveryExhaustedMessage({
327
+ code: typeof failure.code === "string" ? failure.code : null,
328
+ providerCondition: isProviderCondition(failure.providerCondition)
329
+ ? failure.providerCondition
330
+ : null,
331
+ modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
332
+ providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
333
+ providerRecoveryCount: recovery.providerRecoveryCount,
334
+ }),
154
335
  retryable: false,
155
336
  recoveryExhausted: true,
156
337
  providerRecoveryCount: recovery.providerRecoveryCount,
157
338
  maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
339
+ ...(recovery.providerRecoveryExhaustedReason
340
+ ? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason }
341
+ : {}),
342
+ lastRetryableError: failure.error,
343
+ };
344
+ }
345
+
346
+ function isProviderCondition(value: unknown): value is ProviderCondition {
347
+ return (
348
+ value === "overloaded" ||
349
+ value === "unavailable" ||
350
+ value === "unresponsive" ||
351
+ value === "rate_limited"
352
+ );
353
+ }
354
+
355
+ /**
356
+ * A retryable failure whose automatic same-turn recovery was not authorized
357
+ * (for example, request custody could not be checkpointed). The classifier's
358
+ * "will retry" sentence would be false on a terminal record, so name the
359
+ * condition and the person's options instead.
360
+ */
361
+ export function automaticRecoveryUnavailableFailure<
362
+ T extends Record<string, unknown> & { error: string },
363
+ >(
364
+ failure: T,
365
+ ): T & {
366
+ retryable: false;
367
+ automaticRecovery: "unavailable";
368
+ lastRetryableError: string;
369
+ } {
370
+ const subject = providerRecoverySubject({
371
+ code: typeof failure.code === "string" ? failure.code : null,
372
+ providerCondition: isProviderCondition(failure.providerCondition)
373
+ ? failure.providerCondition
374
+ : null,
375
+ modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
376
+ providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
377
+ });
378
+ return {
379
+ ...failure,
380
+ error: `${subject}. Opengeni couldn't retry this turn automatically. Try again, or send a message to continue.`,
381
+ retryable: false,
382
+ automaticRecovery: "unavailable",
158
383
  lastRetryableError: failure.error,
159
384
  };
160
385
  }
161
386
 
387
+ /**
388
+ * Name the exact accepted model route on model-provider recovery evidence so a
389
+ * person learns which model is affected. Display labels only: the route's
390
+ * credentials, base URL and upstream ids never enter the event.
391
+ */
392
+ export function withModelRoutePresentation<T extends { code?: string | undefined }>(
393
+ failure: T,
394
+ route: { model: string; modelLabel: string; providerLabel: string } | undefined,
395
+ ): T & { model?: string; modelLabel?: string; providerLabel?: string } {
396
+ if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
397
+ return {
398
+ ...failure,
399
+ model: route.model,
400
+ modelLabel: route.modelLabel,
401
+ providerLabel: route.providerLabel,
402
+ };
403
+ }
404
+
162
405
  export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
163
406
  const value = metadata.providerRecoveryCount;
164
407
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
@@ -194,6 +437,13 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
194
437
  value.error && typeof value.error === "object"
195
438
  ? (value.error as Record<string, unknown>)
196
439
  : null;
440
+ const milliseconds = Number(
441
+ headerValue(value.headers, "retry-after-ms") ??
442
+ headerValue(value.responseHeaders, "retry-after-ms") ??
443
+ headerValue(body?.headers, "retry-after-ms") ??
444
+ undefined,
445
+ );
446
+ if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
197
447
  const directSeconds = Number(
198
448
  value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
199
449
  );
@@ -317,20 +567,199 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
317
567
  });
318
568
  }
319
569
 
570
+ /** Complete structured-cause graph or no recovery authority. Budget distinct
571
+ * objects (including array containers), not queue positions/duplicate refs.
572
+ * The separate link ceiling bounds huge duplicate arrays without truncating
573
+ * them into permission. Cycles are harmless; overflow/unreadable edges hold. */
574
+ function structuredRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
575
+ const graph = new Map<object, Set<object>>();
576
+ const queue: object[] = [];
577
+ let links = 0;
578
+ const add = (value: unknown): boolean => {
579
+ if (!value || typeof value !== "object" || graph.has(value)) return true;
580
+ if (graph.size >= 64) return false;
581
+ graph.set(value, new Set());
582
+ queue.push(value);
583
+ return true;
584
+ };
585
+ if (!add(error)) return null;
586
+ try {
587
+ for (const current of queue) {
588
+ const record = current as Record<string, unknown>;
589
+ const children = Array.isArray(current)
590
+ ? current
591
+ : ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
592
+ for (const child of children) {
593
+ if (++links > 4096 || !add(child)) return null;
594
+ if (child && typeof child === "object") graph.get(current)!.add(child);
595
+ }
596
+ }
597
+ } catch {
598
+ return null;
599
+ }
600
+ return graph;
601
+ }
602
+
320
603
  function retryableDatabaseFailureCode(
321
604
  error: unknown,
605
+ requireDatabaseProvenance = false,
322
606
  ): PostClaimDatabaseRecoveryDetail["code"] | null {
323
- const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
324
- const sqlState = persistenceFailure?.details.sqlState ?? null;
325
- if (sqlState === null && isRetryableDatabaseTransportFailure(error)) {
326
- return "db_failure";
607
+ try {
608
+ const graph = structuredRecoveryCauseGraph(error);
609
+ if (!graph) return null;
610
+ const transports = new Set<object>();
611
+ const boundaries = new Set<object>();
612
+ const ownDatabaseNodes = new Set<object>();
613
+ const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
614
+ const ownChildren = (node: object): object[] =>
615
+ node instanceof DatabaseTransactionError
616
+ ? // Callback failures retained beside a failed rollback supply vetoes,
617
+ // never driver provenance for a provider's connection-looking error.
618
+ [...graph.get(node)!].filter((child) => child === node.cause)
619
+ : [...graph.get(node)!];
620
+ // Ask the canonical transport predicate about ONLY this node's facts. Its
621
+ // recursive search must not pair a DB sibling with an unrelated provider.
622
+ for (const node of graph.keys()) {
623
+ if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
624
+ return null;
625
+ const record = node as Record<string, unknown>;
626
+ if (
627
+ requireDatabaseProvenance
628
+ ? isRunningTurnDatabaseTransportFailure(record)
629
+ : isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno })
630
+ )
631
+ transports.add(node);
632
+ const sqlState = isSessionEventPersistenceError(node)
633
+ ? node.details.sqlState
634
+ : record.name === "PostgresError" && typeof record.code === "string"
635
+ ? record.code
636
+ : null;
637
+ if (
638
+ requireDatabaseProvenance
639
+ ? isRunningTurnDatabaseConnectionSqlState(sqlState)
640
+ : isDatabaseConnectionSqlState(sqlState)
641
+ )
642
+ transports.add(node);
643
+ if (
644
+ node instanceof DrizzleQueryError ||
645
+ node instanceof DatabaseTransactionError ||
646
+ isSessionEventPersistenceError(node)
647
+ ) {
648
+ // Only actual errors raised at our ORM/typed persistence boundary own
649
+ // their driver subtree. A PostgresError name, SDK wrapper or provider
650
+ // socket by itself is never own-client provenance for a running turn.
651
+ const queue: object[] = [node];
652
+ for (const source of queue) {
653
+ if (ownDatabaseNodes.has(source)) continue;
654
+ ownDatabaseNodes.add(source);
655
+ queue.push(...ownChildren(source));
656
+ }
657
+ }
658
+ }
659
+ const hasOwnTransport = (boundary: object): boolean => {
660
+ const seen = new Set<object>();
661
+ const queue = [boundary];
662
+ for (const node of queue) {
663
+ if (seen.has(node)) continue;
664
+ seen.add(node);
665
+ if (transports.has(node)) return true;
666
+ queue.push(...ownChildren(node));
667
+ }
668
+ return false;
669
+ };
670
+ // Inspect ALL trusted DB evidence before returning a permit. A permanent,
671
+ // auth or uncertain SQLSTATE vetoes every sibling regardless of traversal
672
+ // order; a deeper transport/reset cannot override it either.
673
+ for (const node of graph.keys()) {
674
+ const record = node as Record<string, unknown>;
675
+ if (node instanceof DrizzleQueryError || node instanceof DatabaseTransactionError) {
676
+ boundaries.add(node);
677
+ if (hasOwnTransport(node)) codes.add("db_failure");
678
+ }
679
+ const typed = isSessionEventPersistenceError(node);
680
+ if (!typed && record.name !== "PostgresError") continue;
681
+ boundaries.add(node);
682
+ const sqlState = typed
683
+ ? node.details.sqlState
684
+ : typeof record.code === "string"
685
+ ? record.code
686
+ : null;
687
+ const connectionOutage = requireDatabaseProvenance
688
+ ? isRunningTurnDatabaseConnectionSqlState(sqlState)
689
+ : isDatabaseConnectionSqlState(sqlState);
690
+ // PostgreSQL aborts a deadlock or serialization victim's whole
691
+ // transaction: unlike a lost connection, that write certainly did not
692
+ // commit, so it is at least as safe for exact-attempt recovery.
693
+ const rolledBack = isRetryablePersistenceSqlState(sqlState);
694
+ const code = typed
695
+ ? retryablePersistenceFailureCode(sqlState)
696
+ : connectionOutage || rolledBack
697
+ ? databaseFailureCode(sqlState)
698
+ : null;
699
+ if (
700
+ !code ||
701
+ (requireDatabaseProvenance &&
702
+ !connectionOutage &&
703
+ !rolledBack &&
704
+ !(sqlState === null && hasOwnTransport(node)))
705
+ )
706
+ return null;
707
+ if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
708
+ }
709
+ // Preserve the legacy pre-execution transport-only allowance, but never
710
+ // borrow it across an explicit DB boundary with no eligible own evidence.
711
+ if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
712
+ codes.add("db_failure");
713
+ // Stable classification for multiple positive DB siblings too.
714
+ for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"] as const)
715
+ if (codes.has(code)) return code;
716
+ return null;
717
+ } catch {
718
+ // Unreadable structured facts are no more authority than unreadable edges.
719
+ return null;
327
720
  }
721
+ }
722
+
723
+ const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
724
+ // postgres.js reports these for a physically lost connection, including
725
+ // transaction cleanup after its socket has already closed. Own-client
726
+ // provenance and the unknown-outcome veto remain mandatory above.
727
+ "CONNECTION_CLOSED",
728
+ "CONNECTION_DESTROYED",
729
+ "CONNECTION_ENDED",
730
+ "ECONNREFUSED",
731
+ "ECONNRESET",
732
+ "CONNECT_TIMEOUT",
733
+ ]);
734
+
735
+ function isRunningTurnDatabaseTransportFailure(record: Record<string, unknown>): boolean {
736
+ return [record.code, record.errno].some(
737
+ (code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code),
738
+ );
739
+ }
740
+
741
+ function isRunningTurnDatabaseConnectionSqlState(sqlState: string | null): boolean {
742
+ return (
743
+ sqlState !== null &&
744
+ (/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState))
745
+ );
746
+ }
747
+
748
+ function isDatabaseConnectionSqlState(sqlState: string | null): boolean {
749
+ return (
750
+ sqlState !== null &&
751
+ (sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState))
752
+ );
753
+ }
754
+
755
+ function retryablePersistenceFailureCode(
756
+ sqlState: string | null,
757
+ ): PostClaimDatabaseRecoveryDetail["code"] | null {
328
758
  if (
329
- !persistenceFailure ||
330
759
  !(
331
760
  sqlState === null ||
332
761
  sqlState.startsWith("08") ||
333
- sqlState.startsWith("40") ||
762
+ isRetryablePersistenceSqlState(sqlState) ||
334
763
  sqlState.startsWith("53") ||
335
764
  sqlState === "55P03" ||
336
765
  sqlState === "57014" ||
@@ -342,32 +771,59 @@ function retryableDatabaseFailureCode(
342
771
  ) {
343
772
  return null;
344
773
  }
345
- return persistenceFailure.details.code;
774
+ return databaseFailureCode(sqlState);
346
775
  }
347
776
 
348
777
  /**
349
- * Carry one exact claimed-but-not-started attempt into the workflow's DB-only
350
- * recovery lane. Permanent database/state failures remain terminal; only the
351
- * same operational outage classes that are safe before claim are admitted.
778
+ * True when an error is a structured database failure that the exact-attempt
779
+ * database recovery path owns. Callers that translate other failures into
780
+ * user-visible states must rethrow these unchanged.
781
+ */
782
+ export function isPostClaimDatabaseRecoveryCandidate(error: unknown): boolean {
783
+ return retryableDatabaseFailureCode(error, true) !== null;
784
+ }
785
+
786
+ /**
787
+ * Carry one exact claimed attempt into the workflow's DB-only
788
+ * recovery lane. Permanent database/state failures remain terminal. The
789
+ * running-turn lane additionally requires a closed own-client outage class;
790
+ * existing pre-execution recovery classifications are unchanged.
352
791
  */
353
792
  export function postClaimDatabaseRecoveryFailure(input: {
354
793
  error: unknown;
355
794
  turnId: string;
356
795
  triggerEventId: string;
357
796
  executionGeneration: number;
797
+ /** Require an actual own-client boundary and the closed running-turn allowlist. */
798
+ requireDatabaseProvenance?: boolean;
799
+ sandboxSetupOutcomeUnknown?: true;
800
+ sandboxSetupRecoveryExhausted?: true;
358
801
  providerRecovery?: {
359
802
  failureCode: string;
360
803
  providerRecoveryCount: number;
804
+ continueDelayMs?: number;
361
805
  };
362
806
  }): ApplicationFailure | null {
363
- const code = retryableDatabaseFailureCode(input.error);
807
+ const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
364
808
  if (!code || input.executionGeneration < 1) return null;
809
+ if (
810
+ (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
811
+ ((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
812
+ input.providerRecovery)
813
+ ) {
814
+ return null;
815
+ }
365
816
  if (
366
817
  input.providerRecovery &&
367
818
  (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
368
819
  input.providerRecovery.providerRecoveryCount <= 0 ||
369
- input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
370
- !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))
820
+ input.providerRecovery.providerRecoveryCount >
821
+ providerRecoveryLimit(input.providerRecovery.failureCode) ||
822
+ !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) ||
823
+ !validProviderOverloadRecoveryDelay(
824
+ input.providerRecovery.failureCode,
825
+ input.providerRecovery.continueDelayMs,
826
+ ))
371
827
  ) {
372
828
  return null;
373
829
  }
@@ -376,13 +832,21 @@ export function postClaimDatabaseRecoveryFailure(input: {
376
832
  triggerEventId: input.triggerEventId,
377
833
  executionGeneration: input.executionGeneration,
378
834
  code,
835
+ ...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
836
+ ...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
379
837
  ...(input.providerRecovery
380
838
  ? {
381
839
  providerFailureCode: input.providerRecovery.failureCode,
382
840
  providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
841
+ ...(input.providerRecovery.continueDelayMs !== undefined
842
+ ? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs }
843
+ : {}),
383
844
  }
384
845
  : {}),
385
846
  };
847
+ // Known usage whose receipt write failed travels with the exact attempt.
848
+ const sessionUsageReceipts = heldSessionUsageReceiptsForTurn(input.turnId);
849
+ if (sessionUsageReceipts.length > 0) detail.sessionUsageReceipts = sessionUsageReceipts;
386
850
  return ApplicationFailure.create({
387
851
  message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
388
852
  type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
@@ -462,8 +926,43 @@ export function sandboxLifecycleTransitionDiagnostic(
462
926
  return null;
463
927
  }
464
928
 
929
+ export function modelPreparationFailureEventPayload(error: unknown, durationMs: number) {
930
+ const transition = sandboxLifecycleTransitionDiagnostic(error);
931
+ return {
932
+ phase: "model_preparation",
933
+ durationMs: Math.max(0, Math.round(durationMs)),
934
+ expectedTransition: transition !== null,
935
+ ...(transition
936
+ ? {
937
+ failureCategory: "drain_capture_wait",
938
+ failureStage: "lifecycle_wait",
939
+ failureCode: transition.reason,
940
+ retryable: true,
941
+ }
942
+ : {}),
943
+ };
944
+ }
945
+
946
+ /**
947
+ * The turn's first model request could not be dispatched until the context
948
+ * was compacted (proactive threshold or an operator-requested Compact). That
949
+ * is an expected step of preparation, not a failure: the same activity
950
+ * compacts and then reopens preparation for the retried request.
951
+ */
952
+ export function compactionFirstPreparationEventPayload(durationMs: number) {
953
+ return {
954
+ phase: "model_preparation",
955
+ durationMs: Math.max(0, Math.round(durationMs)),
956
+ expectedTransition: true,
957
+ failureCategory: "context_compaction",
958
+ failureStage: "context_compaction",
959
+ failureCode: "compaction_required",
960
+ retryable: true,
961
+ } as const;
962
+ }
963
+
465
964
  /**
466
- * Recognize the one active-route transition that cannot finish inside its
965
+ * Recognize active-route transitions that cannot finish inside their
467
966
  * originating attempt. A Modal-home session may start on a Connected Machine
468
967
  * without creating or leasing its managed home box. When an explicit attach
469
968
  * clears the active pointer back to home, the pointer commit is authoritative,
@@ -474,9 +973,56 @@ export function sandboxLifecycleTransitionDiagnostic(
474
973
  * only those structural error links with a strict bound; never classify from
475
974
  * message text, which could originate in model or tool content.
476
975
  */
976
+ /**
977
+ * Only the home resolver's positive pre-dispatch signal permits a fresh attempt.
978
+ * The same error class can also escape AFTER a provider mutation, so neither its
979
+ * retryable property nor message alone establishes replay safety. Inspect the
980
+ * complete bounded SDK cause graph and let uncertain peer outcomes veto this
981
+ * narrow recovery; unreadable or overflowing graphs fail closed.
982
+ */
983
+ function isPreDispatchHomeBackendRecoveryRequired(error: unknown): boolean {
984
+ const graph = structuredRecoveryCauseGraph(error);
985
+ if (!graph) return false;
986
+ let found = false;
987
+ try {
988
+ for (const node of graph.keys()) {
989
+ if (
990
+ isRoutingMutationOutcomeUnknownError(node) ||
991
+ isRoutingMutationOutputRejectedError(node) ||
992
+ isRawProviderCommandOutcomeUnknown(node) ||
993
+ isModalCommandStartOutcomeUnknownError(node) ||
994
+ isProviderCommandObservationUnavailableError(node)
995
+ )
996
+ return false;
997
+ if (node instanceof RoutingBackendRecoveryRequiredError) {
998
+ if (
999
+ node.op !== "resolve_home_backend" ||
1000
+ !Number.isSafeInteger(node.leaseEpoch) ||
1001
+ node.leaseEpoch < 0 ||
1002
+ !node.retryable ||
1003
+ (node.recovery !== "pending" && node.recovery !== "superseded")
1004
+ )
1005
+ return false;
1006
+ found = true;
1007
+ }
1008
+ }
1009
+ } catch {
1010
+ return false;
1011
+ }
1012
+ return found;
1013
+ }
1014
+
477
1015
  export function sandboxRouteTransitionCode(
478
1016
  error: unknown,
479
- ): "home_unavailable_this_turn" | "workspace_root_changed_this_turn" | null {
1017
+ ):
1018
+ | "home_unavailable_this_turn"
1019
+ | "workspace_root_changed_this_turn"
1020
+ | "native_capabilities_changed_this_attempt"
1021
+ | "home_backend_recovery_pending"
1022
+ | null {
1023
+ if (isPreDispatchHomeBackendRecoveryRequired(error)) {
1024
+ return "home_backend_recovery_pending";
1025
+ }
480
1026
  const pending: unknown[] = [error];
481
1027
  const seen = new WeakSet<object>();
482
1028
  let inspected = 0;
@@ -489,6 +1035,13 @@ export function sandboxRouteTransitionCode(
489
1035
 
490
1036
  try {
491
1037
  const record = current as Record<string, unknown>;
1038
+ if (
1039
+ (current instanceof SandboxCapabilitiesChangedError ||
1040
+ record.name === "SandboxCapabilitiesChangedError") &&
1041
+ record.code === "native_capabilities_changed_this_attempt"
1042
+ ) {
1043
+ return "native_capabilities_changed_this_attempt";
1044
+ }
492
1045
  if (
493
1046
  (current instanceof ActiveBackendUnresolvableError ||
494
1047
  record.name === "ActiveBackendUnresolvableError") &&
@@ -612,6 +1165,14 @@ export function compactionFailureReasonFromError(error: unknown): string {
612
1165
  `the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
613
1166
  );
614
1167
  }
1168
+ // An exhausted provider quota is not retried (see agentRunFailurePayload),
1169
+ // so name the refusal plainly instead of the raw diagnostic envelope.
1170
+ const quota = classifyProviderQuotaExhaustionError(error);
1171
+ if (quota) {
1172
+ return compactionFailureReason(
1173
+ `${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`,
1174
+ );
1175
+ }
615
1176
  if (
616
1177
  error instanceof CompactionProviderResponseError ||
617
1178
  error instanceof EmptyCompactionSummaryError
@@ -638,8 +1199,12 @@ export function compactionFailureTurnEventPayload(
638
1199
  recovery: "user_message";
639
1200
  compacted: false;
640
1201
  providerRejection?: CompactionProviderRejection;
1202
+ quotaScope?: ProviderQuotaScope;
641
1203
  } {
642
1204
  const rejection = compactionProviderRejection(error);
1205
+ // The same closed marker as a `provider_quota_exhausted` turn failure, so
1206
+ // clients can name the exhausted limit and offer another model here too.
1207
+ const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
643
1208
  return {
644
1209
  error: overrides.error ?? compactionFailureReasonFromError(error),
645
1210
  code: "context_compaction_failed",
@@ -647,6 +1212,7 @@ export function compactionFailureTurnEventPayload(
647
1212
  recovery: "user_message",
648
1213
  compacted: false,
649
1214
  ...(rejection ? { providerRejection: rejection } : {}),
1215
+ ...(quota ? { quotaScope: quota.scope } : {}),
650
1216
  };
651
1217
  }
652
1218
 
@@ -658,18 +1224,38 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
658
1224
 
659
1225
  export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
660
1226
  if (!(error instanceof CompactionProviderResponseError)) return false;
1227
+ // The worker's own included-usage refusal (usage exhausted or unverifiable)
1228
+ // happens before dispatch, so the checkpoint request was never sent. It
1229
+ // settles exactly like the same refusal on an ordinary request: recorded
1230
+ // against the account, then the same turn re-places or waits for capacity.
1231
+ if (findCodexCreditPolicyError(error)) return true;
661
1232
  if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
1233
+ // A Claude subscription refusal (rate limit or reconnect) on the checkpoint
1234
+ // request recovers exactly like one on an ordinary request: failure
1235
+ // settlement rotates to another account or waits for capacity on the same
1236
+ // logical turn. Its wording or retry hint never makes it a terminal quota.
1237
+ const claudeRefusal = isClaudeSubscriptionTransportError(error)
1238
+ ? classifyClaudeProviderError(error)
1239
+ : null;
1240
+ if (claudeRefusal?.kind === "unauthorized" || claudeRefusal?.kind === "rate_limited") return true;
662
1241
  // Codex may reject an opaque artifact it minted itself on the compaction
663
1242
  // request exactly as it can on an ordinary request. Failure settlement
664
1243
  // invalidates only the exact participating artifacts and recovers the same
665
1244
  // logical turn; when nothing can be invalidated it fails closed there.
666
1245
  if (classifyCodexEncryptedArtifactRejection(error)) return true;
1246
+ // A ChatGPT plan that no longer includes the model refuses the compaction
1247
+ // request exactly as it refuses an ordinary one (often with an empty 400).
1248
+ // Failure settlement re-checks the plan and, when it proves the loss,
1249
+ // excludes that account for the model and fails the same turn over; an
1250
+ // unexplained rejection still fails there with typed copy.
1251
+ if (classifyCodexEntitlementRejection(error)) return true;
667
1252
  return agentRunFailurePayload(error).retryable === true;
668
1253
  }
669
1254
 
670
1255
  export function classifyContextWindowOverflowError(
671
1256
  error: unknown,
672
1257
  ): { message: string; code?: string; detail?: string } | null {
1258
+ if (isProviderSafetyRefusal(error)) return null;
673
1259
  const fields = collectErrorStrings(error);
674
1260
  const matched = fields.find(
675
1261
  (value) =>
@@ -736,6 +1322,9 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
736
1322
  }
737
1323
  seen.add(value);
738
1324
  const out: string[] = [];
1325
+ // This diagnostic is already provider-owned and byte-bounded. Do not widen
1326
+ // generic detail traversal to arbitrary application payloads.
1327
+ if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
739
1328
  const record = value as Record<string, unknown>;
740
1329
  for (const key of ["message", "code", "type", "name", "param"]) {
741
1330
  const field = record[key];
@@ -792,15 +1381,47 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
792
1381
  */
793
1382
  export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
794
1383
  "unable to connect. is the computer able to access the url?";
1384
+ /** Bun's exact fetch text when an upstream drops an open connection. */
1385
+ export const STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE =
1386
+ "the socket connection was closed unexpectedly. for more information, pass `verbose: true` in the second argument to fetch()";
1387
+ const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES = new Set([
1388
+ STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE,
1389
+ STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE,
1390
+ ]);
795
1391
 
796
1392
  export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
797
- return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
1393
+ return STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES.has(message.trim().toLowerCase());
1394
+ }
1395
+
1396
+ const TRANSIENT_TRANSPORT_CODE = /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i;
1397
+
1398
+ /**
1399
+ * A network fault code anywhere on the Error.cause chain. SDK and runner
1400
+ * wrappers often keep the transport error only as `cause`, so the outermost
1401
+ * error's own `code` is not enough.
1402
+ */
1403
+ function transientTransportCode(error: unknown): string | undefined {
1404
+ let current = error;
1405
+ const seen = new Set<unknown>();
1406
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1407
+ if (seen.has(current)) break;
1408
+ seen.add(current);
1409
+ const value = current as { code?: unknown; cause?: unknown };
1410
+ if (typeof value.code === "string" && TRANSIENT_TRANSPORT_CODE.test(value.code)) {
1411
+ return value.code;
1412
+ }
1413
+ current = value.cause;
1414
+ }
1415
+ return undefined;
798
1416
  }
799
1417
 
800
1418
  function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
1419
+ if (error instanceof ResponsesStreamingTerminalError) {
1420
+ return error.category === "safety" ? error.detail : undefined;
1421
+ }
801
1422
  return collectErrorStrings(error).find(
802
1423
  (value) =>
803
- /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
1424
+ /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
804
1425
  value,
805
1426
  ) || /\bthis request was blocked by our safety systems\b/i.test(value),
806
1427
  );
@@ -810,16 +1431,48 @@ function isProviderSafetyRefusal(error: unknown): boolean {
810
1431
  return providerSafetyRefusalDiagnostic(error) !== undefined;
811
1432
  }
812
1433
 
1434
+ /** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
1435
+ function providerHttpStatus(error: unknown): number | undefined {
1436
+ let current = error;
1437
+ const seen = new Set<unknown>();
1438
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1439
+ if (seen.has(current)) break;
1440
+ seen.add(current);
1441
+ const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
1442
+ const status = Number(value.status ?? value.statusCode);
1443
+ if (Number.isInteger(status) && status >= 100 && status < 600) return status;
1444
+ current = value.cause;
1445
+ }
1446
+ return undefined;
1447
+ }
1448
+
1449
+ /**
1450
+ * A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
1451
+ * "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
1452
+ * Explicit cancellation (`AbortError`) is deliberately not matched.
1453
+ */
1454
+ export function isTransportTimeoutError(error: unknown): boolean {
1455
+ let current: unknown = error;
1456
+ const seen = new Set<unknown>();
1457
+ for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
1458
+ if (seen.has(current)) break;
1459
+ seen.add(current);
1460
+ const value = current as { name?: unknown; message?: unknown; cause?: unknown };
1461
+ if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
1462
+ return true;
1463
+ }
1464
+ current = value.cause;
1465
+ }
1466
+ return false;
1467
+ }
1468
+
813
1469
  export function isTransientProviderError(error: unknown): boolean {
1470
+ if (error instanceof ResponsesStreamingTerminalError) {
1471
+ return error.category === "unavailable";
1472
+ }
814
1473
  // A semantic refusal can arrive inside a 5xx transport envelope.
815
1474
  if (isProviderSafetyRefusal(error)) return false;
816
- const status =
817
- typeof error === "object" && error !== null
818
- ? Number(
819
- (error as { status?: unknown; statusCode?: unknown }).status ??
820
- (error as { statusCode?: unknown }).statusCode,
821
- )
822
- : undefined;
1475
+ const status = providerHttpStatus(error);
823
1476
  // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
824
1477
  // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
825
1478
  // a request fault that must NOT auto-retry — even if its body happens to read like
@@ -828,13 +1481,7 @@ export function isTransientProviderError(error: unknown): boolean {
828
1481
  if (status !== undefined && Number.isFinite(status)) {
829
1482
  return status >= 500 && status < 600;
830
1483
  }
831
- const code =
832
- typeof error === "object" && error !== null && "code" in error
833
- ? String((error as { code?: unknown }).code)
834
- : undefined;
835
- if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
836
- return true;
837
- }
1484
+ if (transientTransportCode(error)) return true;
838
1485
  const message = error instanceof Error ? error.message : String(error);
839
1486
  if (isExactStatuslessUpstreamConnectivityMessage(message)) {
840
1487
  return true;
@@ -844,23 +1491,58 @@ export function isTransientProviderError(error: unknown): boolean {
844
1491
  );
845
1492
  }
846
1493
 
847
- export type XaiCredentialFailure = {
848
- kind: "auth" | "forbidden" | "rate_limit";
849
- cooldownMs: number | null;
850
- };
1494
+ /** An explicit `usage_limit_reached` type or code string anywhere on the error chain. */
1495
+ function hasCodexUsageLimitType(error: unknown): boolean {
1496
+ return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
1497
+ }
851
1498
 
852
1499
  /**
853
- * Only definitive SuperGrok account refusals may move the same logical turn to
854
- * another credential. A marked HTTP 401/403/429, or an HTTP 200 SSE terminal
855
- * that is a rate-limit/capacity diagnostic, proves inference was refused
856
- * without an accepted model response; refresh relogin is equally definitive.
1500
+ * Recognize an exhausted API-key provider quota (a daily or monthly allowance,
1501
+ * a free-tier day cap, or an account out of credits) as distinct from an
1502
+ * ordinary per-minute rate limit. Retrying within the bounded same-turn budget
1503
+ * cannot succeed, so the turn fails promptly instead. Subscription transports
1504
+ * own their quota semantics through credential rotation and durable capacity
1505
+ * waits, so a Codex, SuperGrok or Claude subscription transport error never
1506
+ * classifies here.
857
1507
  */
858
- export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
1508
+ export function classifyProviderQuotaExhaustionError(
1509
+ error: unknown,
1510
+ ): ProviderQuotaExhaustion | null {
1511
+ if (
1512
+ isCodexTransportError(error) ||
1513
+ isXaiSubscriptionTransportError(error) ||
1514
+ isClaudeSubscriptionTransportError(error)
1515
+ )
1516
+ return null;
1517
+ // The same reader the OpenAI SDK retry veto uses, so the two never disagree.
1518
+ return classifyProviderQuotaError(error);
1519
+ }
1520
+
1521
+ /** A refusal raised by a Claude subscription (OAuth) transport, at any wrapper depth. */
1522
+ export function isClaudeSubscriptionTransportError(error: unknown): boolean {
1523
+ let current: unknown = error;
1524
+ for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
1525
+ if (current instanceof AnthropicRequestError && current.subscription) return true;
1526
+ current = current.cause;
1527
+ }
1528
+ return false;
1529
+ }
1530
+
1531
+ /**
1532
+ * SuperGrok's classification of a failed model request as a shared provider
1533
+ * outcome (the adapter's `classifyError`, SuperGrok track X1b): a refresh
1534
+ * relogin or a marked 401 is `unauthorized`, a marked 403 `forbidden`, and a
1535
+ * marked 429 or HTTP 200 SSE capacity terminal `rate_limited` with the
1536
+ * provider's retry time (null when it gave none: the adapter's fallback
1537
+ * applies). Anything else, a provider safety refusal included, is null: not
1538
+ * a refusal of the account.
1539
+ */
1540
+ export function classifyXaiProviderError(error: unknown): ProviderErrorOutcome | null {
859
1541
  if (isProviderSafetyRefusal(error)) return null;
860
1542
  let relogin: unknown = error;
861
1543
  for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
862
1544
  if (relogin instanceof XaiSubscriptionReloginRequired) {
863
- return { kind: "auth", cooldownMs: null };
1545
+ return { kind: "unauthorized" };
864
1546
  }
865
1547
  relogin = (relogin as Record<string, unknown>).cause;
866
1548
  }
@@ -875,10 +1557,10 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
875
1557
  const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
876
1558
  const code = String(value.code ?? body?.code ?? "").toLowerCase();
877
1559
  if (status === 401 || code === "unauthorized" || code === "invalid_token") {
878
- return { kind: "auth", cooldownMs: null };
1560
+ return { kind: "unauthorized" };
879
1561
  }
880
1562
  if (status === 403) {
881
- return { kind: "forbidden", cooldownMs: null };
1563
+ return { kind: "forbidden" };
882
1564
  }
883
1565
  const message = String(value.message ?? body?.message ?? "");
884
1566
  if (
@@ -888,10 +1570,7 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
888
1570
  status: Number.isInteger(status) ? status : null,
889
1571
  })
890
1572
  ) {
891
- return {
892
- kind: "rate_limit",
893
- cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
894
- };
1573
+ return { kind: "rate_limited", retryAfterMs: providerRetryAfterMs(error) };
895
1574
  }
896
1575
  current = value.cause;
897
1576
  }
@@ -918,15 +1597,167 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
918
1597
  return null;
919
1598
  }
920
1599
 
1600
+ function isRawDatabaseQueryError(error: unknown): boolean {
1601
+ return (
1602
+ error instanceof Error &&
1603
+ "query" in error &&
1604
+ typeof error.query === "string" &&
1605
+ "params" in error &&
1606
+ Array.isArray(error.params)
1607
+ );
1608
+ }
1609
+
1610
+ function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
1611
+ if (error instanceof AnthropicRequestError) return error;
1612
+ return error instanceof Error && error.cause instanceof AnthropicRequestError
1613
+ ? error.cause
1614
+ : undefined;
1615
+ }
1616
+
1617
+ function isRawProviderCommandOutcomeUnknown(error: unknown): boolean {
1618
+ try {
1619
+ return (
1620
+ error instanceof ProviderCommandInputOutcomeUnknownError ||
1621
+ error instanceof ProviderCommandStartOutcomeUnknownError
1622
+ );
1623
+ } catch {
1624
+ return false;
1625
+ }
1626
+ }
1627
+
921
1628
  export function agentRunFailurePayload(
922
1629
  error: unknown,
923
1630
  options: { isCodexTurn?: boolean } = {},
924
1631
  ): ReturnType<typeof baseAgentRunFailurePayload> {
1632
+ return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
1633
+ }
1634
+
1635
+ /**
1636
+ * Record the closed provider condition on a retryable model-provider failure so
1637
+ * clients can say "overloaded" rather than a generic outage. Presentation
1638
+ * evidence only: retry authority and pacing are decided by `code`.
1639
+ */
1640
+ function withProviderCondition(
1641
+ error: unknown,
1642
+ failure: ReturnType<typeof baseAgentRunFailurePayload>,
1643
+ ): ReturnType<typeof baseAgentRunFailurePayload> {
1644
+ if (failure.code === "provider_rate_limited") {
1645
+ return { ...failure, providerCondition: "rate_limited" };
1646
+ }
1647
+ if (failure.code !== "provider_unavailable") return failure;
1648
+ if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
1649
+ return {
1650
+ ...failure,
1651
+ providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable",
1652
+ };
1653
+ }
1654
+
1655
+ /** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
1656
+ export function isProviderOverloadError(
1657
+ error: unknown,
1658
+ failure: { error?: string; detail?: string } = {},
1659
+ ): boolean {
1660
+ if (providerHttpStatus(error) === 529) return true;
1661
+ const anthropic = anthropicRequestDiagnostic(error);
1662
+ return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
1663
+ (value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value),
1664
+ );
1665
+ }
1666
+
1667
+ /** Retry authority must not depend on the looser presentation-only overload label. */
1668
+ export function providerRecoveryCode(
1669
+ error: unknown,
1670
+ failure: { code?: string; retryable?: boolean },
1671
+ ): string | undefined {
1672
+ if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
1673
+ const status = providerHttpStatus(error);
1674
+ let current = error;
1675
+ const seen = new Set<unknown>();
1676
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1677
+ if (seen.has(current)) break;
1678
+ seen.add(current);
1679
+ if (
1680
+ current instanceof AnthropicRequestError &&
1681
+ status !== undefined &&
1682
+ status >= 500 &&
1683
+ status < 600 &&
1684
+ (status === 529 || current.errorType === "overloaded_error")
1685
+ ) {
1686
+ return PROVIDER_OVERLOAD_RECOVERY_CODE;
1687
+ }
1688
+ current = (current as { cause?: unknown }).cause;
1689
+ }
1690
+ return failure.code;
1691
+ }
1692
+
1693
+ function classifyAgentRunFailurePayload(
1694
+ error: unknown,
1695
+ options: { isCodexTurn?: boolean } = {},
1696
+ ): ReturnType<typeof baseAgentRunFailurePayload> {
1697
+ const sizeError = findAnthropicRequestSizeError(error);
1698
+ if (sizeError)
1699
+ return {
1700
+ error: sizeError.message,
1701
+ code: sizeError.code,
1702
+ retryable: false,
1703
+ requestSize: sizeError.requestSize,
1704
+ ...(sizeError.request_id ? { requestId: sizeError.request_id } : {}),
1705
+ };
1706
+ const graph = structuredRecoveryCauseGraph(error);
1707
+ const nodes = graph ? [...graph.keys()] : [];
1708
+ const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
1709
+ if (outputRejected) {
1710
+ // A known receipt applies only to its own mutation. Uncertain peers retain
1711
+ // their established terminal classification, regardless of SDK graph order.
1712
+ const unknown = nodes.find(
1713
+ (node) =>
1714
+ isRoutingMutationOutcomeUnknownError(node) ||
1715
+ isModalCommandStartOutcomeUnknownError(node) ||
1716
+ isProviderCommandObservationUnavailableError(node) ||
1717
+ isRawProviderCommandOutcomeUnknown(node),
1718
+ );
1719
+ if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
1720
+ return {
1721
+ error: outputRejected.message,
1722
+ code: outputRejected.code,
1723
+ retryable: false,
1724
+ };
1725
+ }
1726
+ const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
1727
+ if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
925
1728
  const failure = baseAgentRunFailurePayload(error, options);
926
1729
  const diagnostic = materializationVerificationDiagnostic(error);
1730
+ const anthropic = anthropicRequestDiagnostic(error);
1731
+ if (anthropic) {
1732
+ const authenticationRejected =
1733
+ anthropic.status === 401 &&
1734
+ (error === anthropic ||
1735
+ (error instanceof Error && (error as Error & { status?: unknown }).status === 401));
1736
+ return {
1737
+ ...failure,
1738
+ code:
1739
+ failure.code ??
1740
+ (authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
1741
+ retryable: failure.retryable ?? false,
1742
+ ...(anthropic.detail ? { detail: anthropic.detail } : {}),
1743
+ ...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
1744
+ };
1745
+ }
927
1746
  return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
928
1747
  }
929
1748
 
1749
+ /** Keep Anthropic provider text on terminal failures, never recovery events. */
1750
+ export function agentRunRecoveryFailurePayload(
1751
+ error: unknown,
1752
+ failure: ReturnType<typeof agentRunFailurePayload>,
1753
+ ): ReturnType<typeof agentRunFailurePayload> {
1754
+ if (!anthropicRequestDiagnostic(error)) return failure;
1755
+ // Project a copy: retry exhaustion still needs the terminal diagnostic.
1756
+ const recovery = { ...failure };
1757
+ delete recovery.detail;
1758
+ return recovery;
1759
+ }
1760
+
930
1761
  function baseAgentRunFailurePayload(
931
1762
  error: unknown,
932
1763
  options: { isCodexTurn?: boolean } = {},
@@ -950,6 +1781,22 @@ function baseAgentRunFailurePayload(
950
1781
  historyPersistenceStage?: MandatoryHistoryPersistenceStage;
951
1782
  mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
952
1783
  materializationDiagnostic?: MaterializationVerificationDiagnostic;
1784
+ requestSize?: AnthropicRequestSize;
1785
+ quotaScope?: ProviderQuotaScope;
1786
+ /** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
1787
+ planType?: string | null;
1788
+ /** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
1789
+ model?: string | null;
1790
+ /** Display label of the turn's model on model-provider recovery evidence. */
1791
+ modelLabel?: string;
1792
+ /** Display label of the serving provider on model-provider recovery evidence. */
1793
+ providerLabel?: string;
1794
+ /** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
1795
+ providerCondition?: ProviderCondition;
1796
+ recoveryExhausted?: boolean;
1797
+ providerRecoveryCount?: number;
1798
+ maxProviderRecoveryCount?: number;
1799
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
953
1800
  } {
954
1801
  if (error instanceof SandboxMaterializationVerificationError) {
955
1802
  return {
@@ -963,7 +1810,11 @@ function baseAgentRunFailurePayload(
963
1810
  return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
964
1811
  }
965
1812
  if (error instanceof MandatoryHistoryPersistenceError) {
966
- const underlying = isSessionEventPersistenceError(error.cause)
1813
+ const databaseFailure =
1814
+ isSessionEventPersistenceError(error.cause) ||
1815
+ findPostgresDriverError(error.cause) !== null ||
1816
+ isRawDatabaseQueryError(error.cause);
1817
+ const underlying = databaseFailure
967
1818
  ? agentRunFailurePayload(error.cause, options)
968
1819
  : {
969
1820
  error: error.cause instanceof Error ? error.cause.message : String(error.cause),
@@ -973,6 +1824,32 @@ function baseAgentRunFailurePayload(
973
1824
  historyPersistenceStage: error.stage,
974
1825
  };
975
1826
  }
1827
+ // Raw ORM wrappers can contain the full SQL and its parameters. Classify a
1828
+ // real driver before provider message heuristics; the original cause stays
1829
+ // available to internal diagnostics. This payload grants no replay authority.
1830
+ const postgresDriverError = findPostgresDriverError(error);
1831
+ const rawOrmFailure = isRawDatabaseQueryError(error);
1832
+ if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
1833
+ const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
1834
+ const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
1835
+ return {
1836
+ error: "Opengeni encountered a database error.",
1837
+ code: databaseFailureCode(sqlState),
1838
+ sqlState,
1839
+ ...(Object.keys(database).length > 0 ? { database } : {}),
1840
+ };
1841
+ }
1842
+ if (error instanceof AnthropicSizeRecoveryExhaustedError) {
1843
+ return { error: error.message, code: error.code, retryable: false };
1844
+ }
1845
+ if (error instanceof AnthropicProviderRejection) {
1846
+ return {
1847
+ error: error.message,
1848
+ code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
1849
+ retryable: false,
1850
+ ...(error.request_id ? { requestId: error.request_id } : {}),
1851
+ };
1852
+ }
976
1853
  const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
977
1854
  if (safetyRefusalDiagnostic !== undefined) {
978
1855
  return {
@@ -983,14 +1860,43 @@ function baseAgentRunFailurePayload(
983
1860
  detail: safetyRefusalDiagnostic,
984
1861
  };
985
1862
  }
1863
+ if (error instanceof ResponsesStreamingTerminalError) {
1864
+ const quota =
1865
+ error.category === "unknown" || error.category === "rate_limit"
1866
+ ? classifyProviderQuotaExhaustionError({
1867
+ code: error.code,
1868
+ error: { code: error.code, type: error.type, message: error.detail },
1869
+ retryAfterSeconds: error.retryAfterSeconds,
1870
+ })
1871
+ : null;
1872
+ if (quota) {
1873
+ return {
1874
+ error: providerQuotaExhaustedMessage(quota.scope),
1875
+ code: PROVIDER_QUOTA_EXHAUSTED_CODE,
1876
+ retryable: false,
1877
+ quotaScope: quota.scope,
1878
+ detail: error.detail,
1879
+ };
1880
+ }
1881
+ return {
1882
+ error:
1883
+ error.category === "rate_limit"
1884
+ ? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort."
1885
+ : error.category === "unavailable"
1886
+ ? "The model provider is temporarily unavailable. The same turn will retry after a short delay."
1887
+ : "The model provider rejected the response. Automatic retries stopped.",
1888
+ code:
1889
+ error.category === "rate_limit"
1890
+ ? "provider_rate_limited"
1891
+ : error.category === "unavailable"
1892
+ ? "provider_unavailable"
1893
+ : "provider_request_rejected",
1894
+ retryable: error.category === "rate_limit" || error.category === "unavailable",
1895
+ detail: error.detail,
1896
+ };
1897
+ }
986
1898
  const message = error instanceof Error ? error.message : String(error);
987
- const status =
988
- typeof error === "object" && error !== null
989
- ? Number(
990
- (error as { status?: unknown; statusCode?: unknown }).status ??
991
- (error as { statusCode?: unknown }).statusCode,
992
- )
993
- : undefined;
1899
+ const status = providerHttpStatus(error);
994
1900
  const code =
995
1901
  typeof error === "object" && error !== null && "code" in error
996
1902
  ? String((error as { code?: unknown }).code)
@@ -1021,10 +1927,29 @@ function baseAgentRunFailurePayload(
1021
1927
  retryable: true,
1022
1928
  };
1023
1929
  }
1024
- if (isModalTaskExecStartDnsResolutionError(error)) {
1930
+ if (isProviderCommandObservationUnavailableError(error)) {
1025
1931
  return {
1026
1932
  error:
1027
- "The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
1933
+ "A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
1934
+ code: "sandbox_command_observation_unavailable",
1935
+ retryable: false,
1936
+ };
1937
+ }
1938
+ if (isModalCommandStartOutcomeUnknownError(error)) {
1939
+ return {
1940
+ error:
1941
+ "A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
1942
+ code: "sandbox_command_start_outcome_unknown",
1943
+ retryable: false,
1944
+ };
1945
+ }
1946
+ if (
1947
+ !isRoutingMutationOutcomeUnknownError(error) &&
1948
+ isModalTaskExecStartPreDispatchUnavailableError(error)
1949
+ ) {
1950
+ return {
1951
+ error:
1952
+ "The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
1028
1953
  code: "sandbox_command_start_unavailable",
1029
1954
  retryable: true,
1030
1955
  };
@@ -1089,6 +2014,33 @@ function baseAgentRunFailurePayload(
1089
2014
  ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
1090
2015
  };
1091
2016
  }
2017
+ // Codex plan entitlement: the settlement path normally re-checks the plan
2018
+ // and records a precise payload. These branches keep any other path from
2019
+ // surfacing the SDK's raw "400 status code (no body)" text.
2020
+ if (error instanceof CodexPlanEntitlementError) {
2021
+ return error.payload;
2022
+ }
2023
+ // Shared subscription core placement outcomes carry their own typed copy.
2024
+ if (error instanceof SubscriptionCoreCodexTurnError) {
2025
+ return error.payload;
2026
+ }
2027
+ const entitlementRejection = classifyCodexEntitlementRejection(error);
2028
+ if (entitlementRejection) {
2029
+ return entitlementRejection.evidence === "plan_entitlement"
2030
+ ? codexPlanEntitlementFailurePayload({
2031
+ accountLabel: null,
2032
+ planType: null,
2033
+ planChanged: false,
2034
+ modelId: null,
2035
+ rejection: entitlementRejection,
2036
+ })
2037
+ : codexRequestRejectedFailurePayload({
2038
+ accountLabel: null,
2039
+ planType: null,
2040
+ rejection: entitlementRejection,
2041
+ planChecked: false,
2042
+ });
2043
+ }
1092
2044
  // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
1093
2045
  // must NOT be reported as a generic, retryable rate-limit (which would loop a
1094
2046
  // goal against a capped backend). Surface a precise, actionable message with
@@ -1099,8 +2051,15 @@ function baseAgentRunFailurePayload(
1099
2051
  // `usage_limit_reached` shape must still outrank generic 429 retryability.
1100
2052
  // Credential quarantine/failover remains separately provenance-gated by
1101
2053
  // `isCodexTransportError`; this branch only chooses the truthful user payload.
2054
+ // The looser "429 ... usage limit" wording counts only on a Codex transport
2055
+ // error: an API-key provider's 429 that says "usage limit" is provider quota
2056
+ // evidence, not a ChatGPT/Codex subscription cap.
2057
+ const creditPolicy = findCodexCreditPolicyError(error);
2058
+ if (creditPolicy) {
2059
+ return { error: creditPolicy.message, code: creditPolicy.code, retryable: false };
2060
+ }
1102
2061
  const usageLimit = classifyCodexUsageLimitError(error);
1103
- if (usageLimit) {
2062
+ if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
1104
2063
  return codexUsageLimitFailurePayload(usageLimit, message);
1105
2064
  }
1106
2065
  const codexTimeout = classifyCodexResponseTimeoutError(error, {
@@ -1144,6 +2103,39 @@ function baseAgentRunFailurePayload(
1144
2103
  ...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
1145
2104
  };
1146
2105
  }
2106
+ // A generic model stream that went silent mid-response (typed idle bound) or
2107
+ // a transport timeout ("The operation timed out." DOMException TimeoutError
2108
+ // from the fetch layer) is a dead upstream connection, not a request fault:
2109
+ // checkpoint durable truth and recover the same turn within the finite
2110
+ // provider recovery budget. Codex/SuperGrok typed timeouts are classified
2111
+ // above under their own transport policy.
2112
+ const streamIdle = classifyModelStreamIdleTimeoutError(error);
2113
+ if (streamIdle) {
2114
+ const seconds =
2115
+ streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1_000) : null;
2116
+ const window = seconds !== null ? ` for ${seconds}s` : "";
2117
+ return {
2118
+ error:
2119
+ streamIdle.kind === "progress"
2120
+ ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.`
2121
+ : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
2122
+ code: "provider_unavailable",
2123
+ retryable: true,
2124
+ timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
2125
+ responseObserved: true,
2126
+ detail: streamIdle.message,
2127
+ };
2128
+ }
2129
+ if (status === undefined && isTransportTimeoutError(error)) {
2130
+ return {
2131
+ error:
2132
+ "The model provider connection timed out. The same turn will retry after a short delay.",
2133
+ code: "provider_unavailable",
2134
+ retryable: true,
2135
+ timeoutClass: "transport",
2136
+ detail: message,
2137
+ };
2138
+ }
1147
2139
  if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
1148
2140
  return {
1149
2141
  error:
@@ -1152,10 +2144,33 @@ function baseAgentRunFailurePayload(
1152
2144
  retryable: true,
1153
2145
  };
1154
2146
  }
2147
+ // An exhausted quota also arrives as HTTP 429 (or 402), but no retry within
2148
+ // the finite same-turn budget can succeed. Fail the turn promptly with a
2149
+ // distinct code so the client can offer another model; ordinary short rate
2150
+ // limits fall through to the retryable branch below.
2151
+ if (status === 402 && code === "anthropic_billing_error") {
2152
+ return {
2153
+ error: "Claude could not bill this request. Check the account's billing and payment details.",
2154
+ code: "provider_billing_error",
2155
+ retryable: false,
2156
+ ...(message ? { detail: message } : {}),
2157
+ };
2158
+ }
2159
+ const quota = classifyProviderQuotaExhaustionError(error);
2160
+ if (quota) {
2161
+ return {
2162
+ error: providerQuotaExhaustedMessage(quota.scope),
2163
+ code: PROVIDER_QUOTA_EXHAUSTED_CODE,
2164
+ retryable: false,
2165
+ quotaScope: quota.scope,
2166
+ ...(message ? { detail: message } : {}),
2167
+ };
2168
+ }
1155
2169
  if (
1156
2170
  status === 429 ||
1157
- code === "rate_limit_exceeded" ||
1158
- /(?:too many requests|rate.?limit|\b429\b)/i.test(message)
2171
+ ((status === undefined || !Number.isFinite(status)) &&
2172
+ (code === "rate_limit_exceeded" ||
2173
+ /(?:too many requests|rate.?limit|\b429\b)/i.test(message)))
1159
2174
  ) {
1160
2175
  return {
1161
2176
  error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
@@ -1172,31 +2187,24 @@ function baseAgentRunFailurePayload(
1172
2187
  if (isExactStatuslessUpstreamConnectivityMessage(message)) {
1173
2188
  return {
1174
2189
  error:
1175
- "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
2190
+ "Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
1176
2191
  code: "upstream_connectivity_unavailable",
1177
2192
  retryable: true,
1178
2193
  };
1179
2194
  }
1180
2195
  return { error: message, code: "provider_unavailable", retryable: true };
1181
2196
  }
1182
- const postgresDriverError = findPostgresDriverError(error);
1183
- if (postgresDriverError) {
1184
- const database = safeDatabaseErrorFacts(postgresDriverError);
1185
- const sqlState = nestedPostgresSqlState(postgresDriverError);
1186
- if (sqlState !== null || Object.keys(database).length > 0) {
1187
- return {
1188
- error: message,
1189
- sqlState,
1190
- ...(Object.keys(database).length > 0 ? { database } : {}),
1191
- };
1192
- }
1193
- }
1194
2197
  return { error: message };
1195
2198
  }
1196
2199
 
1197
2200
  export type CodexCredentialFailure = {
1198
- kind: "auth" | "forbidden" | "rate_limit" | "quota";
2201
+ /**
2202
+ * `plan_entitlement` is produced only after a plan re-check proves the
2203
+ * serving account's current plan does not include the requested model.
2204
+ */
2205
+ kind: "auth" | "forbidden" | "rate_limit" | "quota" | "plan_entitlement";
1199
2206
  cooldownSeconds: number | null;
2207
+ origin?: "included_usage_policy" | "usage_verification_policy";
1200
2208
  };
1201
2209
 
1202
2210
  export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
@@ -1216,7 +2224,11 @@ export function codexCredentialCooldownUntil(
1216
2224
  > | null,
1217
2225
  now: Date,
1218
2226
  ): Date | null {
1219
- if (failure.kind === "auth" || failure.kind === "forbidden") {
2227
+ if (
2228
+ failure.kind === "auth" ||
2229
+ failure.kind === "forbidden" ||
2230
+ failure.kind === "plan_entitlement"
2231
+ ) {
1220
2232
  return null;
1221
2233
  }
1222
2234
  const providerReset =
@@ -1257,6 +2269,19 @@ export function codexCredentialCooldownUntil(
1257
2269
  * progress and therefore MUST NOT walk the credential pool automatically.
1258
2270
  */
1259
2271
  export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
2272
+ const creditPolicy = findCodexCreditPolicyError(error);
2273
+ if (creditPolicy instanceof CodexIncludedUsageUnknownError) {
2274
+ // Verified pre-dispatch refusal. A short hard hold cannot authorize credit
2275
+ // fallback and does not claim that the provider exhausted its allowance.
2276
+ return { kind: "rate_limit", cooldownSeconds: 60, origin: "usage_verification_policy" };
2277
+ }
2278
+ if (creditPolicy instanceof CodexIncludedUsageExhaustedError) {
2279
+ return {
2280
+ kind: "quota",
2281
+ cooldownSeconds: creditPolicy.resetsInSeconds,
2282
+ origin: "included_usage_policy",
2283
+ };
2284
+ }
1260
2285
  // A request safety refusal is not evidence that another account should run it.
1261
2286
  if (isProviderSafetyRefusal(error)) return null;
1262
2287
  // A permanent OAuth refresh failure is definitive and the shared resolver has
@@ -1276,6 +2301,12 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
1276
2301
  if (!isCodexTransportError(error)) {
1277
2302
  return null;
1278
2303
  }
2304
+ // Plan/model entitlement evidence (an explicit plan refusal, or an empty
2305
+ // HTTP 400) is not account health or quota. The worker re-checks the plan
2306
+ // first; only a proven entitlement loss walks the pool, as `plan_entitlement`.
2307
+ if (classifyCodexEntitlementRejection(error)) {
2308
+ return null;
2309
+ }
1279
2310
  const usageLimit = classifyCodexUsageLimitError(error);
1280
2311
  if (usageLimit) {
1281
2312
  return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
@@ -1380,3 +2411,42 @@ export function codexUsageLimitFailurePayload(
1380
2411
  // open indefinitely for a goal-bearing session; cap the continuation hold so the
1381
2412
  // goal re-evaluates at most this far out (it will re-pause if still capped).
1382
2413
  export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
2414
+
2415
+ /**
2416
+ * Claude's classification of a failed model request as a shared provider
2417
+ * outcome (the adapter's `classifyError`, Claude track C1b), for a request
2418
+ * of the Claude subscription transport only: a reconnect-required refresh or
2419
+ * HTTP 401 is `unauthorized` (the transport already ran the turn's one
2420
+ * forced refresh); 429 is `rate_limited` for the turn's model `modelId`, with
2421
+ * the provider's retry time (null: the adapter's fallback applies); 529 or
2422
+ * `overloaded_error` is `overloaded`, which never moves the turn to another
2423
+ * account; 403 is `fatal`, which never rotates or quarantines the account.
2424
+ * Anything else, a provider safety refusal included, is null: not a refusal
2425
+ * of the account.
2426
+ */
2427
+ export function classifyClaudeProviderError(
2428
+ error: unknown,
2429
+ modelId?: string,
2430
+ ): ProviderErrorOutcome | null {
2431
+ if (isProviderSafetyRefusal(error)) return null;
2432
+ let current: unknown = error;
2433
+ for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
2434
+ if (current instanceof ClaudeSubscriptionReconnectRequired) return { kind: "unauthorized" };
2435
+ if (current instanceof AnthropicRequestError) {
2436
+ if (!current.subscription) return null;
2437
+ if (current.status === 401) return { kind: "unauthorized" };
2438
+ if (current.status === 429)
2439
+ return {
2440
+ kind: "rate_limited",
2441
+ retryAfterMs: providerRetryAfterMs(current),
2442
+ ...(modelId === undefined ? {} : { modelId }),
2443
+ };
2444
+ if (current.status === 529 || current.errorType === "overloaded_error")
2445
+ return { kind: "overloaded" };
2446
+ if (current.status === 403) return { kind: "fatal" };
2447
+ return null;
2448
+ }
2449
+ current = current.cause;
2450
+ }
2451
+ return null;
2452
+ }