cartethyia 2.7.1 → 2.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/.env.example +2 -0
  2. package/dist/dashboard/assets/{BannedPage-D-J75hBM.js → BannedPage-2TJD37Hb.js} +1 -1
  3. package/dist/dashboard/assets/BannedPage-2TJD37Hb.js.gz +0 -0
  4. package/dist/dashboard/assets/CliToolDetailPage-xefJNRmJ.js +1 -0
  5. package/dist/dashboard/assets/CliToolDetailPage-xefJNRmJ.js.gz +0 -0
  6. package/dist/dashboard/assets/CliToolsPage-Rj9iHQ3F.js +1 -0
  7. package/dist/dashboard/assets/CliToolsPage-Rj9iHQ3F.js.gz +0 -0
  8. package/dist/dashboard/assets/{CombosPage-QoJ6foxo.js → CombosPage-JEbYJFxw.js} +1 -1
  9. package/dist/dashboard/assets/CombosPage-JEbYJFxw.js.gz +0 -0
  10. package/dist/dashboard/assets/{ConsoleLogPage-DpNRay4C.js → ConsoleLogPage-C3xx_inM.js} +1 -1
  11. package/dist/dashboard/assets/ConsoleLogPage-C3xx_inM.js.gz +0 -0
  12. package/dist/dashboard/assets/CustomizationPage-Dw306gJU.js +1 -0
  13. package/dist/dashboard/assets/CustomizationPage-Dw306gJU.js.gz +0 -0
  14. package/dist/dashboard/assets/HealthEventsModal-jaBo1TWV.js +1 -0
  15. package/dist/dashboard/assets/HealthEventsModal-jaBo1TWV.js.gz +0 -0
  16. package/dist/dashboard/assets/{LoginPage-DubdQcvX.js → LoginPage-C4w7vea-.js} +1 -1
  17. package/dist/dashboard/assets/LoginPage-C4w7vea-.js.gz +0 -0
  18. package/dist/dashboard/assets/{ModelPicker-BH8Tehdi.js → ModelPicker-BtPcAfot.js} +1 -1
  19. package/dist/dashboard/assets/ModelPicker-BtPcAfot.js.gz +0 -0
  20. package/dist/dashboard/assets/OverviewPage-CGC5gNs1.js +1 -0
  21. package/dist/dashboard/assets/OverviewPage-CGC5gNs1.js.gz +0 -0
  22. package/dist/dashboard/assets/ProviderDetailPage-DQcthNGt.js +4 -0
  23. package/dist/dashboard/assets/ProviderDetailPage-DQcthNGt.js.gz +0 -0
  24. package/dist/dashboard/assets/ProvidersPage-0OsXRGXe.js +1 -0
  25. package/dist/dashboard/assets/ProvidersPage-0OsXRGXe.js.gz +0 -0
  26. package/dist/dashboard/assets/ProxyPage-DdPc8slb.js +9 -0
  27. package/dist/dashboard/assets/ProxyPage-DdPc8slb.js.gz +0 -0
  28. package/dist/dashboard/assets/{QuotaPage-ATpaxrbl.js → QuotaPage-Ci5O4BnH.js} +1 -1
  29. package/dist/dashboard/assets/QuotaPage-Ci5O4BnH.js.gz +0 -0
  30. package/dist/dashboard/assets/{SettingsPage-CETophpx.js → SettingsPage-CDZjTt87.js} +1 -1
  31. package/dist/dashboard/assets/SettingsPage-CDZjTt87.js.gz +0 -0
  32. package/dist/dashboard/assets/SetupPage-BQS0Rx_u.js +1 -0
  33. package/dist/dashboard/assets/SetupPage-BQS0Rx_u.js.gz +0 -0
  34. package/dist/dashboard/assets/{SortableList-CAK-EN9z.js → SortableList-BAL5npBC.js} +1 -1
  35. package/dist/dashboard/assets/SortableList-BAL5npBC.js.gz +0 -0
  36. package/dist/dashboard/assets/{StudioPage-B7W2_RhT.js → StudioPage-TP7voAQ0.js} +3 -3
  37. package/dist/dashboard/assets/StudioPage-TP7voAQ0.js.gz +0 -0
  38. package/dist/dashboard/assets/{UsagePage-6T4EFDjW.js → UsagePage-Cj_fEzhg.js} +13 -13
  39. package/dist/dashboard/assets/UsagePage-Cj_fEzhg.js.gz +0 -0
  40. package/dist/dashboard/assets/{api-keys-CUi6UXsR.js → api-keys-MbI7e7rt.js} +1 -1
  41. package/dist/dashboard/assets/api-keys-MbI7e7rt.js.gz +0 -0
  42. package/dist/dashboard/assets/app-B2vyPoII.css +2 -0
  43. package/dist/dashboard/assets/app-B2vyPoII.css.gz +0 -0
  44. package/dist/dashboard/assets/arrow-up-59ms1iiT.js +1 -0
  45. package/dist/dashboard/assets/arrow-up-59ms1iiT.js.gz +0 -0
  46. package/dist/dashboard/assets/{assert-67xXH5rf.js → assert-C0YYuZJ4.js} +1 -1
  47. package/dist/dashboard/assets/assert-C0YYuZJ4.js.gz +0 -0
  48. package/dist/dashboard/assets/{badge-DC2L43Mj.js → badge-B_DMpMJN.js} +1 -1
  49. package/dist/dashboard/assets/badge-B_DMpMJN.js.gz +0 -0
  50. package/dist/dashboard/assets/{banner-BBVcb0J3.js → banner-DMrMPmth.js} +1 -1
  51. package/dist/dashboard/assets/banner-DMrMPmth.js.gz +0 -0
  52. package/dist/dashboard/assets/{card-Cca1XC6G.js → card-Be5Uzamx.js} +1 -1
  53. package/dist/dashboard/assets/card-Be5Uzamx.js.gz +0 -0
  54. package/dist/dashboard/assets/{cli-tools-KvsWa1Jl.js → cli-tools-muFRphDu.js} +1 -1
  55. package/dist/dashboard/assets/cli-tools-muFRphDu.js.gz +0 -0
  56. package/dist/dashboard/assets/{clipboard-button-DUzEbynq.js → clipboard-button-DOfrVmsi.js} +1 -1
  57. package/dist/dashboard/assets/clipboard-button-DOfrVmsi.js.gz +0 -0
  58. package/dist/dashboard/assets/{dialog-DFvWhwTa.js → dialog-C9omdjQt.js} +1 -1
  59. package/dist/dashboard/assets/dialog-C9omdjQt.js.gz +0 -0
  60. package/dist/dashboard/assets/drawer-Bm8WKaGY.js +1 -0
  61. package/dist/dashboard/assets/drawer-Bm8WKaGY.js.gz +0 -0
  62. package/dist/dashboard/assets/{entry-_xHo5x6r.js → entry-DS90gUks.js} +1 -1
  63. package/dist/dashboard/assets/entry-DS90gUks.js.gz +0 -0
  64. package/dist/dashboard/assets/{entry-BLBLxX0h.js → entry-j40LJotO.js} +3 -3
  65. package/dist/dashboard/assets/entry-j40LJotO.js.gz +0 -0
  66. package/dist/dashboard/assets/{entry-Bqy8RFvM.js → entry-q5HwZ8nN.js} +1 -1
  67. package/dist/dashboard/assets/entry-q5HwZ8nN.js.gz +0 -0
  68. package/dist/dashboard/assets/{concurrency-BDF0ok7y.js → fingerprint-pattern-BWuHtbW4.js} +1 -1
  69. package/dist/dashboard/assets/fingerprint-pattern-BWuHtbW4.js.gz +0 -0
  70. package/dist/dashboard/assets/{formatters-mpBPqAQ8.js → formatters-BTScrQpe.js} +1 -1
  71. package/dist/dashboard/assets/formatters-BTScrQpe.js.gz +0 -0
  72. package/dist/dashboard/assets/{github-badge-C9e8Ts7e.js → github-badge-Orjuzemr.js} +1 -1
  73. package/dist/dashboard/assets/github-badge-Orjuzemr.js.gz +0 -0
  74. package/dist/dashboard/assets/{index-D6QzEC_n.js → index-cPrX5vn6.js} +2 -2
  75. package/dist/dashboard/assets/index-cPrX5vn6.js.gz +0 -0
  76. package/dist/dashboard/assets/{inline-BFUcvllj.js → inline-Dvux8w-0.js} +1 -1
  77. package/dist/dashboard/assets/inline-Dvux8w-0.js.gz +0 -0
  78. package/dist/dashboard/assets/{input-C_ILAvBb.js → input-CU1Rgf3p.js} +1 -1
  79. package/dist/dashboard/assets/input-CU1Rgf3p.js.gz +0 -0
  80. package/dist/dashboard/assets/{layout-CBIyT6Rt.js → layout-C_7A6H55.js} +1 -1
  81. package/dist/dashboard/assets/layout-C_7A6H55.js.gz +0 -0
  82. package/dist/dashboard/assets/{list-row-ebJ_-Nsb.js → list-row-CzFvogbj.js} +1 -1
  83. package/dist/dashboard/assets/list-row-CzFvogbj.js.gz +0 -0
  84. package/dist/dashboard/assets/{live-D0mm--ME.js → live-KaG15HLQ.js} +1 -1
  85. package/dist/dashboard/assets/live-KaG15HLQ.js.gz +0 -0
  86. package/dist/dashboard/assets/{mutation-sMD965oO.js → mutation-JJGQm8TD.js} +1 -1
  87. package/dist/dashboard/assets/mutation-JJGQm8TD.js.gz +0 -0
  88. package/dist/dashboard/assets/{numbers-BO9dKaFr.js → numbers-BNDZnsUE.js} +1 -1
  89. package/dist/dashboard/assets/numbers-BNDZnsUE.js.gz +0 -0
  90. package/dist/dashboard/assets/{query-keys-CfL_Zdoh.js → query-keys-CLaNz8Xz.js} +1 -1
  91. package/dist/dashboard/assets/query-keys-CLaNz8Xz.js.gz +0 -0
  92. package/dist/dashboard/assets/{routing-BnIpi1IJ.js → routing-BEH32QOC.js} +1 -1
  93. package/dist/dashboard/assets/routing-BEH32QOC.js.gz +0 -0
  94. package/dist/dashboard/assets/{select-Cj5Nw43g.js → select-B3iTetHN.js} +1 -1
  95. package/dist/dashboard/assets/select-B3iTetHN.js.gz +0 -0
  96. package/dist/dashboard/assets/settings-2-Ds9DQp0t.js +1 -0
  97. package/dist/dashboard/assets/settings-2-Ds9DQp0t.js.gz +0 -0
  98. package/dist/dashboard/assets/{settings-BAFhDgX8.js → settings-Dt3Eqbnq.js} +1 -1
  99. package/dist/dashboard/assets/settings-Dt3Eqbnq.js.gz +0 -0
  100. package/dist/dashboard/assets/{stack-EMrOMt4I.js → stack-KrOrm7JZ.js} +1 -1
  101. package/dist/dashboard/assets/stack-KrOrm7JZ.js.gz +0 -0
  102. package/dist/dashboard/assets/{state-CWoo7ty5.js → state-DLS7IDMy.js} +1 -1
  103. package/dist/dashboard/assets/state-DLS7IDMy.js.gz +0 -0
  104. package/dist/dashboard/assets/{switch-CCjT6IkF.js → switch-42ItvbAs.js} +1 -1
  105. package/dist/dashboard/assets/switch-42ItvbAs.js.gz +0 -0
  106. package/dist/dashboard/assets/{useMutation-DjkLHUcE.js → useMutation-Dmnfh1Ue.js} +1 -1
  107. package/dist/dashboard/assets/useMutation-Dmnfh1Ue.js.gz +0 -0
  108. package/dist/dashboard/index.html +3 -2
  109. package/dist/dashboard/index.html.gz +0 -0
  110. package/dist/dashboard/precompressed-assets.json +117 -115
  111. package/migrations/full-postgres/0026_add_qoder_native_identity.sql +20 -0
  112. package/migrations/full-postgres/0027_qoder_identity_genuine_seed.sql +22 -0
  113. package/migrations/full-postgres/0028_add_family_template_key_telemetry.sql +40 -0
  114. package/migrations/full-postgres/0029_null_share_child_policy_columns.sql +38 -0
  115. package/migrations/full-postgres/0030_add_api_key_max_children.sql +10 -0
  116. package/migrations/full-postgres/0031_tenant_system_prompts.sql +16 -0
  117. package/migrations/lite-sqlite3/0026_add_qoder_native_identity.sql +15 -0
  118. package/migrations/lite-sqlite3/0027_qoder_identity_genuine_seed.sql +22 -0
  119. package/migrations/lite-sqlite3/0028_add_family_template_key_telemetry.sql +40 -0
  120. package/migrations/lite-sqlite3/0029_null_share_child_policy_columns.sql +38 -0
  121. package/migrations/lite-sqlite3/0030_add_api_key_max_children.sql +10 -0
  122. package/migrations/lite-sqlite3/0031_tenant_system_prompts.sql +10 -0
  123. package/package.json +1 -1
  124. package/src/config/network.ts +1 -1
  125. package/src/console/api-keys/bucket-spend.ts +7 -7
  126. package/src/console/api-keys/contracts.ts +423 -410
  127. package/src/console/api-keys/routes.ts +118 -19
  128. package/src/console/backup/export-endpoint.ts +8 -11
  129. package/src/console/backup/routes.ts +40 -8
  130. package/src/console/backup/service.ts +11 -6
  131. package/src/console/backup/store.ts +384 -88
  132. package/src/console/cli-tools/contracts.ts +10 -4
  133. package/src/console/dashboard-assets.ts +16 -1
  134. package/src/console/domain-registration.ts +2 -0
  135. package/src/console/quota/account-quota-tenant.ts +8 -1
  136. package/src/console/quota/account-quota.ts +4 -1
  137. package/src/console/quota/native-identity.ts +189 -0
  138. package/src/console/remote-support/contracts.ts +12 -6
  139. package/src/console/remote-support/service.ts +2 -2
  140. package/src/console/share/share-router.ts +39 -6
  141. package/src/console/share/share-stats.ts +64 -25
  142. package/src/console/share/share-usage.ts +24 -7
  143. package/src/console/system-prompt/contracts.ts +102 -0
  144. package/src/console/system-prompt/routes.ts +165 -0
  145. package/src/main.ts +11 -0
  146. package/src/network/outbound-fetch.ts +281 -226
  147. package/src/network/pool/agent.ts +7 -0
  148. package/src/observability/event-query.ts +118 -7
  149. package/src/persistence/db-pg.ts +42 -0
  150. package/src/persistence/schema.ts +102 -0
  151. package/src/persistence/stores/api-key-store.ts +753 -581
  152. package/src/persistence/stores/share-store.ts +65 -13
  153. package/src/persistence/stores/telemetry-store.ts +57 -1
  154. package/src/persistence/system-prompt-store.ts +104 -0
  155. package/src/protocol/reasoning.ts +65 -1
  156. package/src/protocol/request/chat.ts +1 -0
  157. package/src/providers/default-registry.ts +1 -0
  158. package/src/providers/integrations/antigravity/antigravity.ts +20 -23
  159. package/src/providers/integrations/buddy/buddy-chat-shared.ts +30 -2
  160. package/src/providers/integrations/buddy/buddy-media.ts +5 -1
  161. package/src/providers/integrations/buddy/codebuddy/provider.ts +5 -1
  162. package/src/providers/integrations/buddy/workbuddy/provider.ts +5 -1
  163. package/src/providers/integrations/cline/cline.ts +7 -0
  164. package/src/providers/integrations/github/github.ts +20 -1
  165. package/src/providers/integrations/hermes.ts +5 -1
  166. package/src/providers/integrations/kilo/kilo.ts +4 -0
  167. package/src/providers/integrations/ollamacloud-models.ts +50 -0
  168. package/src/providers/integrations/openrouter/provider.ts +4 -0
  169. package/src/providers/integrations/qoder/qoder-checkin.ts +224 -51
  170. package/src/providers/integrations/qoder/qoder-native-identity-store.ts +156 -0
  171. package/src/providers/integrations/qoder/qoder-oauth.ts +24 -4
  172. package/src/providers/integrations/qoder/qoder-realm.ts +20 -0
  173. package/src/providers/integrations/qoder/qoder-text-tools.ts +277 -0
  174. package/src/providers/integrations/qoder/qoder-umid.ts +288 -0
  175. package/src/providers/integrations/qoder/qoder.ts +1375 -1641
  176. package/src/providers/operations/account-error-classification.ts +1682 -1660
  177. package/src/providers/operations/account-health-service.ts +619 -621
  178. package/src/providers/operations/client-versions.ts +37 -22
  179. package/src/providers/operations/provider-catalog-service.ts +11 -1
  180. package/src/providers/provider-contracts.ts +15 -2
  181. package/src/security/access-control.ts +19 -0
  182. package/src/security/api-key-auth.ts +14 -1
  183. package/src/security/cli-client-fingerprint.ts +9 -2
  184. package/src/transport/dispatch/empty-stream.ts +224 -224
  185. package/src/transport/dispatch/proxy-request.ts +31 -2
  186. package/src/transport/dispatch/streaming/stream.ts +16 -1
  187. package/src/transport/dispatch/upstream.ts +10 -0
  188. package/src/transport/failure-policy.ts +33 -0
  189. package/src/transport/request/preparer.ts +17 -1
  190. package/src/transport/request/system-prompt-injection.ts +89 -0
  191. package/src/transport/request/tenant-preferences.ts +43 -0
  192. package/src/transport/routing/admission.ts +13 -3
  193. package/src/transport/routing/route-catalog.ts +41 -12
  194. package/src/transport/routing/route-model.ts +53 -0
  195. package/src/transport/routing/router.ts +306 -228
  196. package/src/transport/surface/responses/encode.ts +2 -3
  197. package/src/transport/surface/responses/parse.ts +49 -7
  198. package/src/types/undici.d.ts +33 -0
  199. package/src/workers/daily-checkin.ts +2 -0
  200. package/dist/dashboard/assets/BannedPage-D-J75hBM.js.gz +0 -0
  201. package/dist/dashboard/assets/CliToolDetailPage-CVe6RaBH.js +0 -1
  202. package/dist/dashboard/assets/CliToolDetailPage-CVe6RaBH.js.gz +0 -0
  203. package/dist/dashboard/assets/CliToolsPage-Dd8ZSamp.js +0 -1
  204. package/dist/dashboard/assets/CliToolsPage-Dd8ZSamp.js.gz +0 -0
  205. package/dist/dashboard/assets/CombosPage-QoJ6foxo.js.gz +0 -0
  206. package/dist/dashboard/assets/ConsoleLogPage-DpNRay4C.js.gz +0 -0
  207. package/dist/dashboard/assets/CustomizationPage-CKUFTcW4.js +0 -1
  208. package/dist/dashboard/assets/CustomizationPage-CKUFTcW4.js.gz +0 -0
  209. package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js +0 -1
  210. package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js.gz +0 -0
  211. package/dist/dashboard/assets/LoginPage-DubdQcvX.js.gz +0 -0
  212. package/dist/dashboard/assets/ModelPicker-BH8Tehdi.js.gz +0 -0
  213. package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js +0 -1
  214. package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js.gz +0 -0
  215. package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js +0 -4
  216. package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js.gz +0 -0
  217. package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js +0 -1
  218. package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js.gz +0 -0
  219. package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js +0 -9
  220. package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js.gz +0 -0
  221. package/dist/dashboard/assets/QuotaPage-ATpaxrbl.js.gz +0 -0
  222. package/dist/dashboard/assets/SettingsPage-CETophpx.js.gz +0 -0
  223. package/dist/dashboard/assets/SetupPage-BdgECjqq.js +0 -1
  224. package/dist/dashboard/assets/SetupPage-BdgECjqq.js.gz +0 -0
  225. package/dist/dashboard/assets/SortableList-CAK-EN9z.js.gz +0 -0
  226. package/dist/dashboard/assets/StudioPage-B7W2_RhT.js.gz +0 -0
  227. package/dist/dashboard/assets/UsagePage-6T4EFDjW.js.gz +0 -0
  228. package/dist/dashboard/assets/api-keys-CUi6UXsR.js.gz +0 -0
  229. package/dist/dashboard/assets/app-KvsUJUMF.css +0 -2
  230. package/dist/dashboard/assets/app-KvsUJUMF.css.gz +0 -0
  231. package/dist/dashboard/assets/assert-67xXH5rf.js.gz +0 -0
  232. package/dist/dashboard/assets/badge-DC2L43Mj.js.gz +0 -0
  233. package/dist/dashboard/assets/banner-BBVcb0J3.js.gz +0 -0
  234. package/dist/dashboard/assets/card-Cca1XC6G.js.gz +0 -0
  235. package/dist/dashboard/assets/cli-tools-KvsWa1Jl.js.gz +0 -0
  236. package/dist/dashboard/assets/clipboard-button-DUzEbynq.js.gz +0 -0
  237. package/dist/dashboard/assets/concurrency-BDF0ok7y.js.gz +0 -0
  238. package/dist/dashboard/assets/dialog-DFvWhwTa.js.gz +0 -0
  239. package/dist/dashboard/assets/drawer-CFM2sMX1.js +0 -1
  240. package/dist/dashboard/assets/drawer-CFM2sMX1.js.gz +0 -0
  241. package/dist/dashboard/assets/entry-BLBLxX0h.js.gz +0 -0
  242. package/dist/dashboard/assets/entry-Bqy8RFvM.js.gz +0 -0
  243. package/dist/dashboard/assets/entry-_xHo5x6r.js.gz +0 -0
  244. package/dist/dashboard/assets/formatters-mpBPqAQ8.js.gz +0 -0
  245. package/dist/dashboard/assets/github-badge-C9e8Ts7e.js.gz +0 -0
  246. package/dist/dashboard/assets/index-D6QzEC_n.js.gz +0 -0
  247. package/dist/dashboard/assets/inline-BFUcvllj.js.gz +0 -0
  248. package/dist/dashboard/assets/input-C_ILAvBb.js.gz +0 -0
  249. package/dist/dashboard/assets/layout-CBIyT6Rt.js.gz +0 -0
  250. package/dist/dashboard/assets/list-row-ebJ_-Nsb.js.gz +0 -0
  251. package/dist/dashboard/assets/live-D0mm--ME.js.gz +0 -0
  252. package/dist/dashboard/assets/mutation-sMD965oO.js.gz +0 -0
  253. package/dist/dashboard/assets/numbers-BO9dKaFr.js.gz +0 -0
  254. package/dist/dashboard/assets/query-keys-CfL_Zdoh.js.gz +0 -0
  255. package/dist/dashboard/assets/routing-BnIpi1IJ.js.gz +0 -0
  256. package/dist/dashboard/assets/select-Cj5Nw43g.js.gz +0 -0
  257. package/dist/dashboard/assets/settings-BAFhDgX8.js.gz +0 -0
  258. package/dist/dashboard/assets/stack-EMrOMt4I.js.gz +0 -0
  259. package/dist/dashboard/assets/state-CWoo7ty5.js.gz +0 -0
  260. package/dist/dashboard/assets/switch-CCjT6IkF.js.gz +0 -0
  261. package/dist/dashboard/assets/useMutation-DjkLHUcE.js.gz +0 -0
  262. /package/dist/dashboard/assets/{app-BaWd0puM.js → app-wvwCi4Eh.js} +0 -0
  263. /package/dist/dashboard/assets/{app-BaWd0puM.js.gz → app-wvwCi4Eh.js.gz} +0 -0
@@ -1,1660 +1,1682 @@
1
- /**
2
- * Account error classification: turns one upstream failure into a category,
3
- * a cooldown window, and a verdict on the account.
4
- *
5
- * This is the decision half of the account health machine. It is pure: it
6
- * reads evidence and returns a classification, writes nothing, and knows
7
- * nothing about the database. The writer half lives in
8
- * `account-health-service.ts`.
9
- */
10
- import { isRecord } from "../../protocol/codecs";
11
- import { GatewayError } from "../../protocol/gateway-error";
12
- import { BUDDY_PROVIDER_IDS } from "../provider-metadata";
13
- import { parseProviderResetDuration, parseUpstreamBackoff } from "../../protocol/upstream-parsing";
14
- import {
15
- resolveAccountAuthRejectedCooldownMs,
16
- resolveAccountCooldownEscalationMaxMs,
17
- resolveAccountDisabledRecoveryMs,
18
- resolveAccountModelCapacityCooldownMs,
19
- resolveAccountPolicyBlockCooldownMs,
20
- resolveAccountQuotaCooldownMs,
21
- resolveAccountRateLimitCooldownMs,
22
- resolveAccountTransientCooldownMs,
23
- resolveAccountUnclassifiedCooldownMs,
24
- } from "../../config/cooldowns";
25
- const DEFAULT_RATE_LIMIT_COOLDOWN_MS = resolveAccountRateLimitCooldownMs;
26
- /**
27
- * Inception's per-minute limit window (1 minute).
28
- *
29
- * Inception's Free tier throttles per minute (1 000 requests / 1M input
30
- * tokens / 100K output tokens) and the window resets every minute. The generic
31
- * 15-minute rate-limit fallback would park a healthy key long after the window
32
- * refilled; 1 minute is the provider's own cadence, hardcoded because it is a
33
- * fact about the upstream, not a deployment knob.
34
- */
35
- const INCEPTION_RATE_LIMIT_COOLDOWN_MS = 60_000;
36
- // Fallback when the error states no usable duration: deliberately short (1h,
37
- // not 24h) — quota state is re-probed on the next failure, so an unknown
38
- // reset cadence self-corrects instead of parking the account for a day.
39
- const DEFAULT_QUOTA_COOLDOWN_MS = resolveAccountQuotaCooldownMs;
40
- const MODEL_CAPACITY_COOLDOWN_MS = resolveAccountModelCapacityCooldownMs;
41
- const TRANSIENT_ERROR_COOLDOWN_MS = resolveAccountTransientCooldownMs;
42
- /**
43
- * Accounts one cooldown-recovery pass brings back. The sweep ticks every 30s,
44
- * so a larger backlog drains over successive passes rather than one statement
45
- * covering every expired row.
46
- */
47
- export const COOLDOWN_RECOVERY_BATCH = 500;
48
- /**
49
- * Backoff for a failure no rule matched. Short on purpose: an unclassified
50
- * error is more likely a new upstream shape than a broken account, so the
51
- * account returns to rotation quickly.
52
- */
53
- const UNCLASSIFIED_COOLDOWN_MS = resolveAccountUnclassifiedCooldownMs;
54
- /**
55
- * xAI Grok Build's free tier resets on a rolling 24-hour window, so its
56
- * exhaustion is a full-day quota cooldown — never the generic 1h fallback,
57
- * which let the account re-enter rotation inside the window.
58
- */
59
- const GROK_QUOTA_COOLDOWN_MS = 24 * 60 * 60 * 1000;
60
- /**
61
- * The buddy family's policy/channel block (`11140`, `11128`) is a *base* backoff,
62
- * not a flat penalty — see {@link resolveAccountPolicyBlockCooldownMs}. It used
63
- * to be two constants, 24h for the content block and 6h for the channel one,
64
- * both applied on the first observation. That was wrong in the direction that
65
- * costs most: `11140` is also what a wrong desktop `User-Agent` brand answers
66
- * (`workbuddy/shared.ts`), so after we fixed a header regression every buddy
67
- * account stayed dead for up to a further day, and one prompt that failed a
68
- * safety review cost a whole account a day. As a base it escalates through the
69
- * consecutive-failure ladder, which tells the two causes apart by what they do
70
- * next — a genuinely blocked account is refused again immediately and parks
71
- * itself within minutes.
72
- */
73
- /**
74
- * Backoff for a model the upstream says does not exist for this account
75
- * (Cursor answers Connect `not_found` for a model the credential's plan does
76
- * not reach). Long, because the verdict is a property of the plan and will not
77
- * change on the next request — but bounded, so a plan upgrade is picked up
78
- * without an operator clearing it by hand.
79
- */
80
- const MODEL_UNREACHABLE_COOLDOWN_MS = 6 * 60 * 60 * 1000;
81
- /**
82
- * Backoff for a model the upstream does not serve at all (buddy family
83
- * business code `11102` — `service info not found` — or a `model ... not
84
- * found/supported` message). Mirrors the reference gateways'
85
- * `MODEL_SITE_BLOCK_S` → `MODEL_SITE_BLOCK_MAX_S`.
86
- *
87
- * Escalating rather than flat: the pair is re-dialled after the first window
88
- * and answers the same thing, so a flat 6h let a permanently unsupported model
89
- * consume a request every six hours forever. The ladder tops out at a day,
90
- * which is long enough to stop the redial and short enough that a backend that
91
- * starts serving the model is picked up without an operator clearing it.
92
- *
93
- * This is the one place the references' *strict* behaviour is adopted as a
94
- * backoff rather than as a 400: nothing currently succeeds through it, so no
95
- * client is broken by it.
96
- */
97
- const UNSUPPORTED_MODEL_COOLDOWN_MS = 6 * 60 * 60 * 1000;
98
- const UNSUPPORTED_MODEL_COOLDOWN_MAX_MS = 24 * 60 * 60 * 1000;
99
- /** How many doublings an unsupported-model cooldown may take before the cap. */
100
- const UNSUPPORTED_MODEL_MAX_LEVEL = 8;
101
-
102
- /**
103
- * The cooldown for the *nth* consecutive unsupported-model report (`hits` is
104
- * the new total, so the first report is `hits === 1`).
105
- *
106
- * Exported because the writer, not the classifier, is the only place that
107
- * reads the authoritative hit count — the classifier sees only the count the
108
- * caller supplied, which is absent on the first report of a pair.
109
- */
110
- export function unsupportedModelCooldownMs(hits: number): number {
111
- const level = Math.min(Math.max(0, Math.round(hits) - 1), UNSUPPORTED_MODEL_MAX_LEVEL);
112
- return Math.min(
113
- UNSUPPORTED_MODEL_COOLDOWN_MS * 2 ** level,
114
- UNSUPPORTED_MODEL_COOLDOWN_MAX_MS,
115
- );
116
- }
117
-
118
- /**
119
- * The wording an upstream uses for "we do not serve this model at all", as
120
- * opposed to "this account cannot reach it" (403 entitlement, above) or "this
121
- * route does not exist" (404 `upstream_not_found`).
122
- */
123
- const UNSUPPORTED_MODEL_MESSAGE =
124
- /service info not found|model\s.{0,80}?\bnot\s+(?:found|supported)\b/;
125
-
126
- /**
127
- * True when the evidence says the upstream does not serve the model.
128
- *
129
- * Narrow on purpose, and narrower than `model_unreachable`: only a 400/404
130
- * carrying the buddy family's `11102` or one of the two phrasings above. A
131
- * 4xx that merely *mentions* a model is far more often a payload the caller
132
- * got wrong, and cooling a pair for six hours over that takes a working model
133
- * out of rotation for a request bug.
134
- */
135
- function isUnsupportedModelEvidence(
136
- statusCode: number | null,
137
- providerCode: string,
138
- lower: string,
139
- ): boolean {
140
- if (statusCode !== 400 && statusCode !== 404) return false;
141
- return providerCode === "11102" || UNSUPPORTED_MODEL_MESSAGE.test(lower);
142
- }
143
-
144
- // ===== health/account-recorder.ts =====
145
- export interface AccountHealthEventRecord {
146
- readonly id: string;
147
- readonly accountId: string;
148
- readonly fromStatus: string | null;
149
- readonly toStatus: string;
150
- readonly reason: string | null;
151
- readonly errorCategory: string | null;
152
- readonly modelId: string | null;
153
- readonly createdAt: string;
154
- }
155
-
156
- export type AccountErrorCategory =
157
- | "quota_exhausted"
158
- | "rate_limit_transient"
159
- | "model_capacity"
160
- | "model_unreachable"
161
- | "auth_invalidated"
162
- /**
163
- * Deprecated: an upstream ban. Kept so existing health rows keep rendering
164
- * instead of showing a blank cause. No longer produced — see
165
- * `policy_blocked_banned` and `policy_blocked_invalid_request`.
166
- */
167
- | "policy_blocked"
168
- /**
169
- * The upstream banned this credential: a verdict on the account, e.g. buddy
170
- * `11140` "request illegal" with "Your account has been restricted due to
171
- * unusual activity". Cooldown cannot help — the account is not throttled, it
172
- * is refused. Disabling immediately is a *feature*: a client's retry budget
173
- * is spent in seconds, so surviving attempts burn sibling accounts that are
174
- * still healthy.
175
- */
176
- | "policy_blocked_banned"
177
- /**
178
- * The upstream refused this request: a verdict on the request. e.g. buddy
179
- * `11-128` "request illegal" with no account restriction. Distinguished from
180
- * {@link policy_blocked_banned} because the correct responses differ — a
181
- * banned account must be disabled, a malformed request must only stop being
182
- * retried verbatim.
183
- */
184
- | "policy_blocked_invalid_request"
185
- | "server_error"
186
- | "timeout"
187
- | "validation_required"
188
- | "unknown";
189
- export type AccountFailureOrigin = "cartethyia" | "upstream" | "network";
190
- export type AccountFailureScope =
191
- | "account"
192
- | "provider"
193
- | "model"
194
- | "pool"
195
- | "tenant"
196
- | "request"
197
- | "network"
198
- | "unknown";
199
-
200
- export interface AccountFailureEvidence {
201
- readonly origin?: AccountFailureOrigin;
202
- readonly scope?: AccountFailureScope;
203
- readonly credentialKind?: string;
204
- /**
205
- * The account's credential is a static bearer token used as issued and never
206
- * refreshed. An auth rejection against a static token is not a refreshable
207
- * fault: refreshing cannot help, and disabling the account discards a token
208
- * that may still be valid for other requests. Such a failure cools the
209
- * account down instead of parking it.
210
- */
211
- readonly staticToken?: boolean;
212
- readonly providerCode?: string;
213
- readonly credentialEvidence?: boolean;
214
- readonly retryAfterMs?: number;
215
- readonly statusCode?: number | null;
216
- readonly headers?: Headers | Record<string, string> | null;
217
- readonly modelId?: string;
218
- /**
219
- * Provider that produced the failure. Used only for the one
220
- * provider-family-specific policy rule (the buddy family's `11140`
221
- * content-policy block), never to broaden what disables an account.
222
- */
223
- readonly providerId?: string;
224
- /**
225
- * How many times this (account, model) pair has already been reported
226
- * unsupported, when the caller has read them. Best-effort: the classifier
227
- * states the base window without it, and the writer recomputes against the
228
- * count it reads itself.
229
- */
230
- readonly modelBlockHits?: number;
231
- /**
232
- * The upstream's own error body, when the caller has it.
233
- *
234
- * Providers state a reset window in structured fields far more reliably than
235
- * in prose — a body carrying `resets_at`, `resets_in_seconds` or
236
- * `limit_window_minutes` is an exact statement, while the same body's
237
- * `message` ("The usage limit has been reached") names no window at all.
238
- * Without this the classifier falls back to a fixed 15-minute cooldown for a
239
- * limit the provider said resets in 23 days.
240
- */
241
- readonly upstreamBody?: unknown;
242
- }
243
-
244
- export interface AccountErrorClassification {
245
- readonly category: AccountErrorCategory;
246
- readonly status: "cooldown" | "disabled";
247
- readonly cooldownMs: number;
248
- readonly reason: string;
249
- readonly retryAt: Date | null;
250
- readonly mutatesAccount: boolean;
251
- /**
252
- * True when this write changed a field the routing snapshot actually reads.
253
- *
254
- * The snapshot projects `provider_accounts.status` and
255
- * `.cooldownUntil` and nothing else (`route-catalog.ts:286-287`), so only a
256
- * write that moves one of those has to drop it. Everything else a failure
257
- * persists — the failure streak, the error text, the category, a
258
- * model-scoped cooldown — is invisible to routing, and invalidating for it
259
- * forced the next request to rebuild an 11-SELECT snapshot for no reason.
260
- *
261
- * That made a failure wave self-sustaining: N failed attempts produced N
262
- * invalidations, and every following request paid the full rebuild.
263
- */
264
- readonly routesVisible: boolean;
265
- /**
266
- * True when `cooldownMs` came from the provider's own statement — a parsed
267
- * `Retry-After`-family header, or a reset quoted in its message — rather than
268
- * from one of this module's fallback constants.
269
- *
270
- * A property of the *evidence*, not of the category, so it is the same for
271
- * every branch of one classification. Only {@link escalateFallbackCooldown}
272
- * reads it: a provider-stated window is the truth and is never multiplied.
273
- */
274
- readonly providerStated?: boolean;
275
- /**
276
- * True when the upstream reported the model as one it does not serve (the
277
- * buddy family's `11102`), rather than one this account merely cannot reach.
278
- *
279
- * A property of the classification, because both verdicts land in the same
280
- * `model_unreachable` category and the writer cannot otherwise tell them
281
- * apart — it needs to know whether to count the hit and escalate the window.
282
- */
283
- readonly modelUnsupported?: boolean;
284
- }
285
-
286
- /**
287
- * How many doublings a fallback cooldown may take before the cap takes over.
288
- *
289
- * Four is enough to reach the cap from every fallback default (the longest is
290
- * the 1h quota fallback, which reaches it on the first doubling) while keeping
291
- * the ladder short enough to reason about.
292
- */
293
- const COOLDOWN_ESCALATION_MAX_LEVEL = 4;
294
-
295
- /**
296
- * A provider code that is digits only. Such codes are identifiers, not auth
297
- * vocabulary — providers encode spend/entitlement verdicts as numbers (Qoder's
298
- * `110`, buddy's `6004`) — so the bare-401/403 credential arm must not treat
299
- * an unknown one as evidence that the credential died.
300
- */
301
- const NUMERIC_PROVIDER_CODE = /^\d+$/;
302
-
303
- /**
304
- * Escalates a *fallback* cooldown for an account that keeps failing.
305
- *
306
- * `consecutiveFailures` has been incremented on every account failure since the
307
- * health machine was written, but nothing ever read it — so a broken upstream
308
- * was re-dialled once a minute forever, and an account whose quota had an
309
- * unstated reset came straight back into rotation every time its short fallback
310
- * lapsed. The count is now a backoff level: each further consecutive failure
311
- * doubles the fallback, clamped to {@link resolveAccountCooldownEscalationMaxMs}.
312
- *
313
- * Two guards keep it from overriding the provider. A duration the provider
314
- * *stated* is never multiplied — that window is the truth. And the caller writes
315
- * the result through a monotonic maximum, so a burst of concurrent in-flight
316
- * failures advances the deadline to the longest of them instead of compounding
317
- * once per failure.
318
- */
319
- export function escalateFallbackCooldown(
320
- cooldownMs: number,
321
- providerStated: boolean,
322
- consecutiveFailures: number,
323
- ): number {
324
- if (providerStated || cooldownMs <= 0) return cooldownMs;
325
- const level = Math.min(Math.max(0, consecutiveFailures), COOLDOWN_ESCALATION_MAX_LEVEL);
326
- return Math.min(cooldownMs * 2 ** level, resolveAccountCooldownEscalationMaxMs());
327
- }
328
-
329
- /**
330
- * The later of a candidate deadline and the one already recorded.
331
- *
332
- * Every fallback duration is recomputed from *now* on each failure, so writing
333
- * it unconditionally let a fresh short failure *shorten* a longer window already
334
- * in force: a 60-second unclassified cooldown overwrote the provider's own
335
- * "Try again in 11h" and released the pair back into rotation inside a window
336
- * the upstream had closed, where it failed again — once a minute, forever.
337
- * A cooling deadline may only ever move later.
338
- */
339
- export function latestDeadlineMs(...candidates: readonly (number | undefined | null)[]): number {
340
- let latest = 0;
341
- for (const value of candidates) {
342
- if (typeof value !== "number" || !Number.isFinite(value)) continue;
343
- if (value > latest) latest = value;
344
- }
345
- return latest;
346
- }
347
-
348
- /**
349
- * The account-wide deadline to write, or `null` for "no deadline".
350
- *
351
- * {@link latestDeadlineMs} cannot express *no* deadline — it floors at 0 — and
352
- * the account-wide write has one state that needs it. With
353
- * `CARTETHYIA_ACCOUNT_DISABLED_RECOVERY_MS=0` an automatic disable is meant to
354
- * be terminal again, and the classifier correctly reports `retryAt: null`.
355
- * Deriving the deadline from `Date.now() + cooldownMs` instead stamps *now*.
356
- * The recovery sweep then treats that elapsed deadline as
357
- * an automatic disable awaiting recovery and returns the row to rotation on its
358
- * next tick — the escape hatch enabled exactly the behavior it documents
359
- * itself as disabling.
360
- *
361
- * A candidate only exists when there is a real window to state. Monotonicity is
362
- * preserved: a deadline already in force still wins over a new `null`.
363
- */
364
- export function accountDeadlineMs(
365
- candidates: readonly (number | undefined | null)[],
366
- ): Date | null {
367
- const latest = latestDeadlineMs(...candidates);
368
- return latest > 0 ? new Date(latest) : null;
369
- }
370
-
371
- /**
372
- * Whether the upstream named a restriction on the account itself.
373
- *
374
- * The buddy family reuses "request illegal" for two unrelated rejections, and
375
- * the difference is only in the body:
376
- *
377
- * - `11140` carries `displayMsg` about the account being restricted ("Your
378
- * account has been restricted due to unusual activity") and `actions`
379
- * offering an appeal. That is a ban — the account is refused, not throttled.
380
- * - `11-128` is a refusal of this request; the account is fine.
381
- *
382
- * Proven account-scoped (2026-10-16): two live share-page requests carrying
383
- * byte-identical bodies and headers to the same model differed only in the
384
- * pooled account, and the blocked one 400'd while the other 200'd.
385
- *
386
- * Matched on the code *and* the account-restriction wording, so an unrelated
387
- * upstream that happens to say "restricted" in prose cannot disable accounts.
388
- */
389
- function isBuddyAccountBanned(
390
- lower: string,
391
- providerCode: string,
392
- upstreamBody: unknown,
393
- ): boolean {
394
- if (providerCode !== "11140") return false;
395
- if (banWording(lower)) return true;
396
- // The message alone is not enough, and this is the case that proved it: the
397
- // provider's message for a ban is the same two words it uses for a refused
398
- // request — "request illegal" — while the actual verdict ("Your account has
399
- // been restricted due to unusual activity") lives in `displayMsg`, which the
400
- // message extractor never reads. Matching only the message therefore read
401
- // every ban as a mere request refusal: no category, no disable, and the
402
- // account kept serving.
403
- const body = isRecord(upstreamBody) ? upstreamBody : undefined;
404
- if (body === undefined) return false;
405
- for (const field of ["displayMsg", "displayTips", "msg", "message"]) {
406
- const value = body[field];
407
- if (typeof value === "string" && banWording(value.toLowerCase())) return true;
408
- // `displayMsg` is a per-language map: `{ en: "…", zh: "…" }`. Any
409
- // language states the same verdict, so all of them are read.
410
- if (!isRecord(value)) continue;
411
- for (const nested of Object.values(value)) {
412
- if (typeof nested === "string" && banWording(nested.toLowerCase())) return true;
413
- }
414
- }
415
- // `actions` names the remedy: an appeal can only be offered for a verdict
416
- // about the account, and "edit input" only ever describes a request.
417
- const actions = body["actions"];
418
- if (Array.isArray(actions)) {
419
- for (const action of actions) {
420
- if (typeof action !== "string") continue;
421
- if (action.toLowerCase().includes("appeal") || action.toLowerCase().includes("feedback")) {
422
- return true;
423
- }
424
- }
425
- }
426
- return false;
427
- }
428
-
429
- /** The wording an upstream uses when it restricted the account, not the request. */
430
- function banWording(lower: string): boolean {
431
- return (
432
- lower.includes("restricted") ||
433
- lower.includes("has been banned") ||
434
- lower.includes("unusual activity") ||
435
- lower.includes("appeal this restriction") ||
436
- lower.includes("submit feedback to appeal") ||
437
- lower.includes("account has been") ||
438
- lower.includes("账号存在异常行为") ||
439
- lower.includes("帳號存在異常行為")
440
- );
441
- }
442
-
443
- /**
444
- * Bounded to a day, matching `MAX_COOLDOWN_MS` in the failure policy: a stated
445
- * window longer than that is a monthly or plan-level limit, and parking an
446
- * account for a month on one upstream field is a decision for the operator,
447
- * not an automatic one.
448
- */
449
- const MAX_UPSTREAM_STATED_COOLDOWN_MS = 24 * 3600 * 1000;
450
-
451
- /** Parses the stored upstream text as JSON. Returns `undefined` when it is not. */
452
- function tryParseUpstreamBody(text: string): unknown {
453
- try {
454
- const parsed: unknown = JSON.parse(text);
455
- return typeof parsed === "object" && parsed !== null ? parsed : undefined;
456
- } catch {
457
- return undefined;
458
- }
459
- }
460
-
461
- /**
462
- * Reads a reset window the upstream stated in structured fields.
463
- *
464
- * A body like `{"error":{"type":"usage_limit_reached","resets_at":1795019113,
465
- * "resets_in_seconds":2072028}}` states the window exactly, while its own
466
- * `message` ("The usage limit has been reached") states none — so parsing only
467
- * prose leaves a 23-day limit cooling for the 15-minute fallback, and the
468
- * account re-enters rotation hours before it can serve.
469
- *
470
- * Both spellings are read because providers use both: an absolute epoch
471
- * (`resets_at`, seconds or milliseconds) and a relative delay
472
- * (`resets_in_seconds`). The later of the two wins, since they are the same
473
- * claim computed at slightly different instants.
474
- */
475
- function parseUpstreamBodyReset(body: unknown): number | null {
476
- if (typeof body !== "object" || body === null) return null;
477
- const record = body as Record<string, unknown>;
478
- // Some providers nest under `error`, some return it at the top level.
479
- const candidates = [record, (record["error"] as Record<string, unknown> | undefined) ?? undefined];
480
- let best: number | null = null;
481
- for (const candidate of candidates) {
482
- if (typeof candidate !== "object" || candidate === null) continue;
483
- const now = Date.now();
484
-
485
- const absolute = candidate["resets_at"] ?? candidate["reset_at"];
486
- if (typeof absolute === "number" && Number.isFinite(absolute) && absolute > 0) {
487
- // Seconds vs milliseconds: a plausible epoch in seconds is ~10 digits, in
488
- // milliseconds ~13, and the two are four orders of magnitude apart — so
489
- // the smaller is scaled up rather than guessed from a threshold date.
490
- const epochMs = absolute < 1e11 ? absolute * 1000 : absolute;
491
- const delay = epochMs - now;
492
- if (delay > 0) best = best === null ? delay : Math.max(best, delay);
493
- }
494
-
495
- const relative =
496
- candidate["resets_in_seconds"] ??
497
- candidate["reset_in_seconds"] ??
498
- candidate["retry_after_seconds"];
499
- if (typeof relative === "number" && Number.isFinite(relative) && relative > 0) {
500
- const delay = relative * 1000;
501
- best = best === null ? delay : Math.max(best, delay);
502
- }
503
-
504
- const minutes = candidate["limit_window_minutes"] ?? candidate["reset_in_minutes"];
505
- if (typeof minutes === "number" && Number.isFinite(minutes) && minutes > 0) {
506
- const delay = minutes * 60_000;
507
- best = best === null ? delay : Math.max(best, delay);
508
- }
509
- }
510
- return best === null ? null : Math.min(MAX_UPSTREAM_STATED_COOLDOWN_MS, Math.round(best));
511
- }
512
-
513
-
514
- interface AccountErrorContext {
515
- readonly message: string;
516
- readonly lower: string;
517
- readonly folded: string;
518
- readonly statusCode: number | null;
519
- readonly origin: AccountFailureOrigin;
520
- readonly scope: AccountFailureScope;
521
- readonly revocationNamed: boolean;
522
- readonly accountEvidence: boolean;
523
- readonly canMutate: boolean;
524
- readonly providerCode: string;
525
- readonly statedCooldownMs: number | null;
526
- /** Echo of the caller's evidence; read only by the arms that need it. */
527
- readonly providerId: string | undefined;
528
- /** The raw upstream response body, when the dispatch path stored it. */
529
- readonly rawUpstreamResponse: string | undefined;
530
- /**
531
- * The upstream named a restriction on the account, not a refusal of the
532
- * request. Read by the policy arm to pick between disabling the account and
533
- * merely refusing the request.
534
- */
535
- readonly buddyBanned: boolean;
536
- readonly reason: (prefix: string) => string;
537
- readonly result: (
538
- category: AccountErrorCategory,
539
- status: AccountErrorClassification["status"],
540
- cooldownMs: number,
541
- retryAt: Date | null,
542
- explanation: string,
543
- mutatesAccount?: boolean,
544
- modelUnsupported?: boolean,
545
- ) => AccountErrorClassification;
546
- }
547
-
548
- /**
549
- * One arm of the classifier: an ordered `(match, decide)` pair.
550
- *
551
- * The arms are **not** independent predicates. Later arms are deliberately
552
- * broader than earlier ones — `quotaSignal` matches every price refusal, and
553
- * `authSignal` matches a superset of the request-scoped codes — so a match
554
- * only wins because it is tried first. That is why they live in one ordered
555
- * list rather than behind a dispatch map or a set of independent `if`s: the
556
- * list is the precedence, and reading it top to bottom is reading the
557
- * decision.
558
- *
559
- * `match` is a pure function of the shared signals (no closure over the
560
- * classifier's locals, no allocation per classification): the table is built
561
- * once per call from values already computed for every arm.
562
- */
563
- interface AccountErrorRule {
564
- readonly name: string;
565
- readonly match: (signals: AccountErrorSignals) => boolean;
566
- readonly decide: (signals: AccountErrorSignals) => AccountErrorClassification;
567
- }
568
-
569
- /** The shared signals every arm reads, computed once per classification. */
570
- interface AccountErrorSignals {
571
- readonly message: string;
572
- readonly lower: string;
573
- readonly folded: string;
574
- readonly statusCode: number | null;
575
- readonly origin: AccountFailureOrigin;
576
- readonly scope: AccountFailureScope;
577
- readonly providerCode: string;
578
- readonly statedCooldownMs: number | null;
579
- readonly reason: (prefix: string) => string;
580
- readonly result: AccountErrorContext["result"];
581
- readonly canMutate: boolean;
582
- readonly accountEvidence: boolean;
583
- readonly revocationNamed: boolean;
584
- /** Echo of the caller's evidence; read only by the arms that need it. */
585
- readonly providerId: string | undefined;
586
- readonly modelId: string | undefined;
587
- readonly modelBlockHits: number;
588
- readonly staticToken: boolean;
589
- readonly credentialKind: string | undefined;
590
- /** Buddy-family provider: the only family with provider-specific codes below. */
591
- readonly buddy: boolean;
592
- /**
593
- * The upstream named a restriction on this account, not a refusal of the
594
- * request. Only the `11140` shape carries `displayMsg` about the account
595
- * being restricted, so the policy arm can tell a ban from a bad request
596
- * instead of treating both as one cooldown.
597
- */
598
- readonly buddyBanned: boolean;
599
- readonly priceRefusal: boolean;
600
- readonly deploymentInfraError: boolean;
601
- readonly quotaSignal: boolean;
602
- readonly policyRejection: boolean;
603
- readonly requestScoped: boolean;
604
- readonly channelRejection: boolean;
605
- readonly hostedToolFailure: boolean;
606
- readonly modelEntitlementFailure: boolean;
607
- readonly queuedUpstream: boolean;
608
- readonly validationRequired: boolean;
609
- readonly validationUrl: string | undefined;
610
- }
611
-
612
- const VALIDATION_URL_RE = /https?:\/\/[^\s"'\\]+/;
613
-
614
- /**
615
- * Extracts the Google `validation_url` from an Antigravity rejection.
616
- *
617
- * The URL can appear either in the message itself or nested in the raw
618
- * upstream JSON (`details[].metadata.validation_url`). Returning `undefined`
619
- * means nothing actionable was found — the classifier then just parks the
620
- * account without a link.
621
- */
622
- function extractValidationUrl(message: string, rawUpstreamResponse: string | undefined): string | undefined {
623
- if (rawUpstreamResponse !== undefined && rawUpstreamResponse.includes("validation_url")) {
624
- try {
625
- const parsed = JSON.parse(rawUpstreamResponse) as {
626
- error?: {
627
- details?: Array<{ metadata?: Record<string, unknown> }>;
628
- };
629
- };
630
- for (const detail of parsed?.error?.details ?? []) {
631
- const url = detail?.metadata?.["validation_url"];
632
- if (typeof url === "string" && url.length > 0 && url.startsWith("http")) return url;
633
- }
634
- } catch {
635
- /* fall through to message regex */
636
- }
637
- }
638
- const m = VALIDATION_URL_RE.exec(message);
639
- return m?.[0];
640
- }
641
-
642
- /** Build the request-independent evidence consumed by the ordered classifier. */
643
- function accountErrorContext(error: unknown, options?: AccountFailureEvidence): AccountErrorContext { const errorRecord = isRecord(error) ? error : undefined;
644
- const message =
645
- error instanceof Error
646
- ? error.message
647
- : typeof error === "string"
648
- ? error
649
- : typeof errorRecord?.message === "string"
650
- ? errorRecord.message
651
- : String(error ?? "Unknown error");
652
- const lower = message.toLowerCase();
653
- const statusCode =
654
- options?.statusCode ??
655
- (typeof errorRecord?.status === "number" ? errorRecord.status : null);
656
- const origin = options?.origin ?? "cartethyia";
657
- const scope = options?.scope ?? "unknown";
658
- // Providers write the same phrase with different separators: codex reports
659
- // `token_revoked`, others `token revoked` or `token-revoked`. Folding the
660
- // separators once lets every match below see one spelling.
661
- const folded = lower.replace(/[_-]+/g, " ");
662
- // A revocation named in the message is evidence about the credential itself,
663
- // whichever scope the caller could establish. A stream error frame carries
664
- // no `credentialEvidence` detail and classifies at provider scope, so without
665
- // this the account stays in rotation: the phrase is recognised and then
666
- // discarded because nothing mutates on provider scope.
667
- const revocationNamed = folded.includes("token revoked") || folded.includes("invalidated oauth token");
668
- const accountEvidence =
669
- options?.credentialEvidence === true ||
670
- revocationNamed ||
671
- (options?.origin === "upstream" && options.scope === "account");
672
- // A named revocation mutates even at provider scope: the message is the
673
- // evidence, and provider scope is what a frame with no credential detail
674
- // classifies as. Every other signal still needs account scope.
675
- const canMutate = origin === "upstream" && (revocationNamed || (scope === "account" && accountEvidence));
676
- const providerCode =
677
- options?.providerCode?.toLowerCase() ??
678
- (error instanceof GatewayError && typeof error.details.providerCode === "string"
679
- ? error.details.providerCode.toLowerCase()
680
- : "");
681
-
682
- const getHeader = (name: string): string | null => {
683
- const headers = options?.headers;
684
- if (!headers) return null;
685
- if ("get" in headers && typeof headers.get === "function") return headers.get(name);
686
- const wanted = name.toLowerCase();
687
- const entry = Object.entries(headers).find(([key]) => key.toLowerCase() === wanted);
688
- return entry?.[1] ?? null;
689
- };
690
-
691
- const headerCooldown =
692
- options?.retryAfterMs ??
693
- parseUpstreamBackoff({ get: getHeader });
694
- const messageCooldown = parseProviderResetDuration(message);
695
- const rawUpstreamResponse =
696
- origin === "upstream" &&
697
- error instanceof GatewayError &&
698
- typeof error.details.raw === "string"
699
- ? error.details.raw
700
- : undefined;
701
- // The body's own fields, when they can be reached. Checked alongside the
702
- // header and the prose rather than instead of them: a provider may state the
703
- // window in any of the three, and the longest honest statement wins.
704
- //
705
- // `details.raw` is tried even when no caller passed a parsed body, because
706
- // that is where the upstream's JSON actually reaches this function — the
707
- // dispatch path stores the response text there. A structured window read from
708
- // it is the difference between a 15-minute fallback and the 23 days the
709
- // provider stated.
710
- const bodyCooldown =
711
- parseUpstreamBodyReset(options?.upstreamBody) ??
712
- (rawUpstreamResponse === undefined ? null : parseUpstreamBodyReset(tryParseUpstreamBody(rawUpstreamResponse)));
713
- // The body the ban is named in. The caller's parsed body when there is one,
714
- // and the raw response text otherwise — the dispatch path stores the
715
- // upstream's JSON there, and that is where `displayMsg` lives.
716
- const banBody =
717
- options?.upstreamBody ??
718
- (rawUpstreamResponse === undefined ? undefined : tryParseUpstreamBody(rawUpstreamResponse));
719
- // Whether the provider itself named a window. Decided once, from the
720
- // evidence, so every branch of this classification agrees about it.
721
- const providerStated =
722
- headerCooldown !== null || messageCooldown !== null || bodyCooldown !== null;
723
- const statedCooldownMs =
724
- [headerCooldown, messageCooldown, bodyCooldown].reduce<number | null>(
725
- (best, value) => (value === null ? best : best === null ? value : Math.max(best, value)),
726
- null,
727
- );
728
- const reason = (prefix: string): string => `${prefix}: ${message.slice(0, 160)}`;
729
- const result = (
730
- category: AccountErrorCategory,
731
- status: AccountErrorClassification["status"],
732
- cooldownMs: number,
733
- retryAt: Date | null,
734
- explanation: string,
735
- mutatesAccount = canMutate,
736
- modelUnsupported = false,
737
- ): AccountErrorClassification => ({
738
- category,
739
- status,
740
- cooldownMs,
741
- reason: origin === "upstream" ? rawUpstreamResponse ?? message.slice(0, 160) : explanation,
742
- retryAt,
743
- mutatesAccount,
744
- // Set below by the caller that knows which columns it wrote, not here:
745
- // the same classification feeds both the account-wide path (status +
746
- // cooldownUntil — routing-visible) and the model-scoped path (neither —
747
- // invisible). Defaulting true keeps existing callers conservative.
748
- routesVisible: true,
749
- providerStated,
750
- modelUnsupported,
751
- });
752
-
753
- return {
754
- message,
755
- lower,
756
- folded,
757
- statusCode,
758
- origin,
759
- scope,
760
- revocationNamed,
761
- accountEvidence,
762
- canMutate,
763
- providerCode,
764
- providerId: options?.providerId,
765
- statedCooldownMs,
766
- rawUpstreamResponse,
767
- buddyBanned: isBuddyAccountBanned(lower, providerCode, banBody),
768
- reason,
769
- result,
770
- };
771
- }
772
-
773
- export function classifyAccountError(
774
- error: unknown,
775
- options?: AccountFailureEvidence,
776
- ): AccountErrorClassification {
777
- const {
778
- message,
779
- lower,
780
- folded,
781
- statusCode,
782
- origin,
783
- scope,
784
- revocationNamed,
785
- accountEvidence,
786
- canMutate,
787
- providerCode,
788
- rawUpstreamResponse,
789
- statedCooldownMs,
790
- buddyBanned,
791
- reason,
792
- result,
793
- } = accountErrorContext(error, options);
794
- const modelId = options?.modelId;
795
- const modelBlockHits = Math.max(1, Math.floor(options?.modelBlockHits ?? 1));
796
- const staticToken = options?.staticToken === true;
797
- const credentialKind = options?.credentialKind;
798
- const buddy = BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "");
799
-
800
- // A price refusal is a verdict on the *request*, not on the account. inferhub
801
- // answers 402 with `no provider's ask matches your max-per-mtok bid`: the
802
- // caller's own price ceiling was below every upstream ask, which says nothing
803
- // about the credential or its balance. Classified as quota it cooled a
804
- // fully-credited account down for an hour and the operator reported it as
805
- // "blocked, unusable at all". Detected before the quota branch and recorded
806
- // without mutating the account, so only that request fails.
807
- const priceRefusal =
808
- lower.includes("max-per-mtok") ||
809
- lower.includes("no provider's ask") ||
810
- (lower.includes("ask") && lower.includes("bid"));
811
- // Deployment/proxy infrastructure errors (DEPLOYMENT_DISABLED, etc.) are
812
- // transport-level proxy pool failures, not account quota exhaustion.
813
- // Mutating the account parks it for an hour while the pool itself stays
814
- // healthy — the next request picks the same dying pool and the same error
815
- // repeats. These must not cooldown the account.
816
- const deploymentInfraError =
817
- lower.includes("deployment_disabled") ||
818
- lower.includes("deployment disabled") ||
819
- (lower.includes("sin1::") && lower.includes("deployment"));
820
- if (deploymentInfraError) {
821
- return result("server_error", "cooldown", 60_000, null, `Proxy deployment error: ${message.slice(0, 160)}`, false);
822
- }
823
- // A (CodeBuddy/WorkBuddy) *frequency* window is a verdict on one
824
- // (account, model) pair, not on the account: `{"code":"6004","msg":"usage
825
- // exceeds frequency limit ... your usage will reset at <stamp> UTC+8,
826
- // alternatively, you can switch to the other models to continue using it."}`.
827
- // The provider names the remedy itself — use a different model — which is
828
- // only a remedy if the account still serves every other model. Taken as
829
- // account-wide `quota_exhausted` it parked the whole account until the reset,
830
- // so the switch the upstream offered was refused by our own gateway and every
831
- // sibling model went dark with it; the account also read COOLDOWN /
832
- // quota_exhausted while only that pair was throttled.
833
- //
834
- // Classified `rate_limit_transient` so the writer scopes it to the model:
835
- // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
836
- // account's own status alone. Kept ahead of `quotaSignal`, which matches a
837
- // *broader* slice of the same wording (`usage exceeds` + `frequency limit`)
838
- // and would otherwise claim this message as an account-wide quota window.
839
- // Narrow on purpose — the extra phrase is what says the other models still
840
- // work; without it the failure stays account-wide.
841
- const modelFrequencyWindow =
842
- lower.includes("frequency limit") && lower.includes("switch to the other models");
843
- if (modelFrequencyWindow) {
844
- const cooldownMs = statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
845
- return result(
846
- "rate_limit_transient",
847
- "cooldown",
848
- cooldownMs,
849
- new Date(Date.now() + cooldownMs),
850
- reason("Provider frequency limit for this model"),
851
- // Provider-scoped evidence still has to reach the writer: a stream frame
852
- // carries no credential detail, so `canMutate` is false and without this
853
- // the pair would be retried for the whole window it was closed for.
854
- origin === "upstream" && scope === "provider",
855
- );
856
- }
857
-
858
- // Quota-shaped provider codes/messages (e.g. xAI's
859
- const quotaSignal =
860
- !priceRefusal &&
861
- !deploymentInfraError &&
862
- (statusCode === 402 ||
863
- // Qoder answers an exhausted credit balance with
864
- // `{"code":"112","message":"{\"pricingUrl\":...}"}` — the body points at
865
- // the pricing page, i.e. "buy more", which is a balance verdict, not an
866
- // auth one. It fell through to `auth_invalidated` and the console showed
867
- // "Re-login required" on an account whose credential was fine, sending
868
- // operators to re-authenticate instead of topping up.
869
- // Recognising it here keeps it out of `auth_invalidated`; the branch
870
- // below scopes the resulting cooldown to the model, because the account
871
- // keeps serving its other models.
872
- providerCode === "112" ||
873
- providerCode === "110" ||
874
- providerCode === "insufficient_quota" ||
875
- providerCode === "quota_exceeded" ||
876
- providerCode === "subscription:free-usage-exhausted" ||
877
- providerCode === "14018" ||
878
- // CodeBuddy/WorkBuddy rate limit: `{"code":"6004","msg":"usage exceeds
879
- // frequency limit ... your usage will reset at <stamp> UTC+8"}`. The
880
- // message names a reset, so this is a parked window, not a transient
881
- // 429 — without it the account fell through to `unknown` with
882
- // `mutates=false`, stayed in rotation, and was re-picked for every
883
- // request inside the window the provider had actually named.
884
- //
885
- // Matched on the message's own wording, not on the `6004` code: that code
886
- // is not unique to this surface (the artifact-release API reuses it), so a
887
- // code match would sweep an unrelated failure into a quota cooldown. The
888
- // phrase is the reliable key, and `usage exceeds` + `frequency limit` is
889
- // the pair this family actually prints.
890
- (lower.includes("usage exceeds") && lower.includes("frequency limit")) ||
891
- lower.includes("insufficient_quota") ||
892
- lower.includes("quota_exceeded") ||
893
- lower.includes("usage_limit_reached") ||
894
- lower.includes("usage-exhausted") ||
895
- lower.includes("usage_exhausted") ||
896
- lower.includes("usage exhausted") ||
897
- lower.includes("usage exceeds") ||
898
- lower.includes("frequency limit") ||
899
- lower.includes("free-usage") ||
900
- lower.includes("free usage") ||
901
- lower.includes("exceeded your current quota") ||
902
- // Trae answers an exhausted quota with **HTTP 502** and the body `Your
903
- // requests have exceeded the quota.` The status is what misled the
904
- // classifier: 502 read as a gateway failure, so this fell through to
905
- // `server_error` — a 30s cooldown that does not mutate the account, so
906
- // the account stayed in rotation and every other model on it kept being
907
- // invoked inside a window the provider had already closed. The body is
908
- // the verdict; the 502 is incidental. Matched on the phrase, not on the
909
- // status, which is why this sits in `quotaSignal` ahead of the 5xx arm.
910
- lower.includes("have exceeded the quota") ||
911
- lower.includes("balance exhausted") ||
912
- lower.includes("insufficient balance") ||
913
- lower.includes("out of credits") ||
914
- lower.includes("credits exhausted") ||
915
- lower.includes("spending limit") ||
916
- lower.includes("billing daily count exceeded") ||
917
- // Same Qoder family reached by body text when the code is not parsed out.
918
- (lower.includes("pricingurl") && lower.includes("qoder.com/pricing")));
919
- // A deterministic content-policy rejection is NOT credential invalidation.
920
- // CodeBuddy returns HTTP 403 with provider code 11140 ("request illegal",
921
- // "did not pass the safety review") — refreshing the token cannot change
922
- // that outcome and the account itself is healthy, so it must never flip the
923
- // account to auth_invalidated/disabled. Only real auth signals do.
924
- const policyRejection =
925
- providerCode === "11140" ||
926
- lower.includes("safety review") ||
927
- lower.includes("content did not pass") ||
928
- lower.includes("request illegal") ||
929
- lower.includes("content blocked") ||
930
- lower.includes("account suspended");
931
- // A ban is a verdict on the credential, and cooldown cannot answer it: the
932
- // account is refused, not throttled. Disabling is the fix, and disabling
933
- // *immediately* is a feature — a client's retry budget is spent in seconds,
934
- // so a banned account that merely cooled stayed in the plan and burned
935
- // sibling accounts that were still healthy before the window expired.
936
- //
937
- // The two shapes are told apart by the body, not the code alone: `11140`
938
- // names a restriction on the account, `11-128` refuses the request.
939
- const policyBlock =
940
- policyRejection &&
941
- options?.providerId !== undefined &&
942
- BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
943
- if (policyBlock) {
944
- if (buddyBanned) {
945
- return result(
946
- "policy_blocked_banned",
947
- "disabled",
948
- 0,
949
- null,
950
- reason("Account banned by upstream"),
951
- // A ban is evidence about the credential even at provider scope, for
952
- // the same reason a named revocation is: the message is the evidence.
953
- // Without this a stream frame's missing credential detail would leave a
954
- // banned account serving.
955
- true,
956
- );
957
- }
958
- // Not a ban: the upstream refused this request. The account is fine, so the
959
- // only correct move is to stop retrying it verbatim. Cooling the account
960
- // here was the bug that made one malformed request look like a dead
961
- // credential.
962
- return result(
963
- "policy_blocked_invalid_request",
964
- "cooldown",
965
- 0,
966
- null,
967
- reason("Upstream refused this request"),
968
- // Never mutates: a malformed body fails identically on every account, so
969
- // cooling would burn a healthy credential on the gateway's own fault.
970
- false,
971
- );
972
- }
973
- // These codes are a verdict on the request, not the account. Cooling the
974
- // account for them burns a healthy credential on a body that will fail the
975
- // same way on every other account:
976
- //
977
- // - `11148` tool-call/result mismatch — the transcript is broken; the
978
- // pre-payload repair is supposed to prevent it, and a slip is still the
979
- // body's fault.
980
- // - `11135` the image cannot be processed, `11133` the model rejected a
981
- // parameter, `11101` the body failed to parse — deterministic for that
982
- // payload.
983
- // - `12153` the session is dead — a refresh problem, not a reason to park
984
- // the account on the first observation.
985
- // - `14017` trial not activated — the account may recover once registration
986
- // finishes, so a hard cooldown would keep it out after it becomes usable.
987
- //
988
- // Recognised before the auth and quota branches: several of these arrive as
989
- // 401/403/429, which those branches would otherwise treat as account evidence.
990
- const requestScoped =
991
- providerCode === "11148" ||
992
- providerCode === "11135" ||
993
- providerCode === "11133" ||
994
- providerCode === "11101" ||
995
- providerCode === "12153" ||
996
- providerCode === "14017" ||
997
- lower.includes("tool_call_sequence_broken") ||
998
- lower.includes("unmarshal chat params failed") ||
999
- lower.includes("invalid image_url content") ||
1000
- lower.includes("offline user session not found") ||
1001
- lower.includes("trial not activated") ||
1002
- lower.includes("trial version is not yet activated");
1003
- if (requestScoped && BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "")) {
1004
- return result(
1005
- "unknown",
1006
- "cooldown",
1007
- 0,
1008
- null,
1009
- reason("Request-scoped upstream rejection"),
1010
- false,
1011
- );
1012
- }
1013
- // A channel rejection (`11128`) is the same shape: the upstream is refusing
1014
- // the caller for this account, not this particular prompt, and it keeps
1015
- // refusing on every invocation. Classifying it as `unknown` with
1016
- // `mutates=false` would leave the account in rotation failing every request —
1017
- // and since every account fails the same way, the whole pool looks dead with
1018
- // nothing ever parked.
1019
- const channelRejection =
1020
- providerCode === "11128" || lower.includes("unapproved channel");
1021
- const channelBlock =
1022
- channelRejection &&
1023
- options?.providerId !== undefined &&
1024
- BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
1025
- if (channelBlock) {
1026
- // Same split as the policy arm above: a ban disables, a channel refusal
1027
- // only stops this request. `11-128` is the request shape, so this branch
1028
- // nearly always takes the refusal path — but it shares the ban test so a
1029
- // `11140`-shaped body arriving as a channel rejection still disables.
1030
- if (buddyBanned) {
1031
- return result(
1032
- "policy_blocked_banned",
1033
- "disabled",
1034
- 0,
1035
- null,
1036
- reason("Account banned by upstream"),
1037
- true,
1038
- );
1039
- }
1040
- return result(
1041
- "policy_blocked_invalid_request",
1042
- "cooldown",
1043
- 0,
1044
- null,
1045
- reason("Provider channel rejection"),
1046
- // The credential is valid; the upstream refuses this calling channel.
1047
- // Cooling was the bug: it kept a refused account in the plan, where a
1048
- // client's retry budget burned healthy siblings for the whole window.
1049
- false,
1050
- );
1051
- }
1052
- // server-side tool (web search, x search, web fetch), not evidence about the
1053
- // credential or the account's quota. Some providers surface it as 403, which
1054
- // would otherwise look like an auth signal — refreshing the token cannot fix
1055
- // a search backend, so it must never disable the account.
1056
- const hostedToolFailure =
1057
- lower.includes("web_search") ||
1058
- lower.includes("web search") ||
1059
- lower.includes("web_search_preview") ||
1060
- lower.includes("x_search") ||
1061
- lower.includes("web_fetch") ||
1062
- lower.includes("tool invocation") ||
1063
- lower.includes("tool call failed") ||
1064
- lower.includes("search backend");
1065
- // A model-scoped license or entitlement rejection is NOT credential invalidation.
1066
- // Google Cloud Code / Antigravity answers HTTP 403 with "You do not have a valid
1067
- // license of this product" when an account lacks entitlement for one specific model
1068
- // (e.g. claude-opus-4-6), while the account itself and its tokens remain fully valid
1069
- // for all other models. Disabling the whole account breaks healthy models.
1070
- const modelEntitlementFailure =
1071
- statusCode === 403 &&
1072
- (lower.includes("not have a valid license") ||
1073
- lower.includes("valid license of this product") ||
1074
- lower.includes("model not supported") ||
1075
- lower.includes("not entitled to") ||
1076
- lower.includes("no access to model") ||
1077
- lower.includes("model_not_supported") ||
1078
- lower.includes("not available in your region"));
1079
- /**
1080
- * Google Cloud Code / Antigravity account verification: the upstream answers
1081
- * an unverified account with HTTP 403 `PERMISSION_DENIED` and a
1082
- * `VALIDATION_REQUIRED` reason plus a `validation_url`. The operator must
1083
- * complete verification before the credential can ever serve again, so the
1084
- * account must be parked (`disabled`) — not left `active`, and not a
1085
- * cooldown (no amount of waiting clears it). The validation URL is carried
1086
- * in the recorded reason so the dashboard can surface it as an actionable
1087
- * pill link.
1088
- */
1089
- const validationRequired =
1090
- (statusCode === 403 || lower.includes("permission_denied")) &&
1091
- (lower.includes("validation_required") ||
1092
- lower.includes("verify your account") ||
1093
- lower.includes("validation_url"));
1094
- const validationUrl = validationRequired ? extractValidationUrl(message, rawUpstreamResponse) : undefined;
1095
- /**
1096
- * The upstream accepted the request and put it in a queue behind other
1097
- * callers — it is busy, not rejecting the caller's credential.
1098
- *
1099
- * Qoder answers `403 {"code":"10605","message":"{\"isQueued\":true,
1100
- * \"modelKey\":\"dfmodel\",\"queueCount\":1395,...}"}`. The 403 is what made
1101
- * this misclassify: the generic `(401 || 403) && accountEvidence` arm claimed
1102
- * it as `auth_invalidated`, so a healthy account was parked as "Re-login
1103
- * required" purely because the provider was busy, and an operator was sent
1104
- * to re-authenticate a credential that was never rejected.
1105
- *
1106
- * Matched on the queue evidence rather than on the bare `10605`, so an
1107
- * unrelated 403 that happens to share the code is not swept in. The bodies
1108
- * are nested JSON, so the tell is the key, not a parse of the envelope.
1109
- */
1110
- const queuedUpstream =
1111
- providerCode === "10605" ||
1112
- lower.includes("isqueued") ||
1113
- lower.includes("queuecount") ||
1114
- lower.includes("queuetype") ||
1115
- (lower.includes("queue") && lower.includes("modelkey"));
1116
-
1117
- const signals: AccountErrorSignals = {
1118
- message,
1119
- lower,
1120
- folded,
1121
- statusCode,
1122
- origin,
1123
- scope,
1124
- providerCode,
1125
- statedCooldownMs,
1126
- reason,
1127
- result,
1128
- canMutate,
1129
- accountEvidence,
1130
- revocationNamed,
1131
- providerId: options?.providerId,
1132
- modelId,
1133
- modelBlockHits,
1134
- staticToken,
1135
- credentialKind,
1136
- buddy,
1137
- buddyBanned,
1138
- priceRefusal,
1139
- deploymentInfraError,
1140
- quotaSignal,
1141
- policyRejection,
1142
- requestScoped,
1143
- channelRejection,
1144
- hostedToolFailure,
1145
- modelEntitlementFailure,
1146
- queuedUpstream,
1147
- validationRequired,
1148
- validationUrl,
1149
- };
1150
-
1151
- for (const rule of ACCOUNT_ERROR_RULES) {
1152
- if (rule.match(signals)) return rule.decide(signals);
1153
- }
1154
-
1155
- // Same reasoning as above, with the shortest backoff: an unmapped failure
1156
- // must still be retried automatically rather than parked forever.
1157
- const unclassifiedMs = UNCLASSIFIED_COOLDOWN_MS();
1158
- return result(
1159
- "unknown",
1160
- "cooldown",
1161
- unclassifiedMs,
1162
- new Date(Date.now() + unclassifiedMs),
1163
- reason("Unclassified provider failure"),
1164
- false,
1165
- );
1166
- }
1167
-
1168
- /**
1169
- * The ordered arms of {@link classifyAccountError}, tried top to bottom; the
1170
- * first match decides. Order is behavior, not presentation — see
1171
- * {@link AccountErrorRule}.
1172
- */
1173
- const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
1174
- {
1175
- // Must precede "rate-limit": this rejection is a 429 whose own text says
1176
- // "rate limit", so the broader arm claimed it and cooled the account for
1177
- // the full 15-minute rate-limit window.
1178
- //
1179
- // A full max-inflight is a statement about the gateway's own routing, not
1180
- // about the credential: every account serving the model is busy right now.
1181
- // Cooling them marks healthy accounts as `cooldown` for a limit their
1182
- // provider never applied, which is what made every account read busy while
1183
- // incoming requests piled up — the accounts were never the problem, the
1184
- // ceiling was.
1185
- name: "gateway-capacity-exhausted",
1186
- match: (s) =>
1187
- s.lower.includes("at its concurrency limit") ||
1188
- s.lower.includes("every proxy pool for this provider is at its concurrency limit") ||
1189
- s.lower.includes("the gateway is at its concurrency limit"),
1190
- decide: (s) =>
1191
- s.result(
1192
- "model_capacity",
1193
- // `status` is the status the account would move to; it is never
1194
- // applied because `mutatesAccount` is false below, so this names the
1195
- // shape rather than a transition. Zero cooldown is the point: no wait
1196
- // is attached to the account, so the next request may use it at once.
1197
- "cooldown",
1198
- 0,
1199
- null,
1200
- s.reason("Gateway concurrency ceiling reached; no account was at fault"),
1201
- // Never mutates: there is nothing wrong with the account to record.
1202
- false,
1203
- ),
1204
- },
1205
- {
1206
- // Must precede the unclassified fallback: the fallback is account-blind and
1207
- // would stamp this as a provider-wide failure, which is how one proxy
1208
- // refusing CONNECT put every account of every provider behind that pool
1209
- // into cooldown — opencode's dead front door taking codebuddy down with it.
1210
- //
1211
- // A CONNECT refusal is a statement about the proxy pool, not about a
1212
- // credential: the tunnel was never established, so no upstream ever saw the
1213
- // request and no account can be at fault. Scoped to the pool so the
1214
- // provider's own accounts stay in rotation.
1215
- name: "proxy-connect-refusal",
1216
- match: (s) =>
1217
- s.lower.includes("proxy connect failed") ||
1218
- (s.lower.includes("connect") && s.lower.includes("tunnel") && s.lower.includes("refus")),
1219
- decide: (s) =>
1220
- s.result(
1221
- "model_unreachable",
1222
- "cooldown",
1223
- 0,
1224
- null,
1225
- s.reason("Proxy pool refused the CONNECT tunnel"),
1226
- // Never mutates: the pool owns this failure, not the account.
1227
- false,
1228
- ),
1229
- },
1230
- {
1231
- name: "deployment-infrastructure",
1232
- match: (s) => s.deploymentInfraError,
1233
- decide: (s) =>
1234
- s.result(
1235
- "server_error",
1236
- "cooldown",
1237
- 60_000,
1238
- null,
1239
- `Proxy deployment error: ${s.message.slice(0, 160)}`,
1240
- false,
1241
- ),
1242
- },
1243
- {
1244
- name: "model-frequency-window",
1245
- match: (s) => s.lower.includes("frequency limit") && s.lower.includes("switch to the other models"),
1246
- decide: (s) => {
1247
- const cooldownMs = s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
1248
- return s.result(
1249
- "rate_limit_transient",
1250
- "cooldown",
1251
- cooldownMs,
1252
- new Date(Date.now() + cooldownMs),
1253
- s.reason("Provider frequency limit for this model"),
1254
- // Provider-scoped evidence still has to reach the writer: a stream frame
1255
- // carries no credential detail, so `canMutate` is false and without this
1256
- // the pair would be retried for the whole window it was closed for.
1257
- s.origin === "upstream" && s.scope === "provider",
1258
- );
1259
- },
1260
- },
1261
- {
1262
- name: "content-policy-block",
1263
- match: (s) => s.policyRejection && s.buddy,
1264
- decide: (s) => {
1265
- // A ban is a verdict on the credential, and cooldown cannot answer it:
1266
- // the account is refused, not throttled. Disabling is the fix, and
1267
- // disabling *immediately* is a feature — a client's retry budget is spent
1268
- // in seconds, so a banned account that merely cooled stayed in the plan
1269
- // and burned sibling accounts that were still healthy before the window
1270
- // expired.
1271
- //
1272
- // The two shapes are told apart by the body, not the code alone: `11140`
1273
- // names a restriction on the account, `11-128` refuses the request.
1274
- if (s.buddyBanned) {
1275
- return s.result(
1276
- "policy_blocked_banned",
1277
- "disabled",
1278
- 0,
1279
- null,
1280
- s.reason("Account banned by upstream"),
1281
- // A ban is evidence about the credential even at provider scope, for
1282
- // the same reason a named revocation is: the message is the evidence.
1283
- // Without this a stream frame's missing credential detail would leave
1284
- // a banned account serving.
1285
- true,
1286
- );
1287
- }
1288
- // Not a ban: the upstream refused this request. The account is fine, so
1289
- // the only correct move is to stop retrying it verbatim. Cooling the
1290
- // account here was the bug that made one malformed request look like a
1291
- // dead credential.
1292
- return s.result(
1293
- "policy_blocked_invalid_request",
1294
- "cooldown",
1295
- 0,
1296
- null,
1297
- s.reason("Upstream refused this request"),
1298
- // Never mutates: a malformed body fails identically on every account,
1299
- // so cooling would burn a healthy credential on the gateway's own fault.
1300
- false,
1301
- );
1302
- },
1303
- },
1304
- {
1305
- name: "request-scoped-rejection",
1306
- match: (s) => s.requestScoped && s.buddy,
1307
- decide: (s) =>
1308
- s.result("unknown", "cooldown", 0, null, s.reason("Request-scoped upstream rejection"), false),
1309
- },
1310
- {
1311
- name: "channel-rejection",
1312
- match: (s) => s.channelRejection && s.buddy,
1313
- decide: (s) => {
1314
- const channelBlockMs = resolveAccountPolicyBlockCooldownMs();
1315
- return s.result(
1316
- "policy_blocked",
1317
- "cooldown",
1318
- channelBlockMs,
1319
- new Date(Date.now() + channelBlockMs),
1320
- s.reason("Provider channel rejection"),
1321
- s.origin === "upstream",
1322
- );
1323
- },
1324
- },
1325
- {
1326
- name: "validation-required",
1327
- match: (s) => s.validationRequired,
1328
- decide: (s) =>
1329
- s.result(
1330
- "validation_required",
1331
- "disabled",
1332
- 0,
1333
- null,
1334
- s.validationUrl !== undefined
1335
- ? `${s.reason("Google account verification required")} — ${s.validationUrl}`
1336
- : s.reason("Google account verification required"),
1337
- // The upstream named the verdict; the account cannot serve until the
1338
- // operator completes verification.
1339
- true,
1340
- ),
1341
- },
1342
- {
1343
- name: "credential-rejected",
1344
- match: (s) =>
1345
- !s.quotaSignal &&
1346
- !s.policyRejection &&
1347
- !s.hostedToolFailure &&
1348
- !s.modelEntitlementFailure &&
1349
- !s.queuedUpstream &&
1350
- (s.providerCode === "authentication_failed" ||
1351
- s.folded.includes("invalid api key") ||
1352
- s.folded.includes("incorrect api key") ||
1353
- s.folded.includes("invalid token") ||
1354
- s.folded.includes("token revoked") ||
1355
- s.folded.includes("invalidated oauth token") ||
1356
- s.folded.includes("reauthorization required") ||
1357
- s.folded.includes("account deactivated") ||
1358
- // Trae answers `1001 "we are not able to authenticate you"` when the
1359
- // credential is dead or invalidated — it is the provider's own auth
1360
- // verdict, but the code is numeric, so it is excluded from the bare
1361
- // 401/403 arm below. Recognising it here keeps the account out of the
1362
- // `unknown` fallback (which never mutates and leaves the account in
1363
- // rotation to be re-dialled on every request).
1364
- ((s.providerId === "trae" || s.providerId === "traecn") &&
1365
- s.providerCode === "1001") ||
1366
- // A bare 401/403 only proves the credential when the body actually
1367
- // says so. A purely numeric provider code carries no auth vocabulary —
1368
- // providers encode limit/entitlement verdicts as numbers too (Qoder's
1369
- // `110`, buddy's `6004`) — and treating an unknown one as credential
1370
- // evidence parked healthy accounts as "Re-login required". Unmapped
1371
- // numeric codes fall through to the unclassified cooldown instead of
1372
- // ever invalidating.
1373
- ((s.statusCode === 401 || s.statusCode === 403) &&
1374
- s.accountEvidence &&
1375
- !NUMERIC_PROVIDER_CODE.test(s.providerCode))),
1376
- decide: (s) => {
1377
- // A static token has no refresh to run, so an auth rejection must not park
1378
- // the account: disabling it throws away a token that may still work (a
1379
- // single upstream 401 can be transient). A cooldown lets the next attempt
1380
- // retry the same token without losing the account.
1381
- const oauthRecovery =
1382
- !s.staticToken &&
1383
- s.credentialKind === "oauth" &&
1384
- (s.lower.includes("expired") || s.lower.includes("refresh"));
1385
- // A *named* revocation is the one auth outcome that still parks the account.
1386
- // The upstream said the credential itself is dead — not that it expired, not
1387
- // that this request was refused — so there is no window to wait out and no
1388
- // refresh to run, and a cooldown would only return it to rotation to fail
1389
- // again. Every other auth rejection is weak evidence (clock skew, a key
1390
- // rotated a moment ago, an edge answering before the credential loads) and
1391
- // cools down instead.
1392
- //
1393
- // This distinction is the whole fix. Reaching `disabled` from the generic
1394
- // `401`/`403` branch would be unrecoverable by any sweep, so one transient
1395
- // rejection would remove the account from rotation until an operator
1396
- // noticed — the exact outcome the comment above says a static token must
1397
- // not suffer.
1398
- const revoked = s.revocationNamed;
1399
- const recoverable = s.staticToken || oauthRecovery || !revoked;
1400
- // An automatic disable is **not** terminal any more, but the deadline is what
1401
- // makes that safe: `sweepExpiredCooldowns` recovers a disabled row only when
1402
- // its `cooldownUntil` has elapsed, and an operator-disabled row keeps that
1403
- // column null. So the machine can undo its own verdict while an operator's
1404
- // decision still outranks it — see `resolveAccountDisabledRecoveryMs`.
1405
- const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
1406
- const disabledRetryAt =
1407
- disabledRecoveryMs > 0 ? new Date(Date.now() + disabledRecoveryMs) : null;
1408
- const cooldownMs = recoverable ? resolveAccountAuthRejectedCooldownMs() : disabledRecoveryMs;
1409
- return s.result(
1410
- "auth_invalidated",
1411
- recoverable ? "cooldown" : "disabled",
1412
- cooldownMs,
1413
- recoverable ? new Date(Date.now() + cooldownMs) : disabledRetryAt,
1414
- s.reason(
1415
- !revoked
1416
- ? "Provider credential rejected; retrying after a backoff"
1417
- : s.staticToken
1418
- ? "Static token rejected; retrying without a refresh"
1419
- : oauthRecovery
1420
- ? "OAuth credential requires refresh"
1421
- : "Provider credential revoked",
1422
- ),
1423
- s.canMutate,
1424
- );
1425
- },
1426
- },
1427
- {
1428
- name: "price-refusal",
1429
- match: (s) => s.priceRefusal,
1430
- decide: (s) =>
1431
- s.result("quota_exhausted", "cooldown", 0, null, s.reason("Request price refused by the provider"), false),
1432
- },
1433
- {
1434
- // Qoder's code-112 pricing refusal (`{"code":"112","message":"{\"pricingUrl\":...}"}`)
1435
- // is a "this model is not included in this account's plan, buy more" verdict
1436
- // for one pair — the account keeps serving its other models. The probe path
1437
- // does not always carry the code in its evidence (only the message does), so
1438
- // the match keys on the provider id plus the pricing-page wording as well.
1439
- // It must run before `quota-exhausted`, which would park the whole account.
1440
- name: "qoder-plan-gated",
1441
- match: (s) =>
1442
- (s.providerId === "qoder" || s.providerId === "qodercn") &&
1443
- (s.providerCode === "112" || s.lower.includes("pricingurl")),
1444
- decide: (s) => {
1445
- const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1446
- return s.result(
1447
- "model_unreachable",
1448
- "cooldown",
1449
- cooldownMs,
1450
- new Date(Date.now() + cooldownMs),
1451
- s.reason("Model not included in this account's plan"),
1452
- s.origin === "upstream" && s.modelId !== undefined,
1453
- true,
1454
- );
1455
- },
1456
- },
1457
- {
1458
- name: "model-plan-gated",
1459
- match: (s) => s.quotaSignal && s.providerCode === "112",
1460
- decide: (s) => {
1461
- const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1462
- return s.result(
1463
- "model_unreachable",
1464
- "cooldown",
1465
- cooldownMs,
1466
- new Date(Date.now() + cooldownMs),
1467
- s.reason("Model not included in this account's plan"),
1468
- s.origin === "upstream" && s.modelId !== undefined,
1469
- true,
1470
- );
1471
- },
1472
- },
1473
- {
1474
- name: "quota-exhausted",
1475
- match: (s) => s.quotaSignal,
1476
- decide: (s) => {
1477
- // xAI Grok Build's free tier resets on a rolling window. Use a duration
1478
- // stated by the provider when present; otherwise park for the conservative
1479
- // 24-hour fallback.
1480
- const grokFreeTier =
1481
- s.providerCode === "subscription:free-usage-exhausted" ||
1482
- s.providerCode.startsWith("subscription:") ||
1483
- s.lower.includes("free-usage") ||
1484
- s.lower.includes("free usage") ||
1485
- s.lower.includes("included free usage") ||
1486
- s.lower.includes("rolling 24-hour") ||
1487
- s.lower.includes("rolling 24 hour");
1488
- const buddyCreditsExhausted =
1489
- s.providerCode === "14018" || s.lower.includes("credits exhausted");
1490
- const cooldownMs = grokFreeTier
1491
- ? (s.statedCooldownMs ?? GROK_QUOTA_COOLDOWN_MS)
1492
- : buddyCreditsExhausted
1493
- ? (s.statedCooldownMs ?? 12 * 60 * 60 * 1000)
1494
- : (s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS());
1495
- // The caller records against the concrete account that just failed
1496
- // upstream, so a provider-scoped quota error still cools THAT account
1497
- // down (enables fallback to the next account). Local-origin quota must
1498
- // never mutate an upstream account.
1499
- return s.result(
1500
- "quota_exhausted",
1501
- "cooldown",
1502
- cooldownMs,
1503
- new Date(Date.now() + cooldownMs),
1504
- s.reason("Provider quota exhausted"),
1505
- s.origin === "upstream",
1506
- );
1507
- },
1508
- },
1509
- {
1510
- // Inception's Free tier throttles per minute (1 000 requests / 1M input
1511
- // tokens / 100K output tokens) and the window resets every minute. A
1512
- // generic 429 would cool the account for the 15-minute default, leaving a
1513
- // healthy key parked long after the window refilled — the source of
1514
- // "limitasiya lumayan keras". This arm must run before `rate-limit`: it
1515
- // matches the same signals and shortens the window to 1 minute so routing
1516
- // moves to the next account and the flagged one returns almost at once.
1517
- name: "inception-per-minute-limit",
1518
- match: (s) =>
1519
- s.providerId === "inception" &&
1520
- (s.statusCode === 429 ||
1521
- s.providerCode === "rate_limit_reached" ||
1522
- s.providerCode === "rate_limit_exceeded" ||
1523
- s.lower.includes("rate_limit") ||
1524
- s.lower.includes("rate limit") ||
1525
- s.lower.includes("too many requests") ||
1526
- s.lower.includes("tokens per minute") ||
1527
- s.lower.includes("input token")),
1528
- decide: (s) => {
1529
- const cooldownMs = s.statedCooldownMs ?? INCEPTION_RATE_LIMIT_COOLDOWN_MS;
1530
- return s.result(
1531
- "rate_limit_transient",
1532
- "cooldown",
1533
- cooldownMs,
1534
- new Date(Date.now() + cooldownMs),
1535
- s.reason("Inception per-minute input-token limit"),
1536
- s.origin === "upstream",
1537
- );
1538
- },
1539
- },
1540
- {
1541
- name: "rate-limit",
1542
- match: (s) =>
1543
- s.statusCode === 429 ||
1544
- s.providerCode === "rate_limit_exceeded" ||
1545
- s.lower.includes("rate_limit") ||
1546
- s.lower.includes("rate limit") ||
1547
- s.lower.includes("too many requests") ||
1548
- s.lower.includes("requests per minute") ||
1549
- s.lower.includes("tokens per minute") ||
1550
- s.lower.includes("tpm limit") ||
1551
- s.lower.includes("rpm limit"),
1552
- decide: (s) => {
1553
- const cooldownMs = s.statedCooldownMs ?? DEFAULT_RATE_LIMIT_COOLDOWN_MS();
1554
- // Same per-account reasoning as quota above: the failing account (or
1555
- // its model, when modelId is present) backs off so routing tries the
1556
- // next candidate instead of hammering the same key.
1557
- return s.result(
1558
- "rate_limit_transient",
1559
- "cooldown",
1560
- cooldownMs,
1561
- new Date(Date.now() + cooldownMs),
1562
- s.reason("Provider rate limit"),
1563
- s.origin === "upstream",
1564
- );
1565
- },
1566
- },
1567
- {
1568
- name: "model-capacity",
1569
- match: (s) =>
1570
- s.statusCode === 503 ||
1571
- s.statusCode === 529 ||
1572
- s.providerCode === "model_capacity" ||
1573
- s.queuedUpstream ||
1574
- s.lower.includes("model overloaded") ||
1575
- s.lower.includes("capacity exceeded") ||
1576
- s.lower.includes("temporarily unavailable"),
1577
- decide: (s) => {
1578
- const cooldownMs = s.statedCooldownMs ?? MODEL_CAPACITY_COOLDOWN_MS();
1579
- const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1580
- return s.result(
1581
- "model_capacity",
1582
- "cooldown",
1583
- cooldownMs,
1584
- new Date(Date.now() + cooldownMs),
1585
- s.reason("Provider model capacity"),
1586
- modelScopedMutation,
1587
- );
1588
- },
1589
- },
1590
- {
1591
- name: "model-unsupported",
1592
- match: (s) => isUnsupportedModelEvidence(s.statusCode, s.providerCode, s.folded),
1593
- decide: (s) => {
1594
- const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1595
- return s.result(
1596
- "model_unreachable",
1597
- "cooldown",
1598
- cooldownMs,
1599
- new Date(Date.now() + cooldownMs),
1600
- s.reason("Model not supported by this upstream"),
1601
- s.origin === "upstream" && s.modelId !== undefined,
1602
- true,
1603
- );
1604
- },
1605
- },
1606
- {
1607
- name: "model-unreachable",
1608
- match: (s) => s.providerCode === "not_found" || s.modelEntitlementFailure,
1609
- decide: (s) => {
1610
- const cooldownMs = MODEL_UNREACHABLE_COOLDOWN_MS;
1611
- // Not credential evidence, so `classifyUpstreamFailure` scopes this to the
1612
- // provider and the `scope === "account"` gate the throttles use would
1613
- // forbid the mutation outright — leaving the probe reporting a failure that
1614
- // cooled nothing. The verdict is still about one (account, model) pair:
1615
- // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
1616
- // account's own status untouched, so all that is required is a model to
1617
- // write against.
1618
- const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1619
- return s.result(
1620
- "model_unreachable",
1621
- "cooldown",
1622
- cooldownMs,
1623
- new Date(Date.now() + cooldownMs),
1624
- s.reason("Model unreachable for this account"),
1625
- modelScopedMutation,
1626
- );
1627
- },
1628
- },
1629
- {
1630
- name: "transport-fault",
1631
- match: (s) =>
1632
- (s.statusCode !== null && s.statusCode >= 500 && s.statusCode <= 504) ||
1633
- s.lower.includes("timeout") ||
1634
- s.lower.includes("abort") ||
1635
- s.lower.includes("econnrefused") ||
1636
- s.lower.includes("fetch failed"),
1637
- decide: (s) => {
1638
- // A transport-shaped fault clears on its own, so it is a cooldown with a
1639
- // deadline rather than a state needing an operator. The deadline matters:
1640
- // `sweepExpiredCooldowns` selects on `cooldownUntil IS NOT NULL`, so a row
1641
- // parked without one is never recovered and stays out of rotation.
1642
- //
1643
- // `canMutate` rather than a flat `false`: a 5xx whose failure the caller
1644
- // scoped to the account (an edge/WAF challenge answered to one account's
1645
- // requests while its siblings on the same egress served fine) must cool
1646
- // that account down, or it stays `active` and is re-dialled for as long as
1647
- // the block lasts. Provider-scoped 5xx does not: the upstream is
1648
- // indiscriminately failing and no single account is at fault.
1649
- const transientMs = TRANSIENT_ERROR_COOLDOWN_MS();
1650
- return s.result(
1651
- s.lower.includes("timeout") ? "timeout" : "server_error",
1652
- "cooldown",
1653
- transientMs,
1654
- new Date(Date.now() + transientMs),
1655
- s.reason(`Provider ${s.statusCode ?? "network"} failure`),
1656
- s.canMutate,
1657
- );
1658
- },
1659
- },
1660
- ];
1
+ /**
2
+ * Account error classification: turns one upstream failure into a category,
3
+ * a cooldown window, and a verdict on the account.
4
+ *
5
+ * This is the decision half of the account health machine. It is pure: it
6
+ * reads evidence and returns a classification, writes nothing, and knows
7
+ * nothing about the database. The writer half lives in
8
+ * `account-health-service.ts`.
9
+ */
10
+ import { isRecord } from "../../protocol/codecs";
11
+ import { GatewayError } from "../../protocol/gateway-error";
12
+ import { BUDDY_PROVIDER_IDS } from "../provider-metadata";
13
+ import { parseProviderResetDuration, parseUpstreamBackoff } from "../../protocol/upstream-parsing";
14
+ import {
15
+ resolveAccountAuthRejectedCooldownMs,
16
+ resolveAccountCooldownEscalationMaxMs,
17
+ resolveAccountDisabledRecoveryMs,
18
+ resolveAccountModelCapacityCooldownMs,
19
+ resolveAccountPolicyBlockCooldownMs,
20
+ resolveAccountQuotaCooldownMs,
21
+ resolveAccountRateLimitCooldownMs,
22
+ resolveAccountTransientCooldownMs,
23
+ resolveAccountUnclassifiedCooldownMs,
24
+ } from "../../config/cooldowns";
25
+ const DEFAULT_RATE_LIMIT_COOLDOWN_MS = resolveAccountRateLimitCooldownMs;
26
+ /**
27
+ * Inception's per-minute limit window (1 minute).
28
+ *
29
+ * Inception's Free tier throttles per minute (1 000 requests / 1M input
30
+ * tokens / 100K output tokens) and the window resets every minute. The generic
31
+ * 15-minute rate-limit fallback would park a healthy key long after the window
32
+ * refilled; 1 minute is the provider's own cadence, hardcoded because it is a
33
+ * fact about the upstream, not a deployment knob.
34
+ */
35
+ const INCEPTION_RATE_LIMIT_COOLDOWN_MS = 60_000;
36
+ // Fallback when the error states no usable duration: deliberately short (1h,
37
+ // not 24h) — quota state is re-probed on the next failure, so an unknown
38
+ // reset cadence self-corrects instead of parking the account for a day.
39
+ const DEFAULT_QUOTA_COOLDOWN_MS = resolveAccountQuotaCooldownMs;
40
+ const MODEL_CAPACITY_COOLDOWN_MS = resolveAccountModelCapacityCooldownMs;
41
+ const TRANSIENT_ERROR_COOLDOWN_MS = resolveAccountTransientCooldownMs;
42
+ /**
43
+ * Accounts one cooldown-recovery pass brings back. The sweep ticks every 30s,
44
+ * so a larger backlog drains over successive passes rather than one statement
45
+ * covering every expired row.
46
+ */
47
+ export const COOLDOWN_RECOVERY_BATCH = 500;
48
+ /**
49
+ * Backoff for a failure no rule matched. Short on purpose: an unclassified
50
+ * error is more likely a new upstream shape than a broken account, so the
51
+ * account returns to rotation quickly.
52
+ */
53
+ const UNCLASSIFIED_COOLDOWN_MS = resolveAccountUnclassifiedCooldownMs;
54
+ /**
55
+ * xAI Grok Build's free tier resets on a rolling 24-hour window, so its
56
+ * exhaustion is a full-day quota cooldown — never the generic 1h fallback,
57
+ * which let the account re-enter rotation inside the window.
58
+ */
59
+ const GROK_QUOTA_COOLDOWN_MS = 24 * 60 * 60 * 1000;
60
+ /**
61
+ * The buddy family's policy/channel block (`11140`, `11128`) is a *base* backoff,
62
+ * not a flat penalty — see {@link resolveAccountPolicyBlockCooldownMs}. It used
63
+ * to be two constants, 24h for the content block and 6h for the channel one,
64
+ * both applied on the first observation. That was wrong in the direction that
65
+ * costs most: `11140` is also what a wrong desktop `User-Agent` brand answers
66
+ * (`workbuddy/shared.ts`), so after we fixed a header regression every buddy
67
+ * account stayed dead for up to a further day, and one prompt that failed a
68
+ * safety review cost a whole account a day. As a base it escalates through the
69
+ * consecutive-failure ladder, which tells the two causes apart by what they do
70
+ * next — a genuinely blocked account is refused again immediately and parks
71
+ * itself within minutes.
72
+ */
73
+ /**
74
+ * Backoff for a model the upstream says does not exist for this account
75
+ * (Cursor answers Connect `not_found` for a model the credential's plan does
76
+ * not reach). Long, because the verdict is a property of the plan and will not
77
+ * change on the next request — but bounded, so a plan upgrade is picked up
78
+ * without an operator clearing it by hand.
79
+ */
80
+ const MODEL_UNREACHABLE_COOLDOWN_MS = 6 * 60 * 60 * 1000;
81
+ /**
82
+ * Backoff for a model the upstream does not serve at all (buddy family
83
+ * business code `11102` — `service info not found` — or a `model ... not
84
+ * found/supported` message). Mirrors the reference gateways'
85
+ * `MODEL_SITE_BLOCK_S` → `MODEL_SITE_BLOCK_MAX_S`.
86
+ *
87
+ * Escalating rather than flat: the pair is re-dialled after the first window
88
+ * and answers the same thing, so a flat 6h let a permanently unsupported model
89
+ * consume a request every six hours forever. The ladder tops out at a day,
90
+ * which is long enough to stop the redial and short enough that a backend that
91
+ * starts serving the model is picked up without an operator clearing it.
92
+ *
93
+ * This is the one place the references' *strict* behaviour is adopted as a
94
+ * backoff rather than as a 400: nothing currently succeeds through it, so no
95
+ * client is broken by it.
96
+ */
97
+ const UNSUPPORTED_MODEL_COOLDOWN_MS = 6 * 60 * 60 * 1000;
98
+ const UNSUPPORTED_MODEL_COOLDOWN_MAX_MS = 24 * 60 * 60 * 1000;
99
+ /** How many doublings an unsupported-model cooldown may take before the cap. */
100
+ const UNSUPPORTED_MODEL_MAX_LEVEL = 8;
101
+
102
+ /**
103
+ * The cooldown for the *nth* consecutive unsupported-model report (`hits` is
104
+ * the new total, so the first report is `hits === 1`).
105
+ *
106
+ * Exported because the writer, not the classifier, is the only place that
107
+ * reads the authoritative hit count — the classifier sees only the count the
108
+ * caller supplied, which is absent on the first report of a pair.
109
+ */
110
+ export function unsupportedModelCooldownMs(hits: number): number {
111
+ const level = Math.min(Math.max(0, Math.round(hits) - 1), UNSUPPORTED_MODEL_MAX_LEVEL);
112
+ return Math.min(
113
+ UNSUPPORTED_MODEL_COOLDOWN_MS * 2 ** level,
114
+ UNSUPPORTED_MODEL_COOLDOWN_MAX_MS,
115
+ );
116
+ }
117
+
118
+ /**
119
+ * The wording an upstream uses for "we do not serve this model at all", as
120
+ * opposed to "this account cannot reach it" (403 entitlement, above) or "this
121
+ * route does not exist" (404 `upstream_not_found`).
122
+ */
123
+ const UNSUPPORTED_MODEL_MESSAGE =
124
+ /service info not found|model\s.{0,80}?\bnot\s+(?:found|supported)\b/;
125
+
126
+ /**
127
+ * True when the evidence says the upstream does not serve the model.
128
+ *
129
+ * Narrow on purpose, and narrower than `model_unreachable`: only a 400/404
130
+ * carrying the buddy family's `11102` or one of the two phrasings above. A
131
+ * 4xx that merely *mentions* a model is far more often a payload the caller
132
+ * got wrong, and cooling a pair for six hours over that takes a working model
133
+ * out of rotation for a request bug.
134
+ */
135
+ function isUnsupportedModelEvidence(
136
+ statusCode: number | null,
137
+ providerCode: string,
138
+ lower: string,
139
+ ): boolean {
140
+ if (statusCode !== 400 && statusCode !== 404) return false;
141
+ return providerCode === "11102" || UNSUPPORTED_MODEL_MESSAGE.test(lower);
142
+ }
143
+
144
+ // ===== health/account-recorder.ts =====
145
+ export interface AccountHealthEventRecord {
146
+ readonly id: string;
147
+ readonly accountId: string;
148
+ readonly fromStatus: string | null;
149
+ readonly toStatus: string;
150
+ readonly reason: string | null;
151
+ readonly errorCategory: string | null;
152
+ readonly modelId: string | null;
153
+ readonly createdAt: string;
154
+ }
155
+
156
+ export type AccountErrorCategory =
157
+ | "quota_exhausted"
158
+ | "rate_limit_transient"
159
+ | "model_capacity"
160
+ | "model_unreachable"
161
+ | "auth_invalidated"
162
+ /**
163
+ * Deprecated: an upstream ban. Kept so existing health rows keep rendering
164
+ * instead of showing a blank cause. No longer produced — see
165
+ * `policy_blocked_banned` and `policy_blocked_invalid_request`.
166
+ */
167
+ | "policy_blocked"
168
+ /**
169
+ * The upstream banned this credential: a verdict on the account, e.g. buddy
170
+ * `11140` "request illegal" with "Your account has been restricted due to
171
+ * unusual activity". Cooldown cannot help — the account is not throttled, it
172
+ * is refused. Disabling immediately is a *feature*: a client's retry budget
173
+ * is spent in seconds, so surviving attempts burn sibling accounts that are
174
+ * still healthy.
175
+ */
176
+ | "policy_blocked_banned"
177
+ /**
178
+ * The upstream refused this request: a verdict on the request. e.g. buddy
179
+ * `11-128` "request illegal" with no account restriction. Distinguished from
180
+ * {@link policy_blocked_banned} because the correct responses differ — a
181
+ * banned account must be disabled, a malformed request must only stop being
182
+ * retried verbatim.
183
+ */
184
+ | "policy_blocked_invalid_request"
185
+ | "server_error"
186
+ | "timeout"
187
+ | "validation_required"
188
+ | "unknown";
189
+ export type AccountFailureOrigin = "cartethyia" | "upstream" | "network";
190
+ export type AccountFailureScope =
191
+ | "account"
192
+ | "provider"
193
+ | "model"
194
+ | "pool"
195
+ | "tenant"
196
+ | "request"
197
+ | "network"
198
+ | "unknown";
199
+
200
+ export interface AccountFailureEvidence {
201
+ readonly origin?: AccountFailureOrigin;
202
+ readonly scope?: AccountFailureScope;
203
+ readonly credentialKind?: string;
204
+ /**
205
+ * The account's credential is a static bearer token used as issued and never
206
+ * refreshed. An auth rejection against a static token is not a refreshable
207
+ * fault: refreshing cannot help, and disabling the account discards a token
208
+ * that may still be valid for other requests. Such a failure cools the
209
+ * account down instead of parking it.
210
+ */
211
+ readonly staticToken?: boolean;
212
+ readonly providerCode?: string;
213
+ readonly credentialEvidence?: boolean;
214
+ readonly retryAfterMs?: number;
215
+ readonly statusCode?: number | null;
216
+ readonly headers?: Headers | Record<string, string> | null;
217
+ readonly modelId?: string;
218
+ /**
219
+ * Provider that produced the failure. Used only for the one
220
+ * provider-family-specific policy rule (the buddy family's `11140`
221
+ * content-policy block), never to broaden what disables an account.
222
+ */
223
+ readonly providerId?: string;
224
+ /**
225
+ * How many times this (account, model) pair has already been reported
226
+ * unsupported, when the caller has read them. Best-effort: the classifier
227
+ * states the base window without it, and the writer recomputes against the
228
+ * count it reads itself.
229
+ */
230
+ readonly modelBlockHits?: number;
231
+ /**
232
+ * The upstream's own error body, when the caller has it.
233
+ *
234
+ * Providers state a reset window in structured fields far more reliably than
235
+ * in prose — a body carrying `resets_at`, `resets_in_seconds` or
236
+ * `limit_window_minutes` is an exact statement, while the same body's
237
+ * `message` ("The usage limit has been reached") names no window at all.
238
+ * Without this the classifier falls back to a fixed 15-minute cooldown for a
239
+ * limit the provider said resets in 23 days.
240
+ */
241
+ readonly upstreamBody?: unknown;
242
+ }
243
+
244
+ export interface AccountErrorClassification {
245
+ readonly category: AccountErrorCategory;
246
+ readonly status: "cooldown" | "disabled";
247
+ readonly cooldownMs: number;
248
+ readonly reason: string;
249
+ readonly retryAt: Date | null;
250
+ readonly mutatesAccount: boolean;
251
+ /**
252
+ * True when this write changed a field the routing snapshot actually reads.
253
+ *
254
+ * Routing reads account-wide `status`/`cooldownUntil` and the per-model
255
+ * `modelCooldowns` map (`route-catalog.ts`). Writes to any of those must drop
256
+ * the snapshot; failure streaks and diagnostic text remain invisible to
257
+ * routing.
258
+ *
259
+ * Only a changed deadline is routing-visible. Re-reporting the same active
260
+ * model cooldown must not rebuild the snapshot again.
261
+ */
262
+ readonly routesVisible: boolean;
263
+ /**
264
+ * True when `cooldownMs` came from the provider's own statement — a parsed
265
+ * `Retry-After`-family header, or a reset quoted in its message — rather than
266
+ * from one of this module's fallback constants.
267
+ *
268
+ * A property of the *evidence*, not of the category, so it is the same for
269
+ * every branch of one classification. Only {@link escalateFallbackCooldown}
270
+ * reads it: a provider-stated window is the truth and is never multiplied.
271
+ */
272
+ readonly providerStated?: boolean;
273
+ /**
274
+ * True when the upstream reported the model as one it does not serve (the
275
+ * buddy family's `11102`), rather than one this account merely cannot reach.
276
+ *
277
+ * A property of the classification, because both verdicts land in the same
278
+ * `model_unreachable` category and the writer cannot otherwise tell them
279
+ * apart — it needs to know whether to count the hit and escalate the window.
280
+ */
281
+ readonly modelUnsupported?: boolean;
282
+ }
283
+
284
+ /**
285
+ * How many doublings a fallback cooldown may take before the cap takes over.
286
+ *
287
+ * Four is enough to reach the cap from every fallback default (the longest is
288
+ * the 1h quota fallback, which reaches it on the first doubling) while keeping
289
+ * the ladder short enough to reason about.
290
+ */
291
+ const COOLDOWN_ESCALATION_MAX_LEVEL = 4;
292
+
293
+ /**
294
+ * A provider code that is digits only. Such codes are identifiers, not auth
295
+ * vocabulary — providers encode spend/entitlement verdicts as numbers (Qoder's
296
+ * `110`, buddy's `6004`) — so the bare-401/403 credential arm must not treat
297
+ * an unknown one as evidence that the credential died.
298
+ */
299
+ const NUMERIC_PROVIDER_CODE = /^\d+$/;
300
+
301
+ /**
302
+ * Escalates a *fallback* cooldown for an account that keeps failing.
303
+ *
304
+ * `consecutiveFailures` has been incremented on every account failure since the
305
+ * health machine was written, but nothing ever read it — so a broken upstream
306
+ * was re-dialled once a minute forever, and an account whose quota had an
307
+ * unstated reset came straight back into rotation every time its short fallback
308
+ * lapsed. The count is now a backoff level: each further consecutive failure
309
+ * doubles the fallback, clamped to {@link resolveAccountCooldownEscalationMaxMs}.
310
+ *
311
+ * Two guards keep it from overriding the provider. A duration the provider
312
+ * *stated* is never multiplied — that window is the truth. And the caller writes
313
+ * the result through a monotonic maximum, so a burst of concurrent in-flight
314
+ * failures advances the deadline to the longest of them instead of compounding
315
+ * once per failure.
316
+ */
317
+ export function escalateFallbackCooldown(
318
+ cooldownMs: number,
319
+ providerStated: boolean,
320
+ consecutiveFailures: number,
321
+ ): number {
322
+ if (providerStated || cooldownMs <= 0) return cooldownMs;
323
+ const level = Math.min(Math.max(0, consecutiveFailures), COOLDOWN_ESCALATION_MAX_LEVEL);
324
+ return Math.min(cooldownMs * 2 ** level, resolveAccountCooldownEscalationMaxMs());
325
+ }
326
+
327
+ /**
328
+ * The later of a candidate deadline and the one already recorded.
329
+ *
330
+ * Every fallback duration is recomputed from *now* on each failure, so writing
331
+ * it unconditionally let a fresh short failure *shorten* a longer window already
332
+ * in force: a 60-second unclassified cooldown overwrote the provider's own
333
+ * "Try again in 11h" and released the pair back into rotation inside a window
334
+ * the upstream had closed, where it failed again — once a minute, forever.
335
+ * A cooling deadline may only ever move later.
336
+ */
337
+ export function latestDeadlineMs(...candidates: readonly (number | undefined | null)[]): number {
338
+ let latest = 0;
339
+ for (const value of candidates) {
340
+ if (typeof value !== "number" || !Number.isFinite(value)) continue;
341
+ if (value > latest) latest = value;
342
+ }
343
+ return latest;
344
+ }
345
+
346
+ /**
347
+ * The account-wide deadline to write, or `null` for "no deadline".
348
+ *
349
+ * {@link latestDeadlineMs} cannot express *no* deadline — it floors at 0 — and
350
+ * the account-wide write has one state that needs it. With
351
+ * `CARTETHYIA_ACCOUNT_DISABLED_RECOVERY_MS=0` an automatic disable is meant to
352
+ * be terminal again, and the classifier correctly reports `retryAt: null`.
353
+ * Deriving the deadline from `Date.now() + cooldownMs` instead stamps *now*.
354
+ * The recovery sweep then treats that elapsed deadline as
355
+ * an automatic disable awaiting recovery and returns the row to rotation on its
356
+ * next tick — the escape hatch enabled exactly the behavior it documents
357
+ * itself as disabling.
358
+ *
359
+ * A candidate only exists when there is a real window to state. Monotonicity is
360
+ * preserved: a deadline already in force still wins over a new `null`.
361
+ */
362
+ export function accountDeadlineMs(
363
+ candidates: readonly (number | undefined | null)[],
364
+ ): Date | null {
365
+ const latest = latestDeadlineMs(...candidates);
366
+ return latest > 0 ? new Date(latest) : null;
367
+ }
368
+
369
+ /**
370
+ * Whether the upstream named a restriction on the account itself.
371
+ *
372
+ * The buddy family reuses "request illegal" for two unrelated rejections, and
373
+ * the difference is only in the body:
374
+ *
375
+ * - `11140` carries `displayMsg` about the account being restricted ("Your
376
+ * account has been restricted due to unusual activity") and `actions`
377
+ * offering an appeal. That is a ban — the account is refused, not throttled.
378
+ * - `11-128` is a refusal of this request; the account is fine.
379
+ *
380
+ * Proven account-scoped (2026-10-16): two live share-page requests carrying
381
+ * byte-identical bodies and headers to the same model differed only in the
382
+ * pooled account, and the blocked one 400'd while the other 200'd.
383
+ *
384
+ * Matched on the code *and* the account-restriction wording, so an unrelated
385
+ * upstream that happens to say "restricted" in prose cannot disable accounts.
386
+ */
387
+ function isBuddyAccountBanned(
388
+ lower: string,
389
+ providerCode: string,
390
+ upstreamBody: unknown,
391
+ ): boolean {
392
+ if (providerCode !== "11140") return false;
393
+ if (banWording(lower)) return true;
394
+ // The message alone is not enough, and this is the case that proved it: the
395
+ // provider's message for a ban is the same two words it uses for a refused
396
+ // request — "request illegal" — while the actual verdict ("Your account has
397
+ // been restricted due to unusual activity") lives in `displayMsg`, which the
398
+ // message extractor never reads. Matching only the message therefore read
399
+ // every ban as a mere request refusal: no category, no disable, and the
400
+ // account kept serving.
401
+ const body = isRecord(upstreamBody) ? upstreamBody : undefined;
402
+ if (body === undefined) return false;
403
+ for (const field of ["displayMsg", "displayTips", "msg", "message"]) {
404
+ const value = body[field];
405
+ if (typeof value === "string" && banWording(value.toLowerCase())) return true;
406
+ // `displayMsg` is a per-language map: `{ en: "…", zh: "…" }`. Any
407
+ // language states the same verdict, so all of them are read.
408
+ if (!isRecord(value)) continue;
409
+ for (const nested of Object.values(value)) {
410
+ if (typeof nested === "string" && banWording(nested.toLowerCase())) return true;
411
+ }
412
+ }
413
+ // `actions` names the remedy: an appeal can only be offered for a verdict
414
+ // about the account, and "edit input" only ever describes a request.
415
+ const actions = body["actions"];
416
+ if (Array.isArray(actions)) {
417
+ for (const action of actions) {
418
+ if (typeof action !== "string") continue;
419
+ if (action.toLowerCase().includes("appeal") || action.toLowerCase().includes("feedback")) {
420
+ return true;
421
+ }
422
+ }
423
+ }
424
+ return false;
425
+ }
426
+
427
+ /** The wording an upstream uses when it restricted the account, not the request. */
428
+ function banWording(lower: string): boolean {
429
+ return (
430
+ lower.includes("restricted") ||
431
+ lower.includes("has been banned") ||
432
+ lower.includes("unusual activity") ||
433
+ lower.includes("appeal this restriction") ||
434
+ lower.includes("submit feedback to appeal") ||
435
+ lower.includes("account has been") ||
436
+ lower.includes("账号存在异常行为") ||
437
+ lower.includes("帳號存在異常行為")
438
+ );
439
+ }
440
+
441
+ /**
442
+ * Bounded to a day, matching `MAX_COOLDOWN_MS` in the failure policy: a stated
443
+ * window longer than that is a monthly or plan-level limit, and parking an
444
+ * account for a month on one upstream field is a decision for the operator,
445
+ * not an automatic one.
446
+ */
447
+ const MAX_UPSTREAM_STATED_COOLDOWN_MS = 24 * 3600 * 1000;
448
+
449
+ /** Parses the stored upstream text as JSON. Returns `undefined` when it is not. */
450
+ function tryParseUpstreamBody(text: string): unknown {
451
+ try {
452
+ const parsed: unknown = JSON.parse(text);
453
+ return typeof parsed === "object" && parsed !== null ? parsed : undefined;
454
+ } catch {
455
+ return undefined;
456
+ }
457
+ }
458
+
459
+ /**
460
+ * Reads a reset window the upstream stated in structured fields.
461
+ *
462
+ * A body like `{"error":{"type":"usage_limit_reached","resets_at":1795019113,
463
+ * "resets_in_seconds":2072028}}` states the window exactly, while its own
464
+ * `message` ("The usage limit has been reached") states none — so parsing only
465
+ * prose leaves a 23-day limit cooling for the 15-minute fallback, and the
466
+ * account re-enters rotation hours before it can serve.
467
+ *
468
+ * Both spellings are read because providers use both: an absolute epoch
469
+ * (`resets_at`, seconds or milliseconds) and a relative delay
470
+ * (`resets_in_seconds`). The later of the two wins, since they are the same
471
+ * claim computed at slightly different instants.
472
+ */
473
+ function parseUpstreamBodyReset(body: unknown): number | null {
474
+ if (typeof body !== "object" || body === null) return null;
475
+ const record = body as Record<string, unknown>;
476
+ // Some providers nest under `error`, some return it at the top level.
477
+ const candidates = [record, (record["error"] as Record<string, unknown> | undefined) ?? undefined];
478
+ let best: number | null = null;
479
+ for (const candidate of candidates) {
480
+ if (typeof candidate !== "object" || candidate === null) continue;
481
+ const now = Date.now();
482
+
483
+ const absolute = candidate["resets_at"] ?? candidate["reset_at"];
484
+ if (typeof absolute === "number" && Number.isFinite(absolute) && absolute > 0) {
485
+ // Seconds vs milliseconds: a plausible epoch in seconds is ~10 digits, in
486
+ // milliseconds ~13, and the two are four orders of magnitude apart — so
487
+ // the smaller is scaled up rather than guessed from a threshold date.
488
+ const epochMs = absolute < 1e11 ? absolute * 1000 : absolute;
489
+ const delay = epochMs - now;
490
+ if (delay > 0) best = best === null ? delay : Math.max(best, delay);
491
+ }
492
+
493
+ const relative =
494
+ candidate["resets_in_seconds"] ??
495
+ candidate["reset_in_seconds"] ??
496
+ candidate["retry_after_seconds"];
497
+ if (typeof relative === "number" && Number.isFinite(relative) && relative > 0) {
498
+ const delay = relative * 1000;
499
+ best = best === null ? delay : Math.max(best, delay);
500
+ }
501
+
502
+ const minutes = candidate["limit_window_minutes"] ?? candidate["reset_in_minutes"];
503
+ if (typeof minutes === "number" && Number.isFinite(minutes) && minutes > 0) {
504
+ const delay = minutes * 60_000;
505
+ best = best === null ? delay : Math.max(best, delay);
506
+ }
507
+ }
508
+ return best === null ? null : Math.min(MAX_UPSTREAM_STATED_COOLDOWN_MS, Math.round(best));
509
+ }
510
+
511
+
512
+ interface AccountErrorContext {
513
+ readonly message: string;
514
+ readonly lower: string;
515
+ readonly folded: string;
516
+ readonly statusCode: number | null;
517
+ readonly origin: AccountFailureOrigin;
518
+ readonly scope: AccountFailureScope;
519
+ readonly revocationNamed: boolean;
520
+ readonly accountEvidence: boolean;
521
+ readonly canMutate: boolean;
522
+ readonly providerCode: string;
523
+ readonly statedCooldownMs: number | null;
524
+ /** Echo of the caller's evidence; read only by the arms that need it. */
525
+ readonly providerId: string | undefined;
526
+ /** The raw upstream response body, when the dispatch path stored it. */
527
+ readonly rawUpstreamResponse: string | undefined;
528
+ /**
529
+ * The upstream named a restriction on the account, not a refusal of the
530
+ * request. Read by the policy arm to pick between disabling the account and
531
+ * merely refusing the request.
532
+ */
533
+ readonly buddyBanned: boolean;
534
+ readonly reason: (prefix: string) => string;
535
+ readonly result: (
536
+ category: AccountErrorCategory,
537
+ status: AccountErrorClassification["status"],
538
+ cooldownMs: number,
539
+ retryAt: Date | null,
540
+ explanation: string,
541
+ mutatesAccount?: boolean,
542
+ modelUnsupported?: boolean,
543
+ ) => AccountErrorClassification;
544
+ }
545
+
546
+ /**
547
+ * One arm of the classifier: an ordered `(match, decide)` pair.
548
+ *
549
+ * The arms are **not** independent predicates. Later arms are deliberately
550
+ * broader than earlier ones — `quotaSignal` matches every price refusal, and
551
+ * `authSignal` matches a superset of the request-scoped codes — so a match
552
+ * only wins because it is tried first. That is why they live in one ordered
553
+ * list rather than behind a dispatch map or a set of independent `if`s: the
554
+ * list is the precedence, and reading it top to bottom is reading the
555
+ * decision.
556
+ *
557
+ * `match` is a pure function of the shared signals (no closure over the
558
+ * classifier's locals, no allocation per classification): the table is built
559
+ * once per call from values already computed for every arm.
560
+ */
561
+ interface AccountErrorRule {
562
+ readonly name: string;
563
+ readonly match: (signals: AccountErrorSignals) => boolean;
564
+ readonly decide: (signals: AccountErrorSignals) => AccountErrorClassification;
565
+ }
566
+
567
+ /** The shared signals every arm reads, computed once per classification. */
568
+ interface AccountErrorSignals {
569
+ readonly message: string;
570
+ readonly lower: string;
571
+ readonly folded: string;
572
+ readonly statusCode: number | null;
573
+ readonly origin: AccountFailureOrigin;
574
+ readonly scope: AccountFailureScope;
575
+ readonly providerCode: string;
576
+ readonly statedCooldownMs: number | null;
577
+ readonly reason: (prefix: string) => string;
578
+ readonly result: AccountErrorContext["result"];
579
+ readonly canMutate: boolean;
580
+ readonly accountEvidence: boolean;
581
+ readonly revocationNamed: boolean;
582
+ /** Echo of the caller's evidence; read only by the arms that need it. */
583
+ readonly providerId: string | undefined;
584
+ readonly modelId: string | undefined;
585
+ readonly modelBlockHits: number;
586
+ readonly staticToken: boolean;
587
+ readonly credentialKind: string | undefined;
588
+ /** Buddy-family provider: the only family with provider-specific codes below. */
589
+ readonly buddy: boolean;
590
+ /**
591
+ * The upstream named a restriction on this account, not a refusal of the
592
+ * request. Only the `11140` shape carries `displayMsg` about the account
593
+ * being restricted, so the policy arm can tell a ban from a bad request
594
+ * instead of treating both as one cooldown.
595
+ */
596
+ readonly buddyBanned: boolean;
597
+ readonly priceRefusal: boolean;
598
+ readonly deploymentInfraError: boolean;
599
+ readonly quotaSignal: boolean;
600
+ readonly policyRejection: boolean;
601
+ readonly requestScoped: boolean;
602
+ readonly channelRejection: boolean;
603
+ readonly hostedToolFailure: boolean;
604
+ readonly modelEntitlementFailure: boolean;
605
+ readonly queuedUpstream: boolean;
606
+ readonly validationRequired: boolean;
607
+ readonly validationUrl: string | undefined;
608
+ }
609
+
610
+ const VALIDATION_URL_RE = /https?:\/\/[^\s"'\\]+/;
611
+
612
+ /**
613
+ * Extracts the Google `validation_url` from an Antigravity rejection.
614
+ *
615
+ * The URL can appear either in the message itself or nested in the raw
616
+ * upstream JSON (`details[].metadata.validation_url`). Returning `undefined`
617
+ * means nothing actionable was found — the classifier then just parks the
618
+ * account without a link.
619
+ */
620
+ function extractValidationUrl(message: string, rawUpstreamResponse: string | undefined): string | undefined {
621
+ if (rawUpstreamResponse !== undefined && rawUpstreamResponse.includes("validation_url")) {
622
+ try {
623
+ const parsed = JSON.parse(rawUpstreamResponse) as {
624
+ error?: {
625
+ details?: Array<{ metadata?: Record<string, unknown> }>;
626
+ };
627
+ };
628
+ for (const detail of parsed?.error?.details ?? []) {
629
+ const url = detail?.metadata?.["validation_url"];
630
+ if (typeof url === "string" && url.length > 0 && url.startsWith("http")) return url;
631
+ }
632
+ } catch {
633
+ /* fall through to message regex */
634
+ }
635
+ }
636
+ const m = VALIDATION_URL_RE.exec(message);
637
+ return m?.[0];
638
+ }
639
+
640
+ /** Build the request-independent evidence consumed by the ordered classifier. */
641
+ function accountErrorContext(error: unknown, options?: AccountFailureEvidence): AccountErrorContext { const errorRecord = isRecord(error) ? error : undefined;
642
+ const message =
643
+ error instanceof Error
644
+ ? error.message
645
+ : typeof error === "string"
646
+ ? error
647
+ : typeof errorRecord?.message === "string"
648
+ ? errorRecord.message
649
+ : String(error ?? "Unknown error");
650
+ const lower = message.toLowerCase();
651
+ const statusCode =
652
+ options?.statusCode ??
653
+ (typeof errorRecord?.status === "number" ? errorRecord.status : null);
654
+ const origin = options?.origin ?? "cartethyia";
655
+ const scope = options?.scope ?? "unknown";
656
+ // Providers write the same phrase with different separators: codex reports
657
+ // `token_revoked`, others `token revoked` or `token-revoked`. Folding the
658
+ // separators once lets every match below see one spelling.
659
+ const folded = lower.replace(/[_-]+/g, " ");
660
+ // A revocation named in the message is evidence about the credential itself,
661
+ // whichever scope the caller could establish. A stream error frame carries
662
+ // no `credentialEvidence` detail and classifies at provider scope, so without
663
+ // this the account stays in rotation: the phrase is recognised and then
664
+ // discarded because nothing mutates on provider scope.
665
+ const revocationNamed = folded.includes("token revoked") || folded.includes("invalidated oauth token");
666
+ const accountEvidence =
667
+ options?.credentialEvidence === true ||
668
+ revocationNamed ||
669
+ (options?.origin === "upstream" && options.scope === "account");
670
+ // A named revocation mutates even at provider scope: the message is the
671
+ // evidence, and provider scope is what a frame with no credential detail
672
+ // classifies as. Every other signal still needs account scope.
673
+ const canMutate = origin === "upstream" && (revocationNamed || (scope === "account" && accountEvidence));
674
+ const providerCode =
675
+ options?.providerCode?.toLowerCase() ??
676
+ (error instanceof GatewayError && typeof error.details.providerCode === "string"
677
+ ? error.details.providerCode.toLowerCase()
678
+ : "");
679
+
680
+ const getHeader = (name: string): string | null => {
681
+ const headers = options?.headers;
682
+ if (!headers) return null;
683
+ if ("get" in headers && typeof headers.get === "function") return headers.get(name);
684
+ const wanted = name.toLowerCase();
685
+ const entry = Object.entries(headers).find(([key]) => key.toLowerCase() === wanted);
686
+ return entry?.[1] ?? null;
687
+ };
688
+
689
+ const headerCooldown =
690
+ options?.retryAfterMs ??
691
+ parseUpstreamBackoff({ get: getHeader });
692
+ const messageCooldown = parseProviderResetDuration(message);
693
+ const rawUpstreamResponse =
694
+ origin === "upstream" &&
695
+ error instanceof GatewayError &&
696
+ typeof error.details.raw === "string"
697
+ ? error.details.raw
698
+ : undefined;
699
+ // The body's own fields, when they can be reached. Checked alongside the
700
+ // header and the prose rather than instead of them: a provider may state the
701
+ // window in any of the three, and the longest honest statement wins.
702
+ //
703
+ // `details.raw` is tried even when no caller passed a parsed body, because
704
+ // that is where the upstream's JSON actually reaches this function — the
705
+ // dispatch path stores the response text there. A structured window read from
706
+ // it is the difference between a 15-minute fallback and the 23 days the
707
+ // provider stated.
708
+ const bodyCooldown =
709
+ parseUpstreamBodyReset(options?.upstreamBody) ??
710
+ (rawUpstreamResponse === undefined ? null : parseUpstreamBodyReset(tryParseUpstreamBody(rawUpstreamResponse)));
711
+ // The body the ban is named in. The caller's parsed body when there is one,
712
+ // and the raw response text otherwise — the dispatch path stores the
713
+ // upstream's JSON there, and that is where `displayMsg` lives.
714
+ const banBody =
715
+ options?.upstreamBody ??
716
+ (rawUpstreamResponse === undefined ? undefined : tryParseUpstreamBody(rawUpstreamResponse));
717
+ // Whether the provider itself named a window. Decided once, from the
718
+ // evidence, so every branch of this classification agrees about it.
719
+ const providerStated =
720
+ headerCooldown !== null || messageCooldown !== null || bodyCooldown !== null;
721
+ const statedCooldownMs =
722
+ [headerCooldown, messageCooldown, bodyCooldown].reduce<number | null>(
723
+ (best, value) => (value === null ? best : best === null ? value : Math.max(best, value)),
724
+ null,
725
+ );
726
+ const reason = (prefix: string): string => `${prefix}: ${message.slice(0, 160)}`;
727
+ const result = (
728
+ category: AccountErrorCategory,
729
+ status: AccountErrorClassification["status"],
730
+ cooldownMs: number,
731
+ retryAt: Date | null,
732
+ explanation: string,
733
+ mutatesAccount = canMutate,
734
+ modelUnsupported = false,
735
+ ): AccountErrorClassification => ({
736
+ category,
737
+ status,
738
+ cooldownMs,
739
+ reason: origin === "upstream" ? rawUpstreamResponse ?? message.slice(0, 160) : explanation,
740
+ retryAt,
741
+ mutatesAccount,
742
+ // Set below by the caller that knows which columns it wrote, not here:
743
+ // the same classification feeds both the account-wide path (status +
744
+ // cooldownUntil — routing-visible) and the model-scoped path (neither —
745
+ // invisible). Defaulting true keeps existing callers conservative.
746
+ routesVisible: true,
747
+ providerStated,
748
+ modelUnsupported,
749
+ });
750
+
751
+ return {
752
+ message,
753
+ lower,
754
+ folded,
755
+ statusCode,
756
+ origin,
757
+ scope,
758
+ revocationNamed,
759
+ accountEvidence,
760
+ canMutate,
761
+ providerCode,
762
+ providerId: options?.providerId,
763
+ statedCooldownMs,
764
+ rawUpstreamResponse,
765
+ buddyBanned: isBuddyAccountBanned(lower, providerCode, banBody),
766
+ reason,
767
+ result,
768
+ };
769
+ }
770
+
771
+ export function classifyAccountError(
772
+ error: unknown,
773
+ options?: AccountFailureEvidence,
774
+ ): AccountErrorClassification {
775
+ const {
776
+ message,
777
+ lower,
778
+ folded,
779
+ statusCode,
780
+ origin,
781
+ scope,
782
+ revocationNamed,
783
+ accountEvidence,
784
+ canMutate,
785
+ providerCode,
786
+ rawUpstreamResponse,
787
+ statedCooldownMs,
788
+ buddyBanned,
789
+ reason,
790
+ result,
791
+ } = accountErrorContext(error, options);
792
+ const modelId = options?.modelId;
793
+ const modelBlockHits = Math.max(1, Math.floor(options?.modelBlockHits ?? 1));
794
+ const staticToken = options?.staticToken === true;
795
+ const credentialKind = options?.credentialKind;
796
+ const buddy = BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "");
797
+
798
+ // A price refusal is a verdict on the *request*, not on the account. inferhub
799
+ // answers 402 with `no provider's ask matches your max-per-mtok bid`: the
800
+ // caller's own price ceiling was below every upstream ask, which says nothing
801
+ // about the credential or its balance. Classified as quota it cooled a
802
+ // fully-credited account down for an hour and the operator reported it as
803
+ // "blocked, unusable at all". Detected before the quota branch and recorded
804
+ // without mutating the account, so only that request fails.
805
+ const priceRefusal =
806
+ lower.includes("max-per-mtok") ||
807
+ lower.includes("no provider's ask") ||
808
+ (lower.includes("ask") && lower.includes("bid"));
809
+ // Deployment/proxy infrastructure errors (DEPLOYMENT_DISABLED, etc.) are
810
+ // transport-level proxy pool failures, not account quota exhaustion.
811
+ // Mutating the account parks it for an hour while the pool itself stays
812
+ // healthy — the next request picks the same dying pool and the same error
813
+ // repeats. These must not cooldown the account.
814
+ const deploymentInfraError =
815
+ lower.includes("deployment_disabled") ||
816
+ lower.includes("deployment disabled") ||
817
+ (lower.includes("sin1::") && lower.includes("deployment"));
818
+ if (deploymentInfraError) {
819
+ return result("server_error", "cooldown", 60_000, null, `Proxy deployment error: ${message.slice(0, 160)}`, false);
820
+ }
821
+ // A (CodeBuddy/WorkBuddy) *frequency* window is a verdict on one
822
+ // (account, model) pair, not on the account: `{"code":"6004","msg":"usage
823
+ // exceeds frequency limit ... your usage will reset at <stamp> UTC+8,
824
+ // alternatively, you can switch to the other models to continue using it."}`.
825
+ // The provider names the remedy itself — use a different model — which is
826
+ // only a remedy if the account still serves every other model. Taken as
827
+ // account-wide `quota_exhausted` it parked the whole account until the reset,
828
+ // so the switch the upstream offered was refused by our own gateway and every
829
+ // sibling model went dark with it; the account also read COOLDOWN /
830
+ // quota_exhausted while only that pair was throttled.
831
+ //
832
+ // Classified `rate_limit_transient` so the writer scopes it to the model:
833
+ // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
834
+ // account's own status alone. Kept ahead of `quotaSignal`, which matches a
835
+ // *broader* slice of the same wording (`usage exceeds` + `frequency limit`)
836
+ // and would otherwise claim this message as an account-wide quota window.
837
+ // Narrow on purpose — the extra phrase is what says the other models still
838
+ // work; without it the failure stays account-wide.
839
+ const modelFrequencyWindow =
840
+ lower.includes("frequency limit") && lower.includes("switch to the other models");
841
+ if (modelFrequencyWindow) {
842
+ const cooldownMs = statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
843
+ return result(
844
+ "rate_limit_transient",
845
+ "cooldown",
846
+ cooldownMs,
847
+ new Date(Date.now() + cooldownMs),
848
+ reason("Provider frequency limit for this model"),
849
+ // Provider-scoped evidence still has to reach the writer: a stream frame
850
+ // carries no credential detail, so `canMutate` is false and without this
851
+ // the pair would be retried for the whole window it was closed for.
852
+ origin === "upstream" && scope === "provider",
853
+ );
854
+ }
855
+
856
+ // Quota-shaped provider codes/messages (e.g. xAI's
857
+ const quotaSignal =
858
+ !priceRefusal &&
859
+ !deploymentInfraError &&
860
+ (statusCode === 402 ||
861
+ // Qoder answers an exhausted credit balance with
862
+ // `{"code":"112","message":"{\"pricingUrl\":...}"}` — the body points at
863
+ // the pricing page, i.e. "buy more", which is a balance verdict, not an
864
+ // auth one. It fell through to `auth_invalidated` and the console showed
865
+ // "Re-login required" on an account whose credential was fine, sending
866
+ // operators to re-authenticate instead of topping up.
867
+ // Recognising it here keeps it out of `auth_invalidated`; the branch
868
+ // below scopes the resulting cooldown to the model, because the account
869
+ // keeps serving its other models.
870
+ providerCode === "112" ||
871
+ providerCode === "110" ||
872
+ providerCode === "insufficient_quota" ||
873
+ providerCode === "quota_exceeded" ||
874
+ providerCode === "subscription:free-usage-exhausted" ||
875
+ providerCode === "14018" ||
876
+ // CodeBuddy/WorkBuddy rate limit: `{"code":"6004","msg":"usage exceeds
877
+ // frequency limit ... your usage will reset at <stamp> UTC+8"}`. The
878
+ // message names a reset, so this is a parked window, not a transient
879
+ // 429 — without it the account fell through to `unknown` with
880
+ // `mutates=false`, stayed in rotation, and was re-picked for every
881
+ // request inside the window the provider had actually named.
882
+ //
883
+ // Matched on the message's own wording, not on the `6004` code: that code
884
+ // is not unique to this surface (the artifact-release API reuses it), so a
885
+ // code match would sweep an unrelated failure into a quota cooldown. The
886
+ // phrase is the reliable key, and `usage exceeds` + `frequency limit` is
887
+ // the pair this family actually prints.
888
+ (lower.includes("usage exceeds") && lower.includes("frequency limit")) ||
889
+ lower.includes("insufficient_quota") ||
890
+ lower.includes("quota_exceeded") ||
891
+ lower.includes("usage_limit_reached") ||
892
+ lower.includes("usage-exhausted") ||
893
+ lower.includes("usage_exhausted") ||
894
+ lower.includes("usage exhausted") ||
895
+ lower.includes("usage exceeds") ||
896
+ lower.includes("frequency limit") ||
897
+ lower.includes("free-usage") ||
898
+ lower.includes("free usage") ||
899
+ lower.includes("exceeded your current quota") ||
900
+ // Trae answers an exhausted quota with **HTTP 502** and the body `Your
901
+ // requests have exceeded the quota.` The status is what misled the
902
+ // classifier: 502 read as a gateway failure, so this fell through to
903
+ // `server_error` — a 30s cooldown that does not mutate the account, so
904
+ // the account stayed in rotation and every other model on it kept being
905
+ // invoked inside a window the provider had already closed. The body is
906
+ // the verdict; the 502 is incidental. Matched on the phrase, not on the
907
+ // status, which is why this sits in `quotaSignal` ahead of the 5xx arm.
908
+ lower.includes("have exceeded the quota") ||
909
+ lower.includes("balance exhausted") ||
910
+ lower.includes("insufficient balance") ||
911
+ lower.includes("out of credits") ||
912
+ lower.includes("credits exhausted") ||
913
+ lower.includes("spending limit") ||
914
+ lower.includes("billing daily count exceeded") ||
915
+ // Same Qoder family reached by body text when the code is not parsed out.
916
+ (lower.includes("pricingurl") && lower.includes("qoder.com/pricing")));
917
+ // A deterministic content-policy rejection is NOT credential invalidation.
918
+ // CodeBuddy returns HTTP 403 with provider code 11140 ("request illegal",
919
+ // "did not pass the safety review") — refreshing the token cannot change
920
+ // that outcome and the account itself is healthy, so it must never flip the
921
+ // account to auth_invalidated/disabled. Only real auth signals do.
922
+ const policyRejection =
923
+ providerCode === "11140" ||
924
+ lower.includes("safety review") ||
925
+ lower.includes("content did not pass") ||
926
+ lower.includes("request illegal") ||
927
+ lower.includes("content blocked") ||
928
+ lower.includes("account suspended");
929
+ // A ban is a verdict on the credential, and cooldown cannot answer it: the
930
+ // account is refused, not throttled. Disabling is the fix, and disabling
931
+ // *immediately* is a feature — a client's retry budget is spent in seconds,
932
+ // so a banned account that merely cooled stayed in the plan and burned
933
+ // sibling accounts that were still healthy before the window expired.
934
+ //
935
+ // The two shapes are told apart by the body, not the code alone: `11140`
936
+ // names a restriction on the account, `11-128` refuses the request.
937
+ const policyBlock =
938
+ policyRejection &&
939
+ options?.providerId !== undefined &&
940
+ BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
941
+ if (policyBlock) {
942
+ if (buddyBanned) {
943
+ return result(
944
+ "policy_blocked_banned",
945
+ "disabled",
946
+ 0,
947
+ null,
948
+ reason("Account banned by upstream"),
949
+ // A ban is evidence about the credential even at provider scope, for
950
+ // the same reason a named revocation is: the message is the evidence.
951
+ // Without this a stream frame's missing credential detail would leave a
952
+ // banned account serving.
953
+ true,
954
+ );
955
+ }
956
+ // Not a ban: the upstream refused this request. The account is fine, so the
957
+ // only correct move is to stop retrying it verbatim. Cooling the account
958
+ // here was the bug that made one malformed request look like a dead
959
+ // credential.
960
+ return result(
961
+ "policy_blocked_invalid_request",
962
+ "cooldown",
963
+ 0,
964
+ null,
965
+ reason("Upstream refused this request"),
966
+ // Never mutates: a malformed body fails identically on every account, so
967
+ // cooling would burn a healthy credential on the gateway's own fault.
968
+ false,
969
+ );
970
+ }
971
+ // These codes are a verdict on the request, not the account. Cooling the
972
+ // account for them burns a healthy credential on a body that will fail the
973
+ // same way on every other account:
974
+ //
975
+ // - `11148` tool-call/result mismatch — the transcript is broken; the
976
+ // pre-payload repair is supposed to prevent it, and a slip is still the
977
+ // body's fault.
978
+ // - `11135` the image cannot be processed, `11133` the model rejected a
979
+ // parameter, `11101` the body failed to parse — deterministic for that
980
+ // payload.
981
+ // - `12153` the session is dead — a refresh problem, not a reason to park
982
+ // the account on the first observation.
983
+ // - `14017` trial not activated — the account may recover once registration
984
+ // finishes, so a hard cooldown would keep it out after it becomes usable.
985
+ //
986
+ // Recognised before the auth and quota branches: several of these arrive as
987
+ // 401/403/429, which those branches would otherwise treat as account evidence.
988
+ const requestScoped =
989
+ providerCode === "11148" ||
990
+ providerCode === "11135" ||
991
+ providerCode === "11133" ||
992
+ providerCode === "11101" ||
993
+ providerCode === "12153" ||
994
+ providerCode === "14017" ||
995
+ lower.includes("tool_call_sequence_broken") ||
996
+ lower.includes("unmarshal chat params failed") ||
997
+ lower.includes("invalid image_url content") ||
998
+ lower.includes("offline user session not found") ||
999
+ lower.includes("trial not activated") ||
1000
+ lower.includes("trial version is not yet activated");
1001
+ if (requestScoped && BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "")) {
1002
+ return result(
1003
+ "unknown",
1004
+ "cooldown",
1005
+ 0,
1006
+ null,
1007
+ reason("Request-scoped upstream rejection"),
1008
+ false,
1009
+ );
1010
+ }
1011
+ // A channel rejection (`11128`) is the same shape: the upstream is refusing
1012
+ // the caller for this account, not this particular prompt, and it keeps
1013
+ // refusing on every invocation. Classifying it as `unknown` with
1014
+ // `mutates=false` would leave the account in rotation failing every request —
1015
+ // and since every account fails the same way, the whole pool looks dead with
1016
+ // nothing ever parked.
1017
+ const channelRejection =
1018
+ providerCode === "11128" || lower.includes("unapproved channel");
1019
+ const channelBlock =
1020
+ channelRejection &&
1021
+ options?.providerId !== undefined &&
1022
+ BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
1023
+ if (channelBlock) {
1024
+ // Same split as the policy arm above: a ban disables, a channel refusal
1025
+ // only stops this request. `11-128` is the request shape, so this branch
1026
+ // nearly always takes the refusal path — but it shares the ban test so a
1027
+ // `11140`-shaped body arriving as a channel rejection still disables.
1028
+ if (buddyBanned) {
1029
+ return result(
1030
+ "policy_blocked_banned",
1031
+ "disabled",
1032
+ 0,
1033
+ null,
1034
+ reason("Account banned by upstream"),
1035
+ true,
1036
+ );
1037
+ }
1038
+ return result(
1039
+ "policy_blocked_invalid_request",
1040
+ "cooldown",
1041
+ 0,
1042
+ null,
1043
+ reason("Provider channel rejection"),
1044
+ // The credential is valid; the upstream refuses this calling channel.
1045
+ // Cooling was the bug: it kept a refused account in the plan, where a
1046
+ // client's retry budget burned healthy siblings for the whole window.
1047
+ false,
1048
+ );
1049
+ }
1050
+ // server-side tool (web search, x search, web fetch), not evidence about the
1051
+ // credential or the account's quota. Some providers surface it as 403, which
1052
+ // would otherwise look like an auth signal — refreshing the token cannot fix
1053
+ // a search backend, so it must never disable the account.
1054
+ const hostedToolFailure =
1055
+ lower.includes("web_search") ||
1056
+ lower.includes("web search") ||
1057
+ lower.includes("web_search_preview") ||
1058
+ lower.includes("x_search") ||
1059
+ lower.includes("web_fetch") ||
1060
+ lower.includes("tool invocation") ||
1061
+ lower.includes("tool call failed") ||
1062
+ lower.includes("search backend");
1063
+ // A model-scoped license or entitlement rejection is NOT credential invalidation.
1064
+ // Google Cloud Code / Antigravity answers HTTP 403 with "You do not have a valid
1065
+ // license of this product" when an account lacks entitlement for one specific model
1066
+ // (e.g. claude-opus-4-6), while the account itself and its tokens remain fully valid
1067
+ // for all other models. Disabling the whole account breaks healthy models.
1068
+ const modelEntitlementFailure =
1069
+ statusCode === 403 &&
1070
+ (lower.includes("not have a valid license") ||
1071
+ lower.includes("valid license of this product") ||
1072
+ lower.includes("model not supported") ||
1073
+ lower.includes("not entitled to") ||
1074
+ lower.includes("no access to model") ||
1075
+ lower.includes("model_not_supported") ||
1076
+ lower.includes("not available in your region"));
1077
+ /**
1078
+ * Google Cloud Code / Antigravity account verification: the upstream answers
1079
+ * an unverified account with HTTP 403 `PERMISSION_DENIED` and a
1080
+ * `VALIDATION_REQUIRED` reason plus a `validation_url`. The operator must
1081
+ * complete verification before the credential can ever serve again, so the
1082
+ * account must be parked (`disabled`) — not left `active`, and not a
1083
+ * cooldown (no amount of waiting clears it). The validation URL is carried
1084
+ * in the recorded reason so the dashboard can surface it as an actionable
1085
+ * pill link.
1086
+ */
1087
+ const validationRequired =
1088
+ (statusCode === 403 || lower.includes("permission_denied")) &&
1089
+ (lower.includes("validation_required") ||
1090
+ lower.includes("verify your account") ||
1091
+ lower.includes("validation_url"));
1092
+ const validationUrl = validationRequired ? extractValidationUrl(message, rawUpstreamResponse) : undefined;
1093
+ /**
1094
+ * The upstream accepted the request and put it in a queue behind other
1095
+ * callers — it is busy, not rejecting the caller's credential.
1096
+ *
1097
+ * Qoder answers `403 {"code":"10605","message":"{\"isQueued\":true,
1098
+ * \"modelKey\":\"dfmodel\",\"queueCount\":1395,...}"}`. The 403 is what made
1099
+ * this misclassify: the generic `(401 || 403) && accountEvidence` arm claimed
1100
+ * it as `auth_invalidated`, so a healthy account was parked as "Re-login
1101
+ * required" purely because the provider was busy, and an operator was sent
1102
+ * to re-authenticate a credential that was never rejected.
1103
+ *
1104
+ * Matched on the queue evidence rather than on the bare `10605`, so an
1105
+ * unrelated 403 that happens to share the code is not swept in. The bodies
1106
+ * are nested JSON, so the tell is the key, not a parse of the envelope.
1107
+ */
1108
+ const queuedUpstream =
1109
+ providerCode === "10605" ||
1110
+ lower.includes("isqueued") ||
1111
+ lower.includes("queuecount") ||
1112
+ lower.includes("queuetype") ||
1113
+ (lower.includes("queue") && lower.includes("modelkey"));
1114
+
1115
+ const signals: AccountErrorSignals = {
1116
+ message,
1117
+ lower,
1118
+ folded,
1119
+ statusCode,
1120
+ origin,
1121
+ scope,
1122
+ providerCode,
1123
+ statedCooldownMs,
1124
+ reason,
1125
+ result,
1126
+ canMutate,
1127
+ accountEvidence,
1128
+ revocationNamed,
1129
+ providerId: options?.providerId,
1130
+ modelId,
1131
+ modelBlockHits,
1132
+ staticToken,
1133
+ credentialKind,
1134
+ buddy,
1135
+ buddyBanned,
1136
+ priceRefusal,
1137
+ deploymentInfraError,
1138
+ quotaSignal,
1139
+ policyRejection,
1140
+ requestScoped,
1141
+ channelRejection,
1142
+ hostedToolFailure,
1143
+ modelEntitlementFailure,
1144
+ queuedUpstream,
1145
+ validationRequired,
1146
+ validationUrl,
1147
+ };
1148
+
1149
+ for (const rule of ACCOUNT_ERROR_RULES) {
1150
+ if (rule.match(signals)) return rule.decide(signals);
1151
+ }
1152
+
1153
+ // Same reasoning as above, with the shortest backoff: an unmapped failure
1154
+ // must still be retried automatically rather than parked forever.
1155
+ const unclassifiedMs = UNCLASSIFIED_COOLDOWN_MS();
1156
+ return result(
1157
+ "unknown",
1158
+ "cooldown",
1159
+ unclassifiedMs,
1160
+ new Date(Date.now() + unclassifiedMs),
1161
+ reason("Unclassified provider failure"),
1162
+ false,
1163
+ );
1164
+ }
1165
+
1166
+ /**
1167
+ * The ordered arms of {@link classifyAccountError}, tried top to bottom; the
1168
+ * first match decides. Order is behavior, not presentation — see
1169
+ * {@link AccountErrorRule}.
1170
+ */
1171
+ const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
1172
+ {
1173
+ // Must precede "rate-limit": this rejection is a 429 whose own text says
1174
+ // "rate limit", so the broader arm claimed it and cooled the account for
1175
+ // the full 15-minute rate-limit window.
1176
+ //
1177
+ // A full max-inflight is a statement about the gateway's own routing, not
1178
+ // about the credential: every account serving the model is busy right now.
1179
+ // Cooling them marks healthy accounts as `cooldown` for a limit their
1180
+ // provider never applied, which is what made every account read busy while
1181
+ // incoming requests piled up — the accounts were never the problem, the
1182
+ // ceiling was.
1183
+ name: "gateway-capacity-exhausted",
1184
+ match: (s) =>
1185
+ s.lower.includes("at its concurrency limit") ||
1186
+ s.lower.includes("every proxy pool for this provider is at its concurrency limit") ||
1187
+ s.lower.includes("the gateway is at its concurrency limit"),
1188
+ decide: (s) =>
1189
+ s.result(
1190
+ "model_capacity",
1191
+ // `status` is the status the account would move to; it is never
1192
+ // applied because `mutatesAccount` is false below, so this names the
1193
+ // shape rather than a transition. Zero cooldown is the point: no wait
1194
+ // is attached to the account, so the next request may use it at once.
1195
+ "cooldown",
1196
+ 0,
1197
+ null,
1198
+ s.reason("Gateway concurrency ceiling reached; no account was at fault"),
1199
+ // Never mutates: there is nothing wrong with the account to record.
1200
+ false,
1201
+ ),
1202
+ },
1203
+ {
1204
+ // Must precede the unclassified fallback: the fallback is account-blind and
1205
+ // would stamp this as a provider-wide failure, which is how one proxy
1206
+ // refusing CONNECT put every account of every provider behind that pool
1207
+ // into cooldown — opencode's dead front door taking codebuddy down with it.
1208
+ //
1209
+ // A CONNECT refusal is a statement about the proxy pool, not about a
1210
+ // credential: the tunnel was never established, so no upstream ever saw the
1211
+ // request and no account can be at fault. Scoped to the pool so the
1212
+ // provider's own accounts stay in rotation.
1213
+ name: "proxy-connect-refusal",
1214
+ match: (s) =>
1215
+ s.lower.includes("proxy connect failed") ||
1216
+ (s.lower.includes("connect") && s.lower.includes("tunnel") && s.lower.includes("refus")),
1217
+ decide: (s) =>
1218
+ s.result(
1219
+ "model_unreachable",
1220
+ "cooldown",
1221
+ 0,
1222
+ null,
1223
+ s.reason("Proxy pool refused the CONNECT tunnel"),
1224
+ // Never mutates: the pool owns this failure, not the account.
1225
+ false,
1226
+ ),
1227
+ },
1228
+ {
1229
+ name: "deployment-infrastructure",
1230
+ match: (s) => s.deploymentInfraError,
1231
+ decide: (s) =>
1232
+ s.result(
1233
+ "server_error",
1234
+ "cooldown",
1235
+ 60_000,
1236
+ null,
1237
+ `Proxy deployment error: ${s.message.slice(0, 160)}`,
1238
+ false,
1239
+ ),
1240
+ },
1241
+ {
1242
+ name: "model-frequency-window",
1243
+ match: (s) => s.lower.includes("frequency limit") && s.lower.includes("switch to the other models"),
1244
+ decide: (s) => {
1245
+ const cooldownMs = s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
1246
+ return s.result(
1247
+ "rate_limit_transient",
1248
+ "cooldown",
1249
+ cooldownMs,
1250
+ new Date(Date.now() + cooldownMs),
1251
+ s.reason("Provider frequency limit for this model"),
1252
+ // Provider-scoped evidence still has to reach the writer: a stream frame
1253
+ // carries no credential detail, so `canMutate` is false and without this
1254
+ // the pair would be retried for the whole window it was closed for.
1255
+ s.origin === "upstream" && s.scope === "provider",
1256
+ );
1257
+ },
1258
+ },
1259
+ {
1260
+ name: "content-policy-block",
1261
+ match: (s) => s.policyRejection && s.buddy,
1262
+ decide: (s) => {
1263
+ // A ban is a verdict on the credential, and cooldown cannot answer it:
1264
+ // the account is refused, not throttled. Disabling is the fix, and
1265
+ // disabling *immediately* is a feature — a client's retry budget is spent
1266
+ // in seconds, so a banned account that merely cooled stayed in the plan
1267
+ // and burned sibling accounts that were still healthy before the window
1268
+ // expired.
1269
+ //
1270
+ // The two shapes are told apart by the body, not the code alone: `11140`
1271
+ // names a restriction on the account, `11-128` refuses the request.
1272
+ if (s.buddyBanned) {
1273
+ return s.result(
1274
+ "policy_blocked_banned",
1275
+ "disabled",
1276
+ 0,
1277
+ null,
1278
+ s.reason("Account banned by upstream"),
1279
+ // A ban is evidence about the credential even at provider scope, for
1280
+ // the same reason a named revocation is: the message is the evidence.
1281
+ // Without this a stream frame's missing credential detail would leave
1282
+ // a banned account serving.
1283
+ true,
1284
+ );
1285
+ }
1286
+ // Not a ban: the upstream refused this request. The account is fine, so
1287
+ // the only correct move is to stop retrying it verbatim. Cooling the
1288
+ // account here was the bug that made one malformed request look like a
1289
+ // dead credential.
1290
+ return s.result(
1291
+ "policy_blocked_invalid_request",
1292
+ "cooldown",
1293
+ 0,
1294
+ null,
1295
+ s.reason("Upstream refused this request"),
1296
+ // Never mutates: a malformed body fails identically on every account,
1297
+ // so cooling would burn a healthy credential on the gateway's own fault.
1298
+ false,
1299
+ );
1300
+ },
1301
+ },
1302
+ {
1303
+ name: "request-scoped-rejection",
1304
+ match: (s) => s.requestScoped && s.buddy,
1305
+ decide: (s) =>
1306
+ s.result("unknown", "cooldown", 0, null, s.reason("Request-scoped upstream rejection"), false),
1307
+ },
1308
+ {
1309
+ name: "channel-rejection",
1310
+ match: (s) => s.channelRejection && s.buddy,
1311
+ decide: (s) => {
1312
+ const channelBlockMs = resolveAccountPolicyBlockCooldownMs();
1313
+ return s.result(
1314
+ "policy_blocked",
1315
+ "cooldown",
1316
+ channelBlockMs,
1317
+ new Date(Date.now() + channelBlockMs),
1318
+ s.reason("Provider channel rejection"),
1319
+ s.origin === "upstream",
1320
+ );
1321
+ },
1322
+ },
1323
+ {
1324
+ name: "validation-required",
1325
+ match: (s) => s.validationRequired,
1326
+ decide: (s) =>
1327
+ s.result(
1328
+ "validation_required",
1329
+ "disabled",
1330
+ 0,
1331
+ null,
1332
+ s.validationUrl !== undefined
1333
+ ? `${s.reason("Google account verification required")} — ${s.validationUrl}`
1334
+ : s.reason("Google account verification required"),
1335
+ // The upstream named the verdict; the account cannot serve until the
1336
+ // operator completes verification.
1337
+ true,
1338
+ ),
1339
+ },
1340
+ {
1341
+ name: "credential-rejected",
1342
+ match: (s) =>
1343
+ !s.quotaSignal &&
1344
+ !s.policyRejection &&
1345
+ !s.hostedToolFailure &&
1346
+ !s.modelEntitlementFailure &&
1347
+ !s.queuedUpstream &&
1348
+ (s.providerCode === "authentication_failed" ||
1349
+ s.folded.includes("invalid api key") ||
1350
+ s.folded.includes("incorrect api key") ||
1351
+ s.folded.includes("invalid token") ||
1352
+ s.folded.includes("token revoked") ||
1353
+ s.folded.includes("invalidated oauth token") ||
1354
+ s.folded.includes("reauthorization required") ||
1355
+ s.folded.includes("account deactivated") ||
1356
+ // Trae answers `1001 "we are not able to authenticate you"` when the
1357
+ // credential is dead or invalidated — it is the provider's own auth
1358
+ // verdict, but the code is numeric, so it is excluded from the bare
1359
+ // 401/403 arm below. Recognising it here keeps the account out of the
1360
+ // `unknown` fallback (which never mutates and leaves the account in
1361
+ // rotation to be re-dialled on every request).
1362
+ ((s.providerId === "trae" || s.providerId === "traecn") &&
1363
+ s.providerCode === "1001") ||
1364
+ // A bare 401/403 only proves the credential when the body actually
1365
+ // says so. A purely numeric provider code carries no auth vocabulary —
1366
+ // providers encode limit/entitlement verdicts as numbers too (Qoder's
1367
+ // `110`, buddy's `6004`) — and treating an unknown one as credential
1368
+ // evidence parked healthy accounts as "Re-login required". Unmapped
1369
+ // numeric codes fall through to the unclassified cooldown instead of
1370
+ // ever invalidating.
1371
+ ((s.statusCode === 401 || s.statusCode === 403) &&
1372
+ s.accountEvidence &&
1373
+ !NUMERIC_PROVIDER_CODE.test(s.providerCode))),
1374
+ decide: (s) => {
1375
+ // A static token has no refresh to run, so an auth rejection must not park
1376
+ // the account: disabling it throws away a token that may still work (a
1377
+ // single upstream 401 can be transient). A cooldown lets the next attempt
1378
+ // retry the same token without losing the account.
1379
+ const oauthRecovery =
1380
+ !s.staticToken &&
1381
+ s.credentialKind === "oauth" &&
1382
+ (s.lower.includes("expired") || s.lower.includes("refresh"));
1383
+ // A *named* revocation is the one auth outcome that still parks the account.
1384
+ // The upstream said the credential itself is dead — not that it expired, not
1385
+ // that this request was refused — so there is no window to wait out and no
1386
+ // refresh to run, and a cooldown would only return it to rotation to fail
1387
+ // again. Every other auth rejection is weak evidence (clock skew, a key
1388
+ // rotated a moment ago, an edge answering before the credential loads) and
1389
+ // cools down instead.
1390
+ //
1391
+ // This distinction is the whole fix. Reaching `disabled` from the generic
1392
+ // `401`/`403` branch would be unrecoverable by any sweep, so one transient
1393
+ // rejection would remove the account from rotation until an operator
1394
+ // noticed — the exact outcome the comment above says a static token must
1395
+ // not suffer.
1396
+ const revoked = s.revocationNamed;
1397
+ const recoverable = s.staticToken || oauthRecovery || !revoked;
1398
+ // An automatic disable is **not** terminal any more, but the deadline is what
1399
+ // makes that safe: `sweepExpiredCooldowns` recovers a disabled row only when
1400
+ // its `cooldownUntil` has elapsed, and an operator-disabled row keeps that
1401
+ // column null. So the machine can undo its own verdict while an operator's
1402
+ // decision still outranks it — see `resolveAccountDisabledRecoveryMs`.
1403
+ const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
1404
+ const disabledRetryAt =
1405
+ disabledRecoveryMs > 0 ? new Date(Date.now() + disabledRecoveryMs) : null;
1406
+ const cooldownMs = recoverable ? resolveAccountAuthRejectedCooldownMs() : disabledRecoveryMs;
1407
+ return s.result(
1408
+ "auth_invalidated",
1409
+ recoverable ? "cooldown" : "disabled",
1410
+ cooldownMs,
1411
+ recoverable ? new Date(Date.now() + cooldownMs) : disabledRetryAt,
1412
+ s.reason(
1413
+ !revoked
1414
+ ? "Provider credential rejected; retrying after a backoff"
1415
+ : s.staticToken
1416
+ ? "Static token rejected; retrying without a refresh"
1417
+ : oauthRecovery
1418
+ ? "OAuth credential requires refresh"
1419
+ : "Provider credential revoked",
1420
+ ),
1421
+ s.canMutate,
1422
+ );
1423
+ },
1424
+ },
1425
+ {
1426
+ name: "price-refusal",
1427
+ match: (s) => s.priceRefusal,
1428
+ decide: (s) =>
1429
+ s.result("quota_exhausted", "cooldown", 0, null, s.reason("Request price refused by the provider"), false),
1430
+ },
1431
+ {
1432
+ // Qoder's code-112 pricing refusal (`{"code":"112","message":"{\"pricingUrl\":...}"}`)
1433
+ // is a "this model is not included in this account's plan, buy more" verdict
1434
+ // for one pair — the account keeps serving its other models. The probe path
1435
+ // does not always carry the code in its evidence (only the message does), so
1436
+ // the match keys on the provider id plus the pricing-page wording as well.
1437
+ // It must run before `quota-exhausted`, which would park the whole account.
1438
+ name: "qoder-plan-gated",
1439
+ match: (s) =>
1440
+ (s.providerId === "qoder" || s.providerId === "qodercn") &&
1441
+ (s.providerCode === "112" || s.lower.includes("pricingurl")),
1442
+ decide: (s) => {
1443
+ const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1444
+ return s.result(
1445
+ "model_unreachable",
1446
+ "cooldown",
1447
+ cooldownMs,
1448
+ new Date(Date.now() + cooldownMs),
1449
+ s.reason("Model not included in this account's plan"),
1450
+ s.origin === "upstream" && s.modelId !== undefined,
1451
+ true,
1452
+ );
1453
+ },
1454
+ },
1455
+ {
1456
+ name: "model-plan-gated",
1457
+ match: (s) => s.quotaSignal && s.providerCode === "112",
1458
+ decide: (s) => {
1459
+ const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1460
+ return s.result(
1461
+ "model_unreachable",
1462
+ "cooldown",
1463
+ cooldownMs,
1464
+ new Date(Date.now() + cooldownMs),
1465
+ s.reason("Model not included in this account's plan"),
1466
+ s.origin === "upstream" && s.modelId !== undefined,
1467
+ true,
1468
+ );
1469
+ },
1470
+ },
1471
+ {
1472
+ name: "quota-exhausted",
1473
+ match: (s) => s.quotaSignal,
1474
+ decide: (s) => {
1475
+ // xAI Grok Build's free tier resets on a rolling window. Use a duration
1476
+ // stated by the provider when present; otherwise park for the conservative
1477
+ // 24-hour fallback.
1478
+ const grokFreeTier =
1479
+ s.providerCode === "subscription:free-usage-exhausted" ||
1480
+ s.providerCode.startsWith("subscription:") ||
1481
+ s.lower.includes("free-usage") ||
1482
+ s.lower.includes("free usage") ||
1483
+ s.lower.includes("included free usage") ||
1484
+ s.lower.includes("rolling 24-hour") ||
1485
+ s.lower.includes("rolling 24 hour");
1486
+ const buddyCreditsExhausted =
1487
+ s.providerCode === "14018" || s.lower.includes("credits exhausted");
1488
+ const cooldownMs = grokFreeTier
1489
+ ? (s.statedCooldownMs ?? GROK_QUOTA_COOLDOWN_MS)
1490
+ : buddyCreditsExhausted
1491
+ ? (s.statedCooldownMs ?? 12 * 60 * 60 * 1000)
1492
+ : (s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS());
1493
+ // The caller records against the concrete account that just failed
1494
+ // upstream, so a provider-scoped quota error still cools THAT account
1495
+ // down (enables fallback to the next account). Local-origin quota must
1496
+ // never mutate an upstream account.
1497
+ return s.result(
1498
+ "quota_exhausted",
1499
+ "cooldown",
1500
+ cooldownMs,
1501
+ new Date(Date.now() + cooldownMs),
1502
+ s.reason("Provider quota exhausted"),
1503
+ s.origin === "upstream",
1504
+ );
1505
+ },
1506
+ },
1507
+ {
1508
+ // Inception's Free tier throttles per minute (1 000 requests / 1M input
1509
+ // tokens / 100K output tokens) and the window resets every minute. A
1510
+ // generic 429 would cool the account for the 15-minute default, leaving a
1511
+ // healthy key parked long after the window refilled — the source of
1512
+ // "limitasiya lumayan keras". This arm must run before `rate-limit`: it
1513
+ // matches the same signals and shortens the window to 1 minute so routing
1514
+ // moves to the next account and the flagged one returns almost at once.
1515
+ name: "inception-per-minute-limit",
1516
+ match: (s) =>
1517
+ s.providerId === "inception" &&
1518
+ (s.statusCode === 429 ||
1519
+ s.providerCode === "rate_limit_reached" ||
1520
+ s.providerCode === "rate_limit_exceeded" ||
1521
+ s.lower.includes("rate_limit") ||
1522
+ s.lower.includes("rate limit") ||
1523
+ s.lower.includes("too many requests") ||
1524
+ s.lower.includes("tokens per minute") ||
1525
+ s.lower.includes("input token")),
1526
+ decide: (s) => {
1527
+ const cooldownMs = s.statedCooldownMs ?? INCEPTION_RATE_LIMIT_COOLDOWN_MS;
1528
+ return s.result(
1529
+ "rate_limit_transient",
1530
+ "cooldown",
1531
+ cooldownMs,
1532
+ new Date(Date.now() + cooldownMs),
1533
+ s.reason("Inception per-minute input-token limit"),
1534
+ s.origin === "upstream",
1535
+ );
1536
+ },
1537
+ },
1538
+ {
1539
+ name: "rate-limit",
1540
+ match: (s) =>
1541
+ s.statusCode === 429 ||
1542
+ s.providerCode === "rate_limit_exceeded" ||
1543
+ s.lower.includes("rate_limit") ||
1544
+ s.lower.includes("rate limit") ||
1545
+ s.lower.includes("too many requests") ||
1546
+ s.lower.includes("requests per minute") ||
1547
+ s.lower.includes("tokens per minute") ||
1548
+ s.lower.includes("tpm limit") ||
1549
+ s.lower.includes("rpm limit"),
1550
+ decide: (s) => {
1551
+ const cooldownMs = s.statedCooldownMs ?? DEFAULT_RATE_LIMIT_COOLDOWN_MS();
1552
+ // Same per-account reasoning as quota above: the failing account (or
1553
+ // its model, when modelId is present) backs off so routing tries the
1554
+ // next candidate instead of hammering the same key.
1555
+ return s.result(
1556
+ "rate_limit_transient",
1557
+ "cooldown",
1558
+ cooldownMs,
1559
+ new Date(Date.now() + cooldownMs),
1560
+ s.reason("Provider rate limit"),
1561
+ s.origin === "upstream",
1562
+ );
1563
+ },
1564
+ },
1565
+ {
1566
+ name: "model-capacity",
1567
+ match: (s) =>
1568
+ s.statusCode === 503 ||
1569
+ s.statusCode === 529 ||
1570
+ s.providerCode === "model_capacity" ||
1571
+ s.queuedUpstream ||
1572
+ s.lower.includes("model overloaded") ||
1573
+ s.lower.includes("capacity exceeded") ||
1574
+ s.lower.includes("temporarily unavailable"),
1575
+ decide: (s) => {
1576
+ const cooldownMs = s.statedCooldownMs ?? MODEL_CAPACITY_COOLDOWN_MS();
1577
+ const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1578
+ return s.result(
1579
+ "model_capacity",
1580
+ "cooldown",
1581
+ cooldownMs,
1582
+ new Date(Date.now() + cooldownMs),
1583
+ s.reason("Provider model capacity"),
1584
+ modelScopedMutation,
1585
+ );
1586
+ },
1587
+ },
1588
+ {
1589
+ name: "model-unsupported",
1590
+ match: (s) => isUnsupportedModelEvidence(s.statusCode, s.providerCode, s.folded),
1591
+ decide: (s) => {
1592
+ const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1593
+ return s.result(
1594
+ "model_unreachable",
1595
+ "cooldown",
1596
+ cooldownMs,
1597
+ new Date(Date.now() + cooldownMs),
1598
+ s.reason("Model not supported by this upstream"),
1599
+ s.origin === "upstream" && s.modelId !== undefined,
1600
+ true,
1601
+ );
1602
+ },
1603
+ },
1604
+ {
1605
+ name: "model-unreachable",
1606
+ match: (s) => s.providerCode === "not_found" || s.modelEntitlementFailure,
1607
+ decide: (s) => {
1608
+ const cooldownMs = MODEL_UNREACHABLE_COOLDOWN_MS;
1609
+ // Not credential evidence, so `classifyUpstreamFailure` scopes this to the
1610
+ // provider and the `scope === "account"` gate the throttles use would
1611
+ // forbid the mutation outright — leaving the probe reporting a failure that
1612
+ // cooled nothing. The verdict is still about one (account, model) pair:
1613
+ // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
1614
+ // account's own status untouched, so all that is required is a model to
1615
+ // write against.
1616
+ const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1617
+ return s.result(
1618
+ "model_unreachable",
1619
+ "cooldown",
1620
+ cooldownMs,
1621
+ new Date(Date.now() + cooldownMs),
1622
+ s.reason("Model unreachable for this account"),
1623
+ modelScopedMutation,
1624
+ );
1625
+ },
1626
+ },
1627
+ {
1628
+ // Qoder closes an exhausted-quota stream with upstream 499 (nginx
1629
+ // client-close style). The gateway's own aborts are local `transport_closed`
1630
+ // and never carry a qoder providerId, so a 499 scoped to qoder is a quota
1631
+ // verdict, not a transport fault. It must run before `transport-fault`,
1632
+ // which otherwise reads the status as a 5xx-class outage and cools the
1633
+ // account for the shorter transient window — or leaves it unmutated.
1634
+ name: "qoder-499-quota",
1635
+ match: (s) =>
1636
+ (s.providerId === "qoder" || s.providerId === "qodercn") &&
1637
+ s.statusCode === 499 &&
1638
+ s.origin === "upstream",
1639
+ decide: (s) => {
1640
+ const cooldownMs = s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
1641
+ return s.result(
1642
+ "quota_exhausted",
1643
+ "cooldown",
1644
+ cooldownMs,
1645
+ new Date(Date.now() + cooldownMs),
1646
+ s.reason("Qoder quota exhausted (upstream 499)"),
1647
+ s.origin === "upstream",
1648
+ );
1649
+ },
1650
+ },
1651
+ {
1652
+ name: "transport-fault",
1653
+ match: (s) =>
1654
+ (s.statusCode !== null && s.statusCode >= 500 && s.statusCode <= 504) ||
1655
+ s.lower.includes("timeout") ||
1656
+ s.lower.includes("abort") ||
1657
+ s.lower.includes("econnrefused") ||
1658
+ s.lower.includes("fetch failed"),
1659
+ decide: (s) => {
1660
+ // A transport-shaped fault clears on its own, so it is a cooldown with a
1661
+ // deadline rather than a state needing an operator. The deadline matters:
1662
+ // `sweepExpiredCooldowns` selects on `cooldownUntil IS NOT NULL`, so a row
1663
+ // parked without one is never recovered and stays out of rotation.
1664
+ //
1665
+ // `canMutate` rather than a flat `false`: a 5xx whose failure the caller
1666
+ // scoped to the account (an edge/WAF challenge answered to one account's
1667
+ // requests while its siblings on the same egress served fine) must cool
1668
+ // that account down, or it stays `active` and is re-dialled for as long as
1669
+ // the block lasts. Provider-scoped 5xx does not: the upstream is
1670
+ // indiscriminately failing and no single account is at fault.
1671
+ const transientMs = TRANSIENT_ERROR_COOLDOWN_MS();
1672
+ return s.result(
1673
+ s.lower.includes("timeout") ? "timeout" : "server_error",
1674
+ "cooldown",
1675
+ transientMs,
1676
+ new Date(Date.now() + transientMs),
1677
+ s.reason(`Provider ${s.statusCode ?? "network"} failure`),
1678
+ s.canMutate,
1679
+ );
1680
+ },
1681
+ },
1682
+ ];