cartethyia 2.6.7 → 2.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (335) hide show
  1. package/.env.example +1 -7
  2. package/README.md +25 -2
  3. package/dist/dashboard/assets/{BannedPage-Dskhyp0l.js → BannedPage-D-J75hBM.js} +1 -1
  4. package/dist/dashboard/assets/BannedPage-D-J75hBM.js.gz +0 -0
  5. package/dist/dashboard/assets/{CliToolDetailPage--SJhk7qQ.js → CliToolDetailPage-CVe6RaBH.js} +1 -1
  6. package/dist/dashboard/assets/CliToolDetailPage-CVe6RaBH.js.gz +0 -0
  7. package/dist/dashboard/assets/{CliToolsPage-C203VT-c.js → CliToolsPage-Dd8ZSamp.js} +1 -1
  8. package/dist/dashboard/assets/CliToolsPage-Dd8ZSamp.js.gz +0 -0
  9. package/dist/dashboard/assets/{CombosPage-BcS7kGG1.js → CombosPage-QoJ6foxo.js} +1 -1
  10. package/dist/dashboard/assets/CombosPage-QoJ6foxo.js.gz +0 -0
  11. package/dist/dashboard/assets/ConsoleLogPage-DpNRay4C.js +1 -0
  12. package/dist/dashboard/assets/ConsoleLogPage-DpNRay4C.js.gz +0 -0
  13. package/dist/dashboard/assets/{CustomizationPage-uujbBYwC.js → CustomizationPage-CKUFTcW4.js} +1 -1
  14. package/dist/dashboard/assets/CustomizationPage-CKUFTcW4.js.gz +0 -0
  15. package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js +1 -0
  16. package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js.gz +0 -0
  17. package/dist/dashboard/assets/LoginPage-DubdQcvX.js +1 -0
  18. package/dist/dashboard/assets/LoginPage-DubdQcvX.js.gz +0 -0
  19. package/dist/dashboard/assets/{ModelPicker-CA1kLs45.js → ModelPicker-BH8Tehdi.js} +1 -1
  20. package/dist/dashboard/assets/ModelPicker-BH8Tehdi.js.gz +0 -0
  21. package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js +1 -0
  22. package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js.gz +0 -0
  23. package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js +4 -0
  24. package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js.gz +0 -0
  25. package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js +1 -0
  26. package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js.gz +0 -0
  27. package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js +1 -0
  28. package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js.gz +0 -0
  29. package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js +9 -0
  30. package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js.gz +0 -0
  31. package/dist/dashboard/assets/{QuotaPage-BueSWX5D.js → QuotaPage-ATpaxrbl.js} +1 -1
  32. package/dist/dashboard/assets/QuotaPage-ATpaxrbl.js.gz +0 -0
  33. package/dist/dashboard/assets/SettingsPage-CETophpx.js +1 -0
  34. package/dist/dashboard/assets/SettingsPage-CETophpx.js.gz +0 -0
  35. package/dist/dashboard/assets/SetupPage-BdgECjqq.js +1 -0
  36. package/dist/dashboard/assets/SetupPage-BdgECjqq.js.gz +0 -0
  37. package/dist/dashboard/assets/{SortableList-DrKeMcGN.js → SortableList-CAK-EN9z.js} +1 -1
  38. package/dist/dashboard/assets/SortableList-CAK-EN9z.js.gz +0 -0
  39. package/dist/dashboard/assets/{StudioPage-qjuYtccc.js → StudioPage-B7W2_RhT.js} +3 -3
  40. package/dist/dashboard/assets/StudioPage-B7W2_RhT.js.gz +0 -0
  41. package/dist/dashboard/assets/UsagePage-6T4EFDjW.js +56 -0
  42. package/dist/dashboard/assets/UsagePage-6T4EFDjW.js.gz +0 -0
  43. package/dist/dashboard/assets/{api-keys-C23S9sgp.js → api-keys-CUi6UXsR.js} +1 -1
  44. package/dist/dashboard/assets/api-keys-CUi6UXsR.js.gz +0 -0
  45. package/dist/dashboard/assets/app-KvsUJUMF.css +2 -0
  46. package/dist/dashboard/assets/app-KvsUJUMF.css.gz +0 -0
  47. package/dist/dashboard/assets/{assert-DU6qrMA_.js → assert-67xXH5rf.js} +1 -1
  48. package/dist/dashboard/assets/assert-67xXH5rf.js.gz +0 -0
  49. package/dist/dashboard/assets/{badge-Bkz_-RrM.js → badge-DC2L43Mj.js} +1 -1
  50. package/dist/dashboard/assets/badge-DC2L43Mj.js.gz +0 -0
  51. package/dist/dashboard/assets/{banner-C-7y3VKV.js → banner-BBVcb0J3.js} +1 -1
  52. package/dist/dashboard/assets/banner-BBVcb0J3.js.gz +0 -0
  53. package/dist/dashboard/assets/{card-BOPBuBzD.js → card-Cca1XC6G.js} +1 -1
  54. package/dist/dashboard/assets/card-Cca1XC6G.js.gz +0 -0
  55. package/dist/dashboard/assets/{cli-tools-BaFUuFFF.js → cli-tools-KvsWa1Jl.js} +1 -1
  56. package/dist/dashboard/assets/cli-tools-KvsWa1Jl.js.gz +0 -0
  57. package/dist/dashboard/assets/{clipboard-button-e-Mlk0FH.js → clipboard-button-DUzEbynq.js} +1 -1
  58. package/dist/dashboard/assets/clipboard-button-DUzEbynq.js.gz +0 -0
  59. package/dist/dashboard/assets/{dialog-Iec_ywbD.js → dialog-DFvWhwTa.js} +1 -1
  60. package/dist/dashboard/assets/dialog-DFvWhwTa.js.gz +0 -0
  61. package/dist/dashboard/assets/{drawer-Bs8MIIHc.js → drawer-CFM2sMX1.js} +1 -1
  62. package/dist/dashboard/assets/drawer-CFM2sMX1.js.gz +0 -0
  63. package/dist/dashboard/assets/{entry-DOddwoc0.js → entry-BLBLxX0h.js} +3 -3
  64. package/dist/dashboard/assets/entry-BLBLxX0h.js.gz +0 -0
  65. package/dist/dashboard/assets/{entry-DcobaKFu.js → entry-Bqy8RFvM.js} +1 -1
  66. package/dist/dashboard/assets/entry-Bqy8RFvM.js.gz +0 -0
  67. package/dist/dashboard/assets/{entry-BRszm4_Z.js → entry-_xHo5x6r.js} +1 -1
  68. package/dist/dashboard/assets/entry-_xHo5x6r.js.gz +0 -0
  69. package/dist/dashboard/assets/formatters-mpBPqAQ8.js +1 -0
  70. package/dist/dashboard/assets/formatters-mpBPqAQ8.js.gz +0 -0
  71. package/dist/dashboard/assets/{github-badge-BuDsrsPn.js → github-badge-C9e8Ts7e.js} +1 -1
  72. package/dist/dashboard/assets/github-badge-C9e8Ts7e.js.gz +0 -0
  73. package/dist/dashboard/assets/{index-BeLRhsmt.js → index-D6QzEC_n.js} +2 -2
  74. package/dist/dashboard/assets/index-D6QzEC_n.js.gz +0 -0
  75. package/dist/dashboard/assets/{inline-DToMICwl.js → inline-BFUcvllj.js} +1 -1
  76. package/dist/dashboard/assets/inline-BFUcvllj.js.gz +0 -0
  77. package/dist/dashboard/assets/{input-WlyHWeG2.js → input-C_ILAvBb.js} +1 -1
  78. package/dist/dashboard/assets/input-C_ILAvBb.js.gz +0 -0
  79. package/dist/dashboard/assets/{layout-DaJ5baJr.js → layout-CBIyT6Rt.js} +1 -1
  80. package/dist/dashboard/assets/layout-CBIyT6Rt.js.gz +0 -0
  81. package/dist/dashboard/assets/{list-row-B3oSK13z.js → list-row-ebJ_-Nsb.js} +1 -1
  82. package/dist/dashboard/assets/list-row-ebJ_-Nsb.js.gz +0 -0
  83. package/dist/dashboard/assets/{live-ta6GMb-8.js → live-D0mm--ME.js} +1 -1
  84. package/dist/dashboard/assets/live-D0mm--ME.js.gz +0 -0
  85. package/dist/dashboard/assets/minimize-2-D-MN8W8a.js +1 -0
  86. package/dist/dashboard/assets/minimize-2-D-MN8W8a.js.gz +0 -0
  87. package/dist/dashboard/assets/{mutation-g_fduiNw.js → mutation-sMD965oO.js} +1 -1
  88. package/dist/dashboard/assets/mutation-sMD965oO.js.gz +0 -0
  89. package/dist/dashboard/assets/{numbers-Dulqgys5.js → numbers-BO9dKaFr.js} +1 -1
  90. package/dist/dashboard/assets/numbers-BO9dKaFr.js.gz +0 -0
  91. package/dist/dashboard/assets/providers/inception.svg +5 -0
  92. package/dist/dashboard/assets/providers/inception.svg.gz +0 -0
  93. package/dist/dashboard/assets/{query-keys-UETqfytp.js → query-keys-CfL_Zdoh.js} +1 -1
  94. package/dist/dashboard/assets/query-keys-CfL_Zdoh.js.gz +0 -0
  95. package/dist/dashboard/assets/{routing-BPUo-W8d.js → routing-BnIpi1IJ.js} +1 -1
  96. package/dist/dashboard/assets/routing-BnIpi1IJ.js.gz +0 -0
  97. package/dist/dashboard/assets/{select-BfZ3SCTM.js → select-Cj5Nw43g.js} +1 -1
  98. package/dist/dashboard/assets/select-Cj5Nw43g.js.gz +0 -0
  99. package/dist/dashboard/assets/settings-BAFhDgX8.js +1 -0
  100. package/dist/dashboard/assets/settings-BAFhDgX8.js.gz +0 -0
  101. package/dist/dashboard/assets/{stack-B1w5BM0u.js → stack-EMrOMt4I.js} +1 -1
  102. package/dist/dashboard/assets/stack-EMrOMt4I.js.gz +0 -0
  103. package/dist/dashboard/assets/{state-DhZBfWgx.js → state-CWoo7ty5.js} +1 -1
  104. package/dist/dashboard/assets/state-CWoo7ty5.js.gz +0 -0
  105. package/dist/dashboard/assets/{switch-yD7D99Lg.js → switch-CCjT6IkF.js} +1 -1
  106. package/dist/dashboard/assets/switch-CCjT6IkF.js.gz +0 -0
  107. package/dist/dashboard/assets/{useMutation-EDFRkn1i.js → useMutation-DjkLHUcE.js} +1 -1
  108. package/dist/dashboard/assets/useMutation-DjkLHUcE.js.gz +0 -0
  109. package/dist/dashboard/index.html +1 -1
  110. package/dist/dashboard/index.html.gz +0 -0
  111. package/dist/dashboard/precompressed-assets.json +55 -54
  112. package/migrations/full-postgres/0024_add_pool_provider_cooldowns.sql +12 -0
  113. package/migrations/full-postgres/0025_add_pool_random_strategy.sql +3 -0
  114. package/migrations/lite-sqlite3/0024_add_pool_provider_cooldowns.sql +12 -0
  115. package/migrations/lite-sqlite3/0025_add_pool_random_strategy.sql +20 -0
  116. package/package.json +1 -1
  117. package/scripts/cli/console-menu.ts +5 -2
  118. package/scripts/cli/tray.ts +21 -6
  119. package/src/boot/app.ts +24 -62
  120. package/src/boot/dependencies.ts +732 -750
  121. package/src/config/cooldowns.ts +16 -0
  122. package/src/config/providers.ts +13 -1
  123. package/src/config/resources.ts +13 -80
  124. package/src/config/security.ts +2 -38
  125. package/src/config/spec.ts +16 -23
  126. package/src/console/console-router.ts +7 -7
  127. package/src/console/domain-registration.ts +19 -12
  128. package/src/console/providers/catalog/catalog-projections.ts +2 -6
  129. package/src/console/providers/catalog/contracts.ts +19 -0
  130. package/src/console/providers/catalog/provider-operations.ts +38 -0
  131. package/src/console/providers/catalog/routes.ts +14 -0
  132. package/src/console/providers/catalog/store-helpers.ts +326 -0
  133. package/src/console/providers/catalog/store.ts +1292 -1563
  134. package/src/console/quota/account-quota-global.ts +2 -2
  135. package/src/console/quota/account-quota-shared.ts +1 -1
  136. package/src/console/quota/account-quota-tenant.ts +35 -12
  137. package/src/console/quota/quota-view.ts +2 -2
  138. package/src/console/remote-support/contracts.ts +3 -1
  139. package/src/console/remote-support/service.ts +115 -6
  140. package/src/console/routing/pools/contracts.ts +41 -1
  141. package/src/console/routing/pools/operations.ts +183 -4
  142. package/src/console/routing/pools/relay-deploy.ts +17 -4
  143. package/src/console/routing/pools/routes.ts +30 -0
  144. package/src/console/routing/pools/store.ts +98 -8
  145. package/src/main.ts +16 -2
  146. package/src/network/outbound-fetch.ts +226 -365
  147. package/src/network/pool/agent.ts +8 -0
  148. package/src/network/pool/pool-state.ts +7 -0
  149. package/src/network/pool/selector.ts +110 -2
  150. package/src/network/pool-cooldown-store.ts +252 -0
  151. package/src/network/pool-health-machine.ts +5 -3
  152. package/src/network/pool-health.ts +36 -9
  153. package/src/observability/event-query.ts +57 -0
  154. package/src/observability/metrics.ts +15 -5
  155. package/src/observability/runtime-metrics.ts +30 -15
  156. package/src/persistence/health-event-flush.ts +2 -0
  157. package/src/persistence/schema.ts +19 -0
  158. package/src/persistence/usage-flush.ts +8 -3
  159. package/src/protocol/canonical-model.ts +1 -1
  160. package/src/protocol/dsml.ts +377 -0
  161. package/src/protocol/gateway-error.ts +0 -1
  162. package/src/{providers/provider-reasoning.ts → protocol/reasoning.ts} +10 -9
  163. package/src/protocol/request/chat.ts +7 -7
  164. package/src/protocol/request/codex.ts +2 -2
  165. package/src/protocol/request/gemini.ts +1 -1
  166. package/src/protocol/request/messages.ts +6 -6
  167. package/src/protocol/request/responses.ts +7 -7
  168. package/src/protocol/response/dsml.ts +12 -133
  169. package/src/{transport → protocol}/translation/attachment-integrity.ts +2 -2
  170. package/src/{transport → protocol}/translation/cache-controls.ts +1 -1
  171. package/src/{transport → protocol}/translation/capabilities.ts +2 -2
  172. package/src/{transport → protocol}/translation/normalize.ts +1 -1
  173. package/src/{transport → protocol}/translation/quirks.ts +1 -1
  174. package/src/{transport → protocol}/translation/search.ts +1 -1
  175. package/src/{transport → protocol}/translation/thinking.ts +23 -1
  176. package/src/{transport → protocol}/translation/tool-id.ts +1 -1
  177. package/src/{transport → protocol}/translation/tool-repair.ts +2 -2
  178. package/src/protocol/upstream-parsing.ts +176 -0
  179. package/src/providers/adapters/compatible-adapter.ts +11 -2
  180. package/src/providers/authentication/oauth-refresh-service.ts +9 -5
  181. package/src/providers/default-registry.ts +21 -5
  182. package/src/providers/discovery/probe-phases.ts +16 -3
  183. package/src/providers/discovery/probing-service.ts +4 -1
  184. package/src/providers/integrations/anthropic.ts +1 -1
  185. package/src/providers/integrations/buddy/buddy-catalog-shared.ts +1 -4
  186. package/src/providers/integrations/buddy/buddy-chat-shared.ts +141 -8
  187. package/src/providers/integrations/buddy/buddy-checkin.ts +5 -1
  188. package/src/providers/integrations/buddy/buddy-growth-shared.ts +0 -82
  189. package/src/providers/integrations/buddy/buddy-identity.ts +25 -0
  190. package/src/providers/integrations/buddy/buddy-server-tools.ts +240 -0
  191. package/src/providers/integrations/buddy/codebuddy/cn-catalog.ts +36 -0
  192. package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +6 -31
  193. package/src/providers/integrations/buddy/codebuddy/provider.ts +2 -0
  194. package/src/providers/integrations/buddy/codebuddy/shared.ts +19 -4
  195. package/src/providers/integrations/buddy/turing-device-token.ts +262 -0
  196. package/src/providers/integrations/buddy/workbuddy/provider.ts +2 -0
  197. package/src/providers/integrations/buddy/workbuddy/shared.ts +56 -3
  198. package/src/providers/integrations/claude/claude.ts +1 -1
  199. package/src/providers/integrations/cloudflareai/cloudflareai.ts +1 -1
  200. package/src/providers/integrations/codex/codex-errors.ts +1 -1
  201. package/src/providers/integrations/codex/codex.ts +1 -1
  202. package/src/providers/integrations/commandcode/commandcode.ts +134 -104
  203. package/src/providers/integrations/commandcode/discovery.ts +6 -5
  204. package/src/providers/integrations/factory/factory.ts +1 -1
  205. package/src/providers/integrations/github/github.ts +1 -1
  206. package/src/providers/integrations/grok/grok.ts +1 -1
  207. package/src/providers/integrations/inception/provider.ts +91 -0
  208. package/src/providers/integrations/inferhub.ts +1 -1
  209. package/src/providers/integrations/kiro/kiro-request.ts +1 -1
  210. package/src/providers/integrations/mistral.ts +193 -19
  211. package/src/providers/integrations/opencode/fingerprint.ts +19 -2
  212. package/src/providers/integrations/opencode/prompt.ts +111 -0
  213. package/src/providers/integrations/opencode/provider.ts +101 -23
  214. package/src/providers/integrations/opencode/tool-definitions.ts +4 -4
  215. package/src/providers/integrations/qoder/qoder-checkin.ts +303 -0
  216. package/src/providers/integrations/qoder/qoder.ts +1641 -884
  217. package/src/providers/integrations/shared/install-id.ts +2 -1
  218. package/src/providers/integrations/trae/trae-cache.ts +110 -0
  219. package/src/providers/integrations/trae/trae-oauth.ts +29 -4
  220. package/src/providers/integrations/trae/trae-protocol.ts +2 -1
  221. package/src/providers/integrations/trae/trae.ts +106 -23
  222. package/src/providers/operations/account-error-classification.ts +1660 -0
  223. package/src/providers/operations/account-health-service.ts +621 -2097
  224. package/src/providers/operations/account-reset-service.ts +2 -0
  225. package/src/providers/provider-contracts.ts +0 -2
  226. package/src/providers/provider-metadata.ts +25 -4
  227. package/src/providers/provider-probe.ts +2 -2
  228. package/src/{console → providers}/quota/cache.ts +1 -1
  229. package/src/{console → providers}/quota/refresh.ts +5 -5
  230. package/src/security/abuse.ts +17 -0
  231. package/src/transport/dispatch/attempt-finalize.ts +114 -12
  232. package/src/transport/dispatch/attempt-loop.ts +3 -0
  233. package/src/transport/dispatch/empty-stream.ts +224 -217
  234. package/src/transport/dispatch/generation.ts +37 -55
  235. package/src/transport/dispatch/handler-deps.ts +32 -0
  236. package/src/transport/dispatch/leases.ts +9 -4
  237. package/src/transport/dispatch/proxy-request.ts +6 -30
  238. package/src/transport/dispatch/responses-compact.ts +22 -43
  239. package/src/transport/dispatch/streaming/finalize.ts +3 -0
  240. package/src/transport/dispatch/streaming/stream.ts +0 -8
  241. package/src/transport/dispatch/systemone.ts +22 -42
  242. package/src/transport/dispatch/token-count.ts +23 -39
  243. package/src/transport/dispatch/upstream.ts +1 -3
  244. package/src/transport/dispatch/websearch.ts +22 -41
  245. package/src/transport/failure-policy.ts +59 -178
  246. package/src/transport/guards/abuse-guard.ts +10 -0
  247. package/src/transport/guards/auth-guard.ts +0 -18
  248. package/src/transport/guards/error-lifecycle.ts +1 -1
  249. package/src/transport/guards/pipeline.ts +0 -6
  250. package/src/transport/guards/request-context.ts +18 -91
  251. package/src/transport/request/degradation.ts +20 -1
  252. package/src/transport/request/inflight.ts +4 -95
  253. package/src/transport/request/preparer.ts +6 -6
  254. package/src/transport/request/state.ts +66 -38
  255. package/src/transport/request/tenant-preferences.ts +1 -1
  256. package/src/transport/routing/route-catalog.ts +41 -31
  257. package/src/transport/routing/router.ts +2 -2
  258. package/src/transport/surface/messages/parse.ts +1 -1
  259. package/src/workers/buddy-growth-tasks.ts +346 -425
  260. package/src/workers/daily-checkin.ts +17 -6
  261. package/src/workers/quota-refresh-sweep.ts +2 -2
  262. package/src/workers/scheduler.ts +17 -1
  263. package/dist/dashboard/assets/BannedPage-Dskhyp0l.js.gz +0 -0
  264. package/dist/dashboard/assets/CliToolDetailPage--SJhk7qQ.js.gz +0 -0
  265. package/dist/dashboard/assets/CliToolsPage-C203VT-c.js.gz +0 -0
  266. package/dist/dashboard/assets/CombosPage-BcS7kGG1.js.gz +0 -0
  267. package/dist/dashboard/assets/ConsoleLogPage-ChmiSLwO.js +0 -1
  268. package/dist/dashboard/assets/ConsoleLogPage-ChmiSLwO.js.gz +0 -0
  269. package/dist/dashboard/assets/CustomizationPage-uujbBYwC.js.gz +0 -0
  270. package/dist/dashboard/assets/HealthEventsModal-fKAWd-OQ.js +0 -1
  271. package/dist/dashboard/assets/HealthEventsModal-fKAWd-OQ.js.gz +0 -0
  272. package/dist/dashboard/assets/LoginPage-DoOUthbG.js +0 -1
  273. package/dist/dashboard/assets/LoginPage-DoOUthbG.js.gz +0 -0
  274. package/dist/dashboard/assets/ModelPicker-CA1kLs45.js.gz +0 -0
  275. package/dist/dashboard/assets/OverviewPage-C5Z-UBz1.js +0 -1
  276. package/dist/dashboard/assets/OverviewPage-C5Z-UBz1.js.gz +0 -0
  277. package/dist/dashboard/assets/ProviderDetailPage-CFu1qGcT.js +0 -3
  278. package/dist/dashboard/assets/ProviderDetailPage-CFu1qGcT.js.gz +0 -0
  279. package/dist/dashboard/assets/ProviderIcon-CmfzxpVM.js +0 -1
  280. package/dist/dashboard/assets/ProviderIcon-CmfzxpVM.js.gz +0 -0
  281. package/dist/dashboard/assets/ProvidersPage-CnlJZbxF.js +0 -1
  282. package/dist/dashboard/assets/ProvidersPage-CnlJZbxF.js.gz +0 -0
  283. package/dist/dashboard/assets/ProxyPage-B9mBTweh.js +0 -7
  284. package/dist/dashboard/assets/ProxyPage-B9mBTweh.js.gz +0 -0
  285. package/dist/dashboard/assets/QuotaPage-BueSWX5D.js.gz +0 -0
  286. package/dist/dashboard/assets/SettingsPage-B0x00MVY.js +0 -1
  287. package/dist/dashboard/assets/SettingsPage-B0x00MVY.js.gz +0 -0
  288. package/dist/dashboard/assets/SetupPage-DQ9oDRpi.js +0 -1
  289. package/dist/dashboard/assets/SetupPage-DQ9oDRpi.js.gz +0 -0
  290. package/dist/dashboard/assets/SortableList-DrKeMcGN.js.gz +0 -0
  291. package/dist/dashboard/assets/StudioPage-qjuYtccc.js.gz +0 -0
  292. package/dist/dashboard/assets/UsagePage-Be7aNviQ.js +0 -56
  293. package/dist/dashboard/assets/UsagePage-Be7aNviQ.js.gz +0 -0
  294. package/dist/dashboard/assets/api-keys-C23S9sgp.js.gz +0 -0
  295. package/dist/dashboard/assets/app-BUSiYL6F.css +0 -2
  296. package/dist/dashboard/assets/app-BUSiYL6F.css.gz +0 -0
  297. package/dist/dashboard/assets/assert-DU6qrMA_.js.gz +0 -0
  298. package/dist/dashboard/assets/badge-Bkz_-RrM.js.gz +0 -0
  299. package/dist/dashboard/assets/banner-C-7y3VKV.js.gz +0 -0
  300. package/dist/dashboard/assets/card-BOPBuBzD.js.gz +0 -0
  301. package/dist/dashboard/assets/cli-tools-BaFUuFFF.js.gz +0 -0
  302. package/dist/dashboard/assets/clipboard-button-e-Mlk0FH.js.gz +0 -0
  303. package/dist/dashboard/assets/cog-CdWhsajs.js +0 -1
  304. package/dist/dashboard/assets/cog-CdWhsajs.js.gz +0 -0
  305. package/dist/dashboard/assets/dialog-Iec_ywbD.js.gz +0 -0
  306. package/dist/dashboard/assets/drawer-Bs8MIIHc.js.gz +0 -0
  307. package/dist/dashboard/assets/entry-BRszm4_Z.js.gz +0 -0
  308. package/dist/dashboard/assets/entry-DOddwoc0.js.gz +0 -0
  309. package/dist/dashboard/assets/entry-DcobaKFu.js.gz +0 -0
  310. package/dist/dashboard/assets/github-badge-BuDsrsPn.js.gz +0 -0
  311. package/dist/dashboard/assets/index-BeLRhsmt.js.gz +0 -0
  312. package/dist/dashboard/assets/inline-DToMICwl.js.gz +0 -0
  313. package/dist/dashboard/assets/input-WlyHWeG2.js.gz +0 -0
  314. package/dist/dashboard/assets/layout-DaJ5baJr.js.gz +0 -0
  315. package/dist/dashboard/assets/list-row-B3oSK13z.js.gz +0 -0
  316. package/dist/dashboard/assets/live-ta6GMb-8.js.gz +0 -0
  317. package/dist/dashboard/assets/model-bans-BmYTmbAk.js +0 -1
  318. package/dist/dashboard/assets/model-bans-BmYTmbAk.js.gz +0 -0
  319. package/dist/dashboard/assets/mutation-g_fduiNw.js.gz +0 -0
  320. package/dist/dashboard/assets/numbers-Dulqgys5.js.gz +0 -0
  321. package/dist/dashboard/assets/query-keys-UETqfytp.js.gz +0 -0
  322. package/dist/dashboard/assets/routing-BPUo-W8d.js.gz +0 -0
  323. package/dist/dashboard/assets/select-BfZ3SCTM.js.gz +0 -0
  324. package/dist/dashboard/assets/settings--3hxSyry.js +0 -1
  325. package/dist/dashboard/assets/settings--3hxSyry.js.gz +0 -0
  326. package/dist/dashboard/assets/stack-B1w5BM0u.js.gz +0 -0
  327. package/dist/dashboard/assets/state-DhZBfWgx.js.gz +0 -0
  328. package/dist/dashboard/assets/switch-yD7D99Lg.js.gz +0 -0
  329. package/dist/dashboard/assets/useMutation-EDFRkn1i.js.gz +0 -0
  330. package/src/console/admin/model-abuse/contracts.ts +0 -38
  331. package/src/console/admin/model-abuse/routes.ts +0 -97
  332. package/src/security/model-abuse.ts +0 -704
  333. package/src/transport/guards/concurrency-guard.ts +0 -125
  334. /package/dist/dashboard/assets/{app-MdVfwraB.js → app-BaWd0puM.js} +0 -0
  335. /package/dist/dashboard/assets/{app-MdVfwraB.js.gz → app-BaWd0puM.js.gz} +0 -0
@@ -1,2097 +1,621 @@
1
- import { healthEventBuffer } from "../../persistence/health-event-flush";
2
- // Account health: error classification, failure/success recording, recovery and cooldown sweeps.
3
- import { and, count, desc, eq, inArray, isNotNull, lte, or, sql } from "drizzle-orm";
4
- import { affectedRows, pruneExpiredCooldowns } from "../../persistence/dialect";
5
- import type { CartethyiaDatabase } from "../../persistence/db-handle";
6
- import { isRecord } from "../../protocol/codecs";
7
- import { GatewayError } from "../../protocol/gateway-error";
8
- import { BUDDY_PROVIDER_IDS } from "../provider-metadata";
9
- import { healthEvents, providerAccounts } from "../../persistence/schema";
10
-
11
- import { parseProviderResetDuration, parseUpstreamBackoff } from "../../transport/failure-policy";
12
- import {
13
- resolveAccountAuthRejectedCooldownMs,
14
- resolveAccountCooldownEscalationMaxMs,
15
- resolveAccountDisabledRecoveryMs,
16
- resolveAccountModelCapacityCooldownMs,
17
- resolveAccountPolicyBlockCooldownMs,
18
- resolveAccountQuotaCooldownMs,
19
- resolveAccountRateLimitCooldownMs,
20
- resolveAccountTransientCooldownMs,
21
- resolveAccountUnclassifiedCooldownMs,
22
- } from "../../config/cooldowns";
23
- // ===== health/constants.ts =====
24
- /**
25
- * Account status is authoritative for routing: `disabled` is a hard exclusion
26
- * until an operator restores it, while `cooldown` is a *soft* one — the account
27
- * is deprioritized behind every healthy sibling and reached only when nothing
28
- * better is left, so a cooling credential still serves rather than parking the
29
- * model entirely.
30
- *
31
- * Every delay below is a *fallback*: an upstream `Retry-After`/reset header or
32
- * a duration stated in the provider message always wins. They are read through
33
- * `src/config/cooldowns.ts` so an operator can tune the health machine without a code
34
- * change (`CARTETHYIA_ACCOUNT_*_COOLDOWN_MS`).
35
- */
36
- const DEFAULT_RATE_LIMIT_COOLDOWN_MS = resolveAccountRateLimitCooldownMs;
37
- // Fallback when the error states no usable duration: deliberately short (1h,
38
- // not 24h) — quota state is re-probed on the next failure, so an unknown
39
- // reset cadence self-corrects instead of parking the account for a day.
40
- const DEFAULT_QUOTA_COOLDOWN_MS = resolveAccountQuotaCooldownMs;
41
- const MODEL_CAPACITY_COOLDOWN_MS = resolveAccountModelCapacityCooldownMs;
42
- const TRANSIENT_ERROR_COOLDOWN_MS = resolveAccountTransientCooldownMs;
43
- /**
44
- * Accounts one cooldown-recovery pass brings back. The sweep ticks every 30s,
45
- * so a larger backlog drains over successive passes rather than one statement
46
- * covering every expired row.
47
- */
48
- const COOLDOWN_RECOVERY_BATCH = 500;
49
- /**
50
- * Backoff for a failure no rule matched. Short on purpose: an unclassified
51
- * error is more likely a new upstream shape than a broken account, so the
52
- * account returns to rotation quickly.
53
- */
54
- const UNCLASSIFIED_COOLDOWN_MS = resolveAccountUnclassifiedCooldownMs;
55
- /**
56
- * xAI Grok Build's free tier resets on a rolling 24-hour window, so its
57
- * exhaustion is a full-day quota cooldown — never the generic 1h fallback,
58
- * which let the account re-enter rotation inside the window.
59
- */
60
- const GROK_QUOTA_COOLDOWN_MS = 24 * 60 * 60 * 1000;
61
- /**
62
- * The buddy family's policy/channel block (`11140`, `11128`) is a *base* backoff,
63
- * not a flat penalty — see {@link resolveAccountPolicyBlockCooldownMs}. It used
64
- * to be two constants, 24h for the content block and 6h for the channel one,
65
- * both applied on the first observation. That was wrong in the direction that
66
- * costs most: `11140` is also what a wrong desktop `User-Agent` brand answers
67
- * (`workbuddy/shared.ts`), so after we fixed a header regression every buddy
68
- * account stayed dead for up to a further day, and one prompt that failed a
69
- * safety review cost a whole account a day. As a base it escalates through the
70
- * consecutive-failure ladder, which tells the two causes apart by what they do
71
- * next — a genuinely blocked account is refused again immediately and parks
72
- * itself within minutes.
73
- */
74
- /**
75
- * Backoff for a model the upstream says does not exist for this account
76
- * (Cursor answers Connect `not_found` for a model the credential's plan does
77
- * not reach). Long, because the verdict is a property of the plan and will not
78
- * change on the next request — but bounded, so a plan upgrade is picked up
79
- * without an operator clearing it by hand.
80
- */
81
- const MODEL_UNREACHABLE_COOLDOWN_MS = 6 * 60 * 60 * 1000;
82
- /**
83
- * Backoff for a model the upstream does not serve at all (buddy family
84
- * business code `11102` — `service info not found` — or a `model ... not
85
- * found/supported` message). Mirrors the reference gateways'
86
- * `MODEL_SITE_BLOCK_S` → `MODEL_SITE_BLOCK_MAX_S`.
87
- *
88
- * Escalating rather than flat: the pair is re-dialled after the first window
89
- * and answers the same thing, so a flat 6h let a permanently unsupported model
90
- * consume a request every six hours forever. The ladder tops out at a day,
91
- * which is long enough to stop the redial and short enough that a backend that
92
- * starts serving the model is picked up without an operator clearing it.
93
- *
94
- * This is the one place the references' *strict* behaviour is adopted as a
95
- * backoff rather than as a 400: nothing currently succeeds through it, so no
96
- * client is broken by it.
97
- */
98
- const UNSUPPORTED_MODEL_COOLDOWN_MS = 6 * 60 * 60 * 1000;
99
- const UNSUPPORTED_MODEL_COOLDOWN_MAX_MS = 24 * 60 * 60 * 1000;
100
- /** How many doublings an unsupported-model cooldown may take before the cap. */
101
- const UNSUPPORTED_MODEL_MAX_LEVEL = 8;
102
-
103
- /**
104
- * The cooldown for the *nth* consecutive unsupported-model report (`hits` is
105
- * the new total, so the first report is `hits === 1`).
106
- *
107
- * Exported because the writer, not the classifier, is the only place that
108
- * reads the authoritative hit count — the classifier sees only the count the
109
- * caller supplied, which is absent on the first report of a pair.
110
- */
111
- export function unsupportedModelCooldownMs(hits: number): number {
112
- const level = Math.min(Math.max(0, Math.round(hits) - 1), UNSUPPORTED_MODEL_MAX_LEVEL);
113
- return Math.min(
114
- UNSUPPORTED_MODEL_COOLDOWN_MS * 2 ** level,
115
- UNSUPPORTED_MODEL_COOLDOWN_MAX_MS,
116
- );
117
- }
118
-
119
- /**
120
- * The wording an upstream uses for "we do not serve this model at all", as
121
- * opposed to "this account cannot reach it" (403 entitlement, above) or "this
122
- * route does not exist" (404 `upstream_not_found`).
123
- */
124
- const UNSUPPORTED_MODEL_MESSAGE =
125
- /service info not found|model\s.{0,80}?\bnot\s+(?:found|supported)\b/;
126
-
127
- /**
128
- * True when the evidence says the upstream does not serve the model.
129
- *
130
- * Narrow on purpose, and narrower than `model_unreachable`: only a 400/404
131
- * carrying the buddy family's `11102` or one of the two phrasings above. A
132
- * 4xx that merely *mentions* a model is far more often a payload the caller
133
- * got wrong, and cooling a pair for six hours over that takes a working model
134
- * out of rotation for a request bug.
135
- */
136
- function isUnsupportedModelEvidence(
137
- statusCode: number | null,
138
- providerCode: string,
139
- lower: string,
140
- ): boolean {
141
- if (statusCode !== 400 && statusCode !== 404) return false;
142
- return providerCode === "11102" || UNSUPPORTED_MODEL_MESSAGE.test(lower);
143
- }
144
-
145
- // ===== health/account-recorder.ts =====
146
- export interface AccountHealthEventRecord {
147
- readonly id: string;
148
- readonly accountId: string;
149
- readonly fromStatus: string | null;
150
- readonly toStatus: string;
151
- readonly reason: string | null;
152
- readonly errorCategory: string | null;
153
- readonly modelId: string | null;
154
- readonly createdAt: string;
155
- }
156
-
157
- export type AccountErrorCategory =
158
- | "quota_exhausted"
159
- | "rate_limit_transient"
160
- | "model_capacity"
161
- | "model_unreachable"
162
- | "auth_invalidated"
163
- /**
164
- * Deprecated: an upstream ban. Kept so existing health rows keep rendering
165
- * instead of showing a blank cause. No longer produced — see
166
- * `policy_blocked_banned` and `policy_blocked_invalid_request`.
167
- */
168
- | "policy_blocked"
169
- /**
170
- * The upstream banned this credential: a verdict on the account, e.g. buddy
171
- * `11140` "request illegal" with "Your account has been restricted due to
172
- * unusual activity". Cooldown cannot help — the account is not throttled, it
173
- * is refused. Disabling immediately is a *feature*: a client's retry budget
174
- * is spent in seconds, so surviving attempts burn sibling accounts that are
175
- * still healthy.
176
- */
177
- | "policy_blocked_banned"
178
- /**
179
- * The upstream refused this request: a verdict on the request. e.g. buddy
180
- * `11-128` "request illegal" with no account restriction. Distinguished from
181
- * {@link policy_blocked_banned} because the correct responses differ — a
182
- * banned account must be disabled, a malformed request must only stop being
183
- * retried verbatim.
184
- */
185
- | "policy_blocked_invalid_request"
186
- | "server_error"
187
- | "timeout"
188
- | "unknown";
189
- export type AccountFailureOrigin = "cartethyia" | "upstream" | "network";
190
- export type AccountFailureScope =
191
- | "account"
192
- | "provider"
193
- | "model"
194
- | "pool"
195
- | "tenant"
196
- | "request"
197
- | "network"
198
- | "unknown";
199
-
200
- export interface AccountFailureEvidence {
201
- readonly origin?: AccountFailureOrigin;
202
- readonly scope?: AccountFailureScope;
203
- readonly credentialKind?: string;
204
- /**
205
- * The account's credential is a static bearer token used as issued and never
206
- * refreshed. An auth rejection against a static token is not a refreshable
207
- * fault: refreshing cannot help, and disabling the account discards a token
208
- * that may still be valid for other requests. Such a failure cools the
209
- * account down instead of parking it.
210
- */
211
- readonly staticToken?: boolean;
212
- readonly providerCode?: string;
213
- readonly credentialEvidence?: boolean;
214
- readonly retryAfterMs?: number;
215
- readonly statusCode?: number | null;
216
- readonly headers?: Headers | Record<string, string> | null;
217
- readonly modelId?: string;
218
- /**
219
- * Provider that produced the failure. Used only for the one
220
- * provider-family-specific policy rule (the buddy family's `11140`
221
- * content-policy block), never to broaden what disables an account.
222
- */
223
- readonly providerId?: string;
224
- /**
225
- * How many times this (account, model) pair has already been reported
226
- * unsupported, when the caller has read them. Best-effort: the classifier
227
- * states the base window without it, and the writer recomputes against the
228
- * count it reads itself.
229
- */
230
- readonly modelBlockHits?: number;
231
- /**
232
- * The upstream's own error body, when the caller has it.
233
- *
234
- * Providers state a reset window in structured fields far more reliably than
235
- * in prose — a body carrying `resets_at`, `resets_in_seconds` or
236
- * `limit_window_minutes` is an exact statement, while the same body's
237
- * `message` ("The usage limit has been reached") names no window at all.
238
- * Without this the classifier falls back to a fixed 15-minute cooldown for a
239
- * limit the provider said resets in 23 days.
240
- */
241
- readonly upstreamBody?: unknown;
242
- }
243
-
244
- export interface AccountErrorClassification {
245
- readonly category: AccountErrorCategory;
246
- readonly status: "cooldown" | "disabled";
247
- readonly cooldownMs: number;
248
- readonly reason: string;
249
- readonly retryAt: Date | null;
250
- readonly mutatesAccount: boolean;
251
- /**
252
- * True when this write changed a field the routing snapshot actually reads.
253
- *
254
- * The snapshot projects `provider_accounts.status` and
255
- * `.cooldownUntil` and nothing else (`route-catalog.ts:286-287`), so only a
256
- * write that moves one of those has to drop it. Everything else a failure
257
- * persists — the failure streak, the error text, the category, a
258
- * model-scoped cooldown — is invisible to routing, and invalidating for it
259
- * forced the next request to rebuild an 11-SELECT snapshot for no reason.
260
- *
261
- * That made a failure wave self-sustaining: N failed attempts produced N
262
- * invalidations, and every following request paid the full rebuild.
263
- */
264
- readonly routesVisible: boolean;
265
- /**
266
- * True when `cooldownMs` came from the provider's own statement — a parsed
267
- * `Retry-After`-family header, or a reset quoted in its message — rather than
268
- * from one of this module's fallback constants.
269
- *
270
- * A property of the *evidence*, not of the category, so it is the same for
271
- * every branch of one classification. Only {@link escalateFallbackCooldown}
272
- * reads it: a provider-stated window is the truth and is never multiplied.
273
- */
274
- readonly providerStated?: boolean;
275
- /**
276
- * True when the upstream reported the model as one it does not serve (the
277
- * buddy family's `11102`), rather than one this account merely cannot reach.
278
- *
279
- * A property of the classification, because both verdicts land in the same
280
- * `model_unreachable` category and the writer cannot otherwise tell them
281
- * apart — it needs to know whether to count the hit and escalate the window.
282
- */
283
- readonly modelUnsupported?: boolean;
284
- }
285
-
286
- /**
287
- * How many doublings a fallback cooldown may take before the cap takes over.
288
- *
289
- * Four is enough to reach the cap from every fallback default (the longest is
290
- * the 1h quota fallback, which reaches it on the first doubling) while keeping
291
- * the ladder short enough to reason about.
292
- */
293
- const COOLDOWN_ESCALATION_MAX_LEVEL = 4;
294
-
295
- /**
296
- * Escalates a *fallback* cooldown for an account that keeps failing.
297
- *
298
- * `consecutiveFailures` has been incremented on every account failure since the
299
- * health machine was written, but nothing ever read it — so a broken upstream
300
- * was re-dialled once a minute forever, and an account whose quota had an
301
- * unstated reset came straight back into rotation every time its short fallback
302
- * lapsed. The count is now a backoff level: each further consecutive failure
303
- * doubles the fallback, clamped to {@link resolveAccountCooldownEscalationMaxMs}.
304
- *
305
- * Two guards keep it from overriding the provider. A duration the provider
306
- * *stated* is never multiplied — that window is the truth. And the caller writes
307
- * the result through a monotonic maximum, so a burst of concurrent in-flight
308
- * failures advances the deadline to the longest of them instead of compounding
309
- * once per failure.
310
- */
311
- function escalateFallbackCooldown(
312
- cooldownMs: number,
313
- providerStated: boolean,
314
- consecutiveFailures: number,
315
- ): number {
316
- if (providerStated || cooldownMs <= 0) return cooldownMs;
317
- const level = Math.min(Math.max(0, consecutiveFailures), COOLDOWN_ESCALATION_MAX_LEVEL);
318
- return Math.min(cooldownMs * 2 ** level, resolveAccountCooldownEscalationMaxMs());
319
- }
320
-
321
- /**
322
- * The later of a candidate deadline and the one already recorded.
323
- *
324
- * Every fallback duration is recomputed from *now* on each failure, so writing
325
- * it unconditionally let a fresh short failure *shorten* a longer window already
326
- * in force: a 60-second unclassified cooldown overwrote the provider's own
327
- * "Try again in 11h" and released the pair back into rotation inside a window
328
- * the upstream had closed, where it failed again — once a minute, forever.
329
- * A cooling deadline may only ever move later.
330
- */
331
- function latestDeadlineMs(...candidates: readonly (number | undefined | null)[]): number {
332
- let latest = 0;
333
- for (const value of candidates) {
334
- if (typeof value !== "number" || !Number.isFinite(value)) continue;
335
- if (value > latest) latest = value;
336
- }
337
- return latest;
338
- }
339
-
340
- /**
341
- * The account-wide deadline to write, or `null` for "no deadline".
342
- *
343
- * {@link latestDeadlineMs} cannot express *no* deadline — it floors at 0 — and
344
- * the account-wide write has one state that needs it. With
345
- * `CARTETHYIA_ACCOUNT_DISABLED_RECOVERY_MS=0` an automatic disable is meant to
346
- * be terminal again, and the classifier correctly reports `retryAt: null`.
347
- * Deriving the deadline from `Date.now() + cooldownMs` instead stamps *now*.
348
- * The recovery sweep then treats that elapsed deadline as
349
- * an automatic disable awaiting recovery and returns the row to rotation on its
350
- * next tick — the escape hatch enabled exactly the behavior it documents
351
- * itself as disabling.
352
- *
353
- * A candidate only exists when there is a real window to state. Monotonicity is
354
- * preserved: a deadline already in force still wins over a new `null`.
355
- */
356
- function accountDeadlineMs(
357
- candidates: readonly (number | undefined | null)[],
358
- ): Date | null {
359
- const latest = latestDeadlineMs(...candidates);
360
- return latest > 0 ? new Date(latest) : null;
361
- }
362
-
363
- /**
364
- * Whether the upstream named a restriction on the account itself.
365
- *
366
- * The buddy family reuses "request illegal" for two unrelated rejections, and
367
- * the difference is only in the body:
368
- *
369
- * - `11140` carries `displayMsg` about the account being restricted ("Your
370
- * account has been restricted due to unusual activity") and `actions`
371
- * offering an appeal. That is a ban — the account is refused, not throttled.
372
- * - `11-128` is a refusal of this request; the account is fine.
373
- *
374
- * Proven account-scoped (2026-10-16): two live share-page requests carrying
375
- * byte-identical bodies and headers to the same model differed only in the
376
- * pooled account, and the blocked one 400'd while the other 200'd.
377
- *
378
- * Matched on the code *and* the account-restriction wording, so an unrelated
379
- * upstream that happens to say "restricted" in prose cannot disable accounts.
380
- */
381
- function isBuddyAccountBanned(
382
- lower: string,
383
- providerCode: string,
384
- upstreamBody: unknown,
385
- ): boolean {
386
- if (providerCode !== "11140") return false;
387
- if (banWording(lower)) return true;
388
- // The message alone is not enough, and this is the case that proved it: the
389
- // provider's message for a ban is the same two words it uses for a refused
390
- // request — "request illegal" — while the actual verdict ("Your account has
391
- // been restricted due to unusual activity") lives in `displayMsg`, which the
392
- // message extractor never reads. Matching only the message therefore read
393
- // every ban as a mere request refusal: no category, no disable, and the
394
- // account kept serving.
395
- const body = isRecord(upstreamBody) ? upstreamBody : undefined;
396
- if (body === undefined) return false;
397
- for (const field of ["displayMsg", "displayTips", "msg", "message"]) {
398
- const value = body[field];
399
- if (typeof value === "string" && banWording(value.toLowerCase())) return true;
400
- // `displayMsg` is a per-language map: `{ en: "…", zh: "…" }`. Any
401
- // language states the same verdict, so all of them are read.
402
- if (!isRecord(value)) continue;
403
- for (const nested of Object.values(value)) {
404
- if (typeof nested === "string" && banWording(nested.toLowerCase())) return true;
405
- }
406
- }
407
- // `actions` names the remedy: an appeal can only be offered for a verdict
408
- // about the account, and "edit input" only ever describes a request.
409
- const actions = body["actions"];
410
- if (Array.isArray(actions)) {
411
- for (const action of actions) {
412
- if (typeof action !== "string") continue;
413
- if (action.toLowerCase().includes("appeal") || action.toLowerCase().includes("feedback")) {
414
- return true;
415
- }
416
- }
417
- }
418
- return false;
419
- }
420
-
421
- /** The wording an upstream uses when it restricted the account, not the request. */
422
- function banWording(lower: string): boolean {
423
- return (
424
- lower.includes("restricted") ||
425
- lower.includes("has been banned") ||
426
- lower.includes("unusual activity") ||
427
- lower.includes("appeal this restriction") ||
428
- lower.includes("submit feedback to appeal") ||
429
- lower.includes("account has been") ||
430
- lower.includes("账号存在异常行为") ||
431
- lower.includes("帳號存在異常行為")
432
- );
433
- }
434
-
435
- /**
436
- * Bounded to a day, matching `MAX_COOLDOWN_MS` in the failure policy: a stated
437
- * window longer than that is a monthly or plan-level limit, and parking an
438
- * account for a month on one upstream field is a decision for the operator,
439
- * not an automatic one.
440
- */
441
- const MAX_UPSTREAM_STATED_COOLDOWN_MS = 24 * 3600 * 1000;
442
-
443
- /** Parses the stored upstream text as JSON. Returns `undefined` when it is not. */
444
- function tryParseUpstreamBody(text: string): unknown {
445
- try {
446
- const parsed: unknown = JSON.parse(text);
447
- return typeof parsed === "object" && parsed !== null ? parsed : undefined;
448
- } catch {
449
- return undefined;
450
- }
451
- }
452
-
453
- /**
454
- * Reads a reset window the upstream stated in structured fields.
455
- *
456
- * A body like `{"error":{"type":"usage_limit_reached","resets_at":1795019113,
457
- * "resets_in_seconds":2072028}}` states the window exactly, while its own
458
- * `message` ("The usage limit has been reached") states none — so parsing only
459
- * prose leaves a 23-day limit cooling for the 15-minute fallback, and the
460
- * account re-enters rotation hours before it can serve.
461
- *
462
- * Both spellings are read because providers use both: an absolute epoch
463
- * (`resets_at`, seconds or milliseconds) and a relative delay
464
- * (`resets_in_seconds`). The later of the two wins, since they are the same
465
- * claim computed at slightly different instants.
466
- */
467
- function parseUpstreamBodyReset(body: unknown): number | null {
468
- if (typeof body !== "object" || body === null) return null;
469
- const record = body as Record<string, unknown>;
470
- // Some providers nest under `error`, some return it at the top level.
471
- const candidates = [record, (record["error"] as Record<string, unknown> | undefined) ?? undefined];
472
- let best: number | null = null;
473
- for (const candidate of candidates) {
474
- if (typeof candidate !== "object" || candidate === null) continue;
475
- const now = Date.now();
476
-
477
- const absolute = candidate["resets_at"] ?? candidate["reset_at"];
478
- if (typeof absolute === "number" && Number.isFinite(absolute) && absolute > 0) {
479
- // Seconds vs milliseconds: a plausible epoch in seconds is ~10 digits, in
480
- // milliseconds ~13, and the two are four orders of magnitude apart — so
481
- // the smaller is scaled up rather than guessed from a threshold date.
482
- const epochMs = absolute < 1e11 ? absolute * 1000 : absolute;
483
- const delay = epochMs - now;
484
- if (delay > 0) best = best === null ? delay : Math.max(best, delay);
485
- }
486
-
487
- const relative =
488
- candidate["resets_in_seconds"] ??
489
- candidate["reset_in_seconds"] ??
490
- candidate["retry_after_seconds"];
491
- if (typeof relative === "number" && Number.isFinite(relative) && relative > 0) {
492
- const delay = relative * 1000;
493
- best = best === null ? delay : Math.max(best, delay);
494
- }
495
-
496
- const minutes = candidate["limit_window_minutes"] ?? candidate["reset_in_minutes"];
497
- if (typeof minutes === "number" && Number.isFinite(minutes) && minutes > 0) {
498
- const delay = minutes * 60_000;
499
- best = best === null ? delay : Math.max(best, delay);
500
- }
501
- }
502
- return best === null ? null : Math.min(MAX_UPSTREAM_STATED_COOLDOWN_MS, Math.round(best));
503
- }
504
-
505
-
506
- interface AccountErrorContext {
507
- readonly message: string;
508
- readonly lower: string;
509
- readonly folded: string;
510
- readonly statusCode: number | null;
511
- readonly origin: AccountFailureOrigin;
512
- readonly scope: AccountFailureScope;
513
- readonly revocationNamed: boolean;
514
- readonly accountEvidence: boolean;
515
- readonly canMutate: boolean;
516
- readonly providerCode: string;
517
- readonly statedCooldownMs: number | null;
518
- /**
519
- * The upstream named a restriction on the account, not a refusal of the
520
- * request. Read by the policy arm to pick between disabling the account and
521
- * merely refusing the request.
522
- */
523
- readonly buddyBanned: boolean;
524
- readonly reason: (prefix: string) => string;
525
- readonly result: (
526
- category: AccountErrorCategory,
527
- status: AccountErrorClassification["status"],
528
- cooldownMs: number,
529
- retryAt: Date | null,
530
- explanation: string,
531
- mutatesAccount?: boolean,
532
- modelUnsupported?: boolean,
533
- ) => AccountErrorClassification;
534
- }
535
-
536
- /**
537
- * One arm of the classifier: an ordered `(match, decide)` pair.
538
- *
539
- * The arms are **not** independent predicates. Later arms are deliberately
540
- * broader than earlier ones — `quotaSignal` matches every price refusal, and
541
- * `authSignal` matches a superset of the request-scoped codes — so a match
542
- * only wins because it is tried first. That is why they live in one ordered
543
- * list rather than behind a dispatch map or a set of independent `if`s: the
544
- * list is the precedence, and reading it top to bottom is reading the
545
- * decision.
546
- *
547
- * `match` is a pure function of the shared signals (no closure over the
548
- * classifier's locals, no allocation per classification): the table is built
549
- * once per call from values already computed for every arm.
550
- */
551
- interface AccountErrorRule {
552
- readonly name: string;
553
- readonly match: (signals: AccountErrorSignals) => boolean;
554
- readonly decide: (signals: AccountErrorSignals) => AccountErrorClassification;
555
- }
556
-
557
- /** The shared signals every arm reads, computed once per classification. */
558
- interface AccountErrorSignals {
559
- readonly message: string;
560
- readonly lower: string;
561
- readonly folded: string;
562
- readonly statusCode: number | null;
563
- readonly origin: AccountFailureOrigin;
564
- readonly scope: AccountFailureScope;
565
- readonly providerCode: string;
566
- readonly statedCooldownMs: number | null;
567
- readonly reason: (prefix: string) => string;
568
- readonly result: AccountErrorContext["result"];
569
- readonly canMutate: boolean;
570
- readonly accountEvidence: boolean;
571
- readonly revocationNamed: boolean;
572
- /** Echo of the caller's evidence; read only by the arms that need it. */
573
- readonly modelId: string | undefined;
574
- readonly modelBlockHits: number;
575
- readonly staticToken: boolean;
576
- readonly credentialKind: string | undefined;
577
- /** Buddy-family provider: the only family with provider-specific codes below. */
578
- readonly buddy: boolean;
579
- /**
580
- * The upstream named a restriction on this account, not a refusal of the
581
- * request. Only the `11140` shape carries `displayMsg` about the account
582
- * being restricted, so the policy arm can tell a ban from a bad request
583
- * instead of treating both as one cooldown.
584
- */
585
- readonly buddyBanned: boolean;
586
- readonly priceRefusal: boolean;
587
- readonly deploymentInfraError: boolean;
588
- readonly quotaSignal: boolean;
589
- readonly policyRejection: boolean;
590
- readonly requestScoped: boolean;
591
- readonly channelRejection: boolean;
592
- readonly hostedToolFailure: boolean;
593
- readonly modelEntitlementFailure: boolean;
594
- readonly queuedUpstream: boolean;
595
- }
596
-
597
- /** Build the request-independent evidence consumed by the ordered classifier. */
598
- function accountErrorContext(error: unknown, options?: AccountFailureEvidence): AccountErrorContext {
599
- const errorRecord = isRecord(error) ? error : undefined;
600
- const message =
601
- error instanceof Error
602
- ? error.message
603
- : typeof error === "string"
604
- ? error
605
- : typeof errorRecord?.message === "string"
606
- ? errorRecord.message
607
- : String(error ?? "Unknown error");
608
- const lower = message.toLowerCase();
609
- const statusCode =
610
- options?.statusCode ??
611
- (typeof errorRecord?.status === "number" ? errorRecord.status : null);
612
- const origin = options?.origin ?? "cartethyia";
613
- const scope = options?.scope ?? "unknown";
614
- // Providers write the same phrase with different separators: codex reports
615
- // `token_revoked`, others `token revoked` or `token-revoked`. Folding the
616
- // separators once lets every match below see one spelling.
617
- const folded = lower.replace(/[_-]+/g, " ");
618
- // A revocation named in the message is evidence about the credential itself,
619
- // whichever scope the caller could establish. A stream error frame carries
620
- // no `credentialEvidence` detail and classifies at provider scope, so without
621
- // this the account stays in rotation: the phrase is recognised and then
622
- // discarded because nothing mutates on provider scope.
623
- const revocationNamed = folded.includes("token revoked") || folded.includes("invalidated oauth token");
624
- const accountEvidence =
625
- options?.credentialEvidence === true ||
626
- revocationNamed ||
627
- (options?.origin === "upstream" && options.scope === "account");
628
- // A named revocation mutates even at provider scope: the message is the
629
- // evidence, and provider scope is what a frame with no credential detail
630
- // classifies as. Every other signal still needs account scope.
631
- const canMutate = origin === "upstream" && (revocationNamed || (scope === "account" && accountEvidence));
632
- const providerCode = options?.providerCode?.toLowerCase() ?? "";
633
-
634
- const getHeader = (name: string): string | null => {
635
- const headers = options?.headers;
636
- if (!headers) return null;
637
- if ("get" in headers && typeof headers.get === "function") return headers.get(name);
638
- const wanted = name.toLowerCase();
639
- const entry = Object.entries(headers).find(([key]) => key.toLowerCase() === wanted);
640
- return entry?.[1] ?? null;
641
- };
642
-
643
- const headerCooldown =
644
- options?.retryAfterMs ??
645
- parseUpstreamBackoff({ get: getHeader });
646
- const messageCooldown = parseProviderResetDuration(message);
647
- const rawUpstreamResponse =
648
- origin === "upstream" &&
649
- error instanceof GatewayError &&
650
- typeof error.details.raw === "string"
651
- ? error.details.raw
652
- : undefined;
653
- // The body's own fields, when they can be reached. Checked alongside the
654
- // header and the prose rather than instead of them: a provider may state the
655
- // window in any of the three, and the longest honest statement wins.
656
- //
657
- // `details.raw` is tried even when no caller passed a parsed body, because
658
- // that is where the upstream's JSON actually reaches this function — the
659
- // dispatch path stores the response text there. A structured window read from
660
- // it is the difference between a 15-minute fallback and the 23 days the
661
- // provider stated.
662
- const bodyCooldown =
663
- parseUpstreamBodyReset(options?.upstreamBody) ??
664
- (rawUpstreamResponse === undefined ? null : parseUpstreamBodyReset(tryParseUpstreamBody(rawUpstreamResponse)));
665
- // The body the ban is named in. The caller's parsed body when there is one,
666
- // and the raw response text otherwise — the dispatch path stores the
667
- // upstream's JSON there, and that is where `displayMsg` lives.
668
- const banBody =
669
- options?.upstreamBody ??
670
- (rawUpstreamResponse === undefined ? undefined : tryParseUpstreamBody(rawUpstreamResponse));
671
- // Whether the provider itself named a window. Decided once, from the
672
- // evidence, so every branch of this classification agrees about it.
673
- const providerStated =
674
- headerCooldown !== null || messageCooldown !== null || bodyCooldown !== null;
675
- const statedCooldownMs =
676
- [headerCooldown, messageCooldown, bodyCooldown].reduce<number | null>(
677
- (best, value) => (value === null ? best : best === null ? value : Math.max(best, value)),
678
- null,
679
- );
680
- const reason = (prefix: string): string => `${prefix}: ${message.slice(0, 160)}`;
681
- const result = (
682
- category: AccountErrorCategory,
683
- status: AccountErrorClassification["status"],
684
- cooldownMs: number,
685
- retryAt: Date | null,
686
- explanation: string,
687
- mutatesAccount = canMutate,
688
- modelUnsupported = false,
689
- ): AccountErrorClassification => ({
690
- category,
691
- status,
692
- cooldownMs,
693
- reason: origin === "upstream" ? rawUpstreamResponse ?? message.slice(0, 160) : explanation,
694
- retryAt,
695
- mutatesAccount,
696
- // Set below by the caller that knows which columns it wrote, not here:
697
- // the same classification feeds both the account-wide path (status +
698
- // cooldownUntil — routing-visible) and the model-scoped path (neither —
699
- // invisible). Defaulting true keeps existing callers conservative.
700
- routesVisible: true,
701
- providerStated,
702
- modelUnsupported,
703
- });
704
-
705
- return {
706
- message,
707
- lower,
708
- folded,
709
- statusCode,
710
- origin,
711
- scope,
712
- revocationNamed,
713
- accountEvidence,
714
- canMutate,
715
- providerCode,
716
- statedCooldownMs,
717
- buddyBanned: isBuddyAccountBanned(lower, providerCode, banBody),
718
- reason,
719
- result,
720
- };
721
- }
722
-
723
- export function classifyAccountError(
724
- error: unknown,
725
- options?: AccountFailureEvidence,
726
- ): AccountErrorClassification {
727
- const {
728
- message,
729
- lower,
730
- folded,
731
- statusCode,
732
- origin,
733
- scope,
734
- revocationNamed,
735
- accountEvidence,
736
- canMutate,
737
- providerCode,
738
-
739
- statedCooldownMs,
740
- buddyBanned,
741
- reason,
742
- result,
743
- } = accountErrorContext(error, options);
744
- const modelId = options?.modelId;
745
- const modelBlockHits = Math.max(1, Math.floor(options?.modelBlockHits ?? 1));
746
- const staticToken = options?.staticToken === true;
747
- const credentialKind = options?.credentialKind;
748
- const buddy = BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "");
749
-
750
- // A price refusal is a verdict on the *request*, not on the account. inferhub
751
- // answers 402 with `no provider's ask matches your max-per-mtok bid`: the
752
- // caller's own price ceiling was below every upstream ask, which says nothing
753
- // about the credential or its balance. Classified as quota it cooled a
754
- // fully-credited account down for an hour and the operator reported it as
755
- // "blocked, unusable at all". Detected before the quota branch and recorded
756
- // without mutating the account, so only that request fails.
757
- const priceRefusal =
758
- lower.includes("max-per-mtok") ||
759
- lower.includes("no provider's ask") ||
760
- (lower.includes("ask") && lower.includes("bid"));
761
- // Deployment/proxy infrastructure errors (DEPLOYMENT_DISABLED, etc.) are
762
- // transport-level proxy pool failures, not account quota exhaustion.
763
- // Mutating the account parks it for an hour while the pool itself stays
764
- // healthy — the next request picks the same dying pool and the same error
765
- // repeats. These must not cooldown the account.
766
- const deploymentInfraError =
767
- lower.includes("deployment_disabled") ||
768
- lower.includes("deployment disabled") ||
769
- (lower.includes("sin1::") && lower.includes("deployment"));
770
- if (deploymentInfraError) {
771
- return result("server_error", "cooldown", 60_000, null, `Proxy deployment error: ${message.slice(0, 160)}`, false);
772
- }
773
- // A (CodeBuddy/WorkBuddy) *frequency* window is a verdict on one
774
- // (account, model) pair, not on the account: `{"code":"6004","msg":"usage
775
- // exceeds frequency limit ... your usage will reset at <stamp> UTC+8,
776
- // alternatively, you can switch to the other models to continue using it."}`.
777
- // The provider names the remedy itself — use a different model — which is
778
- // only a remedy if the account still serves every other model. Taken as
779
- // account-wide `quota_exhausted` it parked the whole account until the reset,
780
- // so the switch the upstream offered was refused by our own gateway and every
781
- // sibling model went dark with it; the account also read COOLDOWN /
782
- // quota_exhausted while only that pair was throttled.
783
- //
784
- // Classified `rate_limit_transient` so the writer scopes it to the model:
785
- // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
786
- // account's own status alone. Kept ahead of `quotaSignal`, which matches a
787
- // *broader* slice of the same wording (`usage exceeds` + `frequency limit`)
788
- // and would otherwise claim this message as an account-wide quota window.
789
- // Narrow on purpose — the extra phrase is what says the other models still
790
- // work; without it the failure stays account-wide.
791
- const modelFrequencyWindow =
792
- lower.includes("frequency limit") && lower.includes("switch to the other models");
793
- if (modelFrequencyWindow) {
794
- const cooldownMs = statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
795
- return result(
796
- "rate_limit_transient",
797
- "cooldown",
798
- cooldownMs,
799
- new Date(Date.now() + cooldownMs),
800
- reason("Provider frequency limit for this model"),
801
- // Provider-scoped evidence still has to reach the writer: a stream frame
802
- // carries no credential detail, so `canMutate` is false and without this
803
- // the pair would be retried for the whole window it was closed for.
804
- origin === "upstream" && scope === "provider",
805
- );
806
- }
807
-
808
- // Quota-shaped provider codes/messages (e.g. xAI's
809
- const quotaSignal =
810
- !priceRefusal &&
811
- !deploymentInfraError &&
812
- (statusCode === 402 ||
813
- // Qoder answers an exhausted credit balance with
814
- // `{"code":"112","message":"{\"pricingUrl\":...}"}` — the body points at
815
- // the pricing page, i.e. "buy more", which is a balance verdict, not an
816
- // auth one. It fell through to `auth_invalidated` and the console showed
817
- // "Re-login required" on an account whose credential was fine, sending
818
- // operators to re-authenticate instead of topping up.
819
- // Recognising it here keeps it out of `auth_invalidated`; the branch
820
- // below scopes the resulting cooldown to the model, because the account
821
- // keeps serving its other models.
822
- providerCode === "112" ||
823
- providerCode === "insufficient_quota" ||
824
- providerCode === "quota_exceeded" ||
825
- providerCode === "subscription:free-usage-exhausted" ||
826
- providerCode === "14018" ||
827
- // CodeBuddy/WorkBuddy rate limit: `{"code":"6004","msg":"usage exceeds
828
- // frequency limit ... your usage will reset at <stamp> UTC+8"}`. The
829
- // message names a reset, so this is a parked window, not a transient
830
- // 429 — without it the account fell through to `unknown` with
831
- // `mutates=false`, stayed in rotation, and was re-picked for every
832
- // request inside the window the provider had actually named.
833
- //
834
- // Matched on the message's own wording, not on the `6004` code: that code
835
- // is not unique to this surface (the artifact-release API reuses it), so a
836
- // code match would sweep an unrelated failure into a quota cooldown. The
837
- // phrase is the reliable key, and `usage exceeds` + `frequency limit` is
838
- // the pair this family actually prints.
839
- (lower.includes("usage exceeds") && lower.includes("frequency limit")) ||
840
- lower.includes("insufficient_quota") ||
841
- lower.includes("quota_exceeded") ||
842
- lower.includes("usage_limit_reached") ||
843
- lower.includes("usage-exhausted") ||
844
- lower.includes("usage_exhausted") ||
845
- lower.includes("usage exhausted") ||
846
- lower.includes("usage exceeds") ||
847
- lower.includes("frequency limit") ||
848
- lower.includes("free-usage") ||
849
- lower.includes("free usage") ||
850
- lower.includes("exceeded your current quota") ||
851
- // Trae answers an exhausted quota with **HTTP 502** and the body `Your
852
- // requests have exceeded the quota.` The status is what misled the
853
- // classifier: 502 read as a gateway failure, so this fell through to
854
- // `server_error` — a 30s cooldown that does not mutate the account, so
855
- // the account stayed in rotation and every other model on it kept being
856
- // invoked inside a window the provider had already closed. The body is
857
- // the verdict; the 502 is incidental. Matched on the phrase, not on the
858
- // status, which is why this sits in `quotaSignal` ahead of the 5xx arm.
859
- lower.includes("have exceeded the quota") ||
860
- lower.includes("balance exhausted") ||
861
- lower.includes("insufficient balance") ||
862
- lower.includes("out of credits") ||
863
- lower.includes("credits exhausted") ||
864
- lower.includes("spending limit") ||
865
- // Same Qoder family reached by body text when the code is not parsed out.
866
- (lower.includes("pricingurl") && lower.includes("qoder.com/pricing")));
867
- // A deterministic content-policy rejection is NOT credential invalidation.
868
- // CodeBuddy returns HTTP 403 with provider code 11140 ("request illegal",
869
- // "did not pass the safety review") — refreshing the token cannot change
870
- // that outcome and the account itself is healthy, so it must never flip the
871
- // account to auth_invalidated/disabled. Only real auth signals do.
872
- const policyRejection =
873
- providerCode === "11140" ||
874
- lower.includes("safety review") ||
875
- lower.includes("content did not pass") ||
876
- lower.includes("request illegal") ||
877
- lower.includes("content blocked") ||
878
- lower.includes("account suspended");
879
- // A ban is a verdict on the credential, and cooldown cannot answer it: the
880
- // account is refused, not throttled. Disabling is the fix, and disabling
881
- // *immediately* is a feature — a client's retry budget is spent in seconds,
882
- // so a banned account that merely cooled stayed in the plan and burned
883
- // sibling accounts that were still healthy before the window expired.
884
- //
885
- // The two shapes are told apart by the body, not the code alone: `11140`
886
- // names a restriction on the account, `11-128` refuses the request.
887
- const policyBlock =
888
- policyRejection &&
889
- options?.providerId !== undefined &&
890
- BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
891
- if (policyBlock) {
892
- if (buddyBanned) {
893
- return result(
894
- "policy_blocked_banned",
895
- "disabled",
896
- 0,
897
- null,
898
- reason("Account banned by upstream"),
899
- // A ban is evidence about the credential even at provider scope, for
900
- // the same reason a named revocation is: the message is the evidence.
901
- // Without this a stream frame's missing credential detail would leave a
902
- // banned account serving.
903
- true,
904
- );
905
- }
906
- // Not a ban: the upstream refused this request. The account is fine, so the
907
- // only correct move is to stop retrying it verbatim. Cooling the account
908
- // here was the bug that made one malformed request look like a dead
909
- // credential.
910
- return result(
911
- "policy_blocked_invalid_request",
912
- "cooldown",
913
- 0,
914
- null,
915
- reason("Upstream refused this request"),
916
- // Never mutates: a malformed body fails identically on every account, so
917
- // cooling would burn a healthy credential on the gateway's own fault.
918
- false,
919
- );
920
- }
921
- // These codes are a verdict on the request, not the account. Cooling the
922
- // account for them burns a healthy credential on a body that will fail the
923
- // same way on every other account:
924
- //
925
- // - `11148` tool-call/result mismatch — the transcript is broken; the
926
- // pre-payload repair is supposed to prevent it, and a slip is still the
927
- // body's fault.
928
- // - `11135` the image cannot be processed, `11133` the model rejected a
929
- // parameter, `11101` the body failed to parse — deterministic for that
930
- // payload.
931
- // - `12153` the session is dead — a refresh problem, not a reason to park
932
- // the account on the first observation.
933
- // - `14017` trial not activated — the account may recover once registration
934
- // finishes, so a hard cooldown would keep it out after it becomes usable.
935
- //
936
- // Recognised before the auth and quota branches: several of these arrive as
937
- // 401/403/429, which those branches would otherwise treat as account evidence.
938
- const requestScoped =
939
- providerCode === "11148" ||
940
- providerCode === "11135" ||
941
- providerCode === "11133" ||
942
- providerCode === "11101" ||
943
- providerCode === "12153" ||
944
- providerCode === "14017" ||
945
- lower.includes("tool_call_sequence_broken") ||
946
- lower.includes("unmarshal chat params failed") ||
947
- lower.includes("invalid image_url content") ||
948
- lower.includes("offline user session not found") ||
949
- lower.includes("trial not activated") ||
950
- lower.includes("trial version is not yet activated");
951
- if (requestScoped && BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "")) {
952
- return result(
953
- "unknown",
954
- "cooldown",
955
- 0,
956
- null,
957
- reason("Request-scoped upstream rejection"),
958
- false,
959
- );
960
- }
961
- // A channel rejection (`11128`) is the same shape: the upstream is refusing
962
- // the caller for this account, not this particular prompt, and it keeps
963
- // refusing on every invocation. Classifying it as `unknown` with
964
- // `mutates=false` would leave the account in rotation failing every request —
965
- // and since every account fails the same way, the whole pool looks dead with
966
- // nothing ever parked.
967
- const channelRejection =
968
- providerCode === "11128" || lower.includes("unapproved channel");
969
- const channelBlock =
970
- channelRejection &&
971
- options?.providerId !== undefined &&
972
- BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
973
- if (channelBlock) {
974
- // Same split as the policy arm above: a ban disables, a channel refusal
975
- // only stops this request. `11-128` is the request shape, so this branch
976
- // nearly always takes the refusal path — but it shares the ban test so a
977
- // `11140`-shaped body arriving as a channel rejection still disables.
978
- if (buddyBanned) {
979
- return result(
980
- "policy_blocked_banned",
981
- "disabled",
982
- 0,
983
- null,
984
- reason("Account banned by upstream"),
985
- true,
986
- );
987
- }
988
- return result(
989
- "policy_blocked_invalid_request",
990
- "cooldown",
991
- 0,
992
- null,
993
- reason("Provider channel rejection"),
994
- // The credential is valid; the upstream refuses this calling channel.
995
- // Cooling was the bug: it kept a refused account in the plan, where a
996
- // client's retry budget burned healthy siblings for the whole window.
997
- false,
998
- );
999
- }
1000
- // server-side tool (web search, x search, web fetch), not evidence about the
1001
- // credential or the account's quota. Some providers surface it as 403, which
1002
- // would otherwise look like an auth signal — refreshing the token cannot fix
1003
- // a search backend, so it must never disable the account.
1004
- const hostedToolFailure =
1005
- lower.includes("web_search") ||
1006
- lower.includes("web search") ||
1007
- lower.includes("web_search_preview") ||
1008
- lower.includes("x_search") ||
1009
- lower.includes("web_fetch") ||
1010
- lower.includes("tool invocation") ||
1011
- lower.includes("tool call failed") ||
1012
- lower.includes("search backend");
1013
- // A model-scoped license or entitlement rejection is NOT credential invalidation.
1014
- // Google Cloud Code / Antigravity answers HTTP 403 with "You do not have a valid
1015
- // license of this product" when an account lacks entitlement for one specific model
1016
- // (e.g. claude-opus-4-6), while the account itself and its tokens remain fully valid
1017
- // for all other models. Disabling the whole account breaks healthy models.
1018
- const modelEntitlementFailure =
1019
- statusCode === 403 &&
1020
- (lower.includes("not have a valid license") ||
1021
- lower.includes("valid license of this product") ||
1022
- lower.includes("model not supported") ||
1023
- lower.includes("not entitled to") ||
1024
- lower.includes("no access to model") ||
1025
- lower.includes("model_not_supported") ||
1026
- lower.includes("not available in your region"));
1027
- /**
1028
- * The upstream accepted the request and put it in a queue behind other
1029
- * callers — it is busy, not rejecting the caller's credential.
1030
- *
1031
- * Qoder answers `403 {"code":"10605","message":"{\"isQueued\":true,
1032
- * \"modelKey\":\"dfmodel\",\"queueCount\":1395,...}"}`. The 403 is what made
1033
- * this misclassify: the generic `(401 || 403) && accountEvidence` arm claimed
1034
- * it as `auth_invalidated`, so a healthy account was parked as "Re-login
1035
- * required" purely because the provider was busy, and an operator was sent
1036
- * to re-authenticate a credential that was never rejected.
1037
- *
1038
- * Matched on the queue evidence rather than on the bare `10605`, so an
1039
- * unrelated 403 that happens to share the code is not swept in. The bodies
1040
- * are nested JSON, so the tell is the key, not a parse of the envelope.
1041
- */
1042
- const queuedUpstream =
1043
- providerCode === "10605" ||
1044
- lower.includes("isqueued") ||
1045
- lower.includes("queuecount") ||
1046
- lower.includes("queuetype") ||
1047
- (lower.includes("queue") && lower.includes("modelkey"));
1048
-
1049
- const signals: AccountErrorSignals = {
1050
- message,
1051
- lower,
1052
- folded,
1053
- statusCode,
1054
- origin,
1055
- scope,
1056
- providerCode,
1057
- statedCooldownMs,
1058
- reason,
1059
- result,
1060
- canMutate,
1061
- accountEvidence,
1062
- revocationNamed,
1063
- modelId,
1064
- modelBlockHits,
1065
- staticToken,
1066
- credentialKind,
1067
- buddy,
1068
- buddyBanned,
1069
- priceRefusal,
1070
- deploymentInfraError,
1071
- quotaSignal,
1072
- policyRejection,
1073
- requestScoped,
1074
- channelRejection,
1075
- hostedToolFailure,
1076
- modelEntitlementFailure,
1077
- queuedUpstream,
1078
- };
1079
-
1080
- for (const rule of ACCOUNT_ERROR_RULES) {
1081
- if (rule.match(signals)) return rule.decide(signals);
1082
- }
1083
-
1084
- // Same reasoning as above, with the shortest backoff: an unmapped failure
1085
- // must still be retried automatically rather than parked forever.
1086
- const unclassifiedMs = UNCLASSIFIED_COOLDOWN_MS();
1087
- return result(
1088
- "unknown",
1089
- "cooldown",
1090
- unclassifiedMs,
1091
- new Date(Date.now() + unclassifiedMs),
1092
- reason("Unclassified provider failure"),
1093
- false,
1094
- );
1095
- }
1096
-
1097
- /**
1098
- * The ordered arms of {@link classifyAccountError}, tried top to bottom; the
1099
- * first match decides. Order is behavior, not presentation — see
1100
- * {@link AccountErrorRule}.
1101
- */
1102
- const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
1103
- {
1104
- // Must precede "rate-limit": this rejection is a 429 whose own text says
1105
- // "rate limit", so the broader arm claimed it and cooled the account for
1106
- // the full 15-minute rate-limit window.
1107
- //
1108
- // A full max-inflight is a statement about the gateway's own routing, not
1109
- // about the credential: every account serving the model is busy right now.
1110
- // Cooling them marks healthy accounts as `cooldown` for a limit their
1111
- // provider never applied, which is what made every account read busy while
1112
- // incoming requests piled up — the accounts were never the problem, the
1113
- // ceiling was.
1114
- name: "gateway-capacity-exhausted",
1115
- match: (s) =>
1116
- s.lower.includes("at its concurrency limit") ||
1117
- s.lower.includes("every proxy pool for this provider is at its concurrency limit") ||
1118
- s.lower.includes("the gateway is at its concurrency limit"),
1119
- decide: (s) =>
1120
- s.result(
1121
- "model_capacity",
1122
- // `status` is the status the account would move to; it is never
1123
- // applied because `mutatesAccount` is false below, so this names the
1124
- // shape rather than a transition. Zero cooldown is the point: no wait
1125
- // is attached to the account, so the next request may use it at once.
1126
- "cooldown",
1127
- 0,
1128
- null,
1129
- s.reason("Gateway concurrency ceiling reached; no account was at fault"),
1130
- // Never mutates: there is nothing wrong with the account to record.
1131
- false,
1132
- ),
1133
- },
1134
- {
1135
- // Must precede the unclassified fallback: the fallback is account-blind and
1136
- // would stamp this as a provider-wide failure, which is how one proxy
1137
- // refusing CONNECT put every account of every provider behind that pool
1138
- // into cooldown — opencode's dead front door taking codebuddy down with it.
1139
- //
1140
- // A CONNECT refusal is a statement about the proxy pool, not about a
1141
- // credential: the tunnel was never established, so no upstream ever saw the
1142
- // request and no account can be at fault. Scoped to the pool so the
1143
- // provider's own accounts stay in rotation.
1144
- name: "proxy-connect-refusal",
1145
- match: (s) =>
1146
- s.lower.includes("proxy connect failed") ||
1147
- (s.lower.includes("connect") && s.lower.includes("tunnel") && s.lower.includes("refus")),
1148
- decide: (s) =>
1149
- s.result(
1150
- "model_unreachable",
1151
- "cooldown",
1152
- 0,
1153
- null,
1154
- s.reason("Proxy pool refused the CONNECT tunnel"),
1155
- // Never mutates: the pool owns this failure, not the account.
1156
- false,
1157
- ),
1158
- },
1159
- {
1160
- name: "deployment-infrastructure",
1161
- match: (s) => s.deploymentInfraError,
1162
- decide: (s) =>
1163
- s.result(
1164
- "server_error",
1165
- "cooldown",
1166
- 60_000,
1167
- null,
1168
- `Proxy deployment error: ${s.message.slice(0, 160)}`,
1169
- false,
1170
- ),
1171
- },
1172
- {
1173
- name: "model-frequency-window",
1174
- match: (s) => s.lower.includes("frequency limit") && s.lower.includes("switch to the other models"),
1175
- decide: (s) => {
1176
- const cooldownMs = s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
1177
- return s.result(
1178
- "rate_limit_transient",
1179
- "cooldown",
1180
- cooldownMs,
1181
- new Date(Date.now() + cooldownMs),
1182
- s.reason("Provider frequency limit for this model"),
1183
- // Provider-scoped evidence still has to reach the writer: a stream frame
1184
- // carries no credential detail, so `canMutate` is false and without this
1185
- // the pair would be retried for the whole window it was closed for.
1186
- s.origin === "upstream" && s.scope === "provider",
1187
- );
1188
- },
1189
- },
1190
- {
1191
- name: "content-policy-block",
1192
- match: (s) => s.policyRejection && s.buddy,
1193
- decide: (s) => {
1194
- // A ban is a verdict on the credential, and cooldown cannot answer it:
1195
- // the account is refused, not throttled. Disabling is the fix, and
1196
- // disabling *immediately* is a feature — a client's retry budget is spent
1197
- // in seconds, so a banned account that merely cooled stayed in the plan
1198
- // and burned sibling accounts that were still healthy before the window
1199
- // expired.
1200
- //
1201
- // The two shapes are told apart by the body, not the code alone: `11140`
1202
- // names a restriction on the account, `11-128` refuses the request.
1203
- if (s.buddyBanned) {
1204
- return s.result(
1205
- "policy_blocked_banned",
1206
- "disabled",
1207
- 0,
1208
- null,
1209
- s.reason("Account banned by upstream"),
1210
- // A ban is evidence about the credential even at provider scope, for
1211
- // the same reason a named revocation is: the message is the evidence.
1212
- // Without this a stream frame's missing credential detail would leave
1213
- // a banned account serving.
1214
- true,
1215
- );
1216
- }
1217
- // Not a ban: the upstream refused this request. The account is fine, so
1218
- // the only correct move is to stop retrying it verbatim. Cooling the
1219
- // account here was the bug that made one malformed request look like a
1220
- // dead credential.
1221
- return s.result(
1222
- "policy_blocked_invalid_request",
1223
- "cooldown",
1224
- 0,
1225
- null,
1226
- s.reason("Upstream refused this request"),
1227
- // Never mutates: a malformed body fails identically on every account,
1228
- // so cooling would burn a healthy credential on the gateway's own fault.
1229
- false,
1230
- );
1231
- },
1232
- },
1233
- {
1234
- name: "request-scoped-rejection",
1235
- match: (s) => s.requestScoped && s.buddy,
1236
- decide: (s) =>
1237
- s.result("unknown", "cooldown", 0, null, s.reason("Request-scoped upstream rejection"), false),
1238
- },
1239
- {
1240
- name: "channel-rejection",
1241
- match: (s) => s.channelRejection && s.buddy,
1242
- decide: (s) => {
1243
- const channelBlockMs = resolveAccountPolicyBlockCooldownMs();
1244
- return s.result(
1245
- "policy_blocked",
1246
- "cooldown",
1247
- channelBlockMs,
1248
- new Date(Date.now() + channelBlockMs),
1249
- s.reason("Provider channel rejection"),
1250
- s.origin === "upstream",
1251
- );
1252
- },
1253
- },
1254
- {
1255
- name: "credential-rejected",
1256
- match: (s) =>
1257
- !s.quotaSignal &&
1258
- !s.policyRejection &&
1259
- !s.hostedToolFailure &&
1260
- !s.modelEntitlementFailure &&
1261
- !s.queuedUpstream &&
1262
- (s.providerCode === "authentication_failed" ||
1263
- s.folded.includes("invalid api key") ||
1264
- s.folded.includes("incorrect api key") ||
1265
- s.folded.includes("invalid token") ||
1266
- s.folded.includes("token revoked") ||
1267
- s.folded.includes("invalidated oauth token") ||
1268
- s.folded.includes("reauthorization required") ||
1269
- s.folded.includes("account deactivated") ||
1270
- ((s.statusCode === 401 || s.statusCode === 403) && s.accountEvidence)),
1271
- decide: (s) => {
1272
- // A static token has no refresh to run, so an auth rejection must not park
1273
- // the account: disabling it throws away a token that may still work (a
1274
- // single upstream 401 can be transient). A cooldown lets the next attempt
1275
- // retry the same token without losing the account.
1276
- const oauthRecovery =
1277
- !s.staticToken &&
1278
- s.credentialKind === "oauth" &&
1279
- (s.lower.includes("expired") || s.lower.includes("refresh"));
1280
- // A *named* revocation is the one auth outcome that still parks the account.
1281
- // The upstream said the credential itself is dead — not that it expired, not
1282
- // that this request was refused — so there is no window to wait out and no
1283
- // refresh to run, and a cooldown would only return it to rotation to fail
1284
- // again. Every other auth rejection is weak evidence (clock skew, a key
1285
- // rotated a moment ago, an edge answering before the credential loads) and
1286
- // cools down instead.
1287
- //
1288
- // This distinction is the whole fix. Reaching `disabled` from the generic
1289
- // `401`/`403` branch would be unrecoverable by any sweep, so one transient
1290
- // rejection would remove the account from rotation until an operator
1291
- // noticed — the exact outcome the comment above says a static token must
1292
- // not suffer.
1293
- const revoked = s.revocationNamed;
1294
- const recoverable = s.staticToken || oauthRecovery || !revoked;
1295
- // An automatic disable is **not** terminal any more, but the deadline is what
1296
- // makes that safe: `sweepExpiredCooldowns` recovers a disabled row only when
1297
- // its `cooldownUntil` has elapsed, and an operator-disabled row keeps that
1298
- // column null. So the machine can undo its own verdict while an operator's
1299
- // decision still outranks it — see `resolveAccountDisabledRecoveryMs`.
1300
- const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
1301
- const disabledRetryAt =
1302
- disabledRecoveryMs > 0 ? new Date(Date.now() + disabledRecoveryMs) : null;
1303
- const cooldownMs = recoverable ? resolveAccountAuthRejectedCooldownMs() : disabledRecoveryMs;
1304
- return s.result(
1305
- "auth_invalidated",
1306
- recoverable ? "cooldown" : "disabled",
1307
- cooldownMs,
1308
- recoverable ? new Date(Date.now() + cooldownMs) : disabledRetryAt,
1309
- s.reason(
1310
- !revoked
1311
- ? "Provider credential rejected; retrying after a backoff"
1312
- : s.staticToken
1313
- ? "Static token rejected; retrying without a refresh"
1314
- : oauthRecovery
1315
- ? "OAuth credential requires refresh"
1316
- : "Provider credential revoked",
1317
- ),
1318
- s.canMutate,
1319
- );
1320
- },
1321
- },
1322
- {
1323
- name: "price-refusal",
1324
- match: (s) => s.priceRefusal,
1325
- decide: (s) =>
1326
- s.result("quota_exhausted", "cooldown", 0, null, s.reason("Request price refused by the provider"), false),
1327
- },
1328
- {
1329
- name: "model-plan-gated",
1330
- match: (s) => s.quotaSignal && s.providerCode === "112",
1331
- decide: (s) => {
1332
- const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1333
- return s.result(
1334
- "model_unreachable",
1335
- "cooldown",
1336
- cooldownMs,
1337
- new Date(Date.now() + cooldownMs),
1338
- s.reason("Model not included in this account's plan"),
1339
- s.origin === "upstream" && s.modelId !== undefined,
1340
- true,
1341
- );
1342
- },
1343
- },
1344
- {
1345
- name: "quota-exhausted",
1346
- match: (s) => s.quotaSignal,
1347
- decide: (s) => {
1348
- // xAI Grok Build's free tier resets on a rolling window. Use a duration
1349
- // stated by the provider when present; otherwise park for the conservative
1350
- // 24-hour fallback.
1351
- const grokFreeTier =
1352
- s.providerCode === "subscription:free-usage-exhausted" ||
1353
- s.providerCode.startsWith("subscription:") ||
1354
- s.lower.includes("free-usage") ||
1355
- s.lower.includes("free usage") ||
1356
- s.lower.includes("included free usage") ||
1357
- s.lower.includes("rolling 24-hour") ||
1358
- s.lower.includes("rolling 24 hour");
1359
- const buddyCreditsExhausted =
1360
- s.providerCode === "14018" || s.lower.includes("credits exhausted");
1361
- const cooldownMs = grokFreeTier
1362
- ? (s.statedCooldownMs ?? GROK_QUOTA_COOLDOWN_MS)
1363
- : buddyCreditsExhausted
1364
- ? (s.statedCooldownMs ?? 12 * 60 * 60 * 1000)
1365
- : (s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS());
1366
- // The caller records against the concrete account that just failed
1367
- // upstream, so a provider-scoped quota error still cools THAT account
1368
- // down (enables fallback to the next account). Local-origin quota must
1369
- // never mutate an upstream account.
1370
- return s.result(
1371
- "quota_exhausted",
1372
- "cooldown",
1373
- cooldownMs,
1374
- new Date(Date.now() + cooldownMs),
1375
- s.reason("Provider quota exhausted"),
1376
- s.origin === "upstream",
1377
- );
1378
- },
1379
- },
1380
- {
1381
- name: "rate-limit",
1382
- match: (s) =>
1383
- s.statusCode === 429 ||
1384
- s.providerCode === "rate_limit_exceeded" ||
1385
- s.lower.includes("rate_limit") ||
1386
- s.lower.includes("rate limit") ||
1387
- s.lower.includes("too many requests") ||
1388
- s.lower.includes("requests per minute") ||
1389
- s.lower.includes("tokens per minute") ||
1390
- s.lower.includes("tpm limit") ||
1391
- s.lower.includes("rpm limit"),
1392
- decide: (s) => {
1393
- const cooldownMs = s.statedCooldownMs ?? DEFAULT_RATE_LIMIT_COOLDOWN_MS();
1394
- // Same per-account reasoning as quota above: the failing account (or
1395
- // its model, when modelId is present) backs off so routing tries the
1396
- // next candidate instead of hammering the same key.
1397
- return s.result(
1398
- "rate_limit_transient",
1399
- "cooldown",
1400
- cooldownMs,
1401
- new Date(Date.now() + cooldownMs),
1402
- s.reason("Provider rate limit"),
1403
- s.origin === "upstream",
1404
- );
1405
- },
1406
- },
1407
- {
1408
- name: "model-capacity",
1409
- match: (s) =>
1410
- s.statusCode === 503 ||
1411
- s.statusCode === 529 ||
1412
- s.providerCode === "model_capacity" ||
1413
- s.queuedUpstream ||
1414
- s.lower.includes("model overloaded") ||
1415
- s.lower.includes("capacity exceeded") ||
1416
- s.lower.includes("temporarily unavailable"),
1417
- decide: (s) => {
1418
- const cooldownMs = s.statedCooldownMs ?? MODEL_CAPACITY_COOLDOWN_MS();
1419
- const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1420
- return s.result(
1421
- "model_capacity",
1422
- "cooldown",
1423
- cooldownMs,
1424
- new Date(Date.now() + cooldownMs),
1425
- s.reason("Provider model capacity"),
1426
- modelScopedMutation,
1427
- );
1428
- },
1429
- },
1430
- {
1431
- name: "model-unsupported",
1432
- match: (s) => isUnsupportedModelEvidence(s.statusCode, s.providerCode, s.folded),
1433
- decide: (s) => {
1434
- const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
1435
- return s.result(
1436
- "model_unreachable",
1437
- "cooldown",
1438
- cooldownMs,
1439
- new Date(Date.now() + cooldownMs),
1440
- s.reason("Model not supported by this upstream"),
1441
- s.origin === "upstream" && s.modelId !== undefined,
1442
- true,
1443
- );
1444
- },
1445
- },
1446
- {
1447
- name: "model-unreachable",
1448
- match: (s) => s.providerCode === "not_found" || s.modelEntitlementFailure,
1449
- decide: (s) => {
1450
- const cooldownMs = MODEL_UNREACHABLE_COOLDOWN_MS;
1451
- // Not credential evidence, so `classifyUpstreamFailure` scopes this to the
1452
- // provider and the `scope === "account"` gate the throttles use would
1453
- // forbid the mutation outright — leaving the probe reporting a failure that
1454
- // cooled nothing. The verdict is still about one (account, model) pair:
1455
- // `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
1456
- // account's own status untouched, so all that is required is a model to
1457
- // write against.
1458
- const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
1459
- return s.result(
1460
- "model_unreachable",
1461
- "cooldown",
1462
- cooldownMs,
1463
- new Date(Date.now() + cooldownMs),
1464
- s.reason("Model unreachable for this account"),
1465
- modelScopedMutation,
1466
- );
1467
- },
1468
- },
1469
- {
1470
- name: "transport-fault",
1471
- match: (s) =>
1472
- (s.statusCode !== null && s.statusCode >= 500 && s.statusCode <= 504) ||
1473
- s.lower.includes("timeout") ||
1474
- s.lower.includes("abort") ||
1475
- s.lower.includes("econnrefused") ||
1476
- s.lower.includes("fetch failed"),
1477
- decide: (s) => {
1478
- // A transport-shaped fault clears on its own, so it is a cooldown with a
1479
- // deadline rather than a state needing an operator. The deadline matters:
1480
- // `sweepExpiredCooldowns` selects on `cooldownUntil IS NOT NULL`, so a row
1481
- // parked without one is never recovered and stays out of rotation.
1482
- //
1483
- // `canMutate` rather than a flat `false`: a 5xx whose failure the caller
1484
- // scoped to the account (an edge/WAF challenge answered to one account's
1485
- // requests while its siblings on the same egress served fine) must cool
1486
- // that account down, or it stays `active` and is re-dialled for as long as
1487
- // the block lasts. Provider-scoped 5xx does not: the upstream is
1488
- // indiscriminately failing and no single account is at fault.
1489
- const transientMs = TRANSIENT_ERROR_COOLDOWN_MS();
1490
- return s.result(
1491
- s.lower.includes("timeout") ? "timeout" : "server_error",
1492
- "cooldown",
1493
- transientMs,
1494
- new Date(Date.now() + transientMs),
1495
- s.reason(`Provider ${s.statusCode ?? "network"} failure`),
1496
- s.canMutate,
1497
- );
1498
- },
1499
- },
1500
- ];
1501
-
1502
- /** Persists one classified failure to the account row and appends a health event. */
1503
- export async function recordAccountFailure(
1504
- db: CartethyiaDatabase,
1505
- accountId: string,
1506
- error: unknown,
1507
- options?: AccountFailureEvidence,
1508
- ): Promise<AccountErrorClassification | null> {
1509
- const classification = classifyAccountError(error, options);
1510
- /**
1511
- * Logged even when the classification does not mutate the account.
1512
- *
1513
- * `rate_limit_transient` seen on a stream frame carries no credential
1514
- * detail, so it classifies at provider scope and `mutatesAccount` is false
1515
- * — nothing is cooled and the account stays `active`. That part is correct:
1516
- * the upstream's own remedy ("switch to the other models") only works if the
1517
- * account keeps serving. But this returned without writing anything, so a
1518
- * rate-limited request showed up in the request log while the account's
1519
- * health log stayed empty — a symptom with no recorded cause.
1520
- *
1521
- * The row is the whole point of this branch: it records the failure without
1522
- * moving the account. Both status sides are the classification's own, so the
1523
- * dialog names the cause without claiming a transition that never happened.
1524
- */
1525
- if (!classification.mutatesAccount) {
1526
- void writeHealthEvent(db, {
1527
- entityKind: "account",
1528
- accountId,
1529
- fromStatus: classification.status,
1530
- toStatus: classification.status,
1531
- reason: classification.reason,
1532
- errorCategory: classification.category,
1533
- modelId: options?.modelId ?? null,
1534
- createdAt: new Date(),
1535
- }).catch(() => undefined);
1536
- return classification;
1537
- }
1538
-
1539
- const persist = async (client: CartethyiaDatabase): Promise<AccountErrorClassification | null> => {
1540
- // Projected, not a bare `select()`: this runs on *every* attempt outcome,
1541
- // and a full row drags `credential_ciphertext` (the largest thing stored
1542
- // per account) and `auth_state` over the wire to read a status and a
1543
- // counter. `RouteCatalogRepository` already projects for the same reason.
1544
- const accountRows = await client
1545
- .select({
1546
- status: providerAccounts.status,
1547
- staticToken: providerAccounts.staticToken,
1548
- consecutiveFailures: providerAccounts.consecutiveFailures,
1549
- modelCooldowns: providerAccounts.modelCooldowns,
1550
- // Read only for the unsupported-model ladder; NULL means "never
1551
- // counted", which is the common case and must stay cheap.
1552
- modelBlockHits: providerAccounts.modelBlockHits,
1553
- cooldownUntil: providerAccounts.cooldownUntil,
1554
- })
1555
- .from(providerAccounts)
1556
- .where(eq(providerAccounts.id, accountId))
1557
- .limit(1);
1558
- const account = accountRows[0];
1559
- if (!account || account.status === "disabled") return null;
1560
-
1561
- // Re-classify with the account's own static-token flag. The caller's
1562
- // evidence is request-scoped and does not carry it, but an auth rejection
1563
- // against a static token must cool the account down rather than park it —
1564
- // there is no refresh to run, and the token may still be valid.
1565
- const effective = account.staticToken
1566
- ? classifyAccountError(error, { ...options, staticToken: true })
1567
- : classification;
1568
-
1569
- const fromStatus = account.status;
1570
- const now = new Date();
1571
-
1572
- const modelId = options?.modelId;
1573
- // Every category here is a verdict about one (account, model) pair rather
1574
- // than about the credential, so all three back off that pair and leave the
1575
- // account routable for every other model. `model_unreachable` joins the two
1576
- // throttles on the same reasoning: an upstream that reports the model does
1577
- // not exist for this account will report it again on the next request.
1578
- const isThrottle =
1579
- effective.category === "rate_limit_transient" ||
1580
- effective.category === "model_capacity" ||
1581
- effective.category === "model_unreachable";
1582
- const shouldModelCooldown = modelId !== undefined && isThrottle;
1583
- // The account-wide streak, escalated for a provider-stated window. Read once
1584
- // so the model-scoped and account-wide paths below cannot disagree about it:
1585
- // this is the count *before* this failure, which the UPDATE below increments.
1586
- const streak = Math.max(0, account.consecutiveFailures ?? 0);
1587
- const escalatedCooldownMs = escalateFallbackCooldown(
1588
- effective.cooldownMs,
1589
- effective.providerStated === true,
1590
- streak,
1591
- );
1592
-
1593
- // The unsupported-model ladder, computed before the branch: the classifier
1594
- // could only state the base window because the authoritative hit count
1595
- // lives on the row, and it is read here. `hits` is the new total for this
1596
- // pair, so the first report is 1 and gets the base window.
1597
- const blockHits = (account.modelBlockHits as Record<string, number> | null) ?? {};
1598
- const unsupportedHits =
1599
- effective.modelUnsupported === true && modelId !== undefined
1600
- ? Math.max(1, (blockHits[modelId] ?? 0) + 1)
1601
- : undefined;
1602
- // An unsupported-model window is stated by *us*, not by the provider, so it
1603
- // would otherwise be multiplied a second time by `escalateFallbackCooldown`
1604
- // (which reads `consecutiveFailures`, an account-wide counter). The two
1605
- // ladders are different things: one counts how many times this pair was
1606
- // reported unsupported, the other how many failures this account produced.
1607
- const unsupportedCooldownMs =
1608
- unsupportedHits === undefined ? undefined : unsupportedModelCooldownMs(unsupportedHits);
1609
-
1610
- if (shouldModelCooldown) {
1611
- const existing = (account.modelCooldowns as Record<string, string> | null) ?? {};
1612
- // Monotonic: never move an in-force window earlier (see `latestDeadlineMs`).
1613
- // A provider-stated deadline wins outright because it is already the later
1614
- // of the two whenever the provider really parked the pair longer.
1615
- const nextUntil = new Date(
1616
- latestDeadlineMs(
1617
- effective.retryAt?.getTime(),
1618
- Date.now() + (unsupportedCooldownMs ?? escalatedCooldownMs),
1619
- existing[modelId] ? Date.parse(existing[modelId]!) : undefined,
1620
- ),
1621
- ).toISOString();
1622
- const previousUntil = existing[modelId];
1623
- // A re-statement is not an event: what the audit row reports is the
1624
- // *entry* into a cooling episode, and the deadline itself is not a
1625
- // column — so a row can only ever say "this pair started cooling for
1626
- // this reason". The upstream answers the same "Try again in 11h" on
1627
- // every retry (and an unstated reset is recomputed from now each time),
1628
- // so writing per failure produced seven identical rows in twenty minutes
1629
- // and read as an account flapping when nothing had changed. The entry is
1630
- // still refreshed below — its error fields are what the dialog explains
1631
- // the cooldown with — while an entry that is still in force suppresses
1632
- // the duplicate row.
1633
- const alreadyCooling =
1634
- previousUntil !== undefined && new Date(previousUntil).getTime() > Date.now();
1635
- // `status`/`cooldownUntil` stay untouched: only this (account, model)
1636
- // pair backs off, and the account remains routable for every other model.
1637
- // The error fields ARE written, because they are what provider detail and
1638
- // the health dialog read to explain *why* a model is cooling — without
1639
- // them the row showed "1 model cooling" with no reason at all.
1640
- //
1641
- // The streak advances here too. Without it the most common throttle — a
1642
- // model-scoped rate limit — never escalated, because the counter was only
1643
- // incremented on the account-wide path below, and `consecutiveFailures`
1644
- // stays honest about what it always claimed to count: how many failures in
1645
- // a row this account has just produced.
1646
- await client
1647
- .update(providerAccounts)
1648
- .set({
1649
- modelCooldowns: { ...existing, [modelId]: nextUntil },
1650
- // Counted only for the unsupported-model ladder, and only when this
1651
- // failure is that verdict — so a rate limit or a capacity refusal
1652
- // never advances it. `model_cooldowns`' own shape is untouched, so
1653
- // routing and the console keep parsing it exactly as before.
1654
- ...(unsupportedHits === undefined
1655
- ? {}
1656
- : { modelBlockHits: { ...blockHits, [modelId]: unsupportedHits } }),
1657
- consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
1658
- lastError: effective.reason,
1659
- lastErrorCategory: effective.category,
1660
- lastErrorAt: now,
1661
- })
1662
- .where(eq(providerAccounts.id, accountId));
1663
- if (!alreadyCooling) {
1664
- await writeHealthEvent(client, {
1665
- entityKind: "account",
1666
- accountId,
1667
- // The account's own status, on both sides: a model-scoped throttle
1668
- // never moved it, and writing the classification's status here
1669
- // claimed an `active → cooldown` transition that never happened — the
1670
- // dialog showed "Account status: ACTIVE" above a list of transitions
1671
- // to `cooldown`. The row still names the model and the reason, which
1672
- // is what makes it worth keeping.
1673
- fromStatus: fromStatus as "active" | "cooldown" | "disabled",
1674
- toStatus: fromStatus as "active" | "cooldown" | "disabled",
1675
- reason: effective.reason,
1676
- errorCategory: effective.category,
1677
- modelId: modelId ?? null,
1678
- createdAt: now,
1679
- });
1680
- }
1681
- // `status` and `cooldownUntil` are untouched on this path — only the
1682
- // (account, model) pair backed off — so the snapshot is still accurate
1683
- // and must not be dropped.
1684
- return { ...classification, routesVisible: false };
1685
- }
1686
-
1687
- await client
1688
- .update(providerAccounts)
1689
- .set({
1690
- status: effective.status,
1691
- consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
1692
- lastError: effective.reason,
1693
- lastErrorCategory: effective.category,
1694
- lastErrorAt: now,
1695
- // Monotonic against the account's own in-force deadline, for the same
1696
- // reason as the model path: an unclassified 60-second failure must not
1697
- // release an account the provider parked for hours.
1698
- //
1699
- // A `0` duration contributes no candidate rather than stamping *now*
1700
- // (see `accountDeadlineMs`): the one state that asks for it is a
1701
- // terminal automatic disable, and an elapsed deadline there would hand
1702
- // the row straight back to the recovery sweep.
1703
- cooldownUntil: accountDeadlineMs([
1704
- effective.retryAt?.getTime(),
1705
- escalatedCooldownMs > 0 ? Date.now() + escalatedCooldownMs : undefined,
1706
- account.cooldownUntil?.getTime?.(),
1707
- ]),
1708
- })
1709
- .where(eq(providerAccounts.id, accountId));
1710
-
1711
- {
1712
- await writeHealthEvent(client, {
1713
- entityKind: "account",
1714
- accountId,
1715
- fromStatus: fromStatus as "active" | "cooldown" | "disabled",
1716
- toStatus: effective.status,
1717
- reason: effective.reason,
1718
- errorCategory: effective.category,
1719
- modelId: modelId ?? null,
1720
- createdAt: now,
1721
- });
1722
- }
1723
- // This path wrote `status` and `cooldownUntil`, both of which the snapshot
1724
- // projects, so routing has to see it.
1725
- return { ...classification, routesVisible: true };
1726
- };
1727
-
1728
- try {
1729
- // Kept as a capability probe, not a correctness guard: this function is
1730
- // reachable with a partially-implemented database handle (the dispatch
1731
- // tests pass a narrow double), and a transaction is an optimization here,
1732
- // not a requirement — the same writes are correct without one.
1733
- if (typeof db.transaction === "function") {
1734
- return await db.transaction((tx) => persist(tx as CartethyiaDatabase));
1735
- }
1736
- return await persist(db);
1737
- } catch {
1738
- return null;
1739
- }
1740
- }
1741
-
1742
- /** Clears failure state after a successful dispatch; true when the account changed. */
1743
- export async function recordAccountSuccess(
1744
- db: CartethyiaDatabase,
1745
- accountId: string,
1746
- ): Promise<boolean> {
1747
- const mutate = async (client: CartethyiaDatabase): Promise<boolean> => {
1748
- // `recordAccountSuccess` runs on every successful dispatch, and for a
1749
- // healthy account `hadFailures` is false — the row was fetched purely to
1750
- // decide nothing. Projected so that no-op path does not pull the
1751
- // credential ciphertext across the wire.
1752
- const rows = await client
1753
- .select({
1754
- status: providerAccounts.status,
1755
- consecutiveFailures: providerAccounts.consecutiveFailures,
1756
- // Read only to decide whether there is anything to clear: the common
1757
- // case (no unsupported-model hit ever recorded) must not write.
1758
- modelBlockHits: providerAccounts.modelBlockHits,
1759
- })
1760
- .from(providerAccounts)
1761
- .where(eq(providerAccounts.id, accountId))
1762
- .limit(1);
1763
- const account = rows[0];
1764
- if (!account || account.status === "disabled") return false;
1765
- const fromStatus = account.status;
1766
- const hadFailures = (account.consecutiveFailures ?? 0) > 0 || account.status !== "active";
1767
- const now = new Date();
1768
- // The unsupported-model escalation restarts after a success: a model that
1769
- // just served is not "not supported", so the next report of it must start
1770
- // from the base window again rather than from wherever the ladder had
1771
- // reached. `modelCooldowns` is deliberately NOT cleared — an in-force
1772
- // window the provider stated survives, which is the existing behavior and
1773
- // the reference gateways' too.
1774
- const blockHits = account.modelBlockHits as Record<string, number> | null;
1775
- const hasBlockHits = blockHits !== null && Object.keys(blockHits).length > 0;
1776
- await client
1777
- .update(providerAccounts)
1778
- .set({
1779
- status: "active",
1780
- consecutiveFailures: 0,
1781
- cooldownUntil: null,
1782
- lastSuccessAt: now,
1783
- ...(hadFailures ? { lastRecoveredAt: now } : {}),
1784
- ...(hasBlockHits ? { modelBlockHits: {} } : {}),
1785
- })
1786
- .where(eq(providerAccounts.id, accountId));
1787
- if (hadFailures && typeof client.insert === "function") {
1788
- await client.insert(healthEvents).values({
1789
- entityKind: "account",
1790
- accountId,
1791
- fromStatus: fromStatus as "active" | "cooldown" | "disabled",
1792
- toStatus: "active",
1793
- reason: "Request dispatched successfully",
1794
- errorCategory: null,
1795
- createdAt: now,
1796
- });
1797
- }
1798
- return hadFailures;
1799
- };
1800
- try {
1801
- if (typeof db.transaction === "function") {
1802
- return await db.transaction((tx) => mutate(tx as CartethyiaDatabase));
1803
- }
1804
- return await mutate(db);
1805
- } catch {
1806
- return false;
1807
- }
1808
- }
1809
-
1810
- /** Operator-initiated recovery: forces the account back to active and logs the transition. */
1811
- export async function recoverAccount(
1812
- db: CartethyiaDatabase,
1813
- accountId: string,
1814
- reason = "manual_operator_recovery",
1815
- ): Promise<boolean> {
1816
- const recover = async (client: CartethyiaDatabase): Promise<boolean> => {
1817
- const accountRows = await client
1818
- .select()
1819
- .from(providerAccounts)
1820
- .where(eq(providerAccounts.id, accountId))
1821
- .limit(1);
1822
- const account = accountRows[0];
1823
- if (!account) return false;
1824
- const fromStatus = account.status;
1825
- const now = new Date();
1826
- await client
1827
- .update(providerAccounts)
1828
- .set({
1829
- status: "active",
1830
- consecutiveFailures: 0,
1831
- cooldownUntil: null,
1832
- lastRecoveredAt: now,
1833
- // Recovery is the operator asserting the account works now, so it must
1834
- // clear EVERY routing exclusion, not just the account-wide ones. A
1835
- // per-model cooldown entry excludes the (account, model) pair in
1836
- // `route-catalog.ts`, so leaving it behind kept the account blocked for
1837
- // that model through `/v1` even though the account read `active` and a
1838
- // direct probe (which addresses the account by id, bypassing routing
1839
- // eligibility) succeeded. Clearing the error fields for the same reason:
1840
- // a stale reason on a recovered row misreports the current state.
1841
- modelCooldowns: {},
1842
- lastError: null,
1843
- lastErrorCategory: null,
1844
- lastErrorAt: null,
1845
- })
1846
- .where(eq(providerAccounts.id, accountId));
1847
- {
1848
- await writeHealthEvent(client, {
1849
- entityKind: "account",
1850
- accountId,
1851
- fromStatus: fromStatus as "active" | "cooldown" | "disabled",
1852
- toStatus: "active",
1853
- reason,
1854
- errorCategory: null,
1855
- createdAt: now,
1856
- });
1857
- }
1858
- return true;
1859
- };
1860
- try {
1861
- if (typeof db.transaction === "function") {
1862
- return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
1863
- }
1864
- return await recover(db);
1865
- } catch {
1866
- return false;
1867
- }
1868
- }
1869
-
1870
- /** Recovers accounts whose cooldown elapsed and prunes expired per-model keys. */
1871
- export async function sweepExpiredCooldowns(db: CartethyiaDatabase): Promise<number> {
1872
- const recover = async (client: CartethyiaDatabase): Promise<number> => {
1873
- const now = new Date();
1874
- const nowIso = now.toISOString();
1875
- const expired = await client
1876
- .select({
1877
- id: providerAccounts.id,
1878
- status: providerAccounts.status,
1879
- })
1880
- .from(providerAccounts)
1881
- .where(
1882
- or(
1883
- // An ordinary cooldown, recovered as soon as its window elapses.
1884
- and(
1885
- eq(providerAccounts.status, "cooldown"),
1886
- isNotNull(providerAccounts.cooldownUntil),
1887
- lte(providerAccounts.cooldownUntil, now),
1888
- ),
1889
- // An **automatic** disable. Selected only when it carries a deadline:
1890
- // an operator's disable leaves
1891
- // `cooldownUntil` null, so a human's decision still outranks the
1892
- // machine and is never swept away. The deadline is written by the two
1893
- // paths that disable on evidence — the classifier's revocation branch
1894
- // and a definitively rejected refresh grant.
1895
- and(
1896
- eq(providerAccounts.status, "disabled"),
1897
- isNotNull(providerAccounts.cooldownUntil),
1898
- lte(providerAccounts.cooldownUntil, now),
1899
- ),
1900
- ),
1901
- )
1902
- // One pass recovers a bounded batch. The sweep runs every 30s, so a
1903
- // backlog larger than this drains over successive ticks instead of one
1904
- // statement covering every expired row — that statement holds its locks
1905
- // for as long as the backlog takes.
1906
- .limit(COOLDOWN_RECOVERY_BATCH);
1907
- if (expired.length > 0) {
1908
- await client
1909
- .update(providerAccounts)
1910
- .set({
1911
- status: "active",
1912
- consecutiveFailures: 0,
1913
- cooldownUntil: null,
1914
- lastRecoveredAt: now,
1915
- // Cleared with the state it described. A recovered row that kept
1916
- // `auth_invalidated` still rendered the console's "Re-login required"
1917
- // pill above an `active` status, so the account read healthy to
1918
- // routing and broken to the operator at the same time.
1919
- lastErrorCategory: null,
1920
- })
1921
- .where(inArray(providerAccounts.id, expired.map((account) => account.id)));
1922
- if (typeof client.insert === "function") {
1923
- for (let offset = 0; offset < expired.length; offset += 500) {
1924
- await client.insert(healthEvents).values(
1925
- expired.slice(offset, offset + 500).map((account) => ({
1926
- entityKind: "account" as const,
1927
- accountId: account.id,
1928
- fromStatus: account.status,
1929
- toStatus: "active" as const,
1930
- reason: "Cooldown period elapsed — auto-recovered",
1931
- errorCategory: null,
1932
- createdAt: now,
1933
- })),
1934
- );
1935
- }
1936
- }
1937
- }
1938
-
1939
- // Prune per-model cooldowns: remove entries whose ISO timestamp has passed.
1940
- //
1941
- // One set-based UPDATE rather than a SELECT followed by an UPDATE per row.
1942
- // The per-row form issued one statement per account holding any cooldown —
1943
- // on a deployment where many accounts are throttled at once that is the
1944
- // sweep's dominant cost, and it all runs inside this transaction, holding
1945
- // its locks for the whole pass. Recomputing the object in SQL does the same
1946
- // work in one statement.
1947
- //
1948
- // `jsonb_typeof(...) = 'object'` guards the shape: a malformed value must
1949
- // not abort the whole sweep. `IS DISTINCT FROM` keeps the update from
1950
- // rewriting rows it
1951
- // did not change, so the returned row count is the number of accounts whose
1952
- // cooldown set actually shrank — the same thing the loop counted.
1953
- const pruning = pruneExpiredCooldowns(sql`${providerAccounts.modelCooldowns}`, sql`${nowIso}`);
1954
- const pruned = await client
1955
- .update(providerAccounts)
1956
- .set({ modelCooldowns: pruning.value })
1957
- .where(pruning.changed);
1958
- return expired.length + affectedRows(pruned);
1959
- };
1960
- try {
1961
- if (typeof db.transaction === "function") {
1962
- return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
1963
- }
1964
- return await recover(db);
1965
- } catch {
1966
- return 0;
1967
- }
1968
- }
1969
-
1970
- /**
1971
- * Single attempt health reporter: one home for the success/failure writes
1972
- * every dispatch attempt ends with. Success records recovery; failure
1973
- * records with caller-supplied evidence (origin/scope classification plus
1974
- * model id). A mutating write invalidates the route snapshot so the next
1975
- * plan sees fresh health. Never throws — health must not fail requests.
1976
- */
1977
- export interface AttemptHealthReport {
1978
- readonly accountId: string | undefined;
1979
- readonly modelId?: string;
1980
- readonly error?: unknown;
1981
- readonly evidence?: AccountFailureEvidence;
1982
- readonly snapshotService?: { invalidate(): unknown };
1983
- }
1984
-
1985
- /**
1986
- * Writes one health audit row, off the request path when a buffer is installed.
1987
- *
1988
- * `health_events` is append-only and read by nobody in the request path, so it
1989
- * does not need to land before the request finishes — unlike the
1990
- * `provider_accounts` UPDATE beside it, which routing reads and which therefore
1991
- * stays synchronous and transactional.
1992
- */
1993
- async function writeHealthEvent(
1994
- client: CartethyiaDatabase,
1995
- row: {
1996
- readonly entityKind: "account";
1997
- readonly accountId: string;
1998
- readonly fromStatus: string;
1999
- readonly toStatus: string;
2000
- readonly reason: string;
2001
- readonly errorCategory: string | null;
2002
- readonly modelId?: string | null;
2003
- readonly createdAt: Date;
2004
- },
2005
- ): Promise<void> {
2006
- const buffer = healthEventBuffer();
2007
- if (buffer !== undefined) {
2008
- buffer.enqueue({ ...row, modelId: row.modelId ?? null });
2009
- return;
2010
- }
2011
- if (typeof client.insert !== "function") return;
2012
- await client.insert(healthEvents).values(row as never);
2013
- }
2014
-
2015
- export async function reportAttemptOutcome(
2016
- db: CartethyiaDatabase,
2017
- report: AttemptHealthReport,
2018
- ): Promise<void> {
2019
- if (!report.accountId) return;
2020
- if (report.error === undefined) {
2021
- const recovered = await recordAccountSuccess(db, report.accountId).catch(() => false);
2022
- if (recovered) await report.snapshotService?.invalidate();
2023
- return;
2024
- }
2025
- const classification = await recordAccountFailure(db, report.accountId, report.error, {
2026
- ...report.evidence,
2027
- ...(report.modelId === undefined ? {} : { modelId: report.modelId }),
2028
- }).catch(() => null);
2029
- // Gated on `routesVisible`, not on `mutatesAccount`: a failure that only
2030
- // cooled one (account, model) pair, or only advanced the streak, changed
2031
- // nothing routing reads. Invalidating for those rebuilt an 11-SELECT snapshot
2032
- // on the next request, and under a failure wave each rebuild fed the next
2033
- // invalidation.
2034
- if (classification?.routesVisible) await report.snapshotService?.invalidate();
2035
- }
2036
-
2037
- export async function listAccountHealthEvents(
2038
- db: CartethyiaDatabase,
2039
- accountId: string,
2040
- limit = 50,
2041
- ): Promise<readonly AccountHealthEventRecord[]> {
2042
- try {
2043
- const rows = await db
2044
- .select()
2045
- .from(healthEvents)
2046
- .where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
2047
- .orderBy(desc(healthEvents.createdAt))
2048
- .limit(limit);
2049
-
2050
- return rows.map((r) => ({
2051
- id: r.id,
2052
- accountId: r.accountId as string,
2053
- fromStatus: r.fromStatus,
2054
- toStatus: r.toStatus,
2055
- modelId: r.modelId ?? null,
2056
- reason: r.reason,
2057
- errorCategory: r.errorCategory,
2058
- createdAt: r.createdAt.toISOString(),
2059
- }));
2060
- } catch {
2061
- return [];
2062
- }
2063
- }
2064
-
2065
- /** Every stored row across accounts and pools — the size the purge offers to delete. */
2066
- export async function countAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
2067
- try {
2068
- const rows = await db.select({ total: count() }).from(healthEvents);
2069
- return rows[0]?.total ?? 0;
2070
- } catch {
2071
- return 0;
2072
- }
2073
- }
2074
-
2075
- /**
2076
- * Deletes stored health & error rows.
2077
- *
2078
- * Unlike the read above, a failed delete surfaces: an operator who purged
2079
- * believes the trail is gone, so "it failed" must not read as "it succeeded
2080
- * with zero rows". Callers own the flush-buffer barrier (`drain`) so a queued
2081
- * write cannot land after these rows are gone.
2082
- */
2083
- export async function deleteAccountHealthEvents(
2084
- db: CartethyiaDatabase,
2085
- accountId: string,
2086
- ): Promise<number> {
2087
- const rows = await db
2088
- .delete(healthEvents)
2089
- .where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
2090
- .returning({ id: healthEvents.id });
2091
- return rows.length;
2092
- }
2093
-
2094
- export async function deleteAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
2095
- const rows = await db.delete(healthEvents).returning({ id: healthEvents.id });
2096
- return rows.length;
2097
- }
1
+ import { randomUUID } from "node:crypto";
2
+ import { healthEventBuffer } from "../../persistence/health-event-flush";
3
+ import { and, count, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
4
+ import { affectedRows, pruneExpiredCooldowns } from "../../persistence/dialect";
5
+ import type { CartethyiaDatabase } from "../../persistence/db-handle";
6
+ import { healthEvents, providerAccounts } from "../../persistence/schema";
7
+ import {
8
+ COOLDOWN_RECOVERY_BATCH,
9
+ accountDeadlineMs,
10
+ classifyAccountError,
11
+ escalateFallbackCooldown,
12
+ latestDeadlineMs,
13
+ unsupportedModelCooldownMs,
14
+ } from "./account-error-classification";
15
+ import type { AccountErrorClassification, AccountFailureEvidence } from "./account-error-classification";
16
+
17
+ export { classifyAccountError, unsupportedModelCooldownMs } from "./account-error-classification";
18
+ export type {
19
+ AccountErrorCategory,
20
+ AccountErrorClassification,
21
+ AccountFailureEvidence,
22
+ AccountFailureOrigin,
23
+ AccountFailureScope,
24
+ } from "./account-error-classification";
25
+
26
+ export interface AccountHealthEventRecord {
27
+ readonly id: string;
28
+ readonly accountId: string;
29
+ readonly fromStatus: string | null;
30
+ readonly toStatus: string;
31
+ readonly reason: string | null;
32
+ readonly errorCategory: string | null;
33
+ readonly modelId: string | null;
34
+ readonly createdAt: string;
35
+ }
36
+
37
+ /** Persists one classified failure to the account row and appends a health event. */
38
+ export async function recordAccountFailure(
39
+ db: CartethyiaDatabase,
40
+ accountId: string,
41
+ error: unknown,
42
+ options?: AccountFailureEvidence,
43
+ ): Promise<AccountErrorClassification | null> {
44
+ const classification = classifyAccountError(error, options);
45
+ /**
46
+ * Logged even when the classification does not mutate the account.
47
+ *
48
+ * `rate_limit_transient` seen on a stream frame carries no credential
49
+ * detail, so it classifies at provider scope and `mutatesAccount` is false
50
+ * — nothing is cooled and the account stays `active`. That part is correct:
51
+ * the upstream's own remedy ("switch to the other models") only works if the
52
+ * account keeps serving. But this returned without writing anything, so a
53
+ * rate-limited request showed up in the request log while the account's
54
+ * health log stayed empty — a symptom with no recorded cause.
55
+ *
56
+ * The row is the whole point of this branch: it records the failure without
57
+ * moving the account. Both status sides are the classification's own, so the
58
+ * dialog names the cause without claiming a transition that never happened.
59
+ */
60
+ if (!classification.mutatesAccount) {
61
+ void writeHealthEvent(db, {
62
+ entityKind: "account",
63
+ accountId,
64
+ fromStatus: classification.status,
65
+ toStatus: classification.status,
66
+ reason: classification.reason,
67
+ errorCategory: classification.category,
68
+ modelId: options?.modelId ?? null,
69
+ createdAt: new Date(),
70
+ }).catch(() => undefined);
71
+ return classification;
72
+ }
73
+
74
+ const persist = async (client: CartethyiaDatabase): Promise<AccountErrorClassification | null> => {
75
+ // Projected, not a bare `select()`: this runs on *every* attempt outcome,
76
+ // and a full row drags `credential_ciphertext` (the largest thing stored
77
+ // per account) and `auth_state` over the wire to read a status and a
78
+ // counter. `RouteCatalogRepository` already projects for the same reason.
79
+ const accountRows = await client
80
+ .select({
81
+ status: providerAccounts.status,
82
+ staticToken: providerAccounts.staticToken,
83
+ consecutiveFailures: providerAccounts.consecutiveFailures,
84
+ modelCooldowns: providerAccounts.modelCooldowns,
85
+ // Read only for the unsupported-model ladder; NULL means "never
86
+ // counted", which is the common case and must stay cheap.
87
+ modelBlockHits: providerAccounts.modelBlockHits,
88
+ cooldownUntil: providerAccounts.cooldownUntil,
89
+ })
90
+ .from(providerAccounts)
91
+ .where(eq(providerAccounts.id, accountId))
92
+ .limit(1);
93
+ const account = accountRows[0];
94
+ if (!account || account.status === "disabled") return null;
95
+
96
+ // Re-classify with the account's own static-token flag. The caller's
97
+ // evidence is request-scoped and does not carry it, but an auth rejection
98
+ // against a static token must cool the account down rather than park it —
99
+ // there is no refresh to run, and the token may still be valid.
100
+ const effective = account.staticToken
101
+ ? classifyAccountError(error, { ...options, staticToken: true })
102
+ : classification;
103
+
104
+ const fromStatus = account.status;
105
+ const now = new Date();
106
+
107
+ const modelId = options?.modelId;
108
+ // Every category here is a verdict about one (account, model) pair rather
109
+ // than about the credential, so all three back off that pair and leave the
110
+ // account routable for every other model. `model_unreachable` joins the two
111
+ // throttles on the same reasoning: an upstream that reports the model does
112
+ // not exist for this account will report it again on the next request.
113
+ const isThrottle =
114
+ effective.category === "rate_limit_transient" ||
115
+ effective.category === "model_capacity" ||
116
+ effective.category === "model_unreachable";
117
+ const shouldModelCooldown = modelId !== undefined && isThrottle;
118
+ // The account-wide streak, escalated for a provider-stated window. Read once
119
+ // so the model-scoped and account-wide paths below cannot disagree about it:
120
+ // this is the count *before* this failure, which the UPDATE below increments.
121
+ const streak = Math.max(0, account.consecutiveFailures ?? 0);
122
+ const escalatedCooldownMs = escalateFallbackCooldown(
123
+ effective.cooldownMs,
124
+ effective.providerStated === true,
125
+ streak,
126
+ );
127
+
128
+ // The unsupported-model ladder, computed before the branch: the classifier
129
+ // could only state the base window because the authoritative hit count
130
+ // lives on the row, and it is read here. `hits` is the new total for this
131
+ // pair, so the first report is 1 and gets the base window.
132
+ const blockHits = (account.modelBlockHits as Record<string, number> | null) ?? {};
133
+ const unsupportedHits =
134
+ effective.modelUnsupported === true && modelId !== undefined
135
+ ? Math.max(1, (blockHits[modelId] ?? 0) + 1)
136
+ : undefined;
137
+ // An unsupported-model window is stated by *us*, not by the provider, so it
138
+ // would otherwise be multiplied a second time by `escalateFallbackCooldown`
139
+ // (which reads `consecutiveFailures`, an account-wide counter). The two
140
+ // ladders are different things: one counts how many times this pair was
141
+ // reported unsupported, the other how many failures this account produced.
142
+ const unsupportedCooldownMs =
143
+ unsupportedHits === undefined ? undefined : unsupportedModelCooldownMs(unsupportedHits);
144
+
145
+ if (shouldModelCooldown) {
146
+ const existing = (account.modelCooldowns as Record<string, string> | null) ?? {};
147
+ // Monotonic: never move an in-force window earlier (see `latestDeadlineMs`).
148
+ // A provider-stated deadline wins outright because it is already the later
149
+ // of the two whenever the provider really parked the pair longer.
150
+ const nextUntil = new Date(
151
+ latestDeadlineMs(
152
+ effective.retryAt?.getTime(),
153
+ Date.now() + (unsupportedCooldownMs ?? escalatedCooldownMs),
154
+ existing[modelId] ? Date.parse(existing[modelId]!) : undefined,
155
+ ),
156
+ ).toISOString();
157
+ const previousUntil = existing[modelId];
158
+ // A re-statement is not an event: what the audit row reports is the
159
+ // *entry* into a cooling episode, and the deadline itself is not a
160
+ // column — so a row can only ever say "this pair started cooling for
161
+ // this reason". The upstream answers the same "Try again in 11h" on
162
+ // every retry (and an unstated reset is recomputed from now each time),
163
+ // so writing per failure produced seven identical rows in twenty minutes
164
+ // and read as an account flapping when nothing had changed. The entry is
165
+ // still refreshed below — its error fields are what the dialog explains
166
+ // the cooldown with — while an entry that is still in force suppresses
167
+ // the duplicate row.
168
+ const alreadyCooling =
169
+ previousUntil !== undefined && new Date(previousUntil).getTime() > Date.now();
170
+ // `status`/`cooldownUntil` stay untouched: only this (account, model)
171
+ // pair backs off, and the account remains routable for every other model.
172
+ // The error fields ARE written, because they are what provider detail and
173
+ // the health dialog read to explain *why* a model is cooling — without
174
+ // them the row showed "1 model cooling" with no reason at all.
175
+ //
176
+ // The streak advances here too. Without it the most common throttle — a
177
+ // model-scoped rate limit — never escalated, because the counter was only
178
+ // incremented on the account-wide path below, and `consecutiveFailures`
179
+ // stays honest about what it always claimed to count: how many failures in
180
+ // a row this account has just produced.
181
+ await client
182
+ .update(providerAccounts)
183
+ .set({
184
+ modelCooldowns: { ...existing, [modelId]: nextUntil },
185
+ // Counted only for the unsupported-model ladder, and only when this
186
+ // failure is that verdict — so a rate limit or a capacity refusal
187
+ // never advances it. `model_cooldowns`' own shape is untouched, so
188
+ // routing and the console keep parsing it exactly as before.
189
+ ...(unsupportedHits === undefined
190
+ ? {}
191
+ : { modelBlockHits: { ...blockHits, [modelId]: unsupportedHits } }),
192
+ consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
193
+ lastError: effective.reason,
194
+ lastErrorCategory: effective.category,
195
+ lastErrorAt: now,
196
+ })
197
+ .where(eq(providerAccounts.id, accountId));
198
+ if (!alreadyCooling) {
199
+ await writeHealthEvent(client, {
200
+ entityKind: "account",
201
+ accountId,
202
+ // The account's own status, on both sides: a model-scoped throttle
203
+ // never moved it, and writing the classification's status here
204
+ // claimed an `active → cooldown` transition that never happened — the
205
+ // dialog showed "Account status: ACTIVE" above a list of transitions
206
+ // to `cooldown`. The row still names the model and the reason, which
207
+ // is what makes it worth keeping.
208
+ fromStatus: fromStatus as "active" | "cooldown" | "disabled",
209
+ toStatus: fromStatus as "active" | "cooldown" | "disabled",
210
+ reason: effective.reason,
211
+ errorCategory: effective.category,
212
+ modelId: modelId ?? null,
213
+ createdAt: now,
214
+ });
215
+ }
216
+ // `status` and `cooldownUntil` are untouched on this path — only the
217
+ // (account, model) pair backed off — so the snapshot is still accurate
218
+ // and must not be dropped.
219
+ return { ...classification, routesVisible: false };
220
+ }
221
+
222
+ await client
223
+ .update(providerAccounts)
224
+ .set({
225
+ status: effective.status,
226
+ consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
227
+ lastError: effective.reason,
228
+ lastErrorCategory: effective.category,
229
+ lastErrorAt: now,
230
+ // Monotonic against the account's own in-force deadline, for the same
231
+ // reason as the model path: an unclassified 60-second failure must not
232
+ // release an account the provider parked for hours.
233
+ //
234
+ // A `0` duration contributes no candidate rather than stamping *now*
235
+ // (see `accountDeadlineMs`): the one state that asks for it is a
236
+ // terminal automatic disable, and an elapsed deadline there would hand
237
+ // the row straight back to the recovery sweep.
238
+ cooldownUntil: accountDeadlineMs([
239
+ effective.retryAt?.getTime(),
240
+ escalatedCooldownMs > 0 ? Date.now() + escalatedCooldownMs : undefined,
241
+ account.cooldownUntil?.getTime?.(),
242
+ ]),
243
+ })
244
+ .where(eq(providerAccounts.id, accountId));
245
+
246
+ {
247
+ await writeHealthEvent(client, {
248
+ entityKind: "account",
249
+ accountId,
250
+ fromStatus: fromStatus as "active" | "cooldown" | "disabled",
251
+ toStatus: effective.status,
252
+ reason: effective.reason,
253
+ errorCategory: effective.category,
254
+ modelId: modelId ?? null,
255
+ createdAt: now,
256
+ });
257
+ }
258
+ // This path wrote `status` and `cooldownUntil`, both of which the snapshot
259
+ // projects, so routing has to see it.
260
+ return { ...classification, routesVisible: true };
261
+ };
262
+
263
+ try {
264
+ // Kept as a capability probe, not a correctness guard: this function is
265
+ // reachable with a partially-implemented database handle (the dispatch
266
+ // tests pass a narrow double), and a transaction is an optimization here,
267
+ // not a requirement — the same writes are correct without one.
268
+ if (typeof db.transaction === "function") {
269
+ return await db.transaction((tx) => persist(tx as CartethyiaDatabase));
270
+ }
271
+ return await persist(db);
272
+ } catch {
273
+ return null;
274
+ }
275
+ }
276
+
277
+ /** Clears failure state after a successful dispatch; true when the account changed. */
278
+ export async function recordAccountSuccess(
279
+ db: CartethyiaDatabase,
280
+ accountId: string,
281
+ ): Promise<boolean> {
282
+ const mutate = async (client: CartethyiaDatabase): Promise<boolean> => {
283
+ // `recordAccountSuccess` runs on every successful dispatch, and for a
284
+ // healthy account `hadFailures` is false — the row was fetched purely to
285
+ // decide nothing. Projected so that no-op path does not pull the
286
+ // credential ciphertext across the wire.
287
+ const rows = await client
288
+ .select({
289
+ status: providerAccounts.status,
290
+ consecutiveFailures: providerAccounts.consecutiveFailures,
291
+ // Read only to decide whether there is anything to clear: the common
292
+ // case (no unsupported-model hit ever recorded) must not write.
293
+ modelBlockHits: providerAccounts.modelBlockHits,
294
+ })
295
+ .from(providerAccounts)
296
+ .where(eq(providerAccounts.id, accountId))
297
+ .limit(1);
298
+ const account = rows[0];
299
+ if (!account || account.status === "disabled") return false;
300
+ const fromStatus = account.status;
301
+ const hadFailures = (account.consecutiveFailures ?? 0) > 0 || account.status !== "active";
302
+ const now = new Date();
303
+ // The unsupported-model escalation restarts after a success: a model that
304
+ // just served is not "not supported", so the next report of it must start
305
+ // from the base window again rather than from wherever the ladder had
306
+ // reached. `modelCooldowns` is deliberately NOT cleared — an in-force
307
+ // window the provider stated survives, which is the existing behavior and
308
+ // the reference gateways' too.
309
+ const blockHits = account.modelBlockHits as Record<string, number> | null;
310
+ const hasBlockHits = blockHits !== null && Object.keys(blockHits).length > 0;
311
+ await client
312
+ .update(providerAccounts)
313
+ .set({
314
+ status: "active",
315
+ consecutiveFailures: 0,
316
+ cooldownUntil: null,
317
+ lastSuccessAt: now,
318
+ ...(hadFailures ? { lastRecoveredAt: now } : {}),
319
+ ...(hasBlockHits ? { modelBlockHits: {} } : {}),
320
+ })
321
+ .where(eq(providerAccounts.id, accountId));
322
+ if (hadFailures && typeof client.insert === "function") {
323
+ await client.insert(healthEvents).values({
324
+ id: randomUUID(),
325
+ entityKind: "account",
326
+ accountId,
327
+ fromStatus: fromStatus as "active" | "cooldown" | "disabled",
328
+ toStatus: "active",
329
+ reason: "Request dispatched successfully",
330
+ errorCategory: null,
331
+ createdAt: now,
332
+ });
333
+ }
334
+ return hadFailures;
335
+ };
336
+ try {
337
+ if (typeof db.transaction === "function") {
338
+ return await db.transaction((tx) => mutate(tx as CartethyiaDatabase));
339
+ }
340
+ return await mutate(db);
341
+ } catch {
342
+ return false;
343
+ }
344
+ }
345
+
346
+ /** Operator-initiated recovery: forces the account back to active and logs the transition. */
347
+ export async function recoverAccount(
348
+ db: CartethyiaDatabase,
349
+ accountId: string,
350
+ reason = "manual_operator_recovery",
351
+ ): Promise<boolean> {
352
+ const recover = async (client: CartethyiaDatabase): Promise<boolean> => {
353
+ const accountRows = await client
354
+ .select()
355
+ .from(providerAccounts)
356
+ .where(eq(providerAccounts.id, accountId))
357
+ .limit(1);
358
+ const account = accountRows[0];
359
+ if (!account) return false;
360
+ const fromStatus = account.status;
361
+ const now = new Date();
362
+ await client
363
+ .update(providerAccounts)
364
+ .set({
365
+ status: "active",
366
+ consecutiveFailures: 0,
367
+ cooldownUntil: null,
368
+ lastRecoveredAt: now,
369
+ // Recovery is the operator asserting the account works now, so it must
370
+ // clear EVERY routing exclusion, not just the account-wide ones. A
371
+ // per-model cooldown entry excludes the (account, model) pair in
372
+ // `route-catalog.ts`, so leaving it behind kept the account blocked for
373
+ // that model through `/v1` even though the account read `active` and a
374
+ // direct probe (which addresses the account by id, bypassing routing
375
+ // eligibility) succeeded. Clearing the error fields for the same reason:
376
+ // a stale reason on a recovered row misreports the current state.
377
+ modelCooldowns: {},
378
+ lastError: null,
379
+ lastErrorCategory: null,
380
+ lastErrorAt: null,
381
+ })
382
+ .where(eq(providerAccounts.id, accountId));
383
+ {
384
+ await writeHealthEvent(client, {
385
+ entityKind: "account",
386
+ accountId,
387
+ fromStatus: fromStatus as "active" | "cooldown" | "disabled",
388
+ toStatus: "active",
389
+ reason,
390
+ errorCategory: null,
391
+ createdAt: now,
392
+ });
393
+ }
394
+ return true;
395
+ };
396
+ try {
397
+ if (typeof db.transaction === "function") {
398
+ return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
399
+ }
400
+ return await recover(db);
401
+ } catch {
402
+ return false;
403
+ }
404
+ }
405
+
406
+ /** Recovers accounts whose cooldown elapsed and prunes expired per-model keys. */
407
+ export async function sweepExpiredCooldowns(db: CartethyiaDatabase): Promise<number> {
408
+ const recover = async (client: CartethyiaDatabase): Promise<number> => {
409
+ const now = new Date();
410
+ const nowIso = now.toISOString();
411
+ const expired = await client
412
+ .select({
413
+ id: providerAccounts.id,
414
+ status: providerAccounts.status,
415
+ })
416
+ .from(providerAccounts)
417
+ .where(
418
+ // An ordinary cooldown, recovered as soon as its window elapses.
419
+ and(
420
+ eq(providerAccounts.status, "cooldown"),
421
+ isNotNull(providerAccounts.cooldownUntil),
422
+ lte(providerAccounts.cooldownUntil, now),
423
+ ),
424
+ )
425
+ // One pass recovers a bounded batch. The sweep runs every 30s, so a
426
+ // backlog larger than this drains over successive ticks instead of one
427
+ // statement covering every expired row — that statement holds its locks
428
+ // for as long as the backlog takes.
429
+ .limit(COOLDOWN_RECOVERY_BATCH);
430
+ if (expired.length > 0) {
431
+ await client
432
+ .update(providerAccounts)
433
+ .set({
434
+ status: "active",
435
+ consecutiveFailures: 0,
436
+ cooldownUntil: null,
437
+ lastRecoveredAt: now,
438
+ // Cleared with the state it described. A recovered row that kept
439
+ // `auth_invalidated` still rendered the console's "Re-login required"
440
+ // pill above an `active` status, so the account read healthy to
441
+ // routing and broken to the operator at the same time.
442
+ lastErrorCategory: null,
443
+ })
444
+ .where(inArray(providerAccounts.id, expired.map((account) => account.id)));
445
+ if (typeof client.insert === "function") {
446
+ for (let offset = 0; offset < expired.length; offset += 500) {
447
+ await client.insert(healthEvents).values(
448
+ expired.slice(offset, offset + 500).map((account) => ({
449
+ id: randomUUID(),
450
+ entityKind: "account" as const,
451
+ accountId: account.id,
452
+ fromStatus: account.status,
453
+ toStatus: "active" as const,
454
+ reason: "Cooldown period elapsed — auto-recovered",
455
+ errorCategory: null,
456
+ createdAt: now,
457
+ })),
458
+ );
459
+ }
460
+ }
461
+ }
462
+
463
+ // Prune per-model cooldowns: remove entries whose ISO timestamp has passed.
464
+ //
465
+ // One set-based UPDATE rather than a SELECT followed by an UPDATE per row.
466
+ // The per-row form issued one statement per account holding any cooldown —
467
+ // on a deployment where many accounts are throttled at once that is the
468
+ // sweep's dominant cost, and it all runs inside this transaction, holding
469
+ // its locks for the whole pass. Recomputing the object in SQL does the same
470
+ // work in one statement.
471
+ //
472
+ // `jsonb_typeof(...) = 'object'` guards the shape: a malformed value must
473
+ // not abort the whole sweep. `IS DISTINCT FROM` keeps the update from
474
+ // rewriting rows it
475
+ // did not change, so the returned row count is the number of accounts whose
476
+ // cooldown set actually shrank — the same thing the loop counted.
477
+ const pruning = pruneExpiredCooldowns(sql`${providerAccounts.modelCooldowns}`, sql`${nowIso}`);
478
+ const pruned = await client
479
+ .update(providerAccounts)
480
+ .set({ modelCooldowns: pruning.value })
481
+ .where(pruning.changed);
482
+ return expired.length + affectedRows(pruned);
483
+ };
484
+ try {
485
+ if (typeof db.transaction === "function") {
486
+ return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
487
+ }
488
+ return await recover(db);
489
+ } catch {
490
+ return 0;
491
+ }
492
+ }
493
+
494
+ /**
495
+ * Single attempt health reporter: one home for the success/failure writes
496
+ * every dispatch attempt ends with. Success records recovery; failure
497
+ * records with caller-supplied evidence (origin/scope classification plus
498
+ * model id). A mutating write invalidates the route snapshot so the next
499
+ * plan sees fresh health. Never throws — health must not fail requests.
500
+ */
501
+ export interface AttemptHealthReport {
502
+ readonly accountId: string | undefined;
503
+ readonly modelId?: string;
504
+ readonly error?: unknown;
505
+ readonly evidence?: AccountFailureEvidence;
506
+ readonly snapshotService?: { invalidate(): unknown };
507
+ }
508
+
509
+ /**
510
+ * Writes one health audit row, off the request path when a buffer is installed.
511
+ *
512
+ * `health_events` is append-only and read by nobody in the request path, so it
513
+ * does not need to land before the request finishes — unlike the
514
+ * `provider_accounts` UPDATE beside it, which routing reads and which therefore
515
+ * stays synchronous and transactional.
516
+ */
517
+ async function writeHealthEvent(
518
+ client: CartethyiaDatabase,
519
+ row: {
520
+ readonly entityKind: "account";
521
+ readonly accountId: string;
522
+ readonly fromStatus: string;
523
+ readonly toStatus: string;
524
+ readonly reason: string;
525
+ readonly errorCategory: string | null;
526
+ readonly modelId?: string | null;
527
+ readonly createdAt: Date;
528
+ },
529
+ ): Promise<void> {
530
+ const buffer = healthEventBuffer();
531
+ if (buffer !== undefined) {
532
+ buffer.enqueue({ ...row, modelId: row.modelId ?? null });
533
+ return;
534
+ }
535
+ if (typeof client.insert !== "function") return;
536
+ await client.insert(healthEvents).values({ id: randomUUID(), ...row } as never);
537
+ }
538
+
539
+ export async function reportAttemptOutcome(
540
+ db: CartethyiaDatabase,
541
+ report: AttemptHealthReport,
542
+ ): Promise<void> {
543
+ if (!report.accountId) return;
544
+ if (report.error === undefined) {
545
+ const recovered = await recordAccountSuccess(db, report.accountId).catch(() => false);
546
+ if (recovered) await report.snapshotService?.invalidate();
547
+ return;
548
+ }
549
+ const classification = await recordAccountFailure(db, report.accountId, report.error, {
550
+ ...report.evidence,
551
+ ...(report.modelId === undefined ? {} : { modelId: report.modelId }),
552
+ }).catch(() => null);
553
+ // Gated on `routesVisible`, not on `mutatesAccount`: a failure that only
554
+ // cooled one (account, model) pair, or only advanced the streak, changed
555
+ // nothing routing reads. Invalidating for those rebuilt an 11-SELECT snapshot
556
+ // on the next request, and under a failure wave each rebuild fed the next
557
+ // invalidation.
558
+ if (classification?.routesVisible) await report.snapshotService?.invalidate();
559
+ }
560
+
561
+ export async function listAccountHealthEvents(
562
+ db: CartethyiaDatabase,
563
+ accountId: string,
564
+ limit = 50,
565
+ ): Promise<readonly AccountHealthEventRecord[]> {
566
+ try {
567
+ const rows = await db
568
+ .select()
569
+ .from(healthEvents)
570
+ .where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
571
+ .orderBy(desc(healthEvents.createdAt))
572
+ .limit(limit);
573
+
574
+ return rows.map((r) => ({
575
+ id: r.id,
576
+ accountId: r.accountId as string,
577
+ fromStatus: r.fromStatus,
578
+ toStatus: r.toStatus,
579
+ modelId: r.modelId ?? null,
580
+ reason: r.reason,
581
+ errorCategory: r.errorCategory,
582
+ createdAt: r.createdAt.toISOString(),
583
+ }));
584
+ } catch {
585
+ return [];
586
+ }
587
+ }
588
+
589
+ /** Every stored row across accounts and pools — the size the purge offers to delete. */
590
+ export async function countAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
591
+ try {
592
+ const rows = await db.select({ total: count() }).from(healthEvents);
593
+ return rows[0]?.total ?? 0;
594
+ } catch {
595
+ return 0;
596
+ }
597
+ }
598
+
599
+ /**
600
+ * Deletes stored health & error rows.
601
+ *
602
+ * Unlike the read above, a failed delete surfaces: an operator who purged
603
+ * believes the trail is gone, so "it failed" must not read as "it succeeded
604
+ * with zero rows". Callers own the flush-buffer barrier (`drain`) so a queued
605
+ * write cannot land after these rows are gone.
606
+ */
607
+ export async function deleteAccountHealthEvents(
608
+ db: CartethyiaDatabase,
609
+ accountId: string,
610
+ ): Promise<number> {
611
+ const rows = await db
612
+ .delete(healthEvents)
613
+ .where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
614
+ .returning({ id: healthEvents.id });
615
+ return rows.length;
616
+ }
617
+
618
+ export async function deleteAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
619
+ const rows = await db.delete(healthEvents).returning({ id: healthEvents.id });
620
+ return rows.length;
621
+ }