cartethyia 2.7.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +107 -106
- package/README.md +25 -2
- package/dist/dashboard/assets/{BannedPage-P76QLGJA.js → BannedPage-D-J75hBM.js} +1 -1
- package/dist/dashboard/assets/BannedPage-D-J75hBM.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolDetailPage-DdBggJS9.js → CliToolDetailPage-CVe6RaBH.js} +1 -1
- package/dist/dashboard/assets/CliToolDetailPage-CVe6RaBH.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-aN1iXhhP.js → CliToolsPage-Dd8ZSamp.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-Dd8ZSamp.js.gz +0 -0
- package/dist/dashboard/assets/{CombosPage-DE-F1nxZ.js → CombosPage-QoJ6foxo.js} +1 -1
- package/dist/dashboard/assets/CombosPage-QoJ6foxo.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-DpNRay4C.js +1 -0
- package/dist/dashboard/assets/ConsoleLogPage-DpNRay4C.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-3CUxMy2X.js → CustomizationPage-CKUFTcW4.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-CKUFTcW4.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js +1 -0
- package/dist/dashboard/assets/HealthEventsModal-CzSLiDzr.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-DubdQcvX.js +1 -0
- package/dist/dashboard/assets/LoginPage-DubdQcvX.js.gz +0 -0
- package/dist/dashboard/assets/{ModelPicker-DKVslqG3.js → ModelPicker-BH8Tehdi.js} +1 -1
- package/dist/dashboard/assets/ModelPicker-BH8Tehdi.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js +1 -0
- package/dist/dashboard/assets/OverviewPage-DuWzAQFL.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js +4 -0
- package/dist/dashboard/assets/ProviderDetailPage-BKEKe2Fm.js.gz +0 -0
- package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js +1 -0
- package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js +1 -0
- package/dist/dashboard/assets/ProvidersPage-Bwotu8UJ.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js +9 -0
- package/dist/dashboard/assets/ProxyPage-D7hpWbg9.js.gz +0 -0
- package/dist/dashboard/assets/{QuotaPage-BjldQwIU.js → QuotaPage-ATpaxrbl.js} +1 -1
- package/dist/dashboard/assets/QuotaPage-ATpaxrbl.js.gz +0 -0
- package/dist/dashboard/assets/{SettingsPage-DUIoLRxk.js → SettingsPage-CETophpx.js} +1 -1
- package/dist/dashboard/assets/SettingsPage-CETophpx.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-BdgECjqq.js +1 -0
- package/dist/dashboard/assets/SetupPage-BdgECjqq.js.gz +0 -0
- package/dist/dashboard/assets/{SortableList-fI7JkdIB.js → SortableList-CAK-EN9z.js} +1 -1
- package/dist/dashboard/assets/SortableList-CAK-EN9z.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-CHVZqskZ.js → StudioPage-B7W2_RhT.js} +3 -3
- package/dist/dashboard/assets/StudioPage-B7W2_RhT.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-6T4EFDjW.js +56 -0
- package/dist/dashboard/assets/UsagePage-6T4EFDjW.js.gz +0 -0
- package/dist/dashboard/assets/{api-keys-DFY_fZU4.js → api-keys-CUi6UXsR.js} +1 -1
- package/dist/dashboard/assets/api-keys-CUi6UXsR.js.gz +0 -0
- package/dist/dashboard/assets/app-KvsUJUMF.css +2 -0
- package/dist/dashboard/assets/app-KvsUJUMF.css.gz +0 -0
- package/dist/dashboard/assets/{assert-CwdnZARt.js → assert-67xXH5rf.js} +1 -1
- package/dist/dashboard/assets/assert-67xXH5rf.js.gz +0 -0
- package/dist/dashboard/assets/{badge-Dr-uf0Au.js → badge-DC2L43Mj.js} +1 -1
- package/dist/dashboard/assets/badge-DC2L43Mj.js.gz +0 -0
- package/dist/dashboard/assets/{banner-BddmMdoS.js → banner-BBVcb0J3.js} +1 -1
- package/dist/dashboard/assets/banner-BBVcb0J3.js.gz +0 -0
- package/dist/dashboard/assets/{card-CK0PXWlX.js → card-Cca1XC6G.js} +1 -1
- package/dist/dashboard/assets/card-Cca1XC6G.js.gz +0 -0
- package/dist/dashboard/assets/{cli-tools-CcB6mGh2.js → cli-tools-KvsWa1Jl.js} +1 -1
- package/dist/dashboard/assets/cli-tools-KvsWa1Jl.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-DGb2P9Yx.js → clipboard-button-DUzEbynq.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-DUzEbynq.js.gz +0 -0
- package/dist/dashboard/assets/{dialog-LkfuL50u.js → dialog-DFvWhwTa.js} +1 -1
- package/dist/dashboard/assets/dialog-DFvWhwTa.js.gz +0 -0
- package/dist/dashboard/assets/{drawer-ZbEZ_84c.js → drawer-CFM2sMX1.js} +1 -1
- package/dist/dashboard/assets/drawer-CFM2sMX1.js.gz +0 -0
- package/dist/dashboard/assets/{entry-CWBY5A0f.js → entry-BLBLxX0h.js} +3 -3
- package/dist/dashboard/assets/entry-BLBLxX0h.js.gz +0 -0
- package/dist/dashboard/assets/{entry-BJfsVMvA.js → entry-Bqy8RFvM.js} +1 -1
- package/dist/dashboard/assets/entry-Bqy8RFvM.js.gz +0 -0
- package/dist/dashboard/assets/{entry-CvC1-Zqx.js → entry-_xHo5x6r.js} +1 -1
- package/dist/dashboard/assets/entry-_xHo5x6r.js.gz +0 -0
- package/dist/dashboard/assets/formatters-mpBPqAQ8.js +1 -0
- package/dist/dashboard/assets/formatters-mpBPqAQ8.js.gz +0 -0
- package/dist/dashboard/assets/{github-badge-BxpznFLt.js → github-badge-C9e8Ts7e.js} +1 -1
- package/dist/dashboard/assets/github-badge-C9e8Ts7e.js.gz +0 -0
- package/dist/dashboard/assets/{index-CDzP3ovJ.js → index-D6QzEC_n.js} +2 -2
- package/dist/dashboard/assets/index-D6QzEC_n.js.gz +0 -0
- package/dist/dashboard/assets/{inline-Ae13MQ5w.js → inline-BFUcvllj.js} +1 -1
- package/dist/dashboard/assets/inline-BFUcvllj.js.gz +0 -0
- package/dist/dashboard/assets/{input-D23fo7cp.js → input-C_ILAvBb.js} +1 -1
- package/dist/dashboard/assets/input-C_ILAvBb.js.gz +0 -0
- package/dist/dashboard/assets/{layout-CSsxjj5P.js → layout-CBIyT6Rt.js} +1 -1
- package/dist/dashboard/assets/layout-CBIyT6Rt.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-dmbwZMDY.js → list-row-ebJ_-Nsb.js} +1 -1
- package/dist/dashboard/assets/list-row-ebJ_-Nsb.js.gz +0 -0
- package/dist/dashboard/assets/{live-BG1oUsZZ.js → live-D0mm--ME.js} +1 -1
- package/dist/dashboard/assets/live-D0mm--ME.js.gz +0 -0
- package/dist/dashboard/assets/{mutation-DSF-e4ho.js → mutation-sMD965oO.js} +1 -1
- package/dist/dashboard/assets/mutation-sMD965oO.js.gz +0 -0
- package/dist/dashboard/assets/{numbers-Bm_aECY7.js → numbers-BO9dKaFr.js} +1 -1
- package/dist/dashboard/assets/numbers-BO9dKaFr.js.gz +0 -0
- package/dist/dashboard/assets/providers/inception.svg +5 -0
- package/dist/dashboard/assets/providers/inception.svg.gz +0 -0
- package/dist/dashboard/assets/{query-keys-CF56nYIV.js → query-keys-CfL_Zdoh.js} +1 -1
- package/dist/dashboard/assets/query-keys-CfL_Zdoh.js.gz +0 -0
- package/dist/dashboard/assets/{routing-CeA71n7G.js → routing-BnIpi1IJ.js} +1 -1
- package/dist/dashboard/assets/routing-BnIpi1IJ.js.gz +0 -0
- package/dist/dashboard/assets/{select-KIT0rC6G.js → select-Cj5Nw43g.js} +1 -1
- package/dist/dashboard/assets/select-Cj5Nw43g.js.gz +0 -0
- package/dist/dashboard/assets/{settings-B_2gkePf.js → settings-BAFhDgX8.js} +1 -1
- package/dist/dashboard/assets/settings-BAFhDgX8.js.gz +0 -0
- package/dist/dashboard/assets/{stack-D7WOSzcX.js → stack-EMrOMt4I.js} +1 -1
- package/dist/dashboard/assets/stack-EMrOMt4I.js.gz +0 -0
- package/dist/dashboard/assets/{state-BBxtKZOQ.js → state-CWoo7ty5.js} +1 -1
- package/dist/dashboard/assets/state-CWoo7ty5.js.gz +0 -0
- package/dist/dashboard/assets/{switch-DNS_D5Ea.js → switch-CCjT6IkF.js} +1 -1
- package/dist/dashboard/assets/switch-CCjT6IkF.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-BgoxdoMk.js → useMutation-DjkLHUcE.js} +1 -1
- package/dist/dashboard/assets/useMutation-DjkLHUcE.js.gz +0 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +54 -53
- package/migrations/full-postgres/0024_add_pool_provider_cooldowns.sql +12 -0
- package/migrations/full-postgres/0025_add_pool_random_strategy.sql +3 -0
- package/migrations/lite-sqlite3/0024_add_pool_provider_cooldowns.sql +12 -0
- package/migrations/lite-sqlite3/0025_add_pool_random_strategy.sql +20 -0
- package/package.json +96 -96
- package/scripts/cli/console-menu.ts +5 -2
- package/scripts/cli/tray.ts +21 -6
- package/src/boot/app.ts +684 -714
- package/src/boot/dependencies.ts +65 -62
- package/src/config/cooldowns.ts +16 -0
- package/src/config/providers.ts +13 -1
- package/src/config/resources.ts +70 -70
- package/src/config/security.ts +37 -37
- package/src/config/spec.ts +371 -358
- package/src/console/console-router.ts +233 -225
- package/src/console/domain-registration.ts +398 -381
- package/src/console/providers/catalog/catalog-projections.ts +2 -6
- package/src/console/providers/catalog/contracts.ts +19 -0
- package/src/console/providers/catalog/provider-operations.ts +38 -0
- package/src/console/providers/catalog/routes.ts +14 -0
- package/src/console/providers/catalog/store-helpers.ts +326 -0
- package/src/console/providers/catalog/store.ts +1292 -1563
- package/src/console/quota/account-quota-global.ts +2 -2
- package/src/console/quota/account-quota-shared.ts +1 -1
- package/src/console/quota/account-quota-tenant.ts +35 -12
- package/src/console/quota/quota-view.ts +2 -2
- package/src/console/routing/pools/contracts.ts +41 -1
- package/src/console/routing/pools/operations.ts +183 -4
- package/src/console/routing/pools/relay-deploy.ts +17 -4
- package/src/console/routing/pools/routes.ts +30 -0
- package/src/console/routing/pools/store.ts +98 -8
- package/src/main.ts +183 -167
- package/src/network/outbound-fetch.ts +226 -365
- package/src/network/pool/agent.ts +8 -0
- package/src/network/pool/pool-state.ts +7 -0
- package/src/network/pool/selector.ts +110 -2
- package/src/network/pool-cooldown-store.ts +252 -0
- package/src/network/pool-health-machine.ts +5 -3
- package/src/network/pool-health.ts +36 -9
- package/src/observability/event-query.ts +57 -0
- package/src/observability/metrics.ts +467 -467
- package/src/observability/runtime-metrics.ts +269 -269
- package/src/persistence/health-event-flush.ts +2 -0
- package/src/persistence/schema.ts +19 -0
- package/src/persistence/usage-flush.ts +8 -3
- package/src/protocol/canonical-model.ts +1 -1
- package/src/protocol/dsml.ts +377 -0
- package/src/protocol/gateway-error.ts +232 -232
- package/src/{providers/provider-reasoning.ts → protocol/reasoning.ts} +10 -9
- package/src/protocol/request/chat.ts +7 -7
- package/src/protocol/request/codex.ts +2 -2
- package/src/protocol/request/gemini.ts +1 -1
- package/src/protocol/request/messages.ts +6 -6
- package/src/protocol/request/responses.ts +7 -7
- package/src/protocol/response/dsml.ts +12 -133
- package/src/{transport → protocol}/translation/attachment-integrity.ts +2 -2
- package/src/{transport → protocol}/translation/cache-controls.ts +1 -1
- package/src/{transport → protocol}/translation/capabilities.ts +2 -2
- package/src/{transport → protocol}/translation/normalize.ts +1 -1
- package/src/{transport → protocol}/translation/quirks.ts +1 -1
- package/src/{transport → protocol}/translation/search.ts +1 -1
- package/src/{transport → protocol}/translation/thinking.ts +23 -1
- package/src/{transport → protocol}/translation/tool-id.ts +1 -1
- package/src/{transport → protocol}/translation/tool-repair.ts +2 -2
- package/src/protocol/upstream-parsing.ts +176 -0
- package/src/providers/adapters/compatible-adapter.ts +11 -2
- package/src/providers/authentication/oauth-refresh-service.ts +9 -5
- package/src/providers/default-registry.ts +21 -5
- package/src/providers/discovery/probe-phases.ts +16 -3
- package/src/providers/discovery/probing-service.ts +4 -1
- package/src/providers/integrations/anthropic.ts +1 -1
- package/src/providers/integrations/buddy/buddy-catalog-shared.ts +1 -4
- package/src/providers/integrations/buddy/buddy-chat-shared.ts +141 -8
- package/src/providers/integrations/buddy/buddy-checkin.ts +5 -1
- package/src/providers/integrations/buddy/buddy-growth-shared.ts +0 -82
- package/src/providers/integrations/buddy/buddy-identity.ts +25 -0
- package/src/providers/integrations/buddy/buddy-server-tools.ts +240 -0
- package/src/providers/integrations/buddy/codebuddy/cn-catalog.ts +36 -0
- package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +6 -31
- package/src/providers/integrations/buddy/codebuddy/provider.ts +2 -0
- package/src/providers/integrations/buddy/codebuddy/shared.ts +19 -4
- package/src/providers/integrations/buddy/turing-device-token.ts +262 -0
- package/src/providers/integrations/buddy/workbuddy/provider.ts +2 -0
- package/src/providers/integrations/buddy/workbuddy/shared.ts +56 -3
- package/src/providers/integrations/claude/claude.ts +1 -1
- package/src/providers/integrations/cloudflareai/cloudflareai.ts +1 -1
- package/src/providers/integrations/codex/codex-errors.ts +1 -1
- package/src/providers/integrations/codex/codex.ts +1 -1
- package/src/providers/integrations/factory/factory.ts +1 -1
- package/src/providers/integrations/github/github.ts +1 -1
- package/src/providers/integrations/grok/grok.ts +1 -1
- package/src/providers/integrations/inception/provider.ts +91 -0
- package/src/providers/integrations/inferhub.ts +1 -1
- package/src/providers/integrations/kiro/kiro-request.ts +1 -1
- package/src/providers/integrations/mistral.ts +193 -19
- package/src/providers/integrations/opencode/fingerprint.ts +19 -2
- package/src/providers/integrations/opencode/prompt.ts +111 -0
- package/src/providers/integrations/opencode/provider.ts +101 -23
- package/src/providers/integrations/opencode/tool-definitions.ts +4 -4
- package/src/providers/integrations/qoder/qoder-checkin.ts +303 -0
- package/src/providers/integrations/qoder/qoder.ts +1641 -1458
- package/src/providers/integrations/shared/install-id.ts +2 -1
- package/src/providers/integrations/trae/trae-cache.ts +110 -0
- package/src/providers/integrations/trae/trae-oauth.ts +29 -4
- package/src/providers/integrations/trae/trae-protocol.ts +2 -1
- package/src/providers/integrations/trae/trae.ts +106 -23
- package/src/providers/operations/account-error-classification.ts +1660 -0
- package/src/providers/operations/account-health-service.ts +621 -2097
- package/src/providers/operations/account-reset-service.ts +2 -0
- package/src/providers/provider-contracts.ts +0 -2
- package/src/providers/provider-metadata.ts +25 -4
- package/src/providers/provider-probe.ts +2 -2
- package/src/{console → providers}/quota/cache.ts +1 -1
- package/src/{console → providers}/quota/refresh.ts +5 -5
- package/src/transport/dispatch/attempt-finalize.ts +114 -12
- package/src/transport/dispatch/attempt-loop.ts +3 -0
- package/src/transport/dispatch/empty-stream.ts +224 -217
- package/src/transport/dispatch/generation.ts +37 -55
- package/src/transport/dispatch/handler-deps.ts +32 -0
- package/src/transport/dispatch/leases.ts +9 -4
- package/src/transport/dispatch/proxy-request.ts +6 -30
- package/src/transport/dispatch/responses-compact.ts +22 -43
- package/src/transport/dispatch/streaming/finalize.ts +3 -0
- package/src/transport/dispatch/streaming/stream.ts +0 -8
- package/src/transport/dispatch/systemone.ts +22 -42
- package/src/transport/dispatch/token-count.ts +23 -39
- package/src/transport/dispatch/upstream.ts +1 -3
- package/src/transport/dispatch/websearch.ts +22 -41
- package/src/transport/failure-policy.ts +59 -178
- package/src/transport/guards/abuse-guard.ts +10 -0
- package/src/transport/guards/auth-guard.ts +109 -109
- package/src/transport/guards/error-lifecycle.ts +1 -1
- package/src/transport/guards/pipeline.ts +220 -220
- package/src/transport/guards/request-context.ts +245 -238
- package/src/transport/request/degradation.ts +20 -1
- package/src/transport/request/inflight.ts +4 -114
- package/src/transport/request/preparer.ts +6 -6
- package/src/transport/request/state.ts +66 -38
- package/src/transport/request/tenant-preferences.ts +1 -1
- package/src/transport/routing/route-catalog.ts +41 -31
- package/src/transport/routing/router.ts +2 -2
- package/src/transport/surface/messages/parse.ts +1 -1
- package/src/workers/buddy-growth-tasks.ts +346 -425
- package/src/workers/daily-checkin.ts +17 -6
- package/src/workers/quota-refresh-sweep.ts +2 -2
- package/src/workers/scheduler.ts +17 -1
- package/dist/dashboard/assets/BannedPage-P76QLGJA.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-DdBggJS9.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-aN1iXhhP.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-DE-F1nxZ.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-5SpmfLyw.js +0 -1
- package/dist/dashboard/assets/ConsoleLogPage-5SpmfLyw.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-3CUxMy2X.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-BkmJQKC0.js +0 -1
- package/dist/dashboard/assets/HealthEventsModal-BkmJQKC0.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-B_0FUKWc.js +0 -1
- package/dist/dashboard/assets/LoginPage-B_0FUKWc.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-DKVslqG3.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-D-H4MEiT.js +0 -1
- package/dist/dashboard/assets/OverviewPage-D-H4MEiT.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-CEJw6Guo.js +0 -3
- package/dist/dashboard/assets/ProviderDetailPage-CEJw6Guo.js.gz +0 -0
- package/dist/dashboard/assets/ProviderIcon-CmfzxpVM.js +0 -1
- package/dist/dashboard/assets/ProviderIcon-CmfzxpVM.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-CEplAR-i.js +0 -1
- package/dist/dashboard/assets/ProvidersPage-CEplAR-i.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-CAN7hy0f.js +0 -7
- package/dist/dashboard/assets/ProxyPage-CAN7hy0f.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-BjldQwIU.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-DUIoLRxk.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-_bCaTM0v.js +0 -1
- package/dist/dashboard/assets/SetupPage-_bCaTM0v.js.gz +0 -0
- package/dist/dashboard/assets/SortableList-fI7JkdIB.js.gz +0 -0
- package/dist/dashboard/assets/StudioPage-CHVZqskZ.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-DiFBymHj.js +0 -56
- package/dist/dashboard/assets/UsagePage-DiFBymHj.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-DFY_fZU4.js.gz +0 -0
- package/dist/dashboard/assets/app-BOJ1_xUB.css +0 -2
- package/dist/dashboard/assets/app-BOJ1_xUB.css.gz +0 -0
- package/dist/dashboard/assets/assert-CwdnZARt.js.gz +0 -0
- package/dist/dashboard/assets/badge-Dr-uf0Au.js.gz +0 -0
- package/dist/dashboard/assets/banner-BddmMdoS.js.gz +0 -0
- package/dist/dashboard/assets/card-CK0PXWlX.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-CcB6mGh2.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-DGb2P9Yx.js.gz +0 -0
- package/dist/dashboard/assets/cog-CdWhsajs.js +0 -1
- package/dist/dashboard/assets/cog-CdWhsajs.js.gz +0 -0
- package/dist/dashboard/assets/dialog-LkfuL50u.js.gz +0 -0
- package/dist/dashboard/assets/drawer-ZbEZ_84c.js.gz +0 -0
- package/dist/dashboard/assets/entry-BJfsVMvA.js.gz +0 -0
- package/dist/dashboard/assets/entry-CWBY5A0f.js.gz +0 -0
- package/dist/dashboard/assets/entry-CvC1-Zqx.js.gz +0 -0
- package/dist/dashboard/assets/github-badge-BxpznFLt.js.gz +0 -0
- package/dist/dashboard/assets/index-CDzP3ovJ.js.gz +0 -0
- package/dist/dashboard/assets/inline-Ae13MQ5w.js.gz +0 -0
- package/dist/dashboard/assets/input-D23fo7cp.js.gz +0 -0
- package/dist/dashboard/assets/layout-CSsxjj5P.js.gz +0 -0
- package/dist/dashboard/assets/list-row-dmbwZMDY.js.gz +0 -0
- package/dist/dashboard/assets/live-BG1oUsZZ.js.gz +0 -0
- package/dist/dashboard/assets/mutation-DSF-e4ho.js.gz +0 -0
- package/dist/dashboard/assets/numbers-Bm_aECY7.js.gz +0 -0
- package/dist/dashboard/assets/query-keys-CF56nYIV.js.gz +0 -0
- package/dist/dashboard/assets/routing-CeA71n7G.js.gz +0 -0
- package/dist/dashboard/assets/select-KIT0rC6G.js.gz +0 -0
- package/dist/dashboard/assets/settings-B_2gkePf.js.gz +0 -0
- package/dist/dashboard/assets/stack-D7WOSzcX.js.gz +0 -0
- package/dist/dashboard/assets/state-BBxtKZOQ.js.gz +0 -0
- package/dist/dashboard/assets/switch-DNS_D5Ea.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-BgoxdoMk.js.gz +0 -0
- /package/dist/dashboard/assets/{app-CaXZ9SCn.js → app-BaWd0puM.js} +0 -0
- /package/dist/dashboard/assets/{app-CaXZ9SCn.js.gz → app-BaWd0puM.js.gz} +0 -0
|
@@ -1,2097 +1,621 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import { and, count, desc, eq, inArray, isNotNull, lte,
|
|
4
|
-
import { affectedRows, pruneExpiredCooldowns } from "../../persistence/dialect";
|
|
5
|
-
import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
const
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
// discarded because nothing mutates on provider scope.
|
|
623
|
-
const revocationNamed = folded.includes("token revoked") || folded.includes("invalidated oauth token");
|
|
624
|
-
const accountEvidence =
|
|
625
|
-
options?.credentialEvidence === true ||
|
|
626
|
-
revocationNamed ||
|
|
627
|
-
(options?.origin === "upstream" && options.scope === "account");
|
|
628
|
-
// A named revocation mutates even at provider scope: the message is the
|
|
629
|
-
// evidence, and provider scope is what a frame with no credential detail
|
|
630
|
-
// classifies as. Every other signal still needs account scope.
|
|
631
|
-
const canMutate = origin === "upstream" && (revocationNamed || (scope === "account" && accountEvidence));
|
|
632
|
-
const providerCode = options?.providerCode?.toLowerCase() ?? "";
|
|
633
|
-
|
|
634
|
-
const getHeader = (name: string): string | null => {
|
|
635
|
-
const headers = options?.headers;
|
|
636
|
-
if (!headers) return null;
|
|
637
|
-
if ("get" in headers && typeof headers.get === "function") return headers.get(name);
|
|
638
|
-
const wanted = name.toLowerCase();
|
|
639
|
-
const entry = Object.entries(headers).find(([key]) => key.toLowerCase() === wanted);
|
|
640
|
-
return entry?.[1] ?? null;
|
|
641
|
-
};
|
|
642
|
-
|
|
643
|
-
const headerCooldown =
|
|
644
|
-
options?.retryAfterMs ??
|
|
645
|
-
parseUpstreamBackoff({ get: getHeader });
|
|
646
|
-
const messageCooldown = parseProviderResetDuration(message);
|
|
647
|
-
const rawUpstreamResponse =
|
|
648
|
-
origin === "upstream" &&
|
|
649
|
-
error instanceof GatewayError &&
|
|
650
|
-
typeof error.details.raw === "string"
|
|
651
|
-
? error.details.raw
|
|
652
|
-
: undefined;
|
|
653
|
-
// The body's own fields, when they can be reached. Checked alongside the
|
|
654
|
-
// header and the prose rather than instead of them: a provider may state the
|
|
655
|
-
// window in any of the three, and the longest honest statement wins.
|
|
656
|
-
//
|
|
657
|
-
// `details.raw` is tried even when no caller passed a parsed body, because
|
|
658
|
-
// that is where the upstream's JSON actually reaches this function — the
|
|
659
|
-
// dispatch path stores the response text there. A structured window read from
|
|
660
|
-
// it is the difference between a 15-minute fallback and the 23 days the
|
|
661
|
-
// provider stated.
|
|
662
|
-
const bodyCooldown =
|
|
663
|
-
parseUpstreamBodyReset(options?.upstreamBody) ??
|
|
664
|
-
(rawUpstreamResponse === undefined ? null : parseUpstreamBodyReset(tryParseUpstreamBody(rawUpstreamResponse)));
|
|
665
|
-
// The body the ban is named in. The caller's parsed body when there is one,
|
|
666
|
-
// and the raw response text otherwise — the dispatch path stores the
|
|
667
|
-
// upstream's JSON there, and that is where `displayMsg` lives.
|
|
668
|
-
const banBody =
|
|
669
|
-
options?.upstreamBody ??
|
|
670
|
-
(rawUpstreamResponse === undefined ? undefined : tryParseUpstreamBody(rawUpstreamResponse));
|
|
671
|
-
// Whether the provider itself named a window. Decided once, from the
|
|
672
|
-
// evidence, so every branch of this classification agrees about it.
|
|
673
|
-
const providerStated =
|
|
674
|
-
headerCooldown !== null || messageCooldown !== null || bodyCooldown !== null;
|
|
675
|
-
const statedCooldownMs =
|
|
676
|
-
[headerCooldown, messageCooldown, bodyCooldown].reduce<number | null>(
|
|
677
|
-
(best, value) => (value === null ? best : best === null ? value : Math.max(best, value)),
|
|
678
|
-
null,
|
|
679
|
-
);
|
|
680
|
-
const reason = (prefix: string): string => `${prefix}: ${message.slice(0, 160)}`;
|
|
681
|
-
const result = (
|
|
682
|
-
category: AccountErrorCategory,
|
|
683
|
-
status: AccountErrorClassification["status"],
|
|
684
|
-
cooldownMs: number,
|
|
685
|
-
retryAt: Date | null,
|
|
686
|
-
explanation: string,
|
|
687
|
-
mutatesAccount = canMutate,
|
|
688
|
-
modelUnsupported = false,
|
|
689
|
-
): AccountErrorClassification => ({
|
|
690
|
-
category,
|
|
691
|
-
status,
|
|
692
|
-
cooldownMs,
|
|
693
|
-
reason: origin === "upstream" ? rawUpstreamResponse ?? message.slice(0, 160) : explanation,
|
|
694
|
-
retryAt,
|
|
695
|
-
mutatesAccount,
|
|
696
|
-
// Set below by the caller that knows which columns it wrote, not here:
|
|
697
|
-
// the same classification feeds both the account-wide path (status +
|
|
698
|
-
// cooldownUntil — routing-visible) and the model-scoped path (neither —
|
|
699
|
-
// invisible). Defaulting true keeps existing callers conservative.
|
|
700
|
-
routesVisible: true,
|
|
701
|
-
providerStated,
|
|
702
|
-
modelUnsupported,
|
|
703
|
-
});
|
|
704
|
-
|
|
705
|
-
return {
|
|
706
|
-
message,
|
|
707
|
-
lower,
|
|
708
|
-
folded,
|
|
709
|
-
statusCode,
|
|
710
|
-
origin,
|
|
711
|
-
scope,
|
|
712
|
-
revocationNamed,
|
|
713
|
-
accountEvidence,
|
|
714
|
-
canMutate,
|
|
715
|
-
providerCode,
|
|
716
|
-
statedCooldownMs,
|
|
717
|
-
buddyBanned: isBuddyAccountBanned(lower, providerCode, banBody),
|
|
718
|
-
reason,
|
|
719
|
-
result,
|
|
720
|
-
};
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
export function classifyAccountError(
|
|
724
|
-
error: unknown,
|
|
725
|
-
options?: AccountFailureEvidence,
|
|
726
|
-
): AccountErrorClassification {
|
|
727
|
-
const {
|
|
728
|
-
message,
|
|
729
|
-
lower,
|
|
730
|
-
folded,
|
|
731
|
-
statusCode,
|
|
732
|
-
origin,
|
|
733
|
-
scope,
|
|
734
|
-
revocationNamed,
|
|
735
|
-
accountEvidence,
|
|
736
|
-
canMutate,
|
|
737
|
-
providerCode,
|
|
738
|
-
|
|
739
|
-
statedCooldownMs,
|
|
740
|
-
buddyBanned,
|
|
741
|
-
reason,
|
|
742
|
-
result,
|
|
743
|
-
} = accountErrorContext(error, options);
|
|
744
|
-
const modelId = options?.modelId;
|
|
745
|
-
const modelBlockHits = Math.max(1, Math.floor(options?.modelBlockHits ?? 1));
|
|
746
|
-
const staticToken = options?.staticToken === true;
|
|
747
|
-
const credentialKind = options?.credentialKind;
|
|
748
|
-
const buddy = BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "");
|
|
749
|
-
|
|
750
|
-
// A price refusal is a verdict on the *request*, not on the account. inferhub
|
|
751
|
-
// answers 402 with `no provider's ask matches your max-per-mtok bid`: the
|
|
752
|
-
// caller's own price ceiling was below every upstream ask, which says nothing
|
|
753
|
-
// about the credential or its balance. Classified as quota it cooled a
|
|
754
|
-
// fully-credited account down for an hour and the operator reported it as
|
|
755
|
-
// "blocked, unusable at all". Detected before the quota branch and recorded
|
|
756
|
-
// without mutating the account, so only that request fails.
|
|
757
|
-
const priceRefusal =
|
|
758
|
-
lower.includes("max-per-mtok") ||
|
|
759
|
-
lower.includes("no provider's ask") ||
|
|
760
|
-
(lower.includes("ask") && lower.includes("bid"));
|
|
761
|
-
// Deployment/proxy infrastructure errors (DEPLOYMENT_DISABLED, etc.) are
|
|
762
|
-
// transport-level proxy pool failures, not account quota exhaustion.
|
|
763
|
-
// Mutating the account parks it for an hour while the pool itself stays
|
|
764
|
-
// healthy — the next request picks the same dying pool and the same error
|
|
765
|
-
// repeats. These must not cooldown the account.
|
|
766
|
-
const deploymentInfraError =
|
|
767
|
-
lower.includes("deployment_disabled") ||
|
|
768
|
-
lower.includes("deployment disabled") ||
|
|
769
|
-
(lower.includes("sin1::") && lower.includes("deployment"));
|
|
770
|
-
if (deploymentInfraError) {
|
|
771
|
-
return result("server_error", "cooldown", 60_000, null, `Proxy deployment error: ${message.slice(0, 160)}`, false);
|
|
772
|
-
}
|
|
773
|
-
// A (CodeBuddy/WorkBuddy) *frequency* window is a verdict on one
|
|
774
|
-
// (account, model) pair, not on the account: `{"code":"6004","msg":"usage
|
|
775
|
-
// exceeds frequency limit ... your usage will reset at <stamp> UTC+8,
|
|
776
|
-
// alternatively, you can switch to the other models to continue using it."}`.
|
|
777
|
-
// The provider names the remedy itself — use a different model — which is
|
|
778
|
-
// only a remedy if the account still serves every other model. Taken as
|
|
779
|
-
// account-wide `quota_exhausted` it parked the whole account until the reset,
|
|
780
|
-
// so the switch the upstream offered was refused by our own gateway and every
|
|
781
|
-
// sibling model went dark with it; the account also read COOLDOWN /
|
|
782
|
-
// quota_exhausted while only that pair was throttled.
|
|
783
|
-
//
|
|
784
|
-
// Classified `rate_limit_transient` so the writer scopes it to the model:
|
|
785
|
-
// `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
|
|
786
|
-
// account's own status alone. Kept ahead of `quotaSignal`, which matches a
|
|
787
|
-
// *broader* slice of the same wording (`usage exceeds` + `frequency limit`)
|
|
788
|
-
// and would otherwise claim this message as an account-wide quota window.
|
|
789
|
-
// Narrow on purpose — the extra phrase is what says the other models still
|
|
790
|
-
// work; without it the failure stays account-wide.
|
|
791
|
-
const modelFrequencyWindow =
|
|
792
|
-
lower.includes("frequency limit") && lower.includes("switch to the other models");
|
|
793
|
-
if (modelFrequencyWindow) {
|
|
794
|
-
const cooldownMs = statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
|
|
795
|
-
return result(
|
|
796
|
-
"rate_limit_transient",
|
|
797
|
-
"cooldown",
|
|
798
|
-
cooldownMs,
|
|
799
|
-
new Date(Date.now() + cooldownMs),
|
|
800
|
-
reason("Provider frequency limit for this model"),
|
|
801
|
-
// Provider-scoped evidence still has to reach the writer: a stream frame
|
|
802
|
-
// carries no credential detail, so `canMutate` is false and without this
|
|
803
|
-
// the pair would be retried for the whole window it was closed for.
|
|
804
|
-
origin === "upstream" && scope === "provider",
|
|
805
|
-
);
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
// Quota-shaped provider codes/messages (e.g. xAI's
|
|
809
|
-
const quotaSignal =
|
|
810
|
-
!priceRefusal &&
|
|
811
|
-
!deploymentInfraError &&
|
|
812
|
-
(statusCode === 402 ||
|
|
813
|
-
// Qoder answers an exhausted credit balance with
|
|
814
|
-
// `{"code":"112","message":"{\"pricingUrl\":...}"}` — the body points at
|
|
815
|
-
// the pricing page, i.e. "buy more", which is a balance verdict, not an
|
|
816
|
-
// auth one. It fell through to `auth_invalidated` and the console showed
|
|
817
|
-
// "Re-login required" on an account whose credential was fine, sending
|
|
818
|
-
// operators to re-authenticate instead of topping up.
|
|
819
|
-
// Recognising it here keeps it out of `auth_invalidated`; the branch
|
|
820
|
-
// below scopes the resulting cooldown to the model, because the account
|
|
821
|
-
// keeps serving its other models.
|
|
822
|
-
providerCode === "112" ||
|
|
823
|
-
providerCode === "insufficient_quota" ||
|
|
824
|
-
providerCode === "quota_exceeded" ||
|
|
825
|
-
providerCode === "subscription:free-usage-exhausted" ||
|
|
826
|
-
providerCode === "14018" ||
|
|
827
|
-
// CodeBuddy/WorkBuddy rate limit: `{"code":"6004","msg":"usage exceeds
|
|
828
|
-
// frequency limit ... your usage will reset at <stamp> UTC+8"}`. The
|
|
829
|
-
// message names a reset, so this is a parked window, not a transient
|
|
830
|
-
// 429 — without it the account fell through to `unknown` with
|
|
831
|
-
// `mutates=false`, stayed in rotation, and was re-picked for every
|
|
832
|
-
// request inside the window the provider had actually named.
|
|
833
|
-
//
|
|
834
|
-
// Matched on the message's own wording, not on the `6004` code: that code
|
|
835
|
-
// is not unique to this surface (the artifact-release API reuses it), so a
|
|
836
|
-
// code match would sweep an unrelated failure into a quota cooldown. The
|
|
837
|
-
// phrase is the reliable key, and `usage exceeds` + `frequency limit` is
|
|
838
|
-
// the pair this family actually prints.
|
|
839
|
-
(lower.includes("usage exceeds") && lower.includes("frequency limit")) ||
|
|
840
|
-
lower.includes("insufficient_quota") ||
|
|
841
|
-
lower.includes("quota_exceeded") ||
|
|
842
|
-
lower.includes("usage_limit_reached") ||
|
|
843
|
-
lower.includes("usage-exhausted") ||
|
|
844
|
-
lower.includes("usage_exhausted") ||
|
|
845
|
-
lower.includes("usage exhausted") ||
|
|
846
|
-
lower.includes("usage exceeds") ||
|
|
847
|
-
lower.includes("frequency limit") ||
|
|
848
|
-
lower.includes("free-usage") ||
|
|
849
|
-
lower.includes("free usage") ||
|
|
850
|
-
lower.includes("exceeded your current quota") ||
|
|
851
|
-
// Trae answers an exhausted quota with **HTTP 502** and the body `Your
|
|
852
|
-
// requests have exceeded the quota.` The status is what misled the
|
|
853
|
-
// classifier: 502 read as a gateway failure, so this fell through to
|
|
854
|
-
// `server_error` — a 30s cooldown that does not mutate the account, so
|
|
855
|
-
// the account stayed in rotation and every other model on it kept being
|
|
856
|
-
// invoked inside a window the provider had already closed. The body is
|
|
857
|
-
// the verdict; the 502 is incidental. Matched on the phrase, not on the
|
|
858
|
-
// status, which is why this sits in `quotaSignal` ahead of the 5xx arm.
|
|
859
|
-
lower.includes("have exceeded the quota") ||
|
|
860
|
-
lower.includes("balance exhausted") ||
|
|
861
|
-
lower.includes("insufficient balance") ||
|
|
862
|
-
lower.includes("out of credits") ||
|
|
863
|
-
lower.includes("credits exhausted") ||
|
|
864
|
-
lower.includes("spending limit") ||
|
|
865
|
-
// Same Qoder family reached by body text when the code is not parsed out.
|
|
866
|
-
(lower.includes("pricingurl") && lower.includes("qoder.com/pricing")));
|
|
867
|
-
// A deterministic content-policy rejection is NOT credential invalidation.
|
|
868
|
-
// CodeBuddy returns HTTP 403 with provider code 11140 ("request illegal",
|
|
869
|
-
// "did not pass the safety review") — refreshing the token cannot change
|
|
870
|
-
// that outcome and the account itself is healthy, so it must never flip the
|
|
871
|
-
// account to auth_invalidated/disabled. Only real auth signals do.
|
|
872
|
-
const policyRejection =
|
|
873
|
-
providerCode === "11140" ||
|
|
874
|
-
lower.includes("safety review") ||
|
|
875
|
-
lower.includes("content did not pass") ||
|
|
876
|
-
lower.includes("request illegal") ||
|
|
877
|
-
lower.includes("content blocked") ||
|
|
878
|
-
lower.includes("account suspended");
|
|
879
|
-
// A ban is a verdict on the credential, and cooldown cannot answer it: the
|
|
880
|
-
// account is refused, not throttled. Disabling is the fix, and disabling
|
|
881
|
-
// *immediately* is a feature — a client's retry budget is spent in seconds,
|
|
882
|
-
// so a banned account that merely cooled stayed in the plan and burned
|
|
883
|
-
// sibling accounts that were still healthy before the window expired.
|
|
884
|
-
//
|
|
885
|
-
// The two shapes are told apart by the body, not the code alone: `11140`
|
|
886
|
-
// names a restriction on the account, `11-128` refuses the request.
|
|
887
|
-
const policyBlock =
|
|
888
|
-
policyRejection &&
|
|
889
|
-
options?.providerId !== undefined &&
|
|
890
|
-
BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
|
|
891
|
-
if (policyBlock) {
|
|
892
|
-
if (buddyBanned) {
|
|
893
|
-
return result(
|
|
894
|
-
"policy_blocked_banned",
|
|
895
|
-
"disabled",
|
|
896
|
-
0,
|
|
897
|
-
null,
|
|
898
|
-
reason("Account banned by upstream"),
|
|
899
|
-
// A ban is evidence about the credential even at provider scope, for
|
|
900
|
-
// the same reason a named revocation is: the message is the evidence.
|
|
901
|
-
// Without this a stream frame's missing credential detail would leave a
|
|
902
|
-
// banned account serving.
|
|
903
|
-
true,
|
|
904
|
-
);
|
|
905
|
-
}
|
|
906
|
-
// Not a ban: the upstream refused this request. The account is fine, so the
|
|
907
|
-
// only correct move is to stop retrying it verbatim. Cooling the account
|
|
908
|
-
// here was the bug that made one malformed request look like a dead
|
|
909
|
-
// credential.
|
|
910
|
-
return result(
|
|
911
|
-
"policy_blocked_invalid_request",
|
|
912
|
-
"cooldown",
|
|
913
|
-
0,
|
|
914
|
-
null,
|
|
915
|
-
reason("Upstream refused this request"),
|
|
916
|
-
// Never mutates: a malformed body fails identically on every account, so
|
|
917
|
-
// cooling would burn a healthy credential on the gateway's own fault.
|
|
918
|
-
false,
|
|
919
|
-
);
|
|
920
|
-
}
|
|
921
|
-
// These codes are a verdict on the request, not the account. Cooling the
|
|
922
|
-
// account for them burns a healthy credential on a body that will fail the
|
|
923
|
-
// same way on every other account:
|
|
924
|
-
//
|
|
925
|
-
// - `11148` tool-call/result mismatch — the transcript is broken; the
|
|
926
|
-
// pre-payload repair is supposed to prevent it, and a slip is still the
|
|
927
|
-
// body's fault.
|
|
928
|
-
// - `11135` the image cannot be processed, `11133` the model rejected a
|
|
929
|
-
// parameter, `11101` the body failed to parse — deterministic for that
|
|
930
|
-
// payload.
|
|
931
|
-
// - `12153` the session is dead — a refresh problem, not a reason to park
|
|
932
|
-
// the account on the first observation.
|
|
933
|
-
// - `14017` trial not activated — the account may recover once registration
|
|
934
|
-
// finishes, so a hard cooldown would keep it out after it becomes usable.
|
|
935
|
-
//
|
|
936
|
-
// Recognised before the auth and quota branches: several of these arrive as
|
|
937
|
-
// 401/403/429, which those branches would otherwise treat as account evidence.
|
|
938
|
-
const requestScoped =
|
|
939
|
-
providerCode === "11148" ||
|
|
940
|
-
providerCode === "11135" ||
|
|
941
|
-
providerCode === "11133" ||
|
|
942
|
-
providerCode === "11101" ||
|
|
943
|
-
providerCode === "12153" ||
|
|
944
|
-
providerCode === "14017" ||
|
|
945
|
-
lower.includes("tool_call_sequence_broken") ||
|
|
946
|
-
lower.includes("unmarshal chat params failed") ||
|
|
947
|
-
lower.includes("invalid image_url content") ||
|
|
948
|
-
lower.includes("offline user session not found") ||
|
|
949
|
-
lower.includes("trial not activated") ||
|
|
950
|
-
lower.includes("trial version is not yet activated");
|
|
951
|
-
if (requestScoped && BUDDY_PROVIDER_IDS.has(options?.providerId?.toLowerCase() ?? "")) {
|
|
952
|
-
return result(
|
|
953
|
-
"unknown",
|
|
954
|
-
"cooldown",
|
|
955
|
-
0,
|
|
956
|
-
null,
|
|
957
|
-
reason("Request-scoped upstream rejection"),
|
|
958
|
-
false,
|
|
959
|
-
);
|
|
960
|
-
}
|
|
961
|
-
// A channel rejection (`11128`) is the same shape: the upstream is refusing
|
|
962
|
-
// the caller for this account, not this particular prompt, and it keeps
|
|
963
|
-
// refusing on every invocation. Classifying it as `unknown` with
|
|
964
|
-
// `mutates=false` would leave the account in rotation failing every request —
|
|
965
|
-
// and since every account fails the same way, the whole pool looks dead with
|
|
966
|
-
// nothing ever parked.
|
|
967
|
-
const channelRejection =
|
|
968
|
-
providerCode === "11128" || lower.includes("unapproved channel");
|
|
969
|
-
const channelBlock =
|
|
970
|
-
channelRejection &&
|
|
971
|
-
options?.providerId !== undefined &&
|
|
972
|
-
BUDDY_PROVIDER_IDS.has(options.providerId.toLowerCase());
|
|
973
|
-
if (channelBlock) {
|
|
974
|
-
// Same split as the policy arm above: a ban disables, a channel refusal
|
|
975
|
-
// only stops this request. `11-128` is the request shape, so this branch
|
|
976
|
-
// nearly always takes the refusal path — but it shares the ban test so a
|
|
977
|
-
// `11140`-shaped body arriving as a channel rejection still disables.
|
|
978
|
-
if (buddyBanned) {
|
|
979
|
-
return result(
|
|
980
|
-
"policy_blocked_banned",
|
|
981
|
-
"disabled",
|
|
982
|
-
0,
|
|
983
|
-
null,
|
|
984
|
-
reason("Account banned by upstream"),
|
|
985
|
-
true,
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
return result(
|
|
989
|
-
"policy_blocked_invalid_request",
|
|
990
|
-
"cooldown",
|
|
991
|
-
0,
|
|
992
|
-
null,
|
|
993
|
-
reason("Provider channel rejection"),
|
|
994
|
-
// The credential is valid; the upstream refuses this calling channel.
|
|
995
|
-
// Cooling was the bug: it kept a refused account in the plan, where a
|
|
996
|
-
// client's retry budget burned healthy siblings for the whole window.
|
|
997
|
-
false,
|
|
998
|
-
);
|
|
999
|
-
}
|
|
1000
|
-
// server-side tool (web search, x search, web fetch), not evidence about the
|
|
1001
|
-
// credential or the account's quota. Some providers surface it as 403, which
|
|
1002
|
-
// would otherwise look like an auth signal — refreshing the token cannot fix
|
|
1003
|
-
// a search backend, so it must never disable the account.
|
|
1004
|
-
const hostedToolFailure =
|
|
1005
|
-
lower.includes("web_search") ||
|
|
1006
|
-
lower.includes("web search") ||
|
|
1007
|
-
lower.includes("web_search_preview") ||
|
|
1008
|
-
lower.includes("x_search") ||
|
|
1009
|
-
lower.includes("web_fetch") ||
|
|
1010
|
-
lower.includes("tool invocation") ||
|
|
1011
|
-
lower.includes("tool call failed") ||
|
|
1012
|
-
lower.includes("search backend");
|
|
1013
|
-
// A model-scoped license or entitlement rejection is NOT credential invalidation.
|
|
1014
|
-
// Google Cloud Code / Antigravity answers HTTP 403 with "You do not have a valid
|
|
1015
|
-
// license of this product" when an account lacks entitlement for one specific model
|
|
1016
|
-
// (e.g. claude-opus-4-6), while the account itself and its tokens remain fully valid
|
|
1017
|
-
// for all other models. Disabling the whole account breaks healthy models.
|
|
1018
|
-
const modelEntitlementFailure =
|
|
1019
|
-
statusCode === 403 &&
|
|
1020
|
-
(lower.includes("not have a valid license") ||
|
|
1021
|
-
lower.includes("valid license of this product") ||
|
|
1022
|
-
lower.includes("model not supported") ||
|
|
1023
|
-
lower.includes("not entitled to") ||
|
|
1024
|
-
lower.includes("no access to model") ||
|
|
1025
|
-
lower.includes("model_not_supported") ||
|
|
1026
|
-
lower.includes("not available in your region"));
|
|
1027
|
-
/**
|
|
1028
|
-
* The upstream accepted the request and put it in a queue behind other
|
|
1029
|
-
* callers — it is busy, not rejecting the caller's credential.
|
|
1030
|
-
*
|
|
1031
|
-
* Qoder answers `403 {"code":"10605","message":"{\"isQueued\":true,
|
|
1032
|
-
* \"modelKey\":\"dfmodel\",\"queueCount\":1395,...}"}`. The 403 is what made
|
|
1033
|
-
* this misclassify: the generic `(401 || 403) && accountEvidence` arm claimed
|
|
1034
|
-
* it as `auth_invalidated`, so a healthy account was parked as "Re-login
|
|
1035
|
-
* required" purely because the provider was busy, and an operator was sent
|
|
1036
|
-
* to re-authenticate a credential that was never rejected.
|
|
1037
|
-
*
|
|
1038
|
-
* Matched on the queue evidence rather than on the bare `10605`, so an
|
|
1039
|
-
* unrelated 403 that happens to share the code is not swept in. The bodies
|
|
1040
|
-
* are nested JSON, so the tell is the key, not a parse of the envelope.
|
|
1041
|
-
*/
|
|
1042
|
-
const queuedUpstream =
|
|
1043
|
-
providerCode === "10605" ||
|
|
1044
|
-
lower.includes("isqueued") ||
|
|
1045
|
-
lower.includes("queuecount") ||
|
|
1046
|
-
lower.includes("queuetype") ||
|
|
1047
|
-
(lower.includes("queue") && lower.includes("modelkey"));
|
|
1048
|
-
|
|
1049
|
-
const signals: AccountErrorSignals = {
|
|
1050
|
-
message,
|
|
1051
|
-
lower,
|
|
1052
|
-
folded,
|
|
1053
|
-
statusCode,
|
|
1054
|
-
origin,
|
|
1055
|
-
scope,
|
|
1056
|
-
providerCode,
|
|
1057
|
-
statedCooldownMs,
|
|
1058
|
-
reason,
|
|
1059
|
-
result,
|
|
1060
|
-
canMutate,
|
|
1061
|
-
accountEvidence,
|
|
1062
|
-
revocationNamed,
|
|
1063
|
-
modelId,
|
|
1064
|
-
modelBlockHits,
|
|
1065
|
-
staticToken,
|
|
1066
|
-
credentialKind,
|
|
1067
|
-
buddy,
|
|
1068
|
-
buddyBanned,
|
|
1069
|
-
priceRefusal,
|
|
1070
|
-
deploymentInfraError,
|
|
1071
|
-
quotaSignal,
|
|
1072
|
-
policyRejection,
|
|
1073
|
-
requestScoped,
|
|
1074
|
-
channelRejection,
|
|
1075
|
-
hostedToolFailure,
|
|
1076
|
-
modelEntitlementFailure,
|
|
1077
|
-
queuedUpstream,
|
|
1078
|
-
};
|
|
1079
|
-
|
|
1080
|
-
for (const rule of ACCOUNT_ERROR_RULES) {
|
|
1081
|
-
if (rule.match(signals)) return rule.decide(signals);
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
// Same reasoning as above, with the shortest backoff: an unmapped failure
|
|
1085
|
-
// must still be retried automatically rather than parked forever.
|
|
1086
|
-
const unclassifiedMs = UNCLASSIFIED_COOLDOWN_MS();
|
|
1087
|
-
return result(
|
|
1088
|
-
"unknown",
|
|
1089
|
-
"cooldown",
|
|
1090
|
-
unclassifiedMs,
|
|
1091
|
-
new Date(Date.now() + unclassifiedMs),
|
|
1092
|
-
reason("Unclassified provider failure"),
|
|
1093
|
-
false,
|
|
1094
|
-
);
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
/**
|
|
1098
|
-
* The ordered arms of {@link classifyAccountError}, tried top to bottom; the
|
|
1099
|
-
* first match decides. Order is behavior, not presentation — see
|
|
1100
|
-
* {@link AccountErrorRule}.
|
|
1101
|
-
*/
|
|
1102
|
-
const ACCOUNT_ERROR_RULES: readonly AccountErrorRule[] = [
|
|
1103
|
-
{
|
|
1104
|
-
// Must precede "rate-limit": this rejection is a 429 whose own text says
|
|
1105
|
-
// "rate limit", so the broader arm claimed it and cooled the account for
|
|
1106
|
-
// the full 15-minute rate-limit window.
|
|
1107
|
-
//
|
|
1108
|
-
// A full max-inflight is a statement about the gateway's own routing, not
|
|
1109
|
-
// about the credential: every account serving the model is busy right now.
|
|
1110
|
-
// Cooling them marks healthy accounts as `cooldown` for a limit their
|
|
1111
|
-
// provider never applied, which is what made every account read busy while
|
|
1112
|
-
// incoming requests piled up — the accounts were never the problem, the
|
|
1113
|
-
// ceiling was.
|
|
1114
|
-
name: "gateway-capacity-exhausted",
|
|
1115
|
-
match: (s) =>
|
|
1116
|
-
s.lower.includes("at its concurrency limit") ||
|
|
1117
|
-
s.lower.includes("every proxy pool for this provider is at its concurrency limit") ||
|
|
1118
|
-
s.lower.includes("the gateway is at its concurrency limit"),
|
|
1119
|
-
decide: (s) =>
|
|
1120
|
-
s.result(
|
|
1121
|
-
"model_capacity",
|
|
1122
|
-
// `status` is the status the account would move to; it is never
|
|
1123
|
-
// applied because `mutatesAccount` is false below, so this names the
|
|
1124
|
-
// shape rather than a transition. Zero cooldown is the point: no wait
|
|
1125
|
-
// is attached to the account, so the next request may use it at once.
|
|
1126
|
-
"cooldown",
|
|
1127
|
-
0,
|
|
1128
|
-
null,
|
|
1129
|
-
s.reason("Gateway concurrency ceiling reached; no account was at fault"),
|
|
1130
|
-
// Never mutates: there is nothing wrong with the account to record.
|
|
1131
|
-
false,
|
|
1132
|
-
),
|
|
1133
|
-
},
|
|
1134
|
-
{
|
|
1135
|
-
// Must precede the unclassified fallback: the fallback is account-blind and
|
|
1136
|
-
// would stamp this as a provider-wide failure, which is how one proxy
|
|
1137
|
-
// refusing CONNECT put every account of every provider behind that pool
|
|
1138
|
-
// into cooldown — opencode's dead front door taking codebuddy down with it.
|
|
1139
|
-
//
|
|
1140
|
-
// A CONNECT refusal is a statement about the proxy pool, not about a
|
|
1141
|
-
// credential: the tunnel was never established, so no upstream ever saw the
|
|
1142
|
-
// request and no account can be at fault. Scoped to the pool so the
|
|
1143
|
-
// provider's own accounts stay in rotation.
|
|
1144
|
-
name: "proxy-connect-refusal",
|
|
1145
|
-
match: (s) =>
|
|
1146
|
-
s.lower.includes("proxy connect failed") ||
|
|
1147
|
-
(s.lower.includes("connect") && s.lower.includes("tunnel") && s.lower.includes("refus")),
|
|
1148
|
-
decide: (s) =>
|
|
1149
|
-
s.result(
|
|
1150
|
-
"model_unreachable",
|
|
1151
|
-
"cooldown",
|
|
1152
|
-
0,
|
|
1153
|
-
null,
|
|
1154
|
-
s.reason("Proxy pool refused the CONNECT tunnel"),
|
|
1155
|
-
// Never mutates: the pool owns this failure, not the account.
|
|
1156
|
-
false,
|
|
1157
|
-
),
|
|
1158
|
-
},
|
|
1159
|
-
{
|
|
1160
|
-
name: "deployment-infrastructure",
|
|
1161
|
-
match: (s) => s.deploymentInfraError,
|
|
1162
|
-
decide: (s) =>
|
|
1163
|
-
s.result(
|
|
1164
|
-
"server_error",
|
|
1165
|
-
"cooldown",
|
|
1166
|
-
60_000,
|
|
1167
|
-
null,
|
|
1168
|
-
`Proxy deployment error: ${s.message.slice(0, 160)}`,
|
|
1169
|
-
false,
|
|
1170
|
-
),
|
|
1171
|
-
},
|
|
1172
|
-
{
|
|
1173
|
-
name: "model-frequency-window",
|
|
1174
|
-
match: (s) => s.lower.includes("frequency limit") && s.lower.includes("switch to the other models"),
|
|
1175
|
-
decide: (s) => {
|
|
1176
|
-
const cooldownMs = s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS();
|
|
1177
|
-
return s.result(
|
|
1178
|
-
"rate_limit_transient",
|
|
1179
|
-
"cooldown",
|
|
1180
|
-
cooldownMs,
|
|
1181
|
-
new Date(Date.now() + cooldownMs),
|
|
1182
|
-
s.reason("Provider frequency limit for this model"),
|
|
1183
|
-
// Provider-scoped evidence still has to reach the writer: a stream frame
|
|
1184
|
-
// carries no credential detail, so `canMutate` is false and without this
|
|
1185
|
-
// the pair would be retried for the whole window it was closed for.
|
|
1186
|
-
s.origin === "upstream" && s.scope === "provider",
|
|
1187
|
-
);
|
|
1188
|
-
},
|
|
1189
|
-
},
|
|
1190
|
-
{
|
|
1191
|
-
name: "content-policy-block",
|
|
1192
|
-
match: (s) => s.policyRejection && s.buddy,
|
|
1193
|
-
decide: (s) => {
|
|
1194
|
-
// A ban is a verdict on the credential, and cooldown cannot answer it:
|
|
1195
|
-
// the account is refused, not throttled. Disabling is the fix, and
|
|
1196
|
-
// disabling *immediately* is a feature — a client's retry budget is spent
|
|
1197
|
-
// in seconds, so a banned account that merely cooled stayed in the plan
|
|
1198
|
-
// and burned sibling accounts that were still healthy before the window
|
|
1199
|
-
// expired.
|
|
1200
|
-
//
|
|
1201
|
-
// The two shapes are told apart by the body, not the code alone: `11140`
|
|
1202
|
-
// names a restriction on the account, `11-128` refuses the request.
|
|
1203
|
-
if (s.buddyBanned) {
|
|
1204
|
-
return s.result(
|
|
1205
|
-
"policy_blocked_banned",
|
|
1206
|
-
"disabled",
|
|
1207
|
-
0,
|
|
1208
|
-
null,
|
|
1209
|
-
s.reason("Account banned by upstream"),
|
|
1210
|
-
// A ban is evidence about the credential even at provider scope, for
|
|
1211
|
-
// the same reason a named revocation is: the message is the evidence.
|
|
1212
|
-
// Without this a stream frame's missing credential detail would leave
|
|
1213
|
-
// a banned account serving.
|
|
1214
|
-
true,
|
|
1215
|
-
);
|
|
1216
|
-
}
|
|
1217
|
-
// Not a ban: the upstream refused this request. The account is fine, so
|
|
1218
|
-
// the only correct move is to stop retrying it verbatim. Cooling the
|
|
1219
|
-
// account here was the bug that made one malformed request look like a
|
|
1220
|
-
// dead credential.
|
|
1221
|
-
return s.result(
|
|
1222
|
-
"policy_blocked_invalid_request",
|
|
1223
|
-
"cooldown",
|
|
1224
|
-
0,
|
|
1225
|
-
null,
|
|
1226
|
-
s.reason("Upstream refused this request"),
|
|
1227
|
-
// Never mutates: a malformed body fails identically on every account,
|
|
1228
|
-
// so cooling would burn a healthy credential on the gateway's own fault.
|
|
1229
|
-
false,
|
|
1230
|
-
);
|
|
1231
|
-
},
|
|
1232
|
-
},
|
|
1233
|
-
{
|
|
1234
|
-
name: "request-scoped-rejection",
|
|
1235
|
-
match: (s) => s.requestScoped && s.buddy,
|
|
1236
|
-
decide: (s) =>
|
|
1237
|
-
s.result("unknown", "cooldown", 0, null, s.reason("Request-scoped upstream rejection"), false),
|
|
1238
|
-
},
|
|
1239
|
-
{
|
|
1240
|
-
name: "channel-rejection",
|
|
1241
|
-
match: (s) => s.channelRejection && s.buddy,
|
|
1242
|
-
decide: (s) => {
|
|
1243
|
-
const channelBlockMs = resolveAccountPolicyBlockCooldownMs();
|
|
1244
|
-
return s.result(
|
|
1245
|
-
"policy_blocked",
|
|
1246
|
-
"cooldown",
|
|
1247
|
-
channelBlockMs,
|
|
1248
|
-
new Date(Date.now() + channelBlockMs),
|
|
1249
|
-
s.reason("Provider channel rejection"),
|
|
1250
|
-
s.origin === "upstream",
|
|
1251
|
-
);
|
|
1252
|
-
},
|
|
1253
|
-
},
|
|
1254
|
-
{
|
|
1255
|
-
name: "credential-rejected",
|
|
1256
|
-
match: (s) =>
|
|
1257
|
-
!s.quotaSignal &&
|
|
1258
|
-
!s.policyRejection &&
|
|
1259
|
-
!s.hostedToolFailure &&
|
|
1260
|
-
!s.modelEntitlementFailure &&
|
|
1261
|
-
!s.queuedUpstream &&
|
|
1262
|
-
(s.providerCode === "authentication_failed" ||
|
|
1263
|
-
s.folded.includes("invalid api key") ||
|
|
1264
|
-
s.folded.includes("incorrect api key") ||
|
|
1265
|
-
s.folded.includes("invalid token") ||
|
|
1266
|
-
s.folded.includes("token revoked") ||
|
|
1267
|
-
s.folded.includes("invalidated oauth token") ||
|
|
1268
|
-
s.folded.includes("reauthorization required") ||
|
|
1269
|
-
s.folded.includes("account deactivated") ||
|
|
1270
|
-
((s.statusCode === 401 || s.statusCode === 403) && s.accountEvidence)),
|
|
1271
|
-
decide: (s) => {
|
|
1272
|
-
// A static token has no refresh to run, so an auth rejection must not park
|
|
1273
|
-
// the account: disabling it throws away a token that may still work (a
|
|
1274
|
-
// single upstream 401 can be transient). A cooldown lets the next attempt
|
|
1275
|
-
// retry the same token without losing the account.
|
|
1276
|
-
const oauthRecovery =
|
|
1277
|
-
!s.staticToken &&
|
|
1278
|
-
s.credentialKind === "oauth" &&
|
|
1279
|
-
(s.lower.includes("expired") || s.lower.includes("refresh"));
|
|
1280
|
-
// A *named* revocation is the one auth outcome that still parks the account.
|
|
1281
|
-
// The upstream said the credential itself is dead — not that it expired, not
|
|
1282
|
-
// that this request was refused — so there is no window to wait out and no
|
|
1283
|
-
// refresh to run, and a cooldown would only return it to rotation to fail
|
|
1284
|
-
// again. Every other auth rejection is weak evidence (clock skew, a key
|
|
1285
|
-
// rotated a moment ago, an edge answering before the credential loads) and
|
|
1286
|
-
// cools down instead.
|
|
1287
|
-
//
|
|
1288
|
-
// This distinction is the whole fix. Reaching `disabled` from the generic
|
|
1289
|
-
// `401`/`403` branch would be unrecoverable by any sweep, so one transient
|
|
1290
|
-
// rejection would remove the account from rotation until an operator
|
|
1291
|
-
// noticed — the exact outcome the comment above says a static token must
|
|
1292
|
-
// not suffer.
|
|
1293
|
-
const revoked = s.revocationNamed;
|
|
1294
|
-
const recoverable = s.staticToken || oauthRecovery || !revoked;
|
|
1295
|
-
// An automatic disable is **not** terminal any more, but the deadline is what
|
|
1296
|
-
// makes that safe: `sweepExpiredCooldowns` recovers a disabled row only when
|
|
1297
|
-
// its `cooldownUntil` has elapsed, and an operator-disabled row keeps that
|
|
1298
|
-
// column null. So the machine can undo its own verdict while an operator's
|
|
1299
|
-
// decision still outranks it — see `resolveAccountDisabledRecoveryMs`.
|
|
1300
|
-
const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
|
|
1301
|
-
const disabledRetryAt =
|
|
1302
|
-
disabledRecoveryMs > 0 ? new Date(Date.now() + disabledRecoveryMs) : null;
|
|
1303
|
-
const cooldownMs = recoverable ? resolveAccountAuthRejectedCooldownMs() : disabledRecoveryMs;
|
|
1304
|
-
return s.result(
|
|
1305
|
-
"auth_invalidated",
|
|
1306
|
-
recoverable ? "cooldown" : "disabled",
|
|
1307
|
-
cooldownMs,
|
|
1308
|
-
recoverable ? new Date(Date.now() + cooldownMs) : disabledRetryAt,
|
|
1309
|
-
s.reason(
|
|
1310
|
-
!revoked
|
|
1311
|
-
? "Provider credential rejected; retrying after a backoff"
|
|
1312
|
-
: s.staticToken
|
|
1313
|
-
? "Static token rejected; retrying without a refresh"
|
|
1314
|
-
: oauthRecovery
|
|
1315
|
-
? "OAuth credential requires refresh"
|
|
1316
|
-
: "Provider credential revoked",
|
|
1317
|
-
),
|
|
1318
|
-
s.canMutate,
|
|
1319
|
-
);
|
|
1320
|
-
},
|
|
1321
|
-
},
|
|
1322
|
-
{
|
|
1323
|
-
name: "price-refusal",
|
|
1324
|
-
match: (s) => s.priceRefusal,
|
|
1325
|
-
decide: (s) =>
|
|
1326
|
-
s.result("quota_exhausted", "cooldown", 0, null, s.reason("Request price refused by the provider"), false),
|
|
1327
|
-
},
|
|
1328
|
-
{
|
|
1329
|
-
name: "model-plan-gated",
|
|
1330
|
-
match: (s) => s.quotaSignal && s.providerCode === "112",
|
|
1331
|
-
decide: (s) => {
|
|
1332
|
-
const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
|
|
1333
|
-
return s.result(
|
|
1334
|
-
"model_unreachable",
|
|
1335
|
-
"cooldown",
|
|
1336
|
-
cooldownMs,
|
|
1337
|
-
new Date(Date.now() + cooldownMs),
|
|
1338
|
-
s.reason("Model not included in this account's plan"),
|
|
1339
|
-
s.origin === "upstream" && s.modelId !== undefined,
|
|
1340
|
-
true,
|
|
1341
|
-
);
|
|
1342
|
-
},
|
|
1343
|
-
},
|
|
1344
|
-
{
|
|
1345
|
-
name: "quota-exhausted",
|
|
1346
|
-
match: (s) => s.quotaSignal,
|
|
1347
|
-
decide: (s) => {
|
|
1348
|
-
// xAI Grok Build's free tier resets on a rolling window. Use a duration
|
|
1349
|
-
// stated by the provider when present; otherwise park for the conservative
|
|
1350
|
-
// 24-hour fallback.
|
|
1351
|
-
const grokFreeTier =
|
|
1352
|
-
s.providerCode === "subscription:free-usage-exhausted" ||
|
|
1353
|
-
s.providerCode.startsWith("subscription:") ||
|
|
1354
|
-
s.lower.includes("free-usage") ||
|
|
1355
|
-
s.lower.includes("free usage") ||
|
|
1356
|
-
s.lower.includes("included free usage") ||
|
|
1357
|
-
s.lower.includes("rolling 24-hour") ||
|
|
1358
|
-
s.lower.includes("rolling 24 hour");
|
|
1359
|
-
const buddyCreditsExhausted =
|
|
1360
|
-
s.providerCode === "14018" || s.lower.includes("credits exhausted");
|
|
1361
|
-
const cooldownMs = grokFreeTier
|
|
1362
|
-
? (s.statedCooldownMs ?? GROK_QUOTA_COOLDOWN_MS)
|
|
1363
|
-
: buddyCreditsExhausted
|
|
1364
|
-
? (s.statedCooldownMs ?? 12 * 60 * 60 * 1000)
|
|
1365
|
-
: (s.statedCooldownMs ?? DEFAULT_QUOTA_COOLDOWN_MS());
|
|
1366
|
-
// The caller records against the concrete account that just failed
|
|
1367
|
-
// upstream, so a provider-scoped quota error still cools THAT account
|
|
1368
|
-
// down (enables fallback to the next account). Local-origin quota must
|
|
1369
|
-
// never mutate an upstream account.
|
|
1370
|
-
return s.result(
|
|
1371
|
-
"quota_exhausted",
|
|
1372
|
-
"cooldown",
|
|
1373
|
-
cooldownMs,
|
|
1374
|
-
new Date(Date.now() + cooldownMs),
|
|
1375
|
-
s.reason("Provider quota exhausted"),
|
|
1376
|
-
s.origin === "upstream",
|
|
1377
|
-
);
|
|
1378
|
-
},
|
|
1379
|
-
},
|
|
1380
|
-
{
|
|
1381
|
-
name: "rate-limit",
|
|
1382
|
-
match: (s) =>
|
|
1383
|
-
s.statusCode === 429 ||
|
|
1384
|
-
s.providerCode === "rate_limit_exceeded" ||
|
|
1385
|
-
s.lower.includes("rate_limit") ||
|
|
1386
|
-
s.lower.includes("rate limit") ||
|
|
1387
|
-
s.lower.includes("too many requests") ||
|
|
1388
|
-
s.lower.includes("requests per minute") ||
|
|
1389
|
-
s.lower.includes("tokens per minute") ||
|
|
1390
|
-
s.lower.includes("tpm limit") ||
|
|
1391
|
-
s.lower.includes("rpm limit"),
|
|
1392
|
-
decide: (s) => {
|
|
1393
|
-
const cooldownMs = s.statedCooldownMs ?? DEFAULT_RATE_LIMIT_COOLDOWN_MS();
|
|
1394
|
-
// Same per-account reasoning as quota above: the failing account (or
|
|
1395
|
-
// its model, when modelId is present) backs off so routing tries the
|
|
1396
|
-
// next candidate instead of hammering the same key.
|
|
1397
|
-
return s.result(
|
|
1398
|
-
"rate_limit_transient",
|
|
1399
|
-
"cooldown",
|
|
1400
|
-
cooldownMs,
|
|
1401
|
-
new Date(Date.now() + cooldownMs),
|
|
1402
|
-
s.reason("Provider rate limit"),
|
|
1403
|
-
s.origin === "upstream",
|
|
1404
|
-
);
|
|
1405
|
-
},
|
|
1406
|
-
},
|
|
1407
|
-
{
|
|
1408
|
-
name: "model-capacity",
|
|
1409
|
-
match: (s) =>
|
|
1410
|
-
s.statusCode === 503 ||
|
|
1411
|
-
s.statusCode === 529 ||
|
|
1412
|
-
s.providerCode === "model_capacity" ||
|
|
1413
|
-
s.queuedUpstream ||
|
|
1414
|
-
s.lower.includes("model overloaded") ||
|
|
1415
|
-
s.lower.includes("capacity exceeded") ||
|
|
1416
|
-
s.lower.includes("temporarily unavailable"),
|
|
1417
|
-
decide: (s) => {
|
|
1418
|
-
const cooldownMs = s.statedCooldownMs ?? MODEL_CAPACITY_COOLDOWN_MS();
|
|
1419
|
-
const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
|
|
1420
|
-
return s.result(
|
|
1421
|
-
"model_capacity",
|
|
1422
|
-
"cooldown",
|
|
1423
|
-
cooldownMs,
|
|
1424
|
-
new Date(Date.now() + cooldownMs),
|
|
1425
|
-
s.reason("Provider model capacity"),
|
|
1426
|
-
modelScopedMutation,
|
|
1427
|
-
);
|
|
1428
|
-
},
|
|
1429
|
-
},
|
|
1430
|
-
{
|
|
1431
|
-
name: "model-unsupported",
|
|
1432
|
-
match: (s) => isUnsupportedModelEvidence(s.statusCode, s.providerCode, s.folded),
|
|
1433
|
-
decide: (s) => {
|
|
1434
|
-
const cooldownMs = unsupportedModelCooldownMs(s.modelBlockHits);
|
|
1435
|
-
return s.result(
|
|
1436
|
-
"model_unreachable",
|
|
1437
|
-
"cooldown",
|
|
1438
|
-
cooldownMs,
|
|
1439
|
-
new Date(Date.now() + cooldownMs),
|
|
1440
|
-
s.reason("Model not supported by this upstream"),
|
|
1441
|
-
s.origin === "upstream" && s.modelId !== undefined,
|
|
1442
|
-
true,
|
|
1443
|
-
);
|
|
1444
|
-
},
|
|
1445
|
-
},
|
|
1446
|
-
{
|
|
1447
|
-
name: "model-unreachable",
|
|
1448
|
-
match: (s) => s.providerCode === "not_found" || s.modelEntitlementFailure,
|
|
1449
|
-
decide: (s) => {
|
|
1450
|
-
const cooldownMs = MODEL_UNREACHABLE_COOLDOWN_MS;
|
|
1451
|
-
// Not credential evidence, so `classifyUpstreamFailure` scopes this to the
|
|
1452
|
-
// provider and the `scope === "account"` gate the throttles use would
|
|
1453
|
-
// forbid the mutation outright — leaving the probe reporting a failure that
|
|
1454
|
-
// cooled nothing. The verdict is still about one (account, model) pair:
|
|
1455
|
-
// `recordAccountFailure` writes `modelCooldowns[modelId]` and leaves the
|
|
1456
|
-
// account's own status untouched, so all that is required is a model to
|
|
1457
|
-
// write against.
|
|
1458
|
-
const modelScopedMutation = s.origin === "upstream" && s.modelId !== undefined;
|
|
1459
|
-
return s.result(
|
|
1460
|
-
"model_unreachable",
|
|
1461
|
-
"cooldown",
|
|
1462
|
-
cooldownMs,
|
|
1463
|
-
new Date(Date.now() + cooldownMs),
|
|
1464
|
-
s.reason("Model unreachable for this account"),
|
|
1465
|
-
modelScopedMutation,
|
|
1466
|
-
);
|
|
1467
|
-
},
|
|
1468
|
-
},
|
|
1469
|
-
{
|
|
1470
|
-
name: "transport-fault",
|
|
1471
|
-
match: (s) =>
|
|
1472
|
-
(s.statusCode !== null && s.statusCode >= 500 && s.statusCode <= 504) ||
|
|
1473
|
-
s.lower.includes("timeout") ||
|
|
1474
|
-
s.lower.includes("abort") ||
|
|
1475
|
-
s.lower.includes("econnrefused") ||
|
|
1476
|
-
s.lower.includes("fetch failed"),
|
|
1477
|
-
decide: (s) => {
|
|
1478
|
-
// A transport-shaped fault clears on its own, so it is a cooldown with a
|
|
1479
|
-
// deadline rather than a state needing an operator. The deadline matters:
|
|
1480
|
-
// `sweepExpiredCooldowns` selects on `cooldownUntil IS NOT NULL`, so a row
|
|
1481
|
-
// parked without one is never recovered and stays out of rotation.
|
|
1482
|
-
//
|
|
1483
|
-
// `canMutate` rather than a flat `false`: a 5xx whose failure the caller
|
|
1484
|
-
// scoped to the account (an edge/WAF challenge answered to one account's
|
|
1485
|
-
// requests while its siblings on the same egress served fine) must cool
|
|
1486
|
-
// that account down, or it stays `active` and is re-dialled for as long as
|
|
1487
|
-
// the block lasts. Provider-scoped 5xx does not: the upstream is
|
|
1488
|
-
// indiscriminately failing and no single account is at fault.
|
|
1489
|
-
const transientMs = TRANSIENT_ERROR_COOLDOWN_MS();
|
|
1490
|
-
return s.result(
|
|
1491
|
-
s.lower.includes("timeout") ? "timeout" : "server_error",
|
|
1492
|
-
"cooldown",
|
|
1493
|
-
transientMs,
|
|
1494
|
-
new Date(Date.now() + transientMs),
|
|
1495
|
-
s.reason(`Provider ${s.statusCode ?? "network"} failure`),
|
|
1496
|
-
s.canMutate,
|
|
1497
|
-
);
|
|
1498
|
-
},
|
|
1499
|
-
},
|
|
1500
|
-
];
|
|
1501
|
-
|
|
1502
|
-
/** Persists one classified failure to the account row and appends a health event. */
|
|
1503
|
-
export async function recordAccountFailure(
|
|
1504
|
-
db: CartethyiaDatabase,
|
|
1505
|
-
accountId: string,
|
|
1506
|
-
error: unknown,
|
|
1507
|
-
options?: AccountFailureEvidence,
|
|
1508
|
-
): Promise<AccountErrorClassification | null> {
|
|
1509
|
-
const classification = classifyAccountError(error, options);
|
|
1510
|
-
/**
|
|
1511
|
-
* Logged even when the classification does not mutate the account.
|
|
1512
|
-
*
|
|
1513
|
-
* `rate_limit_transient` seen on a stream frame carries no credential
|
|
1514
|
-
* detail, so it classifies at provider scope and `mutatesAccount` is false
|
|
1515
|
-
* — nothing is cooled and the account stays `active`. That part is correct:
|
|
1516
|
-
* the upstream's own remedy ("switch to the other models") only works if the
|
|
1517
|
-
* account keeps serving. But this returned without writing anything, so a
|
|
1518
|
-
* rate-limited request showed up in the request log while the account's
|
|
1519
|
-
* health log stayed empty — a symptom with no recorded cause.
|
|
1520
|
-
*
|
|
1521
|
-
* The row is the whole point of this branch: it records the failure without
|
|
1522
|
-
* moving the account. Both status sides are the classification's own, so the
|
|
1523
|
-
* dialog names the cause without claiming a transition that never happened.
|
|
1524
|
-
*/
|
|
1525
|
-
if (!classification.mutatesAccount) {
|
|
1526
|
-
void writeHealthEvent(db, {
|
|
1527
|
-
entityKind: "account",
|
|
1528
|
-
accountId,
|
|
1529
|
-
fromStatus: classification.status,
|
|
1530
|
-
toStatus: classification.status,
|
|
1531
|
-
reason: classification.reason,
|
|
1532
|
-
errorCategory: classification.category,
|
|
1533
|
-
modelId: options?.modelId ?? null,
|
|
1534
|
-
createdAt: new Date(),
|
|
1535
|
-
}).catch(() => undefined);
|
|
1536
|
-
return classification;
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
const persist = async (client: CartethyiaDatabase): Promise<AccountErrorClassification | null> => {
|
|
1540
|
-
// Projected, not a bare `select()`: this runs on *every* attempt outcome,
|
|
1541
|
-
// and a full row drags `credential_ciphertext` (the largest thing stored
|
|
1542
|
-
// per account) and `auth_state` over the wire to read a status and a
|
|
1543
|
-
// counter. `RouteCatalogRepository` already projects for the same reason.
|
|
1544
|
-
const accountRows = await client
|
|
1545
|
-
.select({
|
|
1546
|
-
status: providerAccounts.status,
|
|
1547
|
-
staticToken: providerAccounts.staticToken,
|
|
1548
|
-
consecutiveFailures: providerAccounts.consecutiveFailures,
|
|
1549
|
-
modelCooldowns: providerAccounts.modelCooldowns,
|
|
1550
|
-
// Read only for the unsupported-model ladder; NULL means "never
|
|
1551
|
-
// counted", which is the common case and must stay cheap.
|
|
1552
|
-
modelBlockHits: providerAccounts.modelBlockHits,
|
|
1553
|
-
cooldownUntil: providerAccounts.cooldownUntil,
|
|
1554
|
-
})
|
|
1555
|
-
.from(providerAccounts)
|
|
1556
|
-
.where(eq(providerAccounts.id, accountId))
|
|
1557
|
-
.limit(1);
|
|
1558
|
-
const account = accountRows[0];
|
|
1559
|
-
if (!account || account.status === "disabled") return null;
|
|
1560
|
-
|
|
1561
|
-
// Re-classify with the account's own static-token flag. The caller's
|
|
1562
|
-
// evidence is request-scoped and does not carry it, but an auth rejection
|
|
1563
|
-
// against a static token must cool the account down rather than park it —
|
|
1564
|
-
// there is no refresh to run, and the token may still be valid.
|
|
1565
|
-
const effective = account.staticToken
|
|
1566
|
-
? classifyAccountError(error, { ...options, staticToken: true })
|
|
1567
|
-
: classification;
|
|
1568
|
-
|
|
1569
|
-
const fromStatus = account.status;
|
|
1570
|
-
const now = new Date();
|
|
1571
|
-
|
|
1572
|
-
const modelId = options?.modelId;
|
|
1573
|
-
// Every category here is a verdict about one (account, model) pair rather
|
|
1574
|
-
// than about the credential, so all three back off that pair and leave the
|
|
1575
|
-
// account routable for every other model. `model_unreachable` joins the two
|
|
1576
|
-
// throttles on the same reasoning: an upstream that reports the model does
|
|
1577
|
-
// not exist for this account will report it again on the next request.
|
|
1578
|
-
const isThrottle =
|
|
1579
|
-
effective.category === "rate_limit_transient" ||
|
|
1580
|
-
effective.category === "model_capacity" ||
|
|
1581
|
-
effective.category === "model_unreachable";
|
|
1582
|
-
const shouldModelCooldown = modelId !== undefined && isThrottle;
|
|
1583
|
-
// The account-wide streak, escalated for a provider-stated window. Read once
|
|
1584
|
-
// so the model-scoped and account-wide paths below cannot disagree about it:
|
|
1585
|
-
// this is the count *before* this failure, which the UPDATE below increments.
|
|
1586
|
-
const streak = Math.max(0, account.consecutiveFailures ?? 0);
|
|
1587
|
-
const escalatedCooldownMs = escalateFallbackCooldown(
|
|
1588
|
-
effective.cooldownMs,
|
|
1589
|
-
effective.providerStated === true,
|
|
1590
|
-
streak,
|
|
1591
|
-
);
|
|
1592
|
-
|
|
1593
|
-
// The unsupported-model ladder, computed before the branch: the classifier
|
|
1594
|
-
// could only state the base window because the authoritative hit count
|
|
1595
|
-
// lives on the row, and it is read here. `hits` is the new total for this
|
|
1596
|
-
// pair, so the first report is 1 and gets the base window.
|
|
1597
|
-
const blockHits = (account.modelBlockHits as Record<string, number> | null) ?? {};
|
|
1598
|
-
const unsupportedHits =
|
|
1599
|
-
effective.modelUnsupported === true && modelId !== undefined
|
|
1600
|
-
? Math.max(1, (blockHits[modelId] ?? 0) + 1)
|
|
1601
|
-
: undefined;
|
|
1602
|
-
// An unsupported-model window is stated by *us*, not by the provider, so it
|
|
1603
|
-
// would otherwise be multiplied a second time by `escalateFallbackCooldown`
|
|
1604
|
-
// (which reads `consecutiveFailures`, an account-wide counter). The two
|
|
1605
|
-
// ladders are different things: one counts how many times this pair was
|
|
1606
|
-
// reported unsupported, the other how many failures this account produced.
|
|
1607
|
-
const unsupportedCooldownMs =
|
|
1608
|
-
unsupportedHits === undefined ? undefined : unsupportedModelCooldownMs(unsupportedHits);
|
|
1609
|
-
|
|
1610
|
-
if (shouldModelCooldown) {
|
|
1611
|
-
const existing = (account.modelCooldowns as Record<string, string> | null) ?? {};
|
|
1612
|
-
// Monotonic: never move an in-force window earlier (see `latestDeadlineMs`).
|
|
1613
|
-
// A provider-stated deadline wins outright because it is already the later
|
|
1614
|
-
// of the two whenever the provider really parked the pair longer.
|
|
1615
|
-
const nextUntil = new Date(
|
|
1616
|
-
latestDeadlineMs(
|
|
1617
|
-
effective.retryAt?.getTime(),
|
|
1618
|
-
Date.now() + (unsupportedCooldownMs ?? escalatedCooldownMs),
|
|
1619
|
-
existing[modelId] ? Date.parse(existing[modelId]!) : undefined,
|
|
1620
|
-
),
|
|
1621
|
-
).toISOString();
|
|
1622
|
-
const previousUntil = existing[modelId];
|
|
1623
|
-
// A re-statement is not an event: what the audit row reports is the
|
|
1624
|
-
// *entry* into a cooling episode, and the deadline itself is not a
|
|
1625
|
-
// column — so a row can only ever say "this pair started cooling for
|
|
1626
|
-
// this reason". The upstream answers the same "Try again in 11h" on
|
|
1627
|
-
// every retry (and an unstated reset is recomputed from now each time),
|
|
1628
|
-
// so writing per failure produced seven identical rows in twenty minutes
|
|
1629
|
-
// and read as an account flapping when nothing had changed. The entry is
|
|
1630
|
-
// still refreshed below — its error fields are what the dialog explains
|
|
1631
|
-
// the cooldown with — while an entry that is still in force suppresses
|
|
1632
|
-
// the duplicate row.
|
|
1633
|
-
const alreadyCooling =
|
|
1634
|
-
previousUntil !== undefined && new Date(previousUntil).getTime() > Date.now();
|
|
1635
|
-
// `status`/`cooldownUntil` stay untouched: only this (account, model)
|
|
1636
|
-
// pair backs off, and the account remains routable for every other model.
|
|
1637
|
-
// The error fields ARE written, because they are what provider detail and
|
|
1638
|
-
// the health dialog read to explain *why* a model is cooling — without
|
|
1639
|
-
// them the row showed "1 model cooling" with no reason at all.
|
|
1640
|
-
//
|
|
1641
|
-
// The streak advances here too. Without it the most common throttle — a
|
|
1642
|
-
// model-scoped rate limit — never escalated, because the counter was only
|
|
1643
|
-
// incremented on the account-wide path below, and `consecutiveFailures`
|
|
1644
|
-
// stays honest about what it always claimed to count: how many failures in
|
|
1645
|
-
// a row this account has just produced.
|
|
1646
|
-
await client
|
|
1647
|
-
.update(providerAccounts)
|
|
1648
|
-
.set({
|
|
1649
|
-
modelCooldowns: { ...existing, [modelId]: nextUntil },
|
|
1650
|
-
// Counted only for the unsupported-model ladder, and only when this
|
|
1651
|
-
// failure is that verdict — so a rate limit or a capacity refusal
|
|
1652
|
-
// never advances it. `model_cooldowns`' own shape is untouched, so
|
|
1653
|
-
// routing and the console keep parsing it exactly as before.
|
|
1654
|
-
...(unsupportedHits === undefined
|
|
1655
|
-
? {}
|
|
1656
|
-
: { modelBlockHits: { ...blockHits, [modelId]: unsupportedHits } }),
|
|
1657
|
-
consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
|
|
1658
|
-
lastError: effective.reason,
|
|
1659
|
-
lastErrorCategory: effective.category,
|
|
1660
|
-
lastErrorAt: now,
|
|
1661
|
-
})
|
|
1662
|
-
.where(eq(providerAccounts.id, accountId));
|
|
1663
|
-
if (!alreadyCooling) {
|
|
1664
|
-
await writeHealthEvent(client, {
|
|
1665
|
-
entityKind: "account",
|
|
1666
|
-
accountId,
|
|
1667
|
-
// The account's own status, on both sides: a model-scoped throttle
|
|
1668
|
-
// never moved it, and writing the classification's status here
|
|
1669
|
-
// claimed an `active → cooldown` transition that never happened — the
|
|
1670
|
-
// dialog showed "Account status: ACTIVE" above a list of transitions
|
|
1671
|
-
// to `cooldown`. The row still names the model and the reason, which
|
|
1672
|
-
// is what makes it worth keeping.
|
|
1673
|
-
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
1674
|
-
toStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
1675
|
-
reason: effective.reason,
|
|
1676
|
-
errorCategory: effective.category,
|
|
1677
|
-
modelId: modelId ?? null,
|
|
1678
|
-
createdAt: now,
|
|
1679
|
-
});
|
|
1680
|
-
}
|
|
1681
|
-
// `status` and `cooldownUntil` are untouched on this path — only the
|
|
1682
|
-
// (account, model) pair backed off — so the snapshot is still accurate
|
|
1683
|
-
// and must not be dropped.
|
|
1684
|
-
return { ...classification, routesVisible: false };
|
|
1685
|
-
}
|
|
1686
|
-
|
|
1687
|
-
await client
|
|
1688
|
-
.update(providerAccounts)
|
|
1689
|
-
.set({
|
|
1690
|
-
status: effective.status,
|
|
1691
|
-
consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
|
|
1692
|
-
lastError: effective.reason,
|
|
1693
|
-
lastErrorCategory: effective.category,
|
|
1694
|
-
lastErrorAt: now,
|
|
1695
|
-
// Monotonic against the account's own in-force deadline, for the same
|
|
1696
|
-
// reason as the model path: an unclassified 60-second failure must not
|
|
1697
|
-
// release an account the provider parked for hours.
|
|
1698
|
-
//
|
|
1699
|
-
// A `0` duration contributes no candidate rather than stamping *now*
|
|
1700
|
-
// (see `accountDeadlineMs`): the one state that asks for it is a
|
|
1701
|
-
// terminal automatic disable, and an elapsed deadline there would hand
|
|
1702
|
-
// the row straight back to the recovery sweep.
|
|
1703
|
-
cooldownUntil: accountDeadlineMs([
|
|
1704
|
-
effective.retryAt?.getTime(),
|
|
1705
|
-
escalatedCooldownMs > 0 ? Date.now() + escalatedCooldownMs : undefined,
|
|
1706
|
-
account.cooldownUntil?.getTime?.(),
|
|
1707
|
-
]),
|
|
1708
|
-
})
|
|
1709
|
-
.where(eq(providerAccounts.id, accountId));
|
|
1710
|
-
|
|
1711
|
-
{
|
|
1712
|
-
await writeHealthEvent(client, {
|
|
1713
|
-
entityKind: "account",
|
|
1714
|
-
accountId,
|
|
1715
|
-
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
1716
|
-
toStatus: effective.status,
|
|
1717
|
-
reason: effective.reason,
|
|
1718
|
-
errorCategory: effective.category,
|
|
1719
|
-
modelId: modelId ?? null,
|
|
1720
|
-
createdAt: now,
|
|
1721
|
-
});
|
|
1722
|
-
}
|
|
1723
|
-
// This path wrote `status` and `cooldownUntil`, both of which the snapshot
|
|
1724
|
-
// projects, so routing has to see it.
|
|
1725
|
-
return { ...classification, routesVisible: true };
|
|
1726
|
-
};
|
|
1727
|
-
|
|
1728
|
-
try {
|
|
1729
|
-
// Kept as a capability probe, not a correctness guard: this function is
|
|
1730
|
-
// reachable with a partially-implemented database handle (the dispatch
|
|
1731
|
-
// tests pass a narrow double), and a transaction is an optimization here,
|
|
1732
|
-
// not a requirement — the same writes are correct without one.
|
|
1733
|
-
if (typeof db.transaction === "function") {
|
|
1734
|
-
return await db.transaction((tx) => persist(tx as CartethyiaDatabase));
|
|
1735
|
-
}
|
|
1736
|
-
return await persist(db);
|
|
1737
|
-
} catch {
|
|
1738
|
-
return null;
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
|
|
1742
|
-
/** Clears failure state after a successful dispatch; true when the account changed. */
|
|
1743
|
-
export async function recordAccountSuccess(
|
|
1744
|
-
db: CartethyiaDatabase,
|
|
1745
|
-
accountId: string,
|
|
1746
|
-
): Promise<boolean> {
|
|
1747
|
-
const mutate = async (client: CartethyiaDatabase): Promise<boolean> => {
|
|
1748
|
-
// `recordAccountSuccess` runs on every successful dispatch, and for a
|
|
1749
|
-
// healthy account `hadFailures` is false — the row was fetched purely to
|
|
1750
|
-
// decide nothing. Projected so that no-op path does not pull the
|
|
1751
|
-
// credential ciphertext across the wire.
|
|
1752
|
-
const rows = await client
|
|
1753
|
-
.select({
|
|
1754
|
-
status: providerAccounts.status,
|
|
1755
|
-
consecutiveFailures: providerAccounts.consecutiveFailures,
|
|
1756
|
-
// Read only to decide whether there is anything to clear: the common
|
|
1757
|
-
// case (no unsupported-model hit ever recorded) must not write.
|
|
1758
|
-
modelBlockHits: providerAccounts.modelBlockHits,
|
|
1759
|
-
})
|
|
1760
|
-
.from(providerAccounts)
|
|
1761
|
-
.where(eq(providerAccounts.id, accountId))
|
|
1762
|
-
.limit(1);
|
|
1763
|
-
const account = rows[0];
|
|
1764
|
-
if (!account || account.status === "disabled") return false;
|
|
1765
|
-
const fromStatus = account.status;
|
|
1766
|
-
const hadFailures = (account.consecutiveFailures ?? 0) > 0 || account.status !== "active";
|
|
1767
|
-
const now = new Date();
|
|
1768
|
-
// The unsupported-model escalation restarts after a success: a model that
|
|
1769
|
-
// just served is not "not supported", so the next report of it must start
|
|
1770
|
-
// from the base window again rather than from wherever the ladder had
|
|
1771
|
-
// reached. `modelCooldowns` is deliberately NOT cleared — an in-force
|
|
1772
|
-
// window the provider stated survives, which is the existing behavior and
|
|
1773
|
-
// the reference gateways' too.
|
|
1774
|
-
const blockHits = account.modelBlockHits as Record<string, number> | null;
|
|
1775
|
-
const hasBlockHits = blockHits !== null && Object.keys(blockHits).length > 0;
|
|
1776
|
-
await client
|
|
1777
|
-
.update(providerAccounts)
|
|
1778
|
-
.set({
|
|
1779
|
-
status: "active",
|
|
1780
|
-
consecutiveFailures: 0,
|
|
1781
|
-
cooldownUntil: null,
|
|
1782
|
-
lastSuccessAt: now,
|
|
1783
|
-
...(hadFailures ? { lastRecoveredAt: now } : {}),
|
|
1784
|
-
...(hasBlockHits ? { modelBlockHits: {} } : {}),
|
|
1785
|
-
})
|
|
1786
|
-
.where(eq(providerAccounts.id, accountId));
|
|
1787
|
-
if (hadFailures && typeof client.insert === "function") {
|
|
1788
|
-
await client.insert(healthEvents).values({
|
|
1789
|
-
entityKind: "account",
|
|
1790
|
-
accountId,
|
|
1791
|
-
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
1792
|
-
toStatus: "active",
|
|
1793
|
-
reason: "Request dispatched successfully",
|
|
1794
|
-
errorCategory: null,
|
|
1795
|
-
createdAt: now,
|
|
1796
|
-
});
|
|
1797
|
-
}
|
|
1798
|
-
return hadFailures;
|
|
1799
|
-
};
|
|
1800
|
-
try {
|
|
1801
|
-
if (typeof db.transaction === "function") {
|
|
1802
|
-
return await db.transaction((tx) => mutate(tx as CartethyiaDatabase));
|
|
1803
|
-
}
|
|
1804
|
-
return await mutate(db);
|
|
1805
|
-
} catch {
|
|
1806
|
-
return false;
|
|
1807
|
-
}
|
|
1808
|
-
}
|
|
1809
|
-
|
|
1810
|
-
/** Operator-initiated recovery: forces the account back to active and logs the transition. */
|
|
1811
|
-
export async function recoverAccount(
|
|
1812
|
-
db: CartethyiaDatabase,
|
|
1813
|
-
accountId: string,
|
|
1814
|
-
reason = "manual_operator_recovery",
|
|
1815
|
-
): Promise<boolean> {
|
|
1816
|
-
const recover = async (client: CartethyiaDatabase): Promise<boolean> => {
|
|
1817
|
-
const accountRows = await client
|
|
1818
|
-
.select()
|
|
1819
|
-
.from(providerAccounts)
|
|
1820
|
-
.where(eq(providerAccounts.id, accountId))
|
|
1821
|
-
.limit(1);
|
|
1822
|
-
const account = accountRows[0];
|
|
1823
|
-
if (!account) return false;
|
|
1824
|
-
const fromStatus = account.status;
|
|
1825
|
-
const now = new Date();
|
|
1826
|
-
await client
|
|
1827
|
-
.update(providerAccounts)
|
|
1828
|
-
.set({
|
|
1829
|
-
status: "active",
|
|
1830
|
-
consecutiveFailures: 0,
|
|
1831
|
-
cooldownUntil: null,
|
|
1832
|
-
lastRecoveredAt: now,
|
|
1833
|
-
// Recovery is the operator asserting the account works now, so it must
|
|
1834
|
-
// clear EVERY routing exclusion, not just the account-wide ones. A
|
|
1835
|
-
// per-model cooldown entry excludes the (account, model) pair in
|
|
1836
|
-
// `route-catalog.ts`, so leaving it behind kept the account blocked for
|
|
1837
|
-
// that model through `/v1` even though the account read `active` and a
|
|
1838
|
-
// direct probe (which addresses the account by id, bypassing routing
|
|
1839
|
-
// eligibility) succeeded. Clearing the error fields for the same reason:
|
|
1840
|
-
// a stale reason on a recovered row misreports the current state.
|
|
1841
|
-
modelCooldowns: {},
|
|
1842
|
-
lastError: null,
|
|
1843
|
-
lastErrorCategory: null,
|
|
1844
|
-
lastErrorAt: null,
|
|
1845
|
-
})
|
|
1846
|
-
.where(eq(providerAccounts.id, accountId));
|
|
1847
|
-
{
|
|
1848
|
-
await writeHealthEvent(client, {
|
|
1849
|
-
entityKind: "account",
|
|
1850
|
-
accountId,
|
|
1851
|
-
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
1852
|
-
toStatus: "active",
|
|
1853
|
-
reason,
|
|
1854
|
-
errorCategory: null,
|
|
1855
|
-
createdAt: now,
|
|
1856
|
-
});
|
|
1857
|
-
}
|
|
1858
|
-
return true;
|
|
1859
|
-
};
|
|
1860
|
-
try {
|
|
1861
|
-
if (typeof db.transaction === "function") {
|
|
1862
|
-
return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
|
|
1863
|
-
}
|
|
1864
|
-
return await recover(db);
|
|
1865
|
-
} catch {
|
|
1866
|
-
return false;
|
|
1867
|
-
}
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
/** Recovers accounts whose cooldown elapsed and prunes expired per-model keys. */
|
|
1871
|
-
export async function sweepExpiredCooldowns(db: CartethyiaDatabase): Promise<number> {
|
|
1872
|
-
const recover = async (client: CartethyiaDatabase): Promise<number> => {
|
|
1873
|
-
const now = new Date();
|
|
1874
|
-
const nowIso = now.toISOString();
|
|
1875
|
-
const expired = await client
|
|
1876
|
-
.select({
|
|
1877
|
-
id: providerAccounts.id,
|
|
1878
|
-
status: providerAccounts.status,
|
|
1879
|
-
})
|
|
1880
|
-
.from(providerAccounts)
|
|
1881
|
-
.where(
|
|
1882
|
-
or(
|
|
1883
|
-
// An ordinary cooldown, recovered as soon as its window elapses.
|
|
1884
|
-
and(
|
|
1885
|
-
eq(providerAccounts.status, "cooldown"),
|
|
1886
|
-
isNotNull(providerAccounts.cooldownUntil),
|
|
1887
|
-
lte(providerAccounts.cooldownUntil, now),
|
|
1888
|
-
),
|
|
1889
|
-
// An **automatic** disable. Selected only when it carries a deadline:
|
|
1890
|
-
// an operator's disable leaves
|
|
1891
|
-
// `cooldownUntil` null, so a human's decision still outranks the
|
|
1892
|
-
// machine and is never swept away. The deadline is written by the two
|
|
1893
|
-
// paths that disable on evidence — the classifier's revocation branch
|
|
1894
|
-
// and a definitively rejected refresh grant.
|
|
1895
|
-
and(
|
|
1896
|
-
eq(providerAccounts.status, "disabled"),
|
|
1897
|
-
isNotNull(providerAccounts.cooldownUntil),
|
|
1898
|
-
lte(providerAccounts.cooldownUntil, now),
|
|
1899
|
-
),
|
|
1900
|
-
),
|
|
1901
|
-
)
|
|
1902
|
-
// One pass recovers a bounded batch. The sweep runs every 30s, so a
|
|
1903
|
-
// backlog larger than this drains over successive ticks instead of one
|
|
1904
|
-
// statement covering every expired row — that statement holds its locks
|
|
1905
|
-
// for as long as the backlog takes.
|
|
1906
|
-
.limit(COOLDOWN_RECOVERY_BATCH);
|
|
1907
|
-
if (expired.length > 0) {
|
|
1908
|
-
await client
|
|
1909
|
-
.update(providerAccounts)
|
|
1910
|
-
.set({
|
|
1911
|
-
status: "active",
|
|
1912
|
-
consecutiveFailures: 0,
|
|
1913
|
-
cooldownUntil: null,
|
|
1914
|
-
lastRecoveredAt: now,
|
|
1915
|
-
// Cleared with the state it described. A recovered row that kept
|
|
1916
|
-
// `auth_invalidated` still rendered the console's "Re-login required"
|
|
1917
|
-
// pill above an `active` status, so the account read healthy to
|
|
1918
|
-
// routing and broken to the operator at the same time.
|
|
1919
|
-
lastErrorCategory: null,
|
|
1920
|
-
})
|
|
1921
|
-
.where(inArray(providerAccounts.id, expired.map((account) => account.id)));
|
|
1922
|
-
if (typeof client.insert === "function") {
|
|
1923
|
-
for (let offset = 0; offset < expired.length; offset += 500) {
|
|
1924
|
-
await client.insert(healthEvents).values(
|
|
1925
|
-
expired.slice(offset, offset + 500).map((account) => ({
|
|
1926
|
-
entityKind: "account" as const,
|
|
1927
|
-
accountId: account.id,
|
|
1928
|
-
fromStatus: account.status,
|
|
1929
|
-
toStatus: "active" as const,
|
|
1930
|
-
reason: "Cooldown period elapsed — auto-recovered",
|
|
1931
|
-
errorCategory: null,
|
|
1932
|
-
createdAt: now,
|
|
1933
|
-
})),
|
|
1934
|
-
);
|
|
1935
|
-
}
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
|
|
1939
|
-
// Prune per-model cooldowns: remove entries whose ISO timestamp has passed.
|
|
1940
|
-
//
|
|
1941
|
-
// One set-based UPDATE rather than a SELECT followed by an UPDATE per row.
|
|
1942
|
-
// The per-row form issued one statement per account holding any cooldown —
|
|
1943
|
-
// on a deployment where many accounts are throttled at once that is the
|
|
1944
|
-
// sweep's dominant cost, and it all runs inside this transaction, holding
|
|
1945
|
-
// its locks for the whole pass. Recomputing the object in SQL does the same
|
|
1946
|
-
// work in one statement.
|
|
1947
|
-
//
|
|
1948
|
-
// `jsonb_typeof(...) = 'object'` guards the shape: a malformed value must
|
|
1949
|
-
// not abort the whole sweep. `IS DISTINCT FROM` keeps the update from
|
|
1950
|
-
// rewriting rows it
|
|
1951
|
-
// did not change, so the returned row count is the number of accounts whose
|
|
1952
|
-
// cooldown set actually shrank — the same thing the loop counted.
|
|
1953
|
-
const pruning = pruneExpiredCooldowns(sql`${providerAccounts.modelCooldowns}`, sql`${nowIso}`);
|
|
1954
|
-
const pruned = await client
|
|
1955
|
-
.update(providerAccounts)
|
|
1956
|
-
.set({ modelCooldowns: pruning.value })
|
|
1957
|
-
.where(pruning.changed);
|
|
1958
|
-
return expired.length + affectedRows(pruned);
|
|
1959
|
-
};
|
|
1960
|
-
try {
|
|
1961
|
-
if (typeof db.transaction === "function") {
|
|
1962
|
-
return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
|
|
1963
|
-
}
|
|
1964
|
-
return await recover(db);
|
|
1965
|
-
} catch {
|
|
1966
|
-
return 0;
|
|
1967
|
-
}
|
|
1968
|
-
}
|
|
1969
|
-
|
|
1970
|
-
/**
|
|
1971
|
-
* Single attempt health reporter: one home for the success/failure writes
|
|
1972
|
-
* every dispatch attempt ends with. Success records recovery; failure
|
|
1973
|
-
* records with caller-supplied evidence (origin/scope classification plus
|
|
1974
|
-
* model id). A mutating write invalidates the route snapshot so the next
|
|
1975
|
-
* plan sees fresh health. Never throws — health must not fail requests.
|
|
1976
|
-
*/
|
|
1977
|
-
export interface AttemptHealthReport {
|
|
1978
|
-
readonly accountId: string | undefined;
|
|
1979
|
-
readonly modelId?: string;
|
|
1980
|
-
readonly error?: unknown;
|
|
1981
|
-
readonly evidence?: AccountFailureEvidence;
|
|
1982
|
-
readonly snapshotService?: { invalidate(): unknown };
|
|
1983
|
-
}
|
|
1984
|
-
|
|
1985
|
-
/**
|
|
1986
|
-
* Writes one health audit row, off the request path when a buffer is installed.
|
|
1987
|
-
*
|
|
1988
|
-
* `health_events` is append-only and read by nobody in the request path, so it
|
|
1989
|
-
* does not need to land before the request finishes — unlike the
|
|
1990
|
-
* `provider_accounts` UPDATE beside it, which routing reads and which therefore
|
|
1991
|
-
* stays synchronous and transactional.
|
|
1992
|
-
*/
|
|
1993
|
-
async function writeHealthEvent(
|
|
1994
|
-
client: CartethyiaDatabase,
|
|
1995
|
-
row: {
|
|
1996
|
-
readonly entityKind: "account";
|
|
1997
|
-
readonly accountId: string;
|
|
1998
|
-
readonly fromStatus: string;
|
|
1999
|
-
readonly toStatus: string;
|
|
2000
|
-
readonly reason: string;
|
|
2001
|
-
readonly errorCategory: string | null;
|
|
2002
|
-
readonly modelId?: string | null;
|
|
2003
|
-
readonly createdAt: Date;
|
|
2004
|
-
},
|
|
2005
|
-
): Promise<void> {
|
|
2006
|
-
const buffer = healthEventBuffer();
|
|
2007
|
-
if (buffer !== undefined) {
|
|
2008
|
-
buffer.enqueue({ ...row, modelId: row.modelId ?? null });
|
|
2009
|
-
return;
|
|
2010
|
-
}
|
|
2011
|
-
if (typeof client.insert !== "function") return;
|
|
2012
|
-
await client.insert(healthEvents).values(row as never);
|
|
2013
|
-
}
|
|
2014
|
-
|
|
2015
|
-
export async function reportAttemptOutcome(
|
|
2016
|
-
db: CartethyiaDatabase,
|
|
2017
|
-
report: AttemptHealthReport,
|
|
2018
|
-
): Promise<void> {
|
|
2019
|
-
if (!report.accountId) return;
|
|
2020
|
-
if (report.error === undefined) {
|
|
2021
|
-
const recovered = await recordAccountSuccess(db, report.accountId).catch(() => false);
|
|
2022
|
-
if (recovered) await report.snapshotService?.invalidate();
|
|
2023
|
-
return;
|
|
2024
|
-
}
|
|
2025
|
-
const classification = await recordAccountFailure(db, report.accountId, report.error, {
|
|
2026
|
-
...report.evidence,
|
|
2027
|
-
...(report.modelId === undefined ? {} : { modelId: report.modelId }),
|
|
2028
|
-
}).catch(() => null);
|
|
2029
|
-
// Gated on `routesVisible`, not on `mutatesAccount`: a failure that only
|
|
2030
|
-
// cooled one (account, model) pair, or only advanced the streak, changed
|
|
2031
|
-
// nothing routing reads. Invalidating for those rebuilt an 11-SELECT snapshot
|
|
2032
|
-
// on the next request, and under a failure wave each rebuild fed the next
|
|
2033
|
-
// invalidation.
|
|
2034
|
-
if (classification?.routesVisible) await report.snapshotService?.invalidate();
|
|
2035
|
-
}
|
|
2036
|
-
|
|
2037
|
-
export async function listAccountHealthEvents(
|
|
2038
|
-
db: CartethyiaDatabase,
|
|
2039
|
-
accountId: string,
|
|
2040
|
-
limit = 50,
|
|
2041
|
-
): Promise<readonly AccountHealthEventRecord[]> {
|
|
2042
|
-
try {
|
|
2043
|
-
const rows = await db
|
|
2044
|
-
.select()
|
|
2045
|
-
.from(healthEvents)
|
|
2046
|
-
.where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
|
|
2047
|
-
.orderBy(desc(healthEvents.createdAt))
|
|
2048
|
-
.limit(limit);
|
|
2049
|
-
|
|
2050
|
-
return rows.map((r) => ({
|
|
2051
|
-
id: r.id,
|
|
2052
|
-
accountId: r.accountId as string,
|
|
2053
|
-
fromStatus: r.fromStatus,
|
|
2054
|
-
toStatus: r.toStatus,
|
|
2055
|
-
modelId: r.modelId ?? null,
|
|
2056
|
-
reason: r.reason,
|
|
2057
|
-
errorCategory: r.errorCategory,
|
|
2058
|
-
createdAt: r.createdAt.toISOString(),
|
|
2059
|
-
}));
|
|
2060
|
-
} catch {
|
|
2061
|
-
return [];
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
|
|
2065
|
-
/** Every stored row across accounts and pools — the size the purge offers to delete. */
|
|
2066
|
-
export async function countAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
|
|
2067
|
-
try {
|
|
2068
|
-
const rows = await db.select({ total: count() }).from(healthEvents);
|
|
2069
|
-
return rows[0]?.total ?? 0;
|
|
2070
|
-
} catch {
|
|
2071
|
-
return 0;
|
|
2072
|
-
}
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
|
-
/**
|
|
2076
|
-
* Deletes stored health & error rows.
|
|
2077
|
-
*
|
|
2078
|
-
* Unlike the read above, a failed delete surfaces: an operator who purged
|
|
2079
|
-
* believes the trail is gone, so "it failed" must not read as "it succeeded
|
|
2080
|
-
* with zero rows". Callers own the flush-buffer barrier (`drain`) so a queued
|
|
2081
|
-
* write cannot land after these rows are gone.
|
|
2082
|
-
*/
|
|
2083
|
-
export async function deleteAccountHealthEvents(
|
|
2084
|
-
db: CartethyiaDatabase,
|
|
2085
|
-
accountId: string,
|
|
2086
|
-
): Promise<number> {
|
|
2087
|
-
const rows = await db
|
|
2088
|
-
.delete(healthEvents)
|
|
2089
|
-
.where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
|
|
2090
|
-
.returning({ id: healthEvents.id });
|
|
2091
|
-
return rows.length;
|
|
2092
|
-
}
|
|
2093
|
-
|
|
2094
|
-
export async function deleteAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
|
|
2095
|
-
const rows = await db.delete(healthEvents).returning({ id: healthEvents.id });
|
|
2096
|
-
return rows.length;
|
|
2097
|
-
}
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { healthEventBuffer } from "../../persistence/health-event-flush";
|
|
3
|
+
import { and, count, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
|
|
4
|
+
import { affectedRows, pruneExpiredCooldowns } from "../../persistence/dialect";
|
|
5
|
+
import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
6
|
+
import { healthEvents, providerAccounts } from "../../persistence/schema";
|
|
7
|
+
import {
|
|
8
|
+
COOLDOWN_RECOVERY_BATCH,
|
|
9
|
+
accountDeadlineMs,
|
|
10
|
+
classifyAccountError,
|
|
11
|
+
escalateFallbackCooldown,
|
|
12
|
+
latestDeadlineMs,
|
|
13
|
+
unsupportedModelCooldownMs,
|
|
14
|
+
} from "./account-error-classification";
|
|
15
|
+
import type { AccountErrorClassification, AccountFailureEvidence } from "./account-error-classification";
|
|
16
|
+
|
|
17
|
+
export { classifyAccountError, unsupportedModelCooldownMs } from "./account-error-classification";
|
|
18
|
+
export type {
|
|
19
|
+
AccountErrorCategory,
|
|
20
|
+
AccountErrorClassification,
|
|
21
|
+
AccountFailureEvidence,
|
|
22
|
+
AccountFailureOrigin,
|
|
23
|
+
AccountFailureScope,
|
|
24
|
+
} from "./account-error-classification";
|
|
25
|
+
|
|
26
|
+
export interface AccountHealthEventRecord {
|
|
27
|
+
readonly id: string;
|
|
28
|
+
readonly accountId: string;
|
|
29
|
+
readonly fromStatus: string | null;
|
|
30
|
+
readonly toStatus: string;
|
|
31
|
+
readonly reason: string | null;
|
|
32
|
+
readonly errorCategory: string | null;
|
|
33
|
+
readonly modelId: string | null;
|
|
34
|
+
readonly createdAt: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Persists one classified failure to the account row and appends a health event. */
|
|
38
|
+
export async function recordAccountFailure(
|
|
39
|
+
db: CartethyiaDatabase,
|
|
40
|
+
accountId: string,
|
|
41
|
+
error: unknown,
|
|
42
|
+
options?: AccountFailureEvidence,
|
|
43
|
+
): Promise<AccountErrorClassification | null> {
|
|
44
|
+
const classification = classifyAccountError(error, options);
|
|
45
|
+
/**
|
|
46
|
+
* Logged even when the classification does not mutate the account.
|
|
47
|
+
*
|
|
48
|
+
* `rate_limit_transient` seen on a stream frame carries no credential
|
|
49
|
+
* detail, so it classifies at provider scope and `mutatesAccount` is false
|
|
50
|
+
* — nothing is cooled and the account stays `active`. That part is correct:
|
|
51
|
+
* the upstream's own remedy ("switch to the other models") only works if the
|
|
52
|
+
* account keeps serving. But this returned without writing anything, so a
|
|
53
|
+
* rate-limited request showed up in the request log while the account's
|
|
54
|
+
* health log stayed empty — a symptom with no recorded cause.
|
|
55
|
+
*
|
|
56
|
+
* The row is the whole point of this branch: it records the failure without
|
|
57
|
+
* moving the account. Both status sides are the classification's own, so the
|
|
58
|
+
* dialog names the cause without claiming a transition that never happened.
|
|
59
|
+
*/
|
|
60
|
+
if (!classification.mutatesAccount) {
|
|
61
|
+
void writeHealthEvent(db, {
|
|
62
|
+
entityKind: "account",
|
|
63
|
+
accountId,
|
|
64
|
+
fromStatus: classification.status,
|
|
65
|
+
toStatus: classification.status,
|
|
66
|
+
reason: classification.reason,
|
|
67
|
+
errorCategory: classification.category,
|
|
68
|
+
modelId: options?.modelId ?? null,
|
|
69
|
+
createdAt: new Date(),
|
|
70
|
+
}).catch(() => undefined);
|
|
71
|
+
return classification;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const persist = async (client: CartethyiaDatabase): Promise<AccountErrorClassification | null> => {
|
|
75
|
+
// Projected, not a bare `select()`: this runs on *every* attempt outcome,
|
|
76
|
+
// and a full row drags `credential_ciphertext` (the largest thing stored
|
|
77
|
+
// per account) and `auth_state` over the wire to read a status and a
|
|
78
|
+
// counter. `RouteCatalogRepository` already projects for the same reason.
|
|
79
|
+
const accountRows = await client
|
|
80
|
+
.select({
|
|
81
|
+
status: providerAccounts.status,
|
|
82
|
+
staticToken: providerAccounts.staticToken,
|
|
83
|
+
consecutiveFailures: providerAccounts.consecutiveFailures,
|
|
84
|
+
modelCooldowns: providerAccounts.modelCooldowns,
|
|
85
|
+
// Read only for the unsupported-model ladder; NULL means "never
|
|
86
|
+
// counted", which is the common case and must stay cheap.
|
|
87
|
+
modelBlockHits: providerAccounts.modelBlockHits,
|
|
88
|
+
cooldownUntil: providerAccounts.cooldownUntil,
|
|
89
|
+
})
|
|
90
|
+
.from(providerAccounts)
|
|
91
|
+
.where(eq(providerAccounts.id, accountId))
|
|
92
|
+
.limit(1);
|
|
93
|
+
const account = accountRows[0];
|
|
94
|
+
if (!account || account.status === "disabled") return null;
|
|
95
|
+
|
|
96
|
+
// Re-classify with the account's own static-token flag. The caller's
|
|
97
|
+
// evidence is request-scoped and does not carry it, but an auth rejection
|
|
98
|
+
// against a static token must cool the account down rather than park it —
|
|
99
|
+
// there is no refresh to run, and the token may still be valid.
|
|
100
|
+
const effective = account.staticToken
|
|
101
|
+
? classifyAccountError(error, { ...options, staticToken: true })
|
|
102
|
+
: classification;
|
|
103
|
+
|
|
104
|
+
const fromStatus = account.status;
|
|
105
|
+
const now = new Date();
|
|
106
|
+
|
|
107
|
+
const modelId = options?.modelId;
|
|
108
|
+
// Every category here is a verdict about one (account, model) pair rather
|
|
109
|
+
// than about the credential, so all three back off that pair and leave the
|
|
110
|
+
// account routable for every other model. `model_unreachable` joins the two
|
|
111
|
+
// throttles on the same reasoning: an upstream that reports the model does
|
|
112
|
+
// not exist for this account will report it again on the next request.
|
|
113
|
+
const isThrottle =
|
|
114
|
+
effective.category === "rate_limit_transient" ||
|
|
115
|
+
effective.category === "model_capacity" ||
|
|
116
|
+
effective.category === "model_unreachable";
|
|
117
|
+
const shouldModelCooldown = modelId !== undefined && isThrottle;
|
|
118
|
+
// The account-wide streak, escalated for a provider-stated window. Read once
|
|
119
|
+
// so the model-scoped and account-wide paths below cannot disagree about it:
|
|
120
|
+
// this is the count *before* this failure, which the UPDATE below increments.
|
|
121
|
+
const streak = Math.max(0, account.consecutiveFailures ?? 0);
|
|
122
|
+
const escalatedCooldownMs = escalateFallbackCooldown(
|
|
123
|
+
effective.cooldownMs,
|
|
124
|
+
effective.providerStated === true,
|
|
125
|
+
streak,
|
|
126
|
+
);
|
|
127
|
+
|
|
128
|
+
// The unsupported-model ladder, computed before the branch: the classifier
|
|
129
|
+
// could only state the base window because the authoritative hit count
|
|
130
|
+
// lives on the row, and it is read here. `hits` is the new total for this
|
|
131
|
+
// pair, so the first report is 1 and gets the base window.
|
|
132
|
+
const blockHits = (account.modelBlockHits as Record<string, number> | null) ?? {};
|
|
133
|
+
const unsupportedHits =
|
|
134
|
+
effective.modelUnsupported === true && modelId !== undefined
|
|
135
|
+
? Math.max(1, (blockHits[modelId] ?? 0) + 1)
|
|
136
|
+
: undefined;
|
|
137
|
+
// An unsupported-model window is stated by *us*, not by the provider, so it
|
|
138
|
+
// would otherwise be multiplied a second time by `escalateFallbackCooldown`
|
|
139
|
+
// (which reads `consecutiveFailures`, an account-wide counter). The two
|
|
140
|
+
// ladders are different things: one counts how many times this pair was
|
|
141
|
+
// reported unsupported, the other how many failures this account produced.
|
|
142
|
+
const unsupportedCooldownMs =
|
|
143
|
+
unsupportedHits === undefined ? undefined : unsupportedModelCooldownMs(unsupportedHits);
|
|
144
|
+
|
|
145
|
+
if (shouldModelCooldown) {
|
|
146
|
+
const existing = (account.modelCooldowns as Record<string, string> | null) ?? {};
|
|
147
|
+
// Monotonic: never move an in-force window earlier (see `latestDeadlineMs`).
|
|
148
|
+
// A provider-stated deadline wins outright because it is already the later
|
|
149
|
+
// of the two whenever the provider really parked the pair longer.
|
|
150
|
+
const nextUntil = new Date(
|
|
151
|
+
latestDeadlineMs(
|
|
152
|
+
effective.retryAt?.getTime(),
|
|
153
|
+
Date.now() + (unsupportedCooldownMs ?? escalatedCooldownMs),
|
|
154
|
+
existing[modelId] ? Date.parse(existing[modelId]!) : undefined,
|
|
155
|
+
),
|
|
156
|
+
).toISOString();
|
|
157
|
+
const previousUntil = existing[modelId];
|
|
158
|
+
// A re-statement is not an event: what the audit row reports is the
|
|
159
|
+
// *entry* into a cooling episode, and the deadline itself is not a
|
|
160
|
+
// column — so a row can only ever say "this pair started cooling for
|
|
161
|
+
// this reason". The upstream answers the same "Try again in 11h" on
|
|
162
|
+
// every retry (and an unstated reset is recomputed from now each time),
|
|
163
|
+
// so writing per failure produced seven identical rows in twenty minutes
|
|
164
|
+
// and read as an account flapping when nothing had changed. The entry is
|
|
165
|
+
// still refreshed below — its error fields are what the dialog explains
|
|
166
|
+
// the cooldown with — while an entry that is still in force suppresses
|
|
167
|
+
// the duplicate row.
|
|
168
|
+
const alreadyCooling =
|
|
169
|
+
previousUntil !== undefined && new Date(previousUntil).getTime() > Date.now();
|
|
170
|
+
// `status`/`cooldownUntil` stay untouched: only this (account, model)
|
|
171
|
+
// pair backs off, and the account remains routable for every other model.
|
|
172
|
+
// The error fields ARE written, because they are what provider detail and
|
|
173
|
+
// the health dialog read to explain *why* a model is cooling — without
|
|
174
|
+
// them the row showed "1 model cooling" with no reason at all.
|
|
175
|
+
//
|
|
176
|
+
// The streak advances here too. Without it the most common throttle — a
|
|
177
|
+
// model-scoped rate limit — never escalated, because the counter was only
|
|
178
|
+
// incremented on the account-wide path below, and `consecutiveFailures`
|
|
179
|
+
// stays honest about what it always claimed to count: how many failures in
|
|
180
|
+
// a row this account has just produced.
|
|
181
|
+
await client
|
|
182
|
+
.update(providerAccounts)
|
|
183
|
+
.set({
|
|
184
|
+
modelCooldowns: { ...existing, [modelId]: nextUntil },
|
|
185
|
+
// Counted only for the unsupported-model ladder, and only when this
|
|
186
|
+
// failure is that verdict — so a rate limit or a capacity refusal
|
|
187
|
+
// never advances it. `model_cooldowns`' own shape is untouched, so
|
|
188
|
+
// routing and the console keep parsing it exactly as before.
|
|
189
|
+
...(unsupportedHits === undefined
|
|
190
|
+
? {}
|
|
191
|
+
: { modelBlockHits: { ...blockHits, [modelId]: unsupportedHits } }),
|
|
192
|
+
consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
|
|
193
|
+
lastError: effective.reason,
|
|
194
|
+
lastErrorCategory: effective.category,
|
|
195
|
+
lastErrorAt: now,
|
|
196
|
+
})
|
|
197
|
+
.where(eq(providerAccounts.id, accountId));
|
|
198
|
+
if (!alreadyCooling) {
|
|
199
|
+
await writeHealthEvent(client, {
|
|
200
|
+
entityKind: "account",
|
|
201
|
+
accountId,
|
|
202
|
+
// The account's own status, on both sides: a model-scoped throttle
|
|
203
|
+
// never moved it, and writing the classification's status here
|
|
204
|
+
// claimed an `active → cooldown` transition that never happened — the
|
|
205
|
+
// dialog showed "Account status: ACTIVE" above a list of transitions
|
|
206
|
+
// to `cooldown`. The row still names the model and the reason, which
|
|
207
|
+
// is what makes it worth keeping.
|
|
208
|
+
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
209
|
+
toStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
210
|
+
reason: effective.reason,
|
|
211
|
+
errorCategory: effective.category,
|
|
212
|
+
modelId: modelId ?? null,
|
|
213
|
+
createdAt: now,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
// `status` and `cooldownUntil` are untouched on this path — only the
|
|
217
|
+
// (account, model) pair backed off — so the snapshot is still accurate
|
|
218
|
+
// and must not be dropped.
|
|
219
|
+
return { ...classification, routesVisible: false };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
await client
|
|
223
|
+
.update(providerAccounts)
|
|
224
|
+
.set({
|
|
225
|
+
status: effective.status,
|
|
226
|
+
consecutiveFailures: sql`${providerAccounts.consecutiveFailures} + 1`,
|
|
227
|
+
lastError: effective.reason,
|
|
228
|
+
lastErrorCategory: effective.category,
|
|
229
|
+
lastErrorAt: now,
|
|
230
|
+
// Monotonic against the account's own in-force deadline, for the same
|
|
231
|
+
// reason as the model path: an unclassified 60-second failure must not
|
|
232
|
+
// release an account the provider parked for hours.
|
|
233
|
+
//
|
|
234
|
+
// A `0` duration contributes no candidate rather than stamping *now*
|
|
235
|
+
// (see `accountDeadlineMs`): the one state that asks for it is a
|
|
236
|
+
// terminal automatic disable, and an elapsed deadline there would hand
|
|
237
|
+
// the row straight back to the recovery sweep.
|
|
238
|
+
cooldownUntil: accountDeadlineMs([
|
|
239
|
+
effective.retryAt?.getTime(),
|
|
240
|
+
escalatedCooldownMs > 0 ? Date.now() + escalatedCooldownMs : undefined,
|
|
241
|
+
account.cooldownUntil?.getTime?.(),
|
|
242
|
+
]),
|
|
243
|
+
})
|
|
244
|
+
.where(eq(providerAccounts.id, accountId));
|
|
245
|
+
|
|
246
|
+
{
|
|
247
|
+
await writeHealthEvent(client, {
|
|
248
|
+
entityKind: "account",
|
|
249
|
+
accountId,
|
|
250
|
+
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
251
|
+
toStatus: effective.status,
|
|
252
|
+
reason: effective.reason,
|
|
253
|
+
errorCategory: effective.category,
|
|
254
|
+
modelId: modelId ?? null,
|
|
255
|
+
createdAt: now,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
// This path wrote `status` and `cooldownUntil`, both of which the snapshot
|
|
259
|
+
// projects, so routing has to see it.
|
|
260
|
+
return { ...classification, routesVisible: true };
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
try {
|
|
264
|
+
// Kept as a capability probe, not a correctness guard: this function is
|
|
265
|
+
// reachable with a partially-implemented database handle (the dispatch
|
|
266
|
+
// tests pass a narrow double), and a transaction is an optimization here,
|
|
267
|
+
// not a requirement — the same writes are correct without one.
|
|
268
|
+
if (typeof db.transaction === "function") {
|
|
269
|
+
return await db.transaction((tx) => persist(tx as CartethyiaDatabase));
|
|
270
|
+
}
|
|
271
|
+
return await persist(db);
|
|
272
|
+
} catch {
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Clears failure state after a successful dispatch; true when the account changed. */
|
|
278
|
+
export async function recordAccountSuccess(
|
|
279
|
+
db: CartethyiaDatabase,
|
|
280
|
+
accountId: string,
|
|
281
|
+
): Promise<boolean> {
|
|
282
|
+
const mutate = async (client: CartethyiaDatabase): Promise<boolean> => {
|
|
283
|
+
// `recordAccountSuccess` runs on every successful dispatch, and for a
|
|
284
|
+
// healthy account `hadFailures` is false — the row was fetched purely to
|
|
285
|
+
// decide nothing. Projected so that no-op path does not pull the
|
|
286
|
+
// credential ciphertext across the wire.
|
|
287
|
+
const rows = await client
|
|
288
|
+
.select({
|
|
289
|
+
status: providerAccounts.status,
|
|
290
|
+
consecutiveFailures: providerAccounts.consecutiveFailures,
|
|
291
|
+
// Read only to decide whether there is anything to clear: the common
|
|
292
|
+
// case (no unsupported-model hit ever recorded) must not write.
|
|
293
|
+
modelBlockHits: providerAccounts.modelBlockHits,
|
|
294
|
+
})
|
|
295
|
+
.from(providerAccounts)
|
|
296
|
+
.where(eq(providerAccounts.id, accountId))
|
|
297
|
+
.limit(1);
|
|
298
|
+
const account = rows[0];
|
|
299
|
+
if (!account || account.status === "disabled") return false;
|
|
300
|
+
const fromStatus = account.status;
|
|
301
|
+
const hadFailures = (account.consecutiveFailures ?? 0) > 0 || account.status !== "active";
|
|
302
|
+
const now = new Date();
|
|
303
|
+
// The unsupported-model escalation restarts after a success: a model that
|
|
304
|
+
// just served is not "not supported", so the next report of it must start
|
|
305
|
+
// from the base window again rather than from wherever the ladder had
|
|
306
|
+
// reached. `modelCooldowns` is deliberately NOT cleared — an in-force
|
|
307
|
+
// window the provider stated survives, which is the existing behavior and
|
|
308
|
+
// the reference gateways' too.
|
|
309
|
+
const blockHits = account.modelBlockHits as Record<string, number> | null;
|
|
310
|
+
const hasBlockHits = blockHits !== null && Object.keys(blockHits).length > 0;
|
|
311
|
+
await client
|
|
312
|
+
.update(providerAccounts)
|
|
313
|
+
.set({
|
|
314
|
+
status: "active",
|
|
315
|
+
consecutiveFailures: 0,
|
|
316
|
+
cooldownUntil: null,
|
|
317
|
+
lastSuccessAt: now,
|
|
318
|
+
...(hadFailures ? { lastRecoveredAt: now } : {}),
|
|
319
|
+
...(hasBlockHits ? { modelBlockHits: {} } : {}),
|
|
320
|
+
})
|
|
321
|
+
.where(eq(providerAccounts.id, accountId));
|
|
322
|
+
if (hadFailures && typeof client.insert === "function") {
|
|
323
|
+
await client.insert(healthEvents).values({
|
|
324
|
+
id: randomUUID(),
|
|
325
|
+
entityKind: "account",
|
|
326
|
+
accountId,
|
|
327
|
+
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
328
|
+
toStatus: "active",
|
|
329
|
+
reason: "Request dispatched successfully",
|
|
330
|
+
errorCategory: null,
|
|
331
|
+
createdAt: now,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
return hadFailures;
|
|
335
|
+
};
|
|
336
|
+
try {
|
|
337
|
+
if (typeof db.transaction === "function") {
|
|
338
|
+
return await db.transaction((tx) => mutate(tx as CartethyiaDatabase));
|
|
339
|
+
}
|
|
340
|
+
return await mutate(db);
|
|
341
|
+
} catch {
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Operator-initiated recovery: forces the account back to active and logs the transition. */
|
|
347
|
+
export async function recoverAccount(
|
|
348
|
+
db: CartethyiaDatabase,
|
|
349
|
+
accountId: string,
|
|
350
|
+
reason = "manual_operator_recovery",
|
|
351
|
+
): Promise<boolean> {
|
|
352
|
+
const recover = async (client: CartethyiaDatabase): Promise<boolean> => {
|
|
353
|
+
const accountRows = await client
|
|
354
|
+
.select()
|
|
355
|
+
.from(providerAccounts)
|
|
356
|
+
.where(eq(providerAccounts.id, accountId))
|
|
357
|
+
.limit(1);
|
|
358
|
+
const account = accountRows[0];
|
|
359
|
+
if (!account) return false;
|
|
360
|
+
const fromStatus = account.status;
|
|
361
|
+
const now = new Date();
|
|
362
|
+
await client
|
|
363
|
+
.update(providerAccounts)
|
|
364
|
+
.set({
|
|
365
|
+
status: "active",
|
|
366
|
+
consecutiveFailures: 0,
|
|
367
|
+
cooldownUntil: null,
|
|
368
|
+
lastRecoveredAt: now,
|
|
369
|
+
// Recovery is the operator asserting the account works now, so it must
|
|
370
|
+
// clear EVERY routing exclusion, not just the account-wide ones. A
|
|
371
|
+
// per-model cooldown entry excludes the (account, model) pair in
|
|
372
|
+
// `route-catalog.ts`, so leaving it behind kept the account blocked for
|
|
373
|
+
// that model through `/v1` even though the account read `active` and a
|
|
374
|
+
// direct probe (which addresses the account by id, bypassing routing
|
|
375
|
+
// eligibility) succeeded. Clearing the error fields for the same reason:
|
|
376
|
+
// a stale reason on a recovered row misreports the current state.
|
|
377
|
+
modelCooldowns: {},
|
|
378
|
+
lastError: null,
|
|
379
|
+
lastErrorCategory: null,
|
|
380
|
+
lastErrorAt: null,
|
|
381
|
+
})
|
|
382
|
+
.where(eq(providerAccounts.id, accountId));
|
|
383
|
+
{
|
|
384
|
+
await writeHealthEvent(client, {
|
|
385
|
+
entityKind: "account",
|
|
386
|
+
accountId,
|
|
387
|
+
fromStatus: fromStatus as "active" | "cooldown" | "disabled",
|
|
388
|
+
toStatus: "active",
|
|
389
|
+
reason,
|
|
390
|
+
errorCategory: null,
|
|
391
|
+
createdAt: now,
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
return true;
|
|
395
|
+
};
|
|
396
|
+
try {
|
|
397
|
+
if (typeof db.transaction === "function") {
|
|
398
|
+
return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
|
|
399
|
+
}
|
|
400
|
+
return await recover(db);
|
|
401
|
+
} catch {
|
|
402
|
+
return false;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Recovers accounts whose cooldown elapsed and prunes expired per-model keys. */
|
|
407
|
+
export async function sweepExpiredCooldowns(db: CartethyiaDatabase): Promise<number> {
|
|
408
|
+
const recover = async (client: CartethyiaDatabase): Promise<number> => {
|
|
409
|
+
const now = new Date();
|
|
410
|
+
const nowIso = now.toISOString();
|
|
411
|
+
const expired = await client
|
|
412
|
+
.select({
|
|
413
|
+
id: providerAccounts.id,
|
|
414
|
+
status: providerAccounts.status,
|
|
415
|
+
})
|
|
416
|
+
.from(providerAccounts)
|
|
417
|
+
.where(
|
|
418
|
+
// An ordinary cooldown, recovered as soon as its window elapses.
|
|
419
|
+
and(
|
|
420
|
+
eq(providerAccounts.status, "cooldown"),
|
|
421
|
+
isNotNull(providerAccounts.cooldownUntil),
|
|
422
|
+
lte(providerAccounts.cooldownUntil, now),
|
|
423
|
+
),
|
|
424
|
+
)
|
|
425
|
+
// One pass recovers a bounded batch. The sweep runs every 30s, so a
|
|
426
|
+
// backlog larger than this drains over successive ticks instead of one
|
|
427
|
+
// statement covering every expired row — that statement holds its locks
|
|
428
|
+
// for as long as the backlog takes.
|
|
429
|
+
.limit(COOLDOWN_RECOVERY_BATCH);
|
|
430
|
+
if (expired.length > 0) {
|
|
431
|
+
await client
|
|
432
|
+
.update(providerAccounts)
|
|
433
|
+
.set({
|
|
434
|
+
status: "active",
|
|
435
|
+
consecutiveFailures: 0,
|
|
436
|
+
cooldownUntil: null,
|
|
437
|
+
lastRecoveredAt: now,
|
|
438
|
+
// Cleared with the state it described. A recovered row that kept
|
|
439
|
+
// `auth_invalidated` still rendered the console's "Re-login required"
|
|
440
|
+
// pill above an `active` status, so the account read healthy to
|
|
441
|
+
// routing and broken to the operator at the same time.
|
|
442
|
+
lastErrorCategory: null,
|
|
443
|
+
})
|
|
444
|
+
.where(inArray(providerAccounts.id, expired.map((account) => account.id)));
|
|
445
|
+
if (typeof client.insert === "function") {
|
|
446
|
+
for (let offset = 0; offset < expired.length; offset += 500) {
|
|
447
|
+
await client.insert(healthEvents).values(
|
|
448
|
+
expired.slice(offset, offset + 500).map((account) => ({
|
|
449
|
+
id: randomUUID(),
|
|
450
|
+
entityKind: "account" as const,
|
|
451
|
+
accountId: account.id,
|
|
452
|
+
fromStatus: account.status,
|
|
453
|
+
toStatus: "active" as const,
|
|
454
|
+
reason: "Cooldown period elapsed — auto-recovered",
|
|
455
|
+
errorCategory: null,
|
|
456
|
+
createdAt: now,
|
|
457
|
+
})),
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// Prune per-model cooldowns: remove entries whose ISO timestamp has passed.
|
|
464
|
+
//
|
|
465
|
+
// One set-based UPDATE rather than a SELECT followed by an UPDATE per row.
|
|
466
|
+
// The per-row form issued one statement per account holding any cooldown —
|
|
467
|
+
// on a deployment where many accounts are throttled at once that is the
|
|
468
|
+
// sweep's dominant cost, and it all runs inside this transaction, holding
|
|
469
|
+
// its locks for the whole pass. Recomputing the object in SQL does the same
|
|
470
|
+
// work in one statement.
|
|
471
|
+
//
|
|
472
|
+
// `jsonb_typeof(...) = 'object'` guards the shape: a malformed value must
|
|
473
|
+
// not abort the whole sweep. `IS DISTINCT FROM` keeps the update from
|
|
474
|
+
// rewriting rows it
|
|
475
|
+
// did not change, so the returned row count is the number of accounts whose
|
|
476
|
+
// cooldown set actually shrank — the same thing the loop counted.
|
|
477
|
+
const pruning = pruneExpiredCooldowns(sql`${providerAccounts.modelCooldowns}`, sql`${nowIso}`);
|
|
478
|
+
const pruned = await client
|
|
479
|
+
.update(providerAccounts)
|
|
480
|
+
.set({ modelCooldowns: pruning.value })
|
|
481
|
+
.where(pruning.changed);
|
|
482
|
+
return expired.length + affectedRows(pruned);
|
|
483
|
+
};
|
|
484
|
+
try {
|
|
485
|
+
if (typeof db.transaction === "function") {
|
|
486
|
+
return await db.transaction((tx) => recover(tx as CartethyiaDatabase));
|
|
487
|
+
}
|
|
488
|
+
return await recover(db);
|
|
489
|
+
} catch {
|
|
490
|
+
return 0;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Single attempt health reporter: one home for the success/failure writes
|
|
496
|
+
* every dispatch attempt ends with. Success records recovery; failure
|
|
497
|
+
* records with caller-supplied evidence (origin/scope classification plus
|
|
498
|
+
* model id). A mutating write invalidates the route snapshot so the next
|
|
499
|
+
* plan sees fresh health. Never throws — health must not fail requests.
|
|
500
|
+
*/
|
|
501
|
+
export interface AttemptHealthReport {
|
|
502
|
+
readonly accountId: string | undefined;
|
|
503
|
+
readonly modelId?: string;
|
|
504
|
+
readonly error?: unknown;
|
|
505
|
+
readonly evidence?: AccountFailureEvidence;
|
|
506
|
+
readonly snapshotService?: { invalidate(): unknown };
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Writes one health audit row, off the request path when a buffer is installed.
|
|
511
|
+
*
|
|
512
|
+
* `health_events` is append-only and read by nobody in the request path, so it
|
|
513
|
+
* does not need to land before the request finishes — unlike the
|
|
514
|
+
* `provider_accounts` UPDATE beside it, which routing reads and which therefore
|
|
515
|
+
* stays synchronous and transactional.
|
|
516
|
+
*/
|
|
517
|
+
async function writeHealthEvent(
|
|
518
|
+
client: CartethyiaDatabase,
|
|
519
|
+
row: {
|
|
520
|
+
readonly entityKind: "account";
|
|
521
|
+
readonly accountId: string;
|
|
522
|
+
readonly fromStatus: string;
|
|
523
|
+
readonly toStatus: string;
|
|
524
|
+
readonly reason: string;
|
|
525
|
+
readonly errorCategory: string | null;
|
|
526
|
+
readonly modelId?: string | null;
|
|
527
|
+
readonly createdAt: Date;
|
|
528
|
+
},
|
|
529
|
+
): Promise<void> {
|
|
530
|
+
const buffer = healthEventBuffer();
|
|
531
|
+
if (buffer !== undefined) {
|
|
532
|
+
buffer.enqueue({ ...row, modelId: row.modelId ?? null });
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
if (typeof client.insert !== "function") return;
|
|
536
|
+
await client.insert(healthEvents).values({ id: randomUUID(), ...row } as never);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
export async function reportAttemptOutcome(
|
|
540
|
+
db: CartethyiaDatabase,
|
|
541
|
+
report: AttemptHealthReport,
|
|
542
|
+
): Promise<void> {
|
|
543
|
+
if (!report.accountId) return;
|
|
544
|
+
if (report.error === undefined) {
|
|
545
|
+
const recovered = await recordAccountSuccess(db, report.accountId).catch(() => false);
|
|
546
|
+
if (recovered) await report.snapshotService?.invalidate();
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const classification = await recordAccountFailure(db, report.accountId, report.error, {
|
|
550
|
+
...report.evidence,
|
|
551
|
+
...(report.modelId === undefined ? {} : { modelId: report.modelId }),
|
|
552
|
+
}).catch(() => null);
|
|
553
|
+
// Gated on `routesVisible`, not on `mutatesAccount`: a failure that only
|
|
554
|
+
// cooled one (account, model) pair, or only advanced the streak, changed
|
|
555
|
+
// nothing routing reads. Invalidating for those rebuilt an 11-SELECT snapshot
|
|
556
|
+
// on the next request, and under a failure wave each rebuild fed the next
|
|
557
|
+
// invalidation.
|
|
558
|
+
if (classification?.routesVisible) await report.snapshotService?.invalidate();
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
export async function listAccountHealthEvents(
|
|
562
|
+
db: CartethyiaDatabase,
|
|
563
|
+
accountId: string,
|
|
564
|
+
limit = 50,
|
|
565
|
+
): Promise<readonly AccountHealthEventRecord[]> {
|
|
566
|
+
try {
|
|
567
|
+
const rows = await db
|
|
568
|
+
.select()
|
|
569
|
+
.from(healthEvents)
|
|
570
|
+
.where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
|
|
571
|
+
.orderBy(desc(healthEvents.createdAt))
|
|
572
|
+
.limit(limit);
|
|
573
|
+
|
|
574
|
+
return rows.map((r) => ({
|
|
575
|
+
id: r.id,
|
|
576
|
+
accountId: r.accountId as string,
|
|
577
|
+
fromStatus: r.fromStatus,
|
|
578
|
+
toStatus: r.toStatus,
|
|
579
|
+
modelId: r.modelId ?? null,
|
|
580
|
+
reason: r.reason,
|
|
581
|
+
errorCategory: r.errorCategory,
|
|
582
|
+
createdAt: r.createdAt.toISOString(),
|
|
583
|
+
}));
|
|
584
|
+
} catch {
|
|
585
|
+
return [];
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/** Every stored row across accounts and pools — the size the purge offers to delete. */
|
|
590
|
+
export async function countAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
|
|
591
|
+
try {
|
|
592
|
+
const rows = await db.select({ total: count() }).from(healthEvents);
|
|
593
|
+
return rows[0]?.total ?? 0;
|
|
594
|
+
} catch {
|
|
595
|
+
return 0;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Deletes stored health & error rows.
|
|
601
|
+
*
|
|
602
|
+
* Unlike the read above, a failed delete surfaces: an operator who purged
|
|
603
|
+
* believes the trail is gone, so "it failed" must not read as "it succeeded
|
|
604
|
+
* with zero rows". Callers own the flush-buffer barrier (`drain`) so a queued
|
|
605
|
+
* write cannot land after these rows are gone.
|
|
606
|
+
*/
|
|
607
|
+
export async function deleteAccountHealthEvents(
|
|
608
|
+
db: CartethyiaDatabase,
|
|
609
|
+
accountId: string,
|
|
610
|
+
): Promise<number> {
|
|
611
|
+
const rows = await db
|
|
612
|
+
.delete(healthEvents)
|
|
613
|
+
.where(and(eq(healthEvents.entityKind, "account"), eq(healthEvents.accountId, accountId)))
|
|
614
|
+
.returning({ id: healthEvents.id });
|
|
615
|
+
return rows.length;
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
export async function deleteAllHealthEvents(db: CartethyiaDatabase): Promise<number> {
|
|
619
|
+
const rows = await db.delete(healthEvents).returning({ id: healthEvents.id });
|
|
620
|
+
return rows.length;
|
|
621
|
+
}
|