cartethyia 2.7.9 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -2
- package/dist/dashboard/assets/{BannedPage-BxoG1MrL.js → BannedPage-Cc0vKpMG.js} +1 -1
- package/dist/dashboard/assets/BannedPage-Cc0vKpMG.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolDetailPage-ixwgKk53.js → CliToolDetailPage-Bqj2aa1T.js} +1 -1
- package/dist/dashboard/assets/CliToolDetailPage-Bqj2aa1T.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-BmttNV3U.js → CliToolsPage-CXsaOAO2.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-CXsaOAO2.js.gz +0 -0
- package/dist/dashboard/assets/{CombosPage-BE3qwwfC.js → CombosPage-BeLpJ4ma.js} +1 -1
- package/dist/dashboard/assets/CombosPage-BeLpJ4ma.js.gz +0 -0
- package/dist/dashboard/assets/{ConsoleLogPage-CRZdmMIX.js → ConsoleLogPage-DnV0uZ3m.js} +1 -1
- package/dist/dashboard/assets/ConsoleLogPage-DnV0uZ3m.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-Dv_FJ6Er.js → CustomizationPage-CBxAjo63.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-CBxAjo63.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-B5yEE0k6.js +1 -0
- package/dist/dashboard/assets/HealthEventsModal-B5yEE0k6.js.gz +0 -0
- package/dist/dashboard/assets/{InjectSystemPromptPanel-BOoO8rpz.js → InjectSystemPromptPanel-BWFLECmz.js} +2 -2
- package/dist/dashboard/assets/InjectSystemPromptPanel-BWFLECmz.js.gz +0 -0
- package/dist/dashboard/assets/{LoginPage-ChJuNNVN.js → LoginPage-CgoPc5B9.js} +1 -1
- package/dist/dashboard/assets/LoginPage-CgoPc5B9.js.gz +0 -0
- package/dist/dashboard/assets/{ModelPicker-BZAJtPg1.js → ModelPicker-D9bFQern.js} +1 -1
- package/dist/dashboard/assets/ModelPicker-D9bFQern.js.gz +0 -0
- package/dist/dashboard/assets/{OverviewPage-BE40s3u5.js → OverviewPage-B_p4tL8k.js} +1 -1
- package/dist/dashboard/assets/OverviewPage-B_p4tL8k.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-rs5E2rxv.js +8 -0
- package/dist/dashboard/assets/ProviderDetailPage-rs5E2rxv.js.gz +0 -0
- package/dist/dashboard/assets/{ProviderIcon-CQ6bPUf_.js → ProviderIcon-D_0PiZtg.js} +1 -1
- package/dist/dashboard/assets/ProviderIcon-D_0PiZtg.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-CLI2bJTK.js +1 -0
- package/dist/dashboard/assets/ProvidersPage-CLI2bJTK.js.gz +0 -0
- package/dist/dashboard/assets/{ProxyPage-BJ87Pp0U.js → ProxyPage-BwNC_-Cx.js} +1 -1
- package/dist/dashboard/assets/ProxyPage-BwNC_-Cx.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-DA-a1iSN.js +1 -0
- package/dist/dashboard/assets/QuotaPage-DA-a1iSN.js.gz +0 -0
- package/dist/dashboard/assets/{SettingsPage-C5HPWqEv.js → SettingsPage-CctTga2C.js} +1 -1
- package/dist/dashboard/assets/SettingsPage-CctTga2C.js.gz +0 -0
- package/dist/dashboard/assets/{SetupPage-CFNcATJJ.js → SetupPage-febLU4r1.js} +1 -1
- package/dist/dashboard/assets/SetupPage-febLU4r1.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-YoD4tY5l.js → StudioPage-M6fvR5Wz.js} +1 -1
- package/dist/dashboard/assets/StudioPage-M6fvR5Wz.js.gz +0 -0
- package/dist/dashboard/assets/{UsagePage-DxjaAhb_.js → UsagePage-nyeuRqz9.js} +1 -1
- package/dist/dashboard/assets/UsagePage-nyeuRqz9.js.gz +0 -0
- package/dist/dashboard/assets/{api-keys-BTlkq_Uk.js → api-keys-BFOlvrJC.js} +1 -1
- package/dist/dashboard/assets/api-keys-BFOlvrJC.js.gz +0 -0
- package/dist/dashboard/assets/app-Dx2QgsNj.css +2 -0
- package/dist/dashboard/assets/app-Dx2QgsNj.css.gz +0 -0
- package/dist/dashboard/assets/assert-CqTLw5e7.js +1 -0
- package/dist/dashboard/assets/assert-CqTLw5e7.js.gz +0 -0
- package/dist/dashboard/assets/{badge-DdbT_7Am.js → badge-C83EqLB0.js} +1 -1
- package/dist/dashboard/assets/badge-C83EqLB0.js.gz +0 -0
- package/dist/dashboard/assets/{banner-CsGN-nbm.js → banner-D1xn7pIe.js} +1 -1
- package/dist/dashboard/assets/banner-D1xn7pIe.js.gz +0 -0
- package/dist/dashboard/assets/{card-mgZ_TLLZ.js → card-DU3Lq5Lm.js} +1 -1
- package/dist/dashboard/assets/card-DU3Lq5Lm.js.gz +0 -0
- package/dist/dashboard/assets/{cli-tools-A1-v8-03.js → cli-tools-Dw1U_drK.js} +1 -1
- package/dist/dashboard/assets/cli-tools-Dw1U_drK.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-CJ4Zty89.js → clipboard-button-CEL5Sg_J.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-CEL5Sg_J.js.gz +0 -0
- package/dist/dashboard/assets/{dialog-B5L2jPQ_.js → dialog-BjFR5UHy.js} +1 -1
- package/dist/dashboard/assets/dialog-BjFR5UHy.js.gz +0 -0
- package/dist/dashboard/assets/{drawer-DkjUouAj.js → drawer-HvBW0AjI.js} +1 -1
- package/dist/dashboard/assets/drawer-HvBW0AjI.js.gz +0 -0
- package/dist/dashboard/assets/entry-BSu3gFlt.js +4 -0
- package/dist/dashboard/assets/entry-BSu3gFlt.js.gz +0 -0
- package/dist/dashboard/assets/{entry-B-LVLV6o.js → entry-CECVELyy.js} +1 -1
- package/dist/dashboard/assets/entry-CECVELyy.js.gz +0 -0
- package/dist/dashboard/assets/{entry-CAhdYl3B.js → entry-DYYofQ7W.js} +1 -1
- package/dist/dashboard/assets/entry-DYYofQ7W.js.gz +0 -0
- package/dist/dashboard/assets/{formatters-CrsdLVsn.js → formatters-DLW4i8zK.js} +1 -1
- package/dist/dashboard/assets/formatters-DLW4i8zK.js.gz +0 -0
- package/dist/dashboard/assets/{index-BU64Lqvp.js → index-1-fCqe64.js} +2 -2
- package/dist/dashboard/assets/index-1-fCqe64.js.gz +0 -0
- package/dist/dashboard/assets/{inline-B2aDLyyg.js → inline-DGvU2oLK.js} +1 -1
- package/dist/dashboard/assets/inline-DGvU2oLK.js.gz +0 -0
- package/dist/dashboard/assets/{input-BCVkdR9d.js → input-iNnO1X2K.js} +1 -1
- package/dist/dashboard/assets/input-iNnO1X2K.js.gz +0 -0
- package/dist/dashboard/assets/{layout-DNcAdMK-.js → layout-0q8UGypb.js} +1 -1
- package/dist/dashboard/assets/layout-0q8UGypb.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-DgrIH0hw.js → list-row-C0w2bQjJ.js} +1 -1
- package/dist/dashboard/assets/list-row-C0w2bQjJ.js.gz +0 -0
- package/dist/dashboard/assets/{live-Cae01iu8.js → live-M5aJ_xCa.js} +1 -1
- package/dist/dashboard/assets/live-M5aJ_xCa.js.gz +0 -0
- package/dist/dashboard/assets/{network-CRib7H_2.js → network-BKeQGT3K.js} +1 -1
- package/dist/dashboard/assets/network-BKeQGT3K.js.gz +0 -0
- package/dist/dashboard/assets/{numbers-BKQE8Xpz.js → numbers-CivAk3da.js} +1 -1
- package/dist/dashboard/assets/numbers-CivAk3da.js.gz +0 -0
- package/dist/dashboard/assets/{pagination-ecPvG_8g.js → pagination-CpzvI5ep.js} +1 -1
- package/dist/dashboard/assets/pagination-CpzvI5ep.js.gz +0 -0
- package/dist/dashboard/assets/providers/zro.svg +3 -0
- package/dist/dashboard/assets/providers/zro.svg.gz +0 -0
- package/dist/dashboard/assets/quota-DiPD6UKH.js +1 -0
- package/dist/dashboard/assets/quota-DiPD6UKH.js.gz +0 -0
- package/dist/dashboard/assets/{routing-Ckm1UXuC.js → routing-BFQX2VBc.js} +1 -1
- package/dist/dashboard/assets/routing-BFQX2VBc.js.gz +0 -0
- package/dist/dashboard/assets/{select-ChoFWyKz.js → select-CXbck0jG.js} +2 -2
- package/dist/dashboard/assets/select-CXbck0jG.js.gz +0 -0
- package/dist/dashboard/assets/{settings-CKjFPlKW.js → settings-C_XvMHh9.js} +1 -1
- package/dist/dashboard/assets/settings-C_XvMHh9.js.gz +0 -0
- package/dist/dashboard/assets/{stack-DWCwU3fh.js → stack-DggRiCxE.js} +1 -1
- package/dist/dashboard/assets/stack-DggRiCxE.js.gz +0 -0
- package/dist/dashboard/assets/{state-Cg3VL5J_.js → state-ClFzfbYp.js} +1 -1
- package/dist/dashboard/assets/state-ClFzfbYp.js.gz +0 -0
- package/dist/dashboard/assets/{switch-Db70BMix.js → switch-CTIFxKqS.js} +1 -1
- package/dist/dashboard/assets/switch-CTIFxKqS.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-BqdE0LuM.js → useMutation-BClG3yWj.js} +1 -1
- package/dist/dashboard/assets/useMutation-BClG3yWj.js.gz +0 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +54 -53
- package/migrations/full-postgres/0046_add_accounts_quota_plan_fetched_at.sql +7 -0
- package/migrations/lite-sqlite3/0046_add_accounts_quota_plan_fetched_at.sql +4 -0
- package/package.json +1 -1
- package/src/boot/app.ts +10 -13
- package/src/boot/dependencies.ts +1 -1
- package/src/boot/flush.ts +20 -0
- package/src/boot/lifecycle.ts +7 -5
- package/src/config/routing.ts +26 -0
- package/src/config/spec-metadata.ts +72 -0
- package/src/config/spec.ts +20 -0
- package/src/console/api-keys/contracts.ts +12 -1
- package/src/console/api-keys/routes.ts +40 -28
- package/src/console/auth/account-reset.ts +7 -9
- package/src/console/backup/contracts.ts +4 -0
- package/src/console/backup/export-endpoint.ts +7 -5
- package/src/console/cli-tools/service.ts +6 -12
- package/src/console/console-router-guards.ts +23 -16
- package/src/console/domain-registration.ts +0 -2
- package/src/console/inferhub/normalize.ts +24 -28
- package/src/console/inferhub/types.ts +3 -0
- package/src/console/ip-bans/routes.ts +7 -13
- package/src/console/observability/store.ts +0 -53
- package/src/console/providers/catalog/contracts.ts +2 -2
- package/src/console/providers/catalog/discover-constants.ts +6 -0
- package/src/console/providers/catalog/routes.ts +1 -0
- package/src/console/providers/catalog/store.ts +21 -16
- package/src/console/quota/account-quota-global.ts +14 -24
- package/src/console/quota/account-quota-shared.ts +49 -0
- package/src/console/quota/account-quota-tenant.ts +6 -18
- package/src/console/quota/quota-view.ts +19 -3
- package/src/console/routing/pools/operations.ts +31 -46
- package/src/console/shared/errors.ts +33 -6
- package/src/main.ts +2 -3
- package/src/network/pool/byte-accounting.ts +3 -2
- package/src/network/pool/selector.ts +182 -39
- package/src/network/pool-cooldown-store.ts +89 -14
- package/src/network/pool-health.ts +49 -61
- package/src/network/rate-limit-verdict.ts +1 -1
- package/src/network/ssrf.ts +6 -6
- package/src/observability/event-log.ts +7 -361
- package/src/observability/performance-metrics.ts +9 -0
- package/src/observability/runtime-metrics.ts +14 -9
- package/src/observability/telemetry-buffer.ts +15 -25
- package/src/observability/telemetry-reconciliation.ts +9 -37
- package/src/observability/telemetry-row.ts +125 -0
- package/src/persistence/db-mode.ts +4 -1
- package/src/persistence/dialect.ts +18 -0
- package/src/persistence/migrate.ts +6 -3
- package/src/persistence/schema.ts +8 -0
- package/src/persistence/ttl-cache.ts +20 -2
- package/src/protocol/codecs.ts +1 -1
- package/src/protocol/dsml.ts +34 -86
- package/src/protocol/markup-stream-parser.ts +195 -0
- package/src/protocol/reasoning-text.ts +19 -0
- package/src/protocol/request/chat.ts +10 -10
- package/src/protocol/request/gemini.ts +8 -2
- package/src/protocol/request/messages.ts +3 -4
- package/src/protocol/response/chat.ts +12 -20
- package/src/protocol/response/omp-tool-call-markup.ts +43 -86
- package/src/protocol/response/responses.ts +18 -2
- package/src/protocol/stop-reason.ts +23 -1
- package/src/protocol/think-split.ts +112 -0
- package/src/protocol/translation/dialect/qwen.ts +26 -58
- package/src/providers/adapters/compatible-adapter.ts +71 -12
- package/src/providers/authentication/oauth-client.ts +1 -1
- package/src/providers/authentication/oauth-device-flow.ts +98 -8
- package/src/providers/authentication/oauth-refresh-service.ts +30 -49
- package/src/providers/configured-provider.ts +148 -0
- package/src/providers/default-registry.ts +61 -36
- package/src/providers/discovery/probe-phases.ts +14 -4
- package/src/providers/discovery/probing-service.ts +15 -2
- package/src/providers/integrations/alibabacp/alibabacp-models.ts +1 -2
- package/src/providers/integrations/{dashscope-models.ts → alibabacp/dashscope-models.ts} +2 -2
- package/src/providers/integrations/antigravity/antigravity-protocol.ts +5 -1
- package/src/providers/integrations/buddy/buddy-chat-shared.ts +173 -17
- package/src/providers/integrations/buddy/buddy-identity.ts +19 -6
- package/src/providers/integrations/buddy/capabilities.ts +2 -2
- package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +3 -0
- package/src/providers/integrations/buddy/codebuddy/provider.ts +4 -0
- package/src/providers/integrations/buddy/codebuddy/shared.ts +37 -22
- package/src/providers/integrations/claude/claude-quota.ts +14 -17
- package/src/providers/integrations/cline/cline.ts +2 -2
- package/src/providers/integrations/commandcode/quota.ts +22 -28
- package/src/providers/integrations/cursor/cursor-auth.ts +0 -8
- package/src/providers/integrations/cursor/cursor-exec.ts +9 -2
- package/src/providers/integrations/cursor/cursor-request.ts +26 -12
- package/src/providers/integrations/cursor/cursor-shared.ts +7 -4
- package/src/providers/integrations/cursor/cursor.ts +89 -40
- package/src/providers/integrations/cursor/protocol/messages.ts +22 -29
- package/src/providers/integrations/cursor/protocol/protobuf-schema.ts +37 -0
- package/src/providers/integrations/devin/protocol/devin-proto.ts +1 -27
- package/src/providers/integrations/factory/factory-oauth.ts +17 -44
- package/src/providers/integrations/freebuff/assets/official-system-manager.txt +88 -0
- package/src/providers/integrations/freebuff/assets/official-system-worker.txt +57 -0
- package/src/providers/integrations/freebuff/assets/official-tools.json +1070 -0
- package/src/providers/integrations/freebuff/freebuff-catalog.ts +120 -0
- package/src/providers/integrations/freebuff/freebuff-constants.ts +1233 -0
- package/src/providers/integrations/freebuff/freebuff-discovery.ts +108 -0
- package/src/providers/integrations/freebuff/freebuff-oauth.ts +240 -0
- package/src/providers/integrations/freebuff/freebuff-quota.ts +260 -0
- package/src/providers/integrations/freebuff/freebuff-virtual.ts +89 -0
- package/src/providers/integrations/freebuff/freebuff.ts +682 -0
- package/src/providers/integrations/grok/grok-oauth.ts +48 -45
- package/src/providers/integrations/grok/grok-ratelimit-store.ts +10 -17
- package/src/providers/integrations/grok/grok-turn-index.ts +11 -16
- package/src/providers/integrations/{hermes.ts → hermes/hermes.ts} +9 -9
- package/src/providers/integrations/hermes/index.ts +2 -0
- package/src/providers/integrations/kiro/kiro-catalog.ts +17 -1
- package/src/providers/integrations/kiro/kiro-discovery.ts +10 -12
- package/src/providers/integrations/kiro/kiro-quota.ts +8 -4
- package/src/providers/integrations/kiro/kiro.ts +15 -19
- package/src/providers/integrations/ollamacloud/index.ts +2 -0
- package/src/providers/integrations/{ollamacloud-models.ts → ollamacloud/models.ts} +2 -2
- package/src/providers/integrations/{ollamacloud-quota.ts → ollamacloud/quota.ts} +3 -3
- package/src/providers/integrations/opencode/opencode-go-quota.ts +2 -8
- package/src/providers/integrations/perch/perch-credential.ts +25 -0
- package/src/providers/integrations/perch/perch-protocol.ts +0 -3
- package/src/providers/integrations/perch/perch-quota.ts +2 -16
- package/src/providers/integrations/perch/perch.ts +3 -13
- package/src/providers/integrations/qoder/claim-qoder-credits.ts +18 -59
- package/src/providers/integrations/qoder/qoder-adapter.ts +1 -12
- package/src/providers/integrations/qoder/qoder-auth.ts +101 -29
- package/src/providers/integrations/qoder/qoder-catalog.ts +20 -5
- package/src/providers/integrations/qoder/qoder-chat.ts +7 -21
- package/src/providers/integrations/qoder/qoder-checkin.ts +4 -4
- package/src/providers/integrations/qoder/qoder-grpc.ts +0 -1
- package/src/providers/integrations/qoder/qoder-ide.ts +0 -107
- package/src/providers/integrations/qoder/qoder.ts +0 -1
- package/src/providers/integrations/trae/trae-protocol.ts +9 -5
- package/src/providers/integrations/trae/trae-quota.ts +6 -28
- package/src/providers/integrations/trae/trae-realm.ts +38 -0
- package/src/providers/integrations/trae/trae.ts +7 -24
- package/src/providers/integrations/zcode-start-plan/provider.ts +226 -23
- package/src/providers/integrations/zcode-start-plan/quota.ts +19 -2
- package/src/providers/integrations/zro/zro.ts +204 -0
- package/src/providers/operations/account-error-classification.ts +199 -307
- package/src/providers/operations/account-health-service.ts +82 -1
- package/src/providers/operations/account-refresh-status.ts +22 -37
- package/src/providers/operations/account-tier.ts +15 -16
- package/src/providers/operations/client-version-resolver.ts +1 -1
- package/src/providers/operations/client-versions.ts +5 -51
- package/src/providers/operations/cooldown-rules.ts +11 -16
- package/src/providers/operations/{model-catalog-cache.ts → provider-caches.ts} +68 -16
- package/src/providers/operations/provider-catalog-service.ts +1 -1
- package/src/providers/operations/provider-credential-service.ts +102 -32
- package/src/providers/operations/provider-rules/github.ts +48 -0
- package/src/providers/operations/provider-rules/grok.ts +33 -0
- package/src/providers/operations/provider-rules/inception.ts +49 -0
- package/src/providers/operations/provider-rules/index.ts +32 -0
- package/src/providers/operations/provider-rules/perch.ts +37 -0
- package/src/providers/operations/provider-rules/qoder.ts +70 -0
- package/src/providers/operations/provider-rules/trae.ts +81 -0
- package/src/providers/provider-contracts.ts +4 -2
- package/src/providers/{provider-credential-resolver.ts → provider-credentials.ts} +40 -3
- package/src/providers/provider-metadata.ts +40 -10
- package/src/providers/provider-probe.ts +15 -1
- package/src/providers/provider-registry.ts +42 -33
- package/src/providers/quota/quota-limit-windows.ts +16 -5
- package/src/providers/quota/quota-support.ts +1 -1
- package/src/providers/quota/quota-window-parser.ts +90 -18
- package/src/providers/quota/refresh.ts +75 -18
- package/src/security/api-key-auth-cache.ts +1 -1
- package/src/security/api-key-auth.ts +6 -2
- package/src/transport/dispatch/attempt-finalize.ts +14 -13
- package/src/transport/dispatch/attempt-loop.ts +71 -17
- package/src/transport/dispatch/generation.ts +2 -8
- package/src/transport/dispatch/native-shared.ts +25 -0
- package/src/transport/dispatch/proxy-request.ts +8 -46
- package/src/transport/dispatch/responses-compact.ts +2 -8
- package/src/transport/dispatch/streaming/finalize.ts +9 -6
- package/src/transport/dispatch/streaming/stream.ts +8 -36
- package/src/transport/dispatch/systemone.ts +2 -8
- package/src/transport/dispatch/token-count.ts +2 -6
- package/src/transport/dispatch/upstream.ts +63 -0
- package/src/transport/drain-endpoint.ts +31 -4
- package/src/transport/failure-policy.ts +16 -30
- package/src/transport/guards/body-policy.ts +7 -1
- package/src/transport/guards/error-lifecycle.ts +13 -1
- package/src/transport/guards/request-context.ts +22 -14
- package/src/transport/request/preparer.ts +81 -71
- package/src/transport/routing/bounded-lru.ts +14 -0
- package/src/transport/routing/drain-reserve.ts +129 -0
- package/src/transport/routing/plan-gate.ts +7 -1
- package/src/transport/routing/round-robin.ts +60 -0
- package/src/transport/routing/route-catalog.ts +123 -111
- package/src/transport/routing/route-model.ts +51 -9
- package/src/transport/routing/router.ts +393 -32
- package/src/transport/surface/adapters.ts +11 -2
- package/src/transport/surface/chat/adapter.ts +2 -2
- package/src/transport/surface/chat/encode.ts +47 -4
- package/src/transport/surface/chat/parse.ts +2 -12
- package/src/transport/surface/completion.ts +44 -33
- package/src/transport/surface/content-parts.ts +21 -0
- package/src/transport/surface/messages/adapter.ts +7 -1
- package/src/transport/surface/messages/encode.ts +11 -3
- package/src/transport/surface/messages/parse.ts +2 -1
- package/src/transport/surface/messages/stream.ts +33 -17
- package/src/transport/surface/responses/encode.ts +22 -5
- package/src/transport/surface/responses/parse.ts +3 -11
- package/dist/dashboard/assets/BannedPage-BxoG1MrL.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-ixwgKk53.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-BmttNV3U.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-BE3qwwfC.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-CRZdmMIX.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-Dv_FJ6Er.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-CoVrw2sM.js +0 -1
- package/dist/dashboard/assets/HealthEventsModal-CoVrw2sM.js.gz +0 -0
- package/dist/dashboard/assets/InjectSystemPromptPanel-BOoO8rpz.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-ChJuNNVN.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-BZAJtPg1.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-BE40s3u5.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-sfLoQtPV.js +0 -8
- package/dist/dashboard/assets/ProviderDetailPage-sfLoQtPV.js.gz +0 -0
- package/dist/dashboard/assets/ProviderIcon-CQ6bPUf_.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-DcepmPWo.js +0 -1
- package/dist/dashboard/assets/ProvidersPage-DcepmPWo.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-BJ87Pp0U.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-gkSyRAKa.js +0 -1
- package/dist/dashboard/assets/QuotaPage-gkSyRAKa.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-C5HPWqEv.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-CFNcATJJ.js.gz +0 -0
- package/dist/dashboard/assets/StudioPage-YoD4tY5l.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-DxjaAhb_.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-BTlkq_Uk.js.gz +0 -0
- package/dist/dashboard/assets/app-BX2nnKqs.css +0 -2
- package/dist/dashboard/assets/app-BX2nnKqs.css.gz +0 -0
- package/dist/dashboard/assets/assert-CndthXFQ.js +0 -1
- package/dist/dashboard/assets/assert-CndthXFQ.js.gz +0 -0
- package/dist/dashboard/assets/badge-DdbT_7Am.js.gz +0 -0
- package/dist/dashboard/assets/banner-CsGN-nbm.js.gz +0 -0
- package/dist/dashboard/assets/card-mgZ_TLLZ.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-A1-v8-03.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-CJ4Zty89.js.gz +0 -0
- package/dist/dashboard/assets/dialog-B5L2jPQ_.js.gz +0 -0
- package/dist/dashboard/assets/drawer-DkjUouAj.js.gz +0 -0
- package/dist/dashboard/assets/entry-B-LVLV6o.js.gz +0 -0
- package/dist/dashboard/assets/entry-CAhdYl3B.js.gz +0 -0
- package/dist/dashboard/assets/entry-D2xD0Tmd.js +0 -4
- package/dist/dashboard/assets/entry-D2xD0Tmd.js.gz +0 -0
- package/dist/dashboard/assets/formatters-CrsdLVsn.js.gz +0 -0
- package/dist/dashboard/assets/index-BU64Lqvp.js.gz +0 -0
- package/dist/dashboard/assets/inline-B2aDLyyg.js.gz +0 -0
- package/dist/dashboard/assets/input-BCVkdR9d.js.gz +0 -0
- package/dist/dashboard/assets/layout-DNcAdMK-.js.gz +0 -0
- package/dist/dashboard/assets/list-row-DgrIH0hw.js.gz +0 -0
- package/dist/dashboard/assets/live-Cae01iu8.js.gz +0 -0
- package/dist/dashboard/assets/network-CRib7H_2.js.gz +0 -0
- package/dist/dashboard/assets/numbers-BKQE8Xpz.js.gz +0 -0
- package/dist/dashboard/assets/pagination-ecPvG_8g.js.gz +0 -0
- package/dist/dashboard/assets/quota-DJ1QlAOq.js +0 -1
- package/dist/dashboard/assets/quota-DJ1QlAOq.js.gz +0 -0
- package/dist/dashboard/assets/routing-Ckm1UXuC.js.gz +0 -0
- package/dist/dashboard/assets/select-ChoFWyKz.js.gz +0 -0
- package/dist/dashboard/assets/settings-CKjFPlKW.js.gz +0 -0
- package/dist/dashboard/assets/stack-DWCwU3fh.js.gz +0 -0
- package/dist/dashboard/assets/state-Cg3VL5J_.js.gz +0 -0
- package/dist/dashboard/assets/switch-Db70BMix.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-BqdE0LuM.js.gz +0 -0
- package/src/providers/bundled-discovery-providers.ts +0 -98
- package/src/providers/configured-openai-providers.ts +0 -120
- package/src/providers/operations/model-discovery-cache.ts +0 -34
- package/src/providers/operations/probe-fetch.ts +0 -15
- package/src/providers/operations/provider-version-cache.ts +0 -30
- package/src/providers/provider-capabilities.ts +0 -33
- package/src/providers/provider-credential-envelope.ts +0 -43
- /package/dist/dashboard/assets/{app-B7kIWFm5.js → app-DIXLjojg.js} +0 -0
- /package/dist/dashboard/assets/{app-B7kIWFm5.js.gz → app-DIXLjojg.js.gz} +0 -0
|
@@ -13,7 +13,14 @@ import {
|
|
|
13
13
|
/** Shape returned for quota data on dashboard account responses. */
|
|
14
14
|
interface QuotaRefreshView {
|
|
15
15
|
readonly source: string;
|
|
16
|
-
|
|
16
|
+
/**
|
|
17
|
+
* `ready` = a usable quota reading exists (successful fetch, cached or
|
|
18
|
+
* snapshot). `error` = the collector returned an error with no usable
|
|
19
|
+
* reading. `refreshing` = a background fill is in flight. `unknown` = no
|
|
20
|
+
* quota evidence at all (no collector surface, or an empty successful read
|
|
21
|
+
* with no timestamp) — the canonical wire representation for unknown quota.
|
|
22
|
+
*/
|
|
23
|
+
readonly status: "ready" | "error" | "refreshing" | "unknown";
|
|
17
24
|
readonly plan: string | null;
|
|
18
25
|
readonly windows: ProviderQuotaResult["windows"];
|
|
19
26
|
readonly fetchedAt: string | null;
|
|
@@ -94,7 +101,7 @@ export function toQuotaView(
|
|
|
94
101
|
lastAttemptAt?: string | null;
|
|
95
102
|
lastSuccessAt?: string | null;
|
|
96
103
|
refreshError?: string | null;
|
|
97
|
-
status?: "ready" | "error" | "refreshing";
|
|
104
|
+
status?: "ready" | "error" | "refreshing" | "unknown";
|
|
98
105
|
} = {},
|
|
99
106
|
): QuotaRefreshView {
|
|
100
107
|
const fetchedAt = timestamps.fetchedAt ?? account.lastSuccessAt?.toISOString() ?? null;
|
|
@@ -112,9 +119,18 @@ export function toQuotaView(
|
|
|
112
119
|
// that the last attempt failed; only `quota.error` — no usable reading at
|
|
113
120
|
// all — makes the card an error.
|
|
114
121
|
const error = quota.error ?? null;
|
|
122
|
+
// Unknown quota: a successful read that carries no windows AND no timestamp
|
|
123
|
+
// is not evidence of anything — the canonical "no quota evidence" state.
|
|
124
|
+
// (A provider with a real zero-balance still reports windows/timestamps.)
|
|
125
|
+
const derivedStatus =
|
|
126
|
+
error !== null
|
|
127
|
+
? ("error" as const)
|
|
128
|
+
: quota.windows.length === 0 && fetchedAt === null
|
|
129
|
+
? ("unknown" as const)
|
|
130
|
+
: ("ready" as const);
|
|
115
131
|
return {
|
|
116
132
|
source: quota.source,
|
|
117
|
-
status: timestamps.status ??
|
|
133
|
+
status: timestamps.status ?? derivedStatus,
|
|
118
134
|
plan: quota.plan ?? null,
|
|
119
135
|
windows: quota.windows,
|
|
120
136
|
fetchedAt,
|
|
@@ -45,35 +45,40 @@ function isBlockedEntry(entry: { readonly blocked?: boolean }): boolean {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
export function createNetworkPoolOperations(config: NetworkPoolConfig) {
|
|
48
|
+
/**
|
|
49
|
+
* The read-only enrichment projection shared by the pool list and the pool
|
|
50
|
+
* detail page: live inflight (falling back to the stored row counter when no
|
|
51
|
+
* selector is configured), the derived available/utilization figures, and the
|
|
52
|
+
* active provider cooldowns (omitted entirely when there are none).
|
|
53
|
+
*/
|
|
54
|
+
async function enrichPoolResponse(res: NetworkPoolResponse, poolId: string): Promise<NetworkPoolResponse> {
|
|
55
|
+
const cooldowns = await config.poolSelector?.getPoolCooldowns(poolId);
|
|
56
|
+
const inflight = config.poolSelector
|
|
57
|
+
? await config.poolSelector.getInflightAuthoritative(poolId)
|
|
58
|
+
: res.inflight;
|
|
59
|
+
return {
|
|
60
|
+
...res,
|
|
61
|
+
inflight,
|
|
62
|
+
available: Math.max(0, res.maxInflight - inflight),
|
|
63
|
+
utilization: res.maxInflight > 0 ? Math.min(1, inflight / res.maxInflight) : 1,
|
|
64
|
+
...(cooldowns && cooldowns.length > 0
|
|
65
|
+
? {
|
|
66
|
+
providerCooldowns: cooldowns.map((c) => ({
|
|
67
|
+
providerId: c.providerId,
|
|
68
|
+
until: new Date(c.until).toISOString(),
|
|
69
|
+
reason: c.reason,
|
|
70
|
+
blocked: isBlockedEntry(c),
|
|
71
|
+
})),
|
|
72
|
+
}
|
|
73
|
+
: {}),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
48
77
|
const operations = {
|
|
49
78
|
async listPools(access: AccessDecision | undefined): Promise<NetworkPoolResponse[]> {
|
|
50
79
|
const a = requireTenantScope(access, "dashboard:read");
|
|
51
80
|
const rawPools = await config.store.list(a.tenantId);
|
|
52
|
-
return Promise.all(
|
|
53
|
-
rawPools.map(async (r) => {
|
|
54
|
-
const res = sanitizePoolResponse(r);
|
|
55
|
-
const cooldowns = await config.poolSelector?.getPoolCooldowns(r.id);
|
|
56
|
-
const inflight = config.poolSelector
|
|
57
|
-
? await config.poolSelector.getInflightAuthoritative(r.id)
|
|
58
|
-
: res.inflight;
|
|
59
|
-
return {
|
|
60
|
-
...res,
|
|
61
|
-
inflight,
|
|
62
|
-
available: Math.max(0, res.maxInflight - inflight),
|
|
63
|
-
utilization: res.maxInflight > 0 ? Math.min(1, inflight / res.maxInflight) : 1,
|
|
64
|
-
...(cooldowns && cooldowns.length > 0
|
|
65
|
-
? {
|
|
66
|
-
providerCooldowns: cooldowns.map((c) => ({
|
|
67
|
-
providerId: c.providerId,
|
|
68
|
-
until: new Date(c.until).toISOString(),
|
|
69
|
-
reason: c.reason,
|
|
70
|
-
blocked: isBlockedEntry(c),
|
|
71
|
-
})),
|
|
72
|
-
}
|
|
73
|
-
: {}),
|
|
74
|
-
};
|
|
75
|
-
}),
|
|
76
|
-
);
|
|
81
|
+
return Promise.all(rawPools.map(async (r) => enrichPoolResponse(sanitizePoolResponse(r), r.id)));
|
|
77
82
|
},
|
|
78
83
|
async getPoolDetail(
|
|
79
84
|
access: AccessDecision | undefined,
|
|
@@ -82,27 +87,7 @@ export function createNetworkPoolOperations(config: NetworkPoolConfig) {
|
|
|
82
87
|
const a = requireTenantScope(access, "dashboard:read");
|
|
83
88
|
const rec = await config.store.get(a.tenantId, poolId);
|
|
84
89
|
if (!rec) throw new ConsoleDomainError("pool_not_found", 404, `Pool ${poolId} not found`);
|
|
85
|
-
|
|
86
|
-
const cooldowns = await config.poolSelector?.getPoolCooldowns(poolId);
|
|
87
|
-
const inflight = config.poolSelector
|
|
88
|
-
? await config.poolSelector.getInflightAuthoritative(poolId)
|
|
89
|
-
: res.inflight;
|
|
90
|
-
return {
|
|
91
|
-
...res,
|
|
92
|
-
inflight,
|
|
93
|
-
available: Math.max(0, res.maxInflight - inflight),
|
|
94
|
-
utilization: res.maxInflight > 0 ? Math.min(1, inflight / res.maxInflight) : 1,
|
|
95
|
-
...(cooldowns && cooldowns.length > 0
|
|
96
|
-
? {
|
|
97
|
-
providerCooldowns: cooldowns.map((c) => ({
|
|
98
|
-
providerId: c.providerId,
|
|
99
|
-
until: new Date(c.until).toISOString(),
|
|
100
|
-
reason: c.reason,
|
|
101
|
-
blocked: isBlockedEntry(c),
|
|
102
|
-
})),
|
|
103
|
-
}
|
|
104
|
-
: {}),
|
|
105
|
-
};
|
|
90
|
+
return enrichPoolResponse(sanitizePoolResponse(rec), poolId);
|
|
106
91
|
},
|
|
107
92
|
async clearPoolCooldown(
|
|
108
93
|
access: AccessDecision | undefined,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ValidationError } from "elysia";
|
|
2
2
|
import type { AccessDecision, AccessScope } from "../../security/access-control";
|
|
3
3
|
import { isRecord } from "../../protocol/codecs";
|
|
4
|
-
import { formatPublicErrorMessage } from "../../protocol/gateway-error";
|
|
4
|
+
import { formatPublicErrorMessage, GatewayError } from "../../protocol/gateway-error";
|
|
5
5
|
|
|
6
6
|
// Single console error/access contract. Every domain throws ConsoleDomainError
|
|
7
7
|
// (or the storage-layer BackupError, same code/status/message shape) and
|
|
@@ -24,8 +24,27 @@ export class ConsoleDomainError extends Error {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
interface ShapedDomainError {
|
|
28
|
+
readonly code: string;
|
|
29
|
+
readonly status: number;
|
|
30
|
+
readonly message: string;
|
|
31
|
+
readonly details?: unknown;
|
|
32
|
+
/** Upstream/network origin, preserved so the dashboard can tell a provider
|
|
33
|
+
* rejection from a console-session expiry (see `maybeSignalSessionTransition`). */
|
|
34
|
+
readonly origin?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function asDomainError(error: unknown): ShapedDomainError | undefined {
|
|
28
38
|
if (error instanceof ConsoleDomainError) return error;
|
|
39
|
+
if (error instanceof GatewayError) {
|
|
40
|
+
return {
|
|
41
|
+
code: error.code,
|
|
42
|
+
status: error.status,
|
|
43
|
+
message: error.message,
|
|
44
|
+
details: error.details,
|
|
45
|
+
...(error.origin === "cartethyia" ? {} : { origin: error.origin }),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
29
48
|
if (
|
|
30
49
|
typeof error === "object" &&
|
|
31
50
|
error !== null &&
|
|
@@ -33,8 +52,14 @@ function asDomainError(error: unknown): { code: string; status: number; message:
|
|
|
33
52
|
typeof (error as { status?: unknown }).status === "number" &&
|
|
34
53
|
typeof (error as { message?: unknown }).message === "string"
|
|
35
54
|
) {
|
|
36
|
-
const shaped = error as { code: string; status: number; message: string; details?: unknown };
|
|
37
|
-
return {
|
|
55
|
+
const shaped = error as { code: string; status: number; message: string; details?: unknown; origin?: unknown };
|
|
56
|
+
return {
|
|
57
|
+
code: shaped.code,
|
|
58
|
+
status: shaped.status,
|
|
59
|
+
message: shaped.message,
|
|
60
|
+
details: shaped.details,
|
|
61
|
+
...(typeof shaped.origin === "string" && shaped.origin !== "cartethyia" ? { origin: shaped.origin } : {}),
|
|
62
|
+
};
|
|
38
63
|
}
|
|
39
64
|
return undefined;
|
|
40
65
|
}
|
|
@@ -47,6 +72,7 @@ export function errorResponse(
|
|
|
47
72
|
): {
|
|
48
73
|
error: string;
|
|
49
74
|
code: string;
|
|
75
|
+
origin?: string;
|
|
50
76
|
details?: Record<string, unknown>;
|
|
51
77
|
} {
|
|
52
78
|
const shaped = asDomainError(error);
|
|
@@ -63,16 +89,17 @@ export function errorResponse(
|
|
|
63
89
|
const body: Record<string, unknown> = {
|
|
64
90
|
error: message,
|
|
65
91
|
code: shaped.code,
|
|
92
|
+
...(shaped.origin === undefined ? {} : { origin: shaped.origin }),
|
|
66
93
|
};
|
|
67
94
|
if (includeDetails) body.details = details;
|
|
68
|
-
return body as { error: string; code: string; details?: Record<string, unknown> };
|
|
95
|
+
return body as { error: string; code: string; origin?: string; details?: Record<string, unknown> };
|
|
69
96
|
}
|
|
70
97
|
set.status = 500;
|
|
71
98
|
const body: Record<string, unknown> = {
|
|
72
99
|
error: formatPublicErrorMessage("internal_error", fallbackMessage),
|
|
73
100
|
code: "internal_error",
|
|
74
101
|
};
|
|
75
|
-
return body as { error: string; code: string; details?: Record<string, unknown> };
|
|
102
|
+
return body as { error: string; code: string; origin?: string; details?: Record<string, unknown> };
|
|
76
103
|
}
|
|
77
104
|
|
|
78
105
|
export function requireScope(
|
package/src/main.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import "./boot/elysia-typebox-bridge";
|
|
4
4
|
import { createGatewayApp, createGatewayShell } from "./boot/app";
|
|
5
5
|
import { bootstrap } from "./boot/lifecycle";
|
|
6
|
+
import { flushAllBuffers } from "./boot/flush";
|
|
6
7
|
import type { CartethyiaBoot } from "./boot/lifecycle";
|
|
7
8
|
import {
|
|
8
9
|
resolveDrainToken,
|
|
@@ -90,9 +91,7 @@ const app = boot
|
|
|
90
91
|
// for the queues, minus the stop.
|
|
91
92
|
flushBuffers: {
|
|
92
93
|
flush: async () => {
|
|
93
|
-
await boot.deps
|
|
94
|
-
await boot.deps.usageFlush.flush();
|
|
95
|
-
await boot.deps.healthEventFlush.flush();
|
|
94
|
+
await flushAllBuffers(boot.deps);
|
|
96
95
|
},
|
|
97
96
|
},
|
|
98
97
|
},
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
* byte, and nothing is allocated.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
+
import { evictOldestEntry } from "../../transport/routing/bounded-lru";
|
|
17
|
+
|
|
16
18
|
/** Running totals for one pool. */
|
|
17
19
|
interface PoolByteTotals {
|
|
18
20
|
/** Bytes sent to the proxy (uploads, TLS handshake, CONNECT). */
|
|
@@ -33,8 +35,7 @@ function entry(poolId: string): { sent: number; received: number } {
|
|
|
33
35
|
let current = totals.get(poolId);
|
|
34
36
|
if (!current) {
|
|
35
37
|
if (totals.size >= MAX_BYTE_ENTRY_COUNT) {
|
|
36
|
-
|
|
37
|
-
if (oldest !== undefined) totals.delete(oldest);
|
|
38
|
+
evictOldestEntry(totals, MAX_BYTE_ENTRY_COUNT - 1);
|
|
38
39
|
}
|
|
39
40
|
current = { sent: 0, received: 0 };
|
|
40
41
|
totals.set(poolId, current);
|
|
@@ -108,6 +108,13 @@ type PoolSelectionFailureReason =
|
|
|
108
108
|
| "coordination_unavailable"
|
|
109
109
|
| "no_active_pool";
|
|
110
110
|
|
|
111
|
+
interface PoolCooldownState {
|
|
112
|
+
readonly inCooldown: boolean;
|
|
113
|
+
readonly resetsAt: Date | null;
|
|
114
|
+
readonly reason: string | null;
|
|
115
|
+
readonly blocked: boolean;
|
|
116
|
+
}
|
|
117
|
+
|
|
111
118
|
interface PoolCapacitySnapshot {
|
|
112
119
|
readonly poolId: string;
|
|
113
120
|
readonly maxInflight: number;
|
|
@@ -399,12 +406,7 @@ export class NetworkPoolSelector {
|
|
|
399
406
|
async isProviderCooldown(
|
|
400
407
|
poolId: string,
|
|
401
408
|
providerId: string,
|
|
402
|
-
): Promise<{
|
|
403
|
-
inCooldown: boolean;
|
|
404
|
-
resetsAt: Date | null;
|
|
405
|
-
reason: string | null;
|
|
406
|
-
blocked: boolean;
|
|
407
|
-
}> {
|
|
409
|
+
): Promise<PoolCooldownState> {
|
|
408
410
|
const k = this.cooldownKey(poolId, providerId);
|
|
409
411
|
const local = this.state.cooldowns.get(k);
|
|
410
412
|
if (local) {
|
|
@@ -880,6 +882,171 @@ export class NetworkPoolSelector {
|
|
|
880
882
|
}
|
|
881
883
|
return undefined;
|
|
882
884
|
}
|
|
885
|
+
|
|
886
|
+
/**
|
|
887
|
+
* Infight counter from a Redis `MGET` row. `null` (unset counter) is zero;
|
|
888
|
+
* a malformed counter is the same hard error the per-key read raises, so a
|
|
889
|
+
* corrupt key is never silently treated as 0.
|
|
890
|
+
*/
|
|
891
|
+
private parseInflightCounter(raw: string | null, what: string): number {
|
|
892
|
+
if (raw === null) return 0;
|
|
893
|
+
const value = Number(raw);
|
|
894
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
895
|
+
throw new Error(`invalid pool inflight counter: ${what}`);
|
|
896
|
+
}
|
|
897
|
+
return value;
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* Per-pool cooldown resolution with the same semantics as
|
|
902
|
+
* `isProviderCooldown`: the local cooldown map wins when it has a live
|
|
903
|
+
* entry; otherwise a non-expired Redis row is admitted (bounded) and an
|
|
904
|
+
* expired Redis row is cleared. Used by the batched failure path, which
|
|
905
|
+
* reads all cooldown keys in one `MGET` and resolves each row here.
|
|
906
|
+
*/
|
|
907
|
+
private cooldownFromBatchState(
|
|
908
|
+
poolId: string,
|
|
909
|
+
providerId: string,
|
|
910
|
+
raw: string | null,
|
|
911
|
+
): PoolCooldownState {
|
|
912
|
+
const k = this.cooldownKey(poolId, providerId);
|
|
913
|
+
const local = this.state.cooldowns.get(k);
|
|
914
|
+
if (local) {
|
|
915
|
+
if (Date.now() < local.until) {
|
|
916
|
+
return {
|
|
917
|
+
inCooldown: true,
|
|
918
|
+
resetsAt: new Date(local.until),
|
|
919
|
+
reason: local.reason,
|
|
920
|
+
blocked: local.blocked === true,
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
this.state.cooldowns.delete(k);
|
|
924
|
+
}
|
|
925
|
+
const parsed = raw ? parseCooldownEntry(raw) : undefined;
|
|
926
|
+
if (parsed && Date.now() < parsed.until) {
|
|
927
|
+
enforceCooldownLimit(this.state.cooldowns);
|
|
928
|
+
this.state.cooldowns.set(k, parsed);
|
|
929
|
+
return {
|
|
930
|
+
inCooldown: true,
|
|
931
|
+
resetsAt: new Date(parsed.until),
|
|
932
|
+
reason: parsed.reason,
|
|
933
|
+
blocked: parsed.blocked === true,
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
if (raw) {
|
|
937
|
+
void this.clearRedisCooldown(poolId, providerId);
|
|
938
|
+
}
|
|
939
|
+
return { inCooldown: false, resetsAt: null, reason: null, blocked: false };
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* The original per-pool failure check: one Redis read per pool for each of
|
|
944
|
+
* cooldown, pool inflight, and provider inflight. Retained as the fallback
|
|
945
|
+
* for the non-Redis path and for a Redis outage during the batched read.
|
|
946
|
+
*/
|
|
947
|
+
private async perPoolSelectionChecks(
|
|
948
|
+
eligiblePoolIds: readonly string[],
|
|
949
|
+
providerId: string,
|
|
950
|
+
limitsByPool?: Record<string, number>,
|
|
951
|
+
weightsByPool?: Record<string, number>,
|
|
952
|
+
providerCapacity?: number,
|
|
953
|
+
): Promise<Array<{ cooldown: PoolCooldownState; pool: PoolCapacitySnapshot }>> {
|
|
954
|
+
return Promise.all(
|
|
955
|
+
eligiblePoolIds.map(async (poolId) => {
|
|
956
|
+
const cooldown = await this.isProviderCooldown(poolId, providerId);
|
|
957
|
+
const maxInflight = effectiveConcurrency(
|
|
958
|
+
limitsByPool?.[poolId] ?? DEFAULT_PROXY_CONCURRENCY,
|
|
959
|
+
);
|
|
960
|
+
const weight = normalizedWeight(weightsByPool?.[poolId]);
|
|
961
|
+
const currentInflight = await this.getInflightAuthoritative(poolId);
|
|
962
|
+
const providerInflight =
|
|
963
|
+
providerCapacity === undefined
|
|
964
|
+
? undefined
|
|
965
|
+
: await this.getProviderInflightAuthoritative(poolId, providerId);
|
|
966
|
+
return {
|
|
967
|
+
cooldown,
|
|
968
|
+
pool: {
|
|
969
|
+
poolId,
|
|
970
|
+
maxInflight,
|
|
971
|
+
currentInflight,
|
|
972
|
+
available: Math.max(0, maxInflight - currentInflight),
|
|
973
|
+
weight,
|
|
974
|
+
...(cooldown.resetsAt ? { retryAt: cooldown.resetsAt.getTime() } : {}),
|
|
975
|
+
...(cooldown.blocked ? { blocked: true } : {}),
|
|
976
|
+
...(providerCapacity === undefined
|
|
977
|
+
? {}
|
|
978
|
+
: { providerMaxInflight: providerCapacity, providerInflight: providerInflight ?? 0 }),
|
|
979
|
+
} satisfies PoolCapacitySnapshot,
|
|
980
|
+
};
|
|
981
|
+
}),
|
|
982
|
+
);
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* Batched failure check: one `MGET` per key family (cooldowns, pool
|
|
987
|
+
* inflight, provider inflight) instead of three serial GETs per pool. On a
|
|
988
|
+
* Redis error the whole read falls back to per-pool reads so the failure
|
|
989
|
+
* report keeps the "cooldown store unavailable" per-pool detail.
|
|
990
|
+
*/
|
|
991
|
+
private async batchSelectionChecks(
|
|
992
|
+
eligiblePoolIds: readonly string[],
|
|
993
|
+
providerId: string,
|
|
994
|
+
limitsByPool?: Record<string, number>,
|
|
995
|
+
weightsByPool?: Record<string, number>,
|
|
996
|
+
providerCapacity?: number,
|
|
997
|
+
): Promise<Array<{ cooldown: PoolCooldownState; pool: PoolCapacitySnapshot }>> {
|
|
998
|
+
const cooldownKeys = eligiblePoolIds.map((poolId) => proxyCooldownKey(poolId, providerId));
|
|
999
|
+
const inflightKeys = eligiblePoolIds.map((poolId) => proxyInflightKey(poolId));
|
|
1000
|
+
const providerInflightKeys =
|
|
1001
|
+
providerCapacity === undefined
|
|
1002
|
+
? []
|
|
1003
|
+
: eligiblePoolIds.map((poolId) => proxyProviderInflightKey(poolId, providerId));
|
|
1004
|
+
try {
|
|
1005
|
+
const [cooldownRaws, inflightRaws, providerRaws] = await Promise.all([
|
|
1006
|
+
this.redis!.mget(...cooldownKeys),
|
|
1007
|
+
this.redis!.mget(...inflightKeys),
|
|
1008
|
+
providerInflightKeys.length === 0
|
|
1009
|
+
? Promise.resolve([] as Array<string | null>)
|
|
1010
|
+
: this.redis!.mget(...providerInflightKeys),
|
|
1011
|
+
]);
|
|
1012
|
+
return eligiblePoolIds.map((poolId, index) => {
|
|
1013
|
+
const cooldown = this.cooldownFromBatchState(poolId, providerId, cooldownRaws[index] ?? null);
|
|
1014
|
+
const maxInflight = effectiveConcurrency(
|
|
1015
|
+
limitsByPool?.[poolId] ?? DEFAULT_PROXY_CONCURRENCY,
|
|
1016
|
+
);
|
|
1017
|
+
const weight = normalizedWeight(weightsByPool?.[poolId]);
|
|
1018
|
+
const currentInflight = this.parseInflightCounter(inflightRaws[index] ?? null, poolId);
|
|
1019
|
+
const providerInflight =
|
|
1020
|
+
providerCapacity === undefined
|
|
1021
|
+
? undefined
|
|
1022
|
+
: this.parseInflightCounter(providerRaws[index] ?? null, `${poolId}/${providerId}`);
|
|
1023
|
+
return {
|
|
1024
|
+
cooldown,
|
|
1025
|
+
pool: {
|
|
1026
|
+
poolId,
|
|
1027
|
+
maxInflight,
|
|
1028
|
+
currentInflight,
|
|
1029
|
+
available: Math.max(0, maxInflight - currentInflight),
|
|
1030
|
+
weight,
|
|
1031
|
+
...(cooldown.resetsAt ? { retryAt: cooldown.resetsAt.getTime() } : {}),
|
|
1032
|
+
...(cooldown.blocked ? { blocked: true } : {}),
|
|
1033
|
+
...(providerCapacity === undefined
|
|
1034
|
+
? {}
|
|
1035
|
+
: { providerMaxInflight: providerCapacity, providerInflight: providerInflight ?? 0 }),
|
|
1036
|
+
} satisfies PoolCapacitySnapshot,
|
|
1037
|
+
};
|
|
1038
|
+
});
|
|
1039
|
+
} catch {
|
|
1040
|
+
return this.perPoolSelectionChecks(
|
|
1041
|
+
eligiblePoolIds,
|
|
1042
|
+
providerId,
|
|
1043
|
+
limitsByPool,
|
|
1044
|
+
weightsByPool,
|
|
1045
|
+
providerCapacity,
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
883
1050
|
async getSelectionFailure(
|
|
884
1051
|
eligiblePoolIds: readonly string[],
|
|
885
1052
|
providerId: string,
|
|
@@ -891,39 +1058,15 @@ export class NetworkPoolSelector {
|
|
|
891
1058
|
const providerCapacity =
|
|
892
1059
|
providerMaxInflight === undefined ? undefined : effectiveConcurrency(providerMaxInflight);
|
|
893
1060
|
try {
|
|
894
|
-
// One pass
|
|
895
|
-
//
|
|
896
|
-
//
|
|
897
|
-
//
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
);
|
|
904
|
-
const weight = normalizedWeight(weightsByPool?.[poolId]);
|
|
905
|
-
const currentInflight = await this.getInflightAuthoritative(poolId);
|
|
906
|
-
const providerInflight =
|
|
907
|
-
providerCapacity === undefined
|
|
908
|
-
? undefined
|
|
909
|
-
: await this.getProviderInflightAuthoritative(poolId, providerId);
|
|
910
|
-
return {
|
|
911
|
-
cooldown,
|
|
912
|
-
pool: {
|
|
913
|
-
poolId,
|
|
914
|
-
maxInflight,
|
|
915
|
-
currentInflight,
|
|
916
|
-
available: Math.max(0, maxInflight - currentInflight),
|
|
917
|
-
weight,
|
|
918
|
-
...(cooldown.resetsAt ? { retryAt: cooldown.resetsAt.getTime() } : {}),
|
|
919
|
-
...(cooldown.blocked ? { blocked: true } : {}),
|
|
920
|
-
...(providerCapacity === undefined
|
|
921
|
-
? {}
|
|
922
|
-
: { providerMaxInflight: providerCapacity, providerInflight: providerInflight ?? 0 }),
|
|
923
|
-
} satisfies PoolCapacitySnapshot,
|
|
924
|
-
};
|
|
925
|
-
}),
|
|
926
|
-
);
|
|
1061
|
+
// One batch pass instead of per-pool Redis reads: cooldowns and inflight
|
|
1062
|
+
// counters are read in one `MGET` per key family (three total), not
|
|
1063
|
+
// three serial GETs per pool. The per-pool resolution below consumes the
|
|
1064
|
+
// raw rows with the same semantics `isProviderCooldown` /
|
|
1065
|
+
// `getInflightAuthoritative` / `getProviderInflightAuthoritative` apply.
|
|
1066
|
+
const checks =
|
|
1067
|
+
this.redis === undefined
|
|
1068
|
+
? await this.perPoolSelectionChecks(eligiblePoolIds, providerId, limitsByPool, weightsByPool, providerCapacity)
|
|
1069
|
+
: await this.batchSelectionChecks(eligiblePoolIds, providerId, limitsByPool, weightsByPool, providerCapacity);
|
|
927
1070
|
const pools = checks.map((check) => check.pool);
|
|
928
1071
|
const cooldowns = checks.map((check) => check.cooldown);
|
|
929
1072
|
if (cooldowns.some((entry) => entry.reason === "cooldown store unavailable")) {
|
|
@@ -17,6 +17,7 @@ import { eq } from "drizzle-orm";
|
|
|
17
17
|
import type { CartethyiaDatabase } from "../persistence/db-handle";
|
|
18
18
|
import { networkPools, healthEvents } from "../persistence/schema";
|
|
19
19
|
import { log } from "../observability/logger";
|
|
20
|
+
import { metrics } from "../observability/metrics";
|
|
20
21
|
import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
|
|
21
22
|
import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
|
|
22
23
|
import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
|
|
@@ -152,6 +153,88 @@ export async function pruneExpiredProviderCooldownMarkers(
|
|
|
152
153
|
}
|
|
153
154
|
}
|
|
154
155
|
|
|
156
|
+
/** Max simultaneously tracked (pool, provider) coalescing keys. */
|
|
157
|
+
const MAX_TRACKED_PAIRS = 5_000;
|
|
158
|
+
|
|
159
|
+
/** Last coalesced row timestamp per (pool, provider), for the cooldown window only. */
|
|
160
|
+
const lastRecordedByPoolProvider = new Map<string, number>();
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Appends one provider-scoped egress verdict row to `health_events` — the
|
|
164
|
+
* durable audit log the console's pool page reads.
|
|
165
|
+
*
|
|
166
|
+
* Shared by the three writers of this row shape: the dispatch-side 429
|
|
167
|
+
* recorder (`recordPoolCooldownEvent`), the egress-block recorder
|
|
168
|
+
* (`recordPoolBlockEvent`), and the probe verdict recorder
|
|
169
|
+
* (`recordProbeVerdict`). The row is always `entityKind: "pool"`,
|
|
170
|
+
* `fromStatus: null`, `toStatus: "cooldown"`, and the category is chosen by
|
|
171
|
+
* `blocked`; the only divergences are the optional code suffix (block path)
|
|
172
|
+
* and the optional coalescing window (cooldown path).
|
|
173
|
+
*
|
|
174
|
+
* Coalescing: `coalesceWindowMs` suppresses a duplicate row for the same
|
|
175
|
+
* (pool, provider) pair within the window — used by the cooldown path, where
|
|
176
|
+
* dispatch flags the same pair on every throttled request and the audit log
|
|
177
|
+
* needs one row per episode, not one per request. The default `undefined`
|
|
178
|
+
* means no coalescing, so block and probe rows are never deduped; it also
|
|
179
|
+
* means the pair never enters the tracking map, so the eviction sweep below
|
|
180
|
+
* only ever sees coalesced pairs. A failed insert does not stamp the pair, so
|
|
181
|
+
* the next attempt within the window is still tried.
|
|
182
|
+
*
|
|
183
|
+
* Swallow contract: this never throws to dispatch (observability writes are
|
|
184
|
+
* non-fatal); failures bump `pool_cooldown_record_failed` and are logged.
|
|
185
|
+
*/
|
|
186
|
+
export async function appendPoolProviderEgressAuditRow(
|
|
187
|
+
db: CartethyiaDatabase,
|
|
188
|
+
opts: {
|
|
189
|
+
poolId: string;
|
|
190
|
+
providerId: string;
|
|
191
|
+
reason: string;
|
|
192
|
+
blocked: boolean;
|
|
193
|
+
codeSuffix?: string;
|
|
194
|
+
coalesceWindowMs?: number;
|
|
195
|
+
},
|
|
196
|
+
): Promise<void> {
|
|
197
|
+
const { poolId, providerId, reason, blocked, codeSuffix, coalesceWindowMs } = opts;
|
|
198
|
+
let key: string | undefined;
|
|
199
|
+
let now = 0;
|
|
200
|
+
if (coalesceWindowMs !== undefined) {
|
|
201
|
+
key = `${poolId}::${providerId.toLowerCase()}`;
|
|
202
|
+
now = Date.now();
|
|
203
|
+
const last = lastRecordedByPoolProvider.get(key);
|
|
204
|
+
if (last !== undefined && now - last < coalesceWindowMs) return;
|
|
205
|
+
}
|
|
206
|
+
try {
|
|
207
|
+
await db.insert(healthEvents).values({
|
|
208
|
+
id: randomUUID(),
|
|
209
|
+
entityKind: "pool",
|
|
210
|
+
networkPoolId: poolId,
|
|
211
|
+
fromStatus: null,
|
|
212
|
+
toStatus: "cooldown",
|
|
213
|
+
reason: `provider ${providerId}: ${reason}${codeSuffix ?? ""}`,
|
|
214
|
+
errorCategory: blocked ? "egress_blocked" : "rate_limit_transient",
|
|
215
|
+
createdAt: new Date(),
|
|
216
|
+
});
|
|
217
|
+
} catch (error) {
|
|
218
|
+
metrics.pool_cooldown_record_failed.inc(1);
|
|
219
|
+
log.warn("pool provider egress audit row failed", {
|
|
220
|
+
networkPoolId: poolId,
|
|
221
|
+
providerId,
|
|
222
|
+
error: String(error),
|
|
223
|
+
});
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (key !== undefined && coalesceWindowMs !== undefined) {
|
|
227
|
+
lastRecordedByPoolProvider.set(key, now);
|
|
228
|
+
if (lastRecordedByPoolProvider.size > MAX_TRACKED_PAIRS) {
|
|
229
|
+
for (const [trackedKey, recordedAt] of lastRecordedByPoolProvider) {
|
|
230
|
+
if (now - recordedAt >= coalesceWindowMs) {
|
|
231
|
+
lastRecordedByPoolProvider.delete(trackedKey);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
155
238
|
/**
|
|
156
239
|
* Writes a probe-measured verdict the same way a dispatch-side event would be
|
|
157
240
|
* recorded: one mirror marker and one audit row.
|
|
@@ -174,20 +257,12 @@ export async function recordProbeVerdict(
|
|
|
174
257
|
marker: { until: number; reason: string; blocked?: boolean },
|
|
175
258
|
): Promise<void> {
|
|
176
259
|
await upsertProviderCooldownMarker(db, poolId, providerId, marker);
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
toStatus: "cooldown",
|
|
184
|
-
reason: `provider ${providerId}: ${marker.reason}`,
|
|
185
|
-
errorCategory: marker.blocked === true ? "egress_blocked" : "rate_limit_transient",
|
|
186
|
-
createdAt: new Date(),
|
|
187
|
-
});
|
|
188
|
-
} catch (error) {
|
|
189
|
-
log.warn("pool probe event failed", { networkPoolId: poolId, providerId, error: String(error) });
|
|
190
|
-
}
|
|
260
|
+
await appendPoolProviderEgressAuditRow(db, {
|
|
261
|
+
poolId,
|
|
262
|
+
providerId,
|
|
263
|
+
reason: marker.reason,
|
|
264
|
+
blocked: marker.blocked === true,
|
|
265
|
+
});
|
|
191
266
|
}
|
|
192
267
|
|
|
193
268
|
/**
|