cartethyia 2.7.4 → 2.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/README.md +17 -5
- package/dist/dashboard/assets/{BannedPage-ChG5svFb.js → BannedPage-Cj15GsMp.js} +1 -1
- package/dist/dashboard/assets/BannedPage-Cj15GsMp.js.gz +0 -0
- package/dist/dashboard/assets/ClaimGiftsButton-D83pBJ6J.js +1 -0
- package/dist/dashboard/assets/ClaimGiftsButton-D83pBJ6J.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolDetailPage-BIcWQun7.js → CliToolDetailPage-DZKmVuJN.js} +1 -1
- package/dist/dashboard/assets/CliToolDetailPage-DZKmVuJN.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-Bf2VelDe.js → CliToolsPage-BIccZmeL.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-BIccZmeL.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-DEEKkYD-.js +1 -0
- package/dist/dashboard/assets/CombosPage-DEEKkYD-.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-D5rol2j2.js +1 -0
- package/dist/dashboard/assets/ConsoleLogPage-D5rol2j2.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-ggeMSjte.js → CustomizationPage-BKj_r0rp.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-BKj_r0rp.js.gz +0 -0
- package/dist/dashboard/assets/InjectSystemPromptPanel-B5nZvmf0.js +656 -0
- package/dist/dashboard/assets/InjectSystemPromptPanel-B5nZvmf0.js.gz +0 -0
- package/dist/dashboard/assets/{LoginPage-U6AKaJcF.js → LoginPage-DkXb6dx-.js} +1 -1
- package/dist/dashboard/assets/LoginPage-DkXb6dx-.js.gz +0 -0
- package/dist/dashboard/assets/{ModelPicker-BMS2t8BA.js → ModelPicker-DnQPSIA8.js} +1 -1
- package/dist/dashboard/assets/ModelPicker-DnQPSIA8.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-Ci2wQKUz.js +1 -0
- package/dist/dashboard/assets/OverviewPage-Ci2wQKUz.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-odlgYrOg.js +5 -0
- package/dist/dashboard/assets/ProviderDetailPage-odlgYrOg.js.gz +0 -0
- package/dist/dashboard/assets/ProviderIcon-EfeloBn4.js +1 -0
- package/dist/dashboard/assets/ProviderIcon-EfeloBn4.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-CqFBQEB8.js +1 -0
- package/dist/dashboard/assets/ProvidersPage-CqFBQEB8.js.gz +0 -0
- package/dist/dashboard/assets/{ProxyPage-lsrx7NYg.js → ProxyPage-5SnUSy3j.js} +1 -1
- package/dist/dashboard/assets/ProxyPage-5SnUSy3j.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-CmnMDzhP.js +1 -0
- package/dist/dashboard/assets/QuotaPage-CmnMDzhP.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-BJtRZ4qD.js +1 -0
- package/dist/dashboard/assets/SettingsPage-BJtRZ4qD.js.gz +0 -0
- package/dist/dashboard/assets/{SetupPage-f0DE1h0-.js → SetupPage-WTa0sktC.js} +1 -1
- package/dist/dashboard/assets/SetupPage-WTa0sktC.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-Cq5tW7hW.js → StudioPage-BfAb5-o_.js} +3 -3
- package/dist/dashboard/assets/StudioPage-BfAb5-o_.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-BNzeZ2Vk.js +56 -0
- package/dist/dashboard/assets/UsagePage-BNzeZ2Vk.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-l29JROPv.js +1 -0
- package/dist/dashboard/assets/api-keys-l29JROPv.js.gz +0 -0
- package/dist/dashboard/assets/app--PJWq8_C.css +2 -0
- package/dist/dashboard/assets/app--PJWq8_C.css.gz +0 -0
- package/dist/dashboard/assets/assert-C0K-c4eo.js +1 -0
- package/dist/dashboard/assets/assert-C0K-c4eo.js.gz +0 -0
- package/dist/dashboard/assets/{badge-D9CS6dZk.js → badge-DfbhQU4k.js} +1 -1
- package/dist/dashboard/assets/badge-DfbhQU4k.js.gz +0 -0
- package/dist/dashboard/assets/{banner-DympBNq4.js → banner-BllT4LcF.js} +1 -1
- package/dist/dashboard/assets/banner-BllT4LcF.js.gz +0 -0
- package/dist/dashboard/assets/brand/login_register.jpg +0 -0
- package/dist/dashboard/assets/brand/login_register_initial.jpg +0 -0
- package/dist/dashboard/assets/brand/og_bansos.jpg +0 -0
- package/dist/dashboard/assets/brand/og_image.jpg +0 -0
- package/dist/dashboard/assets/{card-ibRWU_uT.js → card-CtkX66Wd.js} +1 -1
- package/dist/dashboard/assets/card-CtkX66Wd.js.gz +0 -0
- package/dist/dashboard/assets/chevron-left-Bf6CaTSJ.js +1 -0
- package/dist/dashboard/assets/chevron-left-Bf6CaTSJ.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-CQXVMn4P.js +1 -0
- package/dist/dashboard/assets/cli-tools-CQXVMn4P.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-DJm-9pz7.js → clipboard-button-BQqBQd2n.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-BQqBQd2n.js.gz +0 -0
- package/dist/dashboard/assets/{dialog-8fiE6tec.js → dialog-BEK7KP0H.js} +1 -1
- package/dist/dashboard/assets/dialog-BEK7KP0H.js.gz +0 -0
- package/dist/dashboard/assets/{drawer-CqGYeOrd.js → drawer-BrTEZ8zu.js} +1 -1
- package/dist/dashboard/assets/drawer-BrTEZ8zu.js.gz +0 -0
- package/dist/dashboard/assets/entry-CHWxbS56.js +1 -0
- package/dist/dashboard/assets/entry-CHWxbS56.js.gz +0 -0
- package/dist/dashboard/assets/{entry-BSAsYsmV.js → entry-EBHJ3DrE.js} +1 -1
- package/dist/dashboard/assets/entry-EBHJ3DrE.js.gz +0 -0
- package/dist/dashboard/assets/entry-LCS85hiq.js +4 -0
- package/dist/dashboard/assets/entry-LCS85hiq.js.gz +0 -0
- package/dist/dashboard/assets/{formatters-D-MTJxMr.js → formatters-BAACA727.js} +1 -1
- package/dist/dashboard/assets/formatters-BAACA727.js.gz +0 -0
- package/dist/dashboard/assets/{github-badge-CPK47xQx.js → github-badge-YSCE4UC2.js} +1 -1
- package/dist/dashboard/assets/github-badge-YSCE4UC2.js.gz +0 -0
- package/dist/dashboard/assets/{index-DBNPOfhV.js → index-FQlj0HUT.js} +2 -2
- package/dist/dashboard/assets/index-FQlj0HUT.js.gz +0 -0
- package/dist/dashboard/assets/{inline-BT0zntEr.js → inline-C4gU5iOL.js} +1 -1
- package/dist/dashboard/assets/inline-C4gU5iOL.js.gz +0 -0
- package/dist/dashboard/assets/{input-Da2ZViWg.js → input-BHHX_Tmm.js} +1 -1
- package/dist/dashboard/assets/input-BHHX_Tmm.js.gz +0 -0
- package/dist/dashboard/assets/{layout-BXNeLw6M.js → layout-CajrYUnD.js} +1 -1
- package/dist/dashboard/assets/layout-CajrYUnD.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-BycOrrIV.js → list-row-Bx2I7XUe.js} +1 -1
- package/dist/dashboard/assets/list-row-Bx2I7XUe.js.gz +0 -0
- package/dist/dashboard/assets/live-DNNqRbj2.js +1 -0
- package/dist/dashboard/assets/live-DNNqRbj2.js.gz +0 -0
- package/dist/dashboard/assets/network-C5UBACrJ.js +1 -0
- package/dist/dashboard/assets/network-C5UBACrJ.js.gz +0 -0
- package/dist/dashboard/assets/numbers-CUWAZK4J.js +2 -0
- package/dist/dashboard/assets/numbers-CUWAZK4J.js.gz +0 -0
- package/dist/dashboard/assets/pagination-Xsi9Qx04.js +1 -0
- package/dist/dashboard/assets/pagination-Xsi9Qx04.js.gz +0 -0
- package/dist/dashboard/assets/quota-DvQbD_sr.js +1 -0
- package/dist/dashboard/assets/quota-DvQbD_sr.js.gz +0 -0
- package/dist/dashboard/assets/routing-in0f_IfK.js +1 -0
- package/dist/dashboard/assets/routing-in0f_IfK.js.gz +0 -0
- package/dist/dashboard/assets/{select-BIYIBkMU.js → select-BLULNqTp.js} +1 -1
- package/dist/dashboard/assets/select-BLULNqTp.js.gz +0 -0
- package/dist/dashboard/assets/settings-CcGczj6I.js +1 -0
- package/dist/dashboard/assets/settings-CcGczj6I.js.gz +0 -0
- package/dist/dashboard/assets/shield-BMQNxORh.js +1 -0
- package/dist/dashboard/assets/shield-BMQNxORh.js.gz +0 -0
- package/dist/dashboard/assets/{stack-DfgnFDWY.js → stack-D2lIR0tg.js} +1 -1
- package/dist/dashboard/assets/stack-D2lIR0tg.js.gz +0 -0
- package/dist/dashboard/assets/{state-CkfdYWel.js → state-DTARjfiW.js} +1 -1
- package/dist/dashboard/assets/state-DTARjfiW.js.gz +0 -0
- package/dist/dashboard/assets/{switch-SYm237HG.js → switch-c0oPSnqW.js} +1 -1
- package/dist/dashboard/assets/switch-c0oPSnqW.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-BQqCRNIi.js → useMutation-m8WGNPrz.js} +1 -1
- package/dist/dashboard/assets/useMutation-m8WGNPrz.js.gz +0 -0
- package/dist/dashboard/console-backgrounds.jpg +0 -0
- package/dist/dashboard/console-backgrounds_initial.jpg +0 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +56 -55
- package/migrations/full-postgres/0036_system_prompt_presets_refs.sql +10 -0
- package/migrations/full-postgres/0037_share_concurrency_scope.sql +3 -0
- package/migrations/full-postgres/0038_add_accounts_quota_snapshot.sql +7 -0
- package/migrations/full-postgres/0039_add_telemetry_provider_model_indexes.sql +13 -0
- package/migrations/full-postgres/0040_add_models_dialect.sql +3 -0
- package/migrations/full-postgres/0041_add_daily_checkin_ledger_account_fk.sql +7 -0
- package/migrations/lite-sqlite3/0036_system_prompt_presets_refs.sql +14 -0
- package/migrations/lite-sqlite3/0037_share_concurrency_scope.sql +3 -0
- package/migrations/lite-sqlite3/0038_add_accounts_quota_snapshot.sql +4 -0
- package/migrations/lite-sqlite3/0039_add_telemetry_provider_model_indexes.sql +6 -0
- package/migrations/lite-sqlite3/0040_add_models_dialect.sql +3 -0
- package/migrations/lite-sqlite3/0041_add_daily_checkin_ledger_account_fk.sql +26 -0
- package/package.json +4 -6
- package/scripts/cli/update.ts +15 -41
- package/src/boot/app.ts +58 -2
- package/src/boot/dependencies.ts +786 -732
- package/src/boot/lifecycle.ts +11 -12
- package/src/config/cooldowns.ts +34 -55
- package/src/config/network.ts +1 -1
- package/src/config/package-paths.ts +1 -1
- package/src/config/providers.ts +13 -14
- package/src/config/resources.ts +0 -5
- package/src/config/runtime.ts +2 -5
- package/src/config/spec.ts +26 -6
- package/src/console/api-keys/ban-constants.ts +38 -0
- package/src/console/api-keys/contracts.ts +71 -62
- package/src/console/api-keys/routes.ts +48 -12
- package/src/console/auth/access.ts +4 -4
- package/src/console/auth/account-reset.ts +1 -1
- package/src/console/auth/audit-recorder.ts +1 -1
- package/src/console/auth/lockout.ts +7 -3
- package/src/console/auth/row-coercion.ts +57 -12
- package/src/console/auth/session-resolver.ts +1 -1
- package/src/console/auth/session-service.ts +8 -2
- package/src/console/auth/session.ts +37 -31
- package/src/console/auth/setup-service.ts +4 -3
- package/src/console/auth/user-store.ts +1 -1
- package/src/console/backup/9router-map.ts +0 -3
- package/src/console/backup/contracts.ts +9 -9
- package/src/console/backup/export-endpoint.ts +5 -6
- package/src/console/backup/nine-router.ts +1 -7
- package/src/console/backup/routes.ts +29 -33
- package/src/console/backup/service.ts +468 -468
- package/src/console/backup/store.ts +47 -29
- package/src/console/backup/validate.ts +11 -1
- package/src/console/cli-tools/contracts.ts +7 -7
- package/src/console/cli-tools/injectors/cline.ts +110 -110
- package/src/console/cli-tools/injectors/codex.ts +181 -181
- package/src/console/cli-tools/injectors/copilot.ts +108 -108
- package/src/console/cli-tools/injectors/driver.ts +247 -247
- package/src/console/cli-tools/injectors/jcode.ts +115 -115
- package/src/console/cli-tools/injectors/openclaw.ts +161 -161
- package/src/console/cli-tools/injectors/opencode.ts +153 -153
- package/src/console/cli-tools/routes.ts +1 -1
- package/src/console/cli-tools/service.ts +1 -1
- package/src/console/cli-tools/store.ts +2 -2
- package/src/console/console-router.ts +32 -0
- package/src/console/dashboard-assets.ts +9 -9
- package/src/console/domain-registration.ts +16 -2
- package/src/console/inferhub/client.ts +0 -7
- package/src/console/inferhub/key-store.ts +1 -1
- package/src/console/inferhub/normalize.ts +2 -2
- package/src/console/inferhub/routes.ts +1 -1
- package/src/console/inferhub/types.ts +1 -1
- package/src/console/observability/contracts.ts +1 -1
- package/src/console/observability/live.ts +31 -18
- package/src/console/observability/logs.ts +1 -1
- package/src/console/observability/sse.ts +62 -8
- package/src/console/observability/store.ts +20 -4
- package/src/console/providers/catalog/catalog-projections.ts +1 -1
- package/src/console/providers/catalog/contracts.ts +39 -0
- package/src/console/providers/catalog/model-operations.ts +1 -1
- package/src/console/providers/catalog/provider-operations.ts +95 -17
- package/src/console/providers/catalog/public-model-store.ts +0 -0
- package/src/console/providers/catalog/routes.ts +27 -3
- package/src/console/providers/catalog/store.ts +147 -8
- package/src/console/providers/detail/routes.ts +1 -1
- package/src/console/providers/oauth/callback-listener.ts +100 -18
- package/src/console/providers/oauth/routes.ts +92 -13
- package/src/console/quota/account-quota-global.ts +41 -3
- package/src/console/quota/account-quota-shared.ts +10 -3
- package/src/console/quota/account-quota-tenant.ts +259 -66
- package/src/console/quota/daily-growth-job.ts +308 -0
- package/src/console/quota/native-identity.ts +2 -2
- package/src/console/quota/quota-view.ts +1 -1
- package/src/console/remote-support/routes.ts +1 -1
- package/src/console/remote-support/service.ts +2 -2
- package/src/console/routing/model/contracts.ts +1 -1
- package/src/console/routing/pools/contracts.ts +2 -2
- package/src/console/routing/pools/import-parse.ts +3 -3
- package/src/console/routing/pools/relay-deploy.ts +3 -3
- package/src/console/settings/contracts.ts +29 -2
- package/src/console/settings/store.ts +20 -0
- package/src/console/share/share-router.ts +100 -32
- package/src/console/share/share-stats.ts +4 -4
- package/src/console/share/share-usage.ts +1 -1
- package/src/console/shared/errors.ts +2 -2
- package/src/console/shared/object.ts +18 -0
- package/src/console/shared/read-through-cache.ts +4 -4
- package/src/console/shared/transaction.ts +14 -0
- package/src/console/system-prompt/contracts.ts +89 -12
- package/src/console/system-prompt/routes.ts +98 -45
- package/src/console/updates/apply-update.ts +244 -0
- package/src/console/updates/routes.ts +252 -97
- package/src/main.ts +11 -5
- package/src/network/http2-bidi.ts +1 -1
- package/src/network/ip-utils.ts +298 -0
- package/src/network/outbound-fetch.ts +1 -1
- package/src/network/pool/agent.ts +7 -6
- package/src/network/pool/byte-accounting.ts +1 -1
- package/src/network/pool/resolver.ts +16 -12
- package/src/network/pool/selector.ts +150 -136
- package/src/network/pool-health-machine.ts +1 -1
- package/src/network/rate-limit-verdict.ts +1 -1
- package/src/network/retry.ts +1 -1
- package/src/network/ssrf.ts +20 -71
- package/src/observability/event-log.ts +2 -2
- package/src/observability/event-query.ts +59 -291
- package/src/observability/log-sanitize.ts +1 -1
- package/src/observability/metrics.ts +4 -4
- package/src/observability/payload-capture.ts +6 -5
- package/src/observability/payload-store.ts +1 -1
- package/src/observability/runtime-metrics.ts +1 -1
- package/src/observability/telemetry-buffer.ts +12 -3
- package/src/observability/telemetry-reconciliation.ts +3 -3
- package/src/observability/token-speed.ts +1 -1
- package/src/persistence/connection-url.ts +1 -1
- package/src/persistence/db-handle.ts +1 -1
- package/src/persistence/db-pg.ts +2 -2
- package/src/persistence/db-sqlite.ts +3 -3
- package/src/persistence/dialect.ts +5 -1
- package/src/persistence/readiness.ts +1 -1
- package/src/persistence/redis.ts +1 -1
- package/src/persistence/schema.ts +113 -40
- package/src/persistence/shared-cache.ts +2 -2
- package/src/persistence/sort-order.ts +1 -1
- package/src/persistence/stores/api-key-store.ts +10 -1
- package/src/persistence/stores/share-store.ts +20 -8
- package/src/persistence/system-prompt-store.ts +63 -18
- package/src/persistence/tenant-preferences.ts +1 -1
- package/src/persistence/ttl-cache.ts +3 -3
- package/src/persistence/usage-flush.ts +7 -2
- package/src/protocol/canonical-model.ts +1 -1
- package/src/protocol/codecs.ts +6 -10
- package/src/protocol/dsml.ts +3 -49
- package/src/protocol/image-clamp.ts +2 -2
- package/src/protocol/reasoning-text.ts +1 -1
- package/src/protocol/reasoning.ts +7 -52
- package/src/protocol/registry.ts +50 -15
- package/src/protocol/request/chat.ts +46 -19
- package/src/protocol/request/codex.ts +1 -1
- package/src/protocol/request/messages.ts +46 -14
- package/src/protocol/request/responses.ts +19 -10
- package/src/protocol/response/chat.ts +25 -10
- package/src/protocol/response/codex.ts +1 -1
- package/src/protocol/response/messages.ts +1 -1
- package/src/protocol/response/omp-tool-call-markup.ts +1 -1
- package/src/protocol/response/responses.ts +1 -1
- package/src/protocol/sse-decoder.ts +50 -1
- package/src/protocol/stop-reason.ts +2 -2
- package/src/protocol/think-split.ts +347 -0
- package/src/protocol/translation/attachment-integrity.ts +3 -3
- package/src/protocol/translation/cache-controls.ts +8 -20
- package/src/protocol/translation/capabilities.ts +6 -6
- package/src/protocol/translation/dialect/deepseek.ts +10 -0
- package/src/protocol/translation/dialect/inband.ts +39 -0
- package/src/protocol/translation/dialect/model-thinking.ts +89 -0
- package/src/protocol/translation/dialect/qwen.ts +114 -0
- package/src/protocol/translation/dialect/resolver.ts +83 -0
- package/src/protocol/translation/dialect/rules.ts +93 -0
- package/src/protocol/translation/dialect/thinking-ladder.ts +245 -0
- package/src/protocol/translation/dialect/types.ts +58 -0
- package/src/protocol/translation/quirks.ts +18 -4
- package/src/protocol/translation/thinking.ts +12 -252
- package/src/protocol/translation/tool-id.ts +3 -3
- package/src/protocol/translation/tool-repair.ts +1 -1
- package/src/protocol/upstream/post-json.ts +6 -1
- package/src/providers/adapters/compatible-adapter.ts +42 -3
- package/src/providers/authentication/jwt-validator.ts +0 -5
- package/src/providers/authentication/oauth-client.ts +254 -253
- package/src/providers/authentication/oauth-flow-store.ts +42 -7
- package/src/providers/bundled-discovery-providers.ts +6 -30
- package/src/providers/configured-openai-providers.ts +82 -97
- package/src/providers/default-registry.ts +23 -35
- package/src/providers/discovery/catalog/catalog-format.ts +13 -2
- package/src/providers/discovery/discovery-types.ts +0 -1
- package/src/providers/discovery/openai-model-discovery.ts +2 -2
- package/src/providers/discovery/probe-phases.ts +48 -13
- package/src/providers/discovery/probing-service.ts +6 -8
- package/src/providers/integrations/agentrouter/agentrouter.ts +12 -2
- package/src/providers/integrations/antigravity/antigravity-oauth.ts +0 -1
- package/src/providers/integrations/antigravity/antigravity-protocol.ts +0 -14
- package/src/providers/integrations/antigravity/antigravity.ts +2 -2
- package/src/providers/integrations/buddy/buddy-catalog-shared.ts +1 -1
- package/src/providers/integrations/buddy/buddy-chat-shared.ts +20 -4
- package/src/providers/integrations/buddy/buddy-cli-headers.ts +19 -52
- package/src/providers/integrations/buddy/buddy-identity.ts +86 -0
- package/src/providers/integrations/buddy/buddy-oauth-shared.ts +1 -1
- package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +1 -2
- package/src/providers/integrations/buddy/codebuddy/provider.ts +4 -15
- package/src/providers/integrations/buddy/codebuddy/shared.ts +30 -9
- package/src/providers/integrations/buddy/workbuddy/provider.ts +5 -9
- package/src/providers/integrations/buddy/workbuddy/shared.ts +21 -3
- package/src/providers/integrations/claude/claude-cch.ts +148 -148
- package/src/providers/integrations/claude/fallback-credit.ts +3 -7
- package/src/providers/integrations/cline/cline.ts +1 -1
- package/src/providers/integrations/codex/codex-device-code.ts +101 -101
- package/src/providers/integrations/codex/codex-oauth.ts +0 -1
- package/src/providers/integrations/commandcode/commandcode.ts +75 -115
- package/src/providers/integrations/commandcode/quota.ts +131 -51
- package/src/providers/integrations/cursor/cursor-http2.ts +0 -3
- package/src/providers/integrations/cursor/cursor-quota.ts +2 -6
- package/src/providers/integrations/cursor/cursor-request.ts +0 -87
- package/src/providers/integrations/cursor/cursor-shared.ts +0 -4
- package/src/providers/integrations/cursor/cursor.ts +38 -5
- package/src/providers/integrations/cursor/protocol/framing.ts +0 -56
- package/src/providers/integrations/cursor/protocol/messages.ts +0 -96
- package/src/providers/integrations/factory/factory.ts +9 -9
- package/src/providers/integrations/github/github-discovery.ts +55 -10
- package/src/providers/integrations/github/github-oauth.ts +6 -5
- package/src/providers/integrations/github/github.ts +41 -14
- package/src/providers/integrations/grok/grok-oauth.ts +261 -261
- package/src/providers/integrations/grok/grok-quota-frame.ts +1 -1
- package/src/providers/integrations/grok/grok-quota.ts +16 -7
- package/src/providers/integrations/grok/grok-ratelimit-store.ts +0 -5
- package/src/providers/integrations/grok/grok-ratelimit.ts +3 -0
- package/src/providers/integrations/grok/grok.ts +86 -36
- package/src/providers/integrations/kilo/kilo-oauth.ts +1 -1
- package/src/providers/integrations/kiro/kiro-cache.ts +31 -0
- package/src/providers/integrations/kiro/kiro-catalog.ts +31 -5
- package/src/providers/integrations/kiro/kiro-discovery.ts +47 -43
- package/src/providers/integrations/kiro/kiro-machine-id.ts +6 -3
- package/src/providers/integrations/kiro/kiro-oauth.ts +523 -255
- package/src/providers/integrations/kiro/kiro-quota.ts +35 -47
- package/src/providers/integrations/kiro/kiro-request.ts +8 -11
- package/src/providers/integrations/kiro/kiro-stream.ts +13 -0
- package/src/providers/integrations/kiro/kiro.ts +81 -68
- package/src/providers/integrations/muse/muse-oauth.ts +0 -1
- package/src/providers/integrations/muse/muse-quota.ts +0 -4
- package/src/providers/integrations/opencode/fingerprint.ts +7 -5
- package/src/providers/integrations/opencode/provider.ts +17 -24
- package/src/providers/integrations/qoder/qoder-catalog.ts +85 -8
- package/src/providers/integrations/qoder/qoder-oauth.ts +165 -25
- package/src/providers/integrations/qoder/qoder-quota.ts +48 -6
- package/src/providers/integrations/qoder/qoder-realm.ts +10 -31
- package/src/providers/integrations/qoder/qoder-sse.ts +159 -0
- package/src/providers/integrations/qoder/qoder-umid.ts +21 -25
- package/src/providers/integrations/qoder/qoder-virtual.ts +12 -0
- package/src/providers/integrations/qoder/qoder.ts +354 -62
- package/src/providers/integrations/shared/connect.ts +0 -54
- package/src/providers/integrations/shared/hash.ts +15 -0
- package/src/providers/integrations/shared/session-cache.ts +136 -0
- package/src/providers/integrations/trae/trae-cache.ts +7 -87
- package/src/providers/integrations/trae/trae-catalog.ts +1 -6
- package/src/providers/integrations/trae/trae-protocol.ts +17 -8
- package/src/providers/integrations/trae/trae-realm.ts +0 -7
- package/src/providers/integrations/trae/trae.ts +53 -10
- package/src/providers/integrations/xai/xai-oauth.ts +5 -5
- package/src/providers/integrations/zcode-start-plan/gifts.ts +265 -0
- package/src/providers/integrations/zcode-start-plan/oauth.ts +154 -0
- package/src/providers/integrations/zcode-start-plan/provider.ts +131 -0
- package/src/providers/integrations/zcode-start-plan/quota.ts +152 -0
- package/src/providers/integrations/zcode-start-plan/zcode_prompt.json +10 -0
- package/src/providers/operations/account-error-classification.ts +131 -116
- package/src/providers/operations/account-health-service.ts +31 -39
- package/src/providers/operations/account-reset-service.ts +58 -1
- package/src/providers/operations/cli-platform.ts +31 -5
- package/src/providers/operations/client-version-resolver.ts +4 -6
- package/src/providers/operations/client-versions.ts +10 -7
- package/src/providers/operations/provider-catalog-seeder.ts +2 -0
- package/src/providers/operations/provider-catalog-service.ts +18 -2
- package/src/providers/operations/provider-credential-service.ts +19 -2
- package/src/providers/provider-contracts.ts +18 -7
- package/src/providers/provider-metadata.ts +17 -2
- package/src/providers/provider-model-definition.ts +22 -1
- package/src/providers/provider-probe.ts +4 -2
- package/src/providers/provider-usage.ts +81 -16
- package/src/providers/quota/cache.ts +29 -0
- package/src/providers/quota/quota-contracts.ts +4 -0
- package/src/providers/quota/refresh.ts +27 -4
- package/src/providers/quota/snapshot.ts +94 -0
- package/src/providers/search/search-provider.ts +1 -3
- package/src/providers/search/search-providers.ts +2 -1
- package/src/security/abuse.ts +4 -4
- package/src/security/admission/service.ts +2 -2
- package/src/security/admission/ttl.ts +61 -61
- package/src/security/api-key-auth.ts +16 -4
- package/src/security/client-router-fingerprint.ts +4 -4
- package/src/security/credential-codec.ts +3 -3
- package/src/security/csrf.ts +2 -2
- package/src/security/ip-boundary.ts +16 -189
- package/src/security/model-access-rule.ts +1 -1
- package/src/security/outbound-headers.ts +18 -0
- package/src/shared/invoke-guard.ts +47 -0
- package/src/transport/contracts.ts +13 -4
- package/src/transport/dispatch/attempt-finalize.ts +1 -1
- package/src/transport/dispatch/capture-policy.ts +34 -37
- package/src/transport/dispatch/empty-stream.ts +23 -20
- package/src/transport/dispatch/generation.ts +25 -45
- package/src/transport/dispatch/native-shared.ts +127 -0
- package/src/transport/dispatch/proxy-request.ts +3 -1
- package/src/transport/dispatch/responses-compact.ts +25 -46
- package/src/transport/dispatch/streaming/encoder.ts +18 -72
- package/src/transport/dispatch/streaming/finalize.ts +2 -2
- package/src/transport/dispatch/streaming/stream.ts +7 -3
- package/src/transport/dispatch/streaming/thinking-loop.ts +1 -1
- package/src/transport/dispatch/systemone.ts +20 -48
- package/src/transport/dispatch/token-count.ts +41 -38
- package/src/transport/dispatch/websearch.ts +19 -45
- package/src/transport/drain-endpoint.ts +1 -1
- package/src/transport/failure-policy.ts +6 -0
- package/src/transport/guards/auth-guard.ts +0 -6
- package/src/transport/guards/body-policy.ts +3 -3
- package/src/transport/guards/headroom-guard.ts +1 -1
- package/src/transport/guards/pipeline.ts +1 -1
- package/src/transport/guards/request-context.ts +12 -17
- package/src/transport/request/preparer.ts +286 -7
- package/src/transport/request/rtk/compress-request.ts +3 -3
- package/src/transport/request/rtk/rtk-constants.ts +2 -2
- package/src/transport/request/rtk/rtk-filters.ts +8 -8
- package/src/transport/request/state.ts +1 -13
- package/src/transport/request/system-prompt-injection.ts +74 -11
- package/src/transport/request/tenant-preferences.ts +22 -0
- package/src/transport/routing/candidate-index.ts +2 -2
- package/src/transport/routing/round-robin.ts +1 -1
- package/src/transport/routing/route-catalog.ts +17 -18
- package/src/transport/routing/route-model.ts +126 -33
- package/src/transport/routing/router.ts +152 -37
- package/src/transport/shutdown-notice.ts +2 -2
- package/src/transport/surface/adapters.ts +3 -3
- package/src/transport/surface/chat/encode.ts +7 -7
- package/src/transport/surface/chat/parse.ts +7 -7
- package/src/transport/surface/completion.ts +2 -2
- package/src/transport/surface/content-parts.ts +1 -1
- package/src/transport/surface/dialects.ts +2 -2
- package/src/transport/surface/messages/parse.ts +3 -3
- package/src/transport/surface/responses/adapter.ts +1 -1
- package/src/transport/surface/responses/errors.ts +0 -9
- package/src/transport/vision-handoff-cache.ts +58 -0
- package/src/transport/vision-handoff.ts +376 -0
- package/src/workers/account-identity-sweep.ts +2 -2
- package/src/workers/buddy-growth-tasks.ts +392 -346
- package/src/workers/daily-checkin.ts +102 -49
- package/src/workers/maintenance-lease.ts +2 -35
- package/src/workers/oauth-refresh-sweep.ts +8 -1
- package/src/workers/quota-refresh-sweep.ts +120 -82
- package/src/workers/scheduler.ts +67 -4
- package/src/workers/sweep-runner.ts +24 -5
- package/src/workers/zcode-gift-claim.ts +108 -0
- package/dist/dashboard/assets/AccountCooldown-Bjo92Qat.js +0 -1
- package/dist/dashboard/assets/AccountCooldown-Bjo92Qat.js.gz +0 -0
- package/dist/dashboard/assets/BannedPage-ChG5svFb.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-BIcWQun7.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-Bf2VelDe.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-DCgm5g6g.js +0 -1
- package/dist/dashboard/assets/CombosPage-DCgm5g6g.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-a8mD0QF1.js +0 -1
- package/dist/dashboard/assets/ConsoleLogPage-a8mD0QF1.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-ggeMSjte.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-DzSWzE_j.js +0 -1
- package/dist/dashboard/assets/HealthEventsModal-DzSWzE_j.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-U6AKaJcF.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-BMS2t8BA.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-D2D4IiHl.js +0 -1
- package/dist/dashboard/assets/OverviewPage-D2D4IiHl.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-CrNf7j_X.js +0 -5
- package/dist/dashboard/assets/ProviderDetailPage-CrNf7j_X.js.gz +0 -0
- package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js +0 -1
- package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-BSDbquc1.js +0 -1
- package/dist/dashboard/assets/ProvidersPage-BSDbquc1.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-lsrx7NYg.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-L0skzML8.js +0 -1
- package/dist/dashboard/assets/QuotaPage-L0skzML8.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-CmAbjRHW.js +0 -1
- package/dist/dashboard/assets/SettingsPage-CmAbjRHW.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-f0DE1h0-.js.gz +0 -0
- package/dist/dashboard/assets/SortableList-ClzpwyET.js +0 -1
- package/dist/dashboard/assets/SortableList-ClzpwyET.js.gz +0 -0
- package/dist/dashboard/assets/StudioPage-Cq5tW7hW.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-Dfc9rWQn.js +0 -56
- package/dist/dashboard/assets/UsagePage-Dfc9rWQn.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-C38iXoO4.js +0 -1
- package/dist/dashboard/assets/api-keys-C38iXoO4.js.gz +0 -0
- package/dist/dashboard/assets/app-a159k0ub.css +0 -2
- package/dist/dashboard/assets/app-a159k0ub.css.gz +0 -0
- package/dist/dashboard/assets/assert-DbRA8EG2.js +0 -15
- package/dist/dashboard/assets/assert-DbRA8EG2.js.gz +0 -0
- package/dist/dashboard/assets/badge-D9CS6dZk.js.gz +0 -0
- package/dist/dashboard/assets/banner-DympBNq4.js.gz +0 -0
- package/dist/dashboard/assets/card-ibRWU_uT.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-63qBGaRw.js +0 -1
- package/dist/dashboard/assets/cli-tools-63qBGaRw.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-DJm-9pz7.js.gz +0 -0
- package/dist/dashboard/assets/dialog-8fiE6tec.js.gz +0 -0
- package/dist/dashboard/assets/drawer-CqGYeOrd.js.gz +0 -0
- package/dist/dashboard/assets/entry-BSAsYsmV.js.gz +0 -0
- package/dist/dashboard/assets/entry-CiIQULZD.js +0 -4
- package/dist/dashboard/assets/entry-CiIQULZD.js.gz +0 -0
- package/dist/dashboard/assets/entry-Daz0lJ-W.js +0 -1
- package/dist/dashboard/assets/entry-Daz0lJ-W.js.gz +0 -0
- package/dist/dashboard/assets/formatters-D-MTJxMr.js.gz +0 -0
- package/dist/dashboard/assets/github-badge-CPK47xQx.js.gz +0 -0
- package/dist/dashboard/assets/index-DBNPOfhV.js.gz +0 -0
- package/dist/dashboard/assets/inline-BT0zntEr.js.gz +0 -0
- package/dist/dashboard/assets/input-Da2ZViWg.js.gz +0 -0
- package/dist/dashboard/assets/layout-BXNeLw6M.js.gz +0 -0
- package/dist/dashboard/assets/list-row-BycOrrIV.js.gz +0 -0
- package/dist/dashboard/assets/live-DCQNZpF7.js +0 -1
- package/dist/dashboard/assets/live-DCQNZpF7.js.gz +0 -0
- package/dist/dashboard/assets/network-9Yj2ZyLo.js +0 -1
- package/dist/dashboard/assets/network-9Yj2ZyLo.js.gz +0 -0
- package/dist/dashboard/assets/numbers-CEagj6ZZ.js +0 -2
- package/dist/dashboard/assets/numbers-CEagj6ZZ.js.gz +0 -0
- package/dist/dashboard/assets/quota-Caw7wqci.js +0 -1
- package/dist/dashboard/assets/quota-Caw7wqci.js.gz +0 -0
- package/dist/dashboard/assets/routing-CO9zCUus.js +0 -1
- package/dist/dashboard/assets/routing-CO9zCUus.js.gz +0 -0
- package/dist/dashboard/assets/select-BIYIBkMU.js.gz +0 -0
- package/dist/dashboard/assets/settings-jS3-PNSA.js +0 -1
- package/dist/dashboard/assets/settings-jS3-PNSA.js.gz +0 -0
- package/dist/dashboard/assets/stack-DfgnFDWY.js.gz +0 -0
- package/dist/dashboard/assets/state-CkfdYWel.js.gz +0 -0
- package/dist/dashboard/assets/stream-retry-5QnVuSSo.js +0 -1
- package/dist/dashboard/assets/stream-retry-5QnVuSSo.js.gz +0 -0
- package/dist/dashboard/assets/switch-SYm237HG.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-BQqCRNIi.js.gz +0 -0
- package/src/network/response-headers.ts +0 -39
- package/src/protocol/response/dsml.ts +0 -38
- package/src/providers/integrations/xiaomi-mimo/think-stream.ts +0 -229
- /package/dist/dashboard/assets/{app-BayTTy01.js → app-Bgd3zANQ.js} +0 -0
- /package/dist/dashboard/assets/{app-BayTTy01.js.gz → app-Bgd3zANQ.js.gz} +0 -0
- /package/dist/dashboard/assets/brand/{og_bansos.webp → og_bansos_initial.webp} +0 -0
- /package/dist/dashboard/assets/brand/{og_image.webp → og_image_initial.webp} +0 -0
package/src/boot/dependencies.ts
CHANGED
|
@@ -1,732 +1,786 @@
|
|
|
1
|
-
|
|
2
|
-
import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
|
|
3
|
-
import { poolMaxFromEnv } from "../persistence/db-pg";
|
|
4
|
-
import type { CartethyiaDatabase } from "../persistence/db-handle";
|
|
5
|
-
import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
|
|
6
|
-
import { resolveProcessTopology, type ProcessTopology } from "../config/runtime";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
refreshActiveProviderCatalogs,
|
|
15
|
-
registerByokProviders,
|
|
16
|
-
retireUnbundledProviders,
|
|
17
|
-
seedBundledProviders,
|
|
18
|
-
} from "../providers/operations/provider-catalog-service";
|
|
19
|
-
import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
|
|
20
|
-
import { createDefaultProviderRegistry } from "../providers/default-registry";
|
|
21
|
-
import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
|
|
22
|
-
import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
|
|
23
|
-
import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
|
|
24
|
-
import { accountIdentitySweep } from "../workers/account-identity-sweep";
|
|
25
|
-
import { pushStructuredConsoleLog } from "../observability/log-ring";
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import {
|
|
40
|
-
import {
|
|
41
|
-
import {
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
45
|
-
import {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
} from "../network/pool/
|
|
50
|
-
import {
|
|
51
|
-
import {
|
|
52
|
-
import {
|
|
53
|
-
import {
|
|
54
|
-
import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
|
|
55
|
-
import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
|
|
56
|
-
import {
|
|
57
|
-
buddyAdoptTask,
|
|
58
|
-
buddyActivityReportTask,
|
|
59
|
-
buddyStreakTask,
|
|
60
|
-
syncBuddyUsage,
|
|
61
|
-
} from "../workers/buddy-growth-tasks";
|
|
62
|
-
import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
|
|
63
|
-
import { quotaCacheSize } from "../providers/quota/cache";
|
|
64
|
-
import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
|
|
65
|
-
import { preferencesReaderFor } from "../persistence/tenant-preferences";
|
|
66
|
-
import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
|
|
67
|
-
import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
|
|
68
|
-
import { TelemetryPayloadCapture } from "../observability/payload-capture";
|
|
69
|
-
import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
|
|
70
|
-
import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
|
|
71
|
-
import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
|
|
72
|
-
import {
|
|
73
|
-
resolveIpRateLimit,
|
|
74
|
-
} from "../config/security";
|
|
75
|
-
import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
|
|
76
|
-
import type { TrustedProxyBoundary } from "../config/network";
|
|
77
|
-
import { resolveTelemetryRetentionDays } from "../config/telemetry";
|
|
78
|
-
import { resolveConcurrencyTolerance } from "../config/resources";
|
|
79
|
-
import { CONFIG_SPEC, readInt, readList } from "../config/spec";
|
|
80
|
-
import type { ProviderAdapter } from "../providers/provider-contracts";
|
|
81
|
-
import type { ProviderRegistry } from "../providers/provider-registry";
|
|
82
|
-
import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
|
|
83
|
-
import type { ReadinessCheckResult } from "../persistence/readiness";
|
|
84
|
-
import { log } from "../observability/logger";
|
|
85
|
-
import { refreshProviderClientVersions } from "../providers/operations/client-versions";
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
*
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
*
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
const
|
|
215
|
-
//
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
//
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
)
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
//
|
|
272
|
-
//
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
(
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
//
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
return (
|
|
307
|
-
},
|
|
308
|
-
//
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
(
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
const
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
//
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
//
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
//
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
scheduledTasks.register({
|
|
549
|
-
name: "
|
|
550
|
-
intervalMs:
|
|
551
|
-
|
|
552
|
-
run
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
//
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
//
|
|
662
|
-
//
|
|
663
|
-
//
|
|
664
|
-
// a
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
};
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
1
|
+
|
|
2
|
+
import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
|
|
3
|
+
import { poolMaxFromEnv } from "../persistence/db-pg";
|
|
4
|
+
import type { CartethyiaDatabase } from "../persistence/db-handle";
|
|
5
|
+
import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
|
|
6
|
+
import { resolveProcessTopology, type ProcessTopology } from "../config/runtime";
|
|
7
|
+
import { withTimeout } from "./timeout";
|
|
8
|
+
import { createRedisMaintenanceLeaseStore } from "../workers/maintenance-lease";
|
|
9
|
+
import { resolveDbMode } from "../persistence/db-mode";
|
|
10
|
+
import type { RedisClient } from "../persistence/redis";
|
|
11
|
+
import {
|
|
12
|
+
bundledModelCatalog,
|
|
13
|
+
liveProviderUpstreamHosts,
|
|
14
|
+
refreshActiveProviderCatalogs,
|
|
15
|
+
registerByokProviders,
|
|
16
|
+
retireUnbundledProviders,
|
|
17
|
+
seedBundledProviders,
|
|
18
|
+
} from "../providers/operations/provider-catalog-service";
|
|
19
|
+
import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
|
|
20
|
+
import { createDefaultProviderRegistry } from "../providers/default-registry";
|
|
21
|
+
import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
|
|
22
|
+
import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
|
|
23
|
+
import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
|
|
24
|
+
import { accountIdentitySweep } from "../workers/account-identity-sweep";
|
|
25
|
+
import { pushStructuredConsoleLog } from "../observability/log-ring";
|
|
26
|
+
import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
|
|
27
|
+
import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
|
|
28
|
+
import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
|
|
29
|
+
import { RoutingEngine } from "../transport/routing/router";
|
|
30
|
+
import { RedisAdmissionController } from "../transport/routing/admission";
|
|
31
|
+
import { ApiKeyAdmissionService } from "../security/admission/service";
|
|
32
|
+
import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
|
|
33
|
+
import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
|
|
34
|
+
import { sweepLeases } from "../security/admission/lease-sweep";
|
|
35
|
+
import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
|
|
36
|
+
import { UsageFlushBuffer } from "../persistence/usage-flush";
|
|
37
|
+
import { setPendingUsageSource } from "../security/api-key-auth";
|
|
38
|
+
import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
|
|
39
|
+
import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
|
|
40
|
+
import { deleteExpiredSessions } from "../console/auth/session-service";
|
|
41
|
+
import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
|
|
42
|
+
import { checkReadiness } from "../persistence/readiness";
|
|
43
|
+
import { ProxyRequestPreparer } from "../transport/request/preparer";
|
|
44
|
+
import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
|
|
45
|
+
import {
|
|
46
|
+
PoolAgentResolver,
|
|
47
|
+
ValidatedNetworkBindingFactory,
|
|
48
|
+
} from "../network/pool/resolver";
|
|
49
|
+
import { NetworkPoolSelector } from "../network/pool/selector";
|
|
50
|
+
import { restorePoolProbeVerdicts, restoreProviderCooldownMarkers } from "../network/pool-cooldown-store";
|
|
51
|
+
import { ScheduledTaskRegistry } from "../workers/scheduler";
|
|
52
|
+
import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
|
|
53
|
+
import { registerZcodeGiftClaimTask } from "../workers/zcode-gift-claim";
|
|
54
|
+
import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
|
|
55
|
+
import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
|
|
56
|
+
import {
|
|
57
|
+
buddyAdoptTask,
|
|
58
|
+
buddyActivityReportTask,
|
|
59
|
+
buddyStreakTask,
|
|
60
|
+
syncBuddyUsage,
|
|
61
|
+
} from "../workers/buddy-growth-tasks";
|
|
62
|
+
import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
|
|
63
|
+
import { quotaCacheSize } from "../providers/quota/cache";
|
|
64
|
+
import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
|
|
65
|
+
import { preferencesReaderFor } from "../persistence/tenant-preferences";
|
|
66
|
+
import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
|
|
67
|
+
import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
|
|
68
|
+
import { TelemetryPayloadCapture } from "../observability/payload-capture";
|
|
69
|
+
import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
|
|
70
|
+
import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
|
|
71
|
+
import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
|
|
72
|
+
import {
|
|
73
|
+
resolveIpRateLimit,
|
|
74
|
+
} from "../config/security";
|
|
75
|
+
import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
|
|
76
|
+
import type { TrustedProxyBoundary } from "../config/network";
|
|
77
|
+
import { resolveTelemetryRetentionDays } from "../config/telemetry";
|
|
78
|
+
import { resolveConcurrencyTolerance } from "../config/resources";
|
|
79
|
+
import { CONFIG_SPEC, readInt, readList } from "../config/spec";
|
|
80
|
+
import type { ProviderAdapter } from "../providers/provider-contracts";
|
|
81
|
+
import type { ProviderRegistry } from "../providers/provider-registry";
|
|
82
|
+
import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
|
|
83
|
+
import type { ReadinessCheckResult } from "../persistence/readiness";
|
|
84
|
+
import { log } from "../observability/logger";
|
|
85
|
+
import { refreshProviderClientVersions } from "../providers/operations/client-versions";
|
|
86
|
+
import type { ModelDefinition } from "../providers/provider-model-definition";
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
export interface ProductionDeps {
|
|
90
|
+
db: CartethyiaDatabase;
|
|
91
|
+
redis: RedisClient | undefined;
|
|
92
|
+
snapshotService: InMemoryRouteSnapshotService;
|
|
93
|
+
/** Live routing admission snapshot scoped to one provider/tenant for console reads. */
|
|
94
|
+
readRoutingAccountInflight: (
|
|
95
|
+
providerId: string,
|
|
96
|
+
tenantId: string | null,
|
|
97
|
+
) => Promise<readonly { accountId: string; inflight: number }[]>;
|
|
98
|
+
/** On-demand adapter resolution keeps heavy providers out of the boot path. */
|
|
99
|
+
resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
|
|
100
|
+
bundledModelCatalog: BundledProviderCatalog;
|
|
101
|
+
/** Live lookup of a provider's SSRF-validated upstream host. */
|
|
102
|
+
byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
|
|
103
|
+
proxyPreparer: ProxyRequestPreparer;
|
|
104
|
+
networkBindingFactory: ValidatedNetworkBindingFactory;
|
|
105
|
+
ipAbuseProtection: IpAbuseProtectionService;
|
|
106
|
+
readiness: () => Promise<ReadinessCheckResult>;
|
|
107
|
+
scheduledTasks: ScheduledTaskRegistry;
|
|
108
|
+
poolAgentResolver: PoolAgentResolver;
|
|
109
|
+
poolSelector: NetworkPoolSelector;
|
|
110
|
+
telemetryBuffer: TelemetryBatchBuffer;
|
|
111
|
+
usageFlush: UsageFlushBuffer;
|
|
112
|
+
healthEventFlush: HealthEventFlushBuffer;
|
|
113
|
+
trustedProxyBoundary: TrustedProxyBoundary;
|
|
114
|
+
providerRegistry: ProviderRegistry;
|
|
115
|
+
resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
|
|
116
|
+
oauthRefreshService: OAuthRefreshService;
|
|
117
|
+
admissionService: ApiKeyAdmissionService;
|
|
118
|
+
/** Graduated strikes for repeated invalid-model requests. */
|
|
119
|
+
/** Memory sampler; in-flight source is bound once the request store exists. */
|
|
120
|
+
runtimeMetricsSampler: RuntimeMetricsSampler;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Pause between accounts in the OAuth refresh sweep. The token endpoints
|
|
125
|
+
* rate-limit a burst of refreshes even at low concurrency.
|
|
126
|
+
*/
|
|
127
|
+
const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Client identities are refreshed out of band so provider dispatch never waits
|
|
131
|
+
* on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
|
|
132
|
+
*/
|
|
133
|
+
// Matched to the resolver's own 30 min cache TTL: a shorter interval makes
|
|
134
|
+
// every second pass a pure cache read that cannot discover anything new.
|
|
135
|
+
const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
|
|
136
|
+
// B-05: first external fetch is pushed out to 90 s so the listener, the route
|
|
137
|
+
// snapshot warm, and the first-minute sweeps all settle first.
|
|
138
|
+
const CLIENT_VERSION_INITIAL_DELAY_MS = 90_000;
|
|
139
|
+
|
|
140
|
+
/** Bound on the boot-time Redis probe: a start must fail, not hang. */
|
|
141
|
+
const REDIS_PROBE_TIMEOUT_MS = 2_000;
|
|
142
|
+
|
|
143
|
+
/** Bounded Redis reachability probe: fails rather than hangs. */
|
|
144
|
+
async function redisIsReachable(): Promise<boolean> {
|
|
145
|
+
const client = resolveRedisClient();
|
|
146
|
+
if (!client) return false;
|
|
147
|
+
try {
|
|
148
|
+
const result = await withTimeout(client.ping(), REDIS_PROBE_TIMEOUT_MS);
|
|
149
|
+
return result === "PONG";
|
|
150
|
+
} catch {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Refuses to start a multi-process deployment whose shared state is not
|
|
157
|
+
* actually shared.
|
|
158
|
+
*
|
|
159
|
+
* Every process shares one port via `SO_REUSEPORT`, so the kernel balances
|
|
160
|
+
* across them — and each process keeps its own in-memory admission counters,
|
|
161
|
+
* quota cache, and cooldown map. Those are only correct when they live in
|
|
162
|
+
* Redis. Starting anyway does not fail loudly; it enforces every per-key limit
|
|
163
|
+
* N times over, which then reads as "the limit did not work".
|
|
164
|
+
*
|
|
165
|
+
* Redis is checked by PING with a hard bound, not merely for being configured:
|
|
166
|
+
* a configured-but-unreachable Redis is the failure mode that matters, and an
|
|
167
|
+
* unbounded connect at boot would hang the start instead of failing it.
|
|
168
|
+
*/
|
|
169
|
+
async function assertMultiProcessPreconditions(
|
|
170
|
+
topology: ProcessTopology,
|
|
171
|
+
redisReachable: () => Promise<boolean>,
|
|
172
|
+
): Promise<void> {
|
|
173
|
+
if (!topology.multiProcess) return;
|
|
174
|
+
if (process.platform !== "linux") {
|
|
175
|
+
throw Object.assign(
|
|
176
|
+
new Error(
|
|
177
|
+
`CARTETHYIA_PROCESS_COUNT=${topology.count} requires Linux (SO_REUSEPORT); ` +
|
|
178
|
+
`this process is on ${process.platform}`,
|
|
179
|
+
),
|
|
180
|
+
{ code: "multi_process_unsupported_platform" },
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
if (resolveDbMode() !== "full") {
|
|
184
|
+
throw Object.assign(
|
|
185
|
+
new Error(
|
|
186
|
+
`CARTETHYIA_PROCESS_COUNT=${topology.count} requires CARTETHYIA_DB_MODE=full (PostgreSQL); ` +
|
|
187
|
+
"two processes on one SQLite file corrupt it — there is no cross-process lock in the driver",
|
|
188
|
+
),
|
|
189
|
+
{ code: "multi_process_requires_postgres" },
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
const reachable = await redisReachable();
|
|
193
|
+
if (!reachable) {
|
|
194
|
+
throw Object.assign(
|
|
195
|
+
new Error(
|
|
196
|
+
`CARTETHYIA_PROCESS_COUNT=${topology.count} requires a REACHABLE Redis: admission counters, ` +
|
|
197
|
+
"quotas, and cooldowns are per-process without one, so every per-key limit would be " +
|
|
198
|
+
`enforced ${topology.count} times over`,
|
|
199
|
+
),
|
|
200
|
+
{ code: "multi_process_requires_redis" },
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export async function buildProductionDeps(): Promise<ProductionDeps> {
|
|
206
|
+
const startupStartedAt = performance.now();
|
|
207
|
+
const startupMemoryBefore = process.memoryUsage();
|
|
208
|
+
// Resolved first and validated before any connection is opened: a
|
|
209
|
+
// half-configured multi-process deploy that starts anyway looks healthy
|
|
210
|
+
// until a per-key limit is enforced N times over.
|
|
211
|
+
const topology = resolveProcessTopology();
|
|
212
|
+
await assertMultiProcessPreconditions(topology, redisIsReachable);
|
|
213
|
+
const handle = await measureStartupPhase("database-connect", bootDatabase);
|
|
214
|
+
const db = handle.db;
|
|
215
|
+
// One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
|
|
216
|
+
const redis = resolveRedisClient();
|
|
217
|
+
const ssrfPolicy = resolveSsrfPolicy();
|
|
218
|
+
await measureStartupPhase("database-migrations", async () => {
|
|
219
|
+
await ensureMigrated();
|
|
220
|
+
// Pool sizing is an external-server concern; the embedded backend has no
|
|
221
|
+
// connection budget to fit into.
|
|
222
|
+
if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv(), topology.count);
|
|
223
|
+
});
|
|
224
|
+
await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
|
|
225
|
+
// Boot-only: retire global rows for providers the bundle has dropped, so a
|
|
226
|
+
// retired provider cannot linger as a card the console can no longer delete.
|
|
227
|
+
// Kept off the test harness path, which shares one database across suites and
|
|
228
|
+
// installs its own global fixtures.
|
|
229
|
+
await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
|
|
230
|
+
const registry = createDefaultProviderRegistry();
|
|
231
|
+
// Live view, not a snapshot: custom providers registered after boot (or
|
|
232
|
+
// edited from the console) must resolve their SSRF binding immediately.
|
|
233
|
+
const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
|
|
234
|
+
await measureStartupPhase("byok-registration", () =>
|
|
235
|
+
registerByokProviders(registry, db, ssrfPolicy),
|
|
236
|
+
);
|
|
237
|
+
// B-01: the bundled model catalog (73 providers' integration modules, ~the
|
|
238
|
+
// largest pre-listen cost) is DEFERRED to the first snapshot build, which
|
|
239
|
+
// runs right after listen (`main.ts` warms the snapshot). Every consumer
|
|
240
|
+
// holds a reference to `bundledCatalogModels`, a shared map that starts
|
|
241
|
+
// empty and is populated by the first resolution — the console store and
|
|
242
|
+
// the inflight reader capture the map reference at construction, so once
|
|
243
|
+
// the build fills it, all of them see the data. The snapshot builder's
|
|
244
|
+
// refresh step seeds the DB from the same map (markCatalogSeeded semantics:
|
|
245
|
+
// lastSeededCatalogRevision is -1 until the first build, so the first build
|
|
246
|
+
// seeds exactly once and later rebuilds see an unchanged revision and skip).
|
|
247
|
+
const bundledCatalogModels = new Map<string, readonly ModelDefinition[]>();
|
|
248
|
+
let bundledCatalogResolved = false;
|
|
249
|
+
const resolveBundledCatalog = async (): Promise<BundledProviderCatalog> => {
|
|
250
|
+
if (bundledCatalogResolved) return { modelsByProvider: bundledCatalogModels };
|
|
251
|
+
const catalog = await bundledModelCatalog(registry);
|
|
252
|
+
for (const [providerId, models] of catalog.modelsByProvider) {
|
|
253
|
+
bundledCatalogModels.set(providerId, models);
|
|
254
|
+
}
|
|
255
|
+
bundledCatalogResolved = true;
|
|
256
|
+
return { modelsByProvider: bundledCatalogModels };
|
|
257
|
+
};
|
|
258
|
+
const snapshotService = new InMemoryRouteSnapshotService(
|
|
259
|
+
createDatabaseSnapshotBuilder(db, async () => {
|
|
260
|
+
// Compute the lazy catalog first so the refresh shares it and the DB
|
|
261
|
+
// seed uses the same map the console reads (no second full walk).
|
|
262
|
+
const catalog = await resolveBundledCatalog();
|
|
263
|
+
await refreshActiveProviderCatalogs(db, registry, catalog);
|
|
264
|
+
}),
|
|
265
|
+
);
|
|
266
|
+
const routingEngine = new RoutingEngine(
|
|
267
|
+
redis ? new RedisAdmissionController(redis) : undefined,
|
|
268
|
+
redis,
|
|
269
|
+
db,
|
|
270
|
+
);
|
|
271
|
+
// Peer processes' settings mutations arrive here. Subscribed once for the
|
|
272
|
+
// process: the revision counter is process-global, and the handler only
|
|
273
|
+
// moves it forward.
|
|
274
|
+
subscribePeerSettingsRevisions();
|
|
275
|
+
const usageFlush = new UsageFlushBuffer(db);
|
|
276
|
+
// Auth reads lifetime usage to seed the admission counter. The persisted
|
|
277
|
+
// value lags by up to one flush interval, so hand it the in-flight deltas
|
|
278
|
+
// too — otherwise the seed is systematically under what has actually been
|
|
279
|
+
// spent, worst exactly when the key is busiest.
|
|
280
|
+
setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
|
|
281
|
+
// Health audit rows are append-only and read by nobody in dispatch, so they
|
|
282
|
+
// are buffered out of the request. The status write they accompany stays
|
|
283
|
+
// synchronous — routing reads it.
|
|
284
|
+
const healthEventFlush = new HealthEventFlushBuffer(db);
|
|
285
|
+
setHealthEventBuffer(healthEventFlush);
|
|
286
|
+
const admissionStore = redis
|
|
287
|
+
? new RedisAdmissionCounterStore(redis)
|
|
288
|
+
: new InMemoryAdmissionCounterStore();
|
|
289
|
+
const admissionService = new ApiKeyAdmissionService(
|
|
290
|
+
admissionStore,
|
|
291
|
+
undefined,
|
|
292
|
+
// The per-tenant concurrency cap is a `console_settings` preference, read
|
|
293
|
+
// on every admitted request. Reading it directly cost one Postgres round
|
|
294
|
+
// trip per request (multiplied by each failover candidate); the shared
|
|
295
|
+
// revision-keyed reader answers from memory and converges the moment an
|
|
296
|
+
// operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
|
|
297
|
+
// revision every cached key embeds.
|
|
298
|
+
(tenantId) =>
|
|
299
|
+
preferencesReaderFor(db)
|
|
300
|
+
.readPreferences(tenantId)
|
|
301
|
+
.then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
|
|
302
|
+
({ apiKeyId, delta }) => {
|
|
303
|
+
// Coalesced: one UPDATE per key per flush instead of one per request.
|
|
304
|
+
// The admission counter is the live total; this only persists it.
|
|
305
|
+
usageFlush.enqueue(apiKeyId, delta);
|
|
306
|
+
return Promise.resolve();
|
|
307
|
+
},
|
|
308
|
+
// Fresh lifetime reader — invoked by the store only when it is about to
|
|
309
|
+
// seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
|
|
310
|
+
// snapshot value cannot freeze a low baseline in place. Family total: the
|
|
311
|
+
// row plus every child (share recipients admit under the parent id).
|
|
312
|
+
async (apiKeyId) => {
|
|
313
|
+
const store = new DrizzleApiKeyStore(db);
|
|
314
|
+
const row = await store.findActiveById(apiKeyId);
|
|
315
|
+
if (!row) return undefined;
|
|
316
|
+
const children = await store.sumChildrenConsumed(apiKeyId);
|
|
317
|
+
return (row.lifetimeTokensConsumed ?? 0) + children;
|
|
318
|
+
},
|
|
319
|
+
// Slack above every key's concurrency limit. A tenant preference an
|
|
320
|
+
// operator sets in the runtime settings modal, falling back to the
|
|
321
|
+
// deployment default so an unset row does not silently lose the allowance.
|
|
322
|
+
(tenantId) =>
|
|
323
|
+
preferencesReaderFor(db)
|
|
324
|
+
.readPreferences(tenantId)
|
|
325
|
+
.then((prefs) => prefs?.concurrencyTolerance ?? resolveConcurrencyTolerance()),
|
|
326
|
+
);
|
|
327
|
+
const ipStore = redis
|
|
328
|
+
? new RedisIpAbuseStore(redis)
|
|
329
|
+
: new InMemoryIpAbuseStore();
|
|
330
|
+
const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
|
|
331
|
+
maxRequestsPerWindow: resolveIpRateLimit(),
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
|
|
335
|
+
// One list covers every periodic task: a small deployment can cut the
|
|
336
|
+
// heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
|
|
337
|
+
// without the codebase growing a flag per task.
|
|
338
|
+
// Leases coordinate the SHARED sweeps across processes. Absent Redis there is
|
|
339
|
+
// nothing to coordinate with, and the registry's local one-at-a-time rule is
|
|
340
|
+
// the correct (and only) guard.
|
|
341
|
+
const scheduledTasks = new ScheduledTaskRegistry(
|
|
342
|
+
readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
|
|
343
|
+
{
|
|
344
|
+
...(redis ? { lease: createRedisMaintenanceLeaseStore(redis) } : {}),
|
|
345
|
+
},
|
|
346
|
+
);
|
|
347
|
+
if (redis) {
|
|
348
|
+
scheduledTasks.register({
|
|
349
|
+
name: "lease-sweep",
|
|
350
|
+
intervalMs: 30_000,
|
|
351
|
+
run: () => sweepLeases(redis),
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
scheduledTasks.register({
|
|
355
|
+
name: "account-health-sweep",
|
|
356
|
+
// G-04: 30 s → 60 s — this is the most frequent snapshot invalidator, and
|
|
357
|
+
// halving its cadence halves the rebuild/cooldown-recovery churn.
|
|
358
|
+
intervalMs: 60_000,
|
|
359
|
+
// Offsets the pass from lease-sweep and runtime-metrics, which share a
|
|
360
|
+
// cadence: under the registry's single-flight gate, aligned ticks make the
|
|
361
|
+
// tasks skip each other when one runs long.
|
|
362
|
+
initialDelayMs: 5_000,
|
|
363
|
+
// G-03: this sweep can invalidate the shared snapshot, so defer it while
|
|
364
|
+
// foreground is busy (bounded) rather than racing a page open.
|
|
365
|
+
deferWhenBusy: true,
|
|
366
|
+
// Recovers cooldowns deployment-wide and invalidates the shared snapshot.
|
|
367
|
+
shared: true,
|
|
368
|
+
run: async () => {
|
|
369
|
+
const recovered = await sweepExpiredCooldowns(db);
|
|
370
|
+
const poolRecovered = await sweepExpiredPoolCooldowns(db);
|
|
371
|
+
if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
|
|
372
|
+
},
|
|
373
|
+
});
|
|
374
|
+
const payloadCapture = new TelemetryPayloadCapture(db);
|
|
375
|
+
const telemetryStore = new DrizzleTelemetryStore(db);
|
|
376
|
+
scheduledTasks.register({
|
|
377
|
+
name: "telemetry-payload-cleanup",
|
|
378
|
+
intervalMs: 15 * 60_000,
|
|
379
|
+
// Deletes payload frames for every tenant: two passes over the same
|
|
380
|
+
// batches is duplicated work, not speedup.
|
|
381
|
+
shared: true,
|
|
382
|
+
run: () => payloadCapture.cleanupExpired(),
|
|
383
|
+
});
|
|
384
|
+
scheduledTasks.register({
|
|
385
|
+
name: "console-session-sweep",
|
|
386
|
+
// Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
|
|
387
|
+
// 48 — each pass a full scan of the session table for rows that cannot yet
|
|
388
|
+
// be expired. Half the session lifetime is the point where a pass can
|
|
389
|
+
// actually find work, and still leaves a full grace period after it.
|
|
390
|
+
intervalMs: 12 * 60 * 60_000,
|
|
391
|
+
shared: true,
|
|
392
|
+
run: () => deleteExpiredSessions(db),
|
|
393
|
+
});
|
|
394
|
+
scheduledTasks.register({
|
|
395
|
+
name: "telemetry-retention",
|
|
396
|
+
intervalMs: 6 * 60 * 60_000,
|
|
397
|
+
// Retention deletes rows and files for the whole deployment; a second
|
|
398
|
+
// concurrent pass would churn the same batches.
|
|
399
|
+
shared: true,
|
|
400
|
+
run: () =>
|
|
401
|
+
telemetryStore.pruneTelemetry(
|
|
402
|
+
new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
|
|
403
|
+
),
|
|
404
|
+
});
|
|
405
|
+
const telemetryBuffer = new TelemetryBatchBuffer(db);
|
|
406
|
+
const poolSelector = new NetworkPoolSelector(redis);
|
|
407
|
+
// Replay the provider cooldown markers the previous process recorded
|
|
408
|
+
// (`network_pools.provider_cooldowns`). Without this a restart made dispatch
|
|
409
|
+
// forget every egress the upstream was still refusing: the flags are
|
|
410
|
+
// process-local otherwise, so the pool re-entered rotation and took 429s the
|
|
411
|
+
// first request after boot re-discovered. Expired entries are dropped by the
|
|
412
|
+
// restore itself. Best-effort: a failed read leaves selection matching what
|
|
413
|
+
// a fresh deployment would do.
|
|
414
|
+
try {
|
|
415
|
+
const restored = await restoreProviderCooldownMarkers(poolSelector, db);
|
|
416
|
+
const replayed = await restorePoolProbeVerdicts(poolSelector, db);
|
|
417
|
+
if (restored > 0 || replayed > 0) {
|
|
418
|
+
log.info("[pool-selector] restored provider cooldown state", { mirrored: restored, probeVerdictsApplied: replayed });
|
|
419
|
+
}
|
|
420
|
+
} catch (error) {
|
|
421
|
+
log.warn("[pool-selector] provider cooldown restore failed", { error: String(error) });
|
|
422
|
+
}
|
|
423
|
+
const poolAgentResolver = new PoolAgentResolver(
|
|
424
|
+
new DrizzleNetworkPoolLoader(db),
|
|
425
|
+
{ ssrfPolicy, db },
|
|
426
|
+
);
|
|
427
|
+
const networkBindingFactory = new ValidatedNetworkBindingFactory(
|
|
428
|
+
ssrfPolicy,
|
|
429
|
+
undefined,
|
|
430
|
+
poolAgentResolver,
|
|
431
|
+
);
|
|
432
|
+
// The account-inflight reader the console queries. It lives after the deps
|
|
433
|
+
// it closes over (telemetry buffer, binding factory), so the closure never
|
|
434
|
+
// touches a TDZ binding; it also reuses one catalog store per tenant instead
|
|
435
|
+
// of constructing a fresh store object on every console read. Multi-tenant
|
|
436
|
+
// console polling rotates the tenant id, so the memo is keyed by tenant with
|
|
437
|
+
// a small bound — a single-entry memo rebuilt the heavy store on every
|
|
438
|
+
// switch, and an unbounded map would leak one store per tenant forever.
|
|
439
|
+
const MAX_INFLIGHT_STORES = 16;
|
|
440
|
+
const inflightStores = new Map<string, DrizzleProviderCatalogStore>();
|
|
441
|
+
const readRoutingAccountInflight = async (
|
|
442
|
+
providerId: string,
|
|
443
|
+
tenantId: string | null,
|
|
444
|
+
): Promise<readonly { accountId: string; inflight: number }[]> => {
|
|
445
|
+
if (providerId === "" || tenantId === null) return [];
|
|
446
|
+
let inflightStore = inflightStores.get(tenantId);
|
|
447
|
+
if (inflightStore === undefined) {
|
|
448
|
+
// B-01: resolve the deferred bundled catalog first so the store's model
|
|
449
|
+
// map is populated (the snapshot warm normally did this already; this
|
|
450
|
+
// is the cold-race fallback).
|
|
451
|
+
await resolveBundledCatalog();
|
|
452
|
+
inflightStore = new DrizzleProviderCatalogStore(db, {
|
|
453
|
+
telemetryBuffer,
|
|
454
|
+
bundledModelCatalog: { modelsByProvider: bundledCatalogModels },
|
|
455
|
+
providerRegistry: registry,
|
|
456
|
+
outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
|
|
457
|
+
snapshotInvalidator: snapshotService,
|
|
458
|
+
});
|
|
459
|
+
inflightStores.set(tenantId, inflightStore);
|
|
460
|
+
if (inflightStores.size > MAX_INFLIGHT_STORES) {
|
|
461
|
+
const oldest = inflightStores.keys().next().value;
|
|
462
|
+
if (oldest !== undefined) inflightStores.delete(oldest);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
// P-02: only the account-id set is needed, so select just `id` — the old
|
|
466
|
+
// `listAccounts` recomputed today/lifetime usage aggregates for every
|
|
467
|
+
// account on every console poll.
|
|
468
|
+
const accountIds = await inflightStore.listAccountIds(tenantId, providerId);
|
|
469
|
+
const owned = new Set(accountIds);
|
|
470
|
+
// Scoped read: only the provider's own candidates for this tenant are
|
|
471
|
+
// queried, so the console read costs one exact MGET over the accounts it
|
|
472
|
+
// is about to report on — not a fleet-wide `KEYS` scan whose cost is the
|
|
473
|
+
// size of every bucket every process ever wrote.
|
|
474
|
+
const routeSnapshot = await snapshotService.getSnapshot();
|
|
475
|
+
const ownedCandidates = routeSnapshot.candidates.filter(
|
|
476
|
+
(candidate) =>
|
|
477
|
+
candidate.provider_id === providerId &&
|
|
478
|
+
candidate.provider_account_id !== undefined &&
|
|
479
|
+
owned.has(candidate.provider_account_id),
|
|
480
|
+
);
|
|
481
|
+
const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
|
|
482
|
+
const totals = new Map<string, number>();
|
|
483
|
+
for (const [bucket, count] of snapshot) {
|
|
484
|
+
const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
|
|
485
|
+
if (!owned.has(accountId)) continue;
|
|
486
|
+
totals.set(accountId, (totals.get(accountId) ?? 0) + count);
|
|
487
|
+
}
|
|
488
|
+
return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
|
|
489
|
+
};
|
|
490
|
+
// Runtime gauges (memory + bounded-collection sizes) sampled on the shared
|
|
491
|
+
// maintenance cadence. The sampler reads live sizes through closures so the
|
|
492
|
+
// observability layer never holds direct references to routing/pool state.
|
|
493
|
+
const runtimeMetricsSampler = new RuntimeMetricsSampler({
|
|
494
|
+
routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
|
|
495
|
+
ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
|
|
496
|
+
quotaCacheEntries: () => quotaCacheSize(),
|
|
497
|
+
modelCatalogEntries: () => modelCatalogCacheSize(),
|
|
498
|
+
// Late-bound via bindInFlight once the request state store exists.
|
|
499
|
+
inFlight: () => 0,
|
|
500
|
+
// The buffer is built below; the closure keeps the sampler from holding a
|
|
501
|
+
// reference during construction.
|
|
502
|
+
telemetryBacklog: () => telemetryBuffer?.snapshot(),
|
|
503
|
+
});
|
|
504
|
+
// One sample NOW, not on the first tick. The headroom guard reads
|
|
505
|
+
// `memorySnapshot()` on every gateway request and treats `undefined` as
|
|
506
|
+
// "unknown" — so without this the guard would spend its first interval
|
|
507
|
+
// (seconds, at the configured cadence) unable to shed at all, which is
|
|
508
|
+
// exactly the window a cold start is most likely to be short of memory.
|
|
509
|
+
runtimeMetricsSampler.sample();
|
|
510
|
+
scheduledTasks.register({
|
|
511
|
+
name: "runtime-metrics",
|
|
512
|
+
intervalMs: readInt(
|
|
513
|
+
"CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
|
|
514
|
+
CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
|
|
515
|
+
),
|
|
516
|
+
run: () => runtimeMetricsSampler.sample(),
|
|
517
|
+
});
|
|
518
|
+
scheduledTasks.register({
|
|
519
|
+
name: "pool-agent-sweep",
|
|
520
|
+
// `resolveAgent` already self-sweeps every 30 s, so a 1 min timer adds a
|
|
521
|
+
// second pass that almost always finds nothing to release. 5 min is well
|
|
522
|
+
// inside the idle timeout this exists to enforce, and outside the path
|
|
523
|
+
// that already runs.
|
|
524
|
+
intervalMs: 5 * 60_000,
|
|
525
|
+
run: () => poolAgentResolver.sweepIdleAgents(),
|
|
526
|
+
});
|
|
527
|
+
const oauthRefreshService = new OAuthRefreshService(db);
|
|
528
|
+
// Converge accounts parked on a dead refresh grant: an account whose stored
|
|
529
|
+
// access token is still usable becomes a static token (and returns to
|
|
530
|
+
// rotation) instead of staying disabled for re-auth.
|
|
531
|
+
await measureStartupPhase("static-token-reconciliation", () =>
|
|
532
|
+
reconcileStaticTokenAccounts(db),
|
|
533
|
+
);
|
|
534
|
+
// Fills in the identity a batch import deliberately skipped, so a paste of
|
|
535
|
+
// many credentials is one instant write and the email/label arrive shortly
|
|
536
|
+
// after rather than blocking the import. Purely CPU on an already-stored
|
|
537
|
+
// credential — no upstream call — so it runs often and cheaply.
|
|
538
|
+
scheduledTasks.register({
|
|
539
|
+
name: "account-enrichment-sweep",
|
|
540
|
+
// Converges via `identity_checked_at`: once every stored credential has
|
|
541
|
+
// been parsed, the listing query returns nothing — a 30 s timer would run
|
|
542
|
+
// that empty query forever, two million times a year. 10 min is still
|
|
543
|
+
// instant for the case that matters (a batch import, where the label and
|
|
544
|
+
// email show up on the next tick) and costs nothing when idle.
|
|
545
|
+
intervalMs: 10 * 60_000,
|
|
546
|
+
run: () => accountIdentitySweep({ db }),
|
|
547
|
+
});
|
|
548
|
+
scheduledTasks.register({
|
|
549
|
+
name: "oauth-refresh-sweep",
|
|
550
|
+
intervalMs: 60_000,
|
|
551
|
+
// G-04: defer when foreground is busy (InvokeGuard), and cap the pass with
|
|
552
|
+
// a budget mirroring quota refresh's 45 s so an OAuth pass cannot run away
|
|
553
|
+
// into a heavy foreground minute.
|
|
554
|
+
deferWhenBusy: true,
|
|
555
|
+
run: () =>
|
|
556
|
+
oauthRefreshSweep({
|
|
557
|
+
loadDueAccounts: () => loadDueOAuthAccounts(db),
|
|
558
|
+
refreshService: oauthRefreshService,
|
|
559
|
+
resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
560
|
+
// Sequential with a pause between accounts: the OAuth token endpoints
|
|
561
|
+
// rate-limit a burst of refreshes, and a pass over a handful of
|
|
562
|
+
// accounts must not look like one. Per-account outcomes are already
|
|
563
|
+
// pushed to the Console Log ring by `OAuthRefreshService`; this only
|
|
564
|
+
// covers an error that escaped before reaching it.
|
|
565
|
+
interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
|
|
566
|
+
budgetMs: 45_000,
|
|
567
|
+
onAccountError: (accountId, providerId, error) => {
|
|
568
|
+
log.error(
|
|
569
|
+
`[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
|
|
570
|
+
error as Error,
|
|
571
|
+
);
|
|
572
|
+
pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
|
|
573
|
+
event: "token_refresh",
|
|
574
|
+
accountId,
|
|
575
|
+
providerId,
|
|
576
|
+
errorCode: "refresh_sweep_failed",
|
|
577
|
+
});
|
|
578
|
+
},
|
|
579
|
+
}),
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
// Client versions are monitor-only metadata; let the listener settle before
|
|
583
|
+
// starting their external fetches. The scheduled task keeps the cache warm.
|
|
584
|
+
scheduledTasks.register({
|
|
585
|
+
name: "client-version-monitor",
|
|
586
|
+
intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
|
|
587
|
+
initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
|
|
588
|
+
run: refreshProviderClientVersions,
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
const quotaResolveCredential = createAccountSecretResolver({
|
|
592
|
+
db,
|
|
593
|
+
resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
594
|
+
refreshService: oauthRefreshService,
|
|
595
|
+
});
|
|
596
|
+
// Keeps every account's cached quota warm so opening the Quota page is a
|
|
597
|
+
// cache read. Without this the only refreshes are user-triggered, so the
|
|
598
|
+
// first open after an expiry always blocks on the provider.
|
|
599
|
+
//
|
|
600
|
+
// It also carries the daily check-in ride-along: each account refreshed
|
|
601
|
+
// here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
|
|
602
|
+
// the same pass (one attempt per account per day, guarded by a day
|
|
603
|
+
// marker), so no second timer duplicates the credential resolution.
|
|
604
|
+
//
|
|
605
|
+
// Registered in **every** mode, not only when Redis is present: gating it on
|
|
606
|
+
// Redis would cost Lite proactive quota warming and the check-in ride-along
|
|
607
|
+
// above that claims the once-per-day WorkBuddy/CodeBuddy credit grant. The
|
|
608
|
+
// whole sweep chain accepts `RedisClient | undefined` and falls back to the
|
|
609
|
+
// in-process TTL cache and an in-process day marker, so Redis is an
|
|
610
|
+
// optimization here rather than a precondition — which is exactly what
|
|
611
|
+
// Lite is for.
|
|
612
|
+
scheduledTasks.register({
|
|
613
|
+
name: "quota-refresh-sweep",
|
|
614
|
+
intervalMs: 60_000,
|
|
615
|
+
// Staggered off the oauth/lease/health ticks so the three sweeps that
|
|
616
|
+
// strike providerAccounts do not land together every minute.
|
|
617
|
+
initialDelayMs: 20_000,
|
|
618
|
+
// G-03: defer while foreground is busy (bounded).
|
|
619
|
+
deferWhenBusy: true,
|
|
620
|
+
run: () =>
|
|
621
|
+
quotaRefreshSweep({
|
|
622
|
+
db,
|
|
623
|
+
redis,
|
|
624
|
+
providerRegistry: registry,
|
|
625
|
+
resolveCredential: quotaResolveCredential,
|
|
626
|
+
snapshotInvalidator: snapshotService,
|
|
627
|
+
// Without this the sweep hands each collector the raw stored secret.
|
|
628
|
+
// For `cline` that means an `sk_...` key reaching the OAuth branch,
|
|
629
|
+
// which prepends `workos:` and is refused with "please make sure
|
|
630
|
+
// you're using the latest version of Cline and re-authenticate" — so
|
|
631
|
+
// every sweep cools a working account with `quota_fetch_failed` while
|
|
632
|
+
// the same account refreshes fine from the console, which binds the
|
|
633
|
+
// same marker through `createQuotaRefreshDeps`.
|
|
634
|
+
markApiKeyCredential: markProviderApiKeyCredential,
|
|
635
|
+
// Below the 60 s interval: a budget longer than the tick guarantees
|
|
636
|
+
// skip/burst under load instead of steady progress.
|
|
637
|
+
passBudgetMs: 45_000,
|
|
638
|
+
// The check-in ride-along rotates egress per account: each account
|
|
639
|
+
// gets the next active pool in its tenant's rotation so check-ins
|
|
640
|
+
// spread across IPs instead of sharing one direct egress. No pool
|
|
641
|
+
// (or an unreadable pool table) falls back to direct egress, and a
|
|
642
|
+
// tenant never borrows another tenant's pool.
|
|
643
|
+
checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
|
|
644
|
+
// Per-account lines and the pass summary are emitted by the sweep
|
|
645
|
+
// itself at `info` level, so they survive the production LOG_LEVEL.
|
|
646
|
+
// `onTick` is left for tests.
|
|
647
|
+
}),
|
|
648
|
+
});
|
|
649
|
+
registerZcodeGiftClaimTask(scheduledTasks, {
|
|
650
|
+
db,
|
|
651
|
+
redis,
|
|
652
|
+
resolveCredential: quotaResolveCredential,
|
|
653
|
+
snapshotInvalidator: snapshotService,
|
|
654
|
+
fetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
|
|
655
|
+
});
|
|
656
|
+
// The buddy growth/credit tasks: one per upstream capability, each
|
|
657
|
+
// independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
|
|
658
|
+
// them is a credit gain the account is entitled to, and none of them can mark
|
|
659
|
+
// an account unhealthy — a growth endpoint that 404s is a display miss, not a
|
|
660
|
+
// dispatch failure.
|
|
661
|
+
//
|
|
662
|
+
// Daily, not hourly: each claim is once per account per day and is
|
|
663
|
+
// ledger-guarded, so at an hourly cadence 23 of every 24 passes are a full
|
|
664
|
+
// scan of active accounts plus a ledger read per account that resolves to
|
|
665
|
+
// nothing. Once a day is the shortest interval where a pass can still find
|
|
666
|
+
// work, which is the only reason to run at all.
|
|
667
|
+
const buddyTaskDeps = {
|
|
668
|
+
db,
|
|
669
|
+
resolveCredential: quotaResolveCredential,
|
|
670
|
+
fetcher: fetch,
|
|
671
|
+
// Spread across accounts so a pass does not look like a burst to the
|
|
672
|
+
// upstream's risk gate.
|
|
673
|
+
interAccountDelayMs: 800,
|
|
674
|
+
// G-04: bound each pass. These are daily/six-hourly tasks, so the budget
|
|
675
|
+
// is deliberately larger than the minute-cadence sweeps' 45 s — it caps a
|
|
676
|
+
// runaway scan without truncating a legitimate fleet mid-pass.
|
|
677
|
+
budgetMs: 5 * 60_000,
|
|
678
|
+
};
|
|
679
|
+
scheduledTasks.register({
|
|
680
|
+
name: "buddy-streak",
|
|
681
|
+
intervalMs: 24 * 60 * 60_000,
|
|
682
|
+
initialDelayMs: 5 * 60_000,
|
|
683
|
+
// G-03: defer while foreground is busy (bounded).
|
|
684
|
+
deferWhenBusy: true,
|
|
685
|
+
run: () => buddyStreakTask(buddyTaskDeps),
|
|
686
|
+
});
|
|
687
|
+
scheduledTasks.register({
|
|
688
|
+
name: "buddy-adopt",
|
|
689
|
+
intervalMs: 24 * 60 * 60_000,
|
|
690
|
+
initialDelayMs: 17 * 60_000,
|
|
691
|
+
deferWhenBusy: true,
|
|
692
|
+
run: () => buddyAdoptTask(buddyTaskDeps),
|
|
693
|
+
});
|
|
694
|
+
// Growth-activity report: the `/v2/report` batch that lights the login streak
|
|
695
|
+
// and scores the daily growth event. Once a day with its own ledger — it used
|
|
696
|
+
// to ride the quota sweep with no marker of its own, so its eight POSTs ran
|
|
697
|
+
// every minute.
|
|
698
|
+
scheduledTasks.register({
|
|
699
|
+
name: "buddy-activity-report",
|
|
700
|
+
intervalMs: 24 * 60 * 60_000,
|
|
701
|
+
initialDelayMs: 15 * 60 * 60_000,
|
|
702
|
+
deferWhenBusy: true,
|
|
703
|
+
run: () => buddyActivityReportTask(buddyTaskDeps),
|
|
704
|
+
});
|
|
705
|
+
// Real credit deductions, stored so the console renders a figure the upstream
|
|
706
|
+
// actually reported. Six-hourly: the upstream's own window is 30 days, so a
|
|
707
|
+
// day's number does not change intraday.
|
|
708
|
+
scheduledTasks.register({
|
|
709
|
+
name: "buddy-usage-sync",
|
|
710
|
+
intervalMs: 6 * 60 * 60_000,
|
|
711
|
+
initialDelayMs: 23 * 60_000,
|
|
712
|
+
deferWhenBusy: true,
|
|
713
|
+
run: () => syncBuddyUsage(buddyTaskDeps),
|
|
714
|
+
});
|
|
715
|
+
// The scheduled tasks are registered here but started by the caller once the
|
|
716
|
+
// listener is actually serving (`main.ts`). Starting them inside this builder
|
|
717
|
+
// would let the first tick — the lease sweep, the health sweep — run against
|
|
718
|
+
// a process that has not begun accepting traffic yet.
|
|
719
|
+
const deps: ProductionDeps = {
|
|
720
|
+
db,
|
|
721
|
+
redis,
|
|
722
|
+
snapshotService,
|
|
723
|
+
readRoutingAccountInflight,
|
|
724
|
+
resolveProviderAdapter: (providerId) => registry.resolve(providerId),
|
|
725
|
+
bundledModelCatalog: { modelsByProvider: bundledCatalogModels },
|
|
726
|
+
byokUpstreamHosts,
|
|
727
|
+
proxyPreparer: new ProxyRequestPreparer({
|
|
728
|
+
snapshotService,
|
|
729
|
+
routingEngine,
|
|
730
|
+
admissionService,
|
|
731
|
+
}),
|
|
732
|
+
networkBindingFactory,
|
|
733
|
+
ipAbuseProtection,
|
|
734
|
+
readiness,
|
|
735
|
+
scheduledTasks,
|
|
736
|
+
poolAgentResolver,
|
|
737
|
+
telemetryBuffer,
|
|
738
|
+
usageFlush,
|
|
739
|
+
healthEventFlush,
|
|
740
|
+
poolSelector,
|
|
741
|
+
admissionService,
|
|
742
|
+
trustedProxyBoundary: resolveTrustedProxyBoundary(),
|
|
743
|
+
providerRegistry: registry,
|
|
744
|
+
resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
745
|
+
oauthRefreshService,
|
|
746
|
+
runtimeMetricsSampler,
|
|
747
|
+
};
|
|
748
|
+
const startupMemoryAfter = process.memoryUsage();
|
|
749
|
+
log.info(
|
|
750
|
+
`[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
|
|
751
|
+
`rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
|
|
752
|
+
`rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
|
|
753
|
+
`heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
|
|
754
|
+
`heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
|
|
755
|
+
);
|
|
756
|
+
return deps;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
|
|
760
|
+
const startedAt = performance.now();
|
|
761
|
+
const before = process.memoryUsage();
|
|
762
|
+
log.info(
|
|
763
|
+
`[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
|
|
764
|
+
);
|
|
765
|
+
try {
|
|
766
|
+
const result = await operation();
|
|
767
|
+
const after = process.memoryUsage();
|
|
768
|
+
log.info(
|
|
769
|
+
`[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
|
|
770
|
+
`rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
|
|
771
|
+
`heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
|
|
772
|
+
`external_mb=${toMiB(after.external)}`,
|
|
773
|
+
);
|
|
774
|
+
return result;
|
|
775
|
+
} catch (error) {
|
|
776
|
+
log.error(
|
|
777
|
+
`[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
|
|
778
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
779
|
+
);
|
|
780
|
+
throw error;
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
function toMiB(bytes: number): string {
|
|
785
|
+
return (bytes / 1024 / 1024).toFixed(1);
|
|
786
|
+
}
|