cartethyia 2.5.4 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -4
- package/README.md +2 -2
- package/dist/dashboard/assets/{BannedPage-C2orBsI2.js → BannedPage-DE2607Tb.js} +1 -1
- package/dist/dashboard/assets/BannedPage-DE2607Tb.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-Duv6mfrl.js +1 -0
- package/dist/dashboard/assets/CliToolDetailPage-Duv6mfrl.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-BV6pxVQo.js → CliToolsPage-C2PNYWzk.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-C2PNYWzk.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-TXP9OfAM.js +1 -0
- package/dist/dashboard/assets/CombosPage-TXP9OfAM.js.gz +0 -0
- package/dist/dashboard/assets/{ConsoleLogPage-CW-wPmJ6.js → ConsoleLogPage-B-9xnAQN.js} +1 -1
- package/dist/dashboard/assets/ConsoleLogPage-B-9xnAQN.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-0p8uX7vB.js → CustomizationPage-xeax9nJt.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-xeax9nJt.js.gz +0 -0
- package/dist/dashboard/assets/{HealthEventsModal-BjqSMfko.js → HealthEventsModal-Df4jjaHJ.js} +1 -1
- package/dist/dashboard/assets/HealthEventsModal-Df4jjaHJ.js.gz +0 -0
- package/dist/dashboard/assets/{LoginPage-BvzCx8Kr.js → LoginPage-Dyojsm84.js} +1 -1
- package/dist/dashboard/assets/LoginPage-Dyojsm84.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-C7pDb41R.js +1 -0
- package/dist/dashboard/assets/ModelPicker-C7pDb41R.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-BJfg4ow6.js +1 -0
- package/dist/dashboard/assets/OverviewPage-BJfg4ow6.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-ChCX1gmT.js +3 -0
- package/dist/dashboard/assets/ProviderDetailPage-ChCX1gmT.js.gz +0 -0
- package/dist/dashboard/assets/{ProvidersPage-Js9jUPSb.js → ProvidersPage-DeWSOEUO.js} +1 -1
- package/dist/dashboard/assets/ProvidersPage-DeWSOEUO.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-CkpWln2S.js +7 -0
- package/dist/dashboard/assets/ProxyPage-CkpWln2S.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-B3VOgQmt.js +1 -0
- package/dist/dashboard/assets/QuotaPage-B3VOgQmt.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-C_klIZom.js +1 -0
- package/dist/dashboard/assets/SettingsPage-C_klIZom.js.gz +0 -0
- package/dist/dashboard/assets/{SetupPage-DDbMQsoo.js → SetupPage-BYph-InK.js} +1 -1
- package/dist/dashboard/assets/SetupPage-BYph-InK.js.gz +0 -0
- package/dist/dashboard/assets/{SortableList-CB3Z7VTf.js → SortableList-CvyQCYWJ.js} +1 -1
- package/dist/dashboard/assets/SortableList-CvyQCYWJ.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-D95UCPHI.js → StudioPage-Cm1JYilI.js} +3 -3
- package/dist/dashboard/assets/StudioPage-Cm1JYilI.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-DlFnc3_S.js +56 -0
- package/dist/dashboard/assets/UsagePage-DlFnc3_S.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-DQYLQxpW.js +1 -0
- package/dist/dashboard/assets/api-keys-DQYLQxpW.js.gz +0 -0
- package/dist/dashboard/assets/app-CX_Z8cMs.css +2 -0
- package/dist/dashboard/assets/app-CX_Z8cMs.css.gz +0 -0
- package/dist/dashboard/assets/{assert-Chi7pCnt.js → assert-C9FD_JwJ.js} +1 -1
- package/dist/dashboard/assets/assert-C9FD_JwJ.js.gz +0 -0
- package/dist/dashboard/assets/{badge-D4pQapKj.js → badge-CR72dasT.js} +1 -1
- package/dist/dashboard/assets/badge-CR72dasT.js.gz +0 -0
- package/dist/dashboard/assets/{banner-DH6cg-_B.js → banner-CI9_eSfO.js} +1 -1
- package/dist/dashboard/assets/banner-CI9_eSfO.js.gz +0 -0
- package/dist/dashboard/assets/{card-BKqlUOe4.js → card-DS7CQEsO.js} +1 -1
- package/dist/dashboard/assets/card-DS7CQEsO.js.gz +0 -0
- package/dist/dashboard/assets/{cli-tools-fTgljTGA.js → cli-tools-CNqzrfdb.js} +1 -1
- package/dist/dashboard/assets/cli-tools-CNqzrfdb.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-D5Ys_sM0.js → clipboard-button-611pSsaX.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-611pSsaX.js.gz +0 -0
- package/dist/dashboard/assets/{fingerprint-pattern-BWuHtbW4.js → concurrency-BDF0ok7y.js} +1 -1
- package/dist/dashboard/assets/concurrency-BDF0ok7y.js.gz +0 -0
- package/dist/dashboard/assets/dialog-k37exjNt.js +1 -0
- package/dist/dashboard/assets/dialog-k37exjNt.js.gz +0 -0
- package/dist/dashboard/assets/drawer-CX7N8ucR.js +1 -0
- package/dist/dashboard/assets/drawer-CX7N8ucR.js.gz +0 -0
- package/dist/dashboard/assets/{entry-CQBr6U9c.js → entry-B9Xvtnfj.js} +1 -1
- package/dist/dashboard/assets/entry-B9Xvtnfj.js.gz +0 -0
- package/dist/dashboard/assets/entry-Bog0p925.js +4 -0
- package/dist/dashboard/assets/entry-Bog0p925.js.gz +0 -0
- package/dist/dashboard/assets/entry-CTP8AyiS.js +1 -0
- package/dist/dashboard/assets/entry-CTP8AyiS.js.gz +0 -0
- package/dist/dashboard/assets/{github-badge-BlptL8Uf.js → github-badge-sltk1qUi.js} +1 -1
- package/dist/dashboard/assets/github-badge-sltk1qUi.js.gz +0 -0
- package/dist/dashboard/assets/{index-js6TJvdr.js → index-CZQIrdoX.js} +2 -2
- package/dist/dashboard/assets/index-CZQIrdoX.js.gz +0 -0
- package/dist/dashboard/assets/{inline-DUWDSmDV.js → inline-BhY2sSnO.js} +1 -1
- package/dist/dashboard/assets/inline-BhY2sSnO.js.gz +0 -0
- package/dist/dashboard/assets/{input-LVjgZq4f.js → input-V1umFiBk.js} +1 -1
- package/dist/dashboard/assets/input-V1umFiBk.js.gz +0 -0
- package/dist/dashboard/assets/layout-BBOYycyS.js +1 -0
- package/dist/dashboard/assets/layout-BBOYycyS.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-DupvLfDx.js → list-row-1zzZYM0B.js} +1 -1
- package/dist/dashboard/assets/list-row-1zzZYM0B.js.gz +0 -0
- package/dist/dashboard/assets/{live-C4pEJUqI.js → live-BivGfuUs.js} +1 -1
- package/dist/dashboard/assets/live-BivGfuUs.js.gz +0 -0
- package/dist/dashboard/assets/{model-bans-BRiuufkB.js → model-bans-CmffjLuG.js} +1 -1
- package/dist/dashboard/assets/model-bans-CmffjLuG.js.gz +0 -0
- package/dist/dashboard/assets/{mutation-B4ROwk8L.js → mutation-46P0oWMe.js} +1 -1
- package/dist/dashboard/assets/mutation-46P0oWMe.js.gz +0 -0
- package/dist/dashboard/assets/numbers-DkfMR5Qs.js +2 -0
- package/dist/dashboard/assets/numbers-DkfMR5Qs.js.gz +0 -0
- package/dist/dashboard/assets/{query-keys-gmjzC3RS.js → query-keys-BFhGpwR1.js} +1 -1
- package/dist/dashboard/assets/query-keys-BFhGpwR1.js.gz +0 -0
- package/dist/dashboard/assets/{routing-C7tBT5o-.js → routing-CAb4F37R.js} +1 -1
- package/dist/dashboard/assets/routing-CAb4F37R.js.gz +0 -0
- package/dist/dashboard/assets/{select-WUCXorXw.js → select-BPpSsBgL.js} +1 -1
- package/dist/dashboard/assets/select-BPpSsBgL.js.gz +0 -0
- package/dist/dashboard/assets/{settings-C_zKl1GS.js → settings-DNX9sydi.js} +1 -1
- package/dist/dashboard/assets/settings-DNX9sydi.js.gz +0 -0
- package/dist/dashboard/assets/{stack-_3O0F8Xn.js → stack-BHTfXdz_.js} +1 -1
- package/dist/dashboard/assets/stack-BHTfXdz_.js.gz +0 -0
- package/dist/dashboard/assets/{state-BgTxHcJS.js → state-mPDRlQkl.js} +1 -1
- package/dist/dashboard/assets/state-mPDRlQkl.js.gz +0 -0
- package/dist/dashboard/assets/{switch-DEjTtOWf.js → switch-DOmJxg7H.js} +1 -1
- package/dist/dashboard/assets/switch-DOmJxg7H.js.gz +0 -0
- package/dist/dashboard/assets/tokens-COzario9.js +1 -0
- package/dist/dashboard/assets/tokens-COzario9.js.gz +0 -0
- package/dist/dashboard/assets/use-modal-focus-By7oJoGd.js +1 -0
- package/dist/dashboard/assets/use-modal-focus-By7oJoGd.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-BWSC0UAS.js → useMutation-Dem0lXQj.js} +1 -1
- package/dist/dashboard/assets/useMutation-Dem0lXQj.js.gz +0 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +54 -54
- package/migrations/README.md +52 -52
- package/migrations/lite-sqlite3/0000_baseline.sql +2 -2
- package/package.json +95 -95
- package/scripts/cli/admin.ts +125 -0
- package/scripts/cli/{ui/background.ts → background.ts} +1 -1
- package/scripts/cli/{command/console-menu.ts → console-menu.ts} +275 -244
- package/scripts/cli/doctor.ts +37 -0
- package/scripts/cli/{support/env.ts → env.ts} +99 -99
- package/scripts/cli/{gateway/spawn.ts → gateway.ts} +112 -24
- package/scripts/cli/instance.ts +224 -0
- package/scripts/cli/main.ts +61 -18
- package/scripts/cli/{support/platform.ts → platform.ts} +64 -64
- package/scripts/cli/port.ts +100 -0
- package/scripts/cli/{support/services.ts → services.ts} +88 -88
- package/scripts/cli/{command/setup.ts → setup.ts} +35 -34
- package/scripts/cli/{command/start.ts → start.ts} +10 -5
- package/scripts/cli/{support/startup-signal.ts → startup-signal.ts} +23 -23
- package/scripts/cli/tray/index.ts +62 -12
- package/scripts/cli/{command/tray.ts → tray.ts} +208 -208
- package/scripts/cli/{command/update.ts → update.ts} +39 -39
- package/scripts/cli/version.ts +67 -0
- package/scripts/cli/{ui/wordmark.ts → wordmark.ts} +58 -58
- package/src/boot/app.ts +697 -693
- package/src/boot/dependencies.ts +647 -646
- package/src/boot/timeout.ts +2 -13
- package/src/config/cooldowns.ts +186 -138
- package/src/config/network.ts +8 -14
- package/src/config/providers.ts +7 -10
- package/src/config/runtime.ts +76 -76
- package/src/config/security.ts +15 -0
- package/src/config/spec.ts +20 -30
- package/src/config/telemetry.ts +0 -20
- package/src/config/timeouts.ts +0 -6
- package/src/console/admin/model-abuse/contracts.ts +28 -6
- package/src/console/admin/model-abuse/routes.ts +56 -3
- package/src/console/api-keys/contracts.ts +410 -414
- package/src/console/api-keys/routes.ts +881 -880
- package/src/console/auth/credentials.ts +3 -3
- package/src/console/auth/setup-service.ts +2 -2
- package/src/console/backup/export-endpoint.ts +6 -2
- package/src/console/backup/nine-router.ts +2 -5
- package/src/console/backup/service.ts +4 -0
- package/src/console/backup/store.ts +192 -29
- package/src/console/backup/validate.ts +3 -45
- package/src/console/cli-tools/fs-ops.ts +487 -487
- package/src/console/console-router-guards.ts +1 -1
- package/src/console/data-management/contracts.ts +13 -6
- package/src/console/data-management/operations.ts +23 -18
- package/src/console/data-management/routes.ts +0 -1
- package/src/console/domain-registration.ts +10 -5
- package/src/console/observability/live.ts +22 -0
- package/src/console/observability/store.ts +39 -16
- package/src/console/observability/usage-dimensions.ts +4 -4
- package/src/console/providers/catalog/catalog-projections.ts +6 -7
- package/src/console/providers/catalog/model-operations.ts +5 -2
- package/src/console/providers/catalog/provider-operations.ts +4 -4
- package/src/console/providers/catalog/public-model-store.ts +8 -11
- package/src/console/providers/catalog/store.ts +1553 -1553
- package/src/console/providers/oauth/routes.ts +8 -12
- package/src/console/quota/account-quota-tenant.ts +6 -6
- package/src/console/quota/quota-view.ts +1 -1
- package/src/console/remote-support/contracts.ts +61 -0
- package/src/console/remote-support/routes.ts +65 -0
- package/src/console/remote-support/service.ts +207 -0
- package/src/console/response-compression.ts +0 -3
- package/src/console/routing/pools/probe-result.ts +3 -2
- package/src/console/routing/pools/probe-verdict.ts +132 -132
- package/src/console/routing/pools/store.ts +8 -8
- package/src/console/settings/contracts.ts +58 -2
- package/src/console/settings/store.ts +30 -3
- package/src/console/shared/cursor.ts +8 -8
- package/src/console/shared/errors.ts +1 -3
- package/src/network/http2-bidi.ts +2 -4
- package/src/network/pool/agent.ts +478 -478
- package/src/network/pool/selector.ts +5 -4
- package/src/network/retry.ts +1 -2
- package/src/network/ssrf.ts +5 -4
- package/src/observability/event-compaction.ts +80 -78
- package/src/observability/event-day-index.ts +13 -15
- package/src/observability/event-log.ts +4 -4
- package/src/observability/logger.ts +6 -6
- package/src/observability/payload-capture.ts +151 -33
- package/src/observability/payload-store.ts +397 -440
- package/src/observability/redaction.ts +5 -6
- package/src/observability/request-notify.ts +43 -0
- package/src/observability/telemetry-status.ts +8 -9
- package/src/observability/token-speed.ts +5 -4
- package/src/persistence/db-mode.ts +7 -7
- package/src/persistence/readiness.ts +1 -1
- package/src/persistence/schema.ts +1406 -1402
- package/src/persistence/shared-cache.ts +1 -2
- package/src/persistence/stores/api-key-store.ts +18 -6
- package/src/persistence/stores/share-store.ts +4 -3
- package/src/persistence/stores/telemetry-store.ts +236 -237
- package/src/protocol/canonical-model.ts +11 -3
- package/src/protocol/codecs.ts +138 -4
- package/src/protocol/gateway-error.ts +6 -4
- package/src/protocol/request/chat.ts +21 -1
- package/src/protocol/request/codex.ts +10 -1
- package/src/protocol/request/messages.ts +4 -5
- package/src/protocol/request/responses.ts +9 -1
- package/src/protocol/response/chat.ts +34 -4
- package/src/protocol/response/codex.ts +7 -4
- package/src/protocol/response/dsml.ts +140 -28
- package/src/protocol/response/responses.ts +11 -5
- package/src/protocol/stream-error-frames.ts +4 -5
- package/src/providers/authentication/jwt-validator.ts +9 -10
- package/src/providers/authentication/oauth-refresh-service.ts +9 -11
- package/src/providers/configured-openai-providers.ts +134 -136
- package/src/providers/discovery/catalog/catalog-blob.ts +18 -18
- package/src/providers/discovery/catalog/catalog-format.ts +3 -6
- package/src/providers/discovery/models-dev-catalog.ts +354 -354
- package/src/providers/discovery/probe-phases.ts +4 -4
- package/src/providers/discovery/probe-wire.ts +6 -13
- package/src/providers/discovery/probing-service.ts +1117 -1045
- package/src/providers/integrations/alibabacp/alibabacp.ts +1 -1
- package/src/providers/integrations/antigravity/antigravity-quota.ts +5 -7
- package/src/providers/integrations/buddy/buddy-chat-shared.ts +23 -2
- package/src/providers/integrations/buddy/buddy-checkin.ts +1 -4
- package/src/providers/integrations/buddy/buddy-cli-headers.ts +23 -6
- package/src/providers/integrations/buddy/buddy-conversation-headers.ts +0 -1
- package/src/providers/integrations/buddy/buddy-identity.ts +52 -0
- package/src/providers/integrations/buddy/buddy-media.ts +2 -2
- package/src/providers/integrations/buddy/buddy-usage-store.ts +3 -4
- package/src/providers/integrations/buddy/codebuddy/quota.ts +2 -13
- package/src/providers/integrations/buddy/codebuddy/shared.ts +118 -115
- package/src/providers/integrations/buddy/workbuddy/quota.ts +0 -11
- package/src/providers/integrations/buddy/workbuddy/shared.ts +5 -2
- package/src/providers/integrations/claude/claude.ts +608 -608
- package/src/providers/integrations/cline/cline-quota.ts +132 -132
- package/src/providers/integrations/cline/cline.ts +537 -539
- package/src/providers/integrations/codex/codex-oauth.ts +1 -1
- package/src/providers/integrations/commandcode/commandcode.ts +674 -675
- package/src/providers/integrations/commandcode/discovery.ts +117 -116
- package/src/providers/integrations/cursor/cursor-auth.ts +418 -418
- package/src/providers/integrations/cursor/cursor-quota.ts +309 -310
- package/src/providers/integrations/cursor/cursor.ts +11 -163
- package/src/providers/integrations/factory/factory-models.ts +3 -5
- package/src/providers/integrations/grok/grok-install-id.ts +2 -3
- package/src/providers/integrations/grok/grok.ts +6 -7
- package/src/providers/integrations/hermes.ts +14 -18
- package/src/providers/integrations/kiro/kiro-discovery.ts +1 -1
- package/src/providers/integrations/kiro/kiro.ts +1 -1
- package/src/providers/integrations/mistral.ts +6 -6
- package/src/providers/integrations/opencode/fingerprint.ts +3 -3
- package/src/providers/integrations/opencode/provider.ts +8 -8
- package/src/providers/integrations/qoder/qoder-catalog.ts +450 -450
- package/src/providers/integrations/qoder/qoder-oauth.ts +388 -310
- package/src/providers/integrations/qoder/qoder.ts +884 -865
- package/src/providers/integrations/shared/connect.ts +6 -6
- package/src/providers/integrations/trae/trae-catalog.ts +1 -2
- package/src/providers/integrations/trae/trae.ts +2 -2
- package/src/providers/integrations/zcode/oauth.ts +6 -1
- package/src/providers/operations/account-health-service.ts +1528 -1523
- package/src/providers/operations/account-reset-service.ts +1 -1
- package/src/providers/operations/client-versions.ts +6 -6
- package/src/providers/operations/custom-cli-headers.ts +3 -4
- package/src/providers/operations/provider-catalog-seeder.ts +95 -95
- package/src/providers/operations/provider-catalog-service.ts +3 -4
- package/src/providers/provider-metadata.ts +4 -8
- package/src/providers/provider-registry.ts +6 -7
- package/src/providers/provider-usage.ts +1 -1
- package/src/security/abuse.ts +15 -10
- package/src/security/access-control.ts +144 -146
- package/src/security/admission/contracts.ts +11 -0
- package/src/security/admission/in-memory-store.ts +4 -3
- package/src/security/admission/lease-sweep.ts +6 -6
- package/src/security/admission/reasons.ts +5 -1
- package/src/security/admission/redis-store.ts +211 -210
- package/src/security/admission/service.ts +4 -0
- package/src/security/api-key-auth.ts +504 -504
- package/src/security/ip-boundary.ts +155 -15
- package/src/security/model-abuse.ts +366 -94
- package/src/security/outbound-headers.ts +3 -4
- package/src/shared/sleep.ts +4 -0
- package/src/transport/dispatch/attempt-finalize.ts +47 -5
- package/src/transport/dispatch/attempt-loop.ts +531 -530
- package/src/transport/dispatch/stream-bridge.ts +1 -3
- package/src/transport/dispatch/streaming/finalize.ts +1 -1
- package/src/transport/dispatch/streaming/stream.ts +5 -6
- package/src/transport/failure-policy.ts +162 -8
- package/src/transport/guards/auth-guard.ts +7 -4
- package/src/transport/guards/concurrency-guard.ts +0 -5
- package/src/transport/guards/error-lifecycle.ts +13 -6
- package/src/transport/guards/pipeline.ts +15 -3
- package/src/transport/guards/request-context.ts +8 -2
- package/src/transport/request/inflight.ts +5 -6
- package/src/transport/request/preparer.ts +2 -2
- package/src/transport/request/state.ts +12 -8
- package/src/transport/routing/admission.ts +4 -0
- package/src/transport/routing/route-catalog.ts +8 -1
- package/src/transport/routing/route-model.ts +46 -14
- package/src/transport/routing/router.ts +21 -14
- package/src/transport/surface/completion.ts +2 -3
- package/src/transport/surface/dialects.ts +5 -6
- package/src/transport/surface/responses/encode.ts +38 -0
- package/src/transport/surface/responses/parse.ts +4 -0
- package/src/transport/translation/capabilities.ts +2 -3
- package/src/transport/translation/quirks.ts +3 -3
- package/src/workers/buddy-growth-tasks.ts +5 -10
- package/src/workers/daily-checkin.ts +17 -23
- package/src/workers/quota-refresh-sweep.ts +26 -26
- package/dist/dashboard/assets/BannedPage-C2orBsI2.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js +0 -1
- package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-BV6pxVQo.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-Bsan4679.js +0 -1
- package/dist/dashboard/assets/CombosPage-Bsan4679.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-CW-wPmJ6.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-0p8uX7vB.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-BjqSMfko.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-BvzCx8Kr.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-BY1aKEkP.js +0 -1
- package/dist/dashboard/assets/ModelPicker-BY1aKEkP.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js +0 -1
- package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js +0 -3
- package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-Js9jUPSb.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js +0 -7
- package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js +0 -1
- package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-CS86ho5X.js +0 -1
- package/dist/dashboard/assets/SettingsPage-CS86ho5X.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-DDbMQsoo.js.gz +0 -0
- package/dist/dashboard/assets/SortableList-CB3Z7VTf.js.gz +0 -0
- package/dist/dashboard/assets/StudioPage-D95UCPHI.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-Cnae9SWA.js +0 -56
- package/dist/dashboard/assets/UsagePage-Cnae9SWA.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-VY68Mhd1.js +0 -1
- package/dist/dashboard/assets/api-keys-VY68Mhd1.js.gz +0 -0
- package/dist/dashboard/assets/app-COlK95oi.css +0 -2
- package/dist/dashboard/assets/app-COlK95oi.css.gz +0 -0
- package/dist/dashboard/assets/assert-Chi7pCnt.js.gz +0 -0
- package/dist/dashboard/assets/badge-D4pQapKj.js.gz +0 -0
- package/dist/dashboard/assets/banner-DH6cg-_B.js.gz +0 -0
- package/dist/dashboard/assets/card-BKqlUOe4.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-fTgljTGA.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-D5Ys_sM0.js.gz +0 -0
- package/dist/dashboard/assets/dialog-BYj-8WuK.js +0 -1
- package/dist/dashboard/assets/dialog-BYj-8WuK.js.gz +0 -0
- package/dist/dashboard/assets/drawer-CmXIZ6H1.js +0 -1
- package/dist/dashboard/assets/drawer-CmXIZ6H1.js.gz +0 -0
- package/dist/dashboard/assets/entry-CBg5zeu8.js +0 -1
- package/dist/dashboard/assets/entry-CBg5zeu8.js.gz +0 -0
- package/dist/dashboard/assets/entry-CQBr6U9c.js.gz +0 -0
- package/dist/dashboard/assets/entry-CgdSAzEV.js +0 -4
- package/dist/dashboard/assets/entry-CgdSAzEV.js.gz +0 -0
- package/dist/dashboard/assets/fingerprint-pattern-BWuHtbW4.js.gz +0 -0
- package/dist/dashboard/assets/github-badge-BlptL8Uf.js.gz +0 -0
- package/dist/dashboard/assets/index-js6TJvdr.js.gz +0 -0
- package/dist/dashboard/assets/inline-DUWDSmDV.js.gz +0 -0
- package/dist/dashboard/assets/input-LVjgZq4f.js.gz +0 -0
- package/dist/dashboard/assets/layout-CPfxpsfy.js +0 -1
- package/dist/dashboard/assets/layout-CPfxpsfy.js.gz +0 -0
- package/dist/dashboard/assets/list-row-DupvLfDx.js.gz +0 -0
- package/dist/dashboard/assets/live-C4pEJUqI.js.gz +0 -0
- package/dist/dashboard/assets/model-bans-BRiuufkB.js.gz +0 -0
- package/dist/dashboard/assets/mutation-B4ROwk8L.js.gz +0 -0
- package/dist/dashboard/assets/numbers-YNcHvRY7.js +0 -2
- package/dist/dashboard/assets/numbers-YNcHvRY7.js.gz +0 -0
- package/dist/dashboard/assets/query-keys-gmjzC3RS.js.gz +0 -0
- package/dist/dashboard/assets/routing-C7tBT5o-.js.gz +0 -0
- package/dist/dashboard/assets/select-WUCXorXw.js.gz +0 -0
- package/dist/dashboard/assets/settings-C_zKl1GS.js.gz +0 -0
- package/dist/dashboard/assets/stack-_3O0F8Xn.js.gz +0 -0
- package/dist/dashboard/assets/state-BgTxHcJS.js.gz +0 -0
- package/dist/dashboard/assets/switch-DEjTtOWf.js.gz +0 -0
- package/dist/dashboard/assets/tokens-DDxMbSRL.js +0 -1
- package/dist/dashboard/assets/tokens-DDxMbSRL.js.gz +0 -0
- package/dist/dashboard/assets/use-presence-BSZEBx7U.js +0 -1
- package/dist/dashboard/assets/use-presence-BSZEBx7U.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-BWSC0UAS.js.gz +0 -0
- package/scripts/cli/command/admin.ts +0 -65
- package/scripts/cli/command/doctor.ts +0 -257
- package/scripts/cli/gateway/drain-token.ts +0 -78
- package/scripts/cli/gateway/instance-lock.ts +0 -105
- package/scripts/cli/gateway/stop.ts +0 -51
- package/scripts/cli/support/bun.ts +0 -27
- package/scripts/cli/support/version.ts +0 -37
- /package/dist/dashboard/assets/{app-DCQZRmHY.js → app-BIt446yE.js} +0 -0
- /package/dist/dashboard/assets/{app-DCQZRmHY.js.gz → app-BIt446yE.js.gz} +0 -0
- /package/scripts/cli/{ui/prompt.ts → prompt.ts} +0 -0
package/src/boot/dependencies.ts
CHANGED
|
@@ -1,646 +1,647 @@
|
|
|
1
|
-
|
|
2
|
-
import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
|
|
3
|
-
import { poolMaxFromEnv } from "../persistence/db-pg";
|
|
4
|
-
import type { CartethyiaDatabase } from "../persistence/db-handle";
|
|
5
|
-
import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
|
|
6
|
-
import type { RedisClient } from "../persistence/redis";
|
|
7
|
-
import {
|
|
8
|
-
bundledModelCatalog,
|
|
9
|
-
liveProviderUpstreamHosts,
|
|
10
|
-
markCatalogSeeded,
|
|
11
|
-
refreshActiveProviderCatalogs,
|
|
12
|
-
registerByokProviders,
|
|
13
|
-
retireUnbundledProviders,
|
|
14
|
-
seedBundledProviders,
|
|
15
|
-
} from "../providers/operations/provider-catalog-service";
|
|
16
|
-
import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
|
|
17
|
-
import { createDefaultProviderRegistry } from "../providers/default-registry";
|
|
18
|
-
import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
|
|
19
|
-
import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
|
|
20
|
-
import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
|
|
21
|
-
import { accountIdentitySweep } from "../workers/account-identity-sweep";
|
|
22
|
-
import { pushStructuredConsoleLog } from "../observability/log-ring";
|
|
23
|
-
import { seedBundledModels } from "../providers/operations/provider-catalog-seeder";
|
|
24
|
-
import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
|
|
25
|
-
import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
|
|
26
|
-
import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
|
|
27
|
-
import { RoutingEngine } from "../transport/routing/router";
|
|
28
|
-
import { RedisAdmissionController } from "../transport/routing/admission";
|
|
29
|
-
import { ApiKeyAdmissionService } from "../security/admission/service";
|
|
30
|
-
import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
|
|
31
|
-
import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
|
|
32
|
-
import { sweepLeases } from "../security/admission/lease-sweep";
|
|
33
|
-
import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
|
|
34
|
-
import { UsageFlushBuffer } from "../persistence/usage-flush";
|
|
35
|
-
import { setPendingUsageSource } from "../security/api-key-auth";
|
|
36
|
-
import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
|
|
37
|
-
import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
|
|
38
|
-
import { deleteExpiredSessions } from "../console/auth/session-service";
|
|
39
|
-
import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
|
|
40
|
-
import {
|
|
41
|
-
InMemoryModelAbuseStore,
|
|
42
|
-
ModelStrikeService,
|
|
43
|
-
RedisModelAbuseStore,
|
|
44
|
-
} from "../security/model-abuse";
|
|
45
|
-
import { checkReadiness } from "../persistence/readiness";
|
|
46
|
-
import { ProxyRequestPreparer } from "../transport/request/preparer";
|
|
47
|
-
import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
|
|
48
|
-
import {
|
|
49
|
-
PoolAgentResolver,
|
|
50
|
-
ValidatedNetworkBindingFactory,
|
|
51
|
-
} from "../network/pool/resolver";
|
|
52
|
-
import { NetworkPoolSelector } from "../network/pool/selector";
|
|
53
|
-
import { ScheduledTaskRegistry } from "../workers/scheduler";
|
|
54
|
-
import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
|
|
55
|
-
import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
|
|
56
|
-
import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
|
|
57
|
-
import {
|
|
58
|
-
buddyAdoptTask,
|
|
59
|
-
buddyActivityReportTask,
|
|
60
|
-
buddyLotteryTask,
|
|
61
|
-
buddyRedeemTask,
|
|
62
|
-
buddyStreakTask,
|
|
63
|
-
buddyTravelTask,
|
|
64
|
-
buddyTrialTask,
|
|
65
|
-
syncBuddyUsage,
|
|
66
|
-
} from "../workers/buddy-growth-tasks";
|
|
67
|
-
import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
|
|
68
|
-
import { quotaCacheSize } from "../console/quota/cache";
|
|
69
|
-
import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
|
|
70
|
-
import { preferencesReaderFor } from "../persistence/tenant-preferences";
|
|
71
|
-
import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
|
|
72
|
-
import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
|
|
73
|
-
import { TelemetryPayloadCapture } from "../observability/payload-capture";
|
|
74
|
-
import { foldDayIndex, listEventDays } from "../observability/event-day-index";
|
|
75
|
-
import { resolveTelemetryCompactIntervalMs } from "../config/telemetry";
|
|
76
|
-
import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
|
|
77
|
-
import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
|
|
78
|
-
import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
|
|
79
|
-
import {
|
|
80
|
-
resolveIpRateLimit,
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
import
|
|
87
|
-
import {
|
|
88
|
-
import {
|
|
89
|
-
import
|
|
90
|
-
import type {
|
|
91
|
-
import type {
|
|
92
|
-
import type {
|
|
93
|
-
import {
|
|
94
|
-
import {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
*
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
//
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
const
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
//
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
//
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
//
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
redis,
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
//
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
.
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
//
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
//
|
|
253
|
-
//
|
|
254
|
-
//
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
//
|
|
271
|
-
//
|
|
272
|
-
//
|
|
273
|
-
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
//
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
//
|
|
315
|
-
//
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
//
|
|
333
|
-
//
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
//
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
//
|
|
380
|
-
// that
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
//
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
)
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
//
|
|
400
|
-
//
|
|
401
|
-
//
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
//
|
|
416
|
-
//
|
|
417
|
-
//
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
//
|
|
454
|
-
//
|
|
455
|
-
//
|
|
456
|
-
//
|
|
457
|
-
//
|
|
458
|
-
//
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
//
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
//
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
//
|
|
482
|
-
//
|
|
483
|
-
//
|
|
484
|
-
//
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
//
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
//
|
|
491
|
-
//
|
|
492
|
-
//
|
|
493
|
-
//
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
//
|
|
497
|
-
// `
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
//
|
|
502
|
-
//
|
|
503
|
-
//
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
//
|
|
507
|
-
//
|
|
508
|
-
//
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
//
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
//
|
|
557
|
-
//
|
|
558
|
-
//
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
//
|
|
576
|
-
//
|
|
577
|
-
//
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
`
|
|
613
|
-
`
|
|
614
|
-
`
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
const
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
const
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
`
|
|
632
|
-
`
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
1
|
+
|
|
2
|
+
import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
|
|
3
|
+
import { poolMaxFromEnv } from "../persistence/db-pg";
|
|
4
|
+
import type { CartethyiaDatabase } from "../persistence/db-handle";
|
|
5
|
+
import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
|
|
6
|
+
import type { RedisClient } from "../persistence/redis";
|
|
7
|
+
import {
|
|
8
|
+
bundledModelCatalog,
|
|
9
|
+
liveProviderUpstreamHosts,
|
|
10
|
+
markCatalogSeeded,
|
|
11
|
+
refreshActiveProviderCatalogs,
|
|
12
|
+
registerByokProviders,
|
|
13
|
+
retireUnbundledProviders,
|
|
14
|
+
seedBundledProviders,
|
|
15
|
+
} from "../providers/operations/provider-catalog-service";
|
|
16
|
+
import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
|
|
17
|
+
import { createDefaultProviderRegistry } from "../providers/default-registry";
|
|
18
|
+
import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
|
|
19
|
+
import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
|
|
20
|
+
import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
|
|
21
|
+
import { accountIdentitySweep } from "../workers/account-identity-sweep";
|
|
22
|
+
import { pushStructuredConsoleLog } from "../observability/log-ring";
|
|
23
|
+
import { seedBundledModels } from "../providers/operations/provider-catalog-seeder";
|
|
24
|
+
import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
|
|
25
|
+
import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
|
|
26
|
+
import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
|
|
27
|
+
import { RoutingEngine } from "../transport/routing/router";
|
|
28
|
+
import { RedisAdmissionController } from "../transport/routing/admission";
|
|
29
|
+
import { ApiKeyAdmissionService } from "../security/admission/service";
|
|
30
|
+
import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
|
|
31
|
+
import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
|
|
32
|
+
import { sweepLeases } from "../security/admission/lease-sweep";
|
|
33
|
+
import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
|
|
34
|
+
import { UsageFlushBuffer } from "../persistence/usage-flush";
|
|
35
|
+
import { setPendingUsageSource } from "../security/api-key-auth";
|
|
36
|
+
import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
|
|
37
|
+
import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
|
|
38
|
+
import { deleteExpiredSessions } from "../console/auth/session-service";
|
|
39
|
+
import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
|
|
40
|
+
import {
|
|
41
|
+
InMemoryModelAbuseStore,
|
|
42
|
+
ModelStrikeService,
|
|
43
|
+
RedisModelAbuseStore,
|
|
44
|
+
} from "../security/model-abuse";
|
|
45
|
+
import { checkReadiness } from "../persistence/readiness";
|
|
46
|
+
import { ProxyRequestPreparer } from "../transport/request/preparer";
|
|
47
|
+
import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
|
|
48
|
+
import {
|
|
49
|
+
PoolAgentResolver,
|
|
50
|
+
ValidatedNetworkBindingFactory,
|
|
51
|
+
} from "../network/pool/resolver";
|
|
52
|
+
import { NetworkPoolSelector } from "../network/pool/selector";
|
|
53
|
+
import { ScheduledTaskRegistry } from "../workers/scheduler";
|
|
54
|
+
import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
|
|
55
|
+
import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
|
|
56
|
+
import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
|
|
57
|
+
import {
|
|
58
|
+
buddyAdoptTask,
|
|
59
|
+
buddyActivityReportTask,
|
|
60
|
+
buddyLotteryTask,
|
|
61
|
+
buddyRedeemTask,
|
|
62
|
+
buddyStreakTask,
|
|
63
|
+
buddyTravelTask,
|
|
64
|
+
buddyTrialTask,
|
|
65
|
+
syncBuddyUsage,
|
|
66
|
+
} from "../workers/buddy-growth-tasks";
|
|
67
|
+
import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
|
|
68
|
+
import { quotaCacheSize } from "../console/quota/cache";
|
|
69
|
+
import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
|
|
70
|
+
import { preferencesReaderFor } from "../persistence/tenant-preferences";
|
|
71
|
+
import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
|
|
72
|
+
import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
|
|
73
|
+
import { TelemetryPayloadCapture } from "../observability/payload-capture";
|
|
74
|
+
import { foldDayIndex, listEventDays } from "../observability/event-day-index";
|
|
75
|
+
import { resolveTelemetryCompactIntervalMs } from "../config/telemetry";
|
|
76
|
+
import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
|
|
77
|
+
import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
|
|
78
|
+
import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
|
|
79
|
+
import {
|
|
80
|
+
resolveIpRateLimit,
|
|
81
|
+
resolveModelBanMaxTtlMs,
|
|
82
|
+
resolveModelBanTtlMs,
|
|
83
|
+
resolveModelStrikeThreshold,
|
|
84
|
+
resolveModelStrikeWindowMs,
|
|
85
|
+
} from "../config/security";
|
|
86
|
+
import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
|
|
87
|
+
import type { TrustedProxyBoundary } from "../config/network";
|
|
88
|
+
import { resolveTelemetryRetentionDays } from "../config/telemetry";
|
|
89
|
+
import { CONFIG_SPEC, readInt, readList } from "../config/spec";
|
|
90
|
+
import type { ProviderAdapter } from "../providers/provider-contracts";
|
|
91
|
+
import type { ProviderRegistry } from "../providers/provider-registry";
|
|
92
|
+
import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
|
|
93
|
+
import type { ReadinessCheckResult } from "../persistence/readiness";
|
|
94
|
+
import { log } from "../observability/logger";
|
|
95
|
+
import { refreshProviderClientVersions } from "../providers/operations/client-versions";
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
export interface ProductionDeps {
|
|
99
|
+
db: CartethyiaDatabase;
|
|
100
|
+
redis: RedisClient | undefined;
|
|
101
|
+
snapshotService: InMemoryRouteSnapshotService;
|
|
102
|
+
/** Live routing admission snapshot scoped to one provider/tenant for console reads. */
|
|
103
|
+
readRoutingAccountInflight: (
|
|
104
|
+
providerId: string,
|
|
105
|
+
tenantId: string | null,
|
|
106
|
+
) => Promise<readonly { accountId: string; inflight: number }[]>;
|
|
107
|
+
/** On-demand adapter resolution keeps heavy providers out of the boot path. */
|
|
108
|
+
resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
|
|
109
|
+
bundledModelCatalog: BundledProviderCatalog;
|
|
110
|
+
/** Live lookup of a provider's SSRF-validated upstream host. */
|
|
111
|
+
byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
|
|
112
|
+
proxyPreparer: ProxyRequestPreparer;
|
|
113
|
+
networkBindingFactory: ValidatedNetworkBindingFactory;
|
|
114
|
+
ipAbuseProtection: IpAbuseProtectionService;
|
|
115
|
+
readiness: () => Promise<ReadinessCheckResult>;
|
|
116
|
+
scheduledTasks: ScheduledTaskRegistry;
|
|
117
|
+
poolAgentResolver: PoolAgentResolver;
|
|
118
|
+
poolSelector: NetworkPoolSelector;
|
|
119
|
+
telemetryBuffer: TelemetryBatchBuffer;
|
|
120
|
+
usageFlush: UsageFlushBuffer;
|
|
121
|
+
healthEventFlush: HealthEventFlushBuffer;
|
|
122
|
+
trustedProxyBoundary: TrustedProxyBoundary;
|
|
123
|
+
providerRegistry: ProviderRegistry;
|
|
124
|
+
resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
|
|
125
|
+
oauthRefreshService: OAuthRefreshService;
|
|
126
|
+
admissionService: ApiKeyAdmissionService;
|
|
127
|
+
/** Graduated strikes for repeated invalid-model requests. */
|
|
128
|
+
modelStrikes: ModelStrikeService;
|
|
129
|
+
/** Memory sampler; in-flight source is bound once the request store exists. */
|
|
130
|
+
runtimeMetricsSampler: RuntimeMetricsSampler;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Pause between accounts in the OAuth refresh sweep. The token endpoints
|
|
135
|
+
* rate-limit a burst of refreshes even at low concurrency.
|
|
136
|
+
*/
|
|
137
|
+
const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Client identities are refreshed out of band so provider dispatch never waits
|
|
141
|
+
* on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
|
|
142
|
+
*/
|
|
143
|
+
// Matched to the resolver's own 30 min cache TTL: a shorter interval makes
|
|
144
|
+
// every second pass a pure cache read that cannot discover anything new.
|
|
145
|
+
const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
|
|
146
|
+
const CLIENT_VERSION_INITIAL_DELAY_MS = 10_000;
|
|
147
|
+
|
|
148
|
+
export async function buildProductionDeps(): Promise<ProductionDeps> {
|
|
149
|
+
const startupStartedAt = performance.now();
|
|
150
|
+
const startupMemoryBefore = process.memoryUsage();
|
|
151
|
+
const handle = await measureStartupPhase("database-connect", bootDatabase);
|
|
152
|
+
const db = handle.db;
|
|
153
|
+
// One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
|
|
154
|
+
const redis = resolveRedisClient();
|
|
155
|
+
const ssrfPolicy = resolveSsrfPolicy();
|
|
156
|
+
await measureStartupPhase("database-migrations", async () => {
|
|
157
|
+
await ensureMigrated();
|
|
158
|
+
// Pool sizing is an external-server concern; the embedded backend has no
|
|
159
|
+
// connection budget to fit into.
|
|
160
|
+
if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv());
|
|
161
|
+
});
|
|
162
|
+
await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
|
|
163
|
+
// Boot-only: retire global rows for providers the bundle has dropped, so a
|
|
164
|
+
// retired provider cannot linger as a card the console can no longer delete.
|
|
165
|
+
// Kept off the test harness path, which shares one database across suites and
|
|
166
|
+
// installs its own global fixtures.
|
|
167
|
+
await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
|
|
168
|
+
const registry = createDefaultProviderRegistry();
|
|
169
|
+
// Live view, not a snapshot: custom providers registered after boot (or
|
|
170
|
+
// edited from the console) must resolve their SSRF binding immediately.
|
|
171
|
+
const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
|
|
172
|
+
await measureStartupPhase("byok-registration", () =>
|
|
173
|
+
registerByokProviders(registry, db, ssrfPolicy),
|
|
174
|
+
);
|
|
175
|
+
const bundledCatalog = await measureStartupPhase("bundled-model-catalog", () =>
|
|
176
|
+
bundledModelCatalog(registry),
|
|
177
|
+
);
|
|
178
|
+
await measureStartupPhase("bundled-model-seed", () =>
|
|
179
|
+
seedBundledModels(db, bundledCatalog.modelsByProvider),
|
|
180
|
+
);
|
|
181
|
+
// The boot seed above is unconditional, so record it: the snapshot builder's
|
|
182
|
+
// first rebuild would otherwise re-seed an identical catalog.
|
|
183
|
+
markCatalogSeeded();
|
|
184
|
+
const snapshotService = new InMemoryRouteSnapshotService(
|
|
185
|
+
createDatabaseSnapshotBuilder(db, () => refreshActiveProviderCatalogs(db, registry)),
|
|
186
|
+
);
|
|
187
|
+
const routingEngine = new RoutingEngine(
|
|
188
|
+
redis ? new RedisAdmissionController(redis) : undefined,
|
|
189
|
+
redis,
|
|
190
|
+
);
|
|
191
|
+
const readRoutingAccountInflight = async (
|
|
192
|
+
providerId: string,
|
|
193
|
+
tenantId: string | null,
|
|
194
|
+
): Promise<readonly { accountId: string; inflight: number }[]> => {
|
|
195
|
+
if (providerId === "" || tenantId === null) return [];
|
|
196
|
+
const snapshot = await routingEngine.accountInflightSnapshot();
|
|
197
|
+
const accounts = await new DrizzleProviderCatalogStore(db, {
|
|
198
|
+
telemetryBuffer,
|
|
199
|
+
bundledModelCatalog: bundledCatalog,
|
|
200
|
+
providerRegistry: registry,
|
|
201
|
+
outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
|
|
202
|
+
snapshotInvalidator: snapshotService,
|
|
203
|
+
}).listAccounts(tenantId, providerId);
|
|
204
|
+
const owned = new Set(accounts.map((account) => account.id));
|
|
205
|
+
const totals = new Map<string, number>();
|
|
206
|
+
for (const [bucket, count] of snapshot) {
|
|
207
|
+
const prefix = `${providerId}:`;
|
|
208
|
+
if (!bucket.startsWith(prefix)) continue;
|
|
209
|
+
const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
|
|
210
|
+
if (!owned.has(accountId)) continue;
|
|
211
|
+
totals.set(accountId, (totals.get(accountId) ?? 0) + count);
|
|
212
|
+
}
|
|
213
|
+
return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
|
|
214
|
+
};
|
|
215
|
+
// Peer processes' settings mutations arrive here. Subscribed once for the
|
|
216
|
+
// process: the revision counter is process-global, and the handler only
|
|
217
|
+
// moves it forward.
|
|
218
|
+
subscribePeerSettingsRevisions();
|
|
219
|
+
const usageFlush = new UsageFlushBuffer(db);
|
|
220
|
+
// Auth reads lifetime usage to seed the admission counter. The persisted
|
|
221
|
+
// value lags by up to one flush interval, so hand it the in-flight deltas
|
|
222
|
+
// too — otherwise the seed is systematically under what has actually been
|
|
223
|
+
// spent, worst exactly when the key is busiest.
|
|
224
|
+
setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
|
|
225
|
+
// Health audit rows are append-only and read by nobody in dispatch, so they
|
|
226
|
+
// are buffered out of the request. The status write they accompany stays
|
|
227
|
+
// synchronous — routing reads it.
|
|
228
|
+
const healthEventFlush = new HealthEventFlushBuffer(db);
|
|
229
|
+
setHealthEventBuffer(healthEventFlush);
|
|
230
|
+
const admissionStore = redis
|
|
231
|
+
? new RedisAdmissionCounterStore(redis)
|
|
232
|
+
: new InMemoryAdmissionCounterStore();
|
|
233
|
+
const admissionService = new ApiKeyAdmissionService(
|
|
234
|
+
admissionStore,
|
|
235
|
+
undefined,
|
|
236
|
+
// The per-tenant concurrency cap is a `console_settings` preference, read
|
|
237
|
+
// on every admitted request. Reading it directly cost one Postgres round
|
|
238
|
+
// trip per request (multiplied by each failover candidate); the shared
|
|
239
|
+
// revision-keyed reader answers from memory and converges the moment an
|
|
240
|
+
// operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
|
|
241
|
+
// revision every cached key embeds.
|
|
242
|
+
(tenantId) =>
|
|
243
|
+
preferencesReaderFor(db)
|
|
244
|
+
.readPreferences(tenantId)
|
|
245
|
+
.then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
|
|
246
|
+
({ apiKeyId, delta }) => {
|
|
247
|
+
// Coalesced: one UPDATE per key per flush instead of one per request.
|
|
248
|
+
// The admission counter is the live total; this only persists it.
|
|
249
|
+
usageFlush.enqueue(apiKeyId, delta);
|
|
250
|
+
return Promise.resolve();
|
|
251
|
+
},
|
|
252
|
+
// Fresh lifetime reader — invoked by the store only when it is about to
|
|
253
|
+
// seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
|
|
254
|
+
// snapshot value cannot freeze a low baseline in place. Family total: the
|
|
255
|
+
// row plus every child (share recipients admit under the parent id).
|
|
256
|
+
async (apiKeyId) => {
|
|
257
|
+
const store = new DrizzleApiKeyStore(db);
|
|
258
|
+
const row = await store.findActiveById(apiKeyId);
|
|
259
|
+
if (!row) return undefined;
|
|
260
|
+
const children = await store.sumChildrenConsumed(apiKeyId);
|
|
261
|
+
return (row.lifetimeTokensConsumed ?? 0) + children;
|
|
262
|
+
},
|
|
263
|
+
);
|
|
264
|
+
const ipStore = redis
|
|
265
|
+
? new RedisIpAbuseStore(redis)
|
|
266
|
+
: new InMemoryIpAbuseStore();
|
|
267
|
+
const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
|
|
268
|
+
maxRequestsPerWindow: resolveIpRateLimit(),
|
|
269
|
+
});
|
|
270
|
+
// Graduated model-abuse strikes: a client that keeps requesting models
|
|
271
|
+
// outside its access is warned, then banned. Redis-backed when available so
|
|
272
|
+
// the ban survives a restart and is shared across instances; in-memory
|
|
273
|
+
// otherwise, where a restart clears it.
|
|
274
|
+
const modelAbuseStore = redis ? new RedisModelAbuseStore(redis) : new InMemoryModelAbuseStore();
|
|
275
|
+
const modelStrikes = new ModelStrikeService(modelAbuseStore, {
|
|
276
|
+
threshold: resolveModelStrikeThreshold(),
|
|
277
|
+
windowMs: resolveModelStrikeWindowMs(),
|
|
278
|
+
banTtlMs: resolveModelBanTtlMs(),
|
|
279
|
+
maxBanTtlMs: resolveModelBanMaxTtlMs(),
|
|
280
|
+
});
|
|
281
|
+
const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
|
|
282
|
+
// One list covers every periodic task: a small deployment can cut the
|
|
283
|
+
// heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
|
|
284
|
+
// without the codebase growing a flag per task.
|
|
285
|
+
const scheduledTasks = new ScheduledTaskRegistry(
|
|
286
|
+
readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
|
|
287
|
+
);
|
|
288
|
+
if (redis) {
|
|
289
|
+
scheduledTasks.register({
|
|
290
|
+
name: "lease-sweep",
|
|
291
|
+
intervalMs: 30_000,
|
|
292
|
+
run: () => sweepLeases(redis),
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
scheduledTasks.register({
|
|
296
|
+
name: "account-health-sweep",
|
|
297
|
+
intervalMs: 30_000,
|
|
298
|
+
run: async () => {
|
|
299
|
+
const recovered = await sweepExpiredCooldowns(db);
|
|
300
|
+
const poolRecovered = await sweepExpiredPoolCooldowns(db);
|
|
301
|
+
if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
const payloadCapture = new TelemetryPayloadCapture(db);
|
|
305
|
+
const telemetryStore = new DrizzleTelemetryStore(db);
|
|
306
|
+
scheduledTasks.register({
|
|
307
|
+
name: "telemetry-payload-cleanup",
|
|
308
|
+
intervalMs: 15 * 60_000,
|
|
309
|
+
run: () => payloadCapture.cleanupExpired(),
|
|
310
|
+
});
|
|
311
|
+
scheduledTasks.register({
|
|
312
|
+
name: "telemetry-index-fold",
|
|
313
|
+
intervalMs: resolveTelemetryCompactIntervalMs(),
|
|
314
|
+
// Brings each day's index up to date with the bytes appended since the last
|
|
315
|
+
// pass, so a usage read is a small JSON read rather than a parse of the
|
|
316
|
+
// whole period. Folding is incremental and idempotent: a crash only leaves
|
|
317
|
+
// the watermark behind, and the next pass re-reads the tail.
|
|
318
|
+
run: async () => {
|
|
319
|
+
const days = await listEventDays();
|
|
320
|
+
let folded = 0;
|
|
321
|
+
for (const day of days) {
|
|
322
|
+
const result = await foldDayIndex(day).catch(() => ({ folded: 0 }));
|
|
323
|
+
folded += result.folded;
|
|
324
|
+
}
|
|
325
|
+
if (folded > 0) {
|
|
326
|
+
log.info(`[telemetry] indexed ${folded} event(s) across ${days.length} day(s)`);
|
|
327
|
+
}
|
|
328
|
+
},
|
|
329
|
+
});
|
|
330
|
+
scheduledTasks.register({
|
|
331
|
+
name: "console-session-sweep",
|
|
332
|
+
// Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
|
|
333
|
+
// 48 — each pass a full scan of the session table for rows that cannot yet
|
|
334
|
+
// be expired. Half the session lifetime is the point where a pass can
|
|
335
|
+
// actually find work, and still leaves a full grace period after it.
|
|
336
|
+
intervalMs: 12 * 60 * 60_000,
|
|
337
|
+
run: () => deleteExpiredSessions(db),
|
|
338
|
+
});
|
|
339
|
+
scheduledTasks.register({
|
|
340
|
+
name: "telemetry-retention",
|
|
341
|
+
intervalMs: 6 * 60 * 60_000,
|
|
342
|
+
run: () =>
|
|
343
|
+
telemetryStore.pruneTelemetry(
|
|
344
|
+
new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
|
|
345
|
+
),
|
|
346
|
+
});
|
|
347
|
+
const telemetryBuffer = new TelemetryBatchBuffer(db);
|
|
348
|
+
const poolSelector = new NetworkPoolSelector(redis);
|
|
349
|
+
const poolAgentResolver = new PoolAgentResolver(
|
|
350
|
+
new DrizzleNetworkPoolLoader(db),
|
|
351
|
+
{ ssrfPolicy, db },
|
|
352
|
+
);
|
|
353
|
+
const networkBindingFactory = new ValidatedNetworkBindingFactory(
|
|
354
|
+
ssrfPolicy,
|
|
355
|
+
undefined,
|
|
356
|
+
poolAgentResolver,
|
|
357
|
+
);
|
|
358
|
+
// Runtime gauges (memory + bounded-collection sizes) sampled on the shared
|
|
359
|
+
// maintenance cadence. The sampler reads live sizes through closures so the
|
|
360
|
+
// observability layer never holds direct references to routing/pool state.
|
|
361
|
+
const runtimeMetricsSampler = new RuntimeMetricsSampler({
|
|
362
|
+
routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
|
|
363
|
+
ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
|
|
364
|
+
quotaCacheEntries: () => quotaCacheSize(),
|
|
365
|
+
modelCatalogEntries: () => modelCatalogCacheSize(),
|
|
366
|
+
// Late-bound via bindInFlight once the request state store exists.
|
|
367
|
+
inFlight: () => 0,
|
|
368
|
+
});
|
|
369
|
+
scheduledTasks.register({
|
|
370
|
+
name: "runtime-metrics",
|
|
371
|
+
intervalMs: readInt(
|
|
372
|
+
"CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
|
|
373
|
+
CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
|
|
374
|
+
),
|
|
375
|
+
run: () => runtimeMetricsSampler.sample(),
|
|
376
|
+
});
|
|
377
|
+
scheduledTasks.register({
|
|
378
|
+
name: "pool-agent-sweep",
|
|
379
|
+
// `resolveAgent` already self-sweeps every 30 s, so a 1 min timer adds a
|
|
380
|
+
// second pass that almost always finds nothing to release. 5 min is well
|
|
381
|
+
// inside the idle timeout this exists to enforce, and outside the path
|
|
382
|
+
// that already runs.
|
|
383
|
+
intervalMs: 5 * 60_000,
|
|
384
|
+
run: () => poolAgentResolver.sweepIdleAgents(),
|
|
385
|
+
});
|
|
386
|
+
const oauthRefreshService = new OAuthRefreshService(db);
|
|
387
|
+
// Converge accounts parked on a dead refresh grant: an account whose stored
|
|
388
|
+
// access token is still usable becomes a static token (and returns to
|
|
389
|
+
// rotation) instead of staying disabled for re-auth.
|
|
390
|
+
await measureStartupPhase("static-token-reconciliation", () =>
|
|
391
|
+
reconcileStaticTokenAccounts(db),
|
|
392
|
+
);
|
|
393
|
+
// Fills in the identity a batch import deliberately skipped, so a paste of
|
|
394
|
+
// many credentials is one instant write and the email/label arrive shortly
|
|
395
|
+
// after rather than blocking the import. Purely CPU on an already-stored
|
|
396
|
+
// credential — no upstream call — so it runs often and cheaply.
|
|
397
|
+
scheduledTasks.register({
|
|
398
|
+
name: "account-enrichment-sweep",
|
|
399
|
+
// Converges via `identity_checked_at`: once every stored credential has
|
|
400
|
+
// been parsed, the listing query returns nothing — a 30 s timer would run
|
|
401
|
+
// that empty query forever, two million times a year. 10 min is still
|
|
402
|
+
// instant for the case that matters (a batch import, where the label and
|
|
403
|
+
// email show up on the next tick) and costs nothing when idle.
|
|
404
|
+
intervalMs: 10 * 60_000,
|
|
405
|
+
run: () => accountIdentitySweep({ db }),
|
|
406
|
+
});
|
|
407
|
+
scheduledTasks.register({
|
|
408
|
+
name: "oauth-refresh-sweep",
|
|
409
|
+
intervalMs: 60_000,
|
|
410
|
+
run: () =>
|
|
411
|
+
oauthRefreshSweep({
|
|
412
|
+
loadDueAccounts: () => loadDueOAuthAccounts(db),
|
|
413
|
+
refreshService: oauthRefreshService,
|
|
414
|
+
resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
415
|
+
// Sequential with a pause between accounts: the OAuth token endpoints
|
|
416
|
+
// rate-limit a burst of refreshes, and a pass over a handful of
|
|
417
|
+
// accounts must not look like one. Per-account outcomes are already
|
|
418
|
+
// pushed to the Console Log ring by `OAuthRefreshService`; this only
|
|
419
|
+
// covers an error that escaped before reaching it.
|
|
420
|
+
interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
|
|
421
|
+
onAccountError: (accountId, providerId, error) => {
|
|
422
|
+
log.error(
|
|
423
|
+
`[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
|
|
424
|
+
error as Error,
|
|
425
|
+
);
|
|
426
|
+
pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
|
|
427
|
+
event: "token_refresh",
|
|
428
|
+
accountId,
|
|
429
|
+
providerId,
|
|
430
|
+
errorCode: "refresh_sweep_failed",
|
|
431
|
+
});
|
|
432
|
+
},
|
|
433
|
+
}),
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
// Client versions are monitor-only metadata; let the listener settle before
|
|
437
|
+
// starting their external fetches. The scheduled task keeps the cache warm.
|
|
438
|
+
scheduledTasks.register({
|
|
439
|
+
name: "client-version-monitor",
|
|
440
|
+
intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
|
|
441
|
+
initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
|
|
442
|
+
run: refreshProviderClientVersions,
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
const quotaResolveCredential = createAccountSecretResolver({
|
|
446
|
+
db,
|
|
447
|
+
resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
448
|
+
refreshService: oauthRefreshService,
|
|
449
|
+
});
|
|
450
|
+
// Keeps every account's cached quota warm so opening the Quota page is a
|
|
451
|
+
// cache read. Without this the only refreshes are user-triggered, so the
|
|
452
|
+
// first open after an expiry always blocks on the provider.
|
|
453
|
+
//
|
|
454
|
+
// It also carries the daily check-in ride-along: each account refreshed
|
|
455
|
+
// here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
|
|
456
|
+
// the same pass (one attempt per account per day, guarded by a day
|
|
457
|
+
// marker), so no second timer duplicates the credential resolution.
|
|
458
|
+
//
|
|
459
|
+
// Registered in **every** mode, not only when Redis is present: gating it on
|
|
460
|
+
// Redis would cost Lite proactive quota warming and the check-in ride-along
|
|
461
|
+
// above that claims the once-per-day WorkBuddy/CodeBuddy credit grant. The
|
|
462
|
+
// whole sweep chain accepts `RedisClient | undefined` and falls back to the
|
|
463
|
+
// in-process TTL cache and an in-process day marker, so Redis is an
|
|
464
|
+
// optimization here rather than a precondition — which is exactly what
|
|
465
|
+
// Lite is for.
|
|
466
|
+
scheduledTasks.register({
|
|
467
|
+
name: "quota-refresh-sweep",
|
|
468
|
+
intervalMs: 60_000,
|
|
469
|
+
// Staggered off the oauth/lease/health ticks so the three sweeps that
|
|
470
|
+
// strike providerAccounts do not land together every minute.
|
|
471
|
+
initialDelayMs: 20_000,
|
|
472
|
+
run: () =>
|
|
473
|
+
quotaRefreshSweep({
|
|
474
|
+
db,
|
|
475
|
+
redis,
|
|
476
|
+
providerRegistry: registry,
|
|
477
|
+
resolveCredential: quotaResolveCredential,
|
|
478
|
+
snapshotInvalidator: snapshotService,
|
|
479
|
+
// Without this the sweep hands each collector the raw stored secret.
|
|
480
|
+
// For `cline` that means an `sk_...` key reaching the OAuth branch,
|
|
481
|
+
// which prepends `workos:` and is refused with "please make sure
|
|
482
|
+
// you're using the latest version of Cline and re-authenticate" — so
|
|
483
|
+
// every sweep cools a working account with `quota_fetch_failed` while
|
|
484
|
+
// the same account refreshes fine from the console, which binds the
|
|
485
|
+
// same marker through `createQuotaRefreshDeps`.
|
|
486
|
+
markApiKeyCredential: markProviderApiKeyCredential,
|
|
487
|
+
// Below the 60 s interval: a budget longer than the tick guarantees
|
|
488
|
+
// skip/burst under load instead of steady progress.
|
|
489
|
+
passBudgetMs: 45_000,
|
|
490
|
+
// The check-in ride-along rotates egress per account: each account
|
|
491
|
+
// gets the next active pool in its tenant's rotation so check-ins
|
|
492
|
+
// spread across IPs instead of sharing one direct egress. No pool
|
|
493
|
+
// (or an unreadable pool table) falls back to direct egress, and a
|
|
494
|
+
// tenant never borrows another tenant's pool.
|
|
495
|
+
checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
|
|
496
|
+
// Per-account lines and the pass summary are emitted by the sweep
|
|
497
|
+
// itself at `info` level, so they survive the production LOG_LEVEL.
|
|
498
|
+
// `onTick` is left for tests.
|
|
499
|
+
}),
|
|
500
|
+
});
|
|
501
|
+
// The buddy growth/credit tasks: one per upstream capability, each
|
|
502
|
+
// independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
|
|
503
|
+
// them is a credit gain the account is entitled to, and none of them can mark
|
|
504
|
+
// an account unhealthy — a growth endpoint that 404s is a display miss, not a
|
|
505
|
+
// dispatch failure.
|
|
506
|
+
//
|
|
507
|
+
// Daily, not hourly: each claim is once per account per day and is
|
|
508
|
+
// ledger-guarded, so at an hourly cadence 23 of every 24 passes are a full
|
|
509
|
+
// scan of active accounts plus a ledger read per account that resolves to
|
|
510
|
+
// nothing. Once a day is the shortest interval where a pass can still find
|
|
511
|
+
// work, which is the only reason to run at all.
|
|
512
|
+
const buddyTaskDeps = {
|
|
513
|
+
db,
|
|
514
|
+
resolveCredential: quotaResolveCredential,
|
|
515
|
+
fetcher: fetch,
|
|
516
|
+
// Spread across accounts so a pass does not look like a burst to the
|
|
517
|
+
// upstream's risk gate.
|
|
518
|
+
interAccountDelayMs: 800,
|
|
519
|
+
};
|
|
520
|
+
scheduledTasks.register({
|
|
521
|
+
name: "buddy-streak",
|
|
522
|
+
intervalMs: 24 * 60 * 60_000,
|
|
523
|
+
initialDelayMs: 5 * 60_000,
|
|
524
|
+
run: () => buddyStreakTask(buddyTaskDeps),
|
|
525
|
+
});
|
|
526
|
+
scheduledTasks.register({
|
|
527
|
+
name: "buddy-redeem",
|
|
528
|
+
intervalMs: 24 * 60 * 60_000,
|
|
529
|
+
initialDelayMs: 8 * 60_000,
|
|
530
|
+
run: () => buddyRedeemTask(buddyTaskDeps),
|
|
531
|
+
});
|
|
532
|
+
scheduledTasks.register({
|
|
533
|
+
name: "buddy-lottery",
|
|
534
|
+
intervalMs: 24 * 60 * 60_000,
|
|
535
|
+
initialDelayMs: 11 * 60_000,
|
|
536
|
+
run: () => buddyLotteryTask(buddyTaskDeps),
|
|
537
|
+
});
|
|
538
|
+
scheduledTasks.register({
|
|
539
|
+
name: "buddy-travel",
|
|
540
|
+
intervalMs: 24 * 60 * 60_000,
|
|
541
|
+
initialDelayMs: 14 * 60_000,
|
|
542
|
+
run: () => buddyTravelTask(buddyTaskDeps),
|
|
543
|
+
});
|
|
544
|
+
scheduledTasks.register({
|
|
545
|
+
name: "buddy-adopt",
|
|
546
|
+
intervalMs: 24 * 60 * 60_000,
|
|
547
|
+
initialDelayMs: 17 * 60_000,
|
|
548
|
+
run: () => buddyAdoptTask(buddyTaskDeps),
|
|
549
|
+
});
|
|
550
|
+
scheduledTasks.register({
|
|
551
|
+
name: "buddy-trial",
|
|
552
|
+
intervalMs: 24 * 60 * 60_000,
|
|
553
|
+
initialDelayMs: 20 * 60_000,
|
|
554
|
+
run: () => buddyTrialTask(buddyTaskDeps),
|
|
555
|
+
});
|
|
556
|
+
// Growth-activity report: the `/v2/report` batch that lights the login streak
|
|
557
|
+
// and scores the daily growth event. Once a day with its own ledger — it used
|
|
558
|
+
// to ride the quota sweep with no marker of its own, so its eight POSTs ran
|
|
559
|
+
// every minute.
|
|
560
|
+
scheduledTasks.register({
|
|
561
|
+
name: "buddy-activity-report",
|
|
562
|
+
intervalMs: 24 * 60 * 60_000,
|
|
563
|
+
initialDelayMs: 15 * 60 * 60_000,
|
|
564
|
+
run: () => buddyActivityReportTask(buddyTaskDeps),
|
|
565
|
+
});
|
|
566
|
+
// Real credit deductions, stored so the console renders a figure the upstream
|
|
567
|
+
// actually reported. Six-hourly: the upstream's own window is 30 days, so a
|
|
568
|
+
// day's number does not change intraday.
|
|
569
|
+
scheduledTasks.register({
|
|
570
|
+
name: "buddy-usage-sync",
|
|
571
|
+
intervalMs: 6 * 60 * 60_000,
|
|
572
|
+
initialDelayMs: 23 * 60_000,
|
|
573
|
+
run: () => syncBuddyUsage(buddyTaskDeps),
|
|
574
|
+
});
|
|
575
|
+
// The scheduled tasks are registered here but started by the caller once the
|
|
576
|
+
// listener is actually serving (`main.ts`). Starting them inside this builder
|
|
577
|
+
// would let the first tick — the lease sweep, the health sweep — run against
|
|
578
|
+
// a process that has not begun accepting traffic yet.
|
|
579
|
+
const deps: ProductionDeps = {
|
|
580
|
+
db,
|
|
581
|
+
redis,
|
|
582
|
+
snapshotService,
|
|
583
|
+
readRoutingAccountInflight,
|
|
584
|
+
resolveProviderAdapter: (providerId) => registry.resolve(providerId),
|
|
585
|
+
bundledModelCatalog: bundledCatalog,
|
|
586
|
+
byokUpstreamHosts,
|
|
587
|
+
proxyPreparer: new ProxyRequestPreparer({
|
|
588
|
+
snapshotService,
|
|
589
|
+
routingEngine,
|
|
590
|
+
admissionService,
|
|
591
|
+
}),
|
|
592
|
+
networkBindingFactory,
|
|
593
|
+
ipAbuseProtection,
|
|
594
|
+
modelStrikes,
|
|
595
|
+
readiness,
|
|
596
|
+
scheduledTasks,
|
|
597
|
+
poolAgentResolver,
|
|
598
|
+
telemetryBuffer,
|
|
599
|
+
usageFlush,
|
|
600
|
+
healthEventFlush,
|
|
601
|
+
poolSelector,
|
|
602
|
+
admissionService,
|
|
603
|
+
trustedProxyBoundary: resolveTrustedProxyBoundary(),
|
|
604
|
+
providerRegistry: registry,
|
|
605
|
+
resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
|
|
606
|
+
oauthRefreshService,
|
|
607
|
+
runtimeMetricsSampler,
|
|
608
|
+
};
|
|
609
|
+
const startupMemoryAfter = process.memoryUsage();
|
|
610
|
+
log.info(
|
|
611
|
+
`[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
|
|
612
|
+
`rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
|
|
613
|
+
`rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
|
|
614
|
+
`heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
|
|
615
|
+
`heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
|
|
616
|
+
);
|
|
617
|
+
return deps;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
|
|
621
|
+
const startedAt = performance.now();
|
|
622
|
+
const before = process.memoryUsage();
|
|
623
|
+
log.info(
|
|
624
|
+
`[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
|
|
625
|
+
);
|
|
626
|
+
try {
|
|
627
|
+
const result = await operation();
|
|
628
|
+
const after = process.memoryUsage();
|
|
629
|
+
log.info(
|
|
630
|
+
`[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
|
|
631
|
+
`rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
|
|
632
|
+
`heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
|
|
633
|
+
`external_mb=${toMiB(after.external)}`,
|
|
634
|
+
);
|
|
635
|
+
return result;
|
|
636
|
+
} catch (error) {
|
|
637
|
+
log.error(
|
|
638
|
+
`[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
|
|
639
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
640
|
+
);
|
|
641
|
+
throw error;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function toMiB(bytes: number): string {
|
|
646
|
+
return (bytes / 1024 / 1024).toFixed(1);
|
|
647
|
+
}
|