cartethyia 2.5.10 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -0
- package/dist/dashboard/assets/{BannedPage-Cd30Z0b9.js → BannedPage-l6XMnfr9.js} +1 -1
- package/dist/dashboard/assets/BannedPage-l6XMnfr9.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolDetailPage-wf5u-ww6.js → CliToolDetailPage-DwOzAW45.js} +1 -1
- package/dist/dashboard/assets/CliToolDetailPage-DwOzAW45.js.gz +0 -0
- package/dist/dashboard/assets/{CliToolsPage-DykPvuck.js → CliToolsPage-BT3Up3QA.js} +1 -1
- package/dist/dashboard/assets/CliToolsPage-BT3Up3QA.js.gz +0 -0
- package/dist/dashboard/assets/{CombosPage-DwvSD3vs.js → CombosPage-6kYIUpzR.js} +1 -1
- package/dist/dashboard/assets/CombosPage-6kYIUpzR.js.gz +0 -0
- package/dist/dashboard/assets/{ConsoleLogPage-CIr7KDex.js → ConsoleLogPage-zRnj1mlR.js} +1 -1
- package/dist/dashboard/assets/ConsoleLogPage-zRnj1mlR.js.gz +0 -0
- package/dist/dashboard/assets/{CustomizationPage-82X0DDdH.js → CustomizationPage-C7m-H1Rz.js} +1 -1
- package/dist/dashboard/assets/CustomizationPage-C7m-H1Rz.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js +1 -0
- package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js.gz +0 -0
- package/dist/dashboard/assets/{LoginPage-CS5mPuVF.js → LoginPage-CbphLvjg.js} +1 -1
- package/dist/dashboard/assets/LoginPage-CbphLvjg.js.gz +0 -0
- package/dist/dashboard/assets/{ModelPicker-C0UCUsVJ.js → ModelPicker-DVny4S5_.js} +1 -1
- package/dist/dashboard/assets/ModelPicker-DVny4S5_.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js +1 -0
- package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js +3 -0
- package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js.gz +0 -0
- package/dist/dashboard/assets/{ProvidersPage-vdja5ZPK.js → ProvidersPage-Dfb2896L.js} +1 -1
- package/dist/dashboard/assets/ProvidersPage-Dfb2896L.js.gz +0 -0
- package/dist/dashboard/assets/{ProxyPage-BB4CNm4-.js → ProxyPage-BaCIdAUt.js} +1 -1
- package/dist/dashboard/assets/ProxyPage-BaCIdAUt.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js +1 -0
- package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js +1 -0
- package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js.gz +0 -0
- package/dist/dashboard/assets/{SetupPage-FadUOl9S.js → SetupPage-D4g6vOGc.js} +1 -1
- package/dist/dashboard/assets/SetupPage-D4g6vOGc.js.gz +0 -0
- package/dist/dashboard/assets/{SortableList-Cfy6_Khv.js → SortableList-C93i14BD.js} +1 -1
- package/dist/dashboard/assets/SortableList-C93i14BD.js.gz +0 -0
- package/dist/dashboard/assets/{StudioPage-CM92fgJf.js → StudioPage-SgQpymjm.js} +1 -1
- package/dist/dashboard/assets/{StudioPage-CM92fgJf.js.gz → StudioPage-SgQpymjm.js.gz} +0 -0
- package/dist/dashboard/assets/{UsagePage-BT4MyQeT.js → UsagePage-BFME-eba.js} +2 -2
- package/dist/dashboard/assets/UsagePage-BFME-eba.js.gz +0 -0
- package/dist/dashboard/assets/{api-keys-DQYLQxpW.js → api-keys-C23S9sgp.js} +1 -1
- package/dist/dashboard/assets/api-keys-C23S9sgp.js.gz +0 -0
- package/dist/dashboard/assets/{app-B7pVRhJ1.js → app-2KhtHCv9.js} +1 -1
- package/dist/dashboard/assets/{app-B7pVRhJ1.js.gz → app-2KhtHCv9.js.gz} +0 -0
- package/dist/dashboard/assets/app-CEeaMHOy.css +2 -0
- package/dist/dashboard/assets/app-CEeaMHOy.css.gz +0 -0
- package/dist/dashboard/assets/{assert-C9FD_JwJ.js → assert-DU6qrMA_.js} +1 -1
- package/dist/dashboard/assets/assert-DU6qrMA_.js.gz +0 -0
- package/dist/dashboard/assets/{badge-Bvyr7D0u.js → badge-CKsF123d.js} +1 -1
- package/dist/dashboard/assets/badge-CKsF123d.js.gz +0 -0
- package/dist/dashboard/assets/{banner-DIvKAZih.js → banner-8j7uChA9.js} +1 -1
- package/dist/dashboard/assets/banner-8j7uChA9.js.gz +0 -0
- package/dist/dashboard/assets/{card-D5_gYIS_.js → card-CdsSlWA3.js} +1 -1
- package/dist/dashboard/assets/card-CdsSlWA3.js.gz +0 -0
- package/dist/dashboard/assets/{cli-tools-CNqzrfdb.js → cli-tools-BaFUuFFF.js} +1 -1
- package/dist/dashboard/assets/cli-tools-BaFUuFFF.js.gz +0 -0
- package/dist/dashboard/assets/{clipboard-button-B6vDBtbm.js → clipboard-button-uL5aYLfW.js} +1 -1
- package/dist/dashboard/assets/clipboard-button-uL5aYLfW.js.gz +0 -0
- package/dist/dashboard/assets/{dialog-CpjrgdwJ.js → dialog-C07hr5_G.js} +1 -1
- package/dist/dashboard/assets/dialog-C07hr5_G.js.gz +0 -0
- package/dist/dashboard/assets/{drawer-BWmt3b5y.js → drawer-BPO1TcKH.js} +1 -1
- package/dist/dashboard/assets/drawer-BPO1TcKH.js.gz +0 -0
- package/dist/dashboard/assets/{entry-C_oakV3z.js → entry-Bz_CtDfC.js} +1 -1
- package/dist/dashboard/assets/entry-Bz_CtDfC.js.gz +0 -0
- package/dist/dashboard/assets/entry-CwPRKk0d.js +4 -0
- package/dist/dashboard/assets/entry-CwPRKk0d.js.gz +0 -0
- package/dist/dashboard/assets/entry-GJRbmrpA.js +1 -0
- package/dist/dashboard/assets/entry-GJRbmrpA.js.gz +0 -0
- package/dist/dashboard/assets/{github-badge-RRHHWGxh.js → github-badge-DmQreSDd.js} +1 -1
- package/dist/dashboard/assets/github-badge-DmQreSDd.js.gz +0 -0
- package/dist/dashboard/assets/{index-Bhey_rg_.js → index-Bblu6erR.js} +2 -2
- package/dist/dashboard/assets/index-Bblu6erR.js.gz +0 -0
- package/dist/dashboard/assets/{inline-C2_EQ5Uf.js → inline-C9dCYaBZ.js} +1 -1
- package/dist/dashboard/assets/inline-C9dCYaBZ.js.gz +0 -0
- package/dist/dashboard/assets/{input-BMRlQ450.js → input-CjCnXweJ.js} +1 -1
- package/dist/dashboard/assets/input-CjCnXweJ.js.gz +0 -0
- package/dist/dashboard/assets/{layout-XR5xP8ew.js → layout-V6y8DAkl.js} +1 -1
- package/dist/dashboard/assets/layout-V6y8DAkl.js.gz +0 -0
- package/dist/dashboard/assets/{list-row-Cw07dioc.js → list-row-CM1AaG_y.js} +1 -1
- package/dist/dashboard/assets/list-row-CM1AaG_y.js.gz +0 -0
- package/dist/dashboard/assets/{live-BivGfuUs.js → live-ta6GMb-8.js} +1 -1
- package/dist/dashboard/assets/live-ta6GMb-8.js.gz +0 -0
- package/dist/dashboard/assets/{model-bans-CmffjLuG.js → model-bans-BmYTmbAk.js} +1 -1
- package/dist/dashboard/assets/model-bans-BmYTmbAk.js.gz +0 -0
- package/dist/dashboard/assets/{mutation-46P0oWMe.js → mutation-g_fduiNw.js} +1 -1
- package/dist/dashboard/assets/mutation-g_fduiNw.js.gz +0 -0
- package/dist/dashboard/assets/{numbers-DrdA9oSM.js → numbers-BOevJzdz.js} +1 -1
- package/dist/dashboard/assets/numbers-BOevJzdz.js.gz +0 -0
- package/dist/dashboard/assets/{query-keys-BFhGpwR1.js → query-keys-UETqfytp.js} +1 -1
- package/dist/dashboard/assets/query-keys-UETqfytp.js.gz +0 -0
- package/dist/dashboard/assets/{routing-CAb4F37R.js → routing-BPUo-W8d.js} +1 -1
- package/dist/dashboard/assets/routing-BPUo-W8d.js.gz +0 -0
- package/dist/dashboard/assets/{select-CxVVB__N.js → select-C980c4cz.js} +1 -1
- package/dist/dashboard/assets/select-C980c4cz.js.gz +0 -0
- package/dist/dashboard/assets/{settings-DNX9sydi.js → settings--3hxSyry.js} +1 -1
- package/dist/dashboard/assets/settings--3hxSyry.js.gz +0 -0
- package/dist/dashboard/assets/{stack-uPOwM9Hs.js → stack-_gfTdLVB.js} +1 -1
- package/dist/dashboard/assets/stack-_gfTdLVB.js.gz +0 -0
- package/dist/dashboard/assets/{state-D1Yz79iL.js → state-2c5V4zo7.js} +1 -1
- package/dist/dashboard/assets/state-2c5V4zo7.js.gz +0 -0
- package/dist/dashboard/assets/{switch-sevsRrRo.js → switch-Dkke7Naf.js} +1 -1
- package/dist/dashboard/assets/switch-Dkke7Naf.js.gz +0 -0
- package/dist/dashboard/assets/{useMutation-Dem0lXQj.js → useMutation-EDFRkn1i.js} +1 -1
- package/dist/dashboard/assets/useMutation-EDFRkn1i.js.gz +0 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/index.html.gz +0 -0
- package/dist/dashboard/precompressed-assets.json +51 -51
- package/migrations/README.md +52 -52
- package/migrations/full-postgres/0020_add_telemetry_event_analytics.sql +77 -0
- package/migrations/full-postgres/0021_add_api_key_concurrency_tolerance.sql +13 -0
- package/migrations/full-postgres/0022_add_inferhub_config_key.sql +17 -0
- package/migrations/full-postgres/0023_drop_api_key_concurrency_tolerance.sql +13 -0
- package/migrations/lite-sqlite3/0020_add_telemetry_event_analytics.sql +69 -0
- package/migrations/lite-sqlite3/0021_add_api_key_concurrency_tolerance.sql +13 -0
- package/migrations/lite-sqlite3/0022_add_inferhub_config_key.sql +17 -0
- package/migrations/lite-sqlite3/0023_drop_api_key_concurrency_tolerance.sql +13 -0
- package/package.json +96 -95
- package/scripts/cli/admin.ts +126 -128
- package/scripts/cli/autostart.ts +237 -237
- package/scripts/cli/background.ts +120 -110
- package/scripts/cli/bin/carte.cjs +8 -8
- package/scripts/cli/bin/cartethyia.cjs +167 -167
- package/scripts/cli/console-menu.ts +39 -30
- package/scripts/cli/controller.ts +145 -0
- package/scripts/cli/env.ts +99 -99
- package/scripts/cli/gateway.ts +264 -260
- package/scripts/cli/instance.ts +234 -224
- package/scripts/cli/main.ts +188 -167
- package/scripts/cli/platform.ts +64 -64
- package/scripts/cli/prompt.ts +217 -79
- package/scripts/cli/services.ts +88 -88
- package/scripts/cli/setup.ts +445 -445
- package/scripts/cli/start.ts +85 -85
- package/scripts/cli/startup-signal.ts +23 -23
- package/scripts/cli/telemetry.ts +70 -0
- package/scripts/cli/tray/index.ts +263 -263
- package/scripts/cli/tray/windows.ps1 +107 -107
- package/scripts/cli/tray.ts +29 -4
- package/scripts/cli/update.ts +39 -39
- package/scripts/cli/version.ts +67 -67
- package/scripts/cli/wordmark.ts +58 -58
- package/src/boot/app.ts +16 -1
- package/src/boot/dependencies.ts +128 -25
- package/src/config/resources.ts +137 -0
- package/src/config/runtime.ts +133 -76
- package/src/config/spec.ts +30 -29
- package/src/config/telemetry.ts +0 -25
- package/src/console/api-keys/bucket-spend.ts +14 -17
- package/src/console/api-keys/contracts.ts +410 -410
- package/src/console/api-keys/routes.ts +12 -0
- package/src/console/audit/store.ts +2 -3
- package/src/console/auth/setup-service.ts +8 -1
- package/src/console/backup/routes.ts +7 -0
- package/src/console/backup/service.ts +463 -463
- package/src/console/backup/store.ts +16 -11
- package/src/console/backup/validate.ts +1 -4
- package/src/console/cli-tools/fs-ops.ts +18 -2
- package/src/console/data-management/contracts.ts +1 -8
- package/src/console/data-management/operations.ts +61 -48
- package/src/console/data-management/routes.ts +4 -2
- package/src/console/domain-registration.ts +25 -3
- package/src/console/inferhub/client.ts +139 -0
- package/src/console/inferhub/key-store.ts +129 -0
- package/src/console/inferhub/normalize.ts +347 -0
- package/src/console/inferhub/routes.ts +411 -0
- package/src/console/inferhub/store.ts +109 -0
- package/src/console/inferhub/types.ts +183 -0
- package/src/console/observability/contracts.ts +2 -41
- package/src/console/observability/store.ts +176 -371
- package/src/console/providers/catalog/public-model-store.ts +0 -0
- package/src/console/providers/catalog/store.ts +23 -36
- package/src/console/quota/account-quota-shared.ts +52 -52
- package/src/console/quota/account-quota-tenant.ts +16 -24
- package/src/console/quota/refresh.ts +441 -441
- package/src/console/routing/pools/import-parse.ts +206 -206
- package/src/console/routing/pools/operations.ts +543 -543
- package/src/console/routing/pools/probe-verdict.ts +132 -132
- package/src/console/routing/pools/routes.ts +181 -181
- package/src/console/settings/contracts.ts +42 -40
- package/src/console/settings/store.ts +25 -25
- package/src/console/share/share-stats.ts +22 -4
- package/src/console/share/share-usage.ts +30 -4
- package/src/console/shared/cursor.ts +15 -0
- package/src/console/updates/routes.ts +12 -0
- package/src/main.ts +13 -5
- package/src/network/pool/resolver.ts +1104 -1104
- package/src/observability/client-name.ts +22 -0
- package/src/observability/event-log.ts +5 -29
- package/src/observability/event-query.ts +690 -0
- package/src/observability/log-ring.ts +26 -2
- package/src/observability/log-sanitize.ts +118 -0
- package/src/observability/logger.ts +17 -13
- package/src/observability/metrics.ts +35 -0
- package/src/observability/payload-capture.ts +114 -41
- package/src/observability/payload-store.ts +249 -24
- package/src/observability/raw-event-writer.ts +118 -0
- package/src/observability/runtime-metrics.ts +71 -1
- package/src/observability/telemetry-buffer.ts +100 -17
- package/src/observability/telemetry-reconciliation.ts +417 -0
- package/src/persistence/boot.ts +20 -9
- package/src/persistence/db-pg.ts +13 -0
- package/src/persistence/db-sqlite.ts +569 -497
- package/src/persistence/dialect.ts +3 -2
- package/src/persistence/migrate.ts +181 -181
- package/src/persistence/schema.ts +1496 -1406
- package/src/persistence/shared-cache.ts +5 -1
- package/src/persistence/stores/api-key-store.ts +581 -581
- package/src/persistence/stores/telemetry-store.ts +124 -63
- package/src/protocol/reasoning-text.ts +30 -0
- package/src/protocol/response/chat.ts +33 -7
- package/src/protocol/response/codex.ts +14 -2
- package/src/protocol/response/messages.ts +5 -2
- package/src/providers/authentication/oauth-flow-store.ts +3 -10
- package/src/providers/bundled-discovery-providers.ts +122 -122
- package/src/providers/default-registry.ts +10 -0
- package/src/providers/discovery/catalog/catalog-blob.ts +18 -18
- package/src/providers/discovery/models-dev-catalog.ts +354 -354
- package/src/providers/discovery/probe-phases.ts +43 -7
- package/src/providers/discovery/probing-service.ts +70 -32
- package/src/providers/integrations/antigravity/antigravity.ts +9 -2
- package/src/providers/integrations/buddy/buddy-catalog-shared.ts +85 -17
- package/src/providers/integrations/buddy/buddy-chat-shared.ts +934 -934
- package/src/providers/integrations/buddy/buddy-checkin.ts +5 -9
- package/src/providers/integrations/buddy/buddy-discovery-shared.ts +3 -7
- package/src/providers/integrations/buddy/buddy-quota-shared.ts +3 -1
- package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +57 -49
- package/src/providers/integrations/buddy/codebuddy/provider.ts +180 -180
- package/src/providers/integrations/buddy/workbuddy/provider.ts +206 -206
- package/src/providers/integrations/buddy/workbuddy/quota.ts +161 -161
- package/src/providers/integrations/claude/claude-quota.ts +2 -2
- package/src/providers/integrations/claude/claude-user-profiles.ts +3 -2
- package/src/providers/integrations/claude/claude.ts +3 -4
- package/src/providers/integrations/cline/cline.ts +94 -8
- package/src/providers/integrations/codex/codex.ts +61 -37
- package/src/providers/integrations/cursor/cursor-catalog.ts +368 -368
- package/src/providers/integrations/cursor/cursor-http2.ts +300 -300
- package/src/providers/integrations/cursor/cursor-identity.ts +2 -4
- package/src/providers/integrations/cursor/cursor-request.ts +11 -0
- package/src/providers/integrations/cursor/cursor.ts +29 -17
- package/src/providers/integrations/cursor/protocol/framing.ts +129 -129
- package/src/providers/integrations/cursor/protocol/messages.ts +2380 -2380
- package/src/providers/integrations/cursor/protocol/struct.ts +305 -305
- package/src/providers/integrations/cursor/protocol/tool-canonical.ts +275 -0
- package/src/providers/integrations/grok/grok-quota.ts +279 -279
- package/src/providers/integrations/grok/grok-ratelimit-store.ts +73 -73
- package/src/providers/integrations/grok/grok-ratelimit.ts +218 -218
- package/src/providers/integrations/hermes.ts +7 -1
- package/src/providers/integrations/inferhub.ts +201 -73
- package/src/providers/integrations/kilo/kilo.ts +152 -152
- package/src/providers/integrations/kiro/kiro-machine-id.ts +2 -4
- package/src/providers/integrations/openrouter/provider.ts +129 -129
- package/src/providers/integrations/qoder/qoder-quota.ts +193 -192
- package/src/providers/integrations/qoder/qoder-realm.ts +106 -106
- package/src/providers/integrations/trae/trae-oauth.ts +4 -26
- package/src/providers/integrations/trae/trae-quota.ts +389 -384
- package/src/providers/integrations/trae/trae-realm.ts +15 -2
- package/src/providers/integrations/xiaomi-mimo/think-stream.ts +100 -24
- package/src/providers/integrations/zcode/oauth.ts +1 -1
- package/src/providers/operations/account-health-service.ts +605 -279
- package/src/providers/provider-metadata.ts +12 -1
- package/src/providers/provider-model-definition.ts +373 -373
- package/src/providers/quota/quota-contracts.ts +3 -7
- package/src/providers/search/search-provider.ts +2 -6
- package/src/security/admission/contracts.ts +10 -0
- package/src/security/admission/in-memory-store.ts +89 -14
- package/src/security/admission/lua.ts +191 -182
- package/src/security/admission/redis-store.ts +30 -0
- package/src/security/admission/service.ts +18 -0
- package/src/security/api-key-auth-cache.ts +96 -0
- package/src/security/api-key-auth.ts +11 -100
- package/src/security/client-router-fingerprint.ts +3 -7
- package/src/security/ip-boundary.ts +7 -3
- package/src/shared/coerce.ts +67 -0
- package/src/shared/hash.ts +6 -0
- package/src/shared/time.ts +7 -0
- package/src/transport/dispatch/attempt-finalize.ts +26 -138
- package/src/transport/dispatch/attempt-loop.ts +103 -81
- package/src/transport/dispatch/capture-policy.ts +78 -0
- package/src/transport/dispatch/proxy-request.ts +4 -2
- package/src/transport/dispatch/streaming/encoder.ts +111 -5
- package/src/transport/dispatch/streaming/stream.ts +19 -2
- package/src/transport/dispatch/streaming/thinking-loop.ts +300 -0
- package/src/transport/failure-policy.ts +27 -0
- package/src/transport/guards/abuse-guard.ts +2 -2
- package/src/transport/guards/auth-guard.ts +17 -0
- package/src/transport/guards/body-policy.ts +3 -2
- package/src/transport/guards/concurrency-guard.ts +89 -13
- package/src/transport/guards/early-rejection-capture.ts +43 -3
- package/src/transport/guards/error-lifecycle.ts +10 -2
- package/src/transport/guards/guard-errors.ts +23 -0
- package/src/transport/guards/headroom-guard.ts +41 -7
- package/src/transport/guards/pipeline.ts +24 -0
- package/src/transport/guards/request-context.ts +2 -1
- package/src/transport/routing/admission.ts +51 -9
- package/src/transport/routing/bounded-lru.ts +49 -0
- package/src/transport/routing/candidate-index.ts +88 -0
- package/src/transport/routing/reservations.ts +121 -7
- package/src/transport/routing/round-robin.ts +10 -22
- package/src/transport/routing/route-catalog.ts +26 -13
- package/src/transport/routing/route-model.ts +22 -2
- package/src/transport/routing/router.ts +53 -31
- package/src/transport/surface/chat/encode.ts +2 -3
- package/src/transport/surface/chat/parse.ts +2 -1
- package/src/transport/surface/dialects.ts +11 -3
- package/src/transport/surface/messages/encode.ts +2 -1
- package/src/transport/surface/messages/parse.ts +3 -1
- package/src/transport/surface/messages/stream.ts +2 -6
- package/src/workers/maintenance-lease.ts +200 -0
- package/src/workers/scheduler.ts +101 -1
- package/dist/dashboard/assets/BannedPage-Cd30Z0b9.js.gz +0 -0
- package/dist/dashboard/assets/CliToolDetailPage-wf5u-ww6.js.gz +0 -0
- package/dist/dashboard/assets/CliToolsPage-DykPvuck.js.gz +0 -0
- package/dist/dashboard/assets/CombosPage-DwvSD3vs.js.gz +0 -0
- package/dist/dashboard/assets/ConsoleLogPage-CIr7KDex.js.gz +0 -0
- package/dist/dashboard/assets/CustomizationPage-82X0DDdH.js.gz +0 -0
- package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js +0 -1
- package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js.gz +0 -0
- package/dist/dashboard/assets/LoginPage-CS5mPuVF.js.gz +0 -0
- package/dist/dashboard/assets/ModelPicker-C0UCUsVJ.js.gz +0 -0
- package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js +0 -1
- package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js.gz +0 -0
- package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js +0 -3
- package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js.gz +0 -0
- package/dist/dashboard/assets/ProvidersPage-vdja5ZPK.js.gz +0 -0
- package/dist/dashboard/assets/ProxyPage-BB4CNm4-.js.gz +0 -0
- package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js +0 -1
- package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js.gz +0 -0
- package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js +0 -1
- package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js.gz +0 -0
- package/dist/dashboard/assets/SetupPage-FadUOl9S.js.gz +0 -0
- package/dist/dashboard/assets/SortableList-Cfy6_Khv.js.gz +0 -0
- package/dist/dashboard/assets/UsagePage-BT4MyQeT.js.gz +0 -0
- package/dist/dashboard/assets/api-keys-DQYLQxpW.js.gz +0 -0
- package/dist/dashboard/assets/app-CX_Z8cMs.css +0 -2
- package/dist/dashboard/assets/app-CX_Z8cMs.css.gz +0 -0
- package/dist/dashboard/assets/assert-C9FD_JwJ.js.gz +0 -0
- package/dist/dashboard/assets/badge-Bvyr7D0u.js.gz +0 -0
- package/dist/dashboard/assets/banner-DIvKAZih.js.gz +0 -0
- package/dist/dashboard/assets/card-D5_gYIS_.js.gz +0 -0
- package/dist/dashboard/assets/cli-tools-CNqzrfdb.js.gz +0 -0
- package/dist/dashboard/assets/clipboard-button-B6vDBtbm.js.gz +0 -0
- package/dist/dashboard/assets/dialog-CpjrgdwJ.js.gz +0 -0
- package/dist/dashboard/assets/drawer-BWmt3b5y.js.gz +0 -0
- package/dist/dashboard/assets/entry-B-gvimrg.js +0 -1
- package/dist/dashboard/assets/entry-B-gvimrg.js.gz +0 -0
- package/dist/dashboard/assets/entry-C_oakV3z.js.gz +0 -0
- package/dist/dashboard/assets/entry-Dj0cHfEc.js +0 -4
- package/dist/dashboard/assets/entry-Dj0cHfEc.js.gz +0 -0
- package/dist/dashboard/assets/github-badge-RRHHWGxh.js.gz +0 -0
- package/dist/dashboard/assets/index-Bhey_rg_.js.gz +0 -0
- package/dist/dashboard/assets/inline-C2_EQ5Uf.js.gz +0 -0
- package/dist/dashboard/assets/input-BMRlQ450.js.gz +0 -0
- package/dist/dashboard/assets/layout-XR5xP8ew.js.gz +0 -0
- package/dist/dashboard/assets/list-row-Cw07dioc.js.gz +0 -0
- package/dist/dashboard/assets/live-BivGfuUs.js.gz +0 -0
- package/dist/dashboard/assets/model-bans-CmffjLuG.js.gz +0 -0
- package/dist/dashboard/assets/mutation-46P0oWMe.js.gz +0 -0
- package/dist/dashboard/assets/numbers-DrdA9oSM.js.gz +0 -0
- package/dist/dashboard/assets/query-keys-BFhGpwR1.js.gz +0 -0
- package/dist/dashboard/assets/routing-CAb4F37R.js.gz +0 -0
- package/dist/dashboard/assets/select-CxVVB__N.js.gz +0 -0
- package/dist/dashboard/assets/settings-DNX9sydi.js.gz +0 -0
- package/dist/dashboard/assets/stack-uPOwM9Hs.js.gz +0 -0
- package/dist/dashboard/assets/state-D1Yz79iL.js.gz +0 -0
- package/dist/dashboard/assets/switch-sevsRrRo.js.gz +0 -0
- package/dist/dashboard/assets/useMutation-Dem0lXQj.js.gz +0 -0
- package/src/observability/event-compaction.ts +0 -317
- package/src/observability/event-day-index.ts +0 -439
|
@@ -5,6 +5,7 @@ import { assertBoundedJsonDepth } from "../../protocol/codecs";
|
|
|
5
5
|
import type { ProxyRequestStateStore } from "../request/state";
|
|
6
6
|
import { fastPathname } from "../pathname";
|
|
7
7
|
import { NATIVE_SERVICE_PATHS, TOKEN_COUNT_PATH } from "../contracts";
|
|
8
|
+
import { bodyTooLargeError } from "./guard-errors";
|
|
8
9
|
|
|
9
10
|
/** Whether this request is a gateway (`/v1/*`) request. */
|
|
10
11
|
export function isProxyRequest(request: Request): boolean {
|
|
@@ -87,7 +88,7 @@ export async function readIngressBody(
|
|
|
87
88
|
const declared = request.headers.get("content-length");
|
|
88
89
|
const parsed = declared === null ? undefined : Number(declared);
|
|
89
90
|
if (declared !== null && (!/^\d+$/.test(declared) || parsed === undefined || parsed > maxBytes))
|
|
90
|
-
throw
|
|
91
|
+
throw bodyTooLargeError(maxBytes);
|
|
91
92
|
|
|
92
93
|
const reader = request.body?.getReader();
|
|
93
94
|
if (!reader) throw new GatewayError("invalid_request", 400, "The request body could not be read. Resend it with a JSON body.");
|
|
@@ -104,7 +105,7 @@ export async function readIngressBody(
|
|
|
104
105
|
totalBytes += chunk.byteLength;
|
|
105
106
|
if (totalBytes > maxBytes) {
|
|
106
107
|
await reader.cancel().catch(() => undefined);
|
|
107
|
-
throw
|
|
108
|
+
throw bodyTooLargeError(maxBytes);
|
|
108
109
|
}
|
|
109
110
|
chunks.push(chunk);
|
|
110
111
|
}
|
|
@@ -6,23 +6,97 @@
|
|
|
6
6
|
* their own cap still open a hundred upstream calls at once, and the event
|
|
7
7
|
* loop, the database pool, and the upstream all feel that as one burst.
|
|
8
8
|
*
|
|
9
|
-
* This
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
9
|
+
* This admits requests into this process and refuses past the cap with 503 and
|
|
10
|
+
* a retry hint. It is deliberately not a queue: queueing would hold the
|
|
11
|
+
* connection open and make the overload look like latency. A client that
|
|
12
|
+
* retries after the hint lands on a process that has drained.
|
|
13
|
+
*
|
|
14
|
+
* The cap is now DERIVED — see `config/resources.ts` — rather than a constant,
|
|
15
|
+
* because a constant cap admits the same concurrency on a 512 MiB container as
|
|
16
|
+
* on a 4 GiB one, and only the latter can actually hold it.
|
|
14
17
|
*/
|
|
15
18
|
import { Elysia } from "elysia";
|
|
16
19
|
import { GatewayError } from "../../protocol/gateway-error";
|
|
20
|
+
import { resolveProcessConcurrency } from "../../config/resources";
|
|
21
|
+
import { resolveMemoryLimitBytes } from "../../observability/runtime-metrics";
|
|
17
22
|
import { fastPathname } from "../pathname";
|
|
18
23
|
|
|
19
24
|
/** In-flight `/v1` requests one process will carry before refusing. */
|
|
20
|
-
export const PROCESS_CONCURRENCY_CAP =
|
|
25
|
+
export const PROCESS_CONCURRENCY_CAP = resolveProcessConcurrency(
|
|
26
|
+
resolveMemoryLimitBytes() ?? 512 * 1024 * 1024,
|
|
27
|
+
);
|
|
21
28
|
const PROCESS_CONCURRENCY_RETRY_MS = 3_000;
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
/**
|
|
31
|
+
* A bounded set of in-flight tokens.
|
|
32
|
+
*
|
|
33
|
+
* Owned by the caller rather than held in a module global: the previous shape
|
|
34
|
+
* was one `let inFlight` for the whole module, so two middleware instances
|
|
35
|
+
* (tests, or a second app in one process) shared and corrupted a single count.
|
|
36
|
+
*/
|
|
37
|
+
export interface ProcessConcurrencyController {
|
|
38
|
+
/**
|
|
39
|
+
* Takes a token, or returns `undefined` when the process is full.
|
|
40
|
+
*
|
|
41
|
+
* The returned release function is idempotent — released twice still frees
|
|
42
|
+
* exactly one token — because a request can reach more than one completion
|
|
43
|
+
* path (an abort racing a normal finish).
|
|
44
|
+
*/
|
|
45
|
+
tryAcquire(): (() => void) | undefined;
|
|
46
|
+
/** Tokens currently held. */
|
|
47
|
+
activeCount(): number;
|
|
48
|
+
/** The effective cap. */
|
|
49
|
+
limit(): number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function createProcessConcurrencyController(
|
|
53
|
+
limit: number = PROCESS_CONCURRENCY_CAP,
|
|
54
|
+
): ProcessConcurrencyController {
|
|
55
|
+
const effective = Math.min(64, Math.max(1, Math.trunc(limit)));
|
|
56
|
+
let active = 0;
|
|
57
|
+
return {
|
|
58
|
+
tryAcquire() {
|
|
59
|
+
if (active >= effective) return undefined;
|
|
60
|
+
active += 1;
|
|
61
|
+
let released = false;
|
|
62
|
+
return () => {
|
|
63
|
+
if (released) return;
|
|
64
|
+
released = true;
|
|
65
|
+
if (active > 0) active -= 1;
|
|
66
|
+
};
|
|
67
|
+
},
|
|
68
|
+
activeCount() {
|
|
69
|
+
return active;
|
|
70
|
+
},
|
|
71
|
+
limit() {
|
|
72
|
+
return effective;
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The controller the middleware below uses when none is injected.
|
|
79
|
+
*
|
|
80
|
+
* Exported so the metrics sampler can publish its live count: in-flight
|
|
81
|
+
* concurrency is the number that explains both a latency rise and a shed, and
|
|
82
|
+
* a gauge nobody sets is worse than no gauge.
|
|
83
|
+
*/
|
|
84
|
+
export const defaultProcessConcurrencyController = createProcessConcurrencyController();
|
|
24
85
|
|
|
25
|
-
|
|
86
|
+
/**
|
|
87
|
+
* The middleware.
|
|
88
|
+
*
|
|
89
|
+
* The token is keyed by the Request object in a `WeakMap` and released only if
|
|
90
|
+
* one was actually taken. The previous code decremented a module-global count
|
|
91
|
+
* in `afterResponse` for every `/v1` request — including the ones this guard
|
|
92
|
+
* had just REJECTED, which never incremented it. Under sustained overload that
|
|
93
|
+
* drove the count down spuriously and let more requests through than the cap
|
|
94
|
+
* allows, which is the opposite of what a cap is for.
|
|
95
|
+
*/
|
|
96
|
+
export function createProcessConcurrencyMiddleware(
|
|
97
|
+
controller: ProcessConcurrencyController = defaultProcessConcurrencyController,
|
|
98
|
+
): Elysia {
|
|
99
|
+
const tokens = new WeakMap<Request, () => void>();
|
|
26
100
|
return new Elysia()
|
|
27
101
|
.beforeHandle(({ request }) => {
|
|
28
102
|
const path = fastPathname(request.url);
|
|
@@ -30,7 +104,8 @@ export function createProcessConcurrencyMiddleware(): Elysia {
|
|
|
30
104
|
// still answer while the process is saturated, or an operator cannot
|
|
31
105
|
// see why it is refusing.
|
|
32
106
|
if (!path.startsWith("/v1/")) return;
|
|
33
|
-
|
|
107
|
+
const release = controller.tryAcquire();
|
|
108
|
+
if (release === undefined) {
|
|
34
109
|
throw new GatewayError(
|
|
35
110
|
"platform_unavailable",
|
|
36
111
|
503,
|
|
@@ -38,12 +113,13 @@ export function createProcessConcurrencyMiddleware(): Elysia {
|
|
|
38
113
|
{ retryAfterMs: PROCESS_CONCURRENCY_RETRY_MS },
|
|
39
114
|
);
|
|
40
115
|
}
|
|
41
|
-
|
|
116
|
+
tokens.set(request, release);
|
|
42
117
|
})
|
|
43
118
|
.afterResponse(({ request }: { request: Request }) => {
|
|
44
|
-
const
|
|
45
|
-
if (
|
|
46
|
-
|
|
119
|
+
const release = tokens.get(request);
|
|
120
|
+
if (release === undefined) return;
|
|
121
|
+
tokens.delete(request);
|
|
122
|
+
release();
|
|
47
123
|
})
|
|
48
124
|
.as("plugin");
|
|
49
125
|
}
|
|
@@ -12,9 +12,15 @@
|
|
|
12
12
|
* error envelope the client received.
|
|
13
13
|
*/
|
|
14
14
|
import type { PayloadCaptureState } from "../../observability/payload-capture";
|
|
15
|
+
import { resolvePayloadCapture } from "../../observability/payload-capture";
|
|
16
|
+
import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
15
17
|
/** The capture surface this helper needs; matches the real capture's method. */
|
|
16
18
|
export interface EarlyRejectionCaptureSink {
|
|
17
|
-
capture(input: EarlyRejectionCapture): Promise<
|
|
19
|
+
capture(input: EarlyRejectionCapture): Promise<{
|
|
20
|
+
id: string;
|
|
21
|
+
expiresAt: Date;
|
|
22
|
+
payloadCaptureState?: string;
|
|
23
|
+
}>;
|
|
18
24
|
}
|
|
19
25
|
|
|
20
26
|
/** One early-rejection payload frame. */
|
|
@@ -34,6 +40,15 @@ export interface EarlyRejectionState {
|
|
|
34
40
|
readonly ingressBody?: unknown;
|
|
35
41
|
readonly clientResponseText?: string;
|
|
36
42
|
readonly payloadCapture?: EarlyRejectionCaptureSink;
|
|
43
|
+
/**
|
|
44
|
+
* The request's memoized capture preference, when the request already
|
|
45
|
+
* resolved one. Early rejections rarely have, so the helper resolves the
|
|
46
|
+
* tenant preference itself through the same `resolvePayloadCapture` path
|
|
47
|
+
* every other capture decision uses — the bypass this field exists to end
|
|
48
|
+
* was an early rejection capturing bodies with the deployment's full cap
|
|
49
|
+
* while the tenant had configured `none` or `metadata`.
|
|
50
|
+
*/
|
|
51
|
+
payloadCapturePreference?: unknown;
|
|
37
52
|
/**
|
|
38
53
|
* Written by this helper so telemetry can record whether the rejection's body
|
|
39
54
|
* was captured. Mutable because the reason is an outcome, not an input.
|
|
@@ -51,12 +66,36 @@ export interface EarlyRejectionState {
|
|
|
51
66
|
export async function captureEarlyRejectionPayload(
|
|
52
67
|
state: EarlyRejectionState,
|
|
53
68
|
authorization: { readonly tenantId: string },
|
|
69
|
+
db?: CartethyiaDatabase,
|
|
54
70
|
): Promise<void> {
|
|
55
71
|
const capture = state.payloadCapture;
|
|
56
72
|
if (capture === undefined) return;
|
|
57
73
|
if (state.ingressBody === undefined) return;
|
|
58
74
|
try {
|
|
59
|
-
|
|
75
|
+
// The same preference path a dispatched request resolves (memoized on the
|
|
76
|
+
// state when the request got far enough to have one). `none` captures
|
|
77
|
+
// nothing at all; `metadata` records stage presence only, without the
|
|
78
|
+
// ingress body; only `full` keeps bodies — and at the tenant's own
|
|
79
|
+
// retention, never the deployment default by accident.
|
|
80
|
+
const preference =
|
|
81
|
+
state.payloadCapturePreference !== undefined
|
|
82
|
+
? (state.payloadCapturePreference as { ok: boolean })
|
|
83
|
+
: db !== undefined
|
|
84
|
+
? await resolvePayloadCapture(db, authorization.tenantId)
|
|
85
|
+
: { ok: false };
|
|
86
|
+
const mode = preference.ok === true ? (preference as { mode: string }).mode : "none";
|
|
87
|
+
if (mode === "none") {
|
|
88
|
+
state.payloadCaptureState = "skipped_mode_none";
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (mode === "metadata") {
|
|
92
|
+
// Stage presence only: the ingress body is exactly what `metadata`
|
|
93
|
+
// mode declines to retain. A frame with no body is pointless, so the
|
|
94
|
+
// state records the mode outcome and nothing is written.
|
|
95
|
+
state.payloadCaptureState = "captured_metadata";
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const result = await capture.capture({
|
|
60
99
|
tenantId: authorization.tenantId,
|
|
61
100
|
requestBody: state.ingressBody,
|
|
62
101
|
// No provider exchange: the request never reached one.
|
|
@@ -66,7 +105,8 @@ export async function captureEarlyRejectionPayload(
|
|
|
66
105
|
tenantOptIn: true,
|
|
67
106
|
requestId: state.requestId,
|
|
68
107
|
});
|
|
69
|
-
state.payloadCaptureState =
|
|
108
|
+
state.payloadCaptureState =
|
|
109
|
+
(result.payloadCaptureState as PayloadCaptureState | undefined) ?? "captured";
|
|
70
110
|
} catch {
|
|
71
111
|
// A capture failure is not a request failure; the telemetry row stands.
|
|
72
112
|
state.payloadCaptureState = "failed_write";
|
|
@@ -6,6 +6,7 @@ import { GATEWAY_SECURITY_HEADERS } from "../../security/outbound-headers";
|
|
|
6
6
|
import type { TelemetryBatchBuffer, TelemetryEventInput } from "../../observability/telemetry-buffer";
|
|
7
7
|
import type { TelemetryPayloadCapture } from "../../observability/payload-capture";
|
|
8
8
|
import { captureEarlyRejectionPayload } from "./early-rejection-capture";
|
|
9
|
+
import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
9
10
|
import { reasoningEffortFromIntent } from "../../providers/provider-reasoning";
|
|
10
11
|
import { metrics } from "../../observability/metrics";
|
|
11
12
|
import { computeTokensPerSec } from "../../observability/token-speed";
|
|
@@ -168,6 +169,7 @@ export interface AfterResponseApp {
|
|
|
168
169
|
export function finalizeRequestTelemetry(
|
|
169
170
|
state: ProxyRequestState,
|
|
170
171
|
telemetryBuffer: TelemetryBatchBuffer,
|
|
172
|
+
db?: CartethyiaDatabase,
|
|
171
173
|
): void {
|
|
172
174
|
// Discovery/surface routes under `/v1` (e.g. `/v1/models`) are authenticated
|
|
173
175
|
// gateway routes that never dispatch upstream. They hold request state but
|
|
@@ -225,7 +227,11 @@ export function finalizeRequestTelemetry(
|
|
|
225
227
|
// would have nothing to read. Capture the ingress body and the error the
|
|
226
228
|
// client saw, keyed to the same request id as the row.
|
|
227
229
|
if (requestData === undefined && state.ingressBody !== undefined) {
|
|
228
|
-
|
|
230
|
+
// `deps.db` so the helper resolves the tenant's capture mode through the
|
|
231
|
+
// same preference path every dispatched request uses — without it, an
|
|
232
|
+
// early rejection captured bodies with the deployment default regardless
|
|
233
|
+
// of the tenant's `none`/`metadata` choice.
|
|
234
|
+
void captureEarlyRejectionPayload(state, authorization, db);
|
|
229
235
|
}
|
|
230
236
|
|
|
231
237
|
telemetryBuffer.enqueue(
|
|
@@ -385,6 +391,8 @@ export function registerTelemetryLifecycle(
|
|
|
385
391
|
* telemetry row without a payload frame rather than failing.
|
|
386
392
|
*/
|
|
387
393
|
readonly payloadCapture?: TelemetryPayloadCapture;
|
|
394
|
+
/** Database handle for early-rejection capture-mode resolution. */
|
|
395
|
+
readonly db?: CartethyiaDatabase;
|
|
388
396
|
},
|
|
389
397
|
): void {
|
|
390
398
|
app.afterResponse(async ({ request }) => {
|
|
@@ -404,7 +412,7 @@ export function registerTelemetryLifecycle(
|
|
|
404
412
|
// finalizer for requests that never reached completion (early
|
|
405
413
|
// rejections). Cleanup below must still run for completed requests:
|
|
406
414
|
// skipping it leaks one in-flight count per request, forever.
|
|
407
|
-
if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer);
|
|
415
|
+
if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer, deps.db);
|
|
408
416
|
} finally {
|
|
409
417
|
state.cleanup();
|
|
410
418
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared construction points for the guard errors that more than one guard
|
|
3
|
+
* raises. Normalisation downstream is single-seam (`error-lifecycle.ts`); the
|
|
4
|
+
* duplication this module removes was on the construction side, where four
|
|
5
|
+
* guards each hand-rolled their own code + message for the same conditions.
|
|
6
|
+
*/
|
|
7
|
+
import { GatewayError } from "../../protocol/gateway-error";
|
|
8
|
+
|
|
9
|
+
/** Thrown when the request's client peer address cannot be established. */
|
|
10
|
+
export function peerAddressUnavailableError(): GatewayError {
|
|
11
|
+
return new GatewayError("admission_unavailable", 503, "client peer address unavailable");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Thrown when the request body exceeds the endpoint's byte budget. */
|
|
15
|
+
export function bodyTooLargeError(maxBytes: number): GatewayError {
|
|
16
|
+
return new GatewayError(
|
|
17
|
+
"invalid_request",
|
|
18
|
+
413,
|
|
19
|
+
`The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`,
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
@@ -17,15 +17,14 @@
|
|
|
17
17
|
* metrics are exempt so an operator can always reach the console that explains
|
|
18
18
|
* the shedding.
|
|
19
19
|
*/
|
|
20
|
-
import { totalmem } from "node:os";
|
|
21
20
|
import { Elysia } from "elysia";
|
|
22
21
|
import { GatewayError } from "../../protocol/gateway-error";
|
|
23
22
|
import type { RedisClient } from "../../persistence/redis";
|
|
23
|
+
import { deriveMemoryHeadroomBytes } from "../../config/resources";
|
|
24
|
+
import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
|
|
24
25
|
import type { ProxyRequestStateStore } from "../request/state";
|
|
25
26
|
import { isProxyRequest } from "./body-policy";
|
|
26
27
|
|
|
27
|
-
/** Free bytes the process must keep below its RSS before it stops accepting. */
|
|
28
|
-
const DEFAULT_MIN_FREE_BYTES = 64 * 1024 * 1024;
|
|
29
28
|
/** How long the shared shed marker outlives the last shed, so peers can see it. */
|
|
30
29
|
const SHED_MARKER_TTL_SECONDS = 5;
|
|
31
30
|
/** Fleet-visible marker: set on the first shed so every instance reports the same state. */
|
|
@@ -34,7 +33,22 @@ const SHED_RETRY_MS = 1_000;
|
|
|
34
33
|
|
|
35
34
|
export interface HeadroomGuardDeps {
|
|
36
35
|
readonly store: ProxyRequestStateStore;
|
|
37
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* The last sampled memory state, published once per sample by
|
|
38
|
+
* `RuntimeMetricsSampler`.
|
|
39
|
+
*
|
|
40
|
+
* Taken from the sampler rather than measured here: this runs on EVERY
|
|
41
|
+
* request, and `process.memoryUsage()` on the request path is a cost paid
|
|
42
|
+
* per request to answer a question that changes over seconds. When absent
|
|
43
|
+
* (before the first sample) the guard falls back to measuring once — an
|
|
44
|
+
* unsampled process is still a process that can be OOM-killed.
|
|
45
|
+
*/
|
|
46
|
+
readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* Free bytes to keep below the resolved limit. Defaults to
|
|
49
|
+
* `MEMORY_HEADROOM_FRACTION` of it, so a 512 MiB container keeps ~102 MiB
|
|
50
|
+
* free rather than the flat 64 MiB a 2 GiB one would keep.
|
|
51
|
+
*/
|
|
38
52
|
readonly minFreeBytes?: number;
|
|
39
53
|
/**
|
|
40
54
|
* Optional shared client. Used only to publish a shed marker for visibility;
|
|
@@ -45,11 +59,27 @@ export interface HeadroomGuardDeps {
|
|
|
45
59
|
}
|
|
46
60
|
|
|
47
61
|
export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
|
|
48
|
-
const
|
|
62
|
+
const fixedHeadroom = deps.minFreeBytes;
|
|
49
63
|
return new Elysia()
|
|
50
64
|
.beforeHandle(({ request }) => {
|
|
51
65
|
if (!isProxyRequest(request)) return;
|
|
52
|
-
|
|
66
|
+
// Shed when RSS plus the headroom would exceed this process's LIMIT —
|
|
67
|
+
// not the host's total memory. A container capped at 512 MiB on a 16 GiB
|
|
68
|
+
// host never approached the old `os.totalmem()` threshold, so the guard
|
|
69
|
+
// never fired and the process was OOM-killed instead of shedding.
|
|
70
|
+
//
|
|
71
|
+
// The snapshot is up to one sample interval old. That is deliberate:
|
|
72
|
+
// this is the hottest path in the process, and the condition it guards
|
|
73
|
+
// develops over seconds.
|
|
74
|
+
const snapshot = deps.memorySnapshot?.();
|
|
75
|
+
const limit = snapshot?.limitBytes;
|
|
76
|
+
if (limit === undefined) {
|
|
77
|
+
// No limit discoverable and no sample yet: nothing to shed against.
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const rss = snapshot?.rssBytes ?? process.memoryUsage().rss;
|
|
81
|
+
const headroom = fixedHeadroom ?? deriveMemoryHeadroomBytes(limit);
|
|
82
|
+
if (rss + headroom <= limit) return;
|
|
53
83
|
if (deps.redis) {
|
|
54
84
|
// Fire-and-forget: a failed marker write must not change this request's
|
|
55
85
|
// outcome, and awaiting it would put Redis on the shed path.
|
|
@@ -61,7 +91,11 @@ export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
|
|
|
61
91
|
"overloaded",
|
|
62
92
|
503,
|
|
63
93
|
"The gateway is at capacity; retry shortly.",
|
|
64
|
-
{
|
|
94
|
+
{
|
|
95
|
+
retryAfterMs: SHED_RETRY_MS,
|
|
96
|
+
minFreeBytes: headroom,
|
|
97
|
+
activeRequests: deps.store.activeCount(),
|
|
98
|
+
},
|
|
65
99
|
);
|
|
66
100
|
})
|
|
67
101
|
.as("plugin");
|
|
@@ -28,6 +28,7 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
|
|
|
28
28
|
import type { RedisClient } from "../../persistence/redis";
|
|
29
29
|
import type { TelemetryBatchBuffer } from "../../observability/telemetry-buffer";
|
|
30
30
|
import type { ModelStrikeService } from "../../security/model-abuse";
|
|
31
|
+
import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
|
|
31
32
|
import {
|
|
32
33
|
TelemetryPayloadCapture,
|
|
33
34
|
payloadCaptureKeepsBodies,
|
|
@@ -58,6 +59,20 @@ export interface TransportPipelineContext {
|
|
|
58
59
|
* decision is local, and Redis is only written for fleet visibility.
|
|
59
60
|
*/
|
|
60
61
|
readonly redis?: RedisClient | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* The sampled memory state the headroom guard sheds against.
|
|
64
|
+
*
|
|
65
|
+
* Supplied by the runtime sampler. When absent the guard cannot compare RSS
|
|
66
|
+
* against a limit and does not shed — so this must be wired in production,
|
|
67
|
+
* and the sampler must be initialized at boot even before its first tick.
|
|
68
|
+
*/
|
|
69
|
+
readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Whether invoke is switched off for a tenant. Optional: without it the
|
|
72
|
+
* gateway never refuses on the kill switch, which is the pre-change
|
|
73
|
+
* behaviour rather than a failure.
|
|
74
|
+
*/
|
|
75
|
+
readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
|
|
61
76
|
}
|
|
62
77
|
|
|
63
78
|
/**
|
|
@@ -84,6 +99,7 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
|
|
|
84
99
|
createProcessConcurrencyMiddleware(),
|
|
85
100
|
createHeadroomGuard({
|
|
86
101
|
store: context.stateStore,
|
|
102
|
+
...(context.memorySnapshot === undefined ? {} : { memorySnapshot: context.memorySnapshot }),
|
|
87
103
|
...(context.redis === undefined ? {} : { redis: context.redis }),
|
|
88
104
|
}),
|
|
89
105
|
createDependencyReadinessMiddleware({
|
|
@@ -103,6 +119,9 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
|
|
|
103
119
|
db: context.db,
|
|
104
120
|
stateStore: context.stateStore,
|
|
105
121
|
...(context.modelStrikes ? { modelStrikes: context.modelStrikes } : {}),
|
|
122
|
+
...(context.isInvokeDisabled
|
|
123
|
+
? { isInvokeDisabled: context.isInvokeDisabled }
|
|
124
|
+
: {}),
|
|
106
125
|
}),
|
|
107
126
|
createCanonicalRequestMiddleware({
|
|
108
127
|
stateStore: context.stateStore,
|
|
@@ -181,6 +200,11 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
|
|
|
181
200
|
(resolved) => (resolved.ok ? resolved.retentionMs : payloadRetentionMs()),
|
|
182
201
|
),
|
|
183
202
|
),
|
|
203
|
+
// The early-rejection capture resolves the tenant's capture mode
|
|
204
|
+
// through the same preference path dispatched requests use, instead of
|
|
205
|
+
// capturing bodies under the deployment default regardless of the
|
|
206
|
+
// tenant's none/metadata choice.
|
|
207
|
+
db: context.db,
|
|
184
208
|
});
|
|
185
209
|
} else {
|
|
186
210
|
registerRequestCleanup(rootApp, { stateStore: context.stateStore });
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type ModelAbuseOutcome,
|
|
21
21
|
type ModelStrikeService,
|
|
22
22
|
} from "../../security/model-abuse";
|
|
23
|
+
import { peerAddressUnavailableError } from "./guard-errors";
|
|
23
24
|
|
|
24
25
|
/**
|
|
25
26
|
* Whether a thrown error is a *model* rejection the strike layer counts: the
|
|
@@ -123,7 +124,7 @@ export function createClientIdentityMiddleware(deps: {
|
|
|
123
124
|
? deps.resolvePeerAddress(request)
|
|
124
125
|
: (server?.requestIP(request)?.address ?? null);
|
|
125
126
|
if (!peer)
|
|
126
|
-
throw
|
|
127
|
+
throw peerAddressUnavailableError();
|
|
127
128
|
const address = resolveClientIdentity(request, deps.trustedProxyBoundary, peer);
|
|
128
129
|
const state = deps.stateStore.require(request);
|
|
129
130
|
state.clientIdentity = {
|
|
@@ -69,8 +69,20 @@ function capacityRejected(): AdmissionDecision {
|
|
|
69
69
|
|
|
70
70
|
export class InMemoryAdmissionController implements AdmissionController {
|
|
71
71
|
private inflight = new Map<string, number>();
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
readAccountInflight(
|
|
73
|
+
candidates: readonly RouteCandidate[],
|
|
74
|
+
): ReadonlyMap<string, number> {
|
|
75
|
+
// Dedupe buckets: one candidate list often repeats a bucket (combo members
|
|
76
|
+
// fan out to several candidates on one account), and the caller only needs
|
|
77
|
+
// one count per bucket to make its ordering decision.
|
|
78
|
+
const snapshot = new Map<string, number>();
|
|
79
|
+
for (const candidate of candidates) {
|
|
80
|
+
const key = admissionKey(candidate);
|
|
81
|
+
if (snapshot.has(key)) continue;
|
|
82
|
+
const value = this.inflight.get(key) ?? 0;
|
|
83
|
+
if (value > 0) snapshot.set(key, value);
|
|
84
|
+
}
|
|
85
|
+
return snapshot;
|
|
74
86
|
}
|
|
75
87
|
async admit(candidate: RouteCandidate): Promise<AdmissionDecision> {
|
|
76
88
|
// Bucket per ACCOUNT, not per provider:model. A provider with N accounts
|
|
@@ -166,17 +178,47 @@ export class RedisAdmissionController implements AdmissionController {
|
|
|
166
178
|
await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, this.key(reservation.candidate));
|
|
167
179
|
}
|
|
168
180
|
|
|
169
|
-
async
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
181
|
+
async readAccountInflight(
|
|
182
|
+
candidates: readonly RouteCandidate[],
|
|
183
|
+
): Promise<ReadonlyMap<string, number>> {
|
|
184
|
+
// Candidate-scoped exact reads replace the fleet-wide `KEYS` + unbounded
|
|
185
|
+
// `MGET` this method used to run on every `plan()` that had any bounded
|
|
186
|
+
// candidate. `KEYS` is O(keyspace) and blocks the single-threaded Redis
|
|
187
|
+
// server; on a shared box the scan was measured as the dominant planning
|
|
188
|
+
// cost. The keys here are exactly the buckets the caller is about to order
|
|
189
|
+
// candidates by, read in bounded sequential chunks.
|
|
190
|
+
//
|
|
191
|
+
// `KEYS` and `SCAN` stay banned on this path by construction: neither can
|
|
192
|
+
// answer "what are the values of these N known keys" in fewer than
|
|
193
|
+
// O(keyspace) server work, and a fleet-wide inflight snapshot has no
|
|
194
|
+
// request-path consumer left. (If one ever appears, it belongs in a
|
|
195
|
+
// maintenance task using `SCAN` with `COUNT`, never here.)
|
|
196
|
+
const buckets = [...new Set(candidates.map(admissionKey))];
|
|
197
|
+
if (buckets.length === 0) return new Map();
|
|
198
|
+
const keys = buckets.map((bucket) => `admission:inflight:${bucket}`);
|
|
199
|
+
const values: number[] = [];
|
|
200
|
+
// Bounded sequential chunks: one huge MGET would move the unbounded
|
|
201
|
+
// payload problem from the KEYS call to the MGET call. 256 keys keeps a
|
|
202
|
+
// single command payload small while the round trips stay proportional to
|
|
203
|
+
// the (deduped, candidate-bounded) request size.
|
|
204
|
+
for (let offset = 0; offset < keys.length; offset += MGET_CHUNK_SIZE) {
|
|
205
|
+
const chunk = keys.slice(offset, offset + MGET_CHUNK_SIZE);
|
|
206
|
+
const raw = (await this.redis.mget(...chunk)) as (string | null)[];
|
|
207
|
+
for (const value of raw) values.push(Number(value ?? 0));
|
|
208
|
+
}
|
|
173
209
|
const snapshot = new Map<string, number>();
|
|
174
|
-
for (const [index,
|
|
175
|
-
const value =
|
|
210
|
+
for (const [index, bucket] of buckets.entries()) {
|
|
211
|
+
const value = values[index] ?? 0;
|
|
176
212
|
if (Number.isFinite(value) && value > 0) {
|
|
177
|
-
snapshot.set(
|
|
213
|
+
snapshot.set(bucket, value);
|
|
178
214
|
}
|
|
215
|
+
// The Redis controller's counter keys are TTL self-healing, so a bucket
|
|
216
|
+
// absent from the reply reads as zero and is omitted — the projection
|
|
217
|
+
// both controllers share: positive-finite counts only, unprefixed names.
|
|
179
218
|
}
|
|
180
219
|
return snapshot;
|
|
181
220
|
}
|
|
182
221
|
}
|
|
222
|
+
|
|
223
|
+
/** Sequential MGET chunk bound for {@link RedisAdmissionController.readAccountInflight}. */
|
|
224
|
+
const MGET_CHUNK_SIZE = 256;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Bounded LRU map: the one shared implementation for routing-layer caches.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A capacity-bounded LRU `Map`.
|
|
5
|
+
*
|
|
6
|
+
* `get()` refreshes recency; `set()` evicts exactly the least-recently-used
|
|
7
|
+
* entry on overflow. Insertion order is the recency order (head = LRU,
|
|
8
|
+
* tail = MRU), which `Map` maintains for free — so eviction is one
|
|
9
|
+
* `keys().next()` and no bookkeeping arrays.
|
|
10
|
+
*
|
|
11
|
+
* The routing layer previously hand-rolled this shape twice (round-robin
|
|
12
|
+
* state, capability profiles) with subtly different code. One implementation
|
|
13
|
+
* means the bound is provable once: the map's size can never exceed
|
|
14
|
+
* `capacity`, and every entry is reachable through the normal `Map` API the
|
|
15
|
+
* callers already use.
|
|
16
|
+
*/
|
|
17
|
+
export class BoundedLruMap<K, V> extends Map<K, V> {
|
|
18
|
+
private readonly capacity: number;
|
|
19
|
+
|
|
20
|
+
constructor(capacity: number) {
|
|
21
|
+
if (!Number.isFinite(capacity) || capacity < 1 || Math.trunc(capacity) !== capacity) {
|
|
22
|
+
throw new Error(`BoundedLruMap capacity must be a positive integer, got ${capacity}`);
|
|
23
|
+
}
|
|
24
|
+
super();
|
|
25
|
+
this.capacity = Math.trunc(capacity);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
override get(key: K): V | undefined {
|
|
29
|
+
if (!super.has(key)) return undefined;
|
|
30
|
+
// Refresh recency: delete + re-insert moves the entry to the tail.
|
|
31
|
+
const value = super.get(key)!;
|
|
32
|
+
super.delete(key);
|
|
33
|
+
super.set(key, value);
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
override set(key: K, value: V): this {
|
|
38
|
+
// An existing key is re-inserted at the tail even when the value is the
|
|
39
|
+
// same object, so a read-modify-write cycle cannot leave a hot entry
|
|
40
|
+
// parked at the head one eviction away from disappearing.
|
|
41
|
+
if (super.has(key)) super.delete(key);
|
|
42
|
+
else if (super.size >= this.capacity) {
|
|
43
|
+
const oldest = super.keys().next().value;
|
|
44
|
+
if (oldest !== undefined) super.delete(oldest);
|
|
45
|
+
}
|
|
46
|
+
super.set(key, value);
|
|
47
|
+
return this;
|
|
48
|
+
}
|
|
49
|
+
}
|