cartethyia 2.5.10 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (367) hide show
  1. package/.env.example +8 -0
  2. package/dist/dashboard/assets/{BannedPage-Cd30Z0b9.js → BannedPage-l6XMnfr9.js} +1 -1
  3. package/dist/dashboard/assets/BannedPage-l6XMnfr9.js.gz +0 -0
  4. package/dist/dashboard/assets/{CliToolDetailPage-wf5u-ww6.js → CliToolDetailPage-DwOzAW45.js} +1 -1
  5. package/dist/dashboard/assets/CliToolDetailPage-DwOzAW45.js.gz +0 -0
  6. package/dist/dashboard/assets/{CliToolsPage-DykPvuck.js → CliToolsPage-BT3Up3QA.js} +1 -1
  7. package/dist/dashboard/assets/CliToolsPage-BT3Up3QA.js.gz +0 -0
  8. package/dist/dashboard/assets/{CombosPage-DwvSD3vs.js → CombosPage-6kYIUpzR.js} +1 -1
  9. package/dist/dashboard/assets/CombosPage-6kYIUpzR.js.gz +0 -0
  10. package/dist/dashboard/assets/{ConsoleLogPage-CIr7KDex.js → ConsoleLogPage-zRnj1mlR.js} +1 -1
  11. package/dist/dashboard/assets/ConsoleLogPage-zRnj1mlR.js.gz +0 -0
  12. package/dist/dashboard/assets/{CustomizationPage-82X0DDdH.js → CustomizationPage-C7m-H1Rz.js} +1 -1
  13. package/dist/dashboard/assets/CustomizationPage-C7m-H1Rz.js.gz +0 -0
  14. package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js +1 -0
  15. package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js.gz +0 -0
  16. package/dist/dashboard/assets/{LoginPage-CS5mPuVF.js → LoginPage-CbphLvjg.js} +1 -1
  17. package/dist/dashboard/assets/LoginPage-CbphLvjg.js.gz +0 -0
  18. package/dist/dashboard/assets/{ModelPicker-C0UCUsVJ.js → ModelPicker-DVny4S5_.js} +1 -1
  19. package/dist/dashboard/assets/ModelPicker-DVny4S5_.js.gz +0 -0
  20. package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js +1 -0
  21. package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js.gz +0 -0
  22. package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js +3 -0
  23. package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js.gz +0 -0
  24. package/dist/dashboard/assets/{ProvidersPage-vdja5ZPK.js → ProvidersPage-Dfb2896L.js} +1 -1
  25. package/dist/dashboard/assets/ProvidersPage-Dfb2896L.js.gz +0 -0
  26. package/dist/dashboard/assets/{ProxyPage-BB4CNm4-.js → ProxyPage-BaCIdAUt.js} +1 -1
  27. package/dist/dashboard/assets/ProxyPage-BaCIdAUt.js.gz +0 -0
  28. package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js +1 -0
  29. package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js.gz +0 -0
  30. package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js +1 -0
  31. package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js.gz +0 -0
  32. package/dist/dashboard/assets/{SetupPage-FadUOl9S.js → SetupPage-D4g6vOGc.js} +1 -1
  33. package/dist/dashboard/assets/SetupPage-D4g6vOGc.js.gz +0 -0
  34. package/dist/dashboard/assets/{SortableList-Cfy6_Khv.js → SortableList-C93i14BD.js} +1 -1
  35. package/dist/dashboard/assets/SortableList-C93i14BD.js.gz +0 -0
  36. package/dist/dashboard/assets/{StudioPage-CM92fgJf.js → StudioPage-SgQpymjm.js} +1 -1
  37. package/dist/dashboard/assets/{StudioPage-CM92fgJf.js.gz → StudioPage-SgQpymjm.js.gz} +0 -0
  38. package/dist/dashboard/assets/{UsagePage-BT4MyQeT.js → UsagePage-BFME-eba.js} +2 -2
  39. package/dist/dashboard/assets/UsagePage-BFME-eba.js.gz +0 -0
  40. package/dist/dashboard/assets/{api-keys-DQYLQxpW.js → api-keys-C23S9sgp.js} +1 -1
  41. package/dist/dashboard/assets/api-keys-C23S9sgp.js.gz +0 -0
  42. package/dist/dashboard/assets/{app-B7pVRhJ1.js → app-2KhtHCv9.js} +1 -1
  43. package/dist/dashboard/assets/{app-B7pVRhJ1.js.gz → app-2KhtHCv9.js.gz} +0 -0
  44. package/dist/dashboard/assets/app-CEeaMHOy.css +2 -0
  45. package/dist/dashboard/assets/app-CEeaMHOy.css.gz +0 -0
  46. package/dist/dashboard/assets/{assert-C9FD_JwJ.js → assert-DU6qrMA_.js} +1 -1
  47. package/dist/dashboard/assets/assert-DU6qrMA_.js.gz +0 -0
  48. package/dist/dashboard/assets/{badge-Bvyr7D0u.js → badge-CKsF123d.js} +1 -1
  49. package/dist/dashboard/assets/badge-CKsF123d.js.gz +0 -0
  50. package/dist/dashboard/assets/{banner-DIvKAZih.js → banner-8j7uChA9.js} +1 -1
  51. package/dist/dashboard/assets/banner-8j7uChA9.js.gz +0 -0
  52. package/dist/dashboard/assets/{card-D5_gYIS_.js → card-CdsSlWA3.js} +1 -1
  53. package/dist/dashboard/assets/card-CdsSlWA3.js.gz +0 -0
  54. package/dist/dashboard/assets/{cli-tools-CNqzrfdb.js → cli-tools-BaFUuFFF.js} +1 -1
  55. package/dist/dashboard/assets/cli-tools-BaFUuFFF.js.gz +0 -0
  56. package/dist/dashboard/assets/{clipboard-button-B6vDBtbm.js → clipboard-button-uL5aYLfW.js} +1 -1
  57. package/dist/dashboard/assets/clipboard-button-uL5aYLfW.js.gz +0 -0
  58. package/dist/dashboard/assets/{dialog-CpjrgdwJ.js → dialog-C07hr5_G.js} +1 -1
  59. package/dist/dashboard/assets/dialog-C07hr5_G.js.gz +0 -0
  60. package/dist/dashboard/assets/{drawer-BWmt3b5y.js → drawer-BPO1TcKH.js} +1 -1
  61. package/dist/dashboard/assets/drawer-BPO1TcKH.js.gz +0 -0
  62. package/dist/dashboard/assets/{entry-C_oakV3z.js → entry-Bz_CtDfC.js} +1 -1
  63. package/dist/dashboard/assets/entry-Bz_CtDfC.js.gz +0 -0
  64. package/dist/dashboard/assets/entry-CwPRKk0d.js +4 -0
  65. package/dist/dashboard/assets/entry-CwPRKk0d.js.gz +0 -0
  66. package/dist/dashboard/assets/entry-GJRbmrpA.js +1 -0
  67. package/dist/dashboard/assets/entry-GJRbmrpA.js.gz +0 -0
  68. package/dist/dashboard/assets/{github-badge-RRHHWGxh.js → github-badge-DmQreSDd.js} +1 -1
  69. package/dist/dashboard/assets/github-badge-DmQreSDd.js.gz +0 -0
  70. package/dist/dashboard/assets/{index-Bhey_rg_.js → index-Bblu6erR.js} +2 -2
  71. package/dist/dashboard/assets/index-Bblu6erR.js.gz +0 -0
  72. package/dist/dashboard/assets/{inline-C2_EQ5Uf.js → inline-C9dCYaBZ.js} +1 -1
  73. package/dist/dashboard/assets/inline-C9dCYaBZ.js.gz +0 -0
  74. package/dist/dashboard/assets/{input-BMRlQ450.js → input-CjCnXweJ.js} +1 -1
  75. package/dist/dashboard/assets/input-CjCnXweJ.js.gz +0 -0
  76. package/dist/dashboard/assets/{layout-XR5xP8ew.js → layout-V6y8DAkl.js} +1 -1
  77. package/dist/dashboard/assets/layout-V6y8DAkl.js.gz +0 -0
  78. package/dist/dashboard/assets/{list-row-Cw07dioc.js → list-row-CM1AaG_y.js} +1 -1
  79. package/dist/dashboard/assets/list-row-CM1AaG_y.js.gz +0 -0
  80. package/dist/dashboard/assets/{live-BivGfuUs.js → live-ta6GMb-8.js} +1 -1
  81. package/dist/dashboard/assets/live-ta6GMb-8.js.gz +0 -0
  82. package/dist/dashboard/assets/{model-bans-CmffjLuG.js → model-bans-BmYTmbAk.js} +1 -1
  83. package/dist/dashboard/assets/model-bans-BmYTmbAk.js.gz +0 -0
  84. package/dist/dashboard/assets/{mutation-46P0oWMe.js → mutation-g_fduiNw.js} +1 -1
  85. package/dist/dashboard/assets/mutation-g_fduiNw.js.gz +0 -0
  86. package/dist/dashboard/assets/{numbers-DrdA9oSM.js → numbers-BOevJzdz.js} +1 -1
  87. package/dist/dashboard/assets/numbers-BOevJzdz.js.gz +0 -0
  88. package/dist/dashboard/assets/{query-keys-BFhGpwR1.js → query-keys-UETqfytp.js} +1 -1
  89. package/dist/dashboard/assets/query-keys-UETqfytp.js.gz +0 -0
  90. package/dist/dashboard/assets/{routing-CAb4F37R.js → routing-BPUo-W8d.js} +1 -1
  91. package/dist/dashboard/assets/routing-BPUo-W8d.js.gz +0 -0
  92. package/dist/dashboard/assets/{select-CxVVB__N.js → select-C980c4cz.js} +1 -1
  93. package/dist/dashboard/assets/select-C980c4cz.js.gz +0 -0
  94. package/dist/dashboard/assets/{settings-DNX9sydi.js → settings--3hxSyry.js} +1 -1
  95. package/dist/dashboard/assets/settings--3hxSyry.js.gz +0 -0
  96. package/dist/dashboard/assets/{stack-uPOwM9Hs.js → stack-_gfTdLVB.js} +1 -1
  97. package/dist/dashboard/assets/stack-_gfTdLVB.js.gz +0 -0
  98. package/dist/dashboard/assets/{state-D1Yz79iL.js → state-2c5V4zo7.js} +1 -1
  99. package/dist/dashboard/assets/state-2c5V4zo7.js.gz +0 -0
  100. package/dist/dashboard/assets/{switch-sevsRrRo.js → switch-Dkke7Naf.js} +1 -1
  101. package/dist/dashboard/assets/switch-Dkke7Naf.js.gz +0 -0
  102. package/dist/dashboard/assets/{useMutation-Dem0lXQj.js → useMutation-EDFRkn1i.js} +1 -1
  103. package/dist/dashboard/assets/useMutation-EDFRkn1i.js.gz +0 -0
  104. package/dist/dashboard/index.html +1 -1
  105. package/dist/dashboard/index.html.gz +0 -0
  106. package/dist/dashboard/precompressed-assets.json +51 -51
  107. package/migrations/README.md +52 -52
  108. package/migrations/full-postgres/0020_add_telemetry_event_analytics.sql +77 -0
  109. package/migrations/full-postgres/0021_add_api_key_concurrency_tolerance.sql +13 -0
  110. package/migrations/full-postgres/0022_add_inferhub_config_key.sql +17 -0
  111. package/migrations/full-postgres/0023_drop_api_key_concurrency_tolerance.sql +13 -0
  112. package/migrations/lite-sqlite3/0020_add_telemetry_event_analytics.sql +69 -0
  113. package/migrations/lite-sqlite3/0021_add_api_key_concurrency_tolerance.sql +13 -0
  114. package/migrations/lite-sqlite3/0022_add_inferhub_config_key.sql +17 -0
  115. package/migrations/lite-sqlite3/0023_drop_api_key_concurrency_tolerance.sql +13 -0
  116. package/package.json +96 -95
  117. package/scripts/cli/admin.ts +126 -128
  118. package/scripts/cli/autostart.ts +237 -237
  119. package/scripts/cli/background.ts +120 -110
  120. package/scripts/cli/bin/carte.cjs +8 -8
  121. package/scripts/cli/bin/cartethyia.cjs +167 -167
  122. package/scripts/cli/console-menu.ts +39 -30
  123. package/scripts/cli/controller.ts +145 -0
  124. package/scripts/cli/env.ts +99 -99
  125. package/scripts/cli/gateway.ts +264 -260
  126. package/scripts/cli/instance.ts +234 -224
  127. package/scripts/cli/main.ts +188 -167
  128. package/scripts/cli/platform.ts +64 -64
  129. package/scripts/cli/prompt.ts +217 -79
  130. package/scripts/cli/services.ts +88 -88
  131. package/scripts/cli/setup.ts +445 -445
  132. package/scripts/cli/start.ts +85 -85
  133. package/scripts/cli/startup-signal.ts +23 -23
  134. package/scripts/cli/telemetry.ts +70 -0
  135. package/scripts/cli/tray/index.ts +263 -263
  136. package/scripts/cli/tray/windows.ps1 +107 -107
  137. package/scripts/cli/tray.ts +29 -4
  138. package/scripts/cli/update.ts +39 -39
  139. package/scripts/cli/version.ts +67 -67
  140. package/scripts/cli/wordmark.ts +58 -58
  141. package/src/boot/app.ts +16 -1
  142. package/src/boot/dependencies.ts +128 -25
  143. package/src/config/resources.ts +137 -0
  144. package/src/config/runtime.ts +133 -76
  145. package/src/config/spec.ts +30 -29
  146. package/src/config/telemetry.ts +0 -25
  147. package/src/console/api-keys/bucket-spend.ts +14 -17
  148. package/src/console/api-keys/contracts.ts +410 -410
  149. package/src/console/api-keys/routes.ts +12 -0
  150. package/src/console/audit/store.ts +2 -3
  151. package/src/console/auth/setup-service.ts +8 -1
  152. package/src/console/backup/routes.ts +7 -0
  153. package/src/console/backup/service.ts +463 -463
  154. package/src/console/backup/store.ts +16 -11
  155. package/src/console/backup/validate.ts +1 -4
  156. package/src/console/cli-tools/fs-ops.ts +18 -2
  157. package/src/console/data-management/contracts.ts +1 -8
  158. package/src/console/data-management/operations.ts +61 -48
  159. package/src/console/data-management/routes.ts +4 -2
  160. package/src/console/domain-registration.ts +25 -3
  161. package/src/console/inferhub/client.ts +139 -0
  162. package/src/console/inferhub/key-store.ts +129 -0
  163. package/src/console/inferhub/normalize.ts +347 -0
  164. package/src/console/inferhub/routes.ts +411 -0
  165. package/src/console/inferhub/store.ts +109 -0
  166. package/src/console/inferhub/types.ts +183 -0
  167. package/src/console/observability/contracts.ts +2 -41
  168. package/src/console/observability/store.ts +176 -371
  169. package/src/console/providers/catalog/public-model-store.ts +0 -0
  170. package/src/console/providers/catalog/store.ts +23 -36
  171. package/src/console/quota/account-quota-shared.ts +52 -52
  172. package/src/console/quota/account-quota-tenant.ts +16 -24
  173. package/src/console/quota/refresh.ts +441 -441
  174. package/src/console/routing/pools/import-parse.ts +206 -206
  175. package/src/console/routing/pools/operations.ts +543 -543
  176. package/src/console/routing/pools/probe-verdict.ts +132 -132
  177. package/src/console/routing/pools/routes.ts +181 -181
  178. package/src/console/settings/contracts.ts +42 -40
  179. package/src/console/settings/store.ts +25 -25
  180. package/src/console/share/share-stats.ts +22 -4
  181. package/src/console/share/share-usage.ts +30 -4
  182. package/src/console/shared/cursor.ts +15 -0
  183. package/src/console/updates/routes.ts +12 -0
  184. package/src/main.ts +13 -5
  185. package/src/network/pool/resolver.ts +1104 -1104
  186. package/src/observability/client-name.ts +22 -0
  187. package/src/observability/event-log.ts +5 -29
  188. package/src/observability/event-query.ts +690 -0
  189. package/src/observability/log-ring.ts +26 -2
  190. package/src/observability/log-sanitize.ts +118 -0
  191. package/src/observability/logger.ts +17 -13
  192. package/src/observability/metrics.ts +35 -0
  193. package/src/observability/payload-capture.ts +114 -41
  194. package/src/observability/payload-store.ts +249 -24
  195. package/src/observability/raw-event-writer.ts +118 -0
  196. package/src/observability/runtime-metrics.ts +71 -1
  197. package/src/observability/telemetry-buffer.ts +100 -17
  198. package/src/observability/telemetry-reconciliation.ts +417 -0
  199. package/src/persistence/boot.ts +20 -9
  200. package/src/persistence/db-pg.ts +13 -0
  201. package/src/persistence/db-sqlite.ts +569 -497
  202. package/src/persistence/dialect.ts +3 -2
  203. package/src/persistence/migrate.ts +181 -181
  204. package/src/persistence/schema.ts +1496 -1406
  205. package/src/persistence/shared-cache.ts +5 -1
  206. package/src/persistence/stores/api-key-store.ts +581 -581
  207. package/src/persistence/stores/telemetry-store.ts +124 -63
  208. package/src/protocol/reasoning-text.ts +30 -0
  209. package/src/protocol/response/chat.ts +33 -7
  210. package/src/protocol/response/codex.ts +14 -2
  211. package/src/protocol/response/messages.ts +5 -2
  212. package/src/providers/authentication/oauth-flow-store.ts +3 -10
  213. package/src/providers/bundled-discovery-providers.ts +122 -122
  214. package/src/providers/default-registry.ts +10 -0
  215. package/src/providers/discovery/catalog/catalog-blob.ts +18 -18
  216. package/src/providers/discovery/models-dev-catalog.ts +354 -354
  217. package/src/providers/discovery/probe-phases.ts +43 -7
  218. package/src/providers/discovery/probing-service.ts +70 -32
  219. package/src/providers/integrations/antigravity/antigravity.ts +9 -2
  220. package/src/providers/integrations/buddy/buddy-catalog-shared.ts +85 -17
  221. package/src/providers/integrations/buddy/buddy-chat-shared.ts +934 -934
  222. package/src/providers/integrations/buddy/buddy-checkin.ts +5 -9
  223. package/src/providers/integrations/buddy/buddy-discovery-shared.ts +3 -7
  224. package/src/providers/integrations/buddy/buddy-quota-shared.ts +3 -1
  225. package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +57 -49
  226. package/src/providers/integrations/buddy/codebuddy/provider.ts +180 -180
  227. package/src/providers/integrations/buddy/workbuddy/provider.ts +206 -206
  228. package/src/providers/integrations/buddy/workbuddy/quota.ts +161 -161
  229. package/src/providers/integrations/claude/claude-quota.ts +2 -2
  230. package/src/providers/integrations/claude/claude-user-profiles.ts +3 -2
  231. package/src/providers/integrations/claude/claude.ts +3 -4
  232. package/src/providers/integrations/cline/cline.ts +94 -8
  233. package/src/providers/integrations/codex/codex.ts +61 -37
  234. package/src/providers/integrations/cursor/cursor-catalog.ts +368 -368
  235. package/src/providers/integrations/cursor/cursor-http2.ts +300 -300
  236. package/src/providers/integrations/cursor/cursor-identity.ts +2 -4
  237. package/src/providers/integrations/cursor/cursor-request.ts +11 -0
  238. package/src/providers/integrations/cursor/cursor.ts +29 -17
  239. package/src/providers/integrations/cursor/protocol/framing.ts +129 -129
  240. package/src/providers/integrations/cursor/protocol/messages.ts +2380 -2380
  241. package/src/providers/integrations/cursor/protocol/struct.ts +305 -305
  242. package/src/providers/integrations/cursor/protocol/tool-canonical.ts +275 -0
  243. package/src/providers/integrations/grok/grok-quota.ts +279 -279
  244. package/src/providers/integrations/grok/grok-ratelimit-store.ts +73 -73
  245. package/src/providers/integrations/grok/grok-ratelimit.ts +218 -218
  246. package/src/providers/integrations/hermes.ts +7 -1
  247. package/src/providers/integrations/inferhub.ts +201 -73
  248. package/src/providers/integrations/kilo/kilo.ts +152 -152
  249. package/src/providers/integrations/kiro/kiro-machine-id.ts +2 -4
  250. package/src/providers/integrations/openrouter/provider.ts +129 -129
  251. package/src/providers/integrations/qoder/qoder-quota.ts +193 -192
  252. package/src/providers/integrations/qoder/qoder-realm.ts +106 -106
  253. package/src/providers/integrations/trae/trae-oauth.ts +4 -26
  254. package/src/providers/integrations/trae/trae-quota.ts +389 -384
  255. package/src/providers/integrations/trae/trae-realm.ts +15 -2
  256. package/src/providers/integrations/xiaomi-mimo/think-stream.ts +100 -24
  257. package/src/providers/integrations/zcode/oauth.ts +1 -1
  258. package/src/providers/operations/account-health-service.ts +605 -279
  259. package/src/providers/provider-metadata.ts +12 -1
  260. package/src/providers/provider-model-definition.ts +373 -373
  261. package/src/providers/quota/quota-contracts.ts +3 -7
  262. package/src/providers/search/search-provider.ts +2 -6
  263. package/src/security/admission/contracts.ts +10 -0
  264. package/src/security/admission/in-memory-store.ts +89 -14
  265. package/src/security/admission/lua.ts +191 -182
  266. package/src/security/admission/redis-store.ts +30 -0
  267. package/src/security/admission/service.ts +18 -0
  268. package/src/security/api-key-auth-cache.ts +96 -0
  269. package/src/security/api-key-auth.ts +11 -100
  270. package/src/security/client-router-fingerprint.ts +3 -7
  271. package/src/security/ip-boundary.ts +7 -3
  272. package/src/shared/coerce.ts +67 -0
  273. package/src/shared/hash.ts +6 -0
  274. package/src/shared/time.ts +7 -0
  275. package/src/transport/dispatch/attempt-finalize.ts +26 -138
  276. package/src/transport/dispatch/attempt-loop.ts +103 -81
  277. package/src/transport/dispatch/capture-policy.ts +78 -0
  278. package/src/transport/dispatch/proxy-request.ts +4 -2
  279. package/src/transport/dispatch/streaming/encoder.ts +111 -5
  280. package/src/transport/dispatch/streaming/stream.ts +19 -2
  281. package/src/transport/dispatch/streaming/thinking-loop.ts +300 -0
  282. package/src/transport/failure-policy.ts +27 -0
  283. package/src/transport/guards/abuse-guard.ts +2 -2
  284. package/src/transport/guards/auth-guard.ts +17 -0
  285. package/src/transport/guards/body-policy.ts +3 -2
  286. package/src/transport/guards/concurrency-guard.ts +89 -13
  287. package/src/transport/guards/early-rejection-capture.ts +43 -3
  288. package/src/transport/guards/error-lifecycle.ts +10 -2
  289. package/src/transport/guards/guard-errors.ts +23 -0
  290. package/src/transport/guards/headroom-guard.ts +41 -7
  291. package/src/transport/guards/pipeline.ts +24 -0
  292. package/src/transport/guards/request-context.ts +2 -1
  293. package/src/transport/routing/admission.ts +51 -9
  294. package/src/transport/routing/bounded-lru.ts +49 -0
  295. package/src/transport/routing/candidate-index.ts +88 -0
  296. package/src/transport/routing/reservations.ts +121 -7
  297. package/src/transport/routing/round-robin.ts +10 -22
  298. package/src/transport/routing/route-catalog.ts +26 -13
  299. package/src/transport/routing/route-model.ts +22 -2
  300. package/src/transport/routing/router.ts +53 -31
  301. package/src/transport/surface/chat/encode.ts +2 -3
  302. package/src/transport/surface/chat/parse.ts +2 -1
  303. package/src/transport/surface/dialects.ts +11 -3
  304. package/src/transport/surface/messages/encode.ts +2 -1
  305. package/src/transport/surface/messages/parse.ts +3 -1
  306. package/src/transport/surface/messages/stream.ts +2 -6
  307. package/src/workers/maintenance-lease.ts +200 -0
  308. package/src/workers/scheduler.ts +101 -1
  309. package/dist/dashboard/assets/BannedPage-Cd30Z0b9.js.gz +0 -0
  310. package/dist/dashboard/assets/CliToolDetailPage-wf5u-ww6.js.gz +0 -0
  311. package/dist/dashboard/assets/CliToolsPage-DykPvuck.js.gz +0 -0
  312. package/dist/dashboard/assets/CombosPage-DwvSD3vs.js.gz +0 -0
  313. package/dist/dashboard/assets/ConsoleLogPage-CIr7KDex.js.gz +0 -0
  314. package/dist/dashboard/assets/CustomizationPage-82X0DDdH.js.gz +0 -0
  315. package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js +0 -1
  316. package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js.gz +0 -0
  317. package/dist/dashboard/assets/LoginPage-CS5mPuVF.js.gz +0 -0
  318. package/dist/dashboard/assets/ModelPicker-C0UCUsVJ.js.gz +0 -0
  319. package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js +0 -1
  320. package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js.gz +0 -0
  321. package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js +0 -3
  322. package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js.gz +0 -0
  323. package/dist/dashboard/assets/ProvidersPage-vdja5ZPK.js.gz +0 -0
  324. package/dist/dashboard/assets/ProxyPage-BB4CNm4-.js.gz +0 -0
  325. package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js +0 -1
  326. package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js.gz +0 -0
  327. package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js +0 -1
  328. package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js.gz +0 -0
  329. package/dist/dashboard/assets/SetupPage-FadUOl9S.js.gz +0 -0
  330. package/dist/dashboard/assets/SortableList-Cfy6_Khv.js.gz +0 -0
  331. package/dist/dashboard/assets/UsagePage-BT4MyQeT.js.gz +0 -0
  332. package/dist/dashboard/assets/api-keys-DQYLQxpW.js.gz +0 -0
  333. package/dist/dashboard/assets/app-CX_Z8cMs.css +0 -2
  334. package/dist/dashboard/assets/app-CX_Z8cMs.css.gz +0 -0
  335. package/dist/dashboard/assets/assert-C9FD_JwJ.js.gz +0 -0
  336. package/dist/dashboard/assets/badge-Bvyr7D0u.js.gz +0 -0
  337. package/dist/dashboard/assets/banner-DIvKAZih.js.gz +0 -0
  338. package/dist/dashboard/assets/card-D5_gYIS_.js.gz +0 -0
  339. package/dist/dashboard/assets/cli-tools-CNqzrfdb.js.gz +0 -0
  340. package/dist/dashboard/assets/clipboard-button-B6vDBtbm.js.gz +0 -0
  341. package/dist/dashboard/assets/dialog-CpjrgdwJ.js.gz +0 -0
  342. package/dist/dashboard/assets/drawer-BWmt3b5y.js.gz +0 -0
  343. package/dist/dashboard/assets/entry-B-gvimrg.js +0 -1
  344. package/dist/dashboard/assets/entry-B-gvimrg.js.gz +0 -0
  345. package/dist/dashboard/assets/entry-C_oakV3z.js.gz +0 -0
  346. package/dist/dashboard/assets/entry-Dj0cHfEc.js +0 -4
  347. package/dist/dashboard/assets/entry-Dj0cHfEc.js.gz +0 -0
  348. package/dist/dashboard/assets/github-badge-RRHHWGxh.js.gz +0 -0
  349. package/dist/dashboard/assets/index-Bhey_rg_.js.gz +0 -0
  350. package/dist/dashboard/assets/inline-C2_EQ5Uf.js.gz +0 -0
  351. package/dist/dashboard/assets/input-BMRlQ450.js.gz +0 -0
  352. package/dist/dashboard/assets/layout-XR5xP8ew.js.gz +0 -0
  353. package/dist/dashboard/assets/list-row-Cw07dioc.js.gz +0 -0
  354. package/dist/dashboard/assets/live-BivGfuUs.js.gz +0 -0
  355. package/dist/dashboard/assets/model-bans-CmffjLuG.js.gz +0 -0
  356. package/dist/dashboard/assets/mutation-46P0oWMe.js.gz +0 -0
  357. package/dist/dashboard/assets/numbers-DrdA9oSM.js.gz +0 -0
  358. package/dist/dashboard/assets/query-keys-BFhGpwR1.js.gz +0 -0
  359. package/dist/dashboard/assets/routing-CAb4F37R.js.gz +0 -0
  360. package/dist/dashboard/assets/select-CxVVB__N.js.gz +0 -0
  361. package/dist/dashboard/assets/settings-DNX9sydi.js.gz +0 -0
  362. package/dist/dashboard/assets/stack-uPOwM9Hs.js.gz +0 -0
  363. package/dist/dashboard/assets/state-D1Yz79iL.js.gz +0 -0
  364. package/dist/dashboard/assets/switch-sevsRrRo.js.gz +0 -0
  365. package/dist/dashboard/assets/useMutation-Dem0lXQj.js.gz +0 -0
  366. package/src/observability/event-compaction.ts +0 -317
  367. package/src/observability/event-day-index.ts +0 -439
@@ -5,6 +5,7 @@ import { assertBoundedJsonDepth } from "../../protocol/codecs";
5
5
  import type { ProxyRequestStateStore } from "../request/state";
6
6
  import { fastPathname } from "../pathname";
7
7
  import { NATIVE_SERVICE_PATHS, TOKEN_COUNT_PATH } from "../contracts";
8
+ import { bodyTooLargeError } from "./guard-errors";
8
9
 
9
10
  /** Whether this request is a gateway (`/v1/*`) request. */
10
11
  export function isProxyRequest(request: Request): boolean {
@@ -87,7 +88,7 @@ export async function readIngressBody(
87
88
  const declared = request.headers.get("content-length");
88
89
  const parsed = declared === null ? undefined : Number(declared);
89
90
  if (declared !== null && (!/^\d+$/.test(declared) || parsed === undefined || parsed > maxBytes))
90
- throw new GatewayError("invalid_request", 413, `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`);
91
+ throw bodyTooLargeError(maxBytes);
91
92
 
92
93
  const reader = request.body?.getReader();
93
94
  if (!reader) throw new GatewayError("invalid_request", 400, "The request body could not be read. Resend it with a JSON body.");
@@ -104,7 +105,7 @@ export async function readIngressBody(
104
105
  totalBytes += chunk.byteLength;
105
106
  if (totalBytes > maxBytes) {
106
107
  await reader.cancel().catch(() => undefined);
107
- throw new GatewayError("invalid_request", 413, `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`);
108
+ throw bodyTooLargeError(maxBytes);
108
109
  }
109
110
  chunks.push(chunk);
110
111
  }
@@ -6,23 +6,97 @@
6
6
  * their own cap still open a hundred upstream calls at once, and the event
7
7
  * loop, the database pool, and the upstream all feel that as one burst.
8
8
  *
9
- * This counts requests in flight in this process and refuses past the cap
10
- * with 503 and a three-second retry hint. It is deliberately not a queue:
11
- * queueing would hold the connection open and make the overload look like
12
- * latency. A client that retries after the hint lands on a process that has
13
- * drained.
9
+ * This admits requests into this process and refuses past the cap with 503 and
10
+ * a retry hint. It is deliberately not a queue: queueing would hold the
11
+ * connection open and make the overload look like latency. A client that
12
+ * retries after the hint lands on a process that has drained.
13
+ *
14
+ * The cap is now DERIVED — see `config/resources.ts` — rather than a constant,
15
+ * because a constant cap admits the same concurrency on a 512 MiB container as
16
+ * on a 4 GiB one, and only the latter can actually hold it.
14
17
  */
15
18
  import { Elysia } from "elysia";
16
19
  import { GatewayError } from "../../protocol/gateway-error";
20
+ import { resolveProcessConcurrency } from "../../config/resources";
21
+ import { resolveMemoryLimitBytes } from "../../observability/runtime-metrics";
17
22
  import { fastPathname } from "../pathname";
18
23
 
19
24
  /** In-flight `/v1` requests one process will carry before refusing. */
20
- export const PROCESS_CONCURRENCY_CAP = 64;
25
+ export const PROCESS_CONCURRENCY_CAP = resolveProcessConcurrency(
26
+ resolveMemoryLimitBytes() ?? 512 * 1024 * 1024,
27
+ );
21
28
  const PROCESS_CONCURRENCY_RETRY_MS = 3_000;
22
29
 
23
- let inFlight = 0;
30
+ /**
31
+ * A bounded set of in-flight tokens.
32
+ *
33
+ * Owned by the caller rather than held in a module global: the previous shape
34
+ * was one `let inFlight` for the whole module, so two middleware instances
35
+ * (tests, or a second app in one process) shared and corrupted a single count.
36
+ */
37
+ export interface ProcessConcurrencyController {
38
+ /**
39
+ * Takes a token, or returns `undefined` when the process is full.
40
+ *
41
+ * The returned release function is idempotent — released twice still frees
42
+ * exactly one token — because a request can reach more than one completion
43
+ * path (an abort racing a normal finish).
44
+ */
45
+ tryAcquire(): (() => void) | undefined;
46
+ /** Tokens currently held. */
47
+ activeCount(): number;
48
+ /** The effective cap. */
49
+ limit(): number;
50
+ }
51
+
52
+ export function createProcessConcurrencyController(
53
+ limit: number = PROCESS_CONCURRENCY_CAP,
54
+ ): ProcessConcurrencyController {
55
+ const effective = Math.min(64, Math.max(1, Math.trunc(limit)));
56
+ let active = 0;
57
+ return {
58
+ tryAcquire() {
59
+ if (active >= effective) return undefined;
60
+ active += 1;
61
+ let released = false;
62
+ return () => {
63
+ if (released) return;
64
+ released = true;
65
+ if (active > 0) active -= 1;
66
+ };
67
+ },
68
+ activeCount() {
69
+ return active;
70
+ },
71
+ limit() {
72
+ return effective;
73
+ },
74
+ };
75
+ }
76
+
77
+ /**
78
+ * The controller the middleware below uses when none is injected.
79
+ *
80
+ * Exported so the metrics sampler can publish its live count: in-flight
81
+ * concurrency is the number that explains both a latency rise and a shed, and
82
+ * a gauge nobody sets is worse than no gauge.
83
+ */
84
+ export const defaultProcessConcurrencyController = createProcessConcurrencyController();
24
85
 
25
- export function createProcessConcurrencyMiddleware(): Elysia {
86
+ /**
87
+ * The middleware.
88
+ *
89
+ * The token is keyed by the Request object in a `WeakMap` and released only if
90
+ * one was actually taken. The previous code decremented a module-global count
91
+ * in `afterResponse` for every `/v1` request — including the ones this guard
92
+ * had just REJECTED, which never incremented it. Under sustained overload that
93
+ * drove the count down spuriously and let more requests through than the cap
94
+ * allows, which is the opposite of what a cap is for.
95
+ */
96
+ export function createProcessConcurrencyMiddleware(
97
+ controller: ProcessConcurrencyController = defaultProcessConcurrencyController,
98
+ ): Elysia {
99
+ const tokens = new WeakMap<Request, () => void>();
26
100
  return new Elysia()
27
101
  .beforeHandle(({ request }) => {
28
102
  const path = fastPathname(request.url);
@@ -30,7 +104,8 @@ export function createProcessConcurrencyMiddleware(): Elysia {
30
104
  // still answer while the process is saturated, or an operator cannot
31
105
  // see why it is refusing.
32
106
  if (!path.startsWith("/v1/")) return;
33
- if (inFlight >= PROCESS_CONCURRENCY_CAP) {
107
+ const release = controller.tryAcquire();
108
+ if (release === undefined) {
34
109
  throw new GatewayError(
35
110
  "platform_unavailable",
36
111
  503,
@@ -38,12 +113,13 @@ export function createProcessConcurrencyMiddleware(): Elysia {
38
113
  { retryAfterMs: PROCESS_CONCURRENCY_RETRY_MS },
39
114
  );
40
115
  }
41
- inFlight += 1;
116
+ tokens.set(request, release);
42
117
  })
43
118
  .afterResponse(({ request }: { request: Request }) => {
44
- const path = fastPathname(request.url);
45
- if (!path.startsWith("/v1/")) return;
46
- if (inFlight > 0) inFlight -= 1;
119
+ const release = tokens.get(request);
120
+ if (release === undefined) return;
121
+ tokens.delete(request);
122
+ release();
47
123
  })
48
124
  .as("plugin");
49
125
  }
@@ -12,9 +12,15 @@
12
12
  * error envelope the client received.
13
13
  */
14
14
  import type { PayloadCaptureState } from "../../observability/payload-capture";
15
+ import { resolvePayloadCapture } from "../../observability/payload-capture";
16
+ import type { CartethyiaDatabase } from "../../persistence/db-handle";
15
17
  /** The capture surface this helper needs; matches the real capture's method. */
16
18
  export interface EarlyRejectionCaptureSink {
17
- capture(input: EarlyRejectionCapture): Promise<unknown>;
19
+ capture(input: EarlyRejectionCapture): Promise<{
20
+ id: string;
21
+ expiresAt: Date;
22
+ payloadCaptureState?: string;
23
+ }>;
18
24
  }
19
25
 
20
26
  /** One early-rejection payload frame. */
@@ -34,6 +40,15 @@ export interface EarlyRejectionState {
34
40
  readonly ingressBody?: unknown;
35
41
  readonly clientResponseText?: string;
36
42
  readonly payloadCapture?: EarlyRejectionCaptureSink;
43
+ /**
44
+ * The request's memoized capture preference, when the request already
45
+ * resolved one. Early rejections rarely have, so the helper resolves the
46
+ * tenant preference itself through the same `resolvePayloadCapture` path
47
+ * every other capture decision uses — the bypass this field exists to end
48
+ * was an early rejection capturing bodies with the deployment's full cap
49
+ * while the tenant had configured `none` or `metadata`.
50
+ */
51
+ payloadCapturePreference?: unknown;
37
52
  /**
38
53
  * Written by this helper so telemetry can record whether the rejection's body
39
54
  * was captured. Mutable because the reason is an outcome, not an input.
@@ -51,12 +66,36 @@ export interface EarlyRejectionState {
51
66
  export async function captureEarlyRejectionPayload(
52
67
  state: EarlyRejectionState,
53
68
  authorization: { readonly tenantId: string },
69
+ db?: CartethyiaDatabase,
54
70
  ): Promise<void> {
55
71
  const capture = state.payloadCapture;
56
72
  if (capture === undefined) return;
57
73
  if (state.ingressBody === undefined) return;
58
74
  try {
59
- await capture.capture({
75
+ // The same preference path a dispatched request resolves (memoized on the
76
+ // state when the request got far enough to have one). `none` captures
77
+ // nothing at all; `metadata` records stage presence only, without the
78
+ // ingress body; only `full` keeps bodies — and at the tenant's own
79
+ // retention, never the deployment default by accident.
80
+ const preference =
81
+ state.payloadCapturePreference !== undefined
82
+ ? (state.payloadCapturePreference as { ok: boolean })
83
+ : db !== undefined
84
+ ? await resolvePayloadCapture(db, authorization.tenantId)
85
+ : { ok: false };
86
+ const mode = preference.ok === true ? (preference as { mode: string }).mode : "none";
87
+ if (mode === "none") {
88
+ state.payloadCaptureState = "skipped_mode_none";
89
+ return;
90
+ }
91
+ if (mode === "metadata") {
92
+ // Stage presence only: the ingress body is exactly what `metadata`
93
+ // mode declines to retain. A frame with no body is pointless, so the
94
+ // state records the mode outcome and nothing is written.
95
+ state.payloadCaptureState = "captured_metadata";
96
+ return;
97
+ }
98
+ const result = await capture.capture({
60
99
  tenantId: authorization.tenantId,
61
100
  requestBody: state.ingressBody,
62
101
  // No provider exchange: the request never reached one.
@@ -66,7 +105,8 @@ export async function captureEarlyRejectionPayload(
66
105
  tenantOptIn: true,
67
106
  requestId: state.requestId,
68
107
  });
69
- state.payloadCaptureState = "captured";
108
+ state.payloadCaptureState =
109
+ (result.payloadCaptureState as PayloadCaptureState | undefined) ?? "captured";
70
110
  } catch {
71
111
  // A capture failure is not a request failure; the telemetry row stands.
72
112
  state.payloadCaptureState = "failed_write";
@@ -6,6 +6,7 @@ import { GATEWAY_SECURITY_HEADERS } from "../../security/outbound-headers";
6
6
  import type { TelemetryBatchBuffer, TelemetryEventInput } from "../../observability/telemetry-buffer";
7
7
  import type { TelemetryPayloadCapture } from "../../observability/payload-capture";
8
8
  import { captureEarlyRejectionPayload } from "./early-rejection-capture";
9
+ import type { CartethyiaDatabase } from "../../persistence/db-handle";
9
10
  import { reasoningEffortFromIntent } from "../../providers/provider-reasoning";
10
11
  import { metrics } from "../../observability/metrics";
11
12
  import { computeTokensPerSec } from "../../observability/token-speed";
@@ -168,6 +169,7 @@ export interface AfterResponseApp {
168
169
  export function finalizeRequestTelemetry(
169
170
  state: ProxyRequestState,
170
171
  telemetryBuffer: TelemetryBatchBuffer,
172
+ db?: CartethyiaDatabase,
171
173
  ): void {
172
174
  // Discovery/surface routes under `/v1` (e.g. `/v1/models`) are authenticated
173
175
  // gateway routes that never dispatch upstream. They hold request state but
@@ -225,7 +227,11 @@ export function finalizeRequestTelemetry(
225
227
  // would have nothing to read. Capture the ingress body and the error the
226
228
  // client saw, keyed to the same request id as the row.
227
229
  if (requestData === undefined && state.ingressBody !== undefined) {
228
- void captureEarlyRejectionPayload(state, authorization);
230
+ // `deps.db` so the helper resolves the tenant's capture mode through the
231
+ // same preference path every dispatched request uses — without it, an
232
+ // early rejection captured bodies with the deployment default regardless
233
+ // of the tenant's `none`/`metadata` choice.
234
+ void captureEarlyRejectionPayload(state, authorization, db);
229
235
  }
230
236
 
231
237
  telemetryBuffer.enqueue(
@@ -385,6 +391,8 @@ export function registerTelemetryLifecycle(
385
391
  * telemetry row without a payload frame rather than failing.
386
392
  */
387
393
  readonly payloadCapture?: TelemetryPayloadCapture;
394
+ /** Database handle for early-rejection capture-mode resolution. */
395
+ readonly db?: CartethyiaDatabase;
388
396
  },
389
397
  ): void {
390
398
  app.afterResponse(async ({ request }) => {
@@ -404,7 +412,7 @@ export function registerTelemetryLifecycle(
404
412
  // finalizer for requests that never reached completion (early
405
413
  // rejections). Cleanup below must still run for completed requests:
406
414
  // skipping it leaks one in-flight count per request, forever.
407
- if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer);
415
+ if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer, deps.db);
408
416
  } finally {
409
417
  state.cleanup();
410
418
  }
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Shared construction points for the guard errors that more than one guard
3
+ * raises. Normalisation downstream is single-seam (`error-lifecycle.ts`); the
4
+ * duplication this module removes was on the construction side, where four
5
+ * guards each hand-rolled their own code + message for the same conditions.
6
+ */
7
+ import { GatewayError } from "../../protocol/gateway-error";
8
+
9
+ /** Thrown when the request's client peer address cannot be established. */
10
+ export function peerAddressUnavailableError(): GatewayError {
11
+ return new GatewayError("admission_unavailable", 503, "client peer address unavailable");
12
+ }
13
+
14
+ /** Thrown when the request body exceeds the endpoint's byte budget. */
15
+ export function bodyTooLargeError(maxBytes: number): GatewayError {
16
+ return new GatewayError(
17
+ "invalid_request",
18
+ 413,
19
+ `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`,
20
+ );
21
+ }
22
+
23
+
@@ -17,15 +17,14 @@
17
17
  * metrics are exempt so an operator can always reach the console that explains
18
18
  * the shedding.
19
19
  */
20
- import { totalmem } from "node:os";
21
20
  import { Elysia } from "elysia";
22
21
  import { GatewayError } from "../../protocol/gateway-error";
23
22
  import type { RedisClient } from "../../persistence/redis";
23
+ import { deriveMemoryHeadroomBytes } from "../../config/resources";
24
+ import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
24
25
  import type { ProxyRequestStateStore } from "../request/state";
25
26
  import { isProxyRequest } from "./body-policy";
26
27
 
27
- /** Free bytes the process must keep below its RSS before it stops accepting. */
28
- const DEFAULT_MIN_FREE_BYTES = 64 * 1024 * 1024;
29
28
  /** How long the shared shed marker outlives the last shed, so peers can see it. */
30
29
  const SHED_MARKER_TTL_SECONDS = 5;
31
30
  /** Fleet-visible marker: set on the first shed so every instance reports the same state. */
@@ -34,7 +33,22 @@ const SHED_RETRY_MS = 1_000;
34
33
 
35
34
  export interface HeadroomGuardDeps {
36
35
  readonly store: ProxyRequestStateStore;
37
- /** Free bytes to keep; defaults to 64 MiB. A smaller box wants a larger value. */
36
+ /**
37
+ * The last sampled memory state, published once per sample by
38
+ * `RuntimeMetricsSampler`.
39
+ *
40
+ * Taken from the sampler rather than measured here: this runs on EVERY
41
+ * request, and `process.memoryUsage()` on the request path is a cost paid
42
+ * per request to answer a question that changes over seconds. When absent
43
+ * (before the first sample) the guard falls back to measuring once — an
44
+ * unsampled process is still a process that can be OOM-killed.
45
+ */
46
+ readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
47
+ /**
48
+ * Free bytes to keep below the resolved limit. Defaults to
49
+ * `MEMORY_HEADROOM_FRACTION` of it, so a 512 MiB container keeps ~102 MiB
50
+ * free rather than the flat 64 MiB a 2 GiB one would keep.
51
+ */
38
52
  readonly minFreeBytes?: number;
39
53
  /**
40
54
  * Optional shared client. Used only to publish a shed marker for visibility;
@@ -45,11 +59,27 @@ export interface HeadroomGuardDeps {
45
59
  }
46
60
 
47
61
  export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
48
- const minFreeBytes = deps.minFreeBytes ?? DEFAULT_MIN_FREE_BYTES;
62
+ const fixedHeadroom = deps.minFreeBytes;
49
63
  return new Elysia()
50
64
  .beforeHandle(({ request }) => {
51
65
  if (!isProxyRequest(request)) return;
52
- if (process.memoryUsage().rss + minFreeBytes <= totalmem()) return;
66
+ // Shed when RSS plus the headroom would exceed this process's LIMIT —
67
+ // not the host's total memory. A container capped at 512 MiB on a 16 GiB
68
+ // host never approached the old `os.totalmem()` threshold, so the guard
69
+ // never fired and the process was OOM-killed instead of shedding.
70
+ //
71
+ // The snapshot is up to one sample interval old. That is deliberate:
72
+ // this is the hottest path in the process, and the condition it guards
73
+ // develops over seconds.
74
+ const snapshot = deps.memorySnapshot?.();
75
+ const limit = snapshot?.limitBytes;
76
+ if (limit === undefined) {
77
+ // No limit discoverable and no sample yet: nothing to shed against.
78
+ return;
79
+ }
80
+ const rss = snapshot?.rssBytes ?? process.memoryUsage().rss;
81
+ const headroom = fixedHeadroom ?? deriveMemoryHeadroomBytes(limit);
82
+ if (rss + headroom <= limit) return;
53
83
  if (deps.redis) {
54
84
  // Fire-and-forget: a failed marker write must not change this request's
55
85
  // outcome, and awaiting it would put Redis on the shed path.
@@ -61,7 +91,11 @@ export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
61
91
  "overloaded",
62
92
  503,
63
93
  "The gateway is at capacity; retry shortly.",
64
- { retryAfterMs: SHED_RETRY_MS, minFreeBytes, activeRequests: deps.store.activeCount() },
94
+ {
95
+ retryAfterMs: SHED_RETRY_MS,
96
+ minFreeBytes: headroom,
97
+ activeRequests: deps.store.activeCount(),
98
+ },
65
99
  );
66
100
  })
67
101
  .as("plugin");
@@ -28,6 +28,7 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
28
28
  import type { RedisClient } from "../../persistence/redis";
29
29
  import type { TelemetryBatchBuffer } from "../../observability/telemetry-buffer";
30
30
  import type { ModelStrikeService } from "../../security/model-abuse";
31
+ import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
31
32
  import {
32
33
  TelemetryPayloadCapture,
33
34
  payloadCaptureKeepsBodies,
@@ -58,6 +59,20 @@ export interface TransportPipelineContext {
58
59
  * decision is local, and Redis is only written for fleet visibility.
59
60
  */
60
61
  readonly redis?: RedisClient | undefined;
62
+ /**
63
+ * The sampled memory state the headroom guard sheds against.
64
+ *
65
+ * Supplied by the runtime sampler. When absent the guard cannot compare RSS
66
+ * against a limit and does not shed — so this must be wired in production,
67
+ * and the sampler must be initialized at boot even before its first tick.
68
+ */
69
+ readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
70
+ /**
71
+ * Whether invoke is switched off for a tenant. Optional: without it the
72
+ * gateway never refuses on the kill switch, which is the pre-change
73
+ * behaviour rather than a failure.
74
+ */
75
+ readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
61
76
  }
62
77
 
63
78
  /**
@@ -84,6 +99,7 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
84
99
  createProcessConcurrencyMiddleware(),
85
100
  createHeadroomGuard({
86
101
  store: context.stateStore,
102
+ ...(context.memorySnapshot === undefined ? {} : { memorySnapshot: context.memorySnapshot }),
87
103
  ...(context.redis === undefined ? {} : { redis: context.redis }),
88
104
  }),
89
105
  createDependencyReadinessMiddleware({
@@ -103,6 +119,9 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
103
119
  db: context.db,
104
120
  stateStore: context.stateStore,
105
121
  ...(context.modelStrikes ? { modelStrikes: context.modelStrikes } : {}),
122
+ ...(context.isInvokeDisabled
123
+ ? { isInvokeDisabled: context.isInvokeDisabled }
124
+ : {}),
106
125
  }),
107
126
  createCanonicalRequestMiddleware({
108
127
  stateStore: context.stateStore,
@@ -181,6 +200,11 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
181
200
  (resolved) => (resolved.ok ? resolved.retentionMs : payloadRetentionMs()),
182
201
  ),
183
202
  ),
203
+ // The early-rejection capture resolves the tenant's capture mode
204
+ // through the same preference path dispatched requests use, instead of
205
+ // capturing bodies under the deployment default regardless of the
206
+ // tenant's none/metadata choice.
207
+ db: context.db,
184
208
  });
185
209
  } else {
186
210
  registerRequestCleanup(rootApp, { stateStore: context.stateStore });
@@ -20,6 +20,7 @@ import {
20
20
  type ModelAbuseOutcome,
21
21
  type ModelStrikeService,
22
22
  } from "../../security/model-abuse";
23
+ import { peerAddressUnavailableError } from "./guard-errors";
23
24
 
24
25
  /**
25
26
  * Whether a thrown error is a *model* rejection the strike layer counts: the
@@ -123,7 +124,7 @@ export function createClientIdentityMiddleware(deps: {
123
124
  ? deps.resolvePeerAddress(request)
124
125
  : (server?.requestIP(request)?.address ?? null);
125
126
  if (!peer)
126
- throw new GatewayError("admission_unavailable", 503, "client peer address unavailable");
127
+ throw peerAddressUnavailableError();
127
128
  const address = resolveClientIdentity(request, deps.trustedProxyBoundary, peer);
128
129
  const state = deps.stateStore.require(request);
129
130
  state.clientIdentity = {
@@ -69,8 +69,20 @@ function capacityRejected(): AdmissionDecision {
69
69
 
70
70
  export class InMemoryAdmissionController implements AdmissionController {
71
71
  private inflight = new Map<string, number>();
72
- snapshotAccountInflight(): ReadonlyMap<string, number> {
73
- return new Map(this.inflight);
72
+ readAccountInflight(
73
+ candidates: readonly RouteCandidate[],
74
+ ): ReadonlyMap<string, number> {
75
+ // Dedupe buckets: one candidate list often repeats a bucket (combo members
76
+ // fan out to several candidates on one account), and the caller only needs
77
+ // one count per bucket to make its ordering decision.
78
+ const snapshot = new Map<string, number>();
79
+ for (const candidate of candidates) {
80
+ const key = admissionKey(candidate);
81
+ if (snapshot.has(key)) continue;
82
+ const value = this.inflight.get(key) ?? 0;
83
+ if (value > 0) snapshot.set(key, value);
84
+ }
85
+ return snapshot;
74
86
  }
75
87
  async admit(candidate: RouteCandidate): Promise<AdmissionDecision> {
76
88
  // Bucket per ACCOUNT, not per provider:model. A provider with N accounts
@@ -166,17 +178,47 @@ export class RedisAdmissionController implements AdmissionController {
166
178
  await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, this.key(reservation.candidate));
167
179
  }
168
180
 
169
- async snapshotAccountInflight(): Promise<ReadonlyMap<string, number>> {
170
- const keys = await this.redis.keys("admission:inflight:*");
171
- if (keys.length === 0) return new Map();
172
- const values = await this.redis.mget(...keys);
181
+ async readAccountInflight(
182
+ candidates: readonly RouteCandidate[],
183
+ ): Promise<ReadonlyMap<string, number>> {
184
+ // Candidate-scoped exact reads replace the fleet-wide `KEYS` + unbounded
185
+ // `MGET` this method used to run on every `plan()` that had any bounded
186
+ // candidate. `KEYS` is O(keyspace) and blocks the single-threaded Redis
187
+ // server; on a shared box the scan was measured as the dominant planning
188
+ // cost. The keys here are exactly the buckets the caller is about to order
189
+ // candidates by, read in bounded sequential chunks.
190
+ //
191
+ // `KEYS` and `SCAN` stay banned on this path by construction: neither can
192
+ // answer "what are the values of these N known keys" in fewer than
193
+ // O(keyspace) server work, and a fleet-wide inflight snapshot has no
194
+ // request-path consumer left. (If one ever appears, it belongs in a
195
+ // maintenance task using `SCAN` with `COUNT`, never here.)
196
+ const buckets = [...new Set(candidates.map(admissionKey))];
197
+ if (buckets.length === 0) return new Map();
198
+ const keys = buckets.map((bucket) => `admission:inflight:${bucket}`);
199
+ const values: number[] = [];
200
+ // Bounded sequential chunks: one huge MGET would move the unbounded
201
+ // payload problem from the KEYS call to the MGET call. 256 keys keeps a
202
+ // single command payload small while the round trips stay proportional to
203
+ // the (deduped, candidate-bounded) request size.
204
+ for (let offset = 0; offset < keys.length; offset += MGET_CHUNK_SIZE) {
205
+ const chunk = keys.slice(offset, offset + MGET_CHUNK_SIZE);
206
+ const raw = (await this.redis.mget(...chunk)) as (string | null)[];
207
+ for (const value of raw) values.push(Number(value ?? 0));
208
+ }
173
209
  const snapshot = new Map<string, number>();
174
- for (const [index, key] of keys.entries()) {
175
- const value = Number(values[index] ?? 0);
210
+ for (const [index, bucket] of buckets.entries()) {
211
+ const value = values[index] ?? 0;
176
212
  if (Number.isFinite(value) && value > 0) {
177
- snapshot.set(key.slice("admission:inflight:".length), value);
213
+ snapshot.set(bucket, value);
178
214
  }
215
+ // The Redis controller's counter keys are TTL self-healing, so a bucket
216
+ // absent from the reply reads as zero and is omitted — the projection
217
+ // both controllers share: positive-finite counts only, unprefixed names.
179
218
  }
180
219
  return snapshot;
181
220
  }
182
221
  }
222
+
223
+ /** Sequential MGET chunk bound for {@link RedisAdmissionController.readAccountInflight}. */
224
+ const MGET_CHUNK_SIZE = 256;
@@ -0,0 +1,49 @@
1
+ // Bounded LRU map: the one shared implementation for routing-layer caches.
2
+
3
+ /**
4
+ * A capacity-bounded LRU `Map`.
5
+ *
6
+ * `get()` refreshes recency; `set()` evicts exactly the least-recently-used
7
+ * entry on overflow. Insertion order is the recency order (head = LRU,
8
+ * tail = MRU), which `Map` maintains for free — so eviction is one
9
+ * `keys().next()` and no bookkeeping arrays.
10
+ *
11
+ * The routing layer previously hand-rolled this shape twice (round-robin
12
+ * state, capability profiles) with subtly different code. One implementation
13
+ * means the bound is provable once: the map's size can never exceed
14
+ * `capacity`, and every entry is reachable through the normal `Map` API the
15
+ * callers already use.
16
+ */
17
+ export class BoundedLruMap<K, V> extends Map<K, V> {
18
+ private readonly capacity: number;
19
+
20
+ constructor(capacity: number) {
21
+ if (!Number.isFinite(capacity) || capacity < 1 || Math.trunc(capacity) !== capacity) {
22
+ throw new Error(`BoundedLruMap capacity must be a positive integer, got ${capacity}`);
23
+ }
24
+ super();
25
+ this.capacity = Math.trunc(capacity);
26
+ }
27
+
28
+ override get(key: K): V | undefined {
29
+ if (!super.has(key)) return undefined;
30
+ // Refresh recency: delete + re-insert moves the entry to the tail.
31
+ const value = super.get(key)!;
32
+ super.delete(key);
33
+ super.set(key, value);
34
+ return value;
35
+ }
36
+
37
+ override set(key: K, value: V): this {
38
+ // An existing key is re-inserted at the tail even when the value is the
39
+ // same object, so a read-modify-write cycle cannot leave a hot entry
40
+ // parked at the head one eviction away from disappearing.
41
+ if (super.has(key)) super.delete(key);
42
+ else if (super.size >= this.capacity) {
43
+ const oldest = super.keys().next().value;
44
+ if (oldest !== undefined) super.delete(oldest);
45
+ }
46
+ super.set(key, value);
47
+ return this;
48
+ }
49
+ }