cartethyia 2.5.10 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (367) hide show
  1. package/.env.example +8 -0
  2. package/dist/dashboard/assets/{BannedPage-Cd30Z0b9.js → BannedPage-l6XMnfr9.js} +1 -1
  3. package/dist/dashboard/assets/BannedPage-l6XMnfr9.js.gz +0 -0
  4. package/dist/dashboard/assets/{CliToolDetailPage-wf5u-ww6.js → CliToolDetailPage-DwOzAW45.js} +1 -1
  5. package/dist/dashboard/assets/CliToolDetailPage-DwOzAW45.js.gz +0 -0
  6. package/dist/dashboard/assets/{CliToolsPage-DykPvuck.js → CliToolsPage-BT3Up3QA.js} +1 -1
  7. package/dist/dashboard/assets/CliToolsPage-BT3Up3QA.js.gz +0 -0
  8. package/dist/dashboard/assets/{CombosPage-DwvSD3vs.js → CombosPage-6kYIUpzR.js} +1 -1
  9. package/dist/dashboard/assets/CombosPage-6kYIUpzR.js.gz +0 -0
  10. package/dist/dashboard/assets/{ConsoleLogPage-CIr7KDex.js → ConsoleLogPage-zRnj1mlR.js} +1 -1
  11. package/dist/dashboard/assets/ConsoleLogPage-zRnj1mlR.js.gz +0 -0
  12. package/dist/dashboard/assets/{CustomizationPage-82X0DDdH.js → CustomizationPage-C7m-H1Rz.js} +1 -1
  13. package/dist/dashboard/assets/CustomizationPage-C7m-H1Rz.js.gz +0 -0
  14. package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js +1 -0
  15. package/dist/dashboard/assets/HealthEventsModal-C8MR_BdB.js.gz +0 -0
  16. package/dist/dashboard/assets/{LoginPage-CS5mPuVF.js → LoginPage-CbphLvjg.js} +1 -1
  17. package/dist/dashboard/assets/LoginPage-CbphLvjg.js.gz +0 -0
  18. package/dist/dashboard/assets/{ModelPicker-C0UCUsVJ.js → ModelPicker-DVny4S5_.js} +1 -1
  19. package/dist/dashboard/assets/ModelPicker-DVny4S5_.js.gz +0 -0
  20. package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js +1 -0
  21. package/dist/dashboard/assets/OverviewPage-DABu2Wgt.js.gz +0 -0
  22. package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js +3 -0
  23. package/dist/dashboard/assets/ProviderDetailPage-C15x9q13.js.gz +0 -0
  24. package/dist/dashboard/assets/{ProvidersPage-vdja5ZPK.js → ProvidersPage-Dfb2896L.js} +1 -1
  25. package/dist/dashboard/assets/ProvidersPage-Dfb2896L.js.gz +0 -0
  26. package/dist/dashboard/assets/{ProxyPage-BB4CNm4-.js → ProxyPage-BaCIdAUt.js} +1 -1
  27. package/dist/dashboard/assets/ProxyPage-BaCIdAUt.js.gz +0 -0
  28. package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js +1 -0
  29. package/dist/dashboard/assets/QuotaPage-BsdoCIw1.js.gz +0 -0
  30. package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js +1 -0
  31. package/dist/dashboard/assets/SettingsPage-KVJFh3Ip.js.gz +0 -0
  32. package/dist/dashboard/assets/{SetupPage-FadUOl9S.js → SetupPage-D4g6vOGc.js} +1 -1
  33. package/dist/dashboard/assets/SetupPage-D4g6vOGc.js.gz +0 -0
  34. package/dist/dashboard/assets/{SortableList-Cfy6_Khv.js → SortableList-C93i14BD.js} +1 -1
  35. package/dist/dashboard/assets/SortableList-C93i14BD.js.gz +0 -0
  36. package/dist/dashboard/assets/{StudioPage-CM92fgJf.js → StudioPage-SgQpymjm.js} +1 -1
  37. package/dist/dashboard/assets/{StudioPage-CM92fgJf.js.gz → StudioPage-SgQpymjm.js.gz} +0 -0
  38. package/dist/dashboard/assets/{UsagePage-BT4MyQeT.js → UsagePage-BFME-eba.js} +2 -2
  39. package/dist/dashboard/assets/UsagePage-BFME-eba.js.gz +0 -0
  40. package/dist/dashboard/assets/{api-keys-DQYLQxpW.js → api-keys-C23S9sgp.js} +1 -1
  41. package/dist/dashboard/assets/api-keys-C23S9sgp.js.gz +0 -0
  42. package/dist/dashboard/assets/{app-B7pVRhJ1.js → app-2KhtHCv9.js} +1 -1
  43. package/dist/dashboard/assets/{app-B7pVRhJ1.js.gz → app-2KhtHCv9.js.gz} +0 -0
  44. package/dist/dashboard/assets/app-CEeaMHOy.css +2 -0
  45. package/dist/dashboard/assets/app-CEeaMHOy.css.gz +0 -0
  46. package/dist/dashboard/assets/{assert-C9FD_JwJ.js → assert-DU6qrMA_.js} +1 -1
  47. package/dist/dashboard/assets/assert-DU6qrMA_.js.gz +0 -0
  48. package/dist/dashboard/assets/{badge-Bvyr7D0u.js → badge-CKsF123d.js} +1 -1
  49. package/dist/dashboard/assets/badge-CKsF123d.js.gz +0 -0
  50. package/dist/dashboard/assets/{banner-DIvKAZih.js → banner-8j7uChA9.js} +1 -1
  51. package/dist/dashboard/assets/banner-8j7uChA9.js.gz +0 -0
  52. package/dist/dashboard/assets/{card-D5_gYIS_.js → card-CdsSlWA3.js} +1 -1
  53. package/dist/dashboard/assets/card-CdsSlWA3.js.gz +0 -0
  54. package/dist/dashboard/assets/{cli-tools-CNqzrfdb.js → cli-tools-BaFUuFFF.js} +1 -1
  55. package/dist/dashboard/assets/cli-tools-BaFUuFFF.js.gz +0 -0
  56. package/dist/dashboard/assets/{clipboard-button-B6vDBtbm.js → clipboard-button-uL5aYLfW.js} +1 -1
  57. package/dist/dashboard/assets/clipboard-button-uL5aYLfW.js.gz +0 -0
  58. package/dist/dashboard/assets/{dialog-CpjrgdwJ.js → dialog-C07hr5_G.js} +1 -1
  59. package/dist/dashboard/assets/dialog-C07hr5_G.js.gz +0 -0
  60. package/dist/dashboard/assets/{drawer-BWmt3b5y.js → drawer-BPO1TcKH.js} +1 -1
  61. package/dist/dashboard/assets/drawer-BPO1TcKH.js.gz +0 -0
  62. package/dist/dashboard/assets/{entry-C_oakV3z.js → entry-Bz_CtDfC.js} +1 -1
  63. package/dist/dashboard/assets/entry-Bz_CtDfC.js.gz +0 -0
  64. package/dist/dashboard/assets/entry-CwPRKk0d.js +4 -0
  65. package/dist/dashboard/assets/entry-CwPRKk0d.js.gz +0 -0
  66. package/dist/dashboard/assets/entry-GJRbmrpA.js +1 -0
  67. package/dist/dashboard/assets/entry-GJRbmrpA.js.gz +0 -0
  68. package/dist/dashboard/assets/{github-badge-RRHHWGxh.js → github-badge-DmQreSDd.js} +1 -1
  69. package/dist/dashboard/assets/github-badge-DmQreSDd.js.gz +0 -0
  70. package/dist/dashboard/assets/{index-Bhey_rg_.js → index-Bblu6erR.js} +2 -2
  71. package/dist/dashboard/assets/index-Bblu6erR.js.gz +0 -0
  72. package/dist/dashboard/assets/{inline-C2_EQ5Uf.js → inline-C9dCYaBZ.js} +1 -1
  73. package/dist/dashboard/assets/inline-C9dCYaBZ.js.gz +0 -0
  74. package/dist/dashboard/assets/{input-BMRlQ450.js → input-CjCnXweJ.js} +1 -1
  75. package/dist/dashboard/assets/input-CjCnXweJ.js.gz +0 -0
  76. package/dist/dashboard/assets/{layout-XR5xP8ew.js → layout-V6y8DAkl.js} +1 -1
  77. package/dist/dashboard/assets/layout-V6y8DAkl.js.gz +0 -0
  78. package/dist/dashboard/assets/{list-row-Cw07dioc.js → list-row-CM1AaG_y.js} +1 -1
  79. package/dist/dashboard/assets/list-row-CM1AaG_y.js.gz +0 -0
  80. package/dist/dashboard/assets/{live-BivGfuUs.js → live-ta6GMb-8.js} +1 -1
  81. package/dist/dashboard/assets/live-ta6GMb-8.js.gz +0 -0
  82. package/dist/dashboard/assets/{model-bans-CmffjLuG.js → model-bans-BmYTmbAk.js} +1 -1
  83. package/dist/dashboard/assets/model-bans-BmYTmbAk.js.gz +0 -0
  84. package/dist/dashboard/assets/{mutation-46P0oWMe.js → mutation-g_fduiNw.js} +1 -1
  85. package/dist/dashboard/assets/mutation-g_fduiNw.js.gz +0 -0
  86. package/dist/dashboard/assets/{numbers-DrdA9oSM.js → numbers-BOevJzdz.js} +1 -1
  87. package/dist/dashboard/assets/numbers-BOevJzdz.js.gz +0 -0
  88. package/dist/dashboard/assets/{query-keys-BFhGpwR1.js → query-keys-UETqfytp.js} +1 -1
  89. package/dist/dashboard/assets/query-keys-UETqfytp.js.gz +0 -0
  90. package/dist/dashboard/assets/{routing-CAb4F37R.js → routing-BPUo-W8d.js} +1 -1
  91. package/dist/dashboard/assets/routing-BPUo-W8d.js.gz +0 -0
  92. package/dist/dashboard/assets/{select-CxVVB__N.js → select-C980c4cz.js} +1 -1
  93. package/dist/dashboard/assets/select-C980c4cz.js.gz +0 -0
  94. package/dist/dashboard/assets/{settings-DNX9sydi.js → settings--3hxSyry.js} +1 -1
  95. package/dist/dashboard/assets/settings--3hxSyry.js.gz +0 -0
  96. package/dist/dashboard/assets/{stack-uPOwM9Hs.js → stack-_gfTdLVB.js} +1 -1
  97. package/dist/dashboard/assets/stack-_gfTdLVB.js.gz +0 -0
  98. package/dist/dashboard/assets/{state-D1Yz79iL.js → state-2c5V4zo7.js} +1 -1
  99. package/dist/dashboard/assets/state-2c5V4zo7.js.gz +0 -0
  100. package/dist/dashboard/assets/{switch-sevsRrRo.js → switch-Dkke7Naf.js} +1 -1
  101. package/dist/dashboard/assets/switch-Dkke7Naf.js.gz +0 -0
  102. package/dist/dashboard/assets/{useMutation-Dem0lXQj.js → useMutation-EDFRkn1i.js} +1 -1
  103. package/dist/dashboard/assets/useMutation-EDFRkn1i.js.gz +0 -0
  104. package/dist/dashboard/index.html +1 -1
  105. package/dist/dashboard/index.html.gz +0 -0
  106. package/dist/dashboard/precompressed-assets.json +51 -51
  107. package/migrations/README.md +52 -52
  108. package/migrations/full-postgres/0020_add_telemetry_event_analytics.sql +77 -0
  109. package/migrations/full-postgres/0021_add_api_key_concurrency_tolerance.sql +13 -0
  110. package/migrations/full-postgres/0022_add_inferhub_config_key.sql +17 -0
  111. package/migrations/full-postgres/0023_drop_api_key_concurrency_tolerance.sql +13 -0
  112. package/migrations/lite-sqlite3/0020_add_telemetry_event_analytics.sql +69 -0
  113. package/migrations/lite-sqlite3/0021_add_api_key_concurrency_tolerance.sql +13 -0
  114. package/migrations/lite-sqlite3/0022_add_inferhub_config_key.sql +17 -0
  115. package/migrations/lite-sqlite3/0023_drop_api_key_concurrency_tolerance.sql +13 -0
  116. package/package.json +96 -95
  117. package/scripts/cli/admin.ts +126 -128
  118. package/scripts/cli/autostart.ts +237 -237
  119. package/scripts/cli/background.ts +120 -110
  120. package/scripts/cli/bin/carte.cjs +8 -8
  121. package/scripts/cli/bin/cartethyia.cjs +167 -167
  122. package/scripts/cli/console-menu.ts +39 -30
  123. package/scripts/cli/controller.ts +145 -0
  124. package/scripts/cli/env.ts +99 -99
  125. package/scripts/cli/gateway.ts +264 -260
  126. package/scripts/cli/instance.ts +234 -224
  127. package/scripts/cli/main.ts +188 -167
  128. package/scripts/cli/platform.ts +64 -64
  129. package/scripts/cli/prompt.ts +217 -79
  130. package/scripts/cli/services.ts +88 -88
  131. package/scripts/cli/setup.ts +445 -445
  132. package/scripts/cli/start.ts +85 -85
  133. package/scripts/cli/startup-signal.ts +23 -23
  134. package/scripts/cli/telemetry.ts +70 -0
  135. package/scripts/cli/tray/index.ts +263 -263
  136. package/scripts/cli/tray/windows.ps1 +107 -107
  137. package/scripts/cli/tray.ts +29 -4
  138. package/scripts/cli/update.ts +39 -39
  139. package/scripts/cli/version.ts +67 -67
  140. package/scripts/cli/wordmark.ts +58 -58
  141. package/src/boot/app.ts +16 -1
  142. package/src/boot/dependencies.ts +128 -25
  143. package/src/config/resources.ts +137 -0
  144. package/src/config/runtime.ts +133 -76
  145. package/src/config/spec.ts +30 -29
  146. package/src/config/telemetry.ts +0 -25
  147. package/src/console/api-keys/bucket-spend.ts +14 -17
  148. package/src/console/api-keys/contracts.ts +410 -410
  149. package/src/console/api-keys/routes.ts +12 -0
  150. package/src/console/audit/store.ts +2 -3
  151. package/src/console/auth/setup-service.ts +8 -1
  152. package/src/console/backup/routes.ts +7 -0
  153. package/src/console/backup/service.ts +463 -463
  154. package/src/console/backup/store.ts +16 -11
  155. package/src/console/backup/validate.ts +1 -4
  156. package/src/console/cli-tools/fs-ops.ts +18 -2
  157. package/src/console/data-management/contracts.ts +1 -8
  158. package/src/console/data-management/operations.ts +61 -48
  159. package/src/console/data-management/routes.ts +4 -2
  160. package/src/console/domain-registration.ts +25 -3
  161. package/src/console/inferhub/client.ts +139 -0
  162. package/src/console/inferhub/key-store.ts +129 -0
  163. package/src/console/inferhub/normalize.ts +347 -0
  164. package/src/console/inferhub/routes.ts +411 -0
  165. package/src/console/inferhub/store.ts +109 -0
  166. package/src/console/inferhub/types.ts +183 -0
  167. package/src/console/observability/contracts.ts +2 -41
  168. package/src/console/observability/store.ts +176 -371
  169. package/src/console/providers/catalog/public-model-store.ts +0 -0
  170. package/src/console/providers/catalog/store.ts +23 -36
  171. package/src/console/quota/account-quota-shared.ts +52 -52
  172. package/src/console/quota/account-quota-tenant.ts +16 -24
  173. package/src/console/quota/refresh.ts +441 -441
  174. package/src/console/routing/pools/import-parse.ts +206 -206
  175. package/src/console/routing/pools/operations.ts +543 -543
  176. package/src/console/routing/pools/probe-verdict.ts +132 -132
  177. package/src/console/routing/pools/routes.ts +181 -181
  178. package/src/console/settings/contracts.ts +42 -40
  179. package/src/console/settings/store.ts +25 -25
  180. package/src/console/share/share-stats.ts +22 -4
  181. package/src/console/share/share-usage.ts +30 -4
  182. package/src/console/shared/cursor.ts +15 -0
  183. package/src/console/updates/routes.ts +12 -0
  184. package/src/main.ts +13 -5
  185. package/src/network/pool/resolver.ts +1104 -1104
  186. package/src/observability/client-name.ts +22 -0
  187. package/src/observability/event-log.ts +5 -29
  188. package/src/observability/event-query.ts +690 -0
  189. package/src/observability/log-ring.ts +26 -2
  190. package/src/observability/log-sanitize.ts +118 -0
  191. package/src/observability/logger.ts +17 -13
  192. package/src/observability/metrics.ts +35 -0
  193. package/src/observability/payload-capture.ts +114 -41
  194. package/src/observability/payload-store.ts +249 -24
  195. package/src/observability/raw-event-writer.ts +118 -0
  196. package/src/observability/runtime-metrics.ts +71 -1
  197. package/src/observability/telemetry-buffer.ts +100 -17
  198. package/src/observability/telemetry-reconciliation.ts +417 -0
  199. package/src/persistence/boot.ts +20 -9
  200. package/src/persistence/db-pg.ts +13 -0
  201. package/src/persistence/db-sqlite.ts +569 -497
  202. package/src/persistence/dialect.ts +3 -2
  203. package/src/persistence/migrate.ts +181 -181
  204. package/src/persistence/schema.ts +1496 -1406
  205. package/src/persistence/shared-cache.ts +5 -1
  206. package/src/persistence/stores/api-key-store.ts +581 -581
  207. package/src/persistence/stores/telemetry-store.ts +124 -63
  208. package/src/protocol/reasoning-text.ts +30 -0
  209. package/src/protocol/response/chat.ts +33 -7
  210. package/src/protocol/response/codex.ts +14 -2
  211. package/src/protocol/response/messages.ts +5 -2
  212. package/src/providers/authentication/oauth-flow-store.ts +3 -10
  213. package/src/providers/bundled-discovery-providers.ts +122 -122
  214. package/src/providers/default-registry.ts +10 -0
  215. package/src/providers/discovery/catalog/catalog-blob.ts +18 -18
  216. package/src/providers/discovery/models-dev-catalog.ts +354 -354
  217. package/src/providers/discovery/probe-phases.ts +43 -7
  218. package/src/providers/discovery/probing-service.ts +70 -32
  219. package/src/providers/integrations/antigravity/antigravity.ts +9 -2
  220. package/src/providers/integrations/buddy/buddy-catalog-shared.ts +85 -17
  221. package/src/providers/integrations/buddy/buddy-chat-shared.ts +934 -934
  222. package/src/providers/integrations/buddy/buddy-checkin.ts +5 -9
  223. package/src/providers/integrations/buddy/buddy-discovery-shared.ts +3 -7
  224. package/src/providers/integrations/buddy/buddy-quota-shared.ts +3 -1
  225. package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +57 -49
  226. package/src/providers/integrations/buddy/codebuddy/provider.ts +180 -180
  227. package/src/providers/integrations/buddy/workbuddy/provider.ts +206 -206
  228. package/src/providers/integrations/buddy/workbuddy/quota.ts +161 -161
  229. package/src/providers/integrations/claude/claude-quota.ts +2 -2
  230. package/src/providers/integrations/claude/claude-user-profiles.ts +3 -2
  231. package/src/providers/integrations/claude/claude.ts +3 -4
  232. package/src/providers/integrations/cline/cline.ts +94 -8
  233. package/src/providers/integrations/codex/codex.ts +61 -37
  234. package/src/providers/integrations/cursor/cursor-catalog.ts +368 -368
  235. package/src/providers/integrations/cursor/cursor-http2.ts +300 -300
  236. package/src/providers/integrations/cursor/cursor-identity.ts +2 -4
  237. package/src/providers/integrations/cursor/cursor-request.ts +11 -0
  238. package/src/providers/integrations/cursor/cursor.ts +29 -17
  239. package/src/providers/integrations/cursor/protocol/framing.ts +129 -129
  240. package/src/providers/integrations/cursor/protocol/messages.ts +2380 -2380
  241. package/src/providers/integrations/cursor/protocol/struct.ts +305 -305
  242. package/src/providers/integrations/cursor/protocol/tool-canonical.ts +275 -0
  243. package/src/providers/integrations/grok/grok-quota.ts +279 -279
  244. package/src/providers/integrations/grok/grok-ratelimit-store.ts +73 -73
  245. package/src/providers/integrations/grok/grok-ratelimit.ts +218 -218
  246. package/src/providers/integrations/hermes.ts +7 -1
  247. package/src/providers/integrations/inferhub.ts +201 -73
  248. package/src/providers/integrations/kilo/kilo.ts +152 -152
  249. package/src/providers/integrations/kiro/kiro-machine-id.ts +2 -4
  250. package/src/providers/integrations/openrouter/provider.ts +129 -129
  251. package/src/providers/integrations/qoder/qoder-quota.ts +193 -192
  252. package/src/providers/integrations/qoder/qoder-realm.ts +106 -106
  253. package/src/providers/integrations/trae/trae-oauth.ts +4 -26
  254. package/src/providers/integrations/trae/trae-quota.ts +389 -384
  255. package/src/providers/integrations/trae/trae-realm.ts +15 -2
  256. package/src/providers/integrations/xiaomi-mimo/think-stream.ts +100 -24
  257. package/src/providers/integrations/zcode/oauth.ts +1 -1
  258. package/src/providers/operations/account-health-service.ts +605 -279
  259. package/src/providers/provider-metadata.ts +12 -1
  260. package/src/providers/provider-model-definition.ts +373 -373
  261. package/src/providers/quota/quota-contracts.ts +3 -7
  262. package/src/providers/search/search-provider.ts +2 -6
  263. package/src/security/admission/contracts.ts +10 -0
  264. package/src/security/admission/in-memory-store.ts +89 -14
  265. package/src/security/admission/lua.ts +191 -182
  266. package/src/security/admission/redis-store.ts +30 -0
  267. package/src/security/admission/service.ts +18 -0
  268. package/src/security/api-key-auth-cache.ts +96 -0
  269. package/src/security/api-key-auth.ts +11 -100
  270. package/src/security/client-router-fingerprint.ts +3 -7
  271. package/src/security/ip-boundary.ts +7 -3
  272. package/src/shared/coerce.ts +67 -0
  273. package/src/shared/hash.ts +6 -0
  274. package/src/shared/time.ts +7 -0
  275. package/src/transport/dispatch/attempt-finalize.ts +26 -138
  276. package/src/transport/dispatch/attempt-loop.ts +103 -81
  277. package/src/transport/dispatch/capture-policy.ts +78 -0
  278. package/src/transport/dispatch/proxy-request.ts +4 -2
  279. package/src/transport/dispatch/streaming/encoder.ts +111 -5
  280. package/src/transport/dispatch/streaming/stream.ts +19 -2
  281. package/src/transport/dispatch/streaming/thinking-loop.ts +300 -0
  282. package/src/transport/failure-policy.ts +27 -0
  283. package/src/transport/guards/abuse-guard.ts +2 -2
  284. package/src/transport/guards/auth-guard.ts +17 -0
  285. package/src/transport/guards/body-policy.ts +3 -2
  286. package/src/transport/guards/concurrency-guard.ts +89 -13
  287. package/src/transport/guards/early-rejection-capture.ts +43 -3
  288. package/src/transport/guards/error-lifecycle.ts +10 -2
  289. package/src/transport/guards/guard-errors.ts +23 -0
  290. package/src/transport/guards/headroom-guard.ts +41 -7
  291. package/src/transport/guards/pipeline.ts +24 -0
  292. package/src/transport/guards/request-context.ts +2 -1
  293. package/src/transport/routing/admission.ts +51 -9
  294. package/src/transport/routing/bounded-lru.ts +49 -0
  295. package/src/transport/routing/candidate-index.ts +88 -0
  296. package/src/transport/routing/reservations.ts +121 -7
  297. package/src/transport/routing/round-robin.ts +10 -22
  298. package/src/transport/routing/route-catalog.ts +26 -13
  299. package/src/transport/routing/route-model.ts +22 -2
  300. package/src/transport/routing/router.ts +53 -31
  301. package/src/transport/surface/chat/encode.ts +2 -3
  302. package/src/transport/surface/chat/parse.ts +2 -1
  303. package/src/transport/surface/dialects.ts +11 -3
  304. package/src/transport/surface/messages/encode.ts +2 -1
  305. package/src/transport/surface/messages/parse.ts +3 -1
  306. package/src/transport/surface/messages/stream.ts +2 -6
  307. package/src/workers/maintenance-lease.ts +200 -0
  308. package/src/workers/scheduler.ts +101 -1
  309. package/dist/dashboard/assets/BannedPage-Cd30Z0b9.js.gz +0 -0
  310. package/dist/dashboard/assets/CliToolDetailPage-wf5u-ww6.js.gz +0 -0
  311. package/dist/dashboard/assets/CliToolsPage-DykPvuck.js.gz +0 -0
  312. package/dist/dashboard/assets/CombosPage-DwvSD3vs.js.gz +0 -0
  313. package/dist/dashboard/assets/ConsoleLogPage-CIr7KDex.js.gz +0 -0
  314. package/dist/dashboard/assets/CustomizationPage-82X0DDdH.js.gz +0 -0
  315. package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js +0 -1
  316. package/dist/dashboard/assets/HealthEventsModal-BYPucTTn.js.gz +0 -0
  317. package/dist/dashboard/assets/LoginPage-CS5mPuVF.js.gz +0 -0
  318. package/dist/dashboard/assets/ModelPicker-C0UCUsVJ.js.gz +0 -0
  319. package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js +0 -1
  320. package/dist/dashboard/assets/OverviewPage-Dk_ew3YU.js.gz +0 -0
  321. package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js +0 -3
  322. package/dist/dashboard/assets/ProviderDetailPage-CAiRn2Q0.js.gz +0 -0
  323. package/dist/dashboard/assets/ProvidersPage-vdja5ZPK.js.gz +0 -0
  324. package/dist/dashboard/assets/ProxyPage-BB4CNm4-.js.gz +0 -0
  325. package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js +0 -1
  326. package/dist/dashboard/assets/QuotaPage-FEc4iNyq.js.gz +0 -0
  327. package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js +0 -1
  328. package/dist/dashboard/assets/SettingsPage-Dlfi5L4N.js.gz +0 -0
  329. package/dist/dashboard/assets/SetupPage-FadUOl9S.js.gz +0 -0
  330. package/dist/dashboard/assets/SortableList-Cfy6_Khv.js.gz +0 -0
  331. package/dist/dashboard/assets/UsagePage-BT4MyQeT.js.gz +0 -0
  332. package/dist/dashboard/assets/api-keys-DQYLQxpW.js.gz +0 -0
  333. package/dist/dashboard/assets/app-CX_Z8cMs.css +0 -2
  334. package/dist/dashboard/assets/app-CX_Z8cMs.css.gz +0 -0
  335. package/dist/dashboard/assets/assert-C9FD_JwJ.js.gz +0 -0
  336. package/dist/dashboard/assets/badge-Bvyr7D0u.js.gz +0 -0
  337. package/dist/dashboard/assets/banner-DIvKAZih.js.gz +0 -0
  338. package/dist/dashboard/assets/card-D5_gYIS_.js.gz +0 -0
  339. package/dist/dashboard/assets/cli-tools-CNqzrfdb.js.gz +0 -0
  340. package/dist/dashboard/assets/clipboard-button-B6vDBtbm.js.gz +0 -0
  341. package/dist/dashboard/assets/dialog-CpjrgdwJ.js.gz +0 -0
  342. package/dist/dashboard/assets/drawer-BWmt3b5y.js.gz +0 -0
  343. package/dist/dashboard/assets/entry-B-gvimrg.js +0 -1
  344. package/dist/dashboard/assets/entry-B-gvimrg.js.gz +0 -0
  345. package/dist/dashboard/assets/entry-C_oakV3z.js.gz +0 -0
  346. package/dist/dashboard/assets/entry-Dj0cHfEc.js +0 -4
  347. package/dist/dashboard/assets/entry-Dj0cHfEc.js.gz +0 -0
  348. package/dist/dashboard/assets/github-badge-RRHHWGxh.js.gz +0 -0
  349. package/dist/dashboard/assets/index-Bhey_rg_.js.gz +0 -0
  350. package/dist/dashboard/assets/inline-C2_EQ5Uf.js.gz +0 -0
  351. package/dist/dashboard/assets/input-BMRlQ450.js.gz +0 -0
  352. package/dist/dashboard/assets/layout-XR5xP8ew.js.gz +0 -0
  353. package/dist/dashboard/assets/list-row-Cw07dioc.js.gz +0 -0
  354. package/dist/dashboard/assets/live-BivGfuUs.js.gz +0 -0
  355. package/dist/dashboard/assets/model-bans-CmffjLuG.js.gz +0 -0
  356. package/dist/dashboard/assets/mutation-46P0oWMe.js.gz +0 -0
  357. package/dist/dashboard/assets/numbers-DrdA9oSM.js.gz +0 -0
  358. package/dist/dashboard/assets/query-keys-BFhGpwR1.js.gz +0 -0
  359. package/dist/dashboard/assets/routing-CAb4F37R.js.gz +0 -0
  360. package/dist/dashboard/assets/select-CxVVB__N.js.gz +0 -0
  361. package/dist/dashboard/assets/settings-DNX9sydi.js.gz +0 -0
  362. package/dist/dashboard/assets/stack-uPOwM9Hs.js.gz +0 -0
  363. package/dist/dashboard/assets/state-D1Yz79iL.js.gz +0 -0
  364. package/dist/dashboard/assets/switch-sevsRrRo.js.gz +0 -0
  365. package/dist/dashboard/assets/useMutation-Dem0lXQj.js.gz +0 -0
  366. package/src/observability/event-compaction.ts +0 -317
  367. package/src/observability/event-day-index.ts +0 -439
@@ -21,8 +21,6 @@ export interface RuntimeSettingsResponse {
21
21
  readonly telemetryPayloads: TelemetryPayloadMode;
22
22
  /** Capture depth for `full` mode; `minimum` when unset. */
23
23
  readonly telemetryPayloadDepth: TelemetryPayloadDepth;
24
- /** How long a captured frame is kept, in minutes; `null` means deployment default. */
25
- readonly telemetryPayloadRetentionMinutes: number | null;
26
24
  readonly privacyMode: PrivacyMode;
27
25
  /** Compacts bulky tool-result text before dispatch (RTK prune). Default false. */
28
26
  readonly rtkPruneEnabled: boolean;
@@ -39,13 +37,45 @@ export interface RuntimeSettingsResponse {
39
37
  * cooldown is still recorded and still recovered by the sweep.
40
38
  */
41
39
  readonly forceDisableCooldown: boolean;
40
+ /**
41
+ * Stops every invoke and route for this tenant while on. The console stays
42
+ * reachable so the switch can be turned back off from the UI that set it.
43
+ */
44
+ readonly invokeDisabled: boolean;
45
+ /**
46
+ * Slack above every key's concurrency limit; `null` means none. The
47
+ * effective ceiling is `maxConcurrentRequests + concurrencyTolerance`.
48
+ */
49
+ readonly concurrencyTolerance: number | null;
42
50
  /** Per-tenant upstream request timeout in seconds; null means deployment default. */
43
51
  readonly upstreamTimeoutSeconds: number | null;
44
- /** Per-tenant telemetry compaction age in days; null means deployment default. */
52
+ /** Captured payload detail lifetime in minutes; null means deployment default. */
45
53
  readonly telemetryWindowMinutes: number | null;
46
54
  /** Per-tenant inline attachment ceiling in bytes; null means deployment default. */
47
55
  readonly imageClampMaxBytes: number | null;
48
56
  readonly updatedAt: string;
57
+ /*
58
+ * What each `null` actually resolves to, in the same units the row stores.
59
+ *
60
+ * A row that says "Deployment default" without the number makes the operator
61
+ * guess, and the guess is the one thing they cannot check from here: the
62
+ * value lives in an env var on a server they are not looking at. Sent in the
63
+ * same response rather than mirrored in the client, because a copied number
64
+ * is right until someone changes the deployment — then it lies quietly.
65
+ */
66
+ readonly defaults: RuntimeSettingsDefaults;
67
+ }
68
+
69
+ /** The deployment-level values a `null` runtime setting falls back to. */
70
+ export interface RuntimeSettingsDefaults {
71
+ /** Seconds; from `CARTETHYIA_UPSTREAM_TIMEOUT_MS`. */
72
+ readonly upstreamTimeoutSeconds: number;
73
+ /** Minutes; what the payload store expires frames after. */
74
+ readonly telemetryWindowMinutes: number;
75
+ /** Bytes; the inline attachment ceiling. */
76
+ readonly imageClampMaxBytes: number;
77
+ /** Days; how long raw event files are kept. */
78
+ readonly telemetryRetentionDays: number;
49
79
  }
50
80
 
51
81
  /**
@@ -92,20 +122,6 @@ export type TelemetryPayloadMode = (typeof TELEMETRY_PAYLOAD_MODES)[number];
92
122
  export const TELEMETRY_PAYLOAD_DEPTHS = ["minimum", "moderate", "maximum"] as const;
93
123
  /** Capture depth for `full` mode accepted by the runtime-settings PATCH. */
94
124
  export type TelemetryPayloadDepth = (typeof TELEMETRY_PAYLOAD_DEPTHS)[number];
95
- /**
96
- * How long a captured payload frame is kept, in whole minutes, or `null` for
97
- * the deployment default.
98
- *
99
- * Only the offered values are accepted, and a value off the list is *rejected*
100
- * rather than clamped: clamping would answer a request for thirty days with
101
- * seven and report success, and an operator who asked for the longer window
102
- * would believe they got it.
103
- */
104
- export const TELEMETRY_PAYLOAD_RETENTION_MINUTES = [15, 30, 60, 360, 720, 1440, 10080] as const;
105
- /** Payload retention in minutes accepted by the runtime-settings PATCH. */
106
- export type TelemetryPayloadRetentionMinutes =
107
- | (typeof TELEMETRY_PAYLOAD_RETENTION_MINUTES)[number]
108
- | null;
109
125
  export const PRIVACY_MODES = ["masked", "full"] as const;
110
126
  /** Provider/model label privacy accepted by the runtime-settings PATCH. */
111
127
  export type PrivacyMode = (typeof PRIVACY_MODES)[number];
@@ -149,19 +165,6 @@ export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
149
165
  if (patch.telemetryPayloadDepth !== undefined && !TELEMETRY_PAYLOAD_DEPTHS.includes(patch.telemetryPayloadDepth)) {
150
166
  throw new ConsoleDomainError("invalid_request", 400, "Invalid telemetryPayloadDepth");
151
167
  }
152
- if (patch.telemetryPayloadRetentionMinutes !== undefined) {
153
- const minutes = patch.telemetryPayloadRetentionMinutes;
154
- if (
155
- minutes !== null &&
156
- !(TELEMETRY_PAYLOAD_RETENTION_MINUTES as readonly number[]).includes(minutes)
157
- ) {
158
- throw new ConsoleDomainError(
159
- "invalid_request",
160
- 400,
161
- "Invalid telemetryPayloadRetentionMinutes",
162
- );
163
- }
164
- }
165
168
  if (patch.privacyMode !== undefined && !PRIVACY_MODES.includes(patch.privacyMode)) {
166
169
  throw new ConsoleDomainError("invalid_request", 400, "Invalid privacyMode");
167
170
  }
@@ -183,10 +186,11 @@ export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
183
186
  detail: patch as Record<string, unknown>,
184
187
  });
185
188
  // Saved preferences are routing changes: `forceDisableCooldown` decides
186
- // candidate eligibility and `tenantConcurrencyLimit` decides admission,
187
- // and both are read from the cached route snapshot. Invalidate after the
188
- // write commits so the next `/v1/*` request uses them instead of the
189
- // revision that was built before the save. A failure here must not fail
189
+ // candidate eligibility, `tenantConcurrencyLimit` decides admission, and
190
+ // `invokeDisabled` decides whether the request is refused at all — all
191
+ // three read from the cached route snapshot. Invalidate after the write
192
+ // commits so the next `/v1/*` request uses them instead of the revision
193
+ // that was built before the save. A failure here must not fail
190
194
  // the save — the row is already correct, and the snapshot rebuilds on
191
195
  // the next read anyway.
192
196
  await Promise.resolve(deps.snapshotInvalidator?.invalidate()).catch(() => undefined);
@@ -206,12 +210,6 @@ export const runtimeUpdateBody = t.Object({
206
210
  responsesReasoningSummary: t.Optional(literalUnion(RESPONSES_REASONING_SUMMARIES)),
207
211
  telemetryPayloads: t.Optional(literalUnion(TELEMETRY_PAYLOAD_MODES)),
208
212
  telemetryPayloadDepth: t.Optional(literalUnion(TELEMETRY_PAYLOAD_DEPTHS)),
209
- telemetryPayloadRetentionMinutes: t.Optional(
210
- t.Union([
211
- t.Null(),
212
- t.Union(TELEMETRY_PAYLOAD_RETENTION_MINUTES.map((minutes) => t.Literal(minutes))),
213
- ]),
214
- ),
215
213
  privacyMode: t.Optional(literalUnion(PRIVACY_MODES)),
216
214
  tenantConcurrencyLimit: t.Optional(t.Union([t.Null(), t.Number()])),
217
215
  thinkingNormalizationEnabled: t.Optional(t.Boolean()),
@@ -234,6 +232,10 @@ export const runtimeUpdateBody = t.Object({
234
232
  * rejected here rather than silently clamped on the way in.
235
233
  */
236
234
  forceDisableCooldown: t.Optional(t.Boolean()),
235
+ invokeDisabled: t.Optional(t.Boolean()),
236
+ concurrencyTolerance: t.Optional(
237
+ t.Union([t.Null(), t.Number({ minimum: 0, maximum: 1024 })]),
238
+ ),
237
239
  upstreamTimeoutSeconds: t.Optional(
238
240
  t.Union([t.Null(), t.Number({ minimum: 5, maximum: 600 })]),
239
241
  ),
@@ -5,13 +5,16 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
5
5
  import { consoleSettings, type ConsoleSettingsPreferences } from "../../persistence/schema";
6
6
  import type { RedisBackend } from "../../persistence/redis";
7
7
  import { bumpSettingsRevision } from "../../persistence/settings-revision";
8
+ import { resolveTelemetryRetentionDays } from "../../config/telemetry";
9
+ import { resolveUpstreamTimeoutMs } from "../../config/timeouts";
10
+ import { payloadRetentionMs } from "../../observability/payload-capture";
11
+ import { IMAGE_CLAMP_MAX_BYTES } from "../../protocol/image-clamp";
8
12
  import {
9
13
  PONYTAIL_LEVELS,
10
14
  RESPONSES_REASONING_SUMMARIES,
11
15
  RTK_LEVELS,
12
16
  TELEMETRY_PAYLOAD_DEPTHS,
13
17
  TELEMETRY_PAYLOAD_MODES,
14
- TELEMETRY_PAYLOAD_RETENTION_MINUTES,
15
18
  type PonyTailLevel,
16
19
  type ResponsesReasoningSummary,
17
20
  type RtkLevel,
@@ -96,7 +99,7 @@ function normalizeUpstreamTimeoutSeconds(value: unknown): number | null {
96
99
  return normalizeBoundedNumber(value, 5, 600);
97
100
  }
98
101
 
99
- /** Source window in minutes: 1 floor, 1440 (one day) ceiling, matching the env var. */
102
+ /** Captured payload detail lifetime in minutes: 1 floor, one-day ceiling. */
100
103
  function normalizeTelemetryWindowMinutes(value: unknown): number | null {
101
104
  return normalizeBoundedNumber(value, 1, 1440);
102
105
  }
@@ -106,23 +109,10 @@ function normalizeImageClampMaxBytes(value: unknown): number | null {
106
109
  return normalizeBoundedNumber(value, 64 * 1024, 100 * 1024 * 1024);
107
110
  }
108
111
 
109
- /**
110
- * How long a captured payload frame is kept, in whole minutes.
111
- *
112
- * Only one of the offered values survives; anything else reads as "deployment
113
- * default" rather than being clamped to the nearest one. A stored value from
114
- * before this list existed would otherwise be silently rewritten to a window
115
- * the operator never chose, and dropping it to `null` is the same outcome the
116
- * operator gets by never having set it.
117
- */
118
- function normalizePayloadRetentionMinutes(value: unknown): number | null {
112
+ /** Concurrency slack: 0–1024, and `null`/absent means no allowance. */
113
+ function normalizeConcurrencyTolerance(value: unknown): number | null {
119
114
  if (value === null || value === undefined) return null;
120
- const parsed = typeof value === "number" ? value : Number(value);
121
- if (!Number.isFinite(parsed)) return null;
122
- const minutes = Math.floor(parsed);
123
- return (TELEMETRY_PAYLOAD_RETENTION_MINUTES as readonly number[]).includes(minutes)
124
- ? minutes
125
- : null;
115
+ return normalizeBoundedNumber(value, 0, 1024);
126
116
  }
127
117
 
128
118
  function mapRuntimeSettingsRow(
@@ -140,9 +130,6 @@ function mapRuntimeSettingsRow(
140
130
  : "detailed",
141
131
  telemetryPayloads: normalizeTelemetryPayloadMode(prefs.telemetryPayloads),
142
132
  telemetryPayloadDepth: normalizeTelemetryPayloadDepth(prefs.telemetryPayloadDepth),
143
- telemetryPayloadRetentionMinutes: normalizePayloadRetentionMinutes(
144
- prefs.telemetryPayloadRetentionMinutes,
145
- ),
146
133
  privacyMode: prefs.privacyMode === "full" ? "full" : "masked",
147
134
  rtkPruneEnabled: prefs.rtkPruneEnabled === true,
148
135
  rtkPruneLevel: normalizeRtkLevel(prefs.rtkPruneLevel),
@@ -156,10 +143,23 @@ function mapRuntimeSettingsRow(
156
143
  ponyTailEnabled: prefs.ponyTailEnabled === true,
157
144
  ponyTailLevel: normalizePonyTailLevel(prefs.ponyTailLevel),
158
145
  forceDisableCooldown: prefs.forceDisableCooldown === true,
146
+ invokeDisabled: prefs.invokeDisabled === true,
147
+ concurrencyTolerance: normalizeConcurrencyTolerance(prefs.concurrencyTolerance),
159
148
  upstreamTimeoutSeconds: normalizeUpstreamTimeoutSeconds(prefs.upstreamTimeoutSeconds),
160
149
  telemetryWindowMinutes: normalizeTelemetryWindowMinutes(prefs.telemetryWindowMinutes),
161
150
  imageClampMaxBytes: normalizeImageClampMaxBytes(prefs.imageClampMaxBytes),
162
151
  updatedAt,
152
+ /*
153
+ * Resolved at request time, not at module load: all four come from env the
154
+ * operator can change without rebuilding, and a cached value would keep
155
+ * quoting a default the deployment no longer has.
156
+ */
157
+ defaults: {
158
+ upstreamTimeoutSeconds: Math.round(resolveUpstreamTimeoutMs() / 1000),
159
+ telemetryWindowMinutes: Math.round(payloadRetentionMs() / 60_000),
160
+ imageClampMaxBytes: IMAGE_CLAMP_MAX_BYTES,
161
+ telemetryRetentionDays: resolveTelemetryRetentionDays(),
162
+ },
163
163
  };
164
164
  }
165
165
 
@@ -192,10 +192,6 @@ export class DrizzleRuntimeSettingsStore implements RuntimeSettingsStore {
192
192
  patchPrefs.responsesReasoningSummary = patch.responsesReasoningSummary;
193
193
  if (patch.telemetryPayloads !== undefined) patchPrefs.telemetryPayloads = patch.telemetryPayloads;
194
194
  if (patch.telemetryPayloadDepth !== undefined) patchPrefs.telemetryPayloadDepth = patch.telemetryPayloadDepth;
195
- if (patch.telemetryPayloadRetentionMinutes !== undefined)
196
- patchPrefs.telemetryPayloadRetentionMinutes = normalizePayloadRetentionMinutes(
197
- patch.telemetryPayloadRetentionMinutes,
198
- );
199
195
  if (patch.privacyMode !== undefined) patchPrefs.privacyMode = patch.privacyMode;
200
196
  if (patch.rtkPruneEnabled !== undefined) patchPrefs.rtkPruneEnabled = patch.rtkPruneEnabled;
201
197
  if (patch.rtkPruneLevel !== undefined) patchPrefs.rtkPruneLevel = patch.rtkPruneLevel;
@@ -203,6 +199,10 @@ export class DrizzleRuntimeSettingsStore implements RuntimeSettingsStore {
203
199
  if (patch.ponyTailLevel !== undefined) patchPrefs.ponyTailLevel = patch.ponyTailLevel;
204
200
  if (patch.forceDisableCooldown !== undefined)
205
201
  patchPrefs.forceDisableCooldown = patch.forceDisableCooldown === true;
202
+ if (patch.invokeDisabled !== undefined)
203
+ patchPrefs.invokeDisabled = patch.invokeDisabled === true;
204
+ if (patch.concurrencyTolerance !== undefined)
205
+ patchPrefs.concurrencyTolerance = normalizeConcurrencyTolerance(patch.concurrencyTolerance);
206
206
  if (patch.upstreamTimeoutSeconds !== undefined)
207
207
  patchPrefs.upstreamTimeoutSeconds = normalizeUpstreamTimeoutSeconds(patch.upstreamTimeoutSeconds);
208
208
  if (patch.telemetryWindowMinutes !== undefined)
@@ -12,7 +12,9 @@
12
12
 
13
13
  import { createHash } from "node:crypto";
14
14
  import { maskClientIp } from "../../observability/redaction";
15
- import { queryTelemetryEvents, type TelemetryEventRecord } from "../../observability/event-log";
15
+ import { type TelemetryEventRecord } from "../../observability/event-log";
16
+ import { queryTelemetryEvents as querySqlEvents } from "../../observability/event-query";
17
+ import type { CartethyiaDatabase } from "../../persistence/db-handle";
16
18
  import { isGatewayError } from "../../observability/telemetry-status";
17
19
  import type { RedisClient } from "../../persistence/redis";
18
20
  import { cachedRead } from "../shared/read-through-cache";
@@ -259,18 +261,22 @@ function shareFamilyStatsKey(
259
261
  return `share:v1:stats:${tenantId}:${hash}`;
260
262
  }
261
263
 
264
+ /** Cap on the bounded SQL read a family rollup performs. */
265
+ const MAX_FAMILY_EVENTS = 5_000;
266
+
262
267
  interface ShareStatsPortDeps {
268
+ readonly db: CartethyiaDatabase;
263
269
  readonly redis?: RedisClient | undefined;
264
270
  }
265
271
 
266
- export function createShareStatsPort(deps: ShareStatsPortDeps = {}): ShareStatsPort {
272
+ export function createShareStatsPort(deps: ShareStatsPortDeps): ShareStatsPort {
267
273
  return {
268
274
  async getFamilyStats(tenantId, keyIds, recipients, allowedModels) {
269
275
  if (keyIds.length === 0) return emptyStats(recipients);
270
276
  const cached = await cachedRead<Omit<ShareFamilyStats, "recipients">>(
271
277
  deps.redis,
272
278
  shareFamilyStatsKey(tenantId, keyIds, allowedModels),
273
- () => computeFamilyStatsBody(tenantId, keyIds, allowedModels),
279
+ () => computeFamilyStatsBody(deps.db, tenantId, keyIds, allowedModels),
274
280
  SHARE_STATS_CACHE_TTL_SECONDS,
275
281
  );
276
282
  return { ...cached, recipients };
@@ -280,6 +286,7 @@ export function createShareStatsPort(deps: ShareStatsPortDeps = {}): ShareStatsP
280
286
 
281
287
  /** The telemetry-derived remainder of a family rollup; `recipients` is Postgres-derived. */
282
288
  async function computeFamilyStatsBody(
289
+ db: CartethyiaDatabase,
283
290
  tenantId: string,
284
291
  keyIds: readonly string[],
285
292
  allowedModels: readonly string[] | undefined,
@@ -302,7 +309,18 @@ async function computeFamilyStatsBody(
302
309
  // would list exactly the invalid names an abuser probed. Filtering before
303
310
  // the top-50 cut means refused names cannot hide a model the recipient may
304
311
  // use.
305
- const events = await queryTelemetryEvents({ tenantId, apiKeyIds: [...keyIds] });
312
+ //
313
+ // Bounded by the retention floor, not by the oldest counter: `totalTokens`
314
+ // is all-time, so cutting at `monthStart` would silently drop the rows that
315
+ // make it differ from `monthTokens`. Retention already caps the window, and
316
+ // the rows are read newest-first under a page cap, so a very large family
317
+ // cannot pull the whole table.
318
+ const page = await querySqlEvents(db, {
319
+ tenantId,
320
+ apiKeyIds: [...keyIds],
321
+ limit: MAX_FAMILY_EVENTS,
322
+ });
323
+ const events = page.events;
306
324
  const rollup = rollupFamily(events, {
307
325
  dayStart: dayStart.getTime(),
308
326
  monthStart: monthStart.getTime(),
@@ -3,7 +3,8 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
3
3
  import type { ApiKeyRecord } from "../../persistence/stores/api-key-store";
4
4
  import { CachedPreferencesReader, DrizzlePreferencesReader } from "../../persistence/tenant-preferences";
5
5
  import { maskClientIp } from "../../observability/redaction";
6
- import { queryTelemetryEvents, type TelemetryEventRecord } from "../../observability/event-log";
6
+ import { type TelemetryEventRecord } from "../../observability/event-log";
7
+ import { queryTelemetryEvents as querySqlEvents } from "../../observability/event-query";
7
8
  import { isGatewayError } from "../../observability/telemetry-status";
8
9
  import type { RedisClient } from "../../persistence/redis";
9
10
  import { cachedRead } from "../shared/read-through-cache";
@@ -114,6 +115,15 @@ function shareKeyDetailCacheKey(tenantId: string, childKeyId: string, hideClient
114
115
  return `share:v1:detail:${tenantId}:${childKeyId}:${hideClientIp ? "m" : "r"}`;
115
116
  }
116
117
 
118
+ /**
119
+ * Caps on the two SQL event reads a share page performs.
120
+ *
121
+ * Both pages render a bounded list (a rolling window of recent requests), so
122
+ * reading past the cap would fetch rows no element of the response shows.
123
+ */
124
+ const MAX_SUMMARY_EVENTS = 2_000;
125
+ const MAX_ACTIVITY_EVENTS = 2_000;
126
+
117
127
  interface ShareUsagePortDeps {
118
128
  readonly db: CartethyiaDatabase;
119
129
  readonly redis?: RedisClient | undefined;
@@ -128,7 +138,12 @@ export function createShareUsagePort(deps: ShareUsagePortDeps): ShareActivityPor
128
138
  const childIds = children.map((child) => child.id);
129
139
  const todayStart = utcDayStart(new Date());
130
140
  const [todayEvents, lifetimeRows, hideClientIp] = await Promise.all([
131
- queryTelemetryEvents({ tenantId, apiKeyIds: childIds, since: todayStart }),
141
+ // Today's per-key totals as one GROUP BY instead of reading every
142
+ // event of the family. `addEventUsage` needs per-event fields, so the
143
+ // groups are projected into the same shape the loop consumes.
144
+ querySqlEvents(db, { tenantId, apiKeyIds: childIds, since: todayStart, limit: MAX_SUMMARY_EVENTS }).then(
145
+ (page) => page.events,
146
+ ),
132
147
  db
133
148
  .select({
134
149
  apiKeyId: telemetryUsageTotals.entityId,
@@ -184,7 +199,7 @@ export function createShareUsagePort(deps: ShareUsagePortDeps): ShareActivityPor
184
199
  return cachedRead<SharedKeyActivityDetail>(
185
200
  redis,
186
201
  shareKeyDetailCacheKey(tenantId, childKeyId, hideClientIp),
187
- () => computeSharedKeyDetail(tenantId, childKeyId, hideClientIp),
202
+ () => computeSharedKeyDetail(db, tenantId, childKeyId, hideClientIp),
188
203
  SHARE_STATS_CACHE_TTL_SECONDS,
189
204
  );
190
205
  },
@@ -197,12 +212,23 @@ export function createShareUsagePort(deps: ShareUsagePortDeps): ShareActivityPor
197
212
  * resolved by the caller so the cache key can separate the two variants.
198
213
  */
199
214
  async function computeSharedKeyDetail(
215
+ db: CartethyiaDatabase,
200
216
  tenantId: string,
201
217
  childKeyId: string,
202
218
  hideClientIp: boolean,
203
219
  ): Promise<SharedKeyActivityDetail> {
204
220
  const todayStart = utcDayStart(new Date()).getTime();
205
- const events = await queryTelemetryEvents({ tenantId, apiKeyIds: [childKeyId] });
221
+ // Bounded SQL read: this page shows a 24-hour chart plus today's counters,
222
+ // so anything older than the hourly window contributes nothing. The old
223
+ // shape read the whole retained window for one child key.
224
+ const windowStart = new Date(todayStart - 24 * 60 * 60_000);
225
+ const page = await querySqlEvents(db, {
226
+ tenantId,
227
+ apiKeyIds: [childKeyId],
228
+ since: windowStart,
229
+ limit: MAX_ACTIVITY_EVENTS,
230
+ });
231
+ const events = page.events;
206
232
  const models = new Map<string, {
207
233
  providerId: string | null;
208
234
  modelId: string;
@@ -149,3 +149,18 @@ export function decodeDatedCursor(cursor: string | undefined): DatedCursor | und
149
149
  if (Number.isNaN(new Date(parsed.createdAt).getTime())) return undefined;
150
150
  return parsed;
151
151
  }
152
+
153
+ /**
154
+ * Trim a `limit + 1` keyset query result and report whether it overflowed.
155
+ * Callers keep cursor encoding specific to their public page contract.
156
+ */
157
+ export function trimKeysetPage<T>(
158
+ rows: readonly T[],
159
+ limit: number,
160
+ ): { readonly rows: readonly T[]; readonly hasMore: boolean } {
161
+ const hasMore = rows.length > limit;
162
+ return {
163
+ rows: hasMore ? rows.slice(0, limit) : rows,
164
+ hasMore,
165
+ };
166
+ }
@@ -12,6 +12,17 @@ const UPDATE_SAMPLE_INTERVAL_MS = 60_000;
12
12
  const HIGH_MEMORY_PERCENT = 85;
13
13
  const HIGH_MEMORY_CLEAR_PERCENT = 80;
14
14
  const MEMORY_GROWTH_WINDOW_MS = 60_000;
15
+ /**
16
+ * Absolute RSS growth required before a growth alert is raised, alongside the
17
+ * ratio below.
18
+ *
19
+ * The ratio alone reported a 2607% "sustained growth" alert for what was a
20
+ * baseline jump: the window's oldest sample predated an idle reclaim, so a
21
+ * gateway that had just given memory back looked like it had leaked 27x. A
22
+ * leak worth an operator's attention is measured in bytes, not multiples —
23
+ * 20MiB to 24MiB is a 20% ratio and means nothing.
24
+ */
25
+ const MEMORY_GROWTH_MIN_BYTES = 64 * 1024 * 1024;
15
26
  const MEMORY_GROWTH_RATIO = 1.2;
16
27
  const MAX_MEMORY_SAMPLES = 8;
17
28
  const RELEASES_URL = "https://github.com/risunCodex/Cartethyia/releases";
@@ -50,6 +61,7 @@ export function memoryNotifications(
50
61
  const oldest = samples.find((sample) => latest.at - sample.at >= MEMORY_GROWTH_WINDOW_MS);
51
62
  const growth = oldest !== undefined &&
52
63
  latest.rss >= oldest.rss * MEMORY_GROWTH_RATIO &&
64
+ latest.rss - oldest.rss >= MEMORY_GROWTH_MIN_BYTES &&
53
65
  samples.filter((sample) => sample.at >= oldest.at).every((sample, index, window) =>
54
66
  index === 0 || sample.rss >= (window[index - 1]?.rss ?? sample.rss),
55
67
  );
package/src/main.ts CHANGED
@@ -4,7 +4,13 @@ import "./boot/elysia-typebox-bridge";
4
4
  import { createGatewayApp, createGatewayShell } from "./boot/app";
5
5
  import { bootstrap } from "./boot/lifecycle";
6
6
  import type { CartethyiaBoot } from "./boot/lifecycle";
7
- import { resolveDrainToken, resolveIdleTimeout, resolveMaxBodyBytes, resolvePort } from "./config/runtime";
7
+ import {
8
+ resolveDrainToken,
9
+ resolveIdleTimeout,
10
+ resolveMaxBodyBytes,
11
+ resolvePort,
12
+ resolveProcessTopology,
13
+ } from "./config/runtime";
8
14
  import { Manifest } from "elysia";
9
15
  import { log } from "./observability/logger";
10
16
 
@@ -116,10 +122,12 @@ if (boot) {
116
122
  // indefinitely. 60 s aligns with the DB pool idle timeout so all
117
123
  // long-idle resources reap together.
118
124
  idleTimeout: resolveIdleTimeout(),
119
- // Enable Bun's SO_REUSEPORT load-balancing so multi-worker deploys
120
- // (or a second omp process) can share the same port without a
121
- // reverse proxy in front. Silent no-op on platforms that lack it.
122
- reusePort: true,
125
+ // SO_REUSEPORT load-balancing, so several processes can share the port
126
+ // without a reverse proxy. Tied to the declared topology rather than
127
+ // left always-on: with one process declared, enabling it lets an
128
+ // unrelated process on the same port silently take half the traffic —
129
+ // which is worse than a bind error, because nothing reports it.
130
+ reusePort: resolveProcessTopology().multiProcess,
123
131
  // 8 MiB cap on request bodies keeps a rogue client from allocating
124
132
  // gigabytes of memory in the ingress path before validation rejects.
125
133
  maxRequestBodySize: resolveMaxBodyBytes(),