cartethyia 2.5.4 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (395) hide show
  1. package/.env.example +13 -4
  2. package/README.md +2 -2
  3. package/dist/dashboard/assets/{BannedPage-C2orBsI2.js → BannedPage-DE2607Tb.js} +1 -1
  4. package/dist/dashboard/assets/BannedPage-DE2607Tb.js.gz +0 -0
  5. package/dist/dashboard/assets/CliToolDetailPage-Duv6mfrl.js +1 -0
  6. package/dist/dashboard/assets/CliToolDetailPage-Duv6mfrl.js.gz +0 -0
  7. package/dist/dashboard/assets/{CliToolsPage-BV6pxVQo.js → CliToolsPage-C2PNYWzk.js} +1 -1
  8. package/dist/dashboard/assets/CliToolsPage-C2PNYWzk.js.gz +0 -0
  9. package/dist/dashboard/assets/CombosPage-TXP9OfAM.js +1 -0
  10. package/dist/dashboard/assets/CombosPage-TXP9OfAM.js.gz +0 -0
  11. package/dist/dashboard/assets/{ConsoleLogPage-CW-wPmJ6.js → ConsoleLogPage-B-9xnAQN.js} +1 -1
  12. package/dist/dashboard/assets/ConsoleLogPage-B-9xnAQN.js.gz +0 -0
  13. package/dist/dashboard/assets/{CustomizationPage-0p8uX7vB.js → CustomizationPage-xeax9nJt.js} +1 -1
  14. package/dist/dashboard/assets/CustomizationPage-xeax9nJt.js.gz +0 -0
  15. package/dist/dashboard/assets/{HealthEventsModal-BjqSMfko.js → HealthEventsModal-Df4jjaHJ.js} +1 -1
  16. package/dist/dashboard/assets/HealthEventsModal-Df4jjaHJ.js.gz +0 -0
  17. package/dist/dashboard/assets/{LoginPage-BvzCx8Kr.js → LoginPage-Dyojsm84.js} +1 -1
  18. package/dist/dashboard/assets/LoginPage-Dyojsm84.js.gz +0 -0
  19. package/dist/dashboard/assets/ModelPicker-C7pDb41R.js +1 -0
  20. package/dist/dashboard/assets/ModelPicker-C7pDb41R.js.gz +0 -0
  21. package/dist/dashboard/assets/OverviewPage-BJfg4ow6.js +1 -0
  22. package/dist/dashboard/assets/OverviewPage-BJfg4ow6.js.gz +0 -0
  23. package/dist/dashboard/assets/ProviderDetailPage-ChCX1gmT.js +3 -0
  24. package/dist/dashboard/assets/ProviderDetailPage-ChCX1gmT.js.gz +0 -0
  25. package/dist/dashboard/assets/{ProvidersPage-Js9jUPSb.js → ProvidersPage-DeWSOEUO.js} +1 -1
  26. package/dist/dashboard/assets/ProvidersPage-DeWSOEUO.js.gz +0 -0
  27. package/dist/dashboard/assets/ProxyPage-CkpWln2S.js +7 -0
  28. package/dist/dashboard/assets/ProxyPage-CkpWln2S.js.gz +0 -0
  29. package/dist/dashboard/assets/QuotaPage-B3VOgQmt.js +1 -0
  30. package/dist/dashboard/assets/QuotaPage-B3VOgQmt.js.gz +0 -0
  31. package/dist/dashboard/assets/SettingsPage-C_klIZom.js +1 -0
  32. package/dist/dashboard/assets/SettingsPage-C_klIZom.js.gz +0 -0
  33. package/dist/dashboard/assets/{SetupPage-DDbMQsoo.js → SetupPage-BYph-InK.js} +1 -1
  34. package/dist/dashboard/assets/SetupPage-BYph-InK.js.gz +0 -0
  35. package/dist/dashboard/assets/{SortableList-CB3Z7VTf.js → SortableList-CvyQCYWJ.js} +1 -1
  36. package/dist/dashboard/assets/SortableList-CvyQCYWJ.js.gz +0 -0
  37. package/dist/dashboard/assets/{StudioPage-D95UCPHI.js → StudioPage-Cm1JYilI.js} +3 -3
  38. package/dist/dashboard/assets/StudioPage-Cm1JYilI.js.gz +0 -0
  39. package/dist/dashboard/assets/UsagePage-DlFnc3_S.js +56 -0
  40. package/dist/dashboard/assets/UsagePage-DlFnc3_S.js.gz +0 -0
  41. package/dist/dashboard/assets/api-keys-DQYLQxpW.js +1 -0
  42. package/dist/dashboard/assets/api-keys-DQYLQxpW.js.gz +0 -0
  43. package/dist/dashboard/assets/app-CX_Z8cMs.css +2 -0
  44. package/dist/dashboard/assets/app-CX_Z8cMs.css.gz +0 -0
  45. package/dist/dashboard/assets/{assert-Chi7pCnt.js → assert-C9FD_JwJ.js} +1 -1
  46. package/dist/dashboard/assets/assert-C9FD_JwJ.js.gz +0 -0
  47. package/dist/dashboard/assets/{badge-D4pQapKj.js → badge-CR72dasT.js} +1 -1
  48. package/dist/dashboard/assets/badge-CR72dasT.js.gz +0 -0
  49. package/dist/dashboard/assets/{banner-DH6cg-_B.js → banner-CI9_eSfO.js} +1 -1
  50. package/dist/dashboard/assets/banner-CI9_eSfO.js.gz +0 -0
  51. package/dist/dashboard/assets/{card-BKqlUOe4.js → card-DS7CQEsO.js} +1 -1
  52. package/dist/dashboard/assets/card-DS7CQEsO.js.gz +0 -0
  53. package/dist/dashboard/assets/{cli-tools-fTgljTGA.js → cli-tools-CNqzrfdb.js} +1 -1
  54. package/dist/dashboard/assets/cli-tools-CNqzrfdb.js.gz +0 -0
  55. package/dist/dashboard/assets/{clipboard-button-D5Ys_sM0.js → clipboard-button-611pSsaX.js} +1 -1
  56. package/dist/dashboard/assets/clipboard-button-611pSsaX.js.gz +0 -0
  57. package/dist/dashboard/assets/{fingerprint-pattern-BWuHtbW4.js → concurrency-BDF0ok7y.js} +1 -1
  58. package/dist/dashboard/assets/concurrency-BDF0ok7y.js.gz +0 -0
  59. package/dist/dashboard/assets/dialog-k37exjNt.js +1 -0
  60. package/dist/dashboard/assets/dialog-k37exjNt.js.gz +0 -0
  61. package/dist/dashboard/assets/drawer-CX7N8ucR.js +1 -0
  62. package/dist/dashboard/assets/drawer-CX7N8ucR.js.gz +0 -0
  63. package/dist/dashboard/assets/{entry-CQBr6U9c.js → entry-B9Xvtnfj.js} +1 -1
  64. package/dist/dashboard/assets/entry-B9Xvtnfj.js.gz +0 -0
  65. package/dist/dashboard/assets/entry-Bog0p925.js +4 -0
  66. package/dist/dashboard/assets/entry-Bog0p925.js.gz +0 -0
  67. package/dist/dashboard/assets/entry-CTP8AyiS.js +1 -0
  68. package/dist/dashboard/assets/entry-CTP8AyiS.js.gz +0 -0
  69. package/dist/dashboard/assets/{github-badge-BlptL8Uf.js → github-badge-sltk1qUi.js} +1 -1
  70. package/dist/dashboard/assets/github-badge-sltk1qUi.js.gz +0 -0
  71. package/dist/dashboard/assets/{index-js6TJvdr.js → index-CZQIrdoX.js} +2 -2
  72. package/dist/dashboard/assets/index-CZQIrdoX.js.gz +0 -0
  73. package/dist/dashboard/assets/{inline-DUWDSmDV.js → inline-BhY2sSnO.js} +1 -1
  74. package/dist/dashboard/assets/inline-BhY2sSnO.js.gz +0 -0
  75. package/dist/dashboard/assets/{input-LVjgZq4f.js → input-V1umFiBk.js} +1 -1
  76. package/dist/dashboard/assets/input-V1umFiBk.js.gz +0 -0
  77. package/dist/dashboard/assets/layout-BBOYycyS.js +1 -0
  78. package/dist/dashboard/assets/layout-BBOYycyS.js.gz +0 -0
  79. package/dist/dashboard/assets/{list-row-DupvLfDx.js → list-row-1zzZYM0B.js} +1 -1
  80. package/dist/dashboard/assets/list-row-1zzZYM0B.js.gz +0 -0
  81. package/dist/dashboard/assets/{live-C4pEJUqI.js → live-BivGfuUs.js} +1 -1
  82. package/dist/dashboard/assets/live-BivGfuUs.js.gz +0 -0
  83. package/dist/dashboard/assets/{model-bans-BRiuufkB.js → model-bans-CmffjLuG.js} +1 -1
  84. package/dist/dashboard/assets/model-bans-CmffjLuG.js.gz +0 -0
  85. package/dist/dashboard/assets/{mutation-B4ROwk8L.js → mutation-46P0oWMe.js} +1 -1
  86. package/dist/dashboard/assets/mutation-46P0oWMe.js.gz +0 -0
  87. package/dist/dashboard/assets/numbers-DkfMR5Qs.js +2 -0
  88. package/dist/dashboard/assets/numbers-DkfMR5Qs.js.gz +0 -0
  89. package/dist/dashboard/assets/{query-keys-gmjzC3RS.js → query-keys-BFhGpwR1.js} +1 -1
  90. package/dist/dashboard/assets/query-keys-BFhGpwR1.js.gz +0 -0
  91. package/dist/dashboard/assets/{routing-C7tBT5o-.js → routing-CAb4F37R.js} +1 -1
  92. package/dist/dashboard/assets/routing-CAb4F37R.js.gz +0 -0
  93. package/dist/dashboard/assets/{select-WUCXorXw.js → select-BPpSsBgL.js} +1 -1
  94. package/dist/dashboard/assets/select-BPpSsBgL.js.gz +0 -0
  95. package/dist/dashboard/assets/{settings-C_zKl1GS.js → settings-DNX9sydi.js} +1 -1
  96. package/dist/dashboard/assets/settings-DNX9sydi.js.gz +0 -0
  97. package/dist/dashboard/assets/{stack-_3O0F8Xn.js → stack-BHTfXdz_.js} +1 -1
  98. package/dist/dashboard/assets/stack-BHTfXdz_.js.gz +0 -0
  99. package/dist/dashboard/assets/{state-BgTxHcJS.js → state-mPDRlQkl.js} +1 -1
  100. package/dist/dashboard/assets/state-mPDRlQkl.js.gz +0 -0
  101. package/dist/dashboard/assets/{switch-DEjTtOWf.js → switch-DOmJxg7H.js} +1 -1
  102. package/dist/dashboard/assets/switch-DOmJxg7H.js.gz +0 -0
  103. package/dist/dashboard/assets/tokens-COzario9.js +1 -0
  104. package/dist/dashboard/assets/tokens-COzario9.js.gz +0 -0
  105. package/dist/dashboard/assets/use-modal-focus-By7oJoGd.js +1 -0
  106. package/dist/dashboard/assets/use-modal-focus-By7oJoGd.js.gz +0 -0
  107. package/dist/dashboard/assets/{useMutation-BWSC0UAS.js → useMutation-Dem0lXQj.js} +1 -1
  108. package/dist/dashboard/assets/useMutation-Dem0lXQj.js.gz +0 -0
  109. package/dist/dashboard/index.html +1 -1
  110. package/dist/dashboard/index.html.gz +0 -0
  111. package/dist/dashboard/precompressed-assets.json +54 -54
  112. package/migrations/README.md +52 -52
  113. package/migrations/lite-sqlite3/0000_baseline.sql +2 -2
  114. package/package.json +95 -95
  115. package/scripts/cli/admin.ts +125 -0
  116. package/scripts/cli/{ui/background.ts → background.ts} +1 -1
  117. package/scripts/cli/{command/console-menu.ts → console-menu.ts} +275 -244
  118. package/scripts/cli/doctor.ts +37 -0
  119. package/scripts/cli/{support/env.ts → env.ts} +99 -99
  120. package/scripts/cli/{gateway/spawn.ts → gateway.ts} +112 -24
  121. package/scripts/cli/instance.ts +224 -0
  122. package/scripts/cli/main.ts +61 -18
  123. package/scripts/cli/{support/platform.ts → platform.ts} +64 -64
  124. package/scripts/cli/port.ts +100 -0
  125. package/scripts/cli/{support/services.ts → services.ts} +88 -88
  126. package/scripts/cli/{command/setup.ts → setup.ts} +35 -34
  127. package/scripts/cli/{command/start.ts → start.ts} +10 -5
  128. package/scripts/cli/{support/startup-signal.ts → startup-signal.ts} +23 -23
  129. package/scripts/cli/tray/index.ts +62 -12
  130. package/scripts/cli/{command/tray.ts → tray.ts} +208 -208
  131. package/scripts/cli/{command/update.ts → update.ts} +39 -39
  132. package/scripts/cli/version.ts +67 -0
  133. package/scripts/cli/{ui/wordmark.ts → wordmark.ts} +58 -58
  134. package/src/boot/app.ts +697 -693
  135. package/src/boot/dependencies.ts +647 -646
  136. package/src/boot/timeout.ts +2 -13
  137. package/src/config/cooldowns.ts +186 -138
  138. package/src/config/network.ts +8 -14
  139. package/src/config/providers.ts +7 -10
  140. package/src/config/runtime.ts +76 -76
  141. package/src/config/security.ts +15 -0
  142. package/src/config/spec.ts +20 -30
  143. package/src/config/telemetry.ts +0 -20
  144. package/src/config/timeouts.ts +0 -6
  145. package/src/console/admin/model-abuse/contracts.ts +28 -6
  146. package/src/console/admin/model-abuse/routes.ts +56 -3
  147. package/src/console/api-keys/contracts.ts +410 -414
  148. package/src/console/api-keys/routes.ts +881 -880
  149. package/src/console/auth/credentials.ts +3 -3
  150. package/src/console/auth/setup-service.ts +2 -2
  151. package/src/console/backup/export-endpoint.ts +6 -2
  152. package/src/console/backup/nine-router.ts +2 -5
  153. package/src/console/backup/service.ts +4 -0
  154. package/src/console/backup/store.ts +192 -29
  155. package/src/console/backup/validate.ts +3 -45
  156. package/src/console/cli-tools/fs-ops.ts +487 -487
  157. package/src/console/console-router-guards.ts +1 -1
  158. package/src/console/data-management/contracts.ts +13 -6
  159. package/src/console/data-management/operations.ts +23 -18
  160. package/src/console/data-management/routes.ts +0 -1
  161. package/src/console/domain-registration.ts +10 -5
  162. package/src/console/observability/live.ts +22 -0
  163. package/src/console/observability/store.ts +39 -16
  164. package/src/console/observability/usage-dimensions.ts +4 -4
  165. package/src/console/providers/catalog/catalog-projections.ts +6 -7
  166. package/src/console/providers/catalog/model-operations.ts +5 -2
  167. package/src/console/providers/catalog/provider-operations.ts +4 -4
  168. package/src/console/providers/catalog/public-model-store.ts +8 -11
  169. package/src/console/providers/catalog/store.ts +1553 -1553
  170. package/src/console/providers/oauth/routes.ts +8 -12
  171. package/src/console/quota/account-quota-tenant.ts +6 -6
  172. package/src/console/quota/quota-view.ts +1 -1
  173. package/src/console/remote-support/contracts.ts +61 -0
  174. package/src/console/remote-support/routes.ts +65 -0
  175. package/src/console/remote-support/service.ts +207 -0
  176. package/src/console/response-compression.ts +0 -3
  177. package/src/console/routing/pools/probe-result.ts +3 -2
  178. package/src/console/routing/pools/probe-verdict.ts +132 -132
  179. package/src/console/routing/pools/store.ts +8 -8
  180. package/src/console/settings/contracts.ts +58 -2
  181. package/src/console/settings/store.ts +30 -3
  182. package/src/console/shared/cursor.ts +8 -8
  183. package/src/console/shared/errors.ts +1 -3
  184. package/src/network/http2-bidi.ts +2 -4
  185. package/src/network/pool/agent.ts +478 -478
  186. package/src/network/pool/selector.ts +5 -4
  187. package/src/network/retry.ts +1 -2
  188. package/src/network/ssrf.ts +5 -4
  189. package/src/observability/event-compaction.ts +80 -78
  190. package/src/observability/event-day-index.ts +13 -15
  191. package/src/observability/event-log.ts +4 -4
  192. package/src/observability/logger.ts +6 -6
  193. package/src/observability/payload-capture.ts +151 -33
  194. package/src/observability/payload-store.ts +397 -440
  195. package/src/observability/redaction.ts +5 -6
  196. package/src/observability/request-notify.ts +43 -0
  197. package/src/observability/telemetry-status.ts +8 -9
  198. package/src/observability/token-speed.ts +5 -4
  199. package/src/persistence/db-mode.ts +7 -7
  200. package/src/persistence/readiness.ts +1 -1
  201. package/src/persistence/schema.ts +1406 -1402
  202. package/src/persistence/shared-cache.ts +1 -2
  203. package/src/persistence/stores/api-key-store.ts +18 -6
  204. package/src/persistence/stores/share-store.ts +4 -3
  205. package/src/persistence/stores/telemetry-store.ts +236 -237
  206. package/src/protocol/canonical-model.ts +11 -3
  207. package/src/protocol/codecs.ts +138 -4
  208. package/src/protocol/gateway-error.ts +6 -4
  209. package/src/protocol/request/chat.ts +21 -1
  210. package/src/protocol/request/codex.ts +10 -1
  211. package/src/protocol/request/messages.ts +4 -5
  212. package/src/protocol/request/responses.ts +9 -1
  213. package/src/protocol/response/chat.ts +34 -4
  214. package/src/protocol/response/codex.ts +7 -4
  215. package/src/protocol/response/dsml.ts +140 -28
  216. package/src/protocol/response/responses.ts +11 -5
  217. package/src/protocol/stream-error-frames.ts +4 -5
  218. package/src/providers/authentication/jwt-validator.ts +9 -10
  219. package/src/providers/authentication/oauth-refresh-service.ts +9 -11
  220. package/src/providers/configured-openai-providers.ts +134 -136
  221. package/src/providers/discovery/catalog/catalog-blob.ts +18 -18
  222. package/src/providers/discovery/catalog/catalog-format.ts +3 -6
  223. package/src/providers/discovery/models-dev-catalog.ts +354 -354
  224. package/src/providers/discovery/probe-phases.ts +4 -4
  225. package/src/providers/discovery/probe-wire.ts +6 -13
  226. package/src/providers/discovery/probing-service.ts +1117 -1045
  227. package/src/providers/integrations/alibabacp/alibabacp.ts +1 -1
  228. package/src/providers/integrations/antigravity/antigravity-quota.ts +5 -7
  229. package/src/providers/integrations/buddy/buddy-chat-shared.ts +23 -2
  230. package/src/providers/integrations/buddy/buddy-checkin.ts +1 -4
  231. package/src/providers/integrations/buddy/buddy-cli-headers.ts +23 -6
  232. package/src/providers/integrations/buddy/buddy-conversation-headers.ts +0 -1
  233. package/src/providers/integrations/buddy/buddy-identity.ts +52 -0
  234. package/src/providers/integrations/buddy/buddy-media.ts +2 -2
  235. package/src/providers/integrations/buddy/buddy-usage-store.ts +3 -4
  236. package/src/providers/integrations/buddy/codebuddy/quota.ts +2 -13
  237. package/src/providers/integrations/buddy/codebuddy/shared.ts +118 -115
  238. package/src/providers/integrations/buddy/workbuddy/quota.ts +0 -11
  239. package/src/providers/integrations/buddy/workbuddy/shared.ts +5 -2
  240. package/src/providers/integrations/claude/claude.ts +608 -608
  241. package/src/providers/integrations/cline/cline-quota.ts +132 -132
  242. package/src/providers/integrations/cline/cline.ts +537 -539
  243. package/src/providers/integrations/codex/codex-oauth.ts +1 -1
  244. package/src/providers/integrations/commandcode/commandcode.ts +674 -675
  245. package/src/providers/integrations/commandcode/discovery.ts +117 -116
  246. package/src/providers/integrations/cursor/cursor-auth.ts +418 -418
  247. package/src/providers/integrations/cursor/cursor-quota.ts +309 -310
  248. package/src/providers/integrations/cursor/cursor.ts +11 -163
  249. package/src/providers/integrations/factory/factory-models.ts +3 -5
  250. package/src/providers/integrations/grok/grok-install-id.ts +2 -3
  251. package/src/providers/integrations/grok/grok.ts +6 -7
  252. package/src/providers/integrations/hermes.ts +14 -18
  253. package/src/providers/integrations/kiro/kiro-discovery.ts +1 -1
  254. package/src/providers/integrations/kiro/kiro.ts +1 -1
  255. package/src/providers/integrations/mistral.ts +6 -6
  256. package/src/providers/integrations/opencode/fingerprint.ts +3 -3
  257. package/src/providers/integrations/opencode/provider.ts +8 -8
  258. package/src/providers/integrations/qoder/qoder-catalog.ts +450 -450
  259. package/src/providers/integrations/qoder/qoder-oauth.ts +388 -310
  260. package/src/providers/integrations/qoder/qoder.ts +884 -865
  261. package/src/providers/integrations/shared/connect.ts +6 -6
  262. package/src/providers/integrations/trae/trae-catalog.ts +1 -2
  263. package/src/providers/integrations/trae/trae.ts +2 -2
  264. package/src/providers/integrations/zcode/oauth.ts +6 -1
  265. package/src/providers/operations/account-health-service.ts +1528 -1523
  266. package/src/providers/operations/account-reset-service.ts +1 -1
  267. package/src/providers/operations/client-versions.ts +6 -6
  268. package/src/providers/operations/custom-cli-headers.ts +3 -4
  269. package/src/providers/operations/provider-catalog-seeder.ts +95 -95
  270. package/src/providers/operations/provider-catalog-service.ts +3 -4
  271. package/src/providers/provider-metadata.ts +4 -8
  272. package/src/providers/provider-registry.ts +6 -7
  273. package/src/providers/provider-usage.ts +1 -1
  274. package/src/security/abuse.ts +15 -10
  275. package/src/security/access-control.ts +144 -146
  276. package/src/security/admission/contracts.ts +11 -0
  277. package/src/security/admission/in-memory-store.ts +4 -3
  278. package/src/security/admission/lease-sweep.ts +6 -6
  279. package/src/security/admission/reasons.ts +5 -1
  280. package/src/security/admission/redis-store.ts +211 -210
  281. package/src/security/admission/service.ts +4 -0
  282. package/src/security/api-key-auth.ts +504 -504
  283. package/src/security/ip-boundary.ts +155 -15
  284. package/src/security/model-abuse.ts +366 -94
  285. package/src/security/outbound-headers.ts +3 -4
  286. package/src/shared/sleep.ts +4 -0
  287. package/src/transport/dispatch/attempt-finalize.ts +47 -5
  288. package/src/transport/dispatch/attempt-loop.ts +531 -530
  289. package/src/transport/dispatch/stream-bridge.ts +1 -3
  290. package/src/transport/dispatch/streaming/finalize.ts +1 -1
  291. package/src/transport/dispatch/streaming/stream.ts +5 -6
  292. package/src/transport/failure-policy.ts +162 -8
  293. package/src/transport/guards/auth-guard.ts +7 -4
  294. package/src/transport/guards/concurrency-guard.ts +0 -5
  295. package/src/transport/guards/error-lifecycle.ts +13 -6
  296. package/src/transport/guards/pipeline.ts +15 -3
  297. package/src/transport/guards/request-context.ts +8 -2
  298. package/src/transport/request/inflight.ts +5 -6
  299. package/src/transport/request/preparer.ts +2 -2
  300. package/src/transport/request/state.ts +12 -8
  301. package/src/transport/routing/admission.ts +4 -0
  302. package/src/transport/routing/route-catalog.ts +8 -1
  303. package/src/transport/routing/route-model.ts +46 -14
  304. package/src/transport/routing/router.ts +21 -14
  305. package/src/transport/surface/completion.ts +2 -3
  306. package/src/transport/surface/dialects.ts +5 -6
  307. package/src/transport/surface/responses/encode.ts +38 -0
  308. package/src/transport/surface/responses/parse.ts +4 -0
  309. package/src/transport/translation/capabilities.ts +2 -3
  310. package/src/transport/translation/quirks.ts +3 -3
  311. package/src/workers/buddy-growth-tasks.ts +5 -10
  312. package/src/workers/daily-checkin.ts +17 -23
  313. package/src/workers/quota-refresh-sweep.ts +26 -26
  314. package/dist/dashboard/assets/BannedPage-C2orBsI2.js.gz +0 -0
  315. package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js +0 -1
  316. package/dist/dashboard/assets/CliToolDetailPage-DUcZ6pMB.js.gz +0 -0
  317. package/dist/dashboard/assets/CliToolsPage-BV6pxVQo.js.gz +0 -0
  318. package/dist/dashboard/assets/CombosPage-Bsan4679.js +0 -1
  319. package/dist/dashboard/assets/CombosPage-Bsan4679.js.gz +0 -0
  320. package/dist/dashboard/assets/ConsoleLogPage-CW-wPmJ6.js.gz +0 -0
  321. package/dist/dashboard/assets/CustomizationPage-0p8uX7vB.js.gz +0 -0
  322. package/dist/dashboard/assets/HealthEventsModal-BjqSMfko.js.gz +0 -0
  323. package/dist/dashboard/assets/LoginPage-BvzCx8Kr.js.gz +0 -0
  324. package/dist/dashboard/assets/ModelPicker-BY1aKEkP.js +0 -1
  325. package/dist/dashboard/assets/ModelPicker-BY1aKEkP.js.gz +0 -0
  326. package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js +0 -1
  327. package/dist/dashboard/assets/OverviewPage-DOQpy1y0.js.gz +0 -0
  328. package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js +0 -3
  329. package/dist/dashboard/assets/ProviderDetailPage-ecxyUMzt.js.gz +0 -0
  330. package/dist/dashboard/assets/ProvidersPage-Js9jUPSb.js.gz +0 -0
  331. package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js +0 -7
  332. package/dist/dashboard/assets/ProxyPage-J3TBi3Kx.js.gz +0 -0
  333. package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js +0 -1
  334. package/dist/dashboard/assets/QuotaPage-C1gZS6xr.js.gz +0 -0
  335. package/dist/dashboard/assets/SettingsPage-CS86ho5X.js +0 -1
  336. package/dist/dashboard/assets/SettingsPage-CS86ho5X.js.gz +0 -0
  337. package/dist/dashboard/assets/SetupPage-DDbMQsoo.js.gz +0 -0
  338. package/dist/dashboard/assets/SortableList-CB3Z7VTf.js.gz +0 -0
  339. package/dist/dashboard/assets/StudioPage-D95UCPHI.js.gz +0 -0
  340. package/dist/dashboard/assets/UsagePage-Cnae9SWA.js +0 -56
  341. package/dist/dashboard/assets/UsagePage-Cnae9SWA.js.gz +0 -0
  342. package/dist/dashboard/assets/api-keys-VY68Mhd1.js +0 -1
  343. package/dist/dashboard/assets/api-keys-VY68Mhd1.js.gz +0 -0
  344. package/dist/dashboard/assets/app-COlK95oi.css +0 -2
  345. package/dist/dashboard/assets/app-COlK95oi.css.gz +0 -0
  346. package/dist/dashboard/assets/assert-Chi7pCnt.js.gz +0 -0
  347. package/dist/dashboard/assets/badge-D4pQapKj.js.gz +0 -0
  348. package/dist/dashboard/assets/banner-DH6cg-_B.js.gz +0 -0
  349. package/dist/dashboard/assets/card-BKqlUOe4.js.gz +0 -0
  350. package/dist/dashboard/assets/cli-tools-fTgljTGA.js.gz +0 -0
  351. package/dist/dashboard/assets/clipboard-button-D5Ys_sM0.js.gz +0 -0
  352. package/dist/dashboard/assets/dialog-BYj-8WuK.js +0 -1
  353. package/dist/dashboard/assets/dialog-BYj-8WuK.js.gz +0 -0
  354. package/dist/dashboard/assets/drawer-CmXIZ6H1.js +0 -1
  355. package/dist/dashboard/assets/drawer-CmXIZ6H1.js.gz +0 -0
  356. package/dist/dashboard/assets/entry-CBg5zeu8.js +0 -1
  357. package/dist/dashboard/assets/entry-CBg5zeu8.js.gz +0 -0
  358. package/dist/dashboard/assets/entry-CQBr6U9c.js.gz +0 -0
  359. package/dist/dashboard/assets/entry-CgdSAzEV.js +0 -4
  360. package/dist/dashboard/assets/entry-CgdSAzEV.js.gz +0 -0
  361. package/dist/dashboard/assets/fingerprint-pattern-BWuHtbW4.js.gz +0 -0
  362. package/dist/dashboard/assets/github-badge-BlptL8Uf.js.gz +0 -0
  363. package/dist/dashboard/assets/index-js6TJvdr.js.gz +0 -0
  364. package/dist/dashboard/assets/inline-DUWDSmDV.js.gz +0 -0
  365. package/dist/dashboard/assets/input-LVjgZq4f.js.gz +0 -0
  366. package/dist/dashboard/assets/layout-CPfxpsfy.js +0 -1
  367. package/dist/dashboard/assets/layout-CPfxpsfy.js.gz +0 -0
  368. package/dist/dashboard/assets/list-row-DupvLfDx.js.gz +0 -0
  369. package/dist/dashboard/assets/live-C4pEJUqI.js.gz +0 -0
  370. package/dist/dashboard/assets/model-bans-BRiuufkB.js.gz +0 -0
  371. package/dist/dashboard/assets/mutation-B4ROwk8L.js.gz +0 -0
  372. package/dist/dashboard/assets/numbers-YNcHvRY7.js +0 -2
  373. package/dist/dashboard/assets/numbers-YNcHvRY7.js.gz +0 -0
  374. package/dist/dashboard/assets/query-keys-gmjzC3RS.js.gz +0 -0
  375. package/dist/dashboard/assets/routing-C7tBT5o-.js.gz +0 -0
  376. package/dist/dashboard/assets/select-WUCXorXw.js.gz +0 -0
  377. package/dist/dashboard/assets/settings-C_zKl1GS.js.gz +0 -0
  378. package/dist/dashboard/assets/stack-_3O0F8Xn.js.gz +0 -0
  379. package/dist/dashboard/assets/state-BgTxHcJS.js.gz +0 -0
  380. package/dist/dashboard/assets/switch-DEjTtOWf.js.gz +0 -0
  381. package/dist/dashboard/assets/tokens-DDxMbSRL.js +0 -1
  382. package/dist/dashboard/assets/tokens-DDxMbSRL.js.gz +0 -0
  383. package/dist/dashboard/assets/use-presence-BSZEBx7U.js +0 -1
  384. package/dist/dashboard/assets/use-presence-BSZEBx7U.js.gz +0 -0
  385. package/dist/dashboard/assets/useMutation-BWSC0UAS.js.gz +0 -0
  386. package/scripts/cli/command/admin.ts +0 -65
  387. package/scripts/cli/command/doctor.ts +0 -257
  388. package/scripts/cli/gateway/drain-token.ts +0 -78
  389. package/scripts/cli/gateway/instance-lock.ts +0 -105
  390. package/scripts/cli/gateway/stop.ts +0 -51
  391. package/scripts/cli/support/bun.ts +0 -27
  392. package/scripts/cli/support/version.ts +0 -37
  393. /package/dist/dashboard/assets/{app-DCQZRmHY.js → app-BIt446yE.js} +0 -0
  394. /package/dist/dashboard/assets/{app-DCQZRmHY.js.gz → app-BIt446yE.js.gz} +0 -0
  395. /package/scripts/cli/{ui/prompt.ts → prompt.ts} +0 -0
@@ -1,646 +1,647 @@
1
-
2
- import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
3
- import { poolMaxFromEnv } from "../persistence/db-pg";
4
- import type { CartethyiaDatabase } from "../persistence/db-handle";
5
- import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
6
- import type { RedisClient } from "../persistence/redis";
7
- import {
8
- bundledModelCatalog,
9
- liveProviderUpstreamHosts,
10
- markCatalogSeeded,
11
- refreshActiveProviderCatalogs,
12
- registerByokProviders,
13
- retireUnbundledProviders,
14
- seedBundledProviders,
15
- } from "../providers/operations/provider-catalog-service";
16
- import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
17
- import { createDefaultProviderRegistry } from "../providers/default-registry";
18
- import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
19
- import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
20
- import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
21
- import { accountIdentitySweep } from "../workers/account-identity-sweep";
22
- import { pushStructuredConsoleLog } from "../observability/log-ring";
23
- import { seedBundledModels } from "../providers/operations/provider-catalog-seeder";
24
- import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
25
- import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
26
- import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
27
- import { RoutingEngine } from "../transport/routing/router";
28
- import { RedisAdmissionController } from "../transport/routing/admission";
29
- import { ApiKeyAdmissionService } from "../security/admission/service";
30
- import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
31
- import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
32
- import { sweepLeases } from "../security/admission/lease-sweep";
33
- import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
34
- import { UsageFlushBuffer } from "../persistence/usage-flush";
35
- import { setPendingUsageSource } from "../security/api-key-auth";
36
- import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
37
- import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
38
- import { deleteExpiredSessions } from "../console/auth/session-service";
39
- import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
40
- import {
41
- InMemoryModelAbuseStore,
42
- ModelStrikeService,
43
- RedisModelAbuseStore,
44
- } from "../security/model-abuse";
45
- import { checkReadiness } from "../persistence/readiness";
46
- import { ProxyRequestPreparer } from "../transport/request/preparer";
47
- import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
48
- import {
49
- PoolAgentResolver,
50
- ValidatedNetworkBindingFactory,
51
- } from "../network/pool/resolver";
52
- import { NetworkPoolSelector } from "../network/pool/selector";
53
- import { ScheduledTaskRegistry } from "../workers/scheduler";
54
- import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
55
- import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
56
- import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
57
- import {
58
- buddyAdoptTask,
59
- buddyActivityReportTask,
60
- buddyLotteryTask,
61
- buddyRedeemTask,
62
- buddyStreakTask,
63
- buddyTravelTask,
64
- buddyTrialTask,
65
- syncBuddyUsage,
66
- } from "../workers/buddy-growth-tasks";
67
- import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
68
- import { quotaCacheSize } from "../console/quota/cache";
69
- import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
70
- import { preferencesReaderFor } from "../persistence/tenant-preferences";
71
- import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
72
- import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
73
- import { TelemetryPayloadCapture } from "../observability/payload-capture";
74
- import { foldDayIndex, listEventDays } from "../observability/event-day-index";
75
- import { resolveTelemetryCompactIntervalMs } from "../config/telemetry";
76
- import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
77
- import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
78
- import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
79
- import {
80
- resolveIpRateLimit,
81
- resolveModelBanTtlMs,
82
- resolveModelStrikeThreshold,
83
- resolveModelStrikeWindowMs,
84
- } from "../config/security";
85
- import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
86
- import type { TrustedProxyBoundary } from "../config/network";
87
- import { resolveTelemetryRetentionDays } from "../config/telemetry";
88
- import { CONFIG_SPEC, readInt, readList } from "../config/spec";
89
- import type { ProviderAdapter } from "../providers/provider-contracts";
90
- import type { ProviderRegistry } from "../providers/provider-registry";
91
- import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
92
- import type { ReadinessCheckResult } from "../persistence/readiness";
93
- import { log } from "../observability/logger";
94
- import { refreshProviderClientVersions } from "../providers/operations/client-versions";
95
-
96
-
97
- export interface ProductionDeps {
98
- db: CartethyiaDatabase;
99
- redis: RedisClient | undefined;
100
- snapshotService: InMemoryRouteSnapshotService;
101
- /** Live routing admission snapshot scoped to one provider/tenant for console reads. */
102
- readRoutingAccountInflight: (
103
- providerId: string,
104
- tenantId: string | null,
105
- ) => Promise<readonly { accountId: string; inflight: number }[]>;
106
- /** On-demand adapter resolution keeps heavy providers out of the boot path. */
107
- resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
108
- bundledModelCatalog: BundledProviderCatalog;
109
- /** Live lookup of a provider's SSRF-validated upstream host. */
110
- byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
111
- proxyPreparer: ProxyRequestPreparer;
112
- networkBindingFactory: ValidatedNetworkBindingFactory;
113
- ipAbuseProtection: IpAbuseProtectionService;
114
- readiness: () => Promise<ReadinessCheckResult>;
115
- scheduledTasks: ScheduledTaskRegistry;
116
- poolAgentResolver: PoolAgentResolver;
117
- poolSelector: NetworkPoolSelector;
118
- telemetryBuffer: TelemetryBatchBuffer;
119
- usageFlush: UsageFlushBuffer;
120
- healthEventFlush: HealthEventFlushBuffer;
121
- trustedProxyBoundary: TrustedProxyBoundary;
122
- providerRegistry: ProviderRegistry;
123
- resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
124
- oauthRefreshService: OAuthRefreshService;
125
- admissionService: ApiKeyAdmissionService;
126
- /** Graduated strikes for repeated invalid-model requests. */
127
- modelStrikes: ModelStrikeService;
128
- /** Memory sampler; in-flight source is bound once the request store exists. */
129
- runtimeMetricsSampler: RuntimeMetricsSampler;
130
- }
131
-
132
- /**
133
- * Pause between accounts in the OAuth refresh sweep. The token endpoints
134
- * rate-limit a burst of refreshes even at low concurrency.
135
- */
136
- const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
137
-
138
- /**
139
- * Client identities are refreshed out of band so provider dispatch never waits
140
- * on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
141
- */
142
- // Matched to the resolver's own 30 min cache TTL: at 15 min every second pass
143
- // was a pure cache read that could not discover anything new.
144
- const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
145
- const CLIENT_VERSION_INITIAL_DELAY_MS = 10_000;
146
-
147
- export async function buildProductionDeps(): Promise<ProductionDeps> {
148
- const startupStartedAt = performance.now();
149
- const startupMemoryBefore = process.memoryUsage();
150
- const handle = await measureStartupPhase("database-connect", bootDatabase);
151
- const db = handle.db;
152
- // One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
153
- const redis = resolveRedisClient();
154
- const ssrfPolicy = resolveSsrfPolicy();
155
- await measureStartupPhase("database-migrations", async () => {
156
- await ensureMigrated();
157
- // Pool sizing is an external-server concern; the embedded backend has no
158
- // connection budget to fit into.
159
- if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv());
160
- });
161
- await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
162
- // Boot-only: retire global rows for providers the bundle has dropped, so a
163
- // retired provider cannot linger as a card the console can no longer delete.
164
- // Kept off the test harness path, which shares one database across suites and
165
- // installs its own global fixtures.
166
- await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
167
- const registry = createDefaultProviderRegistry();
168
- // Live view, not a snapshot: custom providers registered after boot (or
169
- // edited from the console) must resolve their SSRF binding immediately.
170
- const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
171
- await measureStartupPhase("byok-registration", () =>
172
- registerByokProviders(registry, db, ssrfPolicy),
173
- );
174
- const bundledCatalog = await measureStartupPhase("bundled-model-catalog", () =>
175
- bundledModelCatalog(registry),
176
- );
177
- await measureStartupPhase("bundled-model-seed", () =>
178
- seedBundledModels(db, bundledCatalog.modelsByProvider),
179
- );
180
- // The boot seed above is unconditional, so record it: the snapshot builder's
181
- // first rebuild would otherwise re-seed an identical catalog.
182
- markCatalogSeeded();
183
- const snapshotService = new InMemoryRouteSnapshotService(
184
- createDatabaseSnapshotBuilder(db, () => refreshActiveProviderCatalogs(db, registry)),
185
- );
186
- const routingEngine = new RoutingEngine(
187
- redis ? new RedisAdmissionController(redis) : undefined,
188
- redis,
189
- );
190
- const readRoutingAccountInflight = async (
191
- providerId: string,
192
- tenantId: string | null,
193
- ): Promise<readonly { accountId: string; inflight: number }[]> => {
194
- if (providerId === "" || tenantId === null) return [];
195
- const snapshot = await routingEngine.accountInflightSnapshot();
196
- const accounts = await new DrizzleProviderCatalogStore(db, {
197
- telemetryBuffer,
198
- bundledModelCatalog: bundledCatalog,
199
- providerRegistry: registry,
200
- outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
201
- snapshotInvalidator: snapshotService,
202
- }).listAccounts(tenantId, providerId);
203
- const owned = new Set(accounts.map((account) => account.id));
204
- const totals = new Map<string, number>();
205
- for (const [bucket, count] of snapshot) {
206
- const prefix = `${providerId}:`;
207
- if (!bucket.startsWith(prefix)) continue;
208
- const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
209
- if (!owned.has(accountId)) continue;
210
- totals.set(accountId, (totals.get(accountId) ?? 0) + count);
211
- }
212
- return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
213
- };
214
- // Peer processes' settings mutations arrive here. Subscribed once for the
215
- // process: the revision counter is process-global, and the handler only
216
- // moves it forward.
217
- subscribePeerSettingsRevisions();
218
- const usageFlush = new UsageFlushBuffer(db);
219
- // Auth reads lifetime usage to seed the admission counter. The persisted
220
- // value lags by up to one flush interval, so hand it the in-flight deltas
221
- // too — otherwise the seed is systematically under what has actually been
222
- // spent, worst exactly when the key is busiest.
223
- setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
224
- // Health audit rows are append-only and read by nobody in dispatch, so they
225
- // no longer land inside the request. The status write they accompany stays
226
- // synchronous — routing reads it.
227
- const healthEventFlush = new HealthEventFlushBuffer(db);
228
- setHealthEventBuffer(healthEventFlush);
229
- const admissionStore = redis
230
- ? new RedisAdmissionCounterStore(redis)
231
- : new InMemoryAdmissionCounterStore();
232
- const admissionService = new ApiKeyAdmissionService(
233
- admissionStore,
234
- undefined,
235
- // The per-tenant concurrency cap is a `console_settings` preference, read
236
- // on every admitted request. Reading it directly cost one Postgres round
237
- // trip per request (multiplied by each failover candidate); the shared
238
- // revision-keyed reader answers from memory and converges the moment an
239
- // operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
240
- // revision every cached key embeds.
241
- (tenantId) =>
242
- preferencesReaderFor(db)
243
- .readPreferences(tenantId)
244
- .then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
245
- ({ apiKeyId, delta }) => {
246
- // Coalesced: one UPDATE per key per flush instead of one per request.
247
- // The admission counter is the live total; this only persists it.
248
- usageFlush.enqueue(apiKeyId, delta);
249
- return Promise.resolve();
250
- },
251
- // Fresh lifetime reader — invoked by the store only when it is about to
252
- // seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
253
- // snapshot value cannot freeze a low baseline in place. Family total: the
254
- // row plus every child (share recipients admit under the parent id).
255
- async (apiKeyId) => {
256
- const store = new DrizzleApiKeyStore(db);
257
- const row = await store.findActiveById(apiKeyId);
258
- if (!row) return undefined;
259
- const children = await store.sumChildrenConsumed(apiKeyId);
260
- return (row.lifetimeTokensConsumed ?? 0) + children;
261
- },
262
- );
263
- const ipStore = redis
264
- ? new RedisIpAbuseStore(redis)
265
- : new InMemoryIpAbuseStore();
266
- const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
267
- maxRequestsPerWindow: resolveIpRateLimit(),
268
- });
269
- // Graduated model-abuse strikes: a client that keeps requesting models
270
- // outside its access is warned, then banned. Redis-backed when available so
271
- // the ban survives a restart and is shared across instances; in-memory
272
- // otherwise, where a restart clears it.
273
- const modelAbuseStore = redis ? new RedisModelAbuseStore(redis) : new InMemoryModelAbuseStore();
274
- const modelStrikes = new ModelStrikeService(modelAbuseStore, {
275
- threshold: resolveModelStrikeThreshold(),
276
- windowMs: resolveModelStrikeWindowMs(),
277
- banTtlMs: resolveModelBanTtlMs(),
278
- });
279
- const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
280
- // One list covers every periodic task: a small deployment can cut the
281
- // heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
282
- // without the codebase growing a flag per task.
283
- const scheduledTasks = new ScheduledTaskRegistry(
284
- readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
285
- );
286
- if (redis) {
287
- scheduledTasks.register({
288
- name: "lease-sweep",
289
- intervalMs: 30_000,
290
- run: () => sweepLeases(redis),
291
- });
292
- }
293
- scheduledTasks.register({
294
- name: "account-health-sweep",
295
- intervalMs: 30_000,
296
- run: async () => {
297
- const recovered = await sweepExpiredCooldowns(db);
298
- const poolRecovered = await sweepExpiredPoolCooldowns(db);
299
- if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
300
- },
301
- });
302
- const payloadCapture = new TelemetryPayloadCapture(db);
303
- const telemetryStore = new DrizzleTelemetryStore(db);
304
- scheduledTasks.register({
305
- name: "telemetry-payload-cleanup",
306
- intervalMs: 15 * 60_000,
307
- run: () => payloadCapture.cleanupExpired(),
308
- });
309
- scheduledTasks.register({
310
- name: "telemetry-index-fold",
311
- intervalMs: resolveTelemetryCompactIntervalMs(),
312
- // Brings each day's index up to date with the bytes appended since the last
313
- // pass, so a usage read is a small JSON read rather than a parse of the
314
- // whole period. Folding is incremental and idempotent: a crash only leaves
315
- // the watermark behind, and the next pass re-reads the tail.
316
- run: async () => {
317
- const days = await listEventDays();
318
- let folded = 0;
319
- for (const day of days) {
320
- const result = await foldDayIndex(day).catch(() => ({ folded: 0 }));
321
- folded += result.folded;
322
- }
323
- if (folded > 0) {
324
- log.info(`[telemetry] indexed ${folded} event(s) across ${days.length} day(s)`);
325
- }
326
- },
327
- });
328
- scheduledTasks.register({
329
- name: "console-session-sweep",
330
- // Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
331
- // 48 — each pass a full scan of the session table for rows that cannot yet
332
- // be expired. Half the session lifetime is the point where a pass can
333
- // actually find work, and still leaves a full grace period after it.
334
- intervalMs: 12 * 60 * 60_000,
335
- run: () => deleteExpiredSessions(db),
336
- });
337
- scheduledTasks.register({
338
- name: "telemetry-retention",
339
- intervalMs: 6 * 60 * 60_000,
340
- run: () =>
341
- telemetryStore.pruneTelemetry(
342
- new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
343
- ),
344
- });
345
- const telemetryBuffer = new TelemetryBatchBuffer(db);
346
- const poolSelector = new NetworkPoolSelector(redis);
347
- const poolAgentResolver = new PoolAgentResolver(
348
- new DrizzleNetworkPoolLoader(db),
349
- { ssrfPolicy, db },
350
- );
351
- const networkBindingFactory = new ValidatedNetworkBindingFactory(
352
- ssrfPolicy,
353
- undefined,
354
- poolAgentResolver,
355
- );
356
- // Runtime gauges (memory + bounded-collection sizes) sampled on the shared
357
- // maintenance cadence. The sampler reads live sizes through closures so the
358
- // observability layer never holds direct references to routing/pool state.
359
- const runtimeMetricsSampler = new RuntimeMetricsSampler({
360
- routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
361
- ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
362
- quotaCacheEntries: () => quotaCacheSize(),
363
- modelCatalogEntries: () => modelCatalogCacheSize(),
364
- // Late-bound via bindInFlight once the request state store exists.
365
- inFlight: () => 0,
366
- });
367
- scheduledTasks.register({
368
- name: "runtime-metrics",
369
- intervalMs: readInt(
370
- "CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
371
- CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
372
- ),
373
- run: () => runtimeMetricsSampler.sample(),
374
- });
375
- scheduledTasks.register({
376
- name: "pool-agent-sweep",
377
- // `resolveAgent` already self-sweeps every 30 s, so a 1 min timer added a
378
- // second pass that almost always found nothing to release. 5 min is well
379
- // inside the idle timeout this exists to enforce, and outside the path
380
- // that already runs.
381
- intervalMs: 5 * 60_000,
382
- run: () => poolAgentResolver.sweepIdleAgents(),
383
- });
384
- const oauthRefreshService = new OAuthRefreshService(db);
385
- // Converge accounts an earlier build parked on a dead refresh grant: an
386
- // account whose stored access token is still usable becomes a static token
387
- // (and returns to rotation) instead of staying disabled for re-auth.
388
- await measureStartupPhase("static-token-reconciliation", () =>
389
- reconcileStaticTokenAccounts(db),
390
- );
391
- // Fills in the identity a batch import deliberately skipped, so a paste of
392
- // many credentials is one instant write and the email/label arrive shortly
393
- // after rather than blocking the import. Purely CPU on an already-stored
394
- // credential — no upstream call — so it runs often and cheaply.
395
- scheduledTasks.register({
396
- name: "account-enrichment-sweep",
397
- // Converges via `identity_checked_at`: once every stored credential has
398
- // been parsed, the listing query returns nothing — and a 30 s timer kept
399
- // running that empty query forever, two million times a year. 10 min is
400
- // still instant for the case that matters (a batch import, where the label
401
- // and email show up on the next tick) and costs nothing when idle.
402
- intervalMs: 10 * 60_000,
403
- run: () => accountIdentitySweep({ db }),
404
- });
405
- scheduledTasks.register({
406
- name: "oauth-refresh-sweep",
407
- intervalMs: 60_000,
408
- run: () =>
409
- oauthRefreshSweep({
410
- loadDueAccounts: () => loadDueOAuthAccounts(db),
411
- refreshService: oauthRefreshService,
412
- resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
413
- // Sequential with a pause between accounts: the OAuth token endpoints
414
- // rate-limit a burst of refreshes, and a pass over a handful of
415
- // accounts must not look like one. Per-account outcomes are already
416
- // pushed to the Console Log ring by `OAuthRefreshService`; this only
417
- // covers an error that escaped before reaching it.
418
- interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
419
- onAccountError: (accountId, providerId, error) => {
420
- log.error(
421
- `[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
422
- error as Error,
423
- );
424
- pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
425
- event: "token_refresh",
426
- accountId,
427
- providerId,
428
- errorCode: "refresh_sweep_failed",
429
- });
430
- },
431
- }),
432
- });
433
-
434
- // Client versions are monitor-only metadata; let the listener settle before
435
- // starting their external fetches. The scheduled task keeps the cache warm.
436
- scheduledTasks.register({
437
- name: "client-version-monitor",
438
- intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
439
- initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
440
- run: refreshProviderClientVersions,
441
- });
442
-
443
- const quotaResolveCredential = createAccountSecretResolver({
444
- db,
445
- resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
446
- refreshService: oauthRefreshService,
447
- });
448
- // Keeps every account's cached quota warm so opening the Quota page is a
449
- // cache read. Without this the only refreshes are user-triggered, so the
450
- // first open after an expiry always blocks on the provider.
451
- //
452
- // It also carries the daily check-in ride-along: each account refreshed
453
- // here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
454
- // the same pass (one attempt per account per day, guarded by a day
455
- // marker), so no second timer duplicates the credential resolution.
456
- //
457
- // Registered in **every** mode, not only when Redis is present. It used to
458
- // sit behind `if (redis)`, which quietly cost Lite two things: proactive
459
- // quota warming, and the check-in ride-along above that claims the
460
- // once-per-day WorkBuddy/CodeBuddy credit grant. The whole sweep chain
461
- // already accepts `RedisClient | undefined` and falls back to the
462
- // in-process TTL cache and an in-process day marker, so Redis is an
463
- // optimization here rather than a precondition — which is exactly what
464
- // Lite is for.
465
- scheduledTasks.register({
466
- name: "quota-refresh-sweep",
467
- intervalMs: 60_000,
468
- // Staggered off the oauth/lease/health ticks so the three sweeps that
469
- // strike providerAccounts do not land together every minute.
470
- initialDelayMs: 20_000,
471
- run: () =>
472
- quotaRefreshSweep({
473
- db,
474
- redis,
475
- providerRegistry: registry,
476
- resolveCredential: quotaResolveCredential,
477
- snapshotInvalidator: snapshotService,
478
- // Without this the sweep handed each collector the raw stored secret.
479
- // For `cline` that meant an `sk_...` key reaching the OAuth branch,
480
- // which prepends `workos:` and is refused with "please make sure
481
- // you're using the latest version of Cline and re-authenticate" — so
482
- // every sweep cooled a working account with `quota_fetch_failed`
483
- // while the same account refreshed fine from the console, which
484
- // binds the same marker through `createQuotaRefreshDeps`.
485
- markApiKeyCredential: markProviderApiKeyCredential,
486
- // Below the 60 s interval: a budget longer than the tick guarantees
487
- // skip/burst under load instead of steady progress.
488
- passBudgetMs: 45_000,
489
- // The check-in ride-along rotates egress per account: each account
490
- // gets the next active pool in its tenant's rotation so check-ins
491
- // spread across IPs instead of sharing one direct egress. No pool
492
- // (or an unreadable pool table) falls back to direct egress, and a
493
- // tenant never borrows another tenant's pool.
494
- checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
495
- // Per-account lines and the pass summary are emitted by the sweep
496
- // itself at `info` level, so they survive the production LOG_LEVEL.
497
- // `onTick` is left for tests.
498
- }),
499
- });
500
- // The buddy growth/credit tasks: one per upstream capability, each
501
- // independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
502
- // them is a credit gain the account is entitled to, and none of them can mark
503
- // an account unhealthy — a growth endpoint that 404s is a display miss, not a
504
- // dispatch failure.
505
- //
506
- // Daily, not hourly: each claim is once per account per day and is
507
- // ledger-guarded, so at an hourly cadence 23 of every 24 passes were a full
508
- // scan of active accounts plus a ledger read per account that resolved to
509
- // nothing. Once a day is the shortest interval where a pass can still find
510
- // work, which is the only reason to run at all.
511
- const buddyTaskDeps = {
512
- db,
513
- resolveCredential: quotaResolveCredential,
514
- fetcher: fetch,
515
- // Spread across accounts so a pass does not look like a burst to the
516
- // upstream's risk gate.
517
- interAccountDelayMs: 800,
518
- };
519
- scheduledTasks.register({
520
- name: "buddy-streak",
521
- intervalMs: 24 * 60 * 60_000,
522
- initialDelayMs: 5 * 60_000,
523
- run: () => buddyStreakTask(buddyTaskDeps),
524
- });
525
- scheduledTasks.register({
526
- name: "buddy-redeem",
527
- intervalMs: 24 * 60 * 60_000,
528
- initialDelayMs: 8 * 60_000,
529
- run: () => buddyRedeemTask(buddyTaskDeps),
530
- });
531
- scheduledTasks.register({
532
- name: "buddy-lottery",
533
- intervalMs: 24 * 60 * 60_000,
534
- initialDelayMs: 11 * 60_000,
535
- run: () => buddyLotteryTask(buddyTaskDeps),
536
- });
537
- scheduledTasks.register({
538
- name: "buddy-travel",
539
- intervalMs: 24 * 60 * 60_000,
540
- initialDelayMs: 14 * 60_000,
541
- run: () => buddyTravelTask(buddyTaskDeps),
542
- });
543
- scheduledTasks.register({
544
- name: "buddy-adopt",
545
- intervalMs: 24 * 60 * 60_000,
546
- initialDelayMs: 17 * 60_000,
547
- run: () => buddyAdoptTask(buddyTaskDeps),
548
- });
549
- scheduledTasks.register({
550
- name: "buddy-trial",
551
- intervalMs: 24 * 60 * 60_000,
552
- initialDelayMs: 20 * 60_000,
553
- run: () => buddyTrialTask(buddyTaskDeps),
554
- });
555
- // Growth-activity report: the `/v2/report` batch that lights the login streak
556
- // and scores the daily growth event. Once a day with its own ledger — it used
557
- // to ride the quota sweep with no marker of its own, so its eight POSTs ran
558
- // every minute.
559
- scheduledTasks.register({
560
- name: "buddy-activity-report",
561
- intervalMs: 24 * 60 * 60_000,
562
- initialDelayMs: 15 * 60 * 60_000,
563
- run: () => buddyActivityReportTask(buddyTaskDeps),
564
- });
565
- // Real credit deductions, stored so the console renders a figure the upstream
566
- // actually reported. Six-hourly: the upstream's own window is 30 days, so a
567
- // day's number does not change intraday.
568
- scheduledTasks.register({
569
- name: "buddy-usage-sync",
570
- intervalMs: 6 * 60 * 60_000,
571
- initialDelayMs: 23 * 60_000,
572
- run: () => syncBuddyUsage(buddyTaskDeps),
573
- });
574
- // The scheduled tasks are registered here but started by the caller once the
575
- // listener is actually serving (`main.ts`). Starting them inside this builder
576
- // would let the first tick — the lease sweep, the health sweep — run against
577
- // a process that has not begun accepting traffic yet.
578
- const deps: ProductionDeps = {
579
- db,
580
- redis,
581
- snapshotService,
582
- readRoutingAccountInflight,
583
- resolveProviderAdapter: (providerId) => registry.resolve(providerId),
584
- bundledModelCatalog: bundledCatalog,
585
- byokUpstreamHosts,
586
- proxyPreparer: new ProxyRequestPreparer({
587
- snapshotService,
588
- routingEngine,
589
- admissionService,
590
- }),
591
- networkBindingFactory,
592
- ipAbuseProtection,
593
- modelStrikes,
594
- readiness,
595
- scheduledTasks,
596
- poolAgentResolver,
597
- telemetryBuffer,
598
- usageFlush,
599
- healthEventFlush,
600
- poolSelector,
601
- admissionService,
602
- trustedProxyBoundary: resolveTrustedProxyBoundary(),
603
- providerRegistry: registry,
604
- resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
605
- oauthRefreshService,
606
- runtimeMetricsSampler,
607
- };
608
- const startupMemoryAfter = process.memoryUsage();
609
- log.info(
610
- `[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
611
- `rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
612
- `rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
613
- `heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
614
- `heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
615
- );
616
- return deps;
617
- }
618
-
619
- async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
620
- const startedAt = performance.now();
621
- const before = process.memoryUsage();
622
- log.info(
623
- `[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
624
- );
625
- try {
626
- const result = await operation();
627
- const after = process.memoryUsage();
628
- log.info(
629
- `[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
630
- `rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
631
- `heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
632
- `external_mb=${toMiB(after.external)}`,
633
- );
634
- return result;
635
- } catch (error) {
636
- log.error(
637
- `[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
638
- error instanceof Error ? error : new Error(String(error)),
639
- );
640
- throw error;
641
- }
642
- }
643
-
644
- function toMiB(bytes: number): string {
645
- return (bytes / 1024 / 1024).toFixed(1);
646
- }
1
+
2
+ import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
3
+ import { poolMaxFromEnv } from "../persistence/db-pg";
4
+ import type { CartethyiaDatabase } from "../persistence/db-handle";
5
+ import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
6
+ import type { RedisClient } from "../persistence/redis";
7
+ import {
8
+ bundledModelCatalog,
9
+ liveProviderUpstreamHosts,
10
+ markCatalogSeeded,
11
+ refreshActiveProviderCatalogs,
12
+ registerByokProviders,
13
+ retireUnbundledProviders,
14
+ seedBundledProviders,
15
+ } from "../providers/operations/provider-catalog-service";
16
+ import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
17
+ import { createDefaultProviderRegistry } from "../providers/default-registry";
18
+ import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
19
+ import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
20
+ import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
21
+ import { accountIdentitySweep } from "../workers/account-identity-sweep";
22
+ import { pushStructuredConsoleLog } from "../observability/log-ring";
23
+ import { seedBundledModels } from "../providers/operations/provider-catalog-seeder";
24
+ import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
25
+ import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
26
+ import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
27
+ import { RoutingEngine } from "../transport/routing/router";
28
+ import { RedisAdmissionController } from "../transport/routing/admission";
29
+ import { ApiKeyAdmissionService } from "../security/admission/service";
30
+ import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
31
+ import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
32
+ import { sweepLeases } from "../security/admission/lease-sweep";
33
+ import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
34
+ import { UsageFlushBuffer } from "../persistence/usage-flush";
35
+ import { setPendingUsageSource } from "../security/api-key-auth";
36
+ import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
37
+ import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
38
+ import { deleteExpiredSessions } from "../console/auth/session-service";
39
+ import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
40
+ import {
41
+ InMemoryModelAbuseStore,
42
+ ModelStrikeService,
43
+ RedisModelAbuseStore,
44
+ } from "../security/model-abuse";
45
+ import { checkReadiness } from "../persistence/readiness";
46
+ import { ProxyRequestPreparer } from "../transport/request/preparer";
47
+ import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
48
+ import {
49
+ PoolAgentResolver,
50
+ ValidatedNetworkBindingFactory,
51
+ } from "../network/pool/resolver";
52
+ import { NetworkPoolSelector } from "../network/pool/selector";
53
+ import { ScheduledTaskRegistry } from "../workers/scheduler";
54
+ import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
55
+ import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
56
+ import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
57
+ import {
58
+ buddyAdoptTask,
59
+ buddyActivityReportTask,
60
+ buddyLotteryTask,
61
+ buddyRedeemTask,
62
+ buddyStreakTask,
63
+ buddyTravelTask,
64
+ buddyTrialTask,
65
+ syncBuddyUsage,
66
+ } from "../workers/buddy-growth-tasks";
67
+ import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
68
+ import { quotaCacheSize } from "../console/quota/cache";
69
+ import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
70
+ import { preferencesReaderFor } from "../persistence/tenant-preferences";
71
+ import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
72
+ import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
73
+ import { TelemetryPayloadCapture } from "../observability/payload-capture";
74
+ import { foldDayIndex, listEventDays } from "../observability/event-day-index";
75
+ import { resolveTelemetryCompactIntervalMs } from "../config/telemetry";
76
+ import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
77
+ import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
78
+ import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
79
+ import {
80
+ resolveIpRateLimit,
81
+ resolveModelBanMaxTtlMs,
82
+ resolveModelBanTtlMs,
83
+ resolveModelStrikeThreshold,
84
+ resolveModelStrikeWindowMs,
85
+ } from "../config/security";
86
+ import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
87
+ import type { TrustedProxyBoundary } from "../config/network";
88
+ import { resolveTelemetryRetentionDays } from "../config/telemetry";
89
+ import { CONFIG_SPEC, readInt, readList } from "../config/spec";
90
+ import type { ProviderAdapter } from "../providers/provider-contracts";
91
+ import type { ProviderRegistry } from "../providers/provider-registry";
92
+ import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
93
+ import type { ReadinessCheckResult } from "../persistence/readiness";
94
+ import { log } from "../observability/logger";
95
+ import { refreshProviderClientVersions } from "../providers/operations/client-versions";
96
+
97
+
98
+ export interface ProductionDeps {
99
+ db: CartethyiaDatabase;
100
+ redis: RedisClient | undefined;
101
+ snapshotService: InMemoryRouteSnapshotService;
102
+ /** Live routing admission snapshot scoped to one provider/tenant for console reads. */
103
+ readRoutingAccountInflight: (
104
+ providerId: string,
105
+ tenantId: string | null,
106
+ ) => Promise<readonly { accountId: string; inflight: number }[]>;
107
+ /** On-demand adapter resolution keeps heavy providers out of the boot path. */
108
+ resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
109
+ bundledModelCatalog: BundledProviderCatalog;
110
+ /** Live lookup of a provider's SSRF-validated upstream host. */
111
+ byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
112
+ proxyPreparer: ProxyRequestPreparer;
113
+ networkBindingFactory: ValidatedNetworkBindingFactory;
114
+ ipAbuseProtection: IpAbuseProtectionService;
115
+ readiness: () => Promise<ReadinessCheckResult>;
116
+ scheduledTasks: ScheduledTaskRegistry;
117
+ poolAgentResolver: PoolAgentResolver;
118
+ poolSelector: NetworkPoolSelector;
119
+ telemetryBuffer: TelemetryBatchBuffer;
120
+ usageFlush: UsageFlushBuffer;
121
+ healthEventFlush: HealthEventFlushBuffer;
122
+ trustedProxyBoundary: TrustedProxyBoundary;
123
+ providerRegistry: ProviderRegistry;
124
+ resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
125
+ oauthRefreshService: OAuthRefreshService;
126
+ admissionService: ApiKeyAdmissionService;
127
+ /** Graduated strikes for repeated invalid-model requests. */
128
+ modelStrikes: ModelStrikeService;
129
+ /** Memory sampler; in-flight source is bound once the request store exists. */
130
+ runtimeMetricsSampler: RuntimeMetricsSampler;
131
+ }
132
+
133
+ /**
134
+ * Pause between accounts in the OAuth refresh sweep. The token endpoints
135
+ * rate-limit a burst of refreshes even at low concurrency.
136
+ */
137
+ const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
138
+
139
+ /**
140
+ * Client identities are refreshed out of band so provider dispatch never waits
141
+ * on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
142
+ */
143
+ // Matched to the resolver's own 30 min cache TTL: a shorter interval makes
144
+ // every second pass a pure cache read that cannot discover anything new.
145
+ const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
146
+ const CLIENT_VERSION_INITIAL_DELAY_MS = 10_000;
147
+
148
+ export async function buildProductionDeps(): Promise<ProductionDeps> {
149
+ const startupStartedAt = performance.now();
150
+ const startupMemoryBefore = process.memoryUsage();
151
+ const handle = await measureStartupPhase("database-connect", bootDatabase);
152
+ const db = handle.db;
153
+ // One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
154
+ const redis = resolveRedisClient();
155
+ const ssrfPolicy = resolveSsrfPolicy();
156
+ await measureStartupPhase("database-migrations", async () => {
157
+ await ensureMigrated();
158
+ // Pool sizing is an external-server concern; the embedded backend has no
159
+ // connection budget to fit into.
160
+ if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv());
161
+ });
162
+ await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
163
+ // Boot-only: retire global rows for providers the bundle has dropped, so a
164
+ // retired provider cannot linger as a card the console can no longer delete.
165
+ // Kept off the test harness path, which shares one database across suites and
166
+ // installs its own global fixtures.
167
+ await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
168
+ const registry = createDefaultProviderRegistry();
169
+ // Live view, not a snapshot: custom providers registered after boot (or
170
+ // edited from the console) must resolve their SSRF binding immediately.
171
+ const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
172
+ await measureStartupPhase("byok-registration", () =>
173
+ registerByokProviders(registry, db, ssrfPolicy),
174
+ );
175
+ const bundledCatalog = await measureStartupPhase("bundled-model-catalog", () =>
176
+ bundledModelCatalog(registry),
177
+ );
178
+ await measureStartupPhase("bundled-model-seed", () =>
179
+ seedBundledModels(db, bundledCatalog.modelsByProvider),
180
+ );
181
+ // The boot seed above is unconditional, so record it: the snapshot builder's
182
+ // first rebuild would otherwise re-seed an identical catalog.
183
+ markCatalogSeeded();
184
+ const snapshotService = new InMemoryRouteSnapshotService(
185
+ createDatabaseSnapshotBuilder(db, () => refreshActiveProviderCatalogs(db, registry)),
186
+ );
187
+ const routingEngine = new RoutingEngine(
188
+ redis ? new RedisAdmissionController(redis) : undefined,
189
+ redis,
190
+ );
191
+ const readRoutingAccountInflight = async (
192
+ providerId: string,
193
+ tenantId: string | null,
194
+ ): Promise<readonly { accountId: string; inflight: number }[]> => {
195
+ if (providerId === "" || tenantId === null) return [];
196
+ const snapshot = await routingEngine.accountInflightSnapshot();
197
+ const accounts = await new DrizzleProviderCatalogStore(db, {
198
+ telemetryBuffer,
199
+ bundledModelCatalog: bundledCatalog,
200
+ providerRegistry: registry,
201
+ outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
202
+ snapshotInvalidator: snapshotService,
203
+ }).listAccounts(tenantId, providerId);
204
+ const owned = new Set(accounts.map((account) => account.id));
205
+ const totals = new Map<string, number>();
206
+ for (const [bucket, count] of snapshot) {
207
+ const prefix = `${providerId}:`;
208
+ if (!bucket.startsWith(prefix)) continue;
209
+ const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
210
+ if (!owned.has(accountId)) continue;
211
+ totals.set(accountId, (totals.get(accountId) ?? 0) + count);
212
+ }
213
+ return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
214
+ };
215
+ // Peer processes' settings mutations arrive here. Subscribed once for the
216
+ // process: the revision counter is process-global, and the handler only
217
+ // moves it forward.
218
+ subscribePeerSettingsRevisions();
219
+ const usageFlush = new UsageFlushBuffer(db);
220
+ // Auth reads lifetime usage to seed the admission counter. The persisted
221
+ // value lags by up to one flush interval, so hand it the in-flight deltas
222
+ // too — otherwise the seed is systematically under what has actually been
223
+ // spent, worst exactly when the key is busiest.
224
+ setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
225
+ // Health audit rows are append-only and read by nobody in dispatch, so they
226
+ // are buffered out of the request. The status write they accompany stays
227
+ // synchronous — routing reads it.
228
+ const healthEventFlush = new HealthEventFlushBuffer(db);
229
+ setHealthEventBuffer(healthEventFlush);
230
+ const admissionStore = redis
231
+ ? new RedisAdmissionCounterStore(redis)
232
+ : new InMemoryAdmissionCounterStore();
233
+ const admissionService = new ApiKeyAdmissionService(
234
+ admissionStore,
235
+ undefined,
236
+ // The per-tenant concurrency cap is a `console_settings` preference, read
237
+ // on every admitted request. Reading it directly cost one Postgres round
238
+ // trip per request (multiplied by each failover candidate); the shared
239
+ // revision-keyed reader answers from memory and converges the moment an
240
+ // operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
241
+ // revision every cached key embeds.
242
+ (tenantId) =>
243
+ preferencesReaderFor(db)
244
+ .readPreferences(tenantId)
245
+ .then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
246
+ ({ apiKeyId, delta }) => {
247
+ // Coalesced: one UPDATE per key per flush instead of one per request.
248
+ // The admission counter is the live total; this only persists it.
249
+ usageFlush.enqueue(apiKeyId, delta);
250
+ return Promise.resolve();
251
+ },
252
+ // Fresh lifetime reader — invoked by the store only when it is about to
253
+ // seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
254
+ // snapshot value cannot freeze a low baseline in place. Family total: the
255
+ // row plus every child (share recipients admit under the parent id).
256
+ async (apiKeyId) => {
257
+ const store = new DrizzleApiKeyStore(db);
258
+ const row = await store.findActiveById(apiKeyId);
259
+ if (!row) return undefined;
260
+ const children = await store.sumChildrenConsumed(apiKeyId);
261
+ return (row.lifetimeTokensConsumed ?? 0) + children;
262
+ },
263
+ );
264
+ const ipStore = redis
265
+ ? new RedisIpAbuseStore(redis)
266
+ : new InMemoryIpAbuseStore();
267
+ const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
268
+ maxRequestsPerWindow: resolveIpRateLimit(),
269
+ });
270
+ // Graduated model-abuse strikes: a client that keeps requesting models
271
+ // outside its access is warned, then banned. Redis-backed when available so
272
+ // the ban survives a restart and is shared across instances; in-memory
273
+ // otherwise, where a restart clears it.
274
+ const modelAbuseStore = redis ? new RedisModelAbuseStore(redis) : new InMemoryModelAbuseStore();
275
+ const modelStrikes = new ModelStrikeService(modelAbuseStore, {
276
+ threshold: resolveModelStrikeThreshold(),
277
+ windowMs: resolveModelStrikeWindowMs(),
278
+ banTtlMs: resolveModelBanTtlMs(),
279
+ maxBanTtlMs: resolveModelBanMaxTtlMs(),
280
+ });
281
+ const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
282
+ // One list covers every periodic task: a small deployment can cut the
283
+ // heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
284
+ // without the codebase growing a flag per task.
285
+ const scheduledTasks = new ScheduledTaskRegistry(
286
+ readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
287
+ );
288
+ if (redis) {
289
+ scheduledTasks.register({
290
+ name: "lease-sweep",
291
+ intervalMs: 30_000,
292
+ run: () => sweepLeases(redis),
293
+ });
294
+ }
295
+ scheduledTasks.register({
296
+ name: "account-health-sweep",
297
+ intervalMs: 30_000,
298
+ run: async () => {
299
+ const recovered = await sweepExpiredCooldowns(db);
300
+ const poolRecovered = await sweepExpiredPoolCooldowns(db);
301
+ if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
302
+ },
303
+ });
304
+ const payloadCapture = new TelemetryPayloadCapture(db);
305
+ const telemetryStore = new DrizzleTelemetryStore(db);
306
+ scheduledTasks.register({
307
+ name: "telemetry-payload-cleanup",
308
+ intervalMs: 15 * 60_000,
309
+ run: () => payloadCapture.cleanupExpired(),
310
+ });
311
+ scheduledTasks.register({
312
+ name: "telemetry-index-fold",
313
+ intervalMs: resolveTelemetryCompactIntervalMs(),
314
+ // Brings each day's index up to date with the bytes appended since the last
315
+ // pass, so a usage read is a small JSON read rather than a parse of the
316
+ // whole period. Folding is incremental and idempotent: a crash only leaves
317
+ // the watermark behind, and the next pass re-reads the tail.
318
+ run: async () => {
319
+ const days = await listEventDays();
320
+ let folded = 0;
321
+ for (const day of days) {
322
+ const result = await foldDayIndex(day).catch(() => ({ folded: 0 }));
323
+ folded += result.folded;
324
+ }
325
+ if (folded > 0) {
326
+ log.info(`[telemetry] indexed ${folded} event(s) across ${days.length} day(s)`);
327
+ }
328
+ },
329
+ });
330
+ scheduledTasks.register({
331
+ name: "console-session-sweep",
332
+ // Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
333
+ // 48 — each pass a full scan of the session table for rows that cannot yet
334
+ // be expired. Half the session lifetime is the point where a pass can
335
+ // actually find work, and still leaves a full grace period after it.
336
+ intervalMs: 12 * 60 * 60_000,
337
+ run: () => deleteExpiredSessions(db),
338
+ });
339
+ scheduledTasks.register({
340
+ name: "telemetry-retention",
341
+ intervalMs: 6 * 60 * 60_000,
342
+ run: () =>
343
+ telemetryStore.pruneTelemetry(
344
+ new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
345
+ ),
346
+ });
347
+ const telemetryBuffer = new TelemetryBatchBuffer(db);
348
+ const poolSelector = new NetworkPoolSelector(redis);
349
+ const poolAgentResolver = new PoolAgentResolver(
350
+ new DrizzleNetworkPoolLoader(db),
351
+ { ssrfPolicy, db },
352
+ );
353
+ const networkBindingFactory = new ValidatedNetworkBindingFactory(
354
+ ssrfPolicy,
355
+ undefined,
356
+ poolAgentResolver,
357
+ );
358
+ // Runtime gauges (memory + bounded-collection sizes) sampled on the shared
359
+ // maintenance cadence. The sampler reads live sizes through closures so the
360
+ // observability layer never holds direct references to routing/pool state.
361
+ const runtimeMetricsSampler = new RuntimeMetricsSampler({
362
+ routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
363
+ ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
364
+ quotaCacheEntries: () => quotaCacheSize(),
365
+ modelCatalogEntries: () => modelCatalogCacheSize(),
366
+ // Late-bound via bindInFlight once the request state store exists.
367
+ inFlight: () => 0,
368
+ });
369
+ scheduledTasks.register({
370
+ name: "runtime-metrics",
371
+ intervalMs: readInt(
372
+ "CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
373
+ CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
374
+ ),
375
+ run: () => runtimeMetricsSampler.sample(),
376
+ });
377
+ scheduledTasks.register({
378
+ name: "pool-agent-sweep",
379
+ // `resolveAgent` already self-sweeps every 30 s, so a 1 min timer adds a
380
+ // second pass that almost always finds nothing to release. 5 min is well
381
+ // inside the idle timeout this exists to enforce, and outside the path
382
+ // that already runs.
383
+ intervalMs: 5 * 60_000,
384
+ run: () => poolAgentResolver.sweepIdleAgents(),
385
+ });
386
+ const oauthRefreshService = new OAuthRefreshService(db);
387
+ // Converge accounts parked on a dead refresh grant: an account whose stored
388
+ // access token is still usable becomes a static token (and returns to
389
+ // rotation) instead of staying disabled for re-auth.
390
+ await measureStartupPhase("static-token-reconciliation", () =>
391
+ reconcileStaticTokenAccounts(db),
392
+ );
393
+ // Fills in the identity a batch import deliberately skipped, so a paste of
394
+ // many credentials is one instant write and the email/label arrive shortly
395
+ // after rather than blocking the import. Purely CPU on an already-stored
396
+ // credential — no upstream call — so it runs often and cheaply.
397
+ scheduledTasks.register({
398
+ name: "account-enrichment-sweep",
399
+ // Converges via `identity_checked_at`: once every stored credential has
400
+ // been parsed, the listing query returns nothing — a 30 s timer would run
401
+ // that empty query forever, two million times a year. 10 min is still
402
+ // instant for the case that matters (a batch import, where the label and
403
+ // email show up on the next tick) and costs nothing when idle.
404
+ intervalMs: 10 * 60_000,
405
+ run: () => accountIdentitySweep({ db }),
406
+ });
407
+ scheduledTasks.register({
408
+ name: "oauth-refresh-sweep",
409
+ intervalMs: 60_000,
410
+ run: () =>
411
+ oauthRefreshSweep({
412
+ loadDueAccounts: () => loadDueOAuthAccounts(db),
413
+ refreshService: oauthRefreshService,
414
+ resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
415
+ // Sequential with a pause between accounts: the OAuth token endpoints
416
+ // rate-limit a burst of refreshes, and a pass over a handful of
417
+ // accounts must not look like one. Per-account outcomes are already
418
+ // pushed to the Console Log ring by `OAuthRefreshService`; this only
419
+ // covers an error that escaped before reaching it.
420
+ interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
421
+ onAccountError: (accountId, providerId, error) => {
422
+ log.error(
423
+ `[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
424
+ error as Error,
425
+ );
426
+ pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
427
+ event: "token_refresh",
428
+ accountId,
429
+ providerId,
430
+ errorCode: "refresh_sweep_failed",
431
+ });
432
+ },
433
+ }),
434
+ });
435
+
436
+ // Client versions are monitor-only metadata; let the listener settle before
437
+ // starting their external fetches. The scheduled task keeps the cache warm.
438
+ scheduledTasks.register({
439
+ name: "client-version-monitor",
440
+ intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
441
+ initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
442
+ run: refreshProviderClientVersions,
443
+ });
444
+
445
+ const quotaResolveCredential = createAccountSecretResolver({
446
+ db,
447
+ resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
448
+ refreshService: oauthRefreshService,
449
+ });
450
+ // Keeps every account's cached quota warm so opening the Quota page is a
451
+ // cache read. Without this the only refreshes are user-triggered, so the
452
+ // first open after an expiry always blocks on the provider.
453
+ //
454
+ // It also carries the daily check-in ride-along: each account refreshed
455
+ // here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
456
+ // the same pass (one attempt per account per day, guarded by a day
457
+ // marker), so no second timer duplicates the credential resolution.
458
+ //
459
+ // Registered in **every** mode, not only when Redis is present: gating it on
460
+ // Redis would cost Lite proactive quota warming and the check-in ride-along
461
+ // above that claims the once-per-day WorkBuddy/CodeBuddy credit grant. The
462
+ // whole sweep chain accepts `RedisClient | undefined` and falls back to the
463
+ // in-process TTL cache and an in-process day marker, so Redis is an
464
+ // optimization here rather than a precondition — which is exactly what
465
+ // Lite is for.
466
+ scheduledTasks.register({
467
+ name: "quota-refresh-sweep",
468
+ intervalMs: 60_000,
469
+ // Staggered off the oauth/lease/health ticks so the three sweeps that
470
+ // strike providerAccounts do not land together every minute.
471
+ initialDelayMs: 20_000,
472
+ run: () =>
473
+ quotaRefreshSweep({
474
+ db,
475
+ redis,
476
+ providerRegistry: registry,
477
+ resolveCredential: quotaResolveCredential,
478
+ snapshotInvalidator: snapshotService,
479
+ // Without this the sweep hands each collector the raw stored secret.
480
+ // For `cline` that means an `sk_...` key reaching the OAuth branch,
481
+ // which prepends `workos:` and is refused with "please make sure
482
+ // you're using the latest version of Cline and re-authenticate" — so
483
+ // every sweep cools a working account with `quota_fetch_failed` while
484
+ // the same account refreshes fine from the console, which binds the
485
+ // same marker through `createQuotaRefreshDeps`.
486
+ markApiKeyCredential: markProviderApiKeyCredential,
487
+ // Below the 60 s interval: a budget longer than the tick guarantees
488
+ // skip/burst under load instead of steady progress.
489
+ passBudgetMs: 45_000,
490
+ // The check-in ride-along rotates egress per account: each account
491
+ // gets the next active pool in its tenant's rotation so check-ins
492
+ // spread across IPs instead of sharing one direct egress. No pool
493
+ // (or an unreadable pool table) falls back to direct egress, and a
494
+ // tenant never borrows another tenant's pool.
495
+ checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
496
+ // Per-account lines and the pass summary are emitted by the sweep
497
+ // itself at `info` level, so they survive the production LOG_LEVEL.
498
+ // `onTick` is left for tests.
499
+ }),
500
+ });
501
+ // The buddy growth/credit tasks: one per upstream capability, each
502
+ // independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
503
+ // them is a credit gain the account is entitled to, and none of them can mark
504
+ // an account unhealthy — a growth endpoint that 404s is a display miss, not a
505
+ // dispatch failure.
506
+ //
507
+ // Daily, not hourly: each claim is once per account per day and is
508
+ // ledger-guarded, so at an hourly cadence 23 of every 24 passes are a full
509
+ // scan of active accounts plus a ledger read per account that resolves to
510
+ // nothing. Once a day is the shortest interval where a pass can still find
511
+ // work, which is the only reason to run at all.
512
+ const buddyTaskDeps = {
513
+ db,
514
+ resolveCredential: quotaResolveCredential,
515
+ fetcher: fetch,
516
+ // Spread across accounts so a pass does not look like a burst to the
517
+ // upstream's risk gate.
518
+ interAccountDelayMs: 800,
519
+ };
520
+ scheduledTasks.register({
521
+ name: "buddy-streak",
522
+ intervalMs: 24 * 60 * 60_000,
523
+ initialDelayMs: 5 * 60_000,
524
+ run: () => buddyStreakTask(buddyTaskDeps),
525
+ });
526
+ scheduledTasks.register({
527
+ name: "buddy-redeem",
528
+ intervalMs: 24 * 60 * 60_000,
529
+ initialDelayMs: 8 * 60_000,
530
+ run: () => buddyRedeemTask(buddyTaskDeps),
531
+ });
532
+ scheduledTasks.register({
533
+ name: "buddy-lottery",
534
+ intervalMs: 24 * 60 * 60_000,
535
+ initialDelayMs: 11 * 60_000,
536
+ run: () => buddyLotteryTask(buddyTaskDeps),
537
+ });
538
+ scheduledTasks.register({
539
+ name: "buddy-travel",
540
+ intervalMs: 24 * 60 * 60_000,
541
+ initialDelayMs: 14 * 60_000,
542
+ run: () => buddyTravelTask(buddyTaskDeps),
543
+ });
544
+ scheduledTasks.register({
545
+ name: "buddy-adopt",
546
+ intervalMs: 24 * 60 * 60_000,
547
+ initialDelayMs: 17 * 60_000,
548
+ run: () => buddyAdoptTask(buddyTaskDeps),
549
+ });
550
+ scheduledTasks.register({
551
+ name: "buddy-trial",
552
+ intervalMs: 24 * 60 * 60_000,
553
+ initialDelayMs: 20 * 60_000,
554
+ run: () => buddyTrialTask(buddyTaskDeps),
555
+ });
556
+ // Growth-activity report: the `/v2/report` batch that lights the login streak
557
+ // and scores the daily growth event. Once a day with its own ledger — it used
558
+ // to ride the quota sweep with no marker of its own, so its eight POSTs ran
559
+ // every minute.
560
+ scheduledTasks.register({
561
+ name: "buddy-activity-report",
562
+ intervalMs: 24 * 60 * 60_000,
563
+ initialDelayMs: 15 * 60 * 60_000,
564
+ run: () => buddyActivityReportTask(buddyTaskDeps),
565
+ });
566
+ // Real credit deductions, stored so the console renders a figure the upstream
567
+ // actually reported. Six-hourly: the upstream's own window is 30 days, so a
568
+ // day's number does not change intraday.
569
+ scheduledTasks.register({
570
+ name: "buddy-usage-sync",
571
+ intervalMs: 6 * 60 * 60_000,
572
+ initialDelayMs: 23 * 60_000,
573
+ run: () => syncBuddyUsage(buddyTaskDeps),
574
+ });
575
+ // The scheduled tasks are registered here but started by the caller once the
576
+ // listener is actually serving (`main.ts`). Starting them inside this builder
577
+ // would let the first tick — the lease sweep, the health sweep — run against
578
+ // a process that has not begun accepting traffic yet.
579
+ const deps: ProductionDeps = {
580
+ db,
581
+ redis,
582
+ snapshotService,
583
+ readRoutingAccountInflight,
584
+ resolveProviderAdapter: (providerId) => registry.resolve(providerId),
585
+ bundledModelCatalog: bundledCatalog,
586
+ byokUpstreamHosts,
587
+ proxyPreparer: new ProxyRequestPreparer({
588
+ snapshotService,
589
+ routingEngine,
590
+ admissionService,
591
+ }),
592
+ networkBindingFactory,
593
+ ipAbuseProtection,
594
+ modelStrikes,
595
+ readiness,
596
+ scheduledTasks,
597
+ poolAgentResolver,
598
+ telemetryBuffer,
599
+ usageFlush,
600
+ healthEventFlush,
601
+ poolSelector,
602
+ admissionService,
603
+ trustedProxyBoundary: resolveTrustedProxyBoundary(),
604
+ providerRegistry: registry,
605
+ resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
606
+ oauthRefreshService,
607
+ runtimeMetricsSampler,
608
+ };
609
+ const startupMemoryAfter = process.memoryUsage();
610
+ log.info(
611
+ `[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
612
+ `rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
613
+ `rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
614
+ `heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
615
+ `heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
616
+ );
617
+ return deps;
618
+ }
619
+
620
+ async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
621
+ const startedAt = performance.now();
622
+ const before = process.memoryUsage();
623
+ log.info(
624
+ `[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
625
+ );
626
+ try {
627
+ const result = await operation();
628
+ const after = process.memoryUsage();
629
+ log.info(
630
+ `[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
631
+ `rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
632
+ `heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
633
+ `external_mb=${toMiB(after.external)}`,
634
+ );
635
+ return result;
636
+ } catch (error) {
637
+ log.error(
638
+ `[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
639
+ error instanceof Error ? error : new Error(String(error)),
640
+ );
641
+ throw error;
642
+ }
643
+ }
644
+
645
+ function toMiB(bytes: number): string {
646
+ return (bytes / 1024 / 1024).toFixed(1);
647
+ }