cartethyia 2.7.3 → 2.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (625) hide show
  1. package/.env.example +2 -0
  2. package/README.md +17 -5
  3. package/dist/dashboard/assets/{BannedPage-2TJD37Hb.js → BannedPage-Cj15GsMp.js} +1 -1
  4. package/dist/dashboard/assets/BannedPage-Cj15GsMp.js.gz +0 -0
  5. package/dist/dashboard/assets/ClaimGiftsButton-D83pBJ6J.js +1 -0
  6. package/dist/dashboard/assets/ClaimGiftsButton-D83pBJ6J.js.gz +0 -0
  7. package/dist/dashboard/assets/{CliToolDetailPage-xefJNRmJ.js → CliToolDetailPage-DZKmVuJN.js} +1 -1
  8. package/dist/dashboard/assets/CliToolDetailPage-DZKmVuJN.js.gz +0 -0
  9. package/dist/dashboard/assets/{CliToolsPage-Rj9iHQ3F.js → CliToolsPage-BIccZmeL.js} +1 -1
  10. package/dist/dashboard/assets/CliToolsPage-BIccZmeL.js.gz +0 -0
  11. package/dist/dashboard/assets/CombosPage-DEEKkYD-.js +1 -0
  12. package/dist/dashboard/assets/CombosPage-DEEKkYD-.js.gz +0 -0
  13. package/dist/dashboard/assets/ConsoleLogPage-D5rol2j2.js +1 -0
  14. package/dist/dashboard/assets/ConsoleLogPage-D5rol2j2.js.gz +0 -0
  15. package/dist/dashboard/assets/CustomizationPage-BKj_r0rp.js +1 -0
  16. package/dist/dashboard/assets/CustomizationPage-BKj_r0rp.js.gz +0 -0
  17. package/dist/dashboard/assets/InjectSystemPromptPanel-B5nZvmf0.js +656 -0
  18. package/dist/dashboard/assets/InjectSystemPromptPanel-B5nZvmf0.js.gz +0 -0
  19. package/dist/dashboard/assets/{LoginPage-C4w7vea-.js → LoginPage-DkXb6dx-.js} +1 -1
  20. package/dist/dashboard/assets/LoginPage-DkXb6dx-.js.gz +0 -0
  21. package/dist/dashboard/assets/{ModelPicker-BtPcAfot.js → ModelPicker-DnQPSIA8.js} +1 -1
  22. package/dist/dashboard/assets/ModelPicker-DnQPSIA8.js.gz +0 -0
  23. package/dist/dashboard/assets/OverviewPage-Ci2wQKUz.js +1 -0
  24. package/dist/dashboard/assets/OverviewPage-Ci2wQKUz.js.gz +0 -0
  25. package/dist/dashboard/assets/ProviderDetailPage-odlgYrOg.js +5 -0
  26. package/dist/dashboard/assets/ProviderDetailPage-odlgYrOg.js.gz +0 -0
  27. package/dist/dashboard/assets/ProviderIcon-EfeloBn4.js +1 -0
  28. package/dist/dashboard/assets/ProviderIcon-EfeloBn4.js.gz +0 -0
  29. package/dist/dashboard/assets/ProvidersPage-CqFBQEB8.js +1 -0
  30. package/dist/dashboard/assets/ProvidersPage-CqFBQEB8.js.gz +0 -0
  31. package/dist/dashboard/assets/ProxyPage-5SnUSy3j.js +9 -0
  32. package/dist/dashboard/assets/ProxyPage-5SnUSy3j.js.gz +0 -0
  33. package/dist/dashboard/assets/QuotaPage-CmnMDzhP.js +1 -0
  34. package/dist/dashboard/assets/QuotaPage-CmnMDzhP.js.gz +0 -0
  35. package/dist/dashboard/assets/SettingsPage-BJtRZ4qD.js +1 -0
  36. package/dist/dashboard/assets/SettingsPage-BJtRZ4qD.js.gz +0 -0
  37. package/dist/dashboard/assets/{SetupPage-BQS0Rx_u.js → SetupPage-WTa0sktC.js} +1 -1
  38. package/dist/dashboard/assets/SetupPage-WTa0sktC.js.gz +0 -0
  39. package/dist/dashboard/assets/{StudioPage-TP7voAQ0.js → StudioPage-BfAb5-o_.js} +3 -3
  40. package/dist/dashboard/assets/StudioPage-BfAb5-o_.js.gz +0 -0
  41. package/dist/dashboard/assets/UsagePage-BNzeZ2Vk.js +56 -0
  42. package/dist/dashboard/assets/UsagePage-BNzeZ2Vk.js.gz +0 -0
  43. package/dist/dashboard/assets/api-keys-l29JROPv.js +1 -0
  44. package/dist/dashboard/assets/api-keys-l29JROPv.js.gz +0 -0
  45. package/dist/dashboard/assets/app--PJWq8_C.css +2 -0
  46. package/dist/dashboard/assets/app--PJWq8_C.css.gz +0 -0
  47. package/dist/dashboard/assets/{assert-C0YYuZJ4.js → assert-C0K-c4eo.js} +1 -1
  48. package/dist/dashboard/assets/assert-C0K-c4eo.js.gz +0 -0
  49. package/dist/dashboard/assets/{badge-B_DMpMJN.js → badge-DfbhQU4k.js} +1 -1
  50. package/dist/dashboard/assets/badge-DfbhQU4k.js.gz +0 -0
  51. package/dist/dashboard/assets/{banner-DMrMPmth.js → banner-BllT4LcF.js} +1 -1
  52. package/dist/dashboard/assets/banner-BllT4LcF.js.gz +0 -0
  53. package/dist/dashboard/assets/brand/login_register.jpg +0 -0
  54. package/dist/dashboard/assets/brand/login_register_initial.jpg +0 -0
  55. package/dist/dashboard/assets/brand/og_bansos.jpg +0 -0
  56. package/dist/dashboard/assets/brand/og_image.jpg +0 -0
  57. package/dist/dashboard/assets/{card-Be5Uzamx.js → card-CtkX66Wd.js} +1 -1
  58. package/dist/dashboard/assets/card-CtkX66Wd.js.gz +0 -0
  59. package/dist/dashboard/assets/chevron-left-Bf6CaTSJ.js +1 -0
  60. package/dist/dashboard/assets/chevron-left-Bf6CaTSJ.js.gz +0 -0
  61. package/dist/dashboard/assets/{cli-tools-muFRphDu.js → cli-tools-CQXVMn4P.js} +1 -1
  62. package/dist/dashboard/assets/cli-tools-CQXVMn4P.js.gz +0 -0
  63. package/dist/dashboard/assets/{clipboard-button-DOfrVmsi.js → clipboard-button-BQqBQd2n.js} +1 -1
  64. package/dist/dashboard/assets/clipboard-button-BQqBQd2n.js.gz +0 -0
  65. package/dist/dashboard/assets/{dialog-C9omdjQt.js → dialog-BEK7KP0H.js} +1 -1
  66. package/dist/dashboard/assets/dialog-BEK7KP0H.js.gz +0 -0
  67. package/dist/dashboard/assets/{drawer-Bm8WKaGY.js → drawer-BrTEZ8zu.js} +1 -1
  68. package/dist/dashboard/assets/drawer-BrTEZ8zu.js.gz +0 -0
  69. package/dist/dashboard/assets/entry-CHWxbS56.js +1 -0
  70. package/dist/dashboard/assets/entry-CHWxbS56.js.gz +0 -0
  71. package/dist/dashboard/assets/{entry-q5HwZ8nN.js → entry-EBHJ3DrE.js} +1 -1
  72. package/dist/dashboard/assets/entry-EBHJ3DrE.js.gz +0 -0
  73. package/dist/dashboard/assets/entry-LCS85hiq.js +4 -0
  74. package/dist/dashboard/assets/entry-LCS85hiq.js.gz +0 -0
  75. package/dist/dashboard/assets/{formatters-BTScrQpe.js → formatters-BAACA727.js} +1 -1
  76. package/dist/dashboard/assets/formatters-BAACA727.js.gz +0 -0
  77. package/dist/dashboard/assets/{github-badge-Orjuzemr.js → github-badge-YSCE4UC2.js} +1 -1
  78. package/dist/dashboard/assets/github-badge-YSCE4UC2.js.gz +0 -0
  79. package/dist/dashboard/assets/{index-cPrX5vn6.js → index-FQlj0HUT.js} +2 -2
  80. package/dist/dashboard/assets/index-FQlj0HUT.js.gz +0 -0
  81. package/dist/dashboard/assets/{inline-Dvux8w-0.js → inline-C4gU5iOL.js} +1 -1
  82. package/dist/dashboard/assets/inline-C4gU5iOL.js.gz +0 -0
  83. package/dist/dashboard/assets/{input-CU1Rgf3p.js → input-BHHX_Tmm.js} +1 -1
  84. package/dist/dashboard/assets/input-BHHX_Tmm.js.gz +0 -0
  85. package/dist/dashboard/assets/{layout-C_7A6H55.js → layout-CajrYUnD.js} +1 -1
  86. package/dist/dashboard/assets/{layout-C_7A6H55.js.gz → layout-CajrYUnD.js.gz} +0 -0
  87. package/dist/dashboard/assets/{list-row-CzFvogbj.js → list-row-Bx2I7XUe.js} +1 -1
  88. package/dist/dashboard/assets/list-row-Bx2I7XUe.js.gz +0 -0
  89. package/dist/dashboard/assets/live-DNNqRbj2.js +1 -0
  90. package/dist/dashboard/assets/live-DNNqRbj2.js.gz +0 -0
  91. package/dist/dashboard/assets/minimize-2-BTmMTK4o.js +1 -0
  92. package/dist/dashboard/assets/minimize-2-BTmMTK4o.js.gz +0 -0
  93. package/dist/dashboard/assets/{mutation-JJGQm8TD.js → mutation-Rr0TIvcj.js} +1 -1
  94. package/dist/dashboard/assets/mutation-Rr0TIvcj.js.gz +0 -0
  95. package/dist/dashboard/assets/network-C5UBACrJ.js +1 -0
  96. package/dist/dashboard/assets/network-C5UBACrJ.js.gz +0 -0
  97. package/dist/dashboard/assets/numbers-CUWAZK4J.js +2 -0
  98. package/dist/dashboard/assets/numbers-CUWAZK4J.js.gz +0 -0
  99. package/dist/dashboard/assets/pagination-Xsi9Qx04.js +1 -0
  100. package/dist/dashboard/assets/pagination-Xsi9Qx04.js.gz +0 -0
  101. package/dist/dashboard/assets/{query-keys-CLaNz8Xz.js → query-keys-BloKtaXJ.js} +1 -1
  102. package/dist/dashboard/assets/query-keys-BloKtaXJ.js.gz +0 -0
  103. package/dist/dashboard/assets/quota-DvQbD_sr.js +1 -0
  104. package/dist/dashboard/assets/quota-DvQbD_sr.js.gz +0 -0
  105. package/dist/dashboard/assets/{routing-BEH32QOC.js → routing-in0f_IfK.js} +1 -1
  106. package/dist/dashboard/assets/routing-in0f_IfK.js.gz +0 -0
  107. package/dist/dashboard/assets/{select-B3iTetHN.js → select-BLULNqTp.js} +2 -2
  108. package/dist/dashboard/assets/select-BLULNqTp.js.gz +0 -0
  109. package/dist/dashboard/assets/settings-CcGczj6I.js +1 -0
  110. package/dist/dashboard/assets/settings-CcGczj6I.js.gz +0 -0
  111. package/dist/dashboard/assets/shield-BMQNxORh.js +1 -0
  112. package/dist/dashboard/assets/shield-BMQNxORh.js.gz +0 -0
  113. package/dist/dashboard/assets/{stack-KrOrm7JZ.js → stack-D2lIR0tg.js} +1 -1
  114. package/dist/dashboard/assets/stack-D2lIR0tg.js.gz +0 -0
  115. package/dist/dashboard/assets/{state-DLS7IDMy.js → state-DTARjfiW.js} +1 -1
  116. package/dist/dashboard/assets/state-DTARjfiW.js.gz +0 -0
  117. package/dist/dashboard/assets/{switch-42ItvbAs.js → switch-c0oPSnqW.js} +1 -1
  118. package/dist/dashboard/assets/switch-c0oPSnqW.js.gz +0 -0
  119. package/dist/dashboard/assets/{useMutation-Dmnfh1Ue.js → useMutation-m8WGNPrz.js} +1 -1
  120. package/dist/dashboard/assets/useMutation-m8WGNPrz.js.gz +0 -0
  121. package/dist/dashboard/console-backgrounds.jpg +0 -0
  122. package/dist/dashboard/console-backgrounds_initial.jpg +0 -0
  123. package/dist/dashboard/index.html +2 -2
  124. package/dist/dashboard/index.html.gz +0 -0
  125. package/dist/dashboard/precompressed-assets.json +59 -57
  126. package/migrations/full-postgres/0027_qoder_identity_genuine_seed.sql +9 -10
  127. package/migrations/full-postgres/0032_add_provider_pool_accounts.sql +53 -0
  128. package/migrations/full-postgres/0033_add_api_key_ban.sql +9 -0
  129. package/migrations/full-postgres/0034_add_api_key_ban_until.sql +7 -0
  130. package/migrations/full-postgres/0035_align_pool_account_status_enum.sql +21 -0
  131. package/migrations/full-postgres/0036_system_prompt_presets_refs.sql +10 -0
  132. package/migrations/full-postgres/0037_share_concurrency_scope.sql +3 -0
  133. package/migrations/full-postgres/0038_add_accounts_quota_snapshot.sql +7 -0
  134. package/migrations/full-postgres/0039_add_telemetry_provider_model_indexes.sql +13 -0
  135. package/migrations/full-postgres/0040_add_models_dialect.sql +3 -0
  136. package/migrations/full-postgres/0041_add_daily_checkin_ledger_account_fk.sql +7 -0
  137. package/migrations/lite-sqlite3/0027_qoder_identity_genuine_seed.sql +9 -10
  138. package/migrations/lite-sqlite3/0032_add_provider_pool_accounts.sql +44 -0
  139. package/migrations/lite-sqlite3/0033_add_api_key_ban.sql +8 -0
  140. package/migrations/lite-sqlite3/0034_add_api_key_ban_until.sql +5 -0
  141. package/migrations/lite-sqlite3/0035_align_pool_account_status_enum.sql +6 -0
  142. package/migrations/lite-sqlite3/0036_system_prompt_presets_refs.sql +14 -0
  143. package/migrations/lite-sqlite3/0037_share_concurrency_scope.sql +3 -0
  144. package/migrations/lite-sqlite3/0038_add_accounts_quota_snapshot.sql +4 -0
  145. package/migrations/lite-sqlite3/0039_add_telemetry_provider_model_indexes.sql +6 -0
  146. package/migrations/lite-sqlite3/0040_add_models_dialect.sql +3 -0
  147. package/migrations/lite-sqlite3/0041_add_daily_checkin_ledger_account_fk.sql +26 -0
  148. package/package.json +4 -5
  149. package/scripts/cli/update.ts +15 -41
  150. package/src/boot/app.ts +67 -10
  151. package/src/boot/dependencies.ts +786 -732
  152. package/src/boot/lifecycle.ts +11 -12
  153. package/src/config/cooldowns.ts +34 -55
  154. package/src/config/network.ts +8 -4
  155. package/src/config/package-paths.ts +1 -1
  156. package/src/config/providers.ts +13 -14
  157. package/src/config/resources.ts +0 -5
  158. package/src/config/runtime.ts +2 -5
  159. package/src/config/spec.ts +26 -6
  160. package/src/console/api-keys/ban-constants.ts +38 -0
  161. package/src/console/api-keys/contracts.ts +92 -31
  162. package/src/console/api-keys/routes.ts +131 -6
  163. package/src/console/audit/routes.ts +3 -3
  164. package/src/console/auth/access.ts +4 -4
  165. package/src/console/auth/account-reset.ts +5 -5
  166. package/src/console/auth/audit-recorder.ts +1 -1
  167. package/src/console/auth/lockout.ts +7 -3
  168. package/src/console/auth/row-coercion.ts +57 -12
  169. package/src/console/auth/session-resolver.ts +1 -1
  170. package/src/console/auth/session-service.ts +8 -2
  171. package/src/console/auth/session.ts +40 -34
  172. package/src/console/auth/setup-service.ts +4 -3
  173. package/src/console/auth/user-store.ts +1 -1
  174. package/src/console/backup/9router-map.ts +0 -3
  175. package/src/console/backup/contracts.ts +11 -9
  176. package/src/console/backup/export-endpoint.ts +11 -12
  177. package/src/console/backup/nine-router.ts +1 -7
  178. package/src/console/backup/routes.ts +32 -36
  179. package/src/console/backup/service.ts +468 -468
  180. package/src/console/backup/store.ts +47 -29
  181. package/src/console/backup/validate.ts +11 -1
  182. package/src/console/cli-tools/contracts.ts +7 -7
  183. package/src/console/cli-tools/injectors/cline.ts +110 -110
  184. package/src/console/cli-tools/injectors/codex.ts +181 -181
  185. package/src/console/cli-tools/injectors/copilot.ts +108 -108
  186. package/src/console/cli-tools/injectors/driver.ts +247 -247
  187. package/src/console/cli-tools/injectors/jcode.ts +115 -115
  188. package/src/console/cli-tools/injectors/openclaw.ts +161 -161
  189. package/src/console/cli-tools/injectors/opencode.ts +153 -153
  190. package/src/console/cli-tools/routes.ts +4 -4
  191. package/src/console/cli-tools/service.ts +1 -1
  192. package/src/console/cli-tools/store.ts +2 -2
  193. package/src/console/console-router-guards.ts +2 -2
  194. package/src/console/console-router.ts +36 -4
  195. package/src/console/dashboard-assets.ts +9 -9
  196. package/src/console/data-management/routes.ts +3 -3
  197. package/src/console/domain-registration.ts +18 -4
  198. package/src/console/inferhub/client.ts +0 -7
  199. package/src/console/inferhub/key-store.ts +1 -1
  200. package/src/console/inferhub/normalize.ts +2 -2
  201. package/src/console/inferhub/routes.ts +1 -1
  202. package/src/console/inferhub/types.ts +1 -1
  203. package/src/console/observability/contracts.ts +5 -1
  204. package/src/console/observability/live.ts +34 -21
  205. package/src/console/observability/logs.ts +4 -4
  206. package/src/console/observability/performance.ts +3 -3
  207. package/src/console/observability/routes.ts +3 -3
  208. package/src/console/observability/sse.ts +62 -8
  209. package/src/console/observability/store.ts +36 -20
  210. package/src/console/providers/catalog/catalog-projections.ts +1 -1
  211. package/src/console/providers/catalog/contracts.ts +82 -22
  212. package/src/console/providers/catalog/model-operations.ts +1 -1
  213. package/src/console/providers/catalog/provider-operations.ts +110 -17
  214. package/src/console/providers/catalog/public-model-store.ts +0 -0
  215. package/src/console/providers/catalog/routes.ts +30 -6
  216. package/src/console/providers/catalog/store-helpers.ts +349 -326
  217. package/src/console/providers/catalog/store.ts +1662 -1292
  218. package/src/console/providers/detail/routes.ts +4 -4
  219. package/src/console/providers/oauth/callback-listener.ts +100 -18
  220. package/src/console/providers/oauth/routes.ts +95 -16
  221. package/src/console/quota/account-quota-global.ts +41 -3
  222. package/src/console/quota/account-quota-shared.ts +10 -3
  223. package/src/console/quota/account-quota-tenant.ts +285 -67
  224. package/src/console/quota/account-test-egress.ts +4 -4
  225. package/src/console/quota/daily-growth-job.ts +308 -0
  226. package/src/console/quota/native-identity.ts +2 -2
  227. package/src/console/quota/quota-view.ts +1 -1
  228. package/src/console/remote-support/routes.ts +1 -1
  229. package/src/console/remote-support/service.ts +2 -2
  230. package/src/console/routing/model/contracts.ts +1 -1
  231. package/src/console/routing/model/routes.ts +3 -3
  232. package/src/console/routing/pools/contracts.ts +13 -10
  233. package/src/console/routing/pools/import-parse.ts +3 -3
  234. package/src/console/routing/pools/operations.ts +14 -7
  235. package/src/console/routing/pools/probe-verdict.ts +8 -3
  236. package/src/console/routing/pools/relay-deploy.ts +12 -12
  237. package/src/console/routing/pools/routes.ts +3 -3
  238. package/src/console/routing/pools/store.ts +21 -1
  239. package/src/console/settings/contracts.ts +29 -2
  240. package/src/console/settings/routes.ts +3 -3
  241. package/src/console/settings/store.ts +20 -0
  242. package/src/console/share/share-router.ts +112 -44
  243. package/src/console/share/share-stats.ts +109 -39
  244. package/src/console/share/share-usage.ts +1 -1
  245. package/src/console/shared/errors.ts +2 -2
  246. package/src/console/shared/object.ts +18 -0
  247. package/src/console/shared/read-through-cache.ts +42 -6
  248. package/src/console/shared/transaction.ts +14 -0
  249. package/src/console/system-prompt/contracts.ts +89 -12
  250. package/src/console/system-prompt/routes.ts +98 -45
  251. package/src/console/updates/apply-update.ts +244 -0
  252. package/src/console/updates/routes.ts +256 -98
  253. package/src/main.ts +11 -5
  254. package/src/network/http2-bidi.ts +1 -1
  255. package/src/network/ip-utils.ts +298 -0
  256. package/src/network/outbound-fetch.ts +76 -5
  257. package/src/network/pool/agent.ts +7 -6
  258. package/src/network/pool/byte-accounting.ts +1 -1
  259. package/src/network/pool/resolver.ts +55 -51
  260. package/src/network/pool/selector.ts +150 -136
  261. package/src/network/pool-cooldown-store.ts +71 -16
  262. package/src/network/pool-health-machine.ts +22 -2
  263. package/src/network/rate-limit-verdict.ts +68 -0
  264. package/src/network/retry.ts +1 -1
  265. package/src/network/ssrf.ts +24 -75
  266. package/src/observability/event-log.ts +2 -2
  267. package/src/observability/event-query.ts +286 -214
  268. package/src/observability/log-sanitize.ts +1 -1
  269. package/src/observability/metrics.ts +4 -4
  270. package/src/observability/payload-capture.ts +6 -5
  271. package/src/observability/payload-store.ts +1 -1
  272. package/src/observability/runtime-metrics.ts +1 -1
  273. package/src/observability/telemetry-buffer.ts +12 -3
  274. package/src/observability/telemetry-reconciliation.ts +3 -3
  275. package/src/observability/telemetry-status.ts +1 -1
  276. package/src/observability/token-speed.ts +1 -1
  277. package/src/persistence/connection-url.ts +1 -1
  278. package/src/persistence/db-handle.ts +1 -1
  279. package/src/persistence/db-pg.ts +2 -2
  280. package/src/persistence/db-sqlite.ts +10 -5
  281. package/src/persistence/dialect.ts +5 -1
  282. package/src/persistence/readiness.ts +19 -7
  283. package/src/persistence/redis.ts +1 -1
  284. package/src/persistence/schema.ts +177 -45
  285. package/src/persistence/shared-cache.ts +2 -2
  286. package/src/persistence/sort-order.ts +1 -1
  287. package/src/persistence/stores/api-key-store.ts +109 -4
  288. package/src/persistence/stores/share-store.ts +20 -8
  289. package/src/persistence/stores/telemetry-store.ts +1 -1
  290. package/src/persistence/system-prompt-store.ts +63 -18
  291. package/src/persistence/tenant-preferences.ts +1 -1
  292. package/src/persistence/ttl-cache.ts +3 -3
  293. package/src/persistence/usage-flush.ts +7 -2
  294. package/src/protocol/canonical-model.ts +1 -1
  295. package/src/protocol/codecs.ts +12 -16
  296. package/src/protocol/dsml.ts +3 -49
  297. package/src/protocol/gateway-error.ts +6 -3
  298. package/src/protocol/image-clamp.ts +2 -2
  299. package/src/protocol/image-generation.ts +2 -2
  300. package/src/protocol/reasoning-text.ts +1 -1
  301. package/src/protocol/reasoning.ts +7 -52
  302. package/src/protocol/registry.ts +50 -15
  303. package/src/protocol/request/chat.ts +46 -19
  304. package/src/protocol/request/codex.ts +1 -1
  305. package/src/protocol/request/messages.ts +46 -14
  306. package/src/protocol/request/responses.ts +19 -10
  307. package/src/protocol/response/chat.ts +27 -12
  308. package/src/protocol/response/codex.ts +2 -2
  309. package/src/protocol/response/gemini.ts +1 -1
  310. package/src/protocol/response/messages.ts +6 -6
  311. package/src/protocol/response/omp-tool-call-markup.ts +1 -1
  312. package/src/protocol/response/responses.ts +3 -3
  313. package/src/protocol/sse-decoder.ts +64 -4
  314. package/src/protocol/stop-reason.ts +2 -2
  315. package/src/protocol/think-split.ts +347 -0
  316. package/src/protocol/translation/attachment-integrity.ts +3 -3
  317. package/src/protocol/translation/cache-controls.ts +8 -20
  318. package/src/protocol/translation/capabilities.ts +8 -8
  319. package/src/protocol/translation/dialect/deepseek.ts +10 -0
  320. package/src/protocol/translation/dialect/inband.ts +39 -0
  321. package/src/protocol/translation/dialect/model-thinking.ts +89 -0
  322. package/src/protocol/translation/dialect/qwen.ts +114 -0
  323. package/src/protocol/translation/dialect/resolver.ts +83 -0
  324. package/src/protocol/translation/dialect/rules.ts +93 -0
  325. package/src/protocol/translation/dialect/thinking-ladder.ts +245 -0
  326. package/src/protocol/translation/dialect/types.ts +58 -0
  327. package/src/protocol/translation/quirks.ts +18 -4
  328. package/src/protocol/translation/thinking.ts +12 -252
  329. package/src/protocol/translation/tool-id.ts +3 -3
  330. package/src/protocol/translation/tool-repair.ts +1 -1
  331. package/src/protocol/upstream/post-json.ts +6 -1
  332. package/src/protocol/upstream/send-claude-messages.ts +3 -3
  333. package/src/providers/adapters/base-adapter.ts +1 -1
  334. package/src/providers/adapters/compatible-adapter.ts +44 -5
  335. package/src/providers/authentication/jwt-validator.ts +0 -5
  336. package/src/providers/authentication/oauth-client.ts +254 -253
  337. package/src/providers/authentication/oauth-flow-store.ts +42 -7
  338. package/src/providers/authentication/oauth-refresh-service.ts +2 -2
  339. package/src/providers/bundled-discovery-providers.ts +6 -30
  340. package/src/providers/configured-openai-providers.ts +82 -97
  341. package/src/providers/configured-provider.ts +1 -1
  342. package/src/providers/default-registry.ts +23 -35
  343. package/src/providers/discovery/catalog/catalog-format.ts +13 -2
  344. package/src/providers/discovery/discovery-types.ts +0 -1
  345. package/src/providers/discovery/openai-model-discovery.ts +2 -2
  346. package/src/providers/discovery/probe-phases.ts +50 -15
  347. package/src/providers/discovery/probing-service.ts +18 -20
  348. package/src/providers/integrations/agentrouter/agentrouter.ts +14 -5
  349. package/src/providers/integrations/alibabacp/alibabacp.ts +2 -2
  350. package/src/providers/integrations/anthropic.ts +4 -11
  351. package/src/providers/integrations/antigravity/antigravity-oauth.ts +0 -1
  352. package/src/providers/integrations/antigravity/antigravity-protocol.ts +0 -14
  353. package/src/providers/integrations/antigravity/antigravity.ts +15 -26
  354. package/src/providers/integrations/buddy/buddy-catalog-shared.ts +2 -2
  355. package/src/providers/integrations/buddy/buddy-chat-shared.ts +20 -4
  356. package/src/providers/integrations/buddy/buddy-cli-headers.ts +19 -52
  357. package/src/providers/integrations/buddy/buddy-discovery-shared.ts +37 -6
  358. package/src/providers/integrations/buddy/buddy-identity.ts +108 -1
  359. package/src/providers/integrations/buddy/buddy-media.ts +16 -16
  360. package/src/providers/integrations/buddy/buddy-oauth-shared.ts +1 -1
  361. package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +1 -2
  362. package/src/providers/integrations/buddy/codebuddy/provider.ts +5 -16
  363. package/src/providers/integrations/buddy/codebuddy/shared.ts +30 -9
  364. package/src/providers/integrations/buddy/workbuddy/provider.ts +6 -10
  365. package/src/providers/integrations/buddy/workbuddy/shared.ts +21 -3
  366. package/src/providers/integrations/claude/claude-betas.ts +1 -1
  367. package/src/providers/integrations/claude/claude-cch.ts +148 -148
  368. package/src/providers/integrations/claude/claude-credentials.ts +1 -1
  369. package/src/providers/integrations/claude/claude-slow-lane-retry.ts +1 -1
  370. package/src/providers/integrations/claude/claude.ts +3 -10
  371. package/src/providers/integrations/claude/fallback-credit.ts +3 -7
  372. package/src/providers/integrations/cline/cline.ts +2 -2
  373. package/src/providers/integrations/cloudflareai/cloudflareai.ts +4 -7
  374. package/src/providers/integrations/codex/codex-device-code.ts +101 -101
  375. package/src/providers/integrations/codex/codex-errors.ts +1 -1
  376. package/src/providers/integrations/codex/codex-oauth.ts +0 -1
  377. package/src/providers/integrations/codex/codex.ts +14 -15
  378. package/src/providers/integrations/commandcode/commandcode.ts +76 -116
  379. package/src/providers/integrations/commandcode/quota.ts +132 -48
  380. package/src/providers/integrations/cursor/cursor-http2.ts +11 -14
  381. package/src/providers/integrations/cursor/cursor-quota.ts +2 -6
  382. package/src/providers/integrations/cursor/cursor-request.ts +0 -87
  383. package/src/providers/integrations/cursor/cursor-shared.ts +0 -4
  384. package/src/providers/integrations/cursor/cursor.ts +53 -20
  385. package/src/providers/integrations/cursor/protocol/framing.ts +0 -56
  386. package/src/providers/integrations/cursor/protocol/messages.ts +0 -96
  387. package/src/providers/integrations/factory/factory.ts +13 -13
  388. package/src/providers/integrations/gemini.ts +11 -11
  389. package/src/providers/integrations/github/github-discovery.ts +55 -10
  390. package/src/providers/integrations/github/github-oauth.ts +6 -5
  391. package/src/providers/integrations/github/github.ts +47 -20
  392. package/src/providers/integrations/grok/grok-oauth.ts +261 -261
  393. package/src/providers/integrations/grok/grok-quota-frame.ts +1 -1
  394. package/src/providers/integrations/grok/grok-quota.ts +16 -7
  395. package/src/providers/integrations/grok/grok-ratelimit-store.ts +0 -5
  396. package/src/providers/integrations/grok/grok-ratelimit.ts +3 -0
  397. package/src/providers/integrations/grok/grok.ts +88 -42
  398. package/src/providers/integrations/inferhub.ts +4 -11
  399. package/src/providers/integrations/kilo/kilo-oauth.ts +1 -1
  400. package/src/providers/integrations/kimi/kimi.ts +4 -11
  401. package/src/providers/integrations/kiro/kiro-cache.ts +31 -0
  402. package/src/providers/integrations/kiro/kiro-catalog.ts +31 -5
  403. package/src/providers/integrations/kiro/kiro-discovery.ts +47 -43
  404. package/src/providers/integrations/kiro/kiro-machine-id.ts +6 -3
  405. package/src/providers/integrations/kiro/kiro-oauth.ts +523 -255
  406. package/src/providers/integrations/kiro/kiro-quota.ts +35 -47
  407. package/src/providers/integrations/kiro/kiro-request.ts +8 -11
  408. package/src/providers/integrations/kiro/kiro-stream.ts +13 -0
  409. package/src/providers/integrations/kiro/kiro.ts +91 -78
  410. package/src/providers/integrations/muse/muse-oauth.ts +0 -1
  411. package/src/providers/integrations/muse/muse-quota.ts +0 -4
  412. package/src/providers/integrations/muse/muse.ts +6 -9
  413. package/src/providers/integrations/opencode/fingerprint.ts +7 -5
  414. package/src/providers/integrations/opencode/provider.ts +17 -24
  415. package/src/providers/integrations/perplexity.ts +2 -2
  416. package/src/providers/integrations/qoder/qoder-catalog.ts +132 -17
  417. package/src/providers/integrations/qoder/qoder-checkin.ts +26 -8
  418. package/src/providers/integrations/qoder/qoder-native-identity-store.ts +9 -8
  419. package/src/providers/integrations/qoder/qoder-oauth.ts +165 -25
  420. package/src/providers/integrations/qoder/qoder-quota.ts +48 -6
  421. package/src/providers/integrations/qoder/qoder-realm.ts +22 -28
  422. package/src/providers/integrations/qoder/qoder-sse.ts +159 -0
  423. package/src/providers/integrations/qoder/qoder-umid.ts +33 -38
  424. package/src/providers/integrations/qoder/qoder-virtual.ts +14 -2
  425. package/src/providers/integrations/qoder/qoder.ts +521 -63
  426. package/src/providers/integrations/shared/connect.ts +0 -54
  427. package/src/providers/integrations/shared/hash.ts +15 -0
  428. package/src/providers/integrations/shared/session-cache.ts +136 -0
  429. package/src/providers/integrations/tokenharbor.ts +2 -2
  430. package/src/providers/integrations/trae/trae-cache.ts +7 -87
  431. package/src/providers/integrations/trae/trae-catalog.ts +1 -6
  432. package/src/providers/integrations/trae/trae-protocol.ts +17 -8
  433. package/src/providers/integrations/trae/trae-realm.ts +0 -7
  434. package/src/providers/integrations/trae/trae.ts +59 -19
  435. package/src/providers/integrations/xai/xai-oauth.ts +5 -5
  436. package/src/providers/integrations/zcode/oauth.ts +1 -1
  437. package/src/providers/integrations/zcode-start-plan/gifts.ts +265 -0
  438. package/src/providers/integrations/zcode-start-plan/oauth.ts +154 -0
  439. package/src/providers/integrations/zcode-start-plan/provider.ts +131 -0
  440. package/src/providers/integrations/zcode-start-plan/quota.ts +152 -0
  441. package/src/providers/integrations/zcode-start-plan/zcode_prompt.json +10 -0
  442. package/src/providers/operations/account-error-classification.ts +131 -116
  443. package/src/providers/operations/account-health-service.ts +31 -39
  444. package/src/providers/operations/account-reset-service.ts +58 -1
  445. package/src/providers/operations/cli-platform.ts +31 -5
  446. package/src/providers/operations/client-version-resolver.ts +4 -6
  447. package/src/providers/operations/client-versions.ts +10 -7
  448. package/src/providers/operations/provider-catalog-seeder.ts +2 -0
  449. package/src/providers/operations/provider-catalog-service.ts +22 -6
  450. package/src/providers/operations/provider-credential-service.ts +35 -6
  451. package/src/providers/operations/upstream-deadline.ts +3 -3
  452. package/src/providers/provider-contracts.ts +65 -10
  453. package/src/providers/provider-credential-resolver.ts +1 -1
  454. package/src/providers/provider-metadata.ts +17 -2
  455. package/src/providers/provider-model-definition.ts +22 -1
  456. package/src/providers/provider-probe.ts +4 -2
  457. package/src/providers/provider-registry.ts +4 -4
  458. package/src/providers/provider-usage.ts +81 -16
  459. package/src/providers/quota/cache.ts +29 -0
  460. package/src/providers/quota/quota-contracts.ts +28 -10
  461. package/src/providers/quota/refresh.ts +29 -6
  462. package/src/providers/quota/snapshot.ts +94 -0
  463. package/src/providers/search/search-provider.ts +6 -8
  464. package/src/providers/search/search-providers.ts +2 -1
  465. package/src/security/abuse.ts +8 -8
  466. package/src/security/admission/reasons.ts +17 -17
  467. package/src/security/admission/service.ts +2 -2
  468. package/src/security/admission/ttl.ts +61 -61
  469. package/src/security/api-key-auth.ts +20 -8
  470. package/src/security/client-router-fingerprint.ts +4 -4
  471. package/src/security/credential-codec.ts +3 -3
  472. package/src/security/csrf.ts +2 -2
  473. package/src/security/ip-boundary.ts +16 -189
  474. package/src/security/model-access-rule.ts +1 -1
  475. package/src/security/outbound-headers.ts +18 -0
  476. package/src/shared/coerce.ts +19 -0
  477. package/src/shared/invoke-guard.ts +47 -0
  478. package/src/transport/contracts.ts +13 -4
  479. package/src/transport/dispatch/attempt-finalize.ts +5 -4
  480. package/src/transport/dispatch/attempt-loop.ts +10 -7
  481. package/src/transport/dispatch/capture-policy.ts +34 -37
  482. package/src/transport/dispatch/empty-stream.ts +28 -25
  483. package/src/transport/dispatch/generation.ts +33 -53
  484. package/src/transport/dispatch/leases.ts +8 -8
  485. package/src/transport/dispatch/native-shared.ts +127 -0
  486. package/src/transport/dispatch/proxy-request.ts +10 -8
  487. package/src/transport/dispatch/responses-compact.ts +35 -56
  488. package/src/transport/dispatch/retry-policy.ts +4 -67
  489. package/src/transport/dispatch/stream-bridge.ts +1 -1
  490. package/src/transport/dispatch/streaming/encoder.ts +18 -72
  491. package/src/transport/dispatch/streaming/finalize.ts +11 -10
  492. package/src/transport/dispatch/streaming/stream.ts +7 -3
  493. package/src/transport/dispatch/streaming/thinking-loop.ts +1 -1
  494. package/src/transport/dispatch/systemone.ts +27 -55
  495. package/src/transport/dispatch/token-count.ts +49 -46
  496. package/src/transport/dispatch/websearch.ts +24 -50
  497. package/src/transport/drain-endpoint.ts +3 -3
  498. package/src/transport/failure-policy.ts +19 -6
  499. package/src/transport/guards/auth-guard.ts +6 -11
  500. package/src/transport/guards/body-policy.ts +7 -7
  501. package/src/transport/guards/error-lifecycle.ts +18 -3
  502. package/src/transport/guards/guard-errors.ts +7 -3
  503. package/src/transport/guards/headroom-guard.ts +2 -2
  504. package/src/transport/guards/pipeline.ts +7 -7
  505. package/src/transport/guards/readiness-guard.ts +16 -2
  506. package/src/transport/guards/request-context.ts +15 -20
  507. package/src/transport/request/preparer.ts +306 -27
  508. package/src/transport/request/rtk/compress-request.ts +3 -3
  509. package/src/transport/request/rtk/rtk-constants.ts +2 -2
  510. package/src/transport/request/rtk/rtk-filters.ts +8 -8
  511. package/src/transport/request/state.ts +5 -20
  512. package/src/transport/request/system-prompt-injection.ts +74 -11
  513. package/src/transport/request/tenant-preferences.ts +22 -0
  514. package/src/transport/routing/candidate-index.ts +2 -2
  515. package/src/transport/routing/round-robin.ts +1 -1
  516. package/src/transport/routing/route-catalog.ts +113 -19
  517. package/src/transport/routing/route-model.ts +131 -38
  518. package/src/transport/routing/router.ts +154 -39
  519. package/src/transport/shutdown-notice.ts +21 -9
  520. package/src/transport/surface/adapters.ts +3 -3
  521. package/src/transport/surface/chat/encode.ts +7 -7
  522. package/src/transport/surface/chat/parse.ts +7 -7
  523. package/src/transport/surface/completion.ts +3 -3
  524. package/src/transport/surface/content-parts.ts +1 -1
  525. package/src/transport/surface/dialects.ts +2 -2
  526. package/src/transport/surface/messages/encode.ts +1 -1
  527. package/src/transport/surface/messages/parse.ts +3 -3
  528. package/src/transport/surface/responses/adapter.ts +1 -1
  529. package/src/transport/surface/responses/errors.ts +0 -9
  530. package/src/transport/vision-handoff-cache.ts +58 -0
  531. package/src/transport/vision-handoff.ts +376 -0
  532. package/src/workers/account-identity-sweep.ts +2 -2
  533. package/src/workers/buddy-growth-tasks.ts +392 -346
  534. package/src/workers/daily-checkin.ts +102 -49
  535. package/src/workers/maintenance-lease.ts +2 -35
  536. package/src/workers/oauth-refresh-sweep.ts +8 -1
  537. package/src/workers/quota-refresh-sweep.ts +120 -82
  538. package/src/workers/scheduler.ts +67 -4
  539. package/src/workers/sweep-runner.ts +24 -5
  540. package/src/workers/zcode-gift-claim.ts +108 -0
  541. package/dist/dashboard/assets/AccountCooldown-Bjo92Qat.js +0 -1
  542. package/dist/dashboard/assets/AccountCooldown-Bjo92Qat.js.gz +0 -0
  543. package/dist/dashboard/assets/BannedPage-2TJD37Hb.js.gz +0 -0
  544. package/dist/dashboard/assets/CliToolDetailPage-xefJNRmJ.js.gz +0 -0
  545. package/dist/dashboard/assets/CliToolsPage-Rj9iHQ3F.js.gz +0 -0
  546. package/dist/dashboard/assets/CombosPage-JEbYJFxw.js +0 -1
  547. package/dist/dashboard/assets/CombosPage-JEbYJFxw.js.gz +0 -0
  548. package/dist/dashboard/assets/ConsoleLogPage-C3xx_inM.js +0 -1
  549. package/dist/dashboard/assets/ConsoleLogPage-C3xx_inM.js.gz +0 -0
  550. package/dist/dashboard/assets/CustomizationPage-Dw306gJU.js +0 -1
  551. package/dist/dashboard/assets/CustomizationPage-Dw306gJU.js.gz +0 -0
  552. package/dist/dashboard/assets/HealthEventsModal-jaBo1TWV.js +0 -1
  553. package/dist/dashboard/assets/HealthEventsModal-jaBo1TWV.js.gz +0 -0
  554. package/dist/dashboard/assets/LoginPage-C4w7vea-.js.gz +0 -0
  555. package/dist/dashboard/assets/ModelPicker-BtPcAfot.js.gz +0 -0
  556. package/dist/dashboard/assets/OverviewPage-CGC5gNs1.js +0 -1
  557. package/dist/dashboard/assets/OverviewPage-CGC5gNs1.js.gz +0 -0
  558. package/dist/dashboard/assets/ProviderDetailPage-DQcthNGt.js +0 -4
  559. package/dist/dashboard/assets/ProviderDetailPage-DQcthNGt.js.gz +0 -0
  560. package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js +0 -1
  561. package/dist/dashboard/assets/ProviderIcon-FB6yS4v1.js.gz +0 -0
  562. package/dist/dashboard/assets/ProvidersPage-0OsXRGXe.js +0 -1
  563. package/dist/dashboard/assets/ProvidersPage-0OsXRGXe.js.gz +0 -0
  564. package/dist/dashboard/assets/ProxyPage-DdPc8slb.js +0 -9
  565. package/dist/dashboard/assets/ProxyPage-DdPc8slb.js.gz +0 -0
  566. package/dist/dashboard/assets/QuotaPage-Ci5O4BnH.js +0 -1
  567. package/dist/dashboard/assets/QuotaPage-Ci5O4BnH.js.gz +0 -0
  568. package/dist/dashboard/assets/SettingsPage-CDZjTt87.js +0 -1
  569. package/dist/dashboard/assets/SettingsPage-CDZjTt87.js.gz +0 -0
  570. package/dist/dashboard/assets/SetupPage-BQS0Rx_u.js.gz +0 -0
  571. package/dist/dashboard/assets/SortableList-BAL5npBC.js +0 -1
  572. package/dist/dashboard/assets/SortableList-BAL5npBC.js.gz +0 -0
  573. package/dist/dashboard/assets/StudioPage-TP7voAQ0.js.gz +0 -0
  574. package/dist/dashboard/assets/UsagePage-Cj_fEzhg.js +0 -56
  575. package/dist/dashboard/assets/UsagePage-Cj_fEzhg.js.gz +0 -0
  576. package/dist/dashboard/assets/api-keys-MbI7e7rt.js +0 -1
  577. package/dist/dashboard/assets/api-keys-MbI7e7rt.js.gz +0 -0
  578. package/dist/dashboard/assets/app-B2vyPoII.css +0 -2
  579. package/dist/dashboard/assets/app-B2vyPoII.css.gz +0 -0
  580. package/dist/dashboard/assets/assert-C0YYuZJ4.js.gz +0 -0
  581. package/dist/dashboard/assets/badge-B_DMpMJN.js.gz +0 -0
  582. package/dist/dashboard/assets/banner-DMrMPmth.js.gz +0 -0
  583. package/dist/dashboard/assets/card-Be5Uzamx.js.gz +0 -0
  584. package/dist/dashboard/assets/cli-tools-muFRphDu.js.gz +0 -0
  585. package/dist/dashboard/assets/clipboard-button-DOfrVmsi.js.gz +0 -0
  586. package/dist/dashboard/assets/dialog-C9omdjQt.js.gz +0 -0
  587. package/dist/dashboard/assets/drawer-Bm8WKaGY.js.gz +0 -0
  588. package/dist/dashboard/assets/entry-DS90gUks.js +0 -1
  589. package/dist/dashboard/assets/entry-DS90gUks.js.gz +0 -0
  590. package/dist/dashboard/assets/entry-j40LJotO.js +0 -4
  591. package/dist/dashboard/assets/entry-j40LJotO.js.gz +0 -0
  592. package/dist/dashboard/assets/entry-q5HwZ8nN.js.gz +0 -0
  593. package/dist/dashboard/assets/formatters-BTScrQpe.js.gz +0 -0
  594. package/dist/dashboard/assets/github-badge-Orjuzemr.js.gz +0 -0
  595. package/dist/dashboard/assets/index-cPrX5vn6.js.gz +0 -0
  596. package/dist/dashboard/assets/inline-Dvux8w-0.js.gz +0 -0
  597. package/dist/dashboard/assets/input-CU1Rgf3p.js.gz +0 -0
  598. package/dist/dashboard/assets/list-row-CzFvogbj.js.gz +0 -0
  599. package/dist/dashboard/assets/live-KaG15HLQ.js +0 -1
  600. package/dist/dashboard/assets/live-KaG15HLQ.js.gz +0 -0
  601. package/dist/dashboard/assets/minimize-2-D-MN8W8a.js +0 -1
  602. package/dist/dashboard/assets/minimize-2-D-MN8W8a.js.gz +0 -0
  603. package/dist/dashboard/assets/mutation-JJGQm8TD.js.gz +0 -0
  604. package/dist/dashboard/assets/numbers-BNDZnsUE.js +0 -2
  605. package/dist/dashboard/assets/numbers-BNDZnsUE.js.gz +0 -0
  606. package/dist/dashboard/assets/query-keys-CLaNz8Xz.js.gz +0 -0
  607. package/dist/dashboard/assets/quota-DZGyp0N5.js +0 -1
  608. package/dist/dashboard/assets/quota-DZGyp0N5.js.gz +0 -0
  609. package/dist/dashboard/assets/routing-BEH32QOC.js.gz +0 -0
  610. package/dist/dashboard/assets/select-B3iTetHN.js.gz +0 -0
  611. package/dist/dashboard/assets/settings-Dt3Eqbnq.js +0 -1
  612. package/dist/dashboard/assets/settings-Dt3Eqbnq.js.gz +0 -0
  613. package/dist/dashboard/assets/stack-KrOrm7JZ.js.gz +0 -0
  614. package/dist/dashboard/assets/state-DLS7IDMy.js.gz +0 -0
  615. package/dist/dashboard/assets/stream-retry-5QnVuSSo.js +0 -1
  616. package/dist/dashboard/assets/stream-retry-5QnVuSSo.js.gz +0 -0
  617. package/dist/dashboard/assets/switch-42ItvbAs.js.gz +0 -0
  618. package/dist/dashboard/assets/useMutation-Dmnfh1Ue.js.gz +0 -0
  619. package/src/network/response-headers.ts +0 -39
  620. package/src/protocol/response/dsml.ts +0 -38
  621. package/src/providers/integrations/xiaomi-mimo/think-stream.ts +0 -229
  622. /package/dist/dashboard/assets/{app-wvwCi4Eh.js → app-Bgd3zANQ.js} +0 -0
  623. /package/dist/dashboard/assets/{app-wvwCi4Eh.js.gz → app-Bgd3zANQ.js.gz} +0 -0
  624. /package/dist/dashboard/assets/brand/{og_bansos.webp → og_bansos_initial.webp} +0 -0
  625. /package/dist/dashboard/assets/brand/{og_image.webp → og_image_initial.webp} +0 -0
@@ -1,732 +1,786 @@
1
-
2
- import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
3
- import { poolMaxFromEnv } from "../persistence/db-pg";
4
- import type { CartethyiaDatabase } from "../persistence/db-handle";
5
- import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
6
- import { resolveProcessTopology, type ProcessTopology } from "../config/runtime";
7
- import { createRedisMaintenanceLeaseStore } from "../workers/maintenance-lease";
8
- import { resolveDbMode } from "../persistence/db-mode";
9
- import type { RedisClient } from "../persistence/redis";
10
- import {
11
- bundledModelCatalog,
12
- liveProviderUpstreamHosts,
13
- markCatalogSeeded,
14
- refreshActiveProviderCatalogs,
15
- registerByokProviders,
16
- retireUnbundledProviders,
17
- seedBundledProviders,
18
- } from "../providers/operations/provider-catalog-service";
19
- import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
20
- import { createDefaultProviderRegistry } from "../providers/default-registry";
21
- import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
22
- import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
23
- import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
24
- import { accountIdentitySweep } from "../workers/account-identity-sweep";
25
- import { pushStructuredConsoleLog } from "../observability/log-ring";
26
- import { seedBundledModels } from "../providers/operations/provider-catalog-seeder";
27
- import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
28
- import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
29
- import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
30
- import { RoutingEngine } from "../transport/routing/router";
31
- import { RedisAdmissionController } from "../transport/routing/admission";
32
- import { ApiKeyAdmissionService } from "../security/admission/service";
33
- import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
34
- import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
35
- import { sweepLeases } from "../security/admission/lease-sweep";
36
- import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
37
- import { UsageFlushBuffer } from "../persistence/usage-flush";
38
- import { setPendingUsageSource } from "../security/api-key-auth";
39
- import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
40
- import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
41
- import { deleteExpiredSessions } from "../console/auth/session-service";
42
- import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
43
- import { checkReadiness } from "../persistence/readiness";
44
- import { ProxyRequestPreparer } from "../transport/request/preparer";
45
- import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
46
- import {
47
- PoolAgentResolver,
48
- ValidatedNetworkBindingFactory,
49
- } from "../network/pool/resolver";
50
- import { NetworkPoolSelector } from "../network/pool/selector";
51
- import { restorePoolProbeVerdicts, restoreProviderCooldownMarkers } from "../network/pool-cooldown-store";
52
- import { ScheduledTaskRegistry } from "../workers/scheduler";
53
- import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
54
- import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
55
- import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
56
- import {
57
- buddyAdoptTask,
58
- buddyActivityReportTask,
59
- buddyStreakTask,
60
- syncBuddyUsage,
61
- } from "../workers/buddy-growth-tasks";
62
- import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
63
- import { quotaCacheSize } from "../providers/quota/cache";
64
- import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
65
- import { preferencesReaderFor } from "../persistence/tenant-preferences";
66
- import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
67
- import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
68
- import { TelemetryPayloadCapture } from "../observability/payload-capture";
69
- import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
70
- import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
71
- import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
72
- import {
73
- resolveIpRateLimit,
74
- } from "../config/security";
75
- import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
76
- import type { TrustedProxyBoundary } from "../config/network";
77
- import { resolveTelemetryRetentionDays } from "../config/telemetry";
78
- import { resolveConcurrencyTolerance } from "../config/resources";
79
- import { CONFIG_SPEC, readInt, readList } from "../config/spec";
80
- import type { ProviderAdapter } from "../providers/provider-contracts";
81
- import type { ProviderRegistry } from "../providers/provider-registry";
82
- import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
83
- import type { ReadinessCheckResult } from "../persistence/readiness";
84
- import { log } from "../observability/logger";
85
- import { refreshProviderClientVersions } from "../providers/operations/client-versions";
86
-
87
-
88
- export interface ProductionDeps {
89
- db: CartethyiaDatabase;
90
- redis: RedisClient | undefined;
91
- snapshotService: InMemoryRouteSnapshotService;
92
- /** Live routing admission snapshot scoped to one provider/tenant for console reads. */
93
- readRoutingAccountInflight: (
94
- providerId: string,
95
- tenantId: string | null,
96
- ) => Promise<readonly { accountId: string; inflight: number }[]>;
97
- /** On-demand adapter resolution keeps heavy providers out of the boot path. */
98
- resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
99
- bundledModelCatalog: BundledProviderCatalog;
100
- /** Live lookup of a provider's SSRF-validated upstream host. */
101
- byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
102
- proxyPreparer: ProxyRequestPreparer;
103
- networkBindingFactory: ValidatedNetworkBindingFactory;
104
- ipAbuseProtection: IpAbuseProtectionService;
105
- readiness: () => Promise<ReadinessCheckResult>;
106
- scheduledTasks: ScheduledTaskRegistry;
107
- poolAgentResolver: PoolAgentResolver;
108
- poolSelector: NetworkPoolSelector;
109
- telemetryBuffer: TelemetryBatchBuffer;
110
- usageFlush: UsageFlushBuffer;
111
- healthEventFlush: HealthEventFlushBuffer;
112
- trustedProxyBoundary: TrustedProxyBoundary;
113
- providerRegistry: ProviderRegistry;
114
- resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
115
- oauthRefreshService: OAuthRefreshService;
116
- admissionService: ApiKeyAdmissionService;
117
- /** Graduated strikes for repeated invalid-model requests. */
118
- /** Memory sampler; in-flight source is bound once the request store exists. */
119
- runtimeMetricsSampler: RuntimeMetricsSampler;
120
- }
121
-
122
- /**
123
- * Pause between accounts in the OAuth refresh sweep. The token endpoints
124
- * rate-limit a burst of refreshes even at low concurrency.
125
- */
126
- const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
127
-
128
- /**
129
- * Client identities are refreshed out of band so provider dispatch never waits
130
- * on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
131
- */
132
- // Matched to the resolver's own 30 min cache TTL: a shorter interval makes
133
- // every second pass a pure cache read that cannot discover anything new.
134
- const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
135
- const CLIENT_VERSION_INITIAL_DELAY_MS = 10_000;
136
-
137
- /** Bound on the boot-time Redis probe: a start must fail, not hang. */
138
- const REDIS_PROBE_TIMEOUT_MS = 2_000;
139
-
140
- async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
141
- const { promise: timeout, reject } = Promise.withResolvers<never>();
142
- const timer = setTimeout(() => reject(new Error("timeout")), ms);
143
- try {
144
- return await Promise.race([promise, timeout]);
145
- } finally {
146
- clearTimeout(timer);
147
- }
148
- }
149
-
150
- /** Bounded Redis reachability probe: fails rather than hangs. */
151
- async function redisIsReachable(): Promise<boolean> {
152
- const client = resolveRedisClient();
153
- if (!client) return false;
154
- try {
155
- const result = await withTimeout(client.ping(), REDIS_PROBE_TIMEOUT_MS);
156
- return result === "PONG";
157
- } catch {
158
- return false;
159
- }
160
- }
161
-
162
- /**
163
- * Refuses to start a multi-process deployment whose shared state is not
164
- * actually shared.
165
- *
166
- * Every process shares one port via `SO_REUSEPORT`, so the kernel balances
167
- * across them — and each process keeps its own in-memory admission counters,
168
- * quota cache, and cooldown map. Those are only correct when they live in
169
- * Redis. Starting anyway does not fail loudly; it enforces every per-key limit
170
- * N times over, which then reads as "the limit did not work".
171
- *
172
- * Redis is checked by PING with a hard bound, not merely for being configured:
173
- * a configured-but-unreachable Redis is the failure mode that matters, and an
174
- * unbounded connect at boot would hang the start instead of failing it.
175
- */
176
- async function assertMultiProcessPreconditions(
177
- topology: ProcessTopology,
178
- redisReachable: () => Promise<boolean>,
179
- ): Promise<void> {
180
- if (!topology.multiProcess) return;
181
- if (process.platform !== "linux") {
182
- throw Object.assign(
183
- new Error(
184
- `CARTETHYIA_PROCESS_COUNT=${topology.count} requires Linux (SO_REUSEPORT); ` +
185
- `this process is on ${process.platform}`,
186
- ),
187
- { code: "multi_process_unsupported_platform" },
188
- );
189
- }
190
- if (resolveDbMode() !== "full") {
191
- throw Object.assign(
192
- new Error(
193
- `CARTETHYIA_PROCESS_COUNT=${topology.count} requires CARTETHYIA_DB_MODE=full (PostgreSQL); ` +
194
- "two processes on one SQLite file corrupt it — there is no cross-process lock in the driver",
195
- ),
196
- { code: "multi_process_requires_postgres" },
197
- );
198
- }
199
- const reachable = await redisReachable();
200
- if (!reachable) {
201
- throw Object.assign(
202
- new Error(
203
- `CARTETHYIA_PROCESS_COUNT=${topology.count} requires a REACHABLE Redis: admission counters, ` +
204
- "quotas, and cooldowns are per-process without one, so every per-key limit would be " +
205
- `enforced ${topology.count} times over`,
206
- ),
207
- { code: "multi_process_requires_redis" },
208
- );
209
- }
210
- }
211
-
212
- export async function buildProductionDeps(): Promise<ProductionDeps> {
213
- const startupStartedAt = performance.now();
214
- const startupMemoryBefore = process.memoryUsage();
215
- // Resolved first and validated before any connection is opened: a
216
- // half-configured multi-process deploy that starts anyway looks healthy
217
- // until a per-key limit is enforced N times over.
218
- const topology = resolveProcessTopology();
219
- await assertMultiProcessPreconditions(topology, redisIsReachable);
220
- const handle = await measureStartupPhase("database-connect", bootDatabase);
221
- const db = handle.db;
222
- // One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
223
- const redis = resolveRedisClient();
224
- const ssrfPolicy = resolveSsrfPolicy();
225
- await measureStartupPhase("database-migrations", async () => {
226
- await ensureMigrated();
227
- // Pool sizing is an external-server concern; the embedded backend has no
228
- // connection budget to fit into.
229
- if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv(), topology.count);
230
- });
231
- await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
232
- // Boot-only: retire global rows for providers the bundle has dropped, so a
233
- // retired provider cannot linger as a card the console can no longer delete.
234
- // Kept off the test harness path, which shares one database across suites and
235
- // installs its own global fixtures.
236
- await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
237
- const registry = createDefaultProviderRegistry();
238
- // Live view, not a snapshot: custom providers registered after boot (or
239
- // edited from the console) must resolve their SSRF binding immediately.
240
- const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
241
- await measureStartupPhase("byok-registration", () =>
242
- registerByokProviders(registry, db, ssrfPolicy),
243
- );
244
- const bundledCatalog = await measureStartupPhase("bundled-model-catalog", () =>
245
- bundledModelCatalog(registry),
246
- );
247
- await measureStartupPhase("bundled-model-seed", () =>
248
- seedBundledModels(db, bundledCatalog.modelsByProvider),
249
- );
250
- // The boot seed above is unconditional, so record it: the snapshot builder's
251
- // first rebuild would otherwise re-seed an identical catalog.
252
- markCatalogSeeded();
253
- const snapshotService = new InMemoryRouteSnapshotService(
254
- createDatabaseSnapshotBuilder(db, () => refreshActiveProviderCatalogs(db, registry)),
255
- );
256
- const routingEngine = new RoutingEngine(
257
- redis ? new RedisAdmissionController(redis) : undefined,
258
- redis,
259
- );
260
- // Peer processes' settings mutations arrive here. Subscribed once for the
261
- // process: the revision counter is process-global, and the handler only
262
- // moves it forward.
263
- subscribePeerSettingsRevisions();
264
- const usageFlush = new UsageFlushBuffer(db);
265
- // Auth reads lifetime usage to seed the admission counter. The persisted
266
- // value lags by up to one flush interval, so hand it the in-flight deltas
267
- // too — otherwise the seed is systematically under what has actually been
268
- // spent, worst exactly when the key is busiest.
269
- setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
270
- // Health audit rows are append-only and read by nobody in dispatch, so they
271
- // are buffered out of the request. The status write they accompany stays
272
- // synchronous — routing reads it.
273
- const healthEventFlush = new HealthEventFlushBuffer(db);
274
- setHealthEventBuffer(healthEventFlush);
275
- const admissionStore = redis
276
- ? new RedisAdmissionCounterStore(redis)
277
- : new InMemoryAdmissionCounterStore();
278
- const admissionService = new ApiKeyAdmissionService(
279
- admissionStore,
280
- undefined,
281
- // The per-tenant concurrency cap is a `console_settings` preference, read
282
- // on every admitted request. Reading it directly cost one Postgres round
283
- // trip per request (multiplied by each failover candidate); the shared
284
- // revision-keyed reader answers from memory and converges the moment an
285
- // operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
286
- // revision every cached key embeds.
287
- (tenantId) =>
288
- preferencesReaderFor(db)
289
- .readPreferences(tenantId)
290
- .then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
291
- ({ apiKeyId, delta }) => {
292
- // Coalesced: one UPDATE per key per flush instead of one per request.
293
- // The admission counter is the live total; this only persists it.
294
- usageFlush.enqueue(apiKeyId, delta);
295
- return Promise.resolve();
296
- },
297
- // Fresh lifetime reader — invoked by the store only when it is about to
298
- // seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
299
- // snapshot value cannot freeze a low baseline in place. Family total: the
300
- // row plus every child (share recipients admit under the parent id).
301
- async (apiKeyId) => {
302
- const store = new DrizzleApiKeyStore(db);
303
- const row = await store.findActiveById(apiKeyId);
304
- if (!row) return undefined;
305
- const children = await store.sumChildrenConsumed(apiKeyId);
306
- return (row.lifetimeTokensConsumed ?? 0) + children;
307
- },
308
- // Slack above every key's concurrency limit. A tenant preference an
309
- // operator sets in the runtime settings modal, falling back to the
310
- // deployment default so an unset row does not silently lose the allowance.
311
- (tenantId) =>
312
- preferencesReaderFor(db)
313
- .readPreferences(tenantId)
314
- .then((prefs) => prefs?.concurrencyTolerance ?? resolveConcurrencyTolerance()),
315
- );
316
- const ipStore = redis
317
- ? new RedisIpAbuseStore(redis)
318
- : new InMemoryIpAbuseStore();
319
- const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
320
- maxRequestsPerWindow: resolveIpRateLimit(),
321
- });
322
-
323
- const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
324
- // One list covers every periodic task: a small deployment can cut the
325
- // heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
326
- // without the codebase growing a flag per task.
327
- // Leases coordinate the SHARED sweeps across processes. Absent Redis there is
328
- // nothing to coordinate with, and the registry's local one-at-a-time rule is
329
- // the correct (and only) guard.
330
- const scheduledTasks = new ScheduledTaskRegistry(
331
- readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
332
- {
333
- ...(redis ? { lease: createRedisMaintenanceLeaseStore(redis) } : {}),
334
- },
335
- );
336
- if (redis) {
337
- scheduledTasks.register({
338
- name: "lease-sweep",
339
- intervalMs: 30_000,
340
- run: () => sweepLeases(redis),
341
- });
342
- }
343
- scheduledTasks.register({
344
- name: "account-health-sweep",
345
- intervalMs: 30_000,
346
- // Offsets the pass from lease-sweep and runtime-metrics, which share the
347
- // same cadence: under the registry's single-flight gate, aligned ticks
348
- // make the 30s tasks skip each other when one runs long.
349
- initialDelayMs: 5_000,
350
- // Recovers cooldowns deployment-wide and invalidates the shared snapshot.
351
- shared: true,
352
- run: async () => {
353
- const recovered = await sweepExpiredCooldowns(db);
354
- const poolRecovered = await sweepExpiredPoolCooldowns(db);
355
- if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
356
- },
357
- });
358
- const payloadCapture = new TelemetryPayloadCapture(db);
359
- const telemetryStore = new DrizzleTelemetryStore(db);
360
- scheduledTasks.register({
361
- name: "telemetry-payload-cleanup",
362
- intervalMs: 15 * 60_000,
363
- // Deletes payload frames for every tenant: two passes over the same
364
- // batches is duplicated work, not speedup.
365
- shared: true,
366
- run: () => payloadCapture.cleanupExpired(),
367
- });
368
- scheduledTasks.register({
369
- name: "console-session-sweep",
370
- // Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
371
- // 48 — each pass a full scan of the session table for rows that cannot yet
372
- // be expired. Half the session lifetime is the point where a pass can
373
- // actually find work, and still leaves a full grace period after it.
374
- intervalMs: 12 * 60 * 60_000,
375
- shared: true,
376
- run: () => deleteExpiredSessions(db),
377
- });
378
- scheduledTasks.register({
379
- name: "telemetry-retention",
380
- intervalMs: 6 * 60 * 60_000,
381
- // Retention deletes rows and files for the whole deployment; a second
382
- // concurrent pass would churn the same batches.
383
- shared: true,
384
- run: () =>
385
- telemetryStore.pruneTelemetry(
386
- new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
387
- ),
388
- });
389
- const telemetryBuffer = new TelemetryBatchBuffer(db);
390
- const poolSelector = new NetworkPoolSelector(redis);
391
- // Replay the provider cooldown markers the previous process recorded
392
- // (`network_pools.provider_cooldowns`). Without this a restart made dispatch
393
- // forget every egress the upstream was still refusing: the flags are
394
- // process-local otherwise, so the pool re-entered rotation and took 429s the
395
- // first request after boot re-discovered. Expired entries are dropped by the
396
- // restore itself. Best-effort: a failed read leaves selection matching what
397
- // a fresh deployment would do.
398
- try {
399
- const restored = await restoreProviderCooldownMarkers(poolSelector, db);
400
- const replayed = await restorePoolProbeVerdicts(poolSelector, db);
401
- if (restored > 0 || replayed > 0) {
402
- log.info("[pool-selector] restored provider cooldown state", { mirrored: restored, probeVerdictsApplied: replayed });
403
- }
404
- } catch (error) {
405
- log.warn("[pool-selector] provider cooldown restore failed", { error: String(error) });
406
- }
407
- const poolAgentResolver = new PoolAgentResolver(
408
- new DrizzleNetworkPoolLoader(db),
409
- { ssrfPolicy, db },
410
- );
411
- const networkBindingFactory = new ValidatedNetworkBindingFactory(
412
- ssrfPolicy,
413
- undefined,
414
- poolAgentResolver,
415
- );
416
- // The account-inflight reader the console queries. It lives after the deps
417
- // it closes over (telemetry buffer, binding factory), so the closure never
418
- // touches a TDZ binding; it also reuses one catalog store per tenant instead
419
- // of constructing a fresh store object on every console read.
420
- let lastInflightTenant = "";
421
- let lastInflightStore: DrizzleProviderCatalogStore | undefined;
422
- const readRoutingAccountInflight = async (
423
- providerId: string,
424
- tenantId: string | null,
425
- ): Promise<readonly { accountId: string; inflight: number }[]> => {
426
- if (providerId === "" || tenantId === null) return [];
427
- if (tenantId !== lastInflightTenant || lastInflightStore === undefined) {
428
- lastInflightStore = new DrizzleProviderCatalogStore(db, {
429
- telemetryBuffer,
430
- bundledModelCatalog: bundledCatalog,
431
- providerRegistry: registry,
432
- outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
433
- snapshotInvalidator: snapshotService,
434
- });
435
- lastInflightTenant = tenantId;
436
- }
437
- const accounts = await lastInflightStore.listAccounts(tenantId, providerId);
438
- const owned = new Set(accounts.map((account) => account.id));
439
- // Scoped read: only the provider's own candidates for this tenant are
440
- // queried, so the console read costs one exact MGET over the accounts it
441
- // is about to report on — not a fleet-wide `KEYS` scan whose cost is the
442
- // size of every bucket every process ever wrote.
443
- const routeSnapshot = await snapshotService.getSnapshot();
444
- const ownedCandidates = routeSnapshot.candidates.filter(
445
- (candidate) =>
446
- candidate.provider_id === providerId &&
447
- candidate.provider_account_id !== undefined &&
448
- owned.has(candidate.provider_account_id),
449
- );
450
- const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
451
- const totals = new Map<string, number>();
452
- for (const [bucket, count] of snapshot) {
453
- const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
454
- if (!owned.has(accountId)) continue;
455
- totals.set(accountId, (totals.get(accountId) ?? 0) + count);
456
- }
457
- return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
458
- };
459
- // Runtime gauges (memory + bounded-collection sizes) sampled on the shared
460
- // maintenance cadence. The sampler reads live sizes through closures so the
461
- // observability layer never holds direct references to routing/pool state.
462
- const runtimeMetricsSampler = new RuntimeMetricsSampler({
463
- routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
464
- ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
465
- quotaCacheEntries: () => quotaCacheSize(),
466
- modelCatalogEntries: () => modelCatalogCacheSize(),
467
- // Late-bound via bindInFlight once the request state store exists.
468
- inFlight: () => 0,
469
- // The buffer is built below; the closure keeps the sampler from holding a
470
- // reference during construction.
471
- telemetryBacklog: () => telemetryBuffer?.snapshot(),
472
- });
473
- // One sample NOW, not on the first tick. The headroom guard reads
474
- // `memorySnapshot()` on every gateway request and treats `undefined` as
475
- // "unknown" — so without this the guard would spend its first interval
476
- // (seconds, at the configured cadence) unable to shed at all, which is
477
- // exactly the window a cold start is most likely to be short of memory.
478
- runtimeMetricsSampler.sample();
479
- scheduledTasks.register({
480
- name: "runtime-metrics",
481
- intervalMs: readInt(
482
- "CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
483
- CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
484
- ),
485
- run: () => runtimeMetricsSampler.sample(),
486
- });
487
- scheduledTasks.register({
488
- name: "pool-agent-sweep",
489
- // `resolveAgent` already self-sweeps every 30 s, so a 1 min timer adds a
490
- // second pass that almost always finds nothing to release. 5 min is well
491
- // inside the idle timeout this exists to enforce, and outside the path
492
- // that already runs.
493
- intervalMs: 5 * 60_000,
494
- run: () => poolAgentResolver.sweepIdleAgents(),
495
- });
496
- const oauthRefreshService = new OAuthRefreshService(db);
497
- // Converge accounts parked on a dead refresh grant: an account whose stored
498
- // access token is still usable becomes a static token (and returns to
499
- // rotation) instead of staying disabled for re-auth.
500
- await measureStartupPhase("static-token-reconciliation", () =>
501
- reconcileStaticTokenAccounts(db),
502
- );
503
- // Fills in the identity a batch import deliberately skipped, so a paste of
504
- // many credentials is one instant write and the email/label arrive shortly
505
- // after rather than blocking the import. Purely CPU on an already-stored
506
- // credential — no upstream call — so it runs often and cheaply.
507
- scheduledTasks.register({
508
- name: "account-enrichment-sweep",
509
- // Converges via `identity_checked_at`: once every stored credential has
510
- // been parsed, the listing query returns nothing — a 30 s timer would run
511
- // that empty query forever, two million times a year. 10 min is still
512
- // instant for the case that matters (a batch import, where the label and
513
- // email show up on the next tick) and costs nothing when idle.
514
- intervalMs: 10 * 60_000,
515
- run: () => accountIdentitySweep({ db }),
516
- });
517
- scheduledTasks.register({
518
- name: "oauth-refresh-sweep",
519
- intervalMs: 60_000,
520
- run: () =>
521
- oauthRefreshSweep({
522
- loadDueAccounts: () => loadDueOAuthAccounts(db),
523
- refreshService: oauthRefreshService,
524
- resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
525
- // Sequential with a pause between accounts: the OAuth token endpoints
526
- // rate-limit a burst of refreshes, and a pass over a handful of
527
- // accounts must not look like one. Per-account outcomes are already
528
- // pushed to the Console Log ring by `OAuthRefreshService`; this only
529
- // covers an error that escaped before reaching it.
530
- interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
531
- onAccountError: (accountId, providerId, error) => {
532
- log.error(
533
- `[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
534
- error as Error,
535
- );
536
- pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
537
- event: "token_refresh",
538
- accountId,
539
- providerId,
540
- errorCode: "refresh_sweep_failed",
541
- });
542
- },
543
- }),
544
- });
545
-
546
- // Client versions are monitor-only metadata; let the listener settle before
547
- // starting their external fetches. The scheduled task keeps the cache warm.
548
- scheduledTasks.register({
549
- name: "client-version-monitor",
550
- intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
551
- initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
552
- run: refreshProviderClientVersions,
553
- });
554
-
555
- const quotaResolveCredential = createAccountSecretResolver({
556
- db,
557
- resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
558
- refreshService: oauthRefreshService,
559
- });
560
- // Keeps every account's cached quota warm so opening the Quota page is a
561
- // cache read. Without this the only refreshes are user-triggered, so the
562
- // first open after an expiry always blocks on the provider.
563
- //
564
- // It also carries the daily check-in ride-along: each account refreshed
565
- // here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
566
- // the same pass (one attempt per account per day, guarded by a day
567
- // marker), so no second timer duplicates the credential resolution.
568
- //
569
- // Registered in **every** mode, not only when Redis is present: gating it on
570
- // Redis would cost Lite proactive quota warming and the check-in ride-along
571
- // above that claims the once-per-day WorkBuddy/CodeBuddy credit grant. The
572
- // whole sweep chain accepts `RedisClient | undefined` and falls back to the
573
- // in-process TTL cache and an in-process day marker, so Redis is an
574
- // optimization here rather than a precondition — which is exactly what
575
- // Lite is for.
576
- scheduledTasks.register({
577
- name: "quota-refresh-sweep",
578
- intervalMs: 60_000,
579
- // Staggered off the oauth/lease/health ticks so the three sweeps that
580
- // strike providerAccounts do not land together every minute.
581
- initialDelayMs: 20_000,
582
- run: () =>
583
- quotaRefreshSweep({
584
- db,
585
- redis,
586
- providerRegistry: registry,
587
- resolveCredential: quotaResolveCredential,
588
- snapshotInvalidator: snapshotService,
589
- // Without this the sweep hands each collector the raw stored secret.
590
- // For `cline` that means an `sk_...` key reaching the OAuth branch,
591
- // which prepends `workos:` and is refused with "please make sure
592
- // you're using the latest version of Cline and re-authenticate" — so
593
- // every sweep cools a working account with `quota_fetch_failed` while
594
- // the same account refreshes fine from the console, which binds the
595
- // same marker through `createQuotaRefreshDeps`.
596
- markApiKeyCredential: markProviderApiKeyCredential,
597
- // Below the 60 s interval: a budget longer than the tick guarantees
598
- // skip/burst under load instead of steady progress.
599
- passBudgetMs: 45_000,
600
- // The check-in ride-along rotates egress per account: each account
601
- // gets the next active pool in its tenant's rotation so check-ins
602
- // spread across IPs instead of sharing one direct egress. No pool
603
- // (or an unreadable pool table) falls back to direct egress, and a
604
- // tenant never borrows another tenant's pool.
605
- checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
606
- // Per-account lines and the pass summary are emitted by the sweep
607
- // itself at `info` level, so they survive the production LOG_LEVEL.
608
- // `onTick` is left for tests.
609
- }),
610
- });
611
- // The buddy growth/credit tasks: one per upstream capability, each
612
- // independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
613
- // them is a credit gain the account is entitled to, and none of them can mark
614
- // an account unhealthy — a growth endpoint that 404s is a display miss, not a
615
- // dispatch failure.
616
- //
617
- // Daily, not hourly: each claim is once per account per day and is
618
- // ledger-guarded, so at an hourly cadence 23 of every 24 passes are a full
619
- // scan of active accounts plus a ledger read per account that resolves to
620
- // nothing. Once a day is the shortest interval where a pass can still find
621
- // work, which is the only reason to run at all.
622
- const buddyTaskDeps = {
623
- db,
624
- resolveCredential: quotaResolveCredential,
625
- fetcher: fetch,
626
- // Spread across accounts so a pass does not look like a burst to the
627
- // upstream's risk gate.
628
- interAccountDelayMs: 800,
629
- };
630
- scheduledTasks.register({
631
- name: "buddy-streak",
632
- intervalMs: 24 * 60 * 60_000,
633
- initialDelayMs: 5 * 60_000,
634
- run: () => buddyStreakTask(buddyTaskDeps),
635
- });
636
- scheduledTasks.register({
637
- name: "buddy-adopt",
638
- intervalMs: 24 * 60 * 60_000,
639
- initialDelayMs: 17 * 60_000,
640
- run: () => buddyAdoptTask(buddyTaskDeps),
641
- });
642
- // Growth-activity report: the `/v2/report` batch that lights the login streak
643
- // and scores the daily growth event. Once a day with its own ledger — it used
644
- // to ride the quota sweep with no marker of its own, so its eight POSTs ran
645
- // every minute.
646
- scheduledTasks.register({
647
- name: "buddy-activity-report",
648
- intervalMs: 24 * 60 * 60_000,
649
- initialDelayMs: 15 * 60 * 60_000,
650
- run: () => buddyActivityReportTask(buddyTaskDeps),
651
- });
652
- // Real credit deductions, stored so the console renders a figure the upstream
653
- // actually reported. Six-hourly: the upstream's own window is 30 days, so a
654
- // day's number does not change intraday.
655
- scheduledTasks.register({
656
- name: "buddy-usage-sync",
657
- intervalMs: 6 * 60 * 60_000,
658
- initialDelayMs: 23 * 60_000,
659
- run: () => syncBuddyUsage(buddyTaskDeps),
660
- });
661
- // The scheduled tasks are registered here but started by the caller once the
662
- // listener is actually serving (`main.ts`). Starting them inside this builder
663
- // would let the first tick — the lease sweep, the health sweep — run against
664
- // a process that has not begun accepting traffic yet.
665
- const deps: ProductionDeps = {
666
- db,
667
- redis,
668
- snapshotService,
669
- readRoutingAccountInflight,
670
- resolveProviderAdapter: (providerId) => registry.resolve(providerId),
671
- bundledModelCatalog: bundledCatalog,
672
- byokUpstreamHosts,
673
- proxyPreparer: new ProxyRequestPreparer({
674
- snapshotService,
675
- routingEngine,
676
- admissionService,
677
- }),
678
- networkBindingFactory,
679
- ipAbuseProtection,
680
- readiness,
681
- scheduledTasks,
682
- poolAgentResolver,
683
- telemetryBuffer,
684
- usageFlush,
685
- healthEventFlush,
686
- poolSelector,
687
- admissionService,
688
- trustedProxyBoundary: resolveTrustedProxyBoundary(),
689
- providerRegistry: registry,
690
- resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
691
- oauthRefreshService,
692
- runtimeMetricsSampler,
693
- };
694
- const startupMemoryAfter = process.memoryUsage();
695
- log.info(
696
- `[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
697
- `rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
698
- `rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
699
- `heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
700
- `heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
701
- );
702
- return deps;
703
- }
704
-
705
- async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
706
- const startedAt = performance.now();
707
- const before = process.memoryUsage();
708
- log.info(
709
- `[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
710
- );
711
- try {
712
- const result = await operation();
713
- const after = process.memoryUsage();
714
- log.info(
715
- `[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
716
- `rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
717
- `heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
718
- `external_mb=${toMiB(after.external)}`,
719
- );
720
- return result;
721
- } catch (error) {
722
- log.error(
723
- `[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
724
- error instanceof Error ? error : new Error(String(error)),
725
- );
726
- throw error;
727
- }
728
- }
729
-
730
- function toMiB(bytes: number): string {
731
- return (bytes / 1024 / 1024).toFixed(1);
732
- }
1
+
2
+ import { assertPoolFitsServerCapacity, bootDatabase, ensureMigrated } from "../persistence/boot";
3
+ import { poolMaxFromEnv } from "../persistence/db-pg";
4
+ import type { CartethyiaDatabase } from "../persistence/db-handle";
5
+ import { resolveRedisBackend, resolveRedisClient } from "../persistence/redis";
6
+ import { resolveProcessTopology, type ProcessTopology } from "../config/runtime";
7
+ import { withTimeout } from "./timeout";
8
+ import { createRedisMaintenanceLeaseStore } from "../workers/maintenance-lease";
9
+ import { resolveDbMode } from "../persistence/db-mode";
10
+ import type { RedisClient } from "../persistence/redis";
11
+ import {
12
+ bundledModelCatalog,
13
+ liveProviderUpstreamHosts,
14
+ refreshActiveProviderCatalogs,
15
+ registerByokProviders,
16
+ retireUnbundledProviders,
17
+ seedBundledProviders,
18
+ } from "../providers/operations/provider-catalog-service";
19
+ import type { BundledProviderCatalog } from "../providers/operations/provider-catalog-service";
20
+ import { createDefaultProviderRegistry } from "../providers/default-registry";
21
+ import { OAuthRefreshService, loadDueOAuthAccounts, reconcileStaticTokenAccounts } from "../providers/authentication/oauth-refresh-service";
22
+ import type { OAuthTokenRefresher } from "../providers/authentication/oauth-refresh-service";
23
+ import { oauthRefreshSweep } from "../workers/oauth-refresh-sweep";
24
+ import { accountIdentitySweep } from "../workers/account-identity-sweep";
25
+ import { pushStructuredConsoleLog } from "../observability/log-ring";
26
+ import { createDatabaseSnapshotBuilder } from "../transport/routing/route-catalog";
27
+ import { DrizzleProviderCatalogStore } from "../console/providers/catalog/store";
28
+ import { InMemoryRouteSnapshotService } from "../transport/routing/route-model";
29
+ import { RoutingEngine } from "../transport/routing/router";
30
+ import { RedisAdmissionController } from "../transport/routing/admission";
31
+ import { ApiKeyAdmissionService } from "../security/admission/service";
32
+ import { InMemoryAdmissionCounterStore } from "../security/admission/in-memory-store";
33
+ import { RedisAdmissionCounterStore } from "../security/admission/redis-store";
34
+ import { sweepLeases } from "../security/admission/lease-sweep";
35
+ import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
36
+ import { UsageFlushBuffer } from "../persistence/usage-flush";
37
+ import { setPendingUsageSource } from "../security/api-key-auth";
38
+ import { HealthEventFlushBuffer, setHealthEventBuffer } from "../persistence/health-event-flush";
39
+ import { subscribePeerSettingsRevisions } from "../persistence/settings-revision";
40
+ import { deleteExpiredSessions } from "../console/auth/session-service";
41
+ import { InMemoryIpAbuseStore, IpAbuseProtectionService, RedisIpAbuseStore } from "../security/abuse";
42
+ import { checkReadiness } from "../persistence/readiness";
43
+ import { ProxyRequestPreparer } from "../transport/request/preparer";
44
+ import { DrizzleNetworkPoolLoader } from "../network/pool/loader";
45
+ import {
46
+ PoolAgentResolver,
47
+ ValidatedNetworkBindingFactory,
48
+ } from "../network/pool/resolver";
49
+ import { NetworkPoolSelector } from "../network/pool/selector";
50
+ import { restorePoolProbeVerdicts, restoreProviderCooldownMarkers } from "../network/pool-cooldown-store";
51
+ import { ScheduledTaskRegistry } from "../workers/scheduler";
52
+ import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
53
+ import { registerZcodeGiftClaimTask } from "../workers/zcode-gift-claim";
54
+ import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
55
+ import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
56
+ import {
57
+ buddyAdoptTask,
58
+ buddyActivityReportTask,
59
+ buddyStreakTask,
60
+ syncBuddyUsage,
61
+ } from "../workers/buddy-growth-tasks";
62
+ import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
63
+ import { quotaCacheSize } from "../providers/quota/cache";
64
+ import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
65
+ import { preferencesReaderFor } from "../persistence/tenant-preferences";
66
+ import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
67
+ import { DrizzleTelemetryStore } from "../persistence/stores/telemetry-store";
68
+ import { TelemetryPayloadCapture } from "../observability/payload-capture";
69
+ import { sweepExpiredPoolCooldowns } from "../network/pool-health-machine";
70
+ import { TelemetryBatchBuffer } from "../observability/telemetry-buffer";
71
+ import { RuntimeMetricsSampler } from "../observability/runtime-metrics";
72
+ import {
73
+ resolveIpRateLimit,
74
+ } from "../config/security";
75
+ import { resolveSsrfPolicy, resolveTrustedProxyBoundary } from "../config/network";
76
+ import type { TrustedProxyBoundary } from "../config/network";
77
+ import { resolveTelemetryRetentionDays } from "../config/telemetry";
78
+ import { resolveConcurrencyTolerance } from "../config/resources";
79
+ import { CONFIG_SPEC, readInt, readList } from "../config/spec";
80
+ import type { ProviderAdapter } from "../providers/provider-contracts";
81
+ import type { ProviderRegistry } from "../providers/provider-registry";
82
+ import type { ByokUpstreamHost } from "../providers/operations/provider-catalog-service";
83
+ import type { ReadinessCheckResult } from "../persistence/readiness";
84
+ import { log } from "../observability/logger";
85
+ import { refreshProviderClientVersions } from "../providers/operations/client-versions";
86
+ import type { ModelDefinition } from "../providers/provider-model-definition";
87
+
88
+
89
+ export interface ProductionDeps {
90
+ db: CartethyiaDatabase;
91
+ redis: RedisClient | undefined;
92
+ snapshotService: InMemoryRouteSnapshotService;
93
+ /** Live routing admission snapshot scoped to one provider/tenant for console reads. */
94
+ readRoutingAccountInflight: (
95
+ providerId: string,
96
+ tenantId: string | null,
97
+ ) => Promise<readonly { accountId: string; inflight: number }[]>;
98
+ /** On-demand adapter resolution keeps heavy providers out of the boot path. */
99
+ resolveProviderAdapter: (providerId: string) => Promise<ProviderAdapter | undefined>;
100
+ bundledModelCatalog: BundledProviderCatalog;
101
+ /** Live lookup of a provider's SSRF-validated upstream host. */
102
+ byokUpstreamHosts: { readonly get: (providerId: string) => ByokUpstreamHost | undefined };
103
+ proxyPreparer: ProxyRequestPreparer;
104
+ networkBindingFactory: ValidatedNetworkBindingFactory;
105
+ ipAbuseProtection: IpAbuseProtectionService;
106
+ readiness: () => Promise<ReadinessCheckResult>;
107
+ scheduledTasks: ScheduledTaskRegistry;
108
+ poolAgentResolver: PoolAgentResolver;
109
+ poolSelector: NetworkPoolSelector;
110
+ telemetryBuffer: TelemetryBatchBuffer;
111
+ usageFlush: UsageFlushBuffer;
112
+ healthEventFlush: HealthEventFlushBuffer;
113
+ trustedProxyBoundary: TrustedProxyBoundary;
114
+ providerRegistry: ProviderRegistry;
115
+ resolveOAuthRefresher: (providerId: string) => Promise<OAuthTokenRefresher | undefined>;
116
+ oauthRefreshService: OAuthRefreshService;
117
+ admissionService: ApiKeyAdmissionService;
118
+ /** Graduated strikes for repeated invalid-model requests. */
119
+ /** Memory sampler; in-flight source is bound once the request store exists. */
120
+ runtimeMetricsSampler: RuntimeMetricsSampler;
121
+ }
122
+
123
+ /**
124
+ * Pause between accounts in the OAuth refresh sweep. The token endpoints
125
+ * rate-limit a burst of refreshes even at low concurrency.
126
+ */
127
+ const OAUTH_REFRESH_INTER_ITEM_DELAY_MS = 1_500;
128
+
129
+ /**
130
+ * Client identities are refreshed out of band so provider dispatch never waits
131
+ * on npm/provider version endpoints. Resolver TTLs deduplicate the sources.
132
+ */
133
+ // Matched to the resolver's own 30 min cache TTL: a shorter interval makes
134
+ // every second pass a pure cache read that cannot discover anything new.
135
+ const CLIENT_VERSION_MONITOR_INTERVAL_MS = 30 * 60_000;
136
+ // B-05: first external fetch is pushed out to 90 s so the listener, the route
137
+ // snapshot warm, and the first-minute sweeps all settle first.
138
+ const CLIENT_VERSION_INITIAL_DELAY_MS = 90_000;
139
+
140
+ /** Bound on the boot-time Redis probe: a start must fail, not hang. */
141
+ const REDIS_PROBE_TIMEOUT_MS = 2_000;
142
+
143
+ /** Bounded Redis reachability probe: fails rather than hangs. */
144
+ async function redisIsReachable(): Promise<boolean> {
145
+ const client = resolveRedisClient();
146
+ if (!client) return false;
147
+ try {
148
+ const result = await withTimeout(client.ping(), REDIS_PROBE_TIMEOUT_MS);
149
+ return result === "PONG";
150
+ } catch {
151
+ return false;
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Refuses to start a multi-process deployment whose shared state is not
157
+ * actually shared.
158
+ *
159
+ * Every process shares one port via `SO_REUSEPORT`, so the kernel balances
160
+ * across them — and each process keeps its own in-memory admission counters,
161
+ * quota cache, and cooldown map. Those are only correct when they live in
162
+ * Redis. Starting anyway does not fail loudly; it enforces every per-key limit
163
+ * N times over, which then reads as "the limit did not work".
164
+ *
165
+ * Redis is checked by PING with a hard bound, not merely for being configured:
166
+ * a configured-but-unreachable Redis is the failure mode that matters, and an
167
+ * unbounded connect at boot would hang the start instead of failing it.
168
+ */
169
+ async function assertMultiProcessPreconditions(
170
+ topology: ProcessTopology,
171
+ redisReachable: () => Promise<boolean>,
172
+ ): Promise<void> {
173
+ if (!topology.multiProcess) return;
174
+ if (process.platform !== "linux") {
175
+ throw Object.assign(
176
+ new Error(
177
+ `CARTETHYIA_PROCESS_COUNT=${topology.count} requires Linux (SO_REUSEPORT); ` +
178
+ `this process is on ${process.platform}`,
179
+ ),
180
+ { code: "multi_process_unsupported_platform" },
181
+ );
182
+ }
183
+ if (resolveDbMode() !== "full") {
184
+ throw Object.assign(
185
+ new Error(
186
+ `CARTETHYIA_PROCESS_COUNT=${topology.count} requires CARTETHYIA_DB_MODE=full (PostgreSQL); ` +
187
+ "two processes on one SQLite file corrupt it — there is no cross-process lock in the driver",
188
+ ),
189
+ { code: "multi_process_requires_postgres" },
190
+ );
191
+ }
192
+ const reachable = await redisReachable();
193
+ if (!reachable) {
194
+ throw Object.assign(
195
+ new Error(
196
+ `CARTETHYIA_PROCESS_COUNT=${topology.count} requires a REACHABLE Redis: admission counters, ` +
197
+ "quotas, and cooldowns are per-process without one, so every per-key limit would be " +
198
+ `enforced ${topology.count} times over`,
199
+ ),
200
+ { code: "multi_process_requires_redis" },
201
+ );
202
+ }
203
+ }
204
+
205
+ export async function buildProductionDeps(): Promise<ProductionDeps> {
206
+ const startupStartedAt = performance.now();
207
+ const startupMemoryBefore = process.memoryUsage();
208
+ // Resolved first and validated before any connection is opened: a
209
+ // half-configured multi-process deploy that starts anyway looks healthy
210
+ // until a per-key limit is enforced N times over.
211
+ const topology = resolveProcessTopology();
212
+ await assertMultiProcessPreconditions(topology, redisIsReachable);
213
+ const handle = await measureStartupPhase("database-connect", bootDatabase);
214
+ const db = handle.db;
215
+ // One rule, no mode flag: REDIS_URL set → shared client, unset → memory.
216
+ const redis = resolveRedisClient();
217
+ const ssrfPolicy = resolveSsrfPolicy();
218
+ await measureStartupPhase("database-migrations", async () => {
219
+ await ensureMigrated();
220
+ // Pool sizing is an external-server concern; the embedded backend has no
221
+ // connection budget to fit into.
222
+ if (handle.kind === "pg") await assertPoolFitsServerCapacity(poolMaxFromEnv(), topology.count);
223
+ });
224
+ await measureStartupPhase("provider-seed", () => seedBundledProviders(db));
225
+ // Boot-only: retire global rows for providers the bundle has dropped, so a
226
+ // retired provider cannot linger as a card the console can no longer delete.
227
+ // Kept off the test harness path, which shares one database across suites and
228
+ // installs its own global fixtures.
229
+ await measureStartupPhase("retired-provider-cleanup", () => retireUnbundledProviders(db));
230
+ const registry = createDefaultProviderRegistry();
231
+ // Live view, not a snapshot: custom providers registered after boot (or
232
+ // edited from the console) must resolve their SSRF binding immediately.
233
+ const byokUpstreamHosts = liveProviderUpstreamHosts(registry);
234
+ await measureStartupPhase("byok-registration", () =>
235
+ registerByokProviders(registry, db, ssrfPolicy),
236
+ );
237
+ // B-01: the bundled model catalog (73 providers' integration modules, ~the
238
+ // largest pre-listen cost) is DEFERRED to the first snapshot build, which
239
+ // runs right after listen (`main.ts` warms the snapshot). Every consumer
240
+ // holds a reference to `bundledCatalogModels`, a shared map that starts
241
+ // empty and is populated by the first resolution — the console store and
242
+ // the inflight reader capture the map reference at construction, so once
243
+ // the build fills it, all of them see the data. The snapshot builder's
244
+ // refresh step seeds the DB from the same map (markCatalogSeeded semantics:
245
+ // lastSeededCatalogRevision is -1 until the first build, so the first build
246
+ // seeds exactly once and later rebuilds see an unchanged revision and skip).
247
+ const bundledCatalogModels = new Map<string, readonly ModelDefinition[]>();
248
+ let bundledCatalogResolved = false;
249
+ const resolveBundledCatalog = async (): Promise<BundledProviderCatalog> => {
250
+ if (bundledCatalogResolved) return { modelsByProvider: bundledCatalogModels };
251
+ const catalog = await bundledModelCatalog(registry);
252
+ for (const [providerId, models] of catalog.modelsByProvider) {
253
+ bundledCatalogModels.set(providerId, models);
254
+ }
255
+ bundledCatalogResolved = true;
256
+ return { modelsByProvider: bundledCatalogModels };
257
+ };
258
+ const snapshotService = new InMemoryRouteSnapshotService(
259
+ createDatabaseSnapshotBuilder(db, async () => {
260
+ // Compute the lazy catalog first so the refresh shares it and the DB
261
+ // seed uses the same map the console reads (no second full walk).
262
+ const catalog = await resolveBundledCatalog();
263
+ await refreshActiveProviderCatalogs(db, registry, catalog);
264
+ }),
265
+ );
266
+ const routingEngine = new RoutingEngine(
267
+ redis ? new RedisAdmissionController(redis) : undefined,
268
+ redis,
269
+ db,
270
+ );
271
+ // Peer processes' settings mutations arrive here. Subscribed once for the
272
+ // process: the revision counter is process-global, and the handler only
273
+ // moves it forward.
274
+ subscribePeerSettingsRevisions();
275
+ const usageFlush = new UsageFlushBuffer(db);
276
+ // Auth reads lifetime usage to seed the admission counter. The persisted
277
+ // value lags by up to one flush interval, so hand it the in-flight deltas
278
+ // too — otherwise the seed is systematically under what has actually been
279
+ // spent, worst exactly when the key is busiest.
280
+ setPendingUsageSource((apiKeyId) => usageFlush.pendingFor(apiKeyId));
281
+ // Health audit rows are append-only and read by nobody in dispatch, so they
282
+ // are buffered out of the request. The status write they accompany stays
283
+ // synchronous — routing reads it.
284
+ const healthEventFlush = new HealthEventFlushBuffer(db);
285
+ setHealthEventBuffer(healthEventFlush);
286
+ const admissionStore = redis
287
+ ? new RedisAdmissionCounterStore(redis)
288
+ : new InMemoryAdmissionCounterStore();
289
+ const admissionService = new ApiKeyAdmissionService(
290
+ admissionStore,
291
+ undefined,
292
+ // The per-tenant concurrency cap is a `console_settings` preference, read
293
+ // on every admitted request. Reading it directly cost one Postgres round
294
+ // trip per request (multiplied by each failover candidate); the shared
295
+ // revision-keyed reader answers from memory and converges the moment an
296
+ // operator saves, because `DrizzleRuntimeSettingsStore.update` bumps the
297
+ // revision every cached key embeds.
298
+ (tenantId) =>
299
+ preferencesReaderFor(db)
300
+ .readPreferences(tenantId)
301
+ .then((prefs) => prefs?.tenantConcurrencyLimit ?? null),
302
+ ({ apiKeyId, delta }) => {
303
+ // Coalesced: one UPDATE per key per flush instead of one per request.
304
+ // The admission counter is the live total; this only persists it.
305
+ usageFlush.enqueue(apiKeyId, delta);
306
+ return Promise.resolve();
307
+ },
308
+ // Fresh lifetime reader — invoked by the store only when it is about to
309
+ // seed a missing `admission:lifetime:<id>` counter, so the ≤3s-stale auth
310
+ // snapshot value cannot freeze a low baseline in place. Family total: the
311
+ // row plus every child (share recipients admit under the parent id).
312
+ async (apiKeyId) => {
313
+ const store = new DrizzleApiKeyStore(db);
314
+ const row = await store.findActiveById(apiKeyId);
315
+ if (!row) return undefined;
316
+ const children = await store.sumChildrenConsumed(apiKeyId);
317
+ return (row.lifetimeTokensConsumed ?? 0) + children;
318
+ },
319
+ // Slack above every key's concurrency limit. A tenant preference an
320
+ // operator sets in the runtime settings modal, falling back to the
321
+ // deployment default so an unset row does not silently lose the allowance.
322
+ (tenantId) =>
323
+ preferencesReaderFor(db)
324
+ .readPreferences(tenantId)
325
+ .then((prefs) => prefs?.concurrencyTolerance ?? resolveConcurrencyTolerance()),
326
+ );
327
+ const ipStore = redis
328
+ ? new RedisIpAbuseStore(redis)
329
+ : new InMemoryIpAbuseStore();
330
+ const ipAbuseProtection = new IpAbuseProtectionService(ipStore, undefined, {
331
+ maxRequestsPerWindow: resolveIpRateLimit(),
332
+ });
333
+
334
+ const readiness = () => checkReadiness(db, redis, resolveRedisBackend(redis));
335
+ // One list covers every periodic task: a small deployment can cut the
336
+ // heaviest sweeps (quota refresh, OAuth refresh, client-version polling)
337
+ // without the codebase growing a flag per task.
338
+ // Leases coordinate the SHARED sweeps across processes. Absent Redis there is
339
+ // nothing to coordinate with, and the registry's local one-at-a-time rule is
340
+ // the correct (and only) guard.
341
+ const scheduledTasks = new ScheduledTaskRegistry(
342
+ readList("CARTETHYIA_DISABLED_TASKS", CONFIG_SPEC.CARTETHYIA_DISABLED_TASKS) ?? [],
343
+ {
344
+ ...(redis ? { lease: createRedisMaintenanceLeaseStore(redis) } : {}),
345
+ },
346
+ );
347
+ if (redis) {
348
+ scheduledTasks.register({
349
+ name: "lease-sweep",
350
+ intervalMs: 30_000,
351
+ run: () => sweepLeases(redis),
352
+ });
353
+ }
354
+ scheduledTasks.register({
355
+ name: "account-health-sweep",
356
+ // G-04: 30 s → 60 s — this is the most frequent snapshot invalidator, and
357
+ // halving its cadence halves the rebuild/cooldown-recovery churn.
358
+ intervalMs: 60_000,
359
+ // Offsets the pass from lease-sweep and runtime-metrics, which share a
360
+ // cadence: under the registry's single-flight gate, aligned ticks make the
361
+ // tasks skip each other when one runs long.
362
+ initialDelayMs: 5_000,
363
+ // G-03: this sweep can invalidate the shared snapshot, so defer it while
364
+ // foreground is busy (bounded) rather than racing a page open.
365
+ deferWhenBusy: true,
366
+ // Recovers cooldowns deployment-wide and invalidates the shared snapshot.
367
+ shared: true,
368
+ run: async () => {
369
+ const recovered = await sweepExpiredCooldowns(db);
370
+ const poolRecovered = await sweepExpiredPoolCooldowns(db);
371
+ if (recovered > 0 || poolRecovered > 0) await snapshotService.invalidate();
372
+ },
373
+ });
374
+ const payloadCapture = new TelemetryPayloadCapture(db);
375
+ const telemetryStore = new DrizzleTelemetryStore(db);
376
+ scheduledTasks.register({
377
+ name: "telemetry-payload-cleanup",
378
+ intervalMs: 15 * 60_000,
379
+ // Deletes payload frames for every tenant: two passes over the same
380
+ // batches is duplicated work, not speedup.
381
+ shared: true,
382
+ run: () => payloadCapture.cleanupExpired(),
383
+ });
384
+ scheduledTasks.register({
385
+ name: "console-session-sweep",
386
+ // Sessions live 48 h, so an hourly sweep deleted nothing 47 times out of
387
+ // 48 — each pass a full scan of the session table for rows that cannot yet
388
+ // be expired. Half the session lifetime is the point where a pass can
389
+ // actually find work, and still leaves a full grace period after it.
390
+ intervalMs: 12 * 60 * 60_000,
391
+ shared: true,
392
+ run: () => deleteExpiredSessions(db),
393
+ });
394
+ scheduledTasks.register({
395
+ name: "telemetry-retention",
396
+ intervalMs: 6 * 60 * 60_000,
397
+ // Retention deletes rows and files for the whole deployment; a second
398
+ // concurrent pass would churn the same batches.
399
+ shared: true,
400
+ run: () =>
401
+ telemetryStore.pruneTelemetry(
402
+ new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
403
+ ),
404
+ });
405
+ const telemetryBuffer = new TelemetryBatchBuffer(db);
406
+ const poolSelector = new NetworkPoolSelector(redis);
407
+ // Replay the provider cooldown markers the previous process recorded
408
+ // (`network_pools.provider_cooldowns`). Without this a restart made dispatch
409
+ // forget every egress the upstream was still refusing: the flags are
410
+ // process-local otherwise, so the pool re-entered rotation and took 429s the
411
+ // first request after boot re-discovered. Expired entries are dropped by the
412
+ // restore itself. Best-effort: a failed read leaves selection matching what
413
+ // a fresh deployment would do.
414
+ try {
415
+ const restored = await restoreProviderCooldownMarkers(poolSelector, db);
416
+ const replayed = await restorePoolProbeVerdicts(poolSelector, db);
417
+ if (restored > 0 || replayed > 0) {
418
+ log.info("[pool-selector] restored provider cooldown state", { mirrored: restored, probeVerdictsApplied: replayed });
419
+ }
420
+ } catch (error) {
421
+ log.warn("[pool-selector] provider cooldown restore failed", { error: String(error) });
422
+ }
423
+ const poolAgentResolver = new PoolAgentResolver(
424
+ new DrizzleNetworkPoolLoader(db),
425
+ { ssrfPolicy, db },
426
+ );
427
+ const networkBindingFactory = new ValidatedNetworkBindingFactory(
428
+ ssrfPolicy,
429
+ undefined,
430
+ poolAgentResolver,
431
+ );
432
+ // The account-inflight reader the console queries. It lives after the deps
433
+ // it closes over (telemetry buffer, binding factory), so the closure never
434
+ // touches a TDZ binding; it also reuses one catalog store per tenant instead
435
+ // of constructing a fresh store object on every console read. Multi-tenant
436
+ // console polling rotates the tenant id, so the memo is keyed by tenant with
437
+ // a small bound — a single-entry memo rebuilt the heavy store on every
438
+ // switch, and an unbounded map would leak one store per tenant forever.
439
+ const MAX_INFLIGHT_STORES = 16;
440
+ const inflightStores = new Map<string, DrizzleProviderCatalogStore>();
441
+ const readRoutingAccountInflight = async (
442
+ providerId: string,
443
+ tenantId: string | null,
444
+ ): Promise<readonly { accountId: string; inflight: number }[]> => {
445
+ if (providerId === "" || tenantId === null) return [];
446
+ let inflightStore = inflightStores.get(tenantId);
447
+ if (inflightStore === undefined) {
448
+ // B-01: resolve the deferred bundled catalog first so the store's model
449
+ // map is populated (the snapshot warm normally did this already; this
450
+ // is the cold-race fallback).
451
+ await resolveBundledCatalog();
452
+ inflightStore = new DrizzleProviderCatalogStore(db, {
453
+ telemetryBuffer,
454
+ bundledModelCatalog: { modelsByProvider: bundledCatalogModels },
455
+ providerRegistry: registry,
456
+ outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
457
+ snapshotInvalidator: snapshotService,
458
+ });
459
+ inflightStores.set(tenantId, inflightStore);
460
+ if (inflightStores.size > MAX_INFLIGHT_STORES) {
461
+ const oldest = inflightStores.keys().next().value;
462
+ if (oldest !== undefined) inflightStores.delete(oldest);
463
+ }
464
+ }
465
+ // P-02: only the account-id set is needed, so select just `id` — the old
466
+ // `listAccounts` recomputed today/lifetime usage aggregates for every
467
+ // account on every console poll.
468
+ const accountIds = await inflightStore.listAccountIds(tenantId, providerId);
469
+ const owned = new Set(accountIds);
470
+ // Scoped read: only the provider's own candidates for this tenant are
471
+ // queried, so the console read costs one exact MGET over the accounts it
472
+ // is about to report on — not a fleet-wide `KEYS` scan whose cost is the
473
+ // size of every bucket every process ever wrote.
474
+ const routeSnapshot = await snapshotService.getSnapshot();
475
+ const ownedCandidates = routeSnapshot.candidates.filter(
476
+ (candidate) =>
477
+ candidate.provider_id === providerId &&
478
+ candidate.provider_account_id !== undefined &&
479
+ owned.has(candidate.provider_account_id),
480
+ );
481
+ const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
482
+ const totals = new Map<string, number>();
483
+ for (const [bucket, count] of snapshot) {
484
+ const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
485
+ if (!owned.has(accountId)) continue;
486
+ totals.set(accountId, (totals.get(accountId) ?? 0) + count);
487
+ }
488
+ return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
489
+ };
490
+ // Runtime gauges (memory + bounded-collection sizes) sampled on the shared
491
+ // maintenance cadence. The sampler reads live sizes through closures so the
492
+ // observability layer never holds direct references to routing/pool state.
493
+ const runtimeMetricsSampler = new RuntimeMetricsSampler({
494
+ routingRoundRobinEntries: () => routingEngine.roundRobinEntries(),
495
+ ipAbuseKeys: () => (ipStore instanceof InMemoryIpAbuseStore ? ipStore.keyCount() : 0),
496
+ quotaCacheEntries: () => quotaCacheSize(),
497
+ modelCatalogEntries: () => modelCatalogCacheSize(),
498
+ // Late-bound via bindInFlight once the request state store exists.
499
+ inFlight: () => 0,
500
+ // The buffer is built below; the closure keeps the sampler from holding a
501
+ // reference during construction.
502
+ telemetryBacklog: () => telemetryBuffer?.snapshot(),
503
+ });
504
+ // One sample NOW, not on the first tick. The headroom guard reads
505
+ // `memorySnapshot()` on every gateway request and treats `undefined` as
506
+ // "unknown" — so without this the guard would spend its first interval
507
+ // (seconds, at the configured cadence) unable to shed at all, which is
508
+ // exactly the window a cold start is most likely to be short of memory.
509
+ runtimeMetricsSampler.sample();
510
+ scheduledTasks.register({
511
+ name: "runtime-metrics",
512
+ intervalMs: readInt(
513
+ "CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS",
514
+ CONFIG_SPEC.CARTETHYIA_RUNTIME_METRICS_INTERVAL_MS,
515
+ ),
516
+ run: () => runtimeMetricsSampler.sample(),
517
+ });
518
+ scheduledTasks.register({
519
+ name: "pool-agent-sweep",
520
+ // `resolveAgent` already self-sweeps every 30 s, so a 1 min timer adds a
521
+ // second pass that almost always finds nothing to release. 5 min is well
522
+ // inside the idle timeout this exists to enforce, and outside the path
523
+ // that already runs.
524
+ intervalMs: 5 * 60_000,
525
+ run: () => poolAgentResolver.sweepIdleAgents(),
526
+ });
527
+ const oauthRefreshService = new OAuthRefreshService(db);
528
+ // Converge accounts parked on a dead refresh grant: an account whose stored
529
+ // access token is still usable becomes a static token (and returns to
530
+ // rotation) instead of staying disabled for re-auth.
531
+ await measureStartupPhase("static-token-reconciliation", () =>
532
+ reconcileStaticTokenAccounts(db),
533
+ );
534
+ // Fills in the identity a batch import deliberately skipped, so a paste of
535
+ // many credentials is one instant write and the email/label arrive shortly
536
+ // after rather than blocking the import. Purely CPU on an already-stored
537
+ // credential — no upstream call — so it runs often and cheaply.
538
+ scheduledTasks.register({
539
+ name: "account-enrichment-sweep",
540
+ // Converges via `identity_checked_at`: once every stored credential has
541
+ // been parsed, the listing query returns nothing — a 30 s timer would run
542
+ // that empty query forever, two million times a year. 10 min is still
543
+ // instant for the case that matters (a batch import, where the label and
544
+ // email show up on the next tick) and costs nothing when idle.
545
+ intervalMs: 10 * 60_000,
546
+ run: () => accountIdentitySweep({ db }),
547
+ });
548
+ scheduledTasks.register({
549
+ name: "oauth-refresh-sweep",
550
+ intervalMs: 60_000,
551
+ // G-04: defer when foreground is busy (InvokeGuard), and cap the pass with
552
+ // a budget mirroring quota refresh's 45 s so an OAuth pass cannot run away
553
+ // into a heavy foreground minute.
554
+ deferWhenBusy: true,
555
+ run: () =>
556
+ oauthRefreshSweep({
557
+ loadDueAccounts: () => loadDueOAuthAccounts(db),
558
+ refreshService: oauthRefreshService,
559
+ resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
560
+ // Sequential with a pause between accounts: the OAuth token endpoints
561
+ // rate-limit a burst of refreshes, and a pass over a handful of
562
+ // accounts must not look like one. Per-account outcomes are already
563
+ // pushed to the Console Log ring by `OAuthRefreshService`; this only
564
+ // covers an error that escaped before reaching it.
565
+ interItemDelayMs: OAUTH_REFRESH_INTER_ITEM_DELAY_MS,
566
+ budgetMs: 45_000,
567
+ onAccountError: (accountId, providerId, error) => {
568
+ log.error(
569
+ `[oauth-refresh] account=${accountId} provider=${providerId} failed:`,
570
+ error as Error,
571
+ );
572
+ pushStructuredConsoleLog("error", "OAuth refresh sweep: account failed", {
573
+ event: "token_refresh",
574
+ accountId,
575
+ providerId,
576
+ errorCode: "refresh_sweep_failed",
577
+ });
578
+ },
579
+ }),
580
+ });
581
+
582
+ // Client versions are monitor-only metadata; let the listener settle before
583
+ // starting their external fetches. The scheduled task keeps the cache warm.
584
+ scheduledTasks.register({
585
+ name: "client-version-monitor",
586
+ intervalMs: CLIENT_VERSION_MONITOR_INTERVAL_MS,
587
+ initialDelayMs: CLIENT_VERSION_INITIAL_DELAY_MS,
588
+ run: refreshProviderClientVersions,
589
+ });
590
+
591
+ const quotaResolveCredential = createAccountSecretResolver({
592
+ db,
593
+ resolveRefresher: (providerId) => registry.resolveRefresher(providerId),
594
+ refreshService: oauthRefreshService,
595
+ });
596
+ // Keeps every account's cached quota warm so opening the Quota page is a
597
+ // cache read. Without this the only refreshes are user-triggered, so the
598
+ // first open after an expiry always blocks on the provider.
599
+ //
600
+ // It also carries the daily check-in ride-along: each account refreshed
601
+ // here gets its once-per-day WorkBuddy/CodeBuddy credit grant claimed in
602
+ // the same pass (one attempt per account per day, guarded by a day
603
+ // marker), so no second timer duplicates the credential resolution.
604
+ //
605
+ // Registered in **every** mode, not only when Redis is present: gating it on
606
+ // Redis would cost Lite proactive quota warming and the check-in ride-along
607
+ // above that claims the once-per-day WorkBuddy/CodeBuddy credit grant. The
608
+ // whole sweep chain accepts `RedisClient | undefined` and falls back to the
609
+ // in-process TTL cache and an in-process day marker, so Redis is an
610
+ // optimization here rather than a precondition — which is exactly what
611
+ // Lite is for.
612
+ scheduledTasks.register({
613
+ name: "quota-refresh-sweep",
614
+ intervalMs: 60_000,
615
+ // Staggered off the oauth/lease/health ticks so the three sweeps that
616
+ // strike providerAccounts do not land together every minute.
617
+ initialDelayMs: 20_000,
618
+ // G-03: defer while foreground is busy (bounded).
619
+ deferWhenBusy: true,
620
+ run: () =>
621
+ quotaRefreshSweep({
622
+ db,
623
+ redis,
624
+ providerRegistry: registry,
625
+ resolveCredential: quotaResolveCredential,
626
+ snapshotInvalidator: snapshotService,
627
+ // Without this the sweep hands each collector the raw stored secret.
628
+ // For `cline` that means an `sk_...` key reaching the OAuth branch,
629
+ // which prepends `workos:` and is refused with "please make sure
630
+ // you're using the latest version of Cline and re-authenticate" — so
631
+ // every sweep cools a working account with `quota_fetch_failed` while
632
+ // the same account refreshes fine from the console, which binds the
633
+ // same marker through `createQuotaRefreshDeps`.
634
+ markApiKeyCredential: markProviderApiKeyCredential,
635
+ // Below the 60 s interval: a budget longer than the tick guarantees
636
+ // skip/burst under load instead of steady progress.
637
+ passBudgetMs: 45_000,
638
+ // The check-in ride-along rotates egress per account: each account
639
+ // gets the next active pool in its tenant's rotation so check-ins
640
+ // spread across IPs instead of sharing one direct egress. No pool
641
+ // (or an unreadable pool table) falls back to direct egress, and a
642
+ // tenant never borrows another tenant's pool.
643
+ checkinFetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
644
+ // Per-account lines and the pass summary are emitted by the sweep
645
+ // itself at `info` level, so they survive the production LOG_LEVEL.
646
+ // `onTick` is left for tests.
647
+ }),
648
+ });
649
+ registerZcodeGiftClaimTask(scheduledTasks, {
650
+ db,
651
+ redis,
652
+ resolveCredential: quotaResolveCredential,
653
+ snapshotInvalidator: snapshotService,
654
+ fetcherFor: dailyCheckinEgressForPass({ db, networkBindingFactory }),
655
+ });
656
+ // The buddy growth/credit tasks: one per upstream capability, each
657
+ // independently switchable with `CARTETHYIA_DISABLED_TASKS`. Every one of
658
+ // them is a credit gain the account is entitled to, and none of them can mark
659
+ // an account unhealthy — a growth endpoint that 404s is a display miss, not a
660
+ // dispatch failure.
661
+ //
662
+ // Daily, not hourly: each claim is once per account per day and is
663
+ // ledger-guarded, so at an hourly cadence 23 of every 24 passes are a full
664
+ // scan of active accounts plus a ledger read per account that resolves to
665
+ // nothing. Once a day is the shortest interval where a pass can still find
666
+ // work, which is the only reason to run at all.
667
+ const buddyTaskDeps = {
668
+ db,
669
+ resolveCredential: quotaResolveCredential,
670
+ fetcher: fetch,
671
+ // Spread across accounts so a pass does not look like a burst to the
672
+ // upstream's risk gate.
673
+ interAccountDelayMs: 800,
674
+ // G-04: bound each pass. These are daily/six-hourly tasks, so the budget
675
+ // is deliberately larger than the minute-cadence sweeps' 45 s — it caps a
676
+ // runaway scan without truncating a legitimate fleet mid-pass.
677
+ budgetMs: 5 * 60_000,
678
+ };
679
+ scheduledTasks.register({
680
+ name: "buddy-streak",
681
+ intervalMs: 24 * 60 * 60_000,
682
+ initialDelayMs: 5 * 60_000,
683
+ // G-03: defer while foreground is busy (bounded).
684
+ deferWhenBusy: true,
685
+ run: () => buddyStreakTask(buddyTaskDeps),
686
+ });
687
+ scheduledTasks.register({
688
+ name: "buddy-adopt",
689
+ intervalMs: 24 * 60 * 60_000,
690
+ initialDelayMs: 17 * 60_000,
691
+ deferWhenBusy: true,
692
+ run: () => buddyAdoptTask(buddyTaskDeps),
693
+ });
694
+ // Growth-activity report: the `/v2/report` batch that lights the login streak
695
+ // and scores the daily growth event. Once a day with its own ledger — it used
696
+ // to ride the quota sweep with no marker of its own, so its eight POSTs ran
697
+ // every minute.
698
+ scheduledTasks.register({
699
+ name: "buddy-activity-report",
700
+ intervalMs: 24 * 60 * 60_000,
701
+ initialDelayMs: 15 * 60 * 60_000,
702
+ deferWhenBusy: true,
703
+ run: () => buddyActivityReportTask(buddyTaskDeps),
704
+ });
705
+ // Real credit deductions, stored so the console renders a figure the upstream
706
+ // actually reported. Six-hourly: the upstream's own window is 30 days, so a
707
+ // day's number does not change intraday.
708
+ scheduledTasks.register({
709
+ name: "buddy-usage-sync",
710
+ intervalMs: 6 * 60 * 60_000,
711
+ initialDelayMs: 23 * 60_000,
712
+ deferWhenBusy: true,
713
+ run: () => syncBuddyUsage(buddyTaskDeps),
714
+ });
715
+ // The scheduled tasks are registered here but started by the caller once the
716
+ // listener is actually serving (`main.ts`). Starting them inside this builder
717
+ // would let the first tick — the lease sweep, the health sweep — run against
718
+ // a process that has not begun accepting traffic yet.
719
+ const deps: ProductionDeps = {
720
+ db,
721
+ redis,
722
+ snapshotService,
723
+ readRoutingAccountInflight,
724
+ resolveProviderAdapter: (providerId) => registry.resolve(providerId),
725
+ bundledModelCatalog: { modelsByProvider: bundledCatalogModels },
726
+ byokUpstreamHosts,
727
+ proxyPreparer: new ProxyRequestPreparer({
728
+ snapshotService,
729
+ routingEngine,
730
+ admissionService,
731
+ }),
732
+ networkBindingFactory,
733
+ ipAbuseProtection,
734
+ readiness,
735
+ scheduledTasks,
736
+ poolAgentResolver,
737
+ telemetryBuffer,
738
+ usageFlush,
739
+ healthEventFlush,
740
+ poolSelector,
741
+ admissionService,
742
+ trustedProxyBoundary: resolveTrustedProxyBoundary(),
743
+ providerRegistry: registry,
744
+ resolveOAuthRefresher: (providerId) => registry.resolveRefresher(providerId),
745
+ oauthRefreshService,
746
+ runtimeMetricsSampler,
747
+ };
748
+ const startupMemoryAfter = process.memoryUsage();
749
+ log.info(
750
+ `[startup] phase=dependencies-ready elapsed_ms=${Math.round(performance.now() - startupStartedAt)} ` +
751
+ `rss_mb=${toMiB(startupMemoryAfter.rss)} ` +
752
+ `rss_delta_mb=${toMiB(startupMemoryAfter.rss - startupMemoryBefore.rss)} ` +
753
+ `heap_used_mb=${toMiB(startupMemoryAfter.heapUsed)} ` +
754
+ `heap_delta_mb=${toMiB(startupMemoryAfter.heapUsed - startupMemoryBefore.heapUsed)}`,
755
+ );
756
+ return deps;
757
+ }
758
+
759
+ async function measureStartupPhase<T>(name: string, operation: () => Promise<T>): Promise<T> {
760
+ const startedAt = performance.now();
761
+ const before = process.memoryUsage();
762
+ log.info(
763
+ `[startup] phase=${name} status=started rss_mb=${toMiB(before.rss)} heap_used_mb=${toMiB(before.heapUsed)}`,
764
+ );
765
+ try {
766
+ const result = await operation();
767
+ const after = process.memoryUsage();
768
+ log.info(
769
+ `[startup] phase=${name} status=completed elapsed_ms=${Math.round(performance.now() - startedAt)} ` +
770
+ `rss_mb=${toMiB(after.rss)} rss_delta_mb=${toMiB(after.rss - before.rss)} ` +
771
+ `heap_used_mb=${toMiB(after.heapUsed)} heap_delta_mb=${toMiB(after.heapUsed - before.heapUsed)} ` +
772
+ `external_mb=${toMiB(after.external)}`,
773
+ );
774
+ return result;
775
+ } catch (error) {
776
+ log.error(
777
+ `[startup] phase=${name} status=failed elapsed_ms=${Math.round(performance.now() - startedAt)}`,
778
+ error instanceof Error ? error : new Error(String(error)),
779
+ );
780
+ throw error;
781
+ }
782
+ }
783
+
784
+ function toMiB(bytes: number): string {
785
+ return (bytes / 1024 / 1024).toFixed(1);
786
+ }