cartethyia 2.7.9 → 2.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (496) hide show
  1. package/.env.example +6 -5
  2. package/dist/dashboard/assets/{BannedPage-BxoG1MrL.js → BannedPage-Dq530awD.js} +1 -1
  3. package/dist/dashboard/assets/BannedPage-Dq530awD.js.gz +0 -0
  4. package/dist/dashboard/assets/{CliToolDetailPage-ixwgKk53.js → CliToolDetailPage-DZzaS-7S.js} +1 -1
  5. package/dist/dashboard/assets/CliToolDetailPage-DZzaS-7S.js.gz +0 -0
  6. package/dist/dashboard/assets/{CliToolsPage-BmttNV3U.js → CliToolsPage-DaZZ3cqP.js} +1 -1
  7. package/dist/dashboard/assets/CliToolsPage-DaZZ3cqP.js.gz +0 -0
  8. package/dist/dashboard/assets/{CombosPage-BE3qwwfC.js → CombosPage-BlC_xdkX.js} +1 -1
  9. package/dist/dashboard/assets/CombosPage-BlC_xdkX.js.gz +0 -0
  10. package/dist/dashboard/assets/{ConsoleLogPage-CRZdmMIX.js → ConsoleLogPage-D-rqeVtG.js} +1 -1
  11. package/dist/dashboard/assets/ConsoleLogPage-D-rqeVtG.js.gz +0 -0
  12. package/dist/dashboard/assets/{CustomizationPage-Dv_FJ6Er.js → CustomizationPage-7xT5yg2E.js} +1 -1
  13. package/dist/dashboard/assets/CustomizationPage-7xT5yg2E.js.gz +0 -0
  14. package/dist/dashboard/assets/HealthEventsModal-Clq8E9cW.js +1 -0
  15. package/dist/dashboard/assets/HealthEventsModal-Clq8E9cW.js.gz +0 -0
  16. package/dist/dashboard/assets/{InjectSystemPromptPanel-BOoO8rpz.js → InjectSystemPromptPanel-6Teoq9Se.js} +2 -2
  17. package/dist/dashboard/assets/InjectSystemPromptPanel-6Teoq9Se.js.gz +0 -0
  18. package/dist/dashboard/assets/{LoginPage-ChJuNNVN.js → LoginPage-CZuI0MAR.js} +1 -1
  19. package/dist/dashboard/assets/LoginPage-CZuI0MAR.js.gz +0 -0
  20. package/dist/dashboard/assets/{ModelPicker-BZAJtPg1.js → ModelPicker-C1hNTlkr.js} +1 -1
  21. package/dist/dashboard/assets/ModelPicker-C1hNTlkr.js.gz +0 -0
  22. package/dist/dashboard/assets/{OverviewPage-BE40s3u5.js → OverviewPage-0yfhSI1C.js} +1 -1
  23. package/dist/dashboard/assets/OverviewPage-0yfhSI1C.js.gz +0 -0
  24. package/dist/dashboard/assets/ProviderDetailPage-xqTxdQ4c.js +8 -0
  25. package/dist/dashboard/assets/ProviderDetailPage-xqTxdQ4c.js.gz +0 -0
  26. package/dist/dashboard/assets/{ProviderIcon-CQ6bPUf_.js → ProviderIcon-Ddhm1InL.js} +1 -1
  27. package/dist/dashboard/assets/ProviderIcon-Ddhm1InL.js.gz +0 -0
  28. package/dist/dashboard/assets/ProvidersPage-C3fYuu6i.js +1 -0
  29. package/dist/dashboard/assets/ProvidersPage-C3fYuu6i.js.gz +0 -0
  30. package/dist/dashboard/assets/ProxyPage-CHtJtZ3u.js +9 -0
  31. package/dist/dashboard/assets/ProxyPage-CHtJtZ3u.js.gz +0 -0
  32. package/dist/dashboard/assets/QuotaPage-D1RV_2BX.js +1 -0
  33. package/dist/dashboard/assets/QuotaPage-D1RV_2BX.js.gz +0 -0
  34. package/dist/dashboard/assets/{SettingsPage-C5HPWqEv.js → SettingsPage-BzZAvYSX.js} +1 -1
  35. package/dist/dashboard/assets/SettingsPage-BzZAvYSX.js.gz +0 -0
  36. package/dist/dashboard/assets/{SetupPage-CFNcATJJ.js → SetupPage--s-EfEIL.js} +1 -1
  37. package/dist/dashboard/assets/SetupPage--s-EfEIL.js.gz +0 -0
  38. package/dist/dashboard/assets/{StudioPage-YoD4tY5l.js → StudioPage-BjXFGSK4.js} +1 -1
  39. package/dist/dashboard/assets/StudioPage-BjXFGSK4.js.gz +0 -0
  40. package/dist/dashboard/assets/{UsagePage-DxjaAhb_.js → UsagePage-CP8F8BN7.js} +1 -1
  41. package/dist/dashboard/assets/UsagePage-CP8F8BN7.js.gz +0 -0
  42. package/dist/dashboard/assets/{api-keys-BTlkq_Uk.js → api-keys-2hc12Uz2.js} +1 -1
  43. package/dist/dashboard/assets/api-keys-2hc12Uz2.js.gz +0 -0
  44. package/dist/dashboard/assets/app-DGXbNfAR.css +2 -0
  45. package/dist/dashboard/assets/app-DGXbNfAR.css.gz +0 -0
  46. package/dist/dashboard/assets/assert-BMcj1xZE.js +1 -0
  47. package/dist/dashboard/assets/assert-BMcj1xZE.js.gz +0 -0
  48. package/dist/dashboard/assets/{badge-DdbT_7Am.js → badge-DkRUuVfz.js} +1 -1
  49. package/dist/dashboard/assets/badge-DkRUuVfz.js.gz +0 -0
  50. package/dist/dashboard/assets/{banner-CsGN-nbm.js → banner-CDjW841Q.js} +1 -1
  51. package/dist/dashboard/assets/banner-CDjW841Q.js.gz +0 -0
  52. package/dist/dashboard/assets/{card-mgZ_TLLZ.js → card-Bto5UlL9.js} +1 -1
  53. package/dist/dashboard/assets/card-Bto5UlL9.js.gz +0 -0
  54. package/dist/dashboard/assets/{cli-tools-A1-v8-03.js → cli-tools-DaH6g9hL.js} +1 -1
  55. package/dist/dashboard/assets/cli-tools-DaH6g9hL.js.gz +0 -0
  56. package/dist/dashboard/assets/{clipboard-button-CJ4Zty89.js → clipboard-button-BYKBCzjk.js} +1 -1
  57. package/dist/dashboard/assets/clipboard-button-BYKBCzjk.js.gz +0 -0
  58. package/dist/dashboard/assets/{dialog-B5L2jPQ_.js → dialog-DL0Zftwv.js} +1 -1
  59. package/dist/dashboard/assets/dialog-DL0Zftwv.js.gz +0 -0
  60. package/dist/dashboard/assets/{drawer-DkjUouAj.js → drawer-Dpx_UXaS.js} +1 -1
  61. package/dist/dashboard/assets/drawer-Dpx_UXaS.js.gz +0 -0
  62. package/dist/dashboard/assets/{entry-B-LVLV6o.js → entry-BV7-1NoM.js} +1 -1
  63. package/dist/dashboard/assets/entry-BV7-1NoM.js.gz +0 -0
  64. package/dist/dashboard/assets/{entry-CAhdYl3B.js → entry-CGvdNU6r.js} +1 -1
  65. package/dist/dashboard/assets/entry-CGvdNU6r.js.gz +0 -0
  66. package/dist/dashboard/assets/entry-DScKYvZj.js +4 -0
  67. package/dist/dashboard/assets/entry-DScKYvZj.js.gz +0 -0
  68. package/dist/dashboard/assets/{formatters-CrsdLVsn.js → formatters-BS3DZlZj.js} +1 -1
  69. package/dist/dashboard/assets/formatters-BS3DZlZj.js.gz +0 -0
  70. package/dist/dashboard/assets/{index-BU64Lqvp.js → index-SQnvvTWX.js} +2 -2
  71. package/dist/dashboard/assets/index-SQnvvTWX.js.gz +0 -0
  72. package/dist/dashboard/assets/{inline-B2aDLyyg.js → inline-Dl-cuLK0.js} +1 -1
  73. package/dist/dashboard/assets/inline-Dl-cuLK0.js.gz +0 -0
  74. package/dist/dashboard/assets/{input-BCVkdR9d.js → input-Bfx0EzRc.js} +1 -1
  75. package/dist/dashboard/assets/input-Bfx0EzRc.js.gz +0 -0
  76. package/dist/dashboard/assets/{layout-DNcAdMK-.js → layout-BxhbQ_JS.js} +1 -1
  77. package/dist/dashboard/assets/layout-BxhbQ_JS.js.gz +0 -0
  78. package/dist/dashboard/assets/{list-row-DgrIH0hw.js → list-row-D49Y4ptn.js} +1 -1
  79. package/dist/dashboard/assets/list-row-D49Y4ptn.js.gz +0 -0
  80. package/dist/dashboard/assets/{live-Cae01iu8.js → live-CbVTdX9V.js} +1 -1
  81. package/dist/dashboard/assets/live-CbVTdX9V.js.gz +0 -0
  82. package/dist/dashboard/assets/{network-CRib7H_2.js → network-JGvDveof.js} +1 -1
  83. package/dist/dashboard/assets/network-JGvDveof.js.gz +0 -0
  84. package/dist/dashboard/assets/{numbers-BKQE8Xpz.js → numbers-Dw6kiew-.js} +1 -1
  85. package/dist/dashboard/assets/numbers-Dw6kiew-.js.gz +0 -0
  86. package/dist/dashboard/assets/{pagination-ecPvG_8g.js → pagination-iUoI8GeE.js} +1 -1
  87. package/dist/dashboard/assets/pagination-iUoI8GeE.js.gz +0 -0
  88. package/dist/dashboard/assets/providers/sensenova-intl.png +0 -0
  89. package/dist/dashboard/assets/providers/sensenova.png +0 -0
  90. package/dist/dashboard/assets/providers/zro.svg +3 -0
  91. package/dist/dashboard/assets/providers/zro.svg.gz +0 -0
  92. package/dist/dashboard/assets/quota-DiPD6UKH.js +1 -0
  93. package/dist/dashboard/assets/quota-DiPD6UKH.js.gz +0 -0
  94. package/dist/dashboard/assets/{routing-Ckm1UXuC.js → routing-8jZr-3WE.js} +1 -1
  95. package/dist/dashboard/assets/routing-8jZr-3WE.js.gz +0 -0
  96. package/dist/dashboard/assets/{select-ChoFWyKz.js → select-oO1FxMVb.js} +2 -2
  97. package/dist/dashboard/assets/select-oO1FxMVb.js.gz +0 -0
  98. package/dist/dashboard/assets/{settings-CKjFPlKW.js → settings-CLr3kdIx.js} +1 -1
  99. package/dist/dashboard/assets/settings-CLr3kdIx.js.gz +0 -0
  100. package/dist/dashboard/assets/{stack-DWCwU3fh.js → stack-Y3pDteZr.js} +1 -1
  101. package/dist/dashboard/assets/stack-Y3pDteZr.js.gz +0 -0
  102. package/dist/dashboard/assets/{state-Cg3VL5J_.js → state-CArX_Iak.js} +1 -1
  103. package/dist/dashboard/assets/state-CArX_Iak.js.gz +0 -0
  104. package/dist/dashboard/assets/{switch-Db70BMix.js → switch-DwlwmDO-.js} +1 -1
  105. package/dist/dashboard/assets/switch-DwlwmDO-.js.gz +0 -0
  106. package/dist/dashboard/assets/{useMutation-BqdE0LuM.js → useMutation-DJ9GYEgf.js} +1 -1
  107. package/dist/dashboard/assets/useMutation-DJ9GYEgf.js.gz +0 -0
  108. package/dist/dashboard/index.html +2 -2
  109. package/dist/dashboard/index.html.gz +0 -0
  110. package/dist/dashboard/precompressed-assets.json +54 -53
  111. package/migrations/full-postgres/0046_add_accounts_quota_plan_fetched_at.sql +7 -0
  112. package/migrations/full-postgres/0047_add_provider_device_flow_proxy.sql +17 -0
  113. package/migrations/lite-sqlite3/0046_add_accounts_quota_plan_fetched_at.sql +4 -0
  114. package/migrations/lite-sqlite3/0047_add_provider_device_flow_proxy.sql +19 -0
  115. package/package.json +1 -1
  116. package/src/boot/app.ts +11 -14
  117. package/src/boot/dependencies.ts +5 -7
  118. package/src/boot/flush.ts +20 -0
  119. package/src/boot/lifecycle.ts +7 -5
  120. package/src/config/providers.ts +6 -5
  121. package/src/config/routing.ts +26 -0
  122. package/src/config/spec-metadata.ts +72 -0
  123. package/src/config/spec.ts +28 -8
  124. package/src/console/api-keys/contracts.ts +12 -1
  125. package/src/console/api-keys/routes.ts +40 -28
  126. package/src/console/auth/account-reset.ts +7 -9
  127. package/src/console/backup/contracts.ts +4 -0
  128. package/src/console/backup/export-endpoint.ts +7 -5
  129. package/src/console/backup/nine-router.ts +71 -10
  130. package/src/console/backup/service.ts +8 -1
  131. package/src/console/cli-tools/service.ts +6 -12
  132. package/src/console/console-router-guards.ts +23 -16
  133. package/src/console/console-router.ts +1 -1
  134. package/src/console/domain-registration.ts +7 -4
  135. package/src/console/inferhub/client.ts +1 -1
  136. package/src/console/inferhub/normalize.ts +24 -28
  137. package/src/console/inferhub/types.ts +3 -0
  138. package/src/console/ip-bans/routes.ts +7 -13
  139. package/src/console/observability/live.ts +2 -2
  140. package/src/console/observability/store.ts +0 -53
  141. package/src/console/providers/catalog/contracts.ts +9 -3
  142. package/src/console/providers/catalog/model-operations.ts +2 -3
  143. package/src/console/providers/catalog/provider-operations.ts +1 -1
  144. package/src/console/providers/catalog/routes.ts +3 -2
  145. package/src/console/providers/catalog/store.ts +1914 -1902
  146. package/src/console/providers/detail/routes.ts +1 -0
  147. package/src/console/providers/detail/store.ts +4 -0
  148. package/src/console/providers/oauth/callback-listener.ts +30 -11
  149. package/src/console/providers/oauth/device-egress.ts +123 -0
  150. package/src/console/providers/oauth/routes.ts +123 -16
  151. package/src/console/quota/account-quota-global.ts +14 -24
  152. package/src/console/quota/account-quota-shared.ts +49 -0
  153. package/src/console/quota/account-quota-tenant.ts +11 -23
  154. package/src/console/quota/account-test-egress.ts +2 -2
  155. package/src/console/quota/daily-growth-job.ts +1 -1
  156. package/src/console/quota/quota-view.ts +19 -3
  157. package/src/console/routing/pools/contracts.ts +1 -1
  158. package/src/console/routing/pools/operations.ts +32 -47
  159. package/src/console/routing/pools/probe-verdict.ts +2 -2
  160. package/src/console/routing/pools/relay-deploy.ts +1 -1
  161. package/src/console/routing/pools/store.ts +4 -4
  162. package/src/console/shared/errors.ts +33 -6
  163. package/src/main.ts +2 -3
  164. package/src/network/client.ts +67 -0
  165. package/src/network/egress/bidi-session.ts +322 -0
  166. package/src/network/egress/dial.ts +201 -0
  167. package/src/network/egress/factory.ts +191 -0
  168. package/src/network/{outbound-fetch.ts → egress/fetch.ts} +195 -29
  169. package/src/network/egress/tls-profile.ts +123 -0
  170. package/src/network/egress/ws.ts +175 -0
  171. package/src/network/pool/{byte-accounting.ts → accounting.ts} +3 -2
  172. package/src/network/{types.ts → pool/agent-config.ts} +1 -1
  173. package/src/network/pool/agent.ts +2 -2
  174. package/src/network/{pool-cooldown-store.ts → pool/cooldown-store.ts} +95 -20
  175. package/src/network/{pool-health-machine.ts → pool/health.ts} +205 -6
  176. package/src/network/pool/resolver.ts +10 -849
  177. package/src/network/pool/selector.ts +184 -41
  178. package/src/network/ssrf.ts +7 -7
  179. package/src/network/{retry.ts → util/retry.ts} +1 -1
  180. package/src/network/{rate-limit-verdict.ts → verdict.ts} +1 -1
  181. package/src/observability/event-log.ts +7 -361
  182. package/src/observability/performance-metrics.ts +9 -0
  183. package/src/observability/runtime-metrics.ts +14 -9
  184. package/src/observability/telemetry-buffer.ts +16 -26
  185. package/src/observability/telemetry-reconciliation.ts +9 -37
  186. package/src/observability/telemetry-row.ts +125 -0
  187. package/src/persistence/db-mode.ts +4 -1
  188. package/src/persistence/dialect.ts +18 -0
  189. package/src/persistence/migrate.ts +6 -3
  190. package/src/persistence/schema.ts +17 -0
  191. package/src/persistence/ttl-cache.ts +21 -3
  192. package/src/protocol/codecs.ts +59 -2
  193. package/src/protocol/dsml.ts +34 -86
  194. package/src/protocol/markup-stream-parser.ts +195 -0
  195. package/src/protocol/reasoning-text.ts +19 -0
  196. package/src/protocol/request/chat.ts +10 -10
  197. package/src/protocol/request/gemini.ts +8 -2
  198. package/src/protocol/request/messages.ts +34 -22
  199. package/src/protocol/response/chat.ts +12 -20
  200. package/src/protocol/response/messages.ts +34 -7
  201. package/src/protocol/response/omp-tool-call-markup.ts +43 -86
  202. package/src/protocol/response/responses.ts +18 -2
  203. package/src/protocol/stop-reason.ts +23 -1
  204. package/src/protocol/think-split.ts +112 -0
  205. package/src/protocol/translation/dialect/qwen.ts +26 -58
  206. package/src/protocol/translation/quirks.ts +15 -0
  207. package/src/protocol/upstream/send-claude-messages.ts +2 -2
  208. package/src/providers/adapters/compatible-adapter.ts +76 -12
  209. package/src/providers/authentication/oauth-client.ts +40 -6
  210. package/src/providers/authentication/oauth-device-flow.ts +105 -11
  211. package/src/providers/authentication/oauth-flow-store.ts +82 -1
  212. package/src/providers/authentication/oauth-refresh-service.ts +766 -758
  213. package/src/providers/configured-provider.ts +148 -0
  214. package/src/providers/default-registry.ts +137 -36
  215. package/src/providers/discovery/discovery-types.ts +7 -0
  216. package/src/providers/discovery/probe-phases.ts +20 -6
  217. package/src/providers/discovery/probing-service.ts +22 -2
  218. package/src/providers/integrations/alibabacp/alibabacp-models.ts +1 -2
  219. package/src/providers/integrations/{dashscope-models.ts → alibabacp/dashscope-models.ts} +2 -2
  220. package/src/providers/integrations/antigravity/antigravity-protocol.ts +16 -1
  221. package/src/providers/integrations/buddy/account.ts +969 -0
  222. package/src/providers/integrations/buddy/capabilities.ts +12 -12
  223. package/src/providers/integrations/buddy/{buddy-catalog-shared.ts → catalog.ts} +198 -136
  224. package/src/providers/integrations/buddy/{buddy-chat-shared.ts → chat.ts} +176 -20
  225. package/src/providers/integrations/buddy/codebuddy/index.ts +429 -0
  226. package/src/providers/integrations/buddy/codebuddy/shared.ts +41 -26
  227. package/src/providers/integrations/buddy/headers.ts +311 -0
  228. package/src/providers/integrations/buddy/lifecycle.ts +1018 -0
  229. package/src/providers/integrations/buddy/{buddy-media.ts → media.ts} +1 -1
  230. package/src/providers/integrations/claude/claude-quota.ts +14 -17
  231. package/src/providers/integrations/cline/cline-oauth.ts +10 -5
  232. package/src/providers/integrations/cline/cline.ts +5 -6
  233. package/src/providers/integrations/clixad/clixad-oauth.ts +98 -0
  234. package/src/providers/integrations/clixad/clixad-quota.ts +132 -0
  235. package/src/providers/integrations/clixad/provider.ts +145 -0
  236. package/src/providers/integrations/codex/codex-oauth.ts +8 -5
  237. package/src/providers/integrations/commandcode/quota.ts +22 -28
  238. package/src/providers/integrations/cursor/cursor-auth.ts +1 -9
  239. package/src/providers/integrations/cursor/cursor-exec.ts +9 -2
  240. package/src/providers/integrations/cursor/cursor-hardening.ts +279 -0
  241. package/src/providers/integrations/cursor/cursor-http2.ts +605 -6
  242. package/src/providers/integrations/cursor/cursor-request.ts +26 -12
  243. package/src/providers/integrations/cursor/cursor-shared.ts +8 -5
  244. package/src/providers/integrations/cursor/cursor.ts +574 -110
  245. package/src/providers/integrations/cursor/protocol/messages.ts +22 -29
  246. package/src/providers/integrations/cursor/protocol/protobuf-schema.ts +37 -0
  247. package/src/providers/integrations/deepai/deepai-catalog.ts +149 -0
  248. package/src/providers/integrations/deepai/deepai-common.ts +145 -0
  249. package/src/providers/integrations/deepai/deepai-oauth.ts +129 -0
  250. package/src/providers/integrations/deepai/deepai-quota.ts +97 -0
  251. package/src/providers/integrations/deepai/provider.ts +42 -0
  252. package/src/providers/integrations/devin/devin-auth.ts +7 -0
  253. package/src/providers/integrations/devin/protocol/devin-proto.ts +1 -27
  254. package/src/providers/integrations/factory/factory-oauth.ts +33 -51
  255. package/src/providers/integrations/freebuff/assets/official-system-manager.txt +88 -0
  256. package/src/providers/integrations/freebuff/assets/official-system-worker.txt +57 -0
  257. package/src/providers/integrations/freebuff/assets/official-tools.json +1070 -0
  258. package/src/providers/integrations/freebuff/freebuff-catalog.ts +120 -0
  259. package/src/providers/integrations/freebuff/freebuff-constants.ts +1233 -0
  260. package/src/providers/integrations/freebuff/freebuff-discovery.ts +108 -0
  261. package/src/providers/integrations/freebuff/freebuff-oauth.ts +240 -0
  262. package/src/providers/integrations/freebuff/freebuff-quota.ts +260 -0
  263. package/src/providers/integrations/freebuff/freebuff-virtual.ts +89 -0
  264. package/src/providers/integrations/freebuff/freebuff.ts +682 -0
  265. package/src/providers/integrations/freepi/freepi-catalog.ts +5 -0
  266. package/src/providers/integrations/freepi/freepi-discovery.ts +2 -2
  267. package/src/providers/integrations/freepi/freepi-headers.ts +2 -2
  268. package/src/providers/integrations/freepi/freepi-protocol.ts +11 -2
  269. package/src/providers/integrations/freepi/freepi-realm.ts +0 -3
  270. package/src/providers/integrations/freepi/freepi-session.ts +158 -0
  271. package/src/providers/integrations/freepi/freepi.ts +80 -54
  272. package/src/providers/integrations/github/github-discovery.ts +30 -13
  273. package/src/providers/integrations/github/github-oauth.ts +217 -44
  274. package/src/providers/integrations/github/github-quota.ts +22 -5
  275. package/src/providers/integrations/github/github.ts +40 -32
  276. package/src/providers/integrations/grok/grok-oauth.ts +52 -49
  277. package/src/providers/integrations/grok/grok-ratelimit-store.ts +10 -17
  278. package/src/providers/integrations/grok/grok-turn-index.ts +11 -16
  279. package/src/providers/integrations/hermes/hermes-oauth.ts +3 -3
  280. package/src/providers/integrations/{hermes.ts → hermes/hermes.ts} +42 -13
  281. package/src/providers/integrations/hermes/index.ts +2 -0
  282. package/src/providers/integrations/kilo/kilo-oauth.ts +4 -4
  283. package/src/providers/integrations/kilo/kilo.ts +1 -1
  284. package/src/providers/integrations/kimi/kimi-oauth.ts +3 -3
  285. package/src/providers/integrations/kiro/kiro-catalog.ts +17 -1
  286. package/src/providers/integrations/kiro/kiro-discovery.ts +10 -12
  287. package/src/providers/integrations/kiro/kiro-oauth.ts +2 -2
  288. package/src/providers/integrations/kiro/kiro-quota.ts +8 -4
  289. package/src/providers/integrations/kiro/kiro.ts +15 -19
  290. package/src/providers/integrations/muse/muse-oauth.ts +5 -5
  291. package/src/providers/integrations/ollamacloud/index.ts +2 -0
  292. package/src/providers/integrations/{ollamacloud-models.ts → ollamacloud/models.ts} +2 -2
  293. package/src/providers/integrations/{ollamacloud-quota.ts → ollamacloud/quota.ts} +3 -3
  294. package/src/providers/integrations/opencode/fingerprint.ts +1 -1
  295. package/src/providers/integrations/opencode/opencode-go-quota.ts +2 -8
  296. package/src/providers/integrations/openrouter/provider.ts +1 -1
  297. package/src/providers/integrations/perch/perch-credential.ts +25 -0
  298. package/src/providers/integrations/perch/perch-protocol.ts +0 -6
  299. package/src/providers/integrations/perch/perch-quota.ts +2 -16
  300. package/src/providers/integrations/perch/perch.ts +6 -16
  301. package/src/providers/integrations/qoder/claim-qoder-credits.ts +19 -60
  302. package/src/providers/integrations/qoder/qoder-adapter.ts +20 -125
  303. package/src/providers/integrations/qoder/qoder-auth.ts +109 -30
  304. package/src/providers/integrations/qoder/qoder-catalog.ts +21 -6
  305. package/src/providers/integrations/qoder/qoder-chat.ts +32 -29
  306. package/src/providers/integrations/qoder/qoder-checkin.ts +5 -5
  307. package/src/providers/integrations/qoder/qoder-cosy.ts +37 -9
  308. package/src/providers/integrations/qoder/qoder-oauth.ts +9 -2
  309. package/src/providers/integrations/qoder/qoder-realm.ts +13 -36
  310. package/src/providers/integrations/qoder/qoder-sse.ts +447 -68
  311. package/src/providers/integrations/qoder/qoder-types.ts +9 -0
  312. package/src/providers/integrations/qoder/qoder.ts +3 -26
  313. package/src/providers/integrations/sensenova/provider.ts +144 -0
  314. package/src/providers/integrations/trae/trae-protocol.ts +9 -5
  315. package/src/providers/integrations/trae/trae-quota.ts +21 -29
  316. package/src/providers/integrations/trae/trae-realm.ts +38 -0
  317. package/src/providers/integrations/trae/trae.ts +7 -24
  318. package/src/providers/integrations/vercel/provider.ts +117 -0
  319. package/src/providers/integrations/vercel/vercel-quota.ts +96 -0
  320. package/src/providers/integrations/xai/xai-oauth.ts +5 -5
  321. package/src/providers/integrations/zcode-start-plan/gifts.ts +1 -1
  322. package/src/providers/integrations/zcode-start-plan/oauth.ts +3 -3
  323. package/src/providers/integrations/zcode-start-plan/provider.ts +226 -23
  324. package/src/providers/integrations/zcode-start-plan/quota.ts +19 -2
  325. package/src/providers/integrations/zro/zro.ts +205 -0
  326. package/src/providers/operations/account-error-classification.ts +199 -307
  327. package/src/providers/operations/account-health-service.ts +82 -1
  328. package/src/providers/operations/account-refresh-status.ts +36 -37
  329. package/src/providers/operations/account-tier.ts +15 -16
  330. package/src/providers/operations/client-version-resolver.ts +1 -1
  331. package/src/providers/operations/client-versions.ts +100 -81
  332. package/src/providers/operations/cooldown-rules.ts +11 -16
  333. package/src/providers/operations/oauth-refresh-lead.ts +46 -22
  334. package/src/providers/operations/{model-catalog-cache.ts → provider-caches.ts} +68 -16
  335. package/src/providers/operations/provider-catalog-service.ts +1 -1
  336. package/src/providers/operations/provider-credential-service.ts +118 -51
  337. package/src/providers/operations/provider-rules/github.ts +48 -0
  338. package/src/providers/operations/provider-rules/grok.ts +33 -0
  339. package/src/providers/operations/provider-rules/inception.ts +49 -0
  340. package/src/providers/operations/provider-rules/index.ts +32 -0
  341. package/src/providers/operations/provider-rules/perch.ts +37 -0
  342. package/src/providers/operations/provider-rules/qoder.ts +70 -0
  343. package/src/providers/operations/provider-rules/trae.ts +81 -0
  344. package/src/providers/operations/session-resolution.ts +17 -1
  345. package/src/providers/provider-contracts.ts +6 -4
  346. package/src/providers/{provider-credential-resolver.ts → provider-credentials.ts} +40 -3
  347. package/src/providers/provider-metadata.ts +70 -10
  348. package/src/providers/provider-probe.ts +15 -1
  349. package/src/providers/provider-registry.ts +42 -33
  350. package/src/providers/quota/quota-contracts.ts +2 -0
  351. package/src/providers/quota/quota-limit-windows.ts +16 -5
  352. package/src/providers/quota/quota-support.ts +1 -1
  353. package/src/providers/quota/quota-window-parser.ts +90 -18
  354. package/src/providers/quota/refresh.ts +100 -21
  355. package/src/security/api-key-auth-cache.ts +1 -1
  356. package/src/security/api-key-auth.ts +6 -2
  357. package/src/security/ip-boundary.ts +1 -1
  358. package/src/transport/dispatch/attempt-finalize.ts +15 -14
  359. package/src/transport/dispatch/attempt-loop.ts +73 -19
  360. package/src/transport/dispatch/generation.ts +2 -8
  361. package/src/transport/dispatch/handler-deps.ts +1 -1
  362. package/src/transport/dispatch/leases.ts +1 -1
  363. package/src/transport/dispatch/native-shared.ts +25 -0
  364. package/src/transport/dispatch/proxy-request.ts +8 -46
  365. package/src/transport/dispatch/responses-compact.ts +2 -8
  366. package/src/transport/dispatch/retry-policy.ts +1 -1
  367. package/src/transport/dispatch/streaming/finalize.ts +10 -7
  368. package/src/transport/dispatch/streaming/stream.ts +9 -37
  369. package/src/transport/dispatch/systemone.ts +2 -8
  370. package/src/transport/dispatch/token-count.ts +2 -6
  371. package/src/transport/dispatch/upstream.ts +63 -0
  372. package/src/transport/drain-endpoint.ts +31 -4
  373. package/src/transport/failure-policy.ts +16 -30
  374. package/src/transport/guards/body-policy.ts +7 -1
  375. package/src/transport/guards/error-lifecycle.ts +13 -1
  376. package/src/transport/guards/request-context.ts +22 -14
  377. package/src/transport/request/preparer.ts +81 -71
  378. package/src/transport/routing/bounded-lru.ts +14 -0
  379. package/src/transport/routing/drain-reserve.ts +129 -0
  380. package/src/transport/routing/plan-gate.ts +7 -1
  381. package/src/transport/routing/round-robin.ts +60 -0
  382. package/src/transport/routing/route-catalog.ts +123 -111
  383. package/src/transport/routing/route-model.ts +51 -9
  384. package/src/transport/routing/router.ts +393 -32
  385. package/src/transport/shutdown-notice.ts +1 -1
  386. package/src/transport/surface/adapters.ts +11 -2
  387. package/src/transport/surface/chat/adapter.ts +2 -2
  388. package/src/transport/surface/chat/encode.ts +47 -4
  389. package/src/transport/surface/chat/parse.ts +2 -12
  390. package/src/transport/surface/completion.ts +44 -33
  391. package/src/transport/surface/content-parts.ts +21 -0
  392. package/src/transport/surface/messages/adapter.ts +7 -1
  393. package/src/transport/surface/messages/encode.ts +11 -3
  394. package/src/transport/surface/messages/parse.ts +2 -1
  395. package/src/transport/surface/messages/stream.ts +33 -17
  396. package/src/transport/surface/responses/encode.ts +22 -5
  397. package/src/transport/surface/responses/parse.ts +3 -11
  398. package/src/workers/buddy-growth-tasks.ts +8 -8
  399. package/src/workers/daily-checkin.ts +4 -4
  400. package/src/workers/quota-refresh-sweep.ts +1 -1
  401. package/src/workers/zcode-gift-claim.ts +1 -1
  402. package/dist/dashboard/assets/BannedPage-BxoG1MrL.js.gz +0 -0
  403. package/dist/dashboard/assets/CliToolDetailPage-ixwgKk53.js.gz +0 -0
  404. package/dist/dashboard/assets/CliToolsPage-BmttNV3U.js.gz +0 -0
  405. package/dist/dashboard/assets/CombosPage-BE3qwwfC.js.gz +0 -0
  406. package/dist/dashboard/assets/ConsoleLogPage-CRZdmMIX.js.gz +0 -0
  407. package/dist/dashboard/assets/CustomizationPage-Dv_FJ6Er.js.gz +0 -0
  408. package/dist/dashboard/assets/HealthEventsModal-CoVrw2sM.js +0 -1
  409. package/dist/dashboard/assets/HealthEventsModal-CoVrw2sM.js.gz +0 -0
  410. package/dist/dashboard/assets/InjectSystemPromptPanel-BOoO8rpz.js.gz +0 -0
  411. package/dist/dashboard/assets/LoginPage-ChJuNNVN.js.gz +0 -0
  412. package/dist/dashboard/assets/ModelPicker-BZAJtPg1.js.gz +0 -0
  413. package/dist/dashboard/assets/OverviewPage-BE40s3u5.js.gz +0 -0
  414. package/dist/dashboard/assets/ProviderDetailPage-sfLoQtPV.js +0 -8
  415. package/dist/dashboard/assets/ProviderDetailPage-sfLoQtPV.js.gz +0 -0
  416. package/dist/dashboard/assets/ProviderIcon-CQ6bPUf_.js.gz +0 -0
  417. package/dist/dashboard/assets/ProvidersPage-DcepmPWo.js +0 -1
  418. package/dist/dashboard/assets/ProvidersPage-DcepmPWo.js.gz +0 -0
  419. package/dist/dashboard/assets/ProxyPage-BJ87Pp0U.js +0 -9
  420. package/dist/dashboard/assets/ProxyPage-BJ87Pp0U.js.gz +0 -0
  421. package/dist/dashboard/assets/QuotaPage-gkSyRAKa.js +0 -1
  422. package/dist/dashboard/assets/QuotaPage-gkSyRAKa.js.gz +0 -0
  423. package/dist/dashboard/assets/SettingsPage-C5HPWqEv.js.gz +0 -0
  424. package/dist/dashboard/assets/SetupPage-CFNcATJJ.js.gz +0 -0
  425. package/dist/dashboard/assets/StudioPage-YoD4tY5l.js.gz +0 -0
  426. package/dist/dashboard/assets/UsagePage-DxjaAhb_.js.gz +0 -0
  427. package/dist/dashboard/assets/api-keys-BTlkq_Uk.js.gz +0 -0
  428. package/dist/dashboard/assets/app-BX2nnKqs.css +0 -2
  429. package/dist/dashboard/assets/app-BX2nnKqs.css.gz +0 -0
  430. package/dist/dashboard/assets/assert-CndthXFQ.js +0 -1
  431. package/dist/dashboard/assets/assert-CndthXFQ.js.gz +0 -0
  432. package/dist/dashboard/assets/badge-DdbT_7Am.js.gz +0 -0
  433. package/dist/dashboard/assets/banner-CsGN-nbm.js.gz +0 -0
  434. package/dist/dashboard/assets/card-mgZ_TLLZ.js.gz +0 -0
  435. package/dist/dashboard/assets/cli-tools-A1-v8-03.js.gz +0 -0
  436. package/dist/dashboard/assets/clipboard-button-CJ4Zty89.js.gz +0 -0
  437. package/dist/dashboard/assets/dialog-B5L2jPQ_.js.gz +0 -0
  438. package/dist/dashboard/assets/drawer-DkjUouAj.js.gz +0 -0
  439. package/dist/dashboard/assets/entry-B-LVLV6o.js.gz +0 -0
  440. package/dist/dashboard/assets/entry-CAhdYl3B.js.gz +0 -0
  441. package/dist/dashboard/assets/entry-D2xD0Tmd.js +0 -4
  442. package/dist/dashboard/assets/entry-D2xD0Tmd.js.gz +0 -0
  443. package/dist/dashboard/assets/formatters-CrsdLVsn.js.gz +0 -0
  444. package/dist/dashboard/assets/index-BU64Lqvp.js.gz +0 -0
  445. package/dist/dashboard/assets/inline-B2aDLyyg.js.gz +0 -0
  446. package/dist/dashboard/assets/input-BCVkdR9d.js.gz +0 -0
  447. package/dist/dashboard/assets/layout-DNcAdMK-.js.gz +0 -0
  448. package/dist/dashboard/assets/list-row-DgrIH0hw.js.gz +0 -0
  449. package/dist/dashboard/assets/live-Cae01iu8.js.gz +0 -0
  450. package/dist/dashboard/assets/network-CRib7H_2.js.gz +0 -0
  451. package/dist/dashboard/assets/numbers-BKQE8Xpz.js.gz +0 -0
  452. package/dist/dashboard/assets/pagination-ecPvG_8g.js.gz +0 -0
  453. package/dist/dashboard/assets/quota-DJ1QlAOq.js +0 -1
  454. package/dist/dashboard/assets/quota-DJ1QlAOq.js.gz +0 -0
  455. package/dist/dashboard/assets/routing-Ckm1UXuC.js.gz +0 -0
  456. package/dist/dashboard/assets/select-ChoFWyKz.js.gz +0 -0
  457. package/dist/dashboard/assets/settings-CKjFPlKW.js.gz +0 -0
  458. package/dist/dashboard/assets/stack-DWCwU3fh.js.gz +0 -0
  459. package/dist/dashboard/assets/state-Cg3VL5J_.js.gz +0 -0
  460. package/dist/dashboard/assets/switch-Db70BMix.js.gz +0 -0
  461. package/dist/dashboard/assets/useMutation-BqdE0LuM.js.gz +0 -0
  462. package/src/network/pool-health.ts +0 -210
  463. package/src/providers/bundled-discovery-providers.ts +0 -98
  464. package/src/providers/configured-openai-providers.ts +0 -120
  465. package/src/providers/integrations/buddy/buddy-checkin.ts +0 -521
  466. package/src/providers/integrations/buddy/buddy-cli-headers.ts +0 -76
  467. package/src/providers/integrations/buddy/buddy-conversation-headers.ts +0 -130
  468. package/src/providers/integrations/buddy/buddy-discovery-shared.ts +0 -234
  469. package/src/providers/integrations/buddy/buddy-growth-shared.ts +0 -496
  470. package/src/providers/integrations/buddy/buddy-identity.ts +0 -300
  471. package/src/providers/integrations/buddy/buddy-oauth-shared.ts +0 -341
  472. package/src/providers/integrations/buddy/buddy-quota-shared.ts +0 -417
  473. package/src/providers/integrations/buddy/buddy-server-tools.ts +0 -240
  474. package/src/providers/integrations/buddy/buddy-usage-shared.ts +0 -221
  475. package/src/providers/integrations/buddy/buddy-usage-store.ts +0 -181
  476. package/src/providers/integrations/buddy/codebuddy/cn-catalog.ts +0 -37
  477. package/src/providers/integrations/buddy/codebuddy/cn-provider.ts +0 -185
  478. package/src/providers/integrations/buddy/codebuddy/oauth.ts +0 -47
  479. package/src/providers/integrations/buddy/codebuddy/provider.ts +0 -175
  480. package/src/providers/integrations/buddy/codebuddy/quota.ts +0 -154
  481. package/src/providers/integrations/qoder/chat-proto.ts +0 -328
  482. package/src/providers/integrations/qoder/qoder-grpc.ts +0 -871
  483. package/src/providers/integrations/qoder/qoder-ide.ts +0 -588
  484. package/src/providers/operations/model-discovery-cache.ts +0 -34
  485. package/src/providers/operations/probe-fetch.ts +0 -15
  486. package/src/providers/operations/provider-version-cache.ts +0 -30
  487. package/src/providers/provider-capabilities.ts +0 -33
  488. package/src/providers/provider-credential-envelope.ts +0 -43
  489. /package/dist/dashboard/assets/{app-B7kIWFm5.js → app-C0gTXEoN.js} +0 -0
  490. /package/dist/dashboard/assets/{app-B7kIWFm5.js.gz → app-C0gTXEoN.js.gz} +0 -0
  491. /package/src/network/{http2-bidi.ts → bidi.ts} +0 -0
  492. /package/src/network/pool/{pool-scripts.ts → scripts.ts} +0 -0
  493. /package/src/network/pool/{pool-state.ts → state.ts} +0 -0
  494. /package/src/network/{backoff.ts → util/backoff.ts} +0 -0
  495. /package/src/network/{deduplication.ts → util/dedupe.ts} +0 -0
  496. /package/src/network/{ip-utils.ts → util/ip.ts} +0 -0
@@ -1,758 +1,766 @@
1
- /**
2
- * OAuth token refresh orchestration: in-process single-flight plus a
3
- * Postgres lease-fenced compare-and-swap.
4
- *
5
- * Two layers of coordination:
6
- * 1. In-process: a `Map<accountId, Promise>` so concurrent callers in the
7
- * same process share one upstream refresh call instead of racing.
8
- * 2. Cross-process: a lease (`provider_oauth_states.lease_owner`/
9
- * `lease_expires_at`) acquired via a conditional `UPDATE`, and every
10
- * persisted mutation is fenced on `lease_owner = <this process's lease
11
- * id>` so a process whose lease expired mid-refresh cannot clobber a
12
- * peer that took over — its fenced write simply matches zero rows.
13
- *
14
- * Refresh token, expiry, and lease live in `provider_oauth_states`
15
- * (), a 1:0/1 table keyed by `provider_account_id` — split out
16
- * of `provider_accounts` because the lease is a sparse, cross-process
17
- * coordination concern that stayed null on every non-OAuth (`api_key`)
18
- * account row. Credential ciphertext and health-machine fields stay on
19
- * `provider_accounts`; `persistRefreshed`/`disableAccount` update both
20
- * tables inside one transaction, fenced first on the lease table so a
21
- * losing process's write touches zero rows in either table.
22
- */
23
- import { randomUUID } from "node:crypto";
24
- import { and, eq, inArray, isNull, lt, or, sql } from "drizzle-orm";
25
- import type { CartethyiaDatabase } from "../../persistence/db-handle";
26
- import { sleep } from "../../shared/sleep";
27
- import { providerAccounts, providerOauthStates } from "../../persistence/schema";
28
- import { decryptCredentialToString, encryptCredential } from "../../security/crypto";
29
- import { invalidateCredentialCache } from "../operations/provider-credential-service";
30
- import { resolveAccountDisabledRecoveryMs } from "../../config/cooldowns";
31
- import { asJsonbRecord } from "../../shared/coerce";
32
- import {
33
- loadAccountWithFreshness,
34
- type AccountWithFreshnessRow,
35
- } from "../operations/provider-credential-service";
36
- import { refreshLeadMs } from "../operations/oauth-refresh-lead";
37
- import { classifyAccessTokenUsability } from "./static-token-detection";
38
- import { pushStructuredConsoleLog } from "../../observability/log-ring";
39
-
40
- /**
41
- * Classifies an OAuth token-refresh failure as definitive (the credential is
42
- * permanently revoked/expired and must stop being retried) or transient
43
- * (network blip, upstream 5xx/429, temporary outage — safe to retry later).
44
- *
45
- * Uses a regex over the error's message/body, not a bare HTTP-status
46
- * allowlist, since the same status code (401) can mean either "token is
47
- * dead" or "clock skew, try again" depending on the error body's actual
48
- * wording.
49
- */
50
-
51
- const DEFINITIVE_PATTERN =
52
- /invalid_grant|invalid_token|unauthorized_client|revoked|refresh_token.*expired/i;
53
-
54
- interface OAuthRefreshFailure {
55
- readonly status?: number | undefined;
56
- readonly message: string;
57
- }
58
-
59
- type OAuthFailureClassification = "definitive" | "transient";
60
-
61
- /**
62
- * A bare 401 with no body match for the definitive pattern is still treated
63
- * as definitive (an unrecognized-but-clearly-"unauthorized" response from a
64
- * refresh endpoint almost never self-heals); everything else — timeouts,
65
- * 5xx, 429, 403/"forbidden", "temporarily unavailable" — is transient.
66
- */
67
- export function classifyOAuthRefreshFailure(
68
- failure: OAuthRefreshFailure,
69
- ): OAuthFailureClassification {
70
- if (DEFINITIVE_PATTERN.test(failure.message)) return "definitive";
71
- if (failure.status === 401) return "definitive";
72
- return "transient";
73
- }
74
-
75
- /** How long a lease is valid for before another process may reclaim it. */
76
- const LEASE_TTL_MS = 15_000;
77
-
78
- /**
79
- * Bound on a single token-endpoint call. Neither the reactive path nor the
80
- * sweep passes a caller signal, and a hung endpoint must not block either
81
- * indefinitely — timeouts classify as transient failures.
82
- */
83
- const REFRESH_HTTP_TIMEOUT_MS = 30_000;
84
-
85
- export interface OAuthTokenRefreshResult {
86
- readonly access: string;
87
- /** Omitted when the provider keeps the existing refresh token valid. */
88
- readonly refresh?: string;
89
- readonly expiresAt: Date;
90
- /**
91
- * Auth configuration the refresh corrected or discovered — most often the
92
- * upstream profile the account resolves to, which some providers only report
93
- * beside a fresh access token. Omitted means the stored state stands.
94
- */
95
- readonly auth_state?: Readonly<Record<string, unknown>>;
96
- /**
97
- * Display identity the refresh learned (email/username from the fresh token
98
- * or token response). Omitted when the provider reports nothing new.
99
- */
100
- readonly accountLabel?: string;
101
- }
102
-
103
- /**
104
- * What a refresher is told about the account it is refreshing.
105
- *
106
- * A refresh is not authorized by the refresh token alone everywhere: one flow
107
- * replays the client credentials its login registered, another posts to a
108
- * per-account token endpoint. Both belong to the account rather than to the
109
- * token, so they are handed in here instead of being re-read from storage
110
- * inside every provider client.
111
- */
112
- export interface OAuthRefreshContext {
113
- readonly account_id: string;
114
- readonly auth_state?: Readonly<Record<string, unknown>>;
115
- /** Current access token for endpoints that authenticate refresh requests with both tokens. */
116
- readonly access_token?: string;
117
- /** Decrypted companion secret for flows whose login registered a client secret. */
118
- readonly client_secret?: string;
119
- }
120
-
121
- /** Provider-specific token-endpoint client. Implemented per provider in later phases. */
122
- export interface OAuthTokenRefresher {
123
- /** Requests the current access token in refresh context only when required by the endpoint. */
124
- readonly requiresAccessToken?: boolean;
125
- refresh(
126
- refreshToken: string,
127
- signal?: AbortSignal,
128
- context?: OAuthRefreshContext,
129
- ): Promise<OAuthTokenRefreshResult>;
130
- }
131
-
132
- type AccountRow = AccountWithFreshnessRow;
133
-
134
- /** Acquires the refresh lease for `accountId`, or returns false if another live lease holds it. */
135
- async function acquireLease(
136
- db: CartethyiaDatabase,
137
- accountId: string,
138
- owner: string,
139
- ): Promise<boolean> {
140
- const now = new Date();
141
- const expiresAt = new Date(now.getTime() + LEASE_TTL_MS);
142
- const result = await db
143
- .update(providerOauthStates)
144
- .set({ leaseOwner: owner, leaseExpiresAt: expiresAt })
145
- .where(
146
- and(
147
- eq(providerOauthStates.providerAccountId, accountId),
148
- or(isNull(providerOauthStates.leaseOwner), lt(providerOauthStates.leaseExpiresAt, now)),
149
- ),
150
- )
151
- .returning({ id: providerOauthStates.providerAccountId });
152
- return result.length > 0;
153
- }
154
-
155
- /** Releases the lease without mutating credential state (used for transient failures). */
156
- async function releaseLease(
157
- db: CartethyiaDatabase,
158
- accountId: string,
159
- owner: string,
160
- ): Promise<void> {
161
- await db
162
- .update(providerOauthStates)
163
- .set({ leaseOwner: null, leaseExpiresAt: null })
164
- .where(
165
- and(
166
- eq(providerOauthStates.providerAccountId, accountId),
167
- eq(providerOauthStates.leaseOwner, owner),
168
- ),
169
- );
170
- }
171
-
172
- /**
173
- * D21 invariant guard: a refresh is not a health verdict, so mutating the
174
- * account's `status` here would silently re-enable an account a refresh was
175
- * never meant to resurrect. The `.set()` calls in `persistRefreshed` accept
176
- * `status` at the type level (it is a column on `provider_accounts`), so the
177
- * guard is a runtime assertion that fails loudly in development instead of
178
- * letting a future edit change health state from the refresh path.
179
- */
180
- function assertRefreshNeverMutatesStatus(set: Record<string, unknown>): void {
181
- if ("status" in set) {
182
- throw new Error(
183
- "persistRefreshed must never write `status`; a refresh is not a health verdict",
184
- );
185
- }
186
- }
187
-
188
- /** Persists a successful refresh and releases the lease, fenced on still holding it. */
189
- async function persistRefreshed(
190
- db: CartethyiaDatabase,
191
- accountId: string,
192
- owner: string,
193
- result: OAuthTokenRefreshResult,
194
- ): Promise<boolean> {
195
- return db.transaction(async (tx) => {
196
- // This function must never write `status`; a refresh is not a health
197
- // verdict. The guard below makes an accidental status write a loud dev
198
- // failure rather than a silent re-enable.
199
- const oauthSet = {
200
- ...(result.refresh === undefined
201
- ? {}
202
- : { refreshCiphertext: encryptCredential(result.refresh) }),
203
- expiresAt: result.expiresAt,
204
- leaseOwner: null,
205
- leaseExpiresAt: null,
206
- };
207
- assertRefreshNeverMutatesStatus(oauthSet);
208
- const leaseRows = await tx
209
- .update(providerOauthStates)
210
- .set(oauthSet)
211
- .where(
212
- and(
213
- eq(providerOauthStates.providerAccountId, accountId),
214
- eq(providerOauthStates.leaseOwner, owner),
215
- ),
216
- )
217
- .returning({ id: providerOauthStates.providerAccountId });
218
- if (leaseRows.length === 0) return false;
219
- const accountSet = {
220
- credentialCiphertext: encryptCredential(result.access),
221
- // Only what the refresh reported: a refresher that learned nothing new
222
- // leaves the stored configuration alone rather than blanking it.
223
- // A learned identity only replaces a default label — an operator's
224
- // custom rename is never overwritten. The CASE keeps the read and the
225
- // write in one statement, so a concurrent rename cannot lose to the
226
- // refresh's learned identity.
227
- ...(result.accountLabel === undefined
228
- ? {}
229
- : {
230
- label: sql`CASE WHEN lower(trim(${providerAccounts.label})) = lower(trim(${providerAccounts.providerId})) THEN ${result.accountLabel} ELSE ${providerAccounts.label} END`,
231
- }),
232
- ...(result.auth_state === undefined ? {} : { authState: result.auth_state }),
233
- };
234
- // The second half of the D21 invariant: the credential write must never
235
- // touch the account's health state either.
236
- assertRefreshNeverMutatesStatus(accountSet);
237
- await tx
238
- .update(providerAccounts)
239
- .set(accountSet)
240
- .where(eq(providerAccounts.id, accountId));
241
- invalidateCredentialCache(accountId);
242
- return true;
243
- });
244
- }
245
-
246
- /**
247
- * Packages the account-side facts a refresh needs but the refresh token does
248
- * not carry. The companion secret is decrypted here, once, so provider clients
249
- * never reach into the database.
250
- */
251
- function buildRefreshContext(row: AccountRow, includeAccessToken: boolean): OAuthRefreshContext {
252
- const authState = asJsonbRecord(row.authState);
253
- return {
254
- account_id: row.id,
255
- ...(authState === undefined ? {} : { auth_state: authState }),
256
- ...(includeAccessToken && row.credentialCiphertext
257
- ? { access_token: decryptCredentialToString(row.credentialCiphertext) }
258
- : {}),
259
- ...(row.clientSecretCiphertext
260
- ? { client_secret: decryptCredentialToString(row.clientSecretCiphertext) }
261
- : {}),
262
- };
263
- }
264
-
265
- /**
266
- * Marks an account permanently unusable after a definitive refresh failure,
267
- * fenced on the lease.
268
- *
269
- * @param preserveOperatorDisable When true, an account that is already
270
- * `disabled` with no `cooldownUntil` — the operator's own disable — keeps its
271
- * lack of a recovery deadline (D23). A machine disable (this function being
272
- * the machine) always writes a deadline for itself, but it must not convert
273
- * an operator's terminal decision into a recoverable one: adding a deadline
274
- * to an operator-disabled row is what made the Recover button appear on an
275
- * account the operator disabled on purpose.
276
- */
277
- async function disableAccount(
278
- db: CartethyiaDatabase,
279
- accountId: string,
280
- owner: string,
281
- message: string,
282
- options: { readonly preserveOperatorDisable?: boolean } = {},
283
- ): Promise<void> {
284
- await db.transaction(async (tx) => {
285
- const leaseRows = await tx
286
- .update(providerOauthStates)
287
- .set({ leaseOwner: null, leaseExpiresAt: null })
288
- .where(
289
- and(
290
- eq(providerOauthStates.providerAccountId, accountId),
291
- eq(providerOauthStates.leaseOwner, owner),
292
- ),
293
- )
294
- .returning({ id: providerOauthStates.providerAccountId });
295
- if (leaseRows.length === 0) return;
296
- const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
297
- // D23: read the row before writing so an operator disable (status `disabled`
298
- // with no deadline) is not upgraded with a machine recovery deadline. The
299
- // read is cheap and the lease already serializes refresh writers.
300
- let isOperatorDisable = false;
301
- if (options.preserveOperatorDisable === true) {
302
- const rows = await tx
303
- .select({
304
- status: providerAccounts.status,
305
- cooldownUntil: providerAccounts.cooldownUntil,
306
- })
307
- .from(providerAccounts)
308
- .where(eq(providerAccounts.id, accountId))
309
- .limit(1);
310
- const row = rows[0];
311
- isOperatorDisable = row?.status === "disabled" && row.cooldownUntil == null;
312
- }
313
- await tx
314
- .update(providerAccounts)
315
- .set({
316
- status: "disabled",
317
- lastError: message.slice(0, 500),
318
- // `auth_invalidated` is the category the console reads to render the
319
- // "Re-login required" pill, and the one the quota sweep excludes so it
320
- // stops re-probing a credential that cannot repair itself. The refresh
321
- // grant was definitively rejected and the stored access token could not
322
- // be shown to still work, so this is exactly that state.
323
- lastErrorCategory: "auth_invalidated",
324
- lastErrorAt: new Date(),
325
- // Carries a recovery deadline so `sweepExpiredCooldowns` can bring this
326
- // account back once it elapses; without one, one definitive rejection
327
- // parks the credential for good. The deadline is also what distinguishes
328
- // this (machine-disabled) from an operator's own disable, which leaves
329
- // `cooldownUntil` null and is therefore never swept. See
330
- // `resolveAccountDisabledRecoveryMs`. An operator-disabled row keeps its
331
- // `null` deadline (D23): the machine must not make the operator's
332
- // terminal disable look recoverable.
333
- ...(!isOperatorDisable && disabledRecoveryMs > 0
334
- ? { cooldownUntil: new Date(Date.now() + disabledRecoveryMs) }
335
- : {}),
336
- })
337
- .where(eq(providerAccounts.id, accountId));
338
- invalidateCredentialCache(accountId);
339
- });
340
- }
341
-
342
- /**
343
- * Marks an account as carrying a static bearer token: used exactly as issued,
344
- * never refreshed. Called when the refresh path finds no refresh token on
345
- * record, so the account converges to the same state the console's explicit
346
- * "static token" toggle sets — the sweep skips it and its status stops reading
347
- * as a broken/re-auth account. Not lease-fenced: a non-destructive annotation
348
- * that any process may write, and the credential is not touched.
349
- *
350
- * This never changes `status`: a `disabled` account stays `disabled` (only an
351
- * operator or an explicit recovery may re-enable it), while an `active` one
352
- * stays `active`. The static marker and the error clearing are orthogonal to
353
- * the account's health state.
354
- *
355
- * D24: for a `disabled` row the error fields are deliberately kept. Once the
356
- * account is parked, `lastErrorCategory` is the only record of *why* (e.g.
357
- * `auth_invalidated` → "Re-login required"); wiping it on the static
358
- * annotation would leave a disabled account with no diagnosable reason. An
359
- * `active` row still clears them — there the error is stale by definition.
360
- */
361
- async function markStaticToken(db: CartethyiaDatabase, accountId: string): Promise<void> {
362
- const rows = await db
363
- .select({ status: providerAccounts.status })
364
- .from(providerAccounts)
365
- .where(eq(providerAccounts.id, accountId))
366
- .limit(1);
367
- const isDisabled = rows[0]?.status === "disabled";
368
- await db
369
- .update(providerAccounts)
370
- .set({
371
- staticToken: true,
372
- cooldownUntil: null,
373
- // A static token is a normal, usable credential, not an error state: any
374
- // error left by the failed refresh would otherwise keep the row reading
375
- // "Re-login required" while it is in fact serving. Mirrors the console's
376
- // static-token toggle. For a `disabled` row the category is preserved
377
- // (D24), because there it is the diagnosis, not a stale artifact.
378
- ...(isDisabled
379
- ? {}
380
- : { lastError: null, lastErrorCategory: null, lastErrorAt: null }),
381
- })
382
- .where(eq(providerAccounts.id, accountId));
383
- }
384
-
385
- function failureMessage(error: unknown): string {
386
- if (error instanceof Error) return error.message;
387
- return String(error);
388
- }
389
-
390
- /**
391
- * Decrypts a stored credential, returning `undefined` when the ciphertext
392
- * cannot be decrypted. A corrupt row is not evidence about token expiry, so the
393
- * caller treats it the same as a missing token rather than throwing inside the
394
- * refresh failure path.
395
- */
396
- function safeDecryptCredential(ciphertext: Buffer): string | undefined {
397
- try {
398
- return decryptCredentialToString(ciphertext);
399
- } catch {
400
- return undefined;
401
- }
402
- }
403
-
404
- /** Best-effort HTTP status extraction from an OAuth refresh error. Most
405
- * providers surface `error.status` or embed the status in the message; we
406
- * accept either so the classifier can honor "bare 401 -> definitive". */
407
- function failureStatus(error: unknown): number | undefined {
408
- if (error !== null && typeof error === "object" && "status" in error) {
409
- const raw = error.status;
410
- if (typeof raw === "number" && Number.isFinite(raw)) return raw;
411
- }
412
- const msg = failureMessage(error);
413
- const match = /\b(\d{3})\b/.exec(msg);
414
- if (match) {
415
- const status = Number(match[1]);
416
- if (status >= 400 && status < 600) return status;
417
- }
418
- return undefined;
419
- }
420
-
421
- /**
422
- * Boot-time convergence for OAuth accounts already parked by an earlier
423
- * refresh failure.
424
- *
425
- * Before automatic detection existed, a definitive refresh failure disabled the
426
- * account outright (`oauth_revoked`) even when the stored access token was
427
- * still perfectly usable. This pass re-reads those rows once at startup and
428
- * moves each to where it now belongs:
429
- *
430
- * - access token still usable ⇒ static token; the account stays `disabled`;
431
- * only an operator can re-enable it;
432
- * - otherwise ⇒ the `auth_invalidated` category the console renders as
433
- * "Re-login required" and the quota sweep excludes.
434
- *
435
- * Idempotent and cheap: it only inspects OAuth rows that are `disabled` with a
436
- * credential-failure category, so a healthy deployment scans nothing. The
437
- * credential is decoded, never verified — the same structural-only trust the
438
- * dispatch path already places in the stored token.
439
- */
440
- export async function reconcileStaticTokenAccounts(db: CartethyiaDatabase): Promise<number> {
441
- const candidates = await db
442
- .select({
443
- id: providerAccounts.id,
444
- credentialCiphertext: providerAccounts.credentialCiphertext,
445
- staticToken: providerAccounts.staticToken,
446
- })
447
- .from(providerAccounts)
448
- .where(
449
- and(
450
- eq(providerAccounts.credentialKind, "oauth"),
451
- eq(providerAccounts.status, "disabled"),
452
- or(
453
- eq(providerAccounts.lastErrorCategory, "oauth_revoked"),
454
- eq(providerAccounts.lastErrorCategory, "auth_invalidated"),
455
- ),
456
- ),
457
- );
458
- let changed = 0;
459
- for (const row of candidates) {
460
- if (row.staticToken) continue;
461
- const usability = classifyAccessTokenUsability(
462
- row.credentialCiphertext ? safeDecryptCredential(row.credentialCiphertext) : undefined,
463
- );
464
- if (usability === "usable") {
465
- await markStaticToken(db, row.id);
466
- invalidateCredentialCache(row.id);
467
- pushStructuredConsoleLog("info", "OAuth account recovered as a static token at boot", {
468
- event: "token_refresh",
469
- accountId: row.id,
470
- errorCode: "static_token",
471
- });
472
- changed += 1;
473
- continue;
474
- }
475
- // Converge the legacy category name so the console and the quota sweep see
476
- // one spelling of "credential rejected, operator must re-login".
477
- await db
478
- .update(providerAccounts)
479
- .set({ lastErrorCategory: "auth_invalidated" })
480
- .where(eq(providerAccounts.id, row.id));
481
- changed += 1;
482
- }
483
- return changed;
484
- }
485
-
486
- /**
487
- * Rows whose OAuth token is due for proactive refresh (used by the sweep worker).
488
- *
489
- * The due test is applied in JS, not as one SQL `expires_at < cutoff`, because
490
- * the lead is per provider (`refreshLeadMs`): Claude re-mints ~4h ahead, Codex
491
- * ~5 days, Antigravity ~5 min, and a single global cutoff cannot express that.
492
- * Accounts with no recorded expiry are always due — an account the sweep cannot
493
- * see is one whose token dies silently, which is the failure this guards.
494
- */
495
- /**
496
- * Rows one due-scan reads. The expiry test runs in JS because the lead is per
497
- * provider, so the bound has to sit on the query itself.
498
- */
499
- const OAUTH_DUE_SCAN_LIMIT = 2_000;
500
-
501
- export async function loadDueOAuthAccounts(
502
- db: CartethyiaDatabase,
503
- skewMs?: number,
504
- ): Promise<readonly { readonly id: string; readonly providerId: string }[]> {
505
- const rows = await db
506
- .select({
507
- id: providerAccounts.id,
508
- providerId: providerAccounts.providerId,
509
- expiresAt: providerOauthStates.expiresAt,
510
- staticToken: providerAccounts.staticToken,
511
- })
512
- .from(providerAccounts)
513
- .innerJoin(providerOauthStates, eq(providerOauthStates.providerAccountId, providerAccounts.id))
514
- .where(
515
- and(
516
- eq(providerAccounts.credentialKind, "oauth"),
517
- // `cooldown` is included: a cooling account's token still expires, and
518
- // a refresh that lands while it is parked is what lets it serve the
519
- // moment the cooldown ends. `disabled` stays out — that is an
520
- // operator decision, and refreshing it would spend a grant on an
521
- // account that must not return to rotation on its own.
522
- inArray(providerAccounts.status, ["active", "cooldown"]),
523
- ),
524
- )
525
- // The due test is applied in JS, so this limit bounds the scan, not the
526
- // result. One pass cannot read the whole table; accounts past the limit
527
- // are reached on a later tick once the ones in front have been refreshed
528
- // and dropped out of the window.
529
- .limit(OAUTH_DUE_SCAN_LIMIT);
530
- const now = Date.now();
531
- return rows
532
- .filter((row) => {
533
- // A static token is used exactly as issued: there is no refresh grant to
534
- // run, so the sweep must not touch it. Refreshing it every pass only
535
- // floods the log with a guaranteed no-op. The operator clears the flag
536
- // when the account has a real refresh token again.
537
- if (row.staticToken) return false;
538
- if (row.expiresAt == null) return true;
539
- const lead = skewMs ?? refreshLeadMs(row.providerId);
540
- return row.expiresAt.getTime() - lead <= now;
541
- })
542
- .map((row) => ({ id: row.id, providerId: row.providerId }));
543
- }
544
-
545
- export class OAuthRefreshService {
546
- /** Bound on concurrently tracked single-flight refreshes; keys are account ids, so the real bound is the account count. */
547
- private static readonly MAX_INFLIGHT = 1_000;
548
-
549
- readonly #db: CartethyiaDatabase;
550
- readonly #inFlight = new Map<string, Promise<string | null>>();
551
-
552
- constructor(db: CartethyiaDatabase) {
553
- this.#db = db;
554
- }
555
-
556
- /**
557
- * Ensures a fresh access token for an OAuth account, refreshing it first
558
- * if it's within `opts.skewMs` of expiry (or already expired). When
559
- * `opts.force` is `true`, the freshness check is bypassed and the
560
- * refresher is always invoked — used by the 401 dispatch retry so a peer
561
- * that just refreshed cannot cause the caller to reuse a dead cached
562
- * secret. Returns the current decrypted access token, or `null` when the
563
- * account isn't OAuth, has no stored refresh token, or refresh failed.
564
- *
565
- * `opts.origin` labels who asked for the refresh (`ui`/`sweep`/`dispatch`)
566
- * and is carried on the `token_refresh` log lines (D26). The origin is
567
- * observational only — it never changes what a refresh does.
568
- */
569
- async ensureFreshAccessToken(
570
- accountId: string,
571
- refresher: OAuthTokenRefresher,
572
- opts: { force?: boolean; skewMs?: number; origin?: "ui" | "sweep" | "dispatch" } = {},
573
- ): Promise<string | null> {
574
- const force = opts.force === true;
575
- // No explicit skew: the per-provider lead is applied at load time.
576
- const skewMs = opts.skewMs;
577
- const origin = opts.origin ?? "sweep";
578
- // Forced refreshes get their own in-flight slot so they don't merge with
579
- // a raced non-force call that would otherwise short-circuit.
580
- const key = force ? `${accountId}:force` : accountId;
581
- const existing = this.#inFlight.get(key);
582
- if (existing) return existing;
583
- // Eviction guard: keys are account ids (bounded by the accounts table),
584
- // but a runaway caller could still fan out unique keys; drop the oldest
585
- // in-flight task rather than grow without bound. An evicted task's
586
- // `.finally` delete is a no-op for a key no longer present, and a racing
587
- // duplicate refresh is fenced cross-process by the lease below.
588
- if (this.#inFlight.size >= OAuthRefreshService.MAX_INFLIGHT) {
589
- const oldestKey = this.#inFlight.keys().next().value;
590
- if (oldestKey !== undefined) this.#inFlight.delete(oldestKey);
591
- }
592
- const task = this.#refreshIfDue(accountId, refresher, skewMs, force, origin).finally(() => {
593
- this.#inFlight.delete(key);
594
- });
595
- this.#inFlight.set(key, task);
596
- return task;
597
- }
598
-
599
- async #refreshIfDue(
600
- accountId: string,
601
- refresher: OAuthTokenRefresher,
602
- skewMs: number | undefined,
603
- force: boolean,
604
- origin: "ui" | "sweep" | "dispatch",
605
- ): Promise<string | null> {
606
- const account = await loadAccountWithFreshness(this.#db, accountId, skewMs);
607
- if (!account || account.row.credentialKind !== "oauth") return null;
608
- const { row, dueAt } = account;
609
- // A static token is used exactly as issued and never refreshed. There is
610
- // nothing to re-mint, so return null — the same "cannot refresh" signal a
611
- // missing refresh token gives — rather than re-issuing the identical
612
- // credential to a forced (401) retry, which would only repeat the same
613
- // request against the same upstream. Dispatch still uses the stored token
614
- // directly (`dueAt` is undefined for a static token, so the caller's
615
- // refresh branch is skipped and the decrypted credential is used as-is).
616
- if (row.staticToken) return null;
617
- if (!force && dueAt !== undefined && dueAt > Date.now()) {
618
- return row.credentialCiphertext
619
- ? decryptCredentialToString(row.credentialCiphertext)
620
- : null;
621
- }
622
- return this.#refreshNow(row, refresher, skewMs, origin);
623
- }
624
-
625
- async #refreshNow(
626
- row: AccountRow,
627
- refresher: OAuthTokenRefresher,
628
- skewMs: number | undefined,
629
- origin: "ui" | "sweep" | "dispatch",
630
- ): Promise<string | null> {
631
- // D26: statusBefore is read once up front (the row projection used by the
632
- // refresh path does not carry status) and statusAfter is re-read after any
633
- // mutation, so the log proves a refresh never flipped the account.
634
- const statusBefore = await readAccountStatus(this.#db, row.id);
635
- pushStructuredConsoleLog("info", "OAuth token refresh started", {
636
- event: "token_refresh",
637
- accountId: row.id,
638
- providerId: row.providerId,
639
- details: { origin, statusBefore },
640
- });
641
- const owner = randomUUID();
642
- const leased = await acquireLease(this.#db, row.id, owner);
643
- if (!leased) {
644
- // A peer holds the lease — briefly wait for it to finish, then read whatever it left.
645
- await sleep(500);
646
- const fresh = await loadAccountWithFreshness(this.#db, row.id, skewMs);
647
- return fresh?.row.credentialCiphertext && fresh.dueAt !== undefined && fresh.dueAt > Date.now()
648
- ? decryptCredentialToString(fresh.row.credentialCiphertext)
649
- : null;
650
- }
651
- if (!row.refreshCiphertext) {
652
- await releaseLease(this.#db, row.id, owner);
653
- // No refresh token on record: the access token is used exactly as issued
654
- // and cannot be re-minted, so every future refresh attempt is a
655
- // guaranteed no-op. Mark the account static — the same state the
656
- // console's toggle sets — so the sweep stops retrying it and the status
657
- // reads as an informational static token rather than a broken account.
658
- await markStaticToken(this.#db, row.id);
659
- pushStructuredConsoleLog("info", "OAuth token refresh skipped: static token", {
660
- event: "token_refresh",
661
- accountId: row.id,
662
- providerId: row.providerId,
663
- errorCode: "static_token",
664
- details: { origin },
665
- });
666
- return null;
667
- }
668
- const refreshToken = decryptCredentialToString(row.refreshCiphertext);
669
- try {
670
- const result = await refresher.refresh(
671
- refreshToken,
672
- AbortSignal.timeout(REFRESH_HTTP_TIMEOUT_MS),
673
- buildRefreshContext(row, refresher.requiresAccessToken === true),
674
- );
675
- const persisted = await persistRefreshed(this.#db, row.id, owner, result);
676
- // D26: `statusAfter` is re-read from the DB rather than assumed — a
677
- // successful refresh never changes `status`, and the log proves it.
678
- const statusAfter = await readAccountStatus(this.#db, row.id);
679
- pushStructuredConsoleLog(persisted ? "info" : "warn", persisted ? "OAuth token refresh completed" : "OAuth token refresh lost lease", {
680
- event: "token_refresh",
681
- accountId: row.id,
682
- providerId: row.providerId,
683
- ...(persisted ? {} : { errorCode: "refresh_lease_lost" }),
684
- details: {
685
- expiresAt: result.expiresAt.toISOString(),
686
- origin,
687
- statusBefore,
688
- statusAfter,
689
- },
690
- });
691
- return persisted ? result.access : null;
692
- } catch (error) {
693
- const classification = classifyOAuthRefreshFailure({
694
- message: failureMessage(error),
695
- status: failureStatus(error),
696
- });
697
- pushStructuredConsoleLog("error", "OAuth token refresh failed", {
698
- event: "token_refresh",
699
- accountId: row.id,
700
- providerId: row.providerId,
701
- errorCode: classification,
702
- details: { origin, statusBefore },
703
- });
704
- if (classification === "definitive") {
705
- // A definitive refresh failure means the refresh grant is dead — but
706
- // the access token already on the row may still be usable as issued.
707
- // That is exactly the static-token case: stop refreshing, keep serving.
708
- // Only when the token cannot be shown to work (expired or opaque) does
709
- // the account need the operator, and it is parked for re-auth.
710
- const accessUsability = classifyAccessTokenUsability(
711
- row.credentialCiphertext
712
- ? safeDecryptCredential(row.credentialCiphertext)
713
- : undefined,
714
- );
715
- if (accessUsability === "usable") {
716
- await releaseLease(this.#db, row.id, owner);
717
- await markStaticToken(this.#db, row.id);
718
- const statusAfter = await readAccountStatus(this.#db, row.id);
719
- pushStructuredConsoleLog("info", "OAuth refresh failed; access token still valid, marking static", {
720
- event: "token_refresh",
721
- accountId: row.id,
722
- providerId: row.providerId,
723
- errorCode: "static_token",
724
- details: { origin, statusBefore, statusAfter },
725
- });
726
- } else {
727
- await disableAccount(this.#db, row.id, owner, failureMessage(error), {
728
- // D23: a definitive refresh failure must not turn an operator's
729
- // disable into a machine-recoverable one. If the row is already
730
- // `disabled` without a deadline, leave it that way.
731
- preserveOperatorDisable: true,
732
- });
733
- const statusAfter = await readAccountStatus(this.#db, row.id);
734
- pushStructuredConsoleLog("warn", "OAuth refresh failed definitively; account disabled for re-auth", {
735
- event: "token_refresh",
736
- accountId: row.id,
737
- providerId: row.providerId,
738
- errorCode: classification,
739
- details: { origin, statusBefore, statusAfter },
740
- });
741
- }
742
- } else {
743
- await releaseLease(this.#db, row.id, owner);
744
- }
745
- return null;
746
- }
747
- }
748
- }
749
-
750
- /** Reads one account's status (used only for logging/observability, D26). */
751
- async function readAccountStatus(db: CartethyiaDatabase, accountId: string): Promise<string | undefined> {
752
- const rows = await db
753
- .select({ status: providerAccounts.status })
754
- .from(providerAccounts)
755
- .where(eq(providerAccounts.id, accountId))
756
- .limit(1);
757
- return rows[0]?.status;
758
- }
1
+ /**
2
+ * OAuth token refresh orchestration: in-process single-flight plus a
3
+ * Postgres lease-fenced compare-and-swap.
4
+ *
5
+ * Two layers of coordination:
6
+ * 1. In-process: a `Map<accountId, Promise>` so concurrent callers in the
7
+ * same process share one upstream refresh call instead of racing.
8
+ * 2. Cross-process: a lease (`provider_oauth_states.lease_owner`/
9
+ * `lease_expires_at`) acquired via a conditional `UPDATE`, and every
10
+ * persisted mutation is fenced on `lease_owner = <this process's lease
11
+ * id>` so a process whose lease expired mid-refresh cannot clobber a
12
+ * peer that took over — its fenced write simply matches zero rows.
13
+ *
14
+ * Refresh token, expiry, and lease live in `provider_oauth_states`
15
+ * (), a 1:0/1 table keyed by `provider_account_id` — split out
16
+ * of `provider_accounts` because the lease is a sparse, cross-process
17
+ * coordination concern that stayed null on every non-OAuth (`api_key`)
18
+ * account row. Credential ciphertext and health-machine fields stay on
19
+ * `provider_accounts`; `persistRefreshed`/`disableAccount` update both
20
+ * tables inside one transaction, fenced first on the lease table so a
21
+ * losing process's write touches zero rows in either table.
22
+ */
23
+ import { randomUUID } from "node:crypto";
24
+ import { and, eq, inArray, isNull, lt, or, sql } from "drizzle-orm";
25
+ import type { CartethyiaDatabase } from "../../persistence/db-handle";
26
+ import { sleep } from "../../shared/sleep";
27
+ import { providerAccounts, providerOauthStates } from "../../persistence/schema";
28
+ import { decryptCredentialToString, encryptCredential } from "../../security/crypto";
29
+ import { invalidateCredentialCache, loadAccountWithOauthState } from "../operations/provider-credential-service";
30
+ import { applyAccountVerdict } from "../operations/account-health-service";
31
+ import { resolveAccountDisabledRecoveryMs } from "../../config/cooldowns";
32
+ import { asJsonbRecord } from "../../shared/coerce";
33
+ import {
34
+ loadAccountWithFreshness,
35
+ type AccountWithFreshnessRow,
36
+ } from "../operations/provider-credential-service";
37
+ import { refreshLeadMs } from "../operations/oauth-refresh-lead";
38
+ import { classifyAccessTokenUsability } from "./static-token-detection";
39
+ import type { OAuthAccountIdentity } from "./oauth-flow-store";
40
+ import { pushStructuredConsoleLog } from "../../observability/log-ring";
41
+
42
+ /**
43
+ * Classifies an OAuth token-refresh failure as definitive (the credential is
44
+ * permanently revoked/expired and must stop being retried) or transient
45
+ * (network blip, upstream 5xx/429, temporary outage — safe to retry later).
46
+ *
47
+ * Uses a regex over the error's message/body, not a bare HTTP-status
48
+ * allowlist, since the same status code (401) can mean either "token is
49
+ * dead" or "clock skew, try again" depending on the error body's actual
50
+ * wording.
51
+ */
52
+
53
+ const DEFINITIVE_PATTERN =
54
+ /invalid_grant|invalid_token|unauthorized_client|revoked|refresh_token.*expired|github_copilot_profile_mismatch/i;
55
+
56
+ interface OAuthRefreshFailure {
57
+ readonly status?: number | undefined;
58
+ readonly message: string;
59
+ }
60
+
61
+ type OAuthFailureClassification = "definitive" | "transient";
62
+
63
+ /**
64
+ * A bare 401 with no body match for the definitive pattern is still treated
65
+ * as definitive (an unrecognized-but-clearly-"unauthorized" response from a
66
+ * refresh endpoint almost never self-heals); everything else — timeouts,
67
+ * 5xx, 429, 403/"forbidden", "temporarily unavailable" — is transient.
68
+ */
69
+ export function classifyOAuthRefreshFailure(
70
+ failure: OAuthRefreshFailure,
71
+ ): OAuthFailureClassification {
72
+ if (DEFINITIVE_PATTERN.test(failure.message)) return "definitive";
73
+ if (failure.status === 401) return "definitive";
74
+ return "transient";
75
+ }
76
+
77
+ /** How long a lease is valid for before another process may reclaim it. */
78
+ const LEASE_TTL_MS = 15_000;
79
+
80
+ /**
81
+ * Bound on a single token-endpoint call. Neither the reactive path nor the
82
+ * sweep passes a caller signal, and a hung endpoint must not block either
83
+ * indefinitely — timeouts classify as transient failures.
84
+ */
85
+ const REFRESH_HTTP_TIMEOUT_MS = 30_000;
86
+
87
+ export interface OAuthTokenRefreshResult extends OAuthAccountIdentity {
88
+ readonly access: string;
89
+ /** Omitted when the provider keeps the existing refresh token valid. */
90
+ readonly refresh?: string;
91
+ readonly expiresAt: Date;
92
+ /**
93
+ * Auth configuration the refresh corrected or discovered — most often the
94
+ * upstream profile the account resolves to, which some providers only report
95
+ * beside a fresh access token. Omitted means the stored state stands.
96
+ */
97
+ readonly auth_state?: Readonly<Record<string, unknown>>;
98
+ /**
99
+ * Display identity the refresh learned (email/username from the fresh token
100
+ * or token response). Omitted when the provider reports nothing new.
101
+ */
102
+ readonly accountLabel?: string;
103
+ }
104
+
105
+ /**
106
+ * What a refresher is told about the account it is refreshing.
107
+ *
108
+ * A refresh is not authorized by the refresh token alone everywhere: one flow
109
+ * replays the client credentials its login registered, another posts to a
110
+ * per-account token endpoint. Both belong to the account rather than to the
111
+ * token, so they are handed in here instead of being re-read from storage
112
+ * inside every provider client.
113
+ */
114
+ export interface OAuthRefreshContext {
115
+ readonly account_id: string;
116
+ readonly auth_state?: Readonly<Record<string, unknown>>;
117
+ /** Current access token for endpoints that authenticate refresh requests with both tokens. */
118
+ readonly access_token?: string;
119
+ /** Decrypted companion secret for flows whose login registered a client secret. */
120
+ readonly client_secret?: string;
121
+ }
122
+
123
+ /** Provider-specific token-endpoint client. Implemented per provider in later phases. */
124
+ export interface OAuthTokenRefresher {
125
+ /** Requests the current access token in refresh context only when required by the endpoint. */
126
+ readonly requiresAccessToken?: boolean;
127
+ refresh(
128
+ refreshToken: string,
129
+ signal?: AbortSignal,
130
+ context?: OAuthRefreshContext,
131
+ ): Promise<OAuthTokenRefreshResult>;
132
+ }
133
+
134
+ type AccountRow = AccountWithFreshnessRow;
135
+
136
+ /** Acquires the refresh lease for `accountId`, or returns false if another live lease holds it. */
137
+ async function acquireLease(
138
+ db: CartethyiaDatabase,
139
+ accountId: string,
140
+ owner: string,
141
+ ): Promise<boolean> {
142
+ const now = new Date();
143
+ const expiresAt = new Date(now.getTime() + LEASE_TTL_MS);
144
+ const result = await db
145
+ .update(providerOauthStates)
146
+ .set({ leaseOwner: owner, leaseExpiresAt: expiresAt })
147
+ .where(
148
+ and(
149
+ eq(providerOauthStates.providerAccountId, accountId),
150
+ or(isNull(providerOauthStates.leaseOwner), lt(providerOauthStates.leaseExpiresAt, now)),
151
+ ),
152
+ )
153
+ .returning({ id: providerOauthStates.providerAccountId });
154
+ return result.length > 0;
155
+ }
156
+
157
+ /** Releases the lease without mutating credential state (used for transient failures). */
158
+ async function releaseLease(
159
+ db: CartethyiaDatabase,
160
+ accountId: string,
161
+ owner: string,
162
+ ): Promise<void> {
163
+ await db
164
+ .update(providerOauthStates)
165
+ .set({ leaseOwner: null, leaseExpiresAt: null })
166
+ .where(
167
+ and(
168
+ eq(providerOauthStates.providerAccountId, accountId),
169
+ eq(providerOauthStates.leaseOwner, owner),
170
+ ),
171
+ );
172
+ }
173
+
174
+ /**
175
+ * D21 invariant guard: a refresh is not a health verdict, so mutating the
176
+ * account's `status` here would silently re-enable an account a refresh was
177
+ * never meant to resurrect. The `.set()` calls in `persistRefreshed` accept
178
+ * `status` at the type level (it is a column on `provider_accounts`), so the
179
+ * guard is a runtime assertion that fails loudly in development instead of
180
+ * letting a future edit change health state from the refresh path.
181
+ */
182
+ function assertRefreshNeverMutatesStatus(set: Record<string, unknown>): void {
183
+ if ("status" in set) {
184
+ throw new Error(
185
+ "persistRefreshed must never write `status`; a refresh is not a health verdict",
186
+ );
187
+ }
188
+ }
189
+
190
+ /** Persists a successful refresh and releases the lease, fenced on still holding it. */
191
+ async function persistRefreshed(
192
+ db: CartethyiaDatabase,
193
+ accountId: string,
194
+ owner: string,
195
+ result: OAuthTokenRefreshResult,
196
+ ): Promise<boolean> {
197
+ return db.transaction(async (tx) => {
198
+ // This function must never write `status`; a refresh is not a health
199
+ // verdict. The guard below makes an accidental status write a loud dev
200
+ // failure rather than a silent re-enable.
201
+ const oauthSet = {
202
+ ...(result.refresh === undefined
203
+ ? {}
204
+ : { refreshCiphertext: encryptCredential(result.refresh) }),
205
+ expiresAt: result.expiresAt,
206
+ leaseOwner: null,
207
+ leaseExpiresAt: null,
208
+ };
209
+ assertRefreshNeverMutatesStatus(oauthSet);
210
+ const leaseRows = await tx
211
+ .update(providerOauthStates)
212
+ .set(oauthSet)
213
+ .where(
214
+ and(
215
+ eq(providerOauthStates.providerAccountId, accountId),
216
+ eq(providerOauthStates.leaseOwner, owner),
217
+ ),
218
+ )
219
+ .returning({ id: providerOauthStates.providerAccountId });
220
+ if (leaseRows.length === 0) return false;
221
+ const learnedLabel = result.accountLabel ?? result.accountEmail ?? result.accountName;
222
+ const accountSet = {
223
+ credentialCiphertext: encryptCredential(result.access),
224
+ // Only what the refresh reported: a refresher that learned nothing new
225
+ // leaves the stored configuration alone rather than blanking it.
226
+ // A learned identity only replaces a default label � an operator's
227
+ // custom rename is never overwritten. The CASE keeps the read and the
228
+ // write in one statement, so a concurrent rename cannot lose to the
229
+ // refresh's learned identity.
230
+ ...(learnedLabel === undefined
231
+ ? {}
232
+ : {
233
+ label: sql`CASE WHEN lower(trim(${providerAccounts.label})) = lower(trim(${providerAccounts.providerId})) OR lower(trim(${providerAccounts.label})) = lower(trim(${learnedLabel})) THEN ${learnedLabel} ELSE ${providerAccounts.label} END`,
234
+ }),
235
+ ...(result.auth_state === undefined && result.accountName === undefined
236
+ ? {}
237
+ : {
238
+ authState: {
239
+ ...(result.auth_state ?? {}),
240
+ ...(result.accountName === undefined ? {} : { accountName: result.accountName }),
241
+ },
242
+ }),
243
+ ...(result.accountEmail === undefined ? {} : { accountEmail: result.accountEmail }),
244
+ ...(result.accountEmail !== undefined || result.accountName !== undefined
245
+ ? { identityCheckedAt: new Date() }
246
+ : {}),
247
+ };
248
+ // The second half of the D21 invariant: the credential write must never
249
+ // touch the account's health state either.
250
+ assertRefreshNeverMutatesStatus(accountSet);
251
+ await tx
252
+ .update(providerAccounts)
253
+ .set(accountSet)
254
+ .where(eq(providerAccounts.id, accountId));
255
+ invalidateCredentialCache(accountId);
256
+ return true;
257
+ });
258
+ }
259
+
260
+ /**
261
+ * Packages the account-side facts a refresh needs but the refresh token does
262
+ * not carry. The companion secret is decrypted here, once, so provider clients
263
+ * never reach into the database.
264
+ */
265
+ function buildRefreshContext(row: AccountRow, includeAccessToken: boolean): OAuthRefreshContext {
266
+ const authState = asJsonbRecord(row.authState);
267
+ return {
268
+ account_id: row.id,
269
+ ...(authState === undefined ? {} : { auth_state: authState }),
270
+ ...(includeAccessToken && row.credentialCiphertext
271
+ ? { access_token: decryptCredentialToString(row.credentialCiphertext) }
272
+ : {}),
273
+ ...(row.clientSecretCiphertext
274
+ ? { client_secret: decryptCredentialToString(row.clientSecretCiphertext) }
275
+ : {}),
276
+ };
277
+ }
278
+
279
+ /**
280
+ * Marks an account permanently unusable after a definitive refresh failure,
281
+ * fenced on the lease.
282
+ *
283
+ * @param preserveOperatorDisable When true, an account that is already
284
+ * `disabled` with no `cooldownUntil` — the operator's own disable — keeps its
285
+ * lack of a recovery deadline (D23). A machine disable (this function being
286
+ * the machine) always writes a deadline for itself, but it must not convert
287
+ * an operator's terminal decision into a recoverable one: adding a deadline
288
+ * to an operator-disabled row is what made the Recover button appear on an
289
+ * account the operator disabled on purpose.
290
+ */
291
+ async function disableAccount(
292
+ db: CartethyiaDatabase,
293
+ accountId: string,
294
+ owner: string,
295
+ message: string,
296
+ options: { readonly preserveOperatorDisable?: boolean } = {},
297
+ ): Promise<void> {
298
+ await db.transaction(async (tx) => {
299
+ const leaseRows = await tx
300
+ .update(providerOauthStates)
301
+ .set({ leaseOwner: null, leaseExpiresAt: null })
302
+ .where(
303
+ and(
304
+ eq(providerOauthStates.providerAccountId, accountId),
305
+ eq(providerOauthStates.leaseOwner, owner),
306
+ ),
307
+ )
308
+ .returning({ id: providerOauthStates.providerAccountId });
309
+ if (leaseRows.length === 0) return;
310
+ const disabledRecoveryMs = resolveAccountDisabledRecoveryMs();
311
+ // Converge on the health application seam (proposal §3.5): a definitive
312
+ // OAuth refresh failure is a producer into `applyAccountVerdict`, not a
313
+ // separate state machine. The seam preserves D23 (operator-disable rows
314
+ // keep their `null` recovery deadline) via `preserveOperatorDisable`.
315
+ const applied = await applyAccountVerdict(tx, {
316
+ accountId,
317
+ status: "disabled",
318
+ // `auth_invalidated` is the category the console reads to render the
319
+ // "Re-login required" pill, and the one the quota sweep excludes so it
320
+ // stops re-probing a credential that cannot repair itself. The refresh
321
+ // grant was definitively rejected and the stored access token could not
322
+ // be shown to still work, so this is exactly that state.
323
+ reason: message,
324
+ category: "auth_invalidated",
325
+ // Carries a recovery deadline so `sweepExpiredCooldowns` can bring this
326
+ // account back once it elapses; without one, one definitive rejection
327
+ // parks the credential for good. The deadline is also what distinguishes
328
+ // this (machine-disabled) from an operator's own disable, which leaves
329
+ // `cooldownUntil` null and is therefore never swept (D23).
330
+ cooldownUntil:
331
+ disabledRecoveryMs > 0 ? new Date(Date.now() + disabledRecoveryMs) : null,
332
+ ...(options.preserveOperatorDisable === true
333
+ ? { preserveOperatorDisable: true }
334
+ : {}),
335
+ });
336
+ if (applied) invalidateCredentialCache(accountId);
337
+ });
338
+ }
339
+
340
+ /**
341
+ * Marks an account as carrying a static bearer token: used exactly as issued,
342
+ * never refreshed. Called when the refresh path finds no refresh token on
343
+ * record, so the account converges to the same state the console's explicit
344
+ * "static token" toggle sets — the sweep skips it and its status stops reading
345
+ * as a broken/re-auth account. Not lease-fenced: a non-destructive annotation
346
+ * that any process may write, and the credential is not touched.
347
+ *
348
+ * This never changes `status`: a `disabled` account stays `disabled` (only an
349
+ * operator or an explicit recovery may re-enable it), while an `active` one
350
+ * stays `active`. The static marker and the error clearing are orthogonal to
351
+ * the account's health state.
352
+ *
353
+ * D24: for a `disabled` row the error fields are deliberately kept. Once the
354
+ * account is parked, `lastErrorCategory` is the only record of *why* (e.g.
355
+ * `auth_invalidated` → "Re-login required"); wiping it on the static
356
+ * annotation would leave a disabled account with no diagnosable reason. An
357
+ * `active` row still clears them — there the error is stale by definition.
358
+ */
359
+ async function markStaticToken(db: CartethyiaDatabase, accountId: string): Promise<void> {
360
+ const rows = await db
361
+ .select({ status: providerAccounts.status })
362
+ .from(providerAccounts)
363
+ .where(eq(providerAccounts.id, accountId))
364
+ .limit(1);
365
+ const isDisabled = rows[0]?.status === "disabled";
366
+ await db
367
+ .update(providerAccounts)
368
+ .set({
369
+ staticToken: true,
370
+ cooldownUntil: null,
371
+ // A static token is a normal, usable credential, not an error state: any
372
+ // error left by the failed refresh would otherwise keep the row reading
373
+ // "Re-login required" while it is in fact serving. Mirrors the console's
374
+ // static-token toggle. For a `disabled` row the category is preserved
375
+ // (D24), because there it is the diagnosis, not a stale artifact.
376
+ ...(isDisabled
377
+ ? {}
378
+ : { lastError: null, lastErrorCategory: null, lastErrorAt: null }),
379
+ })
380
+ .where(eq(providerAccounts.id, accountId));
381
+ }
382
+
383
+ function failureMessage(error: unknown): string {
384
+ if (error instanceof Error) return error.message;
385
+ return String(error);
386
+ }
387
+
388
+ /**
389
+ * Decrypts a stored credential, returning `undefined` when the ciphertext
390
+ * cannot be decrypted. A corrupt row is not evidence about token expiry, so the
391
+ * caller treats it the same as a missing token rather than throwing inside the
392
+ * refresh failure path.
393
+ */
394
+ function safeDecryptCredential(ciphertext: Buffer): string | undefined {
395
+ try {
396
+ return decryptCredentialToString(ciphertext);
397
+ } catch {
398
+ return undefined;
399
+ }
400
+ }
401
+
402
+ /** Best-effort HTTP status extraction from an OAuth refresh error. Most
403
+ * providers surface `error.status` or embed the status in the message; we
404
+ * accept either so the classifier can honor "bare 401 -> definitive". */
405
+ function failureStatus(error: unknown): number | undefined {
406
+ if (error !== null && typeof error === "object" && "status" in error) {
407
+ const raw = error.status;
408
+ if (typeof raw === "number" && Number.isFinite(raw)) return raw;
409
+ }
410
+ const msg = failureMessage(error);
411
+ const match = /\b(\d{3})\b/.exec(msg);
412
+ if (match) {
413
+ const status = Number(match[1]);
414
+ if (status >= 400 && status < 600) return status;
415
+ }
416
+ return undefined;
417
+ }
418
+
419
+ /**
420
+ * Boot-time convergence for OAuth accounts already parked by an earlier
421
+ * refresh failure.
422
+ *
423
+ * Before automatic detection existed, a definitive refresh failure disabled the
424
+ * account outright (`oauth_revoked`) even when the stored access token was
425
+ * still perfectly usable. This pass re-reads those rows once at startup and
426
+ * moves each to where it now belongs:
427
+ *
428
+ * - access token still usable ⇒ static token; the account stays `disabled`;
429
+ * only an operator can re-enable it;
430
+ * - otherwise ⇒ the `auth_invalidated` category the console renders as
431
+ * "Re-login required" and the quota sweep excludes.
432
+ *
433
+ * Idempotent and cheap: it only inspects OAuth rows that are `disabled` with a
434
+ * credential-failure category, so a healthy deployment scans nothing. The
435
+ * credential is decoded, never verified — the same structural-only trust the
436
+ * dispatch path already places in the stored token.
437
+ */
438
+ export async function reconcileStaticTokenAccounts(db: CartethyiaDatabase): Promise<number> {
439
+ const candidates = await db
440
+ .select({
441
+ id: providerAccounts.id,
442
+ credentialCiphertext: providerAccounts.credentialCiphertext,
443
+ staticToken: providerAccounts.staticToken,
444
+ })
445
+ .from(providerAccounts)
446
+ .where(
447
+ and(
448
+ eq(providerAccounts.credentialKind, "oauth"),
449
+ eq(providerAccounts.status, "disabled"),
450
+ or(
451
+ eq(providerAccounts.lastErrorCategory, "oauth_revoked"),
452
+ eq(providerAccounts.lastErrorCategory, "auth_invalidated"),
453
+ ),
454
+ ),
455
+ );
456
+ let changed = 0;
457
+ for (const row of candidates) {
458
+ if (row.staticToken) continue;
459
+ const usability = classifyAccessTokenUsability(
460
+ row.credentialCiphertext ? safeDecryptCredential(row.credentialCiphertext) : undefined,
461
+ );
462
+ if (usability === "usable") {
463
+ await markStaticToken(db, row.id);
464
+ invalidateCredentialCache(row.id);
465
+ pushStructuredConsoleLog("info", "OAuth account recovered as a static token at boot", {
466
+ event: "token_refresh",
467
+ accountId: row.id,
468
+ errorCode: "static_token",
469
+ });
470
+ changed += 1;
471
+ continue;
472
+ }
473
+ // Converge the legacy category name so the console and the quota sweep see
474
+ // one spelling of "credential rejected, operator must re-login".
475
+ await db
476
+ .update(providerAccounts)
477
+ .set({ lastErrorCategory: "auth_invalidated" })
478
+ .where(eq(providerAccounts.id, row.id));
479
+ changed += 1;
480
+ }
481
+ return changed;
482
+ }
483
+
484
+ /**
485
+ * Rows whose OAuth token is due for proactive refresh (used by the sweep worker).
486
+ *
487
+ * The due test is applied in JS, not as one SQL `expires_at < cutoff`, because
488
+ * the lead is per provider (`refreshLeadMs`): Claude re-mints ~4h ahead, Codex
489
+ * ~5 days, Antigravity ~5 min, and a single global cutoff cannot express that.
490
+ * Accounts with no recorded expiry are always due — an account the sweep cannot
491
+ * see is one whose token dies silently, which is the failure this guards.
492
+ */
493
+ /**
494
+ * Rows one due-scan reads. The expiry test runs in JS because the lead is per
495
+ * provider, so the bound has to sit on the query itself.
496
+ */
497
+ const OAUTH_DUE_SCAN_LIMIT = 2_000;
498
+
499
+ export async function loadDueOAuthAccounts(
500
+ db: CartethyiaDatabase,
501
+ skewMs?: number,
502
+ ): Promise<readonly { readonly id: string; readonly providerId: string }[]> {
503
+ const rows = await db
504
+ .select({
505
+ id: providerAccounts.id,
506
+ providerId: providerAccounts.providerId,
507
+ expiresAt: providerOauthStates.expiresAt,
508
+ staticToken: providerAccounts.staticToken,
509
+ })
510
+ .from(providerAccounts)
511
+ .innerJoin(providerOauthStates, eq(providerOauthStates.providerAccountId, providerAccounts.id))
512
+ .where(
513
+ and(
514
+ eq(providerAccounts.credentialKind, "oauth"),
515
+ // `cooldown` is included: a cooling account's token still expires, and
516
+ // a refresh that lands while it is parked is what lets it serve the
517
+ // moment the cooldown ends. `disabled` stays out — that is an
518
+ // operator decision, and refreshing it would spend a grant on an
519
+ // account that must not return to rotation on its own.
520
+ inArray(providerAccounts.status, ["active", "cooldown"]),
521
+ ),
522
+ )
523
+ // The due test is applied in JS, so this limit bounds the scan, not the
524
+ // result. One pass cannot read the whole table; accounts past the limit
525
+ // are reached on a later tick once the ones in front have been refreshed
526
+ // and dropped out of the window.
527
+ .limit(OAUTH_DUE_SCAN_LIMIT);
528
+ const now = Date.now();
529
+ return rows
530
+ .filter((row) => {
531
+ // A static token is used exactly as issued: there is no refresh grant to
532
+ // run, so the sweep must not touch it. Refreshing it every pass only
533
+ // floods the log with a guaranteed no-op. The operator clears the flag
534
+ // when the account has a real refresh token again.
535
+ if (row.staticToken) return false;
536
+ if (row.expiresAt == null) return true;
537
+ const lead = skewMs ?? refreshLeadMs(row.providerId);
538
+ return row.expiresAt.getTime() - lead <= now;
539
+ })
540
+ .map((row) => ({ id: row.id, providerId: row.providerId }));
541
+ }
542
+
543
+ export class OAuthRefreshService {
544
+ /** Bound on concurrently tracked single-flight refreshes; keys are account ids, so the real bound is the account count. */
545
+ private static readonly MAX_INFLIGHT = 1_000;
546
+
547
+ readonly #db: CartethyiaDatabase;
548
+ readonly #inFlight = new Map<string, Promise<string | null>>();
549
+
550
+ constructor(db: CartethyiaDatabase) {
551
+ this.#db = db;
552
+ }
553
+
554
+ /**
555
+ * Ensures a fresh access token for an OAuth account, refreshing it first
556
+ * if it's within `opts.skewMs` of expiry (or already expired). When
557
+ * `opts.force` is `true`, the freshness check is bypassed and the
558
+ * refresher is always invoked — used by the 401 dispatch retry so a peer
559
+ * that just refreshed cannot cause the caller to reuse a dead cached
560
+ * secret. Returns the current decrypted access token, or `null` when the
561
+ * account isn't OAuth, has no stored refresh token, or refresh failed.
562
+ *
563
+ * `opts.origin` labels who asked for the refresh (`ui`/`sweep`/`dispatch`)
564
+ * and is carried on the `token_refresh` log lines (D26). The origin is
565
+ * observational only — it never changes what a refresh does.
566
+ */
567
+ async ensureFreshAccessToken(
568
+ accountId: string,
569
+ refresher: OAuthTokenRefresher,
570
+ opts: { force?: boolean; skewMs?: number; origin?: "ui" | "sweep" | "dispatch" } = {},
571
+ ): Promise<string | null> {
572
+ const force = opts.force === true;
573
+ // No explicit skew: the per-provider lead is applied at load time.
574
+ const skewMs = opts.skewMs;
575
+ const origin = opts.origin ?? "sweep";
576
+ // Forced refreshes get their own in-flight slot so they don't merge with
577
+ // a raced non-force call that would otherwise short-circuit.
578
+ const key = force ? `${accountId}:force` : accountId;
579
+ const existing = this.#inFlight.get(key);
580
+ if (existing) return existing;
581
+ // Eviction guard: keys are account ids (bounded by the accounts table),
582
+ // but a runaway caller could still fan out unique keys; drop the oldest
583
+ // in-flight task rather than grow without bound. An evicted task's
584
+ // `.finally` delete is a no-op for a key no longer present, and a racing
585
+ // duplicate refresh is fenced cross-process by the lease below.
586
+ if (this.#inFlight.size >= OAuthRefreshService.MAX_INFLIGHT) {
587
+ const oldestKey = this.#inFlight.keys().next().value;
588
+ if (oldestKey !== undefined) this.#inFlight.delete(oldestKey);
589
+ }
590
+ const task = this.#refreshIfDue(accountId, refresher, skewMs, force, origin).finally(() => {
591
+ this.#inFlight.delete(key);
592
+ });
593
+ this.#inFlight.set(key, task);
594
+ return task;
595
+ }
596
+
597
+ async #refreshIfDue(
598
+ accountId: string,
599
+ refresher: OAuthTokenRefresher,
600
+ skewMs: number | undefined,
601
+ force: boolean,
602
+ origin: "ui" | "sweep" | "dispatch",
603
+ ): Promise<string | null> {
604
+ const account = await loadAccountWithFreshness(this.#db, accountId, skewMs);
605
+ if (!account || account.row.credentialKind !== "oauth") return null;
606
+ const health = await loadAccountWithOauthState(this.#db, accountId, {
607
+ includeStatus: true,
608
+ includeLastErrorCategory: true,
609
+ });
610
+ if (health?.status === "disabled" && health.lastErrorCategory === "auth_invalidated") {
611
+ pushStructuredConsoleLog("info", "OAuth token refresh skipped: account auth invalidated", {
612
+ event: "token_refresh",
613
+ accountId,
614
+ providerId: account.row.providerId,
615
+ errorCode: "auth_invalidated",
616
+ details: { origin },
617
+ });
618
+ return null;
619
+ }
620
+ const { row, dueAt } = account;
621
+ // A static token is used exactly as issued and never refreshed. There is
622
+ // nothing to re-mint, so return null — the same "cannot refresh" signal a
623
+ // missing refresh token gives — rather than re-issuing the identical
624
+ // credential to a forced (401) retry, which would only repeat the same
625
+ // request against the same upstream. Dispatch still uses the stored token
626
+ // directly (`dueAt` is undefined for a static token, so the caller's
627
+ // refresh branch is skipped and the decrypted credential is used as-is).
628
+ if (row.staticToken) return null;
629
+ if (!force && dueAt !== undefined && dueAt > Date.now()) {
630
+ return row.credentialCiphertext
631
+ ? decryptCredentialToString(row.credentialCiphertext)
632
+ : null;
633
+ }
634
+ return this.#refreshNow(row, refresher, skewMs, origin);
635
+ }
636
+
637
+ async #refreshNow(
638
+ row: AccountRow,
639
+ refresher: OAuthTokenRefresher,
640
+ skewMs: number | undefined,
641
+ origin: "ui" | "sweep" | "dispatch",
642
+ ): Promise<string | null> {
643
+ // D26: statusBefore is read once up front (the row projection used by the
644
+ // refresh path does not carry status) and statusAfter is re-read after any
645
+ // mutation, so the log proves a refresh never flipped the account.
646
+ const statusBefore = await readAccountStatus(this.#db, row.id);
647
+ pushStructuredConsoleLog("info", "OAuth token refresh started", {
648
+ event: "token_refresh",
649
+ accountId: row.id,
650
+ providerId: row.providerId,
651
+ details: { origin, statusBefore },
652
+ });
653
+ const owner = randomUUID();
654
+ const leased = await acquireLease(this.#db, row.id, owner);
655
+ if (!leased) {
656
+ // A peer holds the lease — briefly wait for it to finish, then read whatever it left.
657
+ await sleep(500);
658
+ const fresh = await loadAccountWithFreshness(this.#db, row.id, skewMs);
659
+ return fresh?.row.credentialCiphertext && fresh.dueAt !== undefined && fresh.dueAt > Date.now()
660
+ ? decryptCredentialToString(fresh.row.credentialCiphertext)
661
+ : null;
662
+ }
663
+ if (!row.refreshCiphertext) {
664
+ await releaseLease(this.#db, row.id, owner);
665
+ // No refresh token on record: the access token is used exactly as issued
666
+ // and cannot be re-minted, so every future refresh attempt is a
667
+ // guaranteed no-op. Mark the account static — the same state the
668
+ // console's toggle sets — so the sweep stops retrying it and the status
669
+ // reads as an informational static token rather than a broken account.
670
+ await markStaticToken(this.#db, row.id);
671
+ pushStructuredConsoleLog("info", "OAuth token refresh skipped: static token", {
672
+ event: "token_refresh",
673
+ accountId: row.id,
674
+ providerId: row.providerId,
675
+ errorCode: "static_token",
676
+ details: { origin },
677
+ });
678
+ return null;
679
+ }
680
+ const refreshToken = decryptCredentialToString(row.refreshCiphertext);
681
+ try {
682
+ const result = await refresher.refresh(
683
+ refreshToken,
684
+ AbortSignal.timeout(REFRESH_HTTP_TIMEOUT_MS),
685
+ buildRefreshContext(row, refresher.requiresAccessToken === true),
686
+ );
687
+ const persisted = await persistRefreshed(this.#db, row.id, owner, result);
688
+ // D26: `statusAfter` is re-read from the DB rather than assumed — a
689
+ // successful refresh never changes `status`, and the log proves it.
690
+ const statusAfter = await readAccountStatus(this.#db, row.id);
691
+ pushStructuredConsoleLog(persisted ? "info" : "warn", persisted ? "OAuth token refresh completed" : "OAuth token refresh lost lease", {
692
+ event: "token_refresh",
693
+ accountId: row.id,
694
+ providerId: row.providerId,
695
+ ...(persisted ? {} : { errorCode: "refresh_lease_lost" }),
696
+ details: {
697
+ expiresAt: result.expiresAt.toISOString(),
698
+ origin,
699
+ statusBefore,
700
+ statusAfter,
701
+ },
702
+ });
703
+ return persisted ? result.access : null;
704
+ } catch (error) {
705
+ const classification = classifyOAuthRefreshFailure({
706
+ message: failureMessage(error),
707
+ status: failureStatus(error),
708
+ });
709
+ pushStructuredConsoleLog("error", "OAuth token refresh failed", {
710
+ event: "token_refresh",
711
+ accountId: row.id,
712
+ providerId: row.providerId,
713
+ errorCode: classification,
714
+ details: { origin, statusBefore },
715
+ });
716
+ if (classification === "definitive") {
717
+ // A definitive refresh failure means the refresh grant is dead — but
718
+ // the access token already on the row may still be usable as issued.
719
+ // That is exactly the static-token case: stop refreshing, keep serving.
720
+ // Only when the token cannot be shown to work (expired or opaque) does
721
+ // the account need the operator, and it is parked for re-auth.
722
+ const accessUsability = classifyAccessTokenUsability(
723
+ row.credentialCiphertext
724
+ ? safeDecryptCredential(row.credentialCiphertext)
725
+ : undefined,
726
+ );
727
+ if (accessUsability === "usable") {
728
+ await releaseLease(this.#db, row.id, owner);
729
+ await markStaticToken(this.#db, row.id);
730
+ const statusAfter = await readAccountStatus(this.#db, row.id);
731
+ pushStructuredConsoleLog("info", "OAuth refresh failed; access token still valid, marking static", {
732
+ event: "token_refresh",
733
+ accountId: row.id,
734
+ providerId: row.providerId,
735
+ errorCode: "static_token",
736
+ details: { origin, statusBefore, statusAfter },
737
+ });
738
+ } else {
739
+ await disableAccount(this.#db, row.id, owner, failureMessage(error), {
740
+ // D23: a definitive refresh failure must not turn an operator's
741
+ // disable into a machine-recoverable one. If the row is already
742
+ // `disabled` without a deadline, leave it that way.
743
+ preserveOperatorDisable: true,
744
+ });
745
+ const statusAfter = await readAccountStatus(this.#db, row.id);
746
+ pushStructuredConsoleLog("warn", "OAuth refresh failed definitively; account disabled for re-auth", {
747
+ event: "token_refresh",
748
+ accountId: row.id,
749
+ providerId: row.providerId,
750
+ errorCode: classification,
751
+ details: { origin, statusBefore, statusAfter },
752
+ });
753
+ }
754
+ } else {
755
+ await releaseLease(this.#db, row.id, owner);
756
+ }
757
+ return null;
758
+ }
759
+ }
760
+ }
761
+
762
+ /** Reads one account's status (used only for logging/observability, D26). */
763
+ async function readAccountStatus(db: CartethyiaDatabase, accountId: string): Promise<string | undefined> {
764
+ const loaded = await loadAccountWithOauthState(db, accountId, { includeStatus: true });
765
+ return loaded?.status;
766
+ }