codex-multi-auth 2.3.0-beta.0 → 2.3.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (470) hide show
  1. package/.codex-plugin/plugin.json +1 -1
  2. package/README.md +4 -3
  3. package/config/schema/config.schema.json +268 -21
  4. package/dist/index.js +0 -1
  5. package/dist/lib/account-policy.js +0 -1
  6. package/dist/lib/accounts/rate-limits.js +0 -1
  7. package/dist/lib/accounts.d.ts +3 -8
  8. package/dist/lib/accounts.d.ts.map +1 -1
  9. package/dist/lib/accounts.js +2 -3
  10. package/dist/lib/audit.js +0 -1
  11. package/dist/lib/auth/auth.js +0 -1
  12. package/dist/lib/auth/browser.js +0 -1
  13. package/dist/lib/auth/device-auth.js +0 -1
  14. package/dist/lib/auth/index.js +0 -1
  15. package/dist/lib/auth/org-override.js +0 -1
  16. package/dist/lib/auth/server.js +0 -1
  17. package/dist/lib/auth/token-utils.js +0 -1
  18. package/dist/lib/auth-rate-limit.js +0 -1
  19. package/dist/lib/budget-guard.js +0 -1
  20. package/dist/lib/capability-policy.js +0 -1
  21. package/dist/lib/circuit-breaker.js +0 -1
  22. package/dist/lib/cli.js +1 -2
  23. package/dist/lib/codex-cli/index.js +0 -1
  24. package/dist/lib/codex-cli/observability.js +0 -1
  25. package/dist/lib/codex-cli/state.js +0 -1
  26. package/dist/lib/codex-cli/sync.d.ts +2 -2
  27. package/dist/lib/codex-cli/sync.d.ts.map +1 -1
  28. package/dist/lib/codex-cli/sync.js +1 -2
  29. package/dist/lib/codex-cli/writer.js +0 -1
  30. package/dist/lib/codex-manager/account-credentials.d.ts +13 -0
  31. package/dist/lib/codex-manager/account-credentials.d.ts.map +1 -0
  32. package/dist/lib/codex-manager/account-credentials.js +55 -0
  33. package/dist/lib/codex-manager/account-manager-commands.js +0 -1
  34. package/dist/lib/codex-manager/account-pool-write.d.ts +95 -0
  35. package/dist/lib/codex-manager/account-pool-write.d.ts.map +1 -0
  36. package/dist/lib/codex-manager/account-pool-write.js +165 -0
  37. package/dist/lib/codex-manager/account-pool-write.js.map +1 -0
  38. package/dist/lib/codex-manager/backend-category-entry.js +0 -1
  39. package/dist/lib/codex-manager/backend-category-helpers.js +0 -1
  40. package/dist/lib/codex-manager/backend-category-prompt.js +0 -1
  41. package/dist/lib/codex-manager/backend-settings-controller.js +0 -1
  42. package/dist/lib/codex-manager/backend-settings-entry.js +0 -1
  43. package/dist/lib/codex-manager/backend-settings-helpers.js +0 -1
  44. package/dist/lib/codex-manager/backend-settings-prompt.js +0 -1
  45. package/dist/lib/codex-manager/backend-settings-schema.js +0 -1
  46. package/dist/lib/codex-manager/behavior-settings-panel.d.ts +1 -1
  47. package/dist/lib/codex-manager/behavior-settings-panel.d.ts.map +1 -1
  48. package/dist/lib/codex-manager/behavior-settings-panel.js +0 -1
  49. package/dist/lib/codex-manager/commands/account.js +0 -1
  50. package/dist/lib/codex-manager/commands/best.d.ts +8 -0
  51. package/dist/lib/codex-manager/commands/best.d.ts.map +1 -1
  52. package/dist/lib/codex-manager/commands/best.js +64 -1
  53. package/dist/lib/codex-manager/commands/bridge.js +0 -1
  54. package/dist/lib/codex-manager/commands/budget.js +0 -1
  55. package/dist/lib/codex-manager/commands/check.js +0 -1
  56. package/dist/lib/codex-manager/commands/config-explain.js +0 -1
  57. package/dist/lib/codex-manager/commands/debug-bundle.js +0 -1
  58. package/dist/lib/codex-manager/commands/forecast.js +0 -1
  59. package/dist/lib/codex-manager/commands/init-config.js +0 -1
  60. package/dist/lib/codex-manager/commands/integrations.js +0 -1
  61. package/dist/lib/codex-manager/commands/models.js +0 -1
  62. package/dist/lib/codex-manager/commands/monitor.js +0 -1
  63. package/dist/lib/codex-manager/commands/report.js +0 -1
  64. package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
  65. package/dist/lib/codex-manager/commands/rotation.js +1 -4
  66. package/dist/lib/codex-manager/commands/status.js +0 -1
  67. package/dist/lib/codex-manager/commands/switch.js +0 -1
  68. package/dist/lib/codex-manager/commands/uninstall.d.ts.map +1 -1
  69. package/dist/lib/codex-manager/commands/uninstall.js +2 -33
  70. package/dist/lib/codex-manager/commands/unpin.js +0 -1
  71. package/dist/lib/codex-manager/commands/usage.js +0 -1
  72. package/dist/lib/codex-manager/commands/verify.js +0 -1
  73. package/dist/lib/codex-manager/commands/why-selected.js +0 -1
  74. package/dist/lib/codex-manager/commands/workspace.js +0 -1
  75. package/dist/lib/codex-manager/dashboard-display-panel.d.ts +1 -1
  76. package/dist/lib/codex-manager/dashboard-display-panel.d.ts.map +1 -1
  77. package/dist/lib/codex-manager/dashboard-display-panel.js +0 -1
  78. package/dist/lib/codex-manager/dashboard-formatters.js +0 -1
  79. package/dist/lib/codex-manager/dashboard-settings-controller.js +0 -1
  80. package/dist/lib/codex-manager/dashboard-settings-data.js +0 -1
  81. package/dist/lib/codex-manager/dashboard-settings-entry.js +0 -1
  82. package/dist/lib/codex-manager/experimental-settings-entry.js +0 -1
  83. package/dist/lib/codex-manager/experimental-settings-prompt.js +0 -1
  84. package/dist/lib/codex-manager/experimental-settings-schema.js +1 -2
  85. package/dist/lib/codex-manager/experimental-sync-target-entry.js +0 -1
  86. package/dist/lib/codex-manager/experimental-sync-target.js +0 -1
  87. package/dist/lib/codex-manager/forecast-report-commands.js +0 -1
  88. package/dist/lib/codex-manager/forecast-report-shared.d.ts +2 -2
  89. package/dist/lib/codex-manager/forecast-report-shared.d.ts.map +1 -1
  90. package/dist/lib/codex-manager/forecast-report-shared.js +5 -21
  91. package/dist/lib/codex-manager/formatters/account-formatters.d.ts +11 -0
  92. package/dist/lib/codex-manager/formatters/account-formatters.d.ts.map +1 -0
  93. package/dist/lib/codex-manager/formatters/account-formatters.js +79 -0
  94. package/dist/lib/codex-manager/formatters/index.d.ts +5 -0
  95. package/dist/lib/codex-manager/formatters/index.d.ts.map +1 -0
  96. package/dist/lib/codex-manager/formatters/index.js +4 -0
  97. package/dist/lib/codex-manager/formatters/model-formatters.d.ts +12 -0
  98. package/dist/lib/codex-manager/formatters/model-formatters.d.ts.map +1 -0
  99. package/dist/lib/codex-manager/formatters/model-formatters.js +23 -0
  100. package/dist/lib/codex-manager/formatters/quota-formatters.d.ts +11 -0
  101. package/dist/lib/codex-manager/formatters/quota-formatters.d.ts.map +1 -0
  102. package/dist/lib/codex-manager/formatters/quota-formatters.js +104 -0
  103. package/dist/lib/codex-manager/formatters/text-style.d.ts +14 -0
  104. package/dist/lib/codex-manager/formatters/text-style.d.ts.map +1 -0
  105. package/dist/lib/codex-manager/formatters/text-style.js +111 -0
  106. package/dist/lib/codex-manager/health-check.d.ts +14 -0
  107. package/dist/lib/codex-manager/health-check.d.ts.map +1 -0
  108. package/dist/lib/codex-manager/health-check.js +278 -0
  109. package/dist/lib/codex-manager/help.js +0 -1
  110. package/dist/lib/codex-manager/login-action-panel.d.ts +4 -0
  111. package/dist/lib/codex-manager/login-action-panel.d.ts.map +1 -0
  112. package/dist/lib/codex-manager/login-action-panel.js +238 -0
  113. package/dist/lib/codex-manager/login-flow.d.ts +17 -0
  114. package/dist/lib/codex-manager/login-flow.d.ts.map +1 -0
  115. package/dist/lib/codex-manager/login-flow.js +387 -0
  116. package/dist/lib/codex-manager/login-menu-actions.d.ts +20 -0
  117. package/dist/lib/codex-manager/login-menu-actions.d.ts.map +1 -0
  118. package/dist/lib/codex-manager/login-menu-actions.js +303 -0
  119. package/dist/lib/codex-manager/login-menu-data.d.ts +19 -0
  120. package/dist/lib/codex-manager/login-menu-data.d.ts.map +1 -0
  121. package/dist/lib/codex-manager/login-menu-data.js +378 -0
  122. package/dist/lib/codex-manager/login-oauth.d.ts +55 -0
  123. package/dist/lib/codex-manager/login-oauth.d.ts.map +1 -0
  124. package/dist/lib/codex-manager/login-oauth.js +345 -0
  125. package/dist/lib/codex-manager/manual-callback.d.ts +33 -0
  126. package/dist/lib/codex-manager/manual-callback.d.ts.map +1 -0
  127. package/dist/lib/codex-manager/manual-callback.js +30 -0
  128. package/dist/lib/codex-manager/manual-callback.js.map +1 -0
  129. package/dist/lib/codex-manager/persist-selected-account.d.ts +24 -0
  130. package/dist/lib/codex-manager/persist-selected-account.d.ts.map +1 -0
  131. package/dist/lib/codex-manager/persist-selected-account.js +107 -0
  132. package/dist/lib/codex-manager/quota-cache-helpers.d.ts +18 -0
  133. package/dist/lib/codex-manager/quota-cache-helpers.d.ts.map +1 -0
  134. package/dist/lib/codex-manager/quota-cache-helpers.js +112 -0
  135. package/dist/lib/codex-manager/rate-limit-markers.js +0 -1
  136. package/dist/lib/codex-manager/repair-commands.js +0 -1
  137. package/dist/lib/codex-manager/settings-hub/backend.js +1 -2
  138. package/dist/lib/codex-manager/settings-hub/dashboard.js +1 -2
  139. package/dist/lib/codex-manager/settings-hub/experimental.js +1 -2
  140. package/dist/lib/codex-manager/settings-hub/index.js +1 -2
  141. package/dist/lib/codex-manager/settings-hub/shared.js +0 -1
  142. package/dist/lib/codex-manager/settings-hub-entry.js +0 -1
  143. package/dist/lib/codex-manager/settings-hub-menu.js +0 -1
  144. package/dist/lib/codex-manager/settings-hub-prompt.js +0 -1
  145. package/dist/lib/codex-manager/settings-hub.js +0 -1
  146. package/dist/lib/codex-manager/settings-panels.js +0 -1
  147. package/dist/lib/codex-manager/settings-persist-utils.js +0 -1
  148. package/dist/lib/codex-manager/settings-preview.js +0 -1
  149. package/dist/lib/codex-manager/settings-write-queue.js +0 -1
  150. package/dist/lib/codex-manager/statusline-order.js +0 -1
  151. package/dist/lib/codex-manager/statusline-settings-panel.d.ts +1 -1
  152. package/dist/lib/codex-manager/statusline-settings-panel.d.ts.map +1 -1
  153. package/dist/lib/codex-manager/statusline-settings-panel.js +0 -1
  154. package/dist/lib/codex-manager/theme-settings-panel.d.ts +1 -1
  155. package/dist/lib/codex-manager/theme-settings-panel.d.ts.map +1 -1
  156. package/dist/lib/codex-manager/theme-settings-panel.js +0 -1
  157. package/dist/lib/codex-manager/unified-settings-controller.js +0 -1
  158. package/dist/lib/codex-manager/unified-settings-entry.js +0 -1
  159. package/dist/lib/codex-manager.d.ts +2 -4
  160. package/dist/lib/codex-manager.d.ts.map +1 -1
  161. package/dist/lib/codex-manager.js +219 -2578
  162. package/dist/lib/codex-manager.js.map +1 -1
  163. package/dist/lib/config-schema.d.ts +32 -0
  164. package/dist/lib/config-schema.d.ts.map +1 -0
  165. package/dist/lib/config-schema.js +96 -0
  166. package/dist/lib/config.d.ts.map +1 -1
  167. package/dist/lib/config.js +60 -104
  168. package/dist/lib/constants.js +0 -1
  169. package/dist/lib/context-overflow.js +0 -1
  170. package/dist/lib/dashboard-settings.js +0 -1
  171. package/dist/lib/entitlement-cache.js +0 -1
  172. package/dist/lib/env-parsing.js +0 -1
  173. package/dist/lib/errors.js +0 -1
  174. package/dist/lib/forecast.js +0 -1
  175. package/dist/lib/fs-retry.d.ts +46 -0
  176. package/dist/lib/fs-retry.d.ts.map +1 -1
  177. package/dist/lib/fs-retry.js +74 -6
  178. package/dist/lib/health.js +0 -1
  179. package/dist/lib/index.js +0 -1
  180. package/dist/lib/integration-generators.js +0 -1
  181. package/dist/lib/live-account-sync.js +0 -1
  182. package/dist/lib/local-bridge.js +0 -1
  183. package/dist/lib/local-client-tokens.js +0 -1
  184. package/dist/lib/logger.js +0 -1
  185. package/dist/lib/model-capability-matrix.js +0 -1
  186. package/dist/lib/named-backup-export.js +0 -1
  187. package/dist/lib/oc-chatgpt-import-adapter.js +0 -1
  188. package/dist/lib/oc-chatgpt-orchestrator.js +0 -1
  189. package/dist/lib/oc-chatgpt-target-detection.js +0 -1
  190. package/dist/lib/parallel-probe.js +0 -1
  191. package/dist/lib/policy/runtime-policy.js +0 -1
  192. package/dist/lib/preemptive-quota-scheduler.js +0 -1
  193. package/dist/lib/proactive-refresh.js +0 -1
  194. package/dist/lib/prompts/codex-host-bridge.js +0 -1
  195. package/dist/lib/prompts/codex.d.ts +3 -11
  196. package/dist/lib/prompts/codex.d.ts.map +1 -1
  197. package/dist/lib/prompts/codex.js +4 -12
  198. package/dist/lib/prompts/fetch-utils.js +0 -1
  199. package/dist/lib/prompts/host-codex-prompt.js +0 -1
  200. package/dist/lib/quota-cache.d.ts.map +1 -1
  201. package/dist/lib/quota-cache.js +15 -43
  202. package/dist/lib/quota-probe.js +0 -1
  203. package/dist/lib/quota-readiness.js +0 -1
  204. package/dist/lib/recovery/constants.js +0 -1
  205. package/dist/lib/recovery/index.js +0 -1
  206. package/dist/lib/recovery/storage.d.ts.map +1 -1
  207. package/dist/lib/recovery/storage.js +30 -51
  208. package/dist/lib/recovery/storage.js.map +1 -1
  209. package/dist/lib/recovery/types.js +0 -1
  210. package/dist/lib/recovery.js +0 -1
  211. package/dist/lib/refresh-guardian.js +0 -1
  212. package/dist/lib/refresh-lease.d.ts.map +1 -1
  213. package/dist/lib/refresh-lease.js +1 -4
  214. package/dist/lib/refresh-queue.js +0 -1
  215. package/dist/lib/request/error-classification.d.ts +75 -0
  216. package/dist/lib/request/error-classification.d.ts.map +1 -0
  217. package/dist/lib/request/error-classification.js +295 -0
  218. package/dist/lib/request/failover-config.js +0 -1
  219. package/dist/lib/request/failure-policy.js +0 -1
  220. package/dist/lib/request/fetch-helpers.d.ts +10 -114
  221. package/dist/lib/request/fetch-helpers.d.ts.map +1 -1
  222. package/dist/lib/request/fetch-helpers.js +7 -589
  223. package/dist/lib/request/headers.d.ts +33 -0
  224. package/dist/lib/request/headers.d.ts.map +1 -0
  225. package/dist/lib/request/headers.js +64 -0
  226. package/dist/lib/request/helpers/input-utils.js +0 -1
  227. package/dist/lib/request/helpers/model-map.d.ts +10 -0
  228. package/dist/lib/request/helpers/model-map.d.ts.map +1 -1
  229. package/dist/lib/request/helpers/model-map.js +11 -1
  230. package/dist/lib/request/helpers/tool-utils.js +0 -1
  231. package/dist/lib/request/index.js +0 -1
  232. package/dist/lib/request/rate-limit-backoff.js +0 -1
  233. package/dist/lib/request/rate-limit-decision.d.ts +28 -0
  234. package/dist/lib/request/rate-limit-decision.d.ts.map +1 -0
  235. package/dist/lib/request/rate-limit-decision.js +191 -0
  236. package/dist/lib/request/request-attempt-budget.js +0 -1
  237. package/dist/lib/request/request-init.js +0 -1
  238. package/dist/lib/request/request-resilience.js +0 -1
  239. package/dist/lib/request/request-transformer.js +0 -1
  240. package/dist/lib/request/response-compaction.js +0 -1
  241. package/dist/lib/request/response-handler.js +0 -1
  242. package/dist/lib/request/response-metadata.js +0 -1
  243. package/dist/lib/request/stream-failover-runtime.d.ts +8 -0
  244. package/dist/lib/request/stream-failover-runtime.d.ts.map +1 -0
  245. package/dist/lib/request/stream-failover-runtime.js +149 -0
  246. package/dist/lib/request/stream-failover.js +0 -1
  247. package/dist/lib/request/token-refresh.d.ts +30 -0
  248. package/dist/lib/request/token-refresh.d.ts.map +1 -0
  249. package/dist/lib/request/token-refresh.js +116 -0
  250. package/dist/lib/request/url-rewriting.d.ts +24 -0
  251. package/dist/lib/request/url-rewriting.d.ts.map +1 -0
  252. package/dist/lib/request/url-rewriting.js +153 -0
  253. package/dist/lib/request/wait-utils.js +0 -1
  254. package/dist/lib/rotation.d.ts +1 -1
  255. package/dist/lib/rotation.d.ts.map +1 -1
  256. package/dist/lib/rotation.js +0 -1
  257. package/dist/lib/routing-mutex.js +0 -1
  258. package/dist/lib/routing-profiles.js +0 -1
  259. package/dist/lib/runtime/account-check-helpers.js +0 -1
  260. package/dist/lib/runtime/account-check-types.js +0 -1
  261. package/dist/lib/runtime/account-check.js +0 -1
  262. package/dist/lib/runtime/account-health-check.js +0 -1
  263. package/dist/lib/runtime/account-manager-cache-entry.js +0 -1
  264. package/dist/lib/runtime/account-manager-cache.js +0 -1
  265. package/dist/lib/runtime/account-pool.js +0 -1
  266. package/dist/lib/runtime/account-scope.js +0 -1
  267. package/dist/lib/runtime/account-select-event.js +0 -1
  268. package/dist/lib/runtime/account-selection.js +0 -1
  269. package/dist/lib/runtime/account-state.js +0 -1
  270. package/dist/lib/runtime/account-status.js +0 -1
  271. package/dist/lib/runtime/account-storage-scope-entry.js +0 -1
  272. package/dist/lib/runtime/app-bind.js +0 -1
  273. package/dist/lib/runtime/auth-facade.js +0 -1
  274. package/dist/lib/runtime/browser-oauth-flow.js +0 -1
  275. package/dist/lib/runtime/capability-boost.js +0 -1
  276. package/dist/lib/runtime/config-toml.js +0 -1
  277. package/dist/lib/runtime/event-handler.js +0 -1
  278. package/dist/lib/runtime/first-run.d.ts +80 -0
  279. package/dist/lib/runtime/first-run.d.ts.map +1 -0
  280. package/dist/lib/runtime/first-run.js +327 -0
  281. package/dist/lib/runtime/flagged-verify-types.js +0 -1
  282. package/dist/lib/runtime/hydrate-emails.js +0 -1
  283. package/dist/lib/runtime/live-sync-entry.js +0 -1
  284. package/dist/lib/runtime/live-sync.js +0 -1
  285. package/dist/lib/runtime/loader-setup.js +0 -1
  286. package/dist/lib/runtime/login-menu-accounts.js +0 -1
  287. package/dist/lib/runtime/manual-oauth-flow.js +0 -1
  288. package/dist/lib/runtime/oauth-browser-flow.js +0 -1
  289. package/dist/lib/runtime/preemptive-quota.js +0 -1
  290. package/dist/lib/runtime/quota-headers.js +0 -1
  291. package/dist/lib/runtime/quota-probe.js +0 -1
  292. package/dist/lib/runtime/quota-settings.js +0 -1
  293. package/dist/lib/runtime/refresh-guardian-entry.js +0 -1
  294. package/dist/lib/runtime/refresh-guardian.js +0 -1
  295. package/dist/lib/runtime/request-init.js +0 -1
  296. package/dist/lib/runtime/rotation-account-selection.d.ts +38 -0
  297. package/dist/lib/runtime/rotation-account-selection.d.ts.map +1 -0
  298. package/dist/lib/runtime/rotation-account-selection.js +172 -0
  299. package/dist/lib/runtime/rotation-proxy-state.d.ts +54 -0
  300. package/dist/lib/runtime/rotation-proxy-state.d.ts.map +1 -0
  301. package/dist/lib/runtime/rotation-proxy-state.js +56 -0
  302. package/dist/lib/runtime/rotation-server-types.d.ts +57 -0
  303. package/dist/lib/runtime/rotation-server-types.d.ts.map +1 -0
  304. package/dist/lib/runtime/rotation-server-types.js +1 -0
  305. package/dist/lib/runtime/rotation-storage-meta.d.ts +40 -0
  306. package/dist/lib/runtime/rotation-storage-meta.d.ts.map +1 -0
  307. package/dist/lib/runtime/rotation-storage-meta.js +143 -0
  308. package/dist/lib/runtime/rotation-token-refresh.d.ts +25 -0
  309. package/dist/lib/runtime/rotation-token-refresh.d.ts.map +1 -0
  310. package/dist/lib/runtime/rotation-token-refresh.js +99 -0
  311. package/dist/lib/runtime/runtime-current-account.d.ts.map +1 -1
  312. package/dist/lib/runtime/runtime-current-account.js +1 -4
  313. package/dist/lib/runtime/runtime-observability.js +0 -1
  314. package/dist/lib/runtime/runtime-services.js +0 -1
  315. package/dist/lib/runtime/session-affinity-entry.js +0 -1
  316. package/dist/lib/runtime/session-affinity.js +0 -1
  317. package/dist/lib/runtime/session-recovery.js +0 -1
  318. package/dist/lib/runtime/status-marker.js +0 -1
  319. package/dist/lib/runtime/storage-scope.js +0 -1
  320. package/dist/lib/runtime/toast.js +0 -1
  321. package/dist/lib/runtime/ui-runtime-entry.js +0 -1
  322. package/dist/lib/runtime/ui-runtime.js +0 -1
  323. package/dist/lib/runtime/verify-flagged.js +0 -1
  324. package/dist/lib/runtime-constants.js +0 -1
  325. package/dist/lib/runtime-paths.js +0 -1
  326. package/dist/lib/runtime-rotation-proxy.d.ts +7 -124
  327. package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
  328. package/dist/lib/runtime-rotation-proxy.js +614 -1353
  329. package/dist/lib/schemas.d.ts +3 -3
  330. package/dist/lib/schemas.js +1 -2
  331. package/dist/lib/session-affinity.js +0 -1
  332. package/dist/lib/shutdown.js +0 -1
  333. package/dist/lib/storage/account-clear-entry.js +0 -1
  334. package/dist/lib/storage/account-clear.js +0 -1
  335. package/dist/lib/storage/account-match-utils.js +0 -1
  336. package/dist/lib/storage/account-persistence.d.ts +1 -1
  337. package/dist/lib/storage/account-persistence.d.ts.map +1 -1
  338. package/dist/lib/storage/account-persistence.js +0 -1
  339. package/dist/lib/storage/account-port.d.ts +1 -1
  340. package/dist/lib/storage/account-port.d.ts.map +1 -1
  341. package/dist/lib/storage/account-port.js +0 -1
  342. package/dist/lib/storage/account-save-entry.d.ts +1 -1
  343. package/dist/lib/storage/account-save-entry.d.ts.map +1 -1
  344. package/dist/lib/storage/account-save-entry.js +0 -1
  345. package/dist/lib/storage/account-save.d.ts +1 -1
  346. package/dist/lib/storage/account-save.d.ts.map +1 -1
  347. package/dist/lib/storage/account-save.js +0 -1
  348. package/dist/lib/storage/account-snapshot.js +0 -1
  349. package/dist/lib/storage/backup-metadata-builder.d.ts +1 -1
  350. package/dist/lib/storage/backup-metadata-builder.d.ts.map +1 -1
  351. package/dist/lib/storage/backup-metadata-builder.js +0 -1
  352. package/dist/lib/storage/backup-metadata.d.ts +13 -0
  353. package/dist/lib/storage/backup-metadata.d.ts.map +1 -1
  354. package/dist/lib/storage/backup-metadata.js +0 -1
  355. package/dist/lib/storage/backup-paths.js +0 -1
  356. package/dist/lib/storage/backup-restore.d.ts +1 -1
  357. package/dist/lib/storage/backup-restore.d.ts.map +1 -1
  358. package/dist/lib/storage/backup-restore.js +0 -1
  359. package/dist/lib/storage/cache-artifacts.js +0 -1
  360. package/dist/lib/storage/error-hints.js +0 -1
  361. package/dist/lib/storage/file-paths.js +0 -1
  362. package/dist/lib/storage/fixture-guards.d.ts +1 -1
  363. package/dist/lib/storage/fixture-guards.d.ts.map +1 -1
  364. package/dist/lib/storage/fixture-guards.js +0 -1
  365. package/dist/lib/storage/flagged-entry.js +0 -1
  366. package/dist/lib/storage/flagged-load-entry.d.ts +1 -1
  367. package/dist/lib/storage/flagged-load-entry.d.ts.map +1 -1
  368. package/dist/lib/storage/flagged-load-entry.js +0 -1
  369. package/dist/lib/storage/flagged-save-entry.d.ts +1 -1
  370. package/dist/lib/storage/flagged-save-entry.d.ts.map +1 -1
  371. package/dist/lib/storage/flagged-save-entry.js +0 -1
  372. package/dist/lib/storage/flagged-storage-file.d.ts +1 -1
  373. package/dist/lib/storage/flagged-storage-file.d.ts.map +1 -1
  374. package/dist/lib/storage/flagged-storage-file.js +0 -1
  375. package/dist/lib/storage/flagged-storage-io.d.ts +1 -1
  376. package/dist/lib/storage/flagged-storage-io.d.ts.map +1 -1
  377. package/dist/lib/storage/flagged-storage-io.js +0 -1
  378. package/dist/lib/storage/flagged-storage.d.ts +1 -1
  379. package/dist/lib/storage/flagged-storage.d.ts.map +1 -1
  380. package/dist/lib/storage/flagged-storage.js +15 -2
  381. package/dist/lib/storage/flagged-storage.js.map +1 -1
  382. package/dist/lib/storage/gitignore.js +0 -1
  383. package/dist/lib/storage/hash.js +0 -1
  384. package/dist/lib/storage/health.js +0 -1
  385. package/dist/lib/storage/identity.js +0 -1
  386. package/dist/lib/storage/import-export.d.ts +1 -1
  387. package/dist/lib/storage/import-export.d.ts.map +1 -1
  388. package/dist/lib/storage/import-export.js +0 -1
  389. package/dist/lib/storage/metadata-section.js +0 -1
  390. package/dist/lib/storage/migrations.d.ts +5 -48
  391. package/dist/lib/storage/migrations.d.ts.map +1 -1
  392. package/dist/lib/storage/migrations.js +5 -2
  393. package/dist/lib/storage/named-backup-entry.js +0 -1
  394. package/dist/lib/storage/named-backups-entry.d.ts +1 -1
  395. package/dist/lib/storage/named-backups-entry.d.ts.map +1 -1
  396. package/dist/lib/storage/named-backups-entry.js +0 -1
  397. package/dist/lib/storage/named-backups.js +0 -1
  398. package/dist/lib/storage/path-state.js +0 -1
  399. package/dist/lib/storage/paths.js +0 -1
  400. package/dist/lib/storage/project-migration.d.ts +1 -1
  401. package/dist/lib/storage/project-migration.d.ts.map +1 -1
  402. package/dist/lib/storage/project-migration.js +0 -1
  403. package/dist/lib/storage/public-types.d.ts +70 -0
  404. package/dist/lib/storage/public-types.d.ts.map +1 -0
  405. package/dist/lib/storage/public-types.js +6 -0
  406. package/dist/lib/storage/record-utils.js +0 -1
  407. package/dist/lib/storage/restore-assessment.d.ts +1 -1
  408. package/dist/lib/storage/restore-assessment.d.ts.map +1 -1
  409. package/dist/lib/storage/restore-assessment.js +0 -1
  410. package/dist/lib/storage/restore-backup-entry.d.ts +1 -1
  411. package/dist/lib/storage/restore-backup-entry.d.ts.map +1 -1
  412. package/dist/lib/storage/restore-backup-entry.js +0 -1
  413. package/dist/lib/storage/restore-metadata.d.ts +1 -1
  414. package/dist/lib/storage/restore-metadata.d.ts.map +1 -1
  415. package/dist/lib/storage/restore-metadata.js +0 -1
  416. package/dist/lib/storage/restore.d.ts +1 -1
  417. package/dist/lib/storage/restore.d.ts.map +1 -1
  418. package/dist/lib/storage/restore.js +0 -1
  419. package/dist/lib/storage/save-retry.d.ts +10 -0
  420. package/dist/lib/storage/save-retry.d.ts.map +1 -0
  421. package/dist/lib/storage/save-retry.js +26 -0
  422. package/dist/lib/storage/snapshot-inspectors.d.ts +1 -1
  423. package/dist/lib/storage/snapshot-inspectors.d.ts.map +1 -1
  424. package/dist/lib/storage/snapshot-inspectors.js +0 -1
  425. package/dist/lib/storage/storage-parser.d.ts +1 -1
  426. package/dist/lib/storage/storage-parser.d.ts.map +1 -1
  427. package/dist/lib/storage/storage-parser.js +0 -1
  428. package/dist/lib/storage/transactions.d.ts +1 -1
  429. package/dist/lib/storage/transactions.d.ts.map +1 -1
  430. package/dist/lib/storage/transactions.js +0 -1
  431. package/dist/lib/storage.d.ts +3 -24
  432. package/dist/lib/storage.d.ts.map +1 -1
  433. package/dist/lib/storage.js +17 -32
  434. package/dist/lib/table-formatter.js +0 -1
  435. package/dist/lib/tools/hashline-tools.js +0 -1
  436. package/dist/lib/types.js +0 -1
  437. package/dist/lib/ui/ansi.js +0 -1
  438. package/dist/lib/ui/auth-menu-builder.d.ts +89 -0
  439. package/dist/lib/ui/auth-menu-builder.d.ts.map +1 -0
  440. package/dist/lib/ui/auth-menu-builder.js +350 -0
  441. package/dist/lib/ui/auth-menu.d.ts +2 -76
  442. package/dist/lib/ui/auth-menu.d.ts.map +1 -1
  443. package/dist/lib/ui/auth-menu.js +3 -349
  444. package/dist/lib/ui/confirm.js +0 -1
  445. package/dist/lib/ui/copy.d.ts +2 -0
  446. package/dist/lib/ui/copy.d.ts.map +1 -1
  447. package/dist/lib/ui/copy.js +2 -0
  448. package/dist/lib/ui/copy.js.map +1 -1
  449. package/dist/lib/ui/display-width.js +0 -1
  450. package/dist/lib/ui/format.js +0 -1
  451. package/dist/lib/ui/index.d.ts +10 -0
  452. package/dist/lib/ui/index.d.ts.map +1 -0
  453. package/dist/lib/ui/index.js +9 -0
  454. package/dist/lib/ui/runtime.js +0 -1
  455. package/dist/lib/ui/select.js +0 -1
  456. package/dist/lib/ui/theme.js +0 -1
  457. package/dist/lib/ui/ui-copy.d.ts +159 -0
  458. package/dist/lib/ui/ui-copy.d.ts.map +1 -0
  459. package/dist/lib/ui/ui-copy.js +163 -0
  460. package/dist/lib/unified-settings.js +0 -1
  461. package/dist/lib/update-notice.js +0 -1
  462. package/dist/lib/usage/index.js +0 -1
  463. package/dist/lib/usage/ledger.js +0 -1
  464. package/dist/lib/usage/pricing.js +0 -1
  465. package/dist/lib/usage/redaction.js +0 -1
  466. package/dist/lib/usage/types.js +0 -1
  467. package/dist/lib/utils.js +0 -1
  468. package/package.json +8 -8
  469. package/scripts/postinstall.js +34 -265
  470. package/scripts/preuninstall.js +2 -2
@@ -1,20 +1,26 @@
1
- import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
2
- import { existsSync, readFileSync, statSync } from "node:fs";
1
+ import { randomUUID, timingSafeEqual } from "node:crypto";
3
2
  import { createServer } from "node:http";
4
3
  import { AccountManager, extractAccountId, } from "./accounts.js";
5
4
  import { withRoutingMutex } from "./routing-mutex.js";
6
- import { getStoragePath } from "./storage.js";
7
5
  import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getMinRotationIntervalMs, getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, getRoutingMutexMode, getSchedulingStrategy, loadPluginConfig, } from "./config.js";
8
6
  import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
9
7
  import { getModelFamily } from "./prompts/codex.js";
10
8
  import { CURRENT_CODEX_MODEL } from "./request/helpers/model-map.js";
11
- import { queuedRefresh } from "./refresh-queue.js";
12
- import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, recordRuntimeReload, recordRuntimeReset, } from "./runtime/runtime-observability.js";
9
+ import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
13
10
  import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
14
11
  import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
15
12
  import { createLogger, maskString, runWithCorrelationId } from "./logger.js";
13
+ import { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, extractErrorCodeFromBody, getQuotaNearExhaustionWaitMs, isTokenInvalidationError, normalizeExhaustionStatus, parseRetryAfterBodyMs, parseRetryAfterHeaderMs, } from "./request/rate-limit-decision.js";
14
+ import { forwardStreamingResponse, HOP_BY_HOP_HEADERS, readErrorBody, responseHeadersForClient, withTimeout, } from "./request/stream-failover-runtime.js";
15
+ import { chooseAccount } from "./runtime/rotation-account-selection.js";
16
+ import { createRotationProxyState, recoverStaleRuntimeState, } from "./runtime/rotation-proxy-state.js";
17
+ import { readStorageMetaFromDisk } from "./runtime/rotation-storage-meta.js";
18
+ import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, } from "./runtime/rotation-token-refresh.js";
16
19
  import { SessionAffinityStore } from "./session-affinity.js";
17
20
  import { isRecord } from "./utils.js";
21
+ export { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, } from "./request/rate-limit-decision.js";
22
+ export { chooseAccount } from "./runtime/rotation-account-selection.js";
23
+ export { maybeInvalidateAffinityFromDisk, readPinnedAccountIndexFromDisk, readStorageMetaFromDisk, resetPinCacheForTesting, } from "./runtime/rotation-storage-meta.js";
18
24
  const DEFAULT_HOST = "127.0.0.1";
19
25
  function isLoopbackHost(host) {
20
26
  const normalized = host.trim().toLowerCase();
@@ -54,47 +60,9 @@ function toUrlHost(host) {
54
60
  // level-gated and carry the per-request correlation id set in handleRequest.
55
61
  const proxyLog = createLogger("runtime-proxy");
56
62
  const DEFAULT_QUOTA_REMAINING_THRESHOLD = 10;
57
- const DEFAULT_AUTH_FAILURE_COOLDOWN_MS = 30_000;
58
63
  const DEFAULT_MAX_RUNTIME_ACCOUNT_ATTEMPTS = 4;
59
- // Phrases observed in upstream 401 response bodies when OpenAI/Microsoft has
60
- // explicitly revoked an OAuth token (as opposed to a generic expired-token 401
61
- // that can be retried after a refresh). Matching is case-insensitive substring.
62
- // If anti-abuse detection triggers different wording in production, add the new
63
- // phrase here and record the source provider and date. See issue #495.
64
- const TOKEN_INVALIDATION_PHRASES = [
65
- "invalidated oauth token",
66
- "authentication token has been invalidated",
67
- "oauth token has been invalidated",
68
- "token has been invalidated",
69
- ];
70
- function isTokenInvalidationError(bodyText) {
71
- const lower = bodyText.toLowerCase();
72
- return TOKEN_INVALIDATION_PHRASES.some((phrase) => lower.includes(phrase));
73
- }
74
64
  const MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024;
75
65
  const MAX_THREAD_GOAL_FALLBACKS = 512;
76
- const HOP_BY_HOP_HEADERS = new Set([
77
- "connection",
78
- "content-length",
79
- "expect",
80
- "keep-alive",
81
- "proxy-authenticate",
82
- "proxy-authorization",
83
- "te",
84
- "trailer",
85
- "transfer-encoding",
86
- "upgrade",
87
- ]);
88
- const PRIVATE_CLIENT_RESPONSE_HEADERS = new Set([
89
- "x-codex-multi-auth-account-index",
90
- "x-codex-multi-auth-account-label",
91
- "x-codex-multi-auth-account-email",
92
- "x-codex-multi-auth-account-id",
93
- ]);
94
- const DECODED_UPSTREAM_RESPONSE_HEADERS = new Set([
95
- // Node fetch returns decoded bytes while preserving the upstream encoding header.
96
- "content-encoding",
97
- ]);
98
66
  const ALLOWED_RESPONSES_PATHS = new Set([
99
67
  URL_PATHS.RESPONSES,
100
68
  URL_PATHS.CODEX_RESPONSES,
@@ -201,147 +169,7 @@ function recordLastRuntimeAccount(status, identity) {
201
169
  snapshot.lastAccountUpdatedAt = identity.updatedAt;
202
170
  });
203
171
  }
204
- // Keyed by absolute storage path so multiple proxy instances and concurrent
205
- // vitest workers (each pointing at their own temp storage file) cannot
206
- // corrupt each other's snapshots. See issue #474.
207
- const STORAGE_META_CACHE = new Map();
208
- // The mtime+size short-circuit (L3) may only be trusted once the cached
209
- // mtime is far enough in the past that no *subsequent* write could share the
210
- // same coarse mtime tick. Filesystems report mtime at wildly different
211
- // granularities (ext4 ns, FAT 2s, some network/Windows volumes ~1s, and CI
212
- // containers occasionally coarser), and our writers use atomic rename, so two
213
- // rapid CLI bumps can land on an identical mtimeMs. Within this settle window
214
- // we therefore ignore mtime equality and fall back to the read + sha1 path
215
- // (the real source of truth). Outside it, the file has been quiescent long
216
- // enough that mtime equality provably means "unchanged", so we skip the read.
217
- // 2s comfortably exceeds the coarsest mtime granularity we expect in practice.
218
- const MTIME_SHORTCIRCUIT_SETTLE_MS = 2_000;
219
- function hashStorageBytes(bytes) {
220
- return createHash("sha1").update(bytes).digest("hex");
221
- }
222
- function metaFromSnapshot(snapshot) {
223
- return {
224
- pinnedAccountIndex: snapshot.pinnedAccountIndex,
225
- affinityGeneration: snapshot.affinityGeneration,
226
- };
227
- }
228
- /**
229
- * Cheap, hot-path-safe single read with mtime-cache short-circuit. When the
230
- * file's `mtimeMs` and `size` match the cached snapshot for this path we
231
- * return the cached value WITHOUT reading or hashing the file. Only when
232
- * mtime/size differ (or were never cached) do we `readFileSync` + sha1 and,
233
- * if the content hash still matches, skip the `JSON.parse`. Transient
234
- * failures (EBUSY/EPERM/EACCES/EAGAIN, partial-write SyntaxError) fall through
235
- * to the last cached value for this path; defaults are only returned when the
236
- * file has never been successfully read.
237
- *
238
- * Replaces an earlier retry loop with a sub-15ms busy-wait that blocked the
239
- * event loop on every transient failure. The proxy is on the request hot
240
- * path; serving a slightly stale (but consistent) value is strictly better
241
- * than blocking. See #474.
242
- */
243
- export function readStorageMetaFromDisk(storagePath = getStoragePath()) {
244
- if (!existsSync(storagePath)) {
245
- STORAGE_META_CACHE.delete(storagePath);
246
- return { pinnedAccountIndex: null, affinityGeneration: 0 };
247
- }
248
- try {
249
- // mtime+size short-circuit (L3): when neither has changed since the last
250
- // successful read AND the cached mtime has settled (see
251
- // MTIME_SHORTCIRCUIT_SETTLE_MS) we return the cached snapshot without
252
- // reading or hashing the file. During the settle window we deliberately
253
- // fall through to the read + sha1 path below, which stays the source of
254
- // truth and protects against the coarse-mtime collision described on
255
- // StorageMetaSnapshot. So this is a pure fast path for the common
256
- // "file quiescent, proxy polling every request" case.
257
- const stats = statSync(storagePath);
258
- const cachedByStat = STORAGE_META_CACHE.get(storagePath);
259
- if (cachedByStat &&
260
- cachedByStat.mtimeMs === stats.mtimeMs &&
261
- cachedByStat.size === stats.size &&
262
- Date.now() - stats.mtimeMs > MTIME_SHORTCIRCUIT_SETTLE_MS) {
263
- return metaFromSnapshot(cachedByStat);
264
- }
265
- const bytes = readFileSync(storagePath);
266
- const contentHash = hashStorageBytes(bytes);
267
- const cached = STORAGE_META_CACHE.get(storagePath);
268
- if (cached && cached.contentHash === contentHash) {
269
- // Content is byte-identical despite the mtime/size change (e.g. an
270
- // atomic-rename rewrite of the same bytes). Refresh the stat fields so
271
- // the next request takes the fast path, but skip the JSON.parse.
272
- const refreshed = {
273
- ...cached,
274
- mtimeMs: stats.mtimeMs,
275
- size: stats.size,
276
- };
277
- STORAGE_META_CACHE.set(storagePath, refreshed);
278
- return metaFromSnapshot(refreshed);
279
- }
280
- const parsed = JSON.parse(bytes.toString("utf8"));
281
- const pinnedAccountIndex = typeof parsed.pinnedAccountIndex === "number" &&
282
- Number.isFinite(parsed.pinnedAccountIndex)
283
- ? Math.trunc(parsed.pinnedAccountIndex)
284
- : null;
285
- const affinityGeneration = typeof parsed.affinityGeneration === "number" &&
286
- Number.isFinite(parsed.affinityGeneration) &&
287
- Number.isInteger(parsed.affinityGeneration) &&
288
- parsed.affinityGeneration >= 0
289
- ? parsed.affinityGeneration
290
- : 0;
291
- const snapshot = {
292
- mtimeMs: stats.mtimeMs,
293
- size: stats.size,
294
- contentHash,
295
- pinnedAccountIndex,
296
- affinityGeneration,
297
- };
298
- STORAGE_META_CACHE.set(storagePath, snapshot);
299
- return { pinnedAccountIndex, affinityGeneration };
300
- }
301
- catch (error) {
302
- // On any failure, prefer the last good snapshot for this path so we
303
- // don't blow away affinity unnecessarily. Defensive: even non-transient
304
- // errors fall back to the cache when one exists — better stale than
305
- // wrong. Defaults are only returned when this file has never been read
306
- // successfully (cache miss).
307
- void error;
308
- const cached = STORAGE_META_CACHE.get(storagePath);
309
- if (cached) {
310
- return metaFromSnapshot(cached);
311
- }
312
- return { pinnedAccountIndex: null, affinityGeneration: 0 };
313
- }
314
- }
315
- /**
316
- * Backwards-compatible helper retained for tests. Prefer `readStorageMetaFromDisk`
317
- * for new callers that also need `affinityGeneration`.
318
- */
319
- export function readPinnedAccountIndexFromDisk(storagePath = getStoragePath()) {
320
- return readStorageMetaFromDisk(storagePath).pinnedAccountIndex;
321
- }
322
- /**
323
- * Test-only: reset the storage-meta content-hash cache between scenarios so
324
- * each test starts from a clean read-from-disk state.
325
- */
326
- export function resetPinCacheForTesting() {
327
- STORAGE_META_CACHE.clear();
328
- }
329
- /**
330
- * If the on-disk `affinityGeneration` is greater than `lastObservedGeneration`,
331
- * drop every entry in `sessionAffinityStore` and return the new generation so
332
- * the caller can update its tracker. Otherwise returns `lastObservedGeneration`
333
- * unchanged. Extracted so the request-flow logic can be unit-tested without
334
- * spinning up the full proxy. See issue #474.
335
- */
336
- export function maybeInvalidateAffinityFromDisk(sessionAffinityStore, lastObservedGeneration, storagePath = getStoragePath()) {
337
- const meta = readStorageMetaFromDisk(storagePath);
338
- if (meta.affinityGeneration > lastObservedGeneration) {
339
- sessionAffinityStore?.clearAll();
340
- return meta.affinityGeneration;
341
- }
342
- return lastObservedGeneration;
343
- }
344
- async function persistRuntimeActiveAccount(accountManager, account, family, isPinned) {
172
+ async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy) {
345
173
  if (isPinned) {
346
174
  // When the user has manually pinned an account, the proxy MUST NOT
347
175
  // clobber that pin via markSwitched("rotation"), saveToDiskDebounced(),
@@ -367,7 +195,14 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
367
195
  // `saveToDiskDebounced` + CLI sync still run in both modes: they snapshot the
368
196
  // current in-memory state / mirror the CLI selection and do not advance the
369
197
  // in-memory rotation cursor.
370
- if (accountManager.getRoutingMutexMode() !== "enabled") {
198
+ //
199
+ // Sequential scheduling fix (#509): in "sequential" mode a within-request
200
+ // fallback to a different account must NOT advance the drain-first primary
201
+ // pointer — only a true primary-exhaustion in chooseAccount may do that.
202
+ // Apply the same guard here as the in-loop re-commit (lines 939-958) so
203
+ // the legacy branch cannot silently clobber the sequential drain order.
204
+ if (accountManager.getRoutingMutexMode() !== "enabled" &&
205
+ schedulingStrategy !== "sequential") {
371
206
  await accountManager.markSwitchedLocked(account, "rotation", family);
372
207
  }
373
208
  accountManager.saveToDiskDebounced();
@@ -378,20 +213,6 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
378
213
  // because the local status mirrors are temporarily locked.
379
214
  }
380
215
  }
381
- function responseHeadersForClient(upstreamHeaders) {
382
- const headers = {};
383
- for (const [key, value] of upstreamHeaders.entries()) {
384
- const normalizedKey = key.toLowerCase();
385
- if (HOP_BY_HOP_HEADERS.has(normalizedKey))
386
- continue;
387
- if (PRIVATE_CLIENT_RESPONSE_HEADERS.has(normalizedKey))
388
- continue;
389
- if (DECODED_UPSTREAM_RESPONSE_HEADERS.has(normalizedKey))
390
- continue;
391
- headers[key] = value;
392
- }
393
- return headers;
394
- }
395
216
  function createRuntimeProxyHttpError(message, statusCode, code) {
396
217
  return Object.assign(new Error(message), { statusCode, code });
397
218
  }
@@ -548,471 +369,12 @@ function buildUpstreamUrl(req, upstreamBaseUrl, upstreamPath) {
548
369
  upstream.search = incomingUrl.search;
549
370
  return upstream.toString();
550
371
  }
551
- // Monotonic auth-failure cooldown: only extend, never shorten. Two concurrent
552
- // requests on the same account can race so that an invalidation path sets a
553
- // long cooldown (5 min) and a subsequent generic 401 truncates it (30 s).
554
- // Reading the live coolingDownUntil before writing prevents that race.
555
- function applyMonotonicAuthCooldown(accountManager, account, cooldownMs) {
556
- const existing = accountManager.getAccountByIndex(account.index)?.coolingDownUntil ?? 0;
557
- // Intentionally Date.now(), not the proxy's injected now(): coolingDownUntil is
558
- // written by markAccountCoolingDown via nowMs() (== Date.now()), so both sides of
559
- // this comparison must live in the same real-wall-clock domain. Switching to the
560
- // injected now() would mis-compare an injected-clock value against a real-clock
561
- // deadline and silently defeat the monotonic guard.
562
- if (Date.now() + cooldownMs > existing) {
563
- accountManager.markAccountCoolingDown(account, cooldownMs, "auth-failure");
564
- }
565
- }
566
- function hasUsableAccessToken(account, now, skewMs) {
567
- return (typeof account.access === "string" &&
568
- account.access.trim().length > 0 &&
569
- typeof account.expires === "number" &&
570
- account.expires > now + Math.max(0, skewMs));
571
- }
572
- function isTokenRefreshRetryable(result) {
573
- if (result.reason === "network_error" || result.reason === "unknown")
574
- return true;
575
- if (result.reason === "invalid_response")
576
- return true;
577
- if (result.reason === "http_error") {
578
- return !(result.statusCode === HTTP_STATUS.BAD_REQUEST ||
579
- result.statusCode === HTTP_STATUS.UNAUTHORIZED ||
580
- result.statusCode === HTTP_STATUS.FORBIDDEN);
581
- }
582
- return false;
583
- }
584
- const runtimeRefreshCommitQueues = new WeakMap();
585
- async function commitRefreshedAuthOnce(accountManager, account, auth) {
586
- const key = [
587
- account.index,
588
- account.accountId ?? "",
589
- account.email ?? "",
590
- account.refreshToken,
591
- ].join("\0");
592
- let queue = runtimeRefreshCommitQueues.get(accountManager);
593
- if (!queue) {
594
- queue = new Map();
595
- runtimeRefreshCommitQueues.set(accountManager, queue);
596
- }
597
- const existing = queue.get(key);
598
- if (existing)
599
- return existing;
600
- const pending = accountManager
601
- .commitRefreshedAuth(account, auth)
602
- .finally(() => queue?.delete(key));
603
- queue.set(key, pending);
604
- return pending;
605
- }
606
- async function ensureFreshAccessToken(params) {
607
- const { accountManager, account, family, model, now, tokenRefreshSkewMs, tokenInvalidationCooldownMs } = params;
608
- if (hasUsableAccessToken(account, now, tokenRefreshSkewMs)) {
609
- return { ok: true, accessToken: account.access ?? "", account };
610
- }
611
- const refreshResult = await queuedRefresh(account.refreshToken);
612
- if (refreshResult.type === "failed") {
613
- accountManager.recordFailure(account, family, model);
614
- accountManager.incrementAuthFailures(account);
615
- // If the refresh endpoint itself returns an explicit invalidation message
616
- // (e.g. Microsoft/Outlook SSO revokes the refresh token server-side), apply
617
- // the long cooldown and signal to the caller to stop rotating rather than
618
- // presenting other accounts' tokens from the same IP.
619
- const invalidated = isTokenInvalidationError(refreshResult.message ?? "");
620
- applyMonotonicAuthCooldown(accountManager, account, invalidated ? tokenInvalidationCooldownMs : DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
621
- accountManager.saveToDiskDebounced();
622
- return { ok: false, retryable: isTokenRefreshRetryable(refreshResult), invalidated };
623
- }
624
- const auth = {
625
- type: "oauth",
626
- access: refreshResult.access,
627
- refresh: refreshResult.refresh,
628
- expires: refreshResult.expires,
629
- };
630
- try {
631
- const updatedAccount = (await commitRefreshedAuthOnce(accountManager, account, auth)) ?? account;
632
- return {
633
- ok: true,
634
- accessToken: updatedAccount.access ?? refreshResult.access,
635
- account: updatedAccount,
636
- };
637
- }
638
- catch {
639
- accountManager.recordFailure(account, family, model);
640
- accountManager.markAccountCoolingDown(account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
641
- accountManager.saveToDiskDebounced();
642
- return { ok: false, retryable: true };
643
- }
644
- }
645
372
  function resolveAccountId(account, accessToken) {
646
373
  const stored = account.accountId?.trim();
647
374
  if (stored)
648
375
  return stored;
649
376
  return extractAccountId(accessToken)?.trim() || null;
650
377
  }
651
- function parseRetryAfterHeaderMs(headers, now) {
652
- const retryAfterMs = headers.get("retry-after-ms");
653
- if (retryAfterMs) {
654
- const parsed = Number.parseInt(retryAfterMs, 10);
655
- if (Number.isFinite(parsed) && parsed > 0)
656
- return parsed;
657
- }
658
- const retryAfter = headers.get("retry-after");
659
- if (!retryAfter)
660
- return null;
661
- const asSeconds = Number.parseInt(retryAfter, 10);
662
- if (Number.isFinite(asSeconds) && asSeconds > 0)
663
- return asSeconds * 1000;
664
- const asDate = Date.parse(retryAfter);
665
- if (Number.isFinite(asDate) && asDate > now)
666
- return asDate - now;
667
- return null;
668
- }
669
- function parseRetryAfterBodyMs(bodyText, now) {
670
- if (!bodyText.trim())
671
- return null;
672
- try {
673
- const parsed = JSON.parse(bodyText);
674
- if (!isRecord(parsed) || !isRecord(parsed.error))
675
- return null;
676
- const retryAfterMs = Number(parsed.error.retry_after_ms);
677
- if (Number.isFinite(retryAfterMs) && retryAfterMs > 0)
678
- return retryAfterMs;
679
- const retryAfterSeconds = Number(parsed.error.retry_after);
680
- if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) {
681
- return retryAfterSeconds * 1000;
682
- }
683
- const resetAtRaw = Number(parsed.error.resets_at ?? parsed.error.reset_at);
684
- if (Number.isFinite(resetAtRaw) && resetAtRaw > 0) {
685
- const resetAtMs = resetAtRaw < 10_000_000_000 ? resetAtRaw * 1000 : resetAtRaw;
686
- if (resetAtMs > now)
687
- return resetAtMs - now;
688
- }
689
- }
690
- catch {
691
- return null;
692
- }
693
- return null;
694
- }
695
- async function readErrorBody(response, timeoutMs, maxBytes = 1024 * 1024) {
696
- // The outbound fetch's abort timer is cleared once headers arrive, so a
697
- // stalled error body would otherwise hang this handler forever (the success
698
- // path is per-chunk stall-bounded; the error path was not). Read the body via
699
- // a reader, bound it by an idle timeout AND a size cap, and cancel the stream
700
- // on timeout/overflow so the socket is released.
701
- const body = response.body;
702
- if (!body || typeof body.getReader !== "function") {
703
- // Fallback for impls without a streamable body: race text() against a timer.
704
- try {
705
- return await withTimeout(response.text(), timeoutMs, () => undefined, "error body stalled");
706
- }
707
- catch {
708
- return "";
709
- }
710
- }
711
- const reader = body.getReader();
712
- const chunks = [];
713
- let total = 0;
714
- try {
715
- for (;;) {
716
- let idleTimer;
717
- const idle = new Promise((_resolve, reject) => {
718
- idleTimer = setTimeout(() => reject(new Error("error body stalled")), Math.max(1, timeoutMs));
719
- });
720
- let result;
721
- try {
722
- result = await Promise.race([reader.read(), idle]);
723
- }
724
- finally {
725
- if (idleTimer)
726
- clearTimeout(idleTimer);
727
- }
728
- if (result.done)
729
- break;
730
- if (result.value) {
731
- total += result.value.byteLength;
732
- if (total > maxBytes)
733
- break; // cap: enough for diagnostics, no OOM
734
- chunks.push(result.value);
735
- }
736
- }
737
- }
738
- catch {
739
- // stalled or errored — fall through with whatever we have
740
- }
741
- finally {
742
- await reader.cancel().catch(() => undefined);
743
- }
744
- try {
745
- return Buffer.concat(chunks).toString("utf8");
746
- }
747
- catch {
748
- return "";
749
- }
750
- }
751
- const TOKEN_INVALIDATED_CODE = "token_invalidated";
752
- const TOKEN_INVALIDATED_FALLBACK_MESSAGE = "OAuth token has been invalidated. Please re-login.";
753
- // Both invalidation exit paths (refresh-failure and upstream-401) must hand the
754
- // client the same machine-readable shape — { error: { message, code:
755
- // "token_invalidated" } } — so a consumer keying off error.code behaves
756
- // identically regardless of which vector fired. The upstream forwards a raw body
757
- // with no guaranteed code, so we wrap it here while preserving its human-readable
758
- // message when one is present.
759
- export function buildTokenInvalidationBody(upstreamBodyText) {
760
- let message = TOKEN_INVALIDATED_FALLBACK_MESSAGE;
761
- const trimmed = upstreamBodyText.trim();
762
- if (trimmed) {
763
- try {
764
- const parsed = JSON.parse(trimmed);
765
- if (isRecord(parsed)) {
766
- const direct = parsed.message;
767
- if (typeof direct === "string" && direct.trim()) {
768
- message = direct.trim();
769
- }
770
- else if (isRecord(parsed.error)) {
771
- const nested = parsed.error.message;
772
- if (typeof nested === "string" && nested.trim()) {
773
- message = nested.trim();
774
- }
775
- }
776
- }
777
- }
778
- catch {
779
- // Non-JSON upstream body (e.g. HTML error page): keep the stable fallback
780
- // message rather than echoing markup back to the client.
781
- }
782
- }
783
- return JSON.stringify({ error: { message, code: TOKEN_INVALIDATED_CODE } });
784
- }
785
- function extractErrorCodeFromBody(bodyText) {
786
- if (!bodyText.trim())
787
- return null;
788
- try {
789
- const parsed = JSON.parse(bodyText);
790
- if (!isRecord(parsed))
791
- return null;
792
- const directCode = parsed.code;
793
- if (typeof directCode === "string" && directCode.trim()) {
794
- return directCode.trim();
795
- }
796
- const maybeError = parsed.error;
797
- if (!isRecord(maybeError))
798
- return null;
799
- const nestedCode = maybeError.code;
800
- return typeof nestedCode === "string" && nestedCode.trim()
801
- ? nestedCode.trim()
802
- : null;
803
- }
804
- catch {
805
- return null;
806
- }
807
- }
808
- function getQuotaWindowWaitMs(headers, prefix, now) {
809
- const resetAfterSeconds = Number.parseInt(headers.get(`${prefix}-reset-after-seconds`) ?? "", 10);
810
- if (Number.isFinite(resetAfterSeconds) && resetAfterSeconds > 0) {
811
- return resetAfterSeconds * 1000;
812
- }
813
- const resetAtRaw = headers.get(`${prefix}-reset-at`);
814
- if (!resetAtRaw)
815
- return 0;
816
- const trimmed = resetAtRaw.trim();
817
- let resetAtMs = 0;
818
- if (/^\d+$/.test(trimmed)) {
819
- const parsed = Number.parseInt(trimmed, 10);
820
- if (Number.isFinite(parsed) && parsed > 0) {
821
- resetAtMs = parsed < 10_000_000_000 ? parsed * 1000 : parsed;
822
- }
823
- }
824
- else {
825
- const parsedDate = Date.parse(trimmed);
826
- if (Number.isFinite(parsedDate))
827
- resetAtMs = parsedDate;
828
- }
829
- return resetAtMs > now ? resetAtMs - now : 0;
830
- }
831
- function getQuotaNearExhaustionWaitMs(headers, remainingThreshold, now) {
832
- const usedThreshold = 100 - Math.max(0, Math.min(100, remainingThreshold));
833
- const candidates = [];
834
- for (const prefix of ["x-codex-primary", "x-codex-secondary"]) {
835
- const used = Number(headers.get(`${prefix}-used-percent`) ?? "");
836
- if (!Number.isFinite(used) || used < usedThreshold)
837
- continue;
838
- const waitMs = getQuotaWindowWaitMs(headers, prefix, now);
839
- if (waitMs > 0)
840
- candidates.push(waitMs);
841
- }
842
- return candidates.length > 0 ? Math.max(...candidates) : 0;
843
- }
844
- /**
845
- * `chooseAccount` is a SYNC selector that internally advances the rotation
846
- * cursor (the session-affinity-preferred branch and the round-robin fallback
847
- * both call `accountManager.markSwitched(...)`, and the hybrid selector advances
848
- * its own cursor). It does NOT acquire the routing mutex itself.
849
- *
850
- * Concurrency (L4): when `routingMutex === "enabled"`, the proxy hot path runs
851
- * this whole call AND the subsequent `markSwitchedLocked` commit inside a single
852
- * `withRoutingMutex` acquisition, so concurrent requests serialize selection +
853
- * cursor advance and cannot stampede the same account. `withRoutingMutex` is
854
- * reentrant (AsyncLocalStorage), so the nested `markSwitchedLocked` — and the
855
- * later one in `persistRuntimeActiveAccount` — run inline without re-acquiring
856
- * the non-reentrant FIFO queue (no deadlock). In legacy mode the inline
857
- * `markSwitched` calls below are used unchanged and no lock is taken, so default
858
- * behavior and perf are identical. See the hot-path caller in
859
- * `startRuntimeRotationProxy` and the regression in
860
- * `test/runtime-rotation-proxy.test.ts`.
861
- */
862
- export function chooseAccount(params) {
863
- const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, skipReasons, stickyBoostByAccount, pidOffsetEnabled, schedulingStrategy, } = params;
864
- // Manual pin (from `codex-multi-auth switch <n>`) overrides every other
865
- // selection signal. We do NOT call markSwitched here — the proxy must not
866
- // clobber the pin set by the CLI. See issue #474.
867
- if (typeof pinnedIndex === "number") {
868
- if (attemptedIndexes.has(pinnedIndex)) {
869
- skipReasons?.set(pinnedIndex, "already-attempted");
870
- return null;
871
- }
872
- if (pinnedIndex < 0 || pinnedIndex >= accountManager.getAccountCount()) {
873
- skipReasons?.set(pinnedIndex, "missing");
874
- return null;
875
- }
876
- if (policy?.blockedAccountIndexes.has(pinnedIndex)) {
877
- skipReasons?.set(pinnedIndex, "policy-blocked");
878
- return null;
879
- }
880
- const pinned = accountManager.getAccountByIndex(pinnedIndex);
881
- if (!pinned || pinned.enabled === false) {
882
- skipReasons?.set(pinnedIndex, "disabled");
883
- return null;
884
- }
885
- const reason = accountManager.getAccountRuntimeSkipReason(pinnedIndex, family, model);
886
- if (reason) {
887
- skipReasons?.set(pinnedIndex, reason);
888
- return null;
889
- }
890
- return pinned;
891
- }
892
- // Sequential / drain-first mode (issue #509): the active account governs ALL
893
- // new requests, so we deliberately SKIP the per-session affinity tier — there
894
- // is no per-chat stickiness, every request follows the single active account.
895
- // The manual pin above still wins (handled first). When the active account is
896
- // exhausted the selector advances to the next available account and earlier
897
- // accounts reclaim the slot once their quota recovers.
898
- if (schedulingStrategy === "sequential") {
899
- const selected = accountManager.getCurrentOrNextForFamilySequential(family, model, policy?.blockedAccountIndexes);
900
- if (selected &&
901
- !attemptedIndexes.has(selected.index) &&
902
- !policy?.blockedAccountIndexes.has(selected.index)) {
903
- const reason = accountManager.getAccountRuntimeSkipReason(selected.index, family, model);
904
- if (!reason)
905
- return selected;
906
- skipReasons?.set(selected.index, reason);
907
- }
908
- // Active account was attempted/blocked/skipped this request (e.g. it just
909
- // rate-limited mid-loop): fall through to the shared linear scan to find
910
- // the next eligible account to TRY. Pass advanceActivePointer=false so a
911
- // transient, non-exhausting failure on the active account does not
912
- // permanently move the drain-first primary — only
913
- // getCurrentOrNextForFamilySequential advances it, and only on true
914
- // exhaustion.
915
- return chooseLinearScanFallback({
916
- accountManager,
917
- family,
918
- model,
919
- attemptedIndexes,
920
- policy,
921
- skipReasons,
922
- advanceActivePointer: false,
923
- });
924
- }
925
- const preferredIndex = sessionAffinityStore?.getPreferredAccountIndex(sessionKey, now);
926
- if (typeof preferredIndex === "number" &&
927
- !attemptedIndexes.has(preferredIndex) &&
928
- !policy?.blockedAccountIndexes.has(preferredIndex)) {
929
- const preferred = accountManager.getAccountByIndex(preferredIndex);
930
- if (preferred) {
931
- const reason = accountManager.getAccountRuntimeSkipReason(preferred.index, family, model);
932
- if (reason) {
933
- skipReasons?.set(preferred.index, reason);
934
- }
935
- else {
936
- // L4 (deferred): unlocked cursor mutation — see chooseAccount header.
937
- accountManager.markSwitched(preferred, "rotation", family);
938
- return preferred;
939
- }
940
- }
941
- }
942
- const selected = accountManager.getCurrentOrNextForFamilyHybrid(family, model, {
943
- scoreBoostByAccount: {
944
- ...(policy?.scoreBoostByAccount ?? {}),
945
- ...(stickyBoostByAccount ?? {}),
946
- },
947
- // accounts-05: carry the PID-offset distribution into the default-on proxy
948
- // path too (index.ts already does). Without it, parallel proxy processes can
949
- // stampede the same account instead of spreading across the pool.
950
- pidOffsetEnabled,
951
- });
952
- if (selected &&
953
- !attemptedIndexes.has(selected.index) &&
954
- !policy?.blockedAccountIndexes.has(selected.index)) {
955
- const reason = accountManager.getAccountRuntimeSkipReason(selected.index, family, model);
956
- if (!reason)
957
- return selected;
958
- skipReasons?.set(selected.index, reason);
959
- }
960
- return chooseLinearScanFallback({
961
- accountManager,
962
- family,
963
- model,
964
- attemptedIndexes,
965
- policy,
966
- skipReasons,
967
- });
968
- }
969
- /**
970
- * Shared linear-scan fallback used by both the hybrid and sequential selection
971
- * paths in `chooseAccount`. Walks every account in pool order and returns the
972
- * first one that is not already attempted, not policy-blocked, and has no
973
- * runtime skip reason (rate-limited / cooling down / circuit-open), recording a
974
- * skip reason for each rejected candidate. Returns null when no eligible
975
- * account remains.
976
- *
977
- * `advanceActivePointer` (default `true`) controls whether the winner is
978
- * committed as the new active/cursor position via `markSwitched`. The hybrid
979
- * path wants this (round-robin advance). The sequential path passes `false`:
980
- * its within-request fallback only needs an account to TRY this request, and
981
- * must NOT move `currentAccountIndexByFamily` — otherwise a transient,
982
- * non-exhausting failure on the active account (which leaves it `isUsable` but
983
- * present in `attemptedIndexes`) would permanently switch the drain-first
984
- * primary even though it was never exhausted (issue #509 regression caught in
985
- * review). In sequential mode only `getCurrentOrNextForFamilySequential` is
986
- * allowed to advance the active pointer, and only on true exhaustion.
987
- */
988
- function chooseLinearScanFallback(params) {
989
- const { accountManager, family, model, attemptedIndexes, policy, skipReasons, advanceActivePointer = true, } = params;
990
- for (const account of accountManager.getAccountsSnapshot()) {
991
- if (attemptedIndexes.has(account.index)) {
992
- skipReasons?.set(account.index, "already-attempted");
993
- continue;
994
- }
995
- if (policy?.blockedAccountIndexes.has(account.index)) {
996
- skipReasons?.set(account.index, "policy-blocked");
997
- continue;
998
- }
999
- const reason = accountManager.getAccountRuntimeSkipReason(account.index, family, model);
1000
- if (!reason) {
1001
- const live = accountManager.getAccountByIndex(account.index);
1002
- if (!live)
1003
- continue;
1004
- // L4 (deferred): unlocked cursor mutation — see chooseAccount header.
1005
- // Skipped in sequential mode (advanceActivePointer=false) so a
1006
- // within-request retry never reassigns the drain-first primary.
1007
- if (advanceActivePointer) {
1008
- accountManager.markSwitched(live, "rotation", family);
1009
- }
1010
- return live;
1011
- }
1012
- skipReasons?.set(account.index, reason);
1013
- }
1014
- return null;
1015
- }
1016
378
  function writeJson(res, status, payload) {
1017
379
  res.writeHead(status, { "content-type": "application/json; charset=utf-8" });
1018
380
  res.end(`${JSON.stringify(payload)}\n`);
@@ -1033,9 +395,6 @@ function writeUnauthorized(res) {
1033
395
  },
1034
396
  });
1035
397
  }
1036
- function normalizeExhaustionStatus(reason) {
1037
- return reason === "rate-limit" ? HTTP_STATUS.TOO_MANY_REQUESTS : 503;
1038
- }
1039
398
  function writePoolExhausted(params) {
1040
399
  const { res, accountManager, family, model, reason } = params;
1041
400
  const waitMs = accountManager.getMinWaitTimeForFamily(family, model);
@@ -1060,82 +419,9 @@ function writePoolExhausted(params) {
1060
419
  },
1061
420
  });
1062
421
  }
1063
- export function buildPinnedUnavailableErrorBody(pinnedIndex, accountSkipReasons) {
1064
- const normalizedPinnedIndex = typeof pinnedIndex === "number" ? pinnedIndex : null;
1065
- const skipReason = normalizedPinnedIndex !== null
1066
- ? accountSkipReasons.get(normalizedPinnedIndex) ?? null
1067
- : null;
1068
- const reasonSuffix = skipReason ? ` (${skipReason})` : "";
1069
- const displayIndex = (normalizedPinnedIndex ?? 0) + 1;
1070
- return {
1071
- message: `Pinned account ${displayIndex} is currently unavailable${reasonSuffix}; run \`codex-multi-auth status\` for details, or \`codex-multi-auth unpin\` to allow rotation.`,
1072
- code: "codex_pinned_account_unavailable",
1073
- pinnedAccountIndex: normalizedPinnedIndex,
1074
- reason: skipReason,
1075
- account_skip_reasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
1076
- String(index),
1077
- reason,
1078
- ])),
1079
- };
1080
- }
1081
- async function withTimeout(promise, timeoutMs, onTimeout, message) {
1082
- let timeout;
1083
- try {
1084
- return await Promise.race([
1085
- promise,
1086
- new Promise((_resolve, reject) => {
1087
- timeout = setTimeout(() => {
1088
- onTimeout();
1089
- reject(new Error(message));
1090
- }, Math.max(1, timeoutMs));
1091
- }),
1092
- ]);
1093
- }
1094
- finally {
1095
- if (timeout)
1096
- clearTimeout(timeout);
1097
- }
1098
- }
1099
- async function forwardStreamingResponse(upstream, res, status, onStreamError, streamStallTimeoutMs) {
1100
- status.streamsStarted += 1;
1101
- res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
1102
- if (!upstream.body) {
1103
- res.end();
1104
- return true;
1105
- }
1106
- const reader = upstream.body.getReader();
1107
- res.on("close", () => {
1108
- if (!res.writableEnded) {
1109
- void reader.cancel().catch(() => undefined);
1110
- }
1111
- });
1112
- try {
1113
- while (true) {
1114
- const { done, value } = await withTimeout(reader.read(), streamStallTimeoutMs, () => {
1115
- void reader.cancel().catch(() => undefined);
1116
- }, `upstream stream stalled after ${streamStallTimeoutMs}ms`);
1117
- if (done)
1118
- break;
1119
- if (value && value.byteLength > 0) {
1120
- res.write(Buffer.from(value));
1121
- }
1122
- }
1123
- res.end();
1124
- return true;
1125
- }
1126
- catch (error) {
1127
- status.lastError = error instanceof Error ? error.message : String(error);
1128
- onStreamError();
1129
- if (!res.destroyed) {
1130
- res.destroy(error instanceof Error ? error : undefined);
1131
- }
1132
- return false;
1133
- }
1134
- }
1135
422
  export async function startRuntimeRotationProxy(options) {
1136
423
  const pluginConfig = loadPluginConfig();
1137
- let activeAccountManager = options.accountManager ?? (await AccountManager.loadFromDisk());
1138
- const knownAccountManagers = new Set([activeAccountManager]);
424
+ const activeAccountManager = options.accountManager ?? (await AccountManager.loadFromDisk());
1139
425
  // accounts-01/08: apply the configured routing-mutex mode so the proxy's
1140
426
  // async select->commit path (persistRuntimeActiveAccount) can serialize cursor
1141
427
  // mutations when routingMutex="enabled". Legacy mode keeps the inline fast path.
@@ -1174,8 +460,6 @@ export async function startRuntimeRotationProxy(options) {
1174
460
  const tokenInvalidationCooldownMs = getTokenInvalidationCooldownMs(pluginConfig);
1175
461
  const minRotationIntervalMs = getMinRotationIntervalMs(pluginConfig);
1176
462
  const pidOffsetEnabled = getPidOffsetEnabled(pluginConfig);
1177
- let lastGlobalAccountIndex = null;
1178
- let lastGlobalSwitchAt = 0;
1179
463
  const fetchTimeoutMs = options.fetchTimeoutMs ?? getFetchTimeoutMs(pluginConfig);
1180
464
  const streamStallTimeoutMs = options.streamStallTimeoutMs ?? getStreamStallTimeoutMs(pluginConfig);
1181
465
  const configuredMaxRetries = getRetryAllAccountsMaxRetries(pluginConfig);
@@ -1193,692 +477,670 @@ export async function startRuntimeRotationProxy(options) {
1193
477
  // Initialize from disk so the proxy starts in sync with whatever generation
1194
478
  // the storage file already shows. Subsequent disk bumps (from CLI commands)
1195
479
  // are detected per-request via `maybeInvalidateAffinityFromDisk`.
1196
- let lastObservedAffinityGeneration = readStorageMetaFromDisk().affinityGeneration;
1197
- const status = {
1198
- startedAt: now(),
1199
- totalRequests: 0,
1200
- upstreamRequests: 0,
1201
- retries: 0,
1202
- rotations: 0,
1203
- streamsStarted: 0,
1204
- lastError: null,
1205
- lastAccountIndex: null,
1206
- lastAccountLabel: null,
1207
- lastAccountId: null,
1208
- lastAccountUpdatedAt: null,
1209
- };
1210
- const threadGoalFallbacks = new Map();
1211
- let staleRuntimeReloadPromise = null;
1212
- let lastStaleRuntimeReloadAt = 0;
1213
- const staleRuntimeReloadDedupeMs = 1_000;
1214
- const recoverStaleRuntimeState = async () => {
1215
- if (Date.now() - lastStaleRuntimeReloadAt <= staleRuntimeReloadDedupeMs) {
1216
- return activeAccountManager;
1217
- }
1218
- if (staleRuntimeReloadPromise) {
1219
- return staleRuntimeReloadPromise;
1220
- }
1221
- staleRuntimeReloadPromise = (async () => {
1222
- AccountManager.resetVolatileRuntimeState();
1223
- recordRuntimeReset("pool-exhausted-no-account");
1224
- const reloaded = await AccountManager.loadFromDisk();
1225
- reloaded.setRoutingMutexMode(routingMutexMode);
1226
- activeAccountManager = reloaded;
1227
- knownAccountManagers.add(reloaded);
1228
- lastStaleRuntimeReloadAt = Date.now();
1229
- recordRuntimeReload("pool-exhausted-no-account");
1230
- return reloaded;
1231
- })()
1232
- .catch((error) => {
1233
- status.lastError = error instanceof Error ? error.message : String(error);
1234
- return null;
1235
- })
1236
- .finally(() => {
1237
- staleRuntimeReloadPromise = null;
480
+ const lastObservedAffinityGeneration = readStorageMetaFromDisk().affinityGeneration;
481
+ const state = createRotationProxyState({
482
+ activeAccountManager,
483
+ routingMutexMode,
484
+ schedulingStrategy,
485
+ fetchImpl,
486
+ upstreamBaseUrl,
487
+ clientApiKey,
488
+ now,
489
+ tokenRefreshSkewMs,
490
+ networkErrorCooldownMs,
491
+ serverErrorCooldownMs,
492
+ tokenInvalidationCooldownMs,
493
+ minRotationIntervalMs,
494
+ pidOffsetEnabled,
495
+ fetchTimeoutMs,
496
+ streamStallTimeoutMs,
497
+ maxRuntimeAccountAttempts,
498
+ maxRequestBodyBytes,
499
+ quotaRemainingPercentThreshold,
500
+ sessionAffinityStore,
501
+ lastObservedAffinityGeneration,
502
+ });
503
+ const server = createServer((req, res) => {
504
+ void handleRequest(state, req, res);
505
+ });
506
+ const sockets = new Set();
507
+ server.on("connection", (socket) => {
508
+ sockets.add(socket);
509
+ socket.once("close", () => {
510
+ sockets.delete(socket);
1238
511
  });
1239
- return staleRuntimeReloadPromise;
512
+ });
513
+ const onPostStartupServerError = (error) => {
514
+ state.status.lastError = error.message;
1240
515
  };
1241
- const handleRequest = async (req, res) => {
1242
- // Per-request trace id (errors-logging-03): distinct from sessionKey, which
1243
- // is shared across a thread's requests. Bound to this request's async context
1244
- // so every proxyLog line and usage row can be correlated to one request.
1245
- const traceId = randomUUID();
1246
- return runWithCorrelationId(traceId, () => handleRequestInner(req, res, traceId));
516
+ await new Promise((resolve, reject) => {
517
+ const onError = (error) => {
518
+ server.off("listening", onListening);
519
+ reject(error);
520
+ };
521
+ const onListening = () => {
522
+ server.off("error", onError);
523
+ resolve();
524
+ };
525
+ server.once("error", onError);
526
+ server.once("listening", onListening);
527
+ server.listen(port, bindHost);
528
+ });
529
+ server.on("error", onPostStartupServerError);
530
+ const address = server.address();
531
+ const resolvedPort = typeof address === "object" && address ? address.port : port;
532
+ return {
533
+ host: bindHost,
534
+ port: resolvedPort,
535
+ baseUrl: `http://${urlHost}:${resolvedPort}`,
536
+ close: async () => {
537
+ await closeServer(server, sockets);
538
+ await state.activeAccountManager.flushPendingSave();
539
+ },
540
+ getStatus: () => ({
541
+ ...state.status,
542
+ // Redact any email/token material that leaked into a raw upstream or
543
+ // refresh error string before exposing it to status/report consumers
544
+ // (errors-logging-08). maskString is a no-op for clean diagnostic text.
545
+ lastError: state.status.lastError === null ? null : maskString(state.status.lastError),
546
+ }),
1247
547
  };
1248
- const handleRequestInner = async (req, res, traceId) => {
1249
- let usageRecorder = null;
1250
- let accountManager = activeAccountManager;
548
+ }
549
+ async function handleRequest(state, req, res) {
550
+ // Per-request trace id (errors-logging-03): distinct from sessionKey, which
551
+ // is shared across a thread's requests. Bound to this request's async context
552
+ // so every proxyLog line and usage row can be correlated to one request.
553
+ const traceId = randomUUID();
554
+ return runWithCorrelationId(traceId, () => handleRequestInner(state, req, res, traceId));
555
+ }
556
+ async function handleRequestInner(state, req, res, traceId) {
557
+ let usageRecorder = null;
558
+ let accountManager = state.activeAccountManager;
559
+ try {
560
+ const incomingUrl = new URL(req.url ?? "/", "http://127.0.0.1");
561
+ // Authenticate before discriminating path/method so an unauthenticated
562
+ // caller cannot enumerate which endpoints exist: an unknown caller always
563
+ // gets 401, never a 404 that would confirm a path is invalid (vs. just
564
+ // unauthorized). Authorized callers still fall through to the 404 below
565
+ // when they hit an unsupported path/method.
566
+ const incomingHeaders = headersFromIncoming(req);
567
+ if (!isAuthorizedClient(incomingHeaders, state.clientApiKey)) {
568
+ writeUnauthorized(res);
569
+ return;
570
+ }
571
+ const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
572
+ const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
573
+ const isThreadGoalRequest = (req.method === "GET" || req.method === "POST") &&
574
+ isThreadGoalPath(incomingUrl.pathname);
575
+ if (!isResponsesRequest && !isModelsRequest && !isThreadGoalRequest) {
576
+ writeMethodOrPathError(res);
577
+ return;
578
+ }
579
+ state.status.totalRequests += 1;
580
+ const requestBody = isResponsesRequest || (isThreadGoalRequest && req.method === "POST")
581
+ ? await readRequestBody(req, state.maxRequestBodyBytes)
582
+ : Buffer.alloc(0);
583
+ const context = isModelsRequest
584
+ ? buildModelsRequestContext(req)
585
+ : isThreadGoalRequest
586
+ ? buildThreadGoalRequestContext(req, requestBody, incomingUrl.pathname)
587
+ : buildResponsesRequestContext(req, requestBody);
588
+ const requestStartedAt = state.now();
589
+ let policyDecision = null;
590
+ let projectKey = null;
591
+ let policyError = null;
1251
592
  try {
1252
- const incomingUrl = new URL(req.url ?? "/", "http://127.0.0.1");
1253
- // Authenticate before discriminating path/method so an unauthenticated
1254
- // caller cannot enumerate which endpoints exist: an unknown caller always
1255
- // gets 401, never a 404 that would confirm a path is invalid (vs. just
1256
- // unauthorized). Authorized callers still fall through to the 404 below
1257
- // when they hit an unsupported path/method.
1258
- const incomingHeaders = headersFromIncoming(req);
1259
- if (!isAuthorizedClient(incomingHeaders, clientApiKey)) {
1260
- writeUnauthorized(res);
1261
- return;
1262
- }
1263
- const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
1264
- const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
1265
- const isThreadGoalRequest = (req.method === "GET" || req.method === "POST") &&
1266
- isThreadGoalPath(incomingUrl.pathname);
1267
- if (!isResponsesRequest && !isModelsRequest && !isThreadGoalRequest) {
1268
- writeMethodOrPathError(res);
1269
- return;
1270
- }
1271
- status.totalRequests += 1;
1272
- const requestBody = isResponsesRequest || (isThreadGoalRequest && req.method === "POST")
1273
- ? await readRequestBody(req, maxRequestBodyBytes)
1274
- : Buffer.alloc(0);
1275
- const context = isModelsRequest
1276
- ? buildModelsRequestContext(req)
593
+ const policyState = await loadRuntimePolicyState();
594
+ projectKey = policyState.project.projectKey;
595
+ policyDecision = await evaluateRuntimePolicy({
596
+ state: policyState,
597
+ accounts: accountManager.getAccountsSnapshot(),
598
+ model: context.model,
599
+ now: requestStartedAt,
600
+ });
601
+ mutateRuntimeObservabilitySnapshot((snapshot) => {
602
+ snapshot.policyBlockedIndexes = [
603
+ ...(policyDecision?.blockedAccountIndexes ?? new Set()),
604
+ ];
605
+ snapshot.policyBlockedReasons = Object.fromEntries([...(policyDecision?.blockedAccountIndexes ?? new Set())].map((index) => [String(index), "policy-blocked"]));
606
+ });
607
+ }
608
+ catch (error) {
609
+ policyError = error instanceof Error ? error.message : String(error);
610
+ state.status.lastError = policyError;
611
+ }
612
+ usageRecorder = createRuntimeUsageRecorder({
613
+ source: "runtime-proxy",
614
+ operation: isModelsRequest
615
+ ? "models"
1277
616
  : isThreadGoalRequest
1278
- ? buildThreadGoalRequestContext(req, requestBody, incomingUrl.pathname)
1279
- : buildResponsesRequestContext(req, requestBody);
1280
- const requestStartedAt = now();
1281
- let policyDecision = null;
1282
- let projectKey = null;
1283
- let policyError = null;
1284
- try {
1285
- const policyState = await loadRuntimePolicyState();
1286
- projectKey = policyState.project.projectKey;
1287
- policyDecision = await evaluateRuntimePolicy({
1288
- state: policyState,
1289
- accounts: accountManager.getAccountsSnapshot(),
1290
- model: context.model,
1291
- now: requestStartedAt,
1292
- });
1293
- mutateRuntimeObservabilitySnapshot((snapshot) => {
1294
- snapshot.policyBlockedIndexes = [
1295
- ...(policyDecision?.blockedAccountIndexes ?? new Set()),
1296
- ];
1297
- snapshot.policyBlockedReasons = Object.fromEntries([...(policyDecision?.blockedAccountIndexes ?? new Set())].map((index) => [String(index), "policy-blocked"]));
1298
- });
1299
- }
1300
- catch (error) {
1301
- policyError = error instanceof Error ? error.message : String(error);
1302
- status.lastError = policyError;
1303
- }
1304
- usageRecorder = createRuntimeUsageRecorder({
1305
- source: "runtime-proxy",
1306
- operation: isModelsRequest
1307
- ? "models"
1308
- : isThreadGoalRequest
1309
- ? "thread-goal"
1310
- : "responses",
617
+ ? "thread-goal"
618
+ : "responses",
619
+ model: context.model,
620
+ projectKey,
621
+ requestId: traceId,
622
+ startedAt: requestStartedAt,
623
+ });
624
+ if (policyError) {
625
+ await usageRecorder.record({
626
+ outcome: "failure",
627
+ statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
628
+ errorCode: "runtime_policy_unavailable",
629
+ });
630
+ writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
631
+ error: {
632
+ message: "Runtime policy could not be loaded for this local request.",
633
+ code: "runtime_policy_unavailable",
634
+ },
635
+ });
636
+ return;
637
+ }
638
+ if (policyDecision && !policyDecision.allowed) {
639
+ await usageRecorder.record({
640
+ outcome: "blocked",
641
+ statusCode: policyDecision.statusCode,
642
+ errorCode: policyDecision.errorCode,
643
+ });
644
+ writeJson(res, policyDecision.statusCode, {
645
+ error: {
646
+ message: "Runtime policy blocked this local request.",
647
+ code: policyDecision.errorCode ?? "policy_blocked",
648
+ reasons: policyDecision.reasons,
649
+ },
650
+ });
651
+ return;
652
+ }
653
+ const upstreamUrl = buildUpstreamUrl(req, state.upstreamBaseUrl, context.upstreamPath);
654
+ const attemptedIndexes = new Set();
655
+ let exhaustionReason = "no-account";
656
+ let accountCount = accountManager.getAccountCount();
657
+ let transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
658
+ let transientAttempts = 0;
659
+ let transientExhaustionReason = null;
660
+ const accountSkipReasons = new Map();
661
+ let reloadedAfterNoAccount = false;
662
+ // Read the manual pin and affinity generation from disk (mtime-cached)
663
+ // on each request so a `codex-multi-auth switch|unpin|best` invocation
664
+ // in another process is honored without forcing a full AccountManager
665
+ // reload. The CLI bumps `affinityGeneration` on user-initiated changes
666
+ // so the proxy can invalidate sticky session affinity that would
667
+ // otherwise glue an in-flight chat thread to the previously selected
668
+ // account. The proxy itself never bumps the generation, so its own
669
+ // debounced disk writes do not clear affinity. See issue #474.
670
+ const storageMeta = readStorageMetaFromDisk();
671
+ const pinnedIndex = storageMeta.pinnedAccountIndex;
672
+ const isPinned = typeof pinnedIndex === "number";
673
+ if (storageMeta.affinityGeneration > state.lastObservedAffinityGeneration) {
674
+ state.sessionAffinityStore?.clearAll();
675
+ state.lastObservedAffinityGeneration = storageMeta.affinityGeneration;
676
+ }
677
+ while (attemptedIndexes.size < accountCount &&
678
+ transientAttempts < transientAttemptLimit) {
679
+ const rotationStickyBoost = state.minRotationIntervalMs > 0 &&
680
+ state.lastGlobalAccountIndex !== null &&
681
+ state.now() - state.lastGlobalSwitchAt < state.minRotationIntervalMs
682
+ ? { [state.lastGlobalAccountIndex]: 1000 }
683
+ : {};
684
+ // L4 fix (routing mutex): when `routingMutex === "enabled"`, run the
685
+ // selection AND the cursor commit inside ONE mutex acquisition so two
686
+ // concurrent requests cannot read the same cursor and stampede before
687
+ // the locked commit lands. `chooseAccount` is sync and mutates the
688
+ // cursor internally (session-affinity `markSwitched`, the hybrid
689
+ // selector's own advance, and the round-robin fallback `markSwitched`);
690
+ // holding the mutex across the whole call serializes all of those.
691
+ // We then `markSwitchedLocked` the winner to (a) re-commit the cursor
692
+ // under the lock across the await boundary and (b) hand
693
+ // `persistRuntimeActiveAccount` a cursor that is already correct. That
694
+ // later `markSwitchedLocked` runs INLINE (reentrant) within this held
695
+ // section, so there is no double-acquire and no deadlock on the
696
+ // non-reentrant FIFO queue. In legacy mode the inline `markSwitched`
697
+ // calls inside `chooseAccount` are used unchanged and no lock is taken,
698
+ // so default behavior and perf are identical to before.
699
+ const selectAccount = () => chooseAccount({
700
+ accountManager,
701
+ sessionAffinityStore: state.sessionAffinityStore,
702
+ sessionKey: context.sessionKey,
703
+ family: context.family,
1311
704
  model: context.model,
1312
- projectKey,
1313
- requestId: traceId,
1314
- startedAt: requestStartedAt,
705
+ attemptedIndexes,
706
+ now: state.now(),
707
+ policy: policyDecision,
708
+ pinnedIndex,
709
+ skipReasons: accountSkipReasons,
710
+ stickyBoostByAccount: rotationStickyBoost,
711
+ pidOffsetEnabled: state.pidOffsetEnabled,
712
+ schedulingStrategy: state.schedulingStrategy,
1315
713
  });
1316
- if (policyError) {
1317
- await usageRecorder.record({
1318
- outcome: "failure",
1319
- statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
1320
- errorCode: "runtime_policy_unavailable",
1321
- });
1322
- writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
1323
- error: {
1324
- message: "Runtime policy could not be loaded for this local request.",
1325
- code: "runtime_policy_unavailable",
1326
- },
1327
- });
1328
- return;
1329
- }
1330
- if (policyDecision && !policyDecision.allowed) {
1331
- await usageRecorder.record({
1332
- outcome: "blocked",
1333
- statusCode: policyDecision.statusCode,
1334
- errorCode: policyDecision.errorCode,
1335
- });
1336
- writeJson(res, policyDecision.statusCode, {
1337
- error: {
1338
- message: "Runtime policy blocked this local request.",
1339
- code: policyDecision.errorCode ?? "policy_blocked",
1340
- reasons: policyDecision.reasons,
1341
- },
1342
- });
1343
- return;
714
+ const selected = state.routingMutexMode === "enabled"
715
+ ? await withRoutingMutex(state.routingMutexMode, async () => {
716
+ const candidate = selectAccount();
717
+ if (candidate &&
718
+ pinnedIndex === null &&
719
+ state.schedulingStrategy !== "sequential") {
720
+ // Re-commit the cursor under the held mutex. Skipped when a
721
+ // manual pin is active so the proxy never clobbers the pin
722
+ // (see #474); pinned selections are deterministic and need no
723
+ // cursor advance. Also skipped in sequential mode: the
724
+ // sequential selector already committed the correct active
725
+ // index inside this held mutex, and re-committing `candidate`
726
+ // would wrongly advance the drain-first primary when the pick
727
+ // came from the non-advancing linear-scan fallback (#509).
728
+ // Runs inline via reentrancy — see comment above.
729
+ await accountManager.markSwitchedLocked(candidate, "rotation", context.family);
730
+ }
731
+ return candidate;
732
+ })
733
+ : selectAccount();
734
+ if (!selected) {
735
+ if (!reloadedAfterNoAccount &&
736
+ !isPinned &&
737
+ accountCount > 0 &&
738
+ exhaustionReason === "no-account" &&
739
+ (policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
740
+ ![...accountSkipReasons.values()].some((reason) => reason === "rate-limited" ||
741
+ reason.startsWith("cooling-down") ||
742
+ reason === "policy-blocked")) {
743
+ reloadedAfterNoAccount = true;
744
+ const reloadedManager = await recoverStaleRuntimeState(state);
745
+ if (reloadedManager) {
746
+ accountManager = reloadedManager;
747
+ accountCount = accountManager.getAccountCount();
748
+ transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
749
+ accountSkipReasons.clear();
750
+ attemptedIndexes.clear();
751
+ continue;
752
+ }
753
+ }
754
+ break;
1344
755
  }
1345
- const upstreamUrl = buildUpstreamUrl(req, upstreamBaseUrl, context.upstreamPath);
1346
- const attemptedIndexes = new Set();
1347
- let exhaustionReason = "no-account";
1348
- let accountCount = accountManager.getAccountCount();
1349
- let transientAttemptLimit = Math.max(1, Math.min(accountCount, maxRuntimeAccountAttempts));
1350
- let transientAttempts = 0;
1351
- let transientExhaustionReason = null;
1352
- const accountSkipReasons = new Map();
1353
- let reloadedAfterNoAccount = false;
1354
- // Read the manual pin and affinity generation from disk (mtime-cached)
1355
- // on each request so a `codex-multi-auth switch|unpin|best` invocation
1356
- // in another process is honored without forcing a full AccountManager
1357
- // reload. The CLI bumps `affinityGeneration` on user-initiated changes
1358
- // so the proxy can invalidate sticky session affinity that would
1359
- // otherwise glue an in-flight chat thread to the previously selected
1360
- // account. The proxy itself never bumps the generation, so its own
1361
- // debounced disk writes do not clear affinity. See issue #474.
1362
- const storageMeta = readStorageMetaFromDisk();
1363
- const pinnedIndex = storageMeta.pinnedAccountIndex;
1364
- const isPinned = typeof pinnedIndex === "number";
1365
- if (storageMeta.affinityGeneration > lastObservedAffinityGeneration) {
1366
- sessionAffinityStore?.clearAll();
1367
- lastObservedAffinityGeneration = storageMeta.affinityGeneration;
756
+ attemptedIndexes.add(selected.index);
757
+ if (!accountManager.consumeToken(selected, context.family, context.model)) {
758
+ accountSkipReasons.set(selected.index, "token-exhausted");
759
+ exhaustionReason = "rate-limit";
760
+ continue;
1368
761
  }
1369
- while (attemptedIndexes.size < accountCount &&
1370
- transientAttempts < transientAttemptLimit) {
1371
- const rotationStickyBoost = minRotationIntervalMs > 0 &&
1372
- lastGlobalAccountIndex !== null &&
1373
- now() - lastGlobalSwitchAt < minRotationIntervalMs
1374
- ? { [lastGlobalAccountIndex]: 1000 }
1375
- : {};
1376
- // L4 fix (routing mutex): when `routingMutex === "enabled"`, run the
1377
- // selection AND the cursor commit inside ONE mutex acquisition so two
1378
- // concurrent requests cannot read the same cursor and stampede before
1379
- // the locked commit lands. `chooseAccount` is sync and mutates the
1380
- // cursor internally (session-affinity `markSwitched`, the hybrid
1381
- // selector's own advance, and the round-robin fallback `markSwitched`);
1382
- // holding the mutex across the whole call serializes all of those.
1383
- // We then `markSwitchedLocked` the winner to (a) re-commit the cursor
1384
- // under the lock across the await boundary and (b) hand
1385
- // `persistRuntimeActiveAccount` a cursor that is already correct. That
1386
- // later `markSwitchedLocked` runs INLINE (reentrant) within this held
1387
- // section, so there is no double-acquire and no deadlock on the
1388
- // non-reentrant FIFO queue. In legacy mode the inline `markSwitched`
1389
- // calls inside `chooseAccount` are used unchanged and no lock is taken,
1390
- // so default behavior and perf are identical to before.
1391
- const selectAccount = () => chooseAccount({
1392
- accountManager,
1393
- sessionAffinityStore,
1394
- sessionKey: context.sessionKey,
1395
- family: context.family,
1396
- model: context.model,
1397
- attemptedIndexes,
1398
- now: now(),
1399
- policy: policyDecision,
1400
- pinnedIndex,
1401
- skipReasons: accountSkipReasons,
1402
- stickyBoostByAccount: rotationStickyBoost,
1403
- pidOffsetEnabled,
1404
- schedulingStrategy,
1405
- });
1406
- const selected = routingMutexMode === "enabled"
1407
- ? await withRoutingMutex(routingMutexMode, async () => {
1408
- const candidate = selectAccount();
1409
- if (candidate &&
1410
- pinnedIndex === null &&
1411
- schedulingStrategy !== "sequential") {
1412
- // Re-commit the cursor under the held mutex. Skipped when a
1413
- // manual pin is active so the proxy never clobbers the pin
1414
- // (see #474); pinned selections are deterministic and need no
1415
- // cursor advance. Also skipped in sequential mode: the
1416
- // sequential selector already committed the correct active
1417
- // index inside this held mutex, and re-committing `candidate`
1418
- // would wrongly advance the drain-first primary when the pick
1419
- // came from the non-advancing linear-scan fallback (#509).
1420
- // Runs inline via reentrancy — see comment above.
1421
- await accountManager.markSwitchedLocked(candidate, "rotation", context.family);
1422
- }
1423
- return candidate;
1424
- })
1425
- : selectAccount();
1426
- if (!selected) {
1427
- if (!reloadedAfterNoAccount &&
1428
- !isPinned &&
1429
- accountCount > 0 &&
1430
- exhaustionReason === "no-account" &&
1431
- (policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
1432
- ![...accountSkipReasons.values()].some((reason) => reason === "rate-limited" ||
1433
- reason.startsWith("cooling-down") ||
1434
- reason === "policy-blocked")) {
1435
- reloadedAfterNoAccount = true;
1436
- const reloadedManager = await recoverStaleRuntimeState();
1437
- if (reloadedManager) {
1438
- accountManager = reloadedManager;
1439
- accountCount = accountManager.getAccountCount();
1440
- transientAttemptLimit = Math.max(1, Math.min(accountCount, maxRuntimeAccountAttempts));
1441
- accountSkipReasons.clear();
1442
- attemptedIndexes.clear();
1443
- continue;
1444
- }
1445
- }
1446
- break;
762
+ const refreshed = await ensureFreshAccessToken({
763
+ accountManager,
764
+ account: selected,
765
+ family: context.family,
766
+ model: context.model,
767
+ now: state.now(),
768
+ tokenRefreshSkewMs: state.tokenRefreshSkewMs,
769
+ tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs,
770
+ });
771
+ if (!refreshed.ok) {
772
+ accountManager.refundToken(selected, context.family, context.model);
773
+ exhaustionReason = "auth-failure";
774
+ if (refreshed.invalidated) {
775
+ // Refresh endpoint explicitly revoked the token. Stop cascade:
776
+ // return auth error to client instead of rotating to the next account.
777
+ state.sessionAffinityStore?.forgetSession(context.sessionKey);
778
+ res.writeHead(HTTP_STATUS.UNAUTHORIZED, { "content-type": "application/json" });
779
+ // Route through the shared builder so both invalidation exit paths stay
780
+ // in lockstep — empty input yields { error: { message: <fallback>,
781
+ // code: "token_invalidated" } }.
782
+ res.end(buildTokenInvalidationBody(""));
783
+ await usageRecorder.record({
784
+ outcome: "failure",
785
+ statusCode: HTTP_STATUS.UNAUTHORIZED,
786
+ errorCode: "token_invalidated",
787
+ account: selected,
788
+ });
789
+ return;
1447
790
  }
1448
- attemptedIndexes.add(selected.index);
1449
- if (!accountManager.consumeToken(selected, context.family, context.model)) {
1450
- accountSkipReasons.set(selected.index, "token-exhausted");
1451
- exhaustionReason = "rate-limit";
1452
- continue;
1453
- }
1454
- const refreshed = await ensureFreshAccessToken({
1455
- accountManager,
1456
- account: selected,
1457
- family: context.family,
1458
- model: context.model,
1459
- now: now(),
1460
- tokenRefreshSkewMs,
1461
- tokenInvalidationCooldownMs,
1462
- });
1463
- if (!refreshed.ok) {
1464
- accountManager.refundToken(selected, context.family, context.model);
1465
- exhaustionReason = "auth-failure";
1466
- if (refreshed.invalidated) {
1467
- // Refresh endpoint explicitly revoked the token. Stop cascade:
1468
- // return auth error to client instead of rotating to the next account.
1469
- sessionAffinityStore?.forgetSession(context.sessionKey);
1470
- res.writeHead(HTTP_STATUS.UNAUTHORIZED, { "content-type": "application/json" });
1471
- // Route through the shared builder so both invalidation exit paths stay
1472
- // in lockstep — empty input yields { error: { message: <fallback>,
1473
- // code: "token_invalidated" } }.
1474
- res.end(buildTokenInvalidationBody(""));
1475
- await usageRecorder.record({
1476
- outcome: "failure",
1477
- statusCode: HTTP_STATUS.UNAUTHORIZED,
1478
- errorCode: "token_invalidated",
1479
- account: selected,
1480
- });
1481
- return;
1482
- }
1483
- if (!refreshed.retryable)
1484
- continue;
1485
- transientAttempts += 1;
1486
- transientExhaustionReason = "auth-failure";
1487
- status.retries += 1;
1488
- status.rotations += 1;
791
+ if (!refreshed.retryable)
1489
792
  continue;
793
+ transientAttempts += 1;
794
+ transientExhaustionReason = "auth-failure";
795
+ state.status.retries += 1;
796
+ state.status.rotations += 1;
797
+ continue;
798
+ }
799
+ const accountId = resolveAccountId(refreshed.account, refreshed.accessToken);
800
+ if (!accountId) {
801
+ accountManager.refundToken(refreshed.account, context.family, context.model);
802
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
803
+ accountManager.markAccountCoolingDown(refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
804
+ exhaustionReason = "auth-failure";
805
+ transientAttempts += 1;
806
+ transientExhaustionReason = "auth-failure";
807
+ state.status.retries += 1;
808
+ state.status.rotations += 1;
809
+ continue;
810
+ }
811
+ const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
812
+ recordLastRuntimeAccount(state.status, accountIdentity);
813
+ const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
814
+ let upstream;
815
+ try {
816
+ state.status.upstreamRequests += 1;
817
+ const fetchAbortController = new AbortController();
818
+ const upstreamRequestInit = {
819
+ method: context.method,
820
+ headers: outboundHeaders,
821
+ signal: fetchAbortController.signal,
822
+ };
823
+ if (context.method === "POST") {
824
+ upstreamRequestInit.body = context.body;
1490
825
  }
1491
- const accountId = resolveAccountId(refreshed.account, refreshed.accessToken);
1492
- if (!accountId) {
826
+ upstream = await withTimeout(state.fetchImpl(upstreamUrl, upstreamRequestInit), state.fetchTimeoutMs, () => fetchAbortController.abort(), `upstream fetch timed out after ${state.fetchTimeoutMs}ms`);
827
+ }
828
+ catch (error) {
829
+ state.status.lastError = error instanceof Error ? error.message : String(error);
830
+ accountManager.refundToken(refreshed.account, context.family, context.model);
831
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
832
+ accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
833
+ accountManager.saveToDiskDebounced();
834
+ exhaustionReason = "network-error";
835
+ transientAttempts += 1;
836
+ transientExhaustionReason = "network-error";
837
+ state.status.retries += 1;
838
+ state.status.rotations += 1;
839
+ continue;
840
+ }
841
+ if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
842
+ const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
843
+ const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, state.now()) ??
844
+ parseRetryAfterBodyMs(bodyText, state.now()) ??
845
+ 60_000;
846
+ // A 429 is the upstream quota signal for the attempted account, so
847
+ // keep the consumed runtime token drained.
848
+ accountManager.recordRateLimit(refreshed.account, context.family, context.model);
849
+ accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
850
+ accountManager.saveToDiskDebounced();
851
+ exhaustionReason = "rate-limit";
852
+ transientAttempts += 1;
853
+ transientExhaustionReason = "rate-limit";
854
+ state.status.retries += 1;
855
+ state.status.rotations += 1;
856
+ continue;
857
+ }
858
+ if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
859
+ const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
860
+ const errorCode = extractErrorCodeFromBody(bodyText);
861
+ if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
862
+ const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
863
+ false;
1493
864
  accountManager.refundToken(refreshed.account, context.family, context.model);
1494
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1495
- accountManager.markAccountCoolingDown(refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
1496
- exhaustionReason = "auth-failure";
1497
- transientAttempts += 1;
1498
- transientExhaustionReason = "auth-failure";
1499
- status.retries += 1;
1500
- status.rotations += 1;
1501
- continue;
1502
- }
1503
- const accountIdentity = accountIdentityFromAccount(refreshed.account, now());
1504
- recordLastRuntimeAccount(status, accountIdentity);
1505
- const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
1506
- let upstream;
1507
- try {
1508
- status.upstreamRequests += 1;
1509
- const fetchAbortController = new AbortController();
1510
- const upstreamRequestInit = {
1511
- method: context.method,
1512
- headers: outboundHeaders,
1513
- signal: fetchAbortController.signal,
1514
- };
1515
- if (context.method === "POST") {
1516
- upstreamRequestInit.body = context.body;
865
+ if (accountWasEnabled) {
866
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
867
+ accountManager.setAccountEnabled(refreshed.account.index, false);
868
+ accountManager.saveToDiskDebounced();
1517
869
  }
1518
- upstream = await withTimeout(fetchImpl(upstreamUrl, upstreamRequestInit), fetchTimeoutMs, () => fetchAbortController.abort(), `upstream fetch timed out after ${fetchTimeoutMs}ms`);
1519
- }
1520
- catch (error) {
1521
- status.lastError = error instanceof Error ? error.message : String(error);
1522
- accountManager.refundToken(refreshed.account, context.family, context.model);
1523
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1524
- accountManager.markAccountCoolingDown(refreshed.account, networkErrorCooldownMs, "network-error");
1525
- accountManager.saveToDiskDebounced();
1526
- exhaustionReason = "network-error";
1527
- transientAttempts += 1;
1528
- transientExhaustionReason = "network-error";
1529
- status.retries += 1;
1530
- status.rotations += 1;
870
+ state.sessionAffinityStore?.forgetSession(context.sessionKey);
871
+ exhaustionReason = "deactivated";
872
+ state.status.retries += 1;
873
+ state.status.rotations += 1;
1531
874
  continue;
1532
875
  }
1533
- if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
1534
- const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
1535
- const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, now()) ??
1536
- parseRetryAfterBodyMs(bodyText, now()) ??
1537
- 60_000;
1538
- // A 429 is the upstream quota signal for the attempted account, so
1539
- // keep the consumed runtime token drained.
1540
- accountManager.recordRateLimit(refreshed.account, context.family, context.model);
1541
- accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
1542
- accountManager.saveToDiskDebounced();
1543
- exhaustionReason = "rate-limit";
1544
- transientAttempts += 1;
1545
- transientExhaustionReason = "rate-limit";
1546
- status.retries += 1;
1547
- status.rotations += 1;
1548
- continue;
1549
- }
1550
- if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
1551
- const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
1552
- const errorCode = extractErrorCodeFromBody(bodyText);
1553
- if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
1554
- const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
1555
- false;
1556
- accountManager.refundToken(refreshed.account, context.family, context.model);
1557
- if (accountWasEnabled) {
1558
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1559
- accountManager.setAccountEnabled(refreshed.account.index, false);
1560
- accountManager.saveToDiskDebounced();
1561
- }
1562
- sessionAffinityStore?.forgetSession(context.sessionKey);
1563
- exhaustionReason = "deactivated";
1564
- status.retries += 1;
1565
- status.rotations += 1;
1566
- continue;
1567
- }
1568
- if (isThreadGoalRequest && isThreadGoalFallbackStatus(upstream.status)) {
1569
- const parsedGoalBody = parseRequestBody(context.body);
1570
- const fallbackKey = context.sessionKey;
1571
- const goal = typeof parsedGoalBody?.goal === "string" ? parsedGoalBody.goal : null;
1572
- if (!fallbackKey) {
1573
- if (context.upstreamPath.endsWith("/get")) {
1574
- writeJson(res, HTTP_STATUS.OK, { goal: null });
1575
- await usageRecorder.record({
1576
- outcome: "failure",
1577
- statusCode: upstream.status,
1578
- errorCode: "thread_goal_session_key_required",
1579
- account: refreshed.account,
1580
- });
1581
- return;
1582
- }
876
+ if (isThreadGoalRequest && isThreadGoalFallbackStatus(upstream.status)) {
877
+ const parsedGoalBody = parseRequestBody(context.body);
878
+ const fallbackKey = context.sessionKey;
879
+ const goal = typeof parsedGoalBody?.goal === "string" ? parsedGoalBody.goal : null;
880
+ if (!fallbackKey) {
881
+ if (context.upstreamPath.endsWith("/get")) {
882
+ writeJson(res, HTTP_STATUS.OK, { goal: null });
1583
883
  await usageRecorder.record({
1584
884
  outcome: "failure",
1585
- statusCode: HTTP_STATUS.BAD_REQUEST,
885
+ statusCode: upstream.status,
1586
886
  errorCode: "thread_goal_session_key_required",
1587
887
  account: refreshed.account,
1588
888
  });
1589
- writeJson(res, HTTP_STATUS.BAD_REQUEST, {
1590
- error: {
1591
- message: "Thread goal fallback requires a thread_id, threadId, or session header.",
1592
- code: "thread_goal_session_key_required",
1593
- },
1594
- });
1595
889
  return;
1596
890
  }
1597
891
  await usageRecorder.record({
1598
892
  outcome: "failure",
1599
- statusCode: upstream.status,
1600
- errorCode: "thread_goal_upstream_blocked",
893
+ statusCode: HTTP_STATUS.BAD_REQUEST,
894
+ errorCode: "thread_goal_session_key_required",
1601
895
  account: refreshed.account,
1602
896
  });
1603
- if (context.upstreamPath.endsWith("/set")) {
1604
- setThreadGoalFallback(threadGoalFallbacks, fallbackKey, goal);
1605
- writeJson(res, HTTP_STATUS.OK, { ok: true, goal });
1606
- return;
1607
- }
1608
- writeJson(res, HTTP_STATUS.OK, {
1609
- goal: getThreadGoalFallback(threadGoalFallbacks, fallbackKey),
1610
- });
1611
- return;
1612
- }
1613
- if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
1614
- writeJson(res, HTTP_STATUS.OK, { goal: null });
1615
- await usageRecorder.record({
1616
- outcome: "failure",
1617
- statusCode: upstream.status,
1618
- errorCode,
1619
- account: refreshed.account,
897
+ writeJson(res, HTTP_STATUS.BAD_REQUEST, {
898
+ error: {
899
+ message: "Thread goal fallback requires a thread_id, threadId, or session header.",
900
+ code: "thread_goal_session_key_required",
901
+ },
1620
902
  });
1621
903
  return;
1622
904
  }
1623
- res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
1624
- res.end(bodyText);
1625
905
  await usageRecorder.record({
1626
906
  outcome: "failure",
1627
907
  statusCode: upstream.status,
1628
- errorCode,
908
+ errorCode: "thread_goal_upstream_blocked",
1629
909
  account: refreshed.account,
1630
910
  });
1631
- return;
1632
- }
1633
- if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
1634
- const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
1635
- accountManager.refundToken(refreshed.account, context.family, context.model);
1636
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1637
- if (isTokenInvalidationError(bodyText)) {
1638
- // The upstream explicitly revoked this OAuth token. Applying a long
1639
- // cooldown prevents cascade invalidation: rapidly presenting each
1640
- // account's token from the same IP triggers OpenAI's anti-abuse
1641
- // detection and invalidates them in sequence. Return the 401 directly
1642
- // rather than rotating so the client can prompt for re-login.
1643
- applyMonotonicAuthCooldown(accountManager, refreshed.account, tokenInvalidationCooldownMs);
1644
- sessionAffinityStore?.forgetSession(context.sessionKey);
1645
- accountManager.saveToDiskDebounced();
1646
- // Emit the same machine-readable shape as the refresh-failure path
1647
- // (code: "token_invalidated") instead of forwarding the raw upstream
1648
- // body, so the client contract is consistent across both vectors.
1649
- const clientHeaders = responseHeadersForClient(upstream.headers);
1650
- clientHeaders["content-type"] = "application/json";
1651
- res.writeHead(upstream.status, clientHeaders);
1652
- res.end(buildTokenInvalidationBody(bodyText));
1653
- await usageRecorder.record({
1654
- outcome: "failure",
1655
- statusCode: upstream.status,
1656
- errorCode: "token_invalidated",
1657
- account: refreshed.account,
1658
- });
911
+ if (context.upstreamPath.endsWith("/set")) {
912
+ setThreadGoalFallback(state.threadGoalFallbacks, fallbackKey, goal);
913
+ writeJson(res, HTTP_STATUS.OK, { ok: true, goal });
1659
914
  return;
1660
915
  }
1661
- applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
1662
- accountManager.saveToDiskDebounced();
1663
- exhaustionReason = "auth-failure";
1664
- transientAttempts += 1;
1665
- transientExhaustionReason = "auth-failure";
1666
- status.retries += 1;
1667
- status.rotations += 1;
1668
- continue;
1669
- }
1670
- if (upstream.status >= 500) {
1671
- await readErrorBody(upstream, streamStallTimeoutMs);
1672
- accountManager.refundToken(refreshed.account, context.family, context.model);
1673
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1674
- accountManager.markAccountCoolingDown(refreshed.account, serverErrorCooldownMs, "server-error");
1675
- accountManager.saveToDiskDebounced();
1676
- exhaustionReason = "server-error";
1677
- transientAttempts += 1;
1678
- transientExhaustionReason = "server-error";
1679
- status.retries += 1;
1680
- status.rotations += 1;
1681
- continue;
916
+ writeJson(res, HTTP_STATUS.OK, {
917
+ goal: getThreadGoalFallback(state.threadGoalFallbacks, fallbackKey),
918
+ });
919
+ return;
1682
920
  }
1683
- if (isThreadGoalRequest && upstream.status >= 400) {
1684
- if (context.upstreamPath.endsWith("/get")) {
1685
- writeJson(res, HTTP_STATUS.OK, { goal: null });
1686
- await usageRecorder.record({
1687
- outcome: "failure",
1688
- statusCode: upstream.status,
1689
- errorCode: "thread_goal_upstream_error",
1690
- account: refreshed.account,
1691
- });
1692
- return;
1693
- }
1694
- const forwarded = await forwardStreamingResponse(upstream, res, status, () => undefined, streamStallTimeoutMs);
921
+ if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
922
+ writeJson(res, HTTP_STATUS.OK, { goal: null });
1695
923
  await usageRecorder.record({
1696
924
  outcome: "failure",
1697
925
  statusCode: upstream.status,
1698
- errorCode: forwarded ? "thread_goal_upstream_error" : "stream_forward_failed",
926
+ errorCode,
1699
927
  account: refreshed.account,
1700
928
  });
1701
929
  return;
1702
930
  }
1703
- accountManager.recordSuccess(refreshed.account, context.family, context.model);
1704
- // A successful request proves the account is usable, so clear any
1705
- // stale runtime skip reason persisted for it on a prior pool
1706
- // exhaustion. Without this the overlay reason (e.g. "token-exhausted",
1707
- // "rate-limited") lingers on disk until an explicit runtime reset and
1708
- // the forecast keeps reporting this working account as unavailable.
1709
- // No-op when no reason is recorded, so the hot path stays write-free.
1710
- recordRuntimeAccountRecovery(refreshed.account.index);
1711
- const nearExhaustionWaitMs = getQuotaNearExhaustionWaitMs(upstream.headers, quotaRemainingPercentThreshold, now());
1712
- if (nearExhaustionWaitMs > 0) {
1713
- accountManager.markRateLimitedWithReason(refreshed.account, nearExhaustionWaitMs, context.family, "quota", context.model);
1714
- sessionAffinityStore?.forgetSession(context.sessionKey);
1715
- accountManager.saveToDiskDebounced();
1716
- }
1717
- else {
1718
- sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, now());
1719
- if (refreshed.account.index !== lastGlobalAccountIndex) {
1720
- lastGlobalAccountIndex = refreshed.account.index;
1721
- }
1722
- lastGlobalSwitchAt = now();
1723
- }
1724
- await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex);
1725
- const forwarded = await forwardStreamingResponse(upstream, res, status, () => {
1726
- accountManager.recordFailure(refreshed.account, context.family, context.model);
1727
- accountManager.markAccountCoolingDown(refreshed.account, networkErrorCooldownMs, "network-error");
1728
- sessionAffinityStore?.forgetSession(context.sessionKey);
1729
- accountManager.saveToDiskDebounced();
1730
- }, streamStallTimeoutMs);
931
+ res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
932
+ res.end(bodyText);
1731
933
  await usageRecorder.record({
1732
- outcome: forwarded ? "success" : "failure",
934
+ outcome: "failure",
1733
935
  statusCode: upstream.status,
1734
- errorCode: forwarded ? null : "stream_forward_failed",
936
+ errorCode,
1735
937
  account: refreshed.account,
1736
938
  });
1737
939
  return;
1738
940
  }
1739
- if (transientAttempts >= transientAttemptLimit &&
1740
- attemptedIndexes.size < accountCount) {
1741
- exhaustionReason = "budget";
941
+ if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
942
+ const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
943
+ accountManager.refundToken(refreshed.account, context.family, context.model);
944
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
945
+ if (isTokenInvalidationError(bodyText)) {
946
+ // The upstream explicitly revoked this OAuth token. Applying a long
947
+ // cooldown prevents cascade invalidation: rapidly presenting each
948
+ // account's token from the same IP triggers OpenAI's anti-abuse
949
+ // detection and invalidates them in sequence. Return the 401 directly
950
+ // rather than rotating so the client can prompt for re-login.
951
+ applyMonotonicAuthCooldown(accountManager, refreshed.account, state.tokenInvalidationCooldownMs);
952
+ state.sessionAffinityStore?.forgetSession(context.sessionKey);
953
+ accountManager.saveToDiskDebounced();
954
+ // Emit the same machine-readable shape as the refresh-failure path
955
+ // (code: "token_invalidated") instead of forwarding the raw upstream
956
+ // body, so the client contract is consistent across both vectors.
957
+ const clientHeaders = responseHeadersForClient(upstream.headers);
958
+ clientHeaders["content-type"] = "application/json";
959
+ res.writeHead(upstream.status, clientHeaders);
960
+ res.end(buildTokenInvalidationBody(bodyText));
961
+ await usageRecorder.record({
962
+ outcome: "failure",
963
+ statusCode: upstream.status,
964
+ errorCode: "token_invalidated",
965
+ account: refreshed.account,
966
+ });
967
+ return;
968
+ }
969
+ applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
970
+ accountManager.saveToDiskDebounced();
971
+ exhaustionReason = "auth-failure";
972
+ transientAttempts += 1;
973
+ transientExhaustionReason = "auth-failure";
974
+ state.status.retries += 1;
975
+ state.status.rotations += 1;
976
+ continue;
1742
977
  }
1743
- else if (exhaustionReason === "deactivated" &&
1744
- transientExhaustionReason) {
1745
- exhaustionReason = transientExhaustionReason;
978
+ if (upstream.status >= 500) {
979
+ await readErrorBody(upstream, state.streamStallTimeoutMs);
980
+ accountManager.refundToken(refreshed.account, context.family, context.model);
981
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
982
+ accountManager.markAccountCoolingDown(refreshed.account, state.serverErrorCooldownMs, "server-error");
983
+ accountManager.saveToDiskDebounced();
984
+ exhaustionReason = "server-error";
985
+ transientAttempts += 1;
986
+ transientExhaustionReason = "server-error";
987
+ state.status.retries += 1;
988
+ state.status.rotations += 1;
989
+ continue;
1746
990
  }
1747
- // When a manual pin is set and the pinned account is unavailable, do
1748
- // NOT silently fall through to rotation. Hard-fail with a 503 so the
1749
- // user is informed the pin cannot be honored. See issue #474.
1750
- //
1751
- // Surface the runtime skip reason in both the human-readable message
1752
- // and a structured `reason` field, mirroring `writePoolExhausted`. A
1753
- // null reason indicates a forecast/runtime state desync (the pinned
1754
- // account was selected but no skip reason was recorded) — see #486.
1755
- if (isPinned) {
1756
- const errorBody = buildPinnedUnavailableErrorBody(pinnedIndex, accountSkipReasons);
1757
- if (errorBody.reason === null) {
1758
- status.lastError = `pinned-503 missing skip reason (pinnedIndex=${pinnedIndex})`;
991
+ if (isThreadGoalRequest && upstream.status >= 400) {
992
+ if (context.upstreamPath.endsWith("/get")) {
993
+ writeJson(res, HTTP_STATUS.OK, { goal: null });
994
+ await usageRecorder.record({
995
+ outcome: "failure",
996
+ statusCode: upstream.status,
997
+ errorCode: "thread_goal_upstream_error",
998
+ account: refreshed.account,
999
+ });
1000
+ return;
1759
1001
  }
1760
- await usageRecorder?.record({
1002
+ const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => undefined, state.streamStallTimeoutMs);
1003
+ await usageRecorder.record({
1761
1004
  outcome: "failure",
1762
- statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
1763
- errorCode: "codex_pinned_account_unavailable",
1005
+ statusCode: upstream.status,
1006
+ errorCode: forwarded ? "thread_goal_upstream_error" : "stream_forward_failed",
1007
+ account: refreshed.account,
1764
1008
  });
1765
- writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
1766
1009
  return;
1767
1010
  }
1768
- await usageRecorder?.record({
1769
- outcome: "failure",
1770
- statusCode: normalizeExhaustionStatus(exhaustionReason),
1771
- errorCode: isThreadGoalRequest && context.upstreamPath.endsWith("/get") ? "thread_goal_pool_exhausted" : exhaustionReason,
1772
- });
1773
- if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
1774
- writeJson(res, HTTP_STATUS.OK, { goal: null });
1011
+ accountManager.recordSuccess(refreshed.account, context.family, context.model);
1012
+ // A successful request proves the account is usable, so clear any
1013
+ // stale runtime skip reason persisted for it on a prior pool
1014
+ // exhaustion. Without this the overlay reason (e.g. "token-exhausted",
1015
+ // "rate-limited") lingers on disk until an explicit runtime reset and
1016
+ // the forecast keeps reporting this working account as unavailable.
1017
+ // No-op when no reason is recorded, so the hot path stays write-free.
1018
+ recordRuntimeAccountRecovery(refreshed.account.index);
1019
+ const nearExhaustionWaitMs = getQuotaNearExhaustionWaitMs(upstream.headers, state.quotaRemainingPercentThreshold, state.now());
1020
+ if (nearExhaustionWaitMs > 0) {
1021
+ accountManager.markRateLimitedWithReason(refreshed.account, nearExhaustionWaitMs, context.family, "quota", context.model);
1022
+ state.sessionAffinityStore?.forgetSession(context.sessionKey);
1023
+ accountManager.saveToDiskDebounced();
1775
1024
  }
1776
1025
  else {
1777
- writePoolExhausted({
1778
- res,
1779
- accountManager,
1780
- family: context.family,
1781
- model: context.model,
1782
- reason: exhaustionReason,
1783
- accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
1784
- String(index),
1785
- reason,
1786
- ])),
1787
- });
1026
+ state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
1027
+ if (refreshed.account.index !== state.lastGlobalAccountIndex) {
1028
+ state.lastGlobalAccountIndex = refreshed.account.index;
1029
+ }
1030
+ state.lastGlobalSwitchAt = state.now();
1788
1031
  }
1032
+ await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy);
1033
+ const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
1034
+ accountManager.recordFailure(refreshed.account, context.family, context.model);
1035
+ accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
1036
+ state.sessionAffinityStore?.forgetSession(context.sessionKey);
1037
+ accountManager.saveToDiskDebounced();
1038
+ }, state.streamStallTimeoutMs);
1039
+ await usageRecorder.record({
1040
+ outcome: forwarded ? "success" : "failure",
1041
+ statusCode: upstream.status,
1042
+ errorCode: forwarded ? null : "stream_forward_failed",
1043
+ account: refreshed.account,
1044
+ });
1045
+ return;
1789
1046
  }
1790
- catch (error) {
1791
- const rawErrorMessage = error instanceof Error ? error.message : String(error);
1792
- // errors-logging-08: redact any email/token material that leaked into a
1793
- // raw upstream or refresh error string before it reaches status consumers
1794
- // or the structured log. maskString is a no-op for clean diagnostic text.
1795
- const maskedErrorMessage = maskString(rawErrorMessage);
1796
- status.lastError = maskedErrorMessage;
1797
- // errors-logging-01: surface the failure through the structured logger
1798
- // (redaction-safe) with the request trace id, instead of only stashing a
1799
- // last-write-wins status string.
1800
- proxyLog.error("runtime proxy request failed", {
1801
- traceId,
1802
- code: isRuntimeProxyHttpError(error) ? error.code : "codex_runtime_rotation_proxy_error",
1803
- error: maskedErrorMessage,
1047
+ if (transientAttempts >= transientAttemptLimit &&
1048
+ attemptedIndexes.size < accountCount) {
1049
+ exhaustionReason = "budget";
1050
+ }
1051
+ else if (exhaustionReason === "deactivated" &&
1052
+ transientExhaustionReason) {
1053
+ exhaustionReason = transientExhaustionReason;
1054
+ }
1055
+ // When a manual pin is set and the pinned account is unavailable, do
1056
+ // NOT silently fall through to rotation. Hard-fail with a 503 so the
1057
+ // user is informed the pin cannot be honored. See issue #474.
1058
+ //
1059
+ // Surface the runtime skip reason in both the human-readable message
1060
+ // and a structured `reason` field, mirroring `writePoolExhausted`. A
1061
+ // null reason indicates a forecast/runtime state desync (the pinned
1062
+ // account was selected but no skip reason was recorded) — see #486.
1063
+ if (isPinned) {
1064
+ const errorBody = buildPinnedUnavailableErrorBody(pinnedIndex, accountSkipReasons);
1065
+ if (errorBody.reason === null) {
1066
+ state.status.lastError = `pinned-503 missing skip reason (pinnedIndex=${pinnedIndex})`;
1067
+ }
1068
+ await usageRecorder?.record({
1069
+ outcome: "failure",
1070
+ statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
1071
+ errorCode: "codex_pinned_account_unavailable",
1804
1072
  });
1805
- if (!res.headersSent) {
1806
- if (isRuntimeProxyHttpError(error)) {
1807
- await usageRecorder?.record({
1808
- outcome: "failure",
1809
- statusCode: error.statusCode,
1810
- errorCode: error.code,
1811
- });
1812
- writeJson(res, error.statusCode, {
1813
- error: {
1814
- message: error.message,
1815
- code: error.code,
1816
- },
1817
- });
1818
- return;
1819
- }
1073
+ writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
1074
+ return;
1075
+ }
1076
+ await usageRecorder?.record({
1077
+ outcome: "failure",
1078
+ statusCode: normalizeExhaustionStatus(exhaustionReason),
1079
+ errorCode: isThreadGoalRequest && context.upstreamPath.endsWith("/get") ? "thread_goal_pool_exhausted" : exhaustionReason,
1080
+ });
1081
+ if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
1082
+ writeJson(res, HTTP_STATUS.OK, { goal: null });
1083
+ }
1084
+ else {
1085
+ writePoolExhausted({
1086
+ res,
1087
+ accountManager,
1088
+ family: context.family,
1089
+ model: context.model,
1090
+ reason: exhaustionReason,
1091
+ accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
1092
+ String(index),
1093
+ reason,
1094
+ ])),
1095
+ });
1096
+ }
1097
+ }
1098
+ catch (error) {
1099
+ const rawErrorMessage = error instanceof Error ? error.message : String(error);
1100
+ // errors-logging-08: redact any email/token material that leaked into a
1101
+ // raw upstream or refresh error string before it reaches state.status consumers
1102
+ // or the structured log. maskString is a no-op for clean diagnostic text.
1103
+ const maskedErrorMessage = maskString(rawErrorMessage);
1104
+ state.status.lastError = maskedErrorMessage;
1105
+ // errors-logging-01: surface the failure through the structured logger
1106
+ // (redaction-safe) with the request trace id, instead of only stashing a
1107
+ // last-write-wins status string.
1108
+ proxyLog.error("runtime proxy request failed", {
1109
+ traceId,
1110
+ code: isRuntimeProxyHttpError(error) ? error.code : "codex_runtime_rotation_proxy_error",
1111
+ error: maskedErrorMessage,
1112
+ });
1113
+ if (!res.headersSent) {
1114
+ if (isRuntimeProxyHttpError(error)) {
1820
1115
  await usageRecorder?.record({
1821
1116
  outcome: "failure",
1822
- statusCode: 500,
1823
- errorCode: "codex_runtime_rotation_proxy_error",
1117
+ statusCode: error.statusCode,
1118
+ errorCode: error.code,
1824
1119
  });
1825
- writeJson(res, 500, {
1120
+ writeJson(res, error.statusCode, {
1826
1121
  error: {
1827
- message: "Runtime rotation proxy failed before forwarding the request.",
1828
- code: "codex_runtime_rotation_proxy_error",
1122
+ message: error.message,
1123
+ code: error.code,
1829
1124
  },
1830
1125
  });
1126
+ return;
1831
1127
  }
1832
- else if (!res.destroyed) {
1833
- res.destroy(error instanceof Error ? error : undefined);
1834
- }
1128
+ await usageRecorder?.record({
1129
+ outcome: "failure",
1130
+ statusCode: 500,
1131
+ errorCode: "codex_runtime_rotation_proxy_error",
1132
+ });
1133
+ writeJson(res, 500, {
1134
+ error: {
1135
+ message: "Runtime rotation proxy failed before forwarding the request.",
1136
+ code: "codex_runtime_rotation_proxy_error",
1137
+ },
1138
+ });
1835
1139
  }
1836
- };
1837
- const server = createServer((req, res) => {
1838
- void handleRequest(req, res);
1839
- });
1840
- const sockets = new Set();
1841
- server.on("connection", (socket) => {
1842
- sockets.add(socket);
1843
- socket.once("close", () => {
1844
- sockets.delete(socket);
1845
- });
1846
- });
1847
- const onPostStartupServerError = (error) => {
1848
- status.lastError = error.message;
1849
- };
1850
- await new Promise((resolve, reject) => {
1851
- const onError = (error) => {
1852
- server.off("listening", onListening);
1853
- reject(error);
1854
- };
1855
- const onListening = () => {
1856
- server.off("error", onError);
1857
- resolve();
1858
- };
1859
- server.once("error", onError);
1860
- server.once("listening", onListening);
1861
- server.listen(port, bindHost);
1862
- });
1863
- server.on("error", onPostStartupServerError);
1864
- const address = server.address();
1865
- const resolvedPort = typeof address === "object" && address ? address.port : port;
1866
- return {
1867
- host: bindHost,
1868
- port: resolvedPort,
1869
- baseUrl: `http://${urlHost}:${resolvedPort}`,
1870
- close: async () => {
1871
- await closeServer(server, sockets);
1872
- await activeAccountManager.flushPendingSave();
1873
- },
1874
- getStatus: () => ({
1875
- ...status,
1876
- // Redact any email/token material that leaked into a raw upstream or
1877
- // refresh error string before exposing it to status/report consumers
1878
- // (errors-logging-08). maskString is a no-op for clean diagnostic text.
1879
- lastError: status.lastError === null ? null : maskString(status.lastError),
1880
- }),
1881
- };
1140
+ else if (!res.destroyed) {
1141
+ res.destroy(error instanceof Error ? error : undefined);
1142
+ }
1143
+ }
1882
1144
  }
1883
1145
  async function closeServer(server, sockets) {
1884
1146
  if (!server.listening)
@@ -1898,4 +1160,3 @@ async function closeServer(server, sockets) {
1898
1160
  }
1899
1161
  await closed;
1900
1162
  }
1901
- //# sourceMappingURL=runtime-rotation-proxy.js.map