codex-multi-auth 2.3.0-beta.1 → 2.3.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +2 -1
- package/README.md +4 -4
- package/config/schema/config.schema.json +268 -21
- package/dist/index.js +0 -1
- package/dist/lib/account-policy.d.ts.map +1 -1
- package/dist/lib/account-policy.js +2 -2
- package/dist/lib/accounts/rate-limits.js +0 -1
- package/dist/lib/accounts.d.ts +3 -8
- package/dist/lib/accounts.d.ts.map +1 -1
- package/dist/lib/accounts.js +2 -3
- package/dist/lib/audit.d.ts +0 -9
- package/dist/lib/audit.d.ts.map +1 -1
- package/dist/lib/audit.js +0 -1
- package/dist/lib/auth/auth.js +0 -1
- package/dist/lib/auth/browser.js +0 -1
- package/dist/lib/auth/device-auth.d.ts +1 -6
- package/dist/lib/auth/device-auth.d.ts.map +1 -1
- package/dist/lib/auth/device-auth.js +5 -6
- package/dist/lib/auth/index.js +0 -1
- package/dist/lib/auth/org-override.js +0 -1
- package/dist/lib/auth/server.js +0 -1
- package/dist/lib/auth/token-utils.js +0 -1
- package/dist/lib/auth-rate-limit.js +0 -1
- package/dist/lib/budget-guard.d.ts.map +1 -1
- package/dist/lib/budget-guard.js +2 -2
- package/dist/lib/capability-policy.js +0 -1
- package/dist/lib/circuit-breaker.js +0 -1
- package/dist/lib/cli.js +1 -2
- package/dist/lib/codex-cli/index.js +0 -1
- package/dist/lib/codex-cli/observability.js +0 -1
- package/dist/lib/codex-cli/state.js +0 -1
- package/dist/lib/codex-cli/sync.d.ts +2 -2
- package/dist/lib/codex-cli/sync.d.ts.map +1 -1
- package/dist/lib/codex-cli/sync.js +1 -2
- package/dist/lib/codex-cli/writer.d.ts.map +1 -1
- package/dist/lib/codex-cli/writer.js +2 -2
- package/dist/lib/codex-manager/account-credentials.d.ts +13 -0
- package/dist/lib/codex-manager/account-credentials.d.ts.map +1 -0
- package/dist/lib/codex-manager/account-credentials.js +55 -0
- package/dist/lib/codex-manager/account-manager-commands.js +0 -1
- package/dist/lib/codex-manager/account-pool-write.d.ts +1 -1
- package/dist/lib/codex-manager/account-pool-write.d.ts.map +1 -1
- package/dist/lib/codex-manager/account-pool-write.js +0 -1
- package/dist/lib/codex-manager/backend-category-entry.js +0 -1
- package/dist/lib/codex-manager/backend-category-helpers.js +0 -1
- package/dist/lib/codex-manager/backend-category-prompt.js +0 -1
- package/dist/lib/codex-manager/backend-settings-controller.js +0 -1
- package/dist/lib/codex-manager/backend-settings-entry.js +0 -1
- package/dist/lib/codex-manager/backend-settings-helpers.d.ts +0 -1
- package/dist/lib/codex-manager/backend-settings-helpers.d.ts.map +1 -1
- package/dist/lib/codex-manager/backend-settings-helpers.js +1 -2
- package/dist/lib/codex-manager/backend-settings-prompt.js +0 -1
- package/dist/lib/codex-manager/backend-settings-schema.js +0 -1
- package/dist/lib/codex-manager/behavior-settings-panel.d.ts +1 -20
- package/dist/lib/codex-manager/behavior-settings-panel.d.ts.map +1 -1
- package/dist/lib/codex-manager/behavior-settings-panel.js +0 -1
- package/dist/lib/codex-manager/commands/account.js +0 -1
- package/dist/lib/codex-manager/commands/best.d.ts +8 -0
- package/dist/lib/codex-manager/commands/best.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/best.js +64 -1
- package/dist/lib/codex-manager/commands/bridge.js +0 -1
- package/dist/lib/codex-manager/commands/budget.js +0 -1
- package/dist/lib/codex-manager/commands/check.js +0 -1
- package/dist/lib/codex-manager/commands/config-explain.js +0 -1
- package/dist/lib/codex-manager/commands/debug-bundle.js +0 -1
- package/dist/lib/codex-manager/commands/forecast.js +0 -1
- package/dist/lib/codex-manager/commands/init-config.js +0 -1
- package/dist/lib/codex-manager/commands/integrations.js +0 -1
- package/dist/lib/codex-manager/commands/models.js +0 -1
- package/dist/lib/codex-manager/commands/monitor.js +0 -1
- package/dist/lib/codex-manager/commands/report.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/report.js +2 -2
- package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/rotation.js +1 -4
- package/dist/lib/codex-manager/commands/status.js +0 -1
- package/dist/lib/codex-manager/commands/switch.js +0 -1
- package/dist/lib/codex-manager/commands/uninstall.d.ts +2 -2
- package/dist/lib/codex-manager/commands/uninstall.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/uninstall.js +3 -34
- package/dist/lib/codex-manager/commands/unpin.js +0 -1
- package/dist/lib/codex-manager/commands/usage.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/usage.js +2 -2
- package/dist/lib/codex-manager/commands/verify.d.ts +2 -2
- package/dist/lib/codex-manager/commands/verify.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/verify.js +0 -1
- package/dist/lib/codex-manager/commands/why-selected.d.ts +1 -22
- package/dist/lib/codex-manager/commands/why-selected.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/why-selected.js +0 -1
- package/dist/lib/codex-manager/commands/workspace.js +0 -1
- package/dist/lib/codex-manager/dashboard-display-panel.d.ts +1 -15
- package/dist/lib/codex-manager/dashboard-display-panel.d.ts.map +1 -1
- package/dist/lib/codex-manager/dashboard-display-panel.js +0 -1
- package/dist/lib/codex-manager/dashboard-formatters.js +0 -1
- package/dist/lib/codex-manager/dashboard-settings-controller.js +0 -1
- package/dist/lib/codex-manager/dashboard-settings-data.js +0 -1
- package/dist/lib/codex-manager/dashboard-settings-entry.js +0 -1
- package/dist/lib/codex-manager/experimental-settings-entry.js +0 -1
- package/dist/lib/codex-manager/experimental-settings-prompt.d.ts +2 -1
- package/dist/lib/codex-manager/experimental-settings-prompt.d.ts.map +1 -1
- package/dist/lib/codex-manager/experimental-settings-prompt.js +0 -1
- package/dist/lib/codex-manager/experimental-settings-schema.js +1 -2
- package/dist/lib/codex-manager/experimental-sync-target-entry.js +0 -1
- package/dist/lib/codex-manager/experimental-sync-target.js +0 -1
- package/dist/lib/codex-manager/forecast-report-commands.js +0 -1
- package/dist/lib/codex-manager/forecast-report-shared.d.ts +2 -2
- package/dist/lib/codex-manager/forecast-report-shared.d.ts.map +1 -1
- package/dist/lib/codex-manager/forecast-report-shared.js +5 -21
- package/dist/lib/codex-manager/formatters/account-formatters.d.ts +11 -0
- package/dist/lib/codex-manager/formatters/account-formatters.d.ts.map +1 -0
- package/dist/lib/codex-manager/formatters/account-formatters.js +79 -0
- package/dist/lib/codex-manager/formatters/index.d.ts +5 -0
- package/dist/lib/codex-manager/formatters/index.d.ts.map +1 -0
- package/dist/lib/codex-manager/formatters/index.js +4 -0
- package/dist/lib/codex-manager/formatters/model-formatters.d.ts +12 -0
- package/dist/lib/codex-manager/formatters/model-formatters.d.ts.map +1 -0
- package/dist/lib/codex-manager/formatters/model-formatters.js +23 -0
- package/dist/lib/codex-manager/formatters/quota-formatters.d.ts +9 -0
- package/dist/lib/codex-manager/formatters/quota-formatters.d.ts.map +1 -0
- package/dist/lib/codex-manager/formatters/quota-formatters.js +104 -0
- package/dist/lib/codex-manager/formatters/text-style.d.ts +14 -0
- package/dist/lib/codex-manager/formatters/text-style.d.ts.map +1 -0
- package/dist/lib/codex-manager/formatters/text-style.js +111 -0
- package/dist/lib/codex-manager/health-check.d.ts +14 -0
- package/dist/lib/codex-manager/health-check.d.ts.map +1 -0
- package/dist/lib/codex-manager/health-check.js +278 -0
- package/dist/lib/codex-manager/help.d.ts +2 -1
- package/dist/lib/codex-manager/help.d.ts.map +1 -1
- package/dist/lib/codex-manager/help.js +0 -1
- package/dist/lib/codex-manager/login-action-panel.d.ts +4 -0
- package/dist/lib/codex-manager/login-action-panel.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-action-panel.js +238 -0
- package/dist/lib/codex-manager/login-flow.d.ts +17 -0
- package/dist/lib/codex-manager/login-flow.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-flow.js +387 -0
- package/dist/lib/codex-manager/login-menu-actions.d.ts +20 -0
- package/dist/lib/codex-manager/login-menu-actions.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-menu-actions.js +303 -0
- package/dist/lib/codex-manager/login-menu-data.d.ts +19 -0
- package/dist/lib/codex-manager/login-menu-data.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-menu-data.js +378 -0
- package/dist/lib/codex-manager/login-oauth.d.ts +55 -0
- package/dist/lib/codex-manager/login-oauth.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-oauth.js +345 -0
- package/dist/lib/codex-manager/manual-callback.js +0 -1
- package/dist/lib/codex-manager/persist-selected-account.d.ts +24 -0
- package/dist/lib/codex-manager/persist-selected-account.d.ts.map +1 -0
- package/dist/lib/codex-manager/persist-selected-account.js +107 -0
- package/dist/lib/codex-manager/quota-cache-helpers.d.ts +18 -0
- package/dist/lib/codex-manager/quota-cache-helpers.d.ts.map +1 -0
- package/dist/lib/codex-manager/quota-cache-helpers.js +112 -0
- package/dist/lib/codex-manager/rate-limit-markers.js +0 -1
- package/dist/lib/codex-manager/repair-commands.d.ts +0 -15
- package/dist/lib/codex-manager/repair-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/repair-commands.js +5 -6
- package/dist/lib/codex-manager/settings-hub/backend.d.ts +0 -5
- package/dist/lib/codex-manager/settings-hub/backend.d.ts.map +1 -1
- package/dist/lib/codex-manager/settings-hub/backend.js +2 -3
- package/dist/lib/codex-manager/settings-hub/dashboard.d.ts +1 -14
- package/dist/lib/codex-manager/settings-hub/dashboard.d.ts.map +1 -1
- package/dist/lib/codex-manager/settings-hub/dashboard.js +9 -10
- package/dist/lib/codex-manager/settings-hub/experimental.js +1 -2
- package/dist/lib/codex-manager/settings-hub/index.js +1 -2
- package/dist/lib/codex-manager/settings-hub/shared.js +0 -1
- package/dist/lib/codex-manager/settings-hub-entry.js +0 -1
- package/dist/lib/codex-manager/settings-hub-menu.js +0 -1
- package/dist/lib/codex-manager/settings-hub-prompt.js +0 -1
- package/dist/lib/codex-manager/settings-hub.js +0 -1
- package/dist/lib/codex-manager/settings-panels.js +0 -1
- package/dist/lib/codex-manager/settings-persist-utils.js +0 -1
- package/dist/lib/codex-manager/settings-preview.js +0 -1
- package/dist/lib/codex-manager/settings-write-queue.d.ts +0 -10
- package/dist/lib/codex-manager/settings-write-queue.d.ts.map +1 -1
- package/dist/lib/codex-manager/settings-write-queue.js +10 -11
- package/dist/lib/codex-manager/statusline-order.js +0 -1
- package/dist/lib/codex-manager/statusline-settings-panel.d.ts +3 -18
- package/dist/lib/codex-manager/statusline-settings-panel.d.ts.map +1 -1
- package/dist/lib/codex-manager/statusline-settings-panel.js +0 -1
- package/dist/lib/codex-manager/theme-settings-panel.d.ts +1 -14
- package/dist/lib/codex-manager/theme-settings-panel.d.ts.map +1 -1
- package/dist/lib/codex-manager/theme-settings-panel.js +0 -1
- package/dist/lib/codex-manager/unified-settings-controller.js +0 -1
- package/dist/lib/codex-manager/unified-settings-entry.js +0 -1
- package/dist/lib/codex-manager.d.ts +2 -4
- package/dist/lib/codex-manager.d.ts.map +1 -1
- package/dist/lib/codex-manager.js +219 -2570
- package/dist/lib/config-schema.d.ts +32 -0
- package/dist/lib/config-schema.d.ts.map +1 -0
- package/dist/lib/config-schema.js +96 -0
- package/dist/lib/config.d.ts +0 -13
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +82 -123
- package/dist/lib/constants.js +0 -1
- package/dist/lib/context-overflow.js +0 -1
- package/dist/lib/dashboard-settings.d.ts +3 -3
- package/dist/lib/dashboard-settings.d.ts.map +1 -1
- package/dist/lib/dashboard-settings.js +0 -2
- package/dist/lib/entitlement-cache.js +0 -1
- package/dist/lib/env-parsing.js +0 -1
- package/dist/lib/errors.d.ts +0 -1
- package/dist/lib/errors.d.ts.map +1 -1
- package/dist/lib/errors.js +0 -1
- package/dist/lib/forecast.d.ts +4 -3
- package/dist/lib/forecast.d.ts.map +1 -1
- package/dist/lib/forecast.js +0 -1
- package/dist/lib/fs-retry.d.ts +46 -3
- package/dist/lib/fs-retry.d.ts.map +1 -1
- package/dist/lib/fs-retry.js +77 -9
- package/dist/lib/health.js +0 -1
- package/dist/lib/index.d.ts +1 -0
- package/dist/lib/index.d.ts.map +1 -1
- package/dist/lib/index.js +1 -1
- package/dist/lib/integration-generators.js +0 -1
- package/dist/lib/live-account-sync.js +0 -1
- package/dist/lib/local-bridge.js +0 -1
- package/dist/lib/local-client-tokens.d.ts.map +1 -1
- package/dist/lib/local-client-tokens.js +2 -2
- package/dist/lib/logger.js +0 -1
- package/dist/lib/model-capability-matrix.js +0 -1
- package/dist/lib/named-backup-export.js +0 -1
- package/dist/lib/oc-chatgpt-import-adapter.d.ts +1 -1
- package/dist/lib/oc-chatgpt-import-adapter.d.ts.map +1 -1
- package/dist/lib/oc-chatgpt-import-adapter.js +0 -1
- package/dist/lib/oc-chatgpt-orchestrator.d.ts.map +1 -1
- package/dist/lib/oc-chatgpt-orchestrator.js +2 -2
- package/dist/lib/oc-chatgpt-target-detection.d.ts +1 -1
- package/dist/lib/oc-chatgpt-target-detection.d.ts.map +1 -1
- package/dist/lib/oc-chatgpt-target-detection.js +0 -1
- package/dist/lib/parallel-probe.js +0 -1
- package/dist/lib/policy/runtime-policy.js +0 -1
- package/dist/lib/preemptive-quota-scheduler.js +0 -1
- package/dist/lib/proactive-refresh.js +0 -1
- package/dist/lib/prompts/codex-host-bridge.d.ts +0 -8
- package/dist/lib/prompts/codex-host-bridge.d.ts.map +1 -1
- package/dist/lib/prompts/codex-host-bridge.js +0 -21
- package/dist/lib/prompts/codex.d.ts +3 -11
- package/dist/lib/prompts/codex.d.ts.map +1 -1
- package/dist/lib/prompts/codex.js +6 -13
- package/dist/lib/prompts/fetch-utils.d.ts +0 -2
- package/dist/lib/prompts/fetch-utils.d.ts.map +1 -1
- package/dist/lib/prompts/fetch-utils.js +2 -3
- package/dist/lib/prompts/host-codex-prompt.d.ts.map +1 -1
- package/dist/lib/prompts/host-codex-prompt.js +2 -2
- package/dist/lib/quota-cache.d.ts.map +1 -1
- package/dist/lib/quota-cache.js +17 -44
- package/dist/lib/quota-probe.js +0 -1
- package/dist/lib/quota-readiness.js +0 -1
- package/dist/lib/recovery/constants.js +0 -1
- package/dist/lib/recovery/index.js +0 -1
- package/dist/lib/recovery/storage.d.ts.map +1 -1
- package/dist/lib/recovery/storage.js +35 -54
- package/dist/lib/recovery/storage.js.map +1 -1
- package/dist/lib/recovery/types.d.ts +4 -12
- package/dist/lib/recovery/types.d.ts.map +1 -1
- package/dist/lib/recovery/types.js +0 -1
- package/dist/lib/recovery.d.ts +2 -2
- package/dist/lib/recovery.d.ts.map +1 -1
- package/dist/lib/recovery.js +0 -1
- package/dist/lib/refresh-guardian.js +0 -1
- package/dist/lib/refresh-lease.d.ts.map +1 -1
- package/dist/lib/refresh-lease.js +1 -4
- package/dist/lib/refresh-queue.js +0 -1
- package/dist/lib/request/error-classification.d.ts +75 -0
- package/dist/lib/request/error-classification.d.ts.map +1 -0
- package/dist/lib/request/error-classification.js +295 -0
- package/dist/lib/request/failover-config.js +0 -1
- package/dist/lib/request/failure-policy.js +0 -1
- package/dist/lib/request/fetch-helpers.d.ts +10 -114
- package/dist/lib/request/fetch-helpers.d.ts.map +1 -1
- package/dist/lib/request/fetch-helpers.js +7 -589
- package/dist/lib/request/headers.d.ts +33 -0
- package/dist/lib/request/headers.d.ts.map +1 -0
- package/dist/lib/request/headers.js +64 -0
- package/dist/lib/request/helpers/input-utils.js +0 -1
- package/dist/lib/request/helpers/model-map.d.ts +10 -0
- package/dist/lib/request/helpers/model-map.d.ts.map +1 -1
- package/dist/lib/request/helpers/model-map.js +11 -1
- package/dist/lib/request/helpers/tool-utils.js +0 -1
- package/dist/lib/request/index.js +0 -1
- package/dist/lib/request/rate-limit-backoff.js +0 -1
- package/dist/lib/request/rate-limit-decision.d.ts +28 -0
- package/dist/lib/request/rate-limit-decision.d.ts.map +1 -0
- package/dist/lib/request/rate-limit-decision.js +191 -0
- package/dist/lib/request/request-attempt-budget.js +0 -1
- package/dist/lib/request/request-init.js +0 -1
- package/dist/lib/request/request-resilience.js +0 -1
- package/dist/lib/request/request-transformer.js +0 -1
- package/dist/lib/request/response-compaction.d.ts +2 -1
- package/dist/lib/request/response-compaction.d.ts.map +1 -1
- package/dist/lib/request/response-compaction.js +0 -1
- package/dist/lib/request/response-handler.js +0 -1
- package/dist/lib/request/response-metadata.js +0 -1
- package/dist/lib/request/stream-failover-runtime.d.ts +8 -0
- package/dist/lib/request/stream-failover-runtime.d.ts.map +1 -0
- package/dist/lib/request/stream-failover-runtime.js +171 -0
- package/dist/lib/request/stream-failover.js +0 -1
- package/dist/lib/request/token-refresh.d.ts +30 -0
- package/dist/lib/request/token-refresh.d.ts.map +1 -0
- package/dist/lib/request/token-refresh.js +116 -0
- package/dist/lib/request/url-rewriting.d.ts +24 -0
- package/dist/lib/request/url-rewriting.d.ts.map +1 -0
- package/dist/lib/request/url-rewriting.js +153 -0
- package/dist/lib/request/wait-utils.js +0 -1
- package/dist/lib/rotation.d.ts +2 -4
- package/dist/lib/rotation.d.ts.map +1 -1
- package/dist/lib/rotation.js +1 -3
- package/dist/lib/routing-mutex.js +0 -1
- package/dist/lib/routing-profiles.d.ts.map +1 -1
- package/dist/lib/routing-profiles.js +2 -2
- package/dist/lib/runtime/account-check-helpers.js +0 -1
- package/dist/lib/runtime/account-check-types.js +0 -1
- package/dist/lib/runtime/account-check.js +0 -1
- package/dist/lib/runtime/account-health-check.js +0 -1
- package/dist/lib/runtime/account-manager-cache-entry.js +0 -1
- package/dist/lib/runtime/account-manager-cache.js +0 -1
- package/dist/lib/runtime/account-pool.js +0 -1
- package/dist/lib/runtime/account-scope.js +0 -1
- package/dist/lib/runtime/account-select-event.js +0 -1
- package/dist/lib/runtime/account-selection.js +0 -1
- package/dist/lib/runtime/account-state.js +0 -1
- package/dist/lib/runtime/account-status.js +0 -1
- package/dist/lib/runtime/account-storage-scope-entry.js +0 -1
- package/dist/lib/runtime/app-bind.js +0 -1
- package/dist/lib/runtime/auth-facade.js +0 -1
- package/dist/lib/runtime/browser-oauth-flow.js +0 -1
- package/dist/lib/runtime/capability-boost.js +0 -1
- package/dist/lib/runtime/config-toml.d.ts +0 -6
- package/dist/lib/runtime/config-toml.d.ts.map +1 -1
- package/dist/lib/runtime/config-toml.js +6 -7
- package/dist/lib/runtime/event-handler.d.ts +0 -31
- package/dist/lib/runtime/event-handler.d.ts.map +1 -1
- package/dist/lib/runtime/event-handler.js +0 -23
- package/dist/lib/runtime/first-run.d.ts +78 -0
- package/dist/lib/runtime/first-run.d.ts.map +1 -0
- package/dist/lib/runtime/first-run.js +327 -0
- package/dist/lib/runtime/flagged-verify-types.js +0 -1
- package/dist/lib/runtime/hydrate-emails.js +0 -1
- package/dist/lib/runtime/live-sync-entry.js +0 -1
- package/dist/lib/runtime/live-sync.js +0 -1
- package/dist/lib/runtime/loader-setup.js +0 -1
- package/dist/lib/runtime/login-menu-accounts.js +0 -1
- package/dist/lib/runtime/manual-oauth-flow.js +0 -1
- package/dist/lib/runtime/oauth-browser-flow.js +0 -1
- package/dist/lib/runtime/preemptive-quota.js +0 -1
- package/dist/lib/runtime/quota-headers.d.ts +1 -6
- package/dist/lib/runtime/quota-headers.d.ts.map +1 -1
- package/dist/lib/runtime/quota-headers.js +5 -6
- package/dist/lib/runtime/quota-probe.js +0 -1
- package/dist/lib/runtime/quota-settings.js +0 -1
- package/dist/lib/runtime/refresh-guardian-entry.js +0 -1
- package/dist/lib/runtime/refresh-guardian.js +0 -1
- package/dist/lib/runtime/request-init.js +0 -1
- package/dist/lib/runtime/rotation-account-selection.d.ts +38 -0
- package/dist/lib/runtime/rotation-account-selection.d.ts.map +1 -0
- package/dist/lib/runtime/rotation-account-selection.js +172 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts +54 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts.map +1 -0
- package/dist/lib/runtime/rotation-proxy-state.js +56 -0
- package/dist/lib/runtime/rotation-server-types.d.ts +57 -0
- package/dist/lib/runtime/rotation-server-types.d.ts.map +1 -0
- package/dist/lib/runtime/rotation-server-types.js +1 -0
- package/dist/lib/runtime/rotation-storage-meta.d.ts +40 -0
- package/dist/lib/runtime/rotation-storage-meta.d.ts.map +1 -0
- package/dist/lib/runtime/rotation-storage-meta.js +143 -0
- package/dist/lib/runtime/rotation-token-refresh.d.ts +25 -0
- package/dist/lib/runtime/rotation-token-refresh.d.ts.map +1 -0
- package/dist/lib/runtime/rotation-token-refresh.js +99 -0
- package/dist/lib/runtime/runtime-current-account.d.ts +3 -3
- package/dist/lib/runtime/runtime-current-account.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-current-account.js +2 -5
- package/dist/lib/runtime/runtime-observability.d.ts +2 -2
- package/dist/lib/runtime/runtime-observability.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-observability.js +0 -5
- package/dist/lib/runtime/runtime-services.js +0 -1
- package/dist/lib/runtime/session-affinity-entry.js +0 -1
- package/dist/lib/runtime/session-affinity.js +0 -1
- package/dist/lib/runtime/session-recovery.js +0 -1
- package/dist/lib/runtime/status-marker.js +0 -1
- package/dist/lib/runtime/storage-scope.js +0 -1
- package/dist/lib/runtime/toast.js +0 -1
- package/dist/lib/runtime/ui-runtime-entry.js +0 -1
- package/dist/lib/runtime/ui-runtime.js +0 -1
- package/dist/lib/runtime/verify-flagged.js +0 -1
- package/dist/lib/runtime-constants.js +0 -1
- package/dist/lib/runtime-paths.js +0 -1
- package/dist/lib/runtime-rotation-proxy.d.ts +7 -124
- package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
- package/dist/lib/runtime-rotation-proxy.js +618 -1356
- package/dist/lib/schemas.d.ts +5 -93
- package/dist/lib/schemas.d.ts.map +1 -1
- package/dist/lib/schemas.js +9 -10
- package/dist/lib/session-affinity.js +0 -1
- package/dist/lib/shutdown.js +0 -1
- package/dist/lib/storage/account-clear-entry.js +0 -1
- package/dist/lib/storage/account-clear.js +0 -1
- package/dist/lib/storage/account-match-utils.js +0 -1
- package/dist/lib/storage/account-persistence.d.ts +1 -1
- package/dist/lib/storage/account-persistence.d.ts.map +1 -1
- package/dist/lib/storage/account-persistence.js +0 -1
- package/dist/lib/storage/account-port.d.ts +1 -1
- package/dist/lib/storage/account-port.d.ts.map +1 -1
- package/dist/lib/storage/account-port.js +0 -1
- package/dist/lib/storage/account-save-entry.d.ts +1 -1
- package/dist/lib/storage/account-save-entry.d.ts.map +1 -1
- package/dist/lib/storage/account-save-entry.js +0 -1
- package/dist/lib/storage/account-save.d.ts +1 -1
- package/dist/lib/storage/account-save.d.ts.map +1 -1
- package/dist/lib/storage/account-save.js +0 -1
- package/dist/lib/storage/account-snapshot.js +0 -1
- package/dist/lib/storage/backup-metadata-builder.d.ts +1 -1
- package/dist/lib/storage/backup-metadata-builder.d.ts.map +1 -1
- package/dist/lib/storage/backup-metadata-builder.js +0 -1
- package/dist/lib/storage/backup-metadata.d.ts +13 -0
- package/dist/lib/storage/backup-metadata.d.ts.map +1 -1
- package/dist/lib/storage/backup-metadata.js +0 -1
- package/dist/lib/storage/backup-paths.js +0 -1
- package/dist/lib/storage/backup-restore.d.ts +1 -1
- package/dist/lib/storage/backup-restore.d.ts.map +1 -1
- package/dist/lib/storage/backup-restore.js +0 -1
- package/dist/lib/storage/cache-artifacts.js +0 -1
- package/dist/lib/storage/error-hints.js +0 -1
- package/dist/lib/storage/file-paths.d.ts +2 -2
- package/dist/lib/storage/file-paths.d.ts.map +1 -1
- package/dist/lib/storage/file-paths.js +2 -3
- package/dist/lib/storage/fixture-guards.d.ts +1 -1
- package/dist/lib/storage/fixture-guards.d.ts.map +1 -1
- package/dist/lib/storage/fixture-guards.js +0 -1
- package/dist/lib/storage/flagged-entry.js +0 -1
- package/dist/lib/storage/flagged-load-entry.d.ts +1 -1
- package/dist/lib/storage/flagged-load-entry.d.ts.map +1 -1
- package/dist/lib/storage/flagged-load-entry.js +0 -1
- package/dist/lib/storage/flagged-save-entry.d.ts +1 -1
- package/dist/lib/storage/flagged-save-entry.d.ts.map +1 -1
- package/dist/lib/storage/flagged-save-entry.js +0 -1
- package/dist/lib/storage/flagged-storage-file.d.ts +1 -1
- package/dist/lib/storage/flagged-storage-file.d.ts.map +1 -1
- package/dist/lib/storage/flagged-storage-file.js +0 -1
- package/dist/lib/storage/flagged-storage-io.d.ts +1 -1
- package/dist/lib/storage/flagged-storage-io.d.ts.map +1 -1
- package/dist/lib/storage/flagged-storage-io.js +2 -3
- package/dist/lib/storage/flagged-storage.d.ts +1 -1
- package/dist/lib/storage/flagged-storage.d.ts.map +1 -1
- package/dist/lib/storage/flagged-storage.js +15 -2
- package/dist/lib/storage/flagged-storage.js.map +1 -1
- package/dist/lib/storage/gitignore.js +0 -1
- package/dist/lib/storage/hash.js +0 -1
- package/dist/lib/storage/health.js +0 -1
- package/dist/lib/storage/identity.js +0 -1
- package/dist/lib/storage/import-export.d.ts +1 -1
- package/dist/lib/storage/import-export.d.ts.map +1 -1
- package/dist/lib/storage/import-export.js +2 -3
- package/dist/lib/storage/metadata-section.js +0 -1
- package/dist/lib/storage/migrations.d.ts +7 -49
- package/dist/lib/storage/migrations.d.ts.map +1 -1
- package/dist/lib/storage/migrations.js +5 -2
- package/dist/lib/storage/named-backup-entry.js +0 -1
- package/dist/lib/storage/named-backups-entry.d.ts +1 -1
- package/dist/lib/storage/named-backups-entry.d.ts.map +1 -1
- package/dist/lib/storage/named-backups-entry.js +0 -1
- package/dist/lib/storage/named-backups.js +0 -1
- package/dist/lib/storage/path-state.js +0 -1
- package/dist/lib/storage/paths.js +0 -1
- package/dist/lib/storage/project-migration.d.ts +1 -1
- package/dist/lib/storage/project-migration.d.ts.map +1 -1
- package/dist/lib/storage/project-migration.js +0 -1
- package/dist/lib/storage/public-types.d.ts +70 -0
- package/dist/lib/storage/public-types.d.ts.map +1 -0
- package/dist/lib/storage/public-types.js +6 -0
- package/dist/lib/storage/record-utils.js +0 -1
- package/dist/lib/storage/restore-assessment.d.ts +1 -1
- package/dist/lib/storage/restore-assessment.d.ts.map +1 -1
- package/dist/lib/storage/restore-assessment.js +0 -1
- package/dist/lib/storage/restore-backup-entry.d.ts +1 -1
- package/dist/lib/storage/restore-backup-entry.d.ts.map +1 -1
- package/dist/lib/storage/restore-backup-entry.js +0 -1
- package/dist/lib/storage/restore-metadata.d.ts +1 -1
- package/dist/lib/storage/restore-metadata.d.ts.map +1 -1
- package/dist/lib/storage/restore-metadata.js +0 -1
- package/dist/lib/storage/restore.d.ts +1 -1
- package/dist/lib/storage/restore.d.ts.map +1 -1
- package/dist/lib/storage/restore.js +0 -1
- package/dist/lib/storage/save-retry.d.ts +10 -0
- package/dist/lib/storage/save-retry.d.ts.map +1 -0
- package/dist/lib/storage/save-retry.js +26 -0
- package/dist/lib/storage/snapshot-inspectors.d.ts +1 -1
- package/dist/lib/storage/snapshot-inspectors.d.ts.map +1 -1
- package/dist/lib/storage/snapshot-inspectors.js +0 -1
- package/dist/lib/storage/storage-parser.d.ts +1 -1
- package/dist/lib/storage/storage-parser.d.ts.map +1 -1
- package/dist/lib/storage/storage-parser.js +0 -1
- package/dist/lib/storage/transactions.d.ts +1 -1
- package/dist/lib/storage/transactions.d.ts.map +1 -1
- package/dist/lib/storage/transactions.js +0 -1
- package/dist/lib/storage.d.ts +3 -24
- package/dist/lib/storage.d.ts.map +1 -1
- package/dist/lib/storage.js +47 -55
- package/dist/lib/table-formatter.js +0 -1
- package/dist/lib/temp-path.d.ts +26 -0
- package/dist/lib/temp-path.d.ts.map +1 -0
- package/dist/lib/temp-path.js +30 -0
- package/dist/lib/tools/hashline-tools.js +0 -1
- package/dist/lib/types.js +0 -1
- package/dist/lib/ui/ansi.js +0 -1
- package/dist/lib/ui/auth-menu-builder.d.ts +89 -0
- package/dist/lib/ui/auth-menu-builder.d.ts.map +1 -0
- package/dist/lib/ui/auth-menu-builder.js +350 -0
- package/dist/lib/ui/auth-menu.d.ts +2 -76
- package/dist/lib/ui/auth-menu.d.ts.map +1 -1
- package/dist/lib/ui/auth-menu.js +3 -349
- package/dist/lib/ui/confirm.js +0 -1
- package/dist/lib/ui/display-width.js +0 -1
- package/dist/lib/ui/format.js +0 -1
- package/dist/lib/ui/index.d.ts +10 -0
- package/dist/lib/ui/index.d.ts.map +1 -0
- package/dist/lib/ui/index.js +9 -0
- package/dist/lib/ui/runtime.js +0 -1
- package/dist/lib/ui/select.js +0 -1
- package/dist/lib/ui/theme.d.ts +3 -2
- package/dist/lib/ui/theme.d.ts.map +1 -1
- package/dist/lib/ui/theme.js +0 -1
- package/dist/lib/ui/ui-copy.d.ts +159 -0
- package/dist/lib/ui/ui-copy.d.ts.map +1 -0
- package/dist/lib/ui/ui-copy.js +163 -0
- package/dist/lib/unified-settings.d.ts.map +1 -1
- package/dist/lib/unified-settings.js +2 -2
- package/dist/lib/update-notice.d.ts.map +1 -1
- package/dist/lib/update-notice.js +2 -2
- package/dist/lib/usage/index.js +0 -1
- package/dist/lib/usage/ledger.js +0 -1
- package/dist/lib/usage/pricing.js +0 -1
- package/dist/lib/usage/redaction.js +0 -1
- package/dist/lib/usage/types.js +0 -1
- package/dist/lib/utils.js +0 -1
- package/package.json +8 -9
- package/scripts/postinstall.js +34 -265
- package/scripts/preuninstall.js +2 -2
|
@@ -1,20 +1,27 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
1
|
+
import { randomUUID, timingSafeEqual } from "node:crypto";
|
|
3
2
|
import { createServer } from "node:http";
|
|
4
3
|
import { AccountManager, extractAccountId, } from "./accounts.js";
|
|
5
4
|
import { withRoutingMutex } from "./routing-mutex.js";
|
|
6
|
-
import { getStoragePath } from "./storage.js";
|
|
7
5
|
import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getMinRotationIntervalMs, getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, getRoutingMutexMode, getSchedulingStrategy, loadPluginConfig, } from "./config.js";
|
|
8
6
|
import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
|
|
9
7
|
import { getModelFamily } from "./prompts/codex.js";
|
|
10
8
|
import { CURRENT_CODEX_MODEL } from "./request/helpers/model-map.js";
|
|
11
|
-
import {
|
|
12
|
-
import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, recordRuntimeReload, recordRuntimeReset, } from "./runtime/runtime-observability.js";
|
|
9
|
+
import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
|
|
13
10
|
import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
|
|
14
11
|
import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
|
|
15
12
|
import { createLogger, maskString, runWithCorrelationId } from "./logger.js";
|
|
13
|
+
import { CodexValidationError } from "./errors.js";
|
|
14
|
+
import { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, extractErrorCodeFromBody, getQuotaNearExhaustionWaitMs, isTokenInvalidationError, normalizeExhaustionStatus, parseRetryAfterBodyMs, parseRetryAfterHeaderMs, } from "./request/rate-limit-decision.js";
|
|
15
|
+
import { forwardStreamingResponse, HOP_BY_HOP_HEADERS, readErrorBody, responseHeadersForClient, withTimeout, } from "./request/stream-failover-runtime.js";
|
|
16
|
+
import { chooseAccount } from "./runtime/rotation-account-selection.js";
|
|
17
|
+
import { createRotationProxyState, recoverStaleRuntimeState, } from "./runtime/rotation-proxy-state.js";
|
|
18
|
+
import { readStorageMetaFromDisk } from "./runtime/rotation-storage-meta.js";
|
|
19
|
+
import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, } from "./runtime/rotation-token-refresh.js";
|
|
16
20
|
import { SessionAffinityStore } from "./session-affinity.js";
|
|
17
21
|
import { isRecord } from "./utils.js";
|
|
22
|
+
export { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, } from "./request/rate-limit-decision.js";
|
|
23
|
+
export { chooseAccount } from "./runtime/rotation-account-selection.js";
|
|
24
|
+
export { maybeInvalidateAffinityFromDisk, readPinnedAccountIndexFromDisk, readStorageMetaFromDisk, resetPinCacheForTesting, } from "./runtime/rotation-storage-meta.js";
|
|
18
25
|
const DEFAULT_HOST = "127.0.0.1";
|
|
19
26
|
function isLoopbackHost(host) {
|
|
20
27
|
const normalized = host.trim().toLowerCase();
|
|
@@ -54,47 +61,9 @@ function toUrlHost(host) {
|
|
|
54
61
|
// level-gated and carry the per-request correlation id set in handleRequest.
|
|
55
62
|
const proxyLog = createLogger("runtime-proxy");
|
|
56
63
|
const DEFAULT_QUOTA_REMAINING_THRESHOLD = 10;
|
|
57
|
-
const DEFAULT_AUTH_FAILURE_COOLDOWN_MS = 30_000;
|
|
58
64
|
const DEFAULT_MAX_RUNTIME_ACCOUNT_ATTEMPTS = 4;
|
|
59
|
-
// Phrases observed in upstream 401 response bodies when OpenAI/Microsoft has
|
|
60
|
-
// explicitly revoked an OAuth token (as opposed to a generic expired-token 401
|
|
61
|
-
// that can be retried after a refresh). Matching is case-insensitive substring.
|
|
62
|
-
// If anti-abuse detection triggers different wording in production, add the new
|
|
63
|
-
// phrase here and record the source provider and date. See issue #495.
|
|
64
|
-
const TOKEN_INVALIDATION_PHRASES = [
|
|
65
|
-
"invalidated oauth token",
|
|
66
|
-
"authentication token has been invalidated",
|
|
67
|
-
"oauth token has been invalidated",
|
|
68
|
-
"token has been invalidated",
|
|
69
|
-
];
|
|
70
|
-
function isTokenInvalidationError(bodyText) {
|
|
71
|
-
const lower = bodyText.toLowerCase();
|
|
72
|
-
return TOKEN_INVALIDATION_PHRASES.some((phrase) => lower.includes(phrase));
|
|
73
|
-
}
|
|
74
65
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024;
|
|
75
66
|
const MAX_THREAD_GOAL_FALLBACKS = 512;
|
|
76
|
-
const HOP_BY_HOP_HEADERS = new Set([
|
|
77
|
-
"connection",
|
|
78
|
-
"content-length",
|
|
79
|
-
"expect",
|
|
80
|
-
"keep-alive",
|
|
81
|
-
"proxy-authenticate",
|
|
82
|
-
"proxy-authorization",
|
|
83
|
-
"te",
|
|
84
|
-
"trailer",
|
|
85
|
-
"transfer-encoding",
|
|
86
|
-
"upgrade",
|
|
87
|
-
]);
|
|
88
|
-
const PRIVATE_CLIENT_RESPONSE_HEADERS = new Set([
|
|
89
|
-
"x-codex-multi-auth-account-index",
|
|
90
|
-
"x-codex-multi-auth-account-label",
|
|
91
|
-
"x-codex-multi-auth-account-email",
|
|
92
|
-
"x-codex-multi-auth-account-id",
|
|
93
|
-
]);
|
|
94
|
-
const DECODED_UPSTREAM_RESPONSE_HEADERS = new Set([
|
|
95
|
-
// Node fetch returns decoded bytes while preserving the upstream encoding header.
|
|
96
|
-
"content-encoding",
|
|
97
|
-
]);
|
|
98
67
|
const ALLOWED_RESPONSES_PATHS = new Set([
|
|
99
68
|
URL_PATHS.RESPONSES,
|
|
100
69
|
URL_PATHS.CODEX_RESPONSES,
|
|
@@ -201,147 +170,7 @@ function recordLastRuntimeAccount(status, identity) {
|
|
|
201
170
|
snapshot.lastAccountUpdatedAt = identity.updatedAt;
|
|
202
171
|
});
|
|
203
172
|
}
|
|
204
|
-
|
|
205
|
-
// vitest workers (each pointing at their own temp storage file) cannot
|
|
206
|
-
// corrupt each other's snapshots. See issue #474.
|
|
207
|
-
const STORAGE_META_CACHE = new Map();
|
|
208
|
-
// The mtime+size short-circuit (L3) may only be trusted once the cached
|
|
209
|
-
// mtime is far enough in the past that no *subsequent* write could share the
|
|
210
|
-
// same coarse mtime tick. Filesystems report mtime at wildly different
|
|
211
|
-
// granularities (ext4 ns, FAT 2s, some network/Windows volumes ~1s, and CI
|
|
212
|
-
// containers occasionally coarser), and our writers use atomic rename, so two
|
|
213
|
-
// rapid CLI bumps can land on an identical mtimeMs. Within this settle window
|
|
214
|
-
// we therefore ignore mtime equality and fall back to the read + sha1 path
|
|
215
|
-
// (the real source of truth). Outside it, the file has been quiescent long
|
|
216
|
-
// enough that mtime equality provably means "unchanged", so we skip the read.
|
|
217
|
-
// 2s comfortably exceeds the coarsest mtime granularity we expect in practice.
|
|
218
|
-
const MTIME_SHORTCIRCUIT_SETTLE_MS = 2_000;
|
|
219
|
-
function hashStorageBytes(bytes) {
|
|
220
|
-
return createHash("sha1").update(bytes).digest("hex");
|
|
221
|
-
}
|
|
222
|
-
function metaFromSnapshot(snapshot) {
|
|
223
|
-
return {
|
|
224
|
-
pinnedAccountIndex: snapshot.pinnedAccountIndex,
|
|
225
|
-
affinityGeneration: snapshot.affinityGeneration,
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
/**
|
|
229
|
-
* Cheap, hot-path-safe single read with mtime-cache short-circuit. When the
|
|
230
|
-
* file's `mtimeMs` and `size` match the cached snapshot for this path we
|
|
231
|
-
* return the cached value WITHOUT reading or hashing the file. Only when
|
|
232
|
-
* mtime/size differ (or were never cached) do we `readFileSync` + sha1 and,
|
|
233
|
-
* if the content hash still matches, skip the `JSON.parse`. Transient
|
|
234
|
-
* failures (EBUSY/EPERM/EACCES/EAGAIN, partial-write SyntaxError) fall through
|
|
235
|
-
* to the last cached value for this path; defaults are only returned when the
|
|
236
|
-
* file has never been successfully read.
|
|
237
|
-
*
|
|
238
|
-
* Replaces an earlier retry loop with a sub-15ms busy-wait that blocked the
|
|
239
|
-
* event loop on every transient failure. The proxy is on the request hot
|
|
240
|
-
* path; serving a slightly stale (but consistent) value is strictly better
|
|
241
|
-
* than blocking. See #474.
|
|
242
|
-
*/
|
|
243
|
-
export function readStorageMetaFromDisk(storagePath = getStoragePath()) {
|
|
244
|
-
if (!existsSync(storagePath)) {
|
|
245
|
-
STORAGE_META_CACHE.delete(storagePath);
|
|
246
|
-
return { pinnedAccountIndex: null, affinityGeneration: 0 };
|
|
247
|
-
}
|
|
248
|
-
try {
|
|
249
|
-
// mtime+size short-circuit (L3): when neither has changed since the last
|
|
250
|
-
// successful read AND the cached mtime has settled (see
|
|
251
|
-
// MTIME_SHORTCIRCUIT_SETTLE_MS) we return the cached snapshot without
|
|
252
|
-
// reading or hashing the file. During the settle window we deliberately
|
|
253
|
-
// fall through to the read + sha1 path below, which stays the source of
|
|
254
|
-
// truth and protects against the coarse-mtime collision described on
|
|
255
|
-
// StorageMetaSnapshot. So this is a pure fast path for the common
|
|
256
|
-
// "file quiescent, proxy polling every request" case.
|
|
257
|
-
const stats = statSync(storagePath);
|
|
258
|
-
const cachedByStat = STORAGE_META_CACHE.get(storagePath);
|
|
259
|
-
if (cachedByStat &&
|
|
260
|
-
cachedByStat.mtimeMs === stats.mtimeMs &&
|
|
261
|
-
cachedByStat.size === stats.size &&
|
|
262
|
-
Date.now() - stats.mtimeMs > MTIME_SHORTCIRCUIT_SETTLE_MS) {
|
|
263
|
-
return metaFromSnapshot(cachedByStat);
|
|
264
|
-
}
|
|
265
|
-
const bytes = readFileSync(storagePath);
|
|
266
|
-
const contentHash = hashStorageBytes(bytes);
|
|
267
|
-
const cached = STORAGE_META_CACHE.get(storagePath);
|
|
268
|
-
if (cached && cached.contentHash === contentHash) {
|
|
269
|
-
// Content is byte-identical despite the mtime/size change (e.g. an
|
|
270
|
-
// atomic-rename rewrite of the same bytes). Refresh the stat fields so
|
|
271
|
-
// the next request takes the fast path, but skip the JSON.parse.
|
|
272
|
-
const refreshed = {
|
|
273
|
-
...cached,
|
|
274
|
-
mtimeMs: stats.mtimeMs,
|
|
275
|
-
size: stats.size,
|
|
276
|
-
};
|
|
277
|
-
STORAGE_META_CACHE.set(storagePath, refreshed);
|
|
278
|
-
return metaFromSnapshot(refreshed);
|
|
279
|
-
}
|
|
280
|
-
const parsed = JSON.parse(bytes.toString("utf8"));
|
|
281
|
-
const pinnedAccountIndex = typeof parsed.pinnedAccountIndex === "number" &&
|
|
282
|
-
Number.isFinite(parsed.pinnedAccountIndex)
|
|
283
|
-
? Math.trunc(parsed.pinnedAccountIndex)
|
|
284
|
-
: null;
|
|
285
|
-
const affinityGeneration = typeof parsed.affinityGeneration === "number" &&
|
|
286
|
-
Number.isFinite(parsed.affinityGeneration) &&
|
|
287
|
-
Number.isInteger(parsed.affinityGeneration) &&
|
|
288
|
-
parsed.affinityGeneration >= 0
|
|
289
|
-
? parsed.affinityGeneration
|
|
290
|
-
: 0;
|
|
291
|
-
const snapshot = {
|
|
292
|
-
mtimeMs: stats.mtimeMs,
|
|
293
|
-
size: stats.size,
|
|
294
|
-
contentHash,
|
|
295
|
-
pinnedAccountIndex,
|
|
296
|
-
affinityGeneration,
|
|
297
|
-
};
|
|
298
|
-
STORAGE_META_CACHE.set(storagePath, snapshot);
|
|
299
|
-
return { pinnedAccountIndex, affinityGeneration };
|
|
300
|
-
}
|
|
301
|
-
catch (error) {
|
|
302
|
-
// On any failure, prefer the last good snapshot for this path so we
|
|
303
|
-
// don't blow away affinity unnecessarily. Defensive: even non-transient
|
|
304
|
-
// errors fall back to the cache when one exists — better stale than
|
|
305
|
-
// wrong. Defaults are only returned when this file has never been read
|
|
306
|
-
// successfully (cache miss).
|
|
307
|
-
void error;
|
|
308
|
-
const cached = STORAGE_META_CACHE.get(storagePath);
|
|
309
|
-
if (cached) {
|
|
310
|
-
return metaFromSnapshot(cached);
|
|
311
|
-
}
|
|
312
|
-
return { pinnedAccountIndex: null, affinityGeneration: 0 };
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Backwards-compatible helper retained for tests. Prefer `readStorageMetaFromDisk`
|
|
317
|
-
* for new callers that also need `affinityGeneration`.
|
|
318
|
-
*/
|
|
319
|
-
export function readPinnedAccountIndexFromDisk(storagePath = getStoragePath()) {
|
|
320
|
-
return readStorageMetaFromDisk(storagePath).pinnedAccountIndex;
|
|
321
|
-
}
|
|
322
|
-
/**
|
|
323
|
-
* Test-only: reset the storage-meta content-hash cache between scenarios so
|
|
324
|
-
* each test starts from a clean read-from-disk state.
|
|
325
|
-
*/
|
|
326
|
-
export function resetPinCacheForTesting() {
|
|
327
|
-
STORAGE_META_CACHE.clear();
|
|
328
|
-
}
|
|
329
|
-
/**
|
|
330
|
-
* If the on-disk `affinityGeneration` is greater than `lastObservedGeneration`,
|
|
331
|
-
* drop every entry in `sessionAffinityStore` and return the new generation so
|
|
332
|
-
* the caller can update its tracker. Otherwise returns `lastObservedGeneration`
|
|
333
|
-
* unchanged. Extracted so the request-flow logic can be unit-tested without
|
|
334
|
-
* spinning up the full proxy. See issue #474.
|
|
335
|
-
*/
|
|
336
|
-
export function maybeInvalidateAffinityFromDisk(sessionAffinityStore, lastObservedGeneration, storagePath = getStoragePath()) {
|
|
337
|
-
const meta = readStorageMetaFromDisk(storagePath);
|
|
338
|
-
if (meta.affinityGeneration > lastObservedGeneration) {
|
|
339
|
-
sessionAffinityStore?.clearAll();
|
|
340
|
-
return meta.affinityGeneration;
|
|
341
|
-
}
|
|
342
|
-
return lastObservedGeneration;
|
|
343
|
-
}
|
|
344
|
-
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned) {
|
|
173
|
+
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy) {
|
|
345
174
|
if (isPinned) {
|
|
346
175
|
// When the user has manually pinned an account, the proxy MUST NOT
|
|
347
176
|
// clobber that pin via markSwitched("rotation"), saveToDiskDebounced(),
|
|
@@ -367,7 +196,14 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
|
|
|
367
196
|
// `saveToDiskDebounced` + CLI sync still run in both modes: they snapshot the
|
|
368
197
|
// current in-memory state / mirror the CLI selection and do not advance the
|
|
369
198
|
// in-memory rotation cursor.
|
|
370
|
-
|
|
199
|
+
//
|
|
200
|
+
// Sequential scheduling fix (#509): in "sequential" mode a within-request
|
|
201
|
+
// fallback to a different account must NOT advance the drain-first primary
|
|
202
|
+
// pointer — only a true primary-exhaustion in chooseAccount may do that.
|
|
203
|
+
// Apply the same guard here as the in-loop re-commit (lines 939-958) so
|
|
204
|
+
// the legacy branch cannot silently clobber the sequential drain order.
|
|
205
|
+
if (accountManager.getRoutingMutexMode() !== "enabled" &&
|
|
206
|
+
schedulingStrategy !== "sequential") {
|
|
371
207
|
await accountManager.markSwitchedLocked(account, "rotation", family);
|
|
372
208
|
}
|
|
373
209
|
accountManager.saveToDiskDebounced();
|
|
@@ -378,20 +214,6 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
|
|
|
378
214
|
// because the local status mirrors are temporarily locked.
|
|
379
215
|
}
|
|
380
216
|
}
|
|
381
|
-
function responseHeadersForClient(upstreamHeaders) {
|
|
382
|
-
const headers = {};
|
|
383
|
-
for (const [key, value] of upstreamHeaders.entries()) {
|
|
384
|
-
const normalizedKey = key.toLowerCase();
|
|
385
|
-
if (HOP_BY_HOP_HEADERS.has(normalizedKey))
|
|
386
|
-
continue;
|
|
387
|
-
if (PRIVATE_CLIENT_RESPONSE_HEADERS.has(normalizedKey))
|
|
388
|
-
continue;
|
|
389
|
-
if (DECODED_UPSTREAM_RESPONSE_HEADERS.has(normalizedKey))
|
|
390
|
-
continue;
|
|
391
|
-
headers[key] = value;
|
|
392
|
-
}
|
|
393
|
-
return headers;
|
|
394
|
-
}
|
|
395
217
|
function createRuntimeProxyHttpError(message, statusCode, code) {
|
|
396
218
|
return Object.assign(new Error(message), { statusCode, code });
|
|
397
219
|
}
|
|
@@ -548,471 +370,12 @@ function buildUpstreamUrl(req, upstreamBaseUrl, upstreamPath) {
|
|
|
548
370
|
upstream.search = incomingUrl.search;
|
|
549
371
|
return upstream.toString();
|
|
550
372
|
}
|
|
551
|
-
// Monotonic auth-failure cooldown: only extend, never shorten. Two concurrent
|
|
552
|
-
// requests on the same account can race so that an invalidation path sets a
|
|
553
|
-
// long cooldown (5 min) and a subsequent generic 401 truncates it (30 s).
|
|
554
|
-
// Reading the live coolingDownUntil before writing prevents that race.
|
|
555
|
-
function applyMonotonicAuthCooldown(accountManager, account, cooldownMs) {
|
|
556
|
-
const existing = accountManager.getAccountByIndex(account.index)?.coolingDownUntil ?? 0;
|
|
557
|
-
// Intentionally Date.now(), not the proxy's injected now(): coolingDownUntil is
|
|
558
|
-
// written by markAccountCoolingDown via nowMs() (== Date.now()), so both sides of
|
|
559
|
-
// this comparison must live in the same real-wall-clock domain. Switching to the
|
|
560
|
-
// injected now() would mis-compare an injected-clock value against a real-clock
|
|
561
|
-
// deadline and silently defeat the monotonic guard.
|
|
562
|
-
if (Date.now() + cooldownMs > existing) {
|
|
563
|
-
accountManager.markAccountCoolingDown(account, cooldownMs, "auth-failure");
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
function hasUsableAccessToken(account, now, skewMs) {
|
|
567
|
-
return (typeof account.access === "string" &&
|
|
568
|
-
account.access.trim().length > 0 &&
|
|
569
|
-
typeof account.expires === "number" &&
|
|
570
|
-
account.expires > now + Math.max(0, skewMs));
|
|
571
|
-
}
|
|
572
|
-
function isTokenRefreshRetryable(result) {
|
|
573
|
-
if (result.reason === "network_error" || result.reason === "unknown")
|
|
574
|
-
return true;
|
|
575
|
-
if (result.reason === "invalid_response")
|
|
576
|
-
return true;
|
|
577
|
-
if (result.reason === "http_error") {
|
|
578
|
-
return !(result.statusCode === HTTP_STATUS.BAD_REQUEST ||
|
|
579
|
-
result.statusCode === HTTP_STATUS.UNAUTHORIZED ||
|
|
580
|
-
result.statusCode === HTTP_STATUS.FORBIDDEN);
|
|
581
|
-
}
|
|
582
|
-
return false;
|
|
583
|
-
}
|
|
584
|
-
const runtimeRefreshCommitQueues = new WeakMap();
|
|
585
|
-
async function commitRefreshedAuthOnce(accountManager, account, auth) {
|
|
586
|
-
const key = [
|
|
587
|
-
account.index,
|
|
588
|
-
account.accountId ?? "",
|
|
589
|
-
account.email ?? "",
|
|
590
|
-
account.refreshToken,
|
|
591
|
-
].join("\0");
|
|
592
|
-
let queue = runtimeRefreshCommitQueues.get(accountManager);
|
|
593
|
-
if (!queue) {
|
|
594
|
-
queue = new Map();
|
|
595
|
-
runtimeRefreshCommitQueues.set(accountManager, queue);
|
|
596
|
-
}
|
|
597
|
-
const existing = queue.get(key);
|
|
598
|
-
if (existing)
|
|
599
|
-
return existing;
|
|
600
|
-
const pending = accountManager
|
|
601
|
-
.commitRefreshedAuth(account, auth)
|
|
602
|
-
.finally(() => queue?.delete(key));
|
|
603
|
-
queue.set(key, pending);
|
|
604
|
-
return pending;
|
|
605
|
-
}
|
|
606
|
-
async function ensureFreshAccessToken(params) {
|
|
607
|
-
const { accountManager, account, family, model, now, tokenRefreshSkewMs, tokenInvalidationCooldownMs } = params;
|
|
608
|
-
if (hasUsableAccessToken(account, now, tokenRefreshSkewMs)) {
|
|
609
|
-
return { ok: true, accessToken: account.access ?? "", account };
|
|
610
|
-
}
|
|
611
|
-
const refreshResult = await queuedRefresh(account.refreshToken);
|
|
612
|
-
if (refreshResult.type === "failed") {
|
|
613
|
-
accountManager.recordFailure(account, family, model);
|
|
614
|
-
accountManager.incrementAuthFailures(account);
|
|
615
|
-
// If the refresh endpoint itself returns an explicit invalidation message
|
|
616
|
-
// (e.g. Microsoft/Outlook SSO revokes the refresh token server-side), apply
|
|
617
|
-
// the long cooldown and signal to the caller to stop rotating rather than
|
|
618
|
-
// presenting other accounts' tokens from the same IP.
|
|
619
|
-
const invalidated = isTokenInvalidationError(refreshResult.message ?? "");
|
|
620
|
-
applyMonotonicAuthCooldown(accountManager, account, invalidated ? tokenInvalidationCooldownMs : DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
621
|
-
accountManager.saveToDiskDebounced();
|
|
622
|
-
return { ok: false, retryable: isTokenRefreshRetryable(refreshResult), invalidated };
|
|
623
|
-
}
|
|
624
|
-
const auth = {
|
|
625
|
-
type: "oauth",
|
|
626
|
-
access: refreshResult.access,
|
|
627
|
-
refresh: refreshResult.refresh,
|
|
628
|
-
expires: refreshResult.expires,
|
|
629
|
-
};
|
|
630
|
-
try {
|
|
631
|
-
const updatedAccount = (await commitRefreshedAuthOnce(accountManager, account, auth)) ?? account;
|
|
632
|
-
return {
|
|
633
|
-
ok: true,
|
|
634
|
-
accessToken: updatedAccount.access ?? refreshResult.access,
|
|
635
|
-
account: updatedAccount,
|
|
636
|
-
};
|
|
637
|
-
}
|
|
638
|
-
catch {
|
|
639
|
-
accountManager.recordFailure(account, family, model);
|
|
640
|
-
accountManager.markAccountCoolingDown(account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
|
|
641
|
-
accountManager.saveToDiskDebounced();
|
|
642
|
-
return { ok: false, retryable: true };
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
373
|
function resolveAccountId(account, accessToken) {
|
|
646
374
|
const stored = account.accountId?.trim();
|
|
647
375
|
if (stored)
|
|
648
376
|
return stored;
|
|
649
377
|
return extractAccountId(accessToken)?.trim() || null;
|
|
650
378
|
}
|
|
651
|
-
function parseRetryAfterHeaderMs(headers, now) {
|
|
652
|
-
const retryAfterMs = headers.get("retry-after-ms");
|
|
653
|
-
if (retryAfterMs) {
|
|
654
|
-
const parsed = Number.parseInt(retryAfterMs, 10);
|
|
655
|
-
if (Number.isFinite(parsed) && parsed > 0)
|
|
656
|
-
return parsed;
|
|
657
|
-
}
|
|
658
|
-
const retryAfter = headers.get("retry-after");
|
|
659
|
-
if (!retryAfter)
|
|
660
|
-
return null;
|
|
661
|
-
const asSeconds = Number.parseInt(retryAfter, 10);
|
|
662
|
-
if (Number.isFinite(asSeconds) && asSeconds > 0)
|
|
663
|
-
return asSeconds * 1000;
|
|
664
|
-
const asDate = Date.parse(retryAfter);
|
|
665
|
-
if (Number.isFinite(asDate) && asDate > now)
|
|
666
|
-
return asDate - now;
|
|
667
|
-
return null;
|
|
668
|
-
}
|
|
669
|
-
function parseRetryAfterBodyMs(bodyText, now) {
|
|
670
|
-
if (!bodyText.trim())
|
|
671
|
-
return null;
|
|
672
|
-
try {
|
|
673
|
-
const parsed = JSON.parse(bodyText);
|
|
674
|
-
if (!isRecord(parsed) || !isRecord(parsed.error))
|
|
675
|
-
return null;
|
|
676
|
-
const retryAfterMs = Number(parsed.error.retry_after_ms);
|
|
677
|
-
if (Number.isFinite(retryAfterMs) && retryAfterMs > 0)
|
|
678
|
-
return retryAfterMs;
|
|
679
|
-
const retryAfterSeconds = Number(parsed.error.retry_after);
|
|
680
|
-
if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) {
|
|
681
|
-
return retryAfterSeconds * 1000;
|
|
682
|
-
}
|
|
683
|
-
const resetAtRaw = Number(parsed.error.resets_at ?? parsed.error.reset_at);
|
|
684
|
-
if (Number.isFinite(resetAtRaw) && resetAtRaw > 0) {
|
|
685
|
-
const resetAtMs = resetAtRaw < 10_000_000_000 ? resetAtRaw * 1000 : resetAtRaw;
|
|
686
|
-
if (resetAtMs > now)
|
|
687
|
-
return resetAtMs - now;
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
catch {
|
|
691
|
-
return null;
|
|
692
|
-
}
|
|
693
|
-
return null;
|
|
694
|
-
}
|
|
695
|
-
async function readErrorBody(response, timeoutMs, maxBytes = 1024 * 1024) {
|
|
696
|
-
// The outbound fetch's abort timer is cleared once headers arrive, so a
|
|
697
|
-
// stalled error body would otherwise hang this handler forever (the success
|
|
698
|
-
// path is per-chunk stall-bounded; the error path was not). Read the body via
|
|
699
|
-
// a reader, bound it by an idle timeout AND a size cap, and cancel the stream
|
|
700
|
-
// on timeout/overflow so the socket is released.
|
|
701
|
-
const body = response.body;
|
|
702
|
-
if (!body || typeof body.getReader !== "function") {
|
|
703
|
-
// Fallback for impls without a streamable body: race text() against a timer.
|
|
704
|
-
try {
|
|
705
|
-
return await withTimeout(response.text(), timeoutMs, () => undefined, "error body stalled");
|
|
706
|
-
}
|
|
707
|
-
catch {
|
|
708
|
-
return "";
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
const reader = body.getReader();
|
|
712
|
-
const chunks = [];
|
|
713
|
-
let total = 0;
|
|
714
|
-
try {
|
|
715
|
-
for (;;) {
|
|
716
|
-
let idleTimer;
|
|
717
|
-
const idle = new Promise((_resolve, reject) => {
|
|
718
|
-
idleTimer = setTimeout(() => reject(new Error("error body stalled")), Math.max(1, timeoutMs));
|
|
719
|
-
});
|
|
720
|
-
let result;
|
|
721
|
-
try {
|
|
722
|
-
result = await Promise.race([reader.read(), idle]);
|
|
723
|
-
}
|
|
724
|
-
finally {
|
|
725
|
-
if (idleTimer)
|
|
726
|
-
clearTimeout(idleTimer);
|
|
727
|
-
}
|
|
728
|
-
if (result.done)
|
|
729
|
-
break;
|
|
730
|
-
if (result.value) {
|
|
731
|
-
total += result.value.byteLength;
|
|
732
|
-
if (total > maxBytes)
|
|
733
|
-
break; // cap: enough for diagnostics, no OOM
|
|
734
|
-
chunks.push(result.value);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
catch {
|
|
739
|
-
// stalled or errored — fall through with whatever we have
|
|
740
|
-
}
|
|
741
|
-
finally {
|
|
742
|
-
await reader.cancel().catch(() => undefined);
|
|
743
|
-
}
|
|
744
|
-
try {
|
|
745
|
-
return Buffer.concat(chunks).toString("utf8");
|
|
746
|
-
}
|
|
747
|
-
catch {
|
|
748
|
-
return "";
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
const TOKEN_INVALIDATED_CODE = "token_invalidated";
|
|
752
|
-
const TOKEN_INVALIDATED_FALLBACK_MESSAGE = "OAuth token has been invalidated. Please re-login.";
|
|
753
|
-
// Both invalidation exit paths (refresh-failure and upstream-401) must hand the
|
|
754
|
-
// client the same machine-readable shape — { error: { message, code:
|
|
755
|
-
// "token_invalidated" } } — so a consumer keying off error.code behaves
|
|
756
|
-
// identically regardless of which vector fired. The upstream forwards a raw body
|
|
757
|
-
// with no guaranteed code, so we wrap it here while preserving its human-readable
|
|
758
|
-
// message when one is present.
|
|
759
|
-
export function buildTokenInvalidationBody(upstreamBodyText) {
|
|
760
|
-
let message = TOKEN_INVALIDATED_FALLBACK_MESSAGE;
|
|
761
|
-
const trimmed = upstreamBodyText.trim();
|
|
762
|
-
if (trimmed) {
|
|
763
|
-
try {
|
|
764
|
-
const parsed = JSON.parse(trimmed);
|
|
765
|
-
if (isRecord(parsed)) {
|
|
766
|
-
const direct = parsed.message;
|
|
767
|
-
if (typeof direct === "string" && direct.trim()) {
|
|
768
|
-
message = direct.trim();
|
|
769
|
-
}
|
|
770
|
-
else if (isRecord(parsed.error)) {
|
|
771
|
-
const nested = parsed.error.message;
|
|
772
|
-
if (typeof nested === "string" && nested.trim()) {
|
|
773
|
-
message = nested.trim();
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
|
-
catch {
|
|
779
|
-
// Non-JSON upstream body (e.g. HTML error page): keep the stable fallback
|
|
780
|
-
// message rather than echoing markup back to the client.
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
return JSON.stringify({ error: { message, code: TOKEN_INVALIDATED_CODE } });
|
|
784
|
-
}
|
|
785
|
-
function extractErrorCodeFromBody(bodyText) {
|
|
786
|
-
if (!bodyText.trim())
|
|
787
|
-
return null;
|
|
788
|
-
try {
|
|
789
|
-
const parsed = JSON.parse(bodyText);
|
|
790
|
-
if (!isRecord(parsed))
|
|
791
|
-
return null;
|
|
792
|
-
const directCode = parsed.code;
|
|
793
|
-
if (typeof directCode === "string" && directCode.trim()) {
|
|
794
|
-
return directCode.trim();
|
|
795
|
-
}
|
|
796
|
-
const maybeError = parsed.error;
|
|
797
|
-
if (!isRecord(maybeError))
|
|
798
|
-
return null;
|
|
799
|
-
const nestedCode = maybeError.code;
|
|
800
|
-
return typeof nestedCode === "string" && nestedCode.trim()
|
|
801
|
-
? nestedCode.trim()
|
|
802
|
-
: null;
|
|
803
|
-
}
|
|
804
|
-
catch {
|
|
805
|
-
return null;
|
|
806
|
-
}
|
|
807
|
-
}
|
|
808
|
-
function getQuotaWindowWaitMs(headers, prefix, now) {
|
|
809
|
-
const resetAfterSeconds = Number.parseInt(headers.get(`${prefix}-reset-after-seconds`) ?? "", 10);
|
|
810
|
-
if (Number.isFinite(resetAfterSeconds) && resetAfterSeconds > 0) {
|
|
811
|
-
return resetAfterSeconds * 1000;
|
|
812
|
-
}
|
|
813
|
-
const resetAtRaw = headers.get(`${prefix}-reset-at`);
|
|
814
|
-
if (!resetAtRaw)
|
|
815
|
-
return 0;
|
|
816
|
-
const trimmed = resetAtRaw.trim();
|
|
817
|
-
let resetAtMs = 0;
|
|
818
|
-
if (/^\d+$/.test(trimmed)) {
|
|
819
|
-
const parsed = Number.parseInt(trimmed, 10);
|
|
820
|
-
if (Number.isFinite(parsed) && parsed > 0) {
|
|
821
|
-
resetAtMs = parsed < 10_000_000_000 ? parsed * 1000 : parsed;
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
else {
|
|
825
|
-
const parsedDate = Date.parse(trimmed);
|
|
826
|
-
if (Number.isFinite(parsedDate))
|
|
827
|
-
resetAtMs = parsedDate;
|
|
828
|
-
}
|
|
829
|
-
return resetAtMs > now ? resetAtMs - now : 0;
|
|
830
|
-
}
|
|
831
|
-
function getQuotaNearExhaustionWaitMs(headers, remainingThreshold, now) {
|
|
832
|
-
const usedThreshold = 100 - Math.max(0, Math.min(100, remainingThreshold));
|
|
833
|
-
const candidates = [];
|
|
834
|
-
for (const prefix of ["x-codex-primary", "x-codex-secondary"]) {
|
|
835
|
-
const used = Number(headers.get(`${prefix}-used-percent`) ?? "");
|
|
836
|
-
if (!Number.isFinite(used) || used < usedThreshold)
|
|
837
|
-
continue;
|
|
838
|
-
const waitMs = getQuotaWindowWaitMs(headers, prefix, now);
|
|
839
|
-
if (waitMs > 0)
|
|
840
|
-
candidates.push(waitMs);
|
|
841
|
-
}
|
|
842
|
-
return candidates.length > 0 ? Math.max(...candidates) : 0;
|
|
843
|
-
}
|
|
844
|
-
/**
|
|
845
|
-
* `chooseAccount` is a SYNC selector that internally advances the rotation
|
|
846
|
-
* cursor (the session-affinity-preferred branch and the round-robin fallback
|
|
847
|
-
* both call `accountManager.markSwitched(...)`, and the hybrid selector advances
|
|
848
|
-
* its own cursor). It does NOT acquire the routing mutex itself.
|
|
849
|
-
*
|
|
850
|
-
* Concurrency (L4): when `routingMutex === "enabled"`, the proxy hot path runs
|
|
851
|
-
* this whole call AND the subsequent `markSwitchedLocked` commit inside a single
|
|
852
|
-
* `withRoutingMutex` acquisition, so concurrent requests serialize selection +
|
|
853
|
-
* cursor advance and cannot stampede the same account. `withRoutingMutex` is
|
|
854
|
-
* reentrant (AsyncLocalStorage), so the nested `markSwitchedLocked` — and the
|
|
855
|
-
* later one in `persistRuntimeActiveAccount` — run inline without re-acquiring
|
|
856
|
-
* the non-reentrant FIFO queue (no deadlock). In legacy mode the inline
|
|
857
|
-
* `markSwitched` calls below are used unchanged and no lock is taken, so default
|
|
858
|
-
* behavior and perf are identical. See the hot-path caller in
|
|
859
|
-
* `startRuntimeRotationProxy` and the regression in
|
|
860
|
-
* `test/runtime-rotation-proxy.test.ts`.
|
|
861
|
-
*/
|
|
862
|
-
export function chooseAccount(params) {
|
|
863
|
-
const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, skipReasons, stickyBoostByAccount, pidOffsetEnabled, schedulingStrategy, } = params;
|
|
864
|
-
// Manual pin (from `codex-multi-auth switch <n>`) overrides every other
|
|
865
|
-
// selection signal. We do NOT call markSwitched here — the proxy must not
|
|
866
|
-
// clobber the pin set by the CLI. See issue #474.
|
|
867
|
-
if (typeof pinnedIndex === "number") {
|
|
868
|
-
if (attemptedIndexes.has(pinnedIndex)) {
|
|
869
|
-
skipReasons?.set(pinnedIndex, "already-attempted");
|
|
870
|
-
return null;
|
|
871
|
-
}
|
|
872
|
-
if (pinnedIndex < 0 || pinnedIndex >= accountManager.getAccountCount()) {
|
|
873
|
-
skipReasons?.set(pinnedIndex, "missing");
|
|
874
|
-
return null;
|
|
875
|
-
}
|
|
876
|
-
if (policy?.blockedAccountIndexes.has(pinnedIndex)) {
|
|
877
|
-
skipReasons?.set(pinnedIndex, "policy-blocked");
|
|
878
|
-
return null;
|
|
879
|
-
}
|
|
880
|
-
const pinned = accountManager.getAccountByIndex(pinnedIndex);
|
|
881
|
-
if (!pinned || pinned.enabled === false) {
|
|
882
|
-
skipReasons?.set(pinnedIndex, "disabled");
|
|
883
|
-
return null;
|
|
884
|
-
}
|
|
885
|
-
const reason = accountManager.getAccountRuntimeSkipReason(pinnedIndex, family, model);
|
|
886
|
-
if (reason) {
|
|
887
|
-
skipReasons?.set(pinnedIndex, reason);
|
|
888
|
-
return null;
|
|
889
|
-
}
|
|
890
|
-
return pinned;
|
|
891
|
-
}
|
|
892
|
-
// Sequential / drain-first mode (issue #509): the active account governs ALL
|
|
893
|
-
// new requests, so we deliberately SKIP the per-session affinity tier — there
|
|
894
|
-
// is no per-chat stickiness, every request follows the single active account.
|
|
895
|
-
// The manual pin above still wins (handled first). When the active account is
|
|
896
|
-
// exhausted the selector advances to the next available account and earlier
|
|
897
|
-
// accounts reclaim the slot once their quota recovers.
|
|
898
|
-
if (schedulingStrategy === "sequential") {
|
|
899
|
-
const selected = accountManager.getCurrentOrNextForFamilySequential(family, model, policy?.blockedAccountIndexes);
|
|
900
|
-
if (selected &&
|
|
901
|
-
!attemptedIndexes.has(selected.index) &&
|
|
902
|
-
!policy?.blockedAccountIndexes.has(selected.index)) {
|
|
903
|
-
const reason = accountManager.getAccountRuntimeSkipReason(selected.index, family, model);
|
|
904
|
-
if (!reason)
|
|
905
|
-
return selected;
|
|
906
|
-
skipReasons?.set(selected.index, reason);
|
|
907
|
-
}
|
|
908
|
-
// Active account was attempted/blocked/skipped this request (e.g. it just
|
|
909
|
-
// rate-limited mid-loop): fall through to the shared linear scan to find
|
|
910
|
-
// the next eligible account to TRY. Pass advanceActivePointer=false so a
|
|
911
|
-
// transient, non-exhausting failure on the active account does not
|
|
912
|
-
// permanently move the drain-first primary — only
|
|
913
|
-
// getCurrentOrNextForFamilySequential advances it, and only on true
|
|
914
|
-
// exhaustion.
|
|
915
|
-
return chooseLinearScanFallback({
|
|
916
|
-
accountManager,
|
|
917
|
-
family,
|
|
918
|
-
model,
|
|
919
|
-
attemptedIndexes,
|
|
920
|
-
policy,
|
|
921
|
-
skipReasons,
|
|
922
|
-
advanceActivePointer: false,
|
|
923
|
-
});
|
|
924
|
-
}
|
|
925
|
-
const preferredIndex = sessionAffinityStore?.getPreferredAccountIndex(sessionKey, now);
|
|
926
|
-
if (typeof preferredIndex === "number" &&
|
|
927
|
-
!attemptedIndexes.has(preferredIndex) &&
|
|
928
|
-
!policy?.blockedAccountIndexes.has(preferredIndex)) {
|
|
929
|
-
const preferred = accountManager.getAccountByIndex(preferredIndex);
|
|
930
|
-
if (preferred) {
|
|
931
|
-
const reason = accountManager.getAccountRuntimeSkipReason(preferred.index, family, model);
|
|
932
|
-
if (reason) {
|
|
933
|
-
skipReasons?.set(preferred.index, reason);
|
|
934
|
-
}
|
|
935
|
-
else {
|
|
936
|
-
// L4 (deferred): unlocked cursor mutation — see chooseAccount header.
|
|
937
|
-
accountManager.markSwitched(preferred, "rotation", family);
|
|
938
|
-
return preferred;
|
|
939
|
-
}
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
const selected = accountManager.getCurrentOrNextForFamilyHybrid(family, model, {
|
|
943
|
-
scoreBoostByAccount: {
|
|
944
|
-
...(policy?.scoreBoostByAccount ?? {}),
|
|
945
|
-
...(stickyBoostByAccount ?? {}),
|
|
946
|
-
},
|
|
947
|
-
// accounts-05: carry the PID-offset distribution into the default-on proxy
|
|
948
|
-
// path too (index.ts already does). Without it, parallel proxy processes can
|
|
949
|
-
// stampede the same account instead of spreading across the pool.
|
|
950
|
-
pidOffsetEnabled,
|
|
951
|
-
});
|
|
952
|
-
if (selected &&
|
|
953
|
-
!attemptedIndexes.has(selected.index) &&
|
|
954
|
-
!policy?.blockedAccountIndexes.has(selected.index)) {
|
|
955
|
-
const reason = accountManager.getAccountRuntimeSkipReason(selected.index, family, model);
|
|
956
|
-
if (!reason)
|
|
957
|
-
return selected;
|
|
958
|
-
skipReasons?.set(selected.index, reason);
|
|
959
|
-
}
|
|
960
|
-
return chooseLinearScanFallback({
|
|
961
|
-
accountManager,
|
|
962
|
-
family,
|
|
963
|
-
model,
|
|
964
|
-
attemptedIndexes,
|
|
965
|
-
policy,
|
|
966
|
-
skipReasons,
|
|
967
|
-
});
|
|
968
|
-
}
|
|
969
|
-
/**
|
|
970
|
-
* Shared linear-scan fallback used by both the hybrid and sequential selection
|
|
971
|
-
* paths in `chooseAccount`. Walks every account in pool order and returns the
|
|
972
|
-
* first one that is not already attempted, not policy-blocked, and has no
|
|
973
|
-
* runtime skip reason (rate-limited / cooling down / circuit-open), recording a
|
|
974
|
-
* skip reason for each rejected candidate. Returns null when no eligible
|
|
975
|
-
* account remains.
|
|
976
|
-
*
|
|
977
|
-
* `advanceActivePointer` (default `true`) controls whether the winner is
|
|
978
|
-
* committed as the new active/cursor position via `markSwitched`. The hybrid
|
|
979
|
-
* path wants this (round-robin advance). The sequential path passes `false`:
|
|
980
|
-
* its within-request fallback only needs an account to TRY this request, and
|
|
981
|
-
* must NOT move `currentAccountIndexByFamily` — otherwise a transient,
|
|
982
|
-
* non-exhausting failure on the active account (which leaves it `isUsable` but
|
|
983
|
-
* present in `attemptedIndexes`) would permanently switch the drain-first
|
|
984
|
-
* primary even though it was never exhausted (issue #509 regression caught in
|
|
985
|
-
* review). In sequential mode only `getCurrentOrNextForFamilySequential` is
|
|
986
|
-
* allowed to advance the active pointer, and only on true exhaustion.
|
|
987
|
-
*/
|
|
988
|
-
function chooseLinearScanFallback(params) {
|
|
989
|
-
const { accountManager, family, model, attemptedIndexes, policy, skipReasons, advanceActivePointer = true, } = params;
|
|
990
|
-
for (const account of accountManager.getAccountsSnapshot()) {
|
|
991
|
-
if (attemptedIndexes.has(account.index)) {
|
|
992
|
-
skipReasons?.set(account.index, "already-attempted");
|
|
993
|
-
continue;
|
|
994
|
-
}
|
|
995
|
-
if (policy?.blockedAccountIndexes.has(account.index)) {
|
|
996
|
-
skipReasons?.set(account.index, "policy-blocked");
|
|
997
|
-
continue;
|
|
998
|
-
}
|
|
999
|
-
const reason = accountManager.getAccountRuntimeSkipReason(account.index, family, model);
|
|
1000
|
-
if (!reason) {
|
|
1001
|
-
const live = accountManager.getAccountByIndex(account.index);
|
|
1002
|
-
if (!live)
|
|
1003
|
-
continue;
|
|
1004
|
-
// L4 (deferred): unlocked cursor mutation — see chooseAccount header.
|
|
1005
|
-
// Skipped in sequential mode (advanceActivePointer=false) so a
|
|
1006
|
-
// within-request retry never reassigns the drain-first primary.
|
|
1007
|
-
if (advanceActivePointer) {
|
|
1008
|
-
accountManager.markSwitched(live, "rotation", family);
|
|
1009
|
-
}
|
|
1010
|
-
return live;
|
|
1011
|
-
}
|
|
1012
|
-
skipReasons?.set(account.index, reason);
|
|
1013
|
-
}
|
|
1014
|
-
return null;
|
|
1015
|
-
}
|
|
1016
379
|
function writeJson(res, status, payload) {
|
|
1017
380
|
res.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
|
1018
381
|
res.end(`${JSON.stringify(payload)}\n`);
|
|
@@ -1033,9 +396,6 @@ function writeUnauthorized(res) {
|
|
|
1033
396
|
},
|
|
1034
397
|
});
|
|
1035
398
|
}
|
|
1036
|
-
function normalizeExhaustionStatus(reason) {
|
|
1037
|
-
return reason === "rate-limit" ? HTTP_STATUS.TOO_MANY_REQUESTS : 503;
|
|
1038
|
-
}
|
|
1039
399
|
function writePoolExhausted(params) {
|
|
1040
400
|
const { res, accountManager, family, model, reason } = params;
|
|
1041
401
|
const waitMs = accountManager.getMinWaitTimeForFamily(family, model);
|
|
@@ -1060,82 +420,9 @@ function writePoolExhausted(params) {
|
|
|
1060
420
|
},
|
|
1061
421
|
});
|
|
1062
422
|
}
|
|
1063
|
-
export function buildPinnedUnavailableErrorBody(pinnedIndex, accountSkipReasons) {
|
|
1064
|
-
const normalizedPinnedIndex = typeof pinnedIndex === "number" ? pinnedIndex : null;
|
|
1065
|
-
const skipReason = normalizedPinnedIndex !== null
|
|
1066
|
-
? accountSkipReasons.get(normalizedPinnedIndex) ?? null
|
|
1067
|
-
: null;
|
|
1068
|
-
const reasonSuffix = skipReason ? ` (${skipReason})` : "";
|
|
1069
|
-
const displayIndex = (normalizedPinnedIndex ?? 0) + 1;
|
|
1070
|
-
return {
|
|
1071
|
-
message: `Pinned account ${displayIndex} is currently unavailable${reasonSuffix}; run \`codex-multi-auth status\` for details, or \`codex-multi-auth unpin\` to allow rotation.`,
|
|
1072
|
-
code: "codex_pinned_account_unavailable",
|
|
1073
|
-
pinnedAccountIndex: normalizedPinnedIndex,
|
|
1074
|
-
reason: skipReason,
|
|
1075
|
-
account_skip_reasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
1076
|
-
String(index),
|
|
1077
|
-
reason,
|
|
1078
|
-
])),
|
|
1079
|
-
};
|
|
1080
|
-
}
|
|
1081
|
-
async function withTimeout(promise, timeoutMs, onTimeout, message) {
|
|
1082
|
-
let timeout;
|
|
1083
|
-
try {
|
|
1084
|
-
return await Promise.race([
|
|
1085
|
-
promise,
|
|
1086
|
-
new Promise((_resolve, reject) => {
|
|
1087
|
-
timeout = setTimeout(() => {
|
|
1088
|
-
onTimeout();
|
|
1089
|
-
reject(new Error(message));
|
|
1090
|
-
}, Math.max(1, timeoutMs));
|
|
1091
|
-
}),
|
|
1092
|
-
]);
|
|
1093
|
-
}
|
|
1094
|
-
finally {
|
|
1095
|
-
if (timeout)
|
|
1096
|
-
clearTimeout(timeout);
|
|
1097
|
-
}
|
|
1098
|
-
}
|
|
1099
|
-
async function forwardStreamingResponse(upstream, res, status, onStreamError, streamStallTimeoutMs) {
|
|
1100
|
-
status.streamsStarted += 1;
|
|
1101
|
-
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
1102
|
-
if (!upstream.body) {
|
|
1103
|
-
res.end();
|
|
1104
|
-
return true;
|
|
1105
|
-
}
|
|
1106
|
-
const reader = upstream.body.getReader();
|
|
1107
|
-
res.on("close", () => {
|
|
1108
|
-
if (!res.writableEnded) {
|
|
1109
|
-
void reader.cancel().catch(() => undefined);
|
|
1110
|
-
}
|
|
1111
|
-
});
|
|
1112
|
-
try {
|
|
1113
|
-
while (true) {
|
|
1114
|
-
const { done, value } = await withTimeout(reader.read(), streamStallTimeoutMs, () => {
|
|
1115
|
-
void reader.cancel().catch(() => undefined);
|
|
1116
|
-
}, `upstream stream stalled after ${streamStallTimeoutMs}ms`);
|
|
1117
|
-
if (done)
|
|
1118
|
-
break;
|
|
1119
|
-
if (value && value.byteLength > 0) {
|
|
1120
|
-
res.write(Buffer.from(value));
|
|
1121
|
-
}
|
|
1122
|
-
}
|
|
1123
|
-
res.end();
|
|
1124
|
-
return true;
|
|
1125
|
-
}
|
|
1126
|
-
catch (error) {
|
|
1127
|
-
status.lastError = error instanceof Error ? error.message : String(error);
|
|
1128
|
-
onStreamError();
|
|
1129
|
-
if (!res.destroyed) {
|
|
1130
|
-
res.destroy(error instanceof Error ? error : undefined);
|
|
1131
|
-
}
|
|
1132
|
-
return false;
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
423
|
export async function startRuntimeRotationProxy(options) {
|
|
1136
424
|
const pluginConfig = loadPluginConfig();
|
|
1137
|
-
|
|
1138
|
-
const knownAccountManagers = new Set([activeAccountManager]);
|
|
425
|
+
const activeAccountManager = options.accountManager ?? (await AccountManager.loadFromDisk());
|
|
1139
426
|
// accounts-01/08: apply the configured routing-mutex mode so the proxy's
|
|
1140
427
|
// async select->commit path (persistRuntimeActiveAccount) can serialize cursor
|
|
1141
428
|
// mutations when routingMutex="enabled". Legacy mode keeps the inline fast path.
|
|
@@ -1149,8 +436,8 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1149
436
|
// binding a non-loopback host would expose every managed account to the
|
|
1150
437
|
// network, so it is refused unconditionally.
|
|
1151
438
|
if (!isLoopbackHost(host)) {
|
|
1152
|
-
throw new
|
|
1153
|
-
"It forwards managed OAuth tokens and is loopback-only.");
|
|
439
|
+
throw new CodexValidationError(`Runtime rotation proxy refuses to bind non-loopback host "${host}". ` +
|
|
440
|
+
"It forwards managed OAuth tokens and is loopback-only.", { field: "host", expected: "a loopback host", context: { host } });
|
|
1154
441
|
}
|
|
1155
442
|
// Normalize the validated host into its two representations exactly once so the
|
|
1156
443
|
// listen() bind and the emitted baseUrl can never disagree under concurrent
|
|
@@ -1165,7 +452,7 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1165
452
|
? options.clientApiKey.trim()
|
|
1166
453
|
: null;
|
|
1167
454
|
if (!clientApiKey) {
|
|
1168
|
-
throw new
|
|
455
|
+
throw new CodexValidationError("Runtime rotation proxy requires a clientApiKey.", { field: "clientApiKey", expected: "a non-empty string" });
|
|
1169
456
|
}
|
|
1170
457
|
const now = options.now ?? Date.now;
|
|
1171
458
|
const tokenRefreshSkewMs = getTokenRefreshSkewMs(pluginConfig);
|
|
@@ -1174,8 +461,6 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1174
461
|
const tokenInvalidationCooldownMs = getTokenInvalidationCooldownMs(pluginConfig);
|
|
1175
462
|
const minRotationIntervalMs = getMinRotationIntervalMs(pluginConfig);
|
|
1176
463
|
const pidOffsetEnabled = getPidOffsetEnabled(pluginConfig);
|
|
1177
|
-
let lastGlobalAccountIndex = null;
|
|
1178
|
-
let lastGlobalSwitchAt = 0;
|
|
1179
464
|
const fetchTimeoutMs = options.fetchTimeoutMs ?? getFetchTimeoutMs(pluginConfig);
|
|
1180
465
|
const streamStallTimeoutMs = options.streamStallTimeoutMs ?? getStreamStallTimeoutMs(pluginConfig);
|
|
1181
466
|
const configuredMaxRetries = getRetryAllAccountsMaxRetries(pluginConfig);
|
|
@@ -1193,692 +478,670 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1193
478
|
// Initialize from disk so the proxy starts in sync with whatever generation
|
|
1194
479
|
// the storage file already shows. Subsequent disk bumps (from CLI commands)
|
|
1195
480
|
// are detected per-request via `maybeInvalidateAffinityFromDisk`.
|
|
1196
|
-
|
|
1197
|
-
const
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
knownAccountManagers.add(reloaded);
|
|
1228
|
-
lastStaleRuntimeReloadAt = Date.now();
|
|
1229
|
-
recordRuntimeReload("pool-exhausted-no-account");
|
|
1230
|
-
return reloaded;
|
|
1231
|
-
})()
|
|
1232
|
-
.catch((error) => {
|
|
1233
|
-
status.lastError = error instanceof Error ? error.message : String(error);
|
|
1234
|
-
return null;
|
|
1235
|
-
})
|
|
1236
|
-
.finally(() => {
|
|
1237
|
-
staleRuntimeReloadPromise = null;
|
|
481
|
+
const lastObservedAffinityGeneration = readStorageMetaFromDisk().affinityGeneration;
|
|
482
|
+
const state = createRotationProxyState({
|
|
483
|
+
activeAccountManager,
|
|
484
|
+
routingMutexMode,
|
|
485
|
+
schedulingStrategy,
|
|
486
|
+
fetchImpl,
|
|
487
|
+
upstreamBaseUrl,
|
|
488
|
+
clientApiKey,
|
|
489
|
+
now,
|
|
490
|
+
tokenRefreshSkewMs,
|
|
491
|
+
networkErrorCooldownMs,
|
|
492
|
+
serverErrorCooldownMs,
|
|
493
|
+
tokenInvalidationCooldownMs,
|
|
494
|
+
minRotationIntervalMs,
|
|
495
|
+
pidOffsetEnabled,
|
|
496
|
+
fetchTimeoutMs,
|
|
497
|
+
streamStallTimeoutMs,
|
|
498
|
+
maxRuntimeAccountAttempts,
|
|
499
|
+
maxRequestBodyBytes,
|
|
500
|
+
quotaRemainingPercentThreshold,
|
|
501
|
+
sessionAffinityStore,
|
|
502
|
+
lastObservedAffinityGeneration,
|
|
503
|
+
});
|
|
504
|
+
const server = createServer((req, res) => {
|
|
505
|
+
void handleRequest(state, req, res);
|
|
506
|
+
});
|
|
507
|
+
const sockets = new Set();
|
|
508
|
+
server.on("connection", (socket) => {
|
|
509
|
+
sockets.add(socket);
|
|
510
|
+
socket.once("close", () => {
|
|
511
|
+
sockets.delete(socket);
|
|
1238
512
|
});
|
|
1239
|
-
|
|
513
|
+
});
|
|
514
|
+
const onPostStartupServerError = (error) => {
|
|
515
|
+
state.status.lastError = error.message;
|
|
1240
516
|
};
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
517
|
+
await new Promise((resolve, reject) => {
|
|
518
|
+
const onError = (error) => {
|
|
519
|
+
server.off("listening", onListening);
|
|
520
|
+
reject(error);
|
|
521
|
+
};
|
|
522
|
+
const onListening = () => {
|
|
523
|
+
server.off("error", onError);
|
|
524
|
+
resolve();
|
|
525
|
+
};
|
|
526
|
+
server.once("error", onError);
|
|
527
|
+
server.once("listening", onListening);
|
|
528
|
+
server.listen(port, bindHost);
|
|
529
|
+
});
|
|
530
|
+
server.on("error", onPostStartupServerError);
|
|
531
|
+
const address = server.address();
|
|
532
|
+
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
533
|
+
return {
|
|
534
|
+
host: bindHost,
|
|
535
|
+
port: resolvedPort,
|
|
536
|
+
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
537
|
+
close: async () => {
|
|
538
|
+
await closeServer(server, sockets);
|
|
539
|
+
await state.activeAccountManager.flushPendingSave();
|
|
540
|
+
},
|
|
541
|
+
getStatus: () => ({
|
|
542
|
+
...state.status,
|
|
543
|
+
// Redact any email/token material that leaked into a raw upstream or
|
|
544
|
+
// refresh error string before exposing it to status/report consumers
|
|
545
|
+
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
546
|
+
lastError: state.status.lastError === null ? null : maskString(state.status.lastError),
|
|
547
|
+
}),
|
|
1247
548
|
};
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
549
|
+
}
|
|
550
|
+
async function handleRequest(state, req, res) {
|
|
551
|
+
// Per-request trace id (errors-logging-03): distinct from sessionKey, which
|
|
552
|
+
// is shared across a thread's requests. Bound to this request's async context
|
|
553
|
+
// so every proxyLog line and usage row can be correlated to one request.
|
|
554
|
+
const traceId = randomUUID();
|
|
555
|
+
return runWithCorrelationId(traceId, () => handleRequestInner(state, req, res, traceId));
|
|
556
|
+
}
|
|
557
|
+
async function handleRequestInner(state, req, res, traceId) {
|
|
558
|
+
let usageRecorder = null;
|
|
559
|
+
let accountManager = state.activeAccountManager;
|
|
560
|
+
try {
|
|
561
|
+
const incomingUrl = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
562
|
+
// Authenticate before discriminating path/method so an unauthenticated
|
|
563
|
+
// caller cannot enumerate which endpoints exist: an unknown caller always
|
|
564
|
+
// gets 401, never a 404 that would confirm a path is invalid (vs. just
|
|
565
|
+
// unauthorized). Authorized callers still fall through to the 404 below
|
|
566
|
+
// when they hit an unsupported path/method.
|
|
567
|
+
const incomingHeaders = headersFromIncoming(req);
|
|
568
|
+
if (!isAuthorizedClient(incomingHeaders, state.clientApiKey)) {
|
|
569
|
+
writeUnauthorized(res);
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
573
|
+
const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
|
|
574
|
+
const isThreadGoalRequest = (req.method === "GET" || req.method === "POST") &&
|
|
575
|
+
isThreadGoalPath(incomingUrl.pathname);
|
|
576
|
+
if (!isResponsesRequest && !isModelsRequest && !isThreadGoalRequest) {
|
|
577
|
+
writeMethodOrPathError(res);
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
state.status.totalRequests += 1;
|
|
581
|
+
const requestBody = isResponsesRequest || (isThreadGoalRequest && req.method === "POST")
|
|
582
|
+
? await readRequestBody(req, state.maxRequestBodyBytes)
|
|
583
|
+
: Buffer.alloc(0);
|
|
584
|
+
const context = isModelsRequest
|
|
585
|
+
? buildModelsRequestContext(req)
|
|
586
|
+
: isThreadGoalRequest
|
|
587
|
+
? buildThreadGoalRequestContext(req, requestBody, incomingUrl.pathname)
|
|
588
|
+
: buildResponsesRequestContext(req, requestBody);
|
|
589
|
+
const requestStartedAt = state.now();
|
|
590
|
+
let policyDecision = null;
|
|
591
|
+
let projectKey = null;
|
|
592
|
+
let policyError = null;
|
|
1251
593
|
try {
|
|
1252
|
-
const
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
const context = isModelsRequest
|
|
1276
|
-
? buildModelsRequestContext(req)
|
|
594
|
+
const policyState = await loadRuntimePolicyState();
|
|
595
|
+
projectKey = policyState.project.projectKey;
|
|
596
|
+
policyDecision = await evaluateRuntimePolicy({
|
|
597
|
+
state: policyState,
|
|
598
|
+
accounts: accountManager.getAccountsSnapshot(),
|
|
599
|
+
model: context.model,
|
|
600
|
+
now: requestStartedAt,
|
|
601
|
+
});
|
|
602
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
603
|
+
snapshot.policyBlockedIndexes = [
|
|
604
|
+
...(policyDecision?.blockedAccountIndexes ?? new Set()),
|
|
605
|
+
];
|
|
606
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...(policyDecision?.blockedAccountIndexes ?? new Set())].map((index) => [String(index), "policy-blocked"]));
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
catch (error) {
|
|
610
|
+
policyError = error instanceof Error ? error.message : String(error);
|
|
611
|
+
state.status.lastError = policyError;
|
|
612
|
+
}
|
|
613
|
+
usageRecorder = createRuntimeUsageRecorder({
|
|
614
|
+
source: "runtime-proxy",
|
|
615
|
+
operation: isModelsRequest
|
|
616
|
+
? "models"
|
|
1277
617
|
: isThreadGoalRequest
|
|
1278
|
-
?
|
|
1279
|
-
:
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
:
|
|
1309
|
-
|
|
1310
|
-
|
|
618
|
+
? "thread-goal"
|
|
619
|
+
: "responses",
|
|
620
|
+
model: context.model,
|
|
621
|
+
projectKey,
|
|
622
|
+
requestId: traceId,
|
|
623
|
+
startedAt: requestStartedAt,
|
|
624
|
+
});
|
|
625
|
+
if (policyError) {
|
|
626
|
+
await usageRecorder.record({
|
|
627
|
+
outcome: "failure",
|
|
628
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
629
|
+
errorCode: "runtime_policy_unavailable",
|
|
630
|
+
});
|
|
631
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
632
|
+
error: {
|
|
633
|
+
message: "Runtime policy could not be loaded for this local request.",
|
|
634
|
+
code: "runtime_policy_unavailable",
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
if (policyDecision && !policyDecision.allowed) {
|
|
640
|
+
await usageRecorder.record({
|
|
641
|
+
outcome: "blocked",
|
|
642
|
+
statusCode: policyDecision.statusCode,
|
|
643
|
+
errorCode: policyDecision.errorCode,
|
|
644
|
+
});
|
|
645
|
+
writeJson(res, policyDecision.statusCode, {
|
|
646
|
+
error: {
|
|
647
|
+
message: "Runtime policy blocked this local request.",
|
|
648
|
+
code: policyDecision.errorCode ?? "policy_blocked",
|
|
649
|
+
reasons: policyDecision.reasons,
|
|
650
|
+
},
|
|
651
|
+
});
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
const upstreamUrl = buildUpstreamUrl(req, state.upstreamBaseUrl, context.upstreamPath);
|
|
655
|
+
const attemptedIndexes = new Set();
|
|
656
|
+
let exhaustionReason = "no-account";
|
|
657
|
+
let accountCount = accountManager.getAccountCount();
|
|
658
|
+
let transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
659
|
+
let transientAttempts = 0;
|
|
660
|
+
let transientExhaustionReason = null;
|
|
661
|
+
const accountSkipReasons = new Map();
|
|
662
|
+
let reloadedAfterNoAccount = false;
|
|
663
|
+
// Read the manual pin and affinity generation from disk (mtime-cached)
|
|
664
|
+
// on each request so a `codex-multi-auth switch|unpin|best` invocation
|
|
665
|
+
// in another process is honored without forcing a full AccountManager
|
|
666
|
+
// reload. The CLI bumps `affinityGeneration` on user-initiated changes
|
|
667
|
+
// so the proxy can invalidate sticky session affinity that would
|
|
668
|
+
// otherwise glue an in-flight chat thread to the previously selected
|
|
669
|
+
// account. The proxy itself never bumps the generation, so its own
|
|
670
|
+
// debounced disk writes do not clear affinity. See issue #474.
|
|
671
|
+
const storageMeta = readStorageMetaFromDisk();
|
|
672
|
+
const pinnedIndex = storageMeta.pinnedAccountIndex;
|
|
673
|
+
const isPinned = typeof pinnedIndex === "number";
|
|
674
|
+
if (storageMeta.affinityGeneration > state.lastObservedAffinityGeneration) {
|
|
675
|
+
state.sessionAffinityStore?.clearAll();
|
|
676
|
+
state.lastObservedAffinityGeneration = storageMeta.affinityGeneration;
|
|
677
|
+
}
|
|
678
|
+
while (attemptedIndexes.size < accountCount &&
|
|
679
|
+
transientAttempts < transientAttemptLimit) {
|
|
680
|
+
const rotationStickyBoost = state.minRotationIntervalMs > 0 &&
|
|
681
|
+
state.lastGlobalAccountIndex !== null &&
|
|
682
|
+
state.now() - state.lastGlobalSwitchAt < state.minRotationIntervalMs
|
|
683
|
+
? { [state.lastGlobalAccountIndex]: 1000 }
|
|
684
|
+
: {};
|
|
685
|
+
// L4 fix (routing mutex): when `routingMutex === "enabled"`, run the
|
|
686
|
+
// selection AND the cursor commit inside ONE mutex acquisition so two
|
|
687
|
+
// concurrent requests cannot read the same cursor and stampede before
|
|
688
|
+
// the locked commit lands. `chooseAccount` is sync and mutates the
|
|
689
|
+
// cursor internally (session-affinity `markSwitched`, the hybrid
|
|
690
|
+
// selector's own advance, and the round-robin fallback `markSwitched`);
|
|
691
|
+
// holding the mutex across the whole call serializes all of those.
|
|
692
|
+
// We then `markSwitchedLocked` the winner to (a) re-commit the cursor
|
|
693
|
+
// under the lock across the await boundary and (b) hand
|
|
694
|
+
// `persistRuntimeActiveAccount` a cursor that is already correct. That
|
|
695
|
+
// later `markSwitchedLocked` runs INLINE (reentrant) within this held
|
|
696
|
+
// section, so there is no double-acquire and no deadlock on the
|
|
697
|
+
// non-reentrant FIFO queue. In legacy mode the inline `markSwitched`
|
|
698
|
+
// calls inside `chooseAccount` are used unchanged and no lock is taken,
|
|
699
|
+
// so default behavior and perf are identical to before.
|
|
700
|
+
const selectAccount = () => chooseAccount({
|
|
701
|
+
accountManager,
|
|
702
|
+
sessionAffinityStore: state.sessionAffinityStore,
|
|
703
|
+
sessionKey: context.sessionKey,
|
|
704
|
+
family: context.family,
|
|
1311
705
|
model: context.model,
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
706
|
+
attemptedIndexes,
|
|
707
|
+
now: state.now(),
|
|
708
|
+
policy: policyDecision,
|
|
709
|
+
pinnedIndex,
|
|
710
|
+
skipReasons: accountSkipReasons,
|
|
711
|
+
stickyBoostByAccount: rotationStickyBoost,
|
|
712
|
+
pidOffsetEnabled: state.pidOffsetEnabled,
|
|
713
|
+
schedulingStrategy: state.schedulingStrategy,
|
|
1315
714
|
});
|
|
1316
|
-
|
|
1317
|
-
await
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
715
|
+
const selected = state.routingMutexMode === "enabled"
|
|
716
|
+
? await withRoutingMutex(state.routingMutexMode, async () => {
|
|
717
|
+
const candidate = selectAccount();
|
|
718
|
+
if (candidate &&
|
|
719
|
+
pinnedIndex === null &&
|
|
720
|
+
state.schedulingStrategy !== "sequential") {
|
|
721
|
+
// Re-commit the cursor under the held mutex. Skipped when a
|
|
722
|
+
// manual pin is active so the proxy never clobbers the pin
|
|
723
|
+
// (see #474); pinned selections are deterministic and need no
|
|
724
|
+
// cursor advance. Also skipped in sequential mode: the
|
|
725
|
+
// sequential selector already committed the correct active
|
|
726
|
+
// index inside this held mutex, and re-committing `candidate`
|
|
727
|
+
// would wrongly advance the drain-first primary when the pick
|
|
728
|
+
// came from the non-advancing linear-scan fallback (#509).
|
|
729
|
+
// Runs inline via reentrancy — see comment above.
|
|
730
|
+
await accountManager.markSwitchedLocked(candidate, "rotation", context.family);
|
|
731
|
+
}
|
|
732
|
+
return candidate;
|
|
733
|
+
})
|
|
734
|
+
: selectAccount();
|
|
735
|
+
if (!selected) {
|
|
736
|
+
if (!reloadedAfterNoAccount &&
|
|
737
|
+
!isPinned &&
|
|
738
|
+
accountCount > 0 &&
|
|
739
|
+
exhaustionReason === "no-account" &&
|
|
740
|
+
(policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
|
|
741
|
+
![...accountSkipReasons.values()].some((reason) => reason === "rate-limited" ||
|
|
742
|
+
reason.startsWith("cooling-down") ||
|
|
743
|
+
reason === "policy-blocked")) {
|
|
744
|
+
reloadedAfterNoAccount = true;
|
|
745
|
+
const reloadedManager = await recoverStaleRuntimeState(state);
|
|
746
|
+
if (reloadedManager) {
|
|
747
|
+
accountManager = reloadedManager;
|
|
748
|
+
accountCount = accountManager.getAccountCount();
|
|
749
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
750
|
+
accountSkipReasons.clear();
|
|
751
|
+
attemptedIndexes.clear();
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
break;
|
|
1344
756
|
}
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
let transientAttempts = 0;
|
|
1351
|
-
let transientExhaustionReason = null;
|
|
1352
|
-
const accountSkipReasons = new Map();
|
|
1353
|
-
let reloadedAfterNoAccount = false;
|
|
1354
|
-
// Read the manual pin and affinity generation from disk (mtime-cached)
|
|
1355
|
-
// on each request so a `codex-multi-auth switch|unpin|best` invocation
|
|
1356
|
-
// in another process is honored without forcing a full AccountManager
|
|
1357
|
-
// reload. The CLI bumps `affinityGeneration` on user-initiated changes
|
|
1358
|
-
// so the proxy can invalidate sticky session affinity that would
|
|
1359
|
-
// otherwise glue an in-flight chat thread to the previously selected
|
|
1360
|
-
// account. The proxy itself never bumps the generation, so its own
|
|
1361
|
-
// debounced disk writes do not clear affinity. See issue #474.
|
|
1362
|
-
const storageMeta = readStorageMetaFromDisk();
|
|
1363
|
-
const pinnedIndex = storageMeta.pinnedAccountIndex;
|
|
1364
|
-
const isPinned = typeof pinnedIndex === "number";
|
|
1365
|
-
if (storageMeta.affinityGeneration > lastObservedAffinityGeneration) {
|
|
1366
|
-
sessionAffinityStore?.clearAll();
|
|
1367
|
-
lastObservedAffinityGeneration = storageMeta.affinityGeneration;
|
|
757
|
+
attemptedIndexes.add(selected.index);
|
|
758
|
+
if (!accountManager.consumeToken(selected, context.family, context.model)) {
|
|
759
|
+
accountSkipReasons.set(selected.index, "token-exhausted");
|
|
760
|
+
exhaustionReason = "rate-limit";
|
|
761
|
+
continue;
|
|
1368
762
|
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
attemptedIndexes,
|
|
1398
|
-
now: now(),
|
|
1399
|
-
policy: policyDecision,
|
|
1400
|
-
pinnedIndex,
|
|
1401
|
-
skipReasons: accountSkipReasons,
|
|
1402
|
-
stickyBoostByAccount: rotationStickyBoost,
|
|
1403
|
-
pidOffsetEnabled,
|
|
1404
|
-
schedulingStrategy,
|
|
1405
|
-
});
|
|
1406
|
-
const selected = routingMutexMode === "enabled"
|
|
1407
|
-
? await withRoutingMutex(routingMutexMode, async () => {
|
|
1408
|
-
const candidate = selectAccount();
|
|
1409
|
-
if (candidate &&
|
|
1410
|
-
pinnedIndex === null &&
|
|
1411
|
-
schedulingStrategy !== "sequential") {
|
|
1412
|
-
// Re-commit the cursor under the held mutex. Skipped when a
|
|
1413
|
-
// manual pin is active so the proxy never clobbers the pin
|
|
1414
|
-
// (see #474); pinned selections are deterministic and need no
|
|
1415
|
-
// cursor advance. Also skipped in sequential mode: the
|
|
1416
|
-
// sequential selector already committed the correct active
|
|
1417
|
-
// index inside this held mutex, and re-committing `candidate`
|
|
1418
|
-
// would wrongly advance the drain-first primary when the pick
|
|
1419
|
-
// came from the non-advancing linear-scan fallback (#509).
|
|
1420
|
-
// Runs inline via reentrancy — see comment above.
|
|
1421
|
-
await accountManager.markSwitchedLocked(candidate, "rotation", context.family);
|
|
1422
|
-
}
|
|
1423
|
-
return candidate;
|
|
1424
|
-
})
|
|
1425
|
-
: selectAccount();
|
|
1426
|
-
if (!selected) {
|
|
1427
|
-
if (!reloadedAfterNoAccount &&
|
|
1428
|
-
!isPinned &&
|
|
1429
|
-
accountCount > 0 &&
|
|
1430
|
-
exhaustionReason === "no-account" &&
|
|
1431
|
-
(policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
|
|
1432
|
-
![...accountSkipReasons.values()].some((reason) => reason === "rate-limited" ||
|
|
1433
|
-
reason.startsWith("cooling-down") ||
|
|
1434
|
-
reason === "policy-blocked")) {
|
|
1435
|
-
reloadedAfterNoAccount = true;
|
|
1436
|
-
const reloadedManager = await recoverStaleRuntimeState();
|
|
1437
|
-
if (reloadedManager) {
|
|
1438
|
-
accountManager = reloadedManager;
|
|
1439
|
-
accountCount = accountManager.getAccountCount();
|
|
1440
|
-
transientAttemptLimit = Math.max(1, Math.min(accountCount, maxRuntimeAccountAttempts));
|
|
1441
|
-
accountSkipReasons.clear();
|
|
1442
|
-
attemptedIndexes.clear();
|
|
1443
|
-
continue;
|
|
1444
|
-
}
|
|
1445
|
-
}
|
|
1446
|
-
break;
|
|
763
|
+
const refreshed = await ensureFreshAccessToken({
|
|
764
|
+
accountManager,
|
|
765
|
+
account: selected,
|
|
766
|
+
family: context.family,
|
|
767
|
+
model: context.model,
|
|
768
|
+
now: state.now(),
|
|
769
|
+
tokenRefreshSkewMs: state.tokenRefreshSkewMs,
|
|
770
|
+
tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs,
|
|
771
|
+
});
|
|
772
|
+
if (!refreshed.ok) {
|
|
773
|
+
accountManager.refundToken(selected, context.family, context.model);
|
|
774
|
+
exhaustionReason = "auth-failure";
|
|
775
|
+
if (refreshed.invalidated) {
|
|
776
|
+
// Refresh endpoint explicitly revoked the token. Stop cascade:
|
|
777
|
+
// return auth error to client instead of rotating to the next account.
|
|
778
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
779
|
+
res.writeHead(HTTP_STATUS.UNAUTHORIZED, { "content-type": "application/json" });
|
|
780
|
+
// Route through the shared builder so both invalidation exit paths stay
|
|
781
|
+
// in lockstep — empty input yields { error: { message: <fallback>,
|
|
782
|
+
// code: "token_invalidated" } }.
|
|
783
|
+
res.end(buildTokenInvalidationBody(""));
|
|
784
|
+
await usageRecorder.record({
|
|
785
|
+
outcome: "failure",
|
|
786
|
+
statusCode: HTTP_STATUS.UNAUTHORIZED,
|
|
787
|
+
errorCode: "token_invalidated",
|
|
788
|
+
account: selected,
|
|
789
|
+
});
|
|
790
|
+
return;
|
|
1447
791
|
}
|
|
1448
|
-
|
|
1449
|
-
if (!accountManager.consumeToken(selected, context.family, context.model)) {
|
|
1450
|
-
accountSkipReasons.set(selected.index, "token-exhausted");
|
|
1451
|
-
exhaustionReason = "rate-limit";
|
|
1452
|
-
continue;
|
|
1453
|
-
}
|
|
1454
|
-
const refreshed = await ensureFreshAccessToken({
|
|
1455
|
-
accountManager,
|
|
1456
|
-
account: selected,
|
|
1457
|
-
family: context.family,
|
|
1458
|
-
model: context.model,
|
|
1459
|
-
now: now(),
|
|
1460
|
-
tokenRefreshSkewMs,
|
|
1461
|
-
tokenInvalidationCooldownMs,
|
|
1462
|
-
});
|
|
1463
|
-
if (!refreshed.ok) {
|
|
1464
|
-
accountManager.refundToken(selected, context.family, context.model);
|
|
1465
|
-
exhaustionReason = "auth-failure";
|
|
1466
|
-
if (refreshed.invalidated) {
|
|
1467
|
-
// Refresh endpoint explicitly revoked the token. Stop cascade:
|
|
1468
|
-
// return auth error to client instead of rotating to the next account.
|
|
1469
|
-
sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1470
|
-
res.writeHead(HTTP_STATUS.UNAUTHORIZED, { "content-type": "application/json" });
|
|
1471
|
-
// Route through the shared builder so both invalidation exit paths stay
|
|
1472
|
-
// in lockstep — empty input yields { error: { message: <fallback>,
|
|
1473
|
-
// code: "token_invalidated" } }.
|
|
1474
|
-
res.end(buildTokenInvalidationBody(""));
|
|
1475
|
-
await usageRecorder.record({
|
|
1476
|
-
outcome: "failure",
|
|
1477
|
-
statusCode: HTTP_STATUS.UNAUTHORIZED,
|
|
1478
|
-
errorCode: "token_invalidated",
|
|
1479
|
-
account: selected,
|
|
1480
|
-
});
|
|
1481
|
-
return;
|
|
1482
|
-
}
|
|
1483
|
-
if (!refreshed.retryable)
|
|
1484
|
-
continue;
|
|
1485
|
-
transientAttempts += 1;
|
|
1486
|
-
transientExhaustionReason = "auth-failure";
|
|
1487
|
-
status.retries += 1;
|
|
1488
|
-
status.rotations += 1;
|
|
792
|
+
if (!refreshed.retryable)
|
|
1489
793
|
continue;
|
|
794
|
+
transientAttempts += 1;
|
|
795
|
+
transientExhaustionReason = "auth-failure";
|
|
796
|
+
state.status.retries += 1;
|
|
797
|
+
state.status.rotations += 1;
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const accountId = resolveAccountId(refreshed.account, refreshed.accessToken);
|
|
801
|
+
if (!accountId) {
|
|
802
|
+
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
803
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
804
|
+
accountManager.markAccountCoolingDown(refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
|
|
805
|
+
exhaustionReason = "auth-failure";
|
|
806
|
+
transientAttempts += 1;
|
|
807
|
+
transientExhaustionReason = "auth-failure";
|
|
808
|
+
state.status.retries += 1;
|
|
809
|
+
state.status.rotations += 1;
|
|
810
|
+
continue;
|
|
811
|
+
}
|
|
812
|
+
const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
|
|
813
|
+
recordLastRuntimeAccount(state.status, accountIdentity);
|
|
814
|
+
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
815
|
+
let upstream;
|
|
816
|
+
try {
|
|
817
|
+
state.status.upstreamRequests += 1;
|
|
818
|
+
const fetchAbortController = new AbortController();
|
|
819
|
+
const upstreamRequestInit = {
|
|
820
|
+
method: context.method,
|
|
821
|
+
headers: outboundHeaders,
|
|
822
|
+
signal: fetchAbortController.signal,
|
|
823
|
+
};
|
|
824
|
+
if (context.method === "POST") {
|
|
825
|
+
upstreamRequestInit.body = context.body;
|
|
1490
826
|
}
|
|
1491
|
-
|
|
1492
|
-
|
|
827
|
+
upstream = await withTimeout(state.fetchImpl(upstreamUrl, upstreamRequestInit), state.fetchTimeoutMs, () => fetchAbortController.abort(), `upstream fetch timed out after ${state.fetchTimeoutMs}ms`);
|
|
828
|
+
}
|
|
829
|
+
catch (error) {
|
|
830
|
+
state.status.lastError = error instanceof Error ? error.message : String(error);
|
|
831
|
+
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
832
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
833
|
+
accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
|
|
834
|
+
accountManager.saveToDiskDebounced();
|
|
835
|
+
exhaustionReason = "network-error";
|
|
836
|
+
transientAttempts += 1;
|
|
837
|
+
transientExhaustionReason = "network-error";
|
|
838
|
+
state.status.retries += 1;
|
|
839
|
+
state.status.rotations += 1;
|
|
840
|
+
continue;
|
|
841
|
+
}
|
|
842
|
+
if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
|
|
843
|
+
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
844
|
+
const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, state.now()) ??
|
|
845
|
+
parseRetryAfterBodyMs(bodyText, state.now()) ??
|
|
846
|
+
60_000;
|
|
847
|
+
// A 429 is the upstream quota signal for the attempted account, so
|
|
848
|
+
// keep the consumed runtime token drained.
|
|
849
|
+
accountManager.recordRateLimit(refreshed.account, context.family, context.model);
|
|
850
|
+
accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
|
|
851
|
+
accountManager.saveToDiskDebounced();
|
|
852
|
+
exhaustionReason = "rate-limit";
|
|
853
|
+
transientAttempts += 1;
|
|
854
|
+
transientExhaustionReason = "rate-limit";
|
|
855
|
+
state.status.retries += 1;
|
|
856
|
+
state.status.rotations += 1;
|
|
857
|
+
continue;
|
|
858
|
+
}
|
|
859
|
+
if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
|
|
860
|
+
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
861
|
+
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
862
|
+
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
863
|
+
const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
|
|
864
|
+
false;
|
|
1493
865
|
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
transientExhaustionReason = "auth-failure";
|
|
1499
|
-
status.retries += 1;
|
|
1500
|
-
status.rotations += 1;
|
|
1501
|
-
continue;
|
|
1502
|
-
}
|
|
1503
|
-
const accountIdentity = accountIdentityFromAccount(refreshed.account, now());
|
|
1504
|
-
recordLastRuntimeAccount(status, accountIdentity);
|
|
1505
|
-
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
1506
|
-
let upstream;
|
|
1507
|
-
try {
|
|
1508
|
-
status.upstreamRequests += 1;
|
|
1509
|
-
const fetchAbortController = new AbortController();
|
|
1510
|
-
const upstreamRequestInit = {
|
|
1511
|
-
method: context.method,
|
|
1512
|
-
headers: outboundHeaders,
|
|
1513
|
-
signal: fetchAbortController.signal,
|
|
1514
|
-
};
|
|
1515
|
-
if (context.method === "POST") {
|
|
1516
|
-
upstreamRequestInit.body = context.body;
|
|
866
|
+
if (accountWasEnabled) {
|
|
867
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
868
|
+
accountManager.setAccountEnabled(refreshed.account.index, false);
|
|
869
|
+
accountManager.saveToDiskDebounced();
|
|
1517
870
|
}
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
status.
|
|
1522
|
-
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
1523
|
-
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1524
|
-
accountManager.markAccountCoolingDown(refreshed.account, networkErrorCooldownMs, "network-error");
|
|
1525
|
-
accountManager.saveToDiskDebounced();
|
|
1526
|
-
exhaustionReason = "network-error";
|
|
1527
|
-
transientAttempts += 1;
|
|
1528
|
-
transientExhaustionReason = "network-error";
|
|
1529
|
-
status.retries += 1;
|
|
1530
|
-
status.rotations += 1;
|
|
871
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
872
|
+
exhaustionReason = "deactivated";
|
|
873
|
+
state.status.retries += 1;
|
|
874
|
+
state.status.rotations += 1;
|
|
1531
875
|
continue;
|
|
1532
876
|
}
|
|
1533
|
-
if (upstream.status
|
|
1534
|
-
const
|
|
1535
|
-
const
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
accountManager.recordRateLimit(refreshed.account, context.family, context.model);
|
|
1541
|
-
accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
|
|
1542
|
-
accountManager.saveToDiskDebounced();
|
|
1543
|
-
exhaustionReason = "rate-limit";
|
|
1544
|
-
transientAttempts += 1;
|
|
1545
|
-
transientExhaustionReason = "rate-limit";
|
|
1546
|
-
status.retries += 1;
|
|
1547
|
-
status.rotations += 1;
|
|
1548
|
-
continue;
|
|
1549
|
-
}
|
|
1550
|
-
if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
|
|
1551
|
-
const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
|
|
1552
|
-
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
1553
|
-
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
1554
|
-
const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
|
|
1555
|
-
false;
|
|
1556
|
-
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
1557
|
-
if (accountWasEnabled) {
|
|
1558
|
-
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1559
|
-
accountManager.setAccountEnabled(refreshed.account.index, false);
|
|
1560
|
-
accountManager.saveToDiskDebounced();
|
|
1561
|
-
}
|
|
1562
|
-
sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1563
|
-
exhaustionReason = "deactivated";
|
|
1564
|
-
status.retries += 1;
|
|
1565
|
-
status.rotations += 1;
|
|
1566
|
-
continue;
|
|
1567
|
-
}
|
|
1568
|
-
if (isThreadGoalRequest && isThreadGoalFallbackStatus(upstream.status)) {
|
|
1569
|
-
const parsedGoalBody = parseRequestBody(context.body);
|
|
1570
|
-
const fallbackKey = context.sessionKey;
|
|
1571
|
-
const goal = typeof parsedGoalBody?.goal === "string" ? parsedGoalBody.goal : null;
|
|
1572
|
-
if (!fallbackKey) {
|
|
1573
|
-
if (context.upstreamPath.endsWith("/get")) {
|
|
1574
|
-
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1575
|
-
await usageRecorder.record({
|
|
1576
|
-
outcome: "failure",
|
|
1577
|
-
statusCode: upstream.status,
|
|
1578
|
-
errorCode: "thread_goal_session_key_required",
|
|
1579
|
-
account: refreshed.account,
|
|
1580
|
-
});
|
|
1581
|
-
return;
|
|
1582
|
-
}
|
|
877
|
+
if (isThreadGoalRequest && isThreadGoalFallbackStatus(upstream.status)) {
|
|
878
|
+
const parsedGoalBody = parseRequestBody(context.body);
|
|
879
|
+
const fallbackKey = context.sessionKey;
|
|
880
|
+
const goal = typeof parsedGoalBody?.goal === "string" ? parsedGoalBody.goal : null;
|
|
881
|
+
if (!fallbackKey) {
|
|
882
|
+
if (context.upstreamPath.endsWith("/get")) {
|
|
883
|
+
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1583
884
|
await usageRecorder.record({
|
|
1584
885
|
outcome: "failure",
|
|
1585
|
-
statusCode:
|
|
886
|
+
statusCode: upstream.status,
|
|
1586
887
|
errorCode: "thread_goal_session_key_required",
|
|
1587
888
|
account: refreshed.account,
|
|
1588
889
|
});
|
|
1589
|
-
writeJson(res, HTTP_STATUS.BAD_REQUEST, {
|
|
1590
|
-
error: {
|
|
1591
|
-
message: "Thread goal fallback requires a thread_id, threadId, or session header.",
|
|
1592
|
-
code: "thread_goal_session_key_required",
|
|
1593
|
-
},
|
|
1594
|
-
});
|
|
1595
890
|
return;
|
|
1596
891
|
}
|
|
1597
892
|
await usageRecorder.record({
|
|
1598
893
|
outcome: "failure",
|
|
1599
|
-
statusCode:
|
|
1600
|
-
errorCode: "
|
|
894
|
+
statusCode: HTTP_STATUS.BAD_REQUEST,
|
|
895
|
+
errorCode: "thread_goal_session_key_required",
|
|
1601
896
|
account: refreshed.account,
|
|
1602
897
|
});
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
writeJson(res, HTTP_STATUS.OK, {
|
|
1609
|
-
goal: getThreadGoalFallback(threadGoalFallbacks, fallbackKey),
|
|
1610
|
-
});
|
|
1611
|
-
return;
|
|
1612
|
-
}
|
|
1613
|
-
if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
|
|
1614
|
-
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1615
|
-
await usageRecorder.record({
|
|
1616
|
-
outcome: "failure",
|
|
1617
|
-
statusCode: upstream.status,
|
|
1618
|
-
errorCode,
|
|
1619
|
-
account: refreshed.account,
|
|
898
|
+
writeJson(res, HTTP_STATUS.BAD_REQUEST, {
|
|
899
|
+
error: {
|
|
900
|
+
message: "Thread goal fallback requires a thread_id, threadId, or session header.",
|
|
901
|
+
code: "thread_goal_session_key_required",
|
|
902
|
+
},
|
|
1620
903
|
});
|
|
1621
904
|
return;
|
|
1622
905
|
}
|
|
1623
|
-
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
1624
|
-
res.end(bodyText);
|
|
1625
906
|
await usageRecorder.record({
|
|
1626
907
|
outcome: "failure",
|
|
1627
908
|
statusCode: upstream.status,
|
|
1628
|
-
errorCode,
|
|
909
|
+
errorCode: "thread_goal_upstream_blocked",
|
|
1629
910
|
account: refreshed.account,
|
|
1630
911
|
});
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
|
|
1635
|
-
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
1636
|
-
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1637
|
-
if (isTokenInvalidationError(bodyText)) {
|
|
1638
|
-
// The upstream explicitly revoked this OAuth token. Applying a long
|
|
1639
|
-
// cooldown prevents cascade invalidation: rapidly presenting each
|
|
1640
|
-
// account's token from the same IP triggers OpenAI's anti-abuse
|
|
1641
|
-
// detection and invalidates them in sequence. Return the 401 directly
|
|
1642
|
-
// rather than rotating so the client can prompt for re-login.
|
|
1643
|
-
applyMonotonicAuthCooldown(accountManager, refreshed.account, tokenInvalidationCooldownMs);
|
|
1644
|
-
sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1645
|
-
accountManager.saveToDiskDebounced();
|
|
1646
|
-
// Emit the same machine-readable shape as the refresh-failure path
|
|
1647
|
-
// (code: "token_invalidated") instead of forwarding the raw upstream
|
|
1648
|
-
// body, so the client contract is consistent across both vectors.
|
|
1649
|
-
const clientHeaders = responseHeadersForClient(upstream.headers);
|
|
1650
|
-
clientHeaders["content-type"] = "application/json";
|
|
1651
|
-
res.writeHead(upstream.status, clientHeaders);
|
|
1652
|
-
res.end(buildTokenInvalidationBody(bodyText));
|
|
1653
|
-
await usageRecorder.record({
|
|
1654
|
-
outcome: "failure",
|
|
1655
|
-
statusCode: upstream.status,
|
|
1656
|
-
errorCode: "token_invalidated",
|
|
1657
|
-
account: refreshed.account,
|
|
1658
|
-
});
|
|
912
|
+
if (context.upstreamPath.endsWith("/set")) {
|
|
913
|
+
setThreadGoalFallback(state.threadGoalFallbacks, fallbackKey, goal);
|
|
914
|
+
writeJson(res, HTTP_STATUS.OK, { ok: true, goal });
|
|
1659
915
|
return;
|
|
1660
916
|
}
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
transientExhaustionReason = "auth-failure";
|
|
1666
|
-
status.retries += 1;
|
|
1667
|
-
status.rotations += 1;
|
|
1668
|
-
continue;
|
|
1669
|
-
}
|
|
1670
|
-
if (upstream.status >= 500) {
|
|
1671
|
-
await readErrorBody(upstream, streamStallTimeoutMs);
|
|
1672
|
-
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
1673
|
-
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1674
|
-
accountManager.markAccountCoolingDown(refreshed.account, serverErrorCooldownMs, "server-error");
|
|
1675
|
-
accountManager.saveToDiskDebounced();
|
|
1676
|
-
exhaustionReason = "server-error";
|
|
1677
|
-
transientAttempts += 1;
|
|
1678
|
-
transientExhaustionReason = "server-error";
|
|
1679
|
-
status.retries += 1;
|
|
1680
|
-
status.rotations += 1;
|
|
1681
|
-
continue;
|
|
917
|
+
writeJson(res, HTTP_STATUS.OK, {
|
|
918
|
+
goal: getThreadGoalFallback(state.threadGoalFallbacks, fallbackKey),
|
|
919
|
+
});
|
|
920
|
+
return;
|
|
1682
921
|
}
|
|
1683
|
-
if (isThreadGoalRequest &&
|
|
1684
|
-
|
|
1685
|
-
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1686
|
-
await usageRecorder.record({
|
|
1687
|
-
outcome: "failure",
|
|
1688
|
-
statusCode: upstream.status,
|
|
1689
|
-
errorCode: "thread_goal_upstream_error",
|
|
1690
|
-
account: refreshed.account,
|
|
1691
|
-
});
|
|
1692
|
-
return;
|
|
1693
|
-
}
|
|
1694
|
-
const forwarded = await forwardStreamingResponse(upstream, res, status, () => undefined, streamStallTimeoutMs);
|
|
922
|
+
if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
|
|
923
|
+
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1695
924
|
await usageRecorder.record({
|
|
1696
925
|
outcome: "failure",
|
|
1697
926
|
statusCode: upstream.status,
|
|
1698
|
-
errorCode
|
|
927
|
+
errorCode,
|
|
1699
928
|
account: refreshed.account,
|
|
1700
929
|
});
|
|
1701
930
|
return;
|
|
1702
931
|
}
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
// stale runtime skip reason persisted for it on a prior pool
|
|
1706
|
-
// exhaustion. Without this the overlay reason (e.g. "token-exhausted",
|
|
1707
|
-
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
1708
|
-
// the forecast keeps reporting this working account as unavailable.
|
|
1709
|
-
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
1710
|
-
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
1711
|
-
const nearExhaustionWaitMs = getQuotaNearExhaustionWaitMs(upstream.headers, quotaRemainingPercentThreshold, now());
|
|
1712
|
-
if (nearExhaustionWaitMs > 0) {
|
|
1713
|
-
accountManager.markRateLimitedWithReason(refreshed.account, nearExhaustionWaitMs, context.family, "quota", context.model);
|
|
1714
|
-
sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1715
|
-
accountManager.saveToDiskDebounced();
|
|
1716
|
-
}
|
|
1717
|
-
else {
|
|
1718
|
-
sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, now());
|
|
1719
|
-
if (refreshed.account.index !== lastGlobalAccountIndex) {
|
|
1720
|
-
lastGlobalAccountIndex = refreshed.account.index;
|
|
1721
|
-
}
|
|
1722
|
-
lastGlobalSwitchAt = now();
|
|
1723
|
-
}
|
|
1724
|
-
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex);
|
|
1725
|
-
const forwarded = await forwardStreamingResponse(upstream, res, status, () => {
|
|
1726
|
-
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1727
|
-
accountManager.markAccountCoolingDown(refreshed.account, networkErrorCooldownMs, "network-error");
|
|
1728
|
-
sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1729
|
-
accountManager.saveToDiskDebounced();
|
|
1730
|
-
}, streamStallTimeoutMs);
|
|
932
|
+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
933
|
+
res.end(bodyText);
|
|
1731
934
|
await usageRecorder.record({
|
|
1732
|
-
outcome:
|
|
935
|
+
outcome: "failure",
|
|
1733
936
|
statusCode: upstream.status,
|
|
1734
|
-
errorCode
|
|
937
|
+
errorCode,
|
|
1735
938
|
account: refreshed.account,
|
|
1736
939
|
});
|
|
1737
940
|
return;
|
|
1738
941
|
}
|
|
1739
|
-
if (
|
|
1740
|
-
|
|
1741
|
-
|
|
942
|
+
if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
|
|
943
|
+
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
944
|
+
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
945
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
946
|
+
if (isTokenInvalidationError(bodyText)) {
|
|
947
|
+
// The upstream explicitly revoked this OAuth token. Applying a long
|
|
948
|
+
// cooldown prevents cascade invalidation: rapidly presenting each
|
|
949
|
+
// account's token from the same IP triggers OpenAI's anti-abuse
|
|
950
|
+
// detection and invalidates them in sequence. Return the 401 directly
|
|
951
|
+
// rather than rotating so the client can prompt for re-login.
|
|
952
|
+
applyMonotonicAuthCooldown(accountManager, refreshed.account, state.tokenInvalidationCooldownMs);
|
|
953
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
954
|
+
accountManager.saveToDiskDebounced();
|
|
955
|
+
// Emit the same machine-readable shape as the refresh-failure path
|
|
956
|
+
// (code: "token_invalidated") instead of forwarding the raw upstream
|
|
957
|
+
// body, so the client contract is consistent across both vectors.
|
|
958
|
+
const clientHeaders = responseHeadersForClient(upstream.headers);
|
|
959
|
+
clientHeaders["content-type"] = "application/json";
|
|
960
|
+
res.writeHead(upstream.status, clientHeaders);
|
|
961
|
+
res.end(buildTokenInvalidationBody(bodyText));
|
|
962
|
+
await usageRecorder.record({
|
|
963
|
+
outcome: "failure",
|
|
964
|
+
statusCode: upstream.status,
|
|
965
|
+
errorCode: "token_invalidated",
|
|
966
|
+
account: refreshed.account,
|
|
967
|
+
});
|
|
968
|
+
return;
|
|
969
|
+
}
|
|
970
|
+
applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
971
|
+
accountManager.saveToDiskDebounced();
|
|
972
|
+
exhaustionReason = "auth-failure";
|
|
973
|
+
transientAttempts += 1;
|
|
974
|
+
transientExhaustionReason = "auth-failure";
|
|
975
|
+
state.status.retries += 1;
|
|
976
|
+
state.status.rotations += 1;
|
|
977
|
+
continue;
|
|
1742
978
|
}
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
979
|
+
if (upstream.status >= 500) {
|
|
980
|
+
await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
981
|
+
accountManager.refundToken(refreshed.account, context.family, context.model);
|
|
982
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
983
|
+
accountManager.markAccountCoolingDown(refreshed.account, state.serverErrorCooldownMs, "server-error");
|
|
984
|
+
accountManager.saveToDiskDebounced();
|
|
985
|
+
exhaustionReason = "server-error";
|
|
986
|
+
transientAttempts += 1;
|
|
987
|
+
transientExhaustionReason = "server-error";
|
|
988
|
+
state.status.retries += 1;
|
|
989
|
+
state.status.rotations += 1;
|
|
990
|
+
continue;
|
|
1746
991
|
}
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
if (errorBody.reason === null) {
|
|
1758
|
-
status.lastError = `pinned-503 missing skip reason (pinnedIndex=${pinnedIndex})`;
|
|
992
|
+
if (isThreadGoalRequest && upstream.status >= 400) {
|
|
993
|
+
if (context.upstreamPath.endsWith("/get")) {
|
|
994
|
+
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
995
|
+
await usageRecorder.record({
|
|
996
|
+
outcome: "failure",
|
|
997
|
+
statusCode: upstream.status,
|
|
998
|
+
errorCode: "thread_goal_upstream_error",
|
|
999
|
+
account: refreshed.account,
|
|
1000
|
+
});
|
|
1001
|
+
return;
|
|
1759
1002
|
}
|
|
1760
|
-
await
|
|
1003
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => undefined, state.streamStallTimeoutMs);
|
|
1004
|
+
await usageRecorder.record({
|
|
1761
1005
|
outcome: "failure",
|
|
1762
|
-
statusCode:
|
|
1763
|
-
errorCode: "
|
|
1006
|
+
statusCode: upstream.status,
|
|
1007
|
+
errorCode: forwarded ? "thread_goal_upstream_error" : "stream_forward_failed",
|
|
1008
|
+
account: refreshed.account,
|
|
1764
1009
|
});
|
|
1765
|
-
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
|
|
1766
1010
|
return;
|
|
1767
1011
|
}
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1012
|
+
accountManager.recordSuccess(refreshed.account, context.family, context.model);
|
|
1013
|
+
// A successful request proves the account is usable, so clear any
|
|
1014
|
+
// stale runtime skip reason persisted for it on a prior pool
|
|
1015
|
+
// exhaustion. Without this the overlay reason (e.g. "token-exhausted",
|
|
1016
|
+
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
1017
|
+
// the forecast keeps reporting this working account as unavailable.
|
|
1018
|
+
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
1019
|
+
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
1020
|
+
const nearExhaustionWaitMs = getQuotaNearExhaustionWaitMs(upstream.headers, state.quotaRemainingPercentThreshold, state.now());
|
|
1021
|
+
if (nearExhaustionWaitMs > 0) {
|
|
1022
|
+
accountManager.markRateLimitedWithReason(refreshed.account, nearExhaustionWaitMs, context.family, "quota", context.model);
|
|
1023
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1024
|
+
accountManager.saveToDiskDebounced();
|
|
1775
1025
|
}
|
|
1776
1026
|
else {
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
reason: exhaustionReason,
|
|
1783
|
-
accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
1784
|
-
String(index),
|
|
1785
|
-
reason,
|
|
1786
|
-
])),
|
|
1787
|
-
});
|
|
1027
|
+
state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
|
|
1028
|
+
if (refreshed.account.index !== state.lastGlobalAccountIndex) {
|
|
1029
|
+
state.lastGlobalAccountIndex = refreshed.account.index;
|
|
1030
|
+
}
|
|
1031
|
+
state.lastGlobalSwitchAt = state.now();
|
|
1788
1032
|
}
|
|
1033
|
+
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy);
|
|
1034
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
|
|
1035
|
+
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
1036
|
+
accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
|
|
1037
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1038
|
+
accountManager.saveToDiskDebounced();
|
|
1039
|
+
}, state.streamStallTimeoutMs);
|
|
1040
|
+
await usageRecorder.record({
|
|
1041
|
+
outcome: forwarded ? "success" : "failure",
|
|
1042
|
+
statusCode: upstream.status,
|
|
1043
|
+
errorCode: forwarded ? null : "stream_forward_failed",
|
|
1044
|
+
account: refreshed.account,
|
|
1045
|
+
});
|
|
1046
|
+
return;
|
|
1789
1047
|
}
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1048
|
+
if (transientAttempts >= transientAttemptLimit &&
|
|
1049
|
+
attemptedIndexes.size < accountCount) {
|
|
1050
|
+
exhaustionReason = "budget";
|
|
1051
|
+
}
|
|
1052
|
+
else if (exhaustionReason === "deactivated" &&
|
|
1053
|
+
transientExhaustionReason) {
|
|
1054
|
+
exhaustionReason = transientExhaustionReason;
|
|
1055
|
+
}
|
|
1056
|
+
// When a manual pin is set and the pinned account is unavailable, do
|
|
1057
|
+
// NOT silently fall through to rotation. Hard-fail with a 503 so the
|
|
1058
|
+
// user is informed the pin cannot be honored. See issue #474.
|
|
1059
|
+
//
|
|
1060
|
+
// Surface the runtime skip reason in both the human-readable message
|
|
1061
|
+
// and a structured `reason` field, mirroring `writePoolExhausted`. A
|
|
1062
|
+
// null reason indicates a forecast/runtime state desync (the pinned
|
|
1063
|
+
// account was selected but no skip reason was recorded) — see #486.
|
|
1064
|
+
if (isPinned) {
|
|
1065
|
+
const errorBody = buildPinnedUnavailableErrorBody(pinnedIndex, accountSkipReasons);
|
|
1066
|
+
if (errorBody.reason === null) {
|
|
1067
|
+
state.status.lastError = `pinned-503 missing skip reason (pinnedIndex=${pinnedIndex})`;
|
|
1068
|
+
}
|
|
1069
|
+
await usageRecorder?.record({
|
|
1070
|
+
outcome: "failure",
|
|
1071
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
1072
|
+
errorCode: "codex_pinned_account_unavailable",
|
|
1804
1073
|
});
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1074
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
await usageRecorder?.record({
|
|
1078
|
+
outcome: "failure",
|
|
1079
|
+
statusCode: normalizeExhaustionStatus(exhaustionReason),
|
|
1080
|
+
errorCode: isThreadGoalRequest && context.upstreamPath.endsWith("/get") ? "thread_goal_pool_exhausted" : exhaustionReason,
|
|
1081
|
+
});
|
|
1082
|
+
if (isThreadGoalRequest && context.upstreamPath.endsWith("/get")) {
|
|
1083
|
+
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
1084
|
+
}
|
|
1085
|
+
else {
|
|
1086
|
+
writePoolExhausted({
|
|
1087
|
+
res,
|
|
1088
|
+
accountManager,
|
|
1089
|
+
family: context.family,
|
|
1090
|
+
model: context.model,
|
|
1091
|
+
reason: exhaustionReason,
|
|
1092
|
+
accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
1093
|
+
String(index),
|
|
1094
|
+
reason,
|
|
1095
|
+
])),
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
catch (error) {
|
|
1100
|
+
const rawErrorMessage = error instanceof Error ? error.message : String(error);
|
|
1101
|
+
// errors-logging-08: redact any email/token material that leaked into a
|
|
1102
|
+
// raw upstream or refresh error string before it reaches state.status consumers
|
|
1103
|
+
// or the structured log. maskString is a no-op for clean diagnostic text.
|
|
1104
|
+
const maskedErrorMessage = maskString(rawErrorMessage);
|
|
1105
|
+
state.status.lastError = maskedErrorMessage;
|
|
1106
|
+
// errors-logging-01: surface the failure through the structured logger
|
|
1107
|
+
// (redaction-safe) with the request trace id, instead of only stashing a
|
|
1108
|
+
// last-write-wins status string.
|
|
1109
|
+
proxyLog.error("runtime proxy request failed", {
|
|
1110
|
+
traceId,
|
|
1111
|
+
code: isRuntimeProxyHttpError(error) ? error.code : "codex_runtime_rotation_proxy_error",
|
|
1112
|
+
error: maskedErrorMessage,
|
|
1113
|
+
});
|
|
1114
|
+
if (!res.headersSent) {
|
|
1115
|
+
if (isRuntimeProxyHttpError(error)) {
|
|
1820
1116
|
await usageRecorder?.record({
|
|
1821
1117
|
outcome: "failure",
|
|
1822
|
-
statusCode:
|
|
1823
|
-
errorCode:
|
|
1118
|
+
statusCode: error.statusCode,
|
|
1119
|
+
errorCode: error.code,
|
|
1824
1120
|
});
|
|
1825
|
-
writeJson(res,
|
|
1121
|
+
writeJson(res, error.statusCode, {
|
|
1826
1122
|
error: {
|
|
1827
|
-
message:
|
|
1828
|
-
code:
|
|
1123
|
+
message: error.message,
|
|
1124
|
+
code: error.code,
|
|
1829
1125
|
},
|
|
1830
1126
|
});
|
|
1127
|
+
return;
|
|
1831
1128
|
}
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1129
|
+
await usageRecorder?.record({
|
|
1130
|
+
outcome: "failure",
|
|
1131
|
+
statusCode: 500,
|
|
1132
|
+
errorCode: "codex_runtime_rotation_proxy_error",
|
|
1133
|
+
});
|
|
1134
|
+
writeJson(res, 500, {
|
|
1135
|
+
error: {
|
|
1136
|
+
message: "Runtime rotation proxy failed before forwarding the request.",
|
|
1137
|
+
code: "codex_runtime_rotation_proxy_error",
|
|
1138
|
+
},
|
|
1139
|
+
});
|
|
1835
1140
|
}
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
}
|
|
1840
|
-
const sockets = new Set();
|
|
1841
|
-
server.on("connection", (socket) => {
|
|
1842
|
-
sockets.add(socket);
|
|
1843
|
-
socket.once("close", () => {
|
|
1844
|
-
sockets.delete(socket);
|
|
1845
|
-
});
|
|
1846
|
-
});
|
|
1847
|
-
const onPostStartupServerError = (error) => {
|
|
1848
|
-
status.lastError = error.message;
|
|
1849
|
-
};
|
|
1850
|
-
await new Promise((resolve, reject) => {
|
|
1851
|
-
const onError = (error) => {
|
|
1852
|
-
server.off("listening", onListening);
|
|
1853
|
-
reject(error);
|
|
1854
|
-
};
|
|
1855
|
-
const onListening = () => {
|
|
1856
|
-
server.off("error", onError);
|
|
1857
|
-
resolve();
|
|
1858
|
-
};
|
|
1859
|
-
server.once("error", onError);
|
|
1860
|
-
server.once("listening", onListening);
|
|
1861
|
-
server.listen(port, bindHost);
|
|
1862
|
-
});
|
|
1863
|
-
server.on("error", onPostStartupServerError);
|
|
1864
|
-
const address = server.address();
|
|
1865
|
-
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
1866
|
-
return {
|
|
1867
|
-
host: bindHost,
|
|
1868
|
-
port: resolvedPort,
|
|
1869
|
-
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
1870
|
-
close: async () => {
|
|
1871
|
-
await closeServer(server, sockets);
|
|
1872
|
-
await activeAccountManager.flushPendingSave();
|
|
1873
|
-
},
|
|
1874
|
-
getStatus: () => ({
|
|
1875
|
-
...status,
|
|
1876
|
-
// Redact any email/token material that leaked into a raw upstream or
|
|
1877
|
-
// refresh error string before exposing it to status/report consumers
|
|
1878
|
-
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
1879
|
-
lastError: status.lastError === null ? null : maskString(status.lastError),
|
|
1880
|
-
}),
|
|
1881
|
-
};
|
|
1141
|
+
else if (!res.destroyed) {
|
|
1142
|
+
res.destroy(error instanceof Error ? error : undefined);
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1882
1145
|
}
|
|
1883
1146
|
async function closeServer(server, sockets) {
|
|
1884
1147
|
if (!server.listening)
|
|
@@ -1898,4 +1161,3 @@ async function closeServer(server, sockets) {
|
|
|
1898
1161
|
}
|
|
1899
1162
|
await closed;
|
|
1900
1163
|
}
|
|
1901
|
-
//# sourceMappingURL=runtime-rotation-proxy.js.map
|