codex-multi-auth 2.16.0 → 2.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +9 -9
- package/LICENSE +21 -21
- package/README.md +487 -463
- package/assets/codex-multi-auth-icon.svg +39 -39
- package/assets/readme-hero.svg +31 -31
- package/config/README.md +31 -31
- package/config/codex-legacy.json +1221 -1221
- package/config/codex-modern.json +423 -423
- package/config/minimal-codex.json +15 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -3
- package/dist/lib/account-policy.d.ts +4 -0
- package/dist/lib/account-policy.d.ts.map +1 -1
- package/dist/lib/account-policy.js +8 -0
- package/dist/lib/accounts/rate-limits.d.ts.map +1 -1
- package/dist/lib/accounts/rate-limits.js +2 -0
- package/dist/lib/accounts.d.ts +15 -3
- package/dist/lib/accounts.d.ts.map +1 -1
- package/dist/lib/accounts.js +231 -30
- package/dist/lib/api-route-store.d.ts +24 -0
- package/dist/lib/api-route-store.d.ts.map +1 -0
- package/dist/lib/api-route-store.js +145 -0
- package/dist/lib/auth/account-access.d.ts +115 -0
- package/dist/lib/auth/account-access.d.ts.map +1 -0
- package/dist/lib/auth/account-access.js +214 -0
- package/dist/lib/auth/token-utils.d.ts +54 -1
- package/dist/lib/auth/token-utils.d.ts.map +1 -1
- package/dist/lib/auth/token-utils.js +92 -7
- package/dist/lib/cli.d.ts +1 -1
- package/dist/lib/cli.d.ts.map +1 -1
- package/dist/lib/cli.js +4 -0
- package/dist/lib/codex-cli/state.d.ts +7 -0
- package/dist/lib/codex-cli/state.d.ts.map +1 -1
- package/dist/lib/codex-cli/state.js +18 -0
- package/dist/lib/codex-cli/writer.d.ts.map +1 -1
- package/dist/lib/codex-cli/writer.js +60 -8
- package/dist/lib/codex-manager/account-manager-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/account-manager-commands.js +1 -0
- package/dist/lib/codex-manager/account-pool-write.d.ts +3 -1
- package/dist/lib/codex-manager/account-pool-write.d.ts.map +1 -1
- package/dist/lib/codex-manager/account-pool-write.js +4 -0
- package/dist/lib/codex-manager/api-login-menu.d.ts +14 -0
- package/dist/lib/codex-manager/api-login-menu.d.ts.map +1 -0
- package/dist/lib/codex-manager/api-login-menu.js +224 -0
- package/dist/lib/codex-manager/commands/account.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/account.js +27 -0
- package/dist/lib/codex-manager/commands/best.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/best.js +6 -2
- package/dist/lib/codex-manager/commands/check.d.ts +7 -1
- package/dist/lib/codex-manager/commands/check.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/check.js +24 -3
- package/dist/lib/codex-manager/commands/forecast.d.ts +1 -1
- package/dist/lib/codex-manager/commands/forecast.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/forecast.js +7 -3
- package/dist/lib/codex-manager/commands/report.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/report.js +5 -1
- package/dist/lib/codex-manager/commands/resets.d.ts +2 -0
- package/dist/lib/codex-manager/commands/resets.d.ts.map +1 -0
- package/dist/lib/codex-manager/commands/resets.js +67 -0
- package/dist/lib/codex-manager/commands/rotation.d.ts +2 -2
- package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/rotation.js +63 -12
- package/dist/lib/codex-manager/commands/status.d.ts +12 -1
- package/dist/lib/codex-manager/commands/status.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/status.js +166 -22
- package/dist/lib/codex-manager/commands/uninstall.js +1 -1
- package/dist/lib/codex-manager/formatters/quota-formatters.d.ts.map +1 -1
- package/dist/lib/codex-manager/formatters/quota-formatters.js +3 -2
- package/dist/lib/codex-manager/health-check.d.ts +3 -0
- package/dist/lib/codex-manager/health-check.d.ts.map +1 -1
- package/dist/lib/codex-manager/health-check.js +57 -7
- package/dist/lib/codex-manager/help.d.ts.map +1 -1
- package/dist/lib/codex-manager/help.js +4 -2
- package/dist/lib/codex-manager/login-flow.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-flow.js +69 -7
- package/dist/lib/codex-manager/login-menu-data.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-menu-data.js +21 -8
- package/dist/lib/codex-manager/login-oauth.d.ts +3 -0
- package/dist/lib/codex-manager/login-oauth.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-oauth.js +6 -1
- package/dist/lib/codex-manager/login-workspace-choice.d.ts +11 -0
- package/dist/lib/codex-manager/login-workspace-choice.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-workspace-choice.js +37 -0
- package/dist/lib/codex-manager/persist-selected-account.d.ts.map +1 -1
- package/dist/lib/codex-manager/persist-selected-account.js +2 -1
- package/dist/lib/codex-manager/quota-cache-helpers.d.ts +7 -1
- package/dist/lib/codex-manager/quota-cache-helpers.d.ts.map +1 -1
- package/dist/lib/codex-manager/quota-cache-helpers.js +34 -19
- package/dist/lib/codex-manager/repair-commands.d.ts +13 -0
- package/dist/lib/codex-manager/repair-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/repair-commands.js +144 -9
- package/dist/lib/codex-manager.d.ts.map +1 -1
- package/dist/lib/codex-manager.js +25 -6
- package/dist/lib/concurrency.d.ts +3 -0
- package/dist/lib/concurrency.d.ts.map +1 -0
- package/dist/lib/concurrency.js +31 -0
- package/dist/lib/context-budget-response.js +8 -8
- package/dist/lib/context-overflow.js +10 -10
- package/dist/lib/model-route-policy.d.ts +52 -0
- package/dist/lib/model-route-policy.d.ts.map +1 -0
- package/dist/lib/model-route-policy.js +159 -0
- package/dist/lib/oauth-success.html +125 -125
- package/dist/lib/policy/runtime-policy.d.ts +1 -0
- package/dist/lib/policy/runtime-policy.d.ts.map +1 -1
- package/dist/lib/policy/runtime-policy.js +3 -0
- package/dist/lib/preemptive-quota-scheduler.d.ts +2 -0
- package/dist/lib/preemptive-quota-scheduler.d.ts.map +1 -1
- package/dist/lib/preemptive-quota-scheduler.js +4 -0
- package/dist/lib/prompts/codex-host-bridge.js +138 -138
- package/dist/lib/prompts/codex.js +79 -79
- package/dist/lib/quota-cache.d.ts +7 -3
- package/dist/lib/quota-cache.d.ts.map +1 -1
- package/dist/lib/quota-cache.js +91 -59
- package/dist/lib/quota-probe.d.ts +5 -0
- package/dist/lib/quota-probe.d.ts.map +1 -1
- package/dist/lib/quota-probe.js +26 -2
- package/dist/lib/quota-readiness.d.ts +7 -2
- package/dist/lib/quota-readiness.d.ts.map +1 -1
- package/dist/lib/quota-readiness.js +25 -4
- package/dist/lib/request/client-cancellation.d.ts +5 -0
- package/dist/lib/request/client-cancellation.d.ts.map +1 -0
- package/dist/lib/request/client-cancellation.js +7 -0
- package/dist/lib/request/response-outcome.d.ts +25 -0
- package/dist/lib/request/response-outcome.d.ts.map +1 -0
- package/dist/lib/request/response-outcome.js +49 -0
- package/dist/lib/request/stream-failover-runtime.d.ts +2 -1
- package/dist/lib/request/stream-failover-runtime.d.ts.map +1 -1
- package/dist/lib/request/stream-failover-runtime.js +29 -10
- package/dist/lib/runtime/account-check.d.ts.map +1 -1
- package/dist/lib/runtime/account-check.js +2 -8
- package/dist/lib/runtime/account-model-catalog.d.ts +64 -0
- package/dist/lib/runtime/account-model-catalog.d.ts.map +1 -0
- package/dist/lib/runtime/account-model-catalog.js +154 -0
- package/dist/lib/runtime/account-pool.d.ts +3 -1
- package/dist/lib/runtime/account-pool.d.ts.map +1 -1
- package/dist/lib/runtime/account-pool.js +4 -0
- package/dist/lib/runtime/account-reset-credits.d.ts +42 -0
- package/dist/lib/runtime/account-reset-credits.d.ts.map +1 -0
- package/dist/lib/runtime/account-reset-credits.js +82 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts +30 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts.map +1 -0
- package/dist/lib/runtime/api-model-capabilities.js +434 -0
- package/dist/lib/runtime/api-model-runtime.d.ts +42 -0
- package/dist/lib/runtime/api-model-runtime.d.ts.map +1 -0
- package/dist/lib/runtime/api-model-runtime.js +358 -0
- package/dist/lib/runtime/app-bind.d.ts +19 -1
- package/dist/lib/runtime/app-bind.d.ts.map +1 -1
- package/dist/lib/runtime/app-bind.js +89 -11
- package/dist/lib/runtime/automatic-account-checks.d.ts +35 -0
- package/dist/lib/runtime/automatic-account-checks.d.ts.map +1 -0
- package/dist/lib/runtime/automatic-account-checks.js +140 -0
- package/dist/lib/runtime/automatic-subscription-checks.d.ts +7 -0
- package/dist/lib/runtime/automatic-subscription-checks.d.ts.map +1 -0
- package/dist/lib/runtime/automatic-subscription-checks.js +54 -0
- package/dist/lib/runtime/catalog-capabilities.d.ts +8 -0
- package/dist/lib/runtime/catalog-capabilities.d.ts.map +1 -0
- package/dist/lib/runtime/catalog-capabilities.js +61 -0
- package/dist/lib/runtime/inference-activity.d.ts +15 -0
- package/dist/lib/runtime/inference-activity.d.ts.map +1 -0
- package/dist/lib/runtime/inference-activity.js +95 -0
- package/dist/lib/runtime/model-discovery-status.d.ts +100 -0
- package/dist/lib/runtime/model-discovery-status.d.ts.map +1 -0
- package/dist/lib/runtime/model-discovery-status.js +387 -0
- package/dist/lib/runtime/native-account-storage.d.ts +11 -0
- package/dist/lib/runtime/native-account-storage.d.ts.map +1 -0
- package/dist/lib/runtime/native-account-storage.js +50 -0
- package/dist/lib/runtime/native-account-sync.d.ts +19 -0
- package/dist/lib/runtime/native-account-sync.d.ts.map +1 -0
- package/dist/lib/runtime/native-account-sync.js +62 -0
- package/dist/lib/runtime/native-binding-lock.d.ts +3 -0
- package/dist/lib/runtime/native-binding-lock.d.ts.map +1 -0
- package/dist/lib/runtime/native-binding-lock.js +5 -0
- package/dist/lib/runtime/native-client-auth.d.ts +3 -0
- package/dist/lib/runtime/native-client-auth.d.ts.map +1 -0
- package/dist/lib/runtime/native-client-auth.js +42 -0
- package/dist/lib/runtime/native-provider-config.d.ts +4 -0
- package/dist/lib/runtime/native-provider-config.d.ts.map +1 -0
- package/dist/lib/runtime/native-provider-config.js +52 -0
- package/dist/lib/runtime/native-rate-limits.d.ts +18 -0
- package/dist/lib/runtime/native-rate-limits.d.ts.map +1 -0
- package/dist/lib/runtime/native-rate-limits.js +135 -0
- package/dist/lib/runtime/reset-credit-routing.d.ts +41 -0
- package/dist/lib/runtime/reset-credit-routing.d.ts.map +1 -0
- package/dist/lib/runtime/reset-credit-routing.js +66 -0
- package/dist/lib/runtime/reset-credits.d.ts +101 -0
- package/dist/lib/runtime/reset-credits.d.ts.map +1 -0
- package/dist/lib/runtime/reset-credits.js +162 -0
- package/dist/lib/runtime/response-output-history.d.ts +17 -0
- package/dist/lib/runtime/response-output-history.d.ts.map +1 -0
- package/dist/lib/runtime/response-output-history.js +85 -0
- package/dist/lib/runtime/responses-websocket.d.ts +31 -0
- package/dist/lib/runtime/responses-websocket.d.ts.map +1 -0
- package/dist/lib/runtime/responses-websocket.js +636 -0
- package/dist/lib/runtime/resume-picker.d.ts +41 -0
- package/dist/lib/runtime/resume-picker.d.ts.map +1 -0
- package/dist/lib/runtime/resume-picker.js +169 -0
- package/dist/lib/runtime/rotation-account-selection.d.ts +6 -0
- package/dist/lib/runtime/rotation-account-selection.d.ts.map +1 -1
- package/dist/lib/runtime/rotation-account-selection.js +63 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts +33 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts.map +1 -1
- package/dist/lib/runtime/rotation-server-types.d.ts +18 -0
- package/dist/lib/runtime/rotation-server-types.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-capability-failures.d.ts +14 -0
- package/dist/lib/runtime/runtime-capability-failures.d.ts.map +1 -0
- package/dist/lib/runtime/runtime-capability-failures.js +76 -0
- package/dist/lib/runtime/runtime-observability.d.ts +7 -0
- package/dist/lib/runtime/runtime-observability.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-observability.js +42 -2
- package/dist/lib/runtime/subscription-first-use.d.ts +10 -0
- package/dist/lib/runtime/subscription-first-use.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-first-use.js +64 -0
- package/dist/lib/runtime/subscription-quota-event.d.ts +6 -0
- package/dist/lib/runtime/subscription-quota-event.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-quota-event.js +31 -0
- package/dist/lib/runtime/subscription-quota-order.d.ts +17 -0
- package/dist/lib/runtime/subscription-quota-order.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-quota-order.js +31 -0
- package/dist/lib/runtime/workspace-model-scopes.d.ts +15 -0
- package/dist/lib/runtime/workspace-model-scopes.d.ts.map +1 -0
- package/dist/lib/runtime/workspace-model-scopes.js +34 -0
- package/dist/lib/runtime-constants.d.ts +3 -0
- package/dist/lib/runtime-constants.d.ts.map +1 -1
- package/dist/lib/runtime-constants.js +5 -0
- package/dist/lib/runtime-paths.d.ts.map +1 -1
- package/dist/lib/runtime-paths.js +4 -1
- package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
- package/dist/lib/runtime-rotation-proxy.js +852 -42
- package/dist/lib/schemas.d.ts +20 -0
- package/dist/lib/schemas.d.ts.map +1 -1
- package/dist/lib/schemas.js +3 -0
- package/dist/lib/storage/credential-sidecars.d.ts +13 -0
- package/dist/lib/storage/credential-sidecars.d.ts.map +1 -0
- package/dist/lib/storage/credential-sidecars.js +52 -0
- package/dist/lib/storage/file-lock.d.ts +9 -0
- package/dist/lib/storage/file-lock.d.ts.map +1 -0
- package/dist/lib/storage/file-lock.js +186 -0
- package/dist/lib/storage/flagged-storage.d.ts.map +1 -1
- package/dist/lib/storage/flagged-storage.js +12 -0
- package/dist/lib/storage/path-state.d.ts +2 -0
- package/dist/lib/storage/path-state.d.ts.map +1 -1
- package/dist/lib/storage/path-state.js +9 -0
- package/dist/lib/storage/paths.js +4 -4
- package/dist/lib/storage/pending-auth.d.ts +18 -0
- package/dist/lib/storage/pending-auth.d.ts.map +1 -0
- package/dist/lib/storage/pending-auth.js +155 -0
- package/dist/lib/storage/public-types.d.ts +13 -0
- package/dist/lib/storage/public-types.d.ts.map +1 -1
- package/dist/lib/storage/record-identity.d.ts +16 -0
- package/dist/lib/storage/record-identity.d.ts.map +1 -0
- package/dist/lib/storage/record-identity.js +16 -0
- package/dist/lib/storage/snapshot-merge.d.ts +16 -0
- package/dist/lib/storage/snapshot-merge.d.ts.map +1 -0
- package/dist/lib/storage/snapshot-merge.js +206 -0
- package/dist/lib/storage/transactions.d.ts.map +1 -1
- package/dist/lib/storage/transactions.js +5 -4
- package/dist/lib/storage.d.ts +6 -0
- package/dist/lib/storage.d.ts.map +1 -1
- package/dist/lib/storage.js +77 -7
- package/dist/lib/ui/auth-menu-builder.d.ts +2 -0
- package/dist/lib/ui/auth-menu-builder.d.ts.map +1 -1
- package/dist/lib/ui/auth-menu-builder.js +1 -0
- package/dist/lib/ui/auth-menu.d.ts.map +1 -1
- package/dist/lib/ui/auth-menu.js +1 -0
- package/dist/lib/ui/check-progress.d.ts +8 -0
- package/dist/lib/ui/check-progress.d.ts.map +1 -0
- package/dist/lib/ui/check-progress.js +30 -0
- package/dist/lib/ui/format.d.ts +3 -1
- package/dist/lib/ui/format.d.ts.map +1 -1
- package/dist/lib/ui/format.js +8 -0
- package/dist/lib/ui/ui-copy.d.ts +2 -2
- package/dist/lib/ui/ui-copy.js +2 -2
- package/dist/lib/usage/pricing.js +2 -2
- package/dist/lib/usage/usage-extraction.d.ts +3 -1
- package/dist/lib/usage/usage-extraction.d.ts.map +1 -1
- package/dist/lib/usage/usage-extraction.js +46 -9
- package/node_modules/@codex-ai/plugin/dist/index.d.ts +2 -2
- package/node_modules/@codex-ai/plugin/dist/index.js +2 -2
- package/node_modules/@codex-ai/plugin/dist/tool.d.ts +42 -42
- package/node_modules/@codex-ai/plugin/dist/tool.js +29 -29
- package/node_modules/@codex-ai/plugin/package.json +9 -9
- package/node_modules/@codex-ai/sdk/dist/index.d.ts +4 -4
- package/node_modules/@codex-ai/sdk/dist/index.js +2 -2
- package/node_modules/@codex-ai/sdk/package.json +8 -8
- package/package.json +200 -197
- package/scripts/codex-app-launcher.js +653 -653
- package/scripts/codex-app-router.js +373 -366
- package/scripts/codex-bin-resolver.js +313 -281
- package/scripts/codex-multi-auth.js +59 -59
- package/scripts/codex-routing.js +62 -61
- package/scripts/codex.js +6642 -6564
- package/scripts/install-codex-auth-utils.js +213 -213
- package/scripts/mcodex.js +367 -367
- package/scripts/postinstall.js +114 -114
- package/scripts/preuninstall.js +265 -265
- package/vendor/codex-ai-plugin/dist/index.d.ts +2 -2
- package/vendor/codex-ai-plugin/dist/index.js +2 -2
- package/vendor/codex-ai-plugin/dist/tool.d.ts +42 -42
- package/vendor/codex-ai-plugin/dist/tool.js +29 -29
- package/vendor/codex-ai-plugin/package.json +9 -9
- package/vendor/codex-ai-sdk/dist/index.d.ts +4 -4
- package/vendor/codex-ai-sdk/dist/index.js +2 -2
- package/vendor/codex-ai-sdk/package.json +8 -8
|
@@ -1,12 +1,38 @@
|
|
|
1
|
+
import { startAutomaticAccountChecks } from "./runtime/automatic-account-checks.js";
|
|
2
|
+
import { createAutomaticSubscriptionCheck } from "./runtime/automatic-subscription-checks.js";
|
|
3
|
+
import { ClientCancellationError } from "./request/client-cancellation.js";
|
|
4
|
+
import { createNativeAccountStorageReader } from "./runtime/native-account-storage.js";
|
|
5
|
+
import { resetSnapshotQuota } from "./runtime/reset-credits.js";
|
|
6
|
+
import { createResetCreditService, loadResetCreditState } from "./runtime/account-reset-credits.js";
|
|
7
|
+
import { recoverResetQuota, applyConfirmedReset } from "./runtime/reset-credit-routing.js";
|
|
8
|
+
import { readSubscriptionQuotaEvent } from "./runtime/subscription-quota-event.js";
|
|
9
|
+
import { loadQuotaCache } from "./quota-cache.js";
|
|
10
|
+
import { findQuotaCacheEntryForAccount } from "./quota-readiness.js";
|
|
11
|
+
import { subscriptionQuotaPreference, compareSubscriptionQuota } from "./runtime/subscription-quota-order.js";
|
|
12
|
+
import { inferenceAccountKey } from "./runtime/inference-activity.js";
|
|
13
|
+
import { modelScopeId, workspaceModelScopes } from "./runtime/workspace-model-scopes.js";
|
|
14
|
+
import { RuntimeCapabilityFailures, classifyCapabilityFailure } from "./runtime/runtime-capability-failures.js";
|
|
15
|
+
import { ResponseOutcome } from "./request/response-outcome.js";
|
|
16
|
+
import { ResponsesWebSocketGateway } from "./runtime/responses-websocket.js";
|
|
17
|
+
import { ApiModelCapabilities } from "./runtime/api-model-capabilities.js";
|
|
18
|
+
import { saveModelInventory } from "./runtime/model-discovery-status.js";
|
|
19
|
+
import { loadApiRoutes, updateApiModelDiscovery } from "./api-route-store.js";
|
|
20
|
+
import { ApiModelRuntime } from "./runtime/api-model-runtime.js";
|
|
21
|
+
import { buildVisibleModelUnion, parseModelRoute, canonicalServiceTier } from "./model-route-policy.js";
|
|
22
|
+
import { isSameNativeAccount, syncNativeAccountCredentials } from "./runtime/native-account-sync.js";
|
|
23
|
+
import { sanitizeEmail } from "./auth/token-utils.js";
|
|
24
|
+
import { isNativeClientToken } from "./runtime/native-client-auth.js";
|
|
25
|
+
import { CatalogRetryError, AccountModelCatalog, clampCatalogRetryMs } from "./runtime/account-model-catalog.js";
|
|
1
26
|
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
|
|
2
27
|
import { createServer } from "node:http";
|
|
28
|
+
import * as zlib from "node:zlib";
|
|
3
29
|
import { AccountManager, AUTH_INVALIDATION_MARKER, extractAccountId, } from "./accounts.js";
|
|
4
30
|
import { withRoutingMutex } from "./routing-mutex.js";
|
|
5
31
|
import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getMinRotationIntervalMs, getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, getPreemptiveQuotaEnabled, getPreemptiveQuotaMaxDeferralMs, getPreemptiveQuotaRemainingPercent5h, getPreemptiveQuotaRemainingPercent7d, getContextBudgetGuardEnabled, getContextBudgetGuardSoftPercent, getContextBudgetGuardHardPercent, getContextBudgetGuardModelWindowOverrides, getRoutingMutexMode, getSchedulingStrategy, loadPluginConfig, } from "./config.js";
|
|
6
32
|
import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
|
|
7
33
|
import { getModelFamily } from "./prompts/codex.js";
|
|
8
34
|
import { CURRENT_CODEX_MODEL } from "./request/helpers/model-map.js";
|
|
9
|
-
import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
|
|
35
|
+
import { recordRuntimeInferenceRequest, flushRuntimeInferenceActivity, mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
|
|
10
36
|
import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
|
|
11
37
|
import { createUsageStreamScanner } from "./usage/usage-extraction.js";
|
|
12
38
|
import { isModelAtCapacityError, isWorkspaceDisabledError, } from "./request/fetch-helpers.js";
|
|
@@ -167,6 +193,30 @@ const CAPACITY_RETRY_BACKOFF_STEPS_MS = [
|
|
|
167
193
|
const DEFAULT_MODEL_CAPACITY_RETRY_MS = 10 * 60_000;
|
|
168
194
|
const MAX_MODEL_CAPACITY_RETRY_MS = 60 * 60_000;
|
|
169
195
|
const MODEL_CAPACITY_RETRY_ENV = "CODEX_MULTI_AUTH_MODEL_CAPACITY_RETRY_MS";
|
|
196
|
+
function getApiModelRuntime(state) {
|
|
197
|
+
if (state.apiModelRuntime)
|
|
198
|
+
return state.apiModelRuntime;
|
|
199
|
+
const capabilities = state.apiModelCapabilities ??= new ApiModelCapabilities();
|
|
200
|
+
const production = state.readApiRoutes === loadApiRoutes;
|
|
201
|
+
state.apiModelRuntime = new ApiModelRuntime(state.fetchImpl, state.now, production ? updateApiModelDiscovery : undefined, production ? (models, refresh, route, force) => capabilities.enrich(models, refresh, route, force) : undefined, () => {
|
|
202
|
+
const runtime = state.apiModelRuntime;
|
|
203
|
+
if (!runtime)
|
|
204
|
+
return;
|
|
205
|
+
const models = [...(state.catalogOAuthModels ?? []), ...buildVisibleModelUnion(runtime.cachedCatalogs())];
|
|
206
|
+
setCatalogEtag(state, models);
|
|
207
|
+
if (state.catalogInventory) {
|
|
208
|
+
state.catalogInventory = { ...state.catalogInventory, checkedAt: state.now(), entries: [
|
|
209
|
+
...state.catalogInventory.entries.filter(entry => entry.kind === "oauth"),
|
|
210
|
+
...runtime.statuses(state.catalogApiRoutes ?? []).map(entry => ({ id: modelScopeId(entry.kind, entry.id), label: entry.label, kind: entry.kind, enabled: runtime.cachedCatalogs().some(c => c.id === entry.id && c.enabled), checkedAt: entry.checkedAt, error: entry.error, models: entry.availableModels, visibleModels: entry.visibleModels, entitlements: entry.entitlements })),
|
|
211
|
+
] };
|
|
212
|
+
void saveModelInventory(state.catalogInventory).catch(() => { state.status.lastError = "Model discovery status could not be saved"; });
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
return state.apiModelRuntime;
|
|
216
|
+
}
|
|
217
|
+
function setCatalogEtag(state, models) {
|
|
218
|
+
return state.catalogEtag = '"' + createHash("sha256").update(JSON.stringify(models)).digest("hex") + '"';
|
|
219
|
+
}
|
|
170
220
|
function capacityRetryBackoffMs(attempt) {
|
|
171
221
|
const index = Math.min(Math.max(0, attempt - 1), CAPACITY_RETRY_BACKOFF_STEPS_MS.length - 1);
|
|
172
222
|
return CAPACITY_RETRY_BACKOFF_STEPS_MS[index] ?? 0;
|
|
@@ -266,6 +316,23 @@ function createOutboundHeaders(incoming, account, accessToken, accountId) {
|
|
|
266
316
|
headers.set(OPENAI_HEADERS.ORIGINATOR, OPENAI_HEADER_VALUES.ORIGINATOR_CODEX);
|
|
267
317
|
return headers;
|
|
268
318
|
}
|
|
319
|
+
/** Files every native Responses request reads change rarely; share one read per second. */
|
|
320
|
+
const PER_REQUEST_READ_TTL_MS = 1000;
|
|
321
|
+
function cachedRead(state, key, load) {
|
|
322
|
+
const cache = state.readCache ??= new Map();
|
|
323
|
+
const now = Date.now();
|
|
324
|
+
const hit = cache.get(key);
|
|
325
|
+
if (hit && hit.at <= now && now - hit.at < PER_REQUEST_READ_TTL_MS)
|
|
326
|
+
return hit.value;
|
|
327
|
+
const value = load();
|
|
328
|
+
cache.set(key, { at: now, value });
|
|
329
|
+
value.catch(() => { if (cache.get(key)?.value === value)
|
|
330
|
+
cache.delete(key); });
|
|
331
|
+
return value;
|
|
332
|
+
}
|
|
333
|
+
function catalogAccountKey(account) {
|
|
334
|
+
return workspaceModelScopes(account).find(scope => scope.bound)?.id ?? "unavailable";
|
|
335
|
+
}
|
|
269
336
|
function isAuthorizedClient(headers, clientApiKey) {
|
|
270
337
|
const authorization = headers.get("authorization") ?? "";
|
|
271
338
|
const bearerMatch = authorization.match(/^Bearer\s+(.+)$/i);
|
|
@@ -299,20 +366,22 @@ function accountIdentityFromAccount(account, updatedAt) {
|
|
|
299
366
|
updatedAt,
|
|
300
367
|
};
|
|
301
368
|
}
|
|
302
|
-
function recordLastRuntimeAccount(status, identity) {
|
|
369
|
+
function recordLastRuntimeAccount(status, identity, requestedWorkspaceId) {
|
|
303
370
|
status.lastAccountIndex = identity.index;
|
|
304
371
|
status.lastAccountLabel = identity.label;
|
|
305
372
|
status.lastAccountId = identity.accountId;
|
|
373
|
+
status.lastRequestedWorkspaceId = requestedWorkspaceId;
|
|
306
374
|
status.lastAccountUpdatedAt = identity.updatedAt;
|
|
307
375
|
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
308
376
|
snapshot.lastAccountIndex = identity.index;
|
|
309
377
|
snapshot.lastAccountLabel = identity.label;
|
|
310
378
|
snapshot.lastAccountEmail = null;
|
|
311
379
|
snapshot.lastAccountId = identity.accountId;
|
|
380
|
+
snapshot.lastRequestedWorkspaceId = requestedWorkspaceId;
|
|
312
381
|
snapshot.lastAccountUpdatedAt = identity.updatedAt;
|
|
313
382
|
});
|
|
314
383
|
}
|
|
315
|
-
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy) {
|
|
384
|
+
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy, preserveDesktopLogin = false) {
|
|
316
385
|
if (isPinned) {
|
|
317
386
|
// When the user has manually pinned an account, the proxy MUST NOT
|
|
318
387
|
// clobber that pin via markSwitched("rotation"), saveToDiskDebounced(),
|
|
@@ -349,7 +418,8 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
|
|
|
349
418
|
await accountManager.markSwitchedLocked(account, "rotation", family);
|
|
350
419
|
}
|
|
351
420
|
accountManager.saveToDiskDebounced();
|
|
352
|
-
|
|
421
|
+
if (!preserveDesktopLogin)
|
|
422
|
+
await accountManager.syncCodexCliActiveSelectionForIndex(account.index);
|
|
353
423
|
}
|
|
354
424
|
catch {
|
|
355
425
|
// Runtime forwarding must not fail after a valid upstream response just
|
|
@@ -379,6 +449,31 @@ async function readRequestBody(req, maxBytes = MAX_REQUEST_BODY_BYTES) {
|
|
|
379
449
|
}
|
|
380
450
|
return Buffer.concat(chunks);
|
|
381
451
|
}
|
|
452
|
+
/** Inspect the decoded entity before any credential or policy decision. */
|
|
453
|
+
async function decodeRequestBody(body, encoding, maxBytes) {
|
|
454
|
+
const coding = encoding?.trim().toLowerCase() ?? "identity";
|
|
455
|
+
if (!coding || coding === "identity")
|
|
456
|
+
return body;
|
|
457
|
+
// Older supported Node releases lack Zstandard. Reject explicitly there;
|
|
458
|
+
// forwarding opaque bytes would bypass the model/credential boundary.
|
|
459
|
+
const zstd = zlib.zstdDecompress;
|
|
460
|
+
const decode = coding === "gzip" ? zlib.gunzip
|
|
461
|
+
: coding === "deflate" ? zlib.inflate
|
|
462
|
+
: coding === "br" ? zlib.brotliDecompress
|
|
463
|
+
: coding === "zstd" ? zstd : undefined;
|
|
464
|
+
if (!decode)
|
|
465
|
+
throw createRuntimeProxyHttpError("Unsupported request content encoding. Zstandard requires a Node runtime with Zstandard support.", 415, "unsupported_content_encoding");
|
|
466
|
+
return new Promise((resolve, reject) => {
|
|
467
|
+
decode(body, { maxOutputLength: maxBytes }, (error, decoded) => {
|
|
468
|
+
if (error) {
|
|
469
|
+
const tooLarge = "code" in error && error.code === "ERR_BUFFER_TOO_LARGE";
|
|
470
|
+
reject(createRuntimeProxyHttpError(tooLarge ? "Decoded request body is too large." : "Invalid compressed request body.", tooLarge ? 413 : 400, tooLarge ? "runtime_rotation_proxy_payload_too_large" : "invalid_request_body"));
|
|
471
|
+
}
|
|
472
|
+
else
|
|
473
|
+
resolve(decoded);
|
|
474
|
+
});
|
|
475
|
+
});
|
|
476
|
+
}
|
|
382
477
|
function parseRequestBody(body) {
|
|
383
478
|
if (body.length === 0)
|
|
384
479
|
return null;
|
|
@@ -696,9 +791,9 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
696
791
|
const preemptiveQuotaScheduler = new PreemptiveQuotaScheduler({
|
|
697
792
|
enabled: getPreemptiveQuotaEnabled(pluginConfig),
|
|
698
793
|
remainingPercentThresholdPrimary: options.quotaRemainingPercentThreshold ??
|
|
699
|
-
getPreemptiveQuotaRemainingPercent5h(pluginConfig),
|
|
794
|
+
(options.nativeOpenai ? 0 : getPreemptiveQuotaRemainingPercent5h(pluginConfig)),
|
|
700
795
|
remainingPercentThresholdSecondary: options.quotaRemainingPercentThreshold ??
|
|
701
|
-
getPreemptiveQuotaRemainingPercent7d(pluginConfig),
|
|
796
|
+
(options.nativeOpenai ? 0 : getPreemptiveQuotaRemainingPercent7d(pluginConfig)),
|
|
702
797
|
maxDeferralMs: getPreemptiveQuotaMaxDeferralMs(pluginConfig),
|
|
703
798
|
});
|
|
704
799
|
const contextBudgetGuard = new ContextBudgetGuard({
|
|
@@ -718,6 +813,11 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
718
813
|
// are detected per-request via `maybeInvalidateAffinityFromDisk`.
|
|
719
814
|
const lastObservedAffinityGeneration = readStorageMetaFromDisk().affinityGeneration;
|
|
720
815
|
const state = createRotationProxyState({
|
|
816
|
+
nativeOpenai: options.nativeOpenai === true,
|
|
817
|
+
readNativeAccountStorage: options.readNativeAccountStorage || !options.accountManager ? createNativeAccountStorageReader(options.readNativeAccountStorage) : undefined,
|
|
818
|
+
readApiRoutes: options.readApiRoutes ?? loadApiRoutes,
|
|
819
|
+
readSubscriptionQuota: options.readSubscriptionQuota ?? (options.accountManager ? undefined : loadQuotaCache),
|
|
820
|
+
catalogAccount: options.catalogAccount,
|
|
721
821
|
activeAccountManager,
|
|
722
822
|
routingMutexMode,
|
|
723
823
|
schedulingStrategy,
|
|
@@ -743,8 +843,10 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
743
843
|
lastObservedAffinityGeneration,
|
|
744
844
|
forcedAccountIndex,
|
|
745
845
|
});
|
|
846
|
+
const websocketGateway = new ResponsesWebSocketGateway(fetchImpl, { maxPayloadBytes: state.maxRequestBodyBytes });
|
|
847
|
+
state.fetchImpl = websocketGateway.fetch;
|
|
746
848
|
const server = createServer((req, res) => {
|
|
747
|
-
|
|
849
|
+
websocketGateway.handle(req, () => handleRequest(state, req, res));
|
|
748
850
|
});
|
|
749
851
|
const sockets = new Set();
|
|
750
852
|
server.on("connection", (socket) => {
|
|
@@ -772,12 +874,19 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
772
874
|
server.on("error", onPostStartupServerError);
|
|
773
875
|
const address = server.address();
|
|
774
876
|
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
877
|
+
if (state.nativeOpenai)
|
|
878
|
+
websocketGateway.attach(server, `http://${urlHost}:${resolvedPort}`);
|
|
879
|
+
// Embedded managers own their own lifecycle; ordinary CLI/app routers check opted-in accounts periodically.
|
|
880
|
+
const automaticChecks = options.accountManager ? undefined : startAutomaticAccountChecks(createAutomaticSubscriptionCheck());
|
|
775
881
|
return {
|
|
776
882
|
host: bindHost,
|
|
777
883
|
port: resolvedPort,
|
|
778
884
|
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
779
885
|
close: async () => {
|
|
886
|
+
await automaticChecks?.stop();
|
|
887
|
+
websocketGateway.close();
|
|
780
888
|
await closeServer(server, sockets);
|
|
889
|
+
await flushRuntimeInferenceActivity();
|
|
781
890
|
await state.activeAccountManager.flushPendingSave();
|
|
782
891
|
},
|
|
783
892
|
// Live client connections, which the app helper reads as evidence that a
|
|
@@ -786,6 +895,8 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
786
895
|
getOpenConnectionCount: () => sockets.size,
|
|
787
896
|
getStatus: () => ({
|
|
788
897
|
...state.status,
|
|
898
|
+
websocketConnections: websocketGateway.stats.connections,
|
|
899
|
+
websocketUpstreamRequests: websocketGateway.stats.upstreamRequests,
|
|
789
900
|
// Redact any email/token material that leaked into a raw upstream or
|
|
790
901
|
// refresh error string before exposing it to status/report consumers
|
|
791
902
|
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
@@ -811,10 +922,116 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
811
922
|
// unauthorized). Authorized callers still fall through to the 404 below
|
|
812
923
|
// when they hit an unsupported path/method.
|
|
813
924
|
const incomingHeaders = headersFromIncoming(req);
|
|
814
|
-
|
|
925
|
+
const bearer = incomingHeaders.get("authorization")?.match(/^Bearer\s+(.+)$/i)?.[1]?.trim();
|
|
926
|
+
const apiKeyClient = isAuthorizedClient(incomingHeaders, state.clientApiKey);
|
|
927
|
+
const isLiveManagedToken = (access, expires, enabled, invalidatedAt) => !!bearer && enabled !== false && !invalidatedAt && !!access && (expires ?? 0) > state.now() && safeEqual(bearer, access);
|
|
928
|
+
const managedOAuthFor = (manager) => state.nativeOpenai === true && manager.getAccountsSnapshot().some(account => isLiveManagedToken(account.access, account.expires, account.enabled, account.authInvalidatedAt));
|
|
929
|
+
let managedStorageVerified = true;
|
|
930
|
+
// A missing or unreadable store (not a transient lock) must not become an
|
|
931
|
+
// empty pool: that would discard the live manager's learned state and hide
|
|
932
|
+
// the cause behind catalog/403 errors.
|
|
933
|
+
let nativeStorageMissing = false;
|
|
934
|
+
let nativeOAuthResult;
|
|
935
|
+
const nativeOAuth = async () => (nativeOAuthResult ??= !!(state.nativeOpenai && bearer && await isNativeClientToken(bearer, state.now())));
|
|
936
|
+
if (state.nativeOpenai && state.readNativeAccountStorage) {
|
|
937
|
+
// Screen the caller before account storage can reshape the manager: a
|
|
938
|
+
// request with no credential never reads disk, and a bearer that matches
|
|
939
|
+
// nothing we hold (in memory or on disk) is refused before any credential
|
|
940
|
+
// sync or manager swap.
|
|
941
|
+
if (!apiKeyClient && !bearer) {
|
|
942
|
+
writeUnauthorized(res);
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
const snapshot = await state.readNativeAccountStorage();
|
|
946
|
+
const disk = snapshot.storage ?? { version: 3, accounts: [], activeIndex: 0, activeIndexByFamily: {} };
|
|
947
|
+
managedStorageVerified = snapshot.verified && snapshot.storage !== null;
|
|
948
|
+
// A concurrent request may have replaced the manager while this read waited.
|
|
949
|
+
accountManager = state.activeAccountManager;
|
|
950
|
+
const presentsKnownCredential = apiKeyClient || (snapshot.storage !== null && managedOAuthFor(accountManager)) ||
|
|
951
|
+
!!disk?.accounts.some(a => isLiveManagedToken(a.accessToken, a.expiresAt, a.enabled, a.authInvalidatedAt)) ||
|
|
952
|
+
await nativeOAuth();
|
|
953
|
+
if (!presentsKnownCredential) {
|
|
954
|
+
writeUnauthorized(res);
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
if (snapshot.transientFailure && snapshot.routingAvailable === false && (apiKeyClient || await nativeOAuth())) {
|
|
958
|
+
writeJson(res, 503, { error: { code: "native_account_storage_unavailable", message: "Account storage is temporarily unavailable. Retry when it is readable." } });
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
nativeStorageMissing = snapshot.storage === null && !snapshot.transientFailure;
|
|
962
|
+
if (snapshot.verified && snapshot.storage !== null) {
|
|
963
|
+
const accounts = accountManager.getAccountsSnapshot();
|
|
964
|
+
const sameInventory = accounts.length === disk.accounts.length && accounts.every((a, i) => isSameNativeAccount(a, disk.accounts[i]));
|
|
965
|
+
if (!sameInventory) {
|
|
966
|
+
accountManager = new AccountManager(undefined, disk);
|
|
967
|
+
accountManager.setRoutingMutexMode(state.routingMutexMode);
|
|
968
|
+
// Preserve independently learned limits by identity, never by the old index.
|
|
969
|
+
// A record id survives an email change; the account identity survives a
|
|
970
|
+
// re-login that re-derives an unstored record id. Rows with neither an
|
|
971
|
+
// accountId nor an email share no identity and match only by refresh token.
|
|
972
|
+
const rebuilt = accountManager.getAccountsSnapshot();
|
|
973
|
+
const claimed = new Set();
|
|
974
|
+
const identityOf = (a) => {
|
|
975
|
+
const id = a.accountId?.trim(), email = sanitizeEmail(a.email);
|
|
976
|
+
return id || email ? JSON.stringify([id ?? null, email ?? null]) : a.refreshToken?.trim() ? `refresh:${a.refreshToken.trim()}` : null;
|
|
977
|
+
};
|
|
978
|
+
const carryTarget = (previous) => {
|
|
979
|
+
const unclaimed = rebuilt.filter(a => !claimed.has(a.index));
|
|
980
|
+
const identity = identityOf(previous);
|
|
981
|
+
const found = unclaimed.find(a => a.recordId && a.recordId === previous.recordId)
|
|
982
|
+
?? (identity === null ? undefined : unclaimed.find(a => identityOf(a) === identity));
|
|
983
|
+
if (found)
|
|
984
|
+
claimed.add(found.index);
|
|
985
|
+
return found;
|
|
986
|
+
};
|
|
987
|
+
for (const previous of accounts) {
|
|
988
|
+
const current = carryTarget(previous);
|
|
989
|
+
const live = current && accountManager.getAccountByIndex(current.index);
|
|
990
|
+
if (!live)
|
|
991
|
+
continue;
|
|
992
|
+
for (const [key, until] of Object.entries(previous.rateLimitResetTimes)) {
|
|
993
|
+
if (typeof until === "number" && until > state.now())
|
|
994
|
+
live.rateLimitResetTimes[key] = Math.max(live.rateLimitResetTimes[key] ?? 0, until);
|
|
995
|
+
}
|
|
996
|
+
if (previous.access === live.access && (previous.coolingDownUntil ?? 0) > (live.coolingDownUntil ?? 0)) {
|
|
997
|
+
live.coolingDownUntil = previous.coolingDownUntil;
|
|
998
|
+
live.cooldownReason = previous.cooldownReason;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
// Reorders change index-based pins/affinity, so they deliberately reload too.
|
|
1002
|
+
const generation = disk.affinityGeneration ?? 0;
|
|
1003
|
+
if (generation <= state.lastObservedAffinityGeneration)
|
|
1004
|
+
state.sessionAffinityStore?.clearAll();
|
|
1005
|
+
state.activeAccountManager = accountManager;
|
|
1006
|
+
state.knownAccountManagers.add(accountManager);
|
|
1007
|
+
state.modelCatalog = undefined;
|
|
1008
|
+
state.modelCatalogs?.clear();
|
|
1009
|
+
}
|
|
1010
|
+
else if (syncNativeAccountCredentials(accountManager, disk)) {
|
|
1011
|
+
state.modelCatalog = undefined;
|
|
1012
|
+
state.modelCatalogs?.clear();
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
// Decide against the synced manager, so a credential revoked or disabled on
|
|
1017
|
+
// disk is refused even though it passed the screen above.
|
|
1018
|
+
if (!apiKeyClient && !(managedStorageVerified && managedOAuthFor(accountManager)) && !(await nativeOAuth())) {
|
|
815
1019
|
writeUnauthorized(res);
|
|
816
1020
|
return;
|
|
817
1021
|
}
|
|
1022
|
+
// The native client treats 426 as a signal to retry over HTTP/SSE.
|
|
1023
|
+
// Authenticate first; never forward an unsupported WebSocket handshake.
|
|
1024
|
+
if (req.method === "GET" && isResponsesPath(incomingUrl.pathname) &&
|
|
1025
|
+
incomingHeaders.get("upgrade")?.toLowerCase() === "websocket") {
|
|
1026
|
+
res.setHeader("connection", "close");
|
|
1027
|
+
writeJson(res, 426, {
|
|
1028
|
+
error: {
|
|
1029
|
+
message: "Use HTTP streaming for Responses API requests.",
|
|
1030
|
+
code: "runtime_rotation_proxy_http_required",
|
|
1031
|
+
},
|
|
1032
|
+
});
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
818
1035
|
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
819
1036
|
const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
|
|
820
1037
|
const isImageRequest = req.method === "POST" && ALLOWED_IMAGE_PATHS.has(incomingUrl.pathname);
|
|
@@ -826,10 +1043,31 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
826
1043
|
return;
|
|
827
1044
|
}
|
|
828
1045
|
state.status.totalRequests += 1;
|
|
829
|
-
|
|
1046
|
+
let requestBody = isResponsesRequest || isImageRequest ||
|
|
830
1047
|
(isThreadGoalRequest && req.method === "POST")
|
|
831
1048
|
? await readRequestBody(req, state.maxRequestBodyBytes)
|
|
832
1049
|
: Buffer.alloc(0);
|
|
1050
|
+
if (isResponsesRequest) {
|
|
1051
|
+
requestBody = await decodeRequestBody(requestBody, incomingHeaders.get("content-encoding"), state.maxRequestBodyBytes);
|
|
1052
|
+
const parsed = parseRequestBody(requestBody);
|
|
1053
|
+
if (!parsed || typeof parsed.model !== "string" || !parsed.model.trim()) {
|
|
1054
|
+
throw createRuntimeProxyHttpError("Responses requests require a JSON object with a model.", 400, "invalid_request_body");
|
|
1055
|
+
}
|
|
1056
|
+
let route;
|
|
1057
|
+
try {
|
|
1058
|
+
route = parseModelRoute(parsed.model);
|
|
1059
|
+
}
|
|
1060
|
+
catch {
|
|
1061
|
+
throw createRuntimeProxyHttpError("Invalid model route.", 400, "invalid_model_route");
|
|
1062
|
+
}
|
|
1063
|
+
if (route.kind === "oauth" && route.serviceTier) {
|
|
1064
|
+
if (parsed.service_tier && canonicalServiceTier(String(parsed.service_tier)) !== canonicalServiceTier(route.serviceTier))
|
|
1065
|
+
throw createRuntimeProxyHttpError("Conflicting speed selection.", 400, "conflicting_service_tier");
|
|
1066
|
+
parsed.model = route.upstreamModel;
|
|
1067
|
+
parsed.service_tier = route.serviceTier;
|
|
1068
|
+
requestBody = Buffer.from(JSON.stringify(parsed));
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
833
1071
|
const context = isModelsRequest
|
|
834
1072
|
? buildModelsRequestContext(req)
|
|
835
1073
|
: isThreadGoalRequest
|
|
@@ -837,6 +1075,23 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
837
1075
|
: isImageRequest
|
|
838
1076
|
? buildImageRequestContext(req, requestBody, incomingUrl.pathname)
|
|
839
1077
|
: buildResponsesRequestContext(req, requestBody);
|
|
1078
|
+
if (isResponsesRequest) {
|
|
1079
|
+
context.headers.delete("content-encoding");
|
|
1080
|
+
context.headers.delete("content-length");
|
|
1081
|
+
}
|
|
1082
|
+
// Explicit API/ZDR routes never touch the OAuth pool, so they stay available.
|
|
1083
|
+
// Everything else would route through a manager the store no longer backs.
|
|
1084
|
+
// A 401 would tell the native client its login is bad when the store is what
|
|
1085
|
+
// is missing, so independently authenticated clients get a 503 (#702).
|
|
1086
|
+
if (nativeStorageMissing && !(isResponsesRequest && context.model && /^(api|zdr)\//.test(context.model))) {
|
|
1087
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
1088
|
+
error: {
|
|
1089
|
+
message: "Account storage is unavailable. Run codex-multi-auth login or check the account store.",
|
|
1090
|
+
code: "native_account_storage_unavailable",
|
|
1091
|
+
},
|
|
1092
|
+
});
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
840
1095
|
const requestStartedAt = state.now();
|
|
841
1096
|
let policyDecision = null;
|
|
842
1097
|
let projectKey = null;
|
|
@@ -938,8 +1193,100 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
938
1193
|
return;
|
|
939
1194
|
}
|
|
940
1195
|
}
|
|
1196
|
+
// Explicit API aliases are handled before all OAuth selection. An absent
|
|
1197
|
+
// credential or catalog is a hard failure, never an ordinary-model fallback.
|
|
1198
|
+
if (context.model && /^(api|zdr)\//.test(context.model)) {
|
|
1199
|
+
if (!isResponsesRequest) {
|
|
1200
|
+
writeJson(res, 400, { error: { code: "api_route_requires_responses" } });
|
|
1201
|
+
return;
|
|
1202
|
+
}
|
|
1203
|
+
const body = parseRequestBody(requestBody);
|
|
1204
|
+
if (!body) {
|
|
1205
|
+
writeJson(res, 400, { error: { code: "invalid_request_body" } });
|
|
1206
|
+
return;
|
|
1207
|
+
}
|
|
1208
|
+
const apiRuntime = getApiModelRuntime(state);
|
|
1209
|
+
const controller = new AbortController();
|
|
1210
|
+
const abort = () => {
|
|
1211
|
+
if (!res.writableEnded)
|
|
1212
|
+
controller.abort();
|
|
1213
|
+
};
|
|
1214
|
+
res.on("close", abort);
|
|
1215
|
+
try {
|
|
1216
|
+
const apiRoutes = (await state.readApiRoutes?.()) ?? [];
|
|
1217
|
+
let attemptedCredentialIndex;
|
|
1218
|
+
const upstream = await apiRuntime.request(context.model, body, apiRoutes, controller.signal, (credentialIndex) => {
|
|
1219
|
+
attemptedCredentialIndex = credentialIndex;
|
|
1220
|
+
const credential = apiRoutes[credentialIndex];
|
|
1221
|
+
if (credential)
|
|
1222
|
+
recordRuntimeInferenceRequest(`sha256:${modelScopeId(credential.kind, credential.id)}`, state.now());
|
|
1223
|
+
const label = `${context.model?.startsWith("zdr/") ? "ZDR" : "API"} credential ${credentialIndex + 1}`;
|
|
1224
|
+
state.status.upstreamRequests++;
|
|
1225
|
+
state.status.lastAccountIndex = null;
|
|
1226
|
+
state.status.lastAccountId = null;
|
|
1227
|
+
state.status.lastRequestedWorkspaceId = null;
|
|
1228
|
+
state.status.lastAccountLabel = label;
|
|
1229
|
+
state.status.lastAccountUpdatedAt = state.now();
|
|
1230
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
1231
|
+
snapshot.lastAccountIndex = null;
|
|
1232
|
+
snapshot.lastAccountId = null;
|
|
1233
|
+
snapshot.lastRequestedWorkspaceId = null;
|
|
1234
|
+
snapshot.lastAccountEmail = null;
|
|
1235
|
+
snapshot.lastAccountLabel = label;
|
|
1236
|
+
snapshot.lastAccountUpdatedAt = state.now();
|
|
1237
|
+
});
|
|
1238
|
+
});
|
|
1239
|
+
const responseOutcome = new ResponseOutcome(upstream.headers.get("content-type")?.includes("text/event-stream") === true);
|
|
1240
|
+
const scanner = createUsageStreamScanner({
|
|
1241
|
+
contentType: upstream.headers.get("content-type"),
|
|
1242
|
+
onEvent: event => responseOutcome.observe(event),
|
|
1243
|
+
});
|
|
1244
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => controller.abort(), state.streamStallTimeoutMs, scanner.push, state.catalogEtag ? { "x-models-etag": state.catalogEtag } : undefined, () => { scanner.result(); return responseOutcome.finish(); });
|
|
1245
|
+
const attemptedCredential = attemptedCredentialIndex === undefined ? undefined : apiRoutes[attemptedCredentialIndex];
|
|
1246
|
+
if (!forwarded && attemptedCredential)
|
|
1247
|
+
apiRuntime.recordStreamFailure(attemptedCredential, context.model, body, responseOutcome.rejection);
|
|
1248
|
+
if (apiRuntime.lastPersistenceError)
|
|
1249
|
+
state.status.lastError = apiRuntime.lastPersistenceError;
|
|
1250
|
+
await usageRecorder.record({
|
|
1251
|
+
outcome: forwarded && upstream.ok ? "success" : "failure",
|
|
1252
|
+
statusCode: upstream.status,
|
|
1253
|
+
errorCode: responseOutcome.finish().errorCode ?? (forwarded ? null : "stream_forward_failed"),
|
|
1254
|
+
...(scanner.result() ?? {}),
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1257
|
+
catch (error) {
|
|
1258
|
+
if (error instanceof ClientCancellationError || res.destroyed) {
|
|
1259
|
+
if (!res.destroyed)
|
|
1260
|
+
res.destroy();
|
|
1261
|
+
return;
|
|
1262
|
+
}
|
|
1263
|
+
throw error;
|
|
1264
|
+
}
|
|
1265
|
+
finally {
|
|
1266
|
+
res.off("close", abort);
|
|
1267
|
+
}
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
941
1270
|
const upstreamUrl = buildUpstreamUrl(req, state.upstreamBaseUrl, context.upstreamPath);
|
|
942
1271
|
const attemptedIndexes = new Set();
|
|
1272
|
+
// The upstream's own capability 400/403/404, forwarded instead of a
|
|
1273
|
+
// synthetic 503 when no other account can take the request.
|
|
1274
|
+
let lastCapabilityRejection;
|
|
1275
|
+
// True while no other failure has followed the last capability rejection.
|
|
1276
|
+
let capabilityRejectionIsLatest = false;
|
|
1277
|
+
// Entitlements belong to the workspace, so each workspace is tried at most
|
|
1278
|
+
// once per request even when several accounts share it.
|
|
1279
|
+
const rejectedWorkspaces = new Set();
|
|
1280
|
+
const forwardCapabilityRejection = async (rejection) => {
|
|
1281
|
+
res.writeHead(rejection.status, rejection.headers);
|
|
1282
|
+
res.end(rejection.body);
|
|
1283
|
+
await usageRecorder?.record({ outcome: "failure", statusCode: rejection.status, errorCode: "upstream_capability_rejected", account: rejection.account });
|
|
1284
|
+
};
|
|
1285
|
+
let catalogEligibleKeys;
|
|
1286
|
+
const catalogExcludedIndexes = () => {
|
|
1287
|
+
const eligible = catalogEligibleKeys;
|
|
1288
|
+
return eligible ? accountManager.getAccountsSnapshot().filter(account => !workspaceCandidates.has(account.index)).map(account => account.index) : [];
|
|
1289
|
+
};
|
|
943
1290
|
let exhaustionReason = "no-account";
|
|
944
1291
|
let accountCount = accountManager.getAccountCount();
|
|
945
1292
|
let transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
@@ -1010,8 +1357,283 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1010
1357
|
// everything downstream — deterministic pick, no cursor advance, no
|
|
1011
1358
|
// stale-state recovery, the `codex_pinned_account_unavailable` failure —
|
|
1012
1359
|
// applies unchanged because it all keys off `pinnedIndex` / `isPinned`.
|
|
1360
|
+
const preferredIndex = state.nativeOpenai
|
|
1361
|
+
? storageMeta.pinnedAccountIndex ?? accountManager.getCurrentAccountForFamily(context.family)?.index ?? null
|
|
1362
|
+
: null;
|
|
1363
|
+
const workspaceCandidates = new Map();
|
|
1364
|
+
const readSubscriptionQuota = state.readSubscriptionQuota;
|
|
1365
|
+
const subscriptionQuotaCache = state.nativeOpenai && isResponsesRequest && readSubscriptionQuota ? await cachedRead(state, "subscription-quota", readSubscriptionQuota).catch(() => null) ?? null : null;
|
|
1366
|
+
const subscriptionQuotaByAccount = {};
|
|
1367
|
+
const resetCreditState = state.nativeOpenai && isResponsesRequest ? await cachedRead(state, "reset-credits", loadResetCreditState).catch(() => null) : null;
|
|
1368
|
+
const quotaForScope = (account, scope) => {
|
|
1369
|
+
// Scoped observations belong to this credential and workspace; legacy entries only describe the binding.
|
|
1370
|
+
const cached = findQuotaCacheEntryForAccount(subscriptionQuotaCache, account, accountManager.getAccountsSnapshot(), undefined, scope.accountId);
|
|
1371
|
+
const observed = state.subscriptionQuotaObservations?.get(JSON.stringify([scope.id, context.model]));
|
|
1372
|
+
const reset = resetCreditState?.snapshots[scope.id];
|
|
1373
|
+
if (reset && reset.updatedAt > Math.max(cached?.updatedAt ?? 0, observed?.updatedAt ?? 0) && reset.updatedAt <= state.now() && state.now() - reset.updatedAt <= 60000)
|
|
1374
|
+
return resetSnapshotQuota(reset);
|
|
1375
|
+
return observed && observed.updatedAt >= (cached?.updatedAt ?? 0) ? observed : cached;
|
|
1376
|
+
};
|
|
1377
|
+
// A stored `switch` pin is a hard constraint in native mode too (#702), not a
|
|
1378
|
+
// preference: it fails with codex_pinned_account_unavailable rather than
|
|
1379
|
+
// silently serving from another account.
|
|
1013
1380
|
const pinnedIndex = state.forcedAccountIndex ?? storageMeta.pinnedAccountIndex;
|
|
1014
1381
|
const isPinned = typeof pinnedIndex === "number";
|
|
1382
|
+
if (state.nativeOpenai && (isModelsRequest || (isResponsesRequest && context.model))) {
|
|
1383
|
+
const requestedVersion = incomingUrl.searchParams.get("client_version") ?? incomingHeaders.get("version");
|
|
1384
|
+
const catalogClientVersion = requestedVersion && /^[0-9A-Za-z.+_-]{1,80}$/.test(requestedVersion) ? requestedVersion : undefined;
|
|
1385
|
+
const backoff = state.catalogBackoff ??= new Map();
|
|
1386
|
+
const forceCatalogRefresh = isModelsRequest && incomingUrl.searchParams.get("refresh_capabilities") === "1";
|
|
1387
|
+
// An explicit capability check is a deliberate retry: it must not stay
|
|
1388
|
+
// parked behind an earlier throttle's backoff.
|
|
1389
|
+
if (forceCatalogRefresh)
|
|
1390
|
+
backoff.clear();
|
|
1391
|
+
const catalogsByVersion = state.modelCatalogs ??= new Map();
|
|
1392
|
+
const versionKey = catalogClientVersion ?? "";
|
|
1393
|
+
state.modelCatalog = catalogsByVersion.get(versionKey);
|
|
1394
|
+
if (forceCatalogRefresh)
|
|
1395
|
+
state.modelCatalog?.refresh();
|
|
1396
|
+
state.modelCatalog ??= new AccountModelCatalog(async (key) => {
|
|
1397
|
+
const retryAt = backoff.get(key) ?? 0;
|
|
1398
|
+
if (retryAt > state.now())
|
|
1399
|
+
throw new CatalogRetryError(retryAt - state.now());
|
|
1400
|
+
backoff.delete(key);
|
|
1401
|
+
const scope = state.activeAccountManager.getAccountsSnapshot().flatMap(workspaceModelScopes).find(s => s.id === key && s.enabled);
|
|
1402
|
+
const account = scope ? state.activeAccountManager.getAccountByIndex(scope.accountIndex) : null;
|
|
1403
|
+
if (!account || account.enabled === false || account.authInvalidatedAt ||
|
|
1404
|
+
(account.cooldownReason === "auth-failure" && (account.coolingDownUntil ?? 0) > state.now()))
|
|
1405
|
+
throw new Error("Account unavailable");
|
|
1406
|
+
const fresh = await ensureFreshAccessToken({ accountManager: state.activeAccountManager, account,
|
|
1407
|
+
family: "codex", model: null, now: state.now(), tokenRefreshSkewMs: state.tokenRefreshSkewMs,
|
|
1408
|
+
tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs });
|
|
1409
|
+
if (!fresh.ok)
|
|
1410
|
+
throw new Error("Account authentication unavailable");
|
|
1411
|
+
const accountId = scope?.accountId;
|
|
1412
|
+
if (!accountId)
|
|
1413
|
+
throw new Error("Account identity unavailable");
|
|
1414
|
+
const url = new URL(state.upstreamBaseUrl);
|
|
1415
|
+
url.pathname = url.pathname.replace(/\/+$/, "") + "/codex/models";
|
|
1416
|
+
const clientVersion = catalogClientVersion;
|
|
1417
|
+
if (clientVersion && /^[0-9A-Za-z.+_-]{1,80}$/.test(clientVersion))
|
|
1418
|
+
url.searchParams.set("client_version", clientVersion);
|
|
1419
|
+
const response = await state.fetchImpl(url.toString(), { method: "GET", redirect: "error",
|
|
1420
|
+
headers: createOutboundHeaders(new Headers(), fresh.account, fresh.accessToken, accountId),
|
|
1421
|
+
signal: AbortSignal.timeout(Math.min(state.fetchTimeoutMs, 15_000)) });
|
|
1422
|
+
if (!response.ok) {
|
|
1423
|
+
await response.body?.cancel();
|
|
1424
|
+
if (response.status === 429) {
|
|
1425
|
+
const retryMs = clampCatalogRetryMs(parseRetryAfterHeaderMs(response.headers, state.now()));
|
|
1426
|
+
if (backoff.size >= 100)
|
|
1427
|
+
backoff.delete(backoff.keys().next().value ?? "");
|
|
1428
|
+
backoff.set(key, state.now() + retryMs);
|
|
1429
|
+
throw new CatalogRetryError(retryMs);
|
|
1430
|
+
}
|
|
1431
|
+
throw new Error("Catalog unavailable");
|
|
1432
|
+
}
|
|
1433
|
+
// Bound remote bytes, not just the advertised content length.
|
|
1434
|
+
const reader = response.body?.getReader();
|
|
1435
|
+
if (!reader)
|
|
1436
|
+
throw new Error("Empty catalog");
|
|
1437
|
+
const chunks = [];
|
|
1438
|
+
let size = 0;
|
|
1439
|
+
try {
|
|
1440
|
+
for (;;) {
|
|
1441
|
+
const part = await reader.read();
|
|
1442
|
+
if (part.done)
|
|
1443
|
+
break;
|
|
1444
|
+
size += part.value.byteLength;
|
|
1445
|
+
if (size > 8 * 1024 * 1024)
|
|
1446
|
+
throw new Error("Catalog too large");
|
|
1447
|
+
chunks.push(part.value);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
finally {
|
|
1451
|
+
await reader.cancel();
|
|
1452
|
+
}
|
|
1453
|
+
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
1454
|
+
}, state.now);
|
|
1455
|
+
const modelCatalog = state.modelCatalog;
|
|
1456
|
+
if (catalogsByVersion.size >= 4 && !catalogsByVersion.has(versionKey))
|
|
1457
|
+
catalogsByVersion.delete(catalogsByVersion.keys().next().value ?? "");
|
|
1458
|
+
catalogsByVersion.set(versionKey, modelCatalog);
|
|
1459
|
+
const eligible = accountManager.getAccountsSnapshot().filter(a => a.enabled !== false &&
|
|
1460
|
+
(!isPinned || a.index === pinnedIndex) && !policyDecision?.blockedAccountIndexes.has(a.index));
|
|
1461
|
+
if (isModelsRequest) {
|
|
1462
|
+
const reference = state.catalogAccount;
|
|
1463
|
+
const discovery = accountManager
|
|
1464
|
+
.getAccountsSnapshot()
|
|
1465
|
+
.filter((a) => a.enabled !== false)
|
|
1466
|
+
.sort((a, b) => Number(Boolean(reference &&
|
|
1467
|
+
b.email === reference.email &&
|
|
1468
|
+
b.accountId === reference.accountId)) -
|
|
1469
|
+
Number(Boolean(reference &&
|
|
1470
|
+
a.email === reference.email &&
|
|
1471
|
+
a.accountId === reference.accountId)));
|
|
1472
|
+
const scopes = discovery.flatMap(workspaceModelScopes);
|
|
1473
|
+
const enabledKeys = scopes.filter(scope => scope.enabled).map(scope => scope.id);
|
|
1474
|
+
const routableKeys = scopes.filter(scope => scope.routable).map(scope => scope.id);
|
|
1475
|
+
const oauthRefresh = modelCatalog.list(enabledKeys);
|
|
1476
|
+
// Explicit check waits for all workspaces; picker discovery runs API and OAuth in parallel.
|
|
1477
|
+
if (forceCatalogRefresh)
|
|
1478
|
+
await oauthRefresh;
|
|
1479
|
+
const apiRuntime = getApiModelRuntime(state);
|
|
1480
|
+
let apiConfigurationUnavailable = false;
|
|
1481
|
+
const configuredRoutes = await state.readApiRoutes?.().catch(() => {
|
|
1482
|
+
apiConfigurationUnavailable = true;
|
|
1483
|
+
state.status.lastError = "api_configuration_unavailable";
|
|
1484
|
+
return [];
|
|
1485
|
+
}) ?? [];
|
|
1486
|
+
if (!apiConfigurationUnavailable && state.status.lastError === "api_configuration_unavailable")
|
|
1487
|
+
state.status.lastError = null;
|
|
1488
|
+
state.catalogApiRoutes = configuredRoutes;
|
|
1489
|
+
// Paid API probes respect the 15-minute cache unless `check capabilities` forces them.
|
|
1490
|
+
const forceProbes = forceCatalogRefresh && incomingUrl.searchParams.get("force_probes") === "1";
|
|
1491
|
+
const apiRefresh = apiRuntime.catalogs(configuredRoutes, forceCatalogRefresh, forceProbes, forceCatalogRefresh);
|
|
1492
|
+
const pickerSnapshot = () => [
|
|
1493
|
+
...modelCatalog.cachedList(routableKeys).filter(model => !/^(api|zdr)\//.test(model.slug)),
|
|
1494
|
+
...buildVisibleModelUnion(apiRuntime.cachedCatalogs().flatMap(catalog => {
|
|
1495
|
+
const route = configuredRoutes.find(route => route.id === catalog.id && route.enabled);
|
|
1496
|
+
return route ? [{ ...catalog, visibleModels: route.visibleModels, priority: route.priority }] : [];
|
|
1497
|
+
})),
|
|
1498
|
+
];
|
|
1499
|
+
const refresh = (async () => {
|
|
1500
|
+
await oauthRefresh;
|
|
1501
|
+
const apiCatalogs = await apiRefresh;
|
|
1502
|
+
const oauthModels = modelCatalog.cachedList(routableKeys).filter(model => !/^(api|zdr)\//.test(model.slug));
|
|
1503
|
+
const models = [...oauthModels, ...buildVisibleModelUnion(apiCatalogs)];
|
|
1504
|
+
if (state.modelCatalog === modelCatalog) {
|
|
1505
|
+
state.catalogOAuthModels = oauthModels;
|
|
1506
|
+
setCatalogEtag(state, models);
|
|
1507
|
+
}
|
|
1508
|
+
await saveModelInventory(state.catalogInventory = {
|
|
1509
|
+
...(apiConfigurationUnavailable ? { apiConfigurationUnavailable: true } : {}),
|
|
1510
|
+
version: 1,
|
|
1511
|
+
checkedAt: state.now(),
|
|
1512
|
+
clientVersion: catalogClientVersion,
|
|
1513
|
+
entries: [
|
|
1514
|
+
...scopes.map((scope) => {
|
|
1515
|
+
const entry = modelCatalog.snapshot(scope.id) ?? {
|
|
1516
|
+
checkedAt: 0,
|
|
1517
|
+
models: [],
|
|
1518
|
+
error: true,
|
|
1519
|
+
};
|
|
1520
|
+
return {
|
|
1521
|
+
...entry,
|
|
1522
|
+
id: scope.id, label: scope.label, routable: scope.routable, bound: scope.bound, selected: scope.selected,
|
|
1523
|
+
kind: "oauth", enabled: scope.enabled, error: scope.enabled && entry.error,
|
|
1524
|
+
visibleModels: scope.routable ? entry.models : [],
|
|
1525
|
+
};
|
|
1526
|
+
}),
|
|
1527
|
+
...apiRuntime
|
|
1528
|
+
.statuses(configuredRoutes)
|
|
1529
|
+
.map((a) => ({
|
|
1530
|
+
id: modelScopeId(a.kind, a.id), label: a.label,
|
|
1531
|
+
kind: a.kind,
|
|
1532
|
+
enabled: apiCatalogs.some((c) => c.id === a.id && c.enabled),
|
|
1533
|
+
checkedAt: a.checkedAt,
|
|
1534
|
+
error: a.error,
|
|
1535
|
+
models: a.availableModels,
|
|
1536
|
+
visibleModels: a.visibleModels,
|
|
1537
|
+
entitlements: a.entitlements,
|
|
1538
|
+
})),
|
|
1539
|
+
],
|
|
1540
|
+
}).catch(() => {
|
|
1541
|
+
state.status.lastError = "Model discovery status could not be saved";
|
|
1542
|
+
});
|
|
1543
|
+
return models;
|
|
1544
|
+
})();
|
|
1545
|
+
// Observe background failures even after the picker response has finished.
|
|
1546
|
+
void refresh.catch(() => { state.status.lastError = "Background model catalog refresh failed"; });
|
|
1547
|
+
let models;
|
|
1548
|
+
if (forceCatalogRefresh)
|
|
1549
|
+
models = await refresh;
|
|
1550
|
+
else {
|
|
1551
|
+
const cached = pickerSnapshot();
|
|
1552
|
+
if (cached.length)
|
|
1553
|
+
models = cached;
|
|
1554
|
+
else {
|
|
1555
|
+
let timer;
|
|
1556
|
+
try {
|
|
1557
|
+
models = await Promise.race([refresh, new Promise(resolve => {
|
|
1558
|
+
timer = setTimeout(() => resolve(pickerSnapshot()), 2000);
|
|
1559
|
+
})]);
|
|
1560
|
+
}
|
|
1561
|
+
finally {
|
|
1562
|
+
if (timer)
|
|
1563
|
+
clearTimeout(timer);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
res.setHeader("etag", setCatalogEtag(state, models));
|
|
1568
|
+
writeJson(res, models.length ? 200 : 503, models.length
|
|
1569
|
+
? { models }
|
|
1570
|
+
: {
|
|
1571
|
+
error: {
|
|
1572
|
+
code: "account_catalog_unavailable",
|
|
1573
|
+
message: "No eligible account catalog is available.",
|
|
1574
|
+
},
|
|
1575
|
+
});
|
|
1576
|
+
return;
|
|
1577
|
+
}
|
|
1578
|
+
const requestedModel = context.model;
|
|
1579
|
+
if (!requestedModel)
|
|
1580
|
+
throw new Error("Missing requested model");
|
|
1581
|
+
const requestedBody = parseRequestBody(requestBody);
|
|
1582
|
+
const effort = isRecord(requestedBody?.reasoning) && typeof requestedBody.reasoning.effort === "string" ? requestedBody.reasoning.effort : undefined;
|
|
1583
|
+
const requestedTier = typeof requestedBody?.service_tier === "string" ? requestedBody.service_tier : undefined;
|
|
1584
|
+
const failures = state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now);
|
|
1585
|
+
await modelCatalog.prepareRouting(eligible.flatMap(workspaceModelScopes).filter(scope => scope.routable).map(scope => scope.id), requestedModel, effort, requestedTier, key => failures.supports(key, requestedModel, effort, requestedTier));
|
|
1586
|
+
if (res.destroyed || res.writableEnded)
|
|
1587
|
+
return;
|
|
1588
|
+
let supported = 0;
|
|
1589
|
+
catalogEligibleKeys = new Set();
|
|
1590
|
+
for (const account of eligible) {
|
|
1591
|
+
const candidates = [];
|
|
1592
|
+
const scopes = workspaceModelScopes(account).filter(scope => scope.routable).sort((a, b) => Number(b.selected) - Number(a.selected) || Number(b.bound) - Number(a.bound));
|
|
1593
|
+
for (const scope of scopes) {
|
|
1594
|
+
// Fail open: an unknown catalog (outage/throttle) stays routable; only a fetched one excludes.
|
|
1595
|
+
if (failures.supports(scope.id, requestedModel, effort, requestedTier) && modelCatalog.supportsForRouting(scope.id, requestedModel, effort, requestedTier))
|
|
1596
|
+
candidates.push(scope);
|
|
1597
|
+
}
|
|
1598
|
+
if (candidates.length) {
|
|
1599
|
+
candidates.sort((a, b) => compareSubscriptionQuota(subscriptionQuotaPreference(quotaForScope(account, a), state.now()), subscriptionQuotaPreference(quotaForScope(account, b), state.now())));
|
|
1600
|
+
const first = candidates[0];
|
|
1601
|
+
subscriptionQuotaByAccount[account.index] = subscriptionQuotaPreference(first ? quotaForScope(account, first) : null, state.now());
|
|
1602
|
+
workspaceCandidates.set(account.index, candidates);
|
|
1603
|
+
catalogEligibleKeys.add(catalogAccountKey(account));
|
|
1604
|
+
supported++;
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
if (!supported) {
|
|
1608
|
+
writeJson(res, 403, { error: { code: "model_not_available_in_account_catalog", message: "Selected model is not advertised by an eligible account. Refresh account access or account routing settings." } });
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
if (state.nativeOpenai && isResponsesRequest && context.model && resetCreditState?.lastRedemptionAt !== undefined) {
|
|
1613
|
+
for (const [index, scopes] of workspaceCandidates) {
|
|
1614
|
+
const account = accountManager.getAccountByIndex(index);
|
|
1615
|
+
if (!account)
|
|
1616
|
+
continue;
|
|
1617
|
+
for (const scope of scopes)
|
|
1618
|
+
applyConfirmedReset({ account, scope, model: context.model, family: context.family, manager: accountManager, snapshot: resetCreditState.snapshots[scope.id], lastRedemptionAt: resetCreditState.lastRedemptionAt,
|
|
1619
|
+
previous: state.subscriptionQuotaObservations?.get(JSON.stringify([scope.id, context.model])) ?? findQuotaCacheEntryForAccount(subscriptionQuotaCache, account, accountManager.getAccountsSnapshot(), undefined, scope.accountId),
|
|
1620
|
+
observations: state.subscriptionQuotaObservations ??= new Map(), now: state.now(), clearQuotaScheduler: a => state.preemptiveQuotaScheduler.clear(buildQuotaScheduleKey(a, context.family, context.model)) });
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
if (state.nativeOpenai && isResponsesRequest && context.model && !isPinned) {
|
|
1624
|
+
try {
|
|
1625
|
+
await recoverResetQuota({ model: context.model, family: context.family, native: true, pinned: false, manager: accountManager,
|
|
1626
|
+
scopes: workspaceCandidates, quotaForScope, priorityByAccount: policyDecision?.priorityByAccount, preferredIndex: storageMeta.pinnedAccountIndex, service: createResetCreditService(accountManager),
|
|
1627
|
+
observations: state.subscriptionQuotaObservations ??= new Map(), now: state.now,
|
|
1628
|
+
clearQuotaScheduler: account => state.preemptiveQuotaScheduler.clear(buildQuotaScheduleKey(account, context.family, context.model)),
|
|
1629
|
+
});
|
|
1630
|
+
}
|
|
1631
|
+
catch {
|
|
1632
|
+
state.status.lastError = "Subscription reset recovery could not be confirmed; no further automatic redemption attempted.";
|
|
1633
|
+
}
|
|
1634
|
+
// Recovery may have redeemed and written new reset state; the next request re-reads it.
|
|
1635
|
+
state.readCache?.delete("reset-credits");
|
|
1636
|
+
}
|
|
1015
1637
|
// The token bucket spreads load across a selectable pool. A pin has no
|
|
1016
1638
|
// alternative account, so exhausting that local heuristic can only reject a
|
|
1017
1639
|
// request that the pinned account could serve. Keep circuit-breaker admission
|
|
@@ -1141,27 +1763,42 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1141
1763
|
// non-reentrant FIFO queue. In legacy mode the inline `markSwitched`
|
|
1142
1764
|
// calls inside `chooseAccount` are used unchanged and no lock is taken,
|
|
1143
1765
|
// so default behavior and perf are identical to before.
|
|
1144
|
-
const selectAccount = () =>
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1766
|
+
const selectAccount = () => {
|
|
1767
|
+
for (const [index, scopes] of workspaceCandidates) {
|
|
1768
|
+
const account = accountManager.getAccountByIndex(index);
|
|
1769
|
+
if (!account)
|
|
1770
|
+
continue;
|
|
1771
|
+
scopes.sort((a, b) => compareSubscriptionQuota(subscriptionQuotaPreference(quotaForScope(account, a), state.now()), subscriptionQuotaPreference(quotaForScope(account, b), state.now())));
|
|
1772
|
+
const scope = scopes[0];
|
|
1773
|
+
if (scope)
|
|
1774
|
+
subscriptionQuotaByAccount[index] = subscriptionQuotaPreference(quotaForScope(account, scope), state.now());
|
|
1775
|
+
}
|
|
1776
|
+
const result = chooseAccount({
|
|
1777
|
+
accountManager,
|
|
1778
|
+
sessionAffinityStore: state.sessionAffinityStore,
|
|
1779
|
+
sessionKey: context.sessionKey,
|
|
1780
|
+
family: context.family,
|
|
1781
|
+
model: context.model,
|
|
1782
|
+
attemptedIndexes: new Set([...attemptedIndexes, ...catalogExcludedIndexes()]),
|
|
1783
|
+
now: state.now(),
|
|
1784
|
+
policy: policyDecision,
|
|
1785
|
+
pinnedIndex,
|
|
1786
|
+
preferredIndex,
|
|
1787
|
+
subscriptionQuotaByAccount: state.nativeOpenai && isResponsesRequest ? subscriptionQuotaByAccount : undefined,
|
|
1788
|
+
fallbackPinnedIndex: state.nativeOpenai ? storageMeta.pinnedAccountIndex : null,
|
|
1789
|
+
skipReasons: accountSkipReasons,
|
|
1790
|
+
stickyBoostByAccount: rotationStickyBoost,
|
|
1791
|
+
pidOffsetEnabled: state.pidOffsetEnabled,
|
|
1792
|
+
schedulingStrategy: state.schedulingStrategy,
|
|
1793
|
+
// Only on a RETRY pass. The first pass still honors a cooldown
|
|
1794
|
+
// another request left on the pin, so an already-cooling pinned
|
|
1795
|
+
// account 503s immediately as before; from the second pass on,
|
|
1796
|
+
// this request's own retry budget outranks the cooldown it just
|
|
1797
|
+
// created for itself.
|
|
1798
|
+
allowPinnedCooldown: isPinned && transientAttempts > 0,
|
|
1799
|
+
});
|
|
1800
|
+
return result;
|
|
1801
|
+
};
|
|
1165
1802
|
const selected = state.routingMutexMode === "enabled"
|
|
1166
1803
|
? await withRoutingMutex(state.routingMutexMode, async () => {
|
|
1167
1804
|
const candidate = selectAccount();
|
|
@@ -1182,6 +1819,8 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1182
1819
|
return candidate;
|
|
1183
1820
|
})
|
|
1184
1821
|
: selectAccount();
|
|
1822
|
+
for (const index of catalogExcludedIndexes())
|
|
1823
|
+
accountSkipReasons.set(index, "model-not-supported");
|
|
1185
1824
|
if (!selected) {
|
|
1186
1825
|
if (!reloadedAfterNoAccount &&
|
|
1187
1826
|
!isPinned &&
|
|
@@ -1199,8 +1838,15 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1199
1838
|
// state after reloading, so let those reasons through (issue #606).
|
|
1200
1839
|
(reason) => reason === "policy-blocked")) {
|
|
1201
1840
|
reloadedAfterNoAccount = true;
|
|
1841
|
+
const eligibilityIdentity = (manager) => JSON.stringify(manager.getAccountsSnapshot().map(account => [account.recordId, account.enabled, account.authInvalidatedAt, workspaceModelScopes(account)]));
|
|
1842
|
+
const checkedInventory = state.nativeOpenai ? eligibilityIdentity(accountManager) : undefined;
|
|
1202
1843
|
const reloadedManager = await recoverStaleRuntimeState(state);
|
|
1203
1844
|
if (reloadedManager) {
|
|
1845
|
+
if (checkedInventory !== undefined && checkedInventory !== eligibilityIdentity(reloadedManager)) {
|
|
1846
|
+
res.setHeader("retry-after", "1");
|
|
1847
|
+
writeJson(res, 503, { error: { code: "account_catalog_refresh_pending", message: "Account inventory changed; retry with refreshed eligibility." } });
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1204
1850
|
accountManager = reloadedManager;
|
|
1205
1851
|
accountCount = accountManager.getAccountCount();
|
|
1206
1852
|
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
@@ -1217,6 +1863,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1217
1863
|
if (preemptiveDeferral.defer && preemptiveDeferral.waitMs > 0) {
|
|
1218
1864
|
accountSkipReasons.set(selected.index, preemptiveDeferral.reason ?? "quota-near-exhaustion");
|
|
1219
1865
|
exhaustionReason = "rate-limit";
|
|
1866
|
+
capabilityRejectionIsLatest = false;
|
|
1220
1867
|
accountManager.markRateLimitedWithReason(selected, preemptiveDeferral.waitMs, context.family, "quota", context.model);
|
|
1221
1868
|
accountManager.recordRateLimit(selected, context.family, context.model);
|
|
1222
1869
|
accountManager.saveToDiskDebounced();
|
|
@@ -1232,6 +1879,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1232
1879
|
if (!admission.ok) {
|
|
1233
1880
|
accountSkipReasons.set(selected.index, admission.reason);
|
|
1234
1881
|
exhaustionReason = "rate-limit";
|
|
1882
|
+
capabilityRejectionIsLatest = false;
|
|
1235
1883
|
// Neither gate can change inside this loop -- nothing here refills the
|
|
1236
1884
|
// bucket or closes a circuit -- and a pin has no other account to move
|
|
1237
1885
|
// to, so re-selecting would burn the whole 16-iteration ceiling
|
|
@@ -1261,6 +1909,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1261
1909
|
// without reading the map at all, and the budget-boundary block
|
|
1262
1910
|
// below already records "auth-failure" for a pin out of retries.
|
|
1263
1911
|
exhaustionReason = "auth-failure";
|
|
1912
|
+
capabilityRejectionIsLatest = false;
|
|
1264
1913
|
if (refreshed.invalidated) {
|
|
1265
1914
|
// Refresh endpoint explicitly revoked the token. Stop cascade:
|
|
1266
1915
|
// return auth error to client instead of rotating to the next account.
|
|
@@ -1286,7 +1935,21 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1286
1935
|
noteRotation();
|
|
1287
1936
|
continue;
|
|
1288
1937
|
}
|
|
1289
|
-
const
|
|
1938
|
+
const pendingScopes = workspaceCandidates.get(refreshed.account.index);
|
|
1939
|
+
// Revalidate enabled scopes after token refresh; never write a per-request choice to the account.
|
|
1940
|
+
const enabledScopes = new Set(workspaceModelScopes(refreshed.account).filter(s => s.routable).map(s => s.id));
|
|
1941
|
+
while (pendingScopes?.length && !enabledScopes.has(pendingScopes[0]?.id ?? ""))
|
|
1942
|
+
pendingScopes.shift();
|
|
1943
|
+
if (pendingScopes && !pendingScopes.length) {
|
|
1944
|
+
refundConsumedPoolToken(refreshed.account);
|
|
1945
|
+
accountSkipReasons.set(refreshed.account.index, "workspace-disabled");
|
|
1946
|
+
capabilityRejectionIsLatest = false;
|
|
1947
|
+
if (isPinned)
|
|
1948
|
+
break;
|
|
1949
|
+
continue;
|
|
1950
|
+
}
|
|
1951
|
+
const requestScope = pendingScopes?.[0];
|
|
1952
|
+
const accountId = requestScope?.accountId ?? resolveAccountId(refreshed.account, refreshed.accessToken);
|
|
1290
1953
|
if (!accountId) {
|
|
1291
1954
|
refundConsumedPoolToken(refreshed.account);
|
|
1292
1955
|
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
@@ -1298,14 +1961,48 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1298
1961
|
// re-selects the still-broken account.
|
|
1299
1962
|
accountManager.saveToDiskDebounced();
|
|
1300
1963
|
exhaustionReason = "auth-failure";
|
|
1964
|
+
capabilityRejectionIsLatest = false;
|
|
1301
1965
|
transientAttempts += 1;
|
|
1302
1966
|
transientExhaustionReason = "auth-failure";
|
|
1303
1967
|
state.status.retries += 1;
|
|
1304
1968
|
noteRotation();
|
|
1305
1969
|
continue;
|
|
1306
1970
|
}
|
|
1971
|
+
if (rejectedWorkspaces.has(accountId)) {
|
|
1972
|
+
// This workspace already rejected the model or setting for this request.
|
|
1973
|
+
refundConsumedPoolToken(refreshed.account);
|
|
1974
|
+
if (pendingScopes && pendingScopes.length > 1) {
|
|
1975
|
+
pendingScopes.shift();
|
|
1976
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
1977
|
+
}
|
|
1978
|
+
else {
|
|
1979
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
1980
|
+
}
|
|
1981
|
+
continue;
|
|
1982
|
+
}
|
|
1983
|
+
if (state.nativeOpenai && state.readNativeAccountStorage) {
|
|
1984
|
+
const latestSnapshot = await state.readNativeAccountStorage();
|
|
1985
|
+
const latest = latestSnapshot.storage;
|
|
1986
|
+
const retained = latestSnapshot.transientFailure && latestSnapshot.routingAvailable === true && (apiKeyClient || await nativeOAuth())
|
|
1987
|
+
? state.activeAccountManager.getAccountsSnapshot().find(item => item.recordId === refreshed.account.recordId)
|
|
1988
|
+
: undefined;
|
|
1989
|
+
const disk = latest?.accounts.find(item => item.recordId && item.recordId === refreshed.account.recordId)
|
|
1990
|
+
?? latest?.accounts.find(item => item.accountId === refreshed.account.accountId && sanitizeEmail(item.email) === sanitizeEmail(refreshed.account.email)) ?? retained;
|
|
1991
|
+
const workspace = disk?.workspaces?.find(item => item.id.trim() === accountId);
|
|
1992
|
+
// Derive the stored binding exactly as workspaceModelScopes does: trimmed,
|
|
1993
|
+
// falling back to the token's workspace claim when no id was stored.
|
|
1994
|
+
// A stored record carries accessToken; the transient-grace fallback is a live account (access).
|
|
1995
|
+
const diskToken = disk?.accessToken ?? disk?.access;
|
|
1996
|
+
const diskBoundId = disk ? disk.accountId?.trim() || extractAccountId(diskToken)?.trim() : undefined;
|
|
1997
|
+
const stillEligible = disk && disk.enabled !== false && !disk.authInvalidatedAt &&
|
|
1998
|
+
(workspace ? workspace.enabled !== false : accountId === diskBoundId);
|
|
1999
|
+
if (!stillEligible) {
|
|
2000
|
+
writeJson(res, 503, { error: { code: "routing_configuration_changed", message: "Routing eligibility changed while preparing the request. Retry with the current configuration." } });
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
1307
2004
|
const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
|
|
1308
|
-
recordLastRuntimeAccount(state.status, accountIdentity);
|
|
2005
|
+
recordLastRuntimeAccount(state.status, accountIdentity, accountId);
|
|
1309
2006
|
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
1310
2007
|
let upstream;
|
|
1311
2008
|
// Abort the in-flight upstream fetch when the client disconnects
|
|
@@ -1340,9 +2037,17 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1340
2037
|
const fetchTimeoutMs = isImageRequest
|
|
1341
2038
|
? Math.max(state.fetchTimeoutMs, 300_000)
|
|
1342
2039
|
: state.fetchTimeoutMs;
|
|
2040
|
+
if (isResponsesRequest || isImageRequest)
|
|
2041
|
+
recordRuntimeInferenceRequest(inferenceAccountKey(refreshed.account), state.now());
|
|
1343
2042
|
upstream = await withTimeout(state.fetchImpl(upstreamUrl, upstreamRequestInit), fetchTimeoutMs, () => fetchAbortController.abort(), `upstream fetch timed out after ${fetchTimeoutMs}ms`);
|
|
1344
2043
|
}
|
|
1345
2044
|
catch (error) {
|
|
2045
|
+
if (clientGone() || (error instanceof ClientCancellationError)) {
|
|
2046
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2047
|
+
if (!res.destroyed)
|
|
2048
|
+
res.destroy();
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
1346
2051
|
// errors-logging-08: a custom fetchImpl, a proxy agent, or an undici
|
|
1347
2052
|
// cause chain can embed the request URL or credential material in the
|
|
1348
2053
|
// raw message, so mask before it reaches any state.status consumer.
|
|
@@ -1387,6 +2092,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1387
2092
|
accountManager.saveToDiskDebounced();
|
|
1388
2093
|
accountSkipReasons.set(refreshed.account.index, "network-error");
|
|
1389
2094
|
exhaustionReason = "network-error";
|
|
2095
|
+
capabilityRejectionIsLatest = false;
|
|
1390
2096
|
transientAttempts += 1;
|
|
1391
2097
|
transientExhaustionReason = "network-error";
|
|
1392
2098
|
state.status.retries += 1;
|
|
@@ -1397,9 +2103,64 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1397
2103
|
res.off("close", onClientClose);
|
|
1398
2104
|
}
|
|
1399
2105
|
reconcileManualSelection();
|
|
2106
|
+
const observeQuota = (snapshot, planType) => {
|
|
2107
|
+
const previous = requestScope ? quotaForScope(refreshed.account, requestScope) : undefined;
|
|
2108
|
+
const merged = {
|
|
2109
|
+
...snapshot,
|
|
2110
|
+
primary: snapshot.primary.usedPercent === undefined ? previous?.primary ?? snapshot.primary : snapshot.primary,
|
|
2111
|
+
secondary: snapshot.secondary.usedPercent === undefined ? previous?.secondary ?? snapshot.secondary : snapshot.secondary,
|
|
2112
|
+
};
|
|
2113
|
+
state.preemptiveQuotaScheduler.update(quotaScheduleKey, merged);
|
|
2114
|
+
if (state.nativeOpenai && requestScope && isResponsesRequest) {
|
|
2115
|
+
const observations = state.subscriptionQuotaObservations ??= new Map();
|
|
2116
|
+
const key = JSON.stringify([requestScope.id, context.model]);
|
|
2117
|
+
if (observations.size >= 1000 && !observations.has(key))
|
|
2118
|
+
observations.delete(observations.keys().next().value ?? "");
|
|
2119
|
+
observations.set(key, { ...merged, model: context.model ?? "unknown", planType: planType ?? previous?.planType });
|
|
2120
|
+
}
|
|
2121
|
+
};
|
|
1400
2122
|
const quotaSnapshot = readQuotaSchedulerSnapshot(upstream.headers, upstream.status, state.now());
|
|
1401
|
-
if (quotaSnapshot)
|
|
1402
|
-
|
|
2123
|
+
if (quotaSnapshot)
|
|
2124
|
+
observeQuota(quotaSnapshot, upstream.headers.get("x-codex-plan-type") ?? undefined);
|
|
2125
|
+
if (isResponsesRequest && context.model && [400, 403, 404].includes(upstream.status)) {
|
|
2126
|
+
const errorBody = await readErrorBody(upstream, state.streamStallTimeoutMs, 65536);
|
|
2127
|
+
let data;
|
|
2128
|
+
try {
|
|
2129
|
+
data = JSON.parse(errorBody);
|
|
2130
|
+
}
|
|
2131
|
+
catch {
|
|
2132
|
+
data = null;
|
|
2133
|
+
}
|
|
2134
|
+
// Learning and rotating on capability rejections is native-only; other
|
|
2135
|
+
// proxies forward the upstream 4xx as they always have.
|
|
2136
|
+
const failure = state.nativeOpenai ? classifyCapabilityFailure(upstream.status, data) : null;
|
|
2137
|
+
upstream = new Response(errorBody, { status: upstream.status, headers: upstream.headers });
|
|
2138
|
+
if (failure) {
|
|
2139
|
+
lastCapabilityRejection = { status: upstream.status, body: errorBody, headers: responseHeadersForClient(upstream.headers), account: refreshed.account };
|
|
2140
|
+
const body = parseRequestBody(context.body);
|
|
2141
|
+
const effort = isRecord(body?.reasoning) && typeof body.reasoning.effort === "string" ? body.reasoning.effort : undefined;
|
|
2142
|
+
const tier = typeof body?.service_tier === "string" ? body.service_tier : undefined;
|
|
2143
|
+
(state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now)).record(requestScope?.id ?? catalogAccountKey(refreshed.account), context.model, failure, effort, tier);
|
|
2144
|
+
rejectedWorkspaces.add(accountId);
|
|
2145
|
+
capabilityRejectionIsLatest = true;
|
|
2146
|
+
if (pendingScopes && pendingScopes.length > 1) {
|
|
2147
|
+
pendingScopes.shift();
|
|
2148
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
2149
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2150
|
+
state.status.retries++;
|
|
2151
|
+
continue;
|
|
2152
|
+
}
|
|
2153
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
2154
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2155
|
+
if (!isPinned) {
|
|
2156
|
+
state.status.retries++;
|
|
2157
|
+
noteRotation();
|
|
2158
|
+
continue;
|
|
2159
|
+
}
|
|
2160
|
+
writeJson(res, upstream.status, { error: { code: "pinned_model_capability_rejected", message: "The pinned account rejected this model or setting; no other account was used." } });
|
|
2161
|
+
await usageRecorder.record({ outcome: "failure", statusCode: upstream.status, errorCode: "pinned_model_capability_rejected", account: refreshed.account });
|
|
2162
|
+
return;
|
|
2163
|
+
}
|
|
1403
2164
|
}
|
|
1404
2165
|
if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
|
|
1405
2166
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
@@ -1436,6 +2197,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1436
2197
|
accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
|
|
1437
2198
|
accountManager.saveToDiskDebounced();
|
|
1438
2199
|
exhaustionReason = "rate-limit";
|
|
2200
|
+
capabilityRejectionIsLatest = false;
|
|
1439
2201
|
transientAttempts += 1;
|
|
1440
2202
|
transientExhaustionReason = "rate-limit";
|
|
1441
2203
|
state.status.retries += 1;
|
|
@@ -1446,6 +2208,19 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1446
2208
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
1447
2209
|
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
1448
2210
|
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
2211
|
+
if (requestScope && refreshed.account.workspaces?.length) {
|
|
2212
|
+
if (accountManager.disableWorkspace(refreshed.account, requestScope.accountId))
|
|
2213
|
+
accountManager.saveToDiskDebounced();
|
|
2214
|
+
pendingScopes?.shift();
|
|
2215
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2216
|
+
if (pendingScopes?.length)
|
|
2217
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
2218
|
+
else
|
|
2219
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
2220
|
+
state.status.retries++;
|
|
2221
|
+
noteRotation();
|
|
2222
|
+
continue;
|
|
2223
|
+
}
|
|
1449
2224
|
const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
|
|
1450
2225
|
false;
|
|
1451
2226
|
refundConsumedPoolToken(refreshed.account);
|
|
@@ -1456,6 +2231,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1456
2231
|
}
|
|
1457
2232
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1458
2233
|
exhaustionReason = "deactivated";
|
|
2234
|
+
capabilityRejectionIsLatest = false;
|
|
1459
2235
|
state.status.retries += 1;
|
|
1460
2236
|
noteRotation();
|
|
1461
2237
|
continue;
|
|
@@ -1560,6 +2336,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1560
2336
|
applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
1561
2337
|
accountManager.saveToDiskDebounced();
|
|
1562
2338
|
exhaustionReason = "auth-failure";
|
|
2339
|
+
capabilityRejectionIsLatest = false;
|
|
1563
2340
|
transientAttempts += 1;
|
|
1564
2341
|
transientExhaustionReason = "auth-failure";
|
|
1565
2342
|
state.status.retries += 1;
|
|
@@ -1600,6 +2377,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1600
2377
|
accountManager.markAccountCoolingDown(refreshed.account, state.serverErrorCooldownMs, "server-error");
|
|
1601
2378
|
accountManager.saveToDiskDebounced();
|
|
1602
2379
|
exhaustionReason = "server-error";
|
|
2380
|
+
capabilityRejectionIsLatest = false;
|
|
1603
2381
|
transientAttempts += 1;
|
|
1604
2382
|
transientExhaustionReason = "server-error";
|
|
1605
2383
|
state.status.retries += 1;
|
|
@@ -1626,14 +2404,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1626
2404
|
});
|
|
1627
2405
|
return;
|
|
1628
2406
|
}
|
|
1629
|
-
accountManager.recordSuccess(refreshed.account, context.family, context.model);
|
|
1630
2407
|
// A successful request proves the account is usable, so clear any
|
|
1631
2408
|
// stale runtime skip reason persisted for it on a prior pool
|
|
1632
2409
|
// exhaustion. Without this the overlay reason (e.g. "token-exhausted",
|
|
1633
2410
|
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
1634
2411
|
// the forecast keeps reporting this working account as unavailable.
|
|
1635
2412
|
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
1636
|
-
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
1637
2413
|
const quotaDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
1638
2414
|
const nearExhaustionWaitMs = quotaDeferral.defer
|
|
1639
2415
|
? quotaDeferral.waitMs
|
|
@@ -1650,16 +2426,28 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1650
2426
|
}
|
|
1651
2427
|
state.lastGlobalSwitchAt = state.now();
|
|
1652
2428
|
}
|
|
1653
|
-
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy);
|
|
2429
|
+
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
1654
2430
|
// Recover the upstream token counts as the body streams past. Without
|
|
1655
2431
|
// them every ledger row lands with all-zero tokens and a zero cost, so
|
|
1656
2432
|
// evaluateBudgetGuard compares `0 >= limit` for maxTokens/maxCostUsd
|
|
1657
2433
|
// and those caps never fire — `budget set --cost 50` would allow
|
|
1658
2434
|
// unlimited spend, with only --requests actually enforced.
|
|
2435
|
+
const responseOutcome = new ResponseOutcome(isResponsesRequest && upstream.headers.get("content-type")?.includes("text/event-stream") === true);
|
|
1659
2436
|
const usageScanner = createUsageStreamScanner({
|
|
1660
2437
|
contentType: upstream.headers.get("content-type"),
|
|
2438
|
+
onEvent: (event) => {
|
|
2439
|
+
responseOutcome.observe(event);
|
|
2440
|
+
const snapshot = readSubscriptionQuotaEvent(event, state.now());
|
|
2441
|
+
if (!snapshot)
|
|
2442
|
+
return;
|
|
2443
|
+
observeQuota(snapshot, snapshot.planType);
|
|
2444
|
+
state.status.streamQuotaUpdates = (state.status.streamQuotaUpdates ?? 0) + 1;
|
|
2445
|
+
state.status.lastStreamQuotaUpdateAt = snapshot.updatedAt;
|
|
2446
|
+
},
|
|
1661
2447
|
});
|
|
2448
|
+
let streamErrored = false;
|
|
1662
2449
|
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
|
|
2450
|
+
streamErrored = true;
|
|
1663
2451
|
// Deliberately NOT the pre-header transport policy above,
|
|
1664
2452
|
// which skips recordFailure (#677). By this point the
|
|
1665
2453
|
// upstream accepted the request and began responding, so a
|
|
@@ -1670,9 +2458,25 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1670
2458
|
accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
|
|
1671
2459
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1672
2460
|
accountManager.saveToDiskDebounced();
|
|
1673
|
-
}, state.streamStallTimeoutMs, usageScanner.push, budgetAdvisory.level === "soft"
|
|
1674
|
-
|
|
1675
|
-
|
|
2461
|
+
}, state.streamStallTimeoutMs, usageScanner.push, { ...(budgetAdvisory.level === "soft" ? buildContextBudgetHeaders(budgetAdvisory) : {}), ...(state.catalogEtag ? { "x-models-etag": state.catalogEtag } : {}) }, () => { usageScanner.result(); return responseOutcome.finish(); });
|
|
2462
|
+
// The client went away mid-stream: no upstream error, and the response was
|
|
2463
|
+
// neither completed nor failed. That says nothing about the account.
|
|
2464
|
+
const clientDisconnected = !forwarded && !streamErrored && !res.writableEnded;
|
|
2465
|
+
if (forwarded && upstream.ok) {
|
|
2466
|
+
accountManager.recordSuccess(refreshed.account, context.family, context.model);
|
|
2467
|
+
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
2468
|
+
}
|
|
2469
|
+
else if (!clientDisconnected) {
|
|
2470
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2471
|
+
const rejection = classifyCapabilityFailure(400, responseOutcome.rejection);
|
|
2472
|
+
if (rejection && context.model) {
|
|
2473
|
+
const body = parseRequestBody(context.body);
|
|
2474
|
+
const effort = isRecord(body?.reasoning) && typeof body.reasoning.effort === "string" ? body.reasoning.effort : undefined;
|
|
2475
|
+
const tier = typeof body?.service_tier === "string" ? body.service_tier : undefined;
|
|
2476
|
+
const scope = workspaceModelScopes(refreshed.account).find(item => item.accountId === accountId);
|
|
2477
|
+
(state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now)).record(scope?.id ?? catalogAccountKey(refreshed.account), context.model, rejection, effort, tier);
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
1676
2480
|
// A stream that broke mid-flight still bills for whatever the upstream
|
|
1677
2481
|
// reported before the break, so record the counts on both outcomes.
|
|
1678
2482
|
const usageTokens = usageScanner.result();
|
|
@@ -1696,9 +2500,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1696
2500
|
});
|
|
1697
2501
|
}
|
|
1698
2502
|
await usageRecorder.record({
|
|
1699
|
-
outcome: forwarded ? "success" : "failure",
|
|
2503
|
+
outcome: forwarded && upstream.ok ? "success" : clientDisconnected ? "cancelled" : "failure",
|
|
1700
2504
|
statusCode: upstream.status,
|
|
1701
|
-
errorCode: forwarded ? null : "stream_forward_failed",
|
|
2505
|
+
errorCode: clientDisconnected ? "client_disconnected" : responseOutcome.finish().errorCode ?? (forwarded ? null : "stream_forward_failed"),
|
|
1702
2506
|
account: refreshed.account,
|
|
1703
2507
|
...(usageTokens ?? {}),
|
|
1704
2508
|
});
|
|
@@ -1815,6 +2619,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1815
2619
|
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
|
|
1816
2620
|
return;
|
|
1817
2621
|
}
|
|
2622
|
+
// Forward the upstream capability rejection only when it is what ended the
|
|
2623
|
+
// request; a later 429/5xx/transport failure reports itself instead.
|
|
2624
|
+
if (lastCapabilityRejection && capabilityRejectionIsLatest && !isThreadGoalRequest) {
|
|
2625
|
+
await forwardCapabilityRejection(lastCapabilityRejection);
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
1818
2628
|
await usageRecorder?.record({
|
|
1819
2629
|
outcome: "failure",
|
|
1820
2630
|
statusCode: normalizeExhaustionStatus(exhaustionReason),
|