codex-multi-auth 2.15.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +22 -1
- package/config/README.md +2 -2
- package/config/codex-legacy.json +109 -184
- package/config/codex-modern.json +44 -114
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -4
- package/dist/lib/account-policy.d.ts +4 -0
- package/dist/lib/account-policy.d.ts.map +1 -1
- package/dist/lib/account-policy.js +8 -0
- package/dist/lib/accounts/rate-limits.d.ts.map +1 -1
- package/dist/lib/accounts/rate-limits.js +2 -0
- package/dist/lib/accounts.d.ts +21 -3
- package/dist/lib/accounts.d.ts.map +1 -1
- package/dist/lib/accounts.js +229 -17
- package/dist/lib/api-route-store.d.ts +24 -0
- package/dist/lib/api-route-store.d.ts.map +1 -0
- package/dist/lib/api-route-store.js +145 -0
- package/dist/lib/auth/account-access.d.ts +115 -0
- package/dist/lib/auth/account-access.d.ts.map +1 -0
- package/dist/lib/auth/account-access.js +214 -0
- package/dist/lib/auth/token-utils.d.ts +54 -1
- package/dist/lib/auth/token-utils.d.ts.map +1 -1
- package/dist/lib/auth/token-utils.js +92 -7
- package/dist/lib/cli.d.ts +1 -1
- package/dist/lib/cli.d.ts.map +1 -1
- package/dist/lib/cli.js +4 -0
- package/dist/lib/codex-cli/state.d.ts +7 -0
- package/dist/lib/codex-cli/state.d.ts.map +1 -1
- package/dist/lib/codex-cli/state.js +18 -0
- package/dist/lib/codex-cli/writer.d.ts.map +1 -1
- package/dist/lib/codex-cli/writer.js +60 -8
- package/dist/lib/codex-manager/account-manager-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/account-manager-commands.js +1 -0
- package/dist/lib/codex-manager/account-pool-write.d.ts +3 -1
- package/dist/lib/codex-manager/account-pool-write.d.ts.map +1 -1
- package/dist/lib/codex-manager/account-pool-write.js +4 -0
- package/dist/lib/codex-manager/api-login-menu.d.ts +14 -0
- package/dist/lib/codex-manager/api-login-menu.d.ts.map +1 -0
- package/dist/lib/codex-manager/api-login-menu.js +224 -0
- package/dist/lib/codex-manager/commands/account.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/account.js +27 -0
- package/dist/lib/codex-manager/commands/best.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/best.js +6 -2
- package/dist/lib/codex-manager/commands/check.d.ts +7 -1
- package/dist/lib/codex-manager/commands/check.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/check.js +24 -3
- package/dist/lib/codex-manager/commands/forecast.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/forecast.js +5 -1
- package/dist/lib/codex-manager/commands/report.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/report.js +5 -1
- package/dist/lib/codex-manager/commands/resets.d.ts +2 -0
- package/dist/lib/codex-manager/commands/resets.d.ts.map +1 -0
- package/dist/lib/codex-manager/commands/resets.js +67 -0
- package/dist/lib/codex-manager/commands/rotation.d.ts +2 -2
- package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/rotation.js +63 -12
- package/dist/lib/codex-manager/commands/status.d.ts +12 -1
- package/dist/lib/codex-manager/commands/status.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/status.js +166 -22
- package/dist/lib/codex-manager/commands/uninstall.js +1 -1
- package/dist/lib/codex-manager/formatters/quota-formatters.d.ts.map +1 -1
- package/dist/lib/codex-manager/formatters/quota-formatters.js +3 -2
- package/dist/lib/codex-manager/health-check.d.ts +3 -0
- package/dist/lib/codex-manager/health-check.d.ts.map +1 -1
- package/dist/lib/codex-manager/health-check.js +56 -6
- package/dist/lib/codex-manager/help.d.ts.map +1 -1
- package/dist/lib/codex-manager/help.js +4 -2
- package/dist/lib/codex-manager/login-flow.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-flow.js +69 -7
- package/dist/lib/codex-manager/login-menu-data.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-menu-data.js +14 -5
- package/dist/lib/codex-manager/login-oauth.d.ts +3 -0
- package/dist/lib/codex-manager/login-oauth.d.ts.map +1 -1
- package/dist/lib/codex-manager/login-oauth.js +6 -1
- package/dist/lib/codex-manager/login-workspace-choice.d.ts +11 -0
- package/dist/lib/codex-manager/login-workspace-choice.d.ts.map +1 -0
- package/dist/lib/codex-manager/login-workspace-choice.js +37 -0
- package/dist/lib/codex-manager/persist-selected-account.d.ts.map +1 -1
- package/dist/lib/codex-manager/persist-selected-account.js +2 -1
- package/dist/lib/codex-manager/repair-commands.d.ts +13 -0
- package/dist/lib/codex-manager/repair-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/repair-commands.js +143 -8
- package/dist/lib/codex-manager.d.ts.map +1 -1
- package/dist/lib/codex-manager.js +25 -6
- package/dist/lib/concurrency.d.ts +3 -0
- package/dist/lib/concurrency.d.ts.map +1 -0
- package/dist/lib/concurrency.js +31 -0
- package/dist/lib/context-budget/model-context-windows.d.ts +1 -1
- package/dist/lib/context-budget/model-context-windows.d.ts.map +1 -1
- package/dist/lib/context-budget/model-context-windows.js +9 -10
- package/dist/lib/model-route-policy.d.ts +52 -0
- package/dist/lib/model-route-policy.d.ts.map +1 -0
- package/dist/lib/model-route-policy.js +159 -0
- package/dist/lib/policy/runtime-policy.d.ts +1 -0
- package/dist/lib/policy/runtime-policy.d.ts.map +1 -1
- package/dist/lib/policy/runtime-policy.js +3 -0
- package/dist/lib/preemptive-quota-scheduler.d.ts +2 -0
- package/dist/lib/preemptive-quota-scheduler.d.ts.map +1 -1
- package/dist/lib/preemptive-quota-scheduler.js +4 -0
- package/dist/lib/prompts/codex.d.ts.map +1 -1
- package/dist/lib/prompts/codex.js +3 -1
- package/dist/lib/quota-probe.d.ts +5 -0
- package/dist/lib/quota-probe.d.ts.map +1 -1
- package/dist/lib/quota-probe.js +26 -2
- package/dist/lib/request/client-cancellation.d.ts +5 -0
- package/dist/lib/request/client-cancellation.d.ts.map +1 -0
- package/dist/lib/request/client-cancellation.js +7 -0
- package/dist/lib/request/error-classification.d.ts.map +1 -1
- package/dist/lib/request/error-classification.js +39 -25
- package/dist/lib/request/fetch-helpers.js +1 -1
- package/dist/lib/request/helpers/model-map.d.ts +12 -2
- package/dist/lib/request/helpers/model-map.d.ts.map +1 -1
- package/dist/lib/request/helpers/model-map.js +174 -147
- package/dist/lib/request/response-outcome.d.ts +25 -0
- package/dist/lib/request/response-outcome.d.ts.map +1 -0
- package/dist/lib/request/response-outcome.js +49 -0
- package/dist/lib/request/stream-failover-runtime.d.ts +2 -1
- package/dist/lib/request/stream-failover-runtime.d.ts.map +1 -1
- package/dist/lib/request/stream-failover-runtime.js +29 -10
- package/dist/lib/runtime/account-check.d.ts.map +1 -1
- package/dist/lib/runtime/account-check.js +2 -8
- package/dist/lib/runtime/account-model-catalog.d.ts +64 -0
- package/dist/lib/runtime/account-model-catalog.d.ts.map +1 -0
- package/dist/lib/runtime/account-model-catalog.js +154 -0
- package/dist/lib/runtime/account-pool.d.ts +3 -1
- package/dist/lib/runtime/account-pool.d.ts.map +1 -1
- package/dist/lib/runtime/account-pool.js +4 -0
- package/dist/lib/runtime/account-reset-credits.d.ts +42 -0
- package/dist/lib/runtime/account-reset-credits.d.ts.map +1 -0
- package/dist/lib/runtime/account-reset-credits.js +82 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts +30 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts.map +1 -0
- package/dist/lib/runtime/api-model-capabilities.js +434 -0
- package/dist/lib/runtime/api-model-runtime.d.ts +42 -0
- package/dist/lib/runtime/api-model-runtime.d.ts.map +1 -0
- package/dist/lib/runtime/api-model-runtime.js +358 -0
- package/dist/lib/runtime/app-bind.d.ts +19 -1
- package/dist/lib/runtime/app-bind.d.ts.map +1 -1
- package/dist/lib/runtime/app-bind.js +79 -9
- package/dist/lib/runtime/automatic-account-checks.d.ts +31 -0
- package/dist/lib/runtime/automatic-account-checks.d.ts.map +1 -0
- package/dist/lib/runtime/automatic-account-checks.js +113 -0
- package/dist/lib/runtime/automatic-subscription-checks.d.ts +7 -0
- package/dist/lib/runtime/automatic-subscription-checks.d.ts.map +1 -0
- package/dist/lib/runtime/automatic-subscription-checks.js +57 -0
- package/dist/lib/runtime/catalog-capabilities.d.ts +8 -0
- package/dist/lib/runtime/catalog-capabilities.d.ts.map +1 -0
- package/dist/lib/runtime/catalog-capabilities.js +61 -0
- package/dist/lib/runtime/inference-activity.d.ts +15 -0
- package/dist/lib/runtime/inference-activity.d.ts.map +1 -0
- package/dist/lib/runtime/inference-activity.js +95 -0
- package/dist/lib/runtime/model-discovery-status.d.ts +100 -0
- package/dist/lib/runtime/model-discovery-status.d.ts.map +1 -0
- package/dist/lib/runtime/model-discovery-status.js +387 -0
- package/dist/lib/runtime/native-account-storage.d.ts +11 -0
- package/dist/lib/runtime/native-account-storage.d.ts.map +1 -0
- package/dist/lib/runtime/native-account-storage.js +50 -0
- package/dist/lib/runtime/native-account-sync.d.ts +19 -0
- package/dist/lib/runtime/native-account-sync.d.ts.map +1 -0
- package/dist/lib/runtime/native-account-sync.js +62 -0
- package/dist/lib/runtime/native-binding-lock.d.ts +3 -0
- package/dist/lib/runtime/native-binding-lock.d.ts.map +1 -0
- package/dist/lib/runtime/native-binding-lock.js +5 -0
- package/dist/lib/runtime/native-client-auth.d.ts +3 -0
- package/dist/lib/runtime/native-client-auth.d.ts.map +1 -0
- package/dist/lib/runtime/native-client-auth.js +42 -0
- package/dist/lib/runtime/native-provider-config.d.ts +4 -0
- package/dist/lib/runtime/native-provider-config.d.ts.map +1 -0
- package/dist/lib/runtime/native-provider-config.js +52 -0
- package/dist/lib/runtime/native-rate-limits.d.ts +18 -0
- package/dist/lib/runtime/native-rate-limits.d.ts.map +1 -0
- package/dist/lib/runtime/native-rate-limits.js +135 -0
- package/dist/lib/runtime/reset-credit-routing.d.ts +41 -0
- package/dist/lib/runtime/reset-credit-routing.d.ts.map +1 -0
- package/dist/lib/runtime/reset-credit-routing.js +66 -0
- package/dist/lib/runtime/reset-credits.d.ts +101 -0
- package/dist/lib/runtime/reset-credits.d.ts.map +1 -0
- package/dist/lib/runtime/reset-credits.js +162 -0
- package/dist/lib/runtime/response-output-history.d.ts +17 -0
- package/dist/lib/runtime/response-output-history.d.ts.map +1 -0
- package/dist/lib/runtime/response-output-history.js +85 -0
- package/dist/lib/runtime/responses-websocket.d.ts +31 -0
- package/dist/lib/runtime/responses-websocket.d.ts.map +1 -0
- package/dist/lib/runtime/responses-websocket.js +636 -0
- package/dist/lib/runtime/resume-picker.d.ts +41 -0
- package/dist/lib/runtime/resume-picker.d.ts.map +1 -0
- package/dist/lib/runtime/resume-picker.js +169 -0
- package/dist/lib/runtime/rotation-account-selection.d.ts +6 -0
- package/dist/lib/runtime/rotation-account-selection.d.ts.map +1 -1
- package/dist/lib/runtime/rotation-account-selection.js +63 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts +33 -0
- package/dist/lib/runtime/rotation-proxy-state.d.ts.map +1 -1
- package/dist/lib/runtime/rotation-server-types.d.ts +18 -0
- package/dist/lib/runtime/rotation-server-types.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-capability-failures.d.ts +14 -0
- package/dist/lib/runtime/runtime-capability-failures.d.ts.map +1 -0
- package/dist/lib/runtime/runtime-capability-failures.js +76 -0
- package/dist/lib/runtime/runtime-observability.d.ts +7 -0
- package/dist/lib/runtime/runtime-observability.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-observability.js +42 -2
- package/dist/lib/runtime/subscription-first-use.d.ts +10 -0
- package/dist/lib/runtime/subscription-first-use.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-first-use.js +64 -0
- package/dist/lib/runtime/subscription-quota-event.d.ts +6 -0
- package/dist/lib/runtime/subscription-quota-event.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-quota-event.js +31 -0
- package/dist/lib/runtime/subscription-quota-order.d.ts +17 -0
- package/dist/lib/runtime/subscription-quota-order.d.ts.map +1 -0
- package/dist/lib/runtime/subscription-quota-order.js +31 -0
- package/dist/lib/runtime/workspace-model-scopes.d.ts +15 -0
- package/dist/lib/runtime/workspace-model-scopes.d.ts.map +1 -0
- package/dist/lib/runtime/workspace-model-scopes.js +34 -0
- package/dist/lib/runtime-constants.d.ts +3 -0
- package/dist/lib/runtime-constants.d.ts.map +1 -1
- package/dist/lib/runtime-constants.js +5 -0
- package/dist/lib/runtime-paths.d.ts.map +1 -1
- package/dist/lib/runtime-paths.js +4 -1
- package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
- package/dist/lib/runtime-rotation-proxy.js +855 -46
- package/dist/lib/schemas.d.ts +20 -0
- package/dist/lib/schemas.d.ts.map +1 -1
- package/dist/lib/schemas.js +3 -0
- package/dist/lib/storage/credential-sidecars.d.ts +13 -0
- package/dist/lib/storage/credential-sidecars.d.ts.map +1 -0
- package/dist/lib/storage/credential-sidecars.js +52 -0
- package/dist/lib/storage/file-lock.d.ts +9 -0
- package/dist/lib/storage/file-lock.d.ts.map +1 -0
- package/dist/lib/storage/file-lock.js +186 -0
- package/dist/lib/storage/flagged-storage.d.ts.map +1 -1
- package/dist/lib/storage/flagged-storage.js +12 -0
- package/dist/lib/storage/path-state.d.ts +2 -0
- package/dist/lib/storage/path-state.d.ts.map +1 -1
- package/dist/lib/storage/path-state.js +9 -0
- package/dist/lib/storage/paths.js +4 -4
- package/dist/lib/storage/pending-auth.d.ts +17 -0
- package/dist/lib/storage/pending-auth.d.ts.map +1 -0
- package/dist/lib/storage/pending-auth.js +153 -0
- package/dist/lib/storage/public-types.d.ts +13 -0
- package/dist/lib/storage/public-types.d.ts.map +1 -1
- package/dist/lib/storage/snapshot-merge.d.ts +16 -0
- package/dist/lib/storage/snapshot-merge.d.ts.map +1 -0
- package/dist/lib/storage/snapshot-merge.js +206 -0
- package/dist/lib/storage/transactions.d.ts.map +1 -1
- package/dist/lib/storage/transactions.js +5 -4
- package/dist/lib/storage.d.ts +6 -0
- package/dist/lib/storage.d.ts.map +1 -1
- package/dist/lib/storage.js +77 -7
- package/dist/lib/ui/auth-menu-builder.d.ts +2 -0
- package/dist/lib/ui/auth-menu-builder.d.ts.map +1 -1
- package/dist/lib/ui/auth-menu-builder.js +1 -0
- package/dist/lib/ui/auth-menu.d.ts.map +1 -1
- package/dist/lib/ui/auth-menu.js +1 -0
- package/dist/lib/ui/check-progress.d.ts +8 -0
- package/dist/lib/ui/check-progress.d.ts.map +1 -0
- package/dist/lib/ui/check-progress.js +30 -0
- package/dist/lib/ui/format.d.ts +3 -1
- package/dist/lib/ui/format.d.ts.map +1 -1
- package/dist/lib/ui/format.js +8 -0
- package/dist/lib/ui/ui-copy.d.ts +2 -2
- package/dist/lib/ui/ui-copy.js +2 -2
- package/dist/lib/usage/pricing.d.ts +19 -2
- package/dist/lib/usage/pricing.d.ts.map +1 -1
- package/dist/lib/usage/pricing.js +101 -40
- package/dist/lib/usage/usage-extraction.d.ts +3 -1
- package/dist/lib/usage/usage-extraction.d.ts.map +1 -1
- package/dist/lib/usage/usage-extraction.js +46 -9
- package/package.json +7 -4
- package/scripts/codex-app-router.js +7 -0
- package/scripts/codex-bin-resolver.js +40 -8
- package/scripts/codex-routing.js +1 -0
- package/scripts/codex.js +209 -126
|
@@ -1,12 +1,38 @@
|
|
|
1
|
+
import { startAutomaticAccountChecks } from "./runtime/automatic-account-checks.js";
|
|
2
|
+
import { createAutomaticSubscriptionCheck } from "./runtime/automatic-subscription-checks.js";
|
|
3
|
+
import { ClientCancellationError } from "./request/client-cancellation.js";
|
|
4
|
+
import { createNativeAccountStorageReader } from "./runtime/native-account-storage.js";
|
|
5
|
+
import { resetSnapshotQuota } from "./runtime/reset-credits.js";
|
|
6
|
+
import { createResetCreditService, loadResetCreditState } from "./runtime/account-reset-credits.js";
|
|
7
|
+
import { recoverResetQuota, applyConfirmedReset } from "./runtime/reset-credit-routing.js";
|
|
8
|
+
import { readSubscriptionQuotaEvent } from "./runtime/subscription-quota-event.js";
|
|
9
|
+
import { loadQuotaCache } from "./quota-cache.js";
|
|
10
|
+
import { findQuotaCacheEntryForAccount } from "./quota-readiness.js";
|
|
11
|
+
import { subscriptionQuotaPreference, compareSubscriptionQuota } from "./runtime/subscription-quota-order.js";
|
|
12
|
+
import { inferenceAccountKey } from "./runtime/inference-activity.js";
|
|
13
|
+
import { modelScopeId, workspaceModelScopes } from "./runtime/workspace-model-scopes.js";
|
|
14
|
+
import { RuntimeCapabilityFailures, classifyCapabilityFailure } from "./runtime/runtime-capability-failures.js";
|
|
15
|
+
import { ResponseOutcome } from "./request/response-outcome.js";
|
|
16
|
+
import { ResponsesWebSocketGateway } from "./runtime/responses-websocket.js";
|
|
17
|
+
import { ApiModelCapabilities } from "./runtime/api-model-capabilities.js";
|
|
18
|
+
import { saveModelInventory } from "./runtime/model-discovery-status.js";
|
|
19
|
+
import { loadApiRoutes, updateApiModelDiscovery } from "./api-route-store.js";
|
|
20
|
+
import { ApiModelRuntime } from "./runtime/api-model-runtime.js";
|
|
21
|
+
import { buildVisibleModelUnion, parseModelRoute, canonicalServiceTier } from "./model-route-policy.js";
|
|
22
|
+
import { isSameNativeAccount, syncNativeAccountCredentials } from "./runtime/native-account-sync.js";
|
|
23
|
+
import { sanitizeEmail } from "./auth/token-utils.js";
|
|
24
|
+
import { isNativeClientToken } from "./runtime/native-client-auth.js";
|
|
25
|
+
import { CatalogRetryError, AccountModelCatalog, clampCatalogRetryMs } from "./runtime/account-model-catalog.js";
|
|
1
26
|
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
|
|
2
27
|
import { createServer } from "node:http";
|
|
28
|
+
import * as zlib from "node:zlib";
|
|
3
29
|
import { AccountManager, AUTH_INVALIDATION_MARKER, extractAccountId, } from "./accounts.js";
|
|
4
30
|
import { withRoutingMutex } from "./routing-mutex.js";
|
|
5
31
|
import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getMinRotationIntervalMs, getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, getPreemptiveQuotaEnabled, getPreemptiveQuotaMaxDeferralMs, getPreemptiveQuotaRemainingPercent5h, getPreemptiveQuotaRemainingPercent7d, getContextBudgetGuardEnabled, getContextBudgetGuardSoftPercent, getContextBudgetGuardHardPercent, getContextBudgetGuardModelWindowOverrides, getRoutingMutexMode, getSchedulingStrategy, loadPluginConfig, } from "./config.js";
|
|
6
32
|
import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
|
|
7
33
|
import { getModelFamily } from "./prompts/codex.js";
|
|
8
34
|
import { CURRENT_CODEX_MODEL } from "./request/helpers/model-map.js";
|
|
9
|
-
import { mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
|
|
35
|
+
import { recordRuntimeInferenceRequest, flushRuntimeInferenceActivity, mutateRuntimeObservabilitySnapshot, recordRuntimeAccountRecovery, recordRuntimePoolExhaustion, } from "./runtime/runtime-observability.js";
|
|
10
36
|
import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
|
|
11
37
|
import { createUsageStreamScanner } from "./usage/usage-extraction.js";
|
|
12
38
|
import { isModelAtCapacityError, isWorkspaceDisabledError, } from "./request/fetch-helpers.js";
|
|
@@ -167,6 +193,30 @@ const CAPACITY_RETRY_BACKOFF_STEPS_MS = [
|
|
|
167
193
|
const DEFAULT_MODEL_CAPACITY_RETRY_MS = 10 * 60_000;
|
|
168
194
|
const MAX_MODEL_CAPACITY_RETRY_MS = 60 * 60_000;
|
|
169
195
|
const MODEL_CAPACITY_RETRY_ENV = "CODEX_MULTI_AUTH_MODEL_CAPACITY_RETRY_MS";
|
|
196
|
+
function getApiModelRuntime(state) {
|
|
197
|
+
if (state.apiModelRuntime)
|
|
198
|
+
return state.apiModelRuntime;
|
|
199
|
+
const capabilities = state.apiModelCapabilities ??= new ApiModelCapabilities();
|
|
200
|
+
const production = state.readApiRoutes === loadApiRoutes;
|
|
201
|
+
state.apiModelRuntime = new ApiModelRuntime(state.fetchImpl, state.now, production ? updateApiModelDiscovery : undefined, production ? (models, refresh, route, force) => capabilities.enrich(models, refresh, route, force) : undefined, () => {
|
|
202
|
+
const runtime = state.apiModelRuntime;
|
|
203
|
+
if (!runtime)
|
|
204
|
+
return;
|
|
205
|
+
const models = [...(state.catalogOAuthModels ?? []), ...buildVisibleModelUnion(runtime.cachedCatalogs())];
|
|
206
|
+
setCatalogEtag(state, models);
|
|
207
|
+
if (state.catalogInventory) {
|
|
208
|
+
state.catalogInventory = { ...state.catalogInventory, checkedAt: state.now(), entries: [
|
|
209
|
+
...state.catalogInventory.entries.filter(entry => entry.kind === "oauth"),
|
|
210
|
+
...runtime.statuses(state.catalogApiRoutes ?? []).map(entry => ({ id: modelScopeId(entry.kind, entry.id), label: entry.label, kind: entry.kind, enabled: runtime.cachedCatalogs().some(c => c.id === entry.id && c.enabled), checkedAt: entry.checkedAt, error: entry.error, models: entry.availableModels, visibleModels: entry.visibleModels, entitlements: entry.entitlements })),
|
|
211
|
+
] };
|
|
212
|
+
void saveModelInventory(state.catalogInventory).catch(() => { state.status.lastError = "Model discovery status could not be saved"; });
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
return state.apiModelRuntime;
|
|
216
|
+
}
|
|
217
|
+
function setCatalogEtag(state, models) {
|
|
218
|
+
return state.catalogEtag = '"' + createHash("sha256").update(JSON.stringify(models)).digest("hex") + '"';
|
|
219
|
+
}
|
|
170
220
|
function capacityRetryBackoffMs(attempt) {
|
|
171
221
|
const index = Math.min(Math.max(0, attempt - 1), CAPACITY_RETRY_BACKOFF_STEPS_MS.length - 1);
|
|
172
222
|
return CAPACITY_RETRY_BACKOFF_STEPS_MS[index] ?? 0;
|
|
@@ -266,6 +316,23 @@ function createOutboundHeaders(incoming, account, accessToken, accountId) {
|
|
|
266
316
|
headers.set(OPENAI_HEADERS.ORIGINATOR, OPENAI_HEADER_VALUES.ORIGINATOR_CODEX);
|
|
267
317
|
return headers;
|
|
268
318
|
}
|
|
319
|
+
/** Files every native Responses request reads change rarely; share one read per second. */
|
|
320
|
+
const PER_REQUEST_READ_TTL_MS = 1000;
|
|
321
|
+
function cachedRead(state, key, load) {
|
|
322
|
+
const cache = state.readCache ??= new Map();
|
|
323
|
+
const now = Date.now();
|
|
324
|
+
const hit = cache.get(key);
|
|
325
|
+
if (hit && hit.at <= now && now - hit.at < PER_REQUEST_READ_TTL_MS)
|
|
326
|
+
return hit.value;
|
|
327
|
+
const value = load();
|
|
328
|
+
cache.set(key, { at: now, value });
|
|
329
|
+
value.catch(() => { if (cache.get(key)?.value === value)
|
|
330
|
+
cache.delete(key); });
|
|
331
|
+
return value;
|
|
332
|
+
}
|
|
333
|
+
function catalogAccountKey(account) {
|
|
334
|
+
return workspaceModelScopes(account).find(scope => scope.bound)?.id ?? "unavailable";
|
|
335
|
+
}
|
|
269
336
|
function isAuthorizedClient(headers, clientApiKey) {
|
|
270
337
|
const authorization = headers.get("authorization") ?? "";
|
|
271
338
|
const bearerMatch = authorization.match(/^Bearer\s+(.+)$/i);
|
|
@@ -299,20 +366,22 @@ function accountIdentityFromAccount(account, updatedAt) {
|
|
|
299
366
|
updatedAt,
|
|
300
367
|
};
|
|
301
368
|
}
|
|
302
|
-
function recordLastRuntimeAccount(status, identity) {
|
|
369
|
+
function recordLastRuntimeAccount(status, identity, requestedWorkspaceId) {
|
|
303
370
|
status.lastAccountIndex = identity.index;
|
|
304
371
|
status.lastAccountLabel = identity.label;
|
|
305
372
|
status.lastAccountId = identity.accountId;
|
|
373
|
+
status.lastRequestedWorkspaceId = requestedWorkspaceId;
|
|
306
374
|
status.lastAccountUpdatedAt = identity.updatedAt;
|
|
307
375
|
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
308
376
|
snapshot.lastAccountIndex = identity.index;
|
|
309
377
|
snapshot.lastAccountLabel = identity.label;
|
|
310
378
|
snapshot.lastAccountEmail = null;
|
|
311
379
|
snapshot.lastAccountId = identity.accountId;
|
|
380
|
+
snapshot.lastRequestedWorkspaceId = requestedWorkspaceId;
|
|
312
381
|
snapshot.lastAccountUpdatedAt = identity.updatedAt;
|
|
313
382
|
});
|
|
314
383
|
}
|
|
315
|
-
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy) {
|
|
384
|
+
async function persistRuntimeActiveAccount(accountManager, account, family, isPinned, schedulingStrategy, preserveDesktopLogin = false) {
|
|
316
385
|
if (isPinned) {
|
|
317
386
|
// When the user has manually pinned an account, the proxy MUST NOT
|
|
318
387
|
// clobber that pin via markSwitched("rotation"), saveToDiskDebounced(),
|
|
@@ -349,7 +418,8 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
|
|
|
349
418
|
await accountManager.markSwitchedLocked(account, "rotation", family);
|
|
350
419
|
}
|
|
351
420
|
accountManager.saveToDiskDebounced();
|
|
352
|
-
|
|
421
|
+
if (!preserveDesktopLogin)
|
|
422
|
+
await accountManager.syncCodexCliActiveSelectionForIndex(account.index);
|
|
353
423
|
}
|
|
354
424
|
catch {
|
|
355
425
|
// Runtime forwarding must not fail after a valid upstream response just
|
|
@@ -379,6 +449,31 @@ async function readRequestBody(req, maxBytes = MAX_REQUEST_BODY_BYTES) {
|
|
|
379
449
|
}
|
|
380
450
|
return Buffer.concat(chunks);
|
|
381
451
|
}
|
|
452
|
+
/** Inspect the decoded entity before any credential or policy decision. */
|
|
453
|
+
async function decodeRequestBody(body, encoding, maxBytes) {
|
|
454
|
+
const coding = encoding?.trim().toLowerCase() ?? "identity";
|
|
455
|
+
if (!coding || coding === "identity")
|
|
456
|
+
return body;
|
|
457
|
+
// Older supported Node releases lack Zstandard. Reject explicitly there;
|
|
458
|
+
// forwarding opaque bytes would bypass the model/credential boundary.
|
|
459
|
+
const zstd = zlib.zstdDecompress;
|
|
460
|
+
const decode = coding === "gzip" ? zlib.gunzip
|
|
461
|
+
: coding === "deflate" ? zlib.inflate
|
|
462
|
+
: coding === "br" ? zlib.brotliDecompress
|
|
463
|
+
: coding === "zstd" ? zstd : undefined;
|
|
464
|
+
if (!decode)
|
|
465
|
+
throw createRuntimeProxyHttpError("Unsupported request content encoding. Zstandard requires a Node runtime with Zstandard support.", 415, "unsupported_content_encoding");
|
|
466
|
+
return new Promise((resolve, reject) => {
|
|
467
|
+
decode(body, { maxOutputLength: maxBytes }, (error, decoded) => {
|
|
468
|
+
if (error) {
|
|
469
|
+
const tooLarge = "code" in error && error.code === "ERR_BUFFER_TOO_LARGE";
|
|
470
|
+
reject(createRuntimeProxyHttpError(tooLarge ? "Decoded request body is too large." : "Invalid compressed request body.", tooLarge ? 413 : 400, tooLarge ? "runtime_rotation_proxy_payload_too_large" : "invalid_request_body"));
|
|
471
|
+
}
|
|
472
|
+
else
|
|
473
|
+
resolve(decoded);
|
|
474
|
+
});
|
|
475
|
+
});
|
|
476
|
+
}
|
|
382
477
|
function parseRequestBody(body) {
|
|
383
478
|
if (body.length === 0)
|
|
384
479
|
return null;
|
|
@@ -496,10 +591,9 @@ function buildResponsesRequestContext(req, body) {
|
|
|
496
591
|
method: "POST",
|
|
497
592
|
upstreamPath: URL_PATHS.CODEX_RESPONSES,
|
|
498
593
|
model,
|
|
499
|
-
//
|
|
500
|
-
//
|
|
501
|
-
// the
|
|
502
|
-
// family is gpt-5.2 and would mis-account a pass-through codex request.
|
|
594
|
+
// A model-less request buckets under the model a codex request runs on.
|
|
595
|
+
// With every codex model retired that is `gpt-5.6-sol`, whose family is
|
|
596
|
+
// gpt-5.2, the same family as DEFAULT_MODEL.
|
|
503
597
|
family: getModelFamily(model ?? CURRENT_CODEX_MODEL),
|
|
504
598
|
stream: parsedBody?.stream === true,
|
|
505
599
|
sessionKey: resolveSessionKey(headers, parsedBody),
|
|
@@ -697,9 +791,9 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
697
791
|
const preemptiveQuotaScheduler = new PreemptiveQuotaScheduler({
|
|
698
792
|
enabled: getPreemptiveQuotaEnabled(pluginConfig),
|
|
699
793
|
remainingPercentThresholdPrimary: options.quotaRemainingPercentThreshold ??
|
|
700
|
-
getPreemptiveQuotaRemainingPercent5h(pluginConfig),
|
|
794
|
+
(options.nativeOpenai ? 0 : getPreemptiveQuotaRemainingPercent5h(pluginConfig)),
|
|
701
795
|
remainingPercentThresholdSecondary: options.quotaRemainingPercentThreshold ??
|
|
702
|
-
getPreemptiveQuotaRemainingPercent7d(pluginConfig),
|
|
796
|
+
(options.nativeOpenai ? 0 : getPreemptiveQuotaRemainingPercent7d(pluginConfig)),
|
|
703
797
|
maxDeferralMs: getPreemptiveQuotaMaxDeferralMs(pluginConfig),
|
|
704
798
|
});
|
|
705
799
|
const contextBudgetGuard = new ContextBudgetGuard({
|
|
@@ -719,6 +813,11 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
719
813
|
// are detected per-request via `maybeInvalidateAffinityFromDisk`.
|
|
720
814
|
const lastObservedAffinityGeneration = readStorageMetaFromDisk().affinityGeneration;
|
|
721
815
|
const state = createRotationProxyState({
|
|
816
|
+
nativeOpenai: options.nativeOpenai === true,
|
|
817
|
+
readNativeAccountStorage: options.readNativeAccountStorage || !options.accountManager ? createNativeAccountStorageReader(options.readNativeAccountStorage) : undefined,
|
|
818
|
+
readApiRoutes: options.readApiRoutes ?? loadApiRoutes,
|
|
819
|
+
readSubscriptionQuota: options.readSubscriptionQuota ?? (options.accountManager ? undefined : loadQuotaCache),
|
|
820
|
+
catalogAccount: options.catalogAccount,
|
|
722
821
|
activeAccountManager,
|
|
723
822
|
routingMutexMode,
|
|
724
823
|
schedulingStrategy,
|
|
@@ -744,8 +843,10 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
744
843
|
lastObservedAffinityGeneration,
|
|
745
844
|
forcedAccountIndex,
|
|
746
845
|
});
|
|
846
|
+
const websocketGateway = new ResponsesWebSocketGateway(fetchImpl, { maxPayloadBytes: state.maxRequestBodyBytes });
|
|
847
|
+
state.fetchImpl = websocketGateway.fetch;
|
|
747
848
|
const server = createServer((req, res) => {
|
|
748
|
-
|
|
849
|
+
websocketGateway.handle(req, () => handleRequest(state, req, res));
|
|
749
850
|
});
|
|
750
851
|
const sockets = new Set();
|
|
751
852
|
server.on("connection", (socket) => {
|
|
@@ -773,12 +874,19 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
773
874
|
server.on("error", onPostStartupServerError);
|
|
774
875
|
const address = server.address();
|
|
775
876
|
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
877
|
+
if (state.nativeOpenai)
|
|
878
|
+
websocketGateway.attach(server, `http://${urlHost}:${resolvedPort}`);
|
|
879
|
+
// Embedded managers own their own lifecycle; ordinary CLI/app routers check opted-in accounts periodically.
|
|
880
|
+
const automaticChecks = options.accountManager ? undefined : startAutomaticAccountChecks(createAutomaticSubscriptionCheck());
|
|
776
881
|
return {
|
|
777
882
|
host: bindHost,
|
|
778
883
|
port: resolvedPort,
|
|
779
884
|
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
780
885
|
close: async () => {
|
|
886
|
+
await automaticChecks?.stop();
|
|
887
|
+
websocketGateway.close();
|
|
781
888
|
await closeServer(server, sockets);
|
|
889
|
+
await flushRuntimeInferenceActivity();
|
|
782
890
|
await state.activeAccountManager.flushPendingSave();
|
|
783
891
|
},
|
|
784
892
|
// Live client connections, which the app helper reads as evidence that a
|
|
@@ -787,6 +895,8 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
787
895
|
getOpenConnectionCount: () => sockets.size,
|
|
788
896
|
getStatus: () => ({
|
|
789
897
|
...state.status,
|
|
898
|
+
websocketConnections: websocketGateway.stats.connections,
|
|
899
|
+
websocketUpstreamRequests: websocketGateway.stats.upstreamRequests,
|
|
790
900
|
// Redact any email/token material that leaked into a raw upstream or
|
|
791
901
|
// refresh error string before exposing it to status/report consumers
|
|
792
902
|
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
@@ -812,10 +922,116 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
812
922
|
// unauthorized). Authorized callers still fall through to the 404 below
|
|
813
923
|
// when they hit an unsupported path/method.
|
|
814
924
|
const incomingHeaders = headersFromIncoming(req);
|
|
815
|
-
|
|
925
|
+
const bearer = incomingHeaders.get("authorization")?.match(/^Bearer\s+(.+)$/i)?.[1]?.trim();
|
|
926
|
+
const apiKeyClient = isAuthorizedClient(incomingHeaders, state.clientApiKey);
|
|
927
|
+
const isLiveManagedToken = (access, expires, enabled, invalidatedAt) => !!bearer && enabled !== false && !invalidatedAt && !!access && (expires ?? 0) > state.now() && safeEqual(bearer, access);
|
|
928
|
+
const managedOAuthFor = (manager) => state.nativeOpenai === true && manager.getAccountsSnapshot().some(account => isLiveManagedToken(account.access, account.expires, account.enabled, account.authInvalidatedAt));
|
|
929
|
+
let managedStorageVerified = true;
|
|
930
|
+
// A missing or unreadable store (not a transient lock) must not become an
|
|
931
|
+
// empty pool: that would discard the live manager's learned state and hide
|
|
932
|
+
// the cause behind catalog/403 errors.
|
|
933
|
+
let nativeStorageMissing = false;
|
|
934
|
+
let nativeOAuthResult;
|
|
935
|
+
const nativeOAuth = async () => (nativeOAuthResult ??= !!(state.nativeOpenai && bearer && await isNativeClientToken(bearer, state.now())));
|
|
936
|
+
if (state.nativeOpenai && state.readNativeAccountStorage) {
|
|
937
|
+
// Screen the caller before account storage can reshape the manager: a
|
|
938
|
+
// request with no credential never reads disk, and a bearer that matches
|
|
939
|
+
// nothing we hold (in memory or on disk) is refused before any credential
|
|
940
|
+
// sync or manager swap.
|
|
941
|
+
if (!apiKeyClient && !bearer) {
|
|
942
|
+
writeUnauthorized(res);
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
const snapshot = await state.readNativeAccountStorage();
|
|
946
|
+
const disk = snapshot.storage ?? { version: 3, accounts: [], activeIndex: 0, activeIndexByFamily: {} };
|
|
947
|
+
managedStorageVerified = snapshot.verified && snapshot.storage !== null;
|
|
948
|
+
// A concurrent request may have replaced the manager while this read waited.
|
|
949
|
+
accountManager = state.activeAccountManager;
|
|
950
|
+
const presentsKnownCredential = apiKeyClient || (snapshot.storage !== null && managedOAuthFor(accountManager)) ||
|
|
951
|
+
!!disk?.accounts.some(a => isLiveManagedToken(a.accessToken, a.expiresAt, a.enabled, a.authInvalidatedAt)) ||
|
|
952
|
+
await nativeOAuth();
|
|
953
|
+
if (!presentsKnownCredential) {
|
|
954
|
+
writeUnauthorized(res);
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
if (snapshot.transientFailure && snapshot.routingAvailable === false && (apiKeyClient || await nativeOAuth())) {
|
|
958
|
+
writeJson(res, 503, { error: { code: "native_account_storage_unavailable", message: "Account storage is temporarily unavailable. Retry when it is readable." } });
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
nativeStorageMissing = snapshot.storage === null && !snapshot.transientFailure;
|
|
962
|
+
if (snapshot.verified && snapshot.storage !== null) {
|
|
963
|
+
const accounts = accountManager.getAccountsSnapshot();
|
|
964
|
+
const sameInventory = accounts.length === disk.accounts.length && accounts.every((a, i) => isSameNativeAccount(a, disk.accounts[i]));
|
|
965
|
+
if (!sameInventory) {
|
|
966
|
+
accountManager = new AccountManager(undefined, disk);
|
|
967
|
+
accountManager.setRoutingMutexMode(state.routingMutexMode);
|
|
968
|
+
// Preserve independently learned limits by identity, never by the old index.
|
|
969
|
+
// A record id survives an email change; the account identity survives a
|
|
970
|
+
// re-login that re-derives an unstored record id. Rows with neither an
|
|
971
|
+
// accountId nor an email share no identity and match only by refresh token.
|
|
972
|
+
const rebuilt = accountManager.getAccountsSnapshot();
|
|
973
|
+
const claimed = new Set();
|
|
974
|
+
const identityOf = (a) => {
|
|
975
|
+
const id = a.accountId?.trim(), email = sanitizeEmail(a.email);
|
|
976
|
+
return id || email ? JSON.stringify([id ?? null, email ?? null]) : a.refreshToken?.trim() ? `refresh:${a.refreshToken.trim()}` : null;
|
|
977
|
+
};
|
|
978
|
+
const carryTarget = (previous) => {
|
|
979
|
+
const unclaimed = rebuilt.filter(a => !claimed.has(a.index));
|
|
980
|
+
const identity = identityOf(previous);
|
|
981
|
+
const found = unclaimed.find(a => a.recordId && a.recordId === previous.recordId)
|
|
982
|
+
?? (identity === null ? undefined : unclaimed.find(a => identityOf(a) === identity));
|
|
983
|
+
if (found)
|
|
984
|
+
claimed.add(found.index);
|
|
985
|
+
return found;
|
|
986
|
+
};
|
|
987
|
+
for (const previous of accounts) {
|
|
988
|
+
const current = carryTarget(previous);
|
|
989
|
+
const live = current && accountManager.getAccountByIndex(current.index);
|
|
990
|
+
if (!live)
|
|
991
|
+
continue;
|
|
992
|
+
for (const [key, until] of Object.entries(previous.rateLimitResetTimes)) {
|
|
993
|
+
if (typeof until === "number" && until > state.now())
|
|
994
|
+
live.rateLimitResetTimes[key] = Math.max(live.rateLimitResetTimes[key] ?? 0, until);
|
|
995
|
+
}
|
|
996
|
+
if (previous.access === live.access && (previous.coolingDownUntil ?? 0) > (live.coolingDownUntil ?? 0)) {
|
|
997
|
+
live.coolingDownUntil = previous.coolingDownUntil;
|
|
998
|
+
live.cooldownReason = previous.cooldownReason;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
// Reorders change index-based pins/affinity, so they deliberately reload too.
|
|
1002
|
+
const generation = disk.affinityGeneration ?? 0;
|
|
1003
|
+
if (generation <= state.lastObservedAffinityGeneration)
|
|
1004
|
+
state.sessionAffinityStore?.clearAll();
|
|
1005
|
+
state.activeAccountManager = accountManager;
|
|
1006
|
+
state.knownAccountManagers.add(accountManager);
|
|
1007
|
+
state.modelCatalog = undefined;
|
|
1008
|
+
state.modelCatalogs?.clear();
|
|
1009
|
+
}
|
|
1010
|
+
else if (syncNativeAccountCredentials(accountManager, disk)) {
|
|
1011
|
+
state.modelCatalog = undefined;
|
|
1012
|
+
state.modelCatalogs?.clear();
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
// Decide against the synced manager, so a credential revoked or disabled on
|
|
1017
|
+
// disk is refused even though it passed the screen above.
|
|
1018
|
+
if (!apiKeyClient && !(managedStorageVerified && managedOAuthFor(accountManager)) && !(await nativeOAuth())) {
|
|
816
1019
|
writeUnauthorized(res);
|
|
817
1020
|
return;
|
|
818
1021
|
}
|
|
1022
|
+
// The native client treats 426 as a signal to retry over HTTP/SSE.
|
|
1023
|
+
// Authenticate first; never forward an unsupported WebSocket handshake.
|
|
1024
|
+
if (req.method === "GET" && isResponsesPath(incomingUrl.pathname) &&
|
|
1025
|
+
incomingHeaders.get("upgrade")?.toLowerCase() === "websocket") {
|
|
1026
|
+
res.setHeader("connection", "close");
|
|
1027
|
+
writeJson(res, 426, {
|
|
1028
|
+
error: {
|
|
1029
|
+
message: "Use HTTP streaming for Responses API requests.",
|
|
1030
|
+
code: "runtime_rotation_proxy_http_required",
|
|
1031
|
+
},
|
|
1032
|
+
});
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
819
1035
|
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
820
1036
|
const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
|
|
821
1037
|
const isImageRequest = req.method === "POST" && ALLOWED_IMAGE_PATHS.has(incomingUrl.pathname);
|
|
@@ -827,10 +1043,31 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
827
1043
|
return;
|
|
828
1044
|
}
|
|
829
1045
|
state.status.totalRequests += 1;
|
|
830
|
-
|
|
1046
|
+
let requestBody = isResponsesRequest || isImageRequest ||
|
|
831
1047
|
(isThreadGoalRequest && req.method === "POST")
|
|
832
1048
|
? await readRequestBody(req, state.maxRequestBodyBytes)
|
|
833
1049
|
: Buffer.alloc(0);
|
|
1050
|
+
if (isResponsesRequest) {
|
|
1051
|
+
requestBody = await decodeRequestBody(requestBody, incomingHeaders.get("content-encoding"), state.maxRequestBodyBytes);
|
|
1052
|
+
const parsed = parseRequestBody(requestBody);
|
|
1053
|
+
if (!parsed || typeof parsed.model !== "string" || !parsed.model.trim()) {
|
|
1054
|
+
throw createRuntimeProxyHttpError("Responses requests require a JSON object with a model.", 400, "invalid_request_body");
|
|
1055
|
+
}
|
|
1056
|
+
let route;
|
|
1057
|
+
try {
|
|
1058
|
+
route = parseModelRoute(parsed.model);
|
|
1059
|
+
}
|
|
1060
|
+
catch {
|
|
1061
|
+
throw createRuntimeProxyHttpError("Invalid model route.", 400, "invalid_model_route");
|
|
1062
|
+
}
|
|
1063
|
+
if (route.kind === "oauth" && route.serviceTier) {
|
|
1064
|
+
if (parsed.service_tier && canonicalServiceTier(String(parsed.service_tier)) !== canonicalServiceTier(route.serviceTier))
|
|
1065
|
+
throw createRuntimeProxyHttpError("Conflicting speed selection.", 400, "conflicting_service_tier");
|
|
1066
|
+
parsed.model = route.upstreamModel;
|
|
1067
|
+
parsed.service_tier = route.serviceTier;
|
|
1068
|
+
requestBody = Buffer.from(JSON.stringify(parsed));
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
834
1071
|
const context = isModelsRequest
|
|
835
1072
|
? buildModelsRequestContext(req)
|
|
836
1073
|
: isThreadGoalRequest
|
|
@@ -838,6 +1075,23 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
838
1075
|
: isImageRequest
|
|
839
1076
|
? buildImageRequestContext(req, requestBody, incomingUrl.pathname)
|
|
840
1077
|
: buildResponsesRequestContext(req, requestBody);
|
|
1078
|
+
if (isResponsesRequest) {
|
|
1079
|
+
context.headers.delete("content-encoding");
|
|
1080
|
+
context.headers.delete("content-length");
|
|
1081
|
+
}
|
|
1082
|
+
// Explicit API/ZDR routes never touch the OAuth pool, so they stay available.
|
|
1083
|
+
// Everything else would route through a manager the store no longer backs.
|
|
1084
|
+
// A 401 would tell the native client its login is bad when the store is what
|
|
1085
|
+
// is missing, so independently authenticated clients get a 503 (#702).
|
|
1086
|
+
if (nativeStorageMissing && !(isResponsesRequest && context.model && /^(api|zdr)\//.test(context.model))) {
|
|
1087
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
1088
|
+
error: {
|
|
1089
|
+
message: "Account storage is unavailable. Run codex-multi-auth login or check the account store.",
|
|
1090
|
+
code: "native_account_storage_unavailable",
|
|
1091
|
+
},
|
|
1092
|
+
});
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
841
1095
|
const requestStartedAt = state.now();
|
|
842
1096
|
let policyDecision = null;
|
|
843
1097
|
let projectKey = null;
|
|
@@ -939,8 +1193,100 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
939
1193
|
return;
|
|
940
1194
|
}
|
|
941
1195
|
}
|
|
1196
|
+
// Explicit API aliases are handled before all OAuth selection. An absent
|
|
1197
|
+
// credential or catalog is a hard failure, never an ordinary-model fallback.
|
|
1198
|
+
if (context.model && /^(api|zdr)\//.test(context.model)) {
|
|
1199
|
+
if (!isResponsesRequest) {
|
|
1200
|
+
writeJson(res, 400, { error: { code: "api_route_requires_responses" } });
|
|
1201
|
+
return;
|
|
1202
|
+
}
|
|
1203
|
+
const body = parseRequestBody(requestBody);
|
|
1204
|
+
if (!body) {
|
|
1205
|
+
writeJson(res, 400, { error: { code: "invalid_request_body" } });
|
|
1206
|
+
return;
|
|
1207
|
+
}
|
|
1208
|
+
const apiRuntime = getApiModelRuntime(state);
|
|
1209
|
+
const controller = new AbortController();
|
|
1210
|
+
const abort = () => {
|
|
1211
|
+
if (!res.writableEnded)
|
|
1212
|
+
controller.abort();
|
|
1213
|
+
};
|
|
1214
|
+
res.on("close", abort);
|
|
1215
|
+
try {
|
|
1216
|
+
const apiRoutes = (await state.readApiRoutes?.()) ?? [];
|
|
1217
|
+
let attemptedCredentialIndex;
|
|
1218
|
+
const upstream = await apiRuntime.request(context.model, body, apiRoutes, controller.signal, (credentialIndex) => {
|
|
1219
|
+
attemptedCredentialIndex = credentialIndex;
|
|
1220
|
+
const credential = apiRoutes[credentialIndex];
|
|
1221
|
+
if (credential)
|
|
1222
|
+
recordRuntimeInferenceRequest(`sha256:${modelScopeId(credential.kind, credential.id)}`, state.now());
|
|
1223
|
+
const label = `${context.model?.startsWith("zdr/") ? "ZDR" : "API"} credential ${credentialIndex + 1}`;
|
|
1224
|
+
state.status.upstreamRequests++;
|
|
1225
|
+
state.status.lastAccountIndex = null;
|
|
1226
|
+
state.status.lastAccountId = null;
|
|
1227
|
+
state.status.lastRequestedWorkspaceId = null;
|
|
1228
|
+
state.status.lastAccountLabel = label;
|
|
1229
|
+
state.status.lastAccountUpdatedAt = state.now();
|
|
1230
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
1231
|
+
snapshot.lastAccountIndex = null;
|
|
1232
|
+
snapshot.lastAccountId = null;
|
|
1233
|
+
snapshot.lastRequestedWorkspaceId = null;
|
|
1234
|
+
snapshot.lastAccountEmail = null;
|
|
1235
|
+
snapshot.lastAccountLabel = label;
|
|
1236
|
+
snapshot.lastAccountUpdatedAt = state.now();
|
|
1237
|
+
});
|
|
1238
|
+
});
|
|
1239
|
+
const responseOutcome = new ResponseOutcome(upstream.headers.get("content-type")?.includes("text/event-stream") === true);
|
|
1240
|
+
const scanner = createUsageStreamScanner({
|
|
1241
|
+
contentType: upstream.headers.get("content-type"),
|
|
1242
|
+
onEvent: event => responseOutcome.observe(event),
|
|
1243
|
+
});
|
|
1244
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => controller.abort(), state.streamStallTimeoutMs, scanner.push, state.catalogEtag ? { "x-models-etag": state.catalogEtag } : undefined, () => { scanner.result(); return responseOutcome.finish(); });
|
|
1245
|
+
const attemptedCredential = attemptedCredentialIndex === undefined ? undefined : apiRoutes[attemptedCredentialIndex];
|
|
1246
|
+
if (!forwarded && attemptedCredential)
|
|
1247
|
+
apiRuntime.recordStreamFailure(attemptedCredential, context.model, body, responseOutcome.rejection);
|
|
1248
|
+
if (apiRuntime.lastPersistenceError)
|
|
1249
|
+
state.status.lastError = apiRuntime.lastPersistenceError;
|
|
1250
|
+
await usageRecorder.record({
|
|
1251
|
+
outcome: forwarded && upstream.ok ? "success" : "failure",
|
|
1252
|
+
statusCode: upstream.status,
|
|
1253
|
+
errorCode: responseOutcome.finish().errorCode ?? (forwarded ? null : "stream_forward_failed"),
|
|
1254
|
+
...(scanner.result() ?? {}),
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1257
|
+
catch (error) {
|
|
1258
|
+
if (error instanceof ClientCancellationError || res.destroyed) {
|
|
1259
|
+
if (!res.destroyed)
|
|
1260
|
+
res.destroy();
|
|
1261
|
+
return;
|
|
1262
|
+
}
|
|
1263
|
+
throw error;
|
|
1264
|
+
}
|
|
1265
|
+
finally {
|
|
1266
|
+
res.off("close", abort);
|
|
1267
|
+
}
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
942
1270
|
const upstreamUrl = buildUpstreamUrl(req, state.upstreamBaseUrl, context.upstreamPath);
|
|
943
1271
|
const attemptedIndexes = new Set();
|
|
1272
|
+
// The upstream's own capability 400/403/404, forwarded instead of a
|
|
1273
|
+
// synthetic 503 when no other account can take the request.
|
|
1274
|
+
let lastCapabilityRejection;
|
|
1275
|
+
// True while no other failure has followed the last capability rejection.
|
|
1276
|
+
let capabilityRejectionIsLatest = false;
|
|
1277
|
+
// Entitlements belong to the workspace, so each workspace is tried at most
|
|
1278
|
+
// once per request even when several accounts share it.
|
|
1279
|
+
const rejectedWorkspaces = new Set();
|
|
1280
|
+
const forwardCapabilityRejection = async (rejection) => {
|
|
1281
|
+
res.writeHead(rejection.status, rejection.headers);
|
|
1282
|
+
res.end(rejection.body);
|
|
1283
|
+
await usageRecorder?.record({ outcome: "failure", statusCode: rejection.status, errorCode: "upstream_capability_rejected", account: rejection.account });
|
|
1284
|
+
};
|
|
1285
|
+
let catalogEligibleKeys;
|
|
1286
|
+
const catalogExcludedIndexes = () => {
|
|
1287
|
+
const eligible = catalogEligibleKeys;
|
|
1288
|
+
return eligible ? accountManager.getAccountsSnapshot().filter(account => !workspaceCandidates.has(account.index)).map(account => account.index) : [];
|
|
1289
|
+
};
|
|
944
1290
|
let exhaustionReason = "no-account";
|
|
945
1291
|
let accountCount = accountManager.getAccountCount();
|
|
946
1292
|
let transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
@@ -1011,8 +1357,283 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1011
1357
|
// everything downstream — deterministic pick, no cursor advance, no
|
|
1012
1358
|
// stale-state recovery, the `codex_pinned_account_unavailable` failure —
|
|
1013
1359
|
// applies unchanged because it all keys off `pinnedIndex` / `isPinned`.
|
|
1360
|
+
const preferredIndex = state.nativeOpenai
|
|
1361
|
+
? storageMeta.pinnedAccountIndex ?? accountManager.getCurrentAccountForFamily(context.family)?.index ?? null
|
|
1362
|
+
: null;
|
|
1363
|
+
const workspaceCandidates = new Map();
|
|
1364
|
+
const readSubscriptionQuota = state.readSubscriptionQuota;
|
|
1365
|
+
const subscriptionQuotaCache = state.nativeOpenai && isResponsesRequest && readSubscriptionQuota ? await cachedRead(state, "subscription-quota", readSubscriptionQuota).catch(() => null) ?? null : null;
|
|
1366
|
+
const subscriptionQuotaByAccount = {};
|
|
1367
|
+
const resetCreditState = state.nativeOpenai && isResponsesRequest ? await cachedRead(state, "reset-credits", loadResetCreditState).catch(() => null) : null;
|
|
1368
|
+
const quotaForScope = (account, scope) => {
|
|
1369
|
+
// Cached checks describe only the stored account binding, never its sibling workspaces.
|
|
1370
|
+
const cached = scope.bound ? findQuotaCacheEntryForAccount(subscriptionQuotaCache, account, accountManager.getAccountsSnapshot()) : null;
|
|
1371
|
+
const observed = state.subscriptionQuotaObservations?.get(JSON.stringify([scope.id, context.model]));
|
|
1372
|
+
const reset = resetCreditState?.snapshots[scope.id];
|
|
1373
|
+
if (reset && reset.updatedAt > Math.max(cached?.updatedAt ?? 0, observed?.updatedAt ?? 0) && reset.updatedAt <= state.now() && state.now() - reset.updatedAt <= 60000)
|
|
1374
|
+
return resetSnapshotQuota(reset);
|
|
1375
|
+
return observed && observed.updatedAt >= (cached?.updatedAt ?? 0) ? observed : cached;
|
|
1376
|
+
};
|
|
1377
|
+
// A stored `switch` pin is a hard constraint in native mode too (#702), not a
|
|
1378
|
+
// preference: it fails with codex_pinned_account_unavailable rather than
|
|
1379
|
+
// silently serving from another account.
|
|
1014
1380
|
const pinnedIndex = state.forcedAccountIndex ?? storageMeta.pinnedAccountIndex;
|
|
1015
1381
|
const isPinned = typeof pinnedIndex === "number";
|
|
1382
|
+
if (state.nativeOpenai && (isModelsRequest || (isResponsesRequest && context.model))) {
|
|
1383
|
+
const requestedVersion = incomingUrl.searchParams.get("client_version") ?? incomingHeaders.get("version");
|
|
1384
|
+
const catalogClientVersion = requestedVersion && /^[0-9A-Za-z.+_-]{1,80}$/.test(requestedVersion) ? requestedVersion : undefined;
|
|
1385
|
+
const backoff = state.catalogBackoff ??= new Map();
|
|
1386
|
+
const forceCatalogRefresh = isModelsRequest && incomingUrl.searchParams.get("refresh_capabilities") === "1";
|
|
1387
|
+
// An explicit capability check is a deliberate retry: it must not stay
|
|
1388
|
+
// parked behind an earlier throttle's backoff.
|
|
1389
|
+
if (forceCatalogRefresh)
|
|
1390
|
+
backoff.clear();
|
|
1391
|
+
const catalogsByVersion = state.modelCatalogs ??= new Map();
|
|
1392
|
+
const versionKey = catalogClientVersion ?? "";
|
|
1393
|
+
state.modelCatalog = catalogsByVersion.get(versionKey);
|
|
1394
|
+
if (forceCatalogRefresh)
|
|
1395
|
+
state.modelCatalog?.refresh();
|
|
1396
|
+
state.modelCatalog ??= new AccountModelCatalog(async (key) => {
|
|
1397
|
+
const retryAt = backoff.get(key) ?? 0;
|
|
1398
|
+
if (retryAt > state.now())
|
|
1399
|
+
throw new CatalogRetryError(retryAt - state.now());
|
|
1400
|
+
backoff.delete(key);
|
|
1401
|
+
const scope = state.activeAccountManager.getAccountsSnapshot().flatMap(workspaceModelScopes).find(s => s.id === key && s.enabled);
|
|
1402
|
+
const account = scope ? state.activeAccountManager.getAccountByIndex(scope.accountIndex) : null;
|
|
1403
|
+
if (!account || account.enabled === false || account.authInvalidatedAt ||
|
|
1404
|
+
(account.cooldownReason === "auth-failure" && (account.coolingDownUntil ?? 0) > state.now()))
|
|
1405
|
+
throw new Error("Account unavailable");
|
|
1406
|
+
const fresh = await ensureFreshAccessToken({ accountManager: state.activeAccountManager, account,
|
|
1407
|
+
family: "codex", model: null, now: state.now(), tokenRefreshSkewMs: state.tokenRefreshSkewMs,
|
|
1408
|
+
tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs });
|
|
1409
|
+
if (!fresh.ok)
|
|
1410
|
+
throw new Error("Account authentication unavailable");
|
|
1411
|
+
const accountId = scope?.accountId;
|
|
1412
|
+
if (!accountId)
|
|
1413
|
+
throw new Error("Account identity unavailable");
|
|
1414
|
+
const url = new URL(state.upstreamBaseUrl);
|
|
1415
|
+
url.pathname = url.pathname.replace(/\/+$/, "") + "/codex/models";
|
|
1416
|
+
const clientVersion = catalogClientVersion;
|
|
1417
|
+
if (clientVersion && /^[0-9A-Za-z.+_-]{1,80}$/.test(clientVersion))
|
|
1418
|
+
url.searchParams.set("client_version", clientVersion);
|
|
1419
|
+
const response = await state.fetchImpl(url.toString(), { method: "GET", redirect: "error",
|
|
1420
|
+
headers: createOutboundHeaders(new Headers(), fresh.account, fresh.accessToken, accountId),
|
|
1421
|
+
signal: AbortSignal.timeout(Math.min(state.fetchTimeoutMs, 15_000)) });
|
|
1422
|
+
if (!response.ok) {
|
|
1423
|
+
await response.body?.cancel();
|
|
1424
|
+
if (response.status === 429) {
|
|
1425
|
+
const retryMs = clampCatalogRetryMs(parseRetryAfterHeaderMs(response.headers, state.now()));
|
|
1426
|
+
if (backoff.size >= 100)
|
|
1427
|
+
backoff.delete(backoff.keys().next().value ?? "");
|
|
1428
|
+
backoff.set(key, state.now() + retryMs);
|
|
1429
|
+
throw new CatalogRetryError(retryMs);
|
|
1430
|
+
}
|
|
1431
|
+
throw new Error("Catalog unavailable");
|
|
1432
|
+
}
|
|
1433
|
+
// Bound remote bytes, not just the advertised content length.
|
|
1434
|
+
const reader = response.body?.getReader();
|
|
1435
|
+
if (!reader)
|
|
1436
|
+
throw new Error("Empty catalog");
|
|
1437
|
+
const chunks = [];
|
|
1438
|
+
let size = 0;
|
|
1439
|
+
try {
|
|
1440
|
+
for (;;) {
|
|
1441
|
+
const part = await reader.read();
|
|
1442
|
+
if (part.done)
|
|
1443
|
+
break;
|
|
1444
|
+
size += part.value.byteLength;
|
|
1445
|
+
if (size > 8 * 1024 * 1024)
|
|
1446
|
+
throw new Error("Catalog too large");
|
|
1447
|
+
chunks.push(part.value);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
finally {
|
|
1451
|
+
await reader.cancel();
|
|
1452
|
+
}
|
|
1453
|
+
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
1454
|
+
}, state.now);
|
|
1455
|
+
const modelCatalog = state.modelCatalog;
|
|
1456
|
+
if (catalogsByVersion.size >= 4 && !catalogsByVersion.has(versionKey))
|
|
1457
|
+
catalogsByVersion.delete(catalogsByVersion.keys().next().value ?? "");
|
|
1458
|
+
catalogsByVersion.set(versionKey, modelCatalog);
|
|
1459
|
+
const eligible = accountManager.getAccountsSnapshot().filter(a => a.enabled !== false &&
|
|
1460
|
+
(!isPinned || a.index === pinnedIndex) && !policyDecision?.blockedAccountIndexes.has(a.index));
|
|
1461
|
+
if (isModelsRequest) {
|
|
1462
|
+
const reference = state.catalogAccount;
|
|
1463
|
+
const discovery = accountManager
|
|
1464
|
+
.getAccountsSnapshot()
|
|
1465
|
+
.filter((a) => a.enabled !== false)
|
|
1466
|
+
.sort((a, b) => Number(Boolean(reference &&
|
|
1467
|
+
b.email === reference.email &&
|
|
1468
|
+
b.accountId === reference.accountId)) -
|
|
1469
|
+
Number(Boolean(reference &&
|
|
1470
|
+
a.email === reference.email &&
|
|
1471
|
+
a.accountId === reference.accountId)));
|
|
1472
|
+
const scopes = discovery.flatMap(workspaceModelScopes);
|
|
1473
|
+
const enabledKeys = scopes.filter(scope => scope.enabled).map(scope => scope.id);
|
|
1474
|
+
const routableKeys = scopes.filter(scope => scope.routable).map(scope => scope.id);
|
|
1475
|
+
const oauthRefresh = modelCatalog.list(enabledKeys);
|
|
1476
|
+
// Explicit check waits for all workspaces; picker discovery runs API and OAuth in parallel.
|
|
1477
|
+
if (forceCatalogRefresh)
|
|
1478
|
+
await oauthRefresh;
|
|
1479
|
+
const apiRuntime = getApiModelRuntime(state);
|
|
1480
|
+
let apiConfigurationUnavailable = false;
|
|
1481
|
+
const configuredRoutes = await state.readApiRoutes?.().catch(() => {
|
|
1482
|
+
apiConfigurationUnavailable = true;
|
|
1483
|
+
state.status.lastError = "api_configuration_unavailable";
|
|
1484
|
+
return [];
|
|
1485
|
+
}) ?? [];
|
|
1486
|
+
if (!apiConfigurationUnavailable && state.status.lastError === "api_configuration_unavailable")
|
|
1487
|
+
state.status.lastError = null;
|
|
1488
|
+
state.catalogApiRoutes = configuredRoutes;
|
|
1489
|
+
// Paid API probes respect the 15-minute cache unless `check capabilities` forces them.
|
|
1490
|
+
const forceProbes = forceCatalogRefresh && incomingUrl.searchParams.get("force_probes") === "1";
|
|
1491
|
+
const apiRefresh = apiRuntime.catalogs(configuredRoutes, forceCatalogRefresh, forceProbes, forceCatalogRefresh);
|
|
1492
|
+
const pickerSnapshot = () => [
|
|
1493
|
+
...modelCatalog.cachedList(routableKeys).filter(model => !/^(api|zdr)\//.test(model.slug)),
|
|
1494
|
+
...buildVisibleModelUnion(apiRuntime.cachedCatalogs().flatMap(catalog => {
|
|
1495
|
+
const route = configuredRoutes.find(route => route.id === catalog.id && route.enabled);
|
|
1496
|
+
return route ? [{ ...catalog, visibleModels: route.visibleModels, priority: route.priority }] : [];
|
|
1497
|
+
})),
|
|
1498
|
+
];
|
|
1499
|
+
const refresh = (async () => {
|
|
1500
|
+
await oauthRefresh;
|
|
1501
|
+
const apiCatalogs = await apiRefresh;
|
|
1502
|
+
const oauthModels = modelCatalog.cachedList(routableKeys).filter(model => !/^(api|zdr)\//.test(model.slug));
|
|
1503
|
+
const models = [...oauthModels, ...buildVisibleModelUnion(apiCatalogs)];
|
|
1504
|
+
if (state.modelCatalog === modelCatalog) {
|
|
1505
|
+
state.catalogOAuthModels = oauthModels;
|
|
1506
|
+
setCatalogEtag(state, models);
|
|
1507
|
+
}
|
|
1508
|
+
await saveModelInventory(state.catalogInventory = {
|
|
1509
|
+
...(apiConfigurationUnavailable ? { apiConfigurationUnavailable: true } : {}),
|
|
1510
|
+
version: 1,
|
|
1511
|
+
checkedAt: state.now(),
|
|
1512
|
+
clientVersion: catalogClientVersion,
|
|
1513
|
+
entries: [
|
|
1514
|
+
...scopes.map((scope) => {
|
|
1515
|
+
const entry = modelCatalog.snapshot(scope.id) ?? {
|
|
1516
|
+
checkedAt: 0,
|
|
1517
|
+
models: [],
|
|
1518
|
+
error: true,
|
|
1519
|
+
};
|
|
1520
|
+
return {
|
|
1521
|
+
...entry,
|
|
1522
|
+
id: scope.id, label: scope.label, routable: scope.routable, bound: scope.bound, selected: scope.selected,
|
|
1523
|
+
kind: "oauth", enabled: scope.enabled, error: scope.enabled && entry.error,
|
|
1524
|
+
visibleModels: scope.routable ? entry.models : [],
|
|
1525
|
+
};
|
|
1526
|
+
}),
|
|
1527
|
+
...apiRuntime
|
|
1528
|
+
.statuses(configuredRoutes)
|
|
1529
|
+
.map((a) => ({
|
|
1530
|
+
id: modelScopeId(a.kind, a.id), label: a.label,
|
|
1531
|
+
kind: a.kind,
|
|
1532
|
+
enabled: apiCatalogs.some((c) => c.id === a.id && c.enabled),
|
|
1533
|
+
checkedAt: a.checkedAt,
|
|
1534
|
+
error: a.error,
|
|
1535
|
+
models: a.availableModels,
|
|
1536
|
+
visibleModels: a.visibleModels,
|
|
1537
|
+
entitlements: a.entitlements,
|
|
1538
|
+
})),
|
|
1539
|
+
],
|
|
1540
|
+
}).catch(() => {
|
|
1541
|
+
state.status.lastError = "Model discovery status could not be saved";
|
|
1542
|
+
});
|
|
1543
|
+
return models;
|
|
1544
|
+
})();
|
|
1545
|
+
// Observe background failures even after the picker response has finished.
|
|
1546
|
+
void refresh.catch(() => { state.status.lastError = "Background model catalog refresh failed"; });
|
|
1547
|
+
let models;
|
|
1548
|
+
if (forceCatalogRefresh)
|
|
1549
|
+
models = await refresh;
|
|
1550
|
+
else {
|
|
1551
|
+
const cached = pickerSnapshot();
|
|
1552
|
+
if (cached.length)
|
|
1553
|
+
models = cached;
|
|
1554
|
+
else {
|
|
1555
|
+
let timer;
|
|
1556
|
+
try {
|
|
1557
|
+
models = await Promise.race([refresh, new Promise(resolve => {
|
|
1558
|
+
timer = setTimeout(() => resolve(pickerSnapshot()), 2000);
|
|
1559
|
+
})]);
|
|
1560
|
+
}
|
|
1561
|
+
finally {
|
|
1562
|
+
if (timer)
|
|
1563
|
+
clearTimeout(timer);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
res.setHeader("etag", setCatalogEtag(state, models));
|
|
1568
|
+
writeJson(res, models.length ? 200 : 503, models.length
|
|
1569
|
+
? { models }
|
|
1570
|
+
: {
|
|
1571
|
+
error: {
|
|
1572
|
+
code: "account_catalog_unavailable",
|
|
1573
|
+
message: "No eligible account catalog is available.",
|
|
1574
|
+
},
|
|
1575
|
+
});
|
|
1576
|
+
return;
|
|
1577
|
+
}
|
|
1578
|
+
const requestedModel = context.model;
|
|
1579
|
+
if (!requestedModel)
|
|
1580
|
+
throw new Error("Missing requested model");
|
|
1581
|
+
const requestedBody = parseRequestBody(requestBody);
|
|
1582
|
+
const effort = isRecord(requestedBody?.reasoning) && typeof requestedBody.reasoning.effort === "string" ? requestedBody.reasoning.effort : undefined;
|
|
1583
|
+
const requestedTier = typeof requestedBody?.service_tier === "string" ? requestedBody.service_tier : undefined;
|
|
1584
|
+
const failures = state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now);
|
|
1585
|
+
await modelCatalog.prepareRouting(eligible.flatMap(workspaceModelScopes).filter(scope => scope.routable).map(scope => scope.id), requestedModel, effort, requestedTier, key => failures.supports(key, requestedModel, effort, requestedTier));
|
|
1586
|
+
if (res.destroyed || res.writableEnded)
|
|
1587
|
+
return;
|
|
1588
|
+
let supported = 0;
|
|
1589
|
+
catalogEligibleKeys = new Set();
|
|
1590
|
+
for (const account of eligible) {
|
|
1591
|
+
const candidates = [];
|
|
1592
|
+
const scopes = workspaceModelScopes(account).filter(scope => scope.routable).sort((a, b) => Number(b.selected) - Number(a.selected) || Number(b.bound) - Number(a.bound));
|
|
1593
|
+
for (const scope of scopes) {
|
|
1594
|
+
// Fail open: an unknown catalog (outage/throttle) stays routable; only a fetched one excludes.
|
|
1595
|
+
if (failures.supports(scope.id, requestedModel, effort, requestedTier) && modelCatalog.supportsForRouting(scope.id, requestedModel, effort, requestedTier))
|
|
1596
|
+
candidates.push(scope);
|
|
1597
|
+
}
|
|
1598
|
+
if (candidates.length) {
|
|
1599
|
+
candidates.sort((a, b) => compareSubscriptionQuota(subscriptionQuotaPreference(quotaForScope(account, a), state.now()), subscriptionQuotaPreference(quotaForScope(account, b), state.now())));
|
|
1600
|
+
const first = candidates[0];
|
|
1601
|
+
subscriptionQuotaByAccount[account.index] = subscriptionQuotaPreference(first ? quotaForScope(account, first) : null, state.now());
|
|
1602
|
+
workspaceCandidates.set(account.index, candidates);
|
|
1603
|
+
catalogEligibleKeys.add(catalogAccountKey(account));
|
|
1604
|
+
supported++;
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
if (!supported) {
|
|
1608
|
+
writeJson(res, 403, { error: { code: "model_not_available_in_account_catalog", message: "Selected model is not advertised by an eligible account. Refresh account access or account routing settings." } });
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
if (state.nativeOpenai && isResponsesRequest && context.model && resetCreditState?.lastRedemptionAt !== undefined) {
|
|
1613
|
+
for (const [index, scopes] of workspaceCandidates) {
|
|
1614
|
+
const account = accountManager.getAccountByIndex(index);
|
|
1615
|
+
if (!account)
|
|
1616
|
+
continue;
|
|
1617
|
+
for (const scope of scopes)
|
|
1618
|
+
applyConfirmedReset({ account, scope, model: context.model, family: context.family, manager: accountManager, snapshot: resetCreditState.snapshots[scope.id], lastRedemptionAt: resetCreditState.lastRedemptionAt,
|
|
1619
|
+
previous: state.subscriptionQuotaObservations?.get(JSON.stringify([scope.id, context.model])) ?? (scope.bound ? findQuotaCacheEntryForAccount(subscriptionQuotaCache, account, accountManager.getAccountsSnapshot()) : null),
|
|
1620
|
+
observations: state.subscriptionQuotaObservations ??= new Map(), now: state.now(), clearQuotaScheduler: a => state.preemptiveQuotaScheduler.clear(buildQuotaScheduleKey(a, context.family, context.model)) });
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
if (state.nativeOpenai && isResponsesRequest && context.model && !isPinned) {
|
|
1624
|
+
try {
|
|
1625
|
+
await recoverResetQuota({ model: context.model, family: context.family, native: true, pinned: false, manager: accountManager,
|
|
1626
|
+
scopes: workspaceCandidates, quotaForScope, priorityByAccount: policyDecision?.priorityByAccount, preferredIndex: storageMeta.pinnedAccountIndex, service: createResetCreditService(accountManager),
|
|
1627
|
+
observations: state.subscriptionQuotaObservations ??= new Map(), now: state.now,
|
|
1628
|
+
clearQuotaScheduler: account => state.preemptiveQuotaScheduler.clear(buildQuotaScheduleKey(account, context.family, context.model)),
|
|
1629
|
+
});
|
|
1630
|
+
}
|
|
1631
|
+
catch {
|
|
1632
|
+
state.status.lastError = "Subscription reset recovery could not be confirmed; no further automatic redemption attempted.";
|
|
1633
|
+
}
|
|
1634
|
+
// Recovery may have redeemed and written new reset state; the next request re-reads it.
|
|
1635
|
+
state.readCache?.delete("reset-credits");
|
|
1636
|
+
}
|
|
1016
1637
|
// The token bucket spreads load across a selectable pool. A pin has no
|
|
1017
1638
|
// alternative account, so exhausting that local heuristic can only reject a
|
|
1018
1639
|
// request that the pinned account could serve. Keep circuit-breaker admission
|
|
@@ -1142,27 +1763,42 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1142
1763
|
// non-reentrant FIFO queue. In legacy mode the inline `markSwitched`
|
|
1143
1764
|
// calls inside `chooseAccount` are used unchanged and no lock is taken,
|
|
1144
1765
|
// so default behavior and perf are identical to before.
|
|
1145
|
-
const selectAccount = () =>
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1766
|
+
const selectAccount = () => {
|
|
1767
|
+
for (const [index, scopes] of workspaceCandidates) {
|
|
1768
|
+
const account = accountManager.getAccountByIndex(index);
|
|
1769
|
+
if (!account)
|
|
1770
|
+
continue;
|
|
1771
|
+
scopes.sort((a, b) => compareSubscriptionQuota(subscriptionQuotaPreference(quotaForScope(account, a), state.now()), subscriptionQuotaPreference(quotaForScope(account, b), state.now())));
|
|
1772
|
+
const scope = scopes[0];
|
|
1773
|
+
if (scope)
|
|
1774
|
+
subscriptionQuotaByAccount[index] = subscriptionQuotaPreference(quotaForScope(account, scope), state.now());
|
|
1775
|
+
}
|
|
1776
|
+
const result = chooseAccount({
|
|
1777
|
+
accountManager,
|
|
1778
|
+
sessionAffinityStore: state.sessionAffinityStore,
|
|
1779
|
+
sessionKey: context.sessionKey,
|
|
1780
|
+
family: context.family,
|
|
1781
|
+
model: context.model,
|
|
1782
|
+
attemptedIndexes: new Set([...attemptedIndexes, ...catalogExcludedIndexes()]),
|
|
1783
|
+
now: state.now(),
|
|
1784
|
+
policy: policyDecision,
|
|
1785
|
+
pinnedIndex,
|
|
1786
|
+
preferredIndex,
|
|
1787
|
+
subscriptionQuotaByAccount: state.nativeOpenai && isResponsesRequest ? subscriptionQuotaByAccount : undefined,
|
|
1788
|
+
fallbackPinnedIndex: state.nativeOpenai ? storageMeta.pinnedAccountIndex : null,
|
|
1789
|
+
skipReasons: accountSkipReasons,
|
|
1790
|
+
stickyBoostByAccount: rotationStickyBoost,
|
|
1791
|
+
pidOffsetEnabled: state.pidOffsetEnabled,
|
|
1792
|
+
schedulingStrategy: state.schedulingStrategy,
|
|
1793
|
+
// Only on a RETRY pass. The first pass still honors a cooldown
|
|
1794
|
+
// another request left on the pin, so an already-cooling pinned
|
|
1795
|
+
// account 503s immediately as before; from the second pass on,
|
|
1796
|
+
// this request's own retry budget outranks the cooldown it just
|
|
1797
|
+
// created for itself.
|
|
1798
|
+
allowPinnedCooldown: isPinned && transientAttempts > 0,
|
|
1799
|
+
});
|
|
1800
|
+
return result;
|
|
1801
|
+
};
|
|
1166
1802
|
const selected = state.routingMutexMode === "enabled"
|
|
1167
1803
|
? await withRoutingMutex(state.routingMutexMode, async () => {
|
|
1168
1804
|
const candidate = selectAccount();
|
|
@@ -1183,6 +1819,8 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1183
1819
|
return candidate;
|
|
1184
1820
|
})
|
|
1185
1821
|
: selectAccount();
|
|
1822
|
+
for (const index of catalogExcludedIndexes())
|
|
1823
|
+
accountSkipReasons.set(index, "model-not-supported");
|
|
1186
1824
|
if (!selected) {
|
|
1187
1825
|
if (!reloadedAfterNoAccount &&
|
|
1188
1826
|
!isPinned &&
|
|
@@ -1200,8 +1838,15 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1200
1838
|
// state after reloading, so let those reasons through (issue #606).
|
|
1201
1839
|
(reason) => reason === "policy-blocked")) {
|
|
1202
1840
|
reloadedAfterNoAccount = true;
|
|
1841
|
+
const eligibilityIdentity = (manager) => JSON.stringify(manager.getAccountsSnapshot().map(account => [account.recordId, account.enabled, account.authInvalidatedAt, workspaceModelScopes(account)]));
|
|
1842
|
+
const checkedInventory = state.nativeOpenai ? eligibilityIdentity(accountManager) : undefined;
|
|
1203
1843
|
const reloadedManager = await recoverStaleRuntimeState(state);
|
|
1204
1844
|
if (reloadedManager) {
|
|
1845
|
+
if (checkedInventory !== undefined && checkedInventory !== eligibilityIdentity(reloadedManager)) {
|
|
1846
|
+
res.setHeader("retry-after", "1");
|
|
1847
|
+
writeJson(res, 503, { error: { code: "account_catalog_refresh_pending", message: "Account inventory changed; retry with refreshed eligibility." } });
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1205
1850
|
accountManager = reloadedManager;
|
|
1206
1851
|
accountCount = accountManager.getAccountCount();
|
|
1207
1852
|
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
@@ -1218,6 +1863,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1218
1863
|
if (preemptiveDeferral.defer && preemptiveDeferral.waitMs > 0) {
|
|
1219
1864
|
accountSkipReasons.set(selected.index, preemptiveDeferral.reason ?? "quota-near-exhaustion");
|
|
1220
1865
|
exhaustionReason = "rate-limit";
|
|
1866
|
+
capabilityRejectionIsLatest = false;
|
|
1221
1867
|
accountManager.markRateLimitedWithReason(selected, preemptiveDeferral.waitMs, context.family, "quota", context.model);
|
|
1222
1868
|
accountManager.recordRateLimit(selected, context.family, context.model);
|
|
1223
1869
|
accountManager.saveToDiskDebounced();
|
|
@@ -1233,6 +1879,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1233
1879
|
if (!admission.ok) {
|
|
1234
1880
|
accountSkipReasons.set(selected.index, admission.reason);
|
|
1235
1881
|
exhaustionReason = "rate-limit";
|
|
1882
|
+
capabilityRejectionIsLatest = false;
|
|
1236
1883
|
// Neither gate can change inside this loop -- nothing here refills the
|
|
1237
1884
|
// bucket or closes a circuit -- and a pin has no other account to move
|
|
1238
1885
|
// to, so re-selecting would burn the whole 16-iteration ceiling
|
|
@@ -1262,6 +1909,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1262
1909
|
// without reading the map at all, and the budget-boundary block
|
|
1263
1910
|
// below already records "auth-failure" for a pin out of retries.
|
|
1264
1911
|
exhaustionReason = "auth-failure";
|
|
1912
|
+
capabilityRejectionIsLatest = false;
|
|
1265
1913
|
if (refreshed.invalidated) {
|
|
1266
1914
|
// Refresh endpoint explicitly revoked the token. Stop cascade:
|
|
1267
1915
|
// return auth error to client instead of rotating to the next account.
|
|
@@ -1287,7 +1935,21 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1287
1935
|
noteRotation();
|
|
1288
1936
|
continue;
|
|
1289
1937
|
}
|
|
1290
|
-
const
|
|
1938
|
+
const pendingScopes = workspaceCandidates.get(refreshed.account.index);
|
|
1939
|
+
// Revalidate enabled scopes after token refresh; never write a per-request choice to the account.
|
|
1940
|
+
const enabledScopes = new Set(workspaceModelScopes(refreshed.account).filter(s => s.routable).map(s => s.id));
|
|
1941
|
+
while (pendingScopes?.length && !enabledScopes.has(pendingScopes[0]?.id ?? ""))
|
|
1942
|
+
pendingScopes.shift();
|
|
1943
|
+
if (pendingScopes && !pendingScopes.length) {
|
|
1944
|
+
refundConsumedPoolToken(refreshed.account);
|
|
1945
|
+
accountSkipReasons.set(refreshed.account.index, "workspace-disabled");
|
|
1946
|
+
capabilityRejectionIsLatest = false;
|
|
1947
|
+
if (isPinned)
|
|
1948
|
+
break;
|
|
1949
|
+
continue;
|
|
1950
|
+
}
|
|
1951
|
+
const requestScope = pendingScopes?.[0];
|
|
1952
|
+
const accountId = requestScope?.accountId ?? resolveAccountId(refreshed.account, refreshed.accessToken);
|
|
1291
1953
|
if (!accountId) {
|
|
1292
1954
|
refundConsumedPoolToken(refreshed.account);
|
|
1293
1955
|
accountManager.recordFailure(refreshed.account, context.family, context.model);
|
|
@@ -1299,14 +1961,48 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1299
1961
|
// re-selects the still-broken account.
|
|
1300
1962
|
accountManager.saveToDiskDebounced();
|
|
1301
1963
|
exhaustionReason = "auth-failure";
|
|
1964
|
+
capabilityRejectionIsLatest = false;
|
|
1302
1965
|
transientAttempts += 1;
|
|
1303
1966
|
transientExhaustionReason = "auth-failure";
|
|
1304
1967
|
state.status.retries += 1;
|
|
1305
1968
|
noteRotation();
|
|
1306
1969
|
continue;
|
|
1307
1970
|
}
|
|
1971
|
+
if (rejectedWorkspaces.has(accountId)) {
|
|
1972
|
+
// This workspace already rejected the model or setting for this request.
|
|
1973
|
+
refundConsumedPoolToken(refreshed.account);
|
|
1974
|
+
if (pendingScopes && pendingScopes.length > 1) {
|
|
1975
|
+
pendingScopes.shift();
|
|
1976
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
1977
|
+
}
|
|
1978
|
+
else {
|
|
1979
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
1980
|
+
}
|
|
1981
|
+
continue;
|
|
1982
|
+
}
|
|
1983
|
+
if (state.nativeOpenai && state.readNativeAccountStorage) {
|
|
1984
|
+
const latestSnapshot = await state.readNativeAccountStorage();
|
|
1985
|
+
const latest = latestSnapshot.storage;
|
|
1986
|
+
const retained = latestSnapshot.transientFailure && latestSnapshot.routingAvailable === true && (apiKeyClient || await nativeOAuth())
|
|
1987
|
+
? state.activeAccountManager.getAccountsSnapshot().find(item => item.recordId === refreshed.account.recordId)
|
|
1988
|
+
: undefined;
|
|
1989
|
+
const disk = latest?.accounts.find(item => item.recordId && item.recordId === refreshed.account.recordId)
|
|
1990
|
+
?? latest?.accounts.find(item => item.accountId === refreshed.account.accountId && sanitizeEmail(item.email) === sanitizeEmail(refreshed.account.email)) ?? retained;
|
|
1991
|
+
const workspace = disk?.workspaces?.find(item => item.id.trim() === accountId);
|
|
1992
|
+
// Derive the stored binding exactly as workspaceModelScopes does: trimmed,
|
|
1993
|
+
// falling back to the token's workspace claim when no id was stored.
|
|
1994
|
+
// A stored record carries accessToken; the transient-grace fallback is a live account (access).
|
|
1995
|
+
const diskToken = disk?.accessToken ?? disk?.access;
|
|
1996
|
+
const diskBoundId = disk ? disk.accountId?.trim() || extractAccountId(diskToken)?.trim() : undefined;
|
|
1997
|
+
const stillEligible = disk && disk.enabled !== false && !disk.authInvalidatedAt &&
|
|
1998
|
+
(workspace ? workspace.enabled !== false : accountId === diskBoundId);
|
|
1999
|
+
if (!stillEligible) {
|
|
2000
|
+
writeJson(res, 503, { error: { code: "routing_configuration_changed", message: "Routing eligibility changed while preparing the request. Retry with the current configuration." } });
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
1308
2004
|
const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
|
|
1309
|
-
recordLastRuntimeAccount(state.status, accountIdentity);
|
|
2005
|
+
recordLastRuntimeAccount(state.status, accountIdentity, accountId);
|
|
1310
2006
|
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
1311
2007
|
let upstream;
|
|
1312
2008
|
// Abort the in-flight upstream fetch when the client disconnects
|
|
@@ -1341,9 +2037,17 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1341
2037
|
const fetchTimeoutMs = isImageRequest
|
|
1342
2038
|
? Math.max(state.fetchTimeoutMs, 300_000)
|
|
1343
2039
|
: state.fetchTimeoutMs;
|
|
2040
|
+
if (isResponsesRequest || isImageRequest)
|
|
2041
|
+
recordRuntimeInferenceRequest(inferenceAccountKey(refreshed.account), state.now());
|
|
1344
2042
|
upstream = await withTimeout(state.fetchImpl(upstreamUrl, upstreamRequestInit), fetchTimeoutMs, () => fetchAbortController.abort(), `upstream fetch timed out after ${fetchTimeoutMs}ms`);
|
|
1345
2043
|
}
|
|
1346
2044
|
catch (error) {
|
|
2045
|
+
if (clientGone() || (error instanceof ClientCancellationError)) {
|
|
2046
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2047
|
+
if (!res.destroyed)
|
|
2048
|
+
res.destroy();
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
1347
2051
|
// errors-logging-08: a custom fetchImpl, a proxy agent, or an undici
|
|
1348
2052
|
// cause chain can embed the request URL or credential material in the
|
|
1349
2053
|
// raw message, so mask before it reaches any state.status consumer.
|
|
@@ -1388,6 +2092,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1388
2092
|
accountManager.saveToDiskDebounced();
|
|
1389
2093
|
accountSkipReasons.set(refreshed.account.index, "network-error");
|
|
1390
2094
|
exhaustionReason = "network-error";
|
|
2095
|
+
capabilityRejectionIsLatest = false;
|
|
1391
2096
|
transientAttempts += 1;
|
|
1392
2097
|
transientExhaustionReason = "network-error";
|
|
1393
2098
|
state.status.retries += 1;
|
|
@@ -1398,9 +2103,64 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1398
2103
|
res.off("close", onClientClose);
|
|
1399
2104
|
}
|
|
1400
2105
|
reconcileManualSelection();
|
|
2106
|
+
const observeQuota = (snapshot, planType) => {
|
|
2107
|
+
const previous = requestScope ? quotaForScope(refreshed.account, requestScope) : undefined;
|
|
2108
|
+
const merged = {
|
|
2109
|
+
...snapshot,
|
|
2110
|
+
primary: snapshot.primary.usedPercent === undefined ? previous?.primary ?? snapshot.primary : snapshot.primary,
|
|
2111
|
+
secondary: snapshot.secondary.usedPercent === undefined ? previous?.secondary ?? snapshot.secondary : snapshot.secondary,
|
|
2112
|
+
};
|
|
2113
|
+
state.preemptiveQuotaScheduler.update(quotaScheduleKey, merged);
|
|
2114
|
+
if (state.nativeOpenai && requestScope && isResponsesRequest) {
|
|
2115
|
+
const observations = state.subscriptionQuotaObservations ??= new Map();
|
|
2116
|
+
const key = JSON.stringify([requestScope.id, context.model]);
|
|
2117
|
+
if (observations.size >= 1000 && !observations.has(key))
|
|
2118
|
+
observations.delete(observations.keys().next().value ?? "");
|
|
2119
|
+
observations.set(key, { ...merged, model: context.model ?? "unknown", planType: planType ?? previous?.planType });
|
|
2120
|
+
}
|
|
2121
|
+
};
|
|
1401
2122
|
const quotaSnapshot = readQuotaSchedulerSnapshot(upstream.headers, upstream.status, state.now());
|
|
1402
|
-
if (quotaSnapshot)
|
|
1403
|
-
|
|
2123
|
+
if (quotaSnapshot)
|
|
2124
|
+
observeQuota(quotaSnapshot, upstream.headers.get("x-codex-plan-type") ?? undefined);
|
|
2125
|
+
if (isResponsesRequest && context.model && [400, 403, 404].includes(upstream.status)) {
|
|
2126
|
+
const errorBody = await readErrorBody(upstream, state.streamStallTimeoutMs, 65536);
|
|
2127
|
+
let data;
|
|
2128
|
+
try {
|
|
2129
|
+
data = JSON.parse(errorBody);
|
|
2130
|
+
}
|
|
2131
|
+
catch {
|
|
2132
|
+
data = null;
|
|
2133
|
+
}
|
|
2134
|
+
// Learning and rotating on capability rejections is native-only; other
|
|
2135
|
+
// proxies forward the upstream 4xx as they always have.
|
|
2136
|
+
const failure = state.nativeOpenai ? classifyCapabilityFailure(upstream.status, data) : null;
|
|
2137
|
+
upstream = new Response(errorBody, { status: upstream.status, headers: upstream.headers });
|
|
2138
|
+
if (failure) {
|
|
2139
|
+
lastCapabilityRejection = { status: upstream.status, body: errorBody, headers: responseHeadersForClient(upstream.headers), account: refreshed.account };
|
|
2140
|
+
const body = parseRequestBody(context.body);
|
|
2141
|
+
const effort = isRecord(body?.reasoning) && typeof body.reasoning.effort === "string" ? body.reasoning.effort : undefined;
|
|
2142
|
+
const tier = typeof body?.service_tier === "string" ? body.service_tier : undefined;
|
|
2143
|
+
(state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now)).record(requestScope?.id ?? catalogAccountKey(refreshed.account), context.model, failure, effort, tier);
|
|
2144
|
+
rejectedWorkspaces.add(accountId);
|
|
2145
|
+
capabilityRejectionIsLatest = true;
|
|
2146
|
+
if (pendingScopes && pendingScopes.length > 1) {
|
|
2147
|
+
pendingScopes.shift();
|
|
2148
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
2149
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2150
|
+
state.status.retries++;
|
|
2151
|
+
continue;
|
|
2152
|
+
}
|
|
2153
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
2154
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2155
|
+
if (!isPinned) {
|
|
2156
|
+
state.status.retries++;
|
|
2157
|
+
noteRotation();
|
|
2158
|
+
continue;
|
|
2159
|
+
}
|
|
2160
|
+
writeJson(res, upstream.status, { error: { code: "pinned_model_capability_rejected", message: "The pinned account rejected this model or setting; no other account was used." } });
|
|
2161
|
+
await usageRecorder.record({ outcome: "failure", statusCode: upstream.status, errorCode: "pinned_model_capability_rejected", account: refreshed.account });
|
|
2162
|
+
return;
|
|
2163
|
+
}
|
|
1404
2164
|
}
|
|
1405
2165
|
if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
|
|
1406
2166
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
@@ -1437,6 +2197,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1437
2197
|
accountManager.markRateLimitedWithReason(refreshed.account, retryAfterMs, context.family, "quota", context.model);
|
|
1438
2198
|
accountManager.saveToDiskDebounced();
|
|
1439
2199
|
exhaustionReason = "rate-limit";
|
|
2200
|
+
capabilityRejectionIsLatest = false;
|
|
1440
2201
|
transientAttempts += 1;
|
|
1441
2202
|
transientExhaustionReason = "rate-limit";
|
|
1442
2203
|
state.status.retries += 1;
|
|
@@ -1447,6 +2208,19 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1447
2208
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
1448
2209
|
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
1449
2210
|
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
2211
|
+
if (requestScope && refreshed.account.workspaces?.length) {
|
|
2212
|
+
if (accountManager.disableWorkspace(refreshed.account, requestScope.accountId))
|
|
2213
|
+
accountManager.saveToDiskDebounced();
|
|
2214
|
+
pendingScopes?.shift();
|
|
2215
|
+
refundConsumedPoolToken(refreshed.account);
|
|
2216
|
+
if (pendingScopes?.length)
|
|
2217
|
+
attemptedIndexes.delete(refreshed.account.index);
|
|
2218
|
+
else
|
|
2219
|
+
policyDecision?.blockedAccountIndexes.add(refreshed.account.index);
|
|
2220
|
+
state.status.retries++;
|
|
2221
|
+
noteRotation();
|
|
2222
|
+
continue;
|
|
2223
|
+
}
|
|
1450
2224
|
const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
|
|
1451
2225
|
false;
|
|
1452
2226
|
refundConsumedPoolToken(refreshed.account);
|
|
@@ -1457,6 +2231,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1457
2231
|
}
|
|
1458
2232
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1459
2233
|
exhaustionReason = "deactivated";
|
|
2234
|
+
capabilityRejectionIsLatest = false;
|
|
1460
2235
|
state.status.retries += 1;
|
|
1461
2236
|
noteRotation();
|
|
1462
2237
|
continue;
|
|
@@ -1561,6 +2336,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1561
2336
|
applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
1562
2337
|
accountManager.saveToDiskDebounced();
|
|
1563
2338
|
exhaustionReason = "auth-failure";
|
|
2339
|
+
capabilityRejectionIsLatest = false;
|
|
1564
2340
|
transientAttempts += 1;
|
|
1565
2341
|
transientExhaustionReason = "auth-failure";
|
|
1566
2342
|
state.status.retries += 1;
|
|
@@ -1601,6 +2377,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1601
2377
|
accountManager.markAccountCoolingDown(refreshed.account, state.serverErrorCooldownMs, "server-error");
|
|
1602
2378
|
accountManager.saveToDiskDebounced();
|
|
1603
2379
|
exhaustionReason = "server-error";
|
|
2380
|
+
capabilityRejectionIsLatest = false;
|
|
1604
2381
|
transientAttempts += 1;
|
|
1605
2382
|
transientExhaustionReason = "server-error";
|
|
1606
2383
|
state.status.retries += 1;
|
|
@@ -1627,14 +2404,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1627
2404
|
});
|
|
1628
2405
|
return;
|
|
1629
2406
|
}
|
|
1630
|
-
accountManager.recordSuccess(refreshed.account, context.family, context.model);
|
|
1631
2407
|
// A successful request proves the account is usable, so clear any
|
|
1632
2408
|
// stale runtime skip reason persisted for it on a prior pool
|
|
1633
2409
|
// exhaustion. Without this the overlay reason (e.g. "token-exhausted",
|
|
1634
2410
|
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
1635
2411
|
// the forecast keeps reporting this working account as unavailable.
|
|
1636
2412
|
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
1637
|
-
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
1638
2413
|
const quotaDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
1639
2414
|
const nearExhaustionWaitMs = quotaDeferral.defer
|
|
1640
2415
|
? quotaDeferral.waitMs
|
|
@@ -1651,16 +2426,28 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1651
2426
|
}
|
|
1652
2427
|
state.lastGlobalSwitchAt = state.now();
|
|
1653
2428
|
}
|
|
1654
|
-
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy);
|
|
2429
|
+
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
1655
2430
|
// Recover the upstream token counts as the body streams past. Without
|
|
1656
2431
|
// them every ledger row lands with all-zero tokens and a zero cost, so
|
|
1657
2432
|
// evaluateBudgetGuard compares `0 >= limit` for maxTokens/maxCostUsd
|
|
1658
2433
|
// and those caps never fire — `budget set --cost 50` would allow
|
|
1659
2434
|
// unlimited spend, with only --requests actually enforced.
|
|
2435
|
+
const responseOutcome = new ResponseOutcome(isResponsesRequest && upstream.headers.get("content-type")?.includes("text/event-stream") === true);
|
|
1660
2436
|
const usageScanner = createUsageStreamScanner({
|
|
1661
2437
|
contentType: upstream.headers.get("content-type"),
|
|
2438
|
+
onEvent: (event) => {
|
|
2439
|
+
responseOutcome.observe(event);
|
|
2440
|
+
const snapshot = readSubscriptionQuotaEvent(event, state.now());
|
|
2441
|
+
if (!snapshot)
|
|
2442
|
+
return;
|
|
2443
|
+
observeQuota(snapshot, snapshot.planType);
|
|
2444
|
+
state.status.streamQuotaUpdates = (state.status.streamQuotaUpdates ?? 0) + 1;
|
|
2445
|
+
state.status.lastStreamQuotaUpdateAt = snapshot.updatedAt;
|
|
2446
|
+
},
|
|
1662
2447
|
});
|
|
2448
|
+
let streamErrored = false;
|
|
1663
2449
|
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
|
|
2450
|
+
streamErrored = true;
|
|
1664
2451
|
// Deliberately NOT the pre-header transport policy above,
|
|
1665
2452
|
// which skips recordFailure (#677). By this point the
|
|
1666
2453
|
// upstream accepted the request and began responding, so a
|
|
@@ -1671,9 +2458,25 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1671
2458
|
accountManager.markAccountCoolingDown(refreshed.account, state.networkErrorCooldownMs, "network-error");
|
|
1672
2459
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
1673
2460
|
accountManager.saveToDiskDebounced();
|
|
1674
|
-
}, state.streamStallTimeoutMs, usageScanner.push, budgetAdvisory.level === "soft"
|
|
1675
|
-
|
|
1676
|
-
|
|
2461
|
+
}, state.streamStallTimeoutMs, usageScanner.push, { ...(budgetAdvisory.level === "soft" ? buildContextBudgetHeaders(budgetAdvisory) : {}), ...(state.catalogEtag ? { "x-models-etag": state.catalogEtag } : {}) }, () => { usageScanner.result(); return responseOutcome.finish(); });
|
|
2462
|
+
// The client went away mid-stream: no upstream error, and the response was
|
|
2463
|
+
// neither completed nor failed. That says nothing about the account.
|
|
2464
|
+
const clientDisconnected = !forwarded && !streamErrored && !res.writableEnded;
|
|
2465
|
+
if (forwarded && upstream.ok) {
|
|
2466
|
+
accountManager.recordSuccess(refreshed.account, context.family, context.model);
|
|
2467
|
+
recordRuntimeAccountRecovery(refreshed.account.index);
|
|
2468
|
+
}
|
|
2469
|
+
else if (!clientDisconnected) {
|
|
2470
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2471
|
+
const rejection = classifyCapabilityFailure(400, responseOutcome.rejection);
|
|
2472
|
+
if (rejection && context.model) {
|
|
2473
|
+
const body = parseRequestBody(context.body);
|
|
2474
|
+
const effort = isRecord(body?.reasoning) && typeof body.reasoning.effort === "string" ? body.reasoning.effort : undefined;
|
|
2475
|
+
const tier = typeof body?.service_tier === "string" ? body.service_tier : undefined;
|
|
2476
|
+
const scope = workspaceModelScopes(refreshed.account).find(item => item.accountId === accountId);
|
|
2477
|
+
(state.capabilityFailures ??= new RuntimeCapabilityFailures(state.now)).record(scope?.id ?? catalogAccountKey(refreshed.account), context.model, rejection, effort, tier);
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
1677
2480
|
// A stream that broke mid-flight still bills for whatever the upstream
|
|
1678
2481
|
// reported before the break, so record the counts on both outcomes.
|
|
1679
2482
|
const usageTokens = usageScanner.result();
|
|
@@ -1697,9 +2500,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1697
2500
|
});
|
|
1698
2501
|
}
|
|
1699
2502
|
await usageRecorder.record({
|
|
1700
|
-
outcome: forwarded ? "success" : "failure",
|
|
2503
|
+
outcome: forwarded && upstream.ok ? "success" : clientDisconnected ? "cancelled" : "failure",
|
|
1701
2504
|
statusCode: upstream.status,
|
|
1702
|
-
errorCode: forwarded ? null : "stream_forward_failed",
|
|
2505
|
+
errorCode: clientDisconnected ? "client_disconnected" : responseOutcome.finish().errorCode ?? (forwarded ? null : "stream_forward_failed"),
|
|
1703
2506
|
account: refreshed.account,
|
|
1704
2507
|
...(usageTokens ?? {}),
|
|
1705
2508
|
});
|
|
@@ -1816,6 +2619,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1816
2619
|
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
|
|
1817
2620
|
return;
|
|
1818
2621
|
}
|
|
2622
|
+
// Forward the upstream capability rejection only when it is what ended the
|
|
2623
|
+
// request; a later 429/5xx/transport failure reports itself instead.
|
|
2624
|
+
if (lastCapabilityRejection && capabilityRejectionIsLatest && !isThreadGoalRequest) {
|
|
2625
|
+
await forwardCapabilityRejection(lastCapabilityRejection);
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
1819
2628
|
await usageRecorder?.record({
|
|
1820
2629
|
outcome: "failure",
|
|
1821
2630
|
statusCode: normalizeExhaustionStatus(exhaustionReason),
|