@bitkyc08/opencodex 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS_INSTALL.md +77 -0
- package/README.md +4 -10
- package/bin/ocx.mjs +71 -18
- package/gui/dist/assets/index-Cd6_PBKn.css +1 -0
- package/gui/dist/assets/index-ChZQsmBY.js +70 -0
- package/gui/dist/index.html +2 -2
- package/gui/dist/provider-icons/alibaba-color.svg +1 -1
- package/gui/dist/provider-icons/antigravity-color.svg +1 -1
- package/gui/dist/provider-icons/claude-color.svg +1 -1
- package/gui/dist/provider-icons/cline-color.svg +16 -0
- package/gui/dist/provider-icons/cloudflare-ai-gateway-color.svg +1 -1
- package/gui/dist/provider-icons/copilot-color.svg +1 -1
- package/gui/dist/provider-icons/cursor-color.svg +1 -1
- package/gui/dist/provider-icons/deepseek-color.svg +1 -1
- package/gui/dist/provider-icons/firepass-color.svg +1 -1
- package/gui/dist/provider-icons/fireworks-color.svg +1 -1
- package/gui/dist/provider-icons/gemini-color.svg +1 -1
- package/gui/dist/provider-icons/github-copilot-color.svg +1 -1
- package/gui/dist/provider-icons/gitlab-duo-color.svg +1 -1
- package/gui/dist/provider-icons/grok.svg +1 -1
- package/gui/dist/provider-icons/groq-color.svg +1 -1
- package/gui/dist/provider-icons/huggingface-color.svg +1 -1
- package/gui/dist/provider-icons/kimi-color.svg +1 -1
- package/gui/dist/provider-icons/kiro-color.svg +2 -2
- package/gui/dist/provider-icons/lm-studio-color.svg +1 -1
- package/gui/dist/provider-icons/mistral-color.svg +1 -1
- package/gui/dist/provider-icons/moonshot-color.svg +1 -1
- package/gui/dist/provider-icons/nvidia-color.svg +1 -1
- package/gui/dist/provider-icons/ollama-color.svg +1 -1
- package/gui/dist/provider-icons/openai.svg +1 -1
- package/gui/dist/provider-icons/opencode.svg +2 -1
- package/gui/dist/provider-icons/openrouter-color.svg +1 -1
- package/gui/dist/provider-icons/pi.svg +2 -2
- package/gui/dist/provider-icons/qianfan-color.svg +1 -1
- package/gui/dist/provider-icons/qwen-portal-color.svg +1 -1
- package/gui/dist/provider-icons/vercel-ai-gateway-color.svg +1 -1
- package/gui/dist/provider-icons/vllm-color.svg +1 -1
- package/gui/dist/provider-icons/xiaomi-color.svg +1 -1
- package/package.json +8 -4
- package/src/adapters/anthropic.ts +208 -14
- package/src/adapters/base.ts +16 -5
- package/src/adapters/cursor/effort-map.ts +3 -2
- package/src/adapters/cursor/framing.ts +39 -0
- package/src/adapters/cursor/live-transport.ts +105 -95
- package/src/adapters/cursor/native-exec.ts +32 -6
- package/src/adapters/cursor/protobuf-request.ts +20 -15
- package/src/adapters/cursor/request-builder.ts +21 -7
- package/src/adapters/cursor/types.ts +7 -0
- package/src/adapters/google-antigravity-replay.ts +237 -21
- package/src/adapters/google-truncation.ts +11 -0
- package/src/adapters/google.ts +50 -9
- package/src/adapters/identity.ts +39 -6
- package/src/adapters/kiro-errors.ts +11 -0
- package/src/adapters/kiro-events.ts +19 -1
- package/src/adapters/kiro-thinking.ts +10 -2
- package/src/adapters/kiro-tools.ts +10 -1
- package/src/adapters/kiro.ts +37 -11
- package/src/adapters/openai-chat.ts +284 -83
- package/src/adapters/openai-responses.ts +182 -24
- package/src/bridge.ts +177 -7
- package/src/chat/outbound.ts +78 -23
- package/src/claude/agents-inject.ts +27 -5
- package/src/claude/inbound.ts +11 -1
- package/src/claude/model-info.ts +13 -10
- package/src/claude/outbound.ts +17 -0
- package/src/cli/account-api.ts +24 -0
- package/src/cli/account-auth.ts +31 -6
- package/src/cli/account-main.ts +317 -0
- package/src/cli/account.ts +5 -0
- package/src/cli/claude.ts +2 -1
- package/src/cli/doctor.ts +93 -22
- package/src/cli/export-command.ts +26 -12
- package/src/cli/help.ts +8 -6
- package/src/cli/index.ts +56 -22
- package/src/cli/integrations.ts +84 -1
- package/src/cli/observe.ts +54 -1
- package/src/cli/opencode.ts +2 -1
- package/src/cli/provider-runtime.ts +18 -1
- package/src/cli/route-policy.ts +92 -0
- package/src/cli/runtime-api.ts +6 -3
- package/src/cli/star-prompt.ts +71 -15
- package/src/cli/status.ts +1 -1
- package/src/cli/v2.ts +36 -9
- package/src/clients/config-export.ts +687 -10
- package/src/codex/account-lifecycle.ts +30 -5
- package/src/codex/account-usability.ts +22 -2
- package/src/codex/admission.ts +255 -0
- package/src/codex/auth-api.ts +427 -140
- package/src/codex/auth-context.ts +155 -30
- package/src/codex/autostart-health.ts +8 -1
- package/src/codex/catalog/account-models.ts +62 -0
- package/src/codex/catalog/aggregation.ts +14 -1
- package/src/codex/catalog/bundled.ts +282 -32
- package/src/codex/catalog/filesystem-evidence.ts +302 -0
- package/src/codex/catalog/metadata.ts +51 -6
- package/src/codex/catalog/parsing.ts +6 -3
- package/src/codex/catalog/provider-fetch.ts +576 -41
- package/src/codex/catalog/sync.ts +505 -66
- package/src/codex/catalog-admission.ts +197 -0
- package/src/codex/catalog-write-serialization.ts +241 -0
- package/src/codex/catalog.ts +2 -1
- package/src/codex/codex-write-lock.ts +372 -0
- package/src/codex/convergence-types.ts +593 -0
- package/src/codex/convergence.ts +441 -0
- package/src/codex/desired-state.ts +177 -0
- package/src/codex/features.ts +52 -8
- package/src/codex/generation.ts +202 -0
- package/src/codex/history-job.ts +257 -0
- package/src/codex/history-lock.ts +241 -0
- package/src/codex/history-migration-guardian.ts +18 -5
- package/src/codex/history-provider.ts +9 -2
- package/src/codex/history-transition.ts +105 -0
- package/src/codex/history-worker.ts +176 -0
- package/src/codex/inject-coordination.ts +245 -0
- package/src/codex/inject.ts +605 -124
- package/src/codex/integration-record.ts +266 -0
- package/src/codex/internal/catalog-writer.ts +203 -0
- package/src/codex/internal/history-writer.ts +80 -0
- package/src/codex/journal.ts +10 -1
- package/src/codex/main-account-cache.ts +24 -0
- package/src/codex/management-convergence.ts +114 -0
- package/src/codex/native-main-admission.ts +47 -0
- package/src/codex/native-main-auth-temp.ts +187 -0
- package/src/codex/native-main-claim.ts +167 -0
- package/src/codex/native-main-lock-file.ts +158 -0
- package/src/codex/native-main-owner.ts +315 -0
- package/src/codex/native-profile-api.ts +247 -0
- package/src/codex/native-profile-manager.ts +1512 -0
- package/src/codex/native-profile-processes.ts +121 -0
- package/src/codex/native-profile-recovery.ts +99 -0
- package/src/codex/native-profile-stage-store.ts +387 -0
- package/src/codex/native-profile-startup.ts +340 -0
- package/src/codex/native-profile-store.ts +855 -0
- package/src/codex/native-profile-types.ts +120 -0
- package/src/codex/native-residue.ts +557 -0
- package/src/codex/project-config-warnings.ts +18 -4
- package/src/codex/prompt-journal.ts +311 -0
- package/src/codex/prompt-layers.ts +967 -0
- package/src/codex/prompt-lock.ts +143 -0
- package/src/codex/quota-rejection.ts +224 -0
- package/src/codex/quota.ts +86 -3
- package/src/codex/routing.ts +299 -62
- package/src/codex/runtime.ts +159 -38
- package/src/codex/shim.ts +39 -13
- package/src/codex/subagent-model-fallback.ts +73 -12
- package/src/codex/transition-state.ts +604 -0
- package/src/codex/upstream-host-health.ts +70 -0
- package/src/codex/user-identity.ts +266 -0
- package/src/codex/write-coordination.ts +114 -0
- package/src/config.ts +562 -26
- package/src/generated/jawcode-model-metadata.ts +2 -2
- package/src/grok/inject.ts +15 -4
- package/src/grok/inspect.ts +45 -0
- package/src/images/loop.ts +113 -20
- package/src/integrations/config-io.ts +151 -0
- package/src/integrations/journal.ts +315 -0
- package/src/integrations/merge.ts +135 -0
- package/src/integrations/native/ownership-preflight.ts +165 -0
- package/src/integrations/ownership.ts +111 -0
- package/src/integrations/registry.ts +101 -0
- package/src/integrations/serialize.ts +235 -0
- package/src/integrations/state.ts +290 -0
- package/src/integrations/store.ts +103 -0
- package/src/integrations/writer.ts +492 -0
- package/src/lib/bounded-body.ts +46 -8
- package/src/lib/bun-runtime.ts +110 -1
- package/src/lib/bun-stream-caps.ts +2 -1
- package/src/lib/redact.ts +407 -2
- package/src/lib/shadow-call.ts +24 -0
- package/src/lib/translator-budget.ts +10 -0
- package/src/lib/upstream-reachability.ts +91 -0
- package/src/lib/upstream-retry.ts +154 -2
- package/src/lib/windows-secret-acl.ts +212 -11
- package/src/lib/winsw.ts +9 -3
- package/src/oauth/index.ts +61 -3
- package/src/oauth/key-providers.ts +4 -0
- package/src/oauth/kiro.ts +50 -6
- package/src/oauth/store.ts +31 -6
- package/src/oauth/token-guardian.ts +9 -3
- package/src/providers/codex-capacity.ts +288 -0
- package/src/providers/derive.ts +33 -1
- package/src/providers/free-directory.ts +3 -1
- package/src/providers/key-failover.ts +71 -3
- package/src/providers/openai-sidecar.ts +64 -4
- package/src/providers/openai-virtual-models.ts +1 -0
- package/src/providers/quota.ts +334 -26
- package/src/providers/registry.ts +284 -16
- package/src/providers/xai-transport.ts +11 -4
- package/src/responses/compaction.ts +8 -1
- package/src/responses/hosted-tool-policy.ts +9 -0
- package/src/responses/parser.ts +44 -2
- package/src/responses/reasoning-envelope.ts +9 -1
- package/src/responses/reasoning-replay-cache.ts +105 -0
- package/src/responses/spill-store.ts +45 -8
- package/src/responses/state.ts +161 -17
- package/src/router.ts +302 -16
- package/src/routing/analytics.ts +377 -0
- package/src/routing/capability.ts +204 -0
- package/src/routing/cost.ts +77 -0
- package/src/routing/evaluator.ts +444 -0
- package/src/routing/health.ts +401 -0
- package/src/routing/history/cursor.ts +43 -0
- package/src/routing/history/indexer.ts +590 -0
- package/src/routing/history/schema.ts +72 -0
- package/src/routing/profile.ts +423 -0
- package/src/routing/quota.ts +124 -0
- package/src/routing/request-evidence.ts +45 -0
- package/src/routing/trace.ts +686 -0
- package/src/server/auth-cors.ts +46 -6
- package/src/server/chat-completions.ts +28 -13
- package/src/server/claude-messages.ts +23 -15
- package/src/server/gui-static.ts +39 -10
- package/src/server/images.ts +10 -1
- package/src/server/index.ts +238 -52
- package/src/server/lifecycle.ts +228 -9
- package/src/server/live.ts +14 -3
- package/src/server/management/agent-settings-routes.ts +64 -14
- package/src/server/management/combo-routes.ts +5 -5
- package/src/server/management/config-routes.ts +1 -1
- package/src/server/management/context.ts +42 -2
- package/src/server/management/integration-routes.ts +538 -0
- package/src/server/management/logs-usage-routes.ts +1 -1
- package/src/server/management/model-routes.ts +32 -113
- package/src/server/management/model-rows.ts +117 -0
- package/src/server/management/native-integration-routes.ts +587 -0
- package/src/server/management/oauth-account-routes.ts +1 -1
- package/src/server/management/provider-routes.ts +218 -117
- package/src/server/management/request-history-routes.ts +191 -0
- package/src/server/management/routing-analytics-routes.ts +74 -0
- package/src/server/management/routing-profile-routes.ts +177 -0
- package/src/server/management/shared.ts +2 -2
- package/src/server/management/sidebar-routes.ts +47 -31
- package/src/server/management/sync-response.ts +69 -0
- package/src/server/management/system-restart.ts +276 -24
- package/src/server/management/system-routes.ts +4 -0
- package/src/server/management-api.ts +84 -9
- package/src/server/management-auth.ts +43 -5
- package/src/server/relay-eager.ts +82 -42
- package/src/server/relay.ts +120 -6
- package/src/server/request-log.ts +26 -6
- package/src/server/responses/collaboration.ts +63 -8
- package/src/server/responses/compact.ts +272 -41
- package/src/server/responses/core.ts +730 -132
- package/src/server/responses/fetch-helpers.ts +15 -1
- package/src/server/responses-item-id-repair.ts +32 -3
- package/src/server/responses-json-events.ts +52 -0
- package/src/server/responses-snapshot-repair.ts +621 -0
- package/src/server/search.ts +51 -6
- package/src/server/sse-payload-rewrite.ts +89 -12
- package/src/server/startup-health-cache.ts +7 -1
- package/src/server/ws-bridge.ts +11 -17
- package/src/service-manager-probe.ts +297 -0
- package/src/service.ts +222 -32
- package/src/tray/windows-tray.ps1 +9 -0
- package/src/tray/windows.ts +15 -7
- package/src/types.ts +194 -14
- package/src/update/index.ts +13 -13
- package/src/update/job.ts +24 -21
- package/src/update/notify.ts +7 -3
- package/src/usage/cost.ts +0 -0
- package/src/usage/expected-prices.ts +129 -10
- package/src/usage/log.ts +50 -15
- package/src/usage/summary.ts +4 -4
- package/src/vision/index.ts +6 -1
- package/src/web-search/loop.ts +161 -34
- package/gui/dist/assets/index-OY43ubAq.css +0 -1
- package/gui/dist/assets/index-YwNnKZcL.js +0 -67
- package/gui/dist/provider-icons/antigravity.svg +0 -1
- package/gui/dist/provider-icons/claude.svg +0 -1
- package/gui/dist/provider-icons/copilot.svg +0 -1
- package/gui/dist/provider-icons/cursor.svg +0 -2
- package/gui/dist/provider-icons/gemini.svg +0 -1
- package/gui/dist/provider-icons/grok-color.svg +0 -1
- package/gui/dist/provider-icons/kiro.svg +0 -14
|
@@ -16,7 +16,8 @@ import {
|
|
|
16
16
|
previousResponseReplayFailure,
|
|
17
17
|
rememberResponseState,
|
|
18
18
|
} from "../../responses/state";
|
|
19
|
-
import { routeModel, type RouteResult } from "../../router";
|
|
19
|
+
import { comboRouteDecisionTrace, NoEligiblePolicyCandidateError, routeModel, type RouteResult } from "../../router";
|
|
20
|
+
import { evidenceFromBody } from "../../routing/request-evidence";
|
|
20
21
|
import {
|
|
21
22
|
advanceComboAfterFailure,
|
|
22
23
|
comboDefaultEffort,
|
|
@@ -59,14 +60,16 @@ import {
|
|
|
59
60
|
} from "../../oauth/anthropic-routing";
|
|
60
61
|
import { buildWebSearchTool, planWebSearch, runWithWebSearch, shouldResolveOpenAiWebSearchSidecar } from "../../web-search";
|
|
61
62
|
import { buildImageTool, buildVideoTool, planImageBridge, planVideoBridge, runWithImageBridge, clampImageMaxRounds, IMAGE_GEN_TOOL_NAME, VIDEO_GEN_TOOL_NAME } from "../../images";
|
|
62
|
-
import { describeImagesInPlace, planVisionSidecar, shouldResolveOpenAiVisionSidecar, stripImagesInPlace } from "../../vision";
|
|
63
|
+
import { describeImagesInPlace, planVisionSidecar, resolveOpenAiVisionModel, shouldResolveOpenAiVisionSidecar, stripImagesInPlace } from "../../vision";
|
|
63
64
|
import { createAdapterEventQueue, preflightAdapterEvents } from "../../adapters/run-turn-queue";
|
|
64
65
|
import {
|
|
65
66
|
applyCodexAuthContextToProvider,
|
|
66
67
|
CodexAccountCooldownError,
|
|
68
|
+
codexMainProfileDrainingResponse,
|
|
67
69
|
cooldownErrorResponse,
|
|
68
70
|
CodexAuthContextError,
|
|
69
71
|
CodexDirectAuthenticationError,
|
|
72
|
+
CodexMainProfileDrainingError,
|
|
70
73
|
CodexPoolAuthenticationError,
|
|
71
74
|
CodexThreadAffinityExpiredError,
|
|
72
75
|
headersForCodexAuthContext,
|
|
@@ -85,7 +88,12 @@ import {
|
|
|
85
88
|
recordCodexUpstreamOutcome,
|
|
86
89
|
type CodexUpstreamOutcome,
|
|
87
90
|
} from "../../codex/routing";
|
|
88
|
-
import {
|
|
91
|
+
import {
|
|
92
|
+
applyUpstreamRecoveryInit,
|
|
93
|
+
fetchWithResetRetry,
|
|
94
|
+
fetchWithTransientRetry,
|
|
95
|
+
prepareSameTarget429Wait,
|
|
96
|
+
} from "../../lib/upstream-retry";
|
|
89
97
|
import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors";
|
|
90
98
|
import { createTranslatorBudget, isTranslatorBudgetExceededError, type TranslatorBudget } from "../../lib/translator-budget";
|
|
91
99
|
import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar";
|
|
@@ -95,12 +103,18 @@ import { applyOpenAiVirtualModel, resolveOpenAiCompactModel } from "../../provid
|
|
|
95
103
|
import { isUsageDebugEnabled } from "../../usage/debug";
|
|
96
104
|
import { readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../request-decompress";
|
|
97
105
|
import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve";
|
|
98
|
-
import type
|
|
99
|
-
import {
|
|
106
|
+
import { providerModelResponsesUpstreamStreaming, type InboundWire } from "../../providers/registry";
|
|
107
|
+
import type { AdapterRequest } from "../../adapters/base";
|
|
108
|
+
import {
|
|
109
|
+
hasKeyPoolFailover,
|
|
110
|
+
rateLimitRetryDelayMs,
|
|
111
|
+
rateLimitRetryPolicyFor,
|
|
112
|
+
rotateProviderTransportOn429,
|
|
113
|
+
} from "../../providers/key-failover";
|
|
100
114
|
import { shouldAttemptImageTierRetry } from "../image-retry";
|
|
101
115
|
import { resolveProviderTransport } from "../../providers/xai-transport";
|
|
102
116
|
import type { WsData } from "../ws-bridge";
|
|
103
|
-
import { registerTurn, trackStreamLifetime, unregisterTurn } from "../lifecycle";
|
|
117
|
+
import { codexAccountSelectionForTurn, registerTurn, trackStreamLifetime, unregisterTurn } from "../lifecycle";
|
|
104
118
|
import { redactSecretString } from "../../lib/redact";
|
|
105
119
|
import { readBoundedResponseBody } from "../../lib/bounded-body";
|
|
106
120
|
import type { AdmissionLease } from "../../lib/admission";
|
|
@@ -111,6 +125,7 @@ import {
|
|
|
111
125
|
maybePrimeSubagentQuota,
|
|
112
126
|
recordSubagentQuotaFailureForThreadSpawn,
|
|
113
127
|
} from "../../codex/subagent-model-fallback";
|
|
128
|
+
import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup";
|
|
114
129
|
import {
|
|
115
130
|
beginRequestAttempt,
|
|
116
131
|
catalogModelSupportsServiceTier,
|
|
@@ -149,6 +164,7 @@ import { cancelBodyOnAbort } from "../../lib/abort";
|
|
|
149
164
|
import {
|
|
150
165
|
createResponsesItemIdPayloadRewrite,
|
|
151
166
|
hasResponsesItemIdRepair,
|
|
167
|
+
repairResponsesJsonItemIds,
|
|
152
168
|
} from "../responses-item-id-repair";
|
|
153
169
|
import {
|
|
154
170
|
createImageGenCallRestoreRewrite,
|
|
@@ -160,7 +176,20 @@ import type { EffectiveSubagentRoster, SpawnAgentSurface } from "../../codex/cat
|
|
|
160
176
|
|
|
161
177
|
import { buildToolBridgeMaps, collabSurface, injectDeveloperMessage, multiAgentGuidanceText } from "./collaboration";
|
|
162
178
|
import { hasUnreadableEncryptedAgentTask, looksLikeBackendCiphertext, sanitizeEncryptedContentInPlace } from "./encrypted-payload";
|
|
163
|
-
import { fetchWithHeaderTimeout, providerFetch, safeHostLabel } from "./fetch-helpers";
|
|
179
|
+
import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel } from "./fetch-helpers";
|
|
180
|
+
import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability";
|
|
181
|
+
import { recordUpstreamHostFailure, resetUpstreamHostHealth, upstreamHostHealthKey } from "../../codex/upstream-host-health";
|
|
182
|
+
import {
|
|
183
|
+
createResponsesSnapshotBlockRewrite,
|
|
184
|
+
hasResponsesSnapshotRepair,
|
|
185
|
+
repairResponsesSnapshotJson,
|
|
186
|
+
} from "../responses-snapshot-repair";
|
|
187
|
+
import {
|
|
188
|
+
composeSseBlockRewrites,
|
|
189
|
+
payloadRewriteAsBlockRewrite,
|
|
190
|
+
relaySseWithBlockRewrite,
|
|
191
|
+
} from "../sse-payload-rewrite";
|
|
192
|
+
import { responsesJsonToSseBody } from "../responses-json-events";
|
|
164
193
|
import { guardTerminalEventStream } from "./terminal-guard";
|
|
165
194
|
|
|
166
195
|
/**
|
|
@@ -178,7 +207,9 @@ export function sidecarOutcomeRecorder(
|
|
|
178
207
|
return authCtx.kind === "pool" || authCtx.kind === "main-pool"
|
|
179
208
|
? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
|
|
180
209
|
threadId,
|
|
210
|
+
fixedAccount: authCtx.fixedAccount,
|
|
181
211
|
probeLeaseId: authCtx.probeLeaseId,
|
|
212
|
+
probeQuotaScope: authCtx.probeQuotaScope,
|
|
182
213
|
writerGeneration: authCtx.writerGeneration,
|
|
183
214
|
})
|
|
184
215
|
: undefined;
|
|
@@ -186,7 +217,7 @@ export function sidecarOutcomeRecorder(
|
|
|
186
217
|
|
|
187
218
|
|
|
188
219
|
|
|
189
|
-
import { isShadowSourceModel } from "../../lib/shadow-call";
|
|
220
|
+
import { isShadowSourceModel, shouldInterceptShadowCall } from "../../lib/shadow-call";
|
|
190
221
|
|
|
191
222
|
export { DEFAULT_SHADOW_SOURCE_MODELS, isShadowSourceModel, shadowSourceModels } from "../../lib/shadow-call";
|
|
192
223
|
|
|
@@ -196,6 +227,11 @@ export function codexLogAccountId(authCtx: CodexAuthContext): string | null {
|
|
|
196
227
|
return authCtx.kind === "pool" || authCtx.kind === "main-pool" ? authCtx.accountId : null;
|
|
197
228
|
}
|
|
198
229
|
|
|
230
|
+
function isFixedCodexAccount(authCtx: CodexAuthContext): boolean {
|
|
231
|
+
return (authCtx.kind === "pool" || authCtx.kind === "main-pool")
|
|
232
|
+
&& authCtx.fixedAccount === true;
|
|
233
|
+
}
|
|
234
|
+
|
|
199
235
|
|
|
200
236
|
|
|
201
237
|
export function usesCodexForwardPoolAuth(
|
|
@@ -246,8 +282,8 @@ async function shouldRetryCodexPoolAccountModel400(
|
|
|
246
282
|
}
|
|
247
283
|
|
|
248
284
|
/** Pre-stream quota/billing rejections that warrant one alternate-account attempt (#584). */
|
|
249
|
-
function shouldRetryCodexPoolAccountQuota(response: Response): boolean {
|
|
250
|
-
return response.status ===
|
|
285
|
+
export function shouldRetryCodexPoolAccountQuota(response: Response): boolean {
|
|
286
|
+
return response.status === 402 || response.status === 429;
|
|
251
287
|
}
|
|
252
288
|
|
|
253
289
|
interface CodexPoolAccountRetryArgs {
|
|
@@ -265,6 +301,7 @@ interface CodexPoolAccountRetryArgs {
|
|
|
265
301
|
// first attempt.
|
|
266
302
|
inboundWire?: InboundWire;
|
|
267
303
|
translatorBudget: TranslatorBudget;
|
|
304
|
+
turnAdmissionLease?: AdmissionLease;
|
|
268
305
|
};
|
|
269
306
|
firstAuthCtx: Extract<CodexAuthContext, { kind: "pool" | "main-pool" }>;
|
|
270
307
|
firstResponse: Response;
|
|
@@ -326,6 +363,9 @@ async function retryCodexPoolOnAlternateAccount(
|
|
|
326
363
|
req, config, route, parsed, logCtx, options, firstAuthCtx, firstResponse,
|
|
327
364
|
outcomeStatus, upstream, connectMs, passthroughEstimate, stream,
|
|
328
365
|
} = args;
|
|
366
|
+
// Defense in depth: exact account selectors must never reach alternate-account resolution,
|
|
367
|
+
// even if a future caller forgets to guard this helper.
|
|
368
|
+
if (firstAuthCtx.fixedAccount) return { kind: "no-alternate" };
|
|
329
369
|
const inboundWire = options.inboundWire ?? "responses";
|
|
330
370
|
let retryAuthCtx: CodexAuthContext | undefined;
|
|
331
371
|
try {
|
|
@@ -333,13 +373,18 @@ async function retryCodexPoolOnAlternateAccount(
|
|
|
333
373
|
req.headers,
|
|
334
374
|
config,
|
|
335
375
|
"pool",
|
|
336
|
-
{
|
|
376
|
+
{
|
|
377
|
+
excludeAccountId: firstAuthCtx.accountId,
|
|
378
|
+
modelId: route.modelId,
|
|
379
|
+
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
|
|
380
|
+
},
|
|
337
381
|
);
|
|
338
382
|
} catch (error) {
|
|
339
383
|
if (
|
|
340
384
|
!(error instanceof CodexPoolAuthenticationError)
|
|
341
385
|
&& !(error instanceof CodexAuthContextError)
|
|
342
386
|
&& !(error instanceof CodexAccountCooldownError)
|
|
387
|
+
&& !(error instanceof CodexMainProfileDrainingError)
|
|
343
388
|
) throw error;
|
|
344
389
|
}
|
|
345
390
|
if (retryAuthCtx?.kind !== "pool" && retryAuthCtx?.kind !== "main-pool") {
|
|
@@ -406,7 +451,12 @@ async function retryCodexPoolOnAlternateAccount(
|
|
|
406
451
|
connectMs,
|
|
407
452
|
stream,
|
|
408
453
|
providerFetch(route.provider),
|
|
454
|
+
// Credential-bearing forward send: never follow a redirect into a
|
|
455
|
+
// dead-host rejection after the credential was seen (#914).
|
|
456
|
+
route.provider.authMode === "forward",
|
|
409
457
|
);
|
|
458
|
+
// A real HTTP response proves the host was reached (#914).
|
|
459
|
+
resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)));
|
|
410
460
|
return {
|
|
411
461
|
kind: "retried",
|
|
412
462
|
authCtx: retryAuthCtx,
|
|
@@ -440,6 +490,7 @@ export function codexForwardTerminalOutcomeRecorder(
|
|
|
440
490
|
// prior soft-avoid so a healthy account isn't stuck avoided.
|
|
441
491
|
recordCodexUpstreamOutcome(config, authCtx.accountId, 200, {
|
|
442
492
|
threadId,
|
|
493
|
+
fixedAccount: authCtx.fixedAccount,
|
|
443
494
|
modelId,
|
|
444
495
|
probeLeaseId: codexProbeLeaseId(authCtx),
|
|
445
496
|
probeQuotaScope: codexProbeQuotaScope(authCtx),
|
|
@@ -461,6 +512,7 @@ export function codexForwardTerminalOutcomeRecorder(
|
|
|
461
512
|
: (httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502);
|
|
462
513
|
recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
|
|
463
514
|
threadId,
|
|
515
|
+
fixedAccount: authCtx.fixedAccount,
|
|
464
516
|
modelId,
|
|
465
517
|
probeLeaseId: codexProbeLeaseId(authCtx),
|
|
466
518
|
probeQuotaScope: codexProbeQuotaScope(authCtx),
|
|
@@ -537,6 +589,8 @@ export interface HandleResponsesOptions {
|
|
|
537
589
|
* it. Omitted means a genuine Responses inbound.
|
|
538
590
|
*/
|
|
539
591
|
inboundWire?: InboundWire;
|
|
592
|
+
/** Internal transport identity for route-scoped upstream compatibility policy. */
|
|
593
|
+
inboundTransport?: "websocket";
|
|
540
594
|
/** Internal recursion guard; callers outside this module must not set it. */
|
|
541
595
|
comboAttempt?: boolean;
|
|
542
596
|
/** Internal combo handoff: allow a later same-provider model after a reset-derived 429/402. */
|
|
@@ -549,6 +603,10 @@ export interface HandleResponsesOptions {
|
|
|
549
603
|
|
|
550
604
|
|
|
551
605
|
|
|
606
|
+
/**
|
|
607
|
+
* Build the 499 JSON error the proxy returns when the client disconnects before the
|
|
608
|
+
* response completes (`client_cancelled`).
|
|
609
|
+
*/
|
|
552
610
|
export function clientCancelledResponse(): Response {
|
|
553
611
|
return formatErrorResponse(499, "client_cancelled", "Client cancelled request");
|
|
554
612
|
}
|
|
@@ -685,6 +743,15 @@ export function buildComboChildHeaders(parentHeaders: HeadersInit): Headers {
|
|
|
685
743
|
const UNREADABLE_ENCRYPTED_AGENT_TASK_MESSAGE =
|
|
686
744
|
"Routed V2 worker task is encrypted for the native ChatGPT backend and cannot be read by the selected provider. Use plaintext V2 agent-message delivery or select a native ChatGPT model.";
|
|
687
745
|
|
|
746
|
+
// Whole-body policy for non-streaming upstream JSON responses (see the application/json
|
|
747
|
+
// branch of the passthrough return path). 32 MiB matches the continuation snapshot read
|
|
748
|
+
// bound and is far above any legitimate non-streaming completion, including base64 image
|
|
749
|
+
// payloads. The stall deadlines only govern the body transfer — generation time before
|
|
750
|
+
// the response headers is untouched.
|
|
751
|
+
const MAX_UPSTREAM_JSON_BODY_BYTES = 32 * 1024 * 1024;
|
|
752
|
+
const UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS = 180_000;
|
|
753
|
+
const UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS = 30_000;
|
|
754
|
+
|
|
688
755
|
function unreadableEncryptedAgentTaskResponse(): Response {
|
|
689
756
|
return new Response(
|
|
690
757
|
JSON.stringify({
|
|
@@ -716,7 +783,11 @@ async function resolveResponsesCodexAuth(
|
|
|
716
783
|
if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config);
|
|
717
784
|
let authCtx: CodexAuthContext;
|
|
718
785
|
if (route.codexAccountMode) {
|
|
719
|
-
authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, {
|
|
786
|
+
authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, {
|
|
787
|
+
accountId: route.codexAccountId,
|
|
788
|
+
modelId: route.modelId,
|
|
789
|
+
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
|
|
790
|
+
});
|
|
720
791
|
options.onCodexAuthContextResolved?.(authCtx);
|
|
721
792
|
} else {
|
|
722
793
|
authCtx = { kind: "main", accountId: null };
|
|
@@ -736,7 +807,10 @@ async function resolveResponsesCodexAuth(
|
|
|
736
807
|
};
|
|
737
808
|
} catch (err) {
|
|
738
809
|
if (err instanceof CodexAccountCooldownError) {
|
|
739
|
-
return { ok: false, response: cooldownErrorResponse(err) };
|
|
810
|
+
return { ok: false, response: cooldownErrorResponse(err, Date.now(), route.codexAccountNamespace) };
|
|
811
|
+
}
|
|
812
|
+
if (err instanceof CodexMainProfileDrainingError) {
|
|
813
|
+
return { ok: false, response: codexMainProfileDrainingResponse() };
|
|
740
814
|
}
|
|
741
815
|
if (err instanceof CodexThreadAffinityExpiredError) {
|
|
742
816
|
return {
|
|
@@ -745,7 +819,9 @@ async function resolveResponsesCodexAuth(
|
|
|
745
819
|
};
|
|
746
820
|
}
|
|
747
821
|
if (err instanceof CodexAuthContextError) {
|
|
748
|
-
const safeAccountLabel =
|
|
822
|
+
const safeAccountLabel = route.codexAccountNamespace
|
|
823
|
+
? `${route.providerName}-${route.codexAccountNamespace}`
|
|
824
|
+
: formatCodexProviderForLog(route.providerName, err.accountId, config);
|
|
749
825
|
console.error(`[codex-auth] Pool account ${safeAccountLabel} token failed; reauthentication required`);
|
|
750
826
|
return {
|
|
751
827
|
ok: false,
|
|
@@ -776,8 +852,9 @@ async function applyFinalRouteRequestNormalization(args: {
|
|
|
776
852
|
req: Request;
|
|
777
853
|
logCtx: RequestLogContext;
|
|
778
854
|
inboundWire: InboundWire;
|
|
855
|
+
inboundTransport?: "websocket";
|
|
779
856
|
}): Promise<void> {
|
|
780
|
-
const { parsed, route, config, req, logCtx, inboundWire } = args;
|
|
857
|
+
const { parsed, route, config, req, logCtx, inboundWire, inboundTransport } = args;
|
|
781
858
|
|
|
782
859
|
// Apply the routed model id upstream: routing may strip a "<provider>/" namespace.
|
|
783
860
|
if (route.modelId !== parsed.modelId) {
|
|
@@ -786,12 +863,28 @@ async function applyFinalRouteRequestNormalization(args: {
|
|
|
786
863
|
}
|
|
787
864
|
parsed.modelId = route.modelId;
|
|
788
865
|
}
|
|
866
|
+
// Transport-neutral reliability policy (#875): applies to any Responses
|
|
867
|
+
// upstream whose final adapter is openai-responses, not only WS turns.
|
|
868
|
+
const responsesUpstreamStreaming = providerModelResponsesUpstreamStreaming(
|
|
869
|
+
route.providerName,
|
|
870
|
+
route.provider,
|
|
871
|
+
route.modelId,
|
|
872
|
+
);
|
|
873
|
+
|
|
789
874
|
// Settle the wire once so logging, fast-mode, auth, and sidecars read the adapter
|
|
790
875
|
// this request will actually use (#404).
|
|
791
876
|
route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire);
|
|
792
877
|
logCtx.model = route.modelId;
|
|
793
878
|
logCtx.provider = route.providerName;
|
|
794
879
|
logCtx.providerAdapter = route.provider.adapter;
|
|
880
|
+
logCtx.routeDecision = route.routeDecision;
|
|
881
|
+
|
|
882
|
+
if (responsesUpstreamStreaming === false && route.provider.adapter === "openai-responses") {
|
|
883
|
+
parsed.stream = false;
|
|
884
|
+
if (parsed._rawBody && typeof parsed._rawBody === "object") {
|
|
885
|
+
(parsed._rawBody as Record<string, unknown>).stream = false;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
795
888
|
|
|
796
889
|
// Final selected model before virtual wire-model rewriting (Pro aliases).
|
|
797
890
|
const finalSelectedModelId = route.modelId;
|
|
@@ -799,8 +892,9 @@ async function applyFinalRouteRequestNormalization(args: {
|
|
|
799
892
|
// Virtual model rewriting: Pro aliases → base model + reasoning.mode="pro".
|
|
800
893
|
applyOpenAiVirtualModel(parsed, route, logCtx);
|
|
801
894
|
|
|
802
|
-
// Fast mode override for OpenAI-routed models
|
|
803
|
-
|
|
895
|
+
// Fast mode override for OpenAI-routed models, only where the provider's Responses
|
|
896
|
+
// route documents `service_tier` support (capability gate below strips everywhere else).
|
|
897
|
+
if (config.fastMode !== undefined && route.provider.adapter === "openai-responses" && route.provider.supportsServiceTier === true) {
|
|
804
898
|
const tier = config.fastMode ? "priority" : undefined;
|
|
805
899
|
if (parsed._rawBody && typeof parsed._rawBody === "object") {
|
|
806
900
|
if (tier) (parsed._rawBody as Record<string, unknown>).service_tier = tier;
|
|
@@ -808,10 +902,12 @@ async function applyFinalRouteRequestNormalization(args: {
|
|
|
808
902
|
}
|
|
809
903
|
parsed.options.serviceTier = tier;
|
|
810
904
|
}
|
|
905
|
+
applyServiceTierGate(route.provider, parsed._rawBody, parsed.options);
|
|
811
906
|
|
|
812
907
|
{
|
|
813
908
|
const guidance = await multiAgentGuidanceText(parsed, {
|
|
814
909
|
multiAgentGuidanceEnabled: config.multiAgentGuidanceEnabled,
|
|
910
|
+
codexAccountNamespace: route.codexAccountNamespace,
|
|
815
911
|
injectionModel: config.injectionModel,
|
|
816
912
|
injectionEffort: config.injectionEffort,
|
|
817
913
|
subagentModels: config.subagentModels,
|
|
@@ -894,6 +990,7 @@ export async function handleComboResponses(
|
|
|
894
990
|
model: requestedModel,
|
|
895
991
|
provider: "combo",
|
|
896
992
|
comboId,
|
|
993
|
+
routeDecision: logCtx.routeDecision,
|
|
897
994
|
attempts: logCtx.attempts,
|
|
898
995
|
activeAttempt: undefined,
|
|
899
996
|
activeAttemptStartedAt: undefined,
|
|
@@ -928,6 +1025,9 @@ export async function handleComboResponses(
|
|
|
928
1025
|
if (!pick) {
|
|
929
1026
|
return comboUnavailableResponse(`No available targets for combo: ${comboId}`);
|
|
930
1027
|
}
|
|
1028
|
+
// One immutable combo selection trace, before any child dispatch; child
|
|
1029
|
+
// adoption below must never replace it with a concrete child route trace.
|
|
1030
|
+
logCtx.routeDecision = comboRouteDecisionTrace(config, comboId, pick, requestedModel);
|
|
931
1031
|
|
|
932
1032
|
let lastFailure: Response | null = null;
|
|
933
1033
|
while (pick) {
|
|
@@ -1031,6 +1131,7 @@ export async function handleComboResponses(
|
|
|
1031
1131
|
model: requestedModel,
|
|
1032
1132
|
provider: "combo",
|
|
1033
1133
|
comboId,
|
|
1134
|
+
routeDecision: logCtx.routeDecision,
|
|
1034
1135
|
attempts: logCtx.attempts,
|
|
1035
1136
|
activeAttempt: attempt,
|
|
1036
1137
|
activeAttemptStartedAt: started,
|
|
@@ -1143,6 +1244,32 @@ function finalizeOwnedTranslatorBudget(response: Response, budget: TranslatorBud
|
|
|
1143
1244
|
return finalizedResponse;
|
|
1144
1245
|
}
|
|
1145
1246
|
|
|
1247
|
+
/**
|
|
1248
|
+
* Service-tier capability gate, applied after the final route/wire is settled. A
|
|
1249
|
+
* provider explicitly documented as NOT supporting `service_tier` must never
|
|
1250
|
+
* receive it: strip the field and clear the logging value even when the caller
|
|
1251
|
+
* supplied one (fail closed). Tri-state contract: `true` supports (injection
|
|
1252
|
+
* allowed, caller values preserved), `false` strips, and an UNCLASSIFIED custom
|
|
1253
|
+
* provider (`undefined`) preserves caller-supplied values but never gets an
|
|
1254
|
+
* injection — deleting the caller's field there would silently change their
|
|
1255
|
+
* request against a gateway we know nothing about.
|
|
1256
|
+
*/
|
|
1257
|
+
export function applyServiceTierGate(
|
|
1258
|
+
provider: OcxProviderConfig,
|
|
1259
|
+
rawBody: unknown,
|
|
1260
|
+
options: { serviceTier?: string },
|
|
1261
|
+
): void {
|
|
1262
|
+
if (provider.adapter !== "openai-responses" || provider.supportsServiceTier !== false) return;
|
|
1263
|
+
if (rawBody && typeof rawBody === "object") {
|
|
1264
|
+
delete (rawBody as Record<string, unknown>).service_tier;
|
|
1265
|
+
}
|
|
1266
|
+
options.serviceTier = undefined;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
/**
|
|
1270
|
+
* Route one `/v1/responses` request through the adapter pipeline: recovery loop, passthrough
|
|
1271
|
+
* wire, image/web-search bridges, and the terminal-guard continuation.
|
|
1272
|
+
*/
|
|
1146
1273
|
export async function handleResponses(
|
|
1147
1274
|
req: Request,
|
|
1148
1275
|
config: OcxConfig,
|
|
@@ -1160,6 +1287,10 @@ export async function handleResponses(
|
|
|
1160
1287
|
}
|
|
1161
1288
|
}
|
|
1162
1289
|
|
|
1290
|
+
/**
|
|
1291
|
+
* Inner implementation of `handleResponses`; owns the pre-stream recovery loop and the
|
|
1292
|
+
* per-request same-target 429 retry budgets.
|
|
1293
|
+
*/
|
|
1163
1294
|
async function handleResponsesInner(
|
|
1164
1295
|
req: Request,
|
|
1165
1296
|
config: OcxConfig,
|
|
@@ -1247,7 +1378,11 @@ async function handleResponsesInner(
|
|
|
1247
1378
|
// Shadow call intercept: rewrite Codex's hard-coded helper calls
|
|
1248
1379
|
// (gpt-5.4-mini on older clients, gpt-5.6-luna on 0.145.0+)
|
|
1249
1380
|
const _sci = config.shadowCallIntercept;
|
|
1250
|
-
if (_sci?.enabled && _sci.model &&
|
|
1381
|
+
if (_sci?.enabled && _sci.model && shouldInterceptShadowCall(
|
|
1382
|
+
parsed.modelId,
|
|
1383
|
+
_sci.sourceModels,
|
|
1384
|
+
req.headers,
|
|
1385
|
+
)) {
|
|
1251
1386
|
const _sciOriginal = parsed.modelId;
|
|
1252
1387
|
parsed.modelId = _sci.model;
|
|
1253
1388
|
if (parsed._rawBody && typeof parsed._rawBody === "object") {
|
|
@@ -1266,37 +1401,63 @@ async function handleResponsesInner(
|
|
|
1266
1401
|
|
|
1267
1402
|
let route: RouteResult;
|
|
1268
1403
|
try {
|
|
1269
|
-
route = routeModel(config, parsed.modelId);
|
|
1404
|
+
route = routeModel(config, parsed.modelId, evidenceFromBody(parsed._rawBody));
|
|
1405
|
+
logCtx.routeDecision = route.routeDecision;
|
|
1270
1406
|
} catch (err) {
|
|
1271
1407
|
if (err instanceof NoAvailableComboTargetsError) {
|
|
1272
1408
|
return comboUnavailableResponse(err.message);
|
|
1273
1409
|
}
|
|
1410
|
+
if (err instanceof NoEligiblePolicyCandidateError) {
|
|
1411
|
+
// Persist the evaluation trace (per-candidate exclusions + the
|
|
1412
|
+
// no-eligible reason) so failed policy requests stay auditable.
|
|
1413
|
+
logCtx.routeDecision = err.trace;
|
|
1414
|
+
}
|
|
1274
1415
|
return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err));
|
|
1275
1416
|
}
|
|
1276
1417
|
|
|
1277
1418
|
const hasUnexpandedPreviousResponse = !!parsed.previousResponseId
|
|
1278
1419
|
&& parsed._previousResponseInputExpanded !== true;
|
|
1279
|
-
//
|
|
1280
|
-
// Cached fallback state can still select a
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1420
|
+
// Exact account selectors are isolated from Pool-wide quota work. A canonical replay miss must
|
|
1421
|
+
// also fail closed without polling quota upstream. Cached fallback state can still select a
|
|
1422
|
+
// provider with native continuation support below.
|
|
1423
|
+
const threadSpawn = isThreadSpawnRequest(req.headers);
|
|
1424
|
+
const previewSelectionAdmission = threadSpawn && route.codexAccountId === undefined
|
|
1425
|
+
? codexAccountSelectionForTurn(options.turnAdmissionLease)?.()
|
|
1426
|
+
: undefined;
|
|
1427
|
+
const nativeMainRecoveryBlocked = isNativeMainTrafficBlocked();
|
|
1428
|
+
const nativeMainReadsForbidden = nativeMainRecoveryBlocked
|
|
1429
|
+
|| previewSelectionAdmission?.mainProfileDraining === true;
|
|
1430
|
+
const previewSelectionOptions = {
|
|
1431
|
+
nativeMainSelectionOnly: !nativeMainRecoveryBlocked
|
|
1432
|
+
&& previewSelectionAdmission?.mainProfileDraining === true,
|
|
1433
|
+
};
|
|
1288
1434
|
let authCtx: CodexAuthContext = { kind: "main", accountId: null };
|
|
1289
1435
|
let selectedForwardHeaders = req.headers;
|
|
1290
1436
|
let subagentFallbackAccountId = config.activeCodexAccountId ?? null;
|
|
1291
1437
|
let subagentQuotaFailureModel = parsed.modelId;
|
|
1292
1438
|
|
|
1439
|
+
try {
|
|
1440
|
+
if (
|
|
1441
|
+
threadSpawn
|
|
1442
|
+
&& route.codexAccountId === undefined
|
|
1443
|
+
&& !(hasUnexpandedPreviousResponse && isCanonicalOpenAiForwardProvider(route.provider))
|
|
1444
|
+
) {
|
|
1445
|
+
await maybePrimeSubagentQuota(config, Date.now(), { nativeMainReadsForbidden });
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1293
1448
|
// Subagent fallback must settle the final model/provider BEFORE route-dependent
|
|
1294
1449
|
// normalization (virtual models, effort caps, service tier, wire protocol).
|
|
1295
1450
|
// Preview the preferred Codex account without acquiring a probe lease or refreshing
|
|
1296
1451
|
// tokens — auth is resolved only after the final route is selected.
|
|
1297
|
-
if (
|
|
1452
|
+
if (threadSpawn && !options.comboAttempt && route.codexAccountId === undefined) {
|
|
1298
1453
|
const threadId = req.headers.get("x-codex-parent-thread-id");
|
|
1299
|
-
const previewAccountId = previewCodexAccountForRequest(
|
|
1454
|
+
const previewAccountId = previewCodexAccountForRequest(
|
|
1455
|
+
threadId,
|
|
1456
|
+
config,
|
|
1457
|
+
Date.now(),
|
|
1458
|
+
undefined,
|
|
1459
|
+
previewSelectionOptions,
|
|
1460
|
+
);
|
|
1300
1461
|
subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null;
|
|
1301
1462
|
const fallback = applySubagentModelFallback(
|
|
1302
1463
|
parsed,
|
|
@@ -1305,6 +1466,7 @@ async function handleResponsesInner(
|
|
|
1305
1466
|
previewAccountId,
|
|
1306
1467
|
Date.now(),
|
|
1307
1468
|
unreadableEncryptedAgentTask,
|
|
1469
|
+
previewSelectionOptions,
|
|
1308
1470
|
);
|
|
1309
1471
|
if (fallback) {
|
|
1310
1472
|
(logCtx as unknown as Record<string, unknown>).subagentModelFallbackFrom = fallback.from;
|
|
@@ -1317,15 +1479,22 @@ async function handleResponsesInner(
|
|
|
1317
1479
|
|
|
1318
1480
|
if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) {
|
|
1319
1481
|
try {
|
|
1320
|
-
route = routeModel(config, fallback.to);
|
|
1482
|
+
route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody));
|
|
1483
|
+
logCtx.routeDecision = route.routeDecision;
|
|
1321
1484
|
} catch (err) {
|
|
1322
1485
|
if (err instanceof NoAvailableComboTargetsError) {
|
|
1323
1486
|
return comboUnavailableResponse(err.message);
|
|
1324
1487
|
}
|
|
1488
|
+
if (err instanceof NoEligiblePolicyCandidateError) {
|
|
1489
|
+
logCtx.routeDecision = err.trace;
|
|
1490
|
+
}
|
|
1325
1491
|
return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err));
|
|
1326
1492
|
}
|
|
1327
1493
|
}
|
|
1328
1494
|
}
|
|
1495
|
+
} finally {
|
|
1496
|
+
previewSelectionAdmission?.release();
|
|
1497
|
+
}
|
|
1329
1498
|
|
|
1330
1499
|
// Encrypted child tasks may only reach the canonical native backend. This check
|
|
1331
1500
|
// runs against the FINAL route so native-only fallback can rescue a routed primary.
|
|
@@ -1347,7 +1516,25 @@ async function handleResponsesInner(
|
|
|
1347
1516
|
);
|
|
1348
1517
|
}
|
|
1349
1518
|
|
|
1350
|
-
|
|
1519
|
+
// Captured before normalization: whether the CLIENT asked for SSE. The
|
|
1520
|
+
// transport-neutral upstream-streaming policy below may force a bounded JSON
|
|
1521
|
+
// upstream for reliability (#875); the answer must then be reframed to SSE
|
|
1522
|
+
// for streaming clients.
|
|
1523
|
+
const clientRequestedStream = parsed.stream;
|
|
1524
|
+
await applyFinalRouteRequestNormalization({
|
|
1525
|
+
parsed,
|
|
1526
|
+
route,
|
|
1527
|
+
config,
|
|
1528
|
+
req,
|
|
1529
|
+
logCtx,
|
|
1530
|
+
inboundWire,
|
|
1531
|
+
inboundTransport: options.inboundTransport,
|
|
1532
|
+
});
|
|
1533
|
+
// Attribute local auth/cooldown failures to the public selector too; exact auth may fail before
|
|
1534
|
+
// the normal post-resolution provider label is assigned.
|
|
1535
|
+
if (route.codexAccountNamespace) {
|
|
1536
|
+
logCtx.provider = `${route.providerName}-${route.codexAccountNamespace}`;
|
|
1537
|
+
}
|
|
1351
1538
|
|
|
1352
1539
|
{
|
|
1353
1540
|
const finalAuth = await resolveResponsesCodexAuth(req, config, route, options);
|
|
@@ -1357,7 +1544,9 @@ async function handleResponsesInner(
|
|
|
1357
1544
|
}
|
|
1358
1545
|
|
|
1359
1546
|
route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode);
|
|
1360
|
-
logCtx.provider =
|
|
1547
|
+
logCtx.provider = route.codexAccountNamespace
|
|
1548
|
+
? `${route.providerName}-${route.codexAccountNamespace}`
|
|
1549
|
+
: formatCodexProviderForLog(route.providerName, codexLogAccountId(authCtx), config);
|
|
1361
1550
|
// Prefer Codex pool account as the Cursor thread namespace when present. Cursor routes without
|
|
1362
1551
|
// codexAccountMode still get a credential-derived scope inside the Cursor adapter.
|
|
1363
1552
|
const identityScope = codexLogAccountId(authCtx);
|
|
@@ -1460,6 +1649,14 @@ async function handleResponsesInner(
|
|
|
1460
1649
|
listOpenAiForwardSidecarCandidates(config),
|
|
1461
1650
|
req.headers,
|
|
1462
1651
|
config,
|
|
1652
|
+
{
|
|
1653
|
+
// Account-qualified native routes are passthrough, so their in-turn helper is vision.
|
|
1654
|
+
// Scope its cooldown and outcome to the helper model, not the routed text model.
|
|
1655
|
+
...(route.codexAccountId !== undefined
|
|
1656
|
+
? { exactAccount: { accountId: route.codexAccountId, modelId: resolveOpenAiVisionModel(config) } }
|
|
1657
|
+
: {}),
|
|
1658
|
+
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
|
|
1659
|
+
},
|
|
1463
1660
|
);
|
|
1464
1661
|
} catch (err) {
|
|
1465
1662
|
// Sidecars are optional helpers for an otherwise independent routed turn.
|
|
@@ -1470,6 +1667,7 @@ async function handleResponsesInner(
|
|
|
1470
1667
|
&& !(err instanceof CodexAuthContextError)
|
|
1471
1668
|
&& !(err instanceof CodexAccountCooldownError)
|
|
1472
1669
|
&& !(err instanceof CodexThreadAffinityExpiredError)
|
|
1670
|
+
&& !(err instanceof CodexMainProfileDrainingError)
|
|
1473
1671
|
) throw err;
|
|
1474
1672
|
}
|
|
1475
1673
|
}
|
|
@@ -1581,10 +1779,20 @@ async function handleResponsesInner(
|
|
|
1581
1779
|
const transportFailureResponse = (err: unknown): Response => {
|
|
1582
1780
|
upstream.abort();
|
|
1583
1781
|
if (options.abortSignal?.aborted) return clientCancelledResponse();
|
|
1584
|
-
const outcome = err
|
|
1782
|
+
const outcome = classifyTransportFailureKind(err);
|
|
1783
|
+
// Host-level evidence stands regardless of pool membership: a direct
|
|
1784
|
+
// forward send has no pool accounting, but the reachability failure is
|
|
1785
|
+
// still host-wide, not account evidence (#914 review).
|
|
1786
|
+
if (outcome === "connect_neutral") {
|
|
1787
|
+
recordUpstreamHostFailure(
|
|
1788
|
+
upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)),
|
|
1789
|
+
{ code: transportErrorCode(err) },
|
|
1790
|
+
);
|
|
1791
|
+
}
|
|
1585
1792
|
if (usesCodexForwardPoolAuth(authCtx, route.provider)) {
|
|
1586
1793
|
recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
|
|
1587
1794
|
threadId: req.headers.get("x-codex-parent-thread-id"),
|
|
1795
|
+
fixedAccount: authCtx.fixedAccount,
|
|
1588
1796
|
modelId: route.modelId,
|
|
1589
1797
|
probeLeaseId: codexProbeLeaseId(authCtx),
|
|
1590
1798
|
probeQuotaScope: codexProbeQuotaScope(authCtx),
|
|
@@ -1607,7 +1815,14 @@ async function handleResponsesInner(
|
|
|
1607
1815
|
method: request.method,
|
|
1608
1816
|
headers: request.headers,
|
|
1609
1817
|
body: request.body,
|
|
1610
|
-
}, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)
|
|
1818
|
+
}, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider),
|
|
1819
|
+
route.provider.authMode === "forward")
|
|
1820
|
+
// Every real attempt response — including an intermediate 5xx the
|
|
1821
|
+
// retry wrapper replaces — proves the host was reached (#914 review).
|
|
1822
|
+
.then(res => {
|
|
1823
|
+
resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)));
|
|
1824
|
+
return res;
|
|
1825
|
+
});
|
|
1611
1826
|
},
|
|
1612
1827
|
{ abortSignal: upstream.signal, label: safeHostLabel(request.url) },
|
|
1613
1828
|
);
|
|
@@ -1617,7 +1832,65 @@ async function handleResponsesInner(
|
|
|
1617
1832
|
request.releaseBodyObservation?.();
|
|
1618
1833
|
}
|
|
1619
1834
|
|
|
1620
|
-
|
|
1835
|
+
// Same-target 429 wait-and-retry (opt-in `retryOn429`) for key-auth providers on the
|
|
1836
|
+
// passthrough wire. This branch returns before the recovery loop below, so Responses-shaped
|
|
1837
|
+
// key-auth gateways (e.g. the built-in DeepSeek preset) would otherwise surface 429
|
|
1838
|
+
// immediately with no same-key replay. Pre-stream only — nothing has been relayed yet, so
|
|
1839
|
+
// the replay is lossless (same invariant as the recovery loop). Forward/OAuth providers
|
|
1840
|
+
// keep their pool logic below (rateLimitRetryPolicyFor returns null for them).
|
|
1841
|
+
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
|
|
1842
|
+
let rateLimitRetries = 0;
|
|
1843
|
+
while (
|
|
1844
|
+
upstreamResponse.status === 429
|
|
1845
|
+
&& rateLimitPolicy !== null
|
|
1846
|
+
&& rateLimitRetries < rateLimitPolicy.attempts
|
|
1847
|
+
) {
|
|
1848
|
+
rateLimitRetries += 1;
|
|
1849
|
+
// Release unread body + deliberate wait via the shared same-target helper.
|
|
1850
|
+
const retryAfterHeader = upstreamResponse.headers.get("retry-after");
|
|
1851
|
+
try {
|
|
1852
|
+
for await (const _ of prepareSameTarget429Wait({
|
|
1853
|
+
body: upstreamResponse.body,
|
|
1854
|
+
signal: options.abortSignal,
|
|
1855
|
+
delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
|
|
1856
|
+
})) {
|
|
1857
|
+
// pre-stream: no stall watchdog to feed
|
|
1858
|
+
}
|
|
1859
|
+
} catch {
|
|
1860
|
+
upstream.abort();
|
|
1861
|
+
return clientCancelledResponse();
|
|
1862
|
+
}
|
|
1863
|
+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
|
|
1864
|
+
// replay so the wire never starts work for a request the client already abandoned.
|
|
1865
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
1866
|
+
upstream.abort();
|
|
1867
|
+
return clientCancelledResponse();
|
|
1868
|
+
}
|
|
1869
|
+
try {
|
|
1870
|
+
upstreamResponse = await fetchWithTransientRetry(
|
|
1871
|
+
recovery => {
|
|
1872
|
+
// The first send of every replay is itself a rate-limit retry; inner transient-5xx
|
|
1873
|
+
// recoveries keep their own label (recovery is provided for those).
|
|
1874
|
+
noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery ?? "rate-limit-429");
|
|
1875
|
+
return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({
|
|
1876
|
+
method: request.method,
|
|
1877
|
+
headers: request.headers,
|
|
1878
|
+
body: request.body,
|
|
1879
|
+
}, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider),
|
|
1880
|
+
route.provider.authMode === "forward")
|
|
1881
|
+
.then(res => {
|
|
1882
|
+
resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)));
|
|
1883
|
+
return res;
|
|
1884
|
+
});
|
|
1885
|
+
},
|
|
1886
|
+
{ abortSignal: upstream.signal, label: safeHostLabel(request.url) },
|
|
1887
|
+
);
|
|
1888
|
+
} catch (err) {
|
|
1889
|
+
return transportFailureResponse(err);
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
if (usesCodexForwardPoolAuth(authCtx, route.provider) && !authCtx.fixedAccount) {
|
|
1621
1894
|
let poolRetryOutcome: number | undefined;
|
|
1622
1895
|
if (await shouldRetryCodexPoolAccountModel400(
|
|
1623
1896
|
upstreamResponse,
|
|
@@ -1701,7 +1974,7 @@ async function handleResponsesInner(
|
|
|
1701
1974
|
|| logCtx.terminalHttpStatus === 402
|
|
1702
1975
|
? (httpStatusOverride ?? logCtx.terminalHttpStatus)
|
|
1703
1976
|
: undefined;
|
|
1704
|
-
if (quotaFailureMessage !== undefined) {
|
|
1977
|
+
if (!isFixedCodexAccount(authCtx) && quotaFailureMessage !== undefined) {
|
|
1705
1978
|
recordSubagentQuotaFailureForThreadSpawn(
|
|
1706
1979
|
req.headers,
|
|
1707
1980
|
subagentQuotaFailureModel,
|
|
@@ -1720,6 +1993,7 @@ async function handleResponsesInner(
|
|
|
1720
1993
|
recordCodexUpstreamOutcome(config, authCtx.accountId, upstreamResponse.status, {
|
|
1721
1994
|
...quotaMeta,
|
|
1722
1995
|
threadId: req.headers.get("x-codex-parent-thread-id"),
|
|
1996
|
+
fixedAccount: authCtx.fixedAccount,
|
|
1723
1997
|
modelId: route.modelId,
|
|
1724
1998
|
probeLeaseId: codexProbeLeaseId(authCtx),
|
|
1725
1999
|
probeQuotaScope: codexProbeQuotaScope(authCtx),
|
|
@@ -1732,6 +2006,17 @@ async function handleResponsesInner(
|
|
|
1732
2006
|
// Codex renders that as the opaque "Unknown error" (#452). Combo attempts
|
|
1733
2007
|
// keep their typed failure envelope. Non-empty bodies are relayed verbatim
|
|
1734
2008
|
// (headers included) so pool-retry Activation B/D and client diagnostics stay intact.
|
|
2009
|
+
// Manual-redirect policy (#914): a 3xx is relayed as-is (Location preserved
|
|
2010
|
+
// through sanitizePassthroughHeaders) so a redirect to a dead host can never
|
|
2011
|
+
// masquerade as a pre-connection failure after the credential was seen.
|
|
2012
|
+
// The numeric outcome above already classified it neutral — no streak.
|
|
2013
|
+
if (upstreamResponse.status >= 300 && upstreamResponse.status < 400) {
|
|
2014
|
+
return new Response(upstreamResponse.body, {
|
|
2015
|
+
status: upstreamResponse.status,
|
|
2016
|
+
statusText: upstreamResponse.statusText,
|
|
2017
|
+
headers: sanitizePassthroughHeaders(upstreamResponse.headers),
|
|
2018
|
+
});
|
|
2019
|
+
}
|
|
1735
2020
|
if (!upstreamResponse.ok) {
|
|
1736
2021
|
if (options.comboAttempt) {
|
|
1737
2022
|
const failure = await consumeComboFailure(upstreamResponse, options.abortSignal);
|
|
@@ -1758,7 +2043,8 @@ async function handleResponsesInner(
|
|
|
1758
2043
|
// The bundled known-bad runtime remains on tee by default on both platforms.
|
|
1759
2044
|
if (isEventStream && upstreamResponse.body) {
|
|
1760
2045
|
const repairConfig = route.provider.responsesItemIdRepair;
|
|
1761
|
-
const
|
|
2046
|
+
const snapshotRepairEnabled = hasResponsesSnapshotRepair(route.provider.responsesSnapshotRepair);
|
|
2047
|
+
const needsClientRewrite = imageGenCallAliases.size > 0 || hasResponsesItemIdRepair(repairConfig) || snapshotRepairEnabled;
|
|
1762
2048
|
// Compose opt-in payload rewrites into one parse/stringify pass (image-gen restore first).
|
|
1763
2049
|
const payloadRewrites = [
|
|
1764
2050
|
createImageGenCallRestoreRewrite(imageGenCallAliases),
|
|
@@ -1766,6 +2052,23 @@ async function handleResponsesInner(
|
|
|
1766
2052
|
? createResponsesItemIdPayloadRewrite(repairConfig!, translatorBudget)
|
|
1767
2053
|
: undefined,
|
|
1768
2054
|
].filter((rewrite): rewrite is NonNullable<typeof rewrite> => rewrite !== undefined);
|
|
2055
|
+
// #893: sparse-snapshot gateways get field backfills AND lifecycle event
|
|
2056
|
+
// injection at the block level, after payload rewrites. Defaults come
|
|
2057
|
+
// from the finalized OUTBOUND body — the normalized internal tool shapes
|
|
2058
|
+
// are not the Responses wire shapes the snapshot must mirror.
|
|
2059
|
+
const snapshotDefaultsRequest = (() => {
|
|
2060
|
+
try {
|
|
2061
|
+
return JSON.parse(request.body) as unknown;
|
|
2062
|
+
} catch {
|
|
2063
|
+
return undefined;
|
|
2064
|
+
}
|
|
2065
|
+
})();
|
|
2066
|
+
const clientBlockRewrite = snapshotRepairEnabled
|
|
2067
|
+
? composeSseBlockRewrites(
|
|
2068
|
+
payloadRewriteAsBlockRewrite(composeSsePayloadRewrites(...payloadRewrites)),
|
|
2069
|
+
createResponsesSnapshotBlockRewrite(snapshotDefaultsRequest, translatorBudget),
|
|
2070
|
+
)
|
|
2071
|
+
: undefined;
|
|
1769
2072
|
// #864: win32 rewrite traffic must never enter the tee()+JS-pull chain
|
|
1770
2073
|
// (Bun#32111 JS-sink segfault — text frames pass, the terminal block is
|
|
1771
2074
|
// lost). The eager single reader applies the same rewrites inline.
|
|
@@ -1788,7 +2091,7 @@ async function handleResponsesInner(
|
|
|
1788
2091
|
|| logCtx.terminalHttpStatus === 402
|
|
1789
2092
|
? (httpStatusOverride ?? logCtx.terminalHttpStatus)
|
|
1790
2093
|
: undefined;
|
|
1791
|
-
if (quotaFailureMessage !== undefined) {
|
|
2094
|
+
if (!isFixedCodexAccount(authCtx) && quotaFailureMessage !== undefined) {
|
|
1792
2095
|
recordSubagentQuotaFailureForThreadSpawn(
|
|
1793
2096
|
req.headers,
|
|
1794
2097
|
subagentQuotaFailureModel,
|
|
@@ -1811,10 +2114,15 @@ async function handleResponsesInner(
|
|
|
1811
2114
|
inspectChunk: chunk => inspector.feed(chunk),
|
|
1812
2115
|
finishInspection: () => inspector.finish(),
|
|
1813
2116
|
disposeInspection: () => inspector.dispose(),
|
|
1814
|
-
|
|
2117
|
+
// Stream lifetime follows the protocol terminal even when this request
|
|
2118
|
+
// has no outcome callback configured (reported() would stay false).
|
|
2119
|
+
sawTerminal: () => inspector.terminalSeen(),
|
|
1815
2120
|
...(win32EagerRewrite
|
|
1816
2121
|
? { rewritePayload: composeSsePayloadRewrites(...payloadRewrites) }
|
|
1817
2122
|
: {}),
|
|
2123
|
+
...(clientBlockRewrite
|
|
2124
|
+
? { rewriteBlocks: clientBlockRewrite }
|
|
2125
|
+
: {}),
|
|
1818
2126
|
onSynthetic: kind => {
|
|
1819
2127
|
if (!reportNativeTerminal) return;
|
|
1820
2128
|
if (kind === "incomplete") {
|
|
@@ -1829,9 +2137,10 @@ async function handleResponsesInner(
|
|
|
1829
2137
|
onClientCancel: () => options.onNativePassthroughCancel?.(),
|
|
1830
2138
|
onDone: () => unregisterTurn(turnAc),
|
|
1831
2139
|
}, win32EagerRewrite ? { rewriteBudget: translatorBudget } : undefined);
|
|
1832
|
-
//
|
|
1833
|
-
//
|
|
1834
|
-
//
|
|
2140
|
+
// When selected, this relay closes response.completed even if upstream
|
|
2141
|
+
// keeps the connection alive. Windows rewrite traffic applies its
|
|
2142
|
+
// payload transform inline — never via the Bun#32111-unsafe
|
|
2143
|
+
// tee()+JS-pull chain (#864).
|
|
1835
2144
|
if (!headers.has("content-type")) headers.set("content-type", "text/event-stream");
|
|
1836
2145
|
return markEagerRelaySseResponse(
|
|
1837
2146
|
markNativePassthroughSseResponse(new Response(eagerBody, {
|
|
@@ -1863,7 +2172,7 @@ async function handleResponsesInner(
|
|
|
1863
2172
|
|| logCtx.terminalHttpStatus === 402
|
|
1864
2173
|
? (httpStatusOverride ?? logCtx.terminalHttpStatus)
|
|
1865
2174
|
: undefined;
|
|
1866
|
-
if (quotaFailureMessage !== undefined) {
|
|
2175
|
+
if (!isFixedCodexAccount(authCtx) && quotaFailureMessage !== undefined) {
|
|
1867
2176
|
recordSubagentQuotaFailureForThreadSpawn(
|
|
1868
2177
|
req.headers,
|
|
1869
2178
|
subagentQuotaFailureModel,
|
|
@@ -1898,29 +2207,102 @@ async function handleResponsesInner(
|
|
|
1898
2207
|
);
|
|
1899
2208
|
}
|
|
1900
2209
|
if (!headers.has("content-type")) headers.set("content-type", "text/event-stream");
|
|
1901
|
-
//
|
|
1902
|
-
// relay
|
|
1903
|
-
// mid-stream reset
|
|
1904
|
-
const rewrittenBody = payloadRewrites.length > 0
|
|
1905
|
-
?
|
|
2210
|
+
// Windows was handled by the eager terminal-aware branch above. Remaining
|
|
2211
|
+
// tee traffic can use the JS relay to close on a protocol terminal and to
|
|
2212
|
+
// convert a mid-stream reset into a clean response.failed event.
|
|
2213
|
+
const rewrittenBody = clientBlockRewrite !== undefined || payloadRewrites.length > 0
|
|
2214
|
+
? relaySseWithBlockRewrite(nativeBody, clientBlockRewrite ?? payloadRewriteAsBlockRewrite(composeSsePayloadRewrites(...payloadRewrites)), translatorBudget)
|
|
1906
2215
|
: nativeBody;
|
|
1907
|
-
const clientBody =
|
|
1908
|
-
? nativeBody
|
|
1909
|
-
: relaySseWithFailedTail(rewrittenBody, upstream, reason => clientGone.abort(reason));
|
|
2216
|
+
const clientBody = relaySseWithFailedTail(rewrittenBody, upstream, reason => clientGone.abort(reason));
|
|
1910
2217
|
return markNativePassthroughSseResponse(new Response(clientBody, {
|
|
1911
2218
|
status: upstreamResponse.status,
|
|
1912
2219
|
headers,
|
|
1913
2220
|
}));
|
|
1914
2221
|
}
|
|
1915
2222
|
if (headers.get("content-type")?.toLowerCase().includes("application/json")) {
|
|
1916
|
-
|
|
2223
|
+
// Bounded whole-body read: a non-streaming upstream JSON body is fully materialized
|
|
2224
|
+
// here (and again by the request-log finalizer and the WebSocket bridge's reframing),
|
|
2225
|
+
// so an unbounded .text() would let a hostile or stuck upstream grow proxy memory
|
|
2226
|
+
// without limit. This path is no longer rare — WebSocket turns for models whose
|
|
2227
|
+
// streaming terminal event is unreliable are deliberately answered with bounded JSON.
|
|
2228
|
+
// Oversize and stall deadlines both fail closed; a partial body is never parsed.
|
|
2229
|
+
const bounded = await readBoundedResponseBody(upstreamResponse, {
|
|
2230
|
+
maxBytes: MAX_UPSTREAM_JSON_BODY_BYTES,
|
|
2231
|
+
totalTimeoutMs: UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS,
|
|
2232
|
+
inactivityTimeoutMs: UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS,
|
|
2233
|
+
});
|
|
2234
|
+
if (bounded.oversized) {
|
|
2235
|
+
return formatErrorResponse(502, "upstream_error", "upstream JSON response exceeded the safe body limit");
|
|
2236
|
+
}
|
|
2237
|
+
if (bounded.truncated) {
|
|
2238
|
+
return formatErrorResponse(502, "upstream_error", "upstream JSON response stalled before completing");
|
|
2239
|
+
}
|
|
2240
|
+
const text = bounded.text;
|
|
1917
2241
|
inspectResponseLogJson(logCtx, text);
|
|
1918
2242
|
if (rememberPassthroughResponse) {
|
|
1919
2243
|
try {
|
|
1920
2244
|
rememberPassthroughResponse(JSON.parse(text) as { id?: unknown; output?: unknown; status?: unknown });
|
|
1921
2245
|
} catch { /* non-JSON despite content-type; recording is best-effort */ }
|
|
1922
2246
|
}
|
|
1923
|
-
|
|
2247
|
+
const clientJson = (() => {
|
|
2248
|
+
const restored = restoreImageGenCallsInJson(text, imageGenCallAliases);
|
|
2249
|
+
if (!hasResponsesSnapshotRepair(route.provider.responsesSnapshotRepair)) return restored;
|
|
2250
|
+
let outbound: unknown;
|
|
2251
|
+
try {
|
|
2252
|
+
outbound = JSON.parse(request.body);
|
|
2253
|
+
} catch {
|
|
2254
|
+
outbound = undefined;
|
|
2255
|
+
}
|
|
2256
|
+
return repairResponsesSnapshotJson(restored, outbound);
|
|
2257
|
+
})();
|
|
2258
|
+
// #875: the transport-neutral reliability policy forced a bounded JSON
|
|
2259
|
+
// upstream for a client that asked for SSE. Reframe the completed JSON
|
|
2260
|
+
// as the canonical terminal SSE sequence (created → output_item.done →
|
|
2261
|
+
// terminal → [DONE]) so Codex commits the turn instead of hanging on a
|
|
2262
|
+
// stream that never closes. Non-streaming clients keep the plain JSON.
|
|
2263
|
+
if (clientRequestedStream === true
|
|
2264
|
+
&& options.inboundTransport !== "websocket"
|
|
2265
|
+
&& providerModelResponsesUpstreamStreaming(route.providerName, route.provider, route.modelId) === false
|
|
2266
|
+
&& route.provider.adapter === "openai-responses") {
|
|
2267
|
+
try {
|
|
2268
|
+
let completed = JSON.parse(clientJson) as Record<string, unknown>;
|
|
2269
|
+
// The bounded-JSON answer bypasses the SSE relay, so it also bypasses
|
|
2270
|
+
// the SSE item-id rewrite. Apply the same client-facing normalization
|
|
2271
|
+
// here or this policy would silently disable id repair for the very
|
|
2272
|
+
// providers that need it (raw record already happened above).
|
|
2273
|
+
if (hasResponsesItemIdRepair(route.provider.responsesItemIdRepair)) {
|
|
2274
|
+
completed = repairResponsesJsonItemIds(completed, route.provider.responsesItemIdRepair!, translatorBudget);
|
|
2275
|
+
}
|
|
2276
|
+
const sseHeaders = sanitizePassthroughHeaders(headers);
|
|
2277
|
+
sseHeaders.set("content-type", "text/event-stream");
|
|
2278
|
+
sseHeaders.set("cache-control", "no-store");
|
|
2279
|
+
return new Response(responsesJsonToSseBody(completed), {
|
|
2280
|
+
status: upstreamResponse.status,
|
|
2281
|
+
statusText: upstreamResponse.statusText,
|
|
2282
|
+
headers: sseHeaders,
|
|
2283
|
+
});
|
|
2284
|
+
} catch {
|
|
2285
|
+
// Non-JSON despite content-type: fall through to the plain relay.
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
// WS turns reframe this JSON into events in the bridge, which is the
|
|
2289
|
+
// other relay-free path — normalize ids so both bounded-JSON paths agree.
|
|
2290
|
+
const outboundJson = options.inboundTransport === "websocket"
|
|
2291
|
+
&& providerModelResponsesUpstreamStreaming(route.providerName, route.provider, route.modelId) === false
|
|
2292
|
+
&& hasResponsesItemIdRepair(route.provider.responsesItemIdRepair)
|
|
2293
|
+
? (() => {
|
|
2294
|
+
try {
|
|
2295
|
+
return JSON.stringify(repairResponsesJsonItemIds(
|
|
2296
|
+
JSON.parse(clientJson) as Record<string, unknown>,
|
|
2297
|
+
route.provider.responsesItemIdRepair!,
|
|
2298
|
+
translatorBudget,
|
|
2299
|
+
));
|
|
2300
|
+
} catch {
|
|
2301
|
+
return clientJson;
|
|
2302
|
+
}
|
|
2303
|
+
})()
|
|
2304
|
+
: clientJson;
|
|
2305
|
+
return new Response(outboundJson, {
|
|
1924
2306
|
status: upstreamResponse.status,
|
|
1925
2307
|
statusText: upstreamResponse.statusText,
|
|
1926
2308
|
headers,
|
|
@@ -2006,7 +2388,8 @@ async function handleResponsesInner(
|
|
|
2006
2388
|
...(imgPlan ? { plan: imgPlan } : {}),
|
|
2007
2389
|
...(vidPlan ? { videoPlan: vidPlan } : {}),
|
|
2008
2390
|
forwardHeaders: selectedForwardHeaders,
|
|
2009
|
-
onAttemptSend: () =>
|
|
2391
|
+
onAttemptSend: (recovery?: AttemptRecoveryKind) =>
|
|
2392
|
+
noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, recovery),
|
|
2010
2393
|
abortSignal: options.abortSignal,
|
|
2011
2394
|
maxRounds: imgPlan && vidPlan
|
|
2012
2395
|
? clampImageMaxRounds(Math.min(config.images?.maxRounds ?? 3, config.images?.videoMaxRounds ?? 2))
|
|
@@ -2045,6 +2428,7 @@ async function handleResponsesInner(
|
|
|
2045
2428
|
config.cacheRetention,
|
|
2046
2429
|
);
|
|
2047
2430
|
},
|
|
2431
|
+
retryOn429Policy: rateLimitRetryPolicyFor(route.provider),
|
|
2048
2432
|
...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}),
|
|
2049
2433
|
...(options.forceEmptyResponseId ? { forceEmptyResponseId: true } : {}),
|
|
2050
2434
|
onCompletedResponse: (response, providerState) =>
|
|
@@ -2073,7 +2457,6 @@ async function handleResponsesInner(
|
|
|
2073
2457
|
// through web-search instead of being swallowed. runTurn adapters never enter this branch.
|
|
2074
2458
|
if (canRunWebSearch && wsPlan) {
|
|
2075
2459
|
parsed.context.tools = [...(parsed.context.tools ?? []), buildWebSearchTool()];
|
|
2076
|
-
noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens);
|
|
2077
2460
|
const wsResponse = await runWithWebSearch({
|
|
2078
2461
|
parsed, adapter,
|
|
2079
2462
|
incomingMeta: { headers: selectedForwardHeaders, abortSignal: options.abortSignal, translatorBudget },
|
|
@@ -2088,6 +2471,8 @@ async function handleResponsesInner(
|
|
|
2088
2471
|
abortSignal: options.abortSignal,
|
|
2089
2472
|
...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}),
|
|
2090
2473
|
onRequestBuilt: request => recordAdapterReasoning(logCtx, request),
|
|
2474
|
+
onAttemptSend: (recovery?: AttemptRecoveryKind) =>
|
|
2475
|
+
noteAttemptSend(logCtx.activeAttempt, logCtx.usageLogInputTokens, recovery),
|
|
2091
2476
|
onUsage: usage => {
|
|
2092
2477
|
logCtx.usageFromBridge = true;
|
|
2093
2478
|
if (usage) {
|
|
@@ -2113,6 +2498,7 @@ async function handleResponsesInner(
|
|
|
2113
2498
|
config.cacheRetention,
|
|
2114
2499
|
);
|
|
2115
2500
|
},
|
|
2501
|
+
retryOn429Policy: rateLimitRetryPolicyFor(route.provider),
|
|
2116
2502
|
});
|
|
2117
2503
|
// Register the sidecar stream as an active turn so drainAndShutdown waits for (or aborts)
|
|
2118
2504
|
// in-flight web-search turns instead of skipping them during graceful shutdown.
|
|
@@ -2177,6 +2563,7 @@ async function handleResponsesInner(
|
|
|
2177
2563
|
}, 2_000,
|
|
2178
2564
|
{
|
|
2179
2565
|
translatorBudget,
|
|
2566
|
+
replayCacheScope: parsed._clientThreadId ?? "global",
|
|
2180
2567
|
...(options.forceEmptyResponseId ? { responseId: "" } : {}),
|
|
2181
2568
|
stallTimeoutSec: config.stallTimeoutSec,
|
|
2182
2569
|
hideThinkingSummary: parsed.options.hideThinkingSummary,
|
|
@@ -2223,6 +2610,7 @@ async function handleResponsesInner(
|
|
|
2223
2610
|
let providerState: OcxProviderContinuationState | undefined;
|
|
2224
2611
|
const json = buildResponseJSON(events, parsed.modelId, {
|
|
2225
2612
|
translatorBudget,
|
|
2613
|
+
replayCacheScope: parsed._clientThreadId ?? "global",
|
|
2226
2614
|
hideThinkingSummary: parsed.options.hideThinkingSummary,
|
|
2227
2615
|
toolNsMap,
|
|
2228
2616
|
freeformToolNames,
|
|
@@ -2251,19 +2639,58 @@ async function handleResponsesInner(
|
|
|
2251
2639
|
const upstream = new AbortController();
|
|
2252
2640
|
const cleanupUpstreamAbort = linkAbortSignal(upstream, options.abortSignal);
|
|
2253
2641
|
const connectMs = config.connectTimeoutMs ?? 200_000;
|
|
2642
|
+
// Bridge stall budget (seconds of silence before upstream_stall_timeout); the retry backoff
|
|
2643
|
+
// heartbeat interval is derived from it so the watchdog is always fed during deliberate waits.
|
|
2644
|
+
const stallTimeoutMs = typeof config.stallTimeoutSec === "number" && Number.isFinite(config.stallTimeoutSec) && config.stallTimeoutSec > 0
|
|
2645
|
+
? Math.floor(config.stallTimeoutSec * 1000)
|
|
2646
|
+
: 300_000;
|
|
2254
2647
|
let activeAdapter = adapter;
|
|
2255
2648
|
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2649
|
+
// One immutable, body-safe outbound request per same-target sequence (URL, serialized body,
|
|
2650
|
+
// auth headers, generated compat headers). Same-target 429 replays reuse it verbatim; the
|
|
2651
|
+
// builder runs again only after a key/account/adapter rotation, an oauth refresh, or an
|
|
2652
|
+
// image-tier bias change (transportToken bump). `body` is always a serialized string, so
|
|
2653
|
+
// reuse is safe, and releaseBodyObservation is idempotent per build.
|
|
2654
|
+
let initialRequest: AdapterRequest | undefined;
|
|
2655
|
+
let inputTokenEstimate: number | undefined;
|
|
2656
|
+
try {
|
|
2657
|
+
initialRequest = await activeAdapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget });
|
|
2658
|
+
recordAdapterReasoning(logCtx, initialRequest);
|
|
2659
|
+
inputTokenEstimate = typeof initialRequest.usageLog?.inputTokens === "number"
|
|
2660
|
+
? initialRequest.usageLog.inputTokens
|
|
2661
|
+
: undefined;
|
|
2662
|
+
if (inputTokenEstimate !== undefined) logCtx.usageLogInputTokens = inputTokenEstimate;
|
|
2663
|
+
} catch (err) {
|
|
2664
|
+
// A throwing buildRequest never returned a request; if a post-build step threw, release
|
|
2665
|
+
// the serialized-body observation (idempotent) so the translator budget is not leaked.
|
|
2666
|
+
// The build runs after linkAbortSignal, so a failure must also tear the link down and
|
|
2667
|
+
// abort the upstream controller instead of escaping handleResponses unmapped.
|
|
2668
|
+
initialRequest?.releaseBodyObservation?.();
|
|
2669
|
+
cleanupUpstreamAbort();
|
|
2670
|
+
upstream.abort();
|
|
2671
|
+
if (options.abortSignal?.aborted) return clientCancelledResponse();
|
|
2672
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2673
|
+
return formatErrorResponse(400, "invalid_request_error", redactSecretString(msg));
|
|
2674
|
+
}
|
|
2675
|
+
// The catch path above always returns, so the request is definitely assigned here.
|
|
2676
|
+
// Capture it in a const so the fetch callbacks read a narrowed, immutable value
|
|
2677
|
+
// (TypeScript drops narrowing for a `let` captured by a nested function).
|
|
2678
|
+
const builtInitialRequest = initialRequest;
|
|
2679
|
+
let sameTargetRequest: AdapterRequest | undefined = builtInitialRequest;
|
|
2680
|
+
let sameTargetParsed: OcxParsedRequest | undefined = parsed;
|
|
2681
|
+
let sameTargetToken = 0;
|
|
2682
|
+
let transportToken = 0;
|
|
2683
|
+
/**
|
|
2684
|
+
* Invalidate the same-target request cache. Every credential/adapter/parsed mutation MUST
|
|
2685
|
+
* go through here: the cache keys on `parsed` REFERENCE identity, so an in-place mutation
|
|
2686
|
+
* is invisible to it and a missed bump would replay a request built with a stale key.
|
|
2687
|
+
*/
|
|
2688
|
+
const invalidateSameTargetRequest = (): void => { transportToken += 1; };
|
|
2262
2689
|
let upstreamResponse: Response;
|
|
2263
2690
|
try {
|
|
2264
2691
|
if (activeAdapter.fetchResponse) {
|
|
2265
2692
|
noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate);
|
|
2266
|
-
upstreamResponse = await activeAdapter.fetchResponse(
|
|
2693
|
+
upstreamResponse = await activeAdapter.fetchResponse(builtInitialRequest, {
|
|
2267
2694
|
abortSignal: upstream.signal,
|
|
2268
2695
|
timeoutMs: connectMs,
|
|
2269
2696
|
stream: parsed.stream,
|
|
@@ -2272,13 +2699,13 @@ async function handleResponsesInner(
|
|
|
2272
2699
|
upstreamResponse = await fetchWithResetRetry(
|
|
2273
2700
|
recovery => {
|
|
2274
2701
|
noteAttemptSend(logCtx.activeAttempt, inputTokenEstimate, recovery);
|
|
2275
|
-
return fetchWithHeaderTimeout(
|
|
2276
|
-
method:
|
|
2277
|
-
headers:
|
|
2278
|
-
body:
|
|
2702
|
+
return fetchWithHeaderTimeout(builtInitialRequest.url, applyUpstreamRecoveryInit({
|
|
2703
|
+
method: builtInitialRequest.method,
|
|
2704
|
+
headers: builtInitialRequest.headers,
|
|
2705
|
+
body: builtInitialRequest.body,
|
|
2279
2706
|
}, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider));
|
|
2280
2707
|
},
|
|
2281
|
-
{ abortSignal: upstream.signal, label: safeHostLabel(
|
|
2708
|
+
{ abortSignal: upstream.signal, label: safeHostLabel(builtInitialRequest.url) },
|
|
2282
2709
|
);
|
|
2283
2710
|
}
|
|
2284
2711
|
} catch (err) {
|
|
@@ -2288,27 +2715,60 @@ async function handleResponsesInner(
|
|
|
2288
2715
|
const msg = describeUpstreamConnectFailure(err, connectMs);
|
|
2289
2716
|
return formatErrorResponse(502, "upstream_error", msg);
|
|
2290
2717
|
} finally {
|
|
2291
|
-
|
|
2718
|
+
builtInitialRequest.releaseBodyObservation?.();
|
|
2292
2719
|
}
|
|
2293
2720
|
|
|
2721
|
+
// Same-target 429 retry budget is per REQUEST: it lives OUTSIDE the recovery loop (so a 413/401
|
|
2722
|
+
// replay that comes back 429 cannot silently re-arm a fresh budget) and is SHARED with the
|
|
2723
|
+
// terminal-guard continuation below, so the main loop + one continuation can never exceed
|
|
2724
|
+
// `attempts` same-key replays in total (bounded per request).
|
|
2725
|
+
const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider);
|
|
2726
|
+
let rateLimitRetries = 0;
|
|
2727
|
+
// Shared with the terminal-guard continuation below: an image-tier reduction that let the
|
|
2728
|
+
// main request clear a 413 must not be forgotten on the very next continuation build.
|
|
2729
|
+
let imageTierBias = 0;
|
|
2294
2730
|
if (!upstreamResponse.ok) {
|
|
2295
2731
|
// Recovery loop: multi-key 429 failover + at most ONE anthropic 413 tightened retry
|
|
2296
2732
|
// (devlog/260714_image_normalization_pipeline/030). One mutable activeAdapter serves
|
|
2297
2733
|
// both paths so a 429→413 sequence never rebuilds against a stale pre-rotation
|
|
2298
2734
|
// adapter, and imageTierBias — once armed — rides EVERY subsequent rebuild so a
|
|
2299
2735
|
// 413→429 rotation cannot silently undo the tightening.
|
|
2300
|
-
let imageTierBias = 0;
|
|
2301
2736
|
let imageRetryAttempted = false;
|
|
2302
2737
|
let oauth401ReplayAttempted = false;
|
|
2738
|
+
/**
|
|
2739
|
+
* Rebuild the request from the current parsed input (and any image-tier bias) and refetch
|
|
2740
|
+
* it once, tagging the attempt with the given recovery kind. Rebuilds are deterministic
|
|
2741
|
+
* for the same parsed request, so same-target replays stay byte-identical.
|
|
2742
|
+
*/
|
|
2303
2743
|
const rebuildAndRefetch = async (
|
|
2304
2744
|
recovery: AttemptRecoveryKind,
|
|
2305
2745
|
): Promise<Response | { failed: Response }> => {
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
}
|
|
2311
|
-
|
|
2746
|
+
let retryRequest: AdapterRequest;
|
|
2747
|
+
if (sameTargetRequest !== undefined && sameTargetParsed === parsed && sameTargetToken === transportToken) {
|
|
2748
|
+
// Same target (key/adapter/parsed/tier unchanged): replay the exact cached request.
|
|
2749
|
+
retryRequest = sameTargetRequest;
|
|
2750
|
+
} else {
|
|
2751
|
+
try {
|
|
2752
|
+
retryRequest = await activeAdapter.buildRequest(parsed, {
|
|
2753
|
+
headers: selectedForwardHeaders,
|
|
2754
|
+
translatorBudget,
|
|
2755
|
+
...(imageTierBias > 0 ? { imageTierBias } : {}),
|
|
2756
|
+
});
|
|
2757
|
+
recordAdapterReasoning(logCtx, retryRequest);
|
|
2758
|
+
} catch (err) {
|
|
2759
|
+
// A rotated/rebuilt adapter build failure is a request-shaping error, not an
|
|
2760
|
+
// upstream connect failure: tear the abort link down and map it as 400 (no 413
|
|
2761
|
+
// translator-budget mapping here — that stays with parseRequest/buildToolBridgeMaps).
|
|
2762
|
+
cleanupUpstreamAbort();
|
|
2763
|
+
upstream.abort();
|
|
2764
|
+
if (options.abortSignal?.aborted) return { failed: clientCancelledResponse() };
|
|
2765
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2766
|
+
return { failed: formatErrorResponse(400, "invalid_request_error", redactSecretString(msg)) };
|
|
2767
|
+
}
|
|
2768
|
+
sameTargetRequest = retryRequest;
|
|
2769
|
+
sameTargetParsed = parsed;
|
|
2770
|
+
sameTargetToken = transportToken;
|
|
2771
|
+
}
|
|
2312
2772
|
const retryEstimate = typeof retryRequest.usageLog?.inputTokens === "number"
|
|
2313
2773
|
? retryRequest.usageLog.inputTokens
|
|
2314
2774
|
: undefined;
|
|
@@ -2363,6 +2823,7 @@ async function handleResponsesInner(
|
|
|
2363
2823
|
route.providerName === "github-copilot" ? getOAuthCredentialApiBaseUrl(route.providerName) : undefined,
|
|
2364
2824
|
);
|
|
2365
2825
|
route.provider = refreshedProvider;
|
|
2826
|
+
invalidateSameTargetRequest();
|
|
2366
2827
|
activeAdapter = resolveAdapter(
|
|
2367
2828
|
resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire),
|
|
2368
2829
|
config.cacheRetention,
|
|
@@ -2373,6 +2834,45 @@ async function handleResponsesInner(
|
|
|
2373
2834
|
continue recovery;
|
|
2374
2835
|
}
|
|
2375
2836
|
|
|
2837
|
+
// Same-target 429 wait-and-retry (opt-in `retryOn429`, issue #487). Codex never retries
|
|
2838
|
+
// 429 itself (it retries 5xx only), and single-key pools cannot use the failover below,
|
|
2839
|
+
// so wait (Retry-After or the fixed interval) and replay the IDENTICAL request on the
|
|
2840
|
+
// same key first. Pre-stream only: a 429 arrives before any bytes are relayed, so the
|
|
2841
|
+
// replay is lossless. Runs before key failover so "primary-first" setups keep the same
|
|
2842
|
+
// key on rate-limit blips; only after the attempts are exhausted does failover run.
|
|
2843
|
+
while (
|
|
2844
|
+
upstreamResponse.status === 429
|
|
2845
|
+
&& rateLimitPolicy !== null
|
|
2846
|
+
&& rateLimitRetries < rateLimitPolicy.attempts
|
|
2847
|
+
) {
|
|
2848
|
+
rateLimitRetries += 1;
|
|
2849
|
+
// Release unread body + deliberate wait via the shared same-target helper.
|
|
2850
|
+
const retryAfterHeader = upstreamResponse.headers.get("retry-after");
|
|
2851
|
+
try {
|
|
2852
|
+
for await (const _ of prepareSameTarget429Wait({
|
|
2853
|
+
body: upstreamResponse.body,
|
|
2854
|
+
signal: options.abortSignal,
|
|
2855
|
+
delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
|
|
2856
|
+
})) {
|
|
2857
|
+
// pre-stream: no stall watchdog to feed
|
|
2858
|
+
}
|
|
2859
|
+
} catch {
|
|
2860
|
+
cleanupUpstreamAbort();
|
|
2861
|
+
upstream.abort();
|
|
2862
|
+
return clientCancelledResponse();
|
|
2863
|
+
}
|
|
2864
|
+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
|
|
2865
|
+
// replay so an adapter never starts work for a request the client already abandoned.
|
|
2866
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
2867
|
+
cleanupUpstreamAbort();
|
|
2868
|
+
upstream.abort();
|
|
2869
|
+
return clientCancelledResponse();
|
|
2870
|
+
}
|
|
2871
|
+
const result = await rebuildAndRefetch("rate-limit-429");
|
|
2872
|
+
if ("failed" in result) return result.failed;
|
|
2873
|
+
upstreamResponse = result;
|
|
2874
|
+
}
|
|
2875
|
+
|
|
2376
2876
|
// Multi-key 429 failover: rotate to the next pool key (cooldown-aware) and retry the
|
|
2377
2877
|
// SAME request once per remaining key. OAuth/forward providers and single-key pools
|
|
2378
2878
|
// return null immediately, so this stays a no-op for them (src/providers/key-failover.ts).
|
|
@@ -2388,6 +2888,7 @@ async function handleResponsesInner(
|
|
|
2388
2888
|
// until runtime cleanup (one per rotated key under a rate-limit storm).
|
|
2389
2889
|
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
|
|
2390
2890
|
route.provider = rotated;
|
|
2891
|
+
invalidateSameTargetRequest();
|
|
2391
2892
|
activeAdapter = resolveAdapter(
|
|
2392
2893
|
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire),
|
|
2393
2894
|
config.cacheRetention,
|
|
@@ -2418,6 +2919,7 @@ async function handleResponsesInner(
|
|
|
2418
2919
|
anthropicPoolAccountId = nextAccountId;
|
|
2419
2920
|
anthropicPoolFailovers += 1;
|
|
2420
2921
|
route.provider = { ...route.provider, apiKey: accessToken };
|
|
2922
|
+
invalidateSameTargetRequest();
|
|
2421
2923
|
promoteAnthropicActiveAccount(nextAccountId);
|
|
2422
2924
|
logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config);
|
|
2423
2925
|
activeAdapter = resolveAdapter(
|
|
@@ -2442,6 +2944,7 @@ async function handleResponsesInner(
|
|
|
2442
2944
|
})) {
|
|
2443
2945
|
imageRetryAttempted = true;
|
|
2444
2946
|
imageTierBias = 1;
|
|
2947
|
+
invalidateSameTargetRequest();
|
|
2445
2948
|
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
|
|
2446
2949
|
const result = await rebuildAndRefetch("image-413");
|
|
2447
2950
|
if ("failed" in result) return result.failed;
|
|
@@ -2459,15 +2962,17 @@ async function handleResponsesInner(
|
|
|
2459
2962
|
}
|
|
2460
2963
|
const errorText = await upstreamResponse.text().catch(() => "unknown error");
|
|
2461
2964
|
cleanupUpstreamAbort();
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2965
|
+
if (!isFixedCodexAccount(authCtx)) {
|
|
2966
|
+
recordSubagentQuotaFailureForThreadSpawn(
|
|
2967
|
+
req.headers,
|
|
2968
|
+
subagentQuotaFailureModel,
|
|
2969
|
+
upstreamResponse.status === 429 || upstreamResponse.status === 402
|
|
2970
|
+
? upstreamResponse.status
|
|
2971
|
+
: `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`,
|
|
2972
|
+
config,
|
|
2973
|
+
subagentFallbackAccountId,
|
|
2974
|
+
);
|
|
2975
|
+
}
|
|
2471
2976
|
// Upstreams occasionally echo request details in error bodies — scrub token-shaped
|
|
2472
2977
|
// material before it reaches the client-facing error surface.
|
|
2473
2978
|
const message = `Provider error ${upstreamResponse.status}: ${redactSecretString(errorText.slice(0, 500))}`;
|
|
@@ -2484,64 +2989,149 @@ async function handleResponsesInner(
|
|
|
2484
2989
|
|
|
2485
2990
|
cancelBodyOnAbort(upstreamResponse.body, upstream.signal);
|
|
2486
2991
|
|
|
2487
|
-
//
|
|
2488
|
-
//
|
|
2489
|
-
// one internal continuation pass. A continuation failure becomes an in-stream adapter error so
|
|
2490
|
-
// the client never sees a second hidden HTTP response or an unbounded retry loop.
|
|
2992
|
+
// Anthropic-only: one bounded internal continuation re-ask for clean end_turn turns that
|
|
2993
|
+
// announced an edit without emitting a tool call.
|
|
2491
2994
|
const terminalGuardEnabled = activeAdapter.name === "anthropic" && !options.comboAttempt && !routedCompaction;
|
|
2995
|
+
/**
|
|
2996
|
+
* One bounded internal re-ask for Anthropic end_turn-without-tool-call turns. Replays the
|
|
2997
|
+
* continuation on a 429 with the same-key retry budget (hoisted per request), then falls
|
|
2998
|
+
* back to key/account failover; a failure becomes an in-stream adapter error so the client
|
|
2999
|
+
* never sees a second hidden HTTP response or an unbounded retry loop.
|
|
3000
|
+
*/
|
|
2492
3001
|
const fetchTerminalGuardContinuation = async function* (nextParsed: OcxParsedRequest): AsyncGenerator<AdapterEvent> {
|
|
2493
|
-
let imageTierBias = 0;
|
|
2494
3002
|
let response: Response | undefined;
|
|
3003
|
+
// One-shot recovery label for the next top-of-loop continuation send after a failover rotation.
|
|
3004
|
+
let nextContinuationRecoveryKind: AttemptRecoveryKind | undefined;
|
|
3005
|
+
/**
|
|
3006
|
+
* Build and fetch one terminal-guard continuation. `recoveryKind` tags same-target and
|
|
3007
|
+
* failover sends (`rate-limit-429`, `key-429`, `anthropic-oauth-429`, `image-413`); the
|
|
3008
|
+
* adapter rebuild is deterministic for the same parsed request (tests assert byte-identical
|
|
3009
|
+
* replays).
|
|
3010
|
+
*/
|
|
3011
|
+
const fetchContinuation = async (recoveryKind?: AttemptRecoveryKind): Promise<Response> => {
|
|
3012
|
+
let continuationRequest: AdapterRequest | undefined;
|
|
3013
|
+
if (sameTargetRequest !== undefined && sameTargetParsed === nextParsed && sameTargetToken === transportToken) {
|
|
3014
|
+
// Same target (key/adapter/parsed/tier unchanged): replay the exact cached request.
|
|
3015
|
+
continuationRequest = sameTargetRequest;
|
|
3016
|
+
} else {
|
|
3017
|
+
try {
|
|
3018
|
+
continuationRequest = await activeAdapter.buildRequest(nextParsed, {
|
|
3019
|
+
headers: selectedForwardHeaders,
|
|
3020
|
+
translatorBudget,
|
|
3021
|
+
...(imageTierBias > 0 ? { imageTierBias } : {}),
|
|
3022
|
+
});
|
|
3023
|
+
recordAdapterReasoning(logCtx, continuationRequest);
|
|
3024
|
+
} catch (err) {
|
|
3025
|
+
// The main body is already streaming, so there is no HTTP error surface: release
|
|
3026
|
+
// any partial body observation and surface the failure as an in-stream error via
|
|
3027
|
+
// the outer catch (no upstream.abort() — that would kill the live body stream).
|
|
3028
|
+
continuationRequest?.releaseBodyObservation?.();
|
|
3029
|
+
throw err;
|
|
3030
|
+
}
|
|
3031
|
+
sameTargetRequest = continuationRequest;
|
|
3032
|
+
sameTargetParsed = nextParsed;
|
|
3033
|
+
sameTargetToken = transportToken;
|
|
3034
|
+
}
|
|
3035
|
+
// Both branches assign the request (the build catch rethrows), so capture it in a
|
|
3036
|
+
// const for the fetch callback and finally below — a `let` read inside a nested
|
|
3037
|
+
// function keeps its undefined half, which would break the byte-identical replay.
|
|
3038
|
+
const builtContinuationRequest = continuationRequest;
|
|
3039
|
+
const continuationEstimate = typeof builtContinuationRequest.usageLog?.inputTokens === "number"
|
|
3040
|
+
? builtContinuationRequest.usageLog.inputTokens
|
|
3041
|
+
: undefined;
|
|
3042
|
+
if (continuationEstimate !== undefined) logCtx.usageLogInputTokens = continuationEstimate;
|
|
3043
|
+
// Optional recovery label for same-target / failover continuation sends.
|
|
3044
|
+
const replayKind: AttemptRecoveryKind | undefined = recoveryKind;
|
|
3045
|
+
try {
|
|
3046
|
+
if (activeAdapter.fetchResponse) {
|
|
3047
|
+
noteAttemptSend(logCtx.activeAttempt, continuationEstimate, replayKind);
|
|
3048
|
+
return await activeAdapter.fetchResponse(builtContinuationRequest, {
|
|
3049
|
+
abortSignal: upstream.signal,
|
|
3050
|
+
timeoutMs: connectMs,
|
|
3051
|
+
stream: nextParsed.stream,
|
|
3052
|
+
});
|
|
3053
|
+
}
|
|
3054
|
+
return await fetchWithResetRetry(
|
|
3055
|
+
recovery => {
|
|
3056
|
+
noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery ?? replayKind);
|
|
3057
|
+
return fetchWithHeaderTimeout(
|
|
3058
|
+
builtContinuationRequest.url,
|
|
3059
|
+
applyUpstreamRecoveryInit({
|
|
3060
|
+
method: builtContinuationRequest.method,
|
|
3061
|
+
headers: builtContinuationRequest.headers,
|
|
3062
|
+
body: builtContinuationRequest.body,
|
|
3063
|
+
}, recovery),
|
|
3064
|
+
upstream.signal,
|
|
3065
|
+
connectMs,
|
|
3066
|
+
nextParsed.stream,
|
|
3067
|
+
providerFetch(route.provider),
|
|
3068
|
+
);
|
|
3069
|
+
},
|
|
3070
|
+
{ abortSignal: upstream.signal, label: safeHostLabel(builtContinuationRequest.url) },
|
|
3071
|
+
);
|
|
3072
|
+
} finally {
|
|
3073
|
+
builtContinuationRequest.releaseBodyObservation?.();
|
|
3074
|
+
}
|
|
3075
|
+
};
|
|
2495
3076
|
while (true) {
|
|
2496
3077
|
try {
|
|
2497
|
-
const
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
?
|
|
2505
|
-
|
|
2506
|
-
|
|
3078
|
+
const recoveryKind = nextContinuationRecoveryKind;
|
|
3079
|
+
nextContinuationRecoveryKind = undefined;
|
|
3080
|
+
response = await fetchContinuation(recoveryKind);
|
|
3081
|
+
} catch (error) {
|
|
3082
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
3083
|
+
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
3084
|
+
} else {
|
|
3085
|
+
yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` };
|
|
3086
|
+
}
|
|
3087
|
+
return;
|
|
3088
|
+
}
|
|
3089
|
+
|
|
3090
|
+
// Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover:
|
|
3091
|
+
// a primary-key rate-limit blip replays on the SAME key, matching the main recovery
|
|
3092
|
+
// loop; only after the attempts are exhausted does the continuation fail over.
|
|
3093
|
+
while (
|
|
3094
|
+
response.status === 429
|
|
3095
|
+
&& rateLimitPolicy !== null
|
|
3096
|
+
&& rateLimitRetries < rateLimitPolicy.attempts
|
|
3097
|
+
) {
|
|
3098
|
+
rateLimitRetries += 1;
|
|
3099
|
+
// Release unread body + heartbeat-fed wait via the shared same-target helper.
|
|
3100
|
+
const retryAfterHeader = response.headers.get("retry-after");
|
|
2507
3101
|
try {
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
3102
|
+
yield* prepareSameTarget429Wait({
|
|
3103
|
+
body: response.body,
|
|
3104
|
+
// Listen on the upstream signal: once the SSE body is being streamed, a client
|
|
3105
|
+
// cancel aborts `upstream` through the bridge, and upstream is also linked from
|
|
3106
|
+
// options.abortSignal — so this covers both cancellation paths.
|
|
3107
|
+
signal: upstream.signal,
|
|
3108
|
+
delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
|
|
3109
|
+
heartbeatIntervalMs: Math.min(10_000, Math.max(250, stallTimeoutMs / 2)),
|
|
3110
|
+
});
|
|
3111
|
+
} catch {
|
|
3112
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
3113
|
+
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
2515
3114
|
} else {
|
|
2516
|
-
|
|
2517
|
-
recovery => {
|
|
2518
|
-
noteAttemptSend(logCtx.activeAttempt, continuationEstimate, recovery);
|
|
2519
|
-
return fetchWithHeaderTimeout(
|
|
2520
|
-
continuationRequest.url,
|
|
2521
|
-
applyUpstreamRecoveryInit({
|
|
2522
|
-
method: continuationRequest.method,
|
|
2523
|
-
headers: continuationRequest.headers,
|
|
2524
|
-
body: continuationRequest.body,
|
|
2525
|
-
}, recovery),
|
|
2526
|
-
upstream.signal,
|
|
2527
|
-
connectMs,
|
|
2528
|
-
nextParsed.stream,
|
|
2529
|
-
providerFetch(route.provider),
|
|
2530
|
-
);
|
|
2531
|
-
},
|
|
2532
|
-
{ abortSignal: upstream.signal, label: safeHostLabel(continuationRequest.url) },
|
|
2533
|
-
);
|
|
3115
|
+
yield { type: "error", message: "Provider continuation failed: retry wait interrupted" };
|
|
2534
3116
|
}
|
|
2535
|
-
|
|
2536
|
-
continuationRequest.releaseBodyObservation?.();
|
|
3117
|
+
return;
|
|
2537
3118
|
}
|
|
2538
|
-
|
|
2539
|
-
|
|
3119
|
+
// Client cancellation wins over any stale timer edge: re-check before dispatching the
|
|
3120
|
+
// replay so the continuation never starts work for a request the client abandoned.
|
|
3121
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
2540
3122
|
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
2541
|
-
|
|
2542
|
-
|
|
3123
|
+
return;
|
|
3124
|
+
}
|
|
3125
|
+
try {
|
|
3126
|
+
response = await fetchContinuation("rate-limit-429");
|
|
3127
|
+
} catch (error) {
|
|
3128
|
+
if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
3129
|
+
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
3130
|
+
} else {
|
|
3131
|
+
yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` };
|
|
3132
|
+
}
|
|
3133
|
+
return;
|
|
2543
3134
|
}
|
|
2544
|
-
return;
|
|
2545
3135
|
}
|
|
2546
3136
|
|
|
2547
3137
|
if (response.status === 429 && hasKeyPoolFailover(route.provider)) {
|
|
@@ -2554,10 +3144,12 @@ async function handleResponsesInner(
|
|
|
2554
3144
|
if (rotated) {
|
|
2555
3145
|
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
2556
3146
|
route.provider = rotated;
|
|
3147
|
+
invalidateSameTargetRequest();
|
|
2557
3148
|
activeAdapter = resolveAdapter(
|
|
2558
3149
|
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire),
|
|
2559
3150
|
config.cacheRetention,
|
|
2560
3151
|
);
|
|
3152
|
+
nextContinuationRecoveryKind = "key-429";
|
|
2561
3153
|
continue;
|
|
2562
3154
|
}
|
|
2563
3155
|
}
|
|
@@ -2580,6 +3172,7 @@ async function handleResponsesInner(
|
|
|
2580
3172
|
anthropicPoolAccountId = nextAccountId;
|
|
2581
3173
|
anthropicPoolFailovers += 1;
|
|
2582
3174
|
route.provider = { ...route.provider, apiKey: accessToken };
|
|
3175
|
+
invalidateSameTargetRequest();
|
|
2583
3176
|
promoteAnthropicActiveAccount(nextAccountId);
|
|
2584
3177
|
logCtx.provider = formatAnthropicProviderForLog("anthropic", nextAccountId, config);
|
|
2585
3178
|
activeAdapter = resolveAdapter(
|
|
@@ -2587,6 +3180,7 @@ async function handleResponsesInner(
|
|
|
2587
3180
|
config.cacheRetention,
|
|
2588
3181
|
);
|
|
2589
3182
|
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, activeAdapter.name);
|
|
3183
|
+
nextContinuationRecoveryKind = "anthropic-oauth-429";
|
|
2590
3184
|
continue;
|
|
2591
3185
|
} catch {
|
|
2592
3186
|
// fall through to emit continuation error below
|
|
@@ -2600,7 +3194,9 @@ async function handleResponsesInner(
|
|
|
2600
3194
|
alreadyAttempted: imageTierBias > 0,
|
|
2601
3195
|
})) {
|
|
2602
3196
|
imageTierBias = 1;
|
|
3197
|
+
invalidateSameTargetRequest();
|
|
2603
3198
|
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
3199
|
+
nextContinuationRecoveryKind = "image-413";
|
|
2604
3200
|
continue;
|
|
2605
3201
|
}
|
|
2606
3202
|
break;
|
|
@@ -2658,6 +3254,7 @@ async function handleResponsesInner(
|
|
|
2658
3254
|
() => upstream.abort(), 2_000,
|
|
2659
3255
|
{
|
|
2660
3256
|
translatorBudget,
|
|
3257
|
+
replayCacheScope: parsed._clientThreadId ?? "global",
|
|
2661
3258
|
...(options.forceEmptyResponseId ? { responseId: "" } : {}),
|
|
2662
3259
|
stallTimeoutSec: config.stallTimeoutSec,
|
|
2663
3260
|
hideThinkingSummary: parsed.options.hideThinkingSummary,
|
|
@@ -2715,6 +3312,7 @@ async function handleResponsesInner(
|
|
|
2715
3312
|
let providerState: OcxProviderContinuationState | undefined;
|
|
2716
3313
|
const json = buildResponseJSON(events, parsed.modelId, {
|
|
2717
3314
|
translatorBudget,
|
|
3315
|
+
replayCacheScope: parsed._clientThreadId ?? "global",
|
|
2718
3316
|
hideThinkingSummary: parsed.options.hideThinkingSummary,
|
|
2719
3317
|
toolNsMap,
|
|
2720
3318
|
freeformToolNames,
|