@bitkyc08/opencodex 2.57.0 → 2.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -10
- package/gui/dist/assets/index-C5IebErG.js +136 -0
- package/gui/dist/assets/{index-C5-RdDmD.css → index-OESInAjC.css} +1 -1
- package/gui/dist/index.html +2 -2
- package/gui/dist/provider-icons/crusoe.svg +1 -0
- package/gui/dist/provider-icons/opper.svg +3 -0
- package/package.json +2 -2
- package/src/adapters/base.ts +11 -1
- package/src/adapters/codebuddy/scaffold-guard.ts +5 -4
- package/src/adapters/command-code.ts +13 -4
- package/src/adapters/cursor/catalog.ts +11 -0
- package/src/adapters/cursor/cursor-errors.ts +15 -0
- package/src/adapters/cursor/discovery.ts +65 -1
- package/src/adapters/cursor/effort-map.ts +16 -2
- package/src/adapters/cursor/envelope-echo.ts +55 -2
- package/src/adapters/cursor/live-transport.ts +5 -1
- package/src/adapters/cursor/message-mapper.ts +3 -2
- package/src/adapters/cursor/protobuf-events.ts +110 -11
- package/src/adapters/cursor/protobuf-request.ts +27 -6
- package/src/adapters/cursor/request-builder.ts +14 -3
- package/src/adapters/cursor/text-toolcall.ts +230 -0
- package/src/adapters/cursor/thread-continuity.ts +141 -0
- package/src/adapters/cursor/tool-guidance.ts +5 -4
- package/src/adapters/cursor/types.ts +5 -0
- package/src/adapters/cursor.ts +97 -6
- package/src/adapters/devin/cloud-direct/chat.ts +11 -2
- package/src/adapters/devin/cloud-direct/index.ts +7 -0
- package/src/adapters/devin/cloud-direct/stated-reset-retry.ts +103 -0
- package/src/adapters/devin.ts +75 -13
- package/src/adapters/google-antigravity-wire.ts +29 -2
- package/src/adapters/google-http.ts +45 -13
- package/src/adapters/google.ts +23 -4
- package/src/adapters/mimo-free.ts +32 -17
- package/src/adapters/ollama-native.ts +42 -8
- package/src/adapters/openai-chat/response-events.ts +61 -0
- package/src/adapters/openai-chat.ts +5 -10
- package/src/adapters/openai-responses/passthrough.ts +40 -5
- package/src/adapters/openai-responses/request-strips.ts +43 -0
- package/src/adapters/openai-responses/tool-output-recovery.ts +75 -0
- package/src/adapters/openai-responses/tool-schema.ts +19 -7
- package/src/adapters/physical-send.ts +50 -0
- package/src/adapters/responses-tool-schema.ts +76 -46
- package/src/adapters/run-turn-queue.ts +17 -4
- package/src/bridge/response-json.ts +2 -2
- package/src/bridge/sse.ts +166 -25
- package/src/claude/context-windows.ts +22 -0
- package/src/claude/outbound.ts +46 -5
- package/src/cli/account-api.ts +4 -3
- package/src/cli/account-extended.ts +22 -2
- package/src/cli/account-orca-import.ts +63 -0
- package/src/cli/account.ts +32 -4
- package/src/cli/capabilities.ts +40 -0
- package/src/cli/claude.ts +29 -1
- package/src/cli/codex-cli-update.ts +97 -2
- package/src/cli/config-command.ts +35 -18
- package/src/cli/dispatch.ts +71 -4
- package/src/cli/doctor.ts +197 -2
- package/src/cli/help.ts +4 -1
- package/src/cli/index.ts +132 -22
- package/src/cli/models-runtime.ts +33 -4
- package/src/cli/registry.ts +11 -1
- package/src/cli/runtime-api.ts +44 -0
- package/src/cli/start-args.ts +94 -0
- package/src/cli/system-command.ts +72 -1
- package/src/cli/uninstall-client-state.ts +12 -0
- package/src/client/machine-api.ts +4 -3
- package/src/client/machine-listener.ts +14 -1
- package/src/clients/config-export/constants.ts +2 -3
- package/src/clients/config-export.ts +5 -5
- package/src/codex/account-store.ts +81 -5
- package/src/codex/auth-api/pool-quota-probe.ts +14 -3
- package/src/codex/auth-api/routes.ts +17 -2
- package/src/codex/auth-context.ts +58 -20
- package/src/codex/catalog/build-entries.ts +25 -4
- package/src/codex/catalog/derive-entry.ts +8 -1
- package/src/codex/catalog/effort.ts +10 -6
- package/src/codex/catalog/gather-capture.ts +1 -0
- package/src/codex/catalog/model-hints.ts +37 -5
- package/src/codex/catalog/parsing.ts +83 -5
- package/src/codex/catalog/reserve-warn.ts +96 -0
- package/src/codex/catalog/retained-sync.ts +19 -0
- package/src/codex/catalog/routed-gather.ts +42 -3
- package/src/codex/cli-installation-identity.ts +210 -0
- package/src/codex/cli-installation-targets.ts +158 -0
- package/src/codex/convergence.ts +5 -0
- package/src/codex/desktop-switches.ts +145 -0
- package/src/codex/history-job.ts +5 -1
- package/src/codex/history-provider.ts +37 -5
- package/src/codex/history-state-open.ts +105 -0
- package/src/codex/history-worker.ts +14 -1
- package/src/codex/inject/config-toml.ts +44 -2
- package/src/codex/inject/remove.ts +145 -7
- package/src/codex/inject/restore.ts +204 -32
- package/src/codex/inject.ts +6 -9
- package/src/codex/lineage.ts +83 -32
- package/src/codex/loopback-target.ts +40 -0
- package/src/codex/main-account-hard-lock.ts +2 -1
- package/src/codex/main-account.ts +10 -3
- package/src/codex/main-device-reauth.ts +17 -9
- package/src/codex/model-entitlements.ts +60 -1
- package/src/codex/native-profile-startup.ts +64 -20
- package/src/codex/observed-model-denials.ts +137 -0
- package/src/codex/orca-auth-source.ts +94 -0
- package/src/codex/orca-import.ts +219 -0
- package/src/codex/prompt-text-probe.ts +282 -12
- package/src/codex/quota-401-recovery.ts +12 -0
- package/src/codex/quota-types.ts +65 -0
- package/src/codex/quota.ts +24 -19
- package/src/codex/routing/cooldown-math.ts +8 -47
- package/src/codex/routing/pin-drain.ts +57 -0
- package/src/codex/routing.ts +13 -15
- package/src/codex/subagent-model-fallback.ts +94 -0
- package/src/codex/windows-installation-files.ts +224 -0
- package/src/combos/failover.ts +122 -5
- package/src/config/atomic-write.ts +83 -8
- package/src/config/diagnostics.ts +21 -0
- package/src/config/load-degrade.ts +15 -0
- package/src/config/pending-teardown.ts +8 -0
- package/src/config/process-state.ts +36 -3
- package/src/config/provider-relative-send-path.ts +16 -0
- package/src/config/proxy-env.ts +23 -5
- package/src/config/schema/config-schema.ts +23 -0
- package/src/config/schema/leaf-validators.ts +65 -17
- package/src/generated/compatibility-version.json +337 -201
- package/src/generated/model-metadata.ts +1 -1
- package/src/lib/bounded-body.ts +4 -2
- package/src/lib/bounded-subprocess.ts +62 -10
- package/src/lib/destination-policy.ts +48 -6
- package/src/lib/errors.ts +3 -15
- package/src/lib/local-destinations.ts +32 -5
- package/src/lib/provider-outbound.ts +3 -3
- package/src/lib/proxy-env.ts +70 -3
- package/src/lib/request-execution-budget.ts +11 -3
- package/src/lib/response-body-inactivity.ts +193 -0
- package/src/lib/retry-delay.ts +69 -0
- package/src/lib/socks5-fetch.ts +631 -0
- package/src/lib/spend-reservation-ledger.ts +115 -9
- package/src/lib/windows-secret-acl.ts +151 -15
- package/src/lib/windows-user-principal.ts +5 -1
- package/src/lib/workflow-budget.ts +145 -8
- package/src/oauth/account-quota-rank.ts +72 -15
- package/src/oauth/generic-account-failover.ts +40 -27
- package/src/oauth/orcarouter.ts +15 -2
- package/src/oauth/store.ts +8 -0
- package/src/providers/codex-capacity.ts +9 -0
- package/src/providers/derive.ts +6 -0
- package/src/providers/devin-provider-merge-migration.ts +33 -12
- package/src/providers/free-directory.ts +20 -2
- package/src/providers/key-failover.ts +261 -7
- package/src/providers/model-discovery.ts +19 -7
- package/src/providers/model-rename-migration.ts +1 -0
- package/src/providers/openai-sidecar.ts +4 -0
- package/src/providers/opencode-go-transport.ts +14 -5
- package/src/providers/quota/report-cache.ts +3 -0
- package/src/providers/registry/entries-core.ts +11 -0
- package/src/providers/registry/entries-extended.ts +146 -28
- package/src/providers/registry/model-seeds.ts +136 -29
- package/src/providers/registry/types.ts +9 -0
- package/src/responses/apply-patch-envelope.ts +44 -11
- package/src/responses/bridge-search-replay-cache.ts +152 -0
- package/src/responses/code-mode-helper-compat.ts +26 -16
- package/src/responses/custom-tool-compat.ts +1 -1
- package/src/responses/hosted-tool-policy.ts +85 -2
- package/src/responses/schema.ts +9 -2
- package/src/responses/spill-store.ts +17 -0
- package/src/responses/state/body-policy.ts +25 -0
- package/src/responses/state/spill-queue.ts +8 -6
- package/src/responses/state.ts +3 -22
- package/src/router.ts +4 -0
- package/src/server/auth-cors.ts +27 -0
- package/src/server/chat-completions.ts +9 -4
- package/src/server/chat-native-sse.ts +26 -9
- package/src/server/chat-native.ts +10 -4
- package/src/server/claude-messages.ts +24 -2
- package/src/server/gui-static.ts +36 -2
- package/src/server/inbound-body-admission.ts +187 -0
- package/src/server/index/websocket-handler.ts +48 -1
- package/src/server/index.ts +15 -19
- package/src/server/management/api-access.ts +3 -4
- package/src/server/management/config-routes.ts +57 -10
- package/src/server/management/provider-capability-config.ts +35 -7
- package/src/server/management/provider-routes.ts +70 -18
- package/src/server/models-capabilities.ts +24 -3
- package/src/server/proxy-liveness.ts +97 -2
- package/src/server/relay.ts +17 -24
- package/src/server/request-log.ts +25 -1
- package/src/server/responses/adapter-continuation.ts +71 -27
- package/src/server/responses/adapter-delivery.ts +39 -8
- package/src/server/responses/adapter-dispatch.ts +52 -24
- package/src/server/responses/codex-ws-exchange.ts +65 -4
- package/src/server/responses/combo-stream-preflight.ts +68 -5
- package/src/server/responses/compact.ts +60 -11
- package/src/server/responses/core-codex-account.ts +83 -22
- package/src/server/responses/core-combo.ts +26 -0
- package/src/server/responses/core-normalize.ts +12 -5
- package/src/server/responses/core-options.ts +3 -0
- package/src/server/responses/fetch-helpers.ts +72 -3
- package/src/server/responses/native-injection-protocol.ts +42 -0
- package/src/server/responses/native-injection-replay.ts +105 -0
- package/src/server/responses/native-injection.ts +242 -0
- package/src/server/responses/native-response-control.ts +56 -0
- package/src/server/responses/native-response-json.ts +14 -0
- package/src/server/responses/native-response-output.ts +37 -0
- package/src/server/responses/native-steering-log.ts +44 -0
- package/src/server/responses/native-steering-policy.ts +49 -0
- package/src/server/responses/native-steering-replay.ts +126 -0
- package/src/server/responses/native-steering-settings.ts +76 -0
- package/src/server/responses/native-steering.ts +400 -0
- package/src/server/responses/native-tool-results.ts +130 -0
- package/src/server/responses/passthrough-delivery.ts +21 -1
- package/src/server/responses/passthrough-dispatch.ts +146 -49
- package/src/server/responses/passthrough-execution.ts +11 -1
- package/src/server/responses/request-prepare.ts +70 -0
- package/src/server/responses/request-send-budget.ts +84 -7
- package/src/server/responses/request-sidecar-auth.ts +16 -8
- package/src/server/responses/request-spend.ts +38 -9
- package/src/server/responses/request-transport.ts +13 -10
- package/src/server/responses/run-turn-execution.ts +20 -5
- package/src/server/responses/sidecar-execution.ts +2 -0
- package/src/server/responses/ws-upstream.ts +23 -2
- package/src/server/responses-custom-tool-repair.ts +2 -2
- package/src/server/sse-frame-buffer.ts +12 -10
- package/src/server/sse-payload-rewrite.ts +36 -9
- package/src/server/stop-teardown.ts +8 -1
- package/src/server/system-env-shell.ts +5 -1
- package/src/server/system-env.ts +7 -1
- package/src/server/workflow-refusal.ts +56 -2
- package/src/server/ws-bridge.ts +16 -1
- package/src/service/cli.ts +29 -7
- package/src/service/guards.ts +10 -0
- package/src/service/health.ts +43 -0
- package/src/service/state.ts +7 -2
- package/src/types/accounts.ts +4 -0
- package/src/types/config.ts +104 -3
- package/src/types/provider.ts +32 -0
- package/src/types/request.ts +7 -1
- package/src/types/wire.ts +9 -1
- package/src/usage/expected-prices.ts +28 -0
- package/src/usage/log.ts +87 -4
- package/src/web-search/passthrough-bridge.ts +39 -5
- package/gui/dist/assets/index-Cz7CLdif.js +0 -128
package/src/server/relay.ts
CHANGED
|
@@ -22,10 +22,11 @@ import {
|
|
|
22
22
|
} from "./request-log";
|
|
23
23
|
import {
|
|
24
24
|
BoundedSseFrameBuffer,
|
|
25
|
+
EMPTY_BYTES,
|
|
25
26
|
joinSseFrameBytes,
|
|
26
27
|
MAX_CLIENT_SSE_FRAME_BYTES,
|
|
27
28
|
} from "./sse-frame-buffer";
|
|
28
|
-
import { replaceSseDataPayload } from "./sse-payload-rewrite";
|
|
29
|
+
import { replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite";
|
|
29
30
|
import { createBoundedResponseLogBody } from "./response-log-body";
|
|
30
31
|
|
|
31
32
|
const nativePassthroughSseResponses = new WeakSet<Response>();
|
|
@@ -200,7 +201,7 @@ export function createSseTerminalOutputBoundary(
|
|
|
200
201
|
const processFrames = (
|
|
201
202
|
frames: ReturnType<BoundedSseFrameBuffer["feed"]>,
|
|
202
203
|
): Uint8Array => {
|
|
203
|
-
if (disposed || terminal || frames.length === 0) return
|
|
204
|
+
if (disposed || terminal || frames.length === 0) return EMPTY_BYTES;
|
|
204
205
|
const output: Uint8Array[] = [];
|
|
205
206
|
let responsesTerminal = false;
|
|
206
207
|
for (const frame of frames) {
|
|
@@ -259,13 +260,13 @@ export function createSseTerminalOutputBoundary(
|
|
|
259
260
|
|
|
260
261
|
return {
|
|
261
262
|
feed(chunk) {
|
|
262
|
-
if (disposed || terminal) return
|
|
263
|
+
if (disposed || terminal) return EMPTY_BYTES;
|
|
263
264
|
return processFrames(framer.feed(chunk));
|
|
264
265
|
},
|
|
265
266
|
finish() {
|
|
266
|
-
if (disposed || terminal) return
|
|
267
|
+
if (disposed || terminal) return EMPTY_BYTES;
|
|
267
268
|
const tail = framer.finish();
|
|
268
|
-
if (tail.byteLength === 0) return
|
|
269
|
+
if (tail.byteLength === 0) return EMPTY_BYTES;
|
|
269
270
|
// EOF may cut off the final SSE block before its blank-line delimiter.
|
|
270
271
|
// Feed it through the exact same parser/rewrite/terminal path as a
|
|
271
272
|
// complete frame, using a synthetic delimiter so the client receives a
|
|
@@ -359,7 +360,7 @@ export function relaySseWithFailedTail(
|
|
|
359
360
|
if (result !== "buffered") return;
|
|
360
361
|
}
|
|
361
362
|
} catch (err) {
|
|
362
|
-
let partial: Uint8Array =
|
|
363
|
+
let partial: Uint8Array = EMPTY_BYTES;
|
|
363
364
|
let tailTerminal = false;
|
|
364
365
|
try {
|
|
365
366
|
partial = terminalBoundary.finish();
|
|
@@ -403,15 +404,7 @@ export function nextSseBlock(buffer: string): { block: string; delimiter: string
|
|
|
403
404
|
};
|
|
404
405
|
}
|
|
405
406
|
|
|
406
|
-
export
|
|
407
|
-
const data: string[] = [];
|
|
408
|
-
for (const line of block.split(/\r?\n/)) {
|
|
409
|
-
if (!line.startsWith("data:")) continue;
|
|
410
|
-
const value = line.slice(5);
|
|
411
|
-
data.push(value.startsWith(" ") ? value.slice(1) : value);
|
|
412
|
-
}
|
|
413
|
-
return data.length > 0 ? data.join("\n") : null;
|
|
414
|
-
}
|
|
407
|
+
export { sseDataPayload } from "./sse-payload-rewrite";
|
|
415
408
|
|
|
416
409
|
type JsonRecord = Record<string, unknown>;
|
|
417
410
|
|
|
@@ -943,8 +936,8 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
943
936
|
let reported = false;
|
|
944
937
|
let sawTerminal = false;
|
|
945
938
|
let disposed = false;
|
|
946
|
-
let delimiterTail: Uint8Array =
|
|
947
|
-
let candidate: Uint8Array =
|
|
939
|
+
let delimiterTail: Uint8Array = EMPTY_BYTES;
|
|
940
|
+
let candidate: Uint8Array = EMPTY_BYTES;
|
|
948
941
|
let candidateBytes = 0;
|
|
949
942
|
let discardingOversizedFrame = false;
|
|
950
943
|
const reportFirstOutput = createFirstOutputReporter(handlers.onFirstOutput);
|
|
@@ -957,8 +950,8 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
957
950
|
let firstResponseId: string | undefined;
|
|
958
951
|
|
|
959
952
|
const clearFrameState = (): void => {
|
|
960
|
-
delimiterTail =
|
|
961
|
-
candidate =
|
|
953
|
+
delimiterTail = EMPTY_BYTES;
|
|
954
|
+
candidate = EMPTY_BYTES;
|
|
962
955
|
candidateBytes = 0;
|
|
963
956
|
discardingOversizedFrame = false;
|
|
964
957
|
};
|
|
@@ -995,9 +988,9 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
995
988
|
};
|
|
996
989
|
|
|
997
990
|
const takeCandidate = (): Uint8Array => {
|
|
998
|
-
if (candidateBytes === 0) return
|
|
991
|
+
if (candidateBytes === 0) return EMPTY_BYTES;
|
|
999
992
|
const frame = candidate.slice(0, candidateBytes);
|
|
1000
|
-
candidate =
|
|
993
|
+
candidate = EMPTY_BYTES;
|
|
1001
994
|
candidateBytes = 0;
|
|
1002
995
|
return frame;
|
|
1003
996
|
};
|
|
@@ -1010,7 +1003,7 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
1010
1003
|
Math.min(nextBytes, MAX_INSPECTION_SSE_FRAME_BYTES),
|
|
1011
1004
|
);
|
|
1012
1005
|
if (nextBytes > MAX_INSPECTION_SSE_FRAME_BYTES) {
|
|
1013
|
-
candidate =
|
|
1006
|
+
candidate = EMPTY_BYTES;
|
|
1014
1007
|
candidateBytes = 0;
|
|
1015
1008
|
discardingOversizedFrame = true;
|
|
1016
1009
|
inspectionCounters.frameCapOverflows += 1;
|
|
@@ -1173,7 +1166,7 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
1173
1166
|
|
|
1174
1167
|
const scanChunk = (chunk: Uint8Array): void => {
|
|
1175
1168
|
const previousTail = delimiterTail;
|
|
1176
|
-
delimiterTail =
|
|
1169
|
+
delimiterTail = EMPTY_BYTES;
|
|
1177
1170
|
const tailLength = previousTail.byteLength;
|
|
1178
1171
|
const totalLength = tailLength + chunk.byteLength;
|
|
1179
1172
|
const byteAt = (index: number): number => index < tailLength
|
|
@@ -1219,7 +1212,7 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector
|
|
|
1219
1212
|
if (disposed) return;
|
|
1220
1213
|
try {
|
|
1221
1214
|
retainCandidateSlice(delimiterTail);
|
|
1222
|
-
delimiterTail =
|
|
1215
|
+
delimiterTail = EMPTY_BYTES;
|
|
1223
1216
|
if (!discardingOversizedFrame && candidateBytes > 0 && !reported) {
|
|
1224
1217
|
const sourceBytes = candidateBytes;
|
|
1225
1218
|
const decoded = decoder!.decode(takeCandidate());
|
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
usageStatusForFinalLog,
|
|
45
45
|
usageTotalTokens,
|
|
46
46
|
type AttemptRecoveryKind,
|
|
47
|
+
type AttemptRecoveryWithheld,
|
|
47
48
|
type CacheTelemetryProvenance,
|
|
48
49
|
type PersistedRequestSpend,
|
|
49
50
|
type PersistedUsageAttempt,
|
|
@@ -854,7 +855,13 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
|
|
|
854
855
|
* Mark a refusal this proxy synthesized locally. Sets origin to `synthetic` and a
|
|
855
856
|
* distinct local reason so the request log cannot be read as an upstream overload.
|
|
856
857
|
*/
|
|
857
|
-
|
|
858
|
+
// Typed by the two fields it writes rather than by the whole context: the durable-spend tracker
|
|
859
|
+
// has to mark a row from a narrow view of it, and widening that view to the full context there
|
|
860
|
+
// would pull the entire log shape into a module that touches two of its fields.
|
|
861
|
+
export function markLocalRequestLogRefusal(
|
|
862
|
+
logCtx: Pick<RequestLogContext, "localTerminalReason" | "terminalSource">,
|
|
863
|
+
reason: string,
|
|
864
|
+
): void {
|
|
858
865
|
logCtx.localTerminalReason = reason;
|
|
859
866
|
logCtx.terminalSource = "synthetic";
|
|
860
867
|
}
|
|
@@ -1300,6 +1307,7 @@ export function addFinalRequestLog(
|
|
|
1300
1307
|
const attempts = logCtx.attempts?.map(attempt => ({
|
|
1301
1308
|
...attempt,
|
|
1302
1309
|
recoveryKinds: [...attempt.recoveryKinds],
|
|
1310
|
+
...(attempt.recoveryWithheld?.length ? { recoveryWithheld: [...attempt.recoveryWithheld] } : {}),
|
|
1303
1311
|
...(attempt.usage ? { usage: { ...attempt.usage } } : {}),
|
|
1304
1312
|
...(attempt.tierOutcome ? { tierOutcome: { ...attempt.tierOutcome } } : {}),
|
|
1305
1313
|
}));
|
|
@@ -1703,6 +1711,22 @@ export function noteAttemptSend(
|
|
|
1703
1711
|
}
|
|
1704
1712
|
}
|
|
1705
1713
|
|
|
1714
|
+
/**
|
|
1715
|
+
* Record that a recovery this attempt was eligible for did not happen.
|
|
1716
|
+
*
|
|
1717
|
+
* Deliberately NOT `noteAttemptSend`: nothing was sent, so `sendCount` must not move. The two
|
|
1718
|
+
* together are what make a one-send log readable — no kind and no withheld reason means nothing
|
|
1719
|
+
* was eligible, a withheld reason means something was and the budget refused it (#5044).
|
|
1720
|
+
*/
|
|
1721
|
+
export function noteAttemptRecoveryWithheld(
|
|
1722
|
+
attempt: PersistedUsageAttempt | undefined,
|
|
1723
|
+
reason: AttemptRecoveryWithheld,
|
|
1724
|
+
): void {
|
|
1725
|
+
if (!attempt) return;
|
|
1726
|
+
if (!attempt.recoveryWithheld) attempt.recoveryWithheld = [];
|
|
1727
|
+
if (!attempt.recoveryWithheld.includes(reason)) attempt.recoveryWithheld.push(reason);
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1706
1730
|
export function finishRequestAttempt(
|
|
1707
1731
|
attempt: PersistedUsageAttempt,
|
|
1708
1732
|
status: number,
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
sealRequestAttemptIdentity,
|
|
14
14
|
recordAttemptCredentialSource,
|
|
15
15
|
} from "../request-log";
|
|
16
|
+
import { noteAttemptRecoveryWithheld } from "../request-log";
|
|
16
17
|
import { waitForProviderRequestSlot } from "../../providers/request-pacing";
|
|
17
18
|
import { providerFetch, fetchWithHeaderTimeout, safeHostLabel } from "./fetch-helpers";
|
|
18
19
|
import {
|
|
@@ -46,7 +47,12 @@ import {
|
|
|
46
47
|
import { shouldAttemptImageTierRetry } from "../image-retry";
|
|
47
48
|
import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors";
|
|
48
49
|
import { isCyberPolicyCode, CYBER_POLICY_FALLBACK_MESSAGE, CYBER_POLICY_ERROR_CODE } from "../../lib/errors";
|
|
49
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
readResponseBodyWithInactivity,
|
|
52
|
+
readResponseStreamWithInactivity,
|
|
53
|
+
ResponseBodyInactivityError,
|
|
54
|
+
} from "../../lib/response-body-inactivity";
|
|
55
|
+
import { resolveStallTimeoutSec } from "../../stall-timeout";
|
|
50
56
|
import { guardTerminalEventStream } from "./terminal-guard";
|
|
51
57
|
|
|
52
58
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
@@ -78,6 +84,7 @@ export function createAdapterContinuations(
|
|
|
78
84
|
| "genericFailoverAccountId"
|
|
79
85
|
| "genericFailovers"
|
|
80
86
|
| "applyFailoverSnapshot"
|
|
87
|
+
| "replayOAuthCredentialSnapshot"
|
|
81
88
|
| "noteRoutedAttemptSend"
|
|
82
89
|
>,
|
|
83
90
|
sidecarState: Pick<ResponsesSidecarAuth, "routedCompaction">,
|
|
@@ -85,6 +92,7 @@ export function createAdapterContinuations(
|
|
|
85
92
|
ResponsesSendBudget,
|
|
86
93
|
| "adapterDispatchBudget"
|
|
87
94
|
| "noteAdapterPhysicalSend"
|
|
95
|
+
| "noteAdapterRecoveryWithheld"
|
|
88
96
|
| "remainingTransientSendBudget"
|
|
89
97
|
| "noteTransientSends"
|
|
90
98
|
| "reserveCredentialHop"
|
|
@@ -112,9 +120,11 @@ export function createAdapterContinuations(
|
|
|
112
120
|
const { route, translatorBudget, inboundWire, parsed } = requestState;
|
|
113
121
|
const { routedCompaction } = sidecarState;
|
|
114
122
|
const { upstream, connectMs, rateLimitPolicy, stallTimeoutMs } = adapterExchange;
|
|
123
|
+
const bodyInactivityMs = resolveStallTimeoutSec(config.stallTimeoutSec) * 1000;
|
|
115
124
|
const {
|
|
116
125
|
adapterDispatchBudget,
|
|
117
126
|
noteAdapterPhysicalSend,
|
|
127
|
+
noteAdapterRecoveryWithheld,
|
|
118
128
|
remainingTransientSendBudget,
|
|
119
129
|
noteTransientSends,
|
|
120
130
|
reserveCredentialHop,
|
|
@@ -193,6 +203,7 @@ export function createAdapterContinuations(
|
|
|
193
203
|
timeoutMs: connectMs,
|
|
194
204
|
sendBudget: adapterDispatchBudget,
|
|
195
205
|
onPhysicalSend: send => noteAdapterPhysicalSend(continuationEstimate, send),
|
|
206
|
+
onRecoveryWithheld: noteAdapterRecoveryWithheld,
|
|
196
207
|
stream: nextParsed.stream,
|
|
197
208
|
executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
|
|
198
209
|
pacingSlotAcquired: true,
|
|
@@ -263,12 +274,12 @@ export function createAdapterContinuations(
|
|
|
263
274
|
// Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover:
|
|
264
275
|
// a primary-key rate-limit blip replays on the SAME key, matching the main recovery
|
|
265
276
|
// loop; only after the attempts are exhausted does the continuation fail over.
|
|
266
|
-
|
|
267
|
-
|
|
277
|
+
while (
|
|
278
|
+
response.status === 429
|
|
268
279
|
// A synthesized replay refusal is not a rate limit; replaying the continuation on
|
|
269
280
|
// it would re-send a turn whose first send may already have been processed.
|
|
270
281
|
&& !isNonReplayableResponse(response)
|
|
271
|
-
|
|
282
|
+
&& rateLimitPolicy !== null
|
|
272
283
|
&& adapterExchange.rateLimitRetries < rateLimitPolicy.attempts
|
|
273
284
|
// The main recovery loop and the passthrough ladder both consult the shared remainder
|
|
274
285
|
// here; this loop did not, so a request whose budget was already spent could still
|
|
@@ -348,11 +359,11 @@ export function createAdapterContinuations(
|
|
|
348
359
|
continue;
|
|
349
360
|
}
|
|
350
361
|
}
|
|
351
|
-
|
|
352
|
-
|
|
362
|
+
if (
|
|
363
|
+
response.status === 429
|
|
364
|
+
&& transportState.anthropicPoolAccountId
|
|
353
365
|
&& !isNonReplayableResponse(response)
|
|
354
|
-
|
|
355
|
-
&& transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST
|
|
366
|
+
&& transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST
|
|
356
367
|
) {
|
|
357
368
|
const nextAccountId = rotateAnthropicAccountOn429(
|
|
358
369
|
config,
|
|
@@ -390,11 +401,11 @@ export function createAdapterContinuations(
|
|
|
390
401
|
// 429 stayed terminal even with failover fully active -- the same class of divergence the
|
|
391
402
|
// two sidecars already produced once. Request-local state is shared with the other arms so
|
|
392
403
|
// the per-request bound cannot be silently re-armed by reaching a different loop.
|
|
393
|
-
|
|
394
|
-
|
|
404
|
+
if (
|
|
405
|
+
response.status === 429
|
|
406
|
+
&& transportState.genericFailoverAccountId
|
|
395
407
|
&& !isNonReplayableResponse(response)
|
|
396
|
-
|
|
397
|
-
&& transportState.genericFailovers < GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST
|
|
408
|
+
&& transportState.genericFailovers < GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST
|
|
398
409
|
&& isGenericOAuthFailoverEnabled(config, route.providerName)
|
|
399
410
|
) {
|
|
400
411
|
// Intersection with the shared request budget. The continuation loop re-sends the
|
|
@@ -416,8 +427,13 @@ export function createAdapterContinuations(
|
|
|
416
427
|
route.providerName,
|
|
417
428
|
transportState.genericFailoverAccountId,
|
|
418
429
|
response.headers.get("retry-after"),
|
|
430
|
+
Date.now(),
|
|
431
|
+
route.modelId,
|
|
419
432
|
)
|
|
420
433
|
: null;
|
|
434
|
+
// Eligible and refused by the shared budget, as opposed to eligible and finding no next
|
|
435
|
+
// account: the two produce the same response and need different follow-ups (#5044).
|
|
436
|
+
if (!hop.allowed) noteAttemptRecoveryWithheld(logCtx.activeAttempt, "rotation-send-budget");
|
|
421
437
|
if (!nextAccountId) hop.permit?.release();
|
|
422
438
|
if (nextAccountId) {
|
|
423
439
|
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
@@ -436,6 +452,22 @@ export function createAdapterContinuations(
|
|
|
436
452
|
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire),
|
|
437
453
|
config.cacheRetention,
|
|
438
454
|
);
|
|
455
|
+
bindRouteReasoningReplayScope({
|
|
456
|
+
parsed: nextParsed,
|
|
457
|
+
providerName: route.providerName,
|
|
458
|
+
provider: route.provider,
|
|
459
|
+
adapterName: transportState.activeAdapter.name,
|
|
460
|
+
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
|
|
461
|
+
});
|
|
462
|
+
// Response persistence closes over the outer parsed request; keep its owner binding in
|
|
463
|
+
// sync with the terminal-guard clone that builds the rotated continuation request.
|
|
464
|
+
bindRouteReasoningReplayScope({
|
|
465
|
+
parsed,
|
|
466
|
+
providerName: route.providerName,
|
|
467
|
+
provider: route.provider,
|
|
468
|
+
adapterName: transportState.activeAdapter.name,
|
|
469
|
+
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
|
|
470
|
+
});
|
|
439
471
|
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
|
|
440
472
|
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
441
473
|
// The replay goes out on the next iteration. An adapter that owns its ladder
|
|
@@ -493,23 +525,35 @@ export function createAdapterContinuations(
|
|
|
493
525
|
}
|
|
494
526
|
|
|
495
527
|
try {
|
|
496
|
-
//
|
|
497
|
-
// reader
|
|
498
|
-
//
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
528
|
+
// Each successful continuation owns a fresh pending-read deadline and an abort
|
|
529
|
+
// listener that can cancel its reader even while the parser holds the body lock.
|
|
530
|
+
// The shared signal is not aborted on timeout: this generator must still emit
|
|
531
|
+
// the error that the already-live Responses bridge turns into a terminal.
|
|
532
|
+
if (nextParsed.stream) {
|
|
533
|
+
yield* readResponseStreamWithInactivity(
|
|
534
|
+
response,
|
|
535
|
+
upstream.signal,
|
|
536
|
+
bodyInactivityMs,
|
|
537
|
+
guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata),
|
|
538
|
+
);
|
|
539
|
+
} else if (transportState.activeAdapter.parseResponse) {
|
|
540
|
+
yield* await readResponseBodyWithInactivity(
|
|
541
|
+
response,
|
|
542
|
+
upstream.signal,
|
|
543
|
+
bodyInactivityMs,
|
|
544
|
+
guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata),
|
|
545
|
+
);
|
|
546
|
+
} else {
|
|
547
|
+
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
548
|
+
yield { type: "error", message: "Provider continuation does not support response parsing" };
|
|
510
549
|
}
|
|
511
550
|
} catch (error) {
|
|
512
|
-
|
|
551
|
+
// Classify the thrown error before reading signal state: cancelling the stalled
|
|
552
|
+
// source can synchronously abort the shared signal, which would otherwise report
|
|
553
|
+
// this timeout as a client cancellation.
|
|
554
|
+
if (error instanceof ResponseBodyInactivityError) {
|
|
555
|
+
yield { type: "error", message: "Provider continuation response body stalled before completing", status: 504 };
|
|
556
|
+
} else if (options.abortSignal?.aborted || upstream.signal.aborted) {
|
|
513
557
|
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
|
|
514
558
|
} else {
|
|
515
559
|
yield { type: "error", message: `Provider continuation parse failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` };
|
|
@@ -14,6 +14,12 @@ import { rememberResponseState } from "../../responses/state";
|
|
|
14
14
|
import { trackStreamLifetime } from "../lifecycle";
|
|
15
15
|
import { awaitThoughtSignatureDurability } from "../../responses/thought-signature-replay";
|
|
16
16
|
import { adapterResponseReachedServingTerminal } from "./core-replay";
|
|
17
|
+
import {
|
|
18
|
+
readResponseBodyWithInactivity,
|
|
19
|
+
readResponseStreamWithInactivity,
|
|
20
|
+
ResponseBodyInactivityError,
|
|
21
|
+
} from "../../lib/response-body-inactivity";
|
|
22
|
+
import { resolveStallTimeoutSec } from "../../stall-timeout";
|
|
17
23
|
|
|
18
24
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
19
25
|
export async function deliverAdapterResponse(
|
|
@@ -61,14 +67,33 @@ export async function deliverAdapterResponse(
|
|
|
61
67
|
notifyResponseComplete,
|
|
62
68
|
} = responseEffects;
|
|
63
69
|
const { routedCompaction } = sidecarState;
|
|
70
|
+
const bodyInactivityMs = resolveStallTimeoutSec(config.stallTimeoutSec) * 1000;
|
|
64
71
|
|
|
65
72
|
|
|
66
73
|
if (parsed.stream) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
74
|
+
// The continuation legs classify a stalled body themselves; the initial stream needs the
|
|
75
|
+
// same mapping or the bridge catch reports this upstream timeout as a 500 proxy_error.
|
|
76
|
+
const initialEventStream = (async function* (): AsyncGenerator<AdapterEvent> {
|
|
77
|
+
try {
|
|
78
|
+
yield* readResponseStreamWithInactivity(
|
|
79
|
+
upstreamResponse,
|
|
80
|
+
upstream.signal,
|
|
81
|
+
bodyInactivityMs,
|
|
82
|
+
response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata),
|
|
83
|
+
);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if (error instanceof ResponseBodyInactivityError) {
|
|
86
|
+
yield {
|
|
87
|
+
type: "error",
|
|
88
|
+
message: "Upstream response body stalled before completing",
|
|
89
|
+
status: 504,
|
|
90
|
+
errorType: "upstream_error",
|
|
91
|
+
};
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
})();
|
|
72
97
|
const eventStream = terminalGuardEnabled
|
|
73
98
|
? guardTerminalEventStream({
|
|
74
99
|
parsed,
|
|
@@ -137,10 +162,11 @@ export async function deliverAdapterResponse(
|
|
|
137
162
|
if (transportState.activeAdapter.parseResponse) {
|
|
138
163
|
let events: AdapterEvent[];
|
|
139
164
|
try {
|
|
140
|
-
const initialEvents = await
|
|
165
|
+
const initialEvents = await readResponseBodyWithInactivity(
|
|
141
166
|
upstreamResponse,
|
|
142
|
-
|
|
143
|
-
|
|
167
|
+
upstream.signal,
|
|
168
|
+
bodyInactivityMs,
|
|
169
|
+
response => transportState.activeAdapter.parseResponse!(response, translatorBudget, logCtx.activeTierMetadata),
|
|
144
170
|
);
|
|
145
171
|
let guardedEvents: AdapterEvent[];
|
|
146
172
|
if (terminalGuardEnabled) {
|
|
@@ -164,6 +190,11 @@ export async function deliverAdapterResponse(
|
|
|
164
190
|
} else {
|
|
165
191
|
events = guardedEvents;
|
|
166
192
|
}
|
|
193
|
+
} catch (error) {
|
|
194
|
+
if (error instanceof ResponseBodyInactivityError) {
|
|
195
|
+
return formatErrorResponse(504, "upstream_error", "Upstream response body stalled before completing");
|
|
196
|
+
}
|
|
197
|
+
throw error;
|
|
167
198
|
} finally {
|
|
168
199
|
cleanupUpstreamAbort();
|
|
169
200
|
}
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
hasKeyPoolFailover,
|
|
26
26
|
rotateProviderTransportOn401,
|
|
27
27
|
rateLimitRetryDelayMs,
|
|
28
|
+
readQuotaResetAt,
|
|
28
29
|
rotateProviderTransportOn429,
|
|
29
30
|
} from "../../providers/key-failover";
|
|
30
31
|
import {
|
|
@@ -127,6 +128,7 @@ export async function prepareAdapterExchange(
|
|
|
127
128
|
ResponsesSendBudget,
|
|
128
129
|
| "adapterDispatchBudget"
|
|
129
130
|
| "noteAdapterPhysicalSend"
|
|
131
|
+
| "noteAdapterRecoveryWithheld"
|
|
130
132
|
| "remainingTransientSendBudget"
|
|
131
133
|
| "noteTransientSends"
|
|
132
134
|
| "recoverySendAllowance"
|
|
@@ -160,6 +162,7 @@ export async function prepareAdapterExchange(
|
|
|
160
162
|
const {
|
|
161
163
|
adapterDispatchBudget,
|
|
162
164
|
noteAdapterPhysicalSend,
|
|
165
|
+
noteAdapterRecoveryWithheld,
|
|
163
166
|
remainingTransientSendBudget,
|
|
164
167
|
noteTransientSends,
|
|
165
168
|
recoverySendAllowance,
|
|
@@ -286,6 +289,7 @@ export async function prepareAdapterExchange(
|
|
|
286
289
|
timeoutMs: connectMs,
|
|
287
290
|
sendBudget: adapterDispatchBudget,
|
|
288
291
|
onPhysicalSend: send => noteAdapterPhysicalSend(inputTokenEstimate, send),
|
|
292
|
+
onRecoveryWithheld: noteAdapterRecoveryWithheld,
|
|
289
293
|
stream: parsed.stream,
|
|
290
294
|
executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
|
|
291
295
|
pacingSlotAcquired: true,
|
|
@@ -442,6 +446,7 @@ export async function prepareAdapterExchange(
|
|
|
442
446
|
timeoutMs: connectMs,
|
|
443
447
|
sendBudget: adapterDispatchBudget,
|
|
444
448
|
onPhysicalSend: send => noteAdapterPhysicalSend(retryEstimate, send),
|
|
449
|
+
onRecoveryWithheld: noteAdapterRecoveryWithheld,
|
|
445
450
|
stream: parsed.stream,
|
|
446
451
|
executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
|
|
447
452
|
pacingSlotAcquired: true,
|
|
@@ -525,17 +530,17 @@ export async function prepareAdapterExchange(
|
|
|
525
530
|
return { failed: formatErrorResponse(502, "upstream_error", msg) };
|
|
526
531
|
}
|
|
527
532
|
};
|
|
528
|
-
|
|
529
|
-
|
|
533
|
+
// Keep recovery kinds in sync with the native Responses `passthroughRecovery:` loop above.
|
|
534
|
+
recovery: for (;;) {
|
|
530
535
|
// Preserve the terminal verdict through adapter and combo error formatting.
|
|
531
536
|
// This also covers a reset reached by a 401/429/413 recovery refetch.
|
|
532
537
|
if (isNonReplayableResponse(upstreamResponse)) {
|
|
533
538
|
cleanupUpstreamAbort();
|
|
534
539
|
return upstreamResponse;
|
|
535
540
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
541
|
+
if (
|
|
542
|
+
upstreamResponse.status === 401
|
|
543
|
+
&& isOAuth401ReplayProvider
|
|
539
544
|
&& transportState.sentOAuthSnapshot
|
|
540
545
|
&& !oauth401ReplayAttempted
|
|
541
546
|
&& !sendBudgetExhausted()
|
|
@@ -618,15 +623,15 @@ export async function prepareAdapterExchange(
|
|
|
618
623
|
provider: route.provider,
|
|
619
624
|
adapterName: transportState.activeAdapter.name,
|
|
620
625
|
});
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
626
|
+
const result = await rebuildAndRefetch("key-401");
|
|
627
|
+
if ("failed" in result) return result.failed;
|
|
628
|
+
upstreamResponse = result;
|
|
624
629
|
// A recovery refetch can itself die on an ambiguous pre-header reset, and the refusal
|
|
625
630
|
// that answers it is a 429. Every arm below keys on 429, so letting it fall through
|
|
626
631
|
// hands the marked refusal to the next waiting arm and replays the send it exists to
|
|
627
632
|
// stop. Re-enter the loop guard instead, which returns it unchanged.
|
|
628
633
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
629
|
-
|
|
634
|
+
}
|
|
630
635
|
|
|
631
636
|
// Same-target 429 wait-and-retry (opt-in `retryOn429`, issue #487). Codex never retries
|
|
632
637
|
// 429 itself (it retries 5xx only), and single-key pools cannot use the failover below,
|
|
@@ -663,23 +668,44 @@ export async function prepareAdapterExchange(
|
|
|
663
668
|
upstream.abort();
|
|
664
669
|
return clientCancelledResponse();
|
|
665
670
|
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
671
|
+
const result = await rebuildAndRefetch("rate-limit-429");
|
|
672
|
+
if ("failed" in result) return result.failed;
|
|
673
|
+
upstreamResponse = result;
|
|
669
674
|
// The refusal is a 429 too: without this the while condition is still true and the
|
|
670
675
|
// next configured attempt replays it on the same target.
|
|
671
676
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
672
|
-
|
|
677
|
+
}
|
|
673
678
|
|
|
674
679
|
// Multi-key 429 failover: rotate to the next pool key (cooldown-aware) and retry the
|
|
675
680
|
// SAME request once per remaining key. OAuth/forward providers and single-key pools
|
|
676
681
|
// return null immediately, so this stays a no-op for them (src/providers/key-failover.ts).
|
|
677
682
|
while (upstreamResponse.status === 429 && hasKeyPoolFailover(route.provider)) {
|
|
683
|
+
// A quota exhaustion is dated in the BODY, not in `Retry-After` — OpenRouter
|
|
684
|
+
// sends no header for it (#4024). Read a bounded prefix before the socket is
|
|
685
|
+
// released below; a failed or slow read just leaves the header path in charge.
|
|
686
|
+
// Peeks a bounded prefix and hands back a Response still carrying the whole
|
|
687
|
+
// body, so the cancel below still releases the socket.
|
|
688
|
+
let peeked: Awaited<ReturnType<typeof readQuotaResetAt>>;
|
|
689
|
+
try {
|
|
690
|
+
peeked = await readQuotaResetAt(upstreamResponse, { signal: options.abortSignal });
|
|
691
|
+
} catch {
|
|
692
|
+
cleanupUpstreamAbort();
|
|
693
|
+
upstream.abort();
|
|
694
|
+
return clientCancelledResponse();
|
|
695
|
+
}
|
|
696
|
+
if (options.abortSignal?.aborted) {
|
|
697
|
+
cleanupUpstreamAbort();
|
|
698
|
+
upstream.abort();
|
|
699
|
+
return clientCancelledResponse();
|
|
700
|
+
}
|
|
701
|
+
upstreamResponse = peeked.response;
|
|
702
|
+
const quotaResetAt = peeked.at;
|
|
678
703
|
const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, {
|
|
679
704
|
retryAfter: upstreamResponse.headers.get("retry-after"),
|
|
680
705
|
now: Date.now(),
|
|
681
706
|
attemptedKey: route.provider.apiKey,
|
|
682
707
|
promptCacheKey: parsed.options.promptCacheKey,
|
|
708
|
+
quotaResetAt,
|
|
683
709
|
});
|
|
684
710
|
if (!rotated) break;
|
|
685
711
|
// Release the failed response's socket before retrying; unread bodies otherwise linger
|
|
@@ -697,13 +723,13 @@ export async function prepareAdapterExchange(
|
|
|
697
723
|
provider: route.provider,
|
|
698
724
|
adapterName: transportState.activeAdapter.name,
|
|
699
725
|
});
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
726
|
+
const result = await rebuildAndRefetch("key-429");
|
|
727
|
+
if ("failed" in result) return result.failed;
|
|
728
|
+
upstreamResponse = result;
|
|
703
729
|
// Rotating on the refusal would also write a cooldown against a key that rate-limited
|
|
704
730
|
// nothing, which outlives the request.
|
|
705
731
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
706
|
-
|
|
732
|
+
}
|
|
707
733
|
|
|
708
734
|
// Opt-in Anthropic OAuth account pool (#294): cool the failed account and retry
|
|
709
735
|
// with another eligible OAuth account (bounded per request). Disabled by default.
|
|
@@ -736,11 +762,11 @@ export async function prepareAdapterExchange(
|
|
|
736
762
|
);
|
|
737
763
|
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
|
|
738
764
|
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
765
|
+
const result = await rebuildAndRefetch("anthropic-oauth-429");
|
|
766
|
+
if ("failed" in result) return result.failed;
|
|
767
|
+
upstreamResponse = result;
|
|
742
768
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
743
|
-
|
|
769
|
+
} catch {
|
|
744
770
|
break;
|
|
745
771
|
}
|
|
746
772
|
}
|
|
@@ -786,6 +812,8 @@ export async function prepareAdapterExchange(
|
|
|
786
812
|
route.providerName,
|
|
787
813
|
transportState.genericFailoverAccountId,
|
|
788
814
|
upstreamResponse.headers.get("retry-after"),
|
|
815
|
+
Date.now(),
|
|
816
|
+
route.modelId,
|
|
789
817
|
);
|
|
790
818
|
if (!nextAccountId) {
|
|
791
819
|
hop.permit?.release();
|
|
@@ -832,12 +860,12 @@ export async function prepareAdapterExchange(
|
|
|
832
860
|
hop.permit?.release();
|
|
833
861
|
return result.failed;
|
|
834
862
|
}
|
|
835
|
-
|
|
863
|
+
upstreamResponse = result;
|
|
836
864
|
// The hop's permit is already settled by the dispatch boundary above; continuing
|
|
837
865
|
// only skips the remaining arms, it does not abandon a reservation.
|
|
838
866
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
839
|
-
|
|
840
|
-
|
|
867
|
+
} catch {
|
|
868
|
+
// A throw before the send — snapshot fetch, credential application, adapter
|
|
841
869
|
// resolution — must hand the reservation back. Without this the ladder charges the
|
|
842
870
|
// request for a send it never made, and a later recovery in the same request is
|
|
843
871
|
// refused on an allowance nothing spent. release() is idempotent and a no-op once
|