@bitkyc08/opencodex 2.87.0 → 2.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/bin/ocx.mjs +15 -4
- package/gui/dist/assets/App-BnRSujOq.js +58 -0
- package/gui/dist/assets/App-dnkJ85te.css +1 -0
- package/gui/dist/assets/{Tray-DCd-A1az.js → Tray-7z1COnmz.js} +1 -1
- package/gui/dist/assets/index-uchD-ngf.js +94 -0
- package/gui/dist/assets/{quota-summary-ChvmKGOk.js → quota-summary-BY_10IEz.js} +1 -1
- package/gui/dist/index.html +1 -1
- package/package.json +1 -1
- package/src/adapters/anthropic/cache-retention.ts +117 -0
- package/src/adapters/anthropic/passthrough.ts +7 -4
- package/src/adapters/anthropic-rejection-memo.ts +71 -0
- package/src/adapters/anthropic-replay.ts +135 -0
- package/src/adapters/anthropic-tool-catalog.ts +11 -0
- package/src/adapters/anthropic.ts +86 -48
- package/src/adapters/coding-agent/protocol.ts +6 -0
- package/src/adapters/google-http.ts +3 -2
- package/src/adapters/google-progress-history.ts +53 -0
- package/src/adapters/google-vertex-endpoint.ts +67 -0
- package/src/adapters/google.ts +52 -30
- package/src/adapters/openai-responses/compaction-search-history.ts +27 -0
- package/src/adapters/openai-responses/passthrough.ts +17 -12
- package/src/adapters/openai-responses/reasoning.ts +68 -1
- package/src/adapters/registry.ts +1 -1
- package/src/adapters/upstream-http-error.ts +3 -2
- package/src/adapters/zed.ts +7 -4
- package/src/bridge/internal.ts +17 -3
- package/src/bridge/response-json.ts +4 -4
- package/src/bridge/sse.ts +17 -10
- package/src/claude/inbound-content-options.ts +66 -0
- package/src/claude/inbound.ts +12 -26
- package/src/claude/outbound.ts +18 -5
- package/src/cli/account-api.ts +13 -1
- package/src/cli/account-extended.ts +17 -5
- package/src/cli/account-key-quota.ts +4 -1
- package/src/cli/account.ts +1 -1
- package/src/cli/capabilities-agents-routing.ts +45 -8
- package/src/cli/capabilities-base.ts +5 -5
- package/src/cli/capabilities-provider-models.ts +5 -4
- package/src/cli/combo-result.ts +8 -2
- package/src/cli/combo.ts +17 -0
- package/src/cli/cross-home-owner.ts +16 -2
- package/src/cli/help.ts +1 -1
- package/src/cli/loopback-family.ts +31 -0
- package/src/cli/message-command.ts +4 -1
- package/src/cli/message-remote-args.ts +69 -0
- package/src/cli/message-remote-command.ts +77 -0
- package/src/cli/provider-runtime.ts +69 -1
- package/src/cli/registry.ts +3 -3
- package/src/codex/account-pause.ts +6 -1
- package/src/codex/account-runtime-state.ts +5 -0
- package/src/codex/account-store.ts +30 -3
- package/src/codex/auth-api/account-list.ts +3 -0
- package/src/codex/auth-api/main-account-probe.ts +3 -2
- package/src/codex/auth-context.ts +9 -4
- package/src/codex/catalog/aggregation.ts +15 -0
- package/src/codex/catalog/auto-review.ts +2 -1
- package/src/codex/catalog/build-entries.ts +17 -7
- package/src/codex/catalog/derive-entry.ts +3 -1
- package/src/codex/catalog/metadata.ts +2 -0
- package/src/codex/catalog/model-visibility.ts +0 -2
- package/src/codex/catalog/ownership.ts +71 -0
- package/src/codex/catalog/parsing.ts +12 -3
- package/src/codex/catalog/retained-sync.ts +2 -1
- package/src/codex/catalog/routed-gather.ts +3 -1
- package/src/codex/catalog/routed-removal.ts +4 -4
- package/src/codex/convergence.ts +2 -1
- package/src/codex/low-quota-desktop-notifier.ts +125 -0
- package/src/codex/low-quota-protection.ts +57 -31
- package/src/codex/main-account-cache.ts +77 -3
- package/src/codex/main-account.ts +6 -2
- package/src/codex/native-residue.ts +2 -3
- package/src/codex/quota.ts +26 -2
- package/src/codex/websocket-activity.ts +161 -0
- package/src/codex/websocket-registry.ts +25 -1
- package/src/combos/failover.ts +2 -1
- package/src/combos/jev-model-backend.ts +30 -7
- package/src/combos/jev-model-request.ts +20 -0
- package/src/combos/jev-quota-activation.ts +153 -0
- package/src/combos/jev-quota-config.ts +62 -0
- package/src/combos/jev-quota-route.ts +26 -0
- package/src/combos/jev-quota.ts +88 -0
- package/src/combos/jev.ts +25 -12
- package/src/combos/types.ts +27 -0
- package/src/compatibility/index.ts +3 -0
- package/src/compatibility/openai-responses.ts +8 -1
- package/src/compatibility/opengateway-responses.ts +61 -0
- package/src/config/account-auto-pause-revisions.ts +136 -0
- package/src/config/diagnostics.ts +13 -0
- package/src/config/live-reconcile.ts +9 -0
- package/src/config/opencode-zen-upgrade.ts +9 -8
- package/src/config/persist-unlocked.ts +10 -1
- package/src/config/removed-opencode-free.ts +135 -0
- package/src/config/schema/account-auto-pause.ts +14 -0
- package/src/config/schema/config-schema.ts +15 -0
- package/src/config/schema/leaf-validators.ts +7 -0
- package/src/config.ts +10 -3
- package/src/generated/compatibility-version.json +411 -175
- package/src/images/loop.ts +101 -27
- package/src/lib/account-auto-pause-response.ts +16 -0
- package/src/lib/bun-runtime.ts +18 -1
- package/src/lib/bun-transpiler-cache-scope.d.mts +8 -0
- package/src/lib/bun-transpiler-cache-scope.mjs +68 -0
- package/src/lib/decision-quota-sink.ts +76 -0
- package/src/lib/request-failure-model.ts +2 -0
- package/src/lib/state-store-registrations.ts +25 -0
- package/src/link/ssh-argv.ts +3 -0
- package/src/messaging/envelope.ts +6 -3
- package/src/messaging/remote-auth.ts +83 -0
- package/src/messaging/remote-bridge.ts +270 -0
- package/src/messaging/remote-contract.ts +108 -0
- package/src/messaging/remote-enrollment-recovery.ts +59 -0
- package/src/messaging/remote-enrollment.ts +244 -0
- package/src/messaging/remote-envelope.ts +40 -0
- package/src/messaging/remote-files.ts +117 -0
- package/src/messaging/remote-owner.ts +169 -0
- package/src/messaging/remote-ports.ts +56 -0
- package/src/messaging/remote-process.ts +131 -0
- package/src/messaging/remote-replies.ts +98 -0
- package/src/messaging/remote-rpc-admission.ts +32 -0
- package/src/messaging/remote-send.ts +72 -0
- package/src/messaging/remote-store.ts +111 -0
- package/src/messaging/remote-tunnels.ts +65 -0
- package/src/messaging/rpc.ts +13 -1
- package/src/oauth/account-auto-pause-policy.ts +66 -0
- package/src/oauth/account-auto-pause.ts +171 -0
- package/src/oauth/account-quota-rank.ts +2 -1
- package/src/oauth/anthropic-account-refusal.ts +6 -0
- package/src/oauth/anthropic-model-family.ts +8 -0
- package/src/oauth/anthropic-model-quota.ts +2 -6
- package/src/oauth/api-key-pool-preserve.ts +37 -0
- package/src/oauth/generic-account-failover.ts +7 -4
- package/src/oauth/index.ts +5 -35
- package/src/oauth/store.ts +35 -8
- package/src/oauth/types.ts +2 -0
- package/src/plugins/upstream-hooks.ts +14 -1
- package/src/protocols/encoders/adapter-events.ts +5 -5
- package/src/protocols/encoders/messages.ts +2 -2
- package/src/providers/api-key-pause.ts +32 -0
- package/src/providers/api-key-resolve.ts +10 -4
- package/src/providers/api-key-selection.ts +6 -1
- package/src/providers/api-keys.ts +8 -4
- package/src/providers/claude-cli-usage.ts +5 -0
- package/src/providers/key-failover.ts +11 -11
- package/src/providers/model-rename-fields.ts +1 -0
- package/src/providers/opencode-zen-rate-limit.ts +12 -73
- package/src/providers/quota/account-cache.ts +15 -1
- package/src/providers/quota/anthropic-account-quota.ts +2 -1
- package/src/providers/quota/anthropic-cooldown-recovery.ts +22 -4
- package/src/providers/quota/anthropic-decision-windows.ts +47 -0
- package/src/providers/quota/antigravity.ts +39 -6
- package/src/providers/quota/report-cache.ts +14 -0
- package/src/providers/quota/vendor-probes-oauth.ts +3 -3
- package/src/providers/quota-decision-publication.ts +77 -0
- package/src/providers/quota-decision-snapshot.ts +145 -0
- package/src/providers/quota-routing-cache.ts +3 -0
- package/src/providers/quota-types.ts +6 -1
- package/src/providers/quota.ts +3 -2
- package/src/providers/registry/entries-extended.ts +8 -52
- package/src/providers/registry/model-seeds.ts +1 -1
- package/src/providers/registry.ts +1 -2
- package/src/providers/stale-vision-classification-migration.ts +1 -1
- package/src/responses/parser.ts +18 -10
- package/src/responses/reasoning-replay-cache.ts +50 -4
- package/src/responses/web-search-replay.ts +138 -0
- package/src/routing/analytics.ts +7 -15
- package/src/routing/compatibility/behavior.ts +4 -1
- package/src/server/auth-cors.ts +7 -0
- package/src/server/background-lifecycle.ts +2 -1
- package/src/server/chat-native.ts +28 -13
- package/src/server/claude-messages.ts +4 -2
- package/src/server/index/live-sideband.ts +3 -2
- package/src/server/index/optional-listeners.ts +6 -0
- package/src/server/index/serve-options.ts +11 -5
- package/src/server/index/websocket-handler.ts +18 -4
- package/src/server/index.ts +7 -7
- package/src/server/lifecycle.ts +13 -1
- package/src/server/management/combo-routes.ts +17 -2
- package/src/server/management/oauth-account-routes.ts +10 -1
- package/src/server/management/provider-completion-test.ts +289 -0
- package/src/server/management/provider-routes.ts +9 -1
- package/src/server/management/route-registry.ts +2 -0
- package/src/server/messages-native.ts +34 -7
- package/src/server/messages-shared.ts +4 -1
- package/src/server/request-log.ts +7 -2
- package/src/server/responses/account-auto-pause-recovery.ts +26 -0
- package/src/server/responses/adapter-continuation.ts +34 -18
- package/src/server/responses/adapter-delivery.ts +53 -6
- package/src/server/responses/adapter-dispatch.ts +104 -29
- package/src/server/responses/anthropic-thinking-recovery.ts +153 -0
- package/src/server/responses/combo-session-recall.ts +25 -0
- package/src/server/responses/combo-stream-preflight.ts +6 -3
- package/src/server/responses/combo-tool-loop-pin.ts +70 -0
- package/src/server/responses/compact.ts +8 -5
- package/src/server/responses/compaction-recovery.ts +2 -0
- package/src/server/responses/context-overflow.ts +141 -2
- package/src/server/responses/core-codex-account.ts +9 -3
- package/src/server/responses/core-combo.ts +34 -7
- package/src/server/responses/core-lifetime.ts +3 -2
- package/src/server/responses/core-options.ts +2 -0
- package/src/server/responses/core-replay.ts +5 -1
- package/src/server/responses/core.ts +12 -9
- package/src/server/responses/empty-completion-guard.ts +2 -0
- package/src/server/responses/fetch-helpers.ts +35 -4
- package/src/server/responses/jev-model-invoke.ts +13 -14
- package/src/server/responses/native-steering-log.ts +2 -2
- package/src/server/responses/passthrough-delivery.ts +3 -2
- package/src/server/responses/passthrough-dispatch.ts +30 -33
- package/src/server/responses/passthrough-execution.ts +3 -2
- package/src/server/responses/passthrough-observations.ts +38 -0
- package/src/server/responses/request-compression.ts +98 -0
- package/src/server/responses/request-prepare.ts +1 -0
- package/src/server/responses/request-transport.ts +32 -3
- package/src/server/responses/response-effects.ts +1 -0
- package/src/server/responses/response-tool-ownership.ts +21 -0
- package/src/server/responses/run-turn-execution.ts +2 -0
- package/src/server/responses/sidecar-execution.ts +59 -10
- package/src/server/responses/sidecar-send-budget.ts +2 -2
- package/src/server/responses/terminal-guard.ts +2 -0
- package/src/server/responses-undeclared-tool-guard.ts +1 -1
- package/src/server/startup-action-control.ts +2 -2
- package/src/server/startup-health-cache.ts +2 -2
- package/src/server/windows-tray-control.ts +2 -2
- package/src/server/ws-bridge.ts +2 -0
- package/src/service/definition.ts +9 -1
- package/src/types/account-auto-pause.ts +15 -0
- package/src/types/config.ts +23 -1
- package/src/types/opencode-zen.ts +9 -3
- package/src/types/provider.ts +9 -5
- package/src/types/request.ts +30 -4
- package/src/types/wire.ts +4 -2
- package/src/types.ts +2 -0
- package/src/usage/cost.ts +17 -1
- package/src/usage/expected-prices.ts +3 -1
- package/src/usage/jev-stats.ts +4 -0
- package/src/usage/log.ts +2 -0
- package/src/usage/summary.ts +16 -3
- package/src/usage/telemetry-contract.ts +1 -0
- package/src/web-search/anthropic-history.ts +59 -0
- package/src/web-search/loop.ts +212 -73
- package/src/web-search/progress-stream.ts +14 -1
- package/src/web-search/run-turn-loop.ts +9 -3
- package/src/web-search/sidecar-search-replay.ts +96 -0
- package/gui/dist/assets/App-ZfYbLHkB.js +0 -58
- package/gui/dist/assets/App-wT3mXqFz.css +0 -1
- package/gui/dist/assets/index-CfYDMj34.js +0 -94
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* other.
|
|
14
14
|
*/
|
|
15
15
|
import { sniffImageDimensions } from "../adapters/anthropic-image-guard";
|
|
16
|
+
import { anthropicCacheWrite1hTokens } from "../adapters/anthropic/cache-retention";
|
|
16
17
|
import { createToolCallIdAllocator } from "../adapters/tool-call-id";
|
|
17
18
|
import { idleDeadline } from "../lib/abort";
|
|
18
19
|
import {
|
|
@@ -29,12 +30,13 @@ import type { RequestLogContext } from "./request-log";
|
|
|
29
30
|
import { recordGenerationEvent } from "./request-log-generation-window";
|
|
30
31
|
import type { OcxConfig } from "../types";
|
|
31
32
|
|
|
32
|
-
export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: number; outputTokens: number; cachedInputTokens?: number; cacheReadInputTokens?: number; cacheCreationInputTokens?: number } | undefined {
|
|
33
|
+
export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: number; outputTokens: number; cachedInputTokens?: number; cacheReadInputTokens?: number; cacheCreationInputTokens?: number; cacheCreation1hInputTokens?: number } | undefined {
|
|
33
34
|
if (!usage) return undefined;
|
|
34
35
|
const num = (v: unknown) => typeof v === "number" ? v : 0;
|
|
35
36
|
const hasCache = usage.cache_read_input_tokens !== undefined || usage.cache_creation_input_tokens !== undefined;
|
|
36
37
|
const read = num(usage.cache_read_input_tokens);
|
|
37
38
|
const write = num(usage.cache_creation_input_tokens);
|
|
39
|
+
const write1h = anthropicCacheWrite1hTokens(usage, write);
|
|
38
40
|
// Anthropic input_tokens excludes cache read/write; normalize to the canonical
|
|
39
41
|
// inclusive convention (types.ts OcxUsage / devlog 070). cached = READS only.
|
|
40
42
|
return {
|
|
@@ -44,6 +46,7 @@ export function anthropicUsageToOcx(usage: Rec | undefined): { inputTokens: numb
|
|
|
44
46
|
cachedInputTokens: read,
|
|
45
47
|
cacheReadInputTokens: read,
|
|
46
48
|
cacheCreationInputTokens: write,
|
|
49
|
+
...(write1h !== undefined ? { cacheCreation1hInputTokens: write1h } : {}),
|
|
47
50
|
} : {}),
|
|
48
51
|
};
|
|
49
52
|
}
|
|
@@ -982,7 +982,7 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
|
|
|
982
982
|
const raw = usage as {
|
|
983
983
|
input_tokens?: unknown;
|
|
984
984
|
output_tokens?: unknown;
|
|
985
|
-
input_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown };
|
|
985
|
+
input_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown; cache_write_1h_tokens?: unknown };
|
|
986
986
|
output_tokens_details?: { reasoning_tokens?: unknown };
|
|
987
987
|
total_tokens?: unknown;
|
|
988
988
|
prompt_tokens?: unknown;
|
|
@@ -1004,6 +1004,9 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
|
|
|
1004
1004
|
...(typeof raw.input_tokens_details?.cache_write_tokens === "number"
|
|
1005
1005
|
? { cacheCreationInputTokens: raw.input_tokens_details.cache_write_tokens }
|
|
1006
1006
|
: {}),
|
|
1007
|
+
...(typeof raw.input_tokens_details?.cache_write_1h_tokens === "number"
|
|
1008
|
+
? { cacheCreation1hInputTokens: raw.input_tokens_details.cache_write_1h_tokens }
|
|
1009
|
+
: {}),
|
|
1007
1010
|
...(typeof raw.output_tokens_details?.reasoning_tokens === "number"
|
|
1008
1011
|
? { reasoningOutputTokens: raw.output_tokens_details.reasoning_tokens }
|
|
1009
1012
|
: {}),
|
|
@@ -1864,7 +1867,7 @@ export function aggregateAttemptUsage(
|
|
|
1864
1867
|
if (usages.length === 0) return { status };
|
|
1865
1868
|
|
|
1866
1869
|
const sumOptional = (
|
|
1867
|
-
key: "cachedInputTokens" | "cacheReadInputTokens" | "cacheCreationInputTokens"
|
|
1870
|
+
key: "cachedInputTokens" | "cacheReadInputTokens" | "cacheCreationInputTokens" | "cacheCreation1hInputTokens"
|
|
1868
1871
|
| "reasoningOutputTokens" | "providerCredits",
|
|
1869
1872
|
): number | undefined => {
|
|
1870
1873
|
const present = usages.flatMap(usage => (
|
|
@@ -1875,6 +1878,7 @@ export function aggregateAttemptUsage(
|
|
|
1875
1878
|
const cachedInputTokens = sumOptional("cachedInputTokens");
|
|
1876
1879
|
const cacheReadInputTokens = sumOptional("cacheReadInputTokens");
|
|
1877
1880
|
const cacheCreationInputTokens = sumOptional("cacheCreationInputTokens");
|
|
1881
|
+
const cacheCreation1hInputTokens = sumOptional("cacheCreation1hInputTokens");
|
|
1878
1882
|
const reasoningOutputTokens = sumOptional("reasoningOutputTokens");
|
|
1879
1883
|
const providerCredits = sumOptional("providerCredits");
|
|
1880
1884
|
const totalTokens = usages.reduce(
|
|
@@ -1888,6 +1892,7 @@ export function aggregateAttemptUsage(
|
|
|
1888
1892
|
...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
|
|
1889
1893
|
...(cacheReadInputTokens !== undefined ? { cacheReadInputTokens } : {}),
|
|
1890
1894
|
...(cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens } : {}),
|
|
1895
|
+
...(cacheCreation1hInputTokens !== undefined ? { cacheCreation1hInputTokens } : {}),
|
|
1891
1896
|
...(reasoningOutputTokens !== undefined ? { reasoningOutputTokens } : {}),
|
|
1892
1897
|
...(providerCredits !== undefined ? { providerCredits } : {}),
|
|
1893
1898
|
...(status === "estimated" ? { estimated: true } : {}),
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ResponsesSendBudget } from "./request-send-budget";
|
|
2
|
+
|
|
3
|
+
const moves = new WeakMap<object, number>();
|
|
4
|
+
/** One same-provider credential move, funded by the existing recovery owner. */
|
|
5
|
+
export async function recoverAutoPausedAccount<T>(options: {
|
|
6
|
+
requestKey: object;
|
|
7
|
+
remaining?: number;
|
|
8
|
+
reserve: ResponsesSendBudget["reserveCredentialHop"];
|
|
9
|
+
refresh: () => Promise<boolean>;
|
|
10
|
+
refetch: () => Promise<T>;
|
|
11
|
+
setPermit: (permit: ResponsesSendBudget["pendingHopPermit"]) => void;
|
|
12
|
+
}): Promise<T | undefined> {
|
|
13
|
+
const count = moves.get(options.requestKey) ?? 0;
|
|
14
|
+
if (count >= 3 || options.remaining !== undefined && options.remaining <= 0) return;
|
|
15
|
+
const hop = options.reserve("auth-recovery", "account-auto-pause", true);
|
|
16
|
+
if (!hop.allowed) return;
|
|
17
|
+
try {
|
|
18
|
+
if (!await options.refresh()) return;
|
|
19
|
+
moves.set(options.requestKey, count + 1);
|
|
20
|
+
options.setPermit(hop.permit);
|
|
21
|
+
return await options.refetch();
|
|
22
|
+
} finally {
|
|
23
|
+
options.setPermit(undefined);
|
|
24
|
+
hop.permit?.release();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo";
|
|
1
2
|
import { rotateAnthropicAccountOnResponseForInstance } from "../../oauth/anthropic-account-refusal";
|
|
2
3
|
import type { ResponsesRequestContext } from "./core-options";
|
|
3
4
|
import type { PreparedResponsesRequest } from "./request-prepare";
|
|
@@ -29,6 +30,7 @@ import {
|
|
|
29
30
|
applyUpstreamRecoveryInit,
|
|
30
31
|
isNonReplayableResponse,
|
|
31
32
|
prepareSameTarget429Wait,
|
|
33
|
+
SendBudgetExhaustedError,
|
|
32
34
|
} from "../../lib/upstream-retry";
|
|
33
35
|
import { unwrapUpstreamRetryEvidenceError } from "../../codex/auth-context";
|
|
34
36
|
import { AdmissionModelDeniedError, MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope";
|
|
@@ -121,6 +123,7 @@ export function createAdapterContinuations(
|
|
|
121
123
|
| "rateLimitRetries"
|
|
122
124
|
| "stallTimeoutMs"
|
|
123
125
|
| "localUpstream"
|
|
126
|
+
| "thinkingSignatureRecovery"
|
|
124
127
|
>,
|
|
125
128
|
) {
|
|
126
129
|
const { options, logCtx, config } = requestContext;
|
|
@@ -159,6 +162,10 @@ export function createAdapterContinuations(
|
|
|
159
162
|
errorType: MODEL_NOT_ALLOWED_FOR_KEY, code: MODEL_NOT_ALLOWED_FOR_KEY, retryable: false,
|
|
160
163
|
};
|
|
161
164
|
}
|
|
165
|
+
if (refusal instanceof SendBudgetExhaustedError) return {
|
|
166
|
+
type: "error", status: 429, errorType: "rate_limit_error", code: refusal.code,
|
|
167
|
+
message: refusal.message, retryable: false,
|
|
168
|
+
};
|
|
162
169
|
return {
|
|
163
170
|
type: "error",
|
|
164
171
|
message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}`,
|
|
@@ -309,6 +316,14 @@ export function createAdapterContinuations(
|
|
|
309
316
|
const recoveryKind = nextContinuationRecoveryKind;
|
|
310
317
|
nextContinuationRecoveryKind = undefined;
|
|
311
318
|
response = await fetchContinuation(recoveryKind);
|
|
319
|
+
bindAnthropicRejectionResponse(transportState.sameTargetRequest, response);
|
|
320
|
+
if (initialRecoveryKind === "empty-completion" && await adapterExchange.thinkingSignatureRecovery.prepareHttp(
|
|
321
|
+
response, transportState.activeAdapter.name, nextParsed,
|
|
322
|
+
)) {
|
|
323
|
+
try { void response.body?.cancel().catch(() => {}); } catch { /* closed */ }
|
|
324
|
+
invalidateSameTargetRequest();
|
|
325
|
+
response = await fetchContinuation("opaque-blob-rejection");
|
|
326
|
+
}
|
|
312
327
|
if (kiroRefusalPendingReplay) {
|
|
313
328
|
try { void kiroRefusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
314
329
|
kiroRefusalPendingReplay = undefined;
|
|
@@ -704,24 +719,25 @@ export function createAdapterContinuations(
|
|
|
704
719
|
// listener that can cancel its reader even while the parser holds the body lock.
|
|
705
720
|
// The shared signal is not aborted on timeout: this generator must still emit
|
|
706
721
|
// the error that the already-live Responses bridge turns into a terminal.
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
bodyInactivityMs,
|
|
712
|
-
|
|
713
|
-
)
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
yield
|
|
724
|
-
}
|
|
722
|
+
const parse = async function* (current: Response): AsyncGenerator<AdapterEvent> {
|
|
723
|
+
if (!current.ok) {
|
|
724
|
+
yield await adapterExchange.thinkingSignatureRecovery.projectHttpFailure(current, upstream.signal);
|
|
725
|
+
} else if (nextParsed.stream) {
|
|
726
|
+
yield* readResponseStreamWithInactivity(current, upstream.signal, bodyInactivityMs,
|
|
727
|
+
guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata));
|
|
728
|
+
} else if (transportState.activeAdapter.parseResponse) {
|
|
729
|
+
yield* await readResponseBodyWithInactivity(current, upstream.signal, bodyInactivityMs,
|
|
730
|
+
guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata));
|
|
731
|
+
} else {
|
|
732
|
+
try { void current.body?.cancel().catch(() => {}); } catch { /* closed */ }
|
|
733
|
+
yield { type: "error", message: "Provider continuation does not support response parsing" };
|
|
734
|
+
}
|
|
735
|
+
};
|
|
736
|
+
yield* adapterExchange.thinkingSignatureRecovery.wrap(parse(response), async function* () {
|
|
737
|
+
invalidateSameTargetRequest();
|
|
738
|
+
yield* parse(await fetchContinuation("opaque-blob-rejection"));
|
|
739
|
+
}, () => transportState.activeAdapter.name,
|
|
740
|
+
initialRecoveryKind === "empty-completion" && !isNonReplayableResponse(response), response, nextParsed);
|
|
725
741
|
} catch (error) {
|
|
726
742
|
// Classify the thrown error before reading signal state: cancelling the stalled
|
|
727
743
|
// source can synchronously abort the shared signal, which would otherwise report
|
|
@@ -23,6 +23,9 @@ import { resolveStallTimeoutMs } from "../../stall-timeout";
|
|
|
23
23
|
import { clientEncoderForDelivery, deliverClientEncodedResponse } from "../inference/client-encoder-delivery";
|
|
24
24
|
import { noteKiroServedSuccess } from "../../providers/kiro-usage";
|
|
25
25
|
import { persistKiroAccountState } from "../../providers/kiro-account-state-disk";
|
|
26
|
+
import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors";
|
|
27
|
+
import { sanitizeNonReplayableUpstreamError } from "./non-replayable-error";
|
|
28
|
+
import { isNonReplayableResponse } from "../../lib/upstream-retry";
|
|
26
29
|
import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
|
|
27
30
|
import { guardEmptyToolCatalogEvents, guardEmptyToolCatalogStream } from "../../lib/empty-tool-catalog";
|
|
28
31
|
|
|
@@ -49,7 +52,7 @@ export async function deliverAdapterResponse(
|
|
|
49
52
|
| "notifyResponseComplete"
|
|
50
53
|
>,
|
|
51
54
|
completionPolicy: Pick<ResponsesCompletionPolicy, "emptyCompletionGuardEnabled">,
|
|
52
|
-
adapterExchange: Pick<AdapterExchange, "upstreamResponse" | "upstream" | "cleanupUpstreamAbort" | "localUpstream">,
|
|
55
|
+
adapterExchange: Pick<AdapterExchange, "upstreamResponse" | "upstream" | "cleanupUpstreamAbort" | "localUpstream" | "thinkingSignatureRecovery" | "refetchThinkingSignature">,
|
|
53
56
|
continuationState: Pick<AdapterContinuations, "terminalGuardEnabled" | "fetchTerminalGuardContinuation" | "fetchGuardedEmptyCompletionRetry">,
|
|
54
57
|
): Promise<Response> {
|
|
55
58
|
const { logCtx, options, config } = requestContext;
|
|
@@ -61,7 +64,7 @@ export async function deliverAdapterResponse(
|
|
|
61
64
|
rememberKiroDeliveredFinalAnswer,
|
|
62
65
|
responseStateOptions,
|
|
63
66
|
} = requestState;
|
|
64
|
-
const { upstreamResponse, upstream, cleanupUpstreamAbort, localUpstream } = adapterExchange;
|
|
67
|
+
const { upstreamResponse, upstream, cleanupUpstreamAbort, localUpstream, thinkingSignatureRecovery, refetchThinkingSignature } = adapterExchange;
|
|
65
68
|
const {
|
|
66
69
|
terminalGuardEnabled,
|
|
67
70
|
fetchTerminalGuardContinuation,
|
|
@@ -77,6 +80,43 @@ export async function deliverAdapterResponse(
|
|
|
77
80
|
const { routedCompaction } = sidecarState;
|
|
78
81
|
const bodyInactivityMs = resolveStallTimeoutMs(config.stallTimeoutSec, { localUpstream });
|
|
79
82
|
const upstreamRequestsStream = parsed.stream || isCanonicalOpenAiForwardProvider(route.provider);
|
|
83
|
+
const recoverThinkingSignature = (events: AsyncIterable<AdapterEvent>): AsyncIterable<AdapterEvent> =>
|
|
84
|
+
thinkingSignatureRecovery.wrap(events, async function* () {
|
|
85
|
+
const result = await refetchThinkingSignature();
|
|
86
|
+
let response = "failed" in result ? result.failed : result;
|
|
87
|
+
// A replacement can be an HTTP/auth/budget refusal; never feed its JSON body to an SSE parser.
|
|
88
|
+
if (!response.ok) {
|
|
89
|
+
// A local refusal closes the upstream signal before returning its generated JSON.
|
|
90
|
+
// Reading that bounded local body still honors caller cancellation.
|
|
91
|
+
const errorSignal = "failed" in result ? options.abortSignal ?? new AbortController().signal : upstream.signal;
|
|
92
|
+
const nonReplayable = isNonReplayableResponse(response);
|
|
93
|
+
if (nonReplayable) response = await sanitizeNonReplayableUpstreamError(response, errorSignal);
|
|
94
|
+
const body = await readDisplaySafeErrorText(response, errorSignal, "Upstream recovery failed");
|
|
95
|
+
const error = normalizeUpstreamErrorText(body, "Upstream recovery failed");
|
|
96
|
+
yield { type: "error", status: response.status, message: error.message ?? error.safeText,
|
|
97
|
+
errorType: error.type ?? "upstream_error", ...(error.code ? { code: error.code } : {}),
|
|
98
|
+
...(nonReplayable ? { retryable: false } : {}) };
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
try {
|
|
102
|
+
const retryEvents = upstreamRequestsStream
|
|
103
|
+
? readResponseStreamWithInactivity(response, upstream.signal, bodyInactivityMs,
|
|
104
|
+
guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata))
|
|
105
|
+
: (async function* () {
|
|
106
|
+
yield* await readResponseBodyWithInactivity(response, upstream.signal, bodyInactivityMs,
|
|
107
|
+
guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata));
|
|
108
|
+
})();
|
|
109
|
+
for await (const event of retryEvents) {
|
|
110
|
+
options.onCompactionRecoveryAdapterEvent?.(event);
|
|
111
|
+
yield event;
|
|
112
|
+
}
|
|
113
|
+
} catch (error) {
|
|
114
|
+
if (!(error instanceof ResponseBodyInactivityError)) throw error;
|
|
115
|
+
yield { type: "error", message: "Upstream recovery response body stalled before completing",
|
|
116
|
+
status: 504, errorType: "upstream_error" };
|
|
117
|
+
}
|
|
118
|
+
}, () => transportState.activeAdapter.name, !isNonReplayableResponse(upstreamResponse), upstreamResponse);
|
|
119
|
+
|
|
80
120
|
const clientToolCatalog = requestState.clientToolCatalogRestriction && route.provider.authMode !== "forward"
|
|
81
121
|
? requestState.clientToolCatalogRestriction : undefined;
|
|
82
122
|
const deliveryToolBridgeMaps = clientToolCatalog
|
|
@@ -110,15 +150,16 @@ export async function deliverAdapterResponse(
|
|
|
110
150
|
throw error;
|
|
111
151
|
}
|
|
112
152
|
})();
|
|
153
|
+
const recoveredEvents = recoverThinkingSignature(initialEventStream);
|
|
113
154
|
const eventStream = terminalGuardEnabled
|
|
114
155
|
? guardTerminalEventStream({
|
|
115
156
|
parsed,
|
|
116
|
-
firstEvents:
|
|
157
|
+
firstEvents: recoveredEvents,
|
|
117
158
|
adapterName: transportState.activeAdapter.name,
|
|
118
159
|
maxAutoContinuations: 1,
|
|
119
160
|
continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream),
|
|
120
161
|
})
|
|
121
|
-
:
|
|
162
|
+
: recoveredEvents;
|
|
122
163
|
// Optional advisor guard (registered per request by the sidecar planner; absent unless the
|
|
123
164
|
// user enabled the advisor): holds synthetic `advisor` tool calls, consults the configured
|
|
124
165
|
// expert model, and re-dispatches the worker. Sits inside the empty-completion guard so an
|
|
@@ -147,6 +188,7 @@ export async function deliverAdapterResponse(
|
|
|
147
188
|
// One completion owner for both deliveries: the bridge calls it from its terminal, the
|
|
148
189
|
// direct client encoder from the fold of the same events.
|
|
149
190
|
const onCompletedResponse = (response: Record<string, unknown>, providerState?: OcxProviderContinuationState) => {
|
|
191
|
+
if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(response);
|
|
150
192
|
const served = transportState.sentOAuthSnapshot;
|
|
151
193
|
if (transportState.activeAdapter.name === "kiro" && response.status === "completed" && served
|
|
152
194
|
&& noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState();
|
|
@@ -250,18 +292,22 @@ export async function deliverAdapterResponse(
|
|
|
250
292
|
));
|
|
251
293
|
}
|
|
252
294
|
for (const event of initialEvents) options.onCompactionRecoveryAdapterEvent?.(event);
|
|
295
|
+
const recoveredEvents: AdapterEvent[] = [];
|
|
296
|
+
for await (const event of recoverThinkingSignature((async function* () { yield* initialEvents; })())) {
|
|
297
|
+
recoveredEvents.push(event);
|
|
298
|
+
}
|
|
253
299
|
let guardedEvents: AdapterEvent[];
|
|
254
300
|
if (terminalGuardEnabled) {
|
|
255
301
|
guardedEvents = [];
|
|
256
302
|
for await (const event of guardTerminalEventStream({
|
|
257
303
|
parsed,
|
|
258
|
-
firstEvents: (async function* () { yield*
|
|
304
|
+
firstEvents: (async function* () { yield* recoveredEvents; })(),
|
|
259
305
|
adapterName: transportState.activeAdapter.name,
|
|
260
306
|
maxAutoContinuations: 1,
|
|
261
307
|
continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream),
|
|
262
308
|
})) guardedEvents.push(event);
|
|
263
309
|
} else {
|
|
264
|
-
guardedEvents =
|
|
310
|
+
guardedEvents = recoveredEvents;
|
|
265
311
|
}
|
|
266
312
|
// Optional advisor guard — same semantics as the streaming branch above.
|
|
267
313
|
if (parsed._advisorGuard) {
|
|
@@ -325,6 +371,7 @@ export async function deliverAdapterResponse(
|
|
|
325
371
|
// #1926 gap 2: same buffered-path durability bound as the primary branch.
|
|
326
372
|
await awaitThoughtSignatureDurability();
|
|
327
373
|
if (adapterResponseReachedServingTerminal(events, json)) {
|
|
374
|
+
if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(json);
|
|
328
375
|
const served = transportState.sentOAuthSnapshot;
|
|
329
376
|
if (transportState.activeAdapter.name === "kiro" && json.status === "completed" && served
|
|
330
377
|
&& noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState();
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
+
import { recoverAutoPausedAccount } from "./account-auto-pause-recovery";
|
|
2
|
+
import { wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause";
|
|
3
|
+
import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo";
|
|
1
4
|
import { deferPhysicalSendReporter } from "./deferred-send-reporter";
|
|
5
|
+
import { usesApiKeyAccount } from "../../providers/label";
|
|
2
6
|
import { spendLedgerStorageErrorResponse } from "./spend-storage-error";
|
|
3
7
|
import { classifyAnthropic429 } from "../../oauth/anthropic-rate-limit-policy";
|
|
4
8
|
import { rotateAnthropicAccountOnResponseForInstance } from "../../oauth/anthropic-account-refusal";
|
|
@@ -87,6 +91,7 @@ import {
|
|
|
87
91
|
reasoningEffortRejectionText,
|
|
88
92
|
anthropicFastRefused,
|
|
89
93
|
} from "./core-opaque-recovery";
|
|
94
|
+
import { createAnthropicThinkingRecovery } from "./anthropic-thinking-recovery";
|
|
90
95
|
import { shouldAttemptImageTierRetry } from "../image-retry";
|
|
91
96
|
import {
|
|
92
97
|
isTransientConsoleGoUploadRejection,
|
|
@@ -139,6 +144,7 @@ export async function prepareAdapterExchange(
|
|
|
139
144
|
| "sameTargetToken"
|
|
140
145
|
| "transportToken"
|
|
141
146
|
| "oauthDispatch"
|
|
147
|
+
| "recoverAutoPausedSelection"
|
|
142
148
|
| "imageTierBias"
|
|
143
149
|
| "oauth401ReplayAttempted"
|
|
144
150
|
| "isOAuth401ReplayProvider"
|
|
@@ -392,11 +398,12 @@ export async function prepareAdapterExchange(
|
|
|
392
398
|
upstreamResponse = new Response(text, { status: original.status, statusText: original.statusText, headers });
|
|
393
399
|
return upstreamResponse;
|
|
394
400
|
};
|
|
401
|
+
const autoPauseEnabled = config.providers[route.providerName]?.accountAutoPause?.enabled === true;
|
|
395
402
|
const billingPoolRequest = isAnthropicBillingKeyPool(route.provider);
|
|
396
403
|
let billingRecoveryActive = false;
|
|
397
|
-
const genericRefetchCountsSends = () => sendBudgetState.adapterSendBudget?.spendEnforced === true
|
|
404
|
+
const genericRefetchCountsSends = () => options.comboAttempt === true || sendBudgetState.adapterSendBudget?.spendEnforced === true
|
|
398
405
|
|| transientRetryPolicyFor(route.provider) !== null || resetReplayPolicyFor(route.provider) !== null
|
|
399
|
-
|| billingPoolRequest || billingRecoveryActive;
|
|
406
|
+
|| billingPoolRequest || billingRecoveryActive || autoPauseEnabled;
|
|
400
407
|
let handoffPermit: ResponsesSendBudget["pendingHopPermit"];
|
|
401
408
|
try {
|
|
402
409
|
if (transportState.activeAdapter.fetchResponse) {
|
|
@@ -518,10 +525,10 @@ export async function prepareAdapterExchange(
|
|
|
518
525
|
label: safeHostLabel(builtInitialRequest.url),
|
|
519
526
|
claimAmbiguousResend: claimPreHeaderResend,
|
|
520
527
|
...(sendBudgetState.adapterSendBudget?.spendEnforced || options.comboDispatchPermit || transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt
|
|
521
|
-
|| billingPoolRequest
|
|
528
|
+
|| billingPoolRequest || autoPauseEnabled
|
|
522
529
|
? {
|
|
523
530
|
// A pending compaction permit already booked this leg's first physical send.
|
|
524
|
-
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt
|
|
531
|
+
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt || autoPauseEnabled
|
|
525
532
|
? {
|
|
526
533
|
attempts: Math.min(initialSendCap,
|
|
527
534
|
sendBudgetState.initialSendAllowance(initialSendCap) + (compactPrepaid ? 1 : 0)),
|
|
@@ -591,7 +598,7 @@ export async function prepareAdapterExchange(
|
|
|
591
598
|
let rateLimitRetries = 0;
|
|
592
599
|
// Shared with the terminal-guard continuation below: an image-tier reduction that let the
|
|
593
600
|
// main request clear a 413 must not be forgotten on the very next continuation build.
|
|
594
|
-
|
|
601
|
+
// Recovery helpers also serve initial HTTP-200 adapter errors during delivery.
|
|
595
602
|
// Recovery loop: multi-key 429 failover + at most ONE opaque-state rebuild and ONE
|
|
596
603
|
// anthropic 413 tightened retry
|
|
597
604
|
// (devlog/260714_image_normalization_pipeline/030). One mutable activeAdapter serves
|
|
@@ -600,6 +607,9 @@ export async function prepareAdapterExchange(
|
|
|
600
607
|
// 413→429 rotation cannot silently undo the tightening.
|
|
601
608
|
let imageRetryAttempted = false;
|
|
602
609
|
const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false };
|
|
610
|
+
const thinkingSignatureRecovery = createAnthropicThinkingRecovery(
|
|
611
|
+
parsed, upstream.signal, opaqueBlobRecoveryGuard, () => !sendBudgetExhausted(),
|
|
612
|
+
);
|
|
603
613
|
// Console Go answers a transient 400 "Invalid upload request." for bodies it accepts
|
|
604
614
|
// moments later; at most one byte-identical replay is allowed per request.
|
|
605
615
|
const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false };
|
|
@@ -753,7 +763,9 @@ export async function prepareAdapterExchange(
|
|
|
753
763
|
: undefined;
|
|
754
764
|
let refetchPermitUsed = false;
|
|
755
765
|
const refetchReporter = refetchAllowance ? transientSendReporter(refetchAllowance.permit) : undefined;
|
|
756
|
-
const deferredRefetch = billingPoolRequest
|
|
766
|
+
const deferredRefetch = (billingPoolRequest || options.comboAttempt) && refetchReporter
|
|
767
|
+
? deferPhysicalSendReporter(refetchReporter) : undefined;
|
|
768
|
+
const pendingKeyMetadata = deferredRefetch && usesApiKeyAccount(route.provider);
|
|
757
769
|
try {
|
|
758
770
|
return await refetchWithPolicy(
|
|
759
771
|
recoveryKind => {
|
|
@@ -762,14 +774,14 @@ export async function prepareAdapterExchange(
|
|
|
762
774
|
});
|
|
763
775
|
// Prepare key metadata before the guarded transport commits its attempt;
|
|
764
776
|
// counters and permits still wait for the physical executor after pacing.
|
|
765
|
-
if (
|
|
777
|
+
if (pendingKeyMetadata) transportState.noteRoutedAttemptSend(retryEstimate, recoveryKind ?? recovery);
|
|
766
778
|
const confirmDispatch = () => {
|
|
767
779
|
if (refetchAllowance?.permit && !refetchPermitUsed) {
|
|
768
780
|
if (!refetchAllowance.permit.use()) throw new SendBudgetExhaustedError(safeHostLabel(retryRequest.url));
|
|
769
781
|
refetchPermitUsed = true;
|
|
770
782
|
}
|
|
771
|
-
if (!
|
|
772
|
-
//
|
|
783
|
+
if (!pendingKeyMetadata) transportState.noteRoutedAttemptSend(retryEstimate, recoveryKind ?? recovery);
|
|
784
|
+
// Counted Combo/pool sends confirm at the physical executor; legacy
|
|
773
785
|
// helper paths retain their existing thunk admission boundary.
|
|
774
786
|
onDispatch?.();
|
|
775
787
|
if (preserveFailureResponse) replacementAdmitted = true;
|
|
@@ -863,6 +875,11 @@ export async function prepareAdapterExchange(
|
|
|
863
875
|
return { failed: formatErrorResponse(502, "upstream_error", msg) };
|
|
864
876
|
}
|
|
865
877
|
};
|
|
878
|
+
const refetchThinkingSignature = () => {
|
|
879
|
+
invalidateSameTargetRequest();
|
|
880
|
+
return rebuildAndRefetch("opaque-blob-rejection");
|
|
881
|
+
};
|
|
882
|
+
if (!upstreamResponse.ok) {
|
|
866
883
|
const rotateAntigravityAuth = async (
|
|
867
884
|
failedResponse: Response,
|
|
868
885
|
recovery: AttemptRecoveryKind,
|
|
@@ -877,7 +894,7 @@ export async function prepareAdapterExchange(
|
|
|
877
894
|
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
878
895
|
const hop = reserveCredentialHop("auth-recovery",
|
|
879
896
|
`${route.providerName}|${route.modelId}|antigravity-auth`,
|
|
880
|
-
!adapterOwnsDispatch && (
|
|
897
|
+
!adapterOwnsDispatch && genericRefetchCountsSends());
|
|
881
898
|
if (!hop.allowed) return null;
|
|
882
899
|
try {
|
|
883
900
|
const snapshot = await failoverAccountSnapshot(route.providerName, nextId);
|
|
@@ -913,6 +930,15 @@ export async function prepareAdapterExchange(
|
|
|
913
930
|
};
|
|
914
931
|
// Keep recovery kinds in sync with the native Responses `passthroughRecovery:` loop above.
|
|
915
932
|
recovery: for (;;) {
|
|
933
|
+
if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") {
|
|
934
|
+
const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop,
|
|
935
|
+
remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed),
|
|
936
|
+
refresh: () => transportState.recoverAutoPausedSelection(parsed), refetch: () => rebuildAndRefetch("oauth-account-429"), setPermit: permit => { sendBudgetState.pendingHopPermit = permit; } });
|
|
937
|
+
if (!recovered) break;
|
|
938
|
+
if ("failed" in recovered) return recovered.failed;
|
|
939
|
+
upstreamResponse = recovered; continue recovery;
|
|
940
|
+
}
|
|
941
|
+
|
|
916
942
|
// Preserve the terminal verdict through adapter and combo error formatting.
|
|
917
943
|
// This also covers a reset reached by a 401/429/413 recovery refetch.
|
|
918
944
|
if (isNonReplayableResponse(upstreamResponse)) {
|
|
@@ -920,8 +946,19 @@ export async function prepareAdapterExchange(
|
|
|
920
946
|
return await sanitizeNonReplayableUpstreamError(upstreamResponse, upstream.signal);
|
|
921
947
|
} finally { cleanupUpstreamAbort(); }
|
|
922
948
|
}
|
|
949
|
+
bindAnthropicRejectionResponse(transportState.sameTargetRequest, upstreamResponse);
|
|
950
|
+
if (await thinkingSignatureRecovery.prepareHttp(
|
|
951
|
+
upstreamResponse, transportState.activeAdapter.name,
|
|
952
|
+
)) {
|
|
953
|
+
try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* closed */ }
|
|
954
|
+
invalidateSameTargetRequest();
|
|
955
|
+
const result = await rebuildAndRefetch("opaque-blob-rejection");
|
|
956
|
+
if ("failed" in result) return result.failed;
|
|
957
|
+
upstreamResponse = result;
|
|
958
|
+
continue recovery;
|
|
959
|
+
}
|
|
923
960
|
if (
|
|
924
|
-
upstreamResponse.status === 401
|
|
961
|
+
upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
|
|
925
962
|
&& isOAuth401ReplayProvider
|
|
926
963
|
&& transportState.sentOAuthSnapshot
|
|
927
964
|
&& !oauth401ReplayAttempted
|
|
@@ -957,7 +994,7 @@ export async function prepareAdapterExchange(
|
|
|
957
994
|
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
958
995
|
const hop = reserveCredentialHop("auth-recovery",
|
|
959
996
|
`${route.providerName}|${route.modelId}|terminal-refresh-account`,
|
|
960
|
-
!adapterOwnsDispatch && (
|
|
997
|
+
!adapterOwnsDispatch && genericRefetchCountsSends());
|
|
961
998
|
if (hop.allowed) {
|
|
962
999
|
try {
|
|
963
1000
|
const admitted = await applyFailoverSnapshot(alternate);
|
|
@@ -1041,7 +1078,7 @@ export async function prepareAdapterExchange(
|
|
|
1041
1078
|
continue recovery;
|
|
1042
1079
|
}
|
|
1043
1080
|
|
|
1044
|
-
if (route.providerName === "google-antigravity" && upstreamResponse.status === 401
|
|
1081
|
+
if (route.providerName === "google-antigravity" && upstreamResponse.status === 401 && !wasPermanentAccountRefusal(upstreamResponse)
|
|
1045
1082
|
&& oauth401ReplayAttempted && !antigravityAuthRotationAttempted
|
|
1046
1083
|
&& !isNonReplayableResponse(upstreamResponse)) {
|
|
1047
1084
|
antigravityAuthRotationAttempted = true;
|
|
@@ -1327,12 +1364,31 @@ export async function prepareAdapterExchange(
|
|
|
1327
1364
|
&& anthropicInstance
|
|
1328
1365
|
&& transportState.anthropicPoolAccountId
|
|
1329
1366
|
) {
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1367
|
+
// Same admission as the other rotation ladders: the roster cap cannot see the shared
|
|
1368
|
+
// send budget. A canRetry term, not a break -- the call also records the refusal.
|
|
1369
|
+
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
1370
|
+
const hop = reserveCredentialHop(
|
|
1371
|
+
"auth-recovery",
|
|
1372
|
+
`${route.providerName}|${route.modelId}|anthropic-pool-rotation`,
|
|
1373
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1374
|
+
);
|
|
1375
|
+
let nextAccountId: string | null;
|
|
1376
|
+
try {
|
|
1377
|
+
nextAccountId = await rotateAnthropicAccountOnResponseForInstance(anthropicInstance, upstreamResponse, {
|
|
1378
|
+
config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey,
|
|
1379
|
+
model: route.modelId, requestKey: transportState, decision: transportState.anthropicRouteDecision, currentDecision: currentAnthropicRouteDecision, signal: upstream.signal,
|
|
1380
|
+
// A denied reservation must not rotate on base headroom its bounds cannot see.
|
|
1381
|
+
canRetry: hop.allowed && (hop.permit !== undefined || !sendBudgetExhausted())
|
|
1382
|
+
&& transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
|
|
1383
|
+
});
|
|
1384
|
+
} catch (rotationError) {
|
|
1385
|
+
hop.permit?.release();
|
|
1386
|
+
throw rotationError;
|
|
1387
|
+
}
|
|
1388
|
+
if (!nextAccountId) {
|
|
1389
|
+
hop.permit?.release();
|
|
1390
|
+
break;
|
|
1391
|
+
}
|
|
1336
1392
|
try {
|
|
1337
1393
|
const admitted = await commitResolvedOAuthSelection(await anthropicRoutingFor(anthropicInstance).getAnthropicPoolAccessSnapshot(nextAccountId));
|
|
1338
1394
|
if (!admitted) throw new Error("OAuth selection changed during recovery");
|
|
@@ -1355,12 +1411,26 @@ export async function prepareAdapterExchange(
|
|
|
1355
1411
|
});
|
|
1356
1412
|
sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
|
|
1357
1413
|
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1414
|
+
// The replay IS this hop's send; the dispatching layer settles the reservation.
|
|
1415
|
+
sendBudgetState.pendingHopPermit = hop.permit;
|
|
1416
|
+
let result: Response | { failed: Response };
|
|
1417
|
+
try {
|
|
1418
|
+
result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401"
|
|
1419
|
+
: upstreamResponse.status === 403 ? "oauth-account-403" : "anthropic-oauth-429",
|
|
1420
|
+
() => { if (!adapterOwnsDispatch) hop.permit?.use(); });
|
|
1421
|
+
} finally {
|
|
1422
|
+
sendBudgetState.pendingHopPermit = undefined;
|
|
1423
|
+
}
|
|
1424
|
+
if ("failed" in result) {
|
|
1425
|
+
// A no-op once the boundary was reached; a refund when the rebuild died first.
|
|
1426
|
+
hop.permit?.release();
|
|
1427
|
+
return result.failed;
|
|
1428
|
+
}
|
|
1429
|
+
upstreamResponse = result;
|
|
1362
1430
|
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
1363
|
-
|
|
1431
|
+
} catch {
|
|
1432
|
+
// release() refunds a pre-send hop and is a no-op once the send settled it.
|
|
1433
|
+
hop.permit?.release();
|
|
1364
1434
|
break;
|
|
1365
1435
|
}
|
|
1366
1436
|
}
|
|
@@ -1400,9 +1470,9 @@ export async function prepareAdapterExchange(
|
|
|
1400
1470
|
const hop = reserveCredentialHop(
|
|
1401
1471
|
"auth-recovery",
|
|
1402
1472
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-429`,
|
|
1403
|
-
//
|
|
1473
|
+
// Counted generic helpers report physical sends; an adapter-owned
|
|
1404
1474
|
// ladder settles its own booking.
|
|
1405
|
-
!adapterOwnsDispatch && (
|
|
1475
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1406
1476
|
);
|
|
1407
1477
|
if (!hop.allowed) break;
|
|
1408
1478
|
const nextAccountId = rotateGenericOAuthAccountOnRefusal(
|
|
@@ -1504,9 +1574,9 @@ export async function prepareAdapterExchange(
|
|
|
1504
1574
|
const hop = reserveCredentialHop(
|
|
1505
1575
|
"auth-recovery",
|
|
1506
1576
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-429`,
|
|
1507
|
-
//
|
|
1577
|
+
// Counted generic helpers report physical sends; an adapter-owned
|
|
1508
1578
|
// ladder settles its own booking.
|
|
1509
|
-
!adapterOwnsDispatch && (
|
|
1579
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1510
1580
|
);
|
|
1511
1581
|
if (!hop.allowed) break;
|
|
1512
1582
|
const nextAccountId = rotateGenericOAuthAccountOn429(
|
|
@@ -1625,7 +1695,7 @@ export async function prepareAdapterExchange(
|
|
|
1625
1695
|
const hop = reserveCredentialHop(
|
|
1626
1696
|
"auth-recovery",
|
|
1627
1697
|
`${route.providerName}|${route.modelId}|adapter-recovery-oauth-verify`,
|
|
1628
|
-
!adapterOwnsDispatch && (
|
|
1698
|
+
!adapterOwnsDispatch && genericRefetchCountsSends(),
|
|
1629
1699
|
);
|
|
1630
1700
|
if (!hop.allowed) break;
|
|
1631
1701
|
// Auth-refusal rotation, not the rate-limit one: a verification refusal
|
|
@@ -1877,6 +1947,8 @@ export async function prepareAdapterExchange(
|
|
|
1877
1947
|
authMode: route.provider.authMode,
|
|
1878
1948
|
hasApiKey: Boolean(route.provider.apiKey?.trim()),
|
|
1879
1949
|
upstreamRetryAfter,
|
|
1950
|
+
upstreamErrorType: normalized.type,
|
|
1951
|
+
upstreamErrorCode: normalized.code,
|
|
1880
1952
|
// This recovery path is the HTTP Responses wire; custom runTurn transports
|
|
1881
1953
|
// never reach enrichOpenCodeZenUpstreamMessage here.
|
|
1882
1954
|
supportsHttpSameKeyRetry: true,
|
|
@@ -1902,6 +1974,7 @@ export async function prepareAdapterExchange(
|
|
|
1902
1974
|
}
|
|
1903
1975
|
}
|
|
1904
1976
|
|
|
1977
|
+
bindAnthropicRejectionResponse(transportState.sameTargetRequest, upstreamResponse);
|
|
1905
1978
|
cancelBodyOnAbort(upstreamResponse.body, upstream.signal);
|
|
1906
1979
|
|
|
1907
1980
|
return {
|
|
@@ -1911,6 +1984,8 @@ export async function prepareAdapterExchange(
|
|
|
1911
1984
|
stallTimeoutMs,
|
|
1912
1985
|
localUpstream,
|
|
1913
1986
|
upstreamResponse,
|
|
1987
|
+
thinkingSignatureRecovery,
|
|
1988
|
+
refetchThinkingSignature,
|
|
1914
1989
|
rateLimitPolicy,
|
|
1915
1990
|
get rateLimitRetries(): typeof rateLimitRetries {
|
|
1916
1991
|
return rateLimitRetries;
|