@bitkyc08/opencodex 2.100.3 → 2.100.5-preview.20261118.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS_INSTALL.md +23 -1
- package/bin/ocx.mjs +8 -1
- package/gui/dist/assets/{App-Dnb6VMsP.js → App-BiOGtxDS.js} +21 -20
- package/gui/dist/assets/{Tray-BV4ms9U9.js → Tray-BwsMejUO.js} +1 -1
- package/gui/dist/assets/brand-mark-BY1dgACL.js +1 -0
- package/gui/dist/assets/{index-DCdXsMyZ.js → index-CwHSKMdG.js} +46 -13
- package/gui/dist/assets/index-DJxM-imn.css +2 -0
- package/gui/dist/assets/{segment-thumb-features-DXSidY6D.js → segment-thumb-features-Bds9Hb_7.js} +1 -1
- package/gui/dist/classic/assets/{App-DA-scMPq.js → App-CGLOiqQL.js} +20 -19
- package/gui/dist/classic/assets/{Tray-BrHaVRAq.js → Tray-Da4r2O70.js} +1 -1
- package/gui/dist/classic/assets/{index-BqOaGLf5.js → index-CvoKXKL9.js} +46 -13
- package/gui/dist/classic/assets/quota-summary-DvqzfCmr.js +1 -0
- package/gui/dist/classic/index.html +1 -1
- package/gui/dist/index.html +2 -2
- package/native/ocx-child/src/cli.rs +5 -3
- package/native/ocx-child/src/codex_home.rs +1 -0
- package/native/ocx-child/src/link_ws.rs +24 -19
- package/native/ocx-child/src/link_ws_frame.rs +136 -50
- package/package.json +3 -3
- package/src/adapters/anthropic/per-message-effort.ts +296 -0
- package/src/adapters/anthropic-input-schema.ts +270 -0
- package/src/adapters/anthropic.ts +4 -63
- package/src/adapters/claude-cli/adapter.ts +1 -0
- package/src/adapters/command-code-tool-text.ts +9 -1
- package/src/adapters/command-code.ts +18 -2
- package/src/adapters/devin/cloud-direct/chat.ts +95 -14
- package/src/adapters/devin/live-models.ts +11 -1
- package/src/adapters/google-errors.ts +14 -2
- package/src/adapters/kiro/reasoning.ts +3 -0
- package/src/adapters/kiro-tools.ts +14 -9
- package/src/adapters/openai-responses/passthrough.ts +3 -1
- package/src/adapters/openai-responses/request-strips.ts +1 -0
- package/src/adapters/qoder/adapter.ts +9 -1
- package/src/adapters/tool-catalog-nudge.ts +1 -1
- package/src/adapters/zed.ts +4 -2
- package/src/claude/chat-tool-dispatch.ts +4 -1
- package/src/claude/desktop-remote-store.ts +60 -10
- package/src/cli/account-api.ts +4 -0
- package/src/cli/account-auth.ts +30 -2
- package/src/cli/account-extended.ts +33 -2
- package/src/cli/capabilities-access-remote.ts +3 -3
- package/src/cli/capabilities-accounts.ts +3 -3
- package/src/cli/capabilities-base.ts +11 -6
- package/src/cli/capabilities-observe-system.ts +2 -2
- package/src/cli/claude.ts +3 -3
- package/src/cli/disconnect-runtime.ts +16 -6
- package/src/cli/dispatch.ts +7 -1
- package/src/cli/index.ts +2 -2
- package/src/cli/link-apply-restart.ts +4 -2
- package/src/cli/link-routes.ts +4 -1
- package/src/cli/link.ts +5 -4
- package/src/cli/logout-command.ts +23 -4
- package/src/cli/preflight-exemptions.ts +3 -0
- package/src/cli/provider.ts +3 -2
- package/src/cli/registry.ts +5 -2
- package/src/cli/resolve.ts +23 -8
- package/src/cli/restart-failure.ts +11 -1
- package/src/cli/runtime-api.ts +4 -0
- package/src/cli/status.ts +4 -1
- package/src/cli/system-command.ts +1 -1
- package/src/cli/system-restart-client.ts +9 -3
- package/src/cli/system-settings-parity.ts +2 -1
- package/src/client/connect.ts +73 -11
- package/src/client/desktop-refusal.ts +38 -0
- package/src/client/lifecycle-lock.ts +29 -4
- package/src/client/runtime-restart.ts +27 -15
- package/src/client/runtime.ts +4 -2
- package/src/client/state.ts +14 -0
- package/src/clients/config-export.ts +12 -7
- package/src/codex/account-lifecycle.ts +3 -1
- package/src/codex/account-pause.ts +4 -0
- package/src/codex/account-runtime-state.ts +4 -0
- package/src/codex/auth-api/main-account-probe.ts +3 -1
- package/src/codex/auth-api/main-hard-lock-refusal-read.ts +172 -0
- package/src/codex/auth-context.ts +40 -2
- package/src/codex/catalog/build-entries.ts +10 -0
- package/src/codex/catalog/derive-entry.ts +7 -0
- package/src/codex/catalog/metadata.ts +8 -0
- package/src/codex/catalog/native-models.ts +16 -6
- package/src/codex/catalog/pinned-models.ts +1 -0
- package/src/codex/catalog/provider-models.ts +5 -5
- package/src/codex/catalog/reserve-warn.ts +7 -5
- package/src/codex/catalog/reserve.ts +4 -3
- package/src/codex/catalog/retained-sync.ts +2 -0
- package/src/codex/catalog/routed-defaults.ts +35 -0
- package/src/codex/catalog-auto-refresh.ts +13 -2
- package/src/codex/config-drift-heal.ts +40 -12
- package/src/codex/config-write-lock.ts +39 -1
- package/src/codex/convergence.ts +2 -0
- package/src/codex/data/roster-pinned-models.json +346 -1
- package/src/codex/inject/config-write-section.ts +39 -1
- package/src/codex/inject/plan.ts +10 -8
- package/src/codex/inject/remove.ts +21 -19
- package/src/codex/inject/restore.ts +5 -1
- package/src/codex/inject-coordination.ts +58 -7
- package/src/codex/inject.ts +10 -12
- package/src/codex/journal-config-restore.ts +42 -0
- package/src/codex/journal.ts +62 -8
- package/src/codex/loopback-target.ts +22 -8
- package/src/codex/main-account.ts +94 -30
- package/src/codex/native-main-auth-file.ts +14 -0
- package/src/codex/native-main-credentials.ts +276 -0
- package/src/codex/native-main-owner.ts +3 -0
- package/src/codex/native-main-refresh-mode.ts +14 -1
- package/src/codex/quota.ts +63 -14
- package/src/combos/failover.ts +195 -1
- package/src/combos/index.ts +5 -0
- package/src/combos/resolve.ts +115 -19
- package/src/config/diagnostics.ts +3 -0
- package/src/config/live-reconcile.ts +6 -3
- package/src/config/load-degrade.ts +2 -0
- package/src/config/persisted-mutation.ts +2 -2
- package/src/config/persisted-projections.ts +18 -0
- package/src/config/process-state.ts +4 -3
- package/src/config/reserve-capability.ts +96 -0
- package/src/config/schema/config-schema.ts +6 -0
- package/src/config/schema/leaf-validators.ts +2 -0
- package/src/config/schema/routed-catalog-defaults.ts +49 -0
- package/src/config/subagent-models.ts +20 -12
- package/src/config.ts +7 -7
- package/src/desktop-update/acquire.ts +202 -0
- package/src/desktop-update/cli.ts +168 -0
- package/src/desktop-update/drain.ts +40 -0
- package/src/desktop-update/headless.ts +285 -0
- package/src/desktop-update/image.ts +85 -0
- package/src/desktop-update/install.ts +53 -0
- package/src/desktop-update/msi.ts +72 -0
- package/src/desktop-update/release.ts +128 -0
- package/src/desktop-update/runtime.ts +155 -0
- package/src/desktop-update/staging.ts +118 -0
- package/src/desktop-update/trust.ts +45 -0
- package/src/generated/compatibility-version.json +327 -195
- package/src/generated/model-metadata.ts +3 -3
- package/src/images/fulfill-video.ts +3 -3
- package/src/images/loop.ts +6 -2
- package/src/images/plan.ts +147 -11
- package/src/images/types.ts +13 -4
- package/src/images/xai-video-client.ts +72 -10
- package/src/lib/client-error-body.ts +6 -0
- package/src/lib/developer-role-rejection.ts +15 -0
- package/src/lib/gcp-adc.ts +48 -4
- package/src/lib/offline-resend.ts +150 -0
- package/src/lib/paced-stream.ts +3 -3
- package/src/lib/provider-outbound.ts +17 -6
- package/src/lib/request-failure-model.ts +2 -0
- package/src/lib/response-body-inactivity.ts +23 -1
- package/src/lib/upstream-retry.ts +39 -6
- package/src/link/explicit-route.ts +4 -4
- package/src/link/remote-windows.ts +79 -31
- package/src/link/residual-recovery.ts +20 -8
- package/src/messaging/budget.ts +23 -5
- package/src/messaging/remote-ports.ts +22 -10
- package/src/messaging/rpc.ts +4 -4
- package/src/oauth/antigravity-risk.ts +126 -0
- package/src/oauth/credential-siblings.ts +32 -0
- package/src/oauth/generic-account-failover.ts +113 -30
- package/src/oauth/generic-session-affinity.ts +10 -3
- package/src/oauth/index.ts +22 -14
- package/src/oauth/login-cli.ts +35 -1
- package/src/oauth/pool-kernel.ts +3 -3
- package/src/oauth/store.ts +38 -7
- package/src/providers/anthropic-billing-refusal.ts +27 -3
- package/src/providers/anthropic-reset-grants.ts +4 -0
- package/src/providers/claude-cli-usage.ts +2 -0
- package/src/providers/codebuddy-models.ts +11 -0
- package/src/providers/google-vertex-model-discovery.ts +49 -15
- package/src/providers/kiro-models.ts +5 -0
- package/src/providers/model-rename-fields.ts +2 -0
- package/src/providers/quota/workbuddy.ts +6 -2
- package/src/providers/registry/entries-core.ts +9 -3
- package/src/providers/registry/entries-extended.ts +20 -9
- package/src/providers/registry/model-seeds.ts +4 -1
- package/src/providers/stale-context-window-migration.ts +2 -0
- package/src/providers/workbuddy-credit-balance.ts +125 -16
- package/src/responses/citation-markers.ts +26 -23
- package/src/responses/effort-history.ts +732 -0
- package/src/routing/history/integrity.ts +41 -4
- package/src/server/auth-cors.ts +13 -3
- package/src/server/codex-model-list.ts +2 -1
- package/src/server/direct-local-http.ts +26 -3
- package/src/server/index/link-listener.ts +20 -14
- package/src/server/index/serve-options.ts +1 -0
- package/src/server/index.ts +3 -2
- package/src/server/link-turn.ts +17 -5
- package/src/server/management/config-routes.ts +39 -3
- package/src/server/management/link-child-preflight.ts +1 -1
- package/src/server/management/link-enrollment.ts +3 -3
- package/src/server/management/link-route-edit.ts +20 -1
- package/src/server/management/link-route-request.ts +11 -5
- package/src/server/management/oauth-account-routes.ts +43 -13
- package/src/server/management/provider-credential-release.ts +44 -0
- package/src/server/management/provider-overwrite-carry.ts +3 -0
- package/src/server/management/provider-routes.ts +12 -12
- package/src/server/management/static-catalog-probe.ts +33 -0
- package/src/server/management/usage-aggregate-cache.ts +6 -3
- package/src/server/messages-native.ts +16 -2
- package/src/server/proxy-liveness.ts +54 -9
- package/src/server/relay.ts +6 -1
- package/src/server/request-log-conversation.ts +8 -4
- package/src/server/request-log-failure-detail.ts +26 -9
- package/src/server/request-log.ts +5 -1
- package/src/server/reserve-capability-startup.ts +32 -0
- package/src/server/responses/adapter-continuation.ts +59 -25
- package/src/server/responses/adapter-delivery.ts +2 -0
- package/src/server/responses/adapter-dispatch.ts +39 -18
- package/src/server/responses/antigravity-fault-rotation.ts +24 -0
- package/src/server/responses/antigravity-validation-refusal.ts +43 -6
- package/src/server/responses/codex-tool-catalog.ts +47 -10
- package/src/server/responses/combo-selection-keepalive.ts +254 -0
- package/src/server/responses/combo-session-recall.ts +26 -1
- package/src/server/responses/combo-stream-preflight.ts +35 -26
- package/src/server/responses/core-combo-failure.ts +6 -0
- package/src/server/responses/core-combo-native.ts +2 -1
- package/src/server/responses/core-combo.ts +67 -28
- package/src/server/responses/core-errors.ts +9 -6
- package/src/server/responses/core-options.ts +10 -0
- package/src/server/responses/core.ts +21 -0
- package/src/server/responses/estimated-usage.ts +115 -0
- package/src/server/responses/input-admission.ts +41 -3
- package/src/server/responses/jev-model-invoke.ts +7 -2
- package/src/server/responses/passthrough-completion.ts +6 -3
- package/src/server/responses/passthrough-delivery.ts +47 -8
- package/src/server/responses/passthrough-dispatch.ts +3 -0
- package/src/server/responses/request-prepare.ts +9 -5
- package/src/server/responses/request-send-budget.ts +1 -1
- package/src/server/responses/request-transport.ts +10 -2
- package/src/server/responses/sidecar-execution.ts +11 -4
- package/src/service/desktop-supervision.d.mts +2 -0
- package/src/service/desktop-supervision.mjs +5 -0
- package/src/service/windows-checked-stop-live.ts +28 -5
- package/src/stall-timeout.ts +11 -0
- package/src/types/config.ts +15 -92
- package/src/types/provider.ts +13 -0
- package/src/types/request.ts +23 -3
- package/src/types/routed-catalog-defaults.ts +17 -0
- package/src/types/routing-profile.ts +90 -0
- package/src/types.ts +1 -0
- package/src/update/index.ts +1 -1
- package/src/usage/expected-prices.ts +35 -0
- package/src/usage/ledger-scanner.ts +44 -10
- package/src/usage/log.ts +14 -4
- package/src/usage/telemetry-contract.ts +1 -0
- package/src/web-search/loop.ts +20 -2
- package/src/web-search/parse.ts +2 -12
- package/src/web-search/passthrough-bridge.ts +7 -0
- package/src/web-search/sources.ts +13 -0
- package/gui/dist/assets/brand-mark-XOLJbomA.js +0 -1
- package/gui/dist/assets/index-DQtCRNAw.css +0 -2
- package/gui/dist/classic/assets/quota-summary-Dy5_wfTT.js +0 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { upstreamValidationIssues } from "../lib/upstream-validation-detail";
|
|
1
|
+
import { upstreamValidationIssues, type ValidationIssue } from "../lib/upstream-validation-detail";
|
|
2
|
+
import { isDeveloperRoleRejection } from "../lib/developer-role-rejection";
|
|
2
3
|
import { clientFacingModelText, errorDestinationText, providerErrorDestinationLabel } from "../lib/error-destination";
|
|
3
4
|
import { knownModelIdentity } from "../lib/model-identity";
|
|
4
5
|
/**
|
|
@@ -80,20 +81,36 @@ export function projectLocalModelRefusalLog(
|
|
|
80
81
|
|| decision.selected && (!known(decision.selected.model) || !providerKnown(decision.selected.provider)))) logCtx.routeDecision = undefined;
|
|
81
82
|
}
|
|
82
83
|
|
|
83
|
-
|
|
84
|
-
|
|
84
|
+
const isRecord = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
|
|
85
|
+
/**
|
|
86
|
+
* The closed pair a developer-role rejection records (#8864); the dashboard already renders it. A fresh
|
|
87
|
+
* array per row, because a valid list is stored by reference and must not be shared between rows.
|
|
88
|
+
*/
|
|
89
|
+
const developerRoleValidation = (): ValidationIssue[] => [{ field: "messages.role", type: "enum" }];
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Capture bounded schema facts before a 4xx is replaced by the client-facing error policy. Only a caller
|
|
93
|
+
* holding the provider's own body sets `providerBody`, which also reads its message once for a
|
|
94
|
+
* developer-role rejection; a body this proxy formatted keeps validation-only capture.
|
|
95
|
+
*/
|
|
96
|
+
export function recordUpstreamValidationDetail(
|
|
97
|
+
logCtx: { failureDetail?: FailureDetail }, status: number, payload: unknown, options: { providerBody?: boolean } = {},
|
|
98
|
+
): void {
|
|
85
99
|
if (status < 400 || status >= 500) return;
|
|
86
100
|
let parsed = payload;
|
|
87
101
|
if (typeof payload === "string") {
|
|
88
102
|
if (payload.length > 256 * 1024) return;
|
|
89
103
|
try { parsed = JSON.parse(payload); } catch { return; }
|
|
90
104
|
}
|
|
91
|
-
const
|
|
105
|
+
const record = isRecord(parsed) ? parsed : undefined;
|
|
106
|
+
const nested = record && isRecord(record.error) ? record.error : undefined;
|
|
107
|
+
const message = typeof nested?.message === "string" ? nested.message
|
|
108
|
+
: typeof record?.message === "string" ? record.message : typeof record?.error === "string" ? record.error : undefined;
|
|
109
|
+
// Read once to choose the closed pair; the message itself is dropped here.
|
|
110
|
+
const validation = upstreamValidationIssues(parsed)
|
|
111
|
+
?? (options.providerBody && isDeveloperRoleRejection(status, message) ? developerRoleValidation() : undefined);
|
|
92
112
|
if (!validation) return;
|
|
93
|
-
|
|
94
|
-
const nested = record.error !== null && typeof record.error === "object" && !Array.isArray(record.error)
|
|
95
|
-
? record.error as Record<string, unknown> : undefined;
|
|
96
|
-
logCtx.failureDetail = boundedFailureDetail({ code: "provider_error", type: nested?.type ?? record.type, validation });
|
|
113
|
+
logCtx.failureDetail = boundedFailureDetail({ code: "provider_error", type: nested?.type ?? record?.type, validation });
|
|
97
114
|
}
|
|
98
115
|
|
|
99
116
|
/**
|
|
@@ -112,7 +129,7 @@ export function recordUpstreamProviderErrorDetail(
|
|
|
112
129
|
): void {
|
|
113
130
|
if (status < 400 || status >= 600) return;
|
|
114
131
|
const before = logCtx.failureDetail;
|
|
115
|
-
recordUpstreamValidationDetail(logCtx, status, payload);
|
|
132
|
+
recordUpstreamValidationDetail(logCtx, status, payload, { providerBody: true });
|
|
116
133
|
if (logCtx.failureDetail !== before) { providerDetails.set(logCtx, logCtx.failureDetail!); return; }
|
|
117
134
|
let type = response ? providerFailureDetailOf(response)?.type : undefined;
|
|
118
135
|
if (type === undefined) {
|
|
@@ -252,6 +252,10 @@ export interface RequestLogContext extends UpstreamErrorDiagnostics, LinkRequest
|
|
|
252
252
|
* responsesUsage in src/bridge.ts), so SSE/JSON re-parsing must not overwrite the raw
|
|
253
253
|
* provenance — a synthetic cached_tokens:0 is NOT a measured cache read. */
|
|
254
254
|
usageFromBridge?: boolean;
|
|
255
|
+
/** Set once the client-facing body carries an estimated usage the provider never sent (#7427).
|
|
256
|
+
* The provider's own body was inspected before the estimate was added, so a later parse of
|
|
257
|
+
* the client body must not record the estimate as provider-reported usage. */
|
|
258
|
+
usageEstimatedForClient?: boolean;
|
|
255
259
|
/** Secret-redacted upstream error reason (e.g. the granular Cursor "rate limit exceeded…"
|
|
256
260
|
* message) extracted from a `response.failed` SSE payload or non-streaming error body, so the
|
|
257
261
|
* request log / GUI shows the actual upstream failure rather than only the HTTP-mapped code. */
|
|
@@ -1049,7 +1053,7 @@ export function applyResponseLogMetadata(logCtx: RequestLogContext, payload: unk
|
|
|
1049
1053
|
logCtx.activeTierMetadata?.observeResponseServiceTier(serviceTier);
|
|
1050
1054
|
}
|
|
1051
1055
|
const usage = usageFromResponsesPayload((source as { usage?: unknown }).usage);
|
|
1052
|
-
if (usage && !logCtx.usageFromBridge) {
|
|
1056
|
+
if (usage && !logCtx.usageFromBridge && !logCtx.usageEstimatedForClient) {
|
|
1053
1057
|
if (!recordKeyWireAttemptUsage(logCtx, usage)) {
|
|
1054
1058
|
logCtx.usage = usage;
|
|
1055
1059
|
if (logCtx.activeAttempt) logCtx.activeAttempt.usage = usage;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { mutatePersistedConfig } from "../config";
|
|
2
|
+
import { adoptReserveCapabilityMigration, isReserveCapabilityMigrated, migrateReserveCapability } from "../config/reserve-capability";
|
|
3
|
+
import type { OcxConfig } from "../types";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Persist the one-time Reserve seed at startup (#4961), so a later hand edit of authless cannot move
|
|
7
|
+
* Reserve. The seed is the file's current authless value: the value being migrated.
|
|
8
|
+
*
|
|
9
|
+
* Only the two Reserve fields are adopted, in place: earlier startup migrations may hold in-memory
|
|
10
|
+
* results whose persistence was unavailable, and the loaded object carries identity-keyed state
|
|
11
|
+
* (file-backed provenance) that a copy would lose. On failure nothing is marked, so the legacy
|
|
12
|
+
* reading continues and the next persisted write seeds from the file instead.
|
|
13
|
+
*/
|
|
14
|
+
export function migrateStartupReserveCapability(
|
|
15
|
+
config: OcxConfig,
|
|
16
|
+
mutate: typeof mutatePersistedConfig = mutatePersistedConfig,
|
|
17
|
+
): OcxConfig {
|
|
18
|
+
if (isReserveCapabilityMigrated(config)) return config;
|
|
19
|
+
try {
|
|
20
|
+
const outcome = mutate(fresh => ({ changed: migrateReserveCapability(fresh), value: fresh }));
|
|
21
|
+
if (outcome.status === "unavailable") {
|
|
22
|
+
console.warn(`[reserve-capability-migration] Persistence unavailable (${outcome.reason}); Reserve still follows codexDesktopAuthless until the next config write.`);
|
|
23
|
+
return config;
|
|
24
|
+
}
|
|
25
|
+
adoptReserveCapabilityMigration(config, outcome.value);
|
|
26
|
+
return config;
|
|
27
|
+
} catch {
|
|
28
|
+
// Do not log the exception: filesystem errors may contain private paths.
|
|
29
|
+
console.warn("[reserve-capability-migration] Persistence failed; Reserve still follows codexDesktopAuthless until the next config write.");
|
|
30
|
+
return config;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -43,7 +43,6 @@ import { redactSecretString } from "../../lib/redact";
|
|
|
43
43
|
import { resolveWireProtocolOverride } from "../adapter-resolve";
|
|
44
44
|
import { bindRouteReasoningReplayScope } from "./core-replay";
|
|
45
45
|
import {
|
|
46
|
-
ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
|
|
47
46
|
anthropicRoutingFor,
|
|
48
47
|
formatAnthropicProviderForLog,
|
|
49
48
|
} from "../../oauth/anthropic-routing";
|
|
@@ -102,7 +101,7 @@ export function createAdapterContinuations(
|
|
|
102
101
|
| "commitResolvedOAuthSelection"
|
|
103
102
|
| "genericFailoverAccountId"
|
|
104
103
|
| "genericFailovers"
|
|
105
|
-
| "genericFailoverLimit"
|
|
104
|
+
| "genericFailoverLimit" | "anthropicFailoverLimit"
|
|
106
105
|
| "applyFailoverSnapshot"
|
|
107
106
|
| "replayOAuthCredentialSnapshot"
|
|
108
107
|
| "noteRoutedAttemptSend"
|
|
@@ -201,8 +200,16 @@ export function createAdapterContinuations(
|
|
|
201
200
|
allowAccountRefusal = false,
|
|
202
201
|
): AsyncGenerator<AdapterEvent> {
|
|
203
202
|
let response: Response | undefined;
|
|
204
|
-
let
|
|
205
|
-
let
|
|
203
|
+
let refusalPendingReplay: Response | undefined;
|
|
204
|
+
let replaySetupFailed = false;
|
|
205
|
+
// Refused account's request attribution, restored when its replacement fails before sending.
|
|
206
|
+
let refusalAttribution: { provider: string; label?: string; ordinal?: number; sends?: number;
|
|
207
|
+
attemptLabel?: NonNullable<typeof logCtx.activeAttempt>["accountLogLabel"] } | undefined;
|
|
208
|
+
const captureRefusalAttribution = () => {
|
|
209
|
+
const attempt = logCtx.activeAttempt;
|
|
210
|
+
refusalAttribution = { provider: logCtx.provider, label: logCtx.accountLogLabel, ordinal: attempt?.ordinal,
|
|
211
|
+
sends: attempt?.sendCount, attemptLabel: attempt?.accountLogLabel };
|
|
212
|
+
};
|
|
206
213
|
// One-shot recovery label for the next top-of-loop continuation send after a failover rotation.
|
|
207
214
|
let nextContinuationRecoveryKind: AttemptRecoveryKind | undefined = initialRecoveryKind;
|
|
208
215
|
/**
|
|
@@ -313,9 +320,11 @@ export function createAdapterContinuations(
|
|
|
313
320
|
// Same request-scoped budget as the initial send and the 429/rotation refetches:
|
|
314
321
|
// a terminal-guard continuation is another leg of ONE request, so handing it a
|
|
315
322
|
// fresh `attempts` would let one request exceed the configured total-send ceiling.
|
|
316
|
-
...(continuationTransientPolicy || sendBudgetState.adapterSendBudget?.spendEnforced
|
|
323
|
+
...(continuationTransientPolicy || anthropicInstance || sendBudgetState.adapterSendBudget?.spendEnforced
|
|
317
324
|
? {
|
|
318
|
-
attempts:
|
|
325
|
+
attempts: Math.min(continuationTransientPolicy?.attempts ?? 1,
|
|
326
|
+
remainingTransientSendBudget(continuationTransientPolicy?.attempts ?? 1)
|
|
327
|
+
+ (sendBudgetState.pendingHopPermit ? 1 : 0)),
|
|
319
328
|
onSendsConsumed: deferredContinuation?.report ?? continuationReporter,
|
|
320
329
|
}
|
|
321
330
|
: {}),
|
|
@@ -338,20 +347,34 @@ export function createAdapterContinuations(
|
|
|
338
347
|
invalidateSameTargetRequest();
|
|
339
348
|
response = await fetchContinuation("opaque-blob-rejection");
|
|
340
349
|
}
|
|
341
|
-
if (
|
|
342
|
-
try { void
|
|
343
|
-
|
|
350
|
+
if (refusalPendingReplay) {
|
|
351
|
+
try { void refusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* already closed */ }
|
|
352
|
+
refusalPendingReplay = undefined;
|
|
344
353
|
sendBudgetState.pendingHopPermit = undefined;
|
|
345
354
|
}
|
|
346
355
|
} catch (error) {
|
|
347
|
-
if (
|
|
356
|
+
if (refusalPendingReplay && !options.abortSignal?.aborted && !upstream.signal.aborted) {
|
|
348
357
|
sendBudgetState.pendingHopPermit?.release();
|
|
349
358
|
sendBudgetState.pendingHopPermit = undefined;
|
|
350
|
-
response =
|
|
351
|
-
|
|
352
|
-
|
|
359
|
+
response = refusalPendingReplay;
|
|
360
|
+
refusalPendingReplay = undefined;
|
|
361
|
+
replaySetupFailed = true;
|
|
362
|
+
const attempt = logCtx.activeAttempt;
|
|
363
|
+
// Same attempt and send count: the replacement never dispatched, so the refusal stays the refused account's.
|
|
364
|
+
if (attempt && refusalAttribution && attempt.ordinal === refusalAttribution.ordinal && attempt.sendCount === refusalAttribution.sends) {
|
|
365
|
+
logCtx.provider = refusalAttribution.provider;
|
|
366
|
+
if (refusalAttribution.label === undefined) delete logCtx.accountLogLabel;
|
|
367
|
+
else logCtx.accountLogLabel = refusalAttribution.label;
|
|
368
|
+
if (refusalAttribution.attemptLabel === undefined) delete attempt.accountLogLabel;
|
|
369
|
+
else attempt.accountLogLabel = refusalAttribution.attemptLabel;
|
|
370
|
+
}
|
|
353
371
|
} else {
|
|
354
|
-
|
|
372
|
+
if (refusalPendingReplay) {
|
|
373
|
+
sendBudgetState.pendingHopPermit?.release();
|
|
374
|
+
sendBudgetState.pendingHopPermit = undefined;
|
|
375
|
+
try { void refusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* closed */ }
|
|
376
|
+
}
|
|
377
|
+
yield continuationFailureEvent(error); return;
|
|
355
378
|
}
|
|
356
379
|
}
|
|
357
380
|
|
|
@@ -360,7 +383,7 @@ export function createAdapterContinuations(
|
|
|
360
383
|
// loop; only after the attempts are exhausted does the continuation fail over.
|
|
361
384
|
while (
|
|
362
385
|
response.status === 429
|
|
363
|
-
&& !
|
|
386
|
+
&& !replaySetupFailed
|
|
364
387
|
// A synthesized replay refusal is not a rate limit; replaying the continuation on
|
|
365
388
|
// it would re-send a turn whose first send may already have been processed.
|
|
366
389
|
&& !isNonReplayableResponse(response) && !isSameTargetRetryHold(response)
|
|
@@ -447,19 +470,26 @@ export function createAdapterContinuations(
|
|
|
447
470
|
(response.status === 429 || response.status === 403 || response.status === 401)
|
|
448
471
|
&& anthropicInstance
|
|
449
472
|
&& transportState.anthropicPoolAccountId
|
|
450
|
-
&& !isNonReplayableResponse(response)
|
|
473
|
+
&& !replaySetupFailed && !isNonReplayableResponse(response)
|
|
451
474
|
) {
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
475
|
+
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
476
|
+
const hop = reserveCredentialHop("auth-recovery", `${route.providerName}|${route.modelId}|continuation-anthropic-429`,
|
|
477
|
+
!adapterOwnsDispatch);
|
|
478
|
+
let nextAccountId: string | null;
|
|
479
|
+
try {
|
|
480
|
+
nextAccountId = await rotateAnthropicAccountOnResponseForInstance(anthropicInstance, response, {
|
|
481
|
+
config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey,
|
|
482
|
+
model: route.modelId, requestKey: transportState, decision: transportState.anthropicRouteDecision, currentDecision: currentAnthropicRouteDecision, signal: upstream.signal,
|
|
483
|
+
canRetry: hop.allowed && transportState.anthropicPoolFailovers < transportState.anthropicFailoverLimit,
|
|
484
|
+
allowAccountRefusal, allow429Recovery: allowAccountRefusal,
|
|
485
|
+
});
|
|
486
|
+
} catch (error) { hop.permit?.release(); throw error; }
|
|
487
|
+
if (!nextAccountId) hop.permit?.release();
|
|
458
488
|
if (nextAccountId) {
|
|
489
|
+
captureRefusalAttribution();
|
|
459
490
|
try {
|
|
460
491
|
const admitted = await commitResolvedOAuthSelection(await anthropicRoutingFor(anthropicInstance).getAnthropicPoolAccessSnapshot(nextAccountId));
|
|
461
492
|
if (!admitted) throw new Error("OAuth selection changed during recovery");
|
|
462
|
-
try { void response.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
|
|
463
493
|
transportState.anthropicPoolAccountId = admitted.accountId;
|
|
464
494
|
transportState.anthropicPoolFailovers += 1;
|
|
465
495
|
route.provider = { ...route.provider, apiKey: admitted.accessToken };
|
|
@@ -483,8 +513,11 @@ export function createAdapterContinuations(
|
|
|
483
513
|
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
|
|
484
514
|
nextContinuationRecoveryKind = response.status === 401 ? "oauth-401"
|
|
485
515
|
: response.status === 403 ? "oauth-account-403" : "anthropic-oauth-429";
|
|
516
|
+
sendBudgetState.pendingHopPermit = hop.permit;
|
|
517
|
+
refusalPendingReplay = response;
|
|
486
518
|
continue;
|
|
487
519
|
} catch {
|
|
520
|
+
hop.permit?.release();
|
|
488
521
|
// fall through to emit continuation error below
|
|
489
522
|
}
|
|
490
523
|
}
|
|
@@ -499,7 +532,7 @@ export function createAdapterContinuations(
|
|
|
499
532
|
(response.status === 429 || response.status === 400 || response.status === 403)
|
|
500
533
|
&& transportState.genericFailoverAccountId
|
|
501
534
|
&& !isNonReplayableResponse(response)
|
|
502
|
-
&& !
|
|
535
|
+
&& !replaySetupFailed
|
|
503
536
|
) {
|
|
504
537
|
const refusal = classifyKiroRefusal(response.status,
|
|
505
538
|
await readDisplaySafeErrorText(response.clone(), upstream.signal, ""));
|
|
@@ -550,6 +583,7 @@ export function createAdapterContinuations(
|
|
|
550
583
|
// metadata, so a token-only swap would mix one account's credential with another's
|
|
551
584
|
// routing data.
|
|
552
585
|
const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId);
|
|
586
|
+
captureRefusalAttribution();
|
|
553
587
|
const applied = await applyFailoverSnapshot(snapshot, nextParsed);
|
|
554
588
|
if (!applied) hop.permit?.release();
|
|
555
589
|
if (applied) {
|
|
@@ -583,7 +617,7 @@ export function createAdapterContinuations(
|
|
|
583
617
|
// same permit so its reporter settles only this booking.
|
|
584
618
|
sendBudgetState.pendingHopPermit = hop.permit;
|
|
585
619
|
nextContinuationRecoveryKind = "oauth-account-429";
|
|
586
|
-
|
|
620
|
+
refusalPendingReplay = response;
|
|
587
621
|
continue;
|
|
588
622
|
}
|
|
589
623
|
} catch {
|
|
@@ -30,6 +30,7 @@ import { isNonReplayableResponse } from "../../lib/upstream-retry";
|
|
|
30
30
|
import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
|
|
31
31
|
import { currentTurnToolBridgeMaps, guardEmptyToolCatalogEvents, guardEmptyToolCatalogStream } from "../../lib/empty-tool-catalog";
|
|
32
32
|
import { guardCompactionSummary } from "../../responses/compaction-summary";
|
|
33
|
+
import { settleAnthropicPerMessageEffort } from "../../adapters/anthropic/per-message-effort";
|
|
33
34
|
|
|
34
35
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
35
36
|
export async function deliverAdapterResponse(
|
|
@@ -208,6 +209,7 @@ export async function deliverAdapterResponse(
|
|
|
208
209
|
// direct client encoder from the fold of the same events.
|
|
209
210
|
const onCompletedResponse = (response: Record<string, unknown>, providerState?: OcxProviderContinuationState) => {
|
|
210
211
|
if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(response);
|
|
212
|
+
settleAnthropicPerMessageEffort(parsed, response, transportState.activeAdapter.name);
|
|
211
213
|
const served = transportState.sentOAuthSnapshot;
|
|
212
214
|
if (transportState.activeAdapter.name === "kiro" && response.status === "completed" && served
|
|
213
215
|
&& noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState();
|
|
@@ -61,9 +61,10 @@ import {
|
|
|
61
61
|
SendBudgetExhaustedError, errorDestinationLabel,
|
|
62
62
|
UpstreamRetryEvidenceError,
|
|
63
63
|
prepareSameTarget429Wait,
|
|
64
|
-
sleepWithAbort,
|
|
64
|
+
sleepWithAbort, RESET_RETRY_MAX_ATTEMPTS,
|
|
65
65
|
} from "../../lib/upstream-retry";
|
|
66
66
|
import { describeUpstreamConnectFailure } from "./upstream-error";
|
|
67
|
+
import { createProviderOfflineResendWait } from "../../lib/offline-resend";
|
|
67
68
|
import type { OpaqueBlobRecoveryGuard } from "./core-opaque-recovery";
|
|
68
69
|
import type { AttemptRecoveryKind } from "../../usage/log";
|
|
69
70
|
import type { OAuthAccessSnapshot } from "../../oauth";
|
|
@@ -81,7 +82,6 @@ import { resolveWireProtocolOverride } from "../adapter-resolve";
|
|
|
81
82
|
import { bindRouteReasoningReplayScope } from "./core-replay";
|
|
82
83
|
import {
|
|
83
84
|
AnthropicAccountCooldownError,
|
|
84
|
-
ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST,
|
|
85
85
|
anthropicRoutingFor,
|
|
86
86
|
formatAnthropicProviderForLog,
|
|
87
87
|
} from "../../oauth/anthropic-routing";
|
|
@@ -92,6 +92,7 @@ import {
|
|
|
92
92
|
quarantineKiroSuspendedAccount,
|
|
93
93
|
failoverAccountSnapshot,
|
|
94
94
|
rotateAntigravityAccountOnAuthRefusal,
|
|
95
|
+
rotateAntigravityAccountOnUpstreamFault,
|
|
95
96
|
} from "../../oauth/generic-account-failover";
|
|
96
97
|
import {
|
|
97
98
|
attemptOpaqueBlobRecovery,
|
|
@@ -119,10 +120,12 @@ import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
|
|
|
119
120
|
import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error";
|
|
120
121
|
import { chargeWorkflowSends } from "../../lib/workflow-budget";
|
|
121
122
|
import { hasWorkflowSendObserver } from "../../lib/request-execution-budget";
|
|
122
|
-
import { isAntigravityValidationRefusal } from "./antigravity-validation-refusal";
|
|
123
|
+
import { isAntigravityAccessDeniedRefusal, isAntigravityValidationRefusal } from "./antigravity-validation-refusal";
|
|
123
124
|
import { createAntigravityModelDenialRecovery, recoverAntigravityAccount } from "./antigravity-model-denial-recovery";
|
|
125
|
+
import { isAntigravityFaultRotatable } from "./antigravity-fault-rotation";
|
|
124
126
|
import { adapterSendBudgetRefusal } from "../workflow-refusal";
|
|
125
127
|
import { AdmissionModelDeniedError, admissionModelDeniedResponse, projectAdmissionModelDeniedLog } from "../admission-model-scope";
|
|
128
|
+
import { rejectAnthropicPerMessageEffort } from "../../adapters/anthropic/per-message-effort";
|
|
126
129
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
127
130
|
export async function prepareAdapterExchange(
|
|
128
131
|
requestContext: Pick<ResponsesRequestContext, "options" | "config" | "logCtx" | "req">,
|
|
@@ -168,7 +171,7 @@ export async function prepareAdapterExchange(
|
|
|
168
171
|
| "commitResolvedOAuthSelection"
|
|
169
172
|
| "genericFailoverAccountId"
|
|
170
173
|
| "genericFailovers"
|
|
171
|
-
| "genericFailoverLimit"
|
|
174
|
+
| "genericFailoverLimit" | "anthropicFailoverLimit"
|
|
172
175
|
| "applyFailoverSnapshot"
|
|
173
176
|
| "noteRoutedAttemptSend" | "attemptSendDeferred" | "deferPhysicalAttribution"
|
|
174
177
|
>,
|
|
@@ -402,7 +405,7 @@ export async function prepareAdapterExchange(
|
|
|
402
405
|
let billingRecoveryActive = false;
|
|
403
406
|
const genericRefetchCountsSends = () => !!sendBudgetState.workflowRootId || options.comboAttempt === true || sendBudgetState.adapterSendBudget?.spendEnforced === true
|
|
404
407
|
|| transientRetryPolicyFor(route.provider) !== null || resetReplayPolicyFor(route.provider) !== null
|
|
405
|
-
|| billingPoolRequest || billingRecoveryActive || autoPauseEnabled;
|
|
408
|
+
|| !!anthropicInstance || billingPoolRequest || billingRecoveryActive || autoPauseEnabled;
|
|
406
409
|
let handoffPermit: ResponsesSendBudget["pendingHopPermit"];
|
|
407
410
|
try {
|
|
408
411
|
if (transportState.activeAdapter.fetchResponse) {
|
|
@@ -471,7 +474,8 @@ export async function prepareAdapterExchange(
|
|
|
471
474
|
|| (options.comboAttempt && route.provider.adapter === "google")
|
|
472
475
|
? transientSendCapFor(transientPolicy?.attempts,
|
|
473
476
|
(options.comboAttempt || compactPrepaid) ? 0 : sendBudgetState.sendsUsed)
|
|
474
|
-
:
|
|
477
|
+
// Anthropic sends are counted for the funded roster, not narrowed: keep the reset ladder.
|
|
478
|
+
: anthropicInstance && !options.comboAttempt && !compactPrepaid && !autoPauseEnabled ? RESET_RETRY_MAX_ATTEMPTS : 1;
|
|
475
479
|
const fetchWithRetryPolicy = (route.provider.adapter === "google" || transientPolicy)
|
|
476
480
|
? fetchWithTransientRetry
|
|
477
481
|
: fetchWithResetRetry;
|
|
@@ -521,11 +525,12 @@ export async function prepareAdapterExchange(
|
|
|
521
525
|
abortSignal: upstream.signal,
|
|
522
526
|
label: safeHostLabel(builtInitialRequest.url), destination: errorDestinationLabel(builtInitialRequest.url),
|
|
523
527
|
claimAmbiguousResend: claimPreHeaderResend,
|
|
528
|
+
offlineWait: options.comboAttempt ? undefined : createProviderOfflineResendWait(route.providerName, route.provider, builtInitialRequest.url, config.offlineResendWaitSec),
|
|
524
529
|
...(sendBudgetState.workflowRootId || sendBudgetState.adapterSendBudget?.spendEnforced || options.comboDispatchPermit || transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt
|
|
525
|
-
|| billingPoolRequest || autoPauseEnabled
|
|
530
|
+
|| anthropicInstance || billingPoolRequest || autoPauseEnabled
|
|
526
531
|
? {
|
|
527
532
|
// A pending compaction permit already booked this leg's first physical send.
|
|
528
|
-
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt || autoPauseEnabled
|
|
533
|
+
...(transientPolicy || resetPolicy || compactPrepaid || options.comboAttempt || anthropicInstance || autoPauseEnabled
|
|
529
534
|
? {
|
|
530
535
|
attempts: Math.min(initialSendCap,
|
|
531
536
|
sendBudgetState.initialSendAllowance(initialSendCap) + (compactPrepaid ? 1 : 0)),
|
|
@@ -611,6 +616,9 @@ export async function prepareAdapterExchange(
|
|
|
611
616
|
const consoleGoUploadRetryGuard: { attempted: boolean } = { attempted: false };
|
|
612
617
|
let oauth401ReplayAttempted = transportState.oauth401ReplayAttempted;
|
|
613
618
|
let antigravityAuthRotationAttempted = false;
|
|
619
|
+
let antigravityFaultRotationAttempted = false;
|
|
620
|
+
const antigravityFaultRotatable = (response: Response) => isAntigravityFaultRotatable(route.providerName, response,
|
|
621
|
+
transportState, antigravityPoolActivated, antigravityFaultRotationAttempted);
|
|
614
622
|
// Direct requests get one reasoning-effort downgrade. Combo candidates may walk down the
|
|
615
623
|
// published ladder one refused rung at a time; the request send budget bounds those retries.
|
|
616
624
|
const reasoningEffortDowngradeGuard: { attempted: boolean } = { attempted: false };
|
|
@@ -877,14 +885,14 @@ export async function prepareAdapterExchange(
|
|
|
877
885
|
const rotateAntigravityAuth = async (
|
|
878
886
|
failedResponse: Response,
|
|
879
887
|
recovery: AttemptRecoveryKind,
|
|
880
|
-
|
|
888
|
+
// An account id (model-404 recovery already chose it) or a selector; the 403 refusal default.
|
|
889
|
+
pick: string | ((sent: { accountId: string; generation: string }) => string | null) = sent =>
|
|
890
|
+
rotateAntigravityAccountOnAuthRefusal(antigravityPoolActivated, sent.accountId, sent.generation, route.modelId),
|
|
881
891
|
): Promise<Response | null> => {
|
|
882
892
|
const sent = transportState.replayOAuthCredentialSnapshot ?? transportState.sentOAuthSnapshot;
|
|
883
893
|
if (!antigravityPoolActivated || !sent
|
|
884
894
|
|| transportState.genericFailovers >= transportState.genericFailoverLimit) return null;
|
|
885
|
-
const nextId =
|
|
886
|
-
antigravityPoolActivated, sent.accountId, sent.generation, route.modelId,
|
|
887
|
-
);
|
|
895
|
+
const nextId = typeof pick === "string" ? pick : pick(sent);
|
|
888
896
|
if (!nextId) return null;
|
|
889
897
|
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
|
|
890
898
|
return recoverAntigravityAccount({ nextId, failedResponse, adapterOwnsDispatch,
|
|
@@ -931,6 +939,7 @@ export async function prepareAdapterExchange(
|
|
|
931
939
|
|
|
932
940
|
// Preserve the terminal verdict through adapter and combo error formatting.
|
|
933
941
|
// This also covers a reset reached by a 401/429/413 recovery refetch.
|
|
942
|
+
await rejectAnthropicPerMessageEffort(parsed, upstreamResponse, upstream.signal);
|
|
934
943
|
if (isNonReplayableResponse(upstreamResponse)) {
|
|
935
944
|
try {
|
|
936
945
|
return await sanitizeNonReplayableUpstreamError(upstreamResponse, upstream.signal);
|
|
@@ -1076,15 +1085,26 @@ export async function prepareAdapterExchange(
|
|
|
1076
1085
|
|
|
1077
1086
|
if (route.providerName === "google-antigravity" && upstreamResponse.status === 403
|
|
1078
1087
|
&& transportState.activeAdapter.name === "google"
|
|
1079
|
-
&& antigravityPoolActivated
|
|
1088
|
+
&& antigravityPoolActivated
|
|
1080
1089
|
&& !isNonReplayableResponse(upstreamResponse)
|
|
1081
1090
|
&& transportState.genericFailovers < transportState.genericFailoverLimit
|
|
1082
|
-
&& await isAntigravityValidationRefusal(upstreamResponse, options.abortSignal)
|
|
1091
|
+
&& (!antigravityAuthRotationAttempted && await isAntigravityValidationRefusal(upstreamResponse, options.abortSignal)
|
|
1092
|
+
// A plain access-denied 403 keeps rotating while siblings remain: each refused account is
|
|
1093
|
+
// cooled before the hop, so the ring advances and `genericFailoverLimit` bounds the sends.
|
|
1094
|
+
|| await isAntigravityAccessDeniedRefusal(upstreamResponse, options.abortSignal))) {
|
|
1083
1095
|
antigravityAuthRotationAttempted = true;
|
|
1084
1096
|
const rotated = await rotateAntigravityAuth(upstreamResponse, "oauth-account-403");
|
|
1085
1097
|
if (rotated) { upstreamResponse = rotated; continue recovery; }
|
|
1086
1098
|
}
|
|
1087
1099
|
|
|
1100
|
+
// A 502/503/504 that outlived the same-account ladder replays once on a live sibling.
|
|
1101
|
+
if (antigravityFaultRotatable(upstreamResponse)) {
|
|
1102
|
+
antigravityFaultRotationAttempted = true;
|
|
1103
|
+
const rotated = await rotateAntigravityAuth(upstreamResponse, "transient-5xx", sent =>
|
|
1104
|
+
rotateAntigravityAccountOnUpstreamFault(antigravityPoolActivated, sent.accountId, sent.generation, route.modelId));
|
|
1105
|
+
if (rotated) { upstreamResponse = rotated; continue recovery; }
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1088
1108
|
// Static API-key pools can recover a credential-scoped 401 without abandoning the
|
|
1089
1109
|
// provider: one revoked or mistyped key says nothing about its siblings. OAuth providers
|
|
1090
1110
|
// refresh above and never enter here — `hasKeyPoolFailover` rejects oauth/forward modes.
|
|
@@ -1367,7 +1387,7 @@ export async function prepareAdapterExchange(
|
|
|
1367
1387
|
model: route.modelId, requestKey: transportState, decision: transportState.anthropicRouteDecision, currentDecision: currentAnthropicRouteDecision, signal: upstream.signal,
|
|
1368
1388
|
// A denied reservation must not rotate on base headroom its bounds cannot see.
|
|
1369
1389
|
canRetry: hop.allowed && (hop.permit !== undefined || !sendBudgetExhausted())
|
|
1370
|
-
&& transportState.anthropicPoolFailovers <
|
|
1390
|
+
&& transportState.anthropicPoolFailovers < transportState.anthropicFailoverLimit,
|
|
1371
1391
|
});
|
|
1372
1392
|
} catch (rotationError) {
|
|
1373
1393
|
hop.permit?.release();
|
|
@@ -1629,8 +1649,9 @@ export async function prepareAdapterExchange(
|
|
|
1629
1649
|
}
|
|
1630
1650
|
upstreamResponse = result;
|
|
1631
1651
|
// The hop's permit is already settled by the dispatch boundary above; continuing
|
|
1632
|
-
// only skips the remaining arms, it does not abandon a reservation.
|
|
1633
|
-
|
|
1652
|
+
// only skips the remaining arms, it does not abandon a reservation. A replacement's
|
|
1653
|
+
// surviving 502/503/504 re-enters recovery so the one-hop fault rotation can run.
|
|
1654
|
+
if (isNonReplayableResponse(upstreamResponse) || antigravityFaultRotatable(upstreamResponse)) continue recovery;
|
|
1634
1655
|
} catch (err) {
|
|
1635
1656
|
// A local admission denial is terminal for the whole request: rotating to another
|
|
1636
1657
|
// account cannot re-scope the key, so the refusal answers in place of the 429.
|
|
@@ -1743,7 +1764,7 @@ export async function prepareAdapterExchange(
|
|
|
1743
1764
|
transportState.genericFailovers += 1;
|
|
1744
1765
|
try { void failedResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ }
|
|
1745
1766
|
upstreamResponse = result;
|
|
1746
|
-
if (isNonReplayableResponse(upstreamResponse)) continue recovery;
|
|
1767
|
+
if (isNonReplayableResponse(upstreamResponse) || antigravityFaultRotatable(upstreamResponse)) continue recovery;
|
|
1747
1768
|
} catch {
|
|
1748
1769
|
hop.permit?.release();
|
|
1749
1770
|
break;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { isNonReplayableResponse } from "../../lib/upstream-retry";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Whether an Antigravity 502/503/504 that outlived the same-account transient ladder may be
|
|
5
|
+
* replayed once on a live sibling. The failing account is cooled briefly (not marked), and the
|
|
6
|
+
* per-request `attempted` flag keeps a pool-wide outage to one extra ladder per request.
|
|
7
|
+
*
|
|
8
|
+
* The recovery loop asks this for the response it is holding and again for the response a 429
|
|
9
|
+
* account hop or a verify-account 403 hop just produced: that hop's replacement can be the 5xx, and
|
|
10
|
+
* without asking here the hop loop would leave recovery with the one-hop fault rotation still unspent.
|
|
11
|
+
*/
|
|
12
|
+
export function isAntigravityFaultRotatable(
|
|
13
|
+
providerName: string,
|
|
14
|
+
response: Response,
|
|
15
|
+
state: { activeAdapter: { name: string }; genericFailovers: number; genericFailoverLimit: number },
|
|
16
|
+
poolActivated: boolean,
|
|
17
|
+
faultRotationAttempted: boolean,
|
|
18
|
+
): boolean {
|
|
19
|
+
return providerName === "google-antigravity" && [502, 503, 504].includes(response.status)
|
|
20
|
+
&& state.activeAdapter.name === "google"
|
|
21
|
+
&& poolActivated && !faultRotationAttempted
|
|
22
|
+
&& !isNonReplayableResponse(response)
|
|
23
|
+
&& state.genericFailovers < state.genericFailoverLimit;
|
|
24
|
+
}
|
|
@@ -1,23 +1,60 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ANTIGRAVITY_ACCESS_DENIED_PREFIX, ANTIGRAVITY_AUTH_FAILED_PREFIX, ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX,
|
|
3
|
+
GOOGLE_PERMISSION_DENIED_SUFFIX, safeAntigravityHttpErrorMessage,
|
|
4
|
+
} from "../../adapters/google-errors";
|
|
2
5
|
import { readBoundedResponseBody } from "../../lib/bounded-body";
|
|
3
|
-
import { hasAntigravityModelNotFoundProvenance } from "../../adapters/antigravity-refusal";
|
|
6
|
+
import { classifyAntigravityRefusal, hasAntigravityModelNotFoundProvenance } from "../../adapters/antigravity-refusal";
|
|
4
7
|
|
|
5
8
|
/** Match only the normalized Antigravity validation-refusal marker. */
|
|
6
9
|
export function hasAntigravityValidationRefusalMarker(text: string): boolean {
|
|
7
10
|
return text.startsWith(`${ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX}: `);
|
|
8
11
|
}
|
|
9
12
|
|
|
13
|
+
/**
|
|
14
|
+
* A normalized 403 that Google itself attributed to this account: a structured `PERMISSION_DENIED`
|
|
15
|
+
* envelope, worded either as a plain denial or as invalid/expired credentials. `classifyGoogle` keeps
|
|
16
|
+
* the "(PERMISSION_DENIED)" suffix on both prefixes only when it parsed that enum. Not a validation
|
|
17
|
+
* demand and not a "verify your account" conviction (the dedicated quarantine path owns that one). The
|
|
18
|
+
* caller cools the account and tries a sibling; nothing here marks the account durably.
|
|
19
|
+
*
|
|
20
|
+
* `classifyGoogle` writes the bare "Antigravity access denied" prefix for every 403, including an empty
|
|
21
|
+
* body, a proxy or WAF HTML page, or an envelope without the enum, and the bare "Antigravity
|
|
22
|
+
* authentication failed" prefix for any 403 whose text says "expired" or "unauthenticated". Neither
|
|
23
|
+
* bare prefix says anything about this account, so neither qualifies: accepting one would walk the
|
|
24
|
+
* whole pool and cool every account on a failure that is wrong for all of them.
|
|
25
|
+
*
|
|
26
|
+
* Text that carries the validation marker is never an access denial: that wording belongs to the
|
|
27
|
+
* validation path, and an upstream message imitating it must not earn a different recovery.
|
|
28
|
+
*/
|
|
29
|
+
export function hasAntigravityAccessDeniedMarker(text: string): boolean {
|
|
30
|
+
return (text.startsWith(`${ANTIGRAVITY_ACCESS_DENIED_PREFIX}${GOOGLE_PERMISSION_DENIED_SUFFIX}`)
|
|
31
|
+
|| text.startsWith(`${ANTIGRAVITY_AUTH_FAILED_PREFIX}${GOOGLE_PERMISSION_DENIED_SUFFIX}`))
|
|
32
|
+
&& !text.includes(ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX)
|
|
33
|
+
&& classifyAntigravityRefusal(403, text).kind === "other";
|
|
34
|
+
}
|
|
35
|
+
|
|
10
36
|
/** Inspect only the Google adapter's normalized, already bounded 403 response. */
|
|
11
37
|
export async function isAntigravityValidationRefusal(response: Response, signal?: AbortSignal): Promise<boolean> {
|
|
12
|
-
return inspectValidationRefusal(response, signal, false);
|
|
38
|
+
return inspectValidationRefusal(response, signal, false, hasAntigravityValidationRefusalMarker);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Normalized 403 whose account is simply refused: cool it and rotate (credential-rejected). */
|
|
42
|
+
export async function isAntigravityAccessDeniedRefusal(response: Response, signal?: AbortSignal): Promise<boolean> {
|
|
43
|
+
return inspectValidationRefusal(response, signal, false, hasAntigravityAccessDeniedMarker);
|
|
13
44
|
}
|
|
14
45
|
|
|
15
46
|
/** Sidecar dispatch retains the raw Google envelope; normalize only a complete bounded body. */
|
|
16
47
|
export async function isAntigravityRawValidationRefusal(response: Response, signal?: AbortSignal): Promise<boolean> {
|
|
17
|
-
return inspectValidationRefusal(response, signal, true
|
|
48
|
+
return inspectValidationRefusal(response, signal, true,
|
|
49
|
+
text => hasAntigravityValidationRefusalMarker(text) || hasAntigravityAccessDeniedMarker(text));
|
|
18
50
|
}
|
|
19
51
|
|
|
20
|
-
async function inspectValidationRefusal(
|
|
52
|
+
async function inspectValidationRefusal(
|
|
53
|
+
response: Response,
|
|
54
|
+
signal: AbortSignal | undefined,
|
|
55
|
+
raw: boolean,
|
|
56
|
+
accepts: (text: string) => boolean,
|
|
57
|
+
): Promise<boolean> {
|
|
21
58
|
if (response.status !== 403 || signal?.aborted) return false;
|
|
22
59
|
try {
|
|
23
60
|
const body = await readBoundedResponseBody(response.clone(), {
|
|
@@ -28,7 +65,7 @@ async function inspectValidationRefusal(response: Response, signal: AbortSignal
|
|
|
28
65
|
signal,
|
|
29
66
|
});
|
|
30
67
|
return !signal?.aborted && body.displaySafe && !body.truncated && !body.timedOut && !body.oversized
|
|
31
|
-
&&
|
|
68
|
+
&& accepts(raw ? safeAntigravityHttpErrorMessage(403, body.text) : body.text);
|
|
32
69
|
} catch {
|
|
33
70
|
return false;
|
|
34
71
|
}
|