@cortexkit/opencode-antigravity-auth 3.1.7 → 3.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +182 -193
- package/dist/cli.js.map +4 -4
- package/dist/cli.meta.json +83 -54
- package/dist/index.js +6886 -6346
- package/dist/index.js.map +4 -4
- package/dist/index.meta.json +475 -387
- package/dist/server.js +3056 -2446
- package/dist/server.js.map +4 -4
- package/dist/server.meta.json +1 -1
- package/dist/src/common-auth-embedded/claustrum/availability.js +1 -1
- package/dist/src/common-auth-embedded/claustrum/custody.js +1 -1
- package/dist/src/common-auth-embedded/fs/private-files.js +14 -5
- package/dist/src/common-auth-embedded/opencode2/index.d.ts +1 -1
- package/dist/src/common-auth-embedded/opencode2/index.js +1 -1
- package/dist/src/common-auth-embedded/opencode2/install.d.ts +6 -1
- package/dist/src/common-auth-embedded/opencode2/install.js +14 -4
- package/dist/src/common-auth-embedded/opencode2/types.d.ts +10 -7
- package/dist/src/common-auth-embedded/pi-slot/index.d.ts +1 -1
- package/dist/src/common-auth-embedded/pi-slot/index.js +1 -1
- package/dist/src/common-auth-embedded/pi-slot/slot.d.ts +6 -9
- package/dist/src/common-auth-embedded/pi-slot/slot.js +42 -105
- package/dist/src/common-auth-embedded/routing-accessor/index.d.ts +66 -16
- package/dist/src/common-auth-embedded/routing-accessor/index.js +123 -17
- package/dist/src/common-auth-embedded/rpc/index.d.ts +1 -1
- package/dist/src/common-auth-embedded/rpc/index.js +1 -1
- package/dist/src/common-auth-embedded/rpc/port-file.d.ts +4 -0
- package/dist/src/common-auth-embedded/rpc/port-file.js +55 -49
- package/dist/src/common-auth-embedded/rpc/rpc-server.js +3 -2
- package/dist/src/common-auth-embedded/rpc/startup-sweep.js +32 -0
- package/dist/src/common-auth-embedded/source-output.json +77 -68
- package/dist/src/common-auth-embedded/store/mutate.js +34 -13
- package/dist/src/ga/server/index.d.ts +1 -1
- package/dist/src/ga/server/index.d.ts.map +1 -1
- package/dist/src/ga/server/index.js +58 -19
- package/dist/src/ga/server/index.js.map +1 -1
- package/dist/src/plugin/__fixtures__/verified-store.d.ts +1 -1
- package/dist/src/plugin/__fixtures__/verified-store.js +3 -3
- package/dist/src/plugin/auth-loader.d.ts.map +1 -1
- package/dist/src/plugin/auth-loader.js +102 -80
- package/dist/src/plugin/auth-loader.js.map +1 -1
- package/dist/src/plugin/fetch-interceptor.d.ts.map +1 -1
- package/dist/src/plugin/fetch-interceptor.js +27 -13
- package/dist/src/plugin/fetch-interceptor.js.map +1 -1
- package/dist/src/plugin/ga-location-services.d.ts.map +1 -1
- package/dist/src/plugin/ga-location-services.js +21 -11
- package/dist/src/plugin/ga-location-services.js.map +1 -1
- package/dist/src/plugin/index.d.ts.map +1 -1
- package/dist/src/plugin/index.js +13 -5
- package/dist/src/plugin/index.js.map +1 -1
- package/dist/src/plugin/request.d.ts.map +1 -1
- package/dist/src/plugin/request.js +2 -2
- package/dist/src/plugin/request.js.map +1 -1
- package/dist/src/plugin/shared/opencode-retry-hints.d.ts +2 -0
- package/dist/src/plugin/shared/opencode-retry-hints.d.ts.map +1 -0
- package/dist/src/plugin/shared/opencode-retry-hints.js +22 -0
- package/dist/src/plugin/shared/opencode-retry-hints.js.map +1 -0
- package/dist/src/plugin/shared/request-services.d.ts +2 -0
- package/dist/src/plugin/shared/request-services.d.ts.map +1 -1
- package/dist/src/plugin/shared/request-services.js +410 -112
- package/dist/src/plugin/shared/request-services.js.map +1 -1
- package/dist/src/plugin/shared/vault-routing-purpose.d.ts +6 -2
- package/dist/src/plugin/shared/vault-routing-purpose.d.ts.map +1 -1
- package/dist/src/plugin/shared/vault-routing-purpose.js +13 -7
- package/dist/src/plugin/shared/vault-routing-purpose.js.map +1 -1
- package/dist/src/plugin/shared/vault-routing.d.ts +8 -1
- package/dist/src/plugin/shared/vault-routing.d.ts.map +1 -1
- package/dist/src/plugin/shared/vault-routing.js +12 -4
- package/dist/src/plugin/shared/vault-routing.js.map +1 -1
- package/dist/src/plugin/storage.d.ts +1 -0
- package/dist/src/plugin/storage.d.ts.map +1 -1
- package/dist/src/plugin/storage.js +3 -3
- package/dist/src/plugin/storage.js.map +1 -1
- package/dist/src/plugin/vault-custody.d.ts +6 -13
- package/dist/src/plugin/vault-custody.d.ts.map +1 -1
- package/dist/src/plugin/vault-custody.js +19 -16
- package/dist/src/plugin/vault-custody.js.map +1 -1
- package/dist/src/runtime-layout.d.ts.map +1 -1
- package/dist/src/runtime-layout.js +5 -2
- package/dist/src/runtime-layout.js.map +1 -1
- package/dist/src/setup.d.ts.map +1 -1
- package/dist/src/setup.js +2 -7
- package/dist/src/setup.js.map +1 -1
- package/dist/tui-build-map.json +9 -9
- package/dist/tui-ga-build-map.json +1 -1
- package/dist/types/src/common-auth-embedded/opencode2/index.d.ts +1 -1
- package/dist/types/src/common-auth-embedded/opencode2/install.d.ts +6 -1
- package/dist/types/src/common-auth-embedded/opencode2/types.d.ts +10 -7
- package/dist/types/src/common-auth-embedded/routing-accessor/index.d.ts +66 -16
- package/dist/types/src/ga/server/index.d.ts +1 -1
- package/dist/types/src/ga/server/index.d.ts.map +1 -1
- package/dist/types/src/plugin/auth-loader.d.ts.map +1 -1
- package/dist/types/src/plugin/fetch-interceptor.d.ts.map +1 -1
- package/dist/types/src/plugin/ga-location-services.d.ts.map +1 -1
- package/dist/types/src/plugin/index.d.ts.map +1 -1
- package/dist/types/src/plugin/request.d.ts.map +1 -1
- package/dist/types/src/plugin/shared/opencode-retry-hints.d.ts +2 -0
- package/dist/types/src/plugin/shared/opencode-retry-hints.d.ts.map +1 -0
- package/dist/types/src/plugin/shared/request-services.d.ts +2 -0
- package/dist/types/src/plugin/shared/request-services.d.ts.map +1 -1
- package/dist/types/src/plugin/shared/vault-routing-purpose.d.ts +6 -2
- package/dist/types/src/plugin/shared/vault-routing-purpose.d.ts.map +1 -1
- package/dist/types/src/plugin/shared/vault-routing.d.ts +8 -1
- package/dist/types/src/plugin/shared/vault-routing.d.ts.map +1 -1
- package/dist/types/src/plugin/storage.d.ts +1 -0
- package/dist/types/src/plugin/storage.d.ts.map +1 -1
- package/dist/types/src/plugin/vault-custody.d.ts +6 -13
- package/dist/types/src/plugin/vault-custody.d.ts.map +1 -1
- package/dist/types/src/runtime-layout.d.ts.map +1 -1
- package/dist/types/src/setup.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/common-auth-embedded/claustrum/availability.js +1 -1
- package/src/common-auth-embedded/claustrum/custody.js +1 -1
- package/src/common-auth-embedded/fs/private-files.js +14 -5
- package/src/common-auth-embedded/opencode2/index.d.ts +1 -1
- package/src/common-auth-embedded/opencode2/index.js +1 -1
- package/src/common-auth-embedded/opencode2/install.d.ts +6 -1
- package/src/common-auth-embedded/opencode2/install.js +14 -4
- package/src/common-auth-embedded/opencode2/types.d.ts +10 -7
- package/src/common-auth-embedded/pi-slot/index.d.ts +1 -1
- package/src/common-auth-embedded/pi-slot/index.js +1 -1
- package/src/common-auth-embedded/pi-slot/slot.d.ts +6 -9
- package/src/common-auth-embedded/pi-slot/slot.js +42 -105
- package/src/common-auth-embedded/routing-accessor/index.d.ts +66 -16
- package/src/common-auth-embedded/routing-accessor/index.js +123 -17
- package/src/common-auth-embedded/rpc/index.d.ts +1 -1
- package/src/common-auth-embedded/rpc/index.js +1 -1
- package/src/common-auth-embedded/rpc/port-file.d.ts +4 -0
- package/src/common-auth-embedded/rpc/port-file.js +55 -49
- package/src/common-auth-embedded/rpc/rpc-server.js +3 -2
- package/src/common-auth-embedded/rpc/startup-sweep.js +32 -0
- package/src/common-auth-embedded/source-output.json +77 -68
- package/src/common-auth-embedded/store/mutate.js +34 -13
- package/src/tui-compiled/common-auth-embedded/fs/private-files.js +14 -5
- package/src/tui-compiled/common-auth-embedded/rpc/port-file.js +55 -49
- package/src/tui-raw/common-auth-embedded/fs/private-files.js +14 -5
- package/src/tui-raw/common-auth-embedded/rpc/port-file.js +55 -49
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* cache, debug log and logger; the raw sender is called unchanged.
|
|
18
18
|
*/
|
|
19
19
|
import { createHash, randomUUID } from 'node:crypto';
|
|
20
|
-
import { accessTokenExpired, calculateBackoffMs, computeSoftQuotaCacheTtlMs, getPublicModelDefinitions, getQuotaKey, getResolverAliasMap, isClaudeThinkingModel, isImageGenerationModel, isRetiredAgyModel, MAX_UNCONFIRMED_RATE_LIMIT_RETRIES, PRE_RESPONSE_DEADLINE_MS, PreResponseDeadline, PreResponseTimeoutError, parseRateLimitReason, parseRefreshParts, preResponseTimeoutResponse, preserveResponseMetadata, quotaAccountIdentity, quotaExhaustedUntil, RETIRED_AGY_MODEL_MESSAGE, redactCredentialError, redactCredentialResponse, redactCredentialText, resolveQuotaGroup, systemInstructionHasHarnessIdentity, UNCONFIRMED_RATE_LIMIT_MESSAGE, unconfirmedRateLimitDelayMs, VAULT_ACCOUNT_REMEDY, } from '@cortexkit/antigravity-auth-core';
|
|
20
|
+
import { accessTokenExpired, calculateBackoffMs, computeSoftQuotaCacheTtlMs, getPublicModelDefinitions, getQuotaKey, getResolverAliasMap, HostRefusalError, isClaudeThinkingModel, isImageGenerationModel, isRetiredAgyModel, isUpstreamFailure, MAX_UNCONFIRMED_RATE_LIMIT_RETRIES, markUpstreamFailure, PRE_RESPONSE_DEADLINE_MS, PreResponseDeadline, PreResponseTimeoutError, parseRateLimitReason, parseRefreshParts, preResponseTimeoutResponse, preserveResponseMetadata, quotaAccountIdentity, quotaExhaustedUntil, RETIRED_AGY_MODEL_MESSAGE, redactCredentialError, redactCredentialResponse, redactCredentialText, resolveQuotaGroup, systemInstructionHasHarnessIdentity, UNCONFIRMED_RATE_LIMIT_MESSAGE, unconfirmedRateLimitDelayMs, VAULT_ACCOUNT_REMEDY, } from '@cortexkit/antigravity-auth-core';
|
|
21
21
|
import { hasVaultHost } from '../../common-auth-embedded/claustrum/host.js';
|
|
22
22
|
import { askRouting, reportRouting as reportVaultRouting, } from '../../common-auth-embedded/routing-accessor/index.js';
|
|
23
23
|
import { ANTIGRAVITY_ENDPOINT_DAILY, ANTIGRAVITY_ENDPOINT_FALLBACKS, } from '../../constants';
|
|
@@ -34,7 +34,7 @@ import { isLocalGrantError, LocalGrantSupersededError, } from './local-grant';
|
|
|
34
34
|
import { createRequestCancellation, retainResponseLifetime, } from './request-cancellation';
|
|
35
35
|
import { VaultBillingAttempt } from './vault-billing';
|
|
36
36
|
import { hasVaultRoutingAccessor, observeVaultServed, observeVaultUpstream, reportVaultExhaustion, } from './vault-routing';
|
|
37
|
-
import {
|
|
37
|
+
import { isTitleVaultRequest } from './vault-routing-purpose';
|
|
38
38
|
/**
|
|
39
39
|
* Wait before retrying the same selection after its first 429.
|
|
40
40
|
*/
|
|
@@ -46,6 +46,67 @@ const MODEL_UNAVAILABLE_TTL_MS = 60 * 60 * 1000;
|
|
|
46
46
|
// while the wait itself is rare. Use the routing accessor's 2 s ceiling, well
|
|
47
47
|
// above routing's normal admission time.
|
|
48
48
|
const ROUTING_ASK_DEADLINE_MS = 2000;
|
|
49
|
+
const REFUSAL_BODY_TIMEOUT_MS = 2_000;
|
|
50
|
+
const REFUSAL_BODY_MAX_BYTES = 64 * 1024;
|
|
51
|
+
// Replay only a complete body so response processing and debug logging never
|
|
52
|
+
// reread a stalled or oversized upstream stream. Keep the real status and retry
|
|
53
|
+
// headers even when the body cannot be used as evidence.
|
|
54
|
+
async function readRefusalBody(response, deadline) {
|
|
55
|
+
const reader = response.body?.getReader();
|
|
56
|
+
if (!reader)
|
|
57
|
+
return { response, text: '' };
|
|
58
|
+
const buffer = new Uint8Array(REFUSAL_BODY_MAX_BYTES);
|
|
59
|
+
const stopAt = performance.now() + REFUSAL_BODY_TIMEOUT_MS;
|
|
60
|
+
let size = 0;
|
|
61
|
+
let finished = false;
|
|
62
|
+
let timer;
|
|
63
|
+
try {
|
|
64
|
+
finished = await deadline.wait(() => Promise.race([
|
|
65
|
+
(async () => {
|
|
66
|
+
try {
|
|
67
|
+
while (true) {
|
|
68
|
+
const chunk = await reader.read();
|
|
69
|
+
if (performance.now() >= stopAt)
|
|
70
|
+
return false;
|
|
71
|
+
if (chunk.done)
|
|
72
|
+
return true;
|
|
73
|
+
if (size + chunk.value.byteLength > buffer.byteLength)
|
|
74
|
+
return false;
|
|
75
|
+
buffer.set(chunk.value, size);
|
|
76
|
+
size += chunk.value.byteLength;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
})(),
|
|
83
|
+
new Promise((resolve) => {
|
|
84
|
+
timer = setTimeout(() => resolve(false), REFUSAL_BODY_TIMEOUT_MS);
|
|
85
|
+
}),
|
|
86
|
+
]));
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
clearTimeout(timer);
|
|
90
|
+
if (!finished)
|
|
91
|
+
void reader.cancel().catch(() => { });
|
|
92
|
+
reader.releaseLock();
|
|
93
|
+
}
|
|
94
|
+
const body = finished ? buffer.subarray(0, size) : null;
|
|
95
|
+
const headers = new Headers(response.headers);
|
|
96
|
+
if (!finished) {
|
|
97
|
+
headers.delete('content-length');
|
|
98
|
+
headers.delete('content-encoding');
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
response: preserveResponseMetadata(new Response(body, {
|
|
102
|
+
status: response.status,
|
|
103
|
+
statusText: response.statusText,
|
|
104
|
+
headers,
|
|
105
|
+
}), response),
|
|
106
|
+
text: body ? new TextDecoder().decode(body) : undefined,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const routingIdentityKey = ({ accountId, credentialId }) => JSON.stringify([accountId, credentialId]);
|
|
49
110
|
/**
|
|
50
111
|
* Whole seconds a routing refusal asks the client to wait, or undefined when
|
|
51
112
|
* routing gave no usable timing. Routing's own retry delay wins; otherwise the
|
|
@@ -542,8 +603,6 @@ export function createRequestExecutor(deps) {
|
|
|
542
603
|
...sendOptions,
|
|
543
604
|
signal: deadline.signal,
|
|
544
605
|
};
|
|
545
|
-
if (!onSenderRejection)
|
|
546
|
-
return sender(url, sendInit, optionsWithSignal);
|
|
547
606
|
let pending;
|
|
548
607
|
try {
|
|
549
608
|
// Keep this observer at the raw sender boundary. Errors from the
|
|
@@ -552,12 +611,12 @@ export function createRequestExecutor(deps) {
|
|
|
552
611
|
pending = sender(url, sendInit, optionsWithSignal);
|
|
553
612
|
}
|
|
554
613
|
catch (error) {
|
|
555
|
-
onSenderRejection(error);
|
|
556
|
-
throw error;
|
|
614
|
+
onSenderRejection?.(error);
|
|
615
|
+
throw markUpstreamFailure(error);
|
|
557
616
|
}
|
|
558
617
|
return pending.catch((error) => {
|
|
559
|
-
onSenderRejection(error);
|
|
560
|
-
throw error;
|
|
618
|
+
onSenderRejection?.(error);
|
|
619
|
+
throw markUpstreamFailure(error);
|
|
561
620
|
});
|
|
562
621
|
});
|
|
563
622
|
const requestSessionIdentity = options.session ?? extractOpenCodeSessionIdentity(init?.headers);
|
|
@@ -678,11 +737,70 @@ export function createRequestExecutor(deps) {
|
|
|
678
737
|
// cross-request memory during this request. Neither skip is a penalty.
|
|
679
738
|
const modelUnavailableIndexes = new Set();
|
|
680
739
|
let routingPlacementConsidered = false;
|
|
740
|
+
let routedCandidate;
|
|
741
|
+
let continueRouting;
|
|
742
|
+
const refusedRoutes = new Map();
|
|
743
|
+
let routingTitle = false;
|
|
744
|
+
let routingWaitMs = 0;
|
|
745
|
+
const failedRoutedPick = () => createNativeGoogleErrorResponse({
|
|
746
|
+
status: 412,
|
|
747
|
+
reason: 'routing_refused',
|
|
748
|
+
});
|
|
749
|
+
const routingWaitRemainingMs = () => Math.min(deadline.remainingMs(), maxWaitMs > 0 ? Math.max(0, maxWaitMs - routingWaitMs) : Infinity);
|
|
750
|
+
const routingRefusalResponse = (refused) => {
|
|
751
|
+
const { code, routingCode, reason } = refused;
|
|
752
|
+
log.warn('routing-refused', {
|
|
753
|
+
code,
|
|
754
|
+
...(routingCode === undefined ? {} : { routingCode }),
|
|
755
|
+
...(reason === undefined ? {} : { reason }),
|
|
756
|
+
});
|
|
757
|
+
const retryAfterSeconds = routingRefusalRetryAfterSeconds(refused, Date.now());
|
|
758
|
+
return createNativeGoogleErrorResponse({
|
|
759
|
+
status: 412,
|
|
760
|
+
reason: 'routing_refused',
|
|
761
|
+
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
762
|
+
});
|
|
763
|
+
};
|
|
764
|
+
const waitOnRoutedAccount = async (ms, reason, attempt) => {
|
|
765
|
+
if (ms >= deadline.remainingMs() ||
|
|
766
|
+
(maxWaitMs > 0 && routingWaitMs + ms > maxWaitMs))
|
|
767
|
+
return false;
|
|
768
|
+
routingWaitMs += ms;
|
|
769
|
+
if (ms > 0)
|
|
770
|
+
await routingSleep(ms, reason, attempt);
|
|
771
|
+
checkAborted();
|
|
772
|
+
return true;
|
|
773
|
+
};
|
|
774
|
+
const unconfirmedRefusal = () => new Response(JSON.stringify({
|
|
775
|
+
error: {
|
|
776
|
+
code: 412,
|
|
777
|
+
status: 'FAILED_PRECONDITION',
|
|
778
|
+
message: UNCONFIRMED_RATE_LIMIT_MESSAGE,
|
|
779
|
+
},
|
|
780
|
+
}), { status: 412, headers: { 'Content-Type': 'application/json' } });
|
|
781
|
+
const retryAfterRoutedBackoff = async (identity, retryAtMs, response) => {
|
|
782
|
+
checkAborted();
|
|
783
|
+
reportVaultRouting({ ...identity, kind: 'transient_backoff', retryAtMs });
|
|
784
|
+
void response.body?.cancel().catch(() => { });
|
|
785
|
+
if (unconfirmedRetries >= MAX_UNCONFIRMED_RATE_LIMIT_RETRIES)
|
|
786
|
+
return unconfirmedRefusal();
|
|
787
|
+
unconfirmedRetries++;
|
|
788
|
+
if (!(await waitOnRoutedAccount(Math.max(0, retryAtMs - Date.now()), 'rate_limited', unconfirmedRetries)))
|
|
789
|
+
return unconfirmedRefusal();
|
|
790
|
+
};
|
|
681
791
|
while (true) {
|
|
682
792
|
await assertAdmission();
|
|
683
793
|
checkAborted();
|
|
794
|
+
const choice = routedCandidate;
|
|
795
|
+
routedCandidate = undefined;
|
|
796
|
+
const exhausted = continueRouting;
|
|
797
|
+
continueRouting = undefined;
|
|
798
|
+
if (exhausted)
|
|
799
|
+
refusedRoutes.set(routingIdentityKey(exhausted), exhausted);
|
|
684
800
|
const accountCount = accountManager.getAccountCount();
|
|
685
801
|
if (accountCount === 0) {
|
|
802
|
+
if (choice)
|
|
803
|
+
return failedRoutedPick();
|
|
686
804
|
// Mirror the no-account short-circuit inside the retry loop so callers
|
|
687
805
|
// that race with account removal still get a proper 401 instead of a
|
|
688
806
|
// hang or a synthetic 200.
|
|
@@ -694,7 +812,7 @@ export function createRequestExecutor(deps) {
|
|
|
694
812
|
// else looks at it. It is not a rate limit or cooldown, so nothing is
|
|
695
813
|
// marked and no wait estimate counts it as coming back.
|
|
696
814
|
const allowlistExcluded = modelAccessExcluded(model);
|
|
697
|
-
if (allowlistExcluded) {
|
|
815
|
+
if (allowlistExcluded && !exhausted && !choice) {
|
|
698
816
|
const serviceable = accountManager
|
|
699
817
|
.getAccounts()
|
|
700
818
|
.filter((entry) => entry.enabled !== false &&
|
|
@@ -721,6 +839,35 @@ export function createRequestExecutor(deps) {
|
|
|
721
839
|
.getAccounts()
|
|
722
840
|
.filter((entry) => !allowlistExcluded.has(entry.index))
|
|
723
841
|
: accountManager.getAccounts();
|
|
842
|
+
const modelUnavailableForPass = () => new Set([...modelUnavailableIndexes, ...rememberedIndexes(model)]);
|
|
843
|
+
let routingUnavailableIndexes;
|
|
844
|
+
const vaultCredentials = credentials.domain === 'vault' ? credentials : undefined;
|
|
845
|
+
const routingPresent = vaultCredentials !== undefined && hasVaultRoutingAccessor();
|
|
846
|
+
routingTitle ||=
|
|
847
|
+
routingPresent && isTitleVaultRequest(kind, init?.headers);
|
|
848
|
+
let account = null;
|
|
849
|
+
if (routingPresent &&
|
|
850
|
+
accountSessionIdentity &&
|
|
851
|
+
vaultCredentials?.routingCandidate) {
|
|
852
|
+
const candidateFor = vaultCredentials.routingCandidate;
|
|
853
|
+
const row = choice
|
|
854
|
+
? accountManager.getAccounts().find((entry) => {
|
|
855
|
+
const current = candidateFor(entry);
|
|
856
|
+
return (current.credentialId === choice.credentialId &&
|
|
857
|
+
current.accountId === choice.accountId);
|
|
858
|
+
})
|
|
859
|
+
: !routingPlacementConsidered
|
|
860
|
+
? accountManager.getRoutedAccountForFamily(family, accountSessionIdentity)
|
|
861
|
+
: null;
|
|
862
|
+
if (row && !allowlistExcluded?.has(row.index)) {
|
|
863
|
+
routingUnavailableIndexes = modelUnavailableForPass();
|
|
864
|
+
if (!routingUnavailableIndexes.has(row.index))
|
|
865
|
+
account = accountManager.useRoutedAccount(row, family, model ?? undefined, accountSessionIdentity, !routingTitle);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
const routedPlacement = account !== null;
|
|
869
|
+
if (choice && !routedPlacement)
|
|
870
|
+
return failedRoutedPick();
|
|
724
871
|
// Operator killswitch: an account whose cached remaining quota for
|
|
725
872
|
// this model's quota group is below the configured floor is not
|
|
726
873
|
// eligible. Missing or stale quota counts as eligible, so a cold
|
|
@@ -729,7 +876,7 @@ export function createRequestExecutor(deps) {
|
|
|
729
876
|
// end in the operator refusal.
|
|
730
877
|
const operatorKillswitch = operatorSettings?.get().killswitch;
|
|
731
878
|
const floorKeying = operatorSettings?.floorKeying?.();
|
|
732
|
-
const eligibleIndexes = operatorKillswitch?.enabled
|
|
879
|
+
const eligibleIndexes = !routedPlacement && operatorKillswitch?.enabled
|
|
733
880
|
? new Set(permittedAccounts
|
|
734
881
|
.filter((entry) => {
|
|
735
882
|
const decision = evaluateKillswitchForAccount(entry, family, {
|
|
@@ -744,7 +891,8 @@ export function createRequestExecutor(deps) {
|
|
|
744
891
|
})
|
|
745
892
|
.map((entry) => entry.index))
|
|
746
893
|
: null;
|
|
747
|
-
if (
|
|
894
|
+
if (!exhausted &&
|
|
895
|
+
eligibleIndexes !== null &&
|
|
748
896
|
eligibleIndexes.size === 0 &&
|
|
749
897
|
permittedAccounts.length > 0) {
|
|
750
898
|
try {
|
|
@@ -788,11 +936,8 @@ export function createRequestExecutor(deps) {
|
|
|
788
936
|
.getAccounts()
|
|
789
937
|
.map((entry) => entry.index)
|
|
790
938
|
.filter((index) => !eligibleIndexes.has(index)));
|
|
791
|
-
const unavailableIndexes =
|
|
792
|
-
|
|
793
|
-
...rememberedIndexes(model),
|
|
794
|
-
]);
|
|
795
|
-
if (unavailableIndexes.size > 0) {
|
|
939
|
+
const unavailableIndexes = routingUnavailableIndexes ?? modelUnavailableForPass();
|
|
940
|
+
if (!exhausted && !routedPlacement && unavailableIndexes.size > 0) {
|
|
796
941
|
const candidates = accountManager
|
|
797
942
|
.getAccounts()
|
|
798
943
|
.filter((entry) => entry.enabled !== false &&
|
|
@@ -824,77 +969,110 @@ export function createRequestExecutor(deps) {
|
|
|
824
969
|
...unavailableIndexes,
|
|
825
970
|
...(allowlistExcluded ?? []),
|
|
826
971
|
]);
|
|
827
|
-
const routingPresent = credentials.domain === 'vault' && hasVaultRoutingAccessor();
|
|
828
972
|
const firstPlacement = routingPresent &&
|
|
973
|
+
!routedPlacement &&
|
|
829
974
|
!routingPlacementConsidered &&
|
|
830
975
|
accountSessionIdentity &&
|
|
831
976
|
!accountManager.getCurrentAccountForFamily(family, accountSessionIdentity);
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
977
|
+
const isHardEligible = (row) => row.enabled !== false &&
|
|
978
|
+
!row.accountIneligible &&
|
|
979
|
+
!row.verificationRequired &&
|
|
980
|
+
!allowlistExcluded?.has(row.index) &&
|
|
981
|
+
!unavailableIndexes.has(row.index);
|
|
982
|
+
const isRoutingCandidate = (row) => isHardEligible(row) &&
|
|
983
|
+
!killedIndexes?.has(row.index) &&
|
|
984
|
+
!accountManager.isAccountOverSoftQuota(row, family, config.soft_quota_threshold_percent, softQuotaCacheTtlMs, model, excludedIndexes);
|
|
985
|
+
const selectLocally = () => accountManager.getCurrentOrNextForFamily(family, model, config.account_selection_strategy, config.pid_offset_enabled, config.soft_quota_threshold_percent, softQuotaCacheTtlMs, accountSessionIdentity, excludedIndexes, !routingTitle);
|
|
986
|
+
// Reserve a local first choice before asking, but don't replace a routed
|
|
987
|
+
// session pin with a provisional choice during continuation.
|
|
988
|
+
if (!exhausted)
|
|
989
|
+
account ??= selectLocally();
|
|
990
|
+
if (account)
|
|
835
991
|
routingPlacementConsidered = true;
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
992
|
+
if ((firstPlacement || exhausted) &&
|
|
993
|
+
routingPresent &&
|
|
994
|
+
accountSessionIdentity &&
|
|
995
|
+
vaultCredentials?.routingCandidate &&
|
|
996
|
+
model) {
|
|
997
|
+
const roster = accountManager.getAccounts();
|
|
998
|
+
const currentKey = exhausted && routingIdentityKey(exhausted);
|
|
999
|
+
const holdCandidate = exhausted?.limitWindow !== 'signed_out' && currentKey;
|
|
1000
|
+
const candidates = roster.flatMap((row) => {
|
|
1001
|
+
const eligible = isRoutingCandidate(row);
|
|
1002
|
+
if (!eligible && (!holdCandidate || !isHardEligible(row)))
|
|
1003
|
+
return [];
|
|
1004
|
+
const candidate = { ...vaultCredentials.routingCandidate(row) };
|
|
1005
|
+
const key = routingIdentityKey(candidate);
|
|
1006
|
+
// A quota hold may keep the refused credential even when local quota
|
|
1007
|
+
// policy excludes it. It is not a replacement pick.
|
|
1008
|
+
if (key === holdCandidate && isHardEligible(row))
|
|
1009
|
+
return [candidate];
|
|
1010
|
+
const refusal = refusedRoutes.get(key);
|
|
1011
|
+
if (refusal &&
|
|
1012
|
+
(refusal.limitWindow === 'signed_out' ||
|
|
1013
|
+
(refusal.resetsAtMs ?? Infinity) > Date.now()))
|
|
1014
|
+
return [];
|
|
1015
|
+
return eligible ? [candidate] : [];
|
|
1016
|
+
});
|
|
1017
|
+
if (candidates.length > 0 || exhausted) {
|
|
1018
|
+
routingPlacementConsidered = true;
|
|
1019
|
+
const canHold = !!holdCandidate &&
|
|
1020
|
+
candidates.some((candidate) => routingIdentityKey(candidate) === holdCandidate);
|
|
1021
|
+
const answer = await deadline.wait(() => {
|
|
1022
|
+
const input = {
|
|
856
1023
|
sessionId: accountSessionIdentity.id,
|
|
857
1024
|
provider: 'antigravity',
|
|
858
1025
|
modelId: model,
|
|
859
1026
|
candidates,
|
|
860
1027
|
deadlineMs: Math.min(ROUTING_ASK_DEADLINE_MS, deadline.remainingMs()),
|
|
861
1028
|
refusals: true,
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
}
|
|
873
|
-
|
|
1029
|
+
};
|
|
1030
|
+
return askRouting(exhausted
|
|
1031
|
+
? {
|
|
1032
|
+
...input,
|
|
1033
|
+
mode: 'continue',
|
|
1034
|
+
requestId: randomUUID(),
|
|
1035
|
+
exhausted,
|
|
1036
|
+
holdWaitMaxS: canHold
|
|
1037
|
+
? Math.floor(routingWaitRemainingMs() / 1000)
|
|
1038
|
+
: 0,
|
|
1039
|
+
}
|
|
1040
|
+
: input);
|
|
1041
|
+
});
|
|
1042
|
+
checkAborted();
|
|
1043
|
+
if (answer && 'refused' in answer)
|
|
1044
|
+
return routingRefusalResponse(answer.refused);
|
|
1045
|
+
if (exhausted && answer?.hold) {
|
|
1046
|
+
if (!canHold ||
|
|
1047
|
+
routingIdentityKey(answer.candidate) !== holdCandidate)
|
|
1048
|
+
return failedRoutedPick();
|
|
1049
|
+
const delay = answer.hold.retry_after_ms;
|
|
1050
|
+
if (!(await waitOnRoutedAccount(delay, 'account_limit')))
|
|
874
1051
|
return createNativeGoogleErrorResponse({
|
|
875
1052
|
status: 412,
|
|
876
|
-
reason: '
|
|
877
|
-
|
|
878
|
-
? {}
|
|
879
|
-
: { retryAfterSeconds }),
|
|
1053
|
+
reason: 'pool_unavailable',
|
|
1054
|
+
resetAfterMs: delay,
|
|
880
1055
|
});
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
1056
|
+
}
|
|
1057
|
+
else if (answer &&
|
|
1058
|
+
exhausted &&
|
|
1059
|
+
routingIdentityKey(answer.candidate) === holdCandidate) {
|
|
1060
|
+
return failedRoutedPick();
|
|
1061
|
+
}
|
|
1062
|
+
const currentRoster = accountManager.getAccounts();
|
|
1063
|
+
if (answer ||
|
|
1064
|
+
exhausted ||
|
|
1065
|
+
currentRoster.length !== roster.length ||
|
|
1066
|
+
currentRoster.some((row, index) => row !== roster[index])) {
|
|
1067
|
+
// A matched pick is revalidated before it can send. An absent
|
|
1068
|
+
// answer can still fall back to ordinary local selection.
|
|
1069
|
+
routedCandidate = answer?.candidate;
|
|
1070
|
+
continue;
|
|
895
1071
|
}
|
|
896
1072
|
}
|
|
897
1073
|
}
|
|
1074
|
+
if (exhausted)
|
|
1075
|
+
account ??= selectLocally();
|
|
898
1076
|
// The selected account must be inside the eligibility mask and
|
|
899
1077
|
// permitted by its model list.
|
|
900
1078
|
if (account && allowlistExcluded?.has(account.index)) {
|
|
@@ -1038,7 +1216,8 @@ export function createRequestExecutor(deps) {
|
|
|
1038
1216
|
pushDebug(`account-switch: ${previousAccountIndex} → ${account.index}, warmup=${needsCacheWarmup}`);
|
|
1039
1217
|
}
|
|
1040
1218
|
previousAccountIndex = account.index;
|
|
1041
|
-
|
|
1219
|
+
if (!routingTitle)
|
|
1220
|
+
accountManager.recordSessionUsage(account.index, accountSessionIdentity);
|
|
1042
1221
|
if (debug.isDebugEnabled()) {
|
|
1043
1222
|
debug.logAccountContext('Selected', {
|
|
1044
1223
|
index: account.index,
|
|
@@ -1239,6 +1418,9 @@ export function createRequestExecutor(deps) {
|
|
|
1239
1418
|
}
|
|
1240
1419
|
}
|
|
1241
1420
|
const selected = account;
|
|
1421
|
+
const expectedRoutedCredential = routedPlacement
|
|
1422
|
+
? (choice ?? vaultCredentials?.routingCandidate?.(selected))
|
|
1423
|
+
: undefined;
|
|
1242
1424
|
const sendPhysical = async (url, sendInit, admission, sendModel, requestKind, onMainTransportFailure) => {
|
|
1243
1425
|
const source = credentials.domain === 'local'
|
|
1244
1426
|
? 'local'
|
|
@@ -1401,7 +1583,8 @@ export function createRequestExecutor(deps) {
|
|
|
1401
1583
|
if (account.fingerprint) {
|
|
1402
1584
|
pushDebug(`fingerprint: deviceId=${account.fingerprint.deviceId.slice(0, 8)}...`);
|
|
1403
1585
|
}
|
|
1404
|
-
if (
|
|
1586
|
+
if (!routedPlacement &&
|
|
1587
|
+
accountManager.isRateLimited(account, family, model)) {
|
|
1405
1588
|
shouldSwitchAccount = true;
|
|
1406
1589
|
}
|
|
1407
1590
|
let totalCapacityRetries = 0;
|
|
@@ -1435,12 +1618,23 @@ export function createRequestExecutor(deps) {
|
|
|
1435
1618
|
if (credentials.domain === 'vault') {
|
|
1436
1619
|
// A rotated admission taken after a 401 serves exactly the
|
|
1437
1620
|
// retry it was taken for, then is gone.
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
account,
|
|
1442
|
-
|
|
1443
|
-
|
|
1621
|
+
try {
|
|
1622
|
+
sendAdmission =
|
|
1623
|
+
rotatedAdmission ??
|
|
1624
|
+
(await deadline.wait(() => credentials.admit({ account, signal: abortSignal })));
|
|
1625
|
+
}
|
|
1626
|
+
catch (error) {
|
|
1627
|
+
checkAborted();
|
|
1628
|
+
if (routedPlacement)
|
|
1629
|
+
return failedRoutedPick();
|
|
1630
|
+
throw error;
|
|
1631
|
+
}
|
|
1632
|
+
if (expectedRoutedCredential &&
|
|
1633
|
+
(sendAdmission.credentialId !==
|
|
1634
|
+
expectedRoutedCredential.credentialId ||
|
|
1635
|
+
sendAdmission.accountIdentity !==
|
|
1636
|
+
expectedRoutedCredential.accountId))
|
|
1637
|
+
return failedRoutedPick();
|
|
1444
1638
|
rotatedAdmission = undefined;
|
|
1445
1639
|
sendAccessToken = sendAdmission.accessToken;
|
|
1446
1640
|
sendProjectId = sendAdmission.projectId;
|
|
@@ -1536,7 +1730,16 @@ export function createRequestExecutor(deps) {
|
|
|
1536
1730
|
}
|
|
1537
1731
|
pushDebug(`dispatching request via ${prepared.headerStyle} transport`);
|
|
1538
1732
|
physicalAttemptStarted = true;
|
|
1539
|
-
|
|
1733
|
+
let response = await sendPhysical(toUrlString(prepared.request), prepared.init, sendAdmission, prepared.effectiveModel ?? model ?? 'unknown', kind ?? 'primary', retainMainTransportFailure);
|
|
1734
|
+
const receivedAtMs = Date.now();
|
|
1735
|
+
let boundedRefusal;
|
|
1736
|
+
if (routingPresent &&
|
|
1737
|
+
(response.status === 429 ||
|
|
1738
|
+
response.status === 403 ||
|
|
1739
|
+
response.status === 404)) {
|
|
1740
|
+
boundedRefusal = await readRefusalBody(response, deadline);
|
|
1741
|
+
response = boundedRefusal.response;
|
|
1742
|
+
}
|
|
1540
1743
|
if (response.ok)
|
|
1541
1744
|
deadline.acceptHeaders();
|
|
1542
1745
|
apiRequestCount++;
|
|
@@ -1596,6 +1799,20 @@ export function createRequestExecutor(deps) {
|
|
|
1596
1799
|
// actual account/header-style used by this request. Fire-and-
|
|
1597
1800
|
// forget — the dispatch path must not wait on a state write.
|
|
1598
1801
|
const sessionKey = requestSessionIdentity.sessionId;
|
|
1802
|
+
const unreadableLimit = response.status === 429 &&
|
|
1803
|
+
boundedRefusal !== undefined &&
|
|
1804
|
+
boundedRefusal.text === undefined;
|
|
1805
|
+
const routingIdentity = (routedPlacement || unreadableLimit) &&
|
|
1806
|
+
sendAdmission &&
|
|
1807
|
+
sessionKey &&
|
|
1808
|
+
hasVaultRoutingAccessor()
|
|
1809
|
+
? {
|
|
1810
|
+
sessionId: sessionKey,
|
|
1811
|
+
accountId: sendAdmission.accountIdentity,
|
|
1812
|
+
credentialId: sendAdmission.credentialId,
|
|
1813
|
+
}
|
|
1814
|
+
: undefined;
|
|
1815
|
+
const routedIdentity = routedPlacement ? routingIdentity : undefined;
|
|
1599
1816
|
if (sessionKey) {
|
|
1600
1817
|
const routingEntry = {
|
|
1601
1818
|
accountId: `acct-${account.index}`,
|
|
@@ -1617,7 +1834,9 @@ export function createRequestExecutor(deps) {
|
|
|
1617
1834
|
const _maxBackoffMs = (config.max_backoff_seconds ?? 60) * 1000;
|
|
1618
1835
|
const bodyInfo = await (async () => {
|
|
1619
1836
|
try {
|
|
1620
|
-
const text =
|
|
1837
|
+
const text = boundedRefusal
|
|
1838
|
+
? (boundedRefusal.text ?? '')
|
|
1839
|
+
: await deadline.wait(() => response.clone().text());
|
|
1621
1840
|
try {
|
|
1622
1841
|
return JSON.parse(text);
|
|
1623
1842
|
}
|
|
@@ -1633,24 +1852,38 @@ export function createRequestExecutor(deps) {
|
|
|
1633
1852
|
const reasonInfo = extractRateLimitBodyInfo(bodyInfo, response.status, response.headers);
|
|
1634
1853
|
const serverRetryMs = reasonInfo.retryDelayMs ?? defaultRetryMs;
|
|
1635
1854
|
const rateLimitReason = parseRateLimitReason(reasonInfo.reason, reasonInfo.message, response.status);
|
|
1855
|
+
const retryAtMs = Math.min(Number.MAX_SAFE_INTEGER,
|
|
1856
|
+
// Use the short unconfirmed cooldown when the response doesn't
|
|
1857
|
+
// contain a retry time recognized by the executor.
|
|
1858
|
+
receivedAtMs +
|
|
1859
|
+
(reasonInfo.retryDelayMs ??
|
|
1860
|
+
unconfirmedRateLimitDelayMs(response.headers.get('retry-after'), receivedAtMs)));
|
|
1861
|
+
const routed429 = response.status === 429 ? routedIdentity : undefined;
|
|
1636
1862
|
if (response.status === 429 && rateLimitReason === 'UNKNOWN') {
|
|
1637
1863
|
await logResponseBody(debugContext, response, 429);
|
|
1638
1864
|
// Inspect the actual wire instruction, not conversation or tool text.
|
|
1639
1865
|
const outgoing = JSON.parse(String(prepared.init.body));
|
|
1640
1866
|
const recognised = systemInstructionHasHarnessIdentity(outgoing.request?.systemInstruction);
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1867
|
+
if (recognised) {
|
|
1868
|
+
void response.body?.cancel().catch(() => { });
|
|
1869
|
+
return unconfirmedRefusal();
|
|
1870
|
+
}
|
|
1871
|
+
if (routed429) {
|
|
1872
|
+
const refusal = await retryAfterRoutedBackoff(routed429, retryAtMs, response);
|
|
1873
|
+
if (refusal)
|
|
1874
|
+
return refusal;
|
|
1875
|
+
i -= 1;
|
|
1876
|
+
continue;
|
|
1877
|
+
}
|
|
1878
|
+
// An unreadable real refusal is still evidence of a short
|
|
1879
|
+
// backoff, even if the original ask fell back to local order.
|
|
1880
|
+
if (routingIdentity)
|
|
1881
|
+
reportVaultRouting({
|
|
1882
|
+
...routingIdentity,
|
|
1883
|
+
kind: 'transient_backoff',
|
|
1884
|
+
retryAtMs,
|
|
1885
|
+
});
|
|
1651
1886
|
void response.body?.cancel().catch(() => { });
|
|
1652
|
-
if (recognised)
|
|
1653
|
-
return terminal();
|
|
1654
1887
|
const delayMs = unconfirmedRateLimitDelayMs(response.headers.get('retry-after'));
|
|
1655
1888
|
accountManager.markRateLimitedWithReason(account, family, model, 'UNKNOWN', delayMs);
|
|
1656
1889
|
const now = Date.now();
|
|
@@ -1678,7 +1911,7 @@ export function createRequestExecutor(deps) {
|
|
|
1678
1911
|
break;
|
|
1679
1912
|
}
|
|
1680
1913
|
if (unconfirmedRetries >= MAX_UNCONFIRMED_RATE_LIMIT_RETRIES)
|
|
1681
|
-
return
|
|
1914
|
+
return unconfirmedRefusal();
|
|
1682
1915
|
unconfirmedRetries++;
|
|
1683
1916
|
// Selection can rotate immediately to a ready account. Otherwise
|
|
1684
1917
|
// wait only for this short cooldown, never the quota backoff ladder.
|
|
@@ -1711,12 +1944,13 @@ export function createRequestExecutor(deps) {
|
|
|
1711
1944
|
i -= 1;
|
|
1712
1945
|
continue;
|
|
1713
1946
|
}
|
|
1947
|
+
let exhaustion;
|
|
1714
1948
|
if (credentials.domain === 'vault' &&
|
|
1715
1949
|
sendAdmission &&
|
|
1716
1950
|
sessionKey &&
|
|
1717
1951
|
hasVaultRoutingAccessor()) {
|
|
1718
1952
|
const group = resolveQuotaGroup(family, model, account.cachedQuota);
|
|
1719
|
-
reportVaultExhaustion({
|
|
1953
|
+
exhaustion = reportVaultExhaustion({
|
|
1720
1954
|
identity: {
|
|
1721
1955
|
sessionId: sessionKey,
|
|
1722
1956
|
accountId: sendAdmission.accountIdentity,
|
|
@@ -1728,12 +1962,29 @@ export function createRequestExecutor(deps) {
|
|
|
1728
1962
|
quota: account.cachedQuota?.[group],
|
|
1729
1963
|
observedAt: account.cachedQuotaUpdatedAt,
|
|
1730
1964
|
signal: abortSignal,
|
|
1965
|
+
singleReport: !!routed429,
|
|
1731
1966
|
});
|
|
1732
1967
|
}
|
|
1968
|
+
if (routed429 &&
|
|
1969
|
+
rateLimitReason === 'QUOTA_EXHAUSTED' &&
|
|
1970
|
+
!exhaustion) {
|
|
1971
|
+
const refusal = await retryAfterRoutedBackoff(routed429, retryAtMs, response);
|
|
1972
|
+
if (refusal)
|
|
1973
|
+
return refusal;
|
|
1974
|
+
i -= 1;
|
|
1975
|
+
continue;
|
|
1976
|
+
}
|
|
1733
1977
|
if (rateLimitReason === 'MODEL_CAPACITY_EXHAUSTED' ||
|
|
1734
1978
|
rateLimitReason === 'SERVER_ERROR') {
|
|
1735
1979
|
totalCapacityRetries++;
|
|
1736
1980
|
if (isCapacityRetryBudgetExhausted(totalCapacityRetries)) {
|
|
1981
|
+
if (routed429) {
|
|
1982
|
+
const refusal = await retryAfterRoutedBackoff(routed429, retryAtMs, response);
|
|
1983
|
+
if (refusal)
|
|
1984
|
+
return refusal;
|
|
1985
|
+
i -= 1;
|
|
1986
|
+
continue;
|
|
1987
|
+
}
|
|
1737
1988
|
pushDebug(`Total capacity retries (${MAX_TOTAL_CAPACITY_RETRIES}) exhausted, switching account`);
|
|
1738
1989
|
retainHttpFailure(response);
|
|
1739
1990
|
shouldSwitchAccount = true;
|
|
@@ -1795,6 +2046,13 @@ export function createRequestExecutor(deps) {
|
|
|
1795
2046
|
}
|
|
1796
2047
|
if (config.switch_on_first_rate_limit && accountCount > 1) {
|
|
1797
2048
|
accountManager.markRateLimitedWithReason(account, family, model, rateLimitReason, serverRetryMs, config.failure_ttl_seconds * 1000);
|
|
2049
|
+
if (routed429) {
|
|
2050
|
+
const refusal = await retryAfterRoutedBackoff(routed429, retryAtMs, response);
|
|
2051
|
+
if (refusal)
|
|
2052
|
+
return refusal;
|
|
2053
|
+
i -= 1;
|
|
2054
|
+
continue;
|
|
2055
|
+
}
|
|
1798
2056
|
shouldSwitchAccount = true;
|
|
1799
2057
|
break;
|
|
1800
2058
|
}
|
|
@@ -1803,6 +2061,13 @@ export function createRequestExecutor(deps) {
|
|
|
1803
2061
|
}
|
|
1804
2062
|
accountManager.markRateLimitedWithReason(account, family, model, rateLimitReason, serverRetryMs, config.failure_ttl_seconds * 1000);
|
|
1805
2063
|
accountManager.requestSaveToDisk();
|
|
2064
|
+
if (routed429 && !exhaustion) {
|
|
2065
|
+
const refusal = await retryAfterRoutedBackoff(routed429, retryAtMs, response);
|
|
2066
|
+
if (refusal)
|
|
2067
|
+
return refusal;
|
|
2068
|
+
i -= 1;
|
|
2069
|
+
continue;
|
|
2070
|
+
}
|
|
1806
2071
|
const switchAccountDelayMs = config.switch_account_delay_ms ?? 500;
|
|
1807
2072
|
if (accountCount > 1) {
|
|
1808
2073
|
const quotaMsg = reasonInfo.quotaResetTime
|
|
@@ -1820,6 +2085,10 @@ export function createRequestExecutor(deps) {
|
|
|
1820
2085
|
await routingSleep(expBackoffMs, 'rate_limited', backoff.attempt);
|
|
1821
2086
|
}
|
|
1822
2087
|
retainHttpFailure(response);
|
|
2088
|
+
if (routed429 && exhaustion) {
|
|
2089
|
+
const { accountId, credentialId } = routed429;
|
|
2090
|
+
continueRouting = { accountId, credentialId, ...exhaustion };
|
|
2091
|
+
}
|
|
1823
2092
|
shouldSwitchAccount = true;
|
|
1824
2093
|
break;
|
|
1825
2094
|
}
|
|
@@ -1833,10 +2102,12 @@ export function createRequestExecutor(deps) {
|
|
|
1833
2102
|
// below.
|
|
1834
2103
|
if (response.status === 404 &&
|
|
1835
2104
|
currentEndpoint === ANTIGRAVITY_ENDPOINT_DAILY) {
|
|
1836
|
-
const notFoundBody =
|
|
1837
|
-
.
|
|
1838
|
-
.
|
|
1839
|
-
|
|
2105
|
+
const notFoundBody = boundedRefusal
|
|
2106
|
+
? (boundedRefusal.text ?? '')
|
|
2107
|
+
: await deadline.wait(() => response
|
|
2108
|
+
.clone()
|
|
2109
|
+
.text()
|
|
2110
|
+
.catch(() => ''));
|
|
1840
2111
|
const unavailable = parseModelUnavailableError(notFoundBody);
|
|
1841
2112
|
if (unavailable) {
|
|
1842
2113
|
if (tokenConsumed) {
|
|
@@ -1873,12 +2144,28 @@ export function createRequestExecutor(deps) {
|
|
|
1873
2144
|
retryState.resetRateLimitState(account.index, quotaKey);
|
|
1874
2145
|
retryState.resetAccountFailureState(account.index);
|
|
1875
2146
|
if (response.status === 403) {
|
|
1876
|
-
const errorBodyText =
|
|
1877
|
-
.
|
|
1878
|
-
.
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2147
|
+
const errorBodyText = boundedRefusal
|
|
2148
|
+
? boundedRefusal.text
|
|
2149
|
+
: await deadline.wait(() => response
|
|
2150
|
+
.clone()
|
|
2151
|
+
.text()
|
|
2152
|
+
.catch(() => ''));
|
|
2153
|
+
const extracted = errorBodyText === undefined
|
|
2154
|
+
? undefined
|
|
2155
|
+
: classifyAccessError(errorBodyText);
|
|
2156
|
+
if (routedIdentity &&
|
|
2157
|
+
(extracted?.accountIneligible || extracted?.validationRequired)) {
|
|
2158
|
+
// A signed-out observation lets routing move the session without
|
|
2159
|
+
// treating a temporary access refusal as an invalid credential.
|
|
2160
|
+
const { accountId, credentialId } = routedIdentity;
|
|
2161
|
+
continueRouting = {
|
|
2162
|
+
accountId,
|
|
2163
|
+
credentialId,
|
|
2164
|
+
limitWindow: 'signed_out',
|
|
2165
|
+
observedAtMs: receivedAtMs,
|
|
2166
|
+
};
|
|
2167
|
+
}
|
|
2168
|
+
if (extracted?.accountIneligible) {
|
|
1882
2169
|
const ineligibleReason = extracted.message ??
|
|
1883
2170
|
'Google marked this account as ineligible for Antigravity.';
|
|
1884
2171
|
accountManager.markAccountIneligible(account.index, ineligibleReason);
|
|
@@ -1894,7 +2181,7 @@ export function createRequestExecutor(deps) {
|
|
|
1894
2181
|
shouldSwitchAccount = true;
|
|
1895
2182
|
break;
|
|
1896
2183
|
}
|
|
1897
|
-
if (extracted
|
|
2184
|
+
if (extracted?.validationRequired) {
|
|
1898
2185
|
const verificationReason = extracted.message ?? 'Google requires account verification.';
|
|
1899
2186
|
const cooldownMs = 10 * 60 * 1000;
|
|
1900
2187
|
accountManager.markAccountVerificationRequired(account.index, verificationReason, extracted.verifyUrl);
|
|
@@ -1926,6 +2213,7 @@ export function createRequestExecutor(deps) {
|
|
|
1926
2213
|
accountManager.markAccountUsed(account.index);
|
|
1927
2214
|
const proactiveThreshold = config.proactive_rotation_threshold_percent ?? 20;
|
|
1928
2215
|
if (!advisoryClosing &&
|
|
2216
|
+
!routingPresent &&
|
|
1929
2217
|
proactiveThreshold > 0 &&
|
|
1930
2218
|
accountManager.shouldProactivelyRotate(family, model, proactiveThreshold, softQuotaCacheTtlMs, accountSessionIdentity)) {
|
|
1931
2219
|
const rotated = accountManager.proactivelyRotateForFamily(family, model, config.soft_quota_threshold_percent, softQuotaCacheTtlMs, accountSessionIdentity,
|
|
@@ -1951,7 +2239,9 @@ export function createRequestExecutor(deps) {
|
|
|
1951
2239
|
await logResponseBody(debugContext, response, response.status);
|
|
1952
2240
|
if (response.status === 400) {
|
|
1953
2241
|
const cloned = response.clone();
|
|
1954
|
-
const bodyText = await deadline.wait(() => cloned.text())
|
|
2242
|
+
const bodyText = await deadline.wait(() => cloned.text().catch((error) => {
|
|
2243
|
+
throw markUpstreamFailure(error);
|
|
2244
|
+
}));
|
|
1955
2245
|
if (bodyText.includes('Prompt is too long') ||
|
|
1956
2246
|
bodyText.includes('prompt_too_long')) {
|
|
1957
2247
|
await showToast('Context too long - use /compact to reduce size', 'warning');
|
|
@@ -1966,7 +2256,9 @@ export function createRequestExecutor(deps) {
|
|
|
1966
2256
|
const maxAttempts = config.empty_response_max_attempts ?? 4;
|
|
1967
2257
|
const retryDelayMs = config.empty_response_retry_delay_ms ?? 2000;
|
|
1968
2258
|
const clonedForCheck = response.clone();
|
|
1969
|
-
const bodyText = await deadline.wait(() => clonedForCheck.text())
|
|
2259
|
+
const bodyText = await deadline.wait(() => clonedForCheck.text().catch((error) => {
|
|
2260
|
+
throw markUpstreamFailure(error);
|
|
2261
|
+
}));
|
|
1970
2262
|
if (isEmptyResponseBody(bodyText)) {
|
|
1971
2263
|
const emptyAttemptKey = `${prepared.sessionId ?? 'none'}:${prepared.effectiveModel ?? 'unknown'}`;
|
|
1972
2264
|
const currentAttempts = retryState.recordEmptyResponseAttempt(emptyAttemptKey);
|
|
@@ -2190,7 +2482,7 @@ export function createRequestExecutor(deps) {
|
|
|
2190
2482
|
function dispose() {
|
|
2191
2483
|
disposePromise ??= (async () => {
|
|
2192
2484
|
retired = true;
|
|
2193
|
-
shutdown.abort(new
|
|
2485
|
+
shutdown.abort(new HostRefusalError('The Antigravity request executor is disposed'));
|
|
2194
2486
|
await drain();
|
|
2195
2487
|
})();
|
|
2196
2488
|
return disposePromise;
|
|
@@ -2198,9 +2490,9 @@ export function createRequestExecutor(deps) {
|
|
|
2198
2490
|
return {
|
|
2199
2491
|
execute(input, init, requestOptions) {
|
|
2200
2492
|
if (shutdown.signal.aborted)
|
|
2201
|
-
return Promise.reject(new
|
|
2493
|
+
return Promise.reject(new HostRefusalError('The Antigravity request executor is disposed'));
|
|
2202
2494
|
if (retired)
|
|
2203
|
-
return Promise.reject(new
|
|
2495
|
+
return Promise.reject(new HostRefusalError('The Antigravity request executor is retired'));
|
|
2204
2496
|
const requestedSignal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
|
|
2205
2497
|
const cancellation = createRequestCancellation(requestedSignal
|
|
2206
2498
|
? [requestedSignal, shutdown.signal]
|
|
@@ -2213,6 +2505,12 @@ export function createRequestExecutor(deps) {
|
|
|
2213
2505
|
const waitPolicy = {};
|
|
2214
2506
|
const selectedCredentials = new Set();
|
|
2215
2507
|
const selectedProjectIds = new Set();
|
|
2508
|
+
const redactFailure = (error) => {
|
|
2509
|
+
const redacted = redactCredentialError(error, [...selectedCredentials], [...selectedProjectIds]);
|
|
2510
|
+
return isUpstreamFailure(error)
|
|
2511
|
+
? markUpstreamFailure(redacted)
|
|
2512
|
+
: redacted;
|
|
2513
|
+
};
|
|
2216
2514
|
const pending = withRoutingWaitReports(cancellation.signal, (waits) => deadline.wait(() => execute(input, { ...init, signal: deadline.signal }, requestOptions ?? {}, selectedCredentials, selectedProjectIds, deadline, waitPolicy, waits)))
|
|
2217
2515
|
.catch((error) => {
|
|
2218
2516
|
cancellation.signal.throwIfAborted();
|
|
@@ -2253,13 +2551,13 @@ export function createRequestExecutor(deps) {
|
|
|
2253
2551
|
handoff.cancel();
|
|
2254
2552
|
deadline.dispose();
|
|
2255
2553
|
cancellation.release();
|
|
2256
|
-
throw
|
|
2554
|
+
throw redactFailure(error);
|
|
2257
2555
|
})
|
|
2258
2556
|
.catch((error) => {
|
|
2259
2557
|
handoff.cancel();
|
|
2260
2558
|
deadline.dispose();
|
|
2261
2559
|
cancellation.release();
|
|
2262
|
-
throw
|
|
2560
|
+
throw redactFailure(error);
|
|
2263
2561
|
});
|
|
2264
2562
|
inFlight.add(pending);
|
|
2265
2563
|
void pending.then(() => inFlight.delete(pending), () => inFlight.delete(pending));
|