@bitkyc08/opencodex 2.80.0 → 2.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/gui/dist/assets/App-7yhoys8a.js +54 -0
- package/gui/dist/assets/Tray-EwAOsP0S.js +1 -0
- package/gui/dist/assets/index-RsoaAxXY.js +94 -0
- package/gui/dist/assets/quota-summary-D2lN8tDm.js +1 -0
- package/gui/dist/index.html +1 -1
- package/gui/dist/provider-icons/claude-green.svg +1 -0
- package/package.json +3 -2
- package/src/adapters/anthropic-model-contract.ts +8 -2
- package/src/adapters/anthropic.ts +7 -2
- package/src/adapters/codebuddy/mcp-server.ts +2 -25
- package/src/adapters/codebuddy/tool-bridge.ts +1 -7
- package/src/adapters/coding-agent/mcp-server.ts +33 -0
- package/src/adapters/coding-agent/protocol.ts +122 -17
- package/src/adapters/coding-agent/tool-catalog.ts +75 -0
- package/src/adapters/coding-agent/turn.ts +101 -44
- package/src/adapters/cursor/catalog.ts +8 -0
- package/src/adapters/cursor/effort-map.ts +2 -0
- package/src/adapters/devin/live-models.ts +2 -0
- package/src/adapters/kiro-retry.ts +9 -4
- package/src/adapters/openai-chat/messages.ts +7 -6
- package/src/adapters/physical-send.ts +8 -5
- package/src/adapters/qoder/adapter.ts +41 -6
- package/src/adapters/qoder/scaffold-guard.ts +2 -3
- package/src/adapters/zed.ts +16 -1
- package/src/claude/inbound-cache-stabilize.ts +4 -9
- package/src/claude/inbound.ts +1 -0
- package/src/claude/intercept/model-bindings.ts +1 -0
- package/src/claude/intercept/picker-listener.ts +71 -4
- package/src/cli/account-anthropic-threshold.ts +14 -5
- package/src/cli/account-api.ts +3 -1
- package/src/cli/account-extended.ts +13 -9
- package/src/cli/account-policy-dto.ts +1 -1
- package/src/cli/account-policy.ts +13 -4
- package/src/cli/account.ts +3 -3
- package/src/cli/capabilities-accounts.ts +2 -2
- package/src/cli/capabilities-agents-routing.ts +24 -0
- package/src/cli/capabilities-integrations.ts +16 -0
- package/src/cli/codex-shim-autorestore.ts +3 -0
- package/src/cli/dispatch.ts +10 -1
- package/src/cli/help.ts +4 -1
- package/src/cli/index.ts +11 -5
- package/src/cli/integration-plan-dto.ts +1 -0
- package/src/cli/integration-preview.ts +8 -1
- package/src/cli/integrations.ts +32 -3
- package/src/cli/message-args.ts +34 -0
- package/src/cli/message-command.ts +43 -0
- package/src/cli/message-runtime.ts +6 -0
- package/src/cli/observe.ts +3 -1
- package/src/cli/provider.ts +29 -6
- package/src/cli/registry.ts +24 -2
- package/src/clients/config-export/commandcode.ts +131 -0
- package/src/clients/config-export/contracts.ts +12 -0
- package/src/clients/config-export.ts +64 -1
- package/src/codex/auth-api/account-list.ts +11 -3
- package/src/codex/auth-api/main-account-probe.ts +25 -8
- package/src/codex/auth-api/pool-mode-gate.ts +33 -9
- package/src/codex/auth-api/pool-quota-probe.ts +15 -1
- package/src/codex/auth-api/routes.ts +1 -0
- package/src/codex/catalog/auto-review.ts +13 -6
- package/src/codex/catalog/build-entries.ts +15 -7
- package/src/codex/catalog/control-plane.ts +45 -0
- package/src/codex/catalog/effort.ts +11 -1
- package/src/codex/catalog/gather-capture.ts +5 -3
- package/src/codex/catalog/parsing.ts +2 -1
- package/src/codex/catalog/provider-models.ts +15 -9
- package/src/codex/control-plane-models.ts +6 -0
- package/src/combos/jev-decision-contract.ts +25 -2
- package/src/combos/jev.ts +3 -2
- package/src/combos/types.ts +1 -1
- package/src/config/diagnostics.ts +27 -16
- package/src/config/load-degrade.ts +34 -0
- package/src/config/schema/anthropic-account-pool.ts +143 -0
- package/src/config/schema/config-schema.ts +7 -1
- package/src/config/schema/leaf-validators.ts +4 -0
- package/src/config.ts +2 -0
- package/src/generated/compatibility-version.json +320 -184
- package/src/generated/model-metadata.ts +5 -4
- package/src/images/loop.ts +2 -1
- package/src/integrations/catalog-refresh.ts +1 -1
- package/src/integrations/mutation-plan.ts +7 -0
- package/src/integrations/registry.ts +8 -0
- package/src/integrations/state.ts +9 -2
- package/src/integrations/writer.ts +8 -2
- package/src/lib/lab-live-route-production.ts +9 -1
- package/src/lib/request-execution-budget.ts +236 -58
- package/src/lib/spend-pool-alias-validation.ts +79 -0
- package/src/lib/spend-pool-continuity.ts +34 -0
- package/src/lib/spend-reservation-ledger.ts +412 -50
- package/src/lib/state-store-registrations.ts +20 -3
- package/src/lib/upstream-retry.ts +17 -5
- package/src/lib/workflow-budget.ts +53 -11
- package/src/messaging/budget.ts +45 -0
- package/src/messaging/discovery.ts +57 -0
- package/src/messaging/envelope.ts +49 -0
- package/src/messaging/input.ts +37 -0
- package/src/messaging/rpc.ts +196 -0
- package/src/messaging/send.ts +58 -0
- package/src/messaging/socket.ts +72 -0
- package/src/messaging/types.ts +29 -0
- package/src/oauth/anthropic-account-refusal.ts +51 -17
- package/src/oauth/anthropic-account-threshold.ts +8 -2
- package/src/oauth/anthropic-continuity.ts +15 -1
- package/src/oauth/anthropic-model-quota.ts +120 -48
- package/src/oauth/anthropic-model-routes.ts +11 -3
- package/src/oauth/anthropic-oauth-definitions.ts +98 -0
- package/src/oauth/anthropic-pool-config.ts +30 -0
- package/src/oauth/anthropic-rate-limit-policy.ts +64 -9
- package/src/oauth/anthropic-routing.ts +1202 -866
- package/src/oauth/anthropic-send-ownership.ts +41 -0
- package/src/oauth/anthropic.ts +4 -1
- package/src/oauth/generic-account-failover.ts +3 -1
- package/src/oauth/health.ts +8 -6
- package/src/oauth/index.ts +90 -173
- package/src/oauth/kiro.ts +22 -9
- package/src/oauth/login-cli.ts +2 -0
- package/src/oauth/model-discovery-auth.ts +43 -0
- package/src/oauth/pool-kernel.ts +11 -5
- package/src/oauth/pool-settings-capability.ts +5 -3
- package/src/oauth/store-anthropic-instance.ts +87 -0
- package/src/oauth/store.ts +67 -9
- package/src/oauth/token-guardian.ts +2 -0
- package/src/oauth/types.ts +2 -0
- package/src/protocols/plan-snapshot.ts +5 -0
- package/src/protocols/settings.ts +20 -5
- package/src/providers/anthropic-instance-id.ts +47 -0
- package/src/providers/anthropic-instance.ts +48 -0
- package/src/providers/anthropic-reset-grant-ledger.ts +11 -1
- package/src/providers/derive.ts +2 -0
- package/src/providers/kiro-models.ts +3 -0
- package/src/providers/label.ts +1 -1
- package/src/providers/model-rename-fields.ts +2 -0
- package/src/providers/quota/account-cache.ts +37 -11
- package/src/providers/quota/anthropic-account-quota.ts +93 -0
- package/src/providers/quota/anthropic-cooldown-recovery.ts +222 -111
- package/src/providers/quota/report-cache.ts +24 -1
- package/src/providers/quota/vendor-probes-oauth.ts +46 -14
- package/src/providers/quota.ts +22 -32
- package/src/providers/registry/entries-core.ts +35 -30
- package/src/providers/registry/model-ids.ts +1 -0
- package/src/providers/registry/model-seeds.ts +7 -4
- package/src/providers/registry/types.ts +3 -0
- package/src/providers/registry.ts +3 -3
- package/src/responses/spill-store.ts +5 -1
- package/src/responses/state/metrics.ts +87 -0
- package/src/responses/state/snapshot-codec.ts +6 -0
- package/src/responses/state/spill-queue.ts +1 -0
- package/src/responses/state/unforced-store-false.ts +55 -0
- package/src/responses/state.ts +50 -66
- package/src/router.ts +28 -4
- package/src/routing/compatibility/assemble.ts +1 -1
- package/src/routing/quota.ts +7 -3
- package/src/server/auth-cors.ts +5 -0
- package/src/server/chat-completions.ts +6 -1
- package/src/server/chat-native.ts +45 -25
- package/src/server/claude-messages.ts +16 -2
- package/src/server/gui-static.ts +38 -4
- package/src/server/index/spend-ledger-lifecycle.ts +10 -4
- package/src/server/index.ts +4 -2
- package/src/server/inference/context.ts +1 -1
- package/src/server/lifecycle.ts +1 -1
- package/src/server/management/agent-settings-routes.ts +30 -6
- package/src/server/management/anthropic-account-threshold.ts +12 -9
- package/src/server/management/anthropic-pool-settings.ts +72 -0
- package/src/server/management/anthropic-reset-grant-routes.ts +98 -39
- package/src/server/management/config-routes.ts +27 -7
- package/src/server/management/decision-routes.ts +2 -1
- package/src/server/management/integration-routes.ts +25 -2
- package/src/server/management/oauth-account-routes.ts +79 -98
- package/src/server/management/provider-instance-ownership.ts +44 -0
- package/src/server/management/provider-routes.ts +21 -28
- package/src/server/management/route-registry.ts +3 -1
- package/src/server/management/routing-profile-routes.ts +4 -4
- package/src/server/management/system-restart.ts +44 -14
- package/src/server/management/system-routes.ts +24 -2
- package/src/server/management/vision-sidecar-options.ts +9 -6
- package/src/server/management/web-search-sidecar-options.ts +51 -2
- package/src/server/management-auth.ts +6 -0
- package/src/server/messages-native-eligibility.ts +3 -2
- package/src/server/messages-native-oauth.ts +86 -41
- package/src/server/messages-native-selector.ts +39 -0
- package/src/server/messages-native.ts +84 -25
- package/src/server/request-log.ts +5 -0
- package/src/server/responses/adapter-continuation.ts +29 -17
- package/src/server/responses/adapter-dispatch.ts +47 -17
- package/src/server/responses/antigravity-validation-refusal.ts +20 -6
- package/src/server/responses/compact.ts +25 -8
- package/src/server/responses/compaction-recovery.ts +1 -1
- package/src/server/responses/core-combo-native.ts +4 -1
- package/src/server/responses/core-combo.ts +16 -11
- package/src/server/responses/core-normalize.ts +1 -0
- package/src/server/responses/core-options.ts +3 -1
- package/src/server/responses/core-replay.ts +5 -0
- package/src/server/responses/passthrough-delivery.ts +7 -3
- package/src/server/responses/passthrough-dispatch.ts +21 -8
- package/src/server/responses/request-prepare.ts +37 -5
- package/src/server/responses/request-send-budget.ts +74 -37
- package/src/server/responses/request-spend.ts +325 -33
- package/src/server/responses/request-transport.ts +132 -52
- package/src/server/responses/run-turn-execution.ts +11 -3
- package/src/server/responses/sidecar-execution.ts +96 -14
- package/src/server/responses-undeclared-tool-guard.ts +5 -1
- package/src/server/workflow-refusal.ts +30 -0
- package/src/service/ownership-mutation-lease.d.mts +1 -1
- package/src/service/ownership-mutation-lease.mjs +5 -7
- package/src/sidecar/anthropic-binding.ts +79 -0
- package/src/sidecar/auth.ts +76 -2
- package/src/types/anthropic-account-pool.ts +33 -0
- package/src/types/config.ts +12 -28
- package/src/types/provider.ts +5 -0
- package/src/types/tools.ts +7 -0
- package/src/types.ts +4 -0
- package/src/usage/cost.ts +3 -2
- package/src/usage/expected-prices.ts +66 -14
- package/src/vision/anthropic-describe.ts +16 -3
- package/src/vision/index.ts +3 -0
- package/src/vision/plan.ts +19 -6
- package/src/web-search/alpha-search.ts +7 -1
- package/src/web-search/anthropic-executor.ts +16 -4
- package/src/web-search/index.ts +7 -3
- package/src/web-search/loop.ts +27 -5
- package/src/web-search/passthrough-bridge.ts +4 -1
- package/src/web-search/sidecar-providers.ts +4 -3
- package/gui/dist/assets/App-MbDVfUeC.js +0 -54
- package/gui/dist/assets/Tray-CzkxK82A.js +0 -1
- package/gui/dist/assets/index-UJt6npXk.js +0 -94
- package/gui/dist/assets/quota-summary-DU3sD6Z-.js +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { clearAllAnthropicFamilyQuota, anthropicModelQuotaFor, anthropicModelPercents, anthropicModelWeeklyPercent } from "./anthropic-model-quota";
|
|
2
2
|
/**
|
|
3
3
|
* Opt-in Anthropic OAuth account pool (#294).
|
|
4
4
|
*
|
|
@@ -17,7 +17,7 @@ import { anthropicFamilyRejected, anthropicFamilyRetryAt, anthropicModelExhauste
|
|
|
17
17
|
* backoff. Classified entitlement/billing 403s use a ten-minute default cooldown.
|
|
18
18
|
* Token refresh failures retain the existing store needsReauth policy.
|
|
19
19
|
*/
|
|
20
|
-
import {
|
|
20
|
+
import { clearAllAnthropicRatePauses, anthropicRatePolicyFor, classifyAnthropic429, anthropicRetryAfterMs } from "./anthropic-rate-limit-policy";
|
|
21
21
|
import { REFRESH_SKEW_MS } from "./refresh-policy";
|
|
22
22
|
import { createHash } from "node:crypto";
|
|
23
23
|
import { captureOAuthAccountSelection, commitOAuthAccountSelection, credentialGeneration, getAccountSet, getAccountCredential, getAccountCredentialWithStatus } from "./store";
|
|
@@ -31,21 +31,22 @@ import {
|
|
|
31
31
|
notePoolRotationSuccess,
|
|
32
32
|
pickRoundRobinAccount,
|
|
33
33
|
peekRoundRobinAccount,
|
|
34
|
-
POOL_KEY_ANTHROPIC,
|
|
35
34
|
seedPoolRotationAccount,
|
|
36
35
|
} from "../codex/pool-rotation";
|
|
37
36
|
import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConfig } from "../types";
|
|
38
37
|
import { sweepExpiredOnWrite } from "../lib/state-store-sweeper";
|
|
39
38
|
import { retainedUtf8Bytes } from "../lib/admission";
|
|
40
|
-
import {
|
|
39
|
+
import { resolveAnthropicModelRouteForInstance, routeCandidates, type AnthropicRouteDecision } from "./anthropic-model-routes";
|
|
41
40
|
import type { ProviderQuota } from "../providers/quota-types";
|
|
42
|
-
import {
|
|
43
|
-
anthropicCooldownGeneration,
|
|
44
|
-
clearAnthropicCooldownGenerations,
|
|
45
|
-
noteAnthropicCooldownMutation,
|
|
46
|
-
} from "../providers/quota/anthropic-cooldown-recovery";
|
|
41
|
+
import { clearAllAnthropicCooldownGenerations, anthropicCooldownRecoveryFor } from "../providers/quota/anthropic-cooldown-recovery";
|
|
47
42
|
import { subscribeAccountSelections, subscribeOAuthAccountPauseChanges, subscribeOAuthAccountRoutingPolicyChanges } from "../lib/account-selection-events";
|
|
48
|
-
import {
|
|
43
|
+
import { effectiveAnthropicAccountThresholdForInstance } from "./anthropic-account-threshold";
|
|
44
|
+
|
|
45
|
+
import type { AnthropicAccountPoolConfig } from "../types/anthropic-account-pool";
|
|
46
|
+
import { configuredAnthropicInstance, isAnthropicInstanceId, type AnthropicInstanceId } from "../providers/anthropic-instance";
|
|
47
|
+
import { resolveAnthropicAccountPoolConfig } from "./anthropic-pool-config";
|
|
48
|
+
import { anthropicPoolKey } from "./pool-kernel";
|
|
49
|
+
import type { GenerationContext } from "../lib/state-store-sweeper";
|
|
49
50
|
|
|
50
51
|
/**
|
|
51
52
|
* The read side of a `Headers` object, so a caller can pass the live upstream response's
|
|
@@ -54,7 +55,6 @@ import { effectiveAnthropicAccountThreshold } from "./anthropic-account-threshol
|
|
|
54
55
|
*/
|
|
55
56
|
export type AnthropicRateLimitHeaders = Pick<Headers, "get">;
|
|
56
57
|
|
|
57
|
-
const PROVIDER = "anthropic";
|
|
58
58
|
/** Backoff only when upstream supplies no usable deadline. */
|
|
59
59
|
const DEFAULT_COOLDOWN_MS = 60_000;
|
|
60
60
|
const ACCOUNT_REFUSAL_COOLDOWN_MS = 10 * 60_000;
|
|
@@ -68,7 +68,7 @@ const VALID_QUOTA_WINDOWS = new Set<OcxAccountPoolQuotaWindow>(["five-hour", "we
|
|
|
68
68
|
/** Cap same-request 429 rotations so short Retry-After cannot infinite-loop. */
|
|
69
69
|
export const ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST = 3;
|
|
70
70
|
|
|
71
|
-
export type AnthropicAccountPoolConfig
|
|
71
|
+
export type { AnthropicAccountPoolConfig } from "../types/anthropic-account-pool";
|
|
72
72
|
|
|
73
73
|
/** Local admission refusal across async dispatch; never treat it as provider reachability evidence. */
|
|
74
74
|
export class AnthropicAccountCooldownError extends Error {
|
|
@@ -105,59 +105,11 @@ interface AffinityEntry {
|
|
|
105
105
|
lastUsedAt: number;
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
-
const upstreamHealth = new Map<string, AccountHealth>();
|
|
109
|
-
const sessionAffinity = new Map<string, AffinityEntry>();
|
|
110
|
-
type OAuthAccountSelection = NonNullable<ReturnType<typeof captureOAuthAccountSelection>>;
|
|
111
|
-
// Undefined means this runtime has not admitted a selection yet; null means consumed.
|
|
112
|
-
// The startup baseline comes from the authoritative store, never a second persisted pin.
|
|
113
|
-
let manualPreference: OAuthAccountSelection | null | undefined;
|
|
114
|
-
let manualSelectionGeneration = 0;
|
|
115
|
-
/** Automatic pointer moves preserve affinity; manual selection revokes older bindings. */
|
|
116
|
-
export function captureAnthropicManualSelectionGeneration(): number { return manualSelectionGeneration; }
|
|
117
|
-
|
|
118
108
|
function normalizeAffinityComponent(value: string | null | undefined): string {
|
|
119
109
|
const normalized = value?.trim() ?? "";
|
|
120
110
|
return normalized && retainedUtf8Bytes(normalized) <= MAX_AFFINITY_COMPONENT_BYTES ? normalized : "";
|
|
121
111
|
}
|
|
122
112
|
|
|
123
|
-
export function anthropicAccountPoolConfig(config: OcxConfig): AnthropicAccountPoolConfig {
|
|
124
|
-
const raw = config.anthropicAccountPool;
|
|
125
|
-
if (!raw || typeof raw !== "object") return {};
|
|
126
|
-
return raw;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export function isAnthropicAccountPoolEnabled(config: OcxConfig): boolean {
|
|
130
|
-
return anthropicAccountPoolConfig(config).enabled === true;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
export function anthropicAutoSwitchThreshold(config: OcxConfig): number {
|
|
134
|
-
const value = anthropicAccountPoolConfig(config).autoSwitchThreshold;
|
|
135
|
-
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100) return value;
|
|
136
|
-
return DEFAULT_AUTO_SWITCH_THRESHOLD;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/** Read live policy at selection, not a credential snapshot captured before an await. */
|
|
140
|
-
export function anthropicAccountAutoSwitchThreshold(config: OcxConfig, accountId: string): number {
|
|
141
|
-
return effectiveAnthropicAccountThreshold(config, getAccountSet(PROVIDER)?.accounts.find(row => row.id === accountId));
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/** Strict parse for management APIs — returns null instead of defaulting. */
|
|
145
|
-
export function parseAccountPoolQuotaWindow(raw: unknown): OcxAccountPoolQuotaWindow | null {
|
|
146
|
-
if (typeof raw === "string" && VALID_QUOTA_WINDOWS.has(raw as OcxAccountPoolQuotaWindow)) {
|
|
147
|
-
return raw as OcxAccountPoolQuotaWindow;
|
|
148
|
-
}
|
|
149
|
-
return null;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
export function normalizeAccountPoolQuotaWindow(raw: unknown): OcxAccountPoolQuotaWindow {
|
|
153
|
-
return parseAccountPoolQuotaWindow(raw) ?? DEFAULT_QUOTA_WINDOW;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
export function anthropicQuotaWindow(config: AnthropicAccountPoolConfig): OcxAccountPoolQuotaWindow {
|
|
157
|
-
return normalizeAccountPoolQuotaWindow(config.quotaWindow);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/** Accept upstream deadlines within the runtime's date range, without a policy ceiling. */
|
|
161
113
|
function delayUntil(timestamp: number, now: number): number | undefined {
|
|
162
114
|
const delay = timestamp - now;
|
|
163
115
|
return Number.isFinite(new Date(timestamp).getTime()) && Number.isFinite(delay) && delay > 0
|
|
@@ -175,7 +127,6 @@ function parseRetryAfterMs(value: string | null | undefined, now: number): numbe
|
|
|
175
127
|
return delayUntil(Date.parse(text), now);
|
|
176
128
|
}
|
|
177
129
|
|
|
178
|
-
/** Only rejected windows constrain recovery; all must reopen, so take the latest reset. */
|
|
179
130
|
function parseRateLimitReset(
|
|
180
131
|
headers: AnthropicRateLimitHeaders | null | undefined,
|
|
181
132
|
now: number,
|
|
@@ -201,297 +152,6 @@ function parseRateLimitReset(
|
|
|
201
152
|
return { delayMs: latest - now, rejectedQuotaWindows };
|
|
202
153
|
}
|
|
203
154
|
|
|
204
|
-
export type AnthropicCooldownRecoveryClaim = Readonly<{
|
|
205
|
-
accountId: string;
|
|
206
|
-
cooldownGeneration: number;
|
|
207
|
-
claimedAt: number;
|
|
208
|
-
}>;
|
|
209
|
-
|
|
210
|
-
export type AnthropicCooldownRecoverySettlement = "cleared" | "retained" | "superseded";
|
|
211
|
-
|
|
212
|
-
/**
|
|
213
|
-
* Capture the exact reset-derived refusal a fresh usage probe is allowed to recover.
|
|
214
|
-
*
|
|
215
|
-
* The generation fence matters because the usage request is asynchronous: a newer 429 may
|
|
216
|
-
* arrive while it is in flight, and an older answer must never erase that newer refusal.
|
|
217
|
-
*/
|
|
218
|
-
export function captureAnthropicCooldownRecovery(
|
|
219
|
-
accountId: string,
|
|
220
|
-
now = Date.now(),
|
|
221
|
-
): AnthropicCooldownRecoveryClaim | null {
|
|
222
|
-
const entry = upstreamHealth.get(accountId);
|
|
223
|
-
if (!entry || entry.cooldownUntil <= now || entry.cooldownSource !== "reset-derived") return null;
|
|
224
|
-
return { accountId, cooldownGeneration: entry.cooldownGeneration, claimedAt: now };
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
/**
|
|
228
|
-
* Clear only the claimed reset-derived cooldown when a fresh, complete usage result proves
|
|
229
|
-
* headroom in every window that upstream previously reported as rejected.
|
|
230
|
-
*/
|
|
231
|
-
export function settleAnthropicCooldownRecovery(
|
|
232
|
-
claim: AnthropicCooldownRecoveryClaim,
|
|
233
|
-
quota: ProviderQuota,
|
|
234
|
-
): AnthropicCooldownRecoverySettlement {
|
|
235
|
-
const entry = upstreamHealth.get(claim.accountId);
|
|
236
|
-
if (!entry || entry.cooldownSource !== "reset-derived"
|
|
237
|
-
|| entry.cooldownGeneration !== claim.cooldownGeneration
|
|
238
|
-
|| anthropicCooldownGeneration(claim.accountId) !== claim.cooldownGeneration
|
|
239
|
-
|| !Number.isFinite(quota.updatedAt) || quota.updatedAt < claim.claimedAt) return "superseded";
|
|
240
|
-
const windows = entry.rejectedQuotaWindows;
|
|
241
|
-
if (!windows?.length) return "retained";
|
|
242
|
-
const recovered = windows.every(window => {
|
|
243
|
-
const percent = window === "five-hour" ? quota.fiveHourPercent : quota.weeklyPercent;
|
|
244
|
-
return typeof percent === "number" && Number.isFinite(percent) && percent >= 0 && percent < 100;
|
|
245
|
-
});
|
|
246
|
-
if (!recovered) return "retained";
|
|
247
|
-
upstreamHealth.delete(claim.accountId);
|
|
248
|
-
noteAnthropicCooldownMutation(claim.accountId);
|
|
249
|
-
return "cleared";
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
export function getAnthropicAccountHealthSnapshot(
|
|
253
|
-
accountId: string,
|
|
254
|
-
now = Date.now(),
|
|
255
|
-
): { cooldownUntil?: number; cooldownSource?: AccountHealth["cooldownSource"] } | null {
|
|
256
|
-
const entry = upstreamHealth.get(accountId);
|
|
257
|
-
if (!entry) return null;
|
|
258
|
-
if (entry.cooldownUntil <= now) {
|
|
259
|
-
upstreamHealth.delete(accountId);
|
|
260
|
-
noteAnthropicCooldownMutation(accountId);
|
|
261
|
-
return null;
|
|
262
|
-
}
|
|
263
|
-
return { cooldownUntil: entry.cooldownUntil, cooldownSource: entry.cooldownSource };
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
export function clearAnthropicAccountCooldown(accountId: string): boolean {
|
|
267
|
-
const cleared = upstreamHealth.delete(accountId);
|
|
268
|
-
if (cleared) noteAnthropicCooldownMutation(accountId);
|
|
269
|
-
return cleared;
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number {
|
|
273
|
-
let removed = 0;
|
|
274
|
-
for (const [accountId, health] of upstreamHealth) {
|
|
275
|
-
if (health.cooldownUntil > now) continue;
|
|
276
|
-
upstreamHealth.delete(accountId);
|
|
277
|
-
noteAnthropicCooldownMutation(accountId);
|
|
278
|
-
removed += 1;
|
|
279
|
-
}
|
|
280
|
-
return removed;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
/** Test / logout helper. */
|
|
284
|
-
export function clearAnthropicAccountPoolState(): void {
|
|
285
|
-
upstreamHealth.clear();
|
|
286
|
-
clearAnthropicRatePauses();
|
|
287
|
-
clearAnthropicFamilyQuota();
|
|
288
|
-
clearAnthropicCooldownGenerations();
|
|
289
|
-
sessionAffinity.clear();
|
|
290
|
-
manualPreference = undefined;
|
|
291
|
-
manualSelectionGeneration++;
|
|
292
|
-
quorumCache = null;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
export function anthropicSessionAffinitySizeForTests(): number {
|
|
296
|
-
return sessionAffinity.size;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
function isCooled(accountId: string, now: number): boolean {
|
|
300
|
-
return getAnthropicAccountHealthSnapshot(accountId, now) !== null;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
function fiveHourKnown(accountId: string): boolean {
|
|
304
|
-
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.fiveHourPercent;
|
|
305
|
-
return typeof percent === "number" && Number.isFinite(percent);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
function weeklyKnown(accountId: string): boolean {
|
|
309
|
-
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.weeklyPercent;
|
|
310
|
-
return typeof percent === "number" && Number.isFinite(percent);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
function fiveHourScore(accountId: string): number {
|
|
314
|
-
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.fiveHourPercent;
|
|
315
|
-
return typeof percent === "number" && Number.isFinite(percent)
|
|
316
|
-
? Math.max(0, Math.min(100, percent))
|
|
317
|
-
: UNKNOWN_USAGE_SCORE;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
function weeklyScore(accountId: string): number {
|
|
321
|
-
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.weeklyPercent;
|
|
322
|
-
return typeof percent === "number" && Number.isFinite(percent)
|
|
323
|
-
? Math.max(0, Math.min(100, percent))
|
|
324
|
-
: UNKNOWN_USAGE_SCORE;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
function exhausted5h(accountId: string): boolean {
|
|
328
|
-
return fiveHourKnown(accountId) && fiveHourScore(accountId) >= 100;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function hasKnownUsage(config: OcxConfig, accountId: string, model?: string): boolean {
|
|
332
|
-
if (model) return hasKnownUsage(config, accountId) || anthropicModelWeeklyPercent(getCachedProviderAccountQuota(PROVIDER, accountId), model) !== undefined;
|
|
333
|
-
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
334
|
-
switch (window) {
|
|
335
|
-
case "five-hour": return fiveHourKnown(accountId);
|
|
336
|
-
case "weekly": return weeklyKnown(accountId);
|
|
337
|
-
case "max-utilization": return fiveHourKnown(accountId) || weeklyKnown(accountId);
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
function usageScore(config: OcxConfig, accountId: string, model?: string): number {
|
|
342
|
-
if (model) {
|
|
343
|
-
const family = anthropicModelWeeklyPercent(getCachedProviderAccountQuota(PROVIDER, accountId), model);
|
|
344
|
-
if (family === undefined) return usageScore(config, accountId);
|
|
345
|
-
return hasKnownUsage(config, accountId) ? Math.max(family, usageScore(config, accountId)) : family;
|
|
346
|
-
}
|
|
347
|
-
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
348
|
-
switch (window) {
|
|
349
|
-
case "five-hour": return fiveHourScore(accountId);
|
|
350
|
-
case "weekly": return weeklyScore(accountId);
|
|
351
|
-
case "max-utilization": {
|
|
352
|
-
const scores = [
|
|
353
|
-
...(fiveHourKnown(accountId) ? [fiveHourScore(accountId)] : []),
|
|
354
|
-
...(weeklyKnown(accountId) ? [weeklyScore(accountId)] : []),
|
|
355
|
-
];
|
|
356
|
-
return scores.length > 0 ? Math.max(...scores) : UNKNOWN_USAGE_SCORE;
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
/** Background `local-cli` slots with expired access are not pool-eligible (identity adoption risk). */
|
|
363
|
-
function isPoolCredentialUsable(accountId: string, now: number): boolean {
|
|
364
|
-
const cred = getAccountCredential(PROVIDER, accountId);
|
|
365
|
-
if (!cred) return false;
|
|
366
|
-
if (cred.source !== "local-cli") return true;
|
|
367
|
-
if (canRefreshAnthropicPoolAccount(accountId)) return true;
|
|
368
|
-
return cred.expires > now + REFRESH_SKEW_MS;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
export function getEligibleAnthropicAccounts(now = Date.now(), model?: string): string[] {
|
|
372
|
-
const set = getAccountSet(PROVIDER);
|
|
373
|
-
if (!set) return [];
|
|
374
|
-
return set.accounts
|
|
375
|
-
.filter(account =>
|
|
376
|
-
account.paused !== true && account.needsReauth !== true
|
|
377
|
-
&& !isCooled(account.id, now) && !anthropicRatePauseUntil(account.id, now) && !anthropicFamilyRejected(account.id, model, now)
|
|
378
|
-
&& isPoolCredentialUsable(account.id, now))
|
|
379
|
-
.map(account => account.id);
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
/**
|
|
383
|
-
* How long a quorum answer may be reused before the store is consulted again.
|
|
384
|
-
*
|
|
385
|
-
* This predicate now runs on the INITIAL resolution of every Anthropic request, not just after a
|
|
386
|
-
* 429, so an uncached implementation puts a synchronous file read in front of ordinary traffic:
|
|
387
|
-
* `getAccountSet` goes through `loadAuthStore`, which has no cache of its own and chmods the
|
|
388
|
-
* config dir, chmods the secret, reads the whole file and normalizes it on every call.
|
|
389
|
-
*
|
|
390
|
-
* Two seconds matches the generic module's `PRESENCE_CACHE_TTL_MS` for the same reason: short
|
|
391
|
-
* enough that a login in another window is visible before the operator can switch back and send a
|
|
392
|
-
* prompt, long enough that a burst of requests shares one read. The cache holds a BOOLEAN derived
|
|
393
|
-
* from a count — never a credential, never an account id.
|
|
394
|
-
*
|
|
395
|
-
* Staleness is bounded by consequence, not only by the TTL. Explicit invalidation covers the
|
|
396
|
-
* roster mutations this module can see (rotation, pool-state reset, affinity clear on account
|
|
397
|
-
* removal, manual selection), but not one it cannot: a 401 elsewhere flagging an account
|
|
398
|
-
* `needsReauth` drops the real quorum to one while a cached `true` survives for up to 2s.
|
|
399
|
-
*
|
|
400
|
-
* That window is harmless in both directions, which is why it is left rather than plumbed
|
|
401
|
-
* through the store. A stale `true` only lets the caller ASK for an alternate;
|
|
402
|
-
* `pickAlternateAnthropicAccount` re-reads the roster through `getEligibleAnthropicAccounts`,
|
|
403
|
-
* skips the reauth-flagged account and returns `null`, so the 429 surfaces exactly as it would
|
|
404
|
-
* have. A stale `false` costs one un-rotated 429 and self-corrects on the next read. Neither
|
|
405
|
-
* can dispatch on an unusable credential, which is the only outcome worth adding a store hook
|
|
406
|
-
* to prevent.
|
|
407
|
-
*/
|
|
408
|
-
const QUORUM_CACHE_TTL_MS = 2_000;
|
|
409
|
-
|
|
410
|
-
let quorumCache: { value: boolean; readAt: number } | null = null;
|
|
411
|
-
// Pause changes eligibility, not health. Do not reset cooldowns or cancel sent turns.
|
|
412
|
-
subscribeOAuthAccountPauseChanges(provider => { if (provider === PROVIDER) quorumCache = null; });
|
|
413
|
-
// A threshold write must fence in-flight automatic proposals, but it does not
|
|
414
|
-
// revoke an operator's one-dispatch choice. Rebase only that still-owned choice;
|
|
415
|
-
// an intervening account change clears it, so an ABA selection is not resurrected.
|
|
416
|
-
subscribeOAuthAccountRoutingPolicyChanges(event => {
|
|
417
|
-
if (event.provider !== PROVIDER || !manualPreference) return;
|
|
418
|
-
if (manualPreference.accountId !== event.before.accountId
|
|
419
|
-
|| manualPreference.revision !== event.before.revision) return;
|
|
420
|
-
manualPreference = event.after.accountId === manualPreference.accountId ? { ...event.after } : null;
|
|
421
|
-
});
|
|
422
|
-
// Any non-policy selection generation supersedes the pending one-shot choice.
|
|
423
|
-
// Threshold mutations rebase it first, before this generic notification runs.
|
|
424
|
-
subscribeAccountSelections(event => {
|
|
425
|
-
if (event.provider !== PROVIDER || event.kind !== "oauth" || !manualPreference) return;
|
|
426
|
-
const current = captureOAuthAccountSelection(PROVIDER);
|
|
427
|
-
if (current?.accountId !== manualPreference.accountId
|
|
428
|
-
|| current.revision !== manualPreference.revision) manualPreference = null;
|
|
429
|
-
});
|
|
430
|
-
|
|
431
|
-
/**
|
|
432
|
-
* Whether a 429 has somewhere to go: two or more accounts that could serve traffic if asked.
|
|
433
|
-
*
|
|
434
|
-
* Reactive failover is a safety net, not a routing policy. It runs only AFTER upstream refused,
|
|
435
|
-
* it cannot spread load across a healthy session, and it cannot fire at all unless the operator
|
|
436
|
-
* deliberately logged in twice. So it activates on presence, exactly like an `apiKeyPool` of two
|
|
437
|
-
* keys does in `providers/key-failover.ts` -- and unlike the PROACTIVE pool (affinity,
|
|
438
|
-
* quota-ranked new-session picks, `autoSwitchThreshold`, `strategy`), which changes which
|
|
439
|
-
* account serves a healthy request and therefore stays behind `anthropicAccountPool.enabled`.
|
|
440
|
-
*
|
|
441
|
-
* Cooldowns are deliberately ignored here. They are transient and per-request, while this
|
|
442
|
-
* answers the durable question "did the operator store a second account". Counting a cooled
|
|
443
|
-
* account as absent would switch the feature off for the length of the cooldown -- precisely
|
|
444
|
-
* when it is needed.
|
|
445
|
-
*
|
|
446
|
-
* `isPoolCredentialUsable` is still applied, so the fail-closed background `local-cli` rule
|
|
447
|
-
* holds: an expired background slot is not a quorum and cannot be adopted.
|
|
448
|
-
*/
|
|
449
|
-
export function hasAnthropicFailoverQuorum(now = Date.now()): boolean {
|
|
450
|
-
// Monotonic guard: a caller-supplied `now` that predates the cached read (tests pass explicit
|
|
451
|
-
// clocks) must not be served from a future entry.
|
|
452
|
-
if (quorumCache && now >= quorumCache.readAt && now - quorumCache.readAt < QUORUM_CACHE_TTL_MS) {
|
|
453
|
-
return quorumCache.value;
|
|
454
|
-
}
|
|
455
|
-
const set = getAccountSet(PROVIDER);
|
|
456
|
-
let value = false;
|
|
457
|
-
if (set) {
|
|
458
|
-
let usable = 0;
|
|
459
|
-
for (const account of set.accounts) {
|
|
460
|
-
if (account.paused === true || account.needsReauth === true) continue;
|
|
461
|
-
if (!isPoolCredentialUsable(account.id, now)) continue;
|
|
462
|
-
if (++usable >= 2) { value = true; break; }
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
quorumCache = { value, readAt: now };
|
|
466
|
-
return value;
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
/** Test seam and manual-recovery hook: force the next quorum question to re-read the store. */
|
|
470
|
-
export function forgetAnthropicFailoverQuorum(): void {
|
|
471
|
-
quorumCache = null;
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
/** Earliest remaining cooldown among cooled Anthropic accounts, for client Retry-After. */
|
|
475
|
-
export function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): number | null {
|
|
476
|
-
const set = getAccountSet(PROVIDER);
|
|
477
|
-
if (!set) return null;
|
|
478
|
-
// Once an explicit fallback route has no eligible declared account, selection may use
|
|
479
|
-
// the ordinary pool. Its earliest usable cooldown must determine the advertised wait.
|
|
480
|
-
const fallbackExpanded = decision?.fallback === true
|
|
481
|
-
&& !getEligibleAnthropicAccounts(now, model).some(id => decision.accounts.includes(id));
|
|
482
|
-
let earliest: number | null = null;
|
|
483
|
-
for (const account of set.accounts) {
|
|
484
|
-
if (account.paused === true || account.needsReauth === true || !isPoolCredentialUsable(account.id, now)) continue;
|
|
485
|
-
if (decision && !fallbackExpanded && !decision.accounts.includes(account.id)) continue;
|
|
486
|
-
const snap = getAnthropicAccountHealthSnapshot(account.id, now);
|
|
487
|
-
const until = Math.max(snap?.cooldownUntil ?? 0, anthropicRatePauseUntil(account.id, now) ?? 0, anthropicFamilyRetryAt(account.id, model, now) ?? 0);
|
|
488
|
-
if (until <= now) continue;
|
|
489
|
-
if (earliest === null || until < earliest) earliest = until;
|
|
490
|
-
}
|
|
491
|
-
if (earliest === null || earliest <= now) return null;
|
|
492
|
-
return Math.max(1, Math.ceil((earliest - now) / 1000));
|
|
493
|
-
}
|
|
494
|
-
|
|
495
155
|
interface ScoredAccount {
|
|
496
156
|
accountId: string;
|
|
497
157
|
hasKnownUsage: boolean;
|
|
@@ -512,110 +172,71 @@ function compareScoredAccounts(a: ScoredAccount, b: ScoredAccount): number {
|
|
|
512
172
|
return a.score - b.score || a.fiveHourTieBreak - b.fiveHourTieBreak;
|
|
513
173
|
}
|
|
514
174
|
|
|
515
|
-
function pickLowestUsage(config: OcxConfig, excludeId: string | undefined, now: number, decision: AnthropicRouteDecision | null = null, model?: string, excludedAccountIds?: ReadonlySet<string>): string | null {
|
|
516
|
-
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
517
|
-
const unfiltered = routeCandidates(getEligibleAnthropicAccounts(now, model), decision).filter(id => id !== excludeId && !excludedAccountIds?.has(id));
|
|
518
|
-
const available = window === "weekly" ? unfiltered.filter(id => !exhausted5h(id)
|
|
519
|
-
|| isAnthropicAccountPoolEnabled(config) && anthropicAccountAutoSwitchThreshold(config, id) === 0) : unfiltered;
|
|
520
|
-
const modelAvailable = model ? available.filter(id => !anthropicModelExhausted(id, model)) : available;
|
|
521
|
-
const availableOrFallback = modelAvailable.length > 0 ? modelAvailable : available.length > 0 ? available : unfiltered;
|
|
522
|
-
// Thresholds are preferences, never eligibility. Keep the old lowest-usage fallback
|
|
523
|
-
// when every candidate is drained, and keep pool-off reactive recovery policy inert.
|
|
524
|
-
const hasKnownUnderThreshold = isAnthropicAccountPoolEnabled(config)
|
|
525
|
-
&& availableOrFallback.some(id => hasKnownUsage(config, id, model) && isActiveUnderFillFirstThreshold(config, id, model));
|
|
526
|
-
const eligible = hasKnownUnderThreshold
|
|
527
|
-
? availableOrFallback.filter(id => isActiveUnderFillFirstThreshold(config, id, model)) : availableOrFallback;
|
|
528
|
-
if (eligible.length === 0) return null;
|
|
529
|
-
const scored: ScoredAccount[] = eligible.map(accountId => ({
|
|
530
|
-
accountId,
|
|
531
|
-
hasKnownUsage: hasKnownUsage(config, accountId, model),
|
|
532
|
-
score: usageScore(config, accountId, model),
|
|
533
|
-
fiveHourTieBreak: window === "five-hour" ? 0 : fiveHourScore(accountId),
|
|
534
|
-
// Every window EXCEPT the legacy five-hour default is an explicit opt-in, so
|
|
535
|
-
// known-before-unknown applies to all of them and to none of the default path.
|
|
536
|
-
knownFirst: window !== "five-hour",
|
|
537
|
-
}));
|
|
538
|
-
let best = scored[0]!;
|
|
539
|
-
for (let i = 1; i < scored.length; i++) {
|
|
540
|
-
const candidate = scored[i]!;
|
|
541
|
-
// Strict `< 0` keeps the earliest eligible account on an exact tie.
|
|
542
|
-
if (compareScoredAccounts(candidate, best) < 0) best = candidate;
|
|
543
|
-
}
|
|
544
|
-
return best.accountId;
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
/** A fallback route uses ordinary ordering only after it widens beyond its declared accounts. */
|
|
548
175
|
function usesDeclaredRouteOrder(eligible: readonly string[], decision: AnthropicRouteDecision | null): boolean {
|
|
549
176
|
return decision !== null && (!decision.fallback || eligible.some(id => decision.accounts.includes(id)));
|
|
550
177
|
}
|
|
551
178
|
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
decision: AnthropicRouteDecision | null,
|
|
558
|
-
model?: string,
|
|
559
|
-
): string | null {
|
|
560
|
-
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
561
|
-
const available = window === "weekly" ? eligible.filter(id => !exhausted5h(id)
|
|
562
|
-
|| anthropicAccountAutoSwitchThreshold(config, id) === 0) : eligible;
|
|
563
|
-
const modelAvailable = model ? available.filter(id => !anthropicModelExhausted(id, model)) : available;
|
|
564
|
-
const candidates = modelAvailable.length > 0 ? modelAvailable : available.length > 0 ? available : eligible;
|
|
565
|
-
if (candidates.length === 0) return null;
|
|
566
|
-
const routeOrder = usesDeclaredRouteOrder(eligible, decision);
|
|
567
|
-
const ordered = routeOrder ? candidates : [...candidates].sort((a, b) => a.localeCompare(b));
|
|
568
|
-
const set = getAccountSet(PROVIDER);
|
|
569
|
-
const stableAll = routeOrder ? [...decision!.accounts] : set
|
|
570
|
-
? [...set.accounts.map(a => a.id)].sort((a, b) => a.localeCompare(b))
|
|
571
|
-
: ordered;
|
|
572
|
-
const startIdx = stableAll.indexOf(afterId);
|
|
573
|
-
if (startIdx < 0) {
|
|
574
|
-
for (const id of ordered) {
|
|
575
|
-
if (isActiveUnderFillFirstThreshold(config, id, model)) return id;
|
|
576
|
-
}
|
|
577
|
-
return ordered[0] ?? null;
|
|
578
|
-
}
|
|
579
|
-
// Skip successors that are also at/above threshold (known drained usage).
|
|
580
|
-
let fallback: string | null = null;
|
|
581
|
-
for (let step = 1; step <= stableAll.length; step++) {
|
|
582
|
-
const candidate = stableAll[(startIdx + step) % stableAll.length]!;
|
|
583
|
-
if (!candidates.includes(candidate)) continue;
|
|
584
|
-
if (!fallback) fallback = candidate;
|
|
585
|
-
if (isActiveUnderFillFirstThreshold(config, candidate, model)) return candidate;
|
|
179
|
+
type OAuthAccountSelection = NonNullable<ReturnType<typeof captureOAuthAccountSelection>>;
|
|
180
|
+
|
|
181
|
+
export function parseAccountPoolQuotaWindow(raw: unknown): OcxAccountPoolQuotaWindow | null {
|
|
182
|
+
if (typeof raw === "string" && VALID_QUOTA_WINDOWS.has(raw as OcxAccountPoolQuotaWindow)) {
|
|
183
|
+
return raw as OcxAccountPoolQuotaWindow;
|
|
586
184
|
}
|
|
587
|
-
return
|
|
185
|
+
return null;
|
|
588
186
|
}
|
|
589
187
|
|
|
590
|
-
export function
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
188
|
+
export function normalizeAccountPoolQuotaWindow(raw: unknown): OcxAccountPoolQuotaWindow {
|
|
189
|
+
return parseAccountPoolQuotaWindow(raw) ?? DEFAULT_QUOTA_WINDOW;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export function anthropicQuotaWindow(config: AnthropicAccountPoolConfig): OcxAccountPoolQuotaWindow {
|
|
193
|
+
return normalizeAccountPoolQuotaWindow(config.quotaWindow);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function formatAnthropicAccountOrdinal(accountId: string): string {
|
|
197
|
+
return fallbackCodexAccountLogLabel(accountId);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export function formatAnthropicProviderForLog(
|
|
201
|
+
providerName: string,
|
|
202
|
+
accountId: string | null | undefined,
|
|
203
|
+
_config?: OcxConfig,
|
|
204
|
+
): string {
|
|
205
|
+
if (!accountId) return providerName;
|
|
206
|
+
return `${providerName}-${formatAnthropicAccountOrdinal(accountId)}`;
|
|
607
207
|
}
|
|
608
208
|
|
|
609
|
-
function
|
|
610
|
-
|
|
611
|
-
|
|
209
|
+
export function anthropicSessionKeyFromParts(input: {
|
|
210
|
+
sessionIdHeader?: string | null;
|
|
211
|
+
threadIdHeader?: string | null;
|
|
212
|
+
promptCacheKey?: string | null;
|
|
213
|
+
clientThreadId?: string | null;
|
|
214
|
+
/** When true, prompt_cache_key is a shared Desktop cohort — ignore it for affinity. */
|
|
215
|
+
promptCacheKeyIsSharedCohort?: boolean;
|
|
216
|
+
}): string | null {
|
|
217
|
+
const preferred = input.clientThreadId?.trim()
|
|
218
|
+
|| input.sessionIdHeader?.trim()
|
|
219
|
+
|| input.threadIdHeader?.trim()
|
|
220
|
+
|| "";
|
|
221
|
+
if (preferred) {
|
|
222
|
+
return preferred.length <= 128 ? preferred : createHash("sha256").update(preferred).digest("hex");
|
|
612
223
|
}
|
|
613
|
-
if (
|
|
614
|
-
const
|
|
615
|
-
|
|
616
|
-
|
|
224
|
+
if (input.promptCacheKeyIsSharedCohort) return null;
|
|
225
|
+
const cacheKey = input.promptCacheKey?.trim() ?? "";
|
|
226
|
+
if (!cacheKey) return null;
|
|
227
|
+
return cacheKey.length <= 128 ? cacheKey : createHash("sha256").update(cacheKey).digest("hex");
|
|
617
228
|
}
|
|
618
229
|
|
|
230
|
+
|
|
231
|
+
export type AnthropicCooldownRecoveryClaim = Readonly<{
|
|
232
|
+
instance: AnthropicInstanceId;
|
|
233
|
+
accountId: string;
|
|
234
|
+
cooldownGeneration: number;
|
|
235
|
+
claimedAt: number;
|
|
236
|
+
}>;
|
|
237
|
+
|
|
238
|
+
export type AnthropicCooldownRecoverySettlement = "cleared" | "retained" | "superseded";
|
|
239
|
+
|
|
619
240
|
export type AnthropicAccountSelectionReason =
|
|
620
241
|
| "pool-disabled"
|
|
621
242
|
| "affinity"
|
|
@@ -635,242 +256,1158 @@ export interface AnthropicAccountSelection {
|
|
|
635
256
|
routePosition?: number;
|
|
636
257
|
}
|
|
637
258
|
|
|
638
|
-
|
|
639
|
-
|
|
259
|
+
export interface AnthropicSelectionRoutingOptions {
|
|
260
|
+
config: OcxConfig;
|
|
261
|
+
sessionKey?: string | null;
|
|
262
|
+
reason?: AnthropicAccountSelectionReason;
|
|
263
|
+
expectedCredentialGeneration?: string;
|
|
264
|
+
routeDecision?: AnthropicRouteDecision | null;
|
|
265
|
+
model?: string;
|
|
640
266
|
}
|
|
641
267
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
268
|
+
/** No health/affinity bucket is allocated by a read or an event for a dormant instance. */
|
|
269
|
+
class RoutingMap<K, V> implements Iterable<[K, V]> {
|
|
270
|
+
private rows?: Map<K, V>;
|
|
271
|
+
get size(): number { return this.rows?.size ?? 0; }
|
|
272
|
+
get(key: K): V | undefined { return this.rows?.get(key); }
|
|
273
|
+
set(key: K, value: V): void { (this.rows ??= new Map()).set(key, value); }
|
|
274
|
+
delete(key: K): boolean { return this.rows?.delete(key) ?? false; }
|
|
275
|
+
clear(): void { this.rows = undefined; }
|
|
276
|
+
*entries(): IterableIterator<[K, V]> { if (this.rows) yield* this.rows; }
|
|
277
|
+
[Symbol.iterator](): IterableIterator<[K, V]> { return this.entries(); }
|
|
278
|
+
}
|
|
279
|
+
type RoutingFacadeSymbol =
|
|
280
|
+
| "captureAnthropicManualSelectionGeneration"
|
|
281
|
+
| "anthropicAccountPoolConfig"
|
|
282
|
+
| "isAnthropicAccountPoolEnabled"
|
|
283
|
+
| "anthropicAutoSwitchThreshold"
|
|
284
|
+
| "anthropicAccountAutoSwitchThreshold"
|
|
285
|
+
| "captureAnthropicCooldownRecovery"
|
|
286
|
+
| "settleAnthropicCooldownRecovery"
|
|
287
|
+
| "getAnthropicAccountHealthSnapshot"
|
|
288
|
+
| "clearAnthropicAccountCooldown"
|
|
289
|
+
| "sweepExpiredAnthropicRoutingHealth"
|
|
290
|
+
| "clearAnthropicAccountPoolState"
|
|
291
|
+
| "anthropicSessionAffinitySizeForTests"
|
|
292
|
+
| "getEligibleAnthropicAccounts"
|
|
293
|
+
| "hasAnthropicFailoverQuorum"
|
|
294
|
+
| "forgetAnthropicFailoverQuorum"
|
|
295
|
+
| "getAnthropicPoolRetryAfterSeconds"
|
|
296
|
+
| "pickAlternateAnthropicAccount"
|
|
297
|
+
| "resolveAnthropicAccountForSession"
|
|
298
|
+
| "resolveAnthropicDispatchAccountId"
|
|
299
|
+
| "bindAnthropicSessionAffinity"
|
|
300
|
+
| "clearAnthropicSessionAffinityForAccount"
|
|
301
|
+
| "rotateAnthropicAccountOn429"
|
|
302
|
+
| "rotateAnthropicAccountOnRefusal"
|
|
303
|
+
| "recordAnthropicAccount429"
|
|
304
|
+
| "recordAnthropicAccountRefusal"
|
|
305
|
+
| "promoteAnthropicActiveAccount"
|
|
306
|
+
| "commitAnthropicSelectionRouting"
|
|
307
|
+
| "resetAnthropicRoutingForManualSelection"
|
|
308
|
+
| "getAnthropicPoolAccessToken"
|
|
309
|
+
| "getAnthropicPoolAccessSnapshot"
|
|
310
|
+
| "canRefreshAnthropicPoolAccount";
|
|
311
|
+
export type AnthropicRouting = Readonly<{ instance: AnthropicInstanceId } & Pick<typeof import("./anthropic-routing"), RoutingFacadeSymbol>>;
|
|
312
|
+
type RoutingHandlers = { onPause(): void; onPolicy: Parameters<typeof subscribeOAuthAccountRoutingPolicyChanges>[0]; onSelection: Parameters<typeof subscribeAccountSelections>[0]; reconcile(context: GenerationContext, config?: OcxConfig): number };
|
|
313
|
+
const instances = new Map<AnthropicInstanceId, AnthropicRouting>();
|
|
314
|
+
const handlers = new Map<AnthropicInstanceId, RoutingHandlers>();
|
|
315
|
+
function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting {
|
|
316
|
+
const PROVIDER = instance;
|
|
317
|
+
const poolKey = anthropicPoolKey(instance);
|
|
318
|
+
const { anthropicFamilyRejected, anthropicFamilyRetryAt, anthropicModelExhausted, clearAnthropicFamilyQuota } = anthropicModelQuotaFor(instance);
|
|
319
|
+
const { anthropicRatePauseUntil, clearAnthropicRatePauses, pauseAnthropicRateAdmission } = anthropicRatePolicyFor(instance);
|
|
320
|
+
const { anthropicCooldownGeneration, clearAnthropicCooldownGenerations, noteAnthropicCooldownMutation } = anthropicCooldownRecoveryFor(instance);
|
|
321
|
+
// Request-owned config authorizes selection/commit. Namespace-only credential and roster
|
|
322
|
+
// methods carry no ambient disk authority; physical consumers recheck their captured config/target.
|
|
323
|
+
function admitted(config: OcxConfig): boolean { return configuredAnthropicInstance(config, instance) === instance; }
|
|
324
|
+
const upstreamHealth = new RoutingMap<string, AccountHealth>();
|
|
325
|
+
const sessionAffinity = new RoutingMap<string, AffinityEntry>();
|
|
326
|
+
// Undefined means this runtime has not admitted a selection yet; null means consumed.
|
|
327
|
+
// The startup baseline comes from the authoritative store, never a second persisted pin.
|
|
328
|
+
let manualPreference: OAuthAccountSelection | null | undefined;
|
|
329
|
+
let manualSelectionGeneration = 0;
|
|
330
|
+
/** Automatic pointer moves preserve affinity; manual selection revokes older bindings. */
|
|
331
|
+
function captureAnthropicManualSelectionGeneration(): number { return manualSelectionGeneration; }
|
|
332
|
+
|
|
333
|
+
function anthropicAccountPoolConfig(config: OcxConfig): AnthropicAccountPoolConfig {
|
|
334
|
+
return resolveAnthropicAccountPoolConfig(config, instance);
|
|
335
|
+
}
|
|
645
336
|
|
|
646
|
-
function
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
650
|
-
if (window === "weekly" && exhausted5h(accountId)) return false;
|
|
651
|
-
// Unknown usage must not force fill-first to abandon the active account.
|
|
652
|
-
if (!hasKnownUsage(config, accountId, model)) return true;
|
|
653
|
-
return usageScore(config, accountId, model) < threshold;
|
|
654
|
-
}
|
|
337
|
+
function isAnthropicAccountPoolEnabled(config: OcxConfig): boolean {
|
|
338
|
+
return anthropicAccountPoolConfig(config).enabled === true;
|
|
339
|
+
}
|
|
655
340
|
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
function pickFillFirstAnthropicAccount(config: OcxConfig, now: number, decision: AnthropicRouteDecision | null, model?: string): string | null {
|
|
661
|
-
const eligible = routeCandidates(getEligibleAnthropicAccounts(now, model), decision);
|
|
662
|
-
if (eligible.length === 0) return null;
|
|
663
|
-
|
|
664
|
-
const set = getAccountSet(PROVIDER);
|
|
665
|
-
const active = set?.activeAccountId;
|
|
666
|
-
if (active && eligible.includes(active) && (!model || !anthropicModelExhausted(active, model)) && isActiveUnderFillFirstThreshold(config, active, model)) {
|
|
667
|
-
return active;
|
|
341
|
+
function anthropicAutoSwitchThreshold(config: OcxConfig): number {
|
|
342
|
+
const value = anthropicAccountPoolConfig(config).autoSwitchThreshold;
|
|
343
|
+
if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100) return value;
|
|
344
|
+
return DEFAULT_AUTO_SWITCH_THRESHOLD;
|
|
668
345
|
}
|
|
669
346
|
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
if (isActiveUnderFillFirstThreshold(config, id, model)) return id;
|
|
674
|
-
}
|
|
675
|
-
return ordered[0] ?? null;
|
|
347
|
+
/** Read live policy at selection, not a credential snapshot captured before an await. */
|
|
348
|
+
function anthropicAccountAutoSwitchThreshold(config: OcxConfig, accountId: string): number {
|
|
349
|
+
return effectiveAnthropicAccountThresholdForInstance(instance, config, getAccountSet(PROVIDER)?.accounts.find(row => row.id === accountId));
|
|
676
350
|
}
|
|
677
351
|
|
|
678
|
-
return pickNextFillFirstAnthropicAccount(config, active, eligible, decision, model);
|
|
679
|
-
}
|
|
680
352
|
|
|
681
|
-
/**
|
|
682
|
-
* Unbound new-session pick for round-robin / fill-first. Returns null to fall through
|
|
683
|
-
* to the legacy quota path (or when the strategy is quota).
|
|
684
|
-
*/
|
|
685
|
-
function pickUnboundStrategyAccount(
|
|
686
|
-
config: OcxConfig,
|
|
687
|
-
now: number,
|
|
688
|
-
decision: AnthropicRouteDecision | null,
|
|
689
|
-
model?: string,
|
|
690
|
-
): { accountId: string; reason: "round-robin" | "fill-first" } | null {
|
|
691
|
-
const strategy = anthropicPoolStrategy(config);
|
|
692
|
-
if (strategy === "quota") return null;
|
|
353
|
+
/** Accept upstream deadlines within the runtime's date range, without a policy ceiling. */
|
|
693
354
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
355
|
+
/** Only rejected windows constrain recovery; all must reopen, so take the latest reset. */
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Capture the exact reset-derived refusal a fresh usage probe is allowed to recover.
|
|
360
|
+
*
|
|
361
|
+
* The generation fence matters because the usage request is asynchronous: a newer 429 may
|
|
362
|
+
* arrive while it is in flight, and an older answer must never erase that newer refusal.
|
|
363
|
+
*/
|
|
364
|
+
function captureAnthropicCooldownRecovery(
|
|
365
|
+
accountId: string,
|
|
366
|
+
now = Date.now(),
|
|
367
|
+
): AnthropicCooldownRecoveryClaim | null {
|
|
368
|
+
const entry = upstreamHealth.get(accountId);
|
|
369
|
+
if (!entry || entry.cooldownUntil <= now || entry.cooldownSource !== "reset-derived") return null;
|
|
370
|
+
return { instance, accountId, cooldownGeneration: entry.cooldownGeneration, claimedAt: now };
|
|
701
371
|
}
|
|
702
372
|
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
373
|
+
/**
|
|
374
|
+
* Clear only the claimed reset-derived cooldown when a fresh, complete usage result proves
|
|
375
|
+
* headroom in every window that upstream previously reported as rejected.
|
|
376
|
+
*/
|
|
377
|
+
function settleAnthropicCooldownRecovery(
|
|
378
|
+
claim: AnthropicCooldownRecoveryClaim,
|
|
379
|
+
quota: ProviderQuota,
|
|
380
|
+
): AnthropicCooldownRecoverySettlement {
|
|
381
|
+
if (claim.instance !== instance) return "superseded";
|
|
382
|
+
const entry = upstreamHealth.get(claim.accountId);
|
|
383
|
+
if (!entry || entry.cooldownSource !== "reset-derived"
|
|
384
|
+
|| entry.cooldownGeneration !== claim.cooldownGeneration
|
|
385
|
+
|| anthropicCooldownGeneration(claim.accountId) !== claim.cooldownGeneration
|
|
386
|
+
|| !Number.isFinite(quota.updatedAt) || quota.updatedAt < claim.claimedAt) return "superseded";
|
|
387
|
+
const windows = entry.rejectedQuotaWindows;
|
|
388
|
+
if (!windows?.length) return "retained";
|
|
389
|
+
const recovered = windows.every(window => {
|
|
390
|
+
const percent = window === "five-hour" ? quota.fiveHourPercent : quota.weeklyPercent;
|
|
391
|
+
return typeof percent === "number" && Number.isFinite(percent) && percent >= 0 && percent < 100;
|
|
392
|
+
});
|
|
393
|
+
if (!recovered) return "retained";
|
|
394
|
+
upstreamHealth.delete(claim.accountId);
|
|
395
|
+
noteAnthropicCooldownMutation(claim.accountId);
|
|
396
|
+
return "cleared";
|
|
707
397
|
}
|
|
708
398
|
|
|
709
|
-
|
|
710
|
-
|
|
399
|
+
function getAnthropicAccountHealthSnapshot(
|
|
400
|
+
accountId: string,
|
|
401
|
+
now = Date.now(),
|
|
402
|
+
): { cooldownUntil?: number; cooldownSource?: AccountHealth["cooldownSource"] } | null {
|
|
403
|
+
const entry = upstreamHealth.get(accountId);
|
|
404
|
+
if (!entry) return null;
|
|
405
|
+
if (entry.cooldownUntil <= now) {
|
|
406
|
+
upstreamHealth.delete(accountId);
|
|
407
|
+
noteAnthropicCooldownMutation(accountId);
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
return { cooldownUntil: entry.cooldownUntil, cooldownSource: entry.cooldownSource };
|
|
411
|
+
}
|
|
711
412
|
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
export function resolveAnthropicAccountForSession(
|
|
717
|
-
sessionKey: string | null | undefined,
|
|
718
|
-
config: OcxConfig,
|
|
719
|
-
now = Date.now(),
|
|
720
|
-
decision: AnthropicRouteDecision | null = null,
|
|
721
|
-
model?: string,
|
|
722
|
-
): AnthropicAccountSelection {
|
|
723
|
-
pruneExpiredAffinity(now);
|
|
724
|
-
const set = getAccountSet(PROVIDER);
|
|
725
|
-
if (!set || set.accounts.length === 0) return { accountId: null, reason: "none", ...(decision ? { routePosition: decision.position } : {}) };
|
|
726
|
-
const scoped = decision && !decision.fallback
|
|
727
|
-
? set.accounts.filter(account => decision.accounts.includes(account.id)) : set.accounts;
|
|
728
|
-
if (scoped.length > 0 && scoped.every(account => account.paused === true)) {
|
|
729
|
-
return { accountId: null, reason: "paused", routePosition: decision?.position };
|
|
413
|
+
function clearAnthropicAccountCooldown(accountId: string): boolean {
|
|
414
|
+
const cleared = upstreamHealth.delete(accountId);
|
|
415
|
+
if (cleared) noteAnthropicCooldownMutation(accountId);
|
|
416
|
+
return cleared;
|
|
730
417
|
}
|
|
731
418
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
419
|
+
function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number {
|
|
420
|
+
let removed = 0;
|
|
421
|
+
for (const [accountId, health] of upstreamHealth) {
|
|
422
|
+
if (health.cooldownUntil > now) continue;
|
|
423
|
+
upstreamHealth.delete(accountId);
|
|
424
|
+
noteAnthropicCooldownMutation(accountId);
|
|
425
|
+
removed += 1;
|
|
426
|
+
}
|
|
427
|
+
return removed;
|
|
736
428
|
}
|
|
737
429
|
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
430
|
+
/** Test / logout helper. */
|
|
431
|
+
function clearAnthropicAccountPoolState(): void {
|
|
432
|
+
for (const [id] of upstreamHealth) noteAnthropicCooldownMutation(id);
|
|
433
|
+
upstreamHealth.clear();
|
|
434
|
+
clearAnthropicRatePauses();
|
|
435
|
+
clearAnthropicFamilyQuota();
|
|
436
|
+
clearAnthropicCooldownGenerations();
|
|
437
|
+
sessionAffinity.clear();
|
|
438
|
+
manualPreference = undefined;
|
|
439
|
+
manualSelectionGeneration++;
|
|
440
|
+
quorumCache = null;
|
|
747
441
|
}
|
|
748
442
|
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
// Disabled proactive rotation does not authorize a paused slot or an entirely cooled pool.
|
|
752
|
-
if (active?.paused || isCooled(set.activeAccountId, now) || anthropicRatePauseUntil(set.activeAccountId, now) || anthropicFamilyRejected(set.activeAccountId, model, now)) {
|
|
753
|
-
return { accountId: eligible[0] ?? null, reason: eligible.length > 0 ? "only-eligible" : "none" };
|
|
754
|
-
}
|
|
755
|
-
return { accountId: set.activeAccountId, reason: "pool-disabled" };
|
|
443
|
+
function anthropicSessionAffinitySizeForTests(): number {
|
|
444
|
+
return sessionAffinity.size;
|
|
756
445
|
}
|
|
757
446
|
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
if (manualPreference) {
|
|
761
|
-
if (manualPreference.accountId !== set.activeAccountId || manualPreference.revision !== set.selectionRevision) {
|
|
762
|
-
manualPreference = null;
|
|
763
|
-
} else {
|
|
764
|
-
const chosen = manualPreference.accountId;
|
|
765
|
-
const quota = getCachedProviderAccountQuota(PROVIDER, chosen);
|
|
766
|
-
const exhausted = anthropicModelPercents(quota, model).some(percent => percent >= 100);
|
|
767
|
-
if (!exhausted && eligible.includes(chosen)) {
|
|
768
|
-
return { accountId: chosen, reason: "manual", routePosition: decision?.position };
|
|
769
|
-
}
|
|
770
|
-
}
|
|
447
|
+
function isCooled(accountId: string, now: number): boolean {
|
|
448
|
+
return getAnthropicAccountHealthSnapshot(accountId, now) !== null;
|
|
771
449
|
}
|
|
772
450
|
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
if (affined && now - affined.lastUsedAt <= AFFINITY_IDLE_TTL_MS) {
|
|
777
|
-
const stillThere = set.accounts.some(a => a.id === affined.accountId && a.paused !== true && a.needsReauth !== true);
|
|
778
|
-
const stillUsable = stillThere && !isCooled(affined.accountId, now)
|
|
779
|
-
&& isPoolCredentialUsable(affined.accountId, now);
|
|
780
|
-
if (stillUsable && eligible.includes(affined.accountId) && (!model || !anthropicModelExhausted(affined.accountId, model))) {
|
|
781
|
-
return { accountId: affined.accountId, reason: "affinity", routePosition: decision?.position };
|
|
782
|
-
}
|
|
783
|
-
// A model route may exclude a healthy binding only for this request. Keep it for
|
|
784
|
-
// another model; remove bindings only when the account itself became unusable.
|
|
785
|
-
if (!stillUsable) sessionAffinity.delete(key);
|
|
786
|
-
}
|
|
451
|
+
function fiveHourKnown(accountId: string): boolean {
|
|
452
|
+
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.fiveHourPercent;
|
|
453
|
+
return typeof percent === "number" && Number.isFinite(percent);
|
|
787
454
|
}
|
|
788
455
|
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
// Round-robin only when there is a real new-session key (or active is unusable).
|
|
793
|
-
if (!key && (strategy === "round-robin" || strategy === "fill-first")) {
|
|
794
|
-
const activeOk = set.accounts.some(a => a.id === set.activeAccountId && a.needsReauth !== true)
|
|
795
|
-
&& !isCooled(set.activeAccountId, now)
|
|
796
|
-
&& eligible.includes(set.activeAccountId) && (!model || !anthropicModelExhausted(set.activeAccountId, model));
|
|
797
|
-
if (activeOk) {
|
|
798
|
-
return { accountId: set.activeAccountId, reason: "active", routePosition: decision?.position };
|
|
799
|
-
}
|
|
456
|
+
function weeklyKnown(accountId: string): boolean {
|
|
457
|
+
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.weeklyPercent;
|
|
458
|
+
return typeof percent === "number" && Number.isFinite(percent);
|
|
800
459
|
}
|
|
801
460
|
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
return
|
|
461
|
+
function fiveHourScore(accountId: string): number {
|
|
462
|
+
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.fiveHourPercent;
|
|
463
|
+
return typeof percent === "number" && Number.isFinite(percent)
|
|
464
|
+
? Math.max(0, Math.min(100, percent))
|
|
465
|
+
: UNKNOWN_USAGE_SCORE;
|
|
805
466
|
}
|
|
806
467
|
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
&&
|
|
810
|
-
|
|
468
|
+
function weeklyScore(accountId: string): number {
|
|
469
|
+
const percent = getCachedProviderAccountQuota(PROVIDER, accountId)?.weeklyPercent;
|
|
470
|
+
return typeof percent === "number" && Number.isFinite(percent)
|
|
471
|
+
? Math.max(0, Math.min(100, percent))
|
|
472
|
+
: UNKNOWN_USAGE_SCORE;
|
|
473
|
+
}
|
|
811
474
|
|
|
812
|
-
|
|
813
|
-
|
|
475
|
+
function exhausted5h(accountId: string): boolean {
|
|
476
|
+
return fiveHourKnown(accountId) && fiveHourScore(accountId) >= 100;
|
|
477
|
+
}
|
|
814
478
|
|
|
815
|
-
|
|
479
|
+
function hasKnownUsage(config: OcxConfig, accountId: string, model?: string): boolean {
|
|
480
|
+
if (model) return hasKnownUsage(config, accountId) || anthropicModelWeeklyPercent(getCachedProviderAccountQuota(PROVIDER, accountId), model) !== undefined;
|
|
816
481
|
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
482
|
+
switch (window) {
|
|
483
|
+
case "five-hour": return fiveHourKnown(accountId);
|
|
484
|
+
case "weekly": return weeklyKnown(accountId);
|
|
485
|
+
case "max-utilization": return fiveHourKnown(accountId) || weeklyKnown(accountId);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
function usageScore(config: OcxConfig, accountId: string, model?: string): number {
|
|
490
|
+
if (model) {
|
|
491
|
+
const family = anthropicModelWeeklyPercent(getCachedProviderAccountQuota(PROVIDER, accountId), model);
|
|
492
|
+
if (family === undefined) return usageScore(config, accountId);
|
|
493
|
+
return hasKnownUsage(config, accountId) ? Math.max(family, usageScore(config, accountId)) : family;
|
|
494
|
+
}
|
|
495
|
+
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
496
|
+
switch (window) {
|
|
497
|
+
case "five-hour": return fiveHourScore(accountId);
|
|
498
|
+
case "weekly": return weeklyScore(accountId);
|
|
499
|
+
case "max-utilization": {
|
|
500
|
+
const scores = [
|
|
501
|
+
...(fiveHourKnown(accountId) ? [fiveHourScore(accountId)] : []),
|
|
502
|
+
...(weeklyKnown(accountId) ? [weeklyScore(accountId)] : []),
|
|
503
|
+
];
|
|
504
|
+
return scores.length > 0 ? Math.max(...scores) : UNKNOWN_USAGE_SCORE;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
/** Background `local-cli` slots with expired access are not pool-eligible (identity adoption risk). */
|
|
511
|
+
function isPoolCredentialUsable(accountId: string, now: number): boolean {
|
|
512
|
+
const cred = getAccountCredential(PROVIDER, accountId);
|
|
513
|
+
if (!cred) return false;
|
|
514
|
+
if (instance === "anthropic2" && cred.source === "local-cli") return false;
|
|
515
|
+
if (cred.source !== "local-cli") return true;
|
|
516
|
+
if (canRefreshAnthropicPoolAccount(accountId)) return true;
|
|
517
|
+
return cred.expires > now + REFRESH_SKEW_MS;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function getEligibleAnthropicAccounts(now = Date.now(), model?: string): string[] {
|
|
521
|
+
const set = getAccountSet(PROVIDER);
|
|
522
|
+
if (!set) return [];
|
|
523
|
+
return set.accounts
|
|
524
|
+
.filter(account =>
|
|
525
|
+
account.paused !== true && account.needsReauth !== true
|
|
526
|
+
&& !isCooled(account.id, now) && !anthropicRatePauseUntil(account.id, now) && !anthropicFamilyRejected(account.id, model, now)
|
|
527
|
+
&& isPoolCredentialUsable(account.id, now))
|
|
528
|
+
.map(account => account.id);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* How long a quorum answer may be reused before the store is consulted again.
|
|
533
|
+
*
|
|
534
|
+
* This predicate now runs on the INITIAL resolution of every Anthropic request, not just after a
|
|
535
|
+
* 429, so an uncached implementation puts a synchronous file read in front of ordinary traffic:
|
|
536
|
+
* `getAccountSet` goes through `loadAuthStore`, which has no cache of its own and chmods the
|
|
537
|
+
* config dir, chmods the secret, reads the whole file and normalizes it on every call.
|
|
538
|
+
*
|
|
539
|
+
* Two seconds matches the generic module's `PRESENCE_CACHE_TTL_MS` for the same reason: short
|
|
540
|
+
* enough that a login in another window is visible before the operator can switch back and send a
|
|
541
|
+
* prompt, long enough that a burst of requests shares one read. The cache holds a BOOLEAN derived
|
|
542
|
+
* from a count — never a credential, never an account id.
|
|
543
|
+
*
|
|
544
|
+
* Staleness is bounded by consequence, not only by the TTL. Explicit invalidation covers the
|
|
545
|
+
* roster mutations this module can see (rotation, pool-state reset, affinity clear on account
|
|
546
|
+
* removal, manual selection), but not one it cannot: a 401 elsewhere flagging an account
|
|
547
|
+
* `needsReauth` drops the real quorum to one while a cached `true` survives for up to 2s.
|
|
548
|
+
*
|
|
549
|
+
* That window is harmless in both directions, which is why it is left rather than plumbed
|
|
550
|
+
* through the store. A stale `true` only lets the caller ASK for an alternate;
|
|
551
|
+
* `pickAlternateAnthropicAccount` re-reads the roster through `getEligibleAnthropicAccounts`,
|
|
552
|
+
* skips the reauth-flagged account and returns `null`, so the 429 surfaces exactly as it would
|
|
553
|
+
* have. A stale `false` costs one un-rotated 429 and self-corrects on the next read. Neither
|
|
554
|
+
* can dispatch on an unusable credential, which is the only outcome worth adding a store hook
|
|
555
|
+
* to prevent.
|
|
556
|
+
*/
|
|
557
|
+
const QUORUM_CACHE_TTL_MS = 2_000;
|
|
558
|
+
|
|
559
|
+
let quorumCache: { value: boolean; readAt: number } | null = null;
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* Whether a 429 has somewhere to go: two or more accounts that could serve traffic if asked.
|
|
563
|
+
*
|
|
564
|
+
* Reactive failover is a safety net, not a routing policy. It runs only AFTER upstream refused,
|
|
565
|
+
* it cannot spread load across a healthy session, and it cannot fire at all unless the operator
|
|
566
|
+
* deliberately logged in twice. So it activates on presence, exactly like an `apiKeyPool` of two
|
|
567
|
+
* keys does in `providers/key-failover.ts` -- and unlike the PROACTIVE pool (affinity,
|
|
568
|
+
* quota-ranked new-session picks, `autoSwitchThreshold`, `strategy`), which changes which
|
|
569
|
+
* account serves a healthy request and therefore stays behind `anthropicAccountPool.enabled`.
|
|
570
|
+
*
|
|
571
|
+
* Cooldowns are deliberately ignored here. They are transient and per-request, while this
|
|
572
|
+
* answers the durable question "did the operator store a second account". Counting a cooled
|
|
573
|
+
* account as absent would switch the feature off for the length of the cooldown -- precisely
|
|
574
|
+
* when it is needed.
|
|
575
|
+
*
|
|
576
|
+
* `isPoolCredentialUsable` is still applied, so the fail-closed background `local-cli` rule
|
|
577
|
+
* holds: an expired background slot is not a quorum and cannot be adopted.
|
|
578
|
+
*/
|
|
579
|
+
function hasAnthropicFailoverQuorum(now = Date.now()): boolean {
|
|
580
|
+
// Monotonic guard: a caller-supplied `now` that predates the cached read (tests pass explicit
|
|
581
|
+
// clocks) must not be served from a future entry.
|
|
582
|
+
if (quorumCache && now >= quorumCache.readAt && now - quorumCache.readAt < QUORUM_CACHE_TTL_MS) {
|
|
583
|
+
return quorumCache.value;
|
|
584
|
+
}
|
|
585
|
+
const set = getAccountSet(PROVIDER);
|
|
586
|
+
let value = false;
|
|
587
|
+
if (set) {
|
|
588
|
+
let usable = 0;
|
|
589
|
+
for (const account of set.accounts) {
|
|
590
|
+
if (account.paused === true || account.needsReauth === true) continue;
|
|
591
|
+
if (!isPoolCredentialUsable(account.id, now)) continue;
|
|
592
|
+
if (++usable >= 2) { value = true; break; }
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
quorumCache = { value, readAt: now };
|
|
596
|
+
return value;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/** Test seam and manual-recovery hook: force the next quorum question to re-read the store. */
|
|
600
|
+
function forgetAnthropicFailoverQuorum(): void {
|
|
601
|
+
quorumCache = null;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/** Earliest remaining cooldown among cooled Anthropic accounts, for client Retry-After. */
|
|
605
|
+
function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): number | null {
|
|
606
|
+
const set = getAccountSet(PROVIDER);
|
|
607
|
+
if (!set) return null;
|
|
608
|
+
// Once an explicit fallback route has no eligible declared account, selection may use
|
|
609
|
+
// the ordinary pool. Its earliest usable cooldown must determine the advertised wait.
|
|
610
|
+
const fallbackExpanded = decision?.fallback === true
|
|
611
|
+
&& !getEligibleAnthropicAccounts(now, model).some(id => decision.accounts.includes(id));
|
|
612
|
+
let earliest: number | null = null;
|
|
613
|
+
for (const account of set.accounts) {
|
|
614
|
+
if (account.paused === true || account.needsReauth === true || !isPoolCredentialUsable(account.id, now)) continue;
|
|
615
|
+
if (decision && !fallbackExpanded && !decision.accounts.includes(account.id)) continue;
|
|
616
|
+
const snap = getAnthropicAccountHealthSnapshot(account.id, now);
|
|
617
|
+
const until = Math.max(snap?.cooldownUntil ?? 0, anthropicRatePauseUntil(account.id, now) ?? 0, anthropicFamilyRetryAt(account.id, model, now) ?? 0);
|
|
618
|
+
if (until <= now) continue;
|
|
619
|
+
if (earliest === null || until < earliest) earliest = until;
|
|
620
|
+
}
|
|
621
|
+
if (earliest === null || earliest <= now) return null;
|
|
622
|
+
return Math.max(1, Math.ceil((earliest - now) / 1000));
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
function pickLowestUsage(config: OcxConfig, excludeId: string | undefined, now: number, decision: AnthropicRouteDecision | null = null, model?: string, excludedAccountIds?: ReadonlySet<string>): string | null {
|
|
626
|
+
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
627
|
+
const unfiltered = routeCandidates(getEligibleAnthropicAccounts(now, model), decision).filter(id => id !== excludeId && !excludedAccountIds?.has(id));
|
|
628
|
+
const available = window === "weekly" ? unfiltered.filter(id => !exhausted5h(id)
|
|
629
|
+
|| isAnthropicAccountPoolEnabled(config) && anthropicAccountAutoSwitchThreshold(config, id) === 0) : unfiltered;
|
|
630
|
+
const modelAvailable = model ? available.filter(id => !anthropicModelExhausted(id, model)) : available;
|
|
631
|
+
const availableOrFallback = modelAvailable.length > 0 ? modelAvailable : available.length > 0 ? available : unfiltered;
|
|
632
|
+
// Thresholds are preferences, never eligibility. Keep the old lowest-usage fallback
|
|
633
|
+
// when every candidate is drained, and keep pool-off reactive recovery policy inert.
|
|
634
|
+
const hasKnownUnderThreshold = isAnthropicAccountPoolEnabled(config)
|
|
635
|
+
&& availableOrFallback.some(id => hasKnownUsage(config, id, model) && isActiveUnderFillFirstThreshold(config, id, model));
|
|
636
|
+
const eligible = hasKnownUnderThreshold
|
|
637
|
+
? availableOrFallback.filter(id => isActiveUnderFillFirstThreshold(config, id, model)) : availableOrFallback;
|
|
638
|
+
if (eligible.length === 0) return null;
|
|
639
|
+
const scored: ScoredAccount[] = eligible.map(accountId => ({
|
|
640
|
+
accountId,
|
|
641
|
+
hasKnownUsage: hasKnownUsage(config, accountId, model),
|
|
642
|
+
score: usageScore(config, accountId, model),
|
|
643
|
+
fiveHourTieBreak: window === "five-hour" ? 0 : fiveHourScore(accountId),
|
|
644
|
+
// Every window EXCEPT the legacy five-hour default is an explicit opt-in, so
|
|
645
|
+
// known-before-unknown applies to all of them and to none of the default path.
|
|
646
|
+
knownFirst: window !== "five-hour",
|
|
647
|
+
}));
|
|
648
|
+
let best = scored[0]!;
|
|
649
|
+
for (let i = 1; i < scored.length; i++) {
|
|
650
|
+
const candidate = scored[i]!;
|
|
651
|
+
// Strict `< 0` keeps the earliest eligible account on an exact tie.
|
|
652
|
+
if (compareScoredAccounts(candidate, best) < 0) best = candidate;
|
|
653
|
+
}
|
|
654
|
+
return best.accountId;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/** A fallback route uses ordinary ordering only after it widens beyond its declared accounts. */
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
/** Next eligible Anthropic account in stable order after `afterId` (wrapping). */
|
|
661
|
+
function pickNextFillFirstAnthropicAccount(
|
|
662
|
+
config: OcxConfig,
|
|
663
|
+
afterId: string,
|
|
664
|
+
eligible: string[],
|
|
665
|
+
decision: AnthropicRouteDecision | null,
|
|
666
|
+
model?: string,
|
|
667
|
+
): string | null {
|
|
668
|
+
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
669
|
+
const available = window === "weekly" ? eligible.filter(id => !exhausted5h(id)
|
|
670
|
+
|| anthropicAccountAutoSwitchThreshold(config, id) === 0) : eligible;
|
|
671
|
+
const modelAvailable = model ? available.filter(id => !anthropicModelExhausted(id, model)) : available;
|
|
672
|
+
const candidates = modelAvailable.length > 0 ? modelAvailable : available.length > 0 ? available : eligible;
|
|
673
|
+
if (candidates.length === 0) return null;
|
|
674
|
+
const routeOrder = usesDeclaredRouteOrder(eligible, decision);
|
|
675
|
+
const ordered = routeOrder ? candidates : [...candidates].sort((a, b) => a.localeCompare(b));
|
|
676
|
+
const set = getAccountSet(PROVIDER);
|
|
677
|
+
const stableAll = routeOrder ? [...decision!.accounts] : set
|
|
678
|
+
? [...set.accounts.map(a => a.id)].sort((a, b) => a.localeCompare(b))
|
|
679
|
+
: ordered;
|
|
680
|
+
const startIdx = stableAll.indexOf(afterId);
|
|
681
|
+
if (startIdx < 0) {
|
|
682
|
+
for (const id of ordered) {
|
|
683
|
+
if (isActiveUnderFillFirstThreshold(config, id, model)) return id;
|
|
684
|
+
}
|
|
685
|
+
return ordered[0] ?? null;
|
|
686
|
+
}
|
|
687
|
+
// Skip successors that are also at/above threshold (known drained usage).
|
|
688
|
+
let fallback: string | null = null;
|
|
689
|
+
for (let step = 1; step <= stableAll.length; step++) {
|
|
690
|
+
const candidate = stableAll[(startIdx + step) % stableAll.length]!;
|
|
691
|
+
if (!candidates.includes(candidate)) continue;
|
|
692
|
+
if (!fallback) fallback = candidate;
|
|
693
|
+
if (isActiveUnderFillFirstThreshold(config, candidate, model)) return candidate;
|
|
694
|
+
}
|
|
695
|
+
return fallback ?? ordered[0] ?? null;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function pickAlternateAnthropicAccount(
|
|
699
|
+
config: OcxConfig,
|
|
700
|
+
excludeId: string,
|
|
701
|
+
now: number,
|
|
702
|
+
decision: AnthropicRouteDecision | null,
|
|
703
|
+
model?: string,
|
|
704
|
+
excludedAccountIds?: ReadonlySet<string>,
|
|
705
|
+
): string | null {
|
|
706
|
+
if (!admitted(config)) return null;
|
|
707
|
+
const strategy = anthropicPoolStrategy(config);
|
|
708
|
+
const eligible = routeCandidates(getEligibleAnthropicAccounts(now, model), decision).filter(id => id !== excludeId && !excludedAccountIds?.has(id));
|
|
709
|
+
if (strategy === "round-robin") {
|
|
710
|
+
return peekRoundRobinAccount(poolKey, eligible, stickyLimitForPool(config));
|
|
711
|
+
}
|
|
712
|
+
if (strategy === "fill-first") {
|
|
713
|
+
return pickNextFillFirstAnthropicAccount(config, excludeId, eligible, decision, model);
|
|
714
|
+
}
|
|
715
|
+
return pickLowestUsage(config, excludeId, now, decision, model, excludedAccountIds);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
function pruneExpiredAffinity(now: number): void {
|
|
719
|
+
for (const [key, entry] of sessionAffinity) {
|
|
720
|
+
if (now - entry.lastUsedAt > AFFINITY_IDLE_TTL_MS) sessionAffinity.delete(key);
|
|
721
|
+
}
|
|
722
|
+
if (sessionAffinity.size <= MAX_AFFINITY_ENTRIES) return;
|
|
723
|
+
const sorted = [...sessionAffinity.entries()].sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
|
|
724
|
+
const drop = sessionAffinity.size - MAX_AFFINITY_ENTRIES;
|
|
725
|
+
for (let i = 0; i < drop; i++) sessionAffinity.delete(sorted[i]![0]);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
function stickyLimitForPool(config: OcxConfig): number {
|
|
729
|
+
return normalizeAccountPoolStickyLimit(anthropicAccountPoolConfig(config).stickyLimit);
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function anthropicPoolStrategy(config: OcxConfig): OcxAccountPoolRotationStrategy {
|
|
733
|
+
return normalizeAccountPoolStrategy(anthropicAccountPoolConfig(config).strategy);
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
function isActiveUnderFillFirstThreshold(config: OcxConfig, accountId: string, model?: string): boolean {
|
|
737
|
+
const threshold = anthropicAccountAutoSwitchThreshold(config, accountId);
|
|
738
|
+
if (threshold <= 0) return true;
|
|
739
|
+
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
740
|
+
if (window === "weekly" && exhausted5h(accountId)) return false;
|
|
741
|
+
// Unknown usage must not force fill-first to abandon the active account.
|
|
742
|
+
if (!hasKnownUsage(config, accountId, model)) return true;
|
|
743
|
+
return usageScore(config, accountId, model) < threshold;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Fill-first: keep eligible active under threshold; otherwise advance to the next
|
|
748
|
+
* eligible id in stable sorted order after the current active (wrapping).
|
|
749
|
+
*/
|
|
750
|
+
function pickFillFirstAnthropicAccount(config: OcxConfig, now: number, decision: AnthropicRouteDecision | null, model?: string): string | null {
|
|
751
|
+
const eligible = routeCandidates(getEligibleAnthropicAccounts(now, model), decision);
|
|
752
|
+
if (eligible.length === 0) return null;
|
|
753
|
+
|
|
754
|
+
const set = getAccountSet(PROVIDER);
|
|
755
|
+
const active = set?.activeAccountId;
|
|
756
|
+
if (active && eligible.includes(active) && (!model || !anthropicModelExhausted(active, model)) && isActiveUnderFillFirstThreshold(config, active, model)) {
|
|
757
|
+
return active;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
if (!active || !set) {
|
|
761
|
+
const ordered = usesDeclaredRouteOrder(eligible, decision) ? eligible : [...eligible].sort((a, b) => a.localeCompare(b));
|
|
762
|
+
for (const id of ordered) {
|
|
763
|
+
if (isActiveUnderFillFirstThreshold(config, id, model)) return id;
|
|
764
|
+
}
|
|
765
|
+
return ordered[0] ?? null;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
return pickNextFillFirstAnthropicAccount(config, active, eligible, decision, model);
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* Unbound new-session pick for round-robin / fill-first. Returns null to fall through
|
|
773
|
+
* to the legacy quota path (or when the strategy is quota).
|
|
774
|
+
*/
|
|
775
|
+
function pickUnboundStrategyAccount(
|
|
776
|
+
config: OcxConfig,
|
|
777
|
+
now: number,
|
|
778
|
+
decision: AnthropicRouteDecision | null,
|
|
779
|
+
model?: string,
|
|
780
|
+
): { accountId: string; reason: "round-robin" | "fill-first" } | null {
|
|
781
|
+
const strategy = anthropicPoolStrategy(config);
|
|
782
|
+
if (strategy === "quota") return null;
|
|
783
|
+
|
|
784
|
+
if (strategy === "round-robin") {
|
|
785
|
+
const eligible = routeCandidates(getEligibleAnthropicAccounts(now, model), decision);
|
|
786
|
+
const limit = stickyLimitForPool(config);
|
|
787
|
+
const headroom = model ? eligible.filter(id => !anthropicModelExhausted(id, model)) : eligible;
|
|
788
|
+
const picked = peekRoundRobinAccount(poolKey, headroom.length ? headroom : eligible, limit);
|
|
789
|
+
if (!picked) return null;
|
|
790
|
+
return { accountId: picked, reason: "round-robin" };
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
if (strategy === "fill-first") {
|
|
794
|
+
const picked = pickFillFirstAnthropicAccount(config, now, decision, model);
|
|
795
|
+
if (!picked) return null;
|
|
796
|
+
return { accountId: picked, reason: "fill-first" };
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Resolve which Anthropic OAuth account should serve this session.
|
|
804
|
+
* When the pool is disabled, retain the active account unless the operator paused it.
|
|
805
|
+
*/
|
|
806
|
+
function resolveAnthropicAccountForSession(
|
|
807
|
+
sessionKey: string | null | undefined,
|
|
808
|
+
config: OcxConfig,
|
|
809
|
+
now = Date.now(),
|
|
810
|
+
decision: AnthropicRouteDecision | null = null,
|
|
811
|
+
model?: string,
|
|
812
|
+
): AnthropicAccountSelection {
|
|
813
|
+
if (!admitted(config)) return { accountId: null, reason: "none", routePosition: decision?.position };
|
|
814
|
+
pruneExpiredAffinity(now);
|
|
815
|
+
const set = getAccountSet(PROVIDER);
|
|
816
|
+
if (!set || set.accounts.length === 0) return { accountId: null, reason: "none", ...(decision ? { routePosition: decision.position } : {}) };
|
|
817
|
+
const scoped = decision && !decision.fallback
|
|
818
|
+
? set.accounts.filter(account => decision.accounts.includes(account.id)) : set.accounts;
|
|
819
|
+
if (scoped.length > 0 && scoped.every(account => account.paused === true)) {
|
|
820
|
+
return { accountId: null, reason: "paused", routePosition: decision?.position };
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
if (manualPreference === undefined) {
|
|
824
|
+
manualPreference = set.selectionRevision !== undefined
|
|
825
|
+
? { accountId: set.activeAccountId, revision: set.selectionRevision }
|
|
826
|
+
: null;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
const eligible = routeCandidates(getEligibleAnthropicAccounts(now, model), decision);
|
|
830
|
+
if (eligible.length === 0) {
|
|
831
|
+
// Pause, removal and reauthentication are not cooldown evidence. Classify only
|
|
832
|
+
// usable members of this strict route (or the ordinary pool after fallback widens).
|
|
833
|
+
const recoverable = set.accounts.filter(account =>
|
|
834
|
+
(!decision || decision.fallback || decision.accounts.includes(account.id))
|
|
835
|
+
&& account.paused !== true && account.needsReauth !== true && isPoolCredentialUsable(account.id, now));
|
|
836
|
+
const cooled = recoverable.length > 0 && recoverable.every(account => isCooled(account.id, now) || anthropicRatePauseUntil(account.id, now) || anthropicFamilyRejected(account.id, model, now));
|
|
837
|
+
if (cooled || decision) return { accountId: null, reason: cooled ? "all-cooled" : "none", routePosition: decision?.position };
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
if (!isAnthropicAccountPoolEnabled(config)) {
|
|
841
|
+
const active = set.accounts.find(account => account.id === set.activeAccountId);
|
|
842
|
+
// Disabled proactive rotation does not authorize a paused slot or an entirely cooled pool.
|
|
843
|
+
if (active?.paused || isCooled(set.activeAccountId, now) || anthropicRatePauseUntil(set.activeAccountId, now) || anthropicFamilyRejected(set.activeAccountId, model, now)) {
|
|
844
|
+
return { accountId: eligible[0] ?? null, reason: eligible.length > 0 ? "only-eligible" : "none" };
|
|
845
|
+
}
|
|
846
|
+
return { accountId: set.activeAccountId, reason: "pool-disabled" };
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
// A manual choice is a one-dispatch preference, not a lower-priority quota hint.
|
|
850
|
+
// Consume it only after admission commits, so a failed token lookup cannot spend it.
|
|
851
|
+
if (manualPreference) {
|
|
852
|
+
if (manualPreference.accountId !== set.activeAccountId || manualPreference.revision !== set.selectionRevision) {
|
|
853
|
+
manualPreference = null;
|
|
854
|
+
} else {
|
|
855
|
+
const chosen = manualPreference.accountId;
|
|
856
|
+
const quota = getCachedProviderAccountQuota(PROVIDER, chosen);
|
|
857
|
+
const exhausted = anthropicModelPercents(quota, model).some(percent => percent >= 100);
|
|
858
|
+
if (!exhausted && eligible.includes(chosen)) {
|
|
859
|
+
return { accountId: chosen, reason: "manual", routePosition: decision?.position };
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
const key = normalizeAffinityComponent(sessionKey);
|
|
865
|
+
if (key) {
|
|
866
|
+
const affined = sessionAffinity.get(key);
|
|
867
|
+
if (affined && now - affined.lastUsedAt <= AFFINITY_IDLE_TTL_MS) {
|
|
868
|
+
const stillThere = set.accounts.some(a => a.id === affined.accountId && a.paused !== true && a.needsReauth !== true);
|
|
869
|
+
const stillUsable = stillThere && !isCooled(affined.accountId, now)
|
|
870
|
+
&& isPoolCredentialUsable(affined.accountId, now);
|
|
871
|
+
if (stillUsable && eligible.includes(affined.accountId) && (!model || !anthropicModelExhausted(affined.accountId, model))) {
|
|
872
|
+
return { accountId: affined.accountId, reason: "affinity", routePosition: decision?.position };
|
|
873
|
+
}
|
|
874
|
+
// A model route may exclude a healthy binding only for this request. Keep it for
|
|
875
|
+
// another model; remove bindings only when the account itself became unusable.
|
|
876
|
+
if (!stillUsable) sessionAffinity.delete(key);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
const strategy = anthropicPoolStrategy(config);
|
|
881
|
+
// No session identity (Desktop turns without a sticky key): hold the current
|
|
882
|
+
// active under RR/fill-first instead of treating every turn as a new session.
|
|
883
|
+
// Round-robin only when there is a real new-session key (or active is unusable).
|
|
884
|
+
if (!key && (strategy === "round-robin" || strategy === "fill-first")) {
|
|
885
|
+
const activeOk = set.accounts.some(a => a.id === set.activeAccountId && a.needsReauth !== true)
|
|
886
|
+
&& !isCooled(set.activeAccountId, now)
|
|
887
|
+
&& eligible.includes(set.activeAccountId) && (!model || !anthropicModelExhausted(set.activeAccountId, model));
|
|
888
|
+
if (activeOk) {
|
|
889
|
+
return { accountId: set.activeAccountId, reason: "active", routePosition: decision?.position };
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
const strategyPick = pickUnboundStrategyAccount(config, now, decision, model);
|
|
894
|
+
if (strategyPick) {
|
|
895
|
+
return { accountId: strategyPick.accountId, reason: strategyPick.reason, routePosition: decision?.position };
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
const threshold = anthropicAccountAutoSwitchThreshold(config, set.activeAccountId);
|
|
899
|
+
const activeOk = set.accounts.some(a => a.id === set.activeAccountId && a.needsReauth !== true)
|
|
900
|
+
&& !isCooled(set.activeAccountId, now)
|
|
901
|
+
&& eligible.includes(set.activeAccountId) && (!model || !anthropicModelExhausted(set.activeAccountId, model));
|
|
902
|
+
|
|
903
|
+
let accountId: string | null = null;
|
|
904
|
+
let reason: AnthropicAccountSelectionReason = "none";
|
|
905
|
+
|
|
906
|
+
if (threshold > 0) {
|
|
907
|
+
const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config));
|
|
908
|
+
// Unknown usage must NOT force a switch away from the healthy active account.
|
|
909
|
+
if (activeOk
|
|
910
|
+
&& !(window === "weekly" && exhausted5h(set.activeAccountId))
|
|
911
|
+
&& (!hasKnownUsage(config, set.activeAccountId, model) || usageScore(config, set.activeAccountId, model) < threshold)) {
|
|
829
912
|
accountId = set.activeAccountId;
|
|
830
913
|
reason = "active";
|
|
914
|
+
} else {
|
|
915
|
+
const picked = pickLowestUsage(config, undefined, now, decision, model);
|
|
916
|
+
if (picked) {
|
|
917
|
+
accountId = picked;
|
|
918
|
+
reason = activeOk && picked === set.activeAccountId ? "active" : "lowest-usage";
|
|
919
|
+
} else if (activeOk) {
|
|
920
|
+
accountId = set.activeAccountId;
|
|
921
|
+
reason = "active";
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
} else if (activeOk) {
|
|
925
|
+
accountId = set.activeAccountId;
|
|
926
|
+
reason = "active";
|
|
927
|
+
} else {
|
|
928
|
+
const picked = pickLowestUsage(config, set.activeAccountId, now, decision, model);
|
|
929
|
+
if (picked) {
|
|
930
|
+
accountId = picked;
|
|
931
|
+
reason = "only-eligible";
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
if (!accountId) {
|
|
936
|
+
const anyCooled = set.accounts.some(a => !a.paused && (!decision || decision.accounts.includes(a.id)) && isCooled(a.id, now));
|
|
937
|
+
return { accountId: null, reason: anyCooled ? "all-cooled" : "none", routePosition: decision?.position };
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
return { accountId, reason, routePosition: decision?.position };
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/** Shared local refusal policy for Responses and native Messages after asynchronous waits. */
|
|
944
|
+
async function resolveAnthropicDispatchAccountId(
|
|
945
|
+
config: OcxConfig,
|
|
946
|
+
sessionKey: string | null = null,
|
|
947
|
+
decision: AnthropicRouteDecision | null = null,
|
|
948
|
+
model?: string,
|
|
949
|
+
): Promise<string> {
|
|
950
|
+
const now = Date.now();
|
|
951
|
+
const selection = resolveAnthropicAccountForSession(sessionKey, config, now, decision, model);
|
|
952
|
+
if (selection.reason === "all-cooled") {
|
|
953
|
+
throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, decision, model), decision?.position);
|
|
954
|
+
}
|
|
955
|
+
if (selection.reason === "paused" || !selection.accountId || !getEligibleAnthropicAccounts(now, model).includes(selection.accountId)) {
|
|
956
|
+
const { OAuthAccountPausedError, OAuthLoginRequiredError } = await import("./index");
|
|
957
|
+
const active = getAccountSet(PROVIDER)?.activeAccountId;
|
|
958
|
+
// Pool-off fresh admission resolves the active credential first. Preserve that
|
|
959
|
+
// paused-active refusal when no usable survivor could take over from it.
|
|
960
|
+
if (selection.reason === "paused" || (!isAnthropicAccountPoolEnabled(config) && active
|
|
961
|
+
&& getAccountCredentialWithStatus(PROVIDER, active)?.paused)) throw new OAuthAccountPausedError();
|
|
962
|
+
throw new OAuthLoginRequiredError(PROVIDER);
|
|
963
|
+
}
|
|
964
|
+
return selection.accountId;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
function bindAnthropicSessionAffinity(
|
|
968
|
+
sessionKey: string | null | undefined,
|
|
969
|
+
accountId: string,
|
|
970
|
+
now = Date.now(),
|
|
971
|
+
): void {
|
|
972
|
+
const key = normalizeAffinityComponent(sessionKey);
|
|
973
|
+
if (!key || !normalizeAffinityComponent(accountId)) return;
|
|
974
|
+
sessionAffinity.set(key, { accountId, lastUsedAt: now });
|
|
975
|
+
pruneExpiredAffinity(now);
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
function clearAnthropicSessionAffinityForAccount(accountId: string): void {
|
|
979
|
+
for (const [key, entry] of sessionAffinity) {
|
|
980
|
+
if (entry.accountId === accountId) sessionAffinity.delete(key);
|
|
981
|
+
}
|
|
982
|
+
// The roster just lost or changed a member. This is the account-removal path, so the next
|
|
983
|
+
// activation question must re-read rather than answer from a count taken while the account
|
|
984
|
+
// was still present -- otherwise a delete leaves a stale quorum for the length of the TTL.
|
|
985
|
+
quorumCache = null;
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* Record a 429 for `failedAccountId`, cool it, clear its affinity, and pick a failover
|
|
990
|
+
* account. Does NOT promote the store active account — caller should promote only after a
|
|
991
|
+
* successful retry (or token resolve).
|
|
992
|
+
*/
|
|
993
|
+
function rotateAnthropicAccountOn429(
|
|
994
|
+
config: OcxConfig,
|
|
995
|
+
failedAccountId: string,
|
|
996
|
+
retryAfterHeader: string | null | undefined,
|
|
997
|
+
sessionKey?: string | null,
|
|
998
|
+
now = Date.now(),
|
|
999
|
+
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
1000
|
+
decision: AnthropicRouteDecision | null = null,
|
|
1001
|
+
model?: string,
|
|
1002
|
+
): string | null {
|
|
1003
|
+
return rotateAnthropicAccountOnRefusal(config, failedAccountId, 429, retryAfterHeader, sessionKey, now, rateLimitHeaders, decision, model);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
/** The caller proves that a 403 is an account refusal before entering this pool policy. */
|
|
1007
|
+
function rotateAnthropicAccountOnRefusal(
|
|
1008
|
+
config: OcxConfig,
|
|
1009
|
+
failedAccountId: string,
|
|
1010
|
+
status: 429 | 403,
|
|
1011
|
+
retryAfterHeader: string | null | undefined,
|
|
1012
|
+
sessionKey?: string | null,
|
|
1013
|
+
now = Date.now(),
|
|
1014
|
+
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
1015
|
+
decision: AnthropicRouteDecision | null = null,
|
|
1016
|
+
model?: string,
|
|
1017
|
+
excludedAccountIds?: ReadonlySet<string>,
|
|
1018
|
+
): string | null {
|
|
1019
|
+
if (!recordAnthropicAccountRefusal(config, failedAccountId, status, retryAfterHeader, now, rateLimitHeaders)) return null;
|
|
1020
|
+
|
|
1021
|
+
// The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive
|
|
1022
|
+
// presence-only recovery must not silently reactivate round-robin/fill-first merely
|
|
1023
|
+
// because those dormant values remain in config. The quota picker is the neutral
|
|
1024
|
+
// recovery policy already used by the default strategy.
|
|
1025
|
+
const next = isAnthropicAccountPoolEnabled(config)
|
|
1026
|
+
? pickAlternateAnthropicAccount(config, failedAccountId, now, decision, model, excludedAccountIds)
|
|
1027
|
+
: pickLowestUsage(config, failedAccountId, now, null, model, excludedAccountIds);
|
|
1028
|
+
if (!next) {
|
|
1029
|
+
console.warn(`[${instance}-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning ${status}`);
|
|
1030
|
+
return null;
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
console.warn(status === 403
|
|
1034
|
+
? `[${instance}-pool] ${decision ? `route:#${decision.position} ` : ""}account unavailable (403); failing over`
|
|
1035
|
+
: `[${instance}-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`);
|
|
1036
|
+
|
|
1037
|
+
return next;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/** Record the account refusal even when this request has no remaining retry sends. */
|
|
1041
|
+
function recordAnthropicAccount429(
|
|
1042
|
+
config: OcxConfig,
|
|
1043
|
+
failedAccountId: string,
|
|
1044
|
+
retryAfterHeader: string | null | undefined,
|
|
1045
|
+
now = Date.now(),
|
|
1046
|
+
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
1047
|
+
): boolean {
|
|
1048
|
+
return recordAnthropicAccountRefusal(config, failedAccountId, 429, retryAfterHeader, now, rateLimitHeaders);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/** Account entitlement refusals use a finite backoff; renewing a plan does not require login. */
|
|
1052
|
+
function recordAnthropicAccountRefusal(
|
|
1053
|
+
config: OcxConfig,
|
|
1054
|
+
failedAccountId: string,
|
|
1055
|
+
status: 429 | 403,
|
|
1056
|
+
retryAfterHeader: string | null | undefined,
|
|
1057
|
+
now = Date.now(),
|
|
1058
|
+
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
1059
|
+
): boolean {
|
|
1060
|
+
if (!admitted(config)) return false;
|
|
1061
|
+
// Reactive 429 failover is NOT gated on the pool flag. That flag buys PROACTIVE routing --
|
|
1062
|
+
// session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all
|
|
1063
|
+
// of which move a HEALTHY request and stay opt-in. Rotating away from an account upstream has
|
|
1064
|
+
// just rate-limited is a different thing: it only ever runs after a refusal, and stranding a
|
|
1065
|
+
// 429 while a second logged-in account sits idle is a defect, not a configuration choice.
|
|
1066
|
+
// Presence is the activation rule, the same one an apiKeyPool of two keys already uses.
|
|
1067
|
+
// A sent turn may finish after its account is paused; its one remaining successor is
|
|
1068
|
+
// still a valid reactive recovery even though the current unpaused quorum is now one.
|
|
1069
|
+
if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)
|
|
1070
|
+
&& !getAccountCredentialWithStatus(PROVIDER, failedAccountId)?.paused) return false;
|
|
1071
|
+
|
|
1072
|
+
if (status === 429) {
|
|
1073
|
+
const headers = rateLimitHeaders ?? new Headers();
|
|
1074
|
+
const kind = classifyAnthropic429({ get: name => name === "retry-after" ? retryAfterHeader ?? null : headers.get(name) }, now);
|
|
1075
|
+
if (kind !== "shared-quota") {
|
|
1076
|
+
if (kind === "transient-rate") pauseAnthropicRateAdmission(failedAccountId, now + (anthropicRetryAfterMs(retryAfterHeader, now) ?? 100));
|
|
1077
|
+
return false;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
// Retry-After first: it is the header written FOR this decision. The rejected window's
|
|
1082
|
+
// reset is the fallback, because a 429 that omits Retry-After still carries it -- and
|
|
1083
|
+
// without that fallback such a refusal cools for the 60s default and the exhausted
|
|
1084
|
+
// account is back in the rotation a minute later.
|
|
1085
|
+
const parsedRetry = parseRetryAfterMs(retryAfterHeader, now);
|
|
1086
|
+
const resetDerived = status === 429 && parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined;
|
|
1087
|
+
const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? (status === 403 ? ACCOUNT_REFUSAL_COOLDOWN_MS : DEFAULT_COOLDOWN_MS);
|
|
1088
|
+
const cooldownGeneration = noteAnthropicCooldownMutation(failedAccountId);
|
|
1089
|
+
upstreamHealth.set(failedAccountId, {
|
|
1090
|
+
cooldownUntil: now + cooldownMs,
|
|
1091
|
+
cooldownSource: parsedRetry !== undefined
|
|
1092
|
+
? "retry-after"
|
|
1093
|
+
: resetDerived !== undefined ? "reset-derived" : "default",
|
|
1094
|
+
...(resetDerived ? { rejectedQuotaWindows: resetDerived.rejectedQuotaWindows } : {}),
|
|
1095
|
+
cooldownGeneration,
|
|
1096
|
+
});
|
|
1097
|
+
sweepExpiredOnWrite(now);
|
|
1098
|
+
clearAnthropicSessionAffinityForAccount(failedAccountId);
|
|
1099
|
+
notePoolRotationFailure(poolKey, failedAccountId);
|
|
1100
|
+
// The refused account changed the eligible roster even when no retry send remains.
|
|
1101
|
+
quorumCache = null;
|
|
1102
|
+
|
|
1103
|
+
return true;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
/** Commit the selected account before dispatch; rejected proposals have no routing side effects. */
|
|
1107
|
+
async function promoteAnthropicActiveAccount(
|
|
1108
|
+
accountId: string,
|
|
1109
|
+
expectedSelection: OAuthAccountSelection | null,
|
|
1110
|
+
options: AnthropicSelectionRoutingOptions,
|
|
1111
|
+
): Promise<OAuthAccountSelection | null> {
|
|
1112
|
+
if (!admitted(options.config)) return null;
|
|
1113
|
+
if (!expectedSelection || !routeCandidates(getEligibleAnthropicAccounts(Date.now(), options.model), options.routeDecision ?? null).includes(accountId)) return null;
|
|
1114
|
+
const committed = await commitOAuthAccountSelection(PROVIDER, accountId, {
|
|
1115
|
+
expectedSelection,
|
|
1116
|
+
expectedCredentialGeneration: options.expectedCredentialGeneration,
|
|
1117
|
+
requireUsableAccount: true,
|
|
1118
|
+
});
|
|
1119
|
+
if (!committed) return null;
|
|
1120
|
+
return commitAnthropicSelectionRouting(accountId, expectedSelection, committed, options) ? committed : null;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/** Main's shared selection owner calls this only after its authoritative commit succeeds. */
|
|
1124
|
+
function commitAnthropicSelectionRouting(
|
|
1125
|
+
accountId: string,
|
|
1126
|
+
expectedSelection: OAuthAccountSelection,
|
|
1127
|
+
committed: OAuthAccountSelection,
|
|
1128
|
+
options: AnthropicSelectionRoutingOptions,
|
|
1129
|
+
): boolean {
|
|
1130
|
+
if (!admitted(options.config)) return false;
|
|
1131
|
+
if (getAccountCredentialWithStatus(PROVIDER, accountId)?.paused || committed.accountId !== accountId || (options.routeDecision
|
|
1132
|
+
&& !routeCandidates(getEligibleAnthropicAccounts(Date.now(), options.model), options.routeDecision).includes(accountId))) return false;
|
|
1133
|
+
const current = captureOAuthAccountSelection(PROVIDER);
|
|
1134
|
+
if (current?.accountId !== committed.accountId || current.revision !== committed.revision) return false;
|
|
1135
|
+
if (isAnthropicAccountPoolEnabled(options.config)) {
|
|
1136
|
+
if (anthropicPoolStrategy(options.config) === "round-robin" && options.reason !== "affinity") {
|
|
1137
|
+
const limit = stickyLimitForPool(options.config);
|
|
1138
|
+
const picked = pickRoundRobinAccount(poolKey, routeCandidates(getEligibleAnthropicAccounts(Date.now(), options.model), options.routeDecision ?? null), limit);
|
|
1139
|
+
if (picked !== accountId) seedPoolRotationAccount(poolKey, accountId);
|
|
1140
|
+
notePoolRotationSuccess(poolKey, accountId, limit);
|
|
831
1141
|
}
|
|
1142
|
+
const key = normalizeAffinityComponent(options.sessionKey);
|
|
1143
|
+
const bound = key ? sessionAffinity.get(key) : undefined;
|
|
1144
|
+
const eligibleAtCommit = options.routeDecision && bound ? getEligibleAnthropicAccounts() : [];
|
|
1145
|
+
const preserveExcludedAffinity = options.routeDecision && bound && bound.accountId !== accountId
|
|
1146
|
+
&& Date.now() - bound.lastUsedAt <= AFFINITY_IDLE_TTL_MS
|
|
1147
|
+
&& eligibleAtCommit.includes(bound.accountId)
|
|
1148
|
+
&& !routeCandidates(eligibleAtCommit, options.routeDecision).includes(bound.accountId);
|
|
1149
|
+
const preservePausedAffinity = bound && (anthropicRatePauseUntil(bound.accountId) !== undefined
|
|
1150
|
+
|| anthropicFamilyRejected(bound.accountId, options.model));
|
|
1151
|
+
if (!preserveExcludedAffinity && !preservePausedAffinity) bindAnthropicSessionAffinity(options.sessionKey, accountId);
|
|
1152
|
+
}
|
|
1153
|
+
if (manualPreference === undefined || (manualPreference?.accountId === expectedSelection.accountId
|
|
1154
|
+
&& manualPreference.revision === expectedSelection.revision)) manualPreference = null;
|
|
1155
|
+
return true;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
/**
|
|
1159
|
+
* Manual selection resets session affinity and seeds the RR ring so the next
|
|
1160
|
+
* unbound new session honors the operator-chosen account (Codex parity).
|
|
1161
|
+
*/
|
|
1162
|
+
function resetAnthropicRoutingForManualSelection(accountId: string): void {
|
|
1163
|
+
sessionAffinity.clear();
|
|
1164
|
+
manualSelectionGeneration++;
|
|
1165
|
+
manualPreference = captureOAuthAccountSelection(PROVIDER);
|
|
1166
|
+
seedPoolRotationAccount(poolKey, accountId);
|
|
1167
|
+
// A manual account selection is an operator statement about the roster; do not answer the
|
|
1168
|
+
// next activation question from a count read before it.
|
|
1169
|
+
quorumCache = null;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
/**
|
|
1173
|
+
* Resolve a bearer for pool traffic without adopting a newer global Claude CLI
|
|
1174
|
+
* credential into a background multiauth `local-cli` slot (same fail-closed rule
|
|
1175
|
+
* as quota probes).
|
|
1176
|
+
*/
|
|
1177
|
+
async function getAnthropicPoolAccessToken(accountId: string): Promise<string> {
|
|
1178
|
+
const row = getAccountCredentialWithStatus(PROVIDER, accountId);
|
|
1179
|
+
if (row?.paused) {
|
|
1180
|
+
const { OAuthAccountPausedError } = await import("./index");
|
|
1181
|
+
throw new OAuthAccountPausedError();
|
|
1182
|
+
}
|
|
1183
|
+
const stored = row?.credential;
|
|
1184
|
+
if (!stored) {
|
|
1185
|
+
const { OAuthLoginRequiredError } = await import("./index");
|
|
1186
|
+
throw new OAuthLoginRequiredError(PROVIDER);
|
|
1187
|
+
}
|
|
1188
|
+
if (instance === "anthropic2" && stored.source === "local-cli") {
|
|
1189
|
+
const { OAuthLoginRequiredError } = await import("./index");
|
|
1190
|
+
throw new OAuthLoginRequiredError(instance);
|
|
1191
|
+
}
|
|
1192
|
+
if (stored.expires > Date.now() + REFRESH_SKEW_MS) return stored.access;
|
|
1193
|
+
if (!canRefreshAnthropicPoolAccount(accountId)) {
|
|
1194
|
+
throw new Error("background local-cli token expired; refuse CLI-adopting refresh for pool");
|
|
832
1195
|
}
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
1196
|
+
const { getValidAccessTokenForAccount } = await import("./index");
|
|
1197
|
+
return getValidAccessTokenForAccount(PROVIDER, accountId);
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
/** Resolve through the pool's local-CLI restrictions, then bind the exact returned generation. */
|
|
1201
|
+
async function getAnthropicPoolAccessSnapshot(accountId: string): Promise<OAuthAccessSnapshot> {
|
|
1202
|
+
const accessToken = await getAnthropicPoolAccessToken(accountId);
|
|
1203
|
+
const row = getAccountCredentialWithStatus(PROVIDER, accountId);
|
|
1204
|
+
if (row?.paused) {
|
|
1205
|
+
const { OAuthAccountPausedError } = await import("./index");
|
|
1206
|
+
throw new OAuthAccountPausedError();
|
|
1207
|
+
}
|
|
1208
|
+
if (!row || row.needsReauth || row.credential.access !== accessToken
|
|
1209
|
+
|| row.credential.expires <= Date.now()) {
|
|
1210
|
+
throw new Error("Anthropic pool credential changed during account selection");
|
|
841
1211
|
}
|
|
1212
|
+
return { provider: PROVIDER, accountId, accessToken, generation: credentialGeneration(row.credential) };
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
/**
|
|
1216
|
+
* Whether the pool may refresh this account's token. Background `local-cli` slots must not
|
|
1217
|
+
* adopt the global Claude CLI credential (same fail-closed rule as quota probes).
|
|
1218
|
+
*/
|
|
1219
|
+
function canRefreshAnthropicPoolAccount(accountId: string): boolean {
|
|
1220
|
+
const set = getAccountSet(PROVIDER);
|
|
1221
|
+
if (set?.accounts.find(account => account.id === accountId)?.paused) return false;
|
|
1222
|
+
const cred = getAccountCredential(PROVIDER, accountId);
|
|
1223
|
+
if (!cred) return false;
|
|
1224
|
+
if (instance === "anthropic2" && cred.source === "local-cli") return false;
|
|
1225
|
+
if (cred.source !== "local-cli") return true;
|
|
1226
|
+
return set?.activeAccountId === accountId;
|
|
842
1227
|
}
|
|
843
1228
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1229
|
+
// Pause changes eligibility, not health. Do not reset cooldowns or cancel sent turns.
|
|
1230
|
+
function onPause(): void { quorumCache = null; }
|
|
1231
|
+
// A threshold write must fence in-flight automatic proposals, but it does not
|
|
1232
|
+
// revoke an operator's one-dispatch choice. Rebase only that still-owned choice;
|
|
1233
|
+
// an intervening account change clears it, so an ABA selection is not resurrected.
|
|
1234
|
+
function onPolicy(event: Parameters<Parameters<typeof subscribeOAuthAccountRoutingPolicyChanges>[0]>[0]): void {
|
|
1235
|
+
if (event.provider !== PROVIDER || !manualPreference) return;
|
|
1236
|
+
if (manualPreference.accountId !== event.before.accountId
|
|
1237
|
+
|| manualPreference.revision !== event.before.revision) return;
|
|
1238
|
+
manualPreference = event.after.accountId === manualPreference.accountId ? { ...event.after } : null;
|
|
847
1239
|
}
|
|
1240
|
+
// Any non-policy selection generation supersedes the pending one-shot choice.
|
|
1241
|
+
// Threshold mutations rebase it first, before this generic notification runs.
|
|
1242
|
+
function onSelection(event: Parameters<Parameters<typeof subscribeAccountSelections>[0]>[0]): void {
|
|
1243
|
+
if (event.provider !== PROVIDER || event.kind !== "oauth" || !manualPreference) return;
|
|
1244
|
+
const current = captureOAuthAccountSelection(PROVIDER);
|
|
1245
|
+
if (current?.accountId !== manualPreference.accountId
|
|
1246
|
+
|| current.revision !== manualPreference.revision) manualPreference = null;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
848
1249
|
|
|
849
|
-
|
|
1250
|
+
function reconcile(context: GenerationContext, config?: OcxConfig): number {
|
|
1251
|
+
let removed = 0;
|
|
1252
|
+
const valid = (id: string) => context.oauthAccountKeys.has(`${instance}\0${id}`)
|
|
1253
|
+
&& (!config || admitted(config));
|
|
1254
|
+
for (const [id] of upstreamHealth) {
|
|
1255
|
+
if (valid(id)) continue;
|
|
1256
|
+
noteAnthropicCooldownMutation(id);
|
|
1257
|
+
upstreamHealth.delete(id);
|
|
1258
|
+
removed++;
|
|
1259
|
+
}
|
|
1260
|
+
for (const [key, entry] of sessionAffinity) {
|
|
1261
|
+
if (valid(entry.accountId)) continue;
|
|
1262
|
+
sessionAffinity.delete(key);
|
|
1263
|
+
removed++;
|
|
1264
|
+
}
|
|
1265
|
+
if (manualPreference && !valid(manualPreference.accountId)) { manualPreference = null; removed++; }
|
|
1266
|
+
if (removed > 0) manualSelectionGeneration++;
|
|
1267
|
+
quorumCache = null;
|
|
1268
|
+
return removed;
|
|
1269
|
+
}
|
|
1270
|
+
handlers.set(instance, { onPause, onPolicy, onSelection, reconcile });
|
|
1271
|
+
return Object.freeze({
|
|
1272
|
+
instance, captureAnthropicManualSelectionGeneration, anthropicAccountPoolConfig,
|
|
1273
|
+
isAnthropicAccountPoolEnabled, anthropicAutoSwitchThreshold, anthropicAccountAutoSwitchThreshold,
|
|
1274
|
+
captureAnthropicCooldownRecovery, settleAnthropicCooldownRecovery, getAnthropicAccountHealthSnapshot,
|
|
1275
|
+
clearAnthropicAccountCooldown, sweepExpiredAnthropicRoutingHealth, clearAnthropicAccountPoolState,
|
|
1276
|
+
anthropicSessionAffinitySizeForTests, getEligibleAnthropicAccounts, hasAnthropicFailoverQuorum,
|
|
1277
|
+
forgetAnthropicFailoverQuorum, getAnthropicPoolRetryAfterSeconds, pickAlternateAnthropicAccount,
|
|
1278
|
+
resolveAnthropicAccountForSession, resolveAnthropicDispatchAccountId, bindAnthropicSessionAffinity,
|
|
1279
|
+
clearAnthropicSessionAffinityForAccount, rotateAnthropicAccountOn429, rotateAnthropicAccountOnRefusal,
|
|
1280
|
+
recordAnthropicAccount429, recordAnthropicAccountRefusal, promoteAnthropicActiveAccount,
|
|
1281
|
+
commitAnthropicSelectionRouting, resetAnthropicRoutingForManualSelection, getAnthropicPoolAccessToken,
|
|
1282
|
+
getAnthropicPoolAccessSnapshot, canRefreshAnthropicPoolAccount,
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
export function anthropicRoutingFor(instance: AnthropicInstanceId): AnthropicRouting {
|
|
1286
|
+
let routing = instances.get(instance);
|
|
1287
|
+
if (!routing) { routing = createAnthropicRouting(instance); instances.set(instance, routing); }
|
|
1288
|
+
return routing;
|
|
1289
|
+
}
|
|
1290
|
+
export function sweepAllAnthropicRoutingHealth(now = Date.now()): number {
|
|
1291
|
+
let removed = 0;
|
|
1292
|
+
for (const routing of instances.values()) removed += routing.sweepExpiredAnthropicRoutingHealth(now);
|
|
1293
|
+
return removed;
|
|
850
1294
|
}
|
|
1295
|
+
export function reconcileAnthropicRoutingState(context: GenerationContext, config?: OcxConfig): number {
|
|
1296
|
+
let removed = 0;
|
|
1297
|
+
for (const handler of handlers.values()) removed += handler.reconcile(context, config);
|
|
1298
|
+
return removed;
|
|
1299
|
+
}
|
|
1300
|
+
export function clearAllAnthropicAccountPoolState(): void {
|
|
1301
|
+
for (const routing of instances.values()) routing.clearAnthropicAccountPoolState();
|
|
1302
|
+
clearAllAnthropicFamilyQuota();
|
|
1303
|
+
clearAllAnthropicRatePauses();
|
|
1304
|
+
clearAllAnthropicCooldownGenerations();
|
|
1305
|
+
}
|
|
1306
|
+
subscribeOAuthAccountPauseChanges(provider => { if (isAnthropicInstanceId(provider)) handlers.get(provider)?.onPause(); });
|
|
1307
|
+
subscribeOAuthAccountRoutingPolicyChanges(event => { if (isAnthropicInstanceId(event.provider)) handlers.get(event.provider)?.onPolicy(event); });
|
|
1308
|
+
subscribeAccountSelections(event => { if (isAnthropicInstanceId(event.provider)) handlers.get(event.provider)?.onSelection(event); });
|
|
851
1309
|
|
|
852
|
-
|
|
853
|
-
|
|
1310
|
+
export function captureAnthropicManualSelectionGeneration(): number {
|
|
1311
|
+
return anthropicRoutingFor("anthropic").captureAnthropicManualSelectionGeneration();
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
export function anthropicAccountPoolConfig(config: OcxConfig): AnthropicAccountPoolConfig {
|
|
1315
|
+
return anthropicRoutingFor("anthropic").anthropicAccountPoolConfig(config);
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
export function isAnthropicAccountPoolEnabled(config: OcxConfig): boolean {
|
|
1319
|
+
return anthropicRoutingFor("anthropic").isAnthropicAccountPoolEnabled(config);
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
export function anthropicAutoSwitchThreshold(config: OcxConfig): number {
|
|
1323
|
+
return anthropicRoutingFor("anthropic").anthropicAutoSwitchThreshold(config);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
export function anthropicAccountAutoSwitchThreshold(config: OcxConfig, accountId: string): number {
|
|
1327
|
+
return anthropicRoutingFor("anthropic").anthropicAccountAutoSwitchThreshold(config, accountId);
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
export function captureAnthropicCooldownRecovery(
|
|
1331
|
+
accountId: string,
|
|
1332
|
+
now = Date.now(),
|
|
1333
|
+
): AnthropicCooldownRecoveryClaim | null {
|
|
1334
|
+
return anthropicRoutingFor("anthropic").captureAnthropicCooldownRecovery(accountId, now);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
export function settleAnthropicCooldownRecovery(
|
|
1338
|
+
claim: AnthropicCooldownRecoveryClaim,
|
|
1339
|
+
quota: ProviderQuota,
|
|
1340
|
+
): AnthropicCooldownRecoverySettlement {
|
|
1341
|
+
return anthropicRoutingFor("anthropic").settleAnthropicCooldownRecovery(claim, quota);
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
export function getAnthropicAccountHealthSnapshot(
|
|
1345
|
+
accountId: string,
|
|
1346
|
+
now = Date.now(),
|
|
1347
|
+
): { cooldownUntil?: number; cooldownSource?: AccountHealth["cooldownSource"] } | null {
|
|
1348
|
+
return anthropicRoutingFor("anthropic").getAnthropicAccountHealthSnapshot(accountId, now);
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
export function clearAnthropicAccountCooldown(accountId: string): boolean {
|
|
1352
|
+
return anthropicRoutingFor("anthropic").clearAnthropicAccountCooldown(accountId);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number {
|
|
1356
|
+
return anthropicRoutingFor("anthropic").sweepExpiredAnthropicRoutingHealth(now);
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
export function clearAnthropicAccountPoolState(): void {
|
|
1360
|
+
return anthropicRoutingFor("anthropic").clearAnthropicAccountPoolState();
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
export function anthropicSessionAffinitySizeForTests(): number {
|
|
1364
|
+
return anthropicRoutingFor("anthropic").anthropicSessionAffinitySizeForTests();
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
export function getEligibleAnthropicAccounts(now = Date.now(), model?: string): string[] {
|
|
1368
|
+
return anthropicRoutingFor("anthropic").getEligibleAnthropicAccounts(now, model);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
export function hasAnthropicFailoverQuorum(now = Date.now()): boolean {
|
|
1372
|
+
return anthropicRoutingFor("anthropic").hasAnthropicFailoverQuorum(now);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
export function forgetAnthropicFailoverQuorum(): void {
|
|
1376
|
+
return anthropicRoutingFor("anthropic").forgetAnthropicFailoverQuorum();
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
export function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): number | null {
|
|
1380
|
+
return anthropicRoutingFor("anthropic").getAnthropicPoolRetryAfterSeconds(now, decision, model);
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
export function pickAlternateAnthropicAccount(
|
|
1384
|
+
config: OcxConfig,
|
|
1385
|
+
excludeId: string,
|
|
1386
|
+
now: number,
|
|
1387
|
+
decision: AnthropicRouteDecision | null,
|
|
1388
|
+
model?: string,
|
|
1389
|
+
excludedAccountIds?: ReadonlySet<string>,
|
|
1390
|
+
): string | null {
|
|
1391
|
+
return anthropicRoutingFor("anthropic").pickAlternateAnthropicAccount(config, excludeId, now, decision, model, excludedAccountIds);
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
export function resolveAnthropicAccountForSession(
|
|
1395
|
+
sessionKey: string | null | undefined,
|
|
1396
|
+
config: OcxConfig,
|
|
1397
|
+
now = Date.now(),
|
|
1398
|
+
decision: AnthropicRouteDecision | null = null,
|
|
1399
|
+
model?: string,
|
|
1400
|
+
): AnthropicAccountSelection {
|
|
1401
|
+
return anthropicRoutingFor("anthropic").resolveAnthropicAccountForSession(sessionKey, config, now, decision, model);
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
export function resolveAnthropicDispatchAccountId(
|
|
854
1405
|
config: OcxConfig,
|
|
855
1406
|
sessionKey: string | null = null,
|
|
856
1407
|
decision: AnthropicRouteDecision | null = null,
|
|
857
1408
|
model?: string,
|
|
858
1409
|
): Promise<string> {
|
|
859
|
-
|
|
860
|
-
const selection = resolveAnthropicAccountForSession(sessionKey, config, now, decision, model);
|
|
861
|
-
if (selection.reason === "all-cooled") {
|
|
862
|
-
throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, decision, model), decision?.position);
|
|
863
|
-
}
|
|
864
|
-
if (selection.reason === "paused" || !selection.accountId || !getEligibleAnthropicAccounts(now, model).includes(selection.accountId)) {
|
|
865
|
-
const { OAuthAccountPausedError, OAuthLoginRequiredError } = await import("./index");
|
|
866
|
-
const active = getAccountSet(PROVIDER)?.activeAccountId;
|
|
867
|
-
// Pool-off fresh admission resolves the active credential first. Preserve that
|
|
868
|
-
// paused-active refusal when no usable survivor could take over from it.
|
|
869
|
-
if (selection.reason === "paused" || (!isAnthropicAccountPoolEnabled(config) && active
|
|
870
|
-
&& getAccountCredentialWithStatus(PROVIDER, active)?.paused)) throw new OAuthAccountPausedError();
|
|
871
|
-
throw new OAuthLoginRequiredError(PROVIDER);
|
|
872
|
-
}
|
|
873
|
-
return selection.accountId;
|
|
1410
|
+
return anthropicRoutingFor("anthropic").resolveAnthropicDispatchAccountId(config, sessionKey, decision, model);
|
|
874
1411
|
}
|
|
875
1412
|
|
|
876
1413
|
export function bindAnthropicSessionAffinity(
|
|
@@ -878,27 +1415,13 @@ export function bindAnthropicSessionAffinity(
|
|
|
878
1415
|
accountId: string,
|
|
879
1416
|
now = Date.now(),
|
|
880
1417
|
): void {
|
|
881
|
-
|
|
882
|
-
if (!key || !normalizeAffinityComponent(accountId)) return;
|
|
883
|
-
sessionAffinity.set(key, { accountId, lastUsedAt: now });
|
|
884
|
-
pruneExpiredAffinity(now);
|
|
1418
|
+
return anthropicRoutingFor("anthropic").bindAnthropicSessionAffinity(sessionKey, accountId, now);
|
|
885
1419
|
}
|
|
886
1420
|
|
|
887
1421
|
export function clearAnthropicSessionAffinityForAccount(accountId: string): void {
|
|
888
|
-
|
|
889
|
-
if (entry.accountId === accountId) sessionAffinity.delete(key);
|
|
890
|
-
}
|
|
891
|
-
// The roster just lost or changed a member. This is the account-removal path, so the next
|
|
892
|
-
// activation question must re-read rather than answer from a count taken while the account
|
|
893
|
-
// was still present -- otherwise a delete leaves a stale quorum for the length of the TTL.
|
|
894
|
-
quorumCache = null;
|
|
1422
|
+
return anthropicRoutingFor("anthropic").clearAnthropicSessionAffinityForAccount(accountId);
|
|
895
1423
|
}
|
|
896
1424
|
|
|
897
|
-
/**
|
|
898
|
-
* Record a 429 for `failedAccountId`, cool it, clear its affinity, and pick a failover
|
|
899
|
-
* account. Does NOT promote the store active account — caller should promote only after a
|
|
900
|
-
* successful retry (or token resolve).
|
|
901
|
-
*/
|
|
902
1425
|
export function rotateAnthropicAccountOn429(
|
|
903
1426
|
config: OcxConfig,
|
|
904
1427
|
failedAccountId: string,
|
|
@@ -909,10 +1432,9 @@ export function rotateAnthropicAccountOn429(
|
|
|
909
1432
|
decision: AnthropicRouteDecision | null = null,
|
|
910
1433
|
model?: string,
|
|
911
1434
|
): string | null {
|
|
912
|
-
return
|
|
1435
|
+
return anthropicRoutingFor("anthropic").rotateAnthropicAccountOn429(config, failedAccountId, retryAfterHeader, sessionKey, now, rateLimitHeaders, decision, model);
|
|
913
1436
|
}
|
|
914
1437
|
|
|
915
|
-
/** The caller proves that a 403 is an account refusal before entering this pool policy. */
|
|
916
1438
|
export function rotateAnthropicAccountOnRefusal(
|
|
917
1439
|
config: OcxConfig,
|
|
918
1440
|
failedAccountId: string,
|
|
@@ -925,28 +1447,9 @@ export function rotateAnthropicAccountOnRefusal(
|
|
|
925
1447
|
model?: string,
|
|
926
1448
|
excludedAccountIds?: ReadonlySet<string>,
|
|
927
1449
|
): string | null {
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
// The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive
|
|
931
|
-
// presence-only recovery must not silently reactivate round-robin/fill-first merely
|
|
932
|
-
// because those dormant values remain in config. The quota picker is the neutral
|
|
933
|
-
// recovery policy already used by the default strategy.
|
|
934
|
-
const next = isAnthropicAccountPoolEnabled(config)
|
|
935
|
-
? pickAlternateAnthropicAccount(config, failedAccountId, now, decision, model, excludedAccountIds)
|
|
936
|
-
: pickLowestUsage(config, failedAccountId, now, null, model, excludedAccountIds);
|
|
937
|
-
if (!next) {
|
|
938
|
-
console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning ${status}`);
|
|
939
|
-
return null;
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
console.warn(status === 403
|
|
943
|
-
? `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}account unavailable (403); failing over`
|
|
944
|
-
: `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`);
|
|
945
|
-
|
|
946
|
-
return next;
|
|
1450
|
+
return anthropicRoutingFor("anthropic").rotateAnthropicAccountOnRefusal(config, failedAccountId, status, retryAfterHeader, sessionKey, now, rateLimitHeaders, decision, model, excludedAccountIds);
|
|
947
1451
|
}
|
|
948
1452
|
|
|
949
|
-
/** Record the account refusal even when this request has no remaining retry sends. */
|
|
950
1453
|
export function recordAnthropicAccount429(
|
|
951
1454
|
config: OcxConfig,
|
|
952
1455
|
failedAccountId: string,
|
|
@@ -954,10 +1457,9 @@ export function recordAnthropicAccount429(
|
|
|
954
1457
|
now = Date.now(),
|
|
955
1458
|
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
956
1459
|
): boolean {
|
|
957
|
-
return
|
|
1460
|
+
return anthropicRoutingFor("anthropic").recordAnthropicAccount429(config, failedAccountId, retryAfterHeader, now, rateLimitHeaders);
|
|
958
1461
|
}
|
|
959
1462
|
|
|
960
|
-
/** Account entitlement refusals use a finite backoff; renewing a plan does not require login. */
|
|
961
1463
|
export function recordAnthropicAccountRefusal(
|
|
962
1464
|
config: OcxConfig,
|
|
963
1465
|
failedAccountId: string,
|
|
@@ -966,228 +1468,62 @@ export function recordAnthropicAccountRefusal(
|
|
|
966
1468
|
now = Date.now(),
|
|
967
1469
|
rateLimitHeaders?: AnthropicRateLimitHeaders | null,
|
|
968
1470
|
): boolean {
|
|
969
|
-
|
|
970
|
-
// session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all
|
|
971
|
-
// of which move a HEALTHY request and stay opt-in. Rotating away from an account upstream has
|
|
972
|
-
// just rate-limited is a different thing: it only ever runs after a refusal, and stranding a
|
|
973
|
-
// 429 while a second logged-in account sits idle is a defect, not a configuration choice.
|
|
974
|
-
// Presence is the activation rule, the same one an apiKeyPool of two keys already uses.
|
|
975
|
-
// A sent turn may finish after its account is paused; its one remaining successor is
|
|
976
|
-
// still a valid reactive recovery even though the current unpaused quorum is now one.
|
|
977
|
-
if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)
|
|
978
|
-
&& !getAccountCredentialWithStatus(PROVIDER, failedAccountId)?.paused) return false;
|
|
979
|
-
|
|
980
|
-
if (status === 429) {
|
|
981
|
-
const headers = rateLimitHeaders ?? new Headers();
|
|
982
|
-
const kind = classifyAnthropic429({ get: name => name === "retry-after" ? retryAfterHeader ?? null : headers.get(name) }, now);
|
|
983
|
-
if (kind !== "shared-quota") {
|
|
984
|
-
if (kind === "transient-rate") pauseAnthropicRateAdmission(failedAccountId, now + (anthropicRetryAfterMs(retryAfterHeader, now) ?? 100));
|
|
985
|
-
return false;
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
// Retry-After first: it is the header written FOR this decision. The rejected window's
|
|
990
|
-
// reset is the fallback, because a 429 that omits Retry-After still carries it -- and
|
|
991
|
-
// without that fallback such a refusal cools for the 60s default and the exhausted
|
|
992
|
-
// account is back in the rotation a minute later.
|
|
993
|
-
const parsedRetry = parseRetryAfterMs(retryAfterHeader, now);
|
|
994
|
-
const resetDerived = status === 429 && parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined;
|
|
995
|
-
const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? (status === 403 ? ACCOUNT_REFUSAL_COOLDOWN_MS : DEFAULT_COOLDOWN_MS);
|
|
996
|
-
const cooldownGeneration = noteAnthropicCooldownMutation(failedAccountId);
|
|
997
|
-
upstreamHealth.set(failedAccountId, {
|
|
998
|
-
cooldownUntil: now + cooldownMs,
|
|
999
|
-
cooldownSource: parsedRetry !== undefined
|
|
1000
|
-
? "retry-after"
|
|
1001
|
-
: resetDerived !== undefined ? "reset-derived" : "default",
|
|
1002
|
-
...(resetDerived ? { rejectedQuotaWindows: resetDerived.rejectedQuotaWindows } : {}),
|
|
1003
|
-
cooldownGeneration,
|
|
1004
|
-
});
|
|
1005
|
-
sweepExpiredOnWrite(now);
|
|
1006
|
-
clearAnthropicSessionAffinityForAccount(failedAccountId);
|
|
1007
|
-
notePoolRotationFailure(POOL_KEY_ANTHROPIC, failedAccountId);
|
|
1008
|
-
// The refused account changed the eligible roster even when no retry send remains.
|
|
1009
|
-
quorumCache = null;
|
|
1010
|
-
|
|
1011
|
-
return true;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
export interface AnthropicSelectionRoutingOptions {
|
|
1015
|
-
config: OcxConfig;
|
|
1016
|
-
sessionKey?: string | null;
|
|
1017
|
-
reason?: AnthropicAccountSelectionReason;
|
|
1018
|
-
expectedCredentialGeneration?: string;
|
|
1019
|
-
routeDecision?: AnthropicRouteDecision | null;
|
|
1020
|
-
model?: string;
|
|
1471
|
+
return anthropicRoutingFor("anthropic").recordAnthropicAccountRefusal(config, failedAccountId, status, retryAfterHeader, now, rateLimitHeaders);
|
|
1021
1472
|
}
|
|
1022
1473
|
|
|
1023
|
-
|
|
1024
|
-
export async function promoteAnthropicActiveAccount(
|
|
1474
|
+
export function promoteAnthropicActiveAccount(
|
|
1025
1475
|
accountId: string,
|
|
1026
1476
|
expectedSelection: OAuthAccountSelection | null,
|
|
1027
1477
|
options: AnthropicSelectionRoutingOptions,
|
|
1028
1478
|
): Promise<OAuthAccountSelection | null> {
|
|
1029
|
-
|
|
1030
|
-
const committed = await commitOAuthAccountSelection(PROVIDER, accountId, {
|
|
1031
|
-
expectedSelection,
|
|
1032
|
-
expectedCredentialGeneration: options.expectedCredentialGeneration,
|
|
1033
|
-
requireUsableAccount: true,
|
|
1034
|
-
});
|
|
1035
|
-
if (!committed) return null;
|
|
1036
|
-
return commitAnthropicSelectionRouting(accountId, expectedSelection, committed, options) ? committed : null;
|
|
1479
|
+
return anthropicRoutingFor("anthropic").promoteAnthropicActiveAccount(accountId, expectedSelection, options);
|
|
1037
1480
|
}
|
|
1038
1481
|
|
|
1039
|
-
/** Main's shared selection owner calls this only after its authoritative commit succeeds. */
|
|
1040
1482
|
export function commitAnthropicSelectionRouting(
|
|
1041
1483
|
accountId: string,
|
|
1042
1484
|
expectedSelection: OAuthAccountSelection,
|
|
1043
1485
|
committed: OAuthAccountSelection,
|
|
1044
1486
|
options: AnthropicSelectionRoutingOptions,
|
|
1045
1487
|
): boolean {
|
|
1046
|
-
|
|
1047
|
-
&& !routeCandidates(getEligibleAnthropicAccounts(Date.now(), options.model), options.routeDecision).includes(accountId))) return false;
|
|
1048
|
-
const current = captureOAuthAccountSelection(PROVIDER);
|
|
1049
|
-
if (current?.accountId !== committed.accountId || current.revision !== committed.revision) return false;
|
|
1050
|
-
if (isAnthropicAccountPoolEnabled(options.config)) {
|
|
1051
|
-
if (anthropicPoolStrategy(options.config) === "round-robin" && options.reason !== "affinity") {
|
|
1052
|
-
const limit = stickyLimitForPool(options.config);
|
|
1053
|
-
const picked = pickRoundRobinAccount(POOL_KEY_ANTHROPIC, routeCandidates(getEligibleAnthropicAccounts(Date.now(), options.model), options.routeDecision ?? null), limit);
|
|
1054
|
-
if (picked !== accountId) seedPoolRotationAccount(POOL_KEY_ANTHROPIC, accountId);
|
|
1055
|
-
notePoolRotationSuccess(POOL_KEY_ANTHROPIC, accountId, limit);
|
|
1056
|
-
}
|
|
1057
|
-
const key = normalizeAffinityComponent(options.sessionKey);
|
|
1058
|
-
const bound = key ? sessionAffinity.get(key) : undefined;
|
|
1059
|
-
const eligibleAtCommit = options.routeDecision && bound ? getEligibleAnthropicAccounts() : [];
|
|
1060
|
-
const preserveExcludedAffinity = options.routeDecision && bound && bound.accountId !== accountId
|
|
1061
|
-
&& Date.now() - bound.lastUsedAt <= AFFINITY_IDLE_TTL_MS
|
|
1062
|
-
&& eligibleAtCommit.includes(bound.accountId)
|
|
1063
|
-
&& !routeCandidates(eligibleAtCommit, options.routeDecision).includes(bound.accountId);
|
|
1064
|
-
const preservePausedAffinity = bound && (anthropicRatePauseUntil(bound.accountId) !== undefined
|
|
1065
|
-
|| anthropicFamilyRejected(bound.accountId, options.model));
|
|
1066
|
-
if (!preserveExcludedAffinity && !preservePausedAffinity) bindAnthropicSessionAffinity(options.sessionKey, accountId);
|
|
1067
|
-
}
|
|
1068
|
-
if (manualPreference === undefined || (manualPreference?.accountId === expectedSelection.accountId
|
|
1069
|
-
&& manualPreference.revision === expectedSelection.revision)) manualPreference = null;
|
|
1070
|
-
return true;
|
|
1488
|
+
return anthropicRoutingFor("anthropic").commitAnthropicSelectionRouting(accountId, expectedSelection, committed, options);
|
|
1071
1489
|
}
|
|
1072
1490
|
|
|
1073
|
-
/**
|
|
1074
|
-
* Manual selection resets session affinity and seeds the RR ring so the next
|
|
1075
|
-
* unbound new session honors the operator-chosen account (Codex parity).
|
|
1076
|
-
*/
|
|
1077
1491
|
export function resetAnthropicRoutingForManualSelection(accountId: string): void {
|
|
1078
|
-
|
|
1079
|
-
manualSelectionGeneration++;
|
|
1080
|
-
manualPreference = captureOAuthAccountSelection(PROVIDER);
|
|
1081
|
-
seedPoolRotationAccount(POOL_KEY_ANTHROPIC, accountId);
|
|
1082
|
-
// A manual account selection is an operator statement about the roster; do not answer the
|
|
1083
|
-
// next activation question from a count read before it.
|
|
1084
|
-
quorumCache = null;
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
/**
|
|
1088
|
-
* Resolve a bearer for pool traffic without adopting a newer global Claude CLI
|
|
1089
|
-
* credential into a background multiauth `local-cli` slot (same fail-closed rule
|
|
1090
|
-
* as quota probes).
|
|
1091
|
-
*/
|
|
1092
|
-
export async function getAnthropicPoolAccessToken(accountId: string): Promise<string> {
|
|
1093
|
-
const row = getAccountCredentialWithStatus(PROVIDER, accountId);
|
|
1094
|
-
if (row?.paused) {
|
|
1095
|
-
const { OAuthAccountPausedError } = await import("./index");
|
|
1096
|
-
throw new OAuthAccountPausedError();
|
|
1097
|
-
}
|
|
1098
|
-
const stored = row?.credential;
|
|
1099
|
-
if (!stored) {
|
|
1100
|
-
const { OAuthLoginRequiredError } = await import("./index");
|
|
1101
|
-
throw new OAuthLoginRequiredError(PROVIDER);
|
|
1102
|
-
}
|
|
1103
|
-
if (stored.expires > Date.now() + REFRESH_SKEW_MS) return stored.access;
|
|
1104
|
-
if (!canRefreshAnthropicPoolAccount(accountId)) {
|
|
1105
|
-
throw new Error("background local-cli token expired; refuse CLI-adopting refresh for pool");
|
|
1106
|
-
}
|
|
1107
|
-
const { getValidAccessTokenForAccount } = await import("./index");
|
|
1108
|
-
return getValidAccessTokenForAccount(PROVIDER, accountId);
|
|
1492
|
+
return anthropicRoutingFor("anthropic").resetAnthropicRoutingForManualSelection(accountId);
|
|
1109
1493
|
}
|
|
1110
1494
|
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
const accessToken = await getAnthropicPoolAccessToken(accountId);
|
|
1114
|
-
const row = getAccountCredentialWithStatus(PROVIDER, accountId);
|
|
1115
|
-
if (row?.paused) {
|
|
1116
|
-
const { OAuthAccountPausedError } = await import("./index");
|
|
1117
|
-
throw new OAuthAccountPausedError();
|
|
1118
|
-
}
|
|
1119
|
-
if (!row || row.needsReauth || row.credential.access !== accessToken
|
|
1120
|
-
|| row.credential.expires <= Date.now()) {
|
|
1121
|
-
throw new Error("Anthropic pool credential changed during account selection");
|
|
1122
|
-
}
|
|
1123
|
-
return { provider: PROVIDER, accountId, accessToken, generation: credentialGeneration(row.credential) };
|
|
1495
|
+
export function getAnthropicPoolAccessToken(accountId: string): Promise<string> {
|
|
1496
|
+
return anthropicRoutingFor("anthropic").getAnthropicPoolAccessToken(accountId);
|
|
1124
1497
|
}
|
|
1125
1498
|
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
providerName: string,
|
|
1129
|
-
model: string,
|
|
1130
|
-
config?: OcxConfig,
|
|
1131
|
-
): Promise<string> {
|
|
1132
|
-
if (providerName !== PROVIDER || !config || !isAnthropicAccountPoolEnabled(config)) {
|
|
1133
|
-
const { getValidAccessToken } = await import("./index");
|
|
1134
|
-
return getValidAccessToken(providerName);
|
|
1135
|
-
}
|
|
1136
|
-
const route = resolveAnthropicModelRoute(config, model);
|
|
1137
|
-
if (route.error) throw new Error("Invalid Anthropic model route configuration");
|
|
1138
|
-
const selection = resolveAnthropicAccountForSession(null, config, Date.now(), route.decision, model);
|
|
1139
|
-
if (!selection.accountId) throw new Error("No permitted Anthropic account is available for this model");
|
|
1140
|
-
return (await getAnthropicPoolAccessSnapshot(selection.accountId)).accessToken;
|
|
1499
|
+
export function getAnthropicPoolAccessSnapshot(accountId: string): Promise<OAuthAccessSnapshot> {
|
|
1500
|
+
return anthropicRoutingFor("anthropic").getAnthropicPoolAccessSnapshot(accountId);
|
|
1141
1501
|
}
|
|
1142
1502
|
|
|
1143
|
-
/**
|
|
1144
|
-
* Whether the pool may refresh this account's token. Background `local-cli` slots must not
|
|
1145
|
-
* adopt the global Claude CLI credential (same fail-closed rule as quota probes).
|
|
1146
|
-
*/
|
|
1147
1503
|
export function canRefreshAnthropicPoolAccount(accountId: string): boolean {
|
|
1148
|
-
|
|
1149
|
-
if (set?.accounts.find(account => account.id === accountId)?.paused) return false;
|
|
1150
|
-
const cred = getAccountCredential(PROVIDER, accountId);
|
|
1151
|
-
if (!cred) return false;
|
|
1152
|
-
if (cred.source !== "local-cli") return true;
|
|
1153
|
-
return set?.activeAccountId === accountId;
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
export function formatAnthropicAccountOrdinal(accountId: string): string {
|
|
1157
|
-
return fallbackCodexAccountLogLabel(accountId);
|
|
1504
|
+
return anthropicRoutingFor("anthropic").canRefreshAnthropicPoolAccount(accountId);
|
|
1158
1505
|
}
|
|
1159
1506
|
|
|
1160
|
-
export function
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1507
|
+
export async function getAnthropicSidecarAccessTokenForInstance(instance: AnthropicInstanceId, model: string, config: OcxConfig): Promise<string> {
|
|
1508
|
+
if (configuredAnthropicInstance(config, instance) !== instance) {
|
|
1509
|
+
const { OAuthLoginRequiredError } = await import("./index");
|
|
1510
|
+
throw new OAuthLoginRequiredError(instance);
|
|
1511
|
+
}
|
|
1512
|
+
const routing = anthropicRoutingFor(instance);
|
|
1513
|
+
const route = resolveAnthropicModelRouteForInstance(instance, config, model);
|
|
1514
|
+
if (route.error) throw new Error("Invalid Anthropic model route configuration");
|
|
1515
|
+
const selection = routing.resolveAnthropicAccountForSession(null, config, Date.now(), route.decision, model);
|
|
1516
|
+
if (!selection.accountId) throw new Error("No permitted Anthropic account is available for this model");
|
|
1517
|
+
return (await routing.getAnthropicPoolAccessSnapshot(selection.accountId)).accessToken;
|
|
1167
1518
|
}
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
* alone (those collide across conversations).
|
|
1173
|
-
*/
|
|
1174
|
-
export function anthropicSessionKeyFromParts(input: {
|
|
1175
|
-
sessionIdHeader?: string | null;
|
|
1176
|
-
threadIdHeader?: string | null;
|
|
1177
|
-
promptCacheKey?: string | null;
|
|
1178
|
-
clientThreadId?: string | null;
|
|
1179
|
-
/** When true, prompt_cache_key is a shared Desktop cohort — ignore it for affinity. */
|
|
1180
|
-
promptCacheKeyIsSharedCohort?: boolean;
|
|
1181
|
-
}): string | null {
|
|
1182
|
-
const preferred = input.clientThreadId?.trim()
|
|
1183
|
-
|| input.sessionIdHeader?.trim()
|
|
1184
|
-
|| input.threadIdHeader?.trim()
|
|
1185
|
-
|| "";
|
|
1186
|
-
if (preferred) {
|
|
1187
|
-
return preferred.length <= 128 ? preferred : createHash("sha256").update(preferred).digest("hex");
|
|
1519
|
+
export async function getAnthropicSidecarAccessToken(providerName: string, model: string, config?: OcxConfig): Promise<string> {
|
|
1520
|
+
if (providerName === "anthropic2") {
|
|
1521
|
+
if (!config) { const { OAuthLoginRequiredError } = await import("./index"); throw new OAuthLoginRequiredError(providerName); }
|
|
1522
|
+
return getAnthropicSidecarAccessTokenForInstance("anthropic2", model, config);
|
|
1188
1523
|
}
|
|
1189
|
-
if (
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1524
|
+
if (providerName !== "anthropic" || !config || !isAnthropicAccountPoolEnabled(config)) {
|
|
1525
|
+
const { getValidAccessToken } = await import("./index");
|
|
1526
|
+
return getValidAccessToken(providerName);
|
|
1527
|
+
}
|
|
1528
|
+
return getAnthropicSidecarAccessTokenForInstance("anthropic", model, config);
|
|
1193
1529
|
}
|