@bitkyc08/opencodex 2.7.42 → 2.7.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/bin/ocx.mjs +13 -14
- package/gui/dist/assets/index-Czw-jpTU.css +1 -0
- package/gui/dist/assets/index-cmds12BG.js +67 -0
- package/gui/dist/index.html +2 -2
- package/package.json +1 -1
- package/src/AGENTS.md +28 -0
- package/src/adapters/anthropic.ts +15 -6
- package/src/adapters/cursor/native-exec-shell.ts +18 -6
- package/src/adapters/cursor/protobuf-events.ts +24 -2
- package/src/adapters/cursor/protobuf-request.ts +1 -2
- package/src/adapters/cursor/tool-definitions.ts +68 -29
- package/src/adapters/google-wire-compiler.ts +4 -0
- package/src/adapters/google.ts +128 -2
- package/src/adapters/identity.ts +12 -2
- package/src/adapters/kiro.ts +49 -6
- package/src/adapters/mimo-free.ts +2 -0
- package/src/adapters/openai-responses.ts +246 -59
- package/src/claude/agents-inject.ts +5 -0
- package/src/claude/inbound.ts +26 -9
- package/src/cli/account-auth.ts +1 -1
- package/src/cli/agent-driven.ts +37 -0
- package/src/cli/claude.ts +3 -3
- package/src/cli/doctor.ts +23 -18
- package/src/cli/help.ts +42 -6
- package/src/cli/index.ts +88 -19
- package/src/cli/interactive-confirm.ts +129 -0
- package/src/cli/opencode.ts +701 -0
- package/src/cli/provider-runtime.ts +3 -0
- package/src/cli/provider.ts +31 -10
- package/src/cli/star-prompt.ts +55 -16
- package/src/cli/status.ts +47 -13
- package/src/codex/account-id.ts +34 -0
- package/src/codex/account-lifecycle.ts +4 -1
- package/src/codex/account-namespace-match.ts +63 -0
- package/src/codex/account-namespaces.ts +149 -0
- package/src/codex/account-pause.ts +20 -0
- package/src/codex/account-usability.ts +6 -1
- package/src/codex/app-server-processes.ts +511 -0
- package/src/codex/auth-api.ts +293 -34
- package/src/codex/auth-collision.ts +2 -1
- package/src/codex/auth-context.ts +60 -17
- package/src/codex/catalog/parsing.ts +16 -1
- package/src/codex/catalog/provider-fetch.ts +48 -12
- package/src/codex/catalog/sync.ts +18 -3
- package/src/codex/catalog.ts +1 -1
- package/src/codex/history-provider.ts +145 -1
- package/src/codex/inject.ts +114 -14
- package/src/codex/main-account.ts +2 -8
- package/src/codex/pool-rotation.ts +186 -0
- package/src/codex/quota.ts +15 -0
- package/src/codex/routing.ts +695 -106
- package/src/codex/shim.ts +4 -1
- package/src/codex/subagent-defaults.ts +550 -0
- package/src/codex/subagent-model-fallback.ts +2 -0
- package/src/codex/sync.ts +3 -0
- package/src/config.ts +566 -25
- package/src/images/artifacts.ts +516 -0
- package/src/images/fulfill-video.ts +163 -0
- package/src/images/fulfill.ts +111 -0
- package/src/images/index.ts +4 -0
- package/src/images/loop.ts +789 -0
- package/src/images/plan.ts +133 -0
- package/src/images/synthetic-tool.ts +133 -0
- package/src/images/types.ts +41 -0
- package/src/images/xai-client.ts +141 -0
- package/src/images/xai-video-client.ts +163 -0
- package/src/lib/admin-secrets.ts +25 -0
- package/src/lib/config-ownership.ts +327 -0
- package/src/lib/crash-guard.ts +2 -0
- package/src/lib/destination-policy.ts +122 -4
- package/src/lib/pinned-http.ts +151 -0
- package/src/lib/process-control.ts +2 -2
- package/src/lib/provider-outbound.ts +164 -0
- package/src/lib/provider-url.ts +14 -0
- package/src/lib/proxy-env.ts +18 -0
- package/src/lib/windows-elevation.ts +81 -3
- package/src/lib/windows-secret-acl.ts +189 -12
- package/src/lib/winsw.ts +2 -0
- package/src/oauth/anthropic-routing.ts +570 -0
- package/src/oauth/health.ts +6 -0
- package/src/oauth/index.ts +283 -72
- package/src/oauth/key-providers.ts +17 -6
- package/src/oauth/kimi.ts +2 -0
- package/src/oauth/kiro-credentials.ts +322 -10
- package/src/oauth/kiro.ts +423 -43
- package/src/oauth/login-cli.ts +32 -5
- package/src/oauth/store.ts +54 -4
- package/src/oauth/types.ts +11 -0
- package/src/providers/alibaba-region-migration.ts +16 -3
- package/src/providers/antigravity-models.ts +3 -0
- package/src/providers/api-keys.ts +13 -6
- package/src/providers/derive.ts +6 -0
- package/src/providers/key-failover.ts +24 -4
- package/src/providers/quota.ts +233 -29
- package/src/providers/registry.ts +11 -3
- package/src/responses/parser.ts +11 -0
- package/src/responses/state.ts +22 -8
- package/src/responses/tool-groups.ts +19 -0
- package/src/router.ts +14 -4
- package/src/server/auth-cors.ts +110 -22
- package/src/server/claude-messages.ts +8 -1
- package/src/server/gui-static.ts +30 -6
- package/src/server/images.ts +303 -9
- package/src/server/index.ts +77 -9
- package/src/server/lifecycle.ts +25 -1
- package/src/server/management/agent-settings-routes.ts +28 -4
- package/src/server/management/combo-routes.ts +7 -0
- package/src/server/management/config-routes.ts +5 -2
- package/src/server/management/logs-usage-routes.ts +167 -3
- package/src/server/management/oauth-account-routes.ts +163 -17
- package/src/server/management/provider-routes.ts +35 -7
- package/src/server/management/system-restart.ts +172 -0
- package/src/server/management/system-routes.ts +33 -10
- package/src/server/management-api.ts +2 -2
- package/src/server/management-auth.ts +216 -0
- package/src/server/proxy-liveness.ts +14 -3
- package/src/server/responses/compact.ts +21 -13
- package/src/server/responses/core.ts +583 -152
- package/src/server/responses-image-gen-repair.ts +118 -0
- package/src/server/responses-item-id-repair.ts +10 -85
- package/src/server/sse-payload-rewrite.ts +116 -0
- package/src/server/system-env.ts +28 -10
- package/src/service.ts +49 -2
- package/src/storage/cleanup-job.ts +57 -0
- package/src/storage/cleanup.ts +1504 -28
- package/src/storage/policy-job.ts +366 -0
- package/src/storage/policy-scheduler.ts +40 -0
- package/src/storage/policy-worker.ts +53 -0
- package/src/storage/policy.ts +522 -0
- package/src/storage/restore-job.ts +242 -0
- package/src/storage/restore-worker.ts +52 -0
- package/src/storage/storage-mutation-coordinator.ts +109 -0
- package/src/tray/windows.ts +2 -0
- package/src/types.ts +96 -1
- package/src/update/index.ts +36 -18
- package/src/update/job.ts +103 -12
- package/src/update/npm-invocation.d.mts +23 -0
- package/src/update/npm-invocation.mjs +94 -0
- package/src/usage/debug.ts +2 -0
- package/src/usage/log.ts +4 -0
- package/gui/dist/assets/index-Bl_VBGoI.js +0 -65
- package/gui/dist/assets/index-DfVGuN88.css +0 -1
package/src/codex/routing.ts
CHANGED
|
@@ -2,10 +2,22 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { saveConfigPreservingClaudeCode } from "../config";
|
|
3
3
|
import { isCodexAccountGenerationLive, readCodexAccountRecord } from "./account-store";
|
|
4
4
|
import { codexAccountLogLabel } from "./account-label";
|
|
5
|
+
import { isCodexAccountPaused } from "./account-pause";
|
|
5
6
|
import { isCodexAccountUsable } from "./account-usability";
|
|
6
7
|
import { isAccountNeedsReauth, markAccountNeedsReauth } from "./account-runtime-state";
|
|
8
|
+
import {
|
|
9
|
+
POOL_KEY_CODEX,
|
|
10
|
+
normalizeAccountPoolStickyLimit,
|
|
11
|
+
normalizeAccountPoolStrategy,
|
|
12
|
+
notePoolRotationFailure,
|
|
13
|
+
notePoolRotationSuccess,
|
|
14
|
+
peekRoundRobinAccount,
|
|
15
|
+
pickRoundRobinAccount,
|
|
16
|
+
seedPoolRotationAccount,
|
|
17
|
+
} from "./pool-rotation";
|
|
7
18
|
import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota";
|
|
8
19
|
import { MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "./main-account";
|
|
20
|
+
import { isSelectableCodexPoolAccount } from "./account-id";
|
|
9
21
|
import type { OcxConfig } from "../types";
|
|
10
22
|
|
|
11
23
|
type ThreadAffinityEntry = {
|
|
@@ -23,7 +35,14 @@ export type CodexThreadResolution =
|
|
|
23
35
|
| { status: "none" }
|
|
24
36
|
| { status: "expired"; accountId: string };
|
|
25
37
|
|
|
26
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Process-local cursor for automatic RR/fill-first (and quota-429 when not
|
|
40
|
+
* sync-writing) picks. Keeps unrelated `saveConfig` from persisting transient
|
|
41
|
+
* rotation as the operator's `activeCodexAccountId`. Manual selection clears it
|
|
42
|
+
* so disk/`config.activeCodexAccountId` remains authoritative.
|
|
43
|
+
*/
|
|
44
|
+
let runtimeActiveCodexAccountId: string | undefined;
|
|
45
|
+
|
|
27
46
|
type CodexUpstreamHealth = {
|
|
28
47
|
consecutiveFailures: number;
|
|
29
48
|
/** Consecutive healthy terminals observed while recovering from escalation level 2+. */
|
|
@@ -90,27 +109,88 @@ export const CODEX_THREAD_AFFINITY_MAX_ENTRIES = 2048;
|
|
|
90
109
|
export const CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS = 60_000;
|
|
91
110
|
|
|
92
111
|
const upstreamHealth = new Map<string, CodexUpstreamHealth>();
|
|
112
|
+
/**
|
|
113
|
+
* Reset-derived 429s can describe a quota owned by one native model family,
|
|
114
|
+
* rather than the whole ChatGPT account. Keep those advisory cooldowns apart
|
|
115
|
+
* from account-wide Retry-After/default throttles and transient health.
|
|
116
|
+
*/
|
|
117
|
+
const quotaScopedHealth = new Map<string, Map<CodexQuotaScope, CodexUpstreamHealth>>();
|
|
93
118
|
|
|
94
119
|
export type CodexUpstreamOutcome = number | "connect_error" | "timeout";
|
|
95
120
|
export type CodexUpstreamOutcomeClass = "success" | "credential" | "quota" | "transient" | "caller" | "unknown";
|
|
96
121
|
export type CodexCooldownSource = "retry-after" | "reset-derived" | "default";
|
|
122
|
+
/**
|
|
123
|
+
* Native Codex quota groups known to be independent upstream. Keep the mapping
|
|
124
|
+
* deliberately conservative: unlisted models share the normal native group.
|
|
125
|
+
* Add a new explicit group here only when its independent upstream quota is
|
|
126
|
+
* confirmed, so shared limits never receive cross-model bypasses.
|
|
127
|
+
*/
|
|
128
|
+
export type CodexQuotaScope = "shared" | "spark";
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Requests without a resolved native model retain the historic one-account-per-
|
|
132
|
+
* thread behavior. Requests with a known quota scope get an independent
|
|
133
|
+
* affinity so a Spark failover cannot displace the same thread's Terra/Luna
|
|
134
|
+
* account (and vice versa).
|
|
135
|
+
*/
|
|
136
|
+
type ThreadAffinityScope = CodexQuotaScope | "legacy";
|
|
137
|
+
const LEGACY_THREAD_AFFINITY_SCOPE = "legacy" as const;
|
|
138
|
+
const threadAccountMap = new Map<string, Map<ThreadAffinityScope, ThreadAffinityEntry>>();
|
|
139
|
+
|
|
140
|
+
const NATIVE_MODEL_QUOTA_SCOPES: Readonly<Record<string, CodexQuotaScope>> = {
|
|
141
|
+
"gpt-5.3-codex-spark": "spark",
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
// A thread can have one legacy binding plus one binding for each known scope.
|
|
145
|
+
// This upper-bound guard avoids an exact map scan until it can be over capacity.
|
|
146
|
+
const MAX_THREAD_AFFINITY_SCOPES = new Set([
|
|
147
|
+
LEGACY_THREAD_AFFINITY_SCOPE,
|
|
148
|
+
"shared",
|
|
149
|
+
...Object.values(NATIVE_MODEL_QUOTA_SCOPES),
|
|
150
|
+
]).size;
|
|
151
|
+
|
|
152
|
+
export function codexQuotaScopeForModel(modelId: string | undefined): CodexQuotaScope | undefined {
|
|
153
|
+
if (!modelId?.trim()) return undefined;
|
|
154
|
+
return NATIVE_MODEL_QUOTA_SCOPES[modelId.trim().toLowerCase()] ?? "shared";
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Independent quota groups must not mutate the shared active-account cursor. */
|
|
158
|
+
function isIndependentCodexQuotaScope(quotaScope?: CodexQuotaScope): boolean {
|
|
159
|
+
return quotaScope !== undefined && quotaScope !== "shared";
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function codexPoolKeyForScope(quotaScope?: CodexQuotaScope): string {
|
|
163
|
+
return isIndependentCodexQuotaScope(quotaScope) ? `${POOL_KEY_CODEX}:${quotaScope}` : POOL_KEY_CODEX;
|
|
164
|
+
}
|
|
165
|
+
|
|
97
166
|
export type CodexUpstreamOutcomeMeta = {
|
|
98
167
|
retryAfter?: string | null;
|
|
99
168
|
resetAt?: unknown | unknown[];
|
|
100
169
|
now?: number;
|
|
170
|
+
/** Native model selected for this request; used only for confirmed scoped quotas. */
|
|
171
|
+
modelId?: string;
|
|
101
172
|
/** When set, clears affinity for this thread immediately on transient failure. */
|
|
102
173
|
threadId?: string | null;
|
|
103
174
|
/**
|
|
104
175
|
* Probe lease held by this request, when it was admitted through an active
|
|
105
176
|
* quota cooldown. Only the outcome carrying the current lease may clear the
|
|
106
177
|
* cooldown (#433).
|
|
107
|
-
|
|
178
|
+
*/
|
|
108
179
|
probeLeaseId?: string;
|
|
180
|
+
/** Scope of `probeLeaseId` when it was granted against a model-scoped cooldown. */
|
|
181
|
+
probeQuotaScope?: CodexQuotaScope;
|
|
182
|
+
/**
|
|
183
|
+
* Already-chosen alternate for same-request 429 retry. When set, promotion
|
|
184
|
+
* reuses this account instead of calling {@link pickAlternateCodexAccount}
|
|
185
|
+
* again (which would advance a round-robin ring twice).
|
|
186
|
+
*/
|
|
187
|
+
promoteAccountId?: string;
|
|
109
188
|
};
|
|
110
189
|
|
|
111
190
|
function hasConfiguredPoolAccount(config: OcxConfig, accountId: string): boolean {
|
|
112
191
|
if (accountId === MAIN_CODEX_ACCOUNT_ID) return isCodexAccountUsable(config, accountId);
|
|
113
|
-
return (config.codexAccounts ?? [])
|
|
192
|
+
return (config.codexAccounts ?? [])
|
|
193
|
+
.some(account => isSelectableCodexPoolAccount(account) && account.id === accountId);
|
|
114
194
|
}
|
|
115
195
|
|
|
116
196
|
export function clearThreadAccountMap(): void {
|
|
@@ -118,17 +198,23 @@ export function clearThreadAccountMap(): void {
|
|
|
118
198
|
}
|
|
119
199
|
|
|
120
200
|
export function clearThreadAccountMapForAccount(accountId: string): void {
|
|
121
|
-
for (const [threadId,
|
|
122
|
-
|
|
201
|
+
for (const [threadId, affinities] of threadAccountMap) {
|
|
202
|
+
for (const [scope, entry] of affinities) {
|
|
203
|
+
if (entry.accountId === accountId) affinities.delete(scope);
|
|
204
|
+
}
|
|
205
|
+
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
123
206
|
}
|
|
124
207
|
}
|
|
125
208
|
|
|
126
209
|
export function clearCodexUpstreamHealth(): void {
|
|
127
210
|
upstreamHealth.clear();
|
|
211
|
+
quotaScopedHealth.clear();
|
|
212
|
+
runtimeActiveCodexAccountId = undefined;
|
|
128
213
|
}
|
|
129
214
|
|
|
130
215
|
export function clearCodexUpstreamHealthForAccount(accountId: string): void {
|
|
131
216
|
upstreamHealth.delete(accountId);
|
|
217
|
+
quotaScopedHealth.delete(accountId);
|
|
132
218
|
}
|
|
133
219
|
|
|
134
220
|
export function getCodexUpstreamHealth(
|
|
@@ -137,6 +223,26 @@ export function getCodexUpstreamHealth(
|
|
|
137
223
|
return upstreamHealth.get(accountId) ?? null;
|
|
138
224
|
}
|
|
139
225
|
|
|
226
|
+
function scopedHealthFor(accountId: string, scope: CodexQuotaScope): CodexUpstreamHealth | undefined {
|
|
227
|
+
return quotaScopedHealth.get(accountId)?.get(scope);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function setScopedHealth(accountId: string, scope: CodexQuotaScope, health: CodexUpstreamHealth): void {
|
|
231
|
+
let scopes = quotaScopedHealth.get(accountId);
|
|
232
|
+
if (!scopes) {
|
|
233
|
+
scopes = new Map();
|
|
234
|
+
quotaScopedHealth.set(accountId, scopes);
|
|
235
|
+
}
|
|
236
|
+
scopes.set(scope, health);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function deleteScopedHealth(accountId: string, scope: CodexQuotaScope): void {
|
|
240
|
+
const scopes = quotaScopedHealth.get(accountId);
|
|
241
|
+
if (!scopes) return;
|
|
242
|
+
scopes.delete(scope);
|
|
243
|
+
if (scopes.size === 0) quotaScopedHealth.delete(accountId);
|
|
244
|
+
}
|
|
245
|
+
|
|
140
246
|
export function computeCodexUsageScore(quota: {
|
|
141
247
|
weeklyPercent?: number;
|
|
142
248
|
monthlyPercent?: number;
|
|
@@ -158,7 +264,9 @@ export function classifyCodexUpstreamOutcome(outcome: CodexUpstreamOutcome): Cod
|
|
|
158
264
|
if (!Number.isFinite(outcome)) return "unknown";
|
|
159
265
|
if (outcome >= 200 && outcome < 300) return "success";
|
|
160
266
|
if (outcome === 401 || outcome === 403) return "credential";
|
|
161
|
-
|
|
267
|
+
// 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
|
|
268
|
+
// (same-request alternate retry records this outcome for the depleted account).
|
|
269
|
+
if (outcome === 429 || outcome === 402) return "quota";
|
|
162
270
|
if (outcome >= 400 && outcome < 500) return "caller";
|
|
163
271
|
if (outcome >= 500 && outcome < 600) return "transient";
|
|
164
272
|
return "unknown";
|
|
@@ -249,7 +357,10 @@ export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now
|
|
|
249
357
|
|
|
250
358
|
/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */
|
|
251
359
|
export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean {
|
|
252
|
-
|
|
360
|
+
return canAcquireQuotaProbeLease(upstreamHealth.get(accountId), now);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now: number): boolean {
|
|
253
364
|
if (!health) return false;
|
|
254
365
|
const cooldownUntil = health.cooldownUntil;
|
|
255
366
|
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false;
|
|
@@ -259,6 +370,24 @@ export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now
|
|
|
259
370
|
return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS;
|
|
260
371
|
}
|
|
261
372
|
|
|
373
|
+
/** Acquire the recovery probe for one confirmed model-specific quota group. */
|
|
374
|
+
export function tryAcquireCodexQuotaScopeProbeLease(
|
|
375
|
+
accountId: string,
|
|
376
|
+
scope: CodexQuotaScope,
|
|
377
|
+
now = Date.now(),
|
|
378
|
+
): string | null {
|
|
379
|
+
const health = scopedHealthFor(accountId, scope);
|
|
380
|
+
if (!canAcquireQuotaProbeLease(health, now)) return null;
|
|
381
|
+
const probeLeaseId = randomUUID();
|
|
382
|
+
setScopedHealth(accountId, scope, {
|
|
383
|
+
...health!,
|
|
384
|
+
probeLeaseId,
|
|
385
|
+
probeLeaseGeneration: health!.cooldownGeneration ?? 0,
|
|
386
|
+
lastProbeAt: now,
|
|
387
|
+
});
|
|
388
|
+
return probeLeaseId;
|
|
389
|
+
}
|
|
390
|
+
|
|
262
391
|
/**
|
|
263
392
|
* Hand a probe lease back without recording an upstream outcome. Used by paths
|
|
264
393
|
* that take a lease and then fail before any request reaches upstream.
|
|
@@ -269,6 +398,18 @@ export function releaseCodexQuotaProbeLease(accountId: string, leaseId: string,
|
|
|
269
398
|
upstreamHealth.set(accountId, withProbeLeaseReleased(health, now));
|
|
270
399
|
}
|
|
271
400
|
|
|
401
|
+
/** Release a model-specific quota probe when the request never reaches upstream. */
|
|
402
|
+
export function releaseCodexQuotaScopeProbeLease(
|
|
403
|
+
accountId: string,
|
|
404
|
+
scope: CodexQuotaScope,
|
|
405
|
+
leaseId: string,
|
|
406
|
+
now = Date.now(),
|
|
407
|
+
): void {
|
|
408
|
+
const health = scopedHealthFor(accountId, scope);
|
|
409
|
+
if (!health || health.probeLeaseId !== leaseId) return;
|
|
410
|
+
setScopedHealth(accountId, scope, withProbeLeaseReleased(health, now));
|
|
411
|
+
}
|
|
412
|
+
|
|
272
413
|
/**
|
|
273
414
|
* True when this outcome belongs to the account's in-flight probe. The
|
|
274
415
|
* undefined-id guard matters: without it an outcome carrying no lease would match
|
|
@@ -308,6 +449,17 @@ function preservedCooldownFields(health: CodexUpstreamHealth | undefined): Parti
|
|
|
308
449
|
/** Manual selection resets transient routing evidence without bypassing a real 429 cooldown. */
|
|
309
450
|
export function resetCodexRoutingForManualSelection(accountId: string): void {
|
|
310
451
|
clearThreadAccountMap();
|
|
452
|
+
// Manual selection is the operator source of truth — drop any automatic runtime cursor.
|
|
453
|
+
runtimeActiveCodexAccountId = undefined;
|
|
454
|
+
// Seed the RR ring so the next unbound new session honors the manually selected account
|
|
455
|
+
// under round-robin (affinity-cleared threads / null threadId). Fill-first already follows
|
|
456
|
+
// config.activeCodexAccountId, which the caller persists before invoking this.
|
|
457
|
+
seedPoolRotationAccount(POOL_KEY_CODEX, accountId);
|
|
458
|
+
for (const scope of new Set(Object.values(NATIVE_MODEL_QUOTA_SCOPES))) {
|
|
459
|
+
if (isIndependentCodexQuotaScope(scope)) {
|
|
460
|
+
seedPoolRotationAccount(codexPoolKeyForScope(scope), accountId);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
311
463
|
const current = upstreamHealth.get(accountId);
|
|
312
464
|
if (!current) return;
|
|
313
465
|
const preserved = preservedCooldownFields(current);
|
|
@@ -334,6 +486,33 @@ export function getCodexAccountHealthSnapshot(accountId: string, now = Date.now(
|
|
|
334
486
|
};
|
|
335
487
|
}
|
|
336
488
|
|
|
489
|
+
/**
|
|
490
|
+
* Read the cooldown relevant to a routed native model. Account-wide cooldowns
|
|
491
|
+
* (Retry-After/default) always win; reset-derived scoped state applies only to
|
|
492
|
+
* its confirmed quota group.
|
|
493
|
+
*/
|
|
494
|
+
export function getCodexQuotaHealthSnapshot(
|
|
495
|
+
accountId: string,
|
|
496
|
+
quotaScope: CodexQuotaScope | undefined,
|
|
497
|
+
now = Date.now(),
|
|
498
|
+
): {
|
|
499
|
+
cooldownUntil?: number;
|
|
500
|
+
cooldownSource?: CodexCooldownSource;
|
|
501
|
+
quotaScope?: CodexQuotaScope;
|
|
502
|
+
} | null {
|
|
503
|
+
const account = getCodexAccountHealthSnapshot(accountId, now);
|
|
504
|
+
if (account) return account;
|
|
505
|
+
if (!quotaScope) return null;
|
|
506
|
+
const scoped = scopedHealthFor(accountId, quotaScope);
|
|
507
|
+
const cooldownUntil = scoped?.cooldownUntil;
|
|
508
|
+
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null;
|
|
509
|
+
return {
|
|
510
|
+
cooldownUntil,
|
|
511
|
+
...(scoped?.cooldownSource ? { cooldownSource: scoped.cooldownSource } : {}),
|
|
512
|
+
quotaScope,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
337
516
|
export function isCodexAccountInCooldown(accountId: string, now = Date.now()): boolean {
|
|
338
517
|
return getCodexAccountCooldownUntil(accountId, now) !== null;
|
|
339
518
|
}
|
|
@@ -357,24 +536,41 @@ export function isCodexAccountInCooldown(accountId: string, now = Date.now()): b
|
|
|
357
536
|
* Returns false when the account carried no live cooldown (already expired or never set).
|
|
358
537
|
*/
|
|
359
538
|
export function clearCodexAccountCooldown(accountId: string, now = Date.now()): boolean {
|
|
360
|
-
const
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
377
|
-
|
|
539
|
+
const clear = (health: CodexUpstreamHealth): CodexUpstreamHealth | null => {
|
|
540
|
+
const cooldownUntil = health.cooldownUntil;
|
|
541
|
+
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null;
|
|
542
|
+
const {
|
|
543
|
+
cooldownUntil: _until,
|
|
544
|
+
cooldownSince: _since,
|
|
545
|
+
cooldownSource: _source,
|
|
546
|
+
probeLeaseId: _leaseId,
|
|
547
|
+
probeLeaseGeneration: _leaseGeneration,
|
|
548
|
+
...rest
|
|
549
|
+
} = health;
|
|
550
|
+
return {
|
|
551
|
+
...rest,
|
|
552
|
+
cooldownGeneration: (health.cooldownGeneration ?? 0) + 1,
|
|
553
|
+
lastProbeAt: now,
|
|
554
|
+
};
|
|
555
|
+
};
|
|
556
|
+
|
|
557
|
+
let cleared = false;
|
|
558
|
+
const accountHealth = upstreamHealth.get(accountId);
|
|
559
|
+
if (accountHealth) {
|
|
560
|
+
const next = clear(accountHealth);
|
|
561
|
+
if (next) {
|
|
562
|
+
upstreamHealth.set(accountId, next);
|
|
563
|
+
cleared = true;
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
for (const [scope, health] of quotaScopedHealth.get(accountId) ?? []) {
|
|
567
|
+
const next = clear(health);
|
|
568
|
+
if (next) {
|
|
569
|
+
setScopedHealth(accountId, scope, next);
|
|
570
|
+
cleared = true;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
return cleared;
|
|
378
574
|
}
|
|
379
575
|
|
|
380
576
|
export function getCodexAccountSoftAvoidUntil(accountId: string, now = Date.now()): number | null {
|
|
@@ -388,12 +584,49 @@ export function isCodexAccountSoftAvoided(accountId: string, now = Date.now()):
|
|
|
388
584
|
return getCodexAccountSoftAvoidUntil(accountId, now) !== null;
|
|
389
585
|
}
|
|
390
586
|
|
|
391
|
-
function isCodexAccountSelectable(
|
|
392
|
-
|
|
587
|
+
function isCodexAccountSelectable(
|
|
588
|
+
config: OcxConfig,
|
|
589
|
+
accountId: string,
|
|
590
|
+
now: number,
|
|
591
|
+
quotaScope?: CodexQuotaScope,
|
|
592
|
+
): boolean {
|
|
593
|
+
return !isCodexAccountPaused(config, accountId)
|
|
594
|
+
&& getCodexQuotaHealthSnapshot(accountId, quotaScope, now) === null
|
|
393
595
|
&& !isCodexAccountSoftAvoided(accountId, now)
|
|
394
596
|
&& isCodexAccountUsable(config, accountId);
|
|
395
597
|
}
|
|
396
598
|
|
|
599
|
+
function threadAffinityScope(quotaScope?: CodexQuotaScope): ThreadAffinityScope {
|
|
600
|
+
return quotaScope ?? LEGACY_THREAD_AFFINITY_SCOPE;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
function getThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): ThreadAffinityEntry | undefined {
|
|
604
|
+
return threadAccountMap.get(threadId)?.get(threadAffinityScope(quotaScope));
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
function deleteThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): void {
|
|
608
|
+
const affinities = threadAccountMap.get(threadId);
|
|
609
|
+
if (!affinities) return;
|
|
610
|
+
affinities.delete(threadAffinityScope(quotaScope));
|
|
611
|
+
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** Remove only the matching failed account's affinities for one thread. */
|
|
615
|
+
function deleteThreadAffinitiesForAccount(threadId: string, accountId: string): void {
|
|
616
|
+
const affinities = threadAccountMap.get(threadId);
|
|
617
|
+
if (!affinities) return;
|
|
618
|
+
for (const [scope, entry] of affinities) {
|
|
619
|
+
if (entry.accountId === accountId) affinities.delete(scope);
|
|
620
|
+
}
|
|
621
|
+
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
function threadAffinityEntryCount(): number {
|
|
625
|
+
let count = 0;
|
|
626
|
+
for (const affinities of threadAccountMap.values()) count += affinities.size;
|
|
627
|
+
return count;
|
|
628
|
+
}
|
|
629
|
+
|
|
397
630
|
function isThreadAffinityExpired(entry: ThreadAffinityEntry, now: number): boolean {
|
|
398
631
|
return now - entry.lastUsedAt > CODEX_THREAD_AFFINITY_IDLE_TTL_MS;
|
|
399
632
|
}
|
|
@@ -404,45 +637,69 @@ function isThreadAffinityGenerationLive(entry: ThreadAffinityEntry): boolean {
|
|
|
404
637
|
}
|
|
405
638
|
|
|
406
639
|
function pruneExpiredThreadAffinities(now: number): void {
|
|
407
|
-
for (const [threadId,
|
|
408
|
-
|
|
640
|
+
for (const [threadId, affinities] of threadAccountMap) {
|
|
641
|
+
for (const [scope, entry] of affinities) {
|
|
642
|
+
if (isThreadAffinityExpired(entry, now)) affinities.delete(scope);
|
|
643
|
+
}
|
|
644
|
+
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
409
645
|
}
|
|
410
646
|
}
|
|
411
647
|
|
|
412
648
|
function pruneLruThreadAffinities(): void {
|
|
413
|
-
|
|
649
|
+
if (threadAccountMap.size * MAX_THREAD_AFFINITY_SCOPES <= CODEX_THREAD_AFFINITY_MAX_ENTRIES) return;
|
|
650
|
+
while (threadAffinityEntryCount() > CODEX_THREAD_AFFINITY_MAX_ENTRIES) {
|
|
414
651
|
let oldestThreadId: string | null = null;
|
|
652
|
+
let oldestScope: ThreadAffinityScope | null = null;
|
|
415
653
|
let oldestLastUsedAt = Number.POSITIVE_INFINITY;
|
|
416
|
-
for (const [threadId,
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
654
|
+
for (const [threadId, affinities] of threadAccountMap) {
|
|
655
|
+
for (const [scope, entry] of affinities) {
|
|
656
|
+
if (entry.lastUsedAt < oldestLastUsedAt) {
|
|
657
|
+
oldestThreadId = threadId;
|
|
658
|
+
oldestScope = scope;
|
|
659
|
+
oldestLastUsedAt = entry.lastUsedAt;
|
|
660
|
+
}
|
|
420
661
|
}
|
|
421
662
|
}
|
|
422
|
-
if (!oldestThreadId) return;
|
|
423
|
-
|
|
663
|
+
if (!oldestThreadId || !oldestScope) return;
|
|
664
|
+
deleteThreadAffinity(oldestThreadId, oldestScope === LEGACY_THREAD_AFFINITY_SCOPE ? undefined : oldestScope);
|
|
424
665
|
}
|
|
425
666
|
}
|
|
426
667
|
|
|
427
|
-
function bindThreadAffinity(
|
|
668
|
+
function bindThreadAffinity(
|
|
669
|
+
threadId: string,
|
|
670
|
+
accountId: string,
|
|
671
|
+
now: number,
|
|
672
|
+
quotaScope?: CodexQuotaScope,
|
|
673
|
+
): void {
|
|
428
674
|
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
|
|
429
675
|
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return;
|
|
430
676
|
pruneExpiredThreadAffinities(now);
|
|
431
|
-
const
|
|
432
|
-
threadAccountMap.
|
|
677
|
+
const scope = threadAffinityScope(quotaScope);
|
|
678
|
+
const affinities = threadAccountMap.get(threadId) ?? new Map<ThreadAffinityScope, ThreadAffinityEntry>();
|
|
679
|
+
const previous = affinities.get(scope);
|
|
680
|
+
affinities.set(scope, {
|
|
433
681
|
accountId,
|
|
434
682
|
generation: accountId === MAIN_CODEX_ACCOUNT_ID ? 0 : record!.generation,
|
|
435
683
|
createdAt: previous?.createdAt ?? now,
|
|
436
684
|
lastUsedAt: now,
|
|
437
685
|
lastReevalAt: now,
|
|
438
686
|
});
|
|
687
|
+
threadAccountMap.set(threadId, affinities);
|
|
439
688
|
pruneLruThreadAffinities();
|
|
440
689
|
}
|
|
441
690
|
|
|
442
|
-
function getEligiblePoolAccounts(
|
|
691
|
+
function getEligiblePoolAccounts(
|
|
692
|
+
config: OcxConfig,
|
|
693
|
+
excludeId?: string,
|
|
694
|
+
now = Date.now(),
|
|
695
|
+
quotaScope?: CodexQuotaScope,
|
|
696
|
+
): string[] {
|
|
443
697
|
const ids = (config.codexAccounts ?? [])
|
|
444
|
-
.filter(account =>
|
|
445
|
-
|
|
698
|
+
.filter(account => isSelectableCodexPoolAccount(account)
|
|
699
|
+
&& account.id !== excludeId
|
|
700
|
+
&& !isCodexAccountPaused(config, account.id)
|
|
701
|
+
&& !isAccountNeedsReauth(account.id))
|
|
702
|
+
.filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
|
|
446
703
|
.filter(account => !isCodexAccountSoftAvoided(account.id, now))
|
|
447
704
|
.filter(account => isCodexAccountUsable(config, account.id))
|
|
448
705
|
.map(account => account.id);
|
|
@@ -450,8 +707,9 @@ function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Da
|
|
|
450
707
|
// first-class rotation candidate when its read-only token is usable (Option A).
|
|
451
708
|
if (
|
|
452
709
|
excludeId !== MAIN_CODEX_ACCOUNT_ID
|
|
710
|
+
&& !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)
|
|
453
711
|
&& !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)
|
|
454
|
-
&&
|
|
712
|
+
&& getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
|
|
455
713
|
&& !isCodexAccountSoftAvoided(MAIN_CODEX_ACCOUNT_ID, now)
|
|
456
714
|
&& isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID)
|
|
457
715
|
) {
|
|
@@ -460,15 +718,160 @@ function getEligiblePoolAccounts(config: OcxConfig, excludeId?: string, now = Da
|
|
|
460
718
|
return ids;
|
|
461
719
|
}
|
|
462
720
|
|
|
721
|
+
function listEligibleCodexAccountIds(
|
|
722
|
+
config: OcxConfig,
|
|
723
|
+
now: number,
|
|
724
|
+
quotaScope?: CodexQuotaScope,
|
|
725
|
+
): string[] {
|
|
726
|
+
return getEligiblePoolAccounts(config, undefined, now, quotaScope);
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function stickyLimitForConfig(config: OcxConfig): number {
|
|
730
|
+
return normalizeAccountPoolStickyLimit(config.accountPoolStickyLimit);
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
function isActiveUnderFillFirstThreshold(config: OcxConfig, accountId: string): boolean {
|
|
734
|
+
const threshold = config.autoSwitchThreshold ?? 80;
|
|
735
|
+
if (threshold <= 0) return true;
|
|
736
|
+
const usage = computeCodexUsageScore(getAccountQuota(accountId), getPoolAccountPlan(config, accountId));
|
|
737
|
+
// Unknown usage must not force fill-first to abandon the active account.
|
|
738
|
+
if (isUnknownUsage(usage)) return true;
|
|
739
|
+
return usage < threshold;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* Fill-first: keep selectable active under threshold; otherwise advance to the next
|
|
744
|
+
* eligible id in stable sorted order after the current active (wrapping).
|
|
745
|
+
*/
|
|
746
|
+
function pickFillFirstCodexAccount(
|
|
747
|
+
config: OcxConfig,
|
|
748
|
+
now: number,
|
|
749
|
+
quotaScope?: CodexQuotaScope,
|
|
750
|
+
): string | null {
|
|
751
|
+
const eligible = listEligibleCodexAccountIds(config, now, quotaScope);
|
|
752
|
+
if (eligible.length === 0) return null;
|
|
753
|
+
|
|
754
|
+
const active = getEffectiveActiveCodexAccountId(config);
|
|
755
|
+
if (active && eligible.includes(active) && isActiveUnderFillFirstThreshold(config, active)) {
|
|
756
|
+
return active;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
return pickNextFillFirstCodexAccount(config, active ?? null, eligible, now);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/** Next eligible account in stable order after `afterId` (wrapping). */
|
|
763
|
+
function pickNextFillFirstCodexAccount(
|
|
764
|
+
config: OcxConfig,
|
|
765
|
+
afterId: string | null,
|
|
766
|
+
eligible = listEligibleCodexAccountIds(config, Date.now()),
|
|
767
|
+
_now = Date.now(),
|
|
768
|
+
): string | null {
|
|
769
|
+
if (eligible.length === 0) return null;
|
|
770
|
+
const ordered = [...eligible].sort((a, b) => a.localeCompare(b));
|
|
771
|
+
if (!afterId) {
|
|
772
|
+
// Prefer an under-threshold account when starting with no active cursor.
|
|
773
|
+
for (const id of ordered) {
|
|
774
|
+
if (isActiveUnderFillFirstThreshold(config, id)) return id;
|
|
775
|
+
}
|
|
776
|
+
return ordered[0] ?? null;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
const allConfigured = [
|
|
780
|
+
...(isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID) || afterId === MAIN_CODEX_ACCOUNT_ID
|
|
781
|
+
? [MAIN_CODEX_ACCOUNT_ID]
|
|
782
|
+
: []),
|
|
783
|
+
...(config.codexAccounts ?? []).filter(account => !account.isMain).map(account => account.id),
|
|
784
|
+
];
|
|
785
|
+
const stableAll = [...new Set(allConfigured)].sort((a, b) => a.localeCompare(b));
|
|
786
|
+
const startIdx = stableAll.indexOf(afterId);
|
|
787
|
+
if (startIdx < 0) {
|
|
788
|
+
for (const id of ordered) {
|
|
789
|
+
if (isActiveUnderFillFirstThreshold(config, id)) return id;
|
|
790
|
+
}
|
|
791
|
+
return ordered[0] ?? null;
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// Skip successors that are also at/above threshold (known drained usage).
|
|
795
|
+
let fallback: string | null = null;
|
|
796
|
+
for (let step = 1; step <= stableAll.length; step++) {
|
|
797
|
+
const candidate = stableAll[(startIdx + step) % stableAll.length]!;
|
|
798
|
+
if (!eligible.includes(candidate)) continue;
|
|
799
|
+
if (!fallback) fallback = candidate;
|
|
800
|
+
if (isActiveUnderFillFirstThreshold(config, candidate)) return candidate;
|
|
801
|
+
}
|
|
802
|
+
return fallback ?? ordered[0] ?? null;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Unbound new-session pick for round-robin / fill-first. Returns null to fall through
|
|
807
|
+
* to the legacy quota path (or when the strategy is quota).
|
|
808
|
+
*
|
|
809
|
+
* When `commit` is true (resolve path), remembers active in-memory, binds thread affinity, and
|
|
810
|
+
* notes RR success. When `commit` is false (preview), returns the same RR/fill-first
|
|
811
|
+
* account resolve would pick via a dry-run peek — without mutating ring weights,
|
|
812
|
+
* activeKey, sticky counters, config, or affinity.
|
|
813
|
+
*
|
|
814
|
+
* Automatic strategy picks never sync-write config; only manual selection persists active.
|
|
815
|
+
*
|
|
816
|
+
* Known limitation (follow-up): when a subagent preview peeks an RR account and the request
|
|
817
|
+
* then falls back to a non-Codex provider, the ring is not reserved/committed. Prefer seeding
|
|
818
|
+
* the peeked account if that path becomes load-bearing.
|
|
819
|
+
*/
|
|
820
|
+
function pickUnboundStrategyAccount(
|
|
821
|
+
config: OcxConfig,
|
|
822
|
+
threadId: string | null,
|
|
823
|
+
now: number,
|
|
824
|
+
commit: boolean,
|
|
825
|
+
quotaScope?: CodexQuotaScope,
|
|
826
|
+
): string | null {
|
|
827
|
+
const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
|
|
828
|
+
if (strategy === "quota") return null;
|
|
829
|
+
const poolKey = codexPoolKeyForScope(quotaScope);
|
|
830
|
+
|
|
831
|
+
let picked: string | null = null;
|
|
832
|
+
if (strategy === "round-robin") {
|
|
833
|
+
const eligible = listEligibleCodexAccountIds(config, now, quotaScope);
|
|
834
|
+
const limit = stickyLimitForConfig(config);
|
|
835
|
+
if (!commit) {
|
|
836
|
+
return peekRoundRobinAccount(poolKey, eligible, limit);
|
|
837
|
+
}
|
|
838
|
+
picked = pickRoundRobinAccount(poolKey, eligible, limit);
|
|
839
|
+
if (!picked) return null;
|
|
840
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) rememberActiveCodexAccount(config, picked);
|
|
841
|
+
if (threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
842
|
+
notePoolRotationSuccess(poolKey, picked, limit);
|
|
843
|
+
return picked;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
if (strategy === "fill-first") {
|
|
847
|
+
picked = pickFillFirstCodexAccount(config, now, quotaScope);
|
|
848
|
+
if (!picked) return null;
|
|
849
|
+
if (commit) {
|
|
850
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) rememberActiveCodexAccount(config, picked);
|
|
851
|
+
if (threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
852
|
+
}
|
|
853
|
+
return picked;
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
return null;
|
|
857
|
+
}
|
|
858
|
+
|
|
463
859
|
export function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined {
|
|
464
860
|
if (accountId === MAIN_CODEX_ACCOUNT_ID) return getMainAccountPlan();
|
|
465
|
-
return (config.codexAccounts ?? [])
|
|
861
|
+
return (config.codexAccounts ?? [])
|
|
862
|
+
.find(account => isSelectableCodexPoolAccount(account) && account.id === accountId)?.plan;
|
|
466
863
|
}
|
|
467
864
|
|
|
468
|
-
function pickLowerUsageAccount(
|
|
865
|
+
function pickLowerUsageAccount(
|
|
866
|
+
config: OcxConfig,
|
|
867
|
+
active: string,
|
|
868
|
+
activeUsage: number,
|
|
869
|
+
now: number,
|
|
870
|
+
quotaScope?: CodexQuotaScope,
|
|
871
|
+
): string {
|
|
469
872
|
let best = active;
|
|
470
873
|
let bestUsage = activeUsage;
|
|
471
|
-
for (const id of getEligiblePoolAccounts(config, active, now)) {
|
|
874
|
+
for (const id of getEligiblePoolAccounts(config, active, now, quotaScope)) {
|
|
472
875
|
const usage = computeCodexUsageScore(getAccountQuota(id), getPoolAccountPlan(config, id));
|
|
473
876
|
if (usage < bestUsage) {
|
|
474
877
|
best = id;
|
|
@@ -478,10 +881,15 @@ function pickLowerUsageAccount(config: OcxConfig, active: string, activeUsage: n
|
|
|
478
881
|
return best;
|
|
479
882
|
}
|
|
480
883
|
|
|
481
|
-
export function pickLowestUsageCodexAccount(
|
|
884
|
+
export function pickLowestUsageCodexAccount(
|
|
885
|
+
config: OcxConfig,
|
|
886
|
+
excludeId?: string,
|
|
887
|
+
now = Date.now(),
|
|
888
|
+
quotaScope?: CodexQuotaScope,
|
|
889
|
+
): string | null {
|
|
482
890
|
let best: string | null = null;
|
|
483
891
|
let bestUsage = Number.POSITIVE_INFINITY;
|
|
484
|
-
for (const id of getEligiblePoolAccounts(config, excludeId, now)) {
|
|
892
|
+
for (const id of getEligiblePoolAccounts(config, excludeId, now, quotaScope)) {
|
|
485
893
|
const usage = computeCodexUsageScore(getAccountQuota(id), getPoolAccountPlan(config, id));
|
|
486
894
|
if (usage < bestUsage) {
|
|
487
895
|
best = id;
|
|
@@ -491,17 +899,92 @@ export function pickLowestUsageCodexAccount(config: OcxConfig, excludeId?: strin
|
|
|
491
899
|
return best;
|
|
492
900
|
}
|
|
493
901
|
|
|
902
|
+
/**
|
|
903
|
+
* Strategy-aware alternate after a cooled/excluded account (same-request 429 retry
|
|
904
|
+
* and active promotion). Quota keeps lowest-usage; fill-first advances stable order;
|
|
905
|
+
* round-robin takes the next ring pick (caller should have noted the failure).
|
|
906
|
+
*/
|
|
907
|
+
export function pickAlternateCodexAccount(
|
|
908
|
+
config: OcxConfig,
|
|
909
|
+
excludeId: string,
|
|
910
|
+
now = Date.now(),
|
|
911
|
+
quotaScope?: CodexQuotaScope,
|
|
912
|
+
): string | null {
|
|
913
|
+
const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
|
|
914
|
+
if (strategy === "round-robin") {
|
|
915
|
+
const eligible = listEligibleCodexAccountIds(config, now, quotaScope).filter(id => id !== excludeId);
|
|
916
|
+
return pickRoundRobinAccount(codexPoolKeyForScope(quotaScope), eligible, stickyLimitForConfig(config));
|
|
917
|
+
}
|
|
918
|
+
if (strategy === "fill-first") {
|
|
919
|
+
const eligible = listEligibleCodexAccountIds(config, now, quotaScope).filter(id => id !== excludeId);
|
|
920
|
+
return pickNextFillFirstCodexAccount(config, excludeId, eligible, now);
|
|
921
|
+
}
|
|
922
|
+
return pickLowestUsageCodexAccount(config, excludeId, now, quotaScope);
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/** Effective active: automatic runtime cursor, else operator/persisted selection. */
|
|
926
|
+
export function getEffectiveActiveCodexAccountId(config: OcxConfig): string | undefined {
|
|
927
|
+
return runtimeActiveCodexAccountId ?? config.activeCodexAccountId;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* Automatic strategy / failover cursor only — never mutates `config.activeCodexAccountId`
|
|
932
|
+
* so an unrelated `saveConfig` cannot persist transient rotation as operator selection.
|
|
933
|
+
*/
|
|
934
|
+
function rememberActiveCodexAccount(_config: OcxConfig, accountId: string): void {
|
|
935
|
+
runtimeActiveCodexAccountId = accountId;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/** Persist operator (or quota-strategy) active selection to config + disk. */
|
|
494
939
|
function setActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
940
|
+
runtimeActiveCodexAccountId = undefined;
|
|
495
941
|
if (config.activeCodexAccountId === accountId) return;
|
|
496
942
|
config.activeCodexAccountId = accountId;
|
|
497
943
|
saveConfigPreservingClaudeCode(config);
|
|
498
944
|
}
|
|
499
945
|
|
|
946
|
+
/** Quota strategy persists; RR/fill-first keep a process-local cursor only. */
|
|
947
|
+
function promoteActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
948
|
+
if (normalizeAccountPoolStrategy(config.accountPoolStrategy) === "quota") {
|
|
949
|
+
setActiveCodexAccount(config, accountId);
|
|
950
|
+
return;
|
|
951
|
+
}
|
|
952
|
+
rememberActiveCodexAccount(config, accountId);
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Reconcile the effective active account after an administrative exclusion such as pause.
|
|
957
|
+
* The operator's persisted selection is cleared when it names the excluded account; quota
|
|
958
|
+
* keeps its historical persisted promotion, while rotating strategies retain the replacement
|
|
959
|
+
* only in the process-local cursor.
|
|
960
|
+
*/
|
|
961
|
+
export function reconcileCodexActiveAfterExclusion(
|
|
962
|
+
config: OcxConfig,
|
|
963
|
+
excludedAccountId: string,
|
|
964
|
+
now = Date.now(),
|
|
965
|
+
): string | null {
|
|
966
|
+
const wasEffective = (getEffectiveActiveCodexAccountId(config) ?? MAIN_CODEX_ACCOUNT_ID) === excludedAccountId;
|
|
967
|
+
if (config.activeCodexAccountId === excludedAccountId) {
|
|
968
|
+
config.activeCodexAccountId = undefined;
|
|
969
|
+
}
|
|
970
|
+
if (!wasEffective) return getEffectiveActiveCodexAccountId(config) ?? null;
|
|
971
|
+
|
|
972
|
+
runtimeActiveCodexAccountId = undefined;
|
|
973
|
+
const fallback = pickAlternateCodexAccount(config, excludedAccountId, now);
|
|
974
|
+
if (fallback) promoteActiveCodexAccount(config, fallback);
|
|
975
|
+
return fallback;
|
|
976
|
+
}
|
|
977
|
+
|
|
500
978
|
function isUnknownUsage(usage: number): boolean {
|
|
501
979
|
return usage >= CODEX_UNKNOWN_USAGE_SCORE;
|
|
502
980
|
}
|
|
503
981
|
|
|
504
|
-
function applyQuotaAutoSwitch(
|
|
982
|
+
function applyQuotaAutoSwitch(
|
|
983
|
+
config: OcxConfig,
|
|
984
|
+
active: string,
|
|
985
|
+
now: number,
|
|
986
|
+
quotaScope?: CodexQuotaScope,
|
|
987
|
+
): string {
|
|
505
988
|
const threshold = config.autoSwitchThreshold ?? 80;
|
|
506
989
|
if (threshold <= 0) return active;
|
|
507
990
|
const quota = getAccountQuota(active);
|
|
@@ -510,9 +993,9 @@ function applyQuotaAutoSwitch(config: OcxConfig, active: string, now: number): s
|
|
|
510
993
|
// threshold. Wait for quota priming instead of rotating among guesses.
|
|
511
994
|
if (isUnknownUsage(activeUsage)) return active;
|
|
512
995
|
if (activeUsage < threshold) return active;
|
|
513
|
-
const best = pickLowerUsageAccount(config, active, activeUsage, now);
|
|
996
|
+
const best = pickLowerUsageAccount(config, active, activeUsage, now, quotaScope);
|
|
514
997
|
if (best !== active) {
|
|
515
|
-
setActiveCodexAccount(config, best);
|
|
998
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) setActiveCodexAccount(config, best);
|
|
516
999
|
return best;
|
|
517
1000
|
}
|
|
518
1001
|
|
|
@@ -529,9 +1012,9 @@ function shouldFailover(config: OcxConfig, accountId: string, now: number): bool
|
|
|
529
1012
|
|
|
530
1013
|
function applyFailureFailover(config: OcxConfig, active: string, now: number): string {
|
|
531
1014
|
if (!shouldFailover(config, active, now)) return active;
|
|
532
|
-
const best =
|
|
1015
|
+
const best = pickAlternateCodexAccount(config, active, now);
|
|
533
1016
|
if (best) {
|
|
534
|
-
|
|
1017
|
+
promoteActiveCodexAccount(config, best);
|
|
535
1018
|
return best;
|
|
536
1019
|
}
|
|
537
1020
|
return active;
|
|
@@ -541,8 +1024,9 @@ export function resolveCodexAccountForThread(
|
|
|
541
1024
|
threadId: string | null,
|
|
542
1025
|
config: OcxConfig,
|
|
543
1026
|
now = Date.now(),
|
|
1027
|
+
quotaScope?: CodexQuotaScope,
|
|
544
1028
|
): string | null {
|
|
545
|
-
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now);
|
|
1029
|
+
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now, quotaScope);
|
|
546
1030
|
return resolution.status === "selected" ? resolution.accountId : null;
|
|
547
1031
|
}
|
|
548
1032
|
|
|
@@ -558,24 +1042,30 @@ export function previewCodexAccountForRequest(
|
|
|
558
1042
|
threadId: string | null,
|
|
559
1043
|
config: OcxConfig,
|
|
560
1044
|
now = Date.now(),
|
|
1045
|
+
quotaScope?: CodexQuotaScope,
|
|
561
1046
|
): string | null {
|
|
562
|
-
|
|
563
|
-
|
|
1047
|
+
const entry = threadId ? getThreadAffinity(threadId, quotaScope) : undefined;
|
|
1048
|
+
if (threadId && entry) {
|
|
564
1049
|
if (
|
|
565
1050
|
!isThreadAffinityExpired(entry, now)
|
|
566
1051
|
&& isThreadAffinityGenerationLive(entry)
|
|
567
|
-
&& isCodexAccountSelectable(config, entry.accountId, now)
|
|
1052
|
+
&& isCodexAccountSelectable(config, entry.accountId, now, quotaScope)
|
|
568
1053
|
&& !shouldFailover(config, entry.accountId, now)
|
|
569
1054
|
) {
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
)
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
1055
|
+
// Quota strategy only: non-quota strategies keep affinity for ongoing threads
|
|
1056
|
+
// (new-session-only rotation — docs / affinity policy A).
|
|
1057
|
+
const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
|
|
1058
|
+
if (strategy === "quota") {
|
|
1059
|
+
const threshold = config.autoSwitchThreshold ?? 80;
|
|
1060
|
+
if (threshold > 0) {
|
|
1061
|
+
const usage = computeCodexUsageScore(
|
|
1062
|
+
getAccountQuota(entry.accountId),
|
|
1063
|
+
getPoolAccountPlan(config, entry.accountId),
|
|
1064
|
+
);
|
|
1065
|
+
if (!isUnknownUsage(usage) && usage >= threshold) {
|
|
1066
|
+
const best = pickLowerUsageAccount(config, entry.accountId, usage, now, quotaScope);
|
|
1067
|
+
if (best !== entry.accountId) return best;
|
|
1068
|
+
}
|
|
579
1069
|
}
|
|
580
1070
|
}
|
|
581
1071
|
return entry.accountId;
|
|
@@ -583,14 +1073,17 @@ export function previewCodexAccountForRequest(
|
|
|
583
1073
|
// Stale/unusable affinity is ignored for preview (no map mutation).
|
|
584
1074
|
}
|
|
585
1075
|
|
|
586
|
-
|
|
1076
|
+
const strategyPick = pickUnboundStrategyAccount(config, threadId, now, false, quotaScope);
|
|
1077
|
+
if (strategyPick) return strategyPick;
|
|
1078
|
+
|
|
1079
|
+
let active = getEffectiveActiveCodexAccountId(config) ?? null;
|
|
587
1080
|
if (!active) {
|
|
588
|
-
return pickLowestUsageCodexAccount(config, undefined, now);
|
|
1081
|
+
return pickLowestUsageCodexAccount(config, undefined, now, quotaScope);
|
|
589
1082
|
}
|
|
590
|
-
if (!isCodexAccountSelectable(config, active, now)) {
|
|
591
|
-
const fallback = pickLowestUsageCodexAccount(config, active, now);
|
|
1083
|
+
if (!isCodexAccountSelectable(config, active, now, quotaScope)) {
|
|
1084
|
+
const fallback = pickLowestUsageCodexAccount(config, active, now, quotaScope);
|
|
592
1085
|
if (fallback) active = fallback;
|
|
593
|
-
else if (hasConfiguredPoolAccount(config, active)) return active;
|
|
1086
|
+
else if (hasConfiguredPoolAccount(config, active) && !isCodexAccountPaused(config, active)) return active;
|
|
594
1087
|
else return null;
|
|
595
1088
|
}
|
|
596
1089
|
|
|
@@ -598,17 +1091,18 @@ export function previewCodexAccountForRequest(
|
|
|
598
1091
|
if (threshold > 0) {
|
|
599
1092
|
const usage = computeCodexUsageScore(getAccountQuota(active), getPoolAccountPlan(config, active));
|
|
600
1093
|
if (!isUnknownUsage(usage) && usage >= threshold) {
|
|
601
|
-
active = pickLowerUsageAccount(config, active, usage, now);
|
|
1094
|
+
active = pickLowerUsageAccount(config, active, usage, now, quotaScope);
|
|
602
1095
|
}
|
|
603
1096
|
}
|
|
604
1097
|
if (shouldFailover(config, active, now)) {
|
|
605
|
-
const best = pickLowestUsageCodexAccount(config, active, now);
|
|
1098
|
+
const best = pickLowestUsageCodexAccount(config, active, now, quotaScope);
|
|
606
1099
|
if (best) active = best;
|
|
607
1100
|
}
|
|
608
1101
|
if (!isCodexAccountUsable(config, active)) {
|
|
609
1102
|
return hasConfiguredPoolAccount(config, active) ? active : null;
|
|
610
1103
|
}
|
|
611
|
-
if (
|
|
1104
|
+
if (isCodexAccountPaused(config, active)) return null;
|
|
1105
|
+
if (getCodexQuotaHealthSnapshot(active, quotaScope, now)) {
|
|
612
1106
|
return hasConfiguredPoolAccount(config, active) ? active : null;
|
|
613
1107
|
}
|
|
614
1108
|
return active;
|
|
@@ -618,16 +1112,17 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
618
1112
|
threadId: string | null,
|
|
619
1113
|
config: OcxConfig,
|
|
620
1114
|
now = Date.now(),
|
|
1115
|
+
quotaScope?: CodexQuotaScope,
|
|
621
1116
|
): CodexThreadResolution {
|
|
622
|
-
|
|
623
|
-
|
|
1117
|
+
const entry = threadId ? getThreadAffinity(threadId, quotaScope) : undefined;
|
|
1118
|
+
if (threadId && entry) {
|
|
624
1119
|
if (isThreadAffinityExpired(entry, now)) {
|
|
625
|
-
|
|
1120
|
+
deleteThreadAffinity(threadId, quotaScope);
|
|
626
1121
|
return { status: "expired", accountId: entry.accountId };
|
|
627
1122
|
}
|
|
628
1123
|
if (
|
|
629
1124
|
isThreadAffinityGenerationLive(entry)
|
|
630
|
-
&& isCodexAccountSelectable(config, entry.accountId, now)
|
|
1125
|
+
&& isCodexAccountSelectable(config, entry.accountId, now, quotaScope)
|
|
631
1126
|
// Affined threads must leave a failing account once the streak trips failover
|
|
632
1127
|
// (soft-avoid covers the first-hit case; this catches post-avoid residual streaks).
|
|
633
1128
|
&& !shouldFailover(config, entry.accountId, now)
|
|
@@ -637,19 +1132,27 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
637
1132
|
// it crosses autoSwitchThreshold and a strictly-cooler account exists.
|
|
638
1133
|
// Without this the reuse branch returns before applyQuotaAutoSwitch and the
|
|
639
1134
|
// thread stays pinned for the full idle TTL (the WSL "never switches" report).
|
|
640
|
-
|
|
641
|
-
|
|
1135
|
+
// Over-threshold pins re-eval immediately so a depleted primary does not keep
|
|
1136
|
+
// serving for up to 60s after a secondary with quota is available (#584).
|
|
1137
|
+
// Non-quota strategies (RR / fill-first) keep affinity for ongoing threads —
|
|
1138
|
+
// rotation is new-session-only (affinity policy A).
|
|
1139
|
+
const strategy = normalizeAccountPoolStrategy(config.accountPoolStrategy);
|
|
1140
|
+
if (strategy === "quota") {
|
|
642
1141
|
const threshold = config.autoSwitchThreshold ?? 80;
|
|
643
|
-
|
|
644
|
-
|
|
1142
|
+
const usage = threshold > 0
|
|
1143
|
+
? computeCodexUsageScore(
|
|
645
1144
|
getAccountQuota(entry.accountId),
|
|
646
1145
|
getPoolAccountPlan(config, entry.accountId),
|
|
647
|
-
)
|
|
648
|
-
|
|
649
|
-
|
|
1146
|
+
)
|
|
1147
|
+
: 0;
|
|
1148
|
+
const overThreshold = threshold > 0 && !isUnknownUsage(usage) && usage >= threshold;
|
|
1149
|
+
if (overThreshold || now - entry.lastReevalAt >= CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS) {
|
|
1150
|
+
entry.lastReevalAt = now;
|
|
1151
|
+
if (overThreshold) {
|
|
1152
|
+
const best = pickLowerUsageAccount(config, entry.accountId, usage, now, quotaScope);
|
|
650
1153
|
if (best !== entry.accountId) {
|
|
651
|
-
setActiveCodexAccount(config, best);
|
|
652
|
-
bindThreadAffinity(threadId, best, now); // rebinds + resets clocks
|
|
1154
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) setActiveCodexAccount(config, best);
|
|
1155
|
+
bindThreadAffinity(threadId, best, now, quotaScope); // rebinds + resets clocks
|
|
653
1156
|
return { status: "selected", accountId: best };
|
|
654
1157
|
}
|
|
655
1158
|
}
|
|
@@ -657,35 +1160,40 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
657
1160
|
}
|
|
658
1161
|
return { status: "selected", accountId: entry.accountId };
|
|
659
1162
|
}
|
|
660
|
-
|
|
1163
|
+
deleteThreadAffinity(threadId, quotaScope);
|
|
661
1164
|
}
|
|
662
|
-
|
|
1165
|
+
|
|
1166
|
+
const strategyPick = pickUnboundStrategyAccount(config, threadId, now, true, quotaScope);
|
|
1167
|
+
if (strategyPick) return { status: "selected", accountId: strategyPick };
|
|
1168
|
+
|
|
1169
|
+
let active = getEffectiveActiveCodexAccountId(config);
|
|
663
1170
|
if (!active) {
|
|
664
|
-
const selected = pickLowestUsageCodexAccount(config, undefined, now);
|
|
1171
|
+
const selected = pickLowestUsageCodexAccount(config, undefined, now, quotaScope);
|
|
665
1172
|
if (!selected) return { status: "none" };
|
|
666
|
-
setActiveCodexAccount(config, selected);
|
|
1173
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) setActiveCodexAccount(config, selected);
|
|
667
1174
|
active = selected;
|
|
668
1175
|
}
|
|
669
|
-
if (!isCodexAccountSelectable(config, active, now)) {
|
|
670
|
-
const fallback = pickLowestUsageCodexAccount(config, active, now);
|
|
1176
|
+
if (!isCodexAccountSelectable(config, active, now, quotaScope)) {
|
|
1177
|
+
const fallback = pickLowestUsageCodexAccount(config, active, now, quotaScope);
|
|
671
1178
|
if (fallback) {
|
|
672
|
-
setActiveCodexAccount(config, fallback);
|
|
1179
|
+
if (!isIndependentCodexQuotaScope(quotaScope)) setActiveCodexAccount(config, fallback);
|
|
673
1180
|
active = fallback;
|
|
674
|
-
} else if (hasConfiguredPoolAccount(config, active)) {
|
|
1181
|
+
} else if (hasConfiguredPoolAccount(config, active) && !isCodexAccountPaused(config, active)) {
|
|
675
1182
|
return { status: "selected", accountId: active };
|
|
676
1183
|
} else {
|
|
677
1184
|
return { status: "none" };
|
|
678
1185
|
}
|
|
679
1186
|
}
|
|
680
|
-
active = applyQuotaAutoSwitch(config, active, now);
|
|
1187
|
+
active = applyQuotaAutoSwitch(config, active, now, quotaScope);
|
|
681
1188
|
active = applyFailureFailover(config, active, now);
|
|
682
1189
|
if (!isCodexAccountUsable(config, active)) {
|
|
683
1190
|
return hasConfiguredPoolAccount(config, active) ? { status: "selected", accountId: active } : { status: "none" };
|
|
684
1191
|
}
|
|
685
|
-
if (
|
|
1192
|
+
if (isCodexAccountPaused(config, active)) return { status: "none" };
|
|
1193
|
+
if (getCodexQuotaHealthSnapshot(active, quotaScope, now)) {
|
|
686
1194
|
return hasConfiguredPoolAccount(config, active) ? { status: "selected", accountId: active } : { status: "none" };
|
|
687
1195
|
}
|
|
688
|
-
if (threadId) bindThreadAffinity(threadId, active, now);
|
|
1196
|
+
if (threadId) bindThreadAffinity(threadId, active, now, quotaScope);
|
|
689
1197
|
return { status: "selected", accountId: active };
|
|
690
1198
|
}
|
|
691
1199
|
|
|
@@ -698,7 +1206,18 @@ export function recordCodexUpstreamOutcome(
|
|
|
698
1206
|
if (!accountId) return;
|
|
699
1207
|
const now = meta.now ?? Date.now();
|
|
700
1208
|
const outcomeClass = classifyCodexUpstreamOutcome(outcome);
|
|
1209
|
+
const quotaScope = codexQuotaScopeForModel(meta.modelId);
|
|
701
1210
|
if (outcomeClass === "success") {
|
|
1211
|
+
const scopedProbe = meta.probeQuotaScope
|
|
1212
|
+
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
1213
|
+
: undefined;
|
|
1214
|
+
if (scopedProbe && meta.probeQuotaScope) {
|
|
1215
|
+
if (scopedProbe.cooldownUntil && probeMayClearCooldown(scopedProbe, meta)) {
|
|
1216
|
+
deleteScopedHealth(accountId, meta.probeQuotaScope);
|
|
1217
|
+
} else if (ownsProbeLease(scopedProbe, meta)) {
|
|
1218
|
+
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
702
1221
|
const current = upstreamHealth.get(accountId);
|
|
703
1222
|
const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
|
|
704
1223
|
// A leased probe that is still on its own cooldown generation proves the
|
|
@@ -734,6 +1253,12 @@ export function recordCodexUpstreamOutcome(
|
|
|
734
1253
|
// A 4xx does not change account health, but it does conclude an in-flight
|
|
735
1254
|
// probe — otherwise the lease would never be handed back.
|
|
736
1255
|
const current = upstreamHealth.get(accountId);
|
|
1256
|
+
const scopedProbe = meta.probeQuotaScope
|
|
1257
|
+
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
1258
|
+
: undefined;
|
|
1259
|
+
if (scopedProbe && meta.probeQuotaScope && ownsProbeLease(scopedProbe, meta)) {
|
|
1260
|
+
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
1261
|
+
}
|
|
737
1262
|
if (ownsProbeLease(current, meta)) {
|
|
738
1263
|
upstreamHealth.set(accountId, withProbeLeaseReleased(current!, now));
|
|
739
1264
|
}
|
|
@@ -749,14 +1274,65 @@ export function recordCodexUpstreamOutcome(
|
|
|
749
1274
|
lastFailureStatus,
|
|
750
1275
|
lastFailureAt: now,
|
|
751
1276
|
});
|
|
1277
|
+
quotaScopedHealth.delete(accountId);
|
|
752
1278
|
markAccountNeedsReauth(accountId);
|
|
753
1279
|
clearThreadAccountMapForAccount(accountId);
|
|
754
1280
|
return;
|
|
755
1281
|
}
|
|
756
1282
|
|
|
757
1283
|
if (outcomeClass === "quota") {
|
|
758
|
-
const prior = upstreamHealth.get(accountId);
|
|
759
1284
|
const { until, source } = computeQuotaCooldown(meta);
|
|
1285
|
+
// A reset timestamp is an advisory quota-window announcement. When the
|
|
1286
|
+
// selected native model belongs to a confirmed independent group, preserve
|
|
1287
|
+
// it there so a different group (Spark versus the shared native quota) can
|
|
1288
|
+
// still reach upstream. Explicit Retry-After/default 429s remain account-wide.
|
|
1289
|
+
if (source === "reset-derived" && quotaScope) {
|
|
1290
|
+
const prior = scopedHealthFor(accountId, quotaScope);
|
|
1291
|
+
const cooldownGeneration = (prior?.cooldownGeneration ?? 0) + 1;
|
|
1292
|
+
const ownsLease = meta.probeQuotaScope === quotaScope && ownsProbeLease(prior, meta);
|
|
1293
|
+
setScopedHealth(accountId, quotaScope, {
|
|
1294
|
+
consecutiveFailures: 0,
|
|
1295
|
+
lastFailureStatus,
|
|
1296
|
+
lastFailureAt: now,
|
|
1297
|
+
cooldownUntil: until,
|
|
1298
|
+
cooldownSince: now,
|
|
1299
|
+
cooldownSource: source,
|
|
1300
|
+
cooldownGeneration,
|
|
1301
|
+
...(ownsLease
|
|
1302
|
+
? { lastProbeAt: now }
|
|
1303
|
+
: {
|
|
1304
|
+
...(prior?.probeLeaseId !== undefined ? { probeLeaseId: prior.probeLeaseId } : {}),
|
|
1305
|
+
...(prior?.probeLeaseGeneration !== undefined ? { probeLeaseGeneration: prior.probeLeaseGeneration } : {}),
|
|
1306
|
+
...(prior?.lastProbeAt !== undefined ? { lastProbeAt: prior.lastProbeAt } : {}),
|
|
1307
|
+
}),
|
|
1308
|
+
});
|
|
1309
|
+
// The shared native scope is the existing account-wide native behavior:
|
|
1310
|
+
// threads must leave it and new requests should prefer an eligible account.
|
|
1311
|
+
// Spark remains isolated so a same-account Terra/Luna combo fallback can run.
|
|
1312
|
+
if (quotaScope === "shared") {
|
|
1313
|
+
clearThreadAccountMapForAccount(accountId);
|
|
1314
|
+
notePoolRotationFailure(POOL_KEY_CODEX, accountId);
|
|
1315
|
+
if (getEffectiveActiveCodexAccountId(config) === accountId) {
|
|
1316
|
+
// Same-request 429 retry already picked via excludeAccountId — reuse it so
|
|
1317
|
+
// round-robin does not advance the ring a second time.
|
|
1318
|
+
const reused = meta.promoteAccountId && meta.promoteAccountId !== accountId
|
|
1319
|
+
? meta.promoteAccountId
|
|
1320
|
+
: null;
|
|
1321
|
+
const fallback = reused ?? pickAlternateCodexAccount(config, accountId, now, quotaScope);
|
|
1322
|
+
if (fallback) promoteActiveCodexAccount(config, fallback);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
return;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
// A scoped probe that received an account-wide throttle is no longer live.
|
|
1329
|
+
const scopedProbe = meta.probeQuotaScope
|
|
1330
|
+
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
1331
|
+
: undefined;
|
|
1332
|
+
if (scopedProbe && meta.probeQuotaScope && ownsProbeLease(scopedProbe, meta)) {
|
|
1333
|
+
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
1334
|
+
}
|
|
1335
|
+
const prior = upstreamHealth.get(accountId);
|
|
760
1336
|
// Every cooldown write bumps the generation so a probe issued against the
|
|
761
1337
|
// previous cooldown can no longer clear this one (#433).
|
|
762
1338
|
const cooldownGeneration = (prior?.cooldownGeneration ?? 0) + 1;
|
|
@@ -779,15 +1355,28 @@ export function recordCodexUpstreamOutcome(
|
|
|
779
1355
|
}),
|
|
780
1356
|
});
|
|
781
1357
|
clearThreadAccountMapForAccount(accountId);
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
1358
|
+
notePoolRotationFailure(POOL_KEY_CODEX, accountId);
|
|
1359
|
+
const effectiveActive = getEffectiveActiveCodexAccountId(config);
|
|
1360
|
+
if (effectiveActive === accountId) {
|
|
1361
|
+
// Same-request 429 retry already picked via excludeAccountId — reuse it so
|
|
1362
|
+
// round-robin does not advance the ring a second time.
|
|
1363
|
+
const reused = meta.promoteAccountId && meta.promoteAccountId !== accountId
|
|
1364
|
+
? meta.promoteAccountId
|
|
1365
|
+
: null;
|
|
1366
|
+
const fallback = reused ?? pickAlternateCodexAccount(config, accountId, now);
|
|
1367
|
+
if (fallback) promoteActiveCodexAccount(config, fallback);
|
|
785
1368
|
}
|
|
786
1369
|
return;
|
|
787
1370
|
}
|
|
788
1371
|
|
|
789
1372
|
// transient (connect_error / timeout / 5xx)
|
|
790
1373
|
const current = upstreamHealth.get(accountId);
|
|
1374
|
+
const scopedProbe = meta.probeQuotaScope
|
|
1375
|
+
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
1376
|
+
: undefined;
|
|
1377
|
+
if (scopedProbe && meta.probeQuotaScope && ownsProbeLease(scopedProbe, meta)) {
|
|
1378
|
+
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
1379
|
+
}
|
|
791
1380
|
// A transient failure concludes an owning probe; an unrelated 5xx must not
|
|
792
1381
|
// consume someone else's live lease or drop hard-cooldown bookkeeping (#433).
|
|
793
1382
|
const transientBase = ownsProbeLease(current, meta) ? withProbeLeaseReleased(current!, now) : current;
|
|
@@ -821,15 +1410,14 @@ export function recordCodexUpstreamOutcome(
|
|
|
821
1410
|
// must not delete a newer healthy binding to account B (race: T→A, A fails,
|
|
822
1411
|
// T→B, late A failure must not delete B's mapping).
|
|
823
1412
|
if (failoverReady && meta.threadId) {
|
|
824
|
-
|
|
825
|
-
if (bound?.accountId === accountId) threadAccountMap.delete(meta.threadId);
|
|
1413
|
+
deleteThreadAffinitiesForAccount(meta.threadId, accountId);
|
|
826
1414
|
}
|
|
827
1415
|
// Once the account is past the failover streak, clear every thread still pinned
|
|
828
1416
|
// to it — matching 429 affinity behavior so "continue" cannot stay on a bad peer.
|
|
829
1417
|
if (shouldFailover(config, accountId, now)) {
|
|
830
1418
|
clearThreadAccountMapForAccount(accountId);
|
|
831
1419
|
}
|
|
832
|
-
if (config
|
|
1420
|
+
if (getEffectiveActiveCodexAccountId(config) === accountId) applyFailureFailover(config, accountId, now);
|
|
833
1421
|
}
|
|
834
1422
|
|
|
835
1423
|
export function formatCodexProviderForLog(providerName: string, accountId: string | null, config: OcxConfig): string {
|
|
@@ -838,6 +1426,7 @@ export function formatCodexProviderForLog(providerName: string, accountId: strin
|
|
|
838
1426
|
// same physical account as the "main" passthrough (null accountId). Log both under the base provider
|
|
839
1427
|
// name so usage/tokens aggregate into a single row instead of splitting into `chatgpt` + `chatgpt-main`.
|
|
840
1428
|
if (accountId === MAIN_CODEX_ACCOUNT_ID) return providerName;
|
|
841
|
-
const account = (config.codexAccounts ?? [])
|
|
1429
|
+
const account = (config.codexAccounts ?? [])
|
|
1430
|
+
.find(candidate => isSelectableCodexPoolAccount(candidate) && candidate.id === accountId);
|
|
842
1431
|
return account ? `${providerName}-${codexAccountLogLabel(account)}` : providerName;
|
|
843
1432
|
}
|