@bitkyc08/opencodex 2.55.0-preview.20260914 → 2.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gui/dist/assets/{index-DH2PUHqr.js → index-D4zuyIxQ.js} +1 -1
- package/gui/dist/index.html +1 -1
- package/package.json +2 -1
- package/src/adapters/base.ts +21 -0
- package/src/adapters/cursor/transport-retry.ts +46 -1
- package/src/adapters/cursor.ts +4 -0
- package/src/adapters/kiro/adapter.ts +42 -1
- package/src/adapters/kiro-retry.ts +23 -4
- package/src/adapters/openai-chat/errors.ts +116 -0
- package/src/adapters/openai-chat/messages.ts +346 -0
- package/src/adapters/openai-chat/passthrough.ts +146 -0
- package/src/adapters/openai-chat/response-events.ts +117 -0
- package/src/adapters/openai-chat/tool-call-validation.ts +200 -0
- package/src/adapters/openai-chat/tool-schema.ts +477 -0
- package/src/adapters/openai-chat/wire.ts +50 -0
- package/src/adapters/openai-chat.ts +33 -1445
- package/src/adapters/openai-responses/canonical-forward.ts +202 -0
- package/src/adapters/openai-responses/image-gen.ts +406 -0
- package/src/adapters/openai-responses/internal.ts +3 -0
- package/src/adapters/openai-responses/passthrough.ts +611 -0
- package/src/adapters/openai-responses/prompt-cache.ts +83 -0
- package/src/adapters/openai-responses/reasoning.ts +220 -0
- package/src/adapters/openai-responses/request-strips.ts +185 -0
- package/src/adapters/openai-responses/tool-output-recovery.ts +509 -0
- package/src/adapters/openai-responses/tool-schema.ts +293 -0
- package/src/adapters/openai-responses/web-search.ts +156 -0
- package/src/adapters/openai-responses.ts +4 -2625
- package/src/bridge/errors.ts +34 -0
- package/src/bridge/internal.ts +174 -0
- package/src/bridge/response-json.ts +624 -0
- package/src/bridge/sse.ts +1444 -0
- package/src/bridge.ts +5 -2204
- package/src/chat/inbound.ts +12 -1
- package/src/codex/account-lifecycle.ts +3 -0
- package/src/codex/account-store.ts +71 -9
- package/src/codex/auth-api/account-list.ts +507 -0
- package/src/codex/auth-api/http.ts +32 -0
- package/src/codex/auth-api/login-flow.ts +554 -0
- package/src/codex/auth-api/login-state.ts +64 -0
- package/src/codex/auth-api/main-account-probe.ts +331 -0
- package/src/codex/auth-api/pool-mode-gate.ts +274 -0
- package/src/codex/auth-api/pool-quota-probe.ts +512 -0
- package/src/codex/auth-api/reset-credit-service.ts +422 -0
- package/src/codex/auth-api/routes.ts +425 -0
- package/src/codex/auth-api/runtime-config.ts +48 -0
- package/src/codex/auth-api.ts +27 -3118
- package/src/codex/auth-context.ts +95 -28
- package/src/codex/catalog/auto-review.ts +507 -0
- package/src/codex/catalog/build-entries.ts +981 -0
- package/src/codex/catalog/combo-member.ts +375 -0
- package/src/codex/catalog/derive-entry.ts +229 -0
- package/src/codex/catalog/effort.ts +0 -1
- package/src/codex/catalog/gated-native-warn.ts +63 -0
- package/src/codex/catalog/gather-capture.ts +533 -0
- package/src/codex/catalog/model-hints.ts +691 -0
- package/src/codex/catalog/model-visibility.ts +304 -0
- package/src/codex/catalog/provider-fetch.ts +52 -2942
- package/src/codex/catalog/provider-models.ts +685 -0
- package/src/codex/catalog/restore.ts +132 -0
- package/src/codex/catalog/retained-sync.ts +706 -0
- package/src/codex/catalog/routed-gather.ts +858 -0
- package/src/codex/catalog/subagent-roster.ts +176 -0
- package/src/codex/catalog/sync.ts +52 -2698
- package/src/codex/inject/config-toml.ts +563 -0
- package/src/codex/inject/remove.ts +192 -0
- package/src/codex/inject/restore.ts +540 -0
- package/src/codex/inject/routing-classify.ts +109 -0
- package/src/codex/inject/routing-target.ts +125 -0
- package/src/codex/inject.ts +81 -1436
- package/src/codex/lineage.ts +458 -0
- package/src/codex/pool-refresh-backoff.ts +152 -0
- package/src/codex/routing/active-account.ts +194 -0
- package/src/codex/routing/cooldown-math.ts +275 -0
- package/src/codex/routing/health-store.ts +402 -0
- package/src/codex/routing/probe-lease.ts +358 -0
- package/src/codex/routing/selection.ts +703 -0
- package/src/codex/routing/thread-affinity.ts +538 -0
- package/src/codex/routing.ts +353 -2234
- package/src/codex/shim-fingerprint.ts +223 -0
- package/src/codex/shim-inspect.ts +175 -0
- package/src/codex/shim-probe.ts +367 -0
- package/src/codex/shim-restore-lock.ts +169 -0
- package/src/codex/shim-state-file.ts +151 -0
- package/src/codex/shim-templates.ts +265 -0
- package/src/codex/shim.ts +48 -1268
- package/src/config/diagnostics.ts +705 -0
- package/src/config/feature-flags.ts +55 -0
- package/src/config/live-reconcile.ts +403 -0
- package/src/config/load-degrade.ts +880 -0
- package/src/config/mutation-lock.ts +244 -0
- package/src/config/openai-tier-backup.ts +268 -0
- package/src/config/persist-unlocked.ts +92 -0
- package/src/config/proxy-env.ts +188 -0
- package/src/config/salvage.ts +244 -0
- package/src/config/schema/config-schema.ts +640 -0
- package/src/config/schema/leaf-validators.ts +855 -0
- package/src/config/warn-memo.ts +28 -0
- package/src/config.ts +234 -4481
- package/src/generated/compatibility-version.json +539 -39
- package/src/lib/request-execution-budget.ts +69 -20
- package/src/lib/spend-reservation-ledger.ts +940 -0
- package/src/lib/upstream-retry.ts +55 -11
- package/src/lib/workflow-budget.ts +553 -30
- package/src/providers/quota/account-cache.ts +441 -0
- package/src/providers/quota/antigravity.ts +295 -0
- package/src/providers/quota/report-cache.ts +320 -0
- package/src/providers/quota/vendor-probes-key.ts +1243 -0
- package/src/providers/quota/vendor-probes-oauth.ts +590 -0
- package/src/providers/quota.ts +324 -3079
- package/src/providers/registry/entries-core.ts +1221 -0
- package/src/providers/registry/entries-extended.ts +1204 -0
- package/src/providers/registry/model-seeds.ts +908 -0
- package/src/providers/registry/types.ts +352 -0
- package/src/providers/registry.ts +24 -3536
- package/src/responses/continuation-ownership.ts +29 -0
- package/src/responses/state/replay-fingerprint.ts +80 -0
- package/src/responses/state/snapshot-codec.ts +104 -0
- package/src/responses/state/spill-failure.ts +118 -0
- package/src/responses/state/spill-queue.ts +665 -0
- package/src/responses/state/temp-recovery.ts +257 -0
- package/src/responses/state.ts +82 -1143
- package/src/routing/identity-domains.ts +449 -0
- package/src/routing/probe-lease.ts +511 -0
- package/src/server/index/bounded-request.ts +88 -0
- package/src/server/index/live-sideband.ts +565 -0
- package/src/server/index/serve-options.ts +1766 -0
- package/src/server/index/startup-warnings.ts +213 -0
- package/src/server/index/websocket-handler.ts +335 -0
- package/src/server/index.ts +40 -2547
- package/src/server/management/route-registry.ts +26 -23
- package/src/server/management/shared.ts +8 -5
- package/src/server/management/workflow-budget-routes.ts +133 -0
- package/src/server/management-api.ts +12 -0
- package/src/server/request-log-conversation.ts +9 -7
- package/src/server/request-log.ts +245 -1
- package/src/server/responses/account-change-state.ts +233 -0
- package/src/server/responses/adapter-continuation.ts +514 -0
- package/src/server/responses/adapter-delivery.ts +214 -0
- package/src/server/responses/adapter-dispatch.ts +971 -0
- package/src/server/responses/compact.ts +59 -4
- package/src/server/responses/completion-policy.ts +33 -0
- package/src/server/responses/core-auth.ts +527 -0
- package/src/server/responses/core-codex-account.ts +859 -0
- package/src/server/responses/core-combo-failure.ts +210 -0
- package/src/server/responses/core-combo.ts +707 -0
- package/src/server/responses/core-errors.ts +152 -0
- package/src/server/responses/core-lifetime.ts +95 -0
- package/src/server/responses/core-normalize.ts +350 -0
- package/src/server/responses/core-opaque-recovery.ts +380 -0
- package/src/server/responses/core-options.ts +159 -0
- package/src/server/responses/core-replay.ts +225 -0
- package/src/server/responses/core.ts +192 -8893
- package/src/server/responses/passthrough-delivery.ts +856 -0
- package/src/server/responses/passthrough-dispatch.ts +1476 -0
- package/src/server/responses/passthrough-execution.ts +54 -0
- package/src/server/responses/request-prepare.ts +970 -0
- package/src/server/responses/request-send-budget.ts +164 -0
- package/src/server/responses/request-sidecar-auth.ts +149 -0
- package/src/server/responses/request-transport.ts +744 -0
- package/src/server/responses/response-effects.ts +157 -0
- package/src/server/responses/run-turn-execution.ts +448 -0
- package/src/server/responses/sidecar-execution.ts +469 -0
- package/src/server/responses-image-gen-repair.ts +1 -1
- package/src/server/workflow-refusal.ts +84 -0
- package/src/types/config.ts +30 -0
- package/src/usage/log.ts +146 -0
- package/src/usage/summary.ts +171 -21
package/src/codex/routing.ts
CHANGED
|
@@ -1,409 +1,198 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
1
|
import { saveConfigPreservingClaudeCode } from "../config";
|
|
3
|
-
import { isCodexAccountGenerationLive
|
|
2
|
+
import { isCodexAccountGenerationLive } from "./account-store";
|
|
4
3
|
import { codexAccountLogLabel } from "./account-label";
|
|
5
|
-
import { NATIVE_RESERVE_MODEL } from "./catalog/native-models";
|
|
6
4
|
import { isCodexAccountPaused } from "./account-pause";
|
|
7
|
-
import { clearCodexAccountPin,
|
|
5
|
+
import { clearCodexAccountPin, pinnedCodexAccountId } from "./account-priority";
|
|
8
6
|
import { isCodexAccountUsable, type CodexAccountUsabilityOptions } from "./account-usability";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
normalizeCodexAccountPoolStrategy,
|
|
14
|
-
notePoolRotationFailure,
|
|
15
|
-
notePoolRotationSuccess,
|
|
16
|
-
peekRoundRobinAccount,
|
|
17
|
-
pickRoundRobinAccount,
|
|
18
|
-
seedPoolRotationAccount,
|
|
19
|
-
selectPriorityTier,
|
|
20
|
-
} from "./pool-rotation";
|
|
21
|
-
import {
|
|
22
|
-
CODEX_EXHAUSTED_USAGE_PERCENT,
|
|
23
|
-
CODEX_UNKNOWN_USAGE_SCORE,
|
|
24
|
-
getAccountQuota,
|
|
25
|
-
isRetiredCodexSparkModel,
|
|
26
|
-
resetAtToMs,
|
|
27
|
-
} from "./quota";
|
|
28
|
-
import { codexPlanKey, isThirtyDayOnlyCodexPlan } from "./plan";
|
|
29
|
-
import {
|
|
30
|
-
MAIN_CODEX_ACCOUNT_ID,
|
|
31
|
-
getMainAccountPlan,
|
|
32
|
-
hasMainAccountRefreshGrant,
|
|
33
|
-
} from "./main-account";
|
|
7
|
+
import { isAccountNeedsReauth, markAccountNeedsReauth } from "./account-runtime-state";
|
|
8
|
+
import { POOL_KEY_CODEX, notePoolRotationFailure } from "./pool-rotation";
|
|
9
|
+
import { getAccountQuota, isRetiredCodexSparkModel } from "./quota";
|
|
10
|
+
import { MAIN_CODEX_ACCOUNT_ID } from "./main-account";
|
|
34
11
|
import { isSelectableCodexPoolAccount } from "./account-id";
|
|
35
12
|
import type { OcxConfig } from "../types";
|
|
36
13
|
import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper";
|
|
37
|
-
import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers";
|
|
38
|
-
import { retainedUtf8Bytes } from "../lib/admission";
|
|
39
14
|
import { recordUpstreamHostFailure } from "./upstream-host-health";
|
|
15
|
+
import type { CodexThreadLineage } from "./lineage";
|
|
40
16
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
consecutiveFailures: number;
|
|
146
|
-
/** Consecutive healthy terminals observed while recovering from escalation level 2+. */
|
|
147
|
-
consecutiveSuccesses?: number;
|
|
148
|
-
lastFailureStatus?: number;
|
|
149
|
-
lastFailureAt?: number;
|
|
150
|
-
/** Hard cooldown (quota 429). Survives a later 2xx; blocks auth + selection. */
|
|
151
|
-
cooldownUntil?: number;
|
|
152
|
-
/**
|
|
153
|
-
* How long a quota refusal keeps selection away from this account (or this native quota
|
|
154
|
-
* group), as opposed to how long it is hard-blocked.
|
|
155
|
-
*
|
|
156
|
-
* The two are deliberately different lengths. {@link CODEX_MAX_RESET_DERIVED_COOLDOWN_MS}
|
|
157
|
-
* caps the hard cooldown at 15 minutes because a reset announcement is advisory and plan
|
|
158
|
-
* quota usually frees up before it — an account must stay reachable so the pool can find
|
|
159
|
-
* that out (#433). The window the refusal announced is not 15 minutes, though, so once the
|
|
160
|
-
* cooldown lapses the account is selectable again while its burst window is still spent,
|
|
161
|
-
* and the strategy picks it straight back: this proxy reads a weekly bar a burst limit never
|
|
162
|
-
* touches, so a refused account still scores as the coolest in the pool. Every request then
|
|
163
|
-
* earns the same 429 until the process restarts, which is the only thing that drops this map.
|
|
164
|
-
*
|
|
165
|
-
* So the announcement governs avoidance and the cap still governs blocking. Avoidance is soft
|
|
166
|
-
* in the {@link softAvoidUntil} sense: it reorders the pool and releases a bound thread, and
|
|
167
|
-
* the last-resort paths still reach the account when nothing else can serve, so one pessimistic
|
|
168
|
-
* announcement cannot stall routing.
|
|
169
|
-
*/
|
|
170
|
-
quotaAvoidUntil?: number;
|
|
171
|
-
/** When the current cooldown was recorded; origin of the probe interval clock. */
|
|
172
|
-
cooldownSince?: number;
|
|
173
|
-
/**
|
|
174
|
-
* What produced the cooldown. An explicit Retry-After is a literal retry
|
|
175
|
-
* directive and is never probed; a quota resetAt only announces a window
|
|
176
|
-
* refresh, so it may be probed early (#433).
|
|
177
|
-
*/
|
|
178
|
-
cooldownSource?: CodexCooldownSource;
|
|
179
|
-
/**
|
|
180
|
-
* Bumped on every cooldown write. A probe lease records the generation it was
|
|
181
|
-
* issued for so a lease cannot clear a cooldown that a later 429 replaced.
|
|
182
|
-
*/
|
|
183
|
-
cooldownGeneration?: number;
|
|
184
|
-
/**
|
|
185
|
-
* Identity of the in-flight probe. A cooled-down account sends no traffic, so
|
|
186
|
-
* no organic 2xx can prove recovery; only the outcome carrying this id may
|
|
187
|
-
* clear the cooldown.
|
|
188
|
-
*/
|
|
189
|
-
probeLeaseId?: string;
|
|
190
|
-
/** Cooldown generation at the moment the lease was granted. */
|
|
191
|
-
probeLeaseGeneration?: number;
|
|
192
|
-
/** Last probe grant or conclusion; paces the probe interval. */
|
|
193
|
-
lastProbeAt?: number;
|
|
194
|
-
/**
|
|
195
|
-
* Soft avoid after connect_error / timeout / transient 5xx. Cleared on 2xx.
|
|
196
|
-
* Blocks pool selection + thread affinity reuse so a sticky session can leave a
|
|
197
|
-
* flaky account without throwing CodexAccountCooldownError (hard-only).
|
|
198
|
-
*/
|
|
199
|
-
softAvoidUntil?: number;
|
|
200
|
-
/**
|
|
201
|
-
* Credential generation a 401/403 quarantine was derived from (#2892 gap 4).
|
|
202
|
-
*
|
|
203
|
-
* Provenance lives ON the entry rather than in a side map keyed by account id. A side map spends
|
|
204
|
-
* "whatever health is current when the old credential is found dead", which deletes a later
|
|
205
|
-
* unrelated entry: a G1 401, then a G2 save, then a genuine G2 503 would lose the 503. Only the
|
|
206
|
-
* entry that carries this field can be spent, and any later write simply replaces it.
|
|
207
|
-
*/
|
|
208
|
-
credentialFailureGeneration?: number;
|
|
209
|
-
};
|
|
210
|
-
|
|
211
|
-
const CODEX_DEFAULT_QUOTA_COOLDOWN_MS = 60_000;
|
|
212
|
-
const CODEX_MAX_QUOTA_COOLDOWN_MS = 24 * 60 * 60_000;
|
|
213
|
-
/**
|
|
214
|
-
* A weekly/monthly quota `resetAt` announces when the window refreshes; it is not
|
|
215
|
-
* a "come back after this" directive like Retry-After. Plan quota routinely frees
|
|
216
|
-
* up long before the advertised reset, so cap reset-derived cooldowns far below
|
|
217
|
-
* the Retry-After ceiling (#433).
|
|
218
|
-
*/
|
|
219
|
-
const CODEX_MAX_RESET_DERIVED_COOLDOWN_MS = 15 * 60_000;
|
|
220
|
-
/**
|
|
221
|
-
* Ceiling on quota-refusal avoidance. Generous enough to cover a full five-hour burst window,
|
|
222
|
-
* tight enough that a weekly or monthly reset four days out cannot take an account out of
|
|
223
|
-
* rotation for the {@link CODEX_MAX_QUOTA_COOLDOWN_MS} day the Retry-After ceiling allows.
|
|
224
|
-
*/
|
|
225
|
-
const CODEX_MAX_QUOTA_AVOID_MS = 6 * 60 * 60_000;
|
|
226
|
-
/** Minimum gap between probe leases for one cooled-down account. */
|
|
227
|
-
export const CODEX_QUOTA_PROBE_INTERVAL_MS = 5 * 60_000;
|
|
228
|
-
export const CODEX_FAILURE_WINDOW_MS = 5 * 60_000;
|
|
229
|
-
/**
|
|
230
|
-
* How recently a 100% burst reading must have been OBSERVED to exclude an account when it
|
|
231
|
-
* carries no reset timestamp (#3425). Deliberately far tighter than the 6h disk-hydration
|
|
232
|
-
* horizon in `quota.ts`: shorter than any plausible five-hour burst window, so a persisted
|
|
233
|
-
* reading can never strand a recovered account, and long enough that a snapshot taken at
|
|
234
|
-
* admission is still fresh when selection reads it.
|
|
235
|
-
*/
|
|
236
|
-
export const TERMINAL_SHORT_WINDOW_FRESHNESS_MS = 5 * 60_000;
|
|
237
|
-
/** How long a transient failure keeps the account out of pool selection. */
|
|
238
|
-
export const CODEX_TRANSIENT_SOFT_AVOID_MS = 30_000;
|
|
239
|
-
const CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS = [
|
|
17
|
+
import { isCodexPoolRefreshCooling } from "./pool-refresh-backoff";
|
|
18
|
+
import {
|
|
19
|
+
classifyCodexUpstreamOutcome,
|
|
20
|
+
computeCodexUsageScore,
|
|
21
|
+
computeQuotaCooldown,
|
|
22
|
+
quotaAvoidUntilFor,
|
|
23
|
+
CODEX_FAILURE_WINDOW_MS,
|
|
24
|
+
CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS,
|
|
25
|
+
type CodexUpstreamOutcome,
|
|
26
|
+
type CodexUpstreamOutcomeMeta,
|
|
27
|
+
} from "./routing/cooldown-math";
|
|
28
|
+
import {
|
|
29
|
+
codexPoolKeyForScope,
|
|
30
|
+
codexQuotaScopeForModel,
|
|
31
|
+
deleteAccountHealth,
|
|
32
|
+
deleteAllScopedHealth,
|
|
33
|
+
deleteScopedHealth,
|
|
34
|
+
dropSpentCredentialFailure,
|
|
35
|
+
getAccountHealth,
|
|
36
|
+
getCodexAccountCooldownUntil,
|
|
37
|
+
getCodexAccountSoftAvoidUntil,
|
|
38
|
+
getCodexQuotaHealthSnapshot,
|
|
39
|
+
isCodexAccountSoftAvoided,
|
|
40
|
+
isCodexQuotaAvoided,
|
|
41
|
+
isHealthAccountAdmissible,
|
|
42
|
+
isHealthGenerationReconciled,
|
|
43
|
+
isIndependentCodexQuotaScope,
|
|
44
|
+
preservedCooldownFields,
|
|
45
|
+
pruneHealthAccountsForContext,
|
|
46
|
+
commitHealthReconcile,
|
|
47
|
+
clearUpstreamHealthState,
|
|
48
|
+
resetHealthReconcileState,
|
|
49
|
+
deleteAllHealthForAccount,
|
|
50
|
+
scopedHealthFor,
|
|
51
|
+
setAccountHealth,
|
|
52
|
+
setScopedHealth,
|
|
53
|
+
type CodexQuotaScope,
|
|
54
|
+
type CodexUpstreamHealth,
|
|
55
|
+
} from "./routing/health-store";
|
|
56
|
+
import { ownsProbeLease, probeMayClearCooldown, withProbeLeaseReleased } from "./routing/probe-lease";
|
|
57
|
+
import {
|
|
58
|
+
adoptLegacyLineageAffinity,
|
|
59
|
+
affinityAfterRelease,
|
|
60
|
+
affinityOnNoAccount,
|
|
61
|
+
bindModelDetourAffinity,
|
|
62
|
+
bindThreadAffinity,
|
|
63
|
+
clearThreadAccountMapForAccount,
|
|
64
|
+
deleteModelDetourAffinity,
|
|
65
|
+
deleteThreadAffinity,
|
|
66
|
+
deleteThreadAffinitiesForAccount,
|
|
67
|
+
getThreadAffinity,
|
|
68
|
+
getThreadAffinityScopes,
|
|
69
|
+
getModelDetourAffinity,
|
|
70
|
+
isThreadAffinityExpired,
|
|
71
|
+
isThreadAffinityGenerationLive,
|
|
72
|
+
peekPendingReleaseReason,
|
|
73
|
+
CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS,
|
|
74
|
+
CODEX_TRANSIENT_AFFINITY_HOLD_MS,
|
|
75
|
+
type CodexAffinityReason,
|
|
76
|
+
type CodexThreadResolution,
|
|
77
|
+
type ThreadAffinityEntry,
|
|
78
|
+
} from "./routing/thread-affinity";
|
|
79
|
+
import {
|
|
80
|
+
accountPoolStrategyForScope,
|
|
81
|
+
applyFailureFailover,
|
|
82
|
+
applyQuotaAutoSwitch,
|
|
83
|
+
codexAccountBlockReason,
|
|
84
|
+
getEligiblePoolAccounts,
|
|
85
|
+
getPoolAccountPlanForSelection,
|
|
86
|
+
hasCodexQuotaHeadroom,
|
|
87
|
+
isCodexAccountPlanExcluded,
|
|
88
|
+
isCacheAffinityEnabled,
|
|
89
|
+
isCodexAccountSelectable,
|
|
90
|
+
isHealthySharedCodexSelection,
|
|
91
|
+
isUnknownUsage,
|
|
92
|
+
pickAlternateCodexAccount,
|
|
93
|
+
pickLowerUsageAccount,
|
|
94
|
+
pickLowestUsageAmong,
|
|
95
|
+
pickLowestUsageCodexAccount,
|
|
96
|
+
pickPriorityPreemption,
|
|
97
|
+
pickResetFirstCodexAccount,
|
|
98
|
+
pickUnboundStrategyAccount,
|
|
99
|
+
sharedStateSelectionOptions,
|
|
100
|
+
strategySelectionOptionsForModelDetour,
|
|
101
|
+
shouldFailover,
|
|
102
|
+
peekAlternateCodexAccount,
|
|
103
|
+
} from "./routing/selection";
|
|
104
|
+
import {
|
|
105
|
+
clearAllManualPreferences,
|
|
106
|
+
consumeManualPreference,
|
|
107
|
+
forgetManualPreference,
|
|
108
|
+
forgetRoutingPreferencesOutside,
|
|
109
|
+
forgetRuntimeActiveCodexAccount,
|
|
110
|
+
getEffectiveActiveCodexAccountId,
|
|
111
|
+
manualPreferenceBlocks,
|
|
112
|
+
promoteActiveCodexAccount,
|
|
113
|
+
rememberActiveCodexAccount,
|
|
114
|
+
setActiveCodexAccount,
|
|
115
|
+
} from "./routing/active-account";
|
|
116
|
+
|
|
117
|
+
export {
|
|
118
|
+
CODEX_QUOTA_PROBE_INTERVAL_MS,
|
|
119
|
+
CODEX_FAILURE_WINDOW_MS,
|
|
120
|
+
TERMINAL_SHORT_WINDOW_FRESHNESS_MS,
|
|
240
121
|
CODEX_TRANSIENT_SOFT_AVOID_MS,
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
export
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
export type
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
};
|
|
316
|
-
|
|
317
|
-
export type CodexQuotaRecoveryProbeProof = {
|
|
318
|
-
credentialGeneration?: number;
|
|
319
|
-
};
|
|
320
|
-
|
|
321
|
-
/**
|
|
322
|
-
* Requests without a resolved native model retain the historic one-account-per-
|
|
323
|
-
* thread behavior. Requests with a known quota scope get an independent
|
|
324
|
-
* affinity so a Reserve failover cannot displace the same thread's Terra/Luna
|
|
325
|
-
* account (and vice versa).
|
|
326
|
-
*/
|
|
327
|
-
type BaseThreadAffinityScope = CodexQuotaScope | "legacy";
|
|
328
|
-
type ModelDetourAffinityScope = `model-detour:${BaseThreadAffinityScope}:${string}`;
|
|
329
|
-
type ThreadAffinityScope = BaseThreadAffinityScope | ModelDetourAffinityScope;
|
|
330
|
-
const LEGACY_THREAD_AFFINITY_SCOPE = "legacy" as const;
|
|
331
|
-
const threadAccountMap = new Map<string, Map<ThreadAffinityScope, ThreadAffinityEntry>>();
|
|
332
|
-
let threadAffinityEntryTotal = 0;
|
|
333
|
-
|
|
334
|
-
function isModelDetourAffinityScope(scope: ThreadAffinityScope): scope is ModelDetourAffinityScope {
|
|
335
|
-
return scope.startsWith("model-detour:");
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
const NATIVE_MODEL_QUOTA_SCOPES: Readonly<Record<string, CodexQuotaScope>> = {
|
|
339
|
-
[NATIVE_RESERVE_MODEL]: "reserve",
|
|
340
|
-
};
|
|
341
|
-
|
|
342
|
-
export function codexQuotaScopeForModel(modelId: string | undefined): CodexQuotaScope | undefined {
|
|
343
|
-
if (!modelId?.trim()) return undefined;
|
|
344
|
-
return NATIVE_MODEL_QUOTA_SCOPES[modelId.trim().toLowerCase()] ?? "shared";
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
/** Independent quota groups must not mutate the shared active-account cursor. */
|
|
348
|
-
function isIndependentCodexQuotaScope(quotaScope?: CodexQuotaScope): boolean {
|
|
349
|
-
return quotaScope !== undefined && quotaScope !== "shared";
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function codexPoolKeyForScope(quotaScope?: CodexQuotaScope): string {
|
|
353
|
-
return isIndependentCodexQuotaScope(quotaScope) ? `${POOL_KEY_CODEX}:${quotaScope}` : POOL_KEY_CODEX;
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
export type CodexUpstreamOutcomeMeta = {
|
|
357
|
-
retryAfter?: string | null;
|
|
358
|
-
resetAt?: unknown | unknown[];
|
|
359
|
-
now?: number;
|
|
360
|
-
/** (provider, host) ledger key for account-neutral reachability failures (#914). */
|
|
361
|
-
hostKey?: string;
|
|
362
|
-
/**
|
|
363
|
-
* Upstream denial evidence for a 403. A workspace/entitlement denial means the CREDENTIAL
|
|
364
|
-
* is fine and the account simply cannot reach this workspace, so it must not be quarantined
|
|
365
|
-
* for reauthentication (#1789). Absent evidence keeps the historical credential handling.
|
|
366
|
-
*/
|
|
367
|
-
denial?: "workspace" | "entitlement";
|
|
368
|
-
/** Stable transport code recorded alongside a neutral host failure. */
|
|
369
|
-
lastFailureCode?: string;
|
|
370
|
-
/** Native model selected for this request; used only for confirmed scoped quotas. */
|
|
371
|
-
modelId?: string;
|
|
372
|
-
/** When set, clears affinity for this thread immediately on transient failure. */
|
|
373
|
-
threadId?: string | null;
|
|
374
|
-
/**
|
|
375
|
-
* Suppress Pool rotation and quota/transient affinity mutations for an account-qualified
|
|
376
|
-
* request. Credential failures still sweep stale affinities because reauthentication is
|
|
377
|
-
* account-wide.
|
|
378
|
-
*/
|
|
379
|
-
fixedAccount?: boolean;
|
|
380
|
-
/**
|
|
381
|
-
* Probe lease held by this request, when it was admitted through an active
|
|
382
|
-
* quota cooldown. Only the outcome carrying the current lease may clear the
|
|
383
|
-
* cooldown (#433).
|
|
384
|
-
*/
|
|
385
|
-
probeLeaseId?: string;
|
|
386
|
-
/** Scope of `probeLeaseId` when it was granted against a model-scoped cooldown. */
|
|
387
|
-
probeQuotaScope?: CodexQuotaScope;
|
|
388
|
-
/**
|
|
389
|
-
* Already-chosen alternate for same-request 429 retry. When set, promotion
|
|
390
|
-
* reuses this account instead of calling {@link pickAlternateCodexAccount}
|
|
391
|
-
* again (which would advance a round-robin ring twice).
|
|
392
|
-
*/
|
|
393
|
-
promoteAccountId?: string;
|
|
394
|
-
/** Generation captured when this routed account was selected. */
|
|
395
|
-
writerGeneration?: number;
|
|
396
|
-
/**
|
|
397
|
-
* Credential generation this request's bearer was read at. Distinct from
|
|
398
|
-
* `writerGeneration`, which tracks the config store.
|
|
399
|
-
*
|
|
400
|
-
* A 401 that arrives after the credential was already replaced is evidence about a
|
|
401
|
-
* token nobody is using any more, so it must not quarantine the replacement. Absent
|
|
402
|
-
* means the caller cannot supply lineage and the historical unfenced handling stands.
|
|
403
|
-
*/
|
|
404
|
-
credentialGeneration?: number;
|
|
405
|
-
};
|
|
406
|
-
|
|
122
|
+
classifyCodexUpstreamOutcome,
|
|
123
|
+
computeCodexUsageScore,
|
|
124
|
+
computeQuotaCooldown,
|
|
125
|
+
computeQuotaCooldownUntil,
|
|
126
|
+
parseRetryAfterMs,
|
|
127
|
+
parseResetCooldownMs,
|
|
128
|
+
} from "./routing/cooldown-math";
|
|
129
|
+
export type {
|
|
130
|
+
CodexUpstreamOutcome,
|
|
131
|
+
CodexUpstreamOutcomeClass,
|
|
132
|
+
CodexCooldownSource,
|
|
133
|
+
CodexUpstreamOutcomeMeta,
|
|
134
|
+
} from "./routing/cooldown-math";
|
|
135
|
+
export {
|
|
136
|
+
codexQuotaScopeForModel,
|
|
137
|
+
listLiveCodexAccountIds,
|
|
138
|
+
getCodexUpstreamHealth,
|
|
139
|
+
getCodexAccountCooldownUntil,
|
|
140
|
+
getCodexAccountHealthSnapshot,
|
|
141
|
+
getCodexQuotaHealthSnapshot,
|
|
142
|
+
isCodexAccountInCooldown,
|
|
143
|
+
clearCodexAccountCooldown,
|
|
144
|
+
getCodexAccountSoftAvoidUntil,
|
|
145
|
+
isCodexAccountSoftAvoided,
|
|
146
|
+
} from "./routing/health-store";
|
|
147
|
+
export type { CodexQuotaScope } from "./routing/health-store";
|
|
148
|
+
export {
|
|
149
|
+
tryAcquireCodexQuotaProbeLease,
|
|
150
|
+
canAcquireCodexQuotaProbeLease,
|
|
151
|
+
claimDueCodexQuotaRecoveryProbes,
|
|
152
|
+
claimManualResetCooldowns,
|
|
153
|
+
settleManualResetCooldown,
|
|
154
|
+
settleCodexQuotaRecoveryProbe,
|
|
155
|
+
tryAcquireCodexQuotaScopeProbeLease,
|
|
156
|
+
canAcquireCodexQuotaScopeProbeLease,
|
|
157
|
+
releaseCodexQuotaProbeLease,
|
|
158
|
+
releaseCodexQuotaScopeProbeLease,
|
|
159
|
+
} from "./routing/probe-lease";
|
|
160
|
+
export type {
|
|
161
|
+
CodexQuotaRecoveryProbeClaim,
|
|
162
|
+
CodexQuotaRecoveryProbeProof,
|
|
163
|
+
ManualResetCooldownClaim,
|
|
164
|
+
ManualResetRefreshLineage,
|
|
165
|
+
} from "./routing/probe-lease";
|
|
166
|
+
export {
|
|
167
|
+
CODEX_THREAD_AFFINITY_IDLE_TTL_MS,
|
|
168
|
+
CODEX_THREAD_AFFINITY_MAX_ENTRIES,
|
|
169
|
+
CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS,
|
|
170
|
+
CODEX_TRANSIENT_AFFINITY_HOLD_MS,
|
|
171
|
+
clearConversationStateIssuerMap,
|
|
172
|
+
clearThreadAccountMap,
|
|
173
|
+
clearThreadAccountMapForAccount,
|
|
174
|
+
debugCodexAffinityGenerations,
|
|
175
|
+
handOffThreadAffinityGeneration,
|
|
176
|
+
peekConversationStateIssuer,
|
|
177
|
+
rememberConversationStateIssuer,
|
|
178
|
+
} from "./routing/thread-affinity";
|
|
179
|
+
export type {
|
|
180
|
+
CodexThreadResolution,
|
|
181
|
+
CodexAffinityMove,
|
|
182
|
+
CodexAffinityReason,
|
|
183
|
+
CodexAffinityDecision,
|
|
184
|
+
} from "./routing/thread-affinity";
|
|
185
|
+
export {
|
|
186
|
+
isCodexAccountPlanExcluded,
|
|
187
|
+
getPoolAccountPlan,
|
|
188
|
+
pickLowestUsageCodexAccount,
|
|
189
|
+
pickAlternateCodexAccount,
|
|
190
|
+
} from "./routing/selection";
|
|
191
|
+
export {
|
|
192
|
+
resetCodexRoutingForManualSelection,
|
|
193
|
+
getEffectiveActiveCodexAccountId,
|
|
194
|
+
isEffectiveCodexAccountPinned,
|
|
195
|
+
} from "./routing/active-account";
|
|
407
196
|
function hasConfiguredPoolAccount(
|
|
408
197
|
config: OcxConfig,
|
|
409
198
|
accountId: string,
|
|
@@ -416,1284 +205,41 @@ function hasConfiguredPoolAccount(
|
|
|
416
205
|
.some(account => isSelectableCodexPoolAccount(account) && account.id === accountId);
|
|
417
206
|
}
|
|
418
207
|
|
|
419
|
-
export function listLiveCodexAccountIds(config: OcxConfig): ReadonlySet<string> {
|
|
420
|
-
const ids = new Set((config.codexAccounts ?? []).map(account => account.id));
|
|
421
|
-
const openai = config.providers.openai;
|
|
422
|
-
if (openai && openai.disabled !== true && isCanonicalOpenAiForwardProvider(openai)) {
|
|
423
|
-
ids.add(MAIN_CODEX_ACCOUNT_ID);
|
|
424
|
-
}
|
|
425
|
-
return ids;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
export function clearThreadAccountMap(): void {
|
|
429
|
-
threadAccountMap.clear();
|
|
430
|
-
threadAffinityEntryTotal = 0;
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
export function clearThreadAccountMapForAccount(
|
|
434
|
-
accountId: string,
|
|
435
|
-
reason: CodexAffinityReason = "unusable",
|
|
436
|
-
): void {
|
|
437
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
438
|
-
for (const [scope, entry] of affinities) {
|
|
439
|
-
if (entry.accountId === accountId && affinities.delete(scope)) {
|
|
440
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
441
|
-
notePendingReleaseReason(threadId, reason);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
/**
|
|
449
|
-
* Why a binding was released, held until that thread's next resolve can report it (#4546).
|
|
450
|
-
*
|
|
451
|
-
* A release and the request that pays for it are two different moments: a 429 clears the pin
|
|
452
|
-
* inside the outcome recorder, and the next request arrives with nothing left to explain why it
|
|
453
|
-
* is starting cold. Bounded, because it is a diagnostic and must not become a leak.
|
|
454
|
-
*/
|
|
455
|
-
const pendingReleaseReasons = new Map<string, CodexAffinityReason>();
|
|
456
|
-
const MAX_PENDING_RELEASE_REASONS = 4096;
|
|
457
|
-
|
|
458
|
-
function notePendingReleaseReason(threadId: string | null, reason: CodexAffinityReason): void {
|
|
459
|
-
if (threadId === null) return;
|
|
460
|
-
if (!pendingReleaseReasons.has(threadId) && pendingReleaseReasons.size >= MAX_PENDING_RELEASE_REASONS) {
|
|
461
|
-
const oldest = pendingReleaseReasons.keys().next();
|
|
462
|
-
if (!oldest.done) pendingReleaseReasons.delete(oldest.value);
|
|
463
|
-
}
|
|
464
|
-
pendingReleaseReasons.set(threadId, reason);
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
function peekPendingReleaseReason(threadId: string | null): CodexAffinityReason | undefined {
|
|
468
|
-
if (threadId === null) return undefined;
|
|
469
|
-
return pendingReleaseReasons.get(threadId);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/**
|
|
473
|
-
* Forget a release only once it has actually been reported.
|
|
474
|
-
*
|
|
475
|
-
* Consuming it at derivation time lost it whenever selection then failed to produce an account:
|
|
476
|
-
* a no-account return carries no payload, so the release went unrecorded and the next successful
|
|
477
|
-
* resolve claimed a fresh healthy bind (#4598). A release survives until some resolve reports it.
|
|
478
|
-
*/
|
|
479
|
-
function clearPendingReleaseReason(threadId: string | null): void {
|
|
480
|
-
if (threadId !== null) pendingReleaseReasons.delete(threadId);
|
|
481
|
-
}
|
|
482
|
-
|
|
483
208
|
export function clearCodexUpstreamHealth(): void {
|
|
484
209
|
// Operator preferences are routing state, not health, but they live and die with the same
|
|
485
210
|
// reset points. Leaving them behind lets a selection from one context suppress the
|
|
486
211
|
// automatic cursor in the next one.
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
runtimeActiveCodexAccountId = undefined;
|
|
212
|
+
clearAllManualPreferences();
|
|
213
|
+
clearUpstreamHealthState();
|
|
214
|
+
forgetRuntimeActiveCodexAccount();
|
|
491
215
|
// The reconcile watermark is part of this state, not something that outlives it. Keeping
|
|
492
216
|
// it across a full reset is incoherent: there is no health left to protect, yet
|
|
493
|
-
// recordCodexUpstreamOutcome would still drop a writer whose generation predates the
|
|
494
|
-
// watermark for any account missing from the equally stale live set. Left behind, it also
|
|
495
|
-
// leaks between test files, which is how it was found.
|
|
496
|
-
|
|
497
|
-
liveHealthAccountIds = new Set();
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
export function clearCodexUpstreamHealthForAccount(accountId: string): void {
|
|
501
|
-
upstreamHealth.delete(accountId);
|
|
502
|
-
quotaScopedHealth.delete(accountId);
|
|
503
|
-
// Deletion is the third operator exit, next to pause and exclusion, and it is the one
|
|
504
|
-
// with no reconcile path behind it: once the account is gone nothing can succeed on it,
|
|
505
|
-
// so an unspent preference naming it would suppress the automatic cursor for every other
|
|
506
|
-
// account until the process restarts.
|
|
507
|
-
forgetManualPreference(accountId);
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
export function reconcileCodexRoutingHealth(context: GenerationContext): number {
|
|
511
|
-
if (context.generation <= lastReconciledGeneration) return 0;
|
|
512
|
-
let removed = 0;
|
|
513
|
-
for (const accountId of upstreamHealth.keys()) {
|
|
514
|
-
if (context.codexAccountIds.has(accountId)) continue;
|
|
515
|
-
upstreamHealth.delete(accountId);
|
|
516
|
-
removed += 1;
|
|
517
|
-
}
|
|
518
|
-
for (const accountId of quotaScopedHealth.keys()) {
|
|
519
|
-
if (context.codexAccountIds.has(accountId)) continue;
|
|
520
|
-
quotaScopedHealth.delete(accountId);
|
|
521
|
-
removed += 1;
|
|
522
|
-
}
|
|
523
|
-
// Sweep preferences the same way, for the account set this generation actually has. The
|
|
524
|
-
// delete path above is the direct route; this is the one that catches an account removed
|
|
525
|
-
// by an edit the runtime never saw. Deliberately not counted in `removed`, which reports
|
|
526
|
-
// health rows.
|
|
527
|
-
for (const [poolKey, preferred] of manualPreference) {
|
|
528
|
-
if (context.codexAccountIds.has(preferred)) continue;
|
|
529
|
-
manualPreference.delete(poolKey);
|
|
530
|
-
}
|
|
531
|
-
liveHealthAccountIds = new Set(context.codexAccountIds);
|
|
532
|
-
lastReconciledGeneration = context.generation;
|
|
533
|
-
return removed;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
export function getCodexUpstreamHealth(
|
|
537
|
-
accountId: string,
|
|
538
|
-
): CodexUpstreamHealth | null {
|
|
539
|
-
dropSpentCredentialFailure(accountId);
|
|
540
|
-
return upstreamHealth.get(accountId) ?? null;
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
function scopedHealthFor(accountId: string, scope: CodexQuotaScope): CodexUpstreamHealth | undefined {
|
|
544
|
-
return quotaScopedHealth.get(accountId)?.get(scope);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function setScopedHealth(accountId: string, scope: CodexQuotaScope, health: CodexUpstreamHealth): void {
|
|
548
|
-
let scopes = quotaScopedHealth.get(accountId);
|
|
549
|
-
if (!scopes) {
|
|
550
|
-
scopes = new Map();
|
|
551
|
-
quotaScopedHealth.set(accountId, scopes);
|
|
552
|
-
}
|
|
553
|
-
scopes.set(scope, health);
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
function deleteScopedHealth(accountId: string, scope: CodexQuotaScope): void {
|
|
557
|
-
const scopes = quotaScopedHealth.get(accountId);
|
|
558
|
-
if (!scopes) return;
|
|
559
|
-
scopes.delete(scope);
|
|
560
|
-
if (scopes.size === 0) quotaScopedHealth.delete(accountId);
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
export function computeCodexUsageScore(quota: {
|
|
564
|
-
weeklyPercent?: number;
|
|
565
|
-
monthlyPercent?: number;
|
|
566
|
-
shortPercent?: number;
|
|
567
|
-
shortResetAt?: number;
|
|
568
|
-
shortObservedAt?: number;
|
|
569
|
-
} | null, plan?: unknown, now: number = Date.now()): number {
|
|
570
|
-
if (!quota) return CODEX_UNKNOWN_USAGE_SCORE;
|
|
571
|
-
const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
|
|
572
|
-
const longWindows = isThirtyDayOnlyCodexPlan(plan)
|
|
573
|
-
? [quota.monthlyPercent]
|
|
574
|
-
: [quota.weeklyPercent, quota.monthlyPercent];
|
|
575
|
-
const knownLong = longWindows.filter(finite);
|
|
576
|
-
// The short burst window only REFINES a known long-window position; it cannot stand in for
|
|
577
|
-
// one. A snapshot carrying just `shortPercent: 0` would otherwise score a flat 0 and make an
|
|
578
|
-
// account whose weekly/monthly usage is entirely unverified look like the emptiest in the
|
|
579
|
-
// pool, so `pickLowestUsageAmong` would send every request to it. Unknown has to stay
|
|
580
|
-
// unknown until a governing window is actually observed.
|
|
581
|
-
//
|
|
582
|
-
// A FULL burst window is the exception (#3029). It is not an optimistic guess about an
|
|
583
|
-
// unobserved window — it is a direct observation that the account cannot serve a request
|
|
584
|
-
// right now, whatever its monthly position turns out to be. Unknown-means-selectable is
|
|
585
|
-
// correct for uncertainty and wrong for a measured refusal: the account stays selected,
|
|
586
|
-
// `applyQuotaAutoSwitch` never fires, and the pool wedges on an exhausted credential.
|
|
587
|
-
if (knownLong.length === 0) {
|
|
588
|
-
return isTerminalShortWindow(quota, now) ? CODEX_EXHAUSTED_USAGE_PERCENT : CODEX_UNKNOWN_USAGE_SCORE;
|
|
589
|
-
}
|
|
590
|
-
const values = finite(quota.shortPercent) ? [...knownLong, quota.shortPercent] : knownLong;
|
|
591
|
-
return Math.max(...values);
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
/**
|
|
595
|
-
* A short-only reading that proves the account is blocked NOW.
|
|
596
|
-
*
|
|
597
|
-
* Freshness is not optional. `getAccountQuota` performs no expiry check, partial updates
|
|
598
|
-
* carry a still-open short tuple forward, and disk hydration accepts a persisted reading for
|
|
599
|
-
* hours — so scoring 100 from `shortPercent` alone would keep excluding an account whose
|
|
600
|
-
* five-hour window has since reset. Merge no longer carries an elapsed shortResetAt, but an
|
|
601
|
-
* explicit incoming elapsed tuple is still stored, and a missing reset cannot be aged there.
|
|
602
|
-
* That is #3029 pointed the other way: the issue is that
|
|
603
|
-
* an exhausted account stays selected, and "a recovered account stays excluded" trades one
|
|
604
|
-
* unusable pool for another.
|
|
605
|
-
*
|
|
606
|
-
* A reading with no `shortResetAt` cannot be aged, so it stays unknown. The conservative
|
|
607
|
-
* direction here is the one that keeps an account selectable: a wrongly-selected account
|
|
608
|
-
* fails one request, while a wrongly-excluded one is invisible until someone reads the pool
|
|
609
|
-
* by hand.
|
|
610
|
-
*
|
|
611
|
-
* A missing reset can instead be aged by shortObservedAt (#3425). General updatedAt is not
|
|
612
|
-
* sufficient: credit-only updates preserve the old short tuple but advance that timestamp.
|
|
613
|
-
* Old disk snapshots without short-window provenance remain unknown.
|
|
614
|
-
*/
|
|
615
|
-
function isTerminalShortWindow(
|
|
616
|
-
quota: { shortPercent?: number; shortResetAt?: number; shortObservedAt?: number },
|
|
617
|
-
now: number,
|
|
618
|
-
): boolean {
|
|
619
|
-
if (typeof quota.shortPercent !== "number" || !Number.isFinite(quota.shortPercent)) return false;
|
|
620
|
-
if (quota.shortPercent < CODEX_EXHAUSTED_USAGE_PERCENT) return false;
|
|
621
|
-
const resetAt = quota.shortResetAt;
|
|
622
|
-
if (typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt <= 0) {
|
|
623
|
-
const observedAt = quota.shortObservedAt;
|
|
624
|
-
if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return false;
|
|
625
|
-
const age = now - observedAt;
|
|
626
|
-
return age >= 0 && age <= TERMINAL_SHORT_WINDOW_FRESHNESS_MS;
|
|
627
|
-
}
|
|
628
|
-
// Seconds and milliseconds both reach storage, so the split lives in one place next to the
|
|
629
|
-
// merge that also ages a stored reset instant (`resetAtToMs`, src/codex/quota.ts).
|
|
630
|
-
return resetAtToMs(resetAt) > now;
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
export function classifyCodexUpstreamOutcome(
|
|
634
|
-
outcome: CodexUpstreamOutcome,
|
|
635
|
-
denial?: "workspace" | "entitlement",
|
|
636
|
-
): CodexUpstreamOutcomeClass {
|
|
637
|
-
if (outcome === "connect_neutral") return "neutral";
|
|
638
|
-
if (outcome === "connect_error" || outcome === "timeout") return "transient";
|
|
639
|
-
if (!Number.isFinite(outcome)) return "unknown";
|
|
640
|
-
if (outcome >= 200 && outcome < 300) return "success";
|
|
641
|
-
// Explicit 3xx policy (#914): a redirect response is relayed as-is and is
|
|
642
|
-
// never account or host health evidence — it proves the host is reachable
|
|
643
|
-
// and says nothing about the credential. Relayed as the neutral class so a
|
|
644
|
-
// stray 3xx cannot increment an account's transient streak.
|
|
645
|
-
if (outcome >= 300 && outcome < 400) return "neutral";
|
|
646
|
-
// 401 is always a credential problem. A 403 is only a credential problem when nothing
|
|
647
|
-
// tells us otherwise: a workspace/entitlement denial (#1789) means the credential is valid
|
|
648
|
-
// and the account simply lacks access here, so quarantining it for reauth is wrong advice.
|
|
649
|
-
// Absent denial evidence the historical mapping stands, so the change fails safe.
|
|
650
|
-
if (outcome === 403 && denial !== undefined) return "workspace";
|
|
651
|
-
if (outcome === 401 || outcome === 403) return "credential";
|
|
652
|
-
// 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
|
|
653
|
-
// (same-request alternate retry records this outcome for the depleted account).
|
|
654
|
-
if (outcome === 429 || outcome === 402) return "quota";
|
|
655
|
-
if (outcome >= 400 && outcome < 500) return "caller";
|
|
656
|
-
if (outcome >= 500 && outcome < 600) return "transient";
|
|
657
|
-
return "unknown";
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
function clampCooldownMs(ms: number): number {
|
|
661
|
-
return Math.min(Math.max(ms, 1), CODEX_MAX_QUOTA_COOLDOWN_MS);
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
export function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined {
|
|
665
|
-
const text = value?.trim();
|
|
666
|
-
if (!text) return undefined;
|
|
667
|
-
if (/^\d+(?:\.\d+)?$/.test(text)) {
|
|
668
|
-
const seconds = Number(text);
|
|
669
|
-
if (Number.isFinite(seconds) && seconds > 0) return clampCooldownMs(Math.ceil(seconds * 1000));
|
|
670
|
-
}
|
|
671
|
-
const timestamp = Date.parse(text);
|
|
672
|
-
if (!Number.isFinite(timestamp)) return undefined;
|
|
673
|
-
const delay = timestamp - now;
|
|
674
|
-
return delay > 0 ? clampCooldownMs(delay) : undefined;
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
function resetTimestampMs(value: unknown): number | undefined {
|
|
678
|
-
const numeric = typeof value === "number"
|
|
679
|
-
? value
|
|
680
|
-
: typeof value === "string" && value.trim() !== ""
|
|
681
|
-
? Number(value)
|
|
682
|
-
: undefined;
|
|
683
|
-
if (typeof numeric !== "number" || !Number.isFinite(numeric) || numeric <= 0) return undefined;
|
|
684
|
-
return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
export function parseResetCooldownMs(resetAt: unknown | unknown[] | undefined, now = Date.now()): number | undefined {
|
|
688
|
-
const values = Array.isArray(resetAt) ? resetAt : [resetAt];
|
|
689
|
-
let best: number | undefined;
|
|
690
|
-
for (const value of values) {
|
|
691
|
-
const timestamp = resetTimestampMs(value);
|
|
692
|
-
if (timestamp === undefined) continue;
|
|
693
|
-
const delay = timestamp - now;
|
|
694
|
-
if (delay <= 0) continue;
|
|
695
|
-
// A far-future reset must not pin the account for the full Retry-After
|
|
696
|
-
// ceiling: quota usually frees up well before the advertised window (#433).
|
|
697
|
-
const clamped = Math.min(clampCooldownMs(delay), CODEX_MAX_RESET_DERIVED_COOLDOWN_MS);
|
|
698
|
-
if (best === undefined || clamped < best) best = clamped;
|
|
699
|
-
}
|
|
700
|
-
return best;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
export function computeQuotaCooldown(meta: CodexUpstreamOutcomeMeta = {}): {
|
|
704
|
-
until: number;
|
|
705
|
-
source: CodexCooldownSource;
|
|
706
|
-
} {
|
|
707
|
-
const now = meta.now ?? Date.now();
|
|
708
|
-
const retryAfterMs = parseRetryAfterMs(meta.retryAfter, now);
|
|
709
|
-
if (retryAfterMs !== undefined) return { until: now + retryAfterMs, source: "retry-after" };
|
|
710
|
-
const resetCooldownMs = parseResetCooldownMs(meta.resetAt, now);
|
|
711
|
-
if (resetCooldownMs !== undefined) return { until: now + resetCooldownMs, source: "reset-derived" };
|
|
712
|
-
return { until: now + CODEX_DEFAULT_QUOTA_COOLDOWN_MS, source: "default" };
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
/**
|
|
716
|
-
* When the pool should stop preferring an account after it refused on quota.
|
|
717
|
-
*
|
|
718
|
-
* The earliest window the refusal actually announced, bounded by {@link CODEX_MAX_QUOTA_AVOID_MS},
|
|
719
|
-
* and never shorter than the cooldown the same refusal produced — a Retry-After directive that
|
|
720
|
-
* outlasts every announcement still governs.
|
|
721
|
-
*/
|
|
722
|
-
function quotaAvoidUntilFor(meta: CodexUpstreamOutcomeMeta, now: number, cooldownUntil: number): number {
|
|
723
|
-
const values = Array.isArray(meta.resetAt) ? meta.resetAt : [meta.resetAt];
|
|
724
|
-
let announced: number | undefined;
|
|
725
|
-
for (const value of values) {
|
|
726
|
-
const timestamp = resetTimestampMs(value);
|
|
727
|
-
if (timestamp === undefined) continue;
|
|
728
|
-
const delay = timestamp - now;
|
|
729
|
-
if (delay <= 0) continue;
|
|
730
|
-
const until = now + Math.min(delay, CODEX_MAX_QUOTA_AVOID_MS);
|
|
731
|
-
if (announced === undefined || until < announced) announced = until;
|
|
732
|
-
}
|
|
733
|
-
return Math.max(cooldownUntil, announced ?? 0);
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
/** Live quota-refusal avoidance for an account, including the lane the request belongs to. */
|
|
737
|
-
function codexQuotaAvoidUntil(
|
|
738
|
-
accountId: string,
|
|
739
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
740
|
-
now: number,
|
|
741
|
-
): number | null {
|
|
742
|
-
const live = (value: number | undefined): number | null =>
|
|
743
|
-
typeof value === "number" && Number.isFinite(value) && value > now ? value : null;
|
|
744
|
-
const account = live(upstreamHealth.get(accountId)?.quotaAvoidUntil);
|
|
745
|
-
const scoped = quotaScope === undefined
|
|
746
|
-
? null
|
|
747
|
-
: live(scopedHealthFor(accountId, quotaScope)?.quotaAvoidUntil);
|
|
748
|
-
if (account === null) return scoped;
|
|
749
|
-
return scoped === null ? account : Math.max(account, scoped);
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
function isCodexQuotaAvoided(
|
|
753
|
-
accountId: string,
|
|
754
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
755
|
-
now: number,
|
|
756
|
-
): boolean {
|
|
757
|
-
return codexQuotaAvoidUntil(accountId, quotaScope, now) !== null;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): number {
|
|
761
|
-
return computeQuotaCooldown(meta).until;
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
/**
|
|
765
|
-
* Grant at most one probe lease per interval for a cooled-down account.
|
|
766
|
-
*
|
|
767
|
-
* A cooled-down account is short-circuited locally, so it never sends traffic and
|
|
768
|
-
* no organic 2xx can prove that upstream quota recovered — the cooldown can only
|
|
769
|
-
* end by expiry or a proxy restart (#433). Releasing a single probe breaks that
|
|
770
|
-
* deadlock. Explicit Retry-After cooldowns are excluded: those are literal retry
|
|
771
|
-
* directives, not window announcements.
|
|
772
|
-
*
|
|
773
|
-
* Returns the lease id, or null when no probe may go out right now.
|
|
774
|
-
*/
|
|
775
|
-
export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): string | null {
|
|
776
|
-
if (!canAcquireCodexQuotaProbeLease(accountId, now)) return null;
|
|
777
|
-
const health = upstreamHealth.get(accountId)!;
|
|
778
|
-
const probeLeaseId = randomUUID();
|
|
779
|
-
upstreamHealth.set(accountId, {
|
|
780
|
-
...health,
|
|
781
|
-
probeLeaseId,
|
|
782
|
-
probeLeaseGeneration: health.cooldownGeneration ?? 0,
|
|
783
|
-
lastProbeAt: now,
|
|
784
|
-
});
|
|
785
|
-
return probeLeaseId;
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */
|
|
789
|
-
export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean {
|
|
790
|
-
return canAcquireQuotaProbeLease(upstreamHealth.get(accountId), now);
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now: number): boolean {
|
|
794
|
-
if (!health) return false;
|
|
795
|
-
const cooldownUntil = health.cooldownUntil;
|
|
796
|
-
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false;
|
|
797
|
-
if (health.cooldownSource === "retry-after") return false;
|
|
798
|
-
if (health.probeLeaseId !== undefined) return false;
|
|
799
|
-
const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil;
|
|
800
|
-
return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS;
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
/**
|
|
804
|
-
* Claim due reset-derived cooldown probes without consulting account selection.
|
|
805
|
-
* Added Pool credentials only; owned main usage recovery is handled separately.
|
|
806
|
-
*/
|
|
807
|
-
export function claimDueCodexQuotaRecoveryProbes(
|
|
808
|
-
config: OcxConfig,
|
|
809
|
-
limit: number,
|
|
810
|
-
now = Date.now(),
|
|
811
|
-
): CodexQuotaRecoveryProbeClaim[] {
|
|
812
|
-
const boundedLimit = Math.max(0, Math.floor(limit));
|
|
813
|
-
if (boundedLimit === 0) return [];
|
|
814
|
-
const candidates: Array<{
|
|
815
|
-
accountId: string;
|
|
816
|
-
scope?: CodexQuotaScope;
|
|
817
|
-
health: CodexUpstreamHealth;
|
|
818
|
-
credentialGeneration: number;
|
|
819
|
-
credentialReplacedAt?: number;
|
|
820
|
-
order: number;
|
|
821
|
-
}> = [];
|
|
822
|
-
for (const [order, account] of (config.codexAccounts ?? []).entries()) {
|
|
823
|
-
if (!isSelectableCodexPoolAccount(account)
|
|
824
|
-
|| isCodexAccountPaused(config, account.id)
|
|
825
|
-
|| isAccountNeedsReauth(account.id)) continue;
|
|
826
|
-
const record = readCodexAccountRecord(account.id);
|
|
827
|
-
if (!record?.credential || record.deletedAt != null) continue;
|
|
828
|
-
const due = [
|
|
829
|
-
{ scope: undefined, health: upstreamHealth.get(account.id) },
|
|
830
|
-
...[...(quotaScopedHealth.get(account.id) ?? [])].map(([scope, health]) => ({ scope, health })),
|
|
831
|
-
].filter((entry): entry is { scope?: CodexQuotaScope; health: CodexUpstreamHealth } =>
|
|
832
|
-
// Generic WHAM evidence can recover only ordinary quota, never Reserve.
|
|
833
|
-
// Do not spend this account's one claim per pass on an independent scope and
|
|
834
|
-
// delay the shared scope that the response can actually recover.
|
|
835
|
-
(entry.scope === undefined || entry.scope === "shared")
|
|
836
|
-
&& entry.health?.cooldownSource === "reset-derived"
|
|
837
|
-
&& canAcquireQuotaProbeLease(entry.health, now))
|
|
838
|
-
.sort((a, b) =>
|
|
839
|
-
(a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
|
|
840
|
-
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0));
|
|
841
|
-
const candidate = due[0];
|
|
842
|
-
if (candidate) candidates.push({
|
|
843
|
-
accountId: account.id,
|
|
844
|
-
...(candidate.scope ? { scope: candidate.scope } : {}),
|
|
845
|
-
health: candidate.health,
|
|
846
|
-
credentialGeneration: record.generation,
|
|
847
|
-
...(record.replacedAt !== undefined ? { credentialReplacedAt: record.replacedAt } : {}),
|
|
848
|
-
order,
|
|
849
|
-
});
|
|
850
|
-
}
|
|
851
|
-
candidates.sort((a, b) => {
|
|
852
|
-
const age = (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
|
|
853
|
-
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0);
|
|
854
|
-
return age || a.order - b.order;
|
|
855
|
-
});
|
|
856
|
-
return candidates.slice(0, boundedLimit).map(candidate => {
|
|
857
|
-
const leaseId = randomUUID();
|
|
858
|
-
const next = {
|
|
859
|
-
...candidate.health,
|
|
860
|
-
probeLeaseId: leaseId,
|
|
861
|
-
probeLeaseGeneration: candidate.health.cooldownGeneration ?? 0,
|
|
862
|
-
lastProbeAt: now,
|
|
863
|
-
};
|
|
864
|
-
if (candidate.scope) setScopedHealth(candidate.accountId, candidate.scope, next);
|
|
865
|
-
else upstreamHealth.set(candidate.accountId, next);
|
|
866
|
-
return {
|
|
867
|
-
accountId: candidate.accountId,
|
|
868
|
-
...(candidate.scope ? { scope: candidate.scope } : {}),
|
|
869
|
-
leaseId,
|
|
870
|
-
cooldownGeneration: candidate.health.cooldownGeneration ?? 0,
|
|
871
|
-
credentialGeneration: candidate.credentialGeneration,
|
|
872
|
-
...(candidate.credentialReplacedAt !== undefined
|
|
873
|
-
? { credentialReplacedAt: candidate.credentialReplacedAt }
|
|
874
|
-
: {}),
|
|
875
|
-
};
|
|
876
|
-
});
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
type CooldownRecoveryLease = Pick<CodexQuotaRecoveryProbeClaim,
|
|
880
|
-
"accountId" | "scope" | "leaseId" | "cooldownGeneration">;
|
|
881
|
-
|
|
882
|
-
export type ManualResetCooldownClaim =
|
|
883
|
-
| { kind: "pool"; probe: CodexQuotaRecoveryProbeClaim }
|
|
884
|
-
| { kind: "main"; probe: CooldownRecoveryLease };
|
|
885
|
-
|
|
886
|
-
function manualResetAccountEligible(config: OcxConfig, accountId: string): boolean {
|
|
887
|
-
return !isCodexAccountPaused(config, accountId) && !isAccountNeedsReauth(accountId)
|
|
888
|
-
&& (accountId === MAIN_CODEX_ACCOUNT_ID
|
|
889
|
-
|| (config.codexAccounts ?? []).some(account => account.id === accountId && isSelectableCodexPoolAccount(account)));
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
/** Explicit reset bypasses probe pacing, never another owner's lease or quota scope. */
|
|
893
|
-
export function claimManualResetCooldowns(
|
|
894
|
-
config: OcxConfig,
|
|
895
|
-
accountId: string,
|
|
896
|
-
now = Date.now(),
|
|
897
|
-
expectedPoolGeneration?: number,
|
|
898
|
-
): ManualResetCooldownClaim[] {
|
|
899
|
-
if (!manualResetAccountEligible(config, accountId)) return [];
|
|
900
|
-
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
|
|
901
|
-
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return [];
|
|
902
|
-
if (record && expectedPoolGeneration !== undefined && record.generation !== expectedPoolGeneration) return [];
|
|
903
|
-
const claims: ManualResetCooldownClaim[] = [];
|
|
904
|
-
for (const scope of [undefined, "shared"] as const) {
|
|
905
|
-
const health = scope ? scopedHealthFor(accountId, scope) : upstreamHealth.get(accountId);
|
|
906
|
-
if (!health || health.cooldownSource !== "reset-derived" || health.probeLeaseId !== undefined
|
|
907
|
-
|| !Number.isFinite(health.cooldownUntil) || !(health.cooldownUntil! > now)) continue;
|
|
908
|
-
const leaseId = randomUUID();
|
|
909
|
-
const cooldownGeneration = health.cooldownGeneration ?? 0;
|
|
910
|
-
const next = { ...health, probeLeaseId: leaseId, probeLeaseGeneration: cooldownGeneration, lastProbeAt: now };
|
|
911
|
-
if (scope) setScopedHealth(accountId, scope, next);
|
|
912
|
-
else upstreamHealth.set(accountId, next);
|
|
913
|
-
const probe = { accountId, scope, leaseId, cooldownGeneration };
|
|
914
|
-
claims.push(record ? { kind: "pool", probe: {
|
|
915
|
-
...probe, credentialGeneration: record.generation, credentialReplacedAt: record.replacedAt,
|
|
916
|
-
} } : { kind: "main", probe });
|
|
917
|
-
}
|
|
918
|
-
return claims;
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
export type ManualResetRefreshLineage = Readonly<{
|
|
922
|
-
fromGeneration: number;
|
|
923
|
-
toGeneration: number;
|
|
924
|
-
provenance: CodexRefreshProvenance;
|
|
925
|
-
}>;
|
|
926
|
-
|
|
927
|
-
type ManualResetQuotaProof = CodexQuotaRecoveryProbeProof & {
|
|
928
|
-
refreshLineage?: ManualResetRefreshLineage;
|
|
929
|
-
};
|
|
930
|
-
|
|
931
|
-
/** Main proof is checked by the already-owned auth operation, never by a Pool record. */
|
|
932
|
-
export function settleManualResetCooldown(
|
|
933
|
-
config: OcxConfig,
|
|
934
|
-
claim: ManualResetCooldownClaim,
|
|
935
|
-
recovered: boolean,
|
|
936
|
-
proof: ManualResetQuotaProof = {},
|
|
937
|
-
now = Date.now(),
|
|
938
|
-
): boolean {
|
|
939
|
-
if (!recovered) return settleCooldownRecoveryLease(claim.probe, false, now);
|
|
940
|
-
const eligible = manualResetAccountEligible(config, claim.probe.accountId);
|
|
941
|
-
if (claim.kind === "main") return settleCooldownRecoveryLease(claim.probe, eligible, now);
|
|
942
|
-
const lineage = proof.refreshLineage;
|
|
943
|
-
// Equal wall-clock replacement stamps do not establish ancestry. Manual +1
|
|
944
|
-
// recovery additionally needs the actual forced-refresh result for this edge.
|
|
945
|
-
const ownedGeneration = proof.credentialGeneration === claim.probe.credentialGeneration
|
|
946
|
-
|| (proof.credentialGeneration === claim.probe.credentialGeneration + 1
|
|
947
|
-
&& lineage?.fromGeneration === claim.probe.credentialGeneration
|
|
948
|
-
&& lineage.toGeneration === proof.credentialGeneration
|
|
949
|
-
&& (lineage.provenance === "self-refresh" || lineage.provenance === "joined-lineage"));
|
|
950
|
-
return settleCodexQuotaRecoveryProbe(claim.probe, eligible && ownedGeneration, proof, now);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
/** Settle one background recovery claim without mutating account-wide outcome state. */
|
|
954
|
-
export function settleCodexQuotaRecoveryProbe(
|
|
955
|
-
claim: CodexQuotaRecoveryProbeClaim,
|
|
956
|
-
recovered: boolean,
|
|
957
|
-
proof: CodexQuotaRecoveryProbeProof,
|
|
958
|
-
now = Date.now(),
|
|
959
|
-
): boolean {
|
|
960
|
-
const health = claim.scope
|
|
961
|
-
? scopedHealthFor(claim.accountId, claim.scope)
|
|
962
|
-
: upstreamHealth.get(claim.accountId);
|
|
963
|
-
if (!health || health.probeLeaseId !== claim.leaseId) return false;
|
|
964
|
-
const currentRecord = readCodexAccountRecord(claim.accountId);
|
|
965
|
-
const proofGeneration = proof.credentialGeneration;
|
|
966
|
-
// A probe-owned token refresh (getValidCodexToken) advances the credential generation by
|
|
967
|
-
// exactly one while preserving `replacedAt`; an external credential replacement bumps the
|
|
968
|
-
// generation too but stamps a fresh `replacedAt`. Accept the +1 transition only when the
|
|
969
|
-
// claim-time lineage is intact AND the generation the fresh quota was proven under is live.
|
|
970
|
-
const generationFenced = proofGeneration !== undefined
|
|
971
|
-
&& (proofGeneration === claim.credentialGeneration
|
|
972
|
-
? isCodexAccountGenerationLive(claim.accountId, proofGeneration)
|
|
973
|
-
: proofGeneration === claim.credentialGeneration + 1
|
|
974
|
-
&& currentRecord?.replacedAt === claim.credentialReplacedAt
|
|
975
|
-
&& isCodexAccountGenerationLive(claim.accountId, proofGeneration));
|
|
976
|
-
return settleCooldownRecoveryLease(claim, recovered && generationFenced, now);
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
function settleCooldownRecoveryLease(claim: CooldownRecoveryLease, recovered: boolean, now: number): boolean {
|
|
980
|
-
const health = claim.scope ? scopedHealthFor(claim.accountId, claim.scope) : upstreamHealth.get(claim.accountId);
|
|
981
|
-
if (!health || health.probeLeaseId !== claim.leaseId) return false;
|
|
982
|
-
const fenced = (claim.scope === undefined || claim.scope === "shared")
|
|
983
|
-
&& health.cooldownSource === "reset-derived"
|
|
984
|
-
&& (health.cooldownGeneration ?? 0) === claim.cooldownGeneration
|
|
985
|
-
&& (health.probeLeaseGeneration ?? 0) === claim.cooldownGeneration;
|
|
986
|
-
if (!recovered || !fenced) {
|
|
987
|
-
const released = withProbeLeaseReleased(health, now);
|
|
988
|
-
if (claim.scope) setScopedHealth(claim.accountId, claim.scope, released);
|
|
989
|
-
else upstreamHealth.set(claim.accountId, released);
|
|
990
|
-
return false;
|
|
991
|
-
}
|
|
992
|
-
if (claim.scope) {
|
|
993
|
-
deleteScopedHealth(claim.accountId, claim.scope);
|
|
994
|
-
} else {
|
|
995
|
-
const {
|
|
996
|
-
cooldownUntil: _until,
|
|
997
|
-
cooldownSince: _since,
|
|
998
|
-
cooldownSource: _source,
|
|
999
|
-
probeLeaseId: _leaseId,
|
|
1000
|
-
probeLeaseGeneration: _leaseGeneration,
|
|
1001
|
-
// "The quota window moved" is a statement about the whole refusal, so the avoidance it
|
|
1002
|
-
// announced goes with the block it produced. Leaving it would make this escape hatch stop
|
|
1003
|
-
// escaping: the account would still be passed over by every selection it is meant to win.
|
|
1004
|
-
quotaAvoidUntil: _avoid,
|
|
1005
|
-
...rest
|
|
1006
|
-
} = health;
|
|
1007
|
-
upstreamHealth.set(claim.accountId, {
|
|
1008
|
-
...rest,
|
|
1009
|
-
cooldownGeneration: claim.cooldownGeneration + 1,
|
|
1010
|
-
lastProbeAt: now,
|
|
1011
|
-
});
|
|
1012
|
-
}
|
|
1013
|
-
return true;
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
/** Acquire the recovery probe for one confirmed model-specific quota group. */
|
|
1017
|
-
export function tryAcquireCodexQuotaScopeProbeLease(
|
|
1018
|
-
accountId: string,
|
|
1019
|
-
scope: CodexQuotaScope,
|
|
1020
|
-
now = Date.now(),
|
|
1021
|
-
): string | null {
|
|
1022
|
-
const health = scopedHealthFor(accountId, scope);
|
|
1023
|
-
if (!canAcquireQuotaProbeLease(health, now)) return null;
|
|
1024
|
-
const probeLeaseId = randomUUID();
|
|
1025
|
-
setScopedHealth(accountId, scope, {
|
|
1026
|
-
...health!,
|
|
1027
|
-
probeLeaseId,
|
|
1028
|
-
probeLeaseGeneration: health!.cooldownGeneration ?? 0,
|
|
1029
|
-
lastProbeAt: now,
|
|
1030
|
-
});
|
|
1031
|
-
return probeLeaseId;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
/** Side-effect-free check for a confirmed model-specific quota probe. */
|
|
1035
|
-
export function canAcquireCodexQuotaScopeProbeLease(
|
|
1036
|
-
accountId: string,
|
|
1037
|
-
scope: CodexQuotaScope,
|
|
1038
|
-
now = Date.now(),
|
|
1039
|
-
): boolean {
|
|
1040
|
-
return canAcquireQuotaProbeLease(scopedHealthFor(accountId, scope), now);
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
/**
|
|
1044
|
-
* Hand a probe lease back without recording an upstream outcome. Used by paths
|
|
1045
|
-
* that take a lease and then fail before any request reaches upstream.
|
|
1046
|
-
*/
|
|
1047
|
-
export function releaseCodexQuotaProbeLease(accountId: string, leaseId: string, now = Date.now()): void {
|
|
1048
|
-
const health = upstreamHealth.get(accountId);
|
|
1049
|
-
if (!health || health.probeLeaseId !== leaseId) return;
|
|
1050
|
-
upstreamHealth.set(accountId, withProbeLeaseReleased(health, now));
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
/** Release a model-specific quota probe when the request never reaches upstream. */
|
|
1054
|
-
export function releaseCodexQuotaScopeProbeLease(
|
|
1055
|
-
accountId: string,
|
|
1056
|
-
scope: CodexQuotaScope,
|
|
1057
|
-
leaseId: string,
|
|
1058
|
-
now = Date.now(),
|
|
1059
|
-
): void {
|
|
1060
|
-
const health = scopedHealthFor(accountId, scope);
|
|
1061
|
-
if (!health || health.probeLeaseId !== leaseId) return;
|
|
1062
|
-
setScopedHealth(accountId, scope, withProbeLeaseReleased(health, now));
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
/**
|
|
1066
|
-
* True when this outcome belongs to the account's in-flight probe. The
|
|
1067
|
-
* undefined-id guard matters: without it an outcome carrying no lease would match
|
|
1068
|
-
* an account holding no lease and be mistaken for the probe owner.
|
|
1069
|
-
*/
|
|
1070
|
-
function ownsProbeLease(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
|
|
1071
|
-
return meta.probeLeaseId !== undefined && meta.probeLeaseId === health?.probeLeaseId;
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
/**
|
|
1075
|
-
* True when the owning probe may still clear the cooldown. A later 429 bumps the
|
|
1076
|
-
* generation, so a probe that started under an older cooldown must not erase the
|
|
1077
|
-
* newer restriction (which may carry an explicit Retry-After).
|
|
1078
|
-
*/
|
|
1079
|
-
function probeMayClearCooldown(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
|
|
1080
|
-
return ownsProbeLease(health, meta)
|
|
1081
|
-
&& (health!.probeLeaseGeneration ?? 0) === (health!.cooldownGeneration ?? 0);
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
/** Strip the in-flight lease while preserving every hard-cooldown field. */
|
|
1085
|
-
function withProbeLeaseReleased(health: CodexUpstreamHealth, now: number): CodexUpstreamHealth {
|
|
1086
|
-
const { probeLeaseId: _id, probeLeaseGeneration: _gen, ...rest } = health;
|
|
1087
|
-
return { ...rest, lastProbeAt: now };
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
/**
|
|
1091
|
-
* Hard-cooldown bookkeeping that ordinary success/transient transitions rebuild
|
|
1092
|
-
* their health object from. Dropping these would let one late unrelated response
|
|
1093
|
-
* erase a Retry-After source, a cooldown generation, or someone else's live probe.
|
|
1094
|
-
*/
|
|
1095
|
-
function preservedCooldownFields(health: CodexUpstreamHealth | undefined): Partial<CodexUpstreamHealth> {
|
|
1096
|
-
if (!health) return {};
|
|
1097
|
-
// `credentialFailureGeneration` is provenance for ONE credential failure, so it must not survive
|
|
1098
|
-
// into a later transient or quota entry — otherwise that entry inherits the tag and gets spent
|
|
1099
|
-
// when the old credential dies, deleting evidence that was never about it (#2892 gap 4 review).
|
|
1100
|
-
const {
|
|
1101
|
-
consecutiveFailures: _f, consecutiveSuccesses: _s, lastFailureStatus: _st, lastFailureAt: _at,
|
|
1102
|
-
softAvoidUntil: _sa, credentialFailureGeneration: _cg, ...cooldownFields
|
|
1103
|
-
} = health;
|
|
1104
|
-
return cooldownFields;
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
/** Manual selection resets transient routing evidence without bypassing a real 429 cooldown. */
|
|
1108
|
-
export function resetCodexRoutingForManualSelection(accountId: string): void {
|
|
1109
|
-
clearThreadAccountMap();
|
|
1110
|
-
// Manual selection is the operator source of truth — drop any automatic runtime cursor.
|
|
1111
|
-
runtimeActiveCodexAccountId = undefined;
|
|
1112
|
-
// Record the pick as an unspent one-shot on the SHARED scope only. An independent scope
|
|
1113
|
-
// gets no entry on purpose: every write site the guard protects is already skipped for
|
|
1114
|
-
// independent scopes, so an entry there would be state nothing reads — and state nothing
|
|
1115
|
-
// reads is what the next reader mistakes for a rule.
|
|
1116
|
-
//
|
|
1117
|
-
// Seeding happens ONLY here. A pool-driven promote must never create or move a preference,
|
|
1118
|
-
// or the pool would manufacture an operator intent nobody expressed.
|
|
1119
|
-
manualPreference.set(POOL_KEY_CODEX, accountId);
|
|
1120
|
-
// Seed the RR ring so the next unbound new session honors the manually selected account
|
|
1121
|
-
// under round-robin (affinity-cleared threads / null threadId). Fill-first already follows
|
|
1122
|
-
// config.activeCodexAccountId, which the caller persists before invoking this.
|
|
1123
|
-
seedPoolRotationAccount(POOL_KEY_CODEX, accountId);
|
|
1124
|
-
for (const scope of new Set(Object.values(NATIVE_MODEL_QUOTA_SCOPES))) {
|
|
1125
|
-
if (isIndependentCodexQuotaScope(scope)) {
|
|
1126
|
-
seedPoolRotationAccount(codexPoolKeyForScope(scope), accountId);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
// Quota avoidance is a preference, like the soft avoid dropped above, and an operator naming
|
|
1130
|
-
// this account has overruled it. The hard cooldown is the part that survives.
|
|
1131
|
-
const overrule = (health: CodexUpstreamHealth) => {
|
|
1132
|
-
const { quotaAvoidUntil: _avoid, ...retained } = preservedCooldownFields(health);
|
|
1133
|
-
return retained;
|
|
1134
|
-
};
|
|
1135
|
-
const current = upstreamHealth.get(accountId);
|
|
1136
|
-
if (current) {
|
|
1137
|
-
const retained = overrule(current);
|
|
1138
|
-
if (Object.keys(retained).length === 0) upstreamHealth.delete(accountId);
|
|
1139
|
-
else upstreamHealth.set(accountId, { consecutiveFailures: 0, ...retained });
|
|
1140
|
-
}
|
|
1141
|
-
// A reset-derived refusal records its avoidance on the SCOPED map and returns before the
|
|
1142
|
-
// account-wide entry is written, so naming the account has to reach that map too. Stopping
|
|
1143
|
-
// at `upstreamHealth` — and returning early when it holds nothing — overruled nothing in
|
|
1144
|
-
// the case that produces the avoidance this function exists to overrule.
|
|
1145
|
-
for (const [scope, health] of [...(quotaScopedHealth.get(accountId) ?? [])]) {
|
|
1146
|
-
const retained = overrule(health);
|
|
1147
|
-
if (Object.keys(retained).length === 0) deleteScopedHealth(accountId, scope);
|
|
1148
|
-
else setScopedHealth(accountId, scope, { consecutiveFailures: 0, ...retained });
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
export function getCodexAccountCooldownUntil(accountId: string, now = Date.now()): number | null {
|
|
1153
|
-
const cooldownUntil = upstreamHealth.get(accountId)?.cooldownUntil;
|
|
1154
|
-
return typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now ? cooldownUntil : null;
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
/** Read-only cooldown snapshot for shared OAuth health projection (no write side effects). */
|
|
1158
|
-
export function getCodexAccountHealthSnapshot(accountId: string, now = Date.now()): {
|
|
1159
|
-
cooldownUntil?: number;
|
|
1160
|
-
cooldownSource?: CodexCooldownSource;
|
|
1161
|
-
} | null {
|
|
1162
|
-
const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
|
|
1163
|
-
if (cooldownUntil === null) return null;
|
|
1164
|
-
const source = upstreamHealth.get(accountId)?.cooldownSource;
|
|
1165
|
-
return {
|
|
1166
|
-
cooldownUntil,
|
|
1167
|
-
...(source ? { cooldownSource: source } : {}),
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
/**
|
|
1172
|
-
* Read the cooldown relevant to a routed native model. Account-wide cooldowns
|
|
1173
|
-
* (Retry-After/default) always win; reset-derived scoped state applies only to
|
|
1174
|
-
* its confirmed quota group.
|
|
1175
|
-
*/
|
|
1176
|
-
export function getCodexQuotaHealthSnapshot(
|
|
1177
|
-
accountId: string,
|
|
1178
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
1179
|
-
now = Date.now(),
|
|
1180
|
-
): {
|
|
1181
|
-
cooldownUntil?: number;
|
|
1182
|
-
cooldownSource?: CodexCooldownSource;
|
|
1183
|
-
quotaScope?: CodexQuotaScope;
|
|
1184
|
-
} | null {
|
|
1185
|
-
const account = getCodexAccountHealthSnapshot(accountId, now);
|
|
1186
|
-
if (account) return account;
|
|
1187
|
-
if (!quotaScope) return null;
|
|
1188
|
-
const scoped = scopedHealthFor(accountId, quotaScope);
|
|
1189
|
-
const cooldownUntil = scoped?.cooldownUntil;
|
|
1190
|
-
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null;
|
|
1191
|
-
return {
|
|
1192
|
-
cooldownUntil,
|
|
1193
|
-
...(scoped?.cooldownSource ? { cooldownSource: scoped.cooldownSource } : {}),
|
|
1194
|
-
quotaScope,
|
|
1195
|
-
};
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
export function isCodexAccountInCooldown(accountId: string, now = Date.now()): boolean {
|
|
1199
|
-
return getCodexAccountCooldownUntil(accountId, now) !== null;
|
|
1200
|
-
}
|
|
1201
|
-
|
|
1202
|
-
/**
|
|
1203
|
-
* Manually lift a hard quota cooldown without touching failure history.
|
|
1204
|
-
*
|
|
1205
|
-
* Injected Codex routing makes this proxy the ONLY model path for Codex Desktop, so a
|
|
1206
|
-
* cooldown that outlives the real upstream limit reads to the user as "the whole app is
|
|
1207
|
-
* broken" with no escape but editing config.toml. This is that escape hatch.
|
|
1208
|
-
*
|
|
1209
|
-
* Deliberately narrow:
|
|
1210
|
-
* - Failure counters and softAvoid survive. Clearing a cooldown says "the quota window
|
|
1211
|
-
* moved", not "this account is healthy"; failover must keep its knowledge.
|
|
1212
|
-
* - Dropping `probeLeaseId` is what stops a stale in-flight probe from later "proving"
|
|
1213
|
-
* recovery against a NEWER cooldown: {@link ownsProbeLease} needs the id to match.
|
|
1214
|
-
* `cooldownGeneration` is preserved and bumped as redundancy only — a fresh 429 already
|
|
1215
|
-
* bumps it in {@link recordCodexUpstreamOutcome}, so the bump here is not load-bearing
|
|
1216
|
-
* today and is kept so the invariant survives a future change that retains the lease.
|
|
1217
|
-
*
|
|
1218
|
-
* Returns false when the account carried neither a live cooldown nor a live avoidance window.
|
|
1219
|
-
* The window outlives the cooldown by design — the cooldown caps at fifteen minutes and the
|
|
1220
|
-
* window runs up to six hours — so the moment an operator actually reaches for this escape
|
|
1221
|
-
* hatch is usually after the cooldown lapsed and only the window is still keeping the account
|
|
1222
|
-
* out of rotation. Refusing to look at the window then would leave the hatch shut in the one
|
|
1223
|
-
* case it exists for.
|
|
1224
|
-
*/
|
|
1225
|
-
export function clearCodexAccountCooldown(accountId: string, now = Date.now()): boolean {
|
|
1226
|
-
const clear = (health: CodexUpstreamHealth): CodexUpstreamHealth | null => {
|
|
1227
|
-
const cooldownUntil = health.cooldownUntil;
|
|
1228
|
-
const liveCooldown = typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now;
|
|
1229
|
-
const avoidUntil = health.quotaAvoidUntil;
|
|
1230
|
-
const liveAvoidance = typeof avoidUntil === "number" && Number.isFinite(avoidUntil) && avoidUntil > now;
|
|
1231
|
-
if (!liveCooldown && !liveAvoidance) return null;
|
|
1232
|
-
const {
|
|
1233
|
-
cooldownUntil: _until,
|
|
1234
|
-
cooldownSince: _since,
|
|
1235
|
-
cooldownSource: _source,
|
|
1236
|
-
probeLeaseId: _leaseId,
|
|
1237
|
-
probeLeaseGeneration: _leaseGeneration,
|
|
1238
|
-
// Same reasoning as the probe recovery above: "the quota window moved" is a statement
|
|
1239
|
-
// about the whole refusal, so the avoidance it announced goes with the block it
|
|
1240
|
-
// produced. Keeping it would leave this escape hatch not escaping, because selection
|
|
1241
|
-
// would still pass over the account for as long as the announced window runs.
|
|
1242
|
-
quotaAvoidUntil: _avoid,
|
|
1243
|
-
...rest
|
|
1244
|
-
} = health;
|
|
1245
|
-
return {
|
|
1246
|
-
...rest,
|
|
1247
|
-
cooldownGeneration: (health.cooldownGeneration ?? 0) + 1,
|
|
1248
|
-
lastProbeAt: now,
|
|
1249
|
-
};
|
|
1250
|
-
};
|
|
1251
|
-
|
|
1252
|
-
let cleared = false;
|
|
1253
|
-
const accountHealth = upstreamHealth.get(accountId);
|
|
1254
|
-
if (accountHealth) {
|
|
1255
|
-
const next = clear(accountHealth);
|
|
1256
|
-
if (next) {
|
|
1257
|
-
upstreamHealth.set(accountId, next);
|
|
1258
|
-
cleared = true;
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
for (const [scope, health] of quotaScopedHealth.get(accountId) ?? []) {
|
|
1262
|
-
const next = clear(health);
|
|
1263
|
-
if (next) {
|
|
1264
|
-
setScopedHealth(accountId, scope, next);
|
|
1265
|
-
cleared = true;
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
return cleared;
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
export function getCodexAccountSoftAvoidUntil(accountId: string, now = Date.now()): number | null {
|
|
1272
|
-
const softAvoidUntil = upstreamHealth.get(accountId)?.softAvoidUntil;
|
|
1273
|
-
return typeof softAvoidUntil === "number" && Number.isFinite(softAvoidUntil) && softAvoidUntil > now
|
|
1274
|
-
? softAvoidUntil
|
|
1275
|
-
: null;
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
export function isCodexAccountSoftAvoided(accountId: string, now = Date.now()): boolean {
|
|
1279
|
-
return getCodexAccountSoftAvoidUntil(accountId, now) !== null;
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
/**
|
|
1283
|
-
* Plan keys the operator excluded from automatic rotation. Absent or empty means no policy, so an
|
|
1284
|
-
* existing install rotates exactly as before. Compared with `codexPlanKey` because the stored plan
|
|
1285
|
-
* is an unrestricted provider string whose casing this repository does not control.
|
|
1286
|
-
*/
|
|
1287
|
-
function excludedCodexPoolPlanKeys(config: OcxConfig): ReadonlySet<string> | undefined {
|
|
1288
|
-
const configured = config.codexPool?.excludedPlans;
|
|
1289
|
-
if (!configured?.length) return undefined;
|
|
1290
|
-
const keys = configured
|
|
1291
|
-
.map(plan => codexPlanKey(plan))
|
|
1292
|
-
.filter((key): key is string => key !== undefined);
|
|
1293
|
-
return keys.length > 0 ? new Set(keys) : undefined;
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
/**
|
|
1297
|
-
* Whether the operator's plan policy removes this account from automatic selection.
|
|
1298
|
-
*
|
|
1299
|
-
* Modelled on pause rather than usability: an excluded account keeps its credential, quota history,
|
|
1300
|
-
* and affinity, stays visible on the account surface, and is still reachable by explicit account
|
|
1301
|
-
* selection. Only automatic rotation skips it, which is the distinction #4211 asked for.
|
|
1302
|
-
*
|
|
1303
|
-
* It is checked in the same two places pause is checked, and that is not redundancy. The eligible
|
|
1304
|
-
* list is consulted only when routing picks a NEW account; an already-active or already-affined
|
|
1305
|
-
* account is served straight from {@link isCodexAccountSelectable}. A lapsed subscription leaves
|
|
1306
|
-
* behind exactly that account, so a policy that filtered only the eligible list would miss the case
|
|
1307
|
-
* it exists for.
|
|
1308
|
-
*
|
|
1309
|
-
* `__main__` is exempt. {@link getPoolAccountPlanForSelection} withholds the main plan during a
|
|
1310
|
-
* selection-only drain so routing never reads the fenced native credential for it, so a rule that
|
|
1311
|
-
* covered main would disagree with itself between drain and ordinary routing.
|
|
1312
|
-
*/
|
|
1313
|
-
export function isCodexAccountPlanExcluded(
|
|
1314
|
-
config: OcxConfig,
|
|
1315
|
-
accountId: string,
|
|
1316
|
-
precomputed?: ReadonlySet<string>,
|
|
1317
|
-
): boolean {
|
|
1318
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return false;
|
|
1319
|
-
// Callers that test a whole list pass the set once rather than rebuilding it per row.
|
|
1320
|
-
const excluded = precomputed ?? excludedCodexPoolPlanKeys(config);
|
|
1321
|
-
if (!excluded) return false;
|
|
1322
|
-
const plan = codexPlanKey(getPoolAccountPlan(config, accountId));
|
|
1323
|
-
return plan !== undefined && excluded.has(plan);
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
function isCodexAccountSelectable(
|
|
1327
|
-
config: OcxConfig,
|
|
1328
|
-
accountId: string,
|
|
1329
|
-
now: number,
|
|
1330
|
-
quotaScope?: CodexQuotaScope,
|
|
1331
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1332
|
-
): boolean {
|
|
1333
|
-
return !isCodexAccountPaused(config, accountId)
|
|
1334
|
-
&& !isCodexAccountPlanExcluded(config, accountId)
|
|
1335
|
-
&& getCodexQuotaHealthSnapshot(accountId, quotaScope, now) === null
|
|
1336
|
-
&& !isCodexQuotaAvoided(accountId, quotaScope, now)
|
|
1337
|
-
&& !isCodexAccountSoftAvoided(accountId, now)
|
|
1338
|
-
&& isCodexAccountUsable(config, accountId, selectionOptions);
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
|
-
/**
|
|
1342
|
-
* Which guard in {@link isCodexAccountSelectable} refused this account, if any.
|
|
1343
|
-
*
|
|
1344
|
-
* Deliberately the same predicates in the same order as that function, because the point is to
|
|
1345
|
-
* REPORT the guard that actually fired rather than to re-derive a plausible-looking cause. An
|
|
1346
|
-
* earlier version of the release reason checked only a subset and let a paused, plan-excluded,
|
|
1347
|
-
* cooled-down or quota-avoided release fall through to a quota fallback, which named something
|
|
1348
|
-
* routing never used -- a diagnostic that is confidently wrong in exactly the cases an operator
|
|
1349
|
-
* would consult it for (#4598).
|
|
1350
|
-
*/
|
|
1351
|
-
function codexAccountBlockReason(
|
|
1352
|
-
config: OcxConfig,
|
|
1353
|
-
accountId: string,
|
|
1354
|
-
now: number,
|
|
1355
|
-
quotaScope?: CodexQuotaScope,
|
|
1356
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1357
|
-
): CodexAffinityReason | undefined {
|
|
1358
|
-
if (isCodexAccountPaused(config, accountId)) return "paused";
|
|
1359
|
-
if (isCodexAccountPlanExcluded(config, accountId)) return "plan_excluded";
|
|
1360
|
-
if (getCodexQuotaHealthSnapshot(accountId, quotaScope, now) !== null) return "cooldown";
|
|
1361
|
-
if (isCodexQuotaAvoided(accountId, quotaScope, now)) return "quota_avoided";
|
|
1362
|
-
if (isCodexAccountSoftAvoided(accountId, now)) return "transient";
|
|
1363
|
-
if (!isCodexAccountUsable(config, accountId, selectionOptions)) return "unusable";
|
|
1364
|
-
return undefined;
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
function threadAffinityScope(quotaScope?: CodexQuotaScope): BaseThreadAffinityScope {
|
|
1368
|
-
return quotaScope ?? LEGACY_THREAD_AFFINITY_SCOPE;
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
function admissibleAffinityComponent(value: string): boolean {
|
|
1372
|
-
return retainedUtf8Bytes(value) <= MAX_AFFINITY_COMPONENT_BYTES;
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
function modelDetourAffinityScope(
|
|
1376
|
-
modelId: string | undefined,
|
|
1377
|
-
quotaScope?: CodexQuotaScope,
|
|
1378
|
-
): ModelDetourAffinityScope | undefined {
|
|
1379
|
-
const canonicalModelId = modelId?.trim().toLowerCase();
|
|
1380
|
-
if (!canonicalModelId || !admissibleAffinityComponent(canonicalModelId)) return undefined;
|
|
1381
|
-
return `model-detour:${threadAffinityScope(quotaScope)}:${canonicalModelId}`;
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
function getThreadAffinityForScope(
|
|
1385
|
-
threadId: string,
|
|
1386
|
-
scope: ThreadAffinityScope,
|
|
1387
|
-
): ThreadAffinityEntry | undefined {
|
|
1388
|
-
if (!admissibleAffinityComponent(threadId)) return undefined;
|
|
1389
|
-
return threadAccountMap.get(threadId)?.get(scope);
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
function getThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): ThreadAffinityEntry | undefined {
|
|
1393
|
-
return getThreadAffinityForScope(threadId, threadAffinityScope(quotaScope));
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
function getModelDetourAffinity(
|
|
1397
|
-
threadId: string,
|
|
1398
|
-
modelId: string | undefined,
|
|
1399
|
-
quotaScope?: CodexQuotaScope,
|
|
1400
|
-
): ThreadAffinityEntry | undefined {
|
|
1401
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1402
|
-
return scope ? getThreadAffinityForScope(threadId, scope) : undefined;
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
function deleteThreadAffinityForScope(threadId: string, scope: ThreadAffinityScope): void {
|
|
1406
|
-
if (!admissibleAffinityComponent(threadId)) return;
|
|
1407
|
-
const affinities = threadAccountMap.get(threadId);
|
|
1408
|
-
if (!affinities) return;
|
|
1409
|
-
if (affinities.delete(scope)) {
|
|
1410
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1411
|
-
}
|
|
1412
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
function deleteThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): void {
|
|
1416
|
-
deleteThreadAffinityForScope(threadId, threadAffinityScope(quotaScope));
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
function deleteModelDetourAffinity(
|
|
1420
|
-
threadId: string,
|
|
1421
|
-
modelId: string | undefined,
|
|
1422
|
-
quotaScope?: CodexQuotaScope,
|
|
1423
|
-
): void {
|
|
1424
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1425
|
-
if (scope) deleteThreadAffinityForScope(threadId, scope);
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
/** Remove only the matching failed account's affinities for one thread. */
|
|
1429
|
-
function deleteThreadAffinitiesForAccount(threadId: string, accountId: string): void {
|
|
1430
|
-
if (!admissibleAffinityComponent(threadId) || !admissibleAffinityComponent(accountId)) return;
|
|
1431
|
-
const affinities = threadAccountMap.get(threadId);
|
|
1432
|
-
if (!affinities) return;
|
|
1433
|
-
for (const [scope, entry] of affinities) {
|
|
1434
|
-
if (entry.accountId === accountId && affinities.delete(scope)) {
|
|
1435
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1436
|
-
}
|
|
1437
|
-
}
|
|
1438
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1439
|
-
}
|
|
1440
|
-
|
|
1441
|
-
function threadAffinityEntryCount(): number {
|
|
1442
|
-
return threadAffinityEntryTotal;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
function isThreadAffinityExpired(entry: ThreadAffinityEntry, now: number): boolean {
|
|
1446
|
-
return now - entry.lastUsedAt > CODEX_THREAD_AFFINITY_IDLE_TTL_MS;
|
|
1447
|
-
}
|
|
1448
|
-
|
|
1449
|
-
function isThreadAffinityGenerationLive(entry: ThreadAffinityEntry): boolean {
|
|
1450
|
-
if (entry.accountId === MAIN_CODEX_ACCOUNT_ID) return entry.generation === 0;
|
|
1451
|
-
return isCodexAccountGenerationLive(entry.accountId, entry.generation);
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
/** Generations this account's affinity entries are bound at. Test observability only. */
|
|
1455
|
-
export function debugCodexAffinityGenerations(accountId: string): number[] {
|
|
1456
|
-
const generations: number[] = [];
|
|
1457
|
-
for (const affinities of threadAccountMap.values()) {
|
|
1458
|
-
for (const entry of affinities.values()) {
|
|
1459
|
-
if (entry.accountId === accountId) generations.push(entry.generation);
|
|
1460
|
-
}
|
|
1461
|
-
}
|
|
1462
|
-
return generations;
|
|
1463
|
-
}
|
|
1464
|
-
|
|
1465
|
-
/**
|
|
1466
|
-
* Advance this account's affinity entries from the generation a rejected credential
|
|
1467
|
-
* was bound under to the generation its own refresh produced.
|
|
1468
|
-
*
|
|
1469
|
-
* A 401 refresh-and-replay keeps the request on the same account, but the CAS write
|
|
1470
|
-
* moves the credential from G to G+1, and {@link isThreadAffinityGenerationLive}
|
|
1471
|
-
* demands exact equality — so without this the entry the replay just preserved is
|
|
1472
|
-
* dead on the next request. Not quarantining an account is not the same as keeping
|
|
1473
|
-
* its affinity.
|
|
1474
|
-
*
|
|
1475
|
-
* Lineage is proven by the CALLER, which must pass only a generation its own refresh
|
|
1476
|
-
* produced. Re-deriving it here from `replacedAt` cannot work: the caller reads that
|
|
1477
|
-
* field after the refresh and this function would re-read the same record, so the
|
|
1478
|
-
* comparison is tautological and an external replacement passes it. An external
|
|
1479
|
-
* replacement must retire the affinity, because that credential may belong to a
|
|
1480
|
-
* different upstream identity.
|
|
1481
|
-
*/
|
|
1482
|
-
export function handOffThreadAffinityGeneration(
|
|
1483
|
-
accountId: string,
|
|
1484
|
-
fromGeneration: number,
|
|
1485
|
-
toGeneration: number,
|
|
1486
|
-
): boolean {
|
|
1487
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return false;
|
|
1488
|
-
if (toGeneration !== fromGeneration + 1) return false;
|
|
1489
|
-
const record = readCodexAccountRecord(accountId);
|
|
1490
|
-
if (!record?.credential || record.deletedAt != null) return false;
|
|
1491
|
-
if (record.generation !== toGeneration) return false;
|
|
1492
|
-
let handedOff = false;
|
|
1493
|
-
for (const affinities of threadAccountMap.values()) {
|
|
1494
|
-
for (const entry of affinities.values()) {
|
|
1495
|
-
if (entry.accountId !== accountId || entry.generation !== fromGeneration) continue;
|
|
1496
|
-
entry.generation = toGeneration;
|
|
1497
|
-
handedOff = true;
|
|
1498
|
-
}
|
|
1499
|
-
}
|
|
1500
|
-
return handedOff;
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
function pruneExpiredThreadAffinities(now: number): void {
|
|
1504
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
1505
|
-
for (const [scope, entry] of affinities) {
|
|
1506
|
-
if (isThreadAffinityExpired(entry, now) && affinities.delete(scope)) {
|
|
1507
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
function pruneLruThreadAffinities(): void {
|
|
1515
|
-
if (threadAffinityEntryCount() <= CODEX_THREAD_AFFINITY_MAX_ENTRIES) return;
|
|
1516
|
-
while (threadAffinityEntryCount() > CODEX_THREAD_AFFINITY_MAX_ENTRIES) {
|
|
1517
|
-
let oldestThreadId: string | null = null;
|
|
1518
|
-
let oldestScope: ThreadAffinityScope | null = null;
|
|
1519
|
-
let oldestLastUsedAt = Number.POSITIVE_INFINITY;
|
|
1520
|
-
let oldestIsDetour = false;
|
|
1521
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
1522
|
-
for (const [scope, entry] of affinities) {
|
|
1523
|
-
const candidateIsDetour = isModelDetourAffinityScope(scope);
|
|
1524
|
-
if (
|
|
1525
|
-
(candidateIsDetour && !oldestIsDetour)
|
|
1526
|
-
|| (candidateIsDetour === oldestIsDetour && entry.lastUsedAt < oldestLastUsedAt)
|
|
1527
|
-
) {
|
|
1528
|
-
oldestThreadId = threadId;
|
|
1529
|
-
oldestScope = scope;
|
|
1530
|
-
oldestLastUsedAt = entry.lastUsedAt;
|
|
1531
|
-
oldestIsDetour = candidateIsDetour;
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
if (!oldestThreadId || !oldestScope) return;
|
|
1536
|
-
deleteThreadAffinityForScope(oldestThreadId, oldestScope);
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
|
|
1540
|
-
function bindThreadAffinityForScope(
|
|
1541
|
-
threadId: string,
|
|
1542
|
-
accountId: string,
|
|
1543
|
-
now: number,
|
|
1544
|
-
scope: ThreadAffinityScope,
|
|
1545
|
-
): void {
|
|
1546
|
-
if (!admissibleAffinityComponent(threadId) || !admissibleAffinityComponent(accountId)) return;
|
|
1547
|
-
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
|
|
1548
|
-
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return;
|
|
1549
|
-
pruneExpiredThreadAffinities(now);
|
|
1550
|
-
const affinities = threadAccountMap.get(threadId) ?? new Map<ThreadAffinityScope, ThreadAffinityEntry>();
|
|
1551
|
-
const previous = affinities.get(scope);
|
|
1552
|
-
affinities.set(scope, {
|
|
1553
|
-
accountId,
|
|
1554
|
-
generation: accountId === MAIN_CODEX_ACCOUNT_ID ? 0 : record!.generation,
|
|
1555
|
-
createdAt: previous?.createdAt ?? now,
|
|
1556
|
-
lastUsedAt: now,
|
|
1557
|
-
lastReevalAt: now,
|
|
1558
|
-
});
|
|
1559
|
-
if (!previous) threadAffinityEntryTotal += 1;
|
|
1560
|
-
threadAccountMap.set(threadId, affinities);
|
|
1561
|
-
pruneLruThreadAffinities();
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
function bindThreadAffinity(
|
|
1565
|
-
threadId: string,
|
|
1566
|
-
accountId: string,
|
|
1567
|
-
now: number,
|
|
1568
|
-
quotaScope?: CodexQuotaScope,
|
|
1569
|
-
): void {
|
|
1570
|
-
bindThreadAffinityForScope(threadId, accountId, now, threadAffinityScope(quotaScope));
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
function bindModelDetourAffinity(
|
|
1574
|
-
threadId: string,
|
|
1575
|
-
accountId: string,
|
|
1576
|
-
now: number,
|
|
1577
|
-
modelId: string | undefined,
|
|
1578
|
-
quotaScope?: CodexQuotaScope,
|
|
1579
|
-
): void {
|
|
1580
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1581
|
-
if (scope) bindThreadAffinityForScope(threadId, accountId, now, scope);
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
|
-
function getEligiblePoolAccounts(
|
|
1585
|
-
config: OcxConfig,
|
|
1586
|
-
excludeId?: string,
|
|
1587
|
-
now = Date.now(),
|
|
1588
|
-
quotaScope?: CodexQuotaScope,
|
|
1589
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1590
|
-
skipFailoverReadyCandidates = false,
|
|
1591
|
-
): readonly string[] {
|
|
1592
|
-
const excludedPlans = excludedCodexPoolPlanKeys(config);
|
|
1593
|
-
const ids = (config.codexAccounts ?? [])
|
|
1594
|
-
.filter(account => isSelectableCodexPoolAccount(account)
|
|
1595
|
-
&& account.id !== excludeId
|
|
1596
|
-
&& !isCodexAccountPaused(config, account.id)
|
|
1597
|
-
&& !isCodexAccountPlanExcluded(config, account.id, excludedPlans)
|
|
1598
|
-
&& !isAccountNeedsReauth(account.id)
|
|
1599
|
-
&& (!skipFailoverReadyCandidates || !shouldFailover(config, account.id, now)))
|
|
1600
|
-
.filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
|
|
1601
|
-
.filter(account => !isCodexAccountSoftAvoided(account.id, now))
|
|
1602
|
-
.filter(account => !isCodexQuotaAvoided(account.id, quotaScope, now))
|
|
1603
|
-
.filter(account => isCodexAccountUsable(config, account.id, selectionOptions))
|
|
1604
|
-
.map(account => account.id);
|
|
1605
|
-
// The main Codex account is not stored in config.codexAccounts; include it as a
|
|
1606
|
-
// first-class rotation candidate when its read-only token is usable (Option A).
|
|
1607
|
-
if (
|
|
1608
|
-
excludeId !== MAIN_CODEX_ACCOUNT_ID
|
|
1609
|
-
&& !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)
|
|
1610
|
-
&& (!isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) || hasMainAccountRefreshGrant())
|
|
1611
|
-
&& getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
|
|
1612
|
-
&& !isCodexAccountSoftAvoided(MAIN_CODEX_ACCOUNT_ID, now)
|
|
1613
|
-
// The main login is not in `config.codexAccounts`, so it never passes through the
|
|
1614
|
-
// filters above and this is the only place an avoidance window can exclude it. Without
|
|
1615
|
-
// this the window a refusal announced applies to the pool but not to the account that
|
|
1616
|
-
// earned it: the cooldown caps at fifteen minutes, the window runs up to six hours, and
|
|
1617
|
-
// in between the main account returns as a first-class candidate.
|
|
1618
|
-
&& !isCodexQuotaAvoided(MAIN_CODEX_ACCOUNT_ID, quotaScope, now)
|
|
1619
|
-
&& (!skipFailoverReadyCandidates || !shouldFailover(config, MAIN_CODEX_ACCOUNT_ID, now))
|
|
1620
|
-
&& isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions)
|
|
1621
|
-
) {
|
|
1622
|
-
ids.unshift(MAIN_CODEX_ACCOUNT_ID);
|
|
1623
|
-
}
|
|
1624
|
-
// Single choke point for selection order: every strategy, failover, and preview
|
|
1625
|
-
// reaches the pool through here, so tiering applies once rather than per picker.
|
|
1626
|
-
// Eligibility above is unchanged — this only narrows an already-eligible list.
|
|
1627
|
-
return selectPriorityTier(
|
|
1628
|
-
ids,
|
|
1629
|
-
codexAccountPriorityLookup(config),
|
|
1630
|
-
id => hasCodexQuotaHeadroom(config, id, selectionOptions, now),
|
|
1631
|
-
pinnedCodexAccountId(config),
|
|
1632
|
-
);
|
|
1633
|
-
}
|
|
1634
|
-
|
|
1635
|
-
function listEligibleCodexAccountIds(
|
|
1636
|
-
config: OcxConfig,
|
|
1637
|
-
now: number,
|
|
1638
|
-
quotaScope?: CodexQuotaScope,
|
|
1639
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1640
|
-
): readonly string[] {
|
|
1641
|
-
return getEligiblePoolAccounts(config, undefined, now, quotaScope, selectionOptions);
|
|
1642
|
-
}
|
|
1643
|
-
|
|
1644
|
-
/** Shared reset timestamps are not evidence for independent model-quota groups. */
|
|
1645
|
-
function accountPoolStrategyForScope(config: OcxConfig, quotaScope?: CodexQuotaScope) {
|
|
1646
|
-
const strategy = normalizeCodexAccountPoolStrategy(config.accountPoolStrategy);
|
|
1647
|
-
return strategy === "reset-first" && isIndependentCodexQuotaScope(quotaScope) ? "quota" : strategy;
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
function stickyLimitForConfig(config: OcxConfig): number {
|
|
1651
|
-
return normalizeAccountPoolStickyLimit(config.accountPoolStickyLimit);
|
|
1652
|
-
}
|
|
1653
|
-
|
|
1654
|
-
/**
|
|
1655
|
-
* Whether an account still has quota to give under the auto-switch threshold.
|
|
1656
|
-
*
|
|
1657
|
-
* Fill-first and the priority tier filter share this predicate, and share both of
|
|
1658
|
-
* its escape hatches. A disabled threshold means only health, pause, and reauth
|
|
1659
|
-
* may drain an account; unknown usage is a guess, so it must neither force
|
|
1660
|
-
* fill-first off the active account nor drain a tier that was simply never
|
|
1661
|
-
* primed. A genuinely exhausted account 429s into cooldown and leaves
|
|
1662
|
-
* eligibility on its own.
|
|
1663
|
-
*/
|
|
1664
|
-
function hasCodexQuotaHeadroom(
|
|
1665
|
-
config: OcxConfig,
|
|
1666
|
-
accountId: string,
|
|
1667
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1668
|
-
now: number = Date.now(),
|
|
1669
|
-
): boolean {
|
|
1670
|
-
const threshold = config.autoSwitchThreshold ?? 80;
|
|
1671
|
-
if (threshold <= 0) return true;
|
|
1672
|
-
const usage = computeCodexUsageScore(
|
|
1673
|
-
getAccountQuota(accountId),
|
|
1674
|
-
getPoolAccountPlanForSelection(config, accountId, selectionOptions),
|
|
1675
|
-
now,
|
|
1676
|
-
);
|
|
1677
|
-
if (isUnknownUsage(usage)) return true;
|
|
1678
|
-
return usage < threshold;
|
|
217
|
+
// recordCodexUpstreamOutcome would still drop a writer whose generation predates the
|
|
218
|
+
// watermark for any account missing from the equally stale live set. Left behind, it also
|
|
219
|
+
// leaks between test files, which is how it was found.
|
|
220
|
+
resetHealthReconcileState();
|
|
1679
221
|
}
|
|
1680
222
|
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
* the install that gets hurt by it, so the protection cannot be something you have to find.
|
|
1689
|
-
*
|
|
1690
|
-
* `false` restores capacity-first routing byte-for-byte. It is a real choice -- a pinned thread
|
|
1691
|
-
* on a busy account pays latency -- and it stays available; it is just no longer the default.
|
|
1692
|
-
*/
|
|
1693
|
-
function isCacheAffinityEnabled(config: OcxConfig): boolean {
|
|
1694
|
-
return config.pool?.cacheAffinity !== false;
|
|
223
|
+
export function clearCodexUpstreamHealthForAccount(accountId: string): void {
|
|
224
|
+
deleteAllHealthForAccount(accountId);
|
|
225
|
+
// Deletion is the third operator exit, next to pause and exclusion, and it is the one
|
|
226
|
+
// with no reconcile path behind it: once the account is gone nothing can succeed on it,
|
|
227
|
+
// so an unspent preference naming it would suppress the automatic cursor for every other
|
|
228
|
+
// account until the process restarts.
|
|
229
|
+
forgetManualPreference(accountId);
|
|
1695
230
|
}
|
|
1696
231
|
|
|
232
|
+
export function reconcileCodexRoutingHealth(context: GenerationContext): number {
|
|
233
|
+
if (isHealthGenerationReconciled(context.generation)) return 0;
|
|
234
|
+
const removed = pruneHealthAccountsForContext(context.codexAccountIds);
|
|
235
|
+
// Sweep preferences the same way, for the account set this generation actually has. The
|
|
236
|
+
// delete path above is the direct route; this is the one that catches an account removed
|
|
237
|
+
// by an edit the runtime never saw. Deliberately not counted in `removed`, which reports
|
|
238
|
+
// health rows.
|
|
239
|
+
forgetRoutingPreferencesOutside(context.codexAccountIds);
|
|
240
|
+
commitHealthReconcile(context.generation, context.codexAccountIds);
|
|
241
|
+
return removed;
|
|
242
|
+
}
|
|
1697
243
|
/**
|
|
1698
244
|
* Is a transient failure streak the ONLY thing standing between this thread and its account?
|
|
1699
245
|
*
|
|
@@ -1725,7 +271,9 @@ function isTransientOnlyAffinityBlock(
|
|
|
1725
271
|
if (!isCodexAccountUsable(config, entry.accountId, selectionOptions)) return false;
|
|
1726
272
|
if (getCodexQuotaHealthSnapshot(entry.accountId, quotaScope, now) !== null) return false;
|
|
1727
273
|
if (isCodexQuotaAvoided(entry.accountId, quotaScope, now)) return false;
|
|
1728
|
-
return shouldFailover(config, entry.accountId, now)
|
|
274
|
+
return shouldFailover(config, entry.accountId, now)
|
|
275
|
+
|| isCodexAccountSoftAvoided(entry.accountId, now)
|
|
276
|
+
|| isCodexPoolRefreshCooling(entry.accountId, now);
|
|
1729
277
|
}
|
|
1730
278
|
|
|
1731
279
|
/** Has a held binding waited longer than a transient failure can reasonably explain? */
|
|
@@ -1742,7 +290,7 @@ function isTransientHoldExpired(entry: ThreadAffinityEntry, now: number): boolea
|
|
|
1742
290
|
* that chance away.
|
|
1743
291
|
*/
|
|
1744
292
|
function isTransientHoldSpentForAccount(threadId: string, accountId: string, now: number): boolean {
|
|
1745
|
-
const affinities =
|
|
293
|
+
const affinities = getThreadAffinityScopes(threadId);
|
|
1746
294
|
if (!affinities) return false;
|
|
1747
295
|
let matched = false;
|
|
1748
296
|
for (const entry of affinities.values()) {
|
|
@@ -1784,429 +332,76 @@ function transientDetourAccount(
|
|
|
1784
332
|
: pickAlternateCodexAccount(config, entry.accountId, now, quotaScope, selectionOptions);
|
|
1785
333
|
}
|
|
1786
334
|
|
|
1787
|
-
/** Earliest future shared short/weekly reset; missing evidence and ties use usage order. */
|
|
1788
|
-
function pickResetFirstCodexAccount(
|
|
1789
|
-
config: OcxConfig,
|
|
1790
|
-
ids: readonly string[],
|
|
1791
|
-
now: number,
|
|
1792
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1793
|
-
): string | null {
|
|
1794
|
-
const available = ids.filter(id => hasCodexQuotaHeadroom(config, id, selectionOptions, now));
|
|
1795
|
-
if (available.length === 0) return pickLowestUsageAmong(config, ids, selectionOptions, now);
|
|
1796
|
-
let earliest = Number.POSITIVE_INFINITY;
|
|
1797
|
-
let candidates: string[] = [];
|
|
1798
|
-
for (const id of available) {
|
|
1799
|
-
const quota = getAccountQuota(id);
|
|
1800
|
-
const resets = [quota?.shortResetAt, quota?.weeklyResetAt]
|
|
1801
|
-
.filter((reset): reset is number => typeof reset === "number" && Number.isFinite(reset))
|
|
1802
|
-
.map(resetAtToMs)
|
|
1803
|
-
.filter(reset => reset > now);
|
|
1804
|
-
const next = Math.min(...resets);
|
|
1805
|
-
if (next < earliest) {
|
|
1806
|
-
earliest = next;
|
|
1807
|
-
candidates = [id];
|
|
1808
|
-
} else if (next === earliest) candidates.push(id);
|
|
1809
|
-
}
|
|
1810
|
-
return pickLowestUsageAmong(config, candidates, selectionOptions, now);
|
|
1811
|
-
}
|
|
1812
|
-
|
|
1813
|
-
/**
|
|
1814
|
-
* Fill-first: keep selectable active under threshold; otherwise advance to the next
|
|
1815
|
-
* eligible id in stable sorted order after the current active (wrapping).
|
|
1816
|
-
*/
|
|
1817
|
-
function pickFillFirstCodexAccount(
|
|
1818
|
-
config: OcxConfig,
|
|
1819
|
-
now: number,
|
|
1820
|
-
quotaScope?: CodexQuotaScope,
|
|
1821
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1822
|
-
): string | null {
|
|
1823
|
-
const eligible = listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions);
|
|
1824
|
-
if (eligible.length === 0) return null;
|
|
1825
|
-
|
|
1826
|
-
const active = getEffectiveActiveCodexAccountId(config);
|
|
1827
|
-
if (active && eligible.includes(active) && hasCodexQuotaHeadroom(config, active, selectionOptions, now)) {
|
|
1828
|
-
return active;
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
return pickNextFillFirstCodexAccount(config, active ?? null, eligible, now, selectionOptions);
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
|
-
/** Next eligible account in stable order after `afterId` (wrapping). */
|
|
1835
|
-
function pickNextFillFirstCodexAccount(
|
|
1836
|
-
config: OcxConfig,
|
|
1837
|
-
afterId: string | null,
|
|
1838
|
-
eligible: readonly string[] = listEligibleCodexAccountIds(config, Date.now()),
|
|
1839
|
-
now = Date.now(),
|
|
1840
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1841
|
-
): string | null {
|
|
1842
|
-
if (eligible.length === 0) return null;
|
|
1843
|
-
const ordered = [...eligible].sort((a, b) => a.localeCompare(b));
|
|
1844
|
-
if (!afterId) {
|
|
1845
|
-
// Prefer an under-threshold account when starting with no active cursor.
|
|
1846
|
-
for (const id of ordered) {
|
|
1847
|
-
if (hasCodexQuotaHeadroom(config, id, selectionOptions, now)) return id;
|
|
1848
|
-
}
|
|
1849
|
-
return ordered[0] ?? null;
|
|
1850
|
-
}
|
|
1851
|
-
|
|
1852
|
-
const allConfigured = [
|
|
1853
|
-
...(isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions) || afterId === MAIN_CODEX_ACCOUNT_ID
|
|
1854
|
-
? [MAIN_CODEX_ACCOUNT_ID]
|
|
1855
|
-
: []),
|
|
1856
|
-
...(config.codexAccounts ?? []).filter(account => !account.isMain).map(account => account.id),
|
|
1857
|
-
];
|
|
1858
|
-
const stableAll = [...new Set(allConfigured)].sort((a, b) => a.localeCompare(b));
|
|
1859
|
-
const startIdx = stableAll.indexOf(afterId);
|
|
1860
|
-
if (startIdx < 0) {
|
|
1861
|
-
for (const id of ordered) {
|
|
1862
|
-
if (hasCodexQuotaHeadroom(config, id, selectionOptions, now)) return id;
|
|
1863
|
-
}
|
|
1864
|
-
return ordered[0] ?? null;
|
|
1865
|
-
}
|
|
1866
|
-
|
|
1867
|
-
// Skip successors that are also at/above threshold (known drained usage).
|
|
1868
|
-
let fallback: string | null = null;
|
|
1869
|
-
for (let step = 1; step <= stableAll.length; step++) {
|
|
1870
|
-
const candidate = stableAll[(startIdx + step) % stableAll.length]!;
|
|
1871
|
-
if (!eligible.includes(candidate)) continue;
|
|
1872
|
-
if (!fallback) fallback = candidate;
|
|
1873
|
-
if (hasCodexQuotaHeadroom(config, candidate, selectionOptions, now)) return candidate;
|
|
1874
|
-
}
|
|
1875
|
-
return fallback ?? ordered[0] ?? null;
|
|
1876
|
-
}
|
|
1877
|
-
|
|
1878
335
|
/**
|
|
1879
|
-
*
|
|
1880
|
-
* to the legacy quota path (or when the strategy is quota).
|
|
336
|
+
* Which account is ACTUALLY answering for one conversation key right now (#4546, wp8).
|
|
1881
337
|
*
|
|
1882
|
-
*
|
|
1883
|
-
*
|
|
1884
|
-
*
|
|
1885
|
-
*
|
|
338
|
+
* First placement reads this, not the binding alone: a parent parked on a transient detour
|
|
339
|
+
* is being served by the detour, so a new child placed "where the parent lives" would miss
|
|
340
|
+
* the warm account by one hop. A dead binding, an expired hold, and an ineligible serving
|
|
341
|
+
* account all answer null -- the caller then tries a sibling, then falls back to cold
|
|
342
|
+
* placement, which is the correct order because a stale home is worse than no hint.
|
|
1886
343
|
*
|
|
1887
|
-
*
|
|
1888
|
-
*
|
|
1889
|
-
*
|
|
1890
|
-
*
|
|
1891
|
-
* the
|
|
344
|
+
* "Right now" includes the MODEL lane. A parent whose home account is not entitled to this
|
|
345
|
+
* model is being served through a model-scoped detour, which is the same "serving, not stale
|
|
346
|
+
* home" case one level further in: reading only the ordinary binding would hand the child an
|
|
347
|
+
* account this request cannot use, and it would then start cold on the very model whose
|
|
348
|
+
* warm account the family already found. The detour scope embeds the model and the quota
|
|
349
|
+
* scope, so the entry consulted here is compatible by construction.
|
|
1892
350
|
*/
|
|
1893
|
-
function
|
|
1894
|
-
|
|
1895
|
-
threadId: string | null,
|
|
1896
|
-
now: number,
|
|
1897
|
-
commit: boolean,
|
|
1898
|
-
quotaScope?: CodexQuotaScope,
|
|
1899
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1900
|
-
commitSharedActive = commit,
|
|
1901
|
-
commitAffinity = commit,
|
|
1902
|
-
): string | null {
|
|
1903
|
-
const strategy = accountPoolStrategyForScope(config, quotaScope);
|
|
1904
|
-
if (strategy === "quota") return null;
|
|
1905
|
-
const poolKey = codexPoolKeyForScope(quotaScope);
|
|
1906
|
-
|
|
1907
|
-
let picked: string | null = null;
|
|
1908
|
-
if (strategy === "round-robin") {
|
|
1909
|
-
const eligible = listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions);
|
|
1910
|
-
const limit = stickyLimitForConfig(config);
|
|
1911
|
-
if (!commit) {
|
|
1912
|
-
return peekRoundRobinAccount(poolKey, eligible, limit);
|
|
1913
|
-
}
|
|
1914
|
-
picked = pickRoundRobinAccount(poolKey, eligible, limit);
|
|
1915
|
-
if (!picked) return null;
|
|
1916
|
-
if (commitSharedActive) {
|
|
1917
|
-
if (!isIndependentCodexQuotaScope(quotaScope)
|
|
1918
|
-
&& !manualPreferenceBlocks(codexPoolKeyForScope(quotaScope), picked)) {
|
|
1919
|
-
rememberActiveCodexAccount(config, picked);
|
|
1920
|
-
}
|
|
1921
|
-
}
|
|
1922
|
-
if (commitAffinity && threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
1923
|
-
notePoolRotationSuccess(poolKey, picked, limit);
|
|
1924
|
-
return picked;
|
|
1925
|
-
}
|
|
1926
|
-
|
|
1927
|
-
if (strategy === "fill-first" || strategy === "reset-first") {
|
|
1928
|
-
picked = strategy === "reset-first"
|
|
1929
|
-
? pickResetFirstCodexAccount(config, listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions), now, selectionOptions)
|
|
1930
|
-
: pickFillFirstCodexAccount(config, now, quotaScope, selectionOptions);
|
|
1931
|
-
if (!picked) return null;
|
|
1932
|
-
if (commitSharedActive) {
|
|
1933
|
-
if (!isIndependentCodexQuotaScope(quotaScope)
|
|
1934
|
-
&& !manualPreferenceBlocks(codexPoolKeyForScope(quotaScope), picked)) {
|
|
1935
|
-
rememberActiveCodexAccount(config, picked);
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
if (commitAffinity && threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
1939
|
-
return picked;
|
|
1940
|
-
}
|
|
1941
|
-
|
|
1942
|
-
return null;
|
|
1943
|
-
}
|
|
1944
|
-
|
|
1945
|
-
export function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined {
|
|
1946
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return getMainAccountPlan();
|
|
1947
|
-
return (config.codexAccounts ?? [])
|
|
1948
|
-
.find(account => isSelectableCodexPoolAccount(account) && account.id === accountId)?.plan;
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
/** Selection-only main routing must not lazily read the fenced native credential for its plan. */
|
|
1952
|
-
function getPoolAccountPlanForSelection(
|
|
1953
|
-
config: OcxConfig,
|
|
1954
|
-
accountId: string,
|
|
1955
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1956
|
-
): string | undefined {
|
|
1957
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID && selectionOptions?.nativeMainSelectionOnly === true) {
|
|
1958
|
-
return undefined;
|
|
1959
|
-
}
|
|
1960
|
-
return getPoolAccountPlan(config, accountId);
|
|
1961
|
-
}
|
|
1962
|
-
|
|
1963
|
-
/** Shared routing state must ignore a request-scoped entitlement roster. */
|
|
1964
|
-
function sharedStateSelectionOptions(
|
|
1965
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1966
|
-
): Pick<
|
|
1967
|
-
CodexAccountUsabilityOptions,
|
|
1968
|
-
"nativeMainSelectionOnly" | "isMainAccountTokenLive"
|
|
1969
|
-
> | undefined {
|
|
1970
|
-
if (!selectionOptions) return undefined;
|
|
1971
|
-
return {
|
|
1972
|
-
...(selectionOptions.nativeMainSelectionOnly !== undefined
|
|
1973
|
-
? { nativeMainSelectionOnly: selectionOptions.nativeMainSelectionOnly }
|
|
1974
|
-
: {}),
|
|
1975
|
-
...(selectionOptions.isMainAccountTokenLive
|
|
1976
|
-
? { isMainAccountTokenLive: selectionOptions.isMainAccountTokenLive }
|
|
1977
|
-
: {}),
|
|
1978
|
-
};
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
function pickLowerUsageAccount(
|
|
351
|
+
function lineageServingAccountId(
|
|
352
|
+
conversationKey: string,
|
|
1982
353
|
config: OcxConfig,
|
|
1983
|
-
active: string,
|
|
1984
|
-
activeUsage: number,
|
|
1985
354
|
now: number,
|
|
1986
355
|
quotaScope?: CodexQuotaScope,
|
|
1987
356
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1988
|
-
|
|
1989
|
-
): string {
|
|
1990
|
-
let best = active;
|
|
1991
|
-
let bestUsage = activeUsage;
|
|
1992
|
-
for (const id of getEligiblePoolAccounts(
|
|
1993
|
-
config,
|
|
1994
|
-
active,
|
|
1995
|
-
now,
|
|
1996
|
-
quotaScope,
|
|
1997
|
-
selectionOptions,
|
|
1998
|
-
skipFailoverReadyCandidates,
|
|
1999
|
-
)) {
|
|
2000
|
-
const usage = computeCodexUsageScore(
|
|
2001
|
-
getAccountQuota(id),
|
|
2002
|
-
getPoolAccountPlanForSelection(config, id, selectionOptions),
|
|
2003
|
-
now,
|
|
2004
|
-
);
|
|
2005
|
-
if (usage < bestUsage) {
|
|
2006
|
-
best = id;
|
|
2007
|
-
bestUsage = usage;
|
|
2008
|
-
}
|
|
2009
|
-
}
|
|
2010
|
-
return best;
|
|
2011
|
-
}
|
|
2012
|
-
|
|
2013
|
-
/** Coolest account in an already-selected candidate list; first index wins ties. */
|
|
2014
|
-
function pickLowestUsageAmong(
|
|
2015
|
-
config: OcxConfig,
|
|
2016
|
-
ids: readonly string[],
|
|
2017
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2018
|
-
now: number = Date.now(),
|
|
2019
|
-
): string | null {
|
|
2020
|
-
let best: string | null = null;
|
|
2021
|
-
let bestUsage = Number.POSITIVE_INFINITY;
|
|
2022
|
-
for (const id of ids) {
|
|
2023
|
-
const usage = computeCodexUsageScore(
|
|
2024
|
-
getAccountQuota(id),
|
|
2025
|
-
getPoolAccountPlanForSelection(config, id, selectionOptions),
|
|
2026
|
-
now,
|
|
2027
|
-
);
|
|
2028
|
-
if (usage < bestUsage) {
|
|
2029
|
-
best = id;
|
|
2030
|
-
bestUsage = usage;
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
return best;
|
|
2034
|
-
}
|
|
2035
|
-
|
|
2036
|
-
export function pickLowestUsageCodexAccount(
|
|
2037
|
-
config: OcxConfig,
|
|
2038
|
-
excludeId?: string,
|
|
2039
|
-
now = Date.now(),
|
|
2040
|
-
quotaScope?: CodexQuotaScope,
|
|
2041
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2042
|
-
): string | null {
|
|
2043
|
-
return pickLowestUsageAmong(
|
|
2044
|
-
config,
|
|
2045
|
-
getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions),
|
|
2046
|
-
selectionOptions,
|
|
2047
|
-
now,
|
|
2048
|
-
);
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
/**
|
|
2052
|
-
* Strategy-aware alternate after a cooled/excluded account (same-request 429 retry
|
|
2053
|
-
* and active promotion). Quota keeps lowest-usage; fill-first advances stable order;
|
|
2054
|
-
* round-robin takes the next ring pick (caller should have noted the failure).
|
|
2055
|
-
*/
|
|
2056
|
-
export function pickAlternateCodexAccount(
|
|
2057
|
-
config: OcxConfig,
|
|
2058
|
-
excludeId: string,
|
|
2059
|
-
now = Date.now(),
|
|
2060
|
-
quotaScope?: CodexQuotaScope,
|
|
2061
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
357
|
+
modelId?: string,
|
|
2062
358
|
): string | null {
|
|
2063
|
-
const
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
|
|
2070
|
-
return pickRoundRobinAccount(codexPoolKeyForScope(quotaScope), eligible, stickyLimitForConfig(config));
|
|
2071
|
-
}
|
|
2072
|
-
if (strategy === "fill-first") {
|
|
2073
|
-
const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
|
|
2074
|
-
return pickNextFillFirstCodexAccount(config, excludeId, eligible, now, selectionOptions);
|
|
2075
|
-
}
|
|
2076
|
-
if (strategy === "reset-first") {
|
|
2077
|
-
return pickResetFirstCodexAccount(config, getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions), now, selectionOptions);
|
|
359
|
+
const entry = (modelId !== undefined
|
|
360
|
+
? getModelDetourAffinity(conversationKey, modelId, quotaScope)
|
|
361
|
+
: undefined)
|
|
362
|
+
?? getThreadAffinity(conversationKey, quotaScope);
|
|
363
|
+
if (!entry || isThreadAffinityExpired(entry, now) || !isThreadAffinityGenerationLive(entry)) {
|
|
364
|
+
return null;
|
|
2078
365
|
}
|
|
2079
|
-
|
|
366
|
+
const holdLive = entry.transientHoldSince !== undefined && !isTransientHoldExpired(entry, now);
|
|
367
|
+
const serving = holdLive && entry.transientDetourAccountId !== undefined
|
|
368
|
+
? entry.transientDetourAccountId
|
|
369
|
+
: entry.accountId;
|
|
370
|
+
return isCodexAccountSelectable(config, serving, now, quotaScope, selectionOptions)
|
|
371
|
+
&& !hasUnrecoveredCodexQuotaRefusal(serving, quotaScope)
|
|
372
|
+
&& !shouldFailover(config, serving, now)
|
|
373
|
+
&& !isCodexAccountSoftAvoided(serving, now)
|
|
374
|
+
? serving
|
|
375
|
+
: null;
|
|
2080
376
|
}
|
|
2081
377
|
|
|
2082
378
|
/**
|
|
2083
|
-
*
|
|
2084
|
-
*
|
|
2085
|
-
*
|
|
2086
|
-
*
|
|
2087
|
-
* the selection rule that could drift from it.
|
|
2088
|
-
*
|
|
2089
|
-
* This exists because preview and resolve have to agree on the FIRST transient detour, not just
|
|
2090
|
-
* on later ones. Preview feeds subagent model-availability scoring, so a preview that reported
|
|
2091
|
-
* the bound account while resolve was about to serve from a cool sibling could retire a model
|
|
2092
|
-
* over usage the request would never have touched.
|
|
379
|
+
* First placement only: where a child with NO binding of its own should start. Parent's
|
|
380
|
+
* current serving account first, then a compatible sibling's -- "compatible" meaning the
|
|
381
|
+
* same quota-scope slot, since a Reserve sibling says nothing about the shared lane. The
|
|
382
|
+
* child still binds under its own key; this is a hint for turn one, not a root-wide pin.
|
|
2093
383
|
*/
|
|
2094
|
-
function
|
|
384
|
+
function pickLineageServingAccount(
|
|
2095
385
|
config: OcxConfig,
|
|
2096
|
-
|
|
386
|
+
lineage: CodexThreadLineage,
|
|
2097
387
|
now: number,
|
|
2098
388
|
quotaScope?: CodexQuotaScope,
|
|
2099
389
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
}
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
* cannot be detected by comparing values: a pool-driven promote legitimately moves the
|
|
2114
|
-
* persisted active account, and reading that as staleness would silently spend the
|
|
2115
|
-
* operator's one-shot. Invalidation is keyed to the OPERATOR path instead — another manual
|
|
2116
|
-
* selection, the account leaving the pool, or a successful dispatch on it.
|
|
2117
|
-
*/
|
|
2118
|
-
const manualPreference = new Map<string, string>();
|
|
2119
|
-
|
|
2120
|
-
/**
|
|
2121
|
-
* Spend the one-shot for a pool scope once a dispatch on that account actually succeeded.
|
|
2122
|
-
* This is the Codex analogue of `commitAnthropicSelectionRouting`, which Codex lacks.
|
|
2123
|
-
*
|
|
2124
|
-
* Wiring this BEFORE the guard below is not a style choice. Measured: with the guard in
|
|
2125
|
-
* place and no consume site, the first manual selection freezes the automatic cursor
|
|
2126
|
-
* permanently and 15 of 69 rotation tests fail.
|
|
2127
|
-
*/
|
|
2128
|
-
function consumeManualPreference(accountId: string, poolKey: string): void {
|
|
2129
|
-
if (manualPreference.get(poolKey) === accountId) manualPreference.delete(poolKey);
|
|
2130
|
-
}
|
|
2131
|
-
|
|
2132
|
-
/**
|
|
2133
|
-
* Drop an account's preference in every scope. Pause and exclusion do not route through
|
|
2134
|
-
* `resetCodexRoutingForManualSelection`, so without this a preference could outlive the
|
|
2135
|
-
* account it names and keep suppressing the automatic cursor.
|
|
2136
|
-
*/
|
|
2137
|
-
function forgetManualPreference(accountId: string): void {
|
|
2138
|
-
for (const [poolKey, preferred] of manualPreference) {
|
|
2139
|
-
if (preferred === accountId) manualPreference.delete(poolKey);
|
|
2140
|
-
}
|
|
2141
|
-
}
|
|
2142
|
-
|
|
2143
|
-
/**
|
|
2144
|
-
* True while an unspent operator selection for this scope names a DIFFERENT account than
|
|
2145
|
-
* the automatic pick about to be recorded.
|
|
2146
|
-
*
|
|
2147
|
-
* Callers pass their own scope: an independent quota scope keeps its own entry and must
|
|
2148
|
-
* never read the shared one. The failover promote does NOT consult this — see its call
|
|
2149
|
-
* site for why.
|
|
2150
|
-
*/
|
|
2151
|
-
function manualPreferenceBlocks(poolKey: string, accountId: string): boolean {
|
|
2152
|
-
const preferred = manualPreference.get(poolKey);
|
|
2153
|
-
return preferred !== undefined && preferred !== accountId;
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
export function getEffectiveActiveCodexAccountId(config: OcxConfig): string | undefined {
|
|
2157
|
-
return runtimeActiveCodexAccountId ?? config.activeCodexAccountId;
|
|
2158
|
-
}
|
|
2159
|
-
|
|
2160
|
-
/**
|
|
2161
|
-
* Whether the account routing is currently on is there because an operator asked
|
|
2162
|
-
* for it, rather than because a strategy landed on it. Surfaces read this instead
|
|
2163
|
-
* of comparing the stored pin themselves, which would report a pin that a later
|
|
2164
|
-
* automatic pick has already moved past.
|
|
2165
|
-
*/
|
|
2166
|
-
export function isEffectiveCodexAccountPinned(config: OcxConfig): boolean {
|
|
2167
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2168
|
-
return pinned !== undefined && pinned === getEffectiveActiveCodexAccountId(config);
|
|
2169
|
-
}
|
|
2170
|
-
|
|
2171
|
-
/**
|
|
2172
|
-
* Automatic strategy / failover cursor only — never mutates `config.activeCodexAccountId`
|
|
2173
|
-
* so an unrelated `saveConfig` cannot persist transient rotation as operator selection.
|
|
2174
|
-
*/
|
|
2175
|
-
function rememberActiveCodexAccount(_config: OcxConfig, accountId: string): void {
|
|
2176
|
-
runtimeActiveCodexAccountId = accountId;
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
/**
|
|
2180
|
-
* End the manual pin when routing moves to a different account. Returns whether
|
|
2181
|
-
* the pin changed so the caller can fold it into a write it was already making.
|
|
2182
|
-
*/
|
|
2183
|
-
function releaseCodexAccountPinFor(config: OcxConfig, accountId: string): boolean {
|
|
2184
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2185
|
-
if (pinned === undefined || pinned === accountId) return false;
|
|
2186
|
-
clearCodexAccountPin(config);
|
|
2187
|
-
return true;
|
|
2188
|
-
}
|
|
2189
|
-
|
|
2190
|
-
/** Persist operator (or quota-strategy) active selection to config + disk. */
|
|
2191
|
-
function setActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
2192
|
-
runtimeActiveCodexAccountId = undefined;
|
|
2193
|
-
const releasedPin = releaseCodexAccountPinFor(config, accountId);
|
|
2194
|
-
if (config.activeCodexAccountId === accountId && !releasedPin) return;
|
|
2195
|
-
config.activeCodexAccountId = accountId;
|
|
2196
|
-
saveConfigPreservingClaudeCode(config);
|
|
2197
|
-
}
|
|
2198
|
-
|
|
2199
|
-
/** Quota strategy persists; RR/fill-first keep a process-local cursor only. */
|
|
2200
|
-
function promoteActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
2201
|
-
if (normalizeCodexAccountPoolStrategy(config.accountPoolStrategy) === "quota") {
|
|
2202
|
-
setActiveCodexAccount(config, accountId);
|
|
2203
|
-
return;
|
|
390
|
+
modelId?: string,
|
|
391
|
+
): { accountId: string; reason: CodexAffinityReason } | null {
|
|
392
|
+
if (lineage.parentConversationKey !== undefined) {
|
|
393
|
+
const parent = lineageServingAccountId(
|
|
394
|
+
lineage.parentConversationKey, config, now, quotaScope, selectionOptions, modelId,
|
|
395
|
+
);
|
|
396
|
+
if (parent) return { accountId: parent, reason: "lineage_parent" };
|
|
397
|
+
for (const siblingKey of lineage.siblingConversationKeys) {
|
|
398
|
+
const sibling = lineageServingAccountId(
|
|
399
|
+
siblingKey, config, now, quotaScope, selectionOptions, modelId,
|
|
400
|
+
);
|
|
401
|
+
if (sibling) return { accountId: sibling, reason: "lineage_sibling" };
|
|
402
|
+
}
|
|
2204
403
|
}
|
|
2205
|
-
|
|
2206
|
-
// saves this release with its own write; a transient failover does not, so the
|
|
2207
|
-
// pin survives a restart that also clears the failure history behind it.
|
|
2208
|
-
releaseCodexAccountPinFor(config, accountId);
|
|
2209
|
-
rememberActiveCodexAccount(config, accountId);
|
|
404
|
+
return null;
|
|
2210
405
|
}
|
|
2211
406
|
|
|
2212
407
|
/**
|
|
@@ -2234,54 +429,12 @@ export function reconcileCodexActiveAfterExclusion(
|
|
|
2234
429
|
clearCodexAccountPin(config, excludedAccountId);
|
|
2235
430
|
if (!wasEffective) return getEffectiveActiveCodexAccountId(config) ?? null;
|
|
2236
431
|
|
|
2237
|
-
|
|
432
|
+
forgetRuntimeActiveCodexAccount();
|
|
2238
433
|
const fallback = pickAlternateCodexAccount(config, excludedAccountId, now);
|
|
2239
434
|
if (fallback) promoteActiveCodexAccount(config, fallback);
|
|
2240
435
|
return fallback;
|
|
2241
436
|
}
|
|
2242
437
|
|
|
2243
|
-
function isUnknownUsage(usage: number): boolean {
|
|
2244
|
-
return usage >= CODEX_UNKNOWN_USAGE_SCORE;
|
|
2245
|
-
}
|
|
2246
|
-
|
|
2247
|
-
/**
|
|
2248
|
-
* Move an unbound request back up when a higher tier regains headroom — the
|
|
2249
|
-
* weekly-reset case. Returns null when nothing should change.
|
|
2250
|
-
*
|
|
2251
|
-
* Downward moves are deliberately left to {@link applyQuotaAutoSwitch}: this only
|
|
2252
|
-
* fires when the tier filter has already excluded `active`, and only toward a
|
|
2253
|
-
* tier that strictly outranks it. Threads bound by affinity never reach here.
|
|
2254
|
-
*/
|
|
2255
|
-
function pickPriorityPreemption(
|
|
2256
|
-
config: OcxConfig,
|
|
2257
|
-
active: string,
|
|
2258
|
-
now: number,
|
|
2259
|
-
quotaScope?: CodexQuotaScope,
|
|
2260
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2261
|
-
): string | null {
|
|
2262
|
-
const eligible = getEligiblePoolAccounts(config, undefined, now, quotaScope, selectionOptions);
|
|
2263
|
-
if (eligible.length === 0 || eligible.includes(active)) return null;
|
|
2264
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2265
|
-
// A live pin already lowered the tier ceiling; never preempt past an explicit
|
|
2266
|
-
// operator choice. Same liveness test the tier filter applies, so preview and
|
|
2267
|
-
// resolve agree even before the pin is garbage-collected.
|
|
2268
|
-
if (
|
|
2269
|
-
pinned !== undefined
|
|
2270
|
-
&& eligible.includes(pinned)
|
|
2271
|
-
&& hasCodexQuotaHeadroom(config, pinned, selectionOptions, now)
|
|
2272
|
-
) return null;
|
|
2273
|
-
const priorityOf = codexAccountPriorityLookup(config);
|
|
2274
|
-
if (priorityOf(eligible[0]!) <= priorityOf(active)) return null;
|
|
2275
|
-
// Members without headroom are in the tier only because a sibling has some;
|
|
2276
|
-
// picking one would hand the request straight back to a drained account.
|
|
2277
|
-
return pickLowestUsageAmong(
|
|
2278
|
-
config,
|
|
2279
|
-
eligible.filter(id => hasCodexQuotaHeadroom(config, id, selectionOptions, now)),
|
|
2280
|
-
selectionOptions,
|
|
2281
|
-
now,
|
|
2282
|
-
);
|
|
2283
|
-
}
|
|
2284
|
-
|
|
2285
438
|
/**
|
|
2286
439
|
* Release a pin whose account is durably drained. "Use this account now" ends
|
|
2287
440
|
* when the account crosses the auto-switch threshold or stops being selectable
|
|
@@ -2316,114 +469,14 @@ function releaseDrainedCodexAccountPin(
|
|
|
2316
469
|
saveConfigPreservingClaudeCode(config);
|
|
2317
470
|
}
|
|
2318
471
|
|
|
2319
|
-
function applyQuotaAutoSwitch(
|
|
2320
|
-
config: OcxConfig,
|
|
2321
|
-
active: string,
|
|
2322
|
-
now: number,
|
|
2323
|
-
quotaScope?: CodexQuotaScope,
|
|
2324
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2325
|
-
commitSharedSelection = true,
|
|
2326
|
-
): string {
|
|
2327
|
-
const threshold = config.autoSwitchThreshold ?? 80;
|
|
2328
|
-
if (threshold <= 0) return active;
|
|
2329
|
-
const quota = getAccountQuota(active);
|
|
2330
|
-
const activeUsage = computeCodexUsageScore(
|
|
2331
|
-
quota,
|
|
2332
|
-
getPoolAccountPlanForSelection(config, active, selectionOptions),
|
|
2333
|
-
now,
|
|
2334
|
-
);
|
|
2335
|
-
// Unknown usage is not evidence that a user's explicit selection crossed the
|
|
2336
|
-
// threshold. Wait for quota priming instead of rotating among guesses.
|
|
2337
|
-
if (isUnknownUsage(activeUsage)) return active;
|
|
2338
|
-
if (activeUsage < threshold) return active;
|
|
2339
|
-
const best = pickLowerUsageAccount(config, active, activeUsage, now, quotaScope, selectionOptions);
|
|
2340
|
-
if (best !== active) {
|
|
2341
|
-
if (commitSharedSelection && !isIndependentCodexQuotaScope(quotaScope)) {
|
|
2342
|
-
setActiveCodexAccount(config, best);
|
|
2343
|
-
}
|
|
2344
|
-
return best;
|
|
2345
|
-
}
|
|
2346
|
-
|
|
2347
|
-
return active;
|
|
2348
|
-
}
|
|
2349
|
-
|
|
2350
|
-
function shouldFailover(config: OcxConfig, accountId: string, now: number): boolean {
|
|
2351
|
-
const threshold = config.upstreamFailoverThreshold ?? 3;
|
|
2352
|
-
if (threshold <= 0) return false;
|
|
2353
|
-
dropSpentCredentialFailure(accountId);
|
|
2354
|
-
const health = upstreamHealth.get(accountId);
|
|
2355
|
-
if (health?.lastFailureAt && now - health.lastFailureAt > CODEX_FAILURE_WINDOW_MS) return false;
|
|
2356
|
-
return !!health && health.consecutiveFailures >= threshold;
|
|
2357
|
-
}
|
|
2358
|
-
|
|
2359
|
-
function isHealthySharedCodexSelection(
|
|
2360
|
-
config: OcxConfig,
|
|
2361
|
-
accountId: string,
|
|
2362
|
-
now: number,
|
|
2363
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
2364
|
-
selectionOptions: CodexAccountUsabilityOptions | undefined,
|
|
2365
|
-
): boolean {
|
|
2366
|
-
return isCodexAccountSelectable(config, accountId, now, quotaScope, selectionOptions)
|
|
2367
|
-
&& hasCodexQuotaHeadroom(config, accountId, selectionOptions, now)
|
|
2368
|
-
&& !shouldFailover(config, accountId, now);
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
function strategySelectionOptionsForModelDetour(
|
|
2372
|
-
config: OcxConfig,
|
|
2373
|
-
now: number,
|
|
2374
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
2375
|
-
selectionOptions: CodexAccountUsabilityOptions | undefined,
|
|
2376
|
-
): CodexAccountUsabilityOptions | undefined {
|
|
2377
|
-
if (selectionOptions?.modelEligibleAccountIds === undefined) return selectionOptions;
|
|
2378
|
-
const sharedSelectionOptions = sharedStateSelectionOptions(selectionOptions) ?? {};
|
|
2379
|
-
return {
|
|
2380
|
-
...selectionOptions,
|
|
2381
|
-
modelEligibleAccountIds: new Set(
|
|
2382
|
-
[...selectionOptions.modelEligibleAccountIds].filter(accountId =>
|
|
2383
|
-
isHealthySharedCodexSelection(
|
|
2384
|
-
config,
|
|
2385
|
-
accountId,
|
|
2386
|
-
now,
|
|
2387
|
-
quotaScope,
|
|
2388
|
-
sharedSelectionOptions,
|
|
2389
|
-
)
|
|
2390
|
-
),
|
|
2391
|
-
),
|
|
2392
|
-
};
|
|
2393
|
-
}
|
|
2394
|
-
|
|
2395
|
-
function applyFailureFailover(
|
|
2396
|
-
config: OcxConfig,
|
|
2397
|
-
active: string,
|
|
2398
|
-
now: number,
|
|
2399
|
-
quotaScope?: CodexQuotaScope,
|
|
2400
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2401
|
-
commitSharedSelection = true,
|
|
2402
|
-
): string {
|
|
2403
|
-
if (!shouldFailover(config, active, now)) return active;
|
|
2404
|
-
const best = pickAlternateCodexAccount(config, active, now, quotaScope, selectionOptions);
|
|
2405
|
-
if (best) {
|
|
2406
|
-
// The scope still routes away from the failing account — that is this request's
|
|
2407
|
-
// own decision — but an independent one must not persist a new shared active
|
|
2408
|
-
// account. recordCodexUpstreamOutcome only suppresses the promotion it makes at
|
|
2409
|
-
// the moment of the failure; the streak outlives the soft avoid, so a later
|
|
2410
|
-
// scoped resolve reaches here with the streak still tripped and would otherwise
|
|
2411
|
-
// move the shared cursor after all.
|
|
2412
|
-
if (commitSharedSelection && !isIndependentCodexQuotaScope(quotaScope)) {
|
|
2413
|
-
promoteActiveCodexAccount(config, best);
|
|
2414
|
-
}
|
|
2415
|
-
return best;
|
|
2416
|
-
}
|
|
2417
|
-
return active;
|
|
2418
|
-
}
|
|
2419
|
-
|
|
2420
472
|
export function resolveCodexAccountForThread(
|
|
2421
473
|
threadId: string | null,
|
|
2422
474
|
config: OcxConfig,
|
|
2423
475
|
now = Date.now(),
|
|
2424
476
|
quotaScope?: CodexQuotaScope,
|
|
477
|
+
lineage?: CodexThreadLineage,
|
|
2425
478
|
): string | null {
|
|
2426
|
-
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now, quotaScope);
|
|
479
|
+
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now, quotaScope, undefined, undefined, lineage);
|
|
2427
480
|
return resolution.status === "selected" ? resolution.accountId : null;
|
|
2428
481
|
}
|
|
2429
482
|
|
|
@@ -2462,7 +515,7 @@ function carriesQuotaRefusal(health: CodexUpstreamHealth | undefined): boolean {
|
|
|
2462
515
|
* quota group, so a spent Spark window still cannot displace the same thread's Terra binding.
|
|
2463
516
|
*/
|
|
2464
517
|
function hasUnrecoveredCodexQuotaRefusal(accountId: string, quotaScope?: CodexQuotaScope): boolean {
|
|
2465
|
-
if (carriesQuotaRefusal(
|
|
518
|
+
if (carriesQuotaRefusal(getAccountHealth(accountId))) return true;
|
|
2466
519
|
return quotaScope !== undefined && carriesQuotaRefusal(scopedHealthFor(accountId, quotaScope));
|
|
2467
520
|
}
|
|
2468
521
|
|
|
@@ -2699,6 +752,7 @@ export function previewCodexAccountForRequest(
|
|
|
2699
752
|
quotaScope?: CodexQuotaScope,
|
|
2700
753
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2701
754
|
modelId?: string,
|
|
755
|
+
lineage?: CodexThreadLineage,
|
|
2702
756
|
): string | null {
|
|
2703
757
|
// A request-scoped model detour keeps its own serving-account affinity. Preview
|
|
2704
758
|
// reads it before the ordinary lane, but never repairs or deletes it. Roster
|
|
@@ -2724,6 +778,30 @@ export function previewCodexAccountForRequest(
|
|
|
2724
778
|
);
|
|
2725
779
|
if (ordinaryPreview) return ordinaryPreview;
|
|
2726
780
|
|
|
781
|
+
// A conversation carried across an in-process swap is still bound under the pre-#4546 raw
|
|
782
|
+
// parent key, and resolve adopts that binding rather than rebinding cold. Preview has to name
|
|
783
|
+
// the same account. Read-only, as everything here is: it neither adopts nor retires the entry.
|
|
784
|
+
if (threadId && !entry && lineage?.legacyConversationKey !== undefined) {
|
|
785
|
+
const legacyPreview = previewReusableAffinityAccount(
|
|
786
|
+
getThreadAffinity(lineage.legacyConversationKey, quotaScope),
|
|
787
|
+
config,
|
|
788
|
+
now,
|
|
789
|
+
quotaScope,
|
|
790
|
+
selectionOptions,
|
|
791
|
+
);
|
|
792
|
+
if (legacyPreview) return legacyPreview;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
// First placement mirrors resolve: a child with no binding previews the account actually
|
|
796
|
+
// serving its parent (or a compatible sibling), so the subagent fallback does not decide
|
|
797
|
+
// against a cold pick the real request would never make. Read-only: nothing binds here.
|
|
798
|
+
if (threadId && !entry && lineage) {
|
|
799
|
+
const lineagePreview = pickLineageServingAccount(
|
|
800
|
+
config, lineage, now, quotaScope, selectionOptions, modelId,
|
|
801
|
+
);
|
|
802
|
+
if (lineagePreview) return lineagePreview.accountId;
|
|
803
|
+
}
|
|
804
|
+
|
|
2727
805
|
const strategyPick = pickUnboundStrategyAccount(
|
|
2728
806
|
config,
|
|
2729
807
|
threadId,
|
|
@@ -2782,6 +860,7 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2782
860
|
quotaScope?: CodexQuotaScope,
|
|
2783
861
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2784
862
|
modelId?: string,
|
|
863
|
+
lineage?: CodexThreadLineage,
|
|
2785
864
|
): CodexThreadResolution {
|
|
2786
865
|
// An entitlement roster constrains only this model request. It must not rewrite
|
|
2787
866
|
// the operator's shared active/pin choice or the task's ordinary-model affinity.
|
|
@@ -2809,6 +888,13 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2809
888
|
)
|
|
2810
889
|
);
|
|
2811
890
|
|
|
891
|
+
// A conversation that was live across an in-process code swap is still bound under the
|
|
892
|
+
// pre-#4546 raw parent key. Adopt that binding onto this thread's key BEFORE anything below
|
|
893
|
+
// reads an entry, so the conversation arrives here as an ordinary bound thread instead of a
|
|
894
|
+
// cold one: every branch that follows -- detour reuse, transient hold, quota re-eval --
|
|
895
|
+
// should treat it as the continuing conversation it is. No-op on a fresh process.
|
|
896
|
+
if (threadId) adoptLegacyLineageAffinity(threadId, lineage, now, quotaScope, modelId);
|
|
897
|
+
|
|
2812
898
|
if (threadId && modelScopedSelection) {
|
|
2813
899
|
const detourEntry = getModelDetourAffinity(threadId, modelId, quotaScope);
|
|
2814
900
|
if (detourEntry) {
|
|
@@ -2989,6 +1075,38 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2989
1075
|
// arrives) is the reason this request is starting cold, so it outranks having found nothing.
|
|
2990
1076
|
releaseReason ??= peekPendingReleaseReason(threadId);
|
|
2991
1077
|
|
|
1078
|
+
// FIRST PLACEMENT for a child thread (#4546, wp8). A child with no binding of its own used
|
|
1079
|
+
// to bind under the raw parent id -- an entry unrelated to the root's real binding -- or
|
|
1080
|
+
// land cold while its parent was being served warm somewhere. Consult the family's CURRENT
|
|
1081
|
+
// serving account first (detour included), then a compatible sibling's, and only then fall
|
|
1082
|
+
// through to cold placement. The child binds under its OWN key below: this is a warm start,
|
|
1083
|
+
// not a root-wide pin, so a later move of the parent never drags the child with it.
|
|
1084
|
+
//
|
|
1085
|
+
// Guarded on `entry === undefined`, which is strictly narrower than "has no usable binding":
|
|
1086
|
+
// a thread whose binding was just released above still holds its own history and re-decides
|
|
1087
|
+
// through the ordinary path. Only a thread that has never bound takes a family hint. That
|
|
1088
|
+
// also makes `preserveExistingModelScopedAffinity` unreachable here -- it is only ever set
|
|
1089
|
+
// while reusing an existing model-detour entry -- so this binds through the ordinary lane.
|
|
1090
|
+
if (threadId && entry === undefined && lineage) {
|
|
1091
|
+
const lineagePick = pickLineageServingAccount(
|
|
1092
|
+
config, lineage, now, quotaScope, selectionOptions, modelId,
|
|
1093
|
+
);
|
|
1094
|
+
if (lineagePick) {
|
|
1095
|
+
bindThreadAffinity(threadId, lineagePick.accountId, now, quotaScope);
|
|
1096
|
+
// Deliberately no promoteActiveCodexAccount: a family hint places THIS request, it does
|
|
1097
|
+
// not move the operator-visible shared cursor for unrelated new threads.
|
|
1098
|
+
return {
|
|
1099
|
+
status: "selected",
|
|
1100
|
+
accountId: lineagePick.accountId,
|
|
1101
|
+
// A pending release still outranks the hint as the reported reason, and consuming it
|
|
1102
|
+
// here is what stops the next request reporting the same release a second time.
|
|
1103
|
+
affinity: releaseReason === undefined
|
|
1104
|
+
? { move: "new_bind", reason: lineagePick.reason }
|
|
1105
|
+
: affinityAfterRelease(threadId, releaseReason),
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
2992
1110
|
// A request-scoped roster may still contain unhealthy candidates. Non-quota strategies return
|
|
2993
1111
|
// before the quota/failover helpers below, so prefer only shared-healthy roster members here;
|
|
2994
1112
|
// otherwise RR/fill-first can immediately re-pick a known failing account even when another
|
|
@@ -3144,6 +1262,7 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
3144
1262
|
return { status: "selected", accountId: active, affinity: affinityAfterRelease(threadId, releaseReason) };
|
|
3145
1263
|
}
|
|
3146
1264
|
|
|
1265
|
+
|
|
3147
1266
|
export function recordCodexUpstreamOutcome(
|
|
3148
1267
|
config: OcxConfig,
|
|
3149
1268
|
accountId: string | null,
|
|
@@ -3159,7 +1278,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3159
1278
|
}
|
|
3160
1279
|
if (!accountId) return;
|
|
3161
1280
|
const writerGeneration = meta.writerGeneration ?? captureConfigGeneration();
|
|
3162
|
-
if (
|
|
1281
|
+
if (!isHealthAccountAdmissible(accountId, writerGeneration)) return;
|
|
3163
1282
|
const now = meta.now ?? Date.now();
|
|
3164
1283
|
const outcomeClass = classifyCodexUpstreamOutcome(outcome, meta.denial);
|
|
3165
1284
|
// Reject retired quota evidence before stale-credential cleanup or any shared mutation.
|
|
@@ -3204,12 +1323,12 @@ export function recordCodexUpstreamOutcome(
|
|
|
3204
1323
|
if (Object.keys(retained).length > 1) setScopedHealth(accountId, quotaScope, retained);
|
|
3205
1324
|
else deleteScopedHealth(accountId, quotaScope);
|
|
3206
1325
|
}
|
|
3207
|
-
const current =
|
|
1326
|
+
const current = getAccountHealth(accountId);
|
|
3208
1327
|
const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
|
|
3209
1328
|
// A leased probe that is still on its own cooldown generation proves the
|
|
3210
1329
|
// account recovered: clear the hard cooldown outright (#433).
|
|
3211
1330
|
if (cooldownUntil && probeMayClearCooldown(current, meta)) {
|
|
3212
|
-
|
|
1331
|
+
deleteAccountHealth(accountId);
|
|
3213
1332
|
return;
|
|
3214
1333
|
}
|
|
3215
1334
|
// Owning probe on a stale generation: the lease is done, but a newer 429
|
|
@@ -3221,7 +1340,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3221
1340
|
if (failoverEnabled && current && current.consecutiveFailures >= 2) {
|
|
3222
1341
|
const consecutiveSuccesses = (current.consecutiveSuccesses ?? 0) + 1;
|
|
3223
1342
|
if (consecutiveSuccesses < 2) {
|
|
3224
|
-
|
|
1343
|
+
setAccountHealth(accountId, {
|
|
3225
1344
|
...base!,
|
|
3226
1345
|
...preserved,
|
|
3227
1346
|
consecutiveSuccesses,
|
|
@@ -3231,14 +1350,14 @@ export function recordCodexUpstreamOutcome(
|
|
|
3231
1350
|
}
|
|
3232
1351
|
// Level 1 clears immediately; escalated accounts need two consecutive healthy terminals.
|
|
3233
1352
|
// Hard quota cooldown intentionally survives either recovery path.
|
|
3234
|
-
if (cooldownUntil)
|
|
3235
|
-
else
|
|
1353
|
+
if (cooldownUntil) setAccountHealth(accountId, { consecutiveFailures: 0, ...preserved });
|
|
1354
|
+
else deleteAccountHealth(accountId);
|
|
3236
1355
|
return;
|
|
3237
1356
|
}
|
|
3238
1357
|
if (outcomeClass === "caller") {
|
|
3239
1358
|
// A 4xx does not change account health, but it does conclude an in-flight
|
|
3240
1359
|
// probe — otherwise the lease would never be handed back.
|
|
3241
|
-
const current =
|
|
1360
|
+
const current = getAccountHealth(accountId);
|
|
3242
1361
|
const scopedProbe = meta.probeQuotaScope
|
|
3243
1362
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3244
1363
|
: undefined;
|
|
@@ -3246,7 +1365,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3246
1365
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3247
1366
|
}
|
|
3248
1367
|
if (ownsProbeLease(current, meta)) {
|
|
3249
|
-
|
|
1368
|
+
setAccountHealth(accountId, withProbeLeaseReleased(current!, now));
|
|
3250
1369
|
}
|
|
3251
1370
|
return;
|
|
3252
1371
|
}
|
|
@@ -3257,7 +1376,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3257
1376
|
// it and must not happen (#914). Conclude any owned probe lease, record the
|
|
3258
1377
|
// failure under the (provider, host) ledger when one is named, and leave
|
|
3259
1378
|
// account health, thread affinity, and the active account untouched.
|
|
3260
|
-
const current =
|
|
1379
|
+
const current = getAccountHealth(accountId);
|
|
3261
1380
|
const scopedProbe = meta.probeQuotaScope
|
|
3262
1381
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3263
1382
|
: undefined;
|
|
@@ -3265,7 +1384,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3265
1384
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3266
1385
|
}
|
|
3267
1386
|
if (ownsProbeLease(current, meta)) {
|
|
3268
|
-
|
|
1387
|
+
setAccountHealth(accountId, withProbeLeaseReleased(current!, now));
|
|
3269
1388
|
}
|
|
3270
1389
|
return;
|
|
3271
1390
|
}
|
|
@@ -3276,8 +1395,8 @@ export function recordCodexUpstreamOutcome(
|
|
|
3276
1395
|
// Record the failure so routing stops preferring it, but do not mark it for
|
|
3277
1396
|
// reauthentication and do not sweep its thread affinities: telling the user to
|
|
3278
1397
|
// re-login is wrong advice that cannot fix a workspace grant.
|
|
3279
|
-
|
|
3280
|
-
consecutiveFailures: (
|
|
1398
|
+
setAccountHealth(accountId, {
|
|
1399
|
+
consecutiveFailures: (getAccountHealth(accountId)?.consecutiveFailures ?? 0) + 1,
|
|
3281
1400
|
lastFailureStatus,
|
|
3282
1401
|
lastFailureAt: now,
|
|
3283
1402
|
});
|
|
@@ -3315,7 +1434,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3315
1434
|
* Affinity sweeping needs no tag: an affinity entry already carries a credential generation and
|
|
3316
1435
|
* self-invalidates on the next check, and re-adding swept entries would be a worse bug.
|
|
3317
1436
|
*/
|
|
3318
|
-
|
|
1437
|
+
setAccountHealth(accountId, {
|
|
3319
1438
|
consecutiveFailures: 1,
|
|
3320
1439
|
lastFailureStatus,
|
|
3321
1440
|
lastFailureAt: now,
|
|
@@ -3324,7 +1443,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3324
1443
|
? { credentialFailureGeneration: meta.credentialGeneration }
|
|
3325
1444
|
: {}),
|
|
3326
1445
|
});
|
|
3327
|
-
|
|
1446
|
+
deleteAllScopedHealth(accountId);
|
|
3328
1447
|
// The reauth flag carries the same provenance, so a replacement landing after this call cannot
|
|
3329
1448
|
// inherit a quarantine that was never about it.
|
|
3330
1449
|
markAccountNeedsReauth(accountId, writerGeneration, meta.credentialGeneration);
|
|
@@ -3385,13 +1504,13 @@ export function recordCodexUpstreamOutcome(
|
|
|
3385
1504
|
if (scopedProbe && meta.probeQuotaScope && ownsProbeLease(scopedProbe, meta)) {
|
|
3386
1505
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3387
1506
|
}
|
|
3388
|
-
const prior =
|
|
1507
|
+
const prior = getAccountHealth(accountId);
|
|
3389
1508
|
// Every cooldown write bumps the generation so a probe issued against the
|
|
3390
1509
|
// previous cooldown can no longer clear this one (#433).
|
|
3391
1510
|
const cooldownGeneration = (prior?.cooldownGeneration ?? 0) + 1;
|
|
3392
1511
|
// A failed probe concludes its lease; an unrelated 429 leaves the live probe alone.
|
|
3393
1512
|
const ownsLease = ownsProbeLease(prior, meta);
|
|
3394
|
-
|
|
1513
|
+
setAccountHealth(accountId, {
|
|
3395
1514
|
consecutiveFailures: 0,
|
|
3396
1515
|
lastFailureStatus,
|
|
3397
1516
|
lastFailureAt: now,
|
|
@@ -3431,7 +1550,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3431
1550
|
}
|
|
3432
1551
|
|
|
3433
1552
|
// transient (connect_error / timeout / 5xx)
|
|
3434
|
-
const current =
|
|
1553
|
+
const current = getAccountHealth(accountId);
|
|
3435
1554
|
const scopedProbe = meta.probeQuotaScope
|
|
3436
1555
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3437
1556
|
: undefined;
|
|
@@ -3457,7 +1576,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3457
1576
|
now + escalationMs,
|
|
3458
1577
|
)
|
|
3459
1578
|
: undefined;
|
|
3460
|
-
|
|
1579
|
+
setAccountHealth(accountId, {
|
|
3461
1580
|
...preservedCooldownFields(transientBase),
|
|
3462
1581
|
consecutiveFailures,
|
|
3463
1582
|
lastFailureStatus,
|