@bitkyc08/opencodex 2.55.0 → 2.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ocx.mjs +10 -0
- package/gui/dist/assets/{index-BBOZWGB6.css → index-C5-RdDmD.css} +1 -1
- package/gui/dist/assets/{index-VuoiWj9J.js → index-Cz7CLdif.js} +21 -21
- package/gui/dist/index.html +2 -2
- package/package.json +4 -3
- package/src/adapters/base.ts +21 -0
- package/src/adapters/codebuddy/adapter.ts +2 -1
- package/src/adapters/codebuddy/scaffold-guard.ts +248 -0
- package/src/adapters/command-code.ts +1 -1
- package/src/adapters/cursor/envelope-echo.ts +8 -2
- package/src/adapters/cursor/transport-retry.ts +46 -1
- package/src/adapters/cursor.ts +4 -0
- package/src/adapters/google.ts +7 -7
- package/src/adapters/kiro/adapter.ts +42 -1
- package/src/adapters/kiro/payload.ts +17 -3
- package/src/adapters/kiro/reasoning.ts +70 -7
- package/src/adapters/kiro/stream.ts +8 -2
- package/src/adapters/kiro/wire.ts +2 -1
- package/src/adapters/kiro-events.ts +21 -13
- package/src/adapters/kiro-retry.ts +23 -4
- package/src/adapters/openai-chat/errors.ts +116 -0
- package/src/adapters/openai-chat/messages.ts +346 -0
- package/src/adapters/openai-chat/passthrough.ts +146 -0
- package/src/adapters/openai-chat/response-events.ts +117 -0
- package/src/adapters/openai-chat/tool-call-validation.ts +200 -0
- package/src/adapters/openai-chat/tool-name-registry.ts +166 -0
- package/src/adapters/openai-chat/tool-schema.ts +495 -0
- package/src/adapters/openai-chat/wire.ts +50 -0
- package/src/adapters/openai-chat.ts +40 -1452
- package/src/adapters/openai-responses/canonical-forward.ts +202 -0
- package/src/adapters/openai-responses/image-gen.ts +406 -0
- package/src/adapters/openai-responses/internal.ts +3 -0
- package/src/adapters/openai-responses/passthrough.ts +642 -0
- package/src/adapters/openai-responses/prompt-cache.ts +83 -0
- package/src/adapters/openai-responses/reasoning.ts +220 -0
- package/src/adapters/openai-responses/request-strips.ts +185 -0
- package/src/adapters/openai-responses/tool-output-recovery.ts +509 -0
- package/src/adapters/openai-responses/tool-schema.ts +293 -0
- package/src/adapters/openai-responses/web-search.ts +156 -0
- package/src/adapters/openai-responses.ts +4 -2625
- package/src/bridge/errors.ts +58 -0
- package/src/bridge/internal.ts +174 -0
- package/src/bridge/response-json.ts +630 -0
- package/src/bridge/sse.ts +1462 -0
- package/src/bridge.ts +5 -2204
- package/src/chat/inbound.ts +12 -1
- package/src/claude/desktop-profile.ts +66 -9
- package/src/claude/outbound.ts +18 -0
- package/src/cli/account-main.ts +1 -1
- package/src/cli/capabilities.ts +2 -2
- package/src/cli/combo.ts +10 -1
- package/src/cli/index.ts +48 -5
- package/src/cli/registry.ts +2 -1
- package/src/cli/system-command.ts +4 -4
- package/src/clients/config-export.ts +7 -3
- package/src/codex/account-label.ts +14 -3
- package/src/codex/account-lifecycle.ts +3 -0
- package/src/codex/account-store.ts +184 -35
- package/src/codex/account-usability.ts +21 -0
- package/src/codex/auth-api/account-list.ts +507 -0
- package/src/codex/auth-api/http.ts +32 -0
- package/src/codex/auth-api/login-flow.ts +566 -0
- package/src/codex/auth-api/login-state.ts +64 -0
- package/src/codex/auth-api/main-account-probe.ts +331 -0
- package/src/codex/auth-api/pool-mode-gate.ts +274 -0
- package/src/codex/auth-api/pool-quota-probe.ts +512 -0
- package/src/codex/auth-api/reset-credit-service.ts +431 -0
- package/src/codex/auth-api/routes.ts +425 -0
- package/src/codex/auth-api/runtime-config.ts +48 -0
- package/src/codex/auth-api.ts +27 -3118
- package/src/codex/auth-context.ts +252 -35
- package/src/codex/catalog/aggregation.ts +80 -1
- package/src/codex/catalog/auto-review.ts +507 -0
- package/src/codex/catalog/build-entries.ts +981 -0
- package/src/codex/catalog/combo-member.ts +375 -0
- package/src/codex/catalog/derive-entry.ts +229 -0
- package/src/codex/catalog/effort.ts +0 -1
- package/src/codex/catalog/gated-native-warn.ts +63 -0
- package/src/codex/catalog/gather-capture.ts +533 -0
- package/src/codex/catalog/model-hints.ts +691 -0
- package/src/codex/catalog/model-visibility.ts +305 -0
- package/src/codex/catalog/provider-fetch.ts +52 -2942
- package/src/codex/catalog/provider-models.ts +685 -0
- package/src/codex/catalog/remote.ts +30 -0
- package/src/codex/catalog/restore.ts +132 -0
- package/src/codex/catalog/retained-sync.ts +714 -0
- package/src/codex/catalog/routed-gather.ts +895 -0
- package/src/codex/catalog/subagent-roster.ts +176 -0
- package/src/codex/catalog/sync.ts +52 -2698
- package/src/codex/cli-install-provenance.ts +7 -1
- package/src/codex/convergence.ts +7 -2
- package/src/codex/desktop-app/types.ts +11 -2
- package/src/codex/desktop-app/windows.ts +5 -5
- package/src/codex/inject/config-toml.ts +563 -0
- package/src/codex/inject/remove.ts +192 -0
- package/src/codex/inject/restore.ts +567 -0
- package/src/codex/inject/routing-classify.ts +109 -0
- package/src/codex/inject/routing-target.ts +125 -0
- package/src/codex/inject.ts +89 -1444
- package/src/codex/lineage.ts +458 -0
- package/src/codex/model-entitlements.ts +152 -15
- package/src/codex/pool-refresh-backoff.ts +161 -0
- package/src/codex/quota-rejection.ts +104 -15
- package/src/codex/routing/active-account.ts +194 -0
- package/src/codex/routing/cache-affinity.ts +70 -0
- package/src/codex/routing/cooldown-math.ts +285 -0
- package/src/codex/routing/health-store.ts +402 -0
- package/src/codex/routing/probe-lease.ts +358 -0
- package/src/codex/routing/selection.ts +780 -0
- package/src/codex/routing/thread-affinity.ts +586 -0
- package/src/codex/routing/transient-hold-dispatch.ts +141 -0
- package/src/codex/routing.ts +370 -2271
- package/src/codex/shim-fingerprint.ts +223 -0
- package/src/codex/shim-inspect.ts +175 -0
- package/src/codex/shim-probe.ts +367 -0
- package/src/codex/shim-restore-lock.ts +169 -0
- package/src/codex/shim-state-file.ts +151 -0
- package/src/codex/shim-templates.ts +265 -0
- package/src/codex/shim.ts +48 -1268
- package/src/codex/warmup.ts +1 -1
- package/src/combos/failover.ts +85 -0
- package/src/combos/request.ts +17 -10
- package/src/combos/types.ts +23 -2
- package/src/config/diagnostics.ts +705 -0
- package/src/config/feature-flags.ts +55 -0
- package/src/config/live-reconcile.ts +403 -0
- package/src/config/load-degrade.ts +880 -0
- package/src/config/mutation-lock.ts +244 -0
- package/src/config/openai-tier-backup.ts +268 -0
- package/src/config/pending-teardown.ts +31 -0
- package/src/config/persist-unlocked.ts +92 -0
- package/src/config/proxy-env.ts +188 -0
- package/src/config/salvage.ts +244 -0
- package/src/config/schema/config-schema.ts +640 -0
- package/src/config/schema/leaf-validators.ts +855 -0
- package/src/config/warn-memo.ts +28 -0
- package/src/config.ts +234 -4481
- package/src/generated/compatibility-version.json +649 -121
- package/src/images/loop.ts +1 -1
- package/src/lib/errors.ts +17 -0
- package/src/lib/request-execution-budget.ts +198 -23
- package/src/lib/spend-reservation-ledger.ts +958 -0
- package/src/lib/state-store-registrations.ts +6 -2
- package/src/lib/test-home-guard.ts +85 -1
- package/src/lib/upstream-retry.ts +132 -21
- package/src/lib/windows-elevation.ts +76 -14
- package/src/lib/workflow-budget.ts +553 -30
- package/src/oauth/index.ts +2 -2
- package/src/oauth/key-providers.ts +2 -2
- package/src/providers/kiro-models.ts +4 -3
- package/src/providers/label.ts +19 -1
- package/src/providers/model-discovery.ts +16 -0
- package/src/providers/quota/account-cache.ts +441 -0
- package/src/providers/quota/antigravity.ts +295 -0
- package/src/providers/quota/report-cache.ts +320 -0
- package/src/providers/quota/vendor-probes-key.ts +1243 -0
- package/src/providers/quota/vendor-probes-oauth.ts +590 -0
- package/src/providers/quota.ts +324 -3079
- package/src/providers/registry/entries-core.ts +1228 -0
- package/src/providers/registry/entries-extended.ts +1213 -0
- package/src/providers/registry/model-seeds.ts +912 -0
- package/src/providers/registry/types.ts +352 -0
- package/src/providers/registry.ts +24 -3536
- package/src/responses/continuation-ownership.ts +29 -0
- package/src/responses/reasoning-envelope.ts +6 -3
- package/src/responses/state/replay-fingerprint.ts +80 -0
- package/src/responses/state/snapshot-codec.ts +104 -0
- package/src/responses/state/spill-failure.ts +118 -0
- package/src/responses/state/spill-queue.ts +665 -0
- package/src/responses/state/temp-recovery.ts +257 -0
- package/src/responses/state.ts +82 -1143
- package/src/routing/identity-domains.ts +456 -0
- package/src/routing/probe-lease.ts +613 -0
- package/src/server/chat-completions.ts +3 -1
- package/src/server/chat-native.ts +37 -9
- package/src/server/index/bounded-request.ts +88 -0
- package/src/server/index/live-sideband.ts +601 -0
- package/src/server/index/serve-options.ts +1766 -0
- package/src/server/index/startup-warnings.ts +213 -0
- package/src/server/index/websocket-handler.ts +339 -0
- package/src/server/index.ts +45 -2552
- package/src/server/inspection-tee.ts +107 -0
- package/src/server/live.ts +46 -1
- package/src/server/management/combo-routes.ts +10 -1
- package/src/server/management/route-registry.ts +26 -23
- package/src/server/management/shared.ts +8 -5
- package/src/server/management/workflow-budget-routes.ts +133 -0
- package/src/server/management-api.ts +12 -0
- package/src/server/relay-eager.ts +2 -0
- package/src/server/relay.ts +14 -19
- package/src/server/request-log-conversation.ts +9 -7
- package/src/server/request-log.ts +372 -4
- package/src/server/response-log-body.ts +153 -0
- package/src/server/responses/account-change-state.ts +307 -0
- package/src/server/responses/adapter-continuation.ts +540 -0
- package/src/server/responses/adapter-delivery.ts +208 -0
- package/src/server/responses/adapter-dispatch.ts +1042 -0
- package/src/server/responses/codex-ws-wire.ts +5 -0
- package/src/server/responses/collaboration.ts +74 -4
- package/src/server/responses/combo-session-recall.ts +68 -8
- package/src/server/responses/compact.ts +113 -17
- package/src/server/responses/completion-policy.ts +33 -0
- package/src/server/responses/core-auth.ts +529 -0
- package/src/server/responses/core-codex-account.ts +907 -0
- package/src/server/responses/core-combo-failure.ts +210 -0
- package/src/server/responses/core-combo.ts +787 -0
- package/src/server/responses/core-errors.ts +170 -0
- package/src/server/responses/core-lifetime.ts +95 -0
- package/src/server/responses/core-normalize.ts +350 -0
- package/src/server/responses/core-opaque-recovery.ts +380 -0
- package/src/server/responses/core-options.ts +159 -0
- package/src/server/responses/core-replay.ts +298 -0
- package/src/server/responses/core.ts +192 -8893
- package/src/server/responses/encrypted-payload.ts +0 -1
- package/src/server/responses/input-admission.ts +126 -6
- package/src/server/responses/passthrough-delivery.ts +869 -0
- package/src/server/responses/passthrough-dispatch.ts +1494 -0
- package/src/server/responses/passthrough-error.ts +38 -2
- package/src/server/responses/passthrough-execution.ts +54 -0
- package/src/server/responses/request-prepare.ts +1080 -0
- package/src/server/responses/request-send-budget.ts +259 -0
- package/src/server/responses/request-sidecar-auth.ts +149 -0
- package/src/server/responses/request-spend.ts +147 -0
- package/src/server/responses/request-transport.ts +803 -0
- package/src/server/responses/response-effects.ts +157 -0
- package/src/server/responses/run-turn-execution.ts +476 -0
- package/src/server/responses/sidecar-execution.ts +463 -0
- package/src/server/responses/terminal-guard.ts +65 -4
- package/src/server/responses-image-gen-repair.ts +1 -1
- package/src/server/responses-undeclared-tool-guard.ts +9 -5
- package/src/server/workflow-refusal.ts +84 -0
- package/src/service/windows-ops.ts +210 -16
- package/src/service/windows-scheduler.ts +28 -21
- package/src/service.ts +1 -1
- package/src/types/config.ts +34 -1
- package/src/types/request.ts +8 -5
- package/src/types/tools.ts +24 -0
- package/src/types.ts +2 -0
- package/src/update/index.ts +10 -0
- package/src/update/stop-contract.d.mts +1 -0
- package/src/update/stop-contract.mjs +19 -0
- package/src/update/stop-decision.d.mts +1 -1
- package/src/update/stop-decision.mjs +12 -3
- package/src/usage/log.ts +147 -1
- package/src/usage/summary.ts +171 -21
- package/src/vision/anthropic-describe.ts +1 -1
- package/src/vision/describe.ts +5 -5
- package/src/web-search/anthropic-executor.ts +1 -1
- package/src/web-search/exa-executor.ts +1 -1
- package/src/web-search/executor.ts +1 -1
- package/src/web-search/gemini-executor.ts +1 -1
- package/src/web-search/loop.ts +1 -1
- package/src/web-search/ollama-executor.ts +1 -1
- package/src/web-search/parse.ts +67 -14
- package/src/web-search/passthrough-bridge.ts +64 -31
- package/src/web-search/xai-executor.ts +1 -1
package/src/codex/routing.ts
CHANGED
|
@@ -1,409 +1,207 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
1
|
import { saveConfigPreservingClaudeCode } from "../config";
|
|
3
|
-
import { isCodexAccountGenerationLive
|
|
2
|
+
import { isCodexAccountGenerationLive } from "./account-store";
|
|
4
3
|
import { codexAccountLogLabel } from "./account-label";
|
|
5
|
-
import { NATIVE_RESERVE_MODEL } from "./catalog/native-models";
|
|
6
4
|
import { isCodexAccountPaused } from "./account-pause";
|
|
7
|
-
import { clearCodexAccountPin,
|
|
5
|
+
import { clearCodexAccountPin, pinnedCodexAccountId } from "./account-priority";
|
|
8
6
|
import { isCodexAccountUsable, type CodexAccountUsabilityOptions } from "./account-usability";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
normalizeCodexAccountPoolStrategy,
|
|
14
|
-
notePoolRotationFailure,
|
|
15
|
-
notePoolRotationSuccess,
|
|
16
|
-
peekRoundRobinAccount,
|
|
17
|
-
pickRoundRobinAccount,
|
|
18
|
-
seedPoolRotationAccount,
|
|
19
|
-
selectPriorityTier,
|
|
20
|
-
} from "./pool-rotation";
|
|
21
|
-
import {
|
|
22
|
-
CODEX_EXHAUSTED_USAGE_PERCENT,
|
|
23
|
-
CODEX_UNKNOWN_USAGE_SCORE,
|
|
24
|
-
getAccountQuota,
|
|
25
|
-
isRetiredCodexSparkModel,
|
|
26
|
-
resetAtToMs,
|
|
27
|
-
} from "./quota";
|
|
28
|
-
import { codexPlanKey, isThirtyDayOnlyCodexPlan } from "./plan";
|
|
29
|
-
import {
|
|
30
|
-
MAIN_CODEX_ACCOUNT_ID,
|
|
31
|
-
getMainAccountPlan,
|
|
32
|
-
hasMainAccountRefreshGrant,
|
|
33
|
-
} from "./main-account";
|
|
7
|
+
import { isAccountNeedsReauth, markAccountNeedsReauth } from "./account-runtime-state";
|
|
8
|
+
import { POOL_KEY_CODEX, notePoolRotationFailure } from "./pool-rotation";
|
|
9
|
+
import { getAccountQuota, isRetiredCodexSparkModel } from "./quota";
|
|
10
|
+
import { MAIN_CODEX_ACCOUNT_ID } from "./main-account";
|
|
34
11
|
import { isSelectableCodexPoolAccount } from "./account-id";
|
|
35
12
|
import type { OcxConfig } from "../types";
|
|
36
13
|
import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper";
|
|
37
|
-
import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers";
|
|
38
|
-
import { retainedUtf8Bytes } from "../lib/admission";
|
|
39
14
|
import { recordUpstreamHostFailure } from "./upstream-host-health";
|
|
15
|
+
import type { CodexThreadLineage } from "./lineage";
|
|
40
16
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
* How long a quota refusal keeps selection away from this account (or this native quota
|
|
154
|
-
* group), as opposed to how long it is hard-blocked.
|
|
155
|
-
*
|
|
156
|
-
* The two are deliberately different lengths. {@link CODEX_MAX_RESET_DERIVED_COOLDOWN_MS}
|
|
157
|
-
* caps the hard cooldown at 15 minutes because a reset announcement is advisory and plan
|
|
158
|
-
* quota usually frees up before it — an account must stay reachable so the pool can find
|
|
159
|
-
* that out (#433). The window the refusal announced is not 15 minutes, though, so once the
|
|
160
|
-
* cooldown lapses the account is selectable again while its burst window is still spent,
|
|
161
|
-
* and the strategy picks it straight back: this proxy reads a weekly bar a burst limit never
|
|
162
|
-
* touches, so a refused account still scores as the coolest in the pool. Every request then
|
|
163
|
-
* earns the same 429 until the process restarts, which is the only thing that drops this map.
|
|
164
|
-
*
|
|
165
|
-
* So the announcement governs avoidance and the cap still governs blocking. Avoidance is soft
|
|
166
|
-
* in the {@link softAvoidUntil} sense: it reorders the pool and releases a bound thread, and
|
|
167
|
-
* the last-resort paths still reach the account when nothing else can serve, so one pessimistic
|
|
168
|
-
* announcement cannot stall routing.
|
|
169
|
-
*/
|
|
170
|
-
quotaAvoidUntil?: number;
|
|
171
|
-
/** When the current cooldown was recorded; origin of the probe interval clock. */
|
|
172
|
-
cooldownSince?: number;
|
|
173
|
-
/**
|
|
174
|
-
* What produced the cooldown. An explicit Retry-After is a literal retry
|
|
175
|
-
* directive and is never probed; a quota resetAt only announces a window
|
|
176
|
-
* refresh, so it may be probed early (#433).
|
|
177
|
-
*/
|
|
178
|
-
cooldownSource?: CodexCooldownSource;
|
|
179
|
-
/**
|
|
180
|
-
* Bumped on every cooldown write. A probe lease records the generation it was
|
|
181
|
-
* issued for so a lease cannot clear a cooldown that a later 429 replaced.
|
|
182
|
-
*/
|
|
183
|
-
cooldownGeneration?: number;
|
|
184
|
-
/**
|
|
185
|
-
* Identity of the in-flight probe. A cooled-down account sends no traffic, so
|
|
186
|
-
* no organic 2xx can prove recovery; only the outcome carrying this id may
|
|
187
|
-
* clear the cooldown.
|
|
188
|
-
*/
|
|
189
|
-
probeLeaseId?: string;
|
|
190
|
-
/** Cooldown generation at the moment the lease was granted. */
|
|
191
|
-
probeLeaseGeneration?: number;
|
|
192
|
-
/** Last probe grant or conclusion; paces the probe interval. */
|
|
193
|
-
lastProbeAt?: number;
|
|
194
|
-
/**
|
|
195
|
-
* Soft avoid after connect_error / timeout / transient 5xx. Cleared on 2xx.
|
|
196
|
-
* Blocks pool selection + thread affinity reuse so a sticky session can leave a
|
|
197
|
-
* flaky account without throwing CodexAccountCooldownError (hard-only).
|
|
198
|
-
*/
|
|
199
|
-
softAvoidUntil?: number;
|
|
200
|
-
/**
|
|
201
|
-
* Credential generation a 401/403 quarantine was derived from (#2892 gap 4).
|
|
202
|
-
*
|
|
203
|
-
* Provenance lives ON the entry rather than in a side map keyed by account id. A side map spends
|
|
204
|
-
* "whatever health is current when the old credential is found dead", which deletes a later
|
|
205
|
-
* unrelated entry: a G1 401, then a G2 save, then a genuine G2 503 would lose the 503. Only the
|
|
206
|
-
* entry that carries this field can be spent, and any later write simply replaces it.
|
|
207
|
-
*/
|
|
208
|
-
credentialFailureGeneration?: number;
|
|
209
|
-
};
|
|
210
|
-
|
|
211
|
-
const CODEX_DEFAULT_QUOTA_COOLDOWN_MS = 60_000;
|
|
212
|
-
const CODEX_MAX_QUOTA_COOLDOWN_MS = 24 * 60 * 60_000;
|
|
213
|
-
/**
|
|
214
|
-
* A weekly/monthly quota `resetAt` announces when the window refreshes; it is not
|
|
215
|
-
* a "come back after this" directive like Retry-After. Plan quota routinely frees
|
|
216
|
-
* up long before the advertised reset, so cap reset-derived cooldowns far below
|
|
217
|
-
* the Retry-After ceiling (#433).
|
|
218
|
-
*/
|
|
219
|
-
const CODEX_MAX_RESET_DERIVED_COOLDOWN_MS = 15 * 60_000;
|
|
220
|
-
/**
|
|
221
|
-
* Ceiling on quota-refusal avoidance. Generous enough to cover a full five-hour burst window,
|
|
222
|
-
* tight enough that a weekly or monthly reset four days out cannot take an account out of
|
|
223
|
-
* rotation for the {@link CODEX_MAX_QUOTA_COOLDOWN_MS} day the Retry-After ceiling allows.
|
|
224
|
-
*/
|
|
225
|
-
const CODEX_MAX_QUOTA_AVOID_MS = 6 * 60 * 60_000;
|
|
226
|
-
/** Minimum gap between probe leases for one cooled-down account. */
|
|
227
|
-
export const CODEX_QUOTA_PROBE_INTERVAL_MS = 5 * 60_000;
|
|
228
|
-
export const CODEX_FAILURE_WINDOW_MS = 5 * 60_000;
|
|
229
|
-
/**
|
|
230
|
-
* How recently a 100% burst reading must have been OBSERVED to exclude an account when it
|
|
231
|
-
* carries no reset timestamp (#3425). Deliberately far tighter than the 6h disk-hydration
|
|
232
|
-
* horizon in `quota.ts`: shorter than any plausible five-hour burst window, so a persisted
|
|
233
|
-
* reading can never strand a recovered account, and long enough that a snapshot taken at
|
|
234
|
-
* admission is still fresh when selection reads it.
|
|
235
|
-
*/
|
|
236
|
-
export const TERMINAL_SHORT_WINDOW_FRESHNESS_MS = 5 * 60_000;
|
|
237
|
-
/** How long a transient failure keeps the account out of pool selection. */
|
|
238
|
-
export const CODEX_TRANSIENT_SOFT_AVOID_MS = 30_000;
|
|
239
|
-
const CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS = [
|
|
17
|
+
import { isCodexPoolRefreshCooling } from "./pool-refresh-backoff";
|
|
18
|
+
import {
|
|
19
|
+
classifyCodexUpstreamOutcome,
|
|
20
|
+
computeCodexUsageScore,
|
|
21
|
+
computeQuotaCooldown,
|
|
22
|
+
quotaAvoidUntilFor,
|
|
23
|
+
CODEX_FAILURE_WINDOW_MS,
|
|
24
|
+
CODEX_TRANSIENT_SOFT_AVOID_ESCALATION_MS,
|
|
25
|
+
type CodexUpstreamOutcome,
|
|
26
|
+
type CodexUpstreamOutcomeMeta,
|
|
27
|
+
} from "./routing/cooldown-math";
|
|
28
|
+
import {
|
|
29
|
+
codexPoolKeyForScope,
|
|
30
|
+
codexQuotaScopeForModel,
|
|
31
|
+
deleteAccountHealth,
|
|
32
|
+
deleteAllScopedHealth,
|
|
33
|
+
deleteScopedHealth,
|
|
34
|
+
dropSpentCredentialFailure,
|
|
35
|
+
getAccountHealth,
|
|
36
|
+
getCodexAccountCooldownUntil,
|
|
37
|
+
getCodexAccountSoftAvoidUntil,
|
|
38
|
+
getCodexQuotaHealthSnapshot,
|
|
39
|
+
isCodexAccountSoftAvoided,
|
|
40
|
+
isCodexQuotaAvoided,
|
|
41
|
+
isHealthAccountAdmissible,
|
|
42
|
+
isHealthGenerationReconciled,
|
|
43
|
+
isIndependentCodexQuotaScope,
|
|
44
|
+
preservedCooldownFields,
|
|
45
|
+
pruneHealthAccountsForContext,
|
|
46
|
+
commitHealthReconcile,
|
|
47
|
+
clearUpstreamHealthState,
|
|
48
|
+
resetHealthReconcileState,
|
|
49
|
+
deleteAllHealthForAccount,
|
|
50
|
+
scopedHealthFor,
|
|
51
|
+
setAccountHealth,
|
|
52
|
+
setScopedHealth,
|
|
53
|
+
type CodexQuotaScope,
|
|
54
|
+
type CodexUpstreamHealth,
|
|
55
|
+
} from "./routing/health-store";
|
|
56
|
+
import { ownsProbeLease, probeMayClearCooldown, withProbeLeaseReleased } from "./routing/probe-lease";
|
|
57
|
+
// `./routing/probe-lease` above is the QUOTA-COOLDOWN lease; the module below owns the
|
|
58
|
+
// unrelated TRANSIENT-HOLD trial and the pool-wide recovery bound above it (#4701).
|
|
59
|
+
import {
|
|
60
|
+
isTransientHoldExpired,
|
|
61
|
+
resolveTransientHoldDispatch,
|
|
62
|
+
settleTransientProbeForOutcome,
|
|
63
|
+
} from "./routing/transient-hold-dispatch";
|
|
64
|
+
import {
|
|
65
|
+
adoptLegacyLineageAffinity,
|
|
66
|
+
affinityAfterRelease,
|
|
67
|
+
affinityOnNoAccount,
|
|
68
|
+
bindModelDetourAffinity,
|
|
69
|
+
bindThreadAffinity,
|
|
70
|
+
clearThreadAccountMapForAccount,
|
|
71
|
+
deleteModelDetourAffinity,
|
|
72
|
+
deleteThreadAffinity,
|
|
73
|
+
deleteThreadAffinitiesForAccount,
|
|
74
|
+
getThreadAffinity,
|
|
75
|
+
getThreadAffinityScopes,
|
|
76
|
+
getModelDetourAffinity,
|
|
77
|
+
isThreadAffinityExpired,
|
|
78
|
+
isThreadAffinityGenerationLive,
|
|
79
|
+
peekPendingReleaseReason,
|
|
80
|
+
CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS,
|
|
81
|
+
CODEX_TRANSIENT_AFFINITY_HOLD_MS,
|
|
82
|
+
type CodexAffinityReason,
|
|
83
|
+
type CodexThreadResolution,
|
|
84
|
+
type ThreadAffinityEntry,
|
|
85
|
+
} from "./routing/thread-affinity";
|
|
86
|
+
import {
|
|
87
|
+
accountPoolStrategyForScope,
|
|
88
|
+
applyFailureFailover,
|
|
89
|
+
applyQuotaAutoSwitch,
|
|
90
|
+
codexAccountBlockReason,
|
|
91
|
+
getEligiblePoolAccounts,
|
|
92
|
+
getPoolAccountPlanForSelection,
|
|
93
|
+
hasCodexQuotaHeadroom,
|
|
94
|
+
isCodexAccountPlanExcluded,
|
|
95
|
+
isCodexAccountSelectable,
|
|
96
|
+
isHealthySharedCodexSelection,
|
|
97
|
+
isUnknownUsage,
|
|
98
|
+
pickAlternateCodexAccount,
|
|
99
|
+
pickLowerUsageAccount,
|
|
100
|
+
pickLowestUsageAmong,
|
|
101
|
+
pickLowestUsageCodexAccount,
|
|
102
|
+
pickPriorityPreemption,
|
|
103
|
+
pickResetFirstCodexAccount,
|
|
104
|
+
pickUnboundStrategyAccount,
|
|
105
|
+
preferModelEntitledAccount,
|
|
106
|
+
sharedStateSelectionOptions,
|
|
107
|
+
strategySelectionOptionsForModelDetour,
|
|
108
|
+
shouldFailover,
|
|
109
|
+
peekAlternateCodexAccount,
|
|
110
|
+
} from "./routing/selection";
|
|
111
|
+
import { mayRebindAffinityForQuota } from "./routing/cache-affinity";
|
|
112
|
+
import {
|
|
113
|
+
clearAllManualPreferences,
|
|
114
|
+
consumeManualPreference,
|
|
115
|
+
forgetManualPreference,
|
|
116
|
+
forgetRoutingPreferencesOutside,
|
|
117
|
+
forgetRuntimeActiveCodexAccount,
|
|
118
|
+
getEffectiveActiveCodexAccountId,
|
|
119
|
+
manualPreferenceBlocks,
|
|
120
|
+
promoteActiveCodexAccount,
|
|
121
|
+
rememberActiveCodexAccount,
|
|
122
|
+
setActiveCodexAccount,
|
|
123
|
+
} from "./routing/active-account";
|
|
124
|
+
|
|
125
|
+
export {
|
|
126
|
+
CODEX_QUOTA_PROBE_INTERVAL_MS,
|
|
127
|
+
CODEX_FAILURE_WINDOW_MS,
|
|
128
|
+
TERMINAL_SHORT_WINDOW_FRESHNESS_MS,
|
|
240
129
|
CODEX_TRANSIENT_SOFT_AVOID_MS,
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
export
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
export type
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
export
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
};
|
|
316
|
-
|
|
317
|
-
export type CodexQuotaRecoveryProbeProof = {
|
|
318
|
-
credentialGeneration?: number;
|
|
319
|
-
};
|
|
320
|
-
|
|
321
|
-
/**
|
|
322
|
-
* Requests without a resolved native model retain the historic one-account-per-
|
|
323
|
-
* thread behavior. Requests with a known quota scope get an independent
|
|
324
|
-
* affinity so a Reserve failover cannot displace the same thread's Terra/Luna
|
|
325
|
-
* account (and vice versa).
|
|
326
|
-
*/
|
|
327
|
-
type BaseThreadAffinityScope = CodexQuotaScope | "legacy";
|
|
328
|
-
type ModelDetourAffinityScope = `model-detour:${BaseThreadAffinityScope}:${string}`;
|
|
329
|
-
type ThreadAffinityScope = BaseThreadAffinityScope | ModelDetourAffinityScope;
|
|
330
|
-
const LEGACY_THREAD_AFFINITY_SCOPE = "legacy" as const;
|
|
331
|
-
const threadAccountMap = new Map<string, Map<ThreadAffinityScope, ThreadAffinityEntry>>();
|
|
332
|
-
let threadAffinityEntryTotal = 0;
|
|
333
|
-
|
|
334
|
-
function isModelDetourAffinityScope(scope: ThreadAffinityScope): scope is ModelDetourAffinityScope {
|
|
335
|
-
return scope.startsWith("model-detour:");
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
const NATIVE_MODEL_QUOTA_SCOPES: Readonly<Record<string, CodexQuotaScope>> = {
|
|
339
|
-
[NATIVE_RESERVE_MODEL]: "reserve",
|
|
340
|
-
};
|
|
341
|
-
|
|
342
|
-
export function codexQuotaScopeForModel(modelId: string | undefined): CodexQuotaScope | undefined {
|
|
343
|
-
if (!modelId?.trim()) return undefined;
|
|
344
|
-
return NATIVE_MODEL_QUOTA_SCOPES[modelId.trim().toLowerCase()] ?? "shared";
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
/** Independent quota groups must not mutate the shared active-account cursor. */
|
|
348
|
-
function isIndependentCodexQuotaScope(quotaScope?: CodexQuotaScope): boolean {
|
|
349
|
-
return quotaScope !== undefined && quotaScope !== "shared";
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function codexPoolKeyForScope(quotaScope?: CodexQuotaScope): string {
|
|
353
|
-
return isIndependentCodexQuotaScope(quotaScope) ? `${POOL_KEY_CODEX}:${quotaScope}` : POOL_KEY_CODEX;
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
export type CodexUpstreamOutcomeMeta = {
|
|
357
|
-
retryAfter?: string | null;
|
|
358
|
-
resetAt?: unknown | unknown[];
|
|
359
|
-
now?: number;
|
|
360
|
-
/** (provider, host) ledger key for account-neutral reachability failures (#914). */
|
|
361
|
-
hostKey?: string;
|
|
362
|
-
/**
|
|
363
|
-
* Upstream denial evidence for a 403. A workspace/entitlement denial means the CREDENTIAL
|
|
364
|
-
* is fine and the account simply cannot reach this workspace, so it must not be quarantined
|
|
365
|
-
* for reauthentication (#1789). Absent evidence keeps the historical credential handling.
|
|
366
|
-
*/
|
|
367
|
-
denial?: "workspace" | "entitlement";
|
|
368
|
-
/** Stable transport code recorded alongside a neutral host failure. */
|
|
369
|
-
lastFailureCode?: string;
|
|
370
|
-
/** Native model selected for this request; used only for confirmed scoped quotas. */
|
|
371
|
-
modelId?: string;
|
|
372
|
-
/** When set, clears affinity for this thread immediately on transient failure. */
|
|
373
|
-
threadId?: string | null;
|
|
374
|
-
/**
|
|
375
|
-
* Suppress Pool rotation and quota/transient affinity mutations for an account-qualified
|
|
376
|
-
* request. Credential failures still sweep stale affinities because reauthentication is
|
|
377
|
-
* account-wide.
|
|
378
|
-
*/
|
|
379
|
-
fixedAccount?: boolean;
|
|
380
|
-
/**
|
|
381
|
-
* Probe lease held by this request, when it was admitted through an active
|
|
382
|
-
* quota cooldown. Only the outcome carrying the current lease may clear the
|
|
383
|
-
* cooldown (#433).
|
|
384
|
-
*/
|
|
385
|
-
probeLeaseId?: string;
|
|
386
|
-
/** Scope of `probeLeaseId` when it was granted against a model-scoped cooldown. */
|
|
387
|
-
probeQuotaScope?: CodexQuotaScope;
|
|
388
|
-
/**
|
|
389
|
-
* Already-chosen alternate for same-request 429 retry. When set, promotion
|
|
390
|
-
* reuses this account instead of calling {@link pickAlternateCodexAccount}
|
|
391
|
-
* again (which would advance a round-robin ring twice).
|
|
392
|
-
*/
|
|
393
|
-
promoteAccountId?: string;
|
|
394
|
-
/** Generation captured when this routed account was selected. */
|
|
395
|
-
writerGeneration?: number;
|
|
396
|
-
/**
|
|
397
|
-
* Credential generation this request's bearer was read at. Distinct from
|
|
398
|
-
* `writerGeneration`, which tracks the config store.
|
|
399
|
-
*
|
|
400
|
-
* A 401 that arrives after the credential was already replaced is evidence about a
|
|
401
|
-
* token nobody is using any more, so it must not quarantine the replacement. Absent
|
|
402
|
-
* means the caller cannot supply lineage and the historical unfenced handling stands.
|
|
403
|
-
*/
|
|
404
|
-
credentialGeneration?: number;
|
|
405
|
-
};
|
|
406
|
-
|
|
130
|
+
classifyCodexUpstreamOutcome,
|
|
131
|
+
computeCodexUsageScore,
|
|
132
|
+
computeQuotaCooldown,
|
|
133
|
+
computeQuotaCooldownUntil,
|
|
134
|
+
parseRetryAfterMs,
|
|
135
|
+
parseResetCooldownMs,
|
|
136
|
+
} from "./routing/cooldown-math";
|
|
137
|
+
export type {
|
|
138
|
+
CodexUpstreamOutcome,
|
|
139
|
+
CodexUpstreamOutcomeClass,
|
|
140
|
+
CodexCooldownSource,
|
|
141
|
+
CodexUpstreamOutcomeMeta,
|
|
142
|
+
} from "./routing/cooldown-math";
|
|
143
|
+
export {
|
|
144
|
+
codexQuotaScopeForModel,
|
|
145
|
+
listLiveCodexAccountIds,
|
|
146
|
+
getCodexUpstreamHealth,
|
|
147
|
+
getCodexAccountCooldownUntil,
|
|
148
|
+
getCodexAccountHealthSnapshot,
|
|
149
|
+
getCodexQuotaHealthSnapshot,
|
|
150
|
+
isCodexAccountInCooldown,
|
|
151
|
+
clearCodexAccountCooldown,
|
|
152
|
+
getCodexAccountSoftAvoidUntil,
|
|
153
|
+
isCodexAccountSoftAvoided,
|
|
154
|
+
} from "./routing/health-store";
|
|
155
|
+
export type { CodexQuotaScope } from "./routing/health-store";
|
|
156
|
+
export {
|
|
157
|
+
tryAcquireCodexQuotaProbeLease,
|
|
158
|
+
canAcquireCodexQuotaProbeLease,
|
|
159
|
+
claimDueCodexQuotaRecoveryProbes,
|
|
160
|
+
claimManualResetCooldowns,
|
|
161
|
+
settleManualResetCooldown,
|
|
162
|
+
settleCodexQuotaRecoveryProbe,
|
|
163
|
+
tryAcquireCodexQuotaScopeProbeLease,
|
|
164
|
+
canAcquireCodexQuotaScopeProbeLease,
|
|
165
|
+
releaseCodexQuotaProbeLease,
|
|
166
|
+
releaseCodexQuotaScopeProbeLease,
|
|
167
|
+
} from "./routing/probe-lease";
|
|
168
|
+
export type {
|
|
169
|
+
CodexQuotaRecoveryProbeClaim,
|
|
170
|
+
CodexQuotaRecoveryProbeProof,
|
|
171
|
+
ManualResetCooldownClaim,
|
|
172
|
+
ManualResetRefreshLineage,
|
|
173
|
+
} from "./routing/probe-lease";
|
|
174
|
+
export {
|
|
175
|
+
CODEX_THREAD_AFFINITY_IDLE_TTL_MS,
|
|
176
|
+
CODEX_THREAD_AFFINITY_MAX_ENTRIES,
|
|
177
|
+
CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS,
|
|
178
|
+
CODEX_TRANSIENT_AFFINITY_HOLD_MS,
|
|
179
|
+
clearConversationStateIssuerMap,
|
|
180
|
+
clearThreadAccountMap,
|
|
181
|
+
clearThreadAccountMapForAccount,
|
|
182
|
+
debugCodexAffinityGenerations,
|
|
183
|
+
handOffThreadAffinityGeneration,
|
|
184
|
+
peekConversationStateIssuer,
|
|
185
|
+
rememberConversationStateIssuer,
|
|
186
|
+
} from "./routing/thread-affinity";
|
|
187
|
+
export type {
|
|
188
|
+
CodexThreadResolution,
|
|
189
|
+
CodexAffinityMove,
|
|
190
|
+
CodexAffinityReason,
|
|
191
|
+
CodexAffinityDecision,
|
|
192
|
+
TransientProbeGrant,
|
|
193
|
+
} from "./routing/thread-affinity";
|
|
194
|
+
export {
|
|
195
|
+
isCodexAccountPlanExcluded,
|
|
196
|
+
getPoolAccountPlan,
|
|
197
|
+
pickLowestUsageCodexAccount,
|
|
198
|
+
pickAlternateCodexAccount,
|
|
199
|
+
} from "./routing/selection";
|
|
200
|
+
export {
|
|
201
|
+
resetCodexRoutingForManualSelection,
|
|
202
|
+
getEffectiveActiveCodexAccountId,
|
|
203
|
+
isEffectiveCodexAccountPinned,
|
|
204
|
+
} from "./routing/active-account";
|
|
407
205
|
function hasConfiguredPoolAccount(
|
|
408
206
|
config: OcxConfig,
|
|
409
207
|
accountId: string,
|
|
@@ -416,90 +214,23 @@ function hasConfiguredPoolAccount(
|
|
|
416
214
|
.some(account => isSelectableCodexPoolAccount(account) && account.id === accountId);
|
|
417
215
|
}
|
|
418
216
|
|
|
419
|
-
export function listLiveCodexAccountIds(config: OcxConfig): ReadonlySet<string> {
|
|
420
|
-
const ids = new Set((config.codexAccounts ?? []).map(account => account.id));
|
|
421
|
-
const openai = config.providers.openai;
|
|
422
|
-
if (openai && openai.disabled !== true && isCanonicalOpenAiForwardProvider(openai)) {
|
|
423
|
-
ids.add(MAIN_CODEX_ACCOUNT_ID);
|
|
424
|
-
}
|
|
425
|
-
return ids;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
export function clearThreadAccountMap(): void {
|
|
429
|
-
threadAccountMap.clear();
|
|
430
|
-
threadAffinityEntryTotal = 0;
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
export function clearThreadAccountMapForAccount(
|
|
434
|
-
accountId: string,
|
|
435
|
-
reason: CodexAffinityReason = "unusable",
|
|
436
|
-
): void {
|
|
437
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
438
|
-
for (const [scope, entry] of affinities) {
|
|
439
|
-
if (entry.accountId === accountId && affinities.delete(scope)) {
|
|
440
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
441
|
-
notePendingReleaseReason(threadId, reason);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
/**
|
|
449
|
-
* Why a binding was released, held until that thread's next resolve can report it (#4546).
|
|
450
|
-
*
|
|
451
|
-
* A release and the request that pays for it are two different moments: a 429 clears the pin
|
|
452
|
-
* inside the outcome recorder, and the next request arrives with nothing left to explain why it
|
|
453
|
-
* is starting cold. Bounded, because it is a diagnostic and must not become a leak.
|
|
454
|
-
*/
|
|
455
|
-
const pendingReleaseReasons = new Map<string, CodexAffinityReason>();
|
|
456
|
-
const MAX_PENDING_RELEASE_REASONS = 4096;
|
|
457
|
-
|
|
458
|
-
function notePendingReleaseReason(threadId: string | null, reason: CodexAffinityReason): void {
|
|
459
|
-
if (threadId === null) return;
|
|
460
|
-
if (!pendingReleaseReasons.has(threadId) && pendingReleaseReasons.size >= MAX_PENDING_RELEASE_REASONS) {
|
|
461
|
-
const oldest = pendingReleaseReasons.keys().next();
|
|
462
|
-
if (!oldest.done) pendingReleaseReasons.delete(oldest.value);
|
|
463
|
-
}
|
|
464
|
-
pendingReleaseReasons.set(threadId, reason);
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
function peekPendingReleaseReason(threadId: string | null): CodexAffinityReason | undefined {
|
|
468
|
-
if (threadId === null) return undefined;
|
|
469
|
-
return pendingReleaseReasons.get(threadId);
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/**
|
|
473
|
-
* Forget a release only once it has actually been reported.
|
|
474
|
-
*
|
|
475
|
-
* Consuming it at derivation time lost it whenever selection then failed to produce an account:
|
|
476
|
-
* a no-account return carries no payload, so the release went unrecorded and the next successful
|
|
477
|
-
* resolve claimed a fresh healthy bind (#4598). A release survives until some resolve reports it.
|
|
478
|
-
*/
|
|
479
|
-
function clearPendingReleaseReason(threadId: string | null): void {
|
|
480
|
-
if (threadId !== null) pendingReleaseReasons.delete(threadId);
|
|
481
|
-
}
|
|
482
|
-
|
|
483
217
|
export function clearCodexUpstreamHealth(): void {
|
|
484
218
|
// Operator preferences are routing state, not health, but they live and die with the same
|
|
485
219
|
// reset points. Leaving them behind lets a selection from one context suppress the
|
|
486
220
|
// automatic cursor in the next one.
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
runtimeActiveCodexAccountId = undefined;
|
|
221
|
+
clearAllManualPreferences();
|
|
222
|
+
clearUpstreamHealthState();
|
|
223
|
+
forgetRuntimeActiveCodexAccount();
|
|
491
224
|
// The reconcile watermark is part of this state, not something that outlives it. Keeping
|
|
492
225
|
// it across a full reset is incoherent: there is no health left to protect, yet
|
|
493
226
|
// recordCodexUpstreamOutcome would still drop a writer whose generation predates the
|
|
494
227
|
// watermark for any account missing from the equally stale live set. Left behind, it also
|
|
495
228
|
// leaks between test files, which is how it was found.
|
|
496
|
-
|
|
497
|
-
liveHealthAccountIds = new Set();
|
|
229
|
+
resetHealthReconcileState();
|
|
498
230
|
}
|
|
499
231
|
|
|
500
232
|
export function clearCodexUpstreamHealthForAccount(accountId: string): void {
|
|
501
|
-
|
|
502
|
-
quotaScopedHealth.delete(accountId);
|
|
233
|
+
deleteAllHealthForAccount(accountId);
|
|
503
234
|
// Deletion is the third operator exit, next to pause and exclusion, and it is the one
|
|
504
235
|
// with no reconcile path behind it: once the account is gone nothing can succeed on it,
|
|
505
236
|
// so an unspent preference naming it would suppress the automatic cursor for every other
|
|
@@ -508,1192 +239,16 @@ export function clearCodexUpstreamHealthForAccount(accountId: string): void {
|
|
|
508
239
|
}
|
|
509
240
|
|
|
510
241
|
export function reconcileCodexRoutingHealth(context: GenerationContext): number {
|
|
511
|
-
if (context.generation
|
|
512
|
-
|
|
513
|
-
for
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
quotaScopedHealth.delete(accountId);
|
|
521
|
-
removed += 1;
|
|
522
|
-
}
|
|
523
|
-
// Sweep preferences the same way, for the account set this generation actually has. The
|
|
524
|
-
// delete path above is the direct route; this is the one that catches an account removed
|
|
525
|
-
// by an edit the runtime never saw. Deliberately not counted in `removed`, which reports
|
|
526
|
-
// health rows.
|
|
527
|
-
for (const [poolKey, preferred] of manualPreference) {
|
|
528
|
-
if (context.codexAccountIds.has(preferred)) continue;
|
|
529
|
-
manualPreference.delete(poolKey);
|
|
530
|
-
}
|
|
531
|
-
liveHealthAccountIds = new Set(context.codexAccountIds);
|
|
532
|
-
lastReconciledGeneration = context.generation;
|
|
533
|
-
return removed;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
export function getCodexUpstreamHealth(
|
|
537
|
-
accountId: string,
|
|
538
|
-
): CodexUpstreamHealth | null {
|
|
539
|
-
dropSpentCredentialFailure(accountId);
|
|
540
|
-
return upstreamHealth.get(accountId) ?? null;
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
function scopedHealthFor(accountId: string, scope: CodexQuotaScope): CodexUpstreamHealth | undefined {
|
|
544
|
-
return quotaScopedHealth.get(accountId)?.get(scope);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function setScopedHealth(accountId: string, scope: CodexQuotaScope, health: CodexUpstreamHealth): void {
|
|
548
|
-
let scopes = quotaScopedHealth.get(accountId);
|
|
549
|
-
if (!scopes) {
|
|
550
|
-
scopes = new Map();
|
|
551
|
-
quotaScopedHealth.set(accountId, scopes);
|
|
552
|
-
}
|
|
553
|
-
scopes.set(scope, health);
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
function deleteScopedHealth(accountId: string, scope: CodexQuotaScope): void {
|
|
557
|
-
const scopes = quotaScopedHealth.get(accountId);
|
|
558
|
-
if (!scopes) return;
|
|
559
|
-
scopes.delete(scope);
|
|
560
|
-
if (scopes.size === 0) quotaScopedHealth.delete(accountId);
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
export function computeCodexUsageScore(quota: {
|
|
564
|
-
weeklyPercent?: number;
|
|
565
|
-
monthlyPercent?: number;
|
|
566
|
-
shortPercent?: number;
|
|
567
|
-
shortResetAt?: number;
|
|
568
|
-
shortObservedAt?: number;
|
|
569
|
-
} | null, plan?: unknown, now: number = Date.now()): number {
|
|
570
|
-
if (!quota) return CODEX_UNKNOWN_USAGE_SCORE;
|
|
571
|
-
const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
|
|
572
|
-
const longWindows = isThirtyDayOnlyCodexPlan(plan)
|
|
573
|
-
? [quota.monthlyPercent]
|
|
574
|
-
: [quota.weeklyPercent, quota.monthlyPercent];
|
|
575
|
-
const knownLong = longWindows.filter(finite);
|
|
576
|
-
// The short burst window only REFINES a known long-window position; it cannot stand in for
|
|
577
|
-
// one. A snapshot carrying just `shortPercent: 0` would otherwise score a flat 0 and make an
|
|
578
|
-
// account whose weekly/monthly usage is entirely unverified look like the emptiest in the
|
|
579
|
-
// pool, so `pickLowestUsageAmong` would send every request to it. Unknown has to stay
|
|
580
|
-
// unknown until a governing window is actually observed.
|
|
581
|
-
//
|
|
582
|
-
// A FULL burst window is the exception (#3029). It is not an optimistic guess about an
|
|
583
|
-
// unobserved window — it is a direct observation that the account cannot serve a request
|
|
584
|
-
// right now, whatever its monthly position turns out to be. Unknown-means-selectable is
|
|
585
|
-
// correct for uncertainty and wrong for a measured refusal: the account stays selected,
|
|
586
|
-
// `applyQuotaAutoSwitch` never fires, and the pool wedges on an exhausted credential.
|
|
587
|
-
if (knownLong.length === 0) {
|
|
588
|
-
return isTerminalShortWindow(quota, now) ? CODEX_EXHAUSTED_USAGE_PERCENT : CODEX_UNKNOWN_USAGE_SCORE;
|
|
589
|
-
}
|
|
590
|
-
const values = finite(quota.shortPercent) ? [...knownLong, quota.shortPercent] : knownLong;
|
|
591
|
-
return Math.max(...values);
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
/**
|
|
595
|
-
* A short-only reading that proves the account is blocked NOW.
|
|
596
|
-
*
|
|
597
|
-
* Freshness is not optional. `getAccountQuota` performs no expiry check, partial updates
|
|
598
|
-
* carry a still-open short tuple forward, and disk hydration accepts a persisted reading for
|
|
599
|
-
* hours — so scoring 100 from `shortPercent` alone would keep excluding an account whose
|
|
600
|
-
* five-hour window has since reset. Merge no longer carries an elapsed shortResetAt, but an
|
|
601
|
-
* explicit incoming elapsed tuple is still stored, and a missing reset cannot be aged there.
|
|
602
|
-
* That is #3029 pointed the other way: the issue is that
|
|
603
|
-
* an exhausted account stays selected, and "a recovered account stays excluded" trades one
|
|
604
|
-
* unusable pool for another.
|
|
605
|
-
*
|
|
606
|
-
* A reading with no `shortResetAt` cannot be aged, so it stays unknown. The conservative
|
|
607
|
-
* direction here is the one that keeps an account selectable: a wrongly-selected account
|
|
608
|
-
* fails one request, while a wrongly-excluded one is invisible until someone reads the pool
|
|
609
|
-
* by hand.
|
|
610
|
-
*
|
|
611
|
-
* A missing reset can instead be aged by shortObservedAt (#3425). General updatedAt is not
|
|
612
|
-
* sufficient: credit-only updates preserve the old short tuple but advance that timestamp.
|
|
613
|
-
* Old disk snapshots without short-window provenance remain unknown.
|
|
614
|
-
*/
|
|
615
|
-
function isTerminalShortWindow(
|
|
616
|
-
quota: { shortPercent?: number; shortResetAt?: number; shortObservedAt?: number },
|
|
617
|
-
now: number,
|
|
618
|
-
): boolean {
|
|
619
|
-
if (typeof quota.shortPercent !== "number" || !Number.isFinite(quota.shortPercent)) return false;
|
|
620
|
-
if (quota.shortPercent < CODEX_EXHAUSTED_USAGE_PERCENT) return false;
|
|
621
|
-
const resetAt = quota.shortResetAt;
|
|
622
|
-
if (typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt <= 0) {
|
|
623
|
-
const observedAt = quota.shortObservedAt;
|
|
624
|
-
if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return false;
|
|
625
|
-
const age = now - observedAt;
|
|
626
|
-
return age >= 0 && age <= TERMINAL_SHORT_WINDOW_FRESHNESS_MS;
|
|
627
|
-
}
|
|
628
|
-
// Seconds and milliseconds both reach storage, so the split lives in one place next to the
|
|
629
|
-
// merge that also ages a stored reset instant (`resetAtToMs`, src/codex/quota.ts).
|
|
630
|
-
return resetAtToMs(resetAt) > now;
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
export function classifyCodexUpstreamOutcome(
|
|
634
|
-
outcome: CodexUpstreamOutcome,
|
|
635
|
-
denial?: "workspace" | "entitlement",
|
|
636
|
-
): CodexUpstreamOutcomeClass {
|
|
637
|
-
if (outcome === "connect_neutral") return "neutral";
|
|
638
|
-
if (outcome === "connect_error" || outcome === "timeout") return "transient";
|
|
639
|
-
if (!Number.isFinite(outcome)) return "unknown";
|
|
640
|
-
if (outcome >= 200 && outcome < 300) return "success";
|
|
641
|
-
// Explicit 3xx policy (#914): a redirect response is relayed as-is and is
|
|
642
|
-
// never account or host health evidence — it proves the host is reachable
|
|
643
|
-
// and says nothing about the credential. Relayed as the neutral class so a
|
|
644
|
-
// stray 3xx cannot increment an account's transient streak.
|
|
645
|
-
if (outcome >= 300 && outcome < 400) return "neutral";
|
|
646
|
-
// 401 is always a credential problem. A 403 is only a credential problem when nothing
|
|
647
|
-
// tells us otherwise: a workspace/entitlement denial (#1789) means the credential is valid
|
|
648
|
-
// and the account simply lacks access here, so quarantining it for reauth is wrong advice.
|
|
649
|
-
// Absent denial evidence the historical mapping stands, so the change fails safe.
|
|
650
|
-
if (outcome === 403 && denial !== undefined) return "workspace";
|
|
651
|
-
if (outcome === 401 || outcome === 403) return "credential";
|
|
652
|
-
// 402 Payment Required is treated as quota exhaustion for pool cooldown/failover
|
|
653
|
-
// (same-request alternate retry records this outcome for the depleted account).
|
|
654
|
-
if (outcome === 429 || outcome === 402) return "quota";
|
|
655
|
-
if (outcome >= 400 && outcome < 500) return "caller";
|
|
656
|
-
if (outcome >= 500 && outcome < 600) return "transient";
|
|
657
|
-
return "unknown";
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
function clampCooldownMs(ms: number): number {
|
|
661
|
-
return Math.min(Math.max(ms, 1), CODEX_MAX_QUOTA_COOLDOWN_MS);
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
export function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined {
|
|
665
|
-
const text = value?.trim();
|
|
666
|
-
if (!text) return undefined;
|
|
667
|
-
if (/^\d+(?:\.\d+)?$/.test(text)) {
|
|
668
|
-
const seconds = Number(text);
|
|
669
|
-
if (Number.isFinite(seconds) && seconds > 0) return clampCooldownMs(Math.ceil(seconds * 1000));
|
|
670
|
-
}
|
|
671
|
-
const timestamp = Date.parse(text);
|
|
672
|
-
if (!Number.isFinite(timestamp)) return undefined;
|
|
673
|
-
const delay = timestamp - now;
|
|
674
|
-
return delay > 0 ? clampCooldownMs(delay) : undefined;
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
function resetTimestampMs(value: unknown): number | undefined {
|
|
678
|
-
const numeric = typeof value === "number"
|
|
679
|
-
? value
|
|
680
|
-
: typeof value === "string" && value.trim() !== ""
|
|
681
|
-
? Number(value)
|
|
682
|
-
: undefined;
|
|
683
|
-
if (typeof numeric !== "number" || !Number.isFinite(numeric) || numeric <= 0) return undefined;
|
|
684
|
-
return numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
export function parseResetCooldownMs(resetAt: unknown | unknown[] | undefined, now = Date.now()): number | undefined {
|
|
688
|
-
const values = Array.isArray(resetAt) ? resetAt : [resetAt];
|
|
689
|
-
let best: number | undefined;
|
|
690
|
-
for (const value of values) {
|
|
691
|
-
const timestamp = resetTimestampMs(value);
|
|
692
|
-
if (timestamp === undefined) continue;
|
|
693
|
-
const delay = timestamp - now;
|
|
694
|
-
if (delay <= 0) continue;
|
|
695
|
-
// A far-future reset must not pin the account for the full Retry-After
|
|
696
|
-
// ceiling: quota usually frees up well before the advertised window (#433).
|
|
697
|
-
const clamped = Math.min(clampCooldownMs(delay), CODEX_MAX_RESET_DERIVED_COOLDOWN_MS);
|
|
698
|
-
if (best === undefined || clamped < best) best = clamped;
|
|
699
|
-
}
|
|
700
|
-
return best;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
export function computeQuotaCooldown(meta: CodexUpstreamOutcomeMeta = {}): {
|
|
704
|
-
until: number;
|
|
705
|
-
source: CodexCooldownSource;
|
|
706
|
-
} {
|
|
707
|
-
const now = meta.now ?? Date.now();
|
|
708
|
-
const retryAfterMs = parseRetryAfterMs(meta.retryAfter, now);
|
|
709
|
-
if (retryAfterMs !== undefined) return { until: now + retryAfterMs, source: "retry-after" };
|
|
710
|
-
const resetCooldownMs = parseResetCooldownMs(meta.resetAt, now);
|
|
711
|
-
if (resetCooldownMs !== undefined) return { until: now + resetCooldownMs, source: "reset-derived" };
|
|
712
|
-
return { until: now + CODEX_DEFAULT_QUOTA_COOLDOWN_MS, source: "default" };
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
/**
|
|
716
|
-
* When the pool should stop preferring an account after it refused on quota.
|
|
717
|
-
*
|
|
718
|
-
* The earliest window the refusal actually announced, bounded by {@link CODEX_MAX_QUOTA_AVOID_MS},
|
|
719
|
-
* and never shorter than the cooldown the same refusal produced — a Retry-After directive that
|
|
720
|
-
* outlasts every announcement still governs.
|
|
721
|
-
*/
|
|
722
|
-
function quotaAvoidUntilFor(meta: CodexUpstreamOutcomeMeta, now: number, cooldownUntil: number): number {
|
|
723
|
-
const values = Array.isArray(meta.resetAt) ? meta.resetAt : [meta.resetAt];
|
|
724
|
-
let announced: number | undefined;
|
|
725
|
-
for (const value of values) {
|
|
726
|
-
const timestamp = resetTimestampMs(value);
|
|
727
|
-
if (timestamp === undefined) continue;
|
|
728
|
-
const delay = timestamp - now;
|
|
729
|
-
if (delay <= 0) continue;
|
|
730
|
-
const until = now + Math.min(delay, CODEX_MAX_QUOTA_AVOID_MS);
|
|
731
|
-
if (announced === undefined || until < announced) announced = until;
|
|
732
|
-
}
|
|
733
|
-
return Math.max(cooldownUntil, announced ?? 0);
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
/** Live quota-refusal avoidance for an account, including the lane the request belongs to. */
|
|
737
|
-
function codexQuotaAvoidUntil(
|
|
738
|
-
accountId: string,
|
|
739
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
740
|
-
now: number,
|
|
741
|
-
): number | null {
|
|
742
|
-
const live = (value: number | undefined): number | null =>
|
|
743
|
-
typeof value === "number" && Number.isFinite(value) && value > now ? value : null;
|
|
744
|
-
const account = live(upstreamHealth.get(accountId)?.quotaAvoidUntil);
|
|
745
|
-
const scoped = quotaScope === undefined
|
|
746
|
-
? null
|
|
747
|
-
: live(scopedHealthFor(accountId, quotaScope)?.quotaAvoidUntil);
|
|
748
|
-
if (account === null) return scoped;
|
|
749
|
-
return scoped === null ? account : Math.max(account, scoped);
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
function isCodexQuotaAvoided(
|
|
753
|
-
accountId: string,
|
|
754
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
755
|
-
now: number,
|
|
756
|
-
): boolean {
|
|
757
|
-
return codexQuotaAvoidUntil(accountId, quotaScope, now) !== null;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
export function computeQuotaCooldownUntil(meta: CodexUpstreamOutcomeMeta = {}): number {
|
|
761
|
-
return computeQuotaCooldown(meta).until;
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
/**
|
|
765
|
-
* Grant at most one probe lease per interval for a cooled-down account.
|
|
766
|
-
*
|
|
767
|
-
* A cooled-down account is short-circuited locally, so it never sends traffic and
|
|
768
|
-
* no organic 2xx can prove that upstream quota recovered — the cooldown can only
|
|
769
|
-
* end by expiry or a proxy restart (#433). Releasing a single probe breaks that
|
|
770
|
-
* deadlock. Explicit Retry-After cooldowns are excluded: those are literal retry
|
|
771
|
-
* directives, not window announcements.
|
|
772
|
-
*
|
|
773
|
-
* Returns the lease id, or null when no probe may go out right now.
|
|
774
|
-
*/
|
|
775
|
-
export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): string | null {
|
|
776
|
-
if (!canAcquireCodexQuotaProbeLease(accountId, now)) return null;
|
|
777
|
-
const health = upstreamHealth.get(accountId)!;
|
|
778
|
-
const probeLeaseId = randomUUID();
|
|
779
|
-
upstreamHealth.set(accountId, {
|
|
780
|
-
...health,
|
|
781
|
-
probeLeaseId,
|
|
782
|
-
probeLeaseGeneration: health.cooldownGeneration ?? 0,
|
|
783
|
-
lastProbeAt: now,
|
|
784
|
-
});
|
|
785
|
-
return probeLeaseId;
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */
|
|
789
|
-
export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean {
|
|
790
|
-
return canAcquireQuotaProbeLease(upstreamHealth.get(accountId), now);
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now: number): boolean {
|
|
794
|
-
if (!health) return false;
|
|
795
|
-
const cooldownUntil = health.cooldownUntil;
|
|
796
|
-
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false;
|
|
797
|
-
if (health.cooldownSource === "retry-after") return false;
|
|
798
|
-
if (health.probeLeaseId !== undefined) return false;
|
|
799
|
-
const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil;
|
|
800
|
-
return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS;
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
/**
|
|
804
|
-
* Claim due reset-derived cooldown probes without consulting account selection.
|
|
805
|
-
* Added Pool credentials only; owned main usage recovery is handled separately.
|
|
806
|
-
*/
|
|
807
|
-
export function claimDueCodexQuotaRecoveryProbes(
|
|
808
|
-
config: OcxConfig,
|
|
809
|
-
limit: number,
|
|
810
|
-
now = Date.now(),
|
|
811
|
-
): CodexQuotaRecoveryProbeClaim[] {
|
|
812
|
-
const boundedLimit = Math.max(0, Math.floor(limit));
|
|
813
|
-
if (boundedLimit === 0) return [];
|
|
814
|
-
const candidates: Array<{
|
|
815
|
-
accountId: string;
|
|
816
|
-
scope?: CodexQuotaScope;
|
|
817
|
-
health: CodexUpstreamHealth;
|
|
818
|
-
credentialGeneration: number;
|
|
819
|
-
credentialReplacedAt?: number;
|
|
820
|
-
order: number;
|
|
821
|
-
}> = [];
|
|
822
|
-
for (const [order, account] of (config.codexAccounts ?? []).entries()) {
|
|
823
|
-
if (!isSelectableCodexPoolAccount(account)
|
|
824
|
-
|| isCodexAccountPaused(config, account.id)
|
|
825
|
-
|| isAccountNeedsReauth(account.id)) continue;
|
|
826
|
-
const record = readCodexAccountRecord(account.id);
|
|
827
|
-
if (!record?.credential || record.deletedAt != null) continue;
|
|
828
|
-
const due = [
|
|
829
|
-
{ scope: undefined, health: upstreamHealth.get(account.id) },
|
|
830
|
-
...[...(quotaScopedHealth.get(account.id) ?? [])].map(([scope, health]) => ({ scope, health })),
|
|
831
|
-
].filter((entry): entry is { scope?: CodexQuotaScope; health: CodexUpstreamHealth } =>
|
|
832
|
-
// Generic WHAM evidence can recover only ordinary quota, never Reserve.
|
|
833
|
-
// Do not spend this account's one claim per pass on an independent scope and
|
|
834
|
-
// delay the shared scope that the response can actually recover.
|
|
835
|
-
(entry.scope === undefined || entry.scope === "shared")
|
|
836
|
-
&& entry.health?.cooldownSource === "reset-derived"
|
|
837
|
-
&& canAcquireQuotaProbeLease(entry.health, now))
|
|
838
|
-
.sort((a, b) =>
|
|
839
|
-
(a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
|
|
840
|
-
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0));
|
|
841
|
-
const candidate = due[0];
|
|
842
|
-
if (candidate) candidates.push({
|
|
843
|
-
accountId: account.id,
|
|
844
|
-
...(candidate.scope ? { scope: candidate.scope } : {}),
|
|
845
|
-
health: candidate.health,
|
|
846
|
-
credentialGeneration: record.generation,
|
|
847
|
-
...(record.replacedAt !== undefined ? { credentialReplacedAt: record.replacedAt } : {}),
|
|
848
|
-
order,
|
|
849
|
-
});
|
|
850
|
-
}
|
|
851
|
-
candidates.sort((a, b) => {
|
|
852
|
-
const age = (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
|
|
853
|
-
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0);
|
|
854
|
-
return age || a.order - b.order;
|
|
855
|
-
});
|
|
856
|
-
return candidates.slice(0, boundedLimit).map(candidate => {
|
|
857
|
-
const leaseId = randomUUID();
|
|
858
|
-
const next = {
|
|
859
|
-
...candidate.health,
|
|
860
|
-
probeLeaseId: leaseId,
|
|
861
|
-
probeLeaseGeneration: candidate.health.cooldownGeneration ?? 0,
|
|
862
|
-
lastProbeAt: now,
|
|
863
|
-
};
|
|
864
|
-
if (candidate.scope) setScopedHealth(candidate.accountId, candidate.scope, next);
|
|
865
|
-
else upstreamHealth.set(candidate.accountId, next);
|
|
866
|
-
return {
|
|
867
|
-
accountId: candidate.accountId,
|
|
868
|
-
...(candidate.scope ? { scope: candidate.scope } : {}),
|
|
869
|
-
leaseId,
|
|
870
|
-
cooldownGeneration: candidate.health.cooldownGeneration ?? 0,
|
|
871
|
-
credentialGeneration: candidate.credentialGeneration,
|
|
872
|
-
...(candidate.credentialReplacedAt !== undefined
|
|
873
|
-
? { credentialReplacedAt: candidate.credentialReplacedAt }
|
|
874
|
-
: {}),
|
|
875
|
-
};
|
|
876
|
-
});
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
type CooldownRecoveryLease = Pick<CodexQuotaRecoveryProbeClaim,
|
|
880
|
-
"accountId" | "scope" | "leaseId" | "cooldownGeneration">;
|
|
881
|
-
|
|
882
|
-
export type ManualResetCooldownClaim =
|
|
883
|
-
| { kind: "pool"; probe: CodexQuotaRecoveryProbeClaim }
|
|
884
|
-
| { kind: "main"; probe: CooldownRecoveryLease };
|
|
885
|
-
|
|
886
|
-
function manualResetAccountEligible(config: OcxConfig, accountId: string): boolean {
|
|
887
|
-
return !isCodexAccountPaused(config, accountId) && !isAccountNeedsReauth(accountId)
|
|
888
|
-
&& (accountId === MAIN_CODEX_ACCOUNT_ID
|
|
889
|
-
|| (config.codexAccounts ?? []).some(account => account.id === accountId && isSelectableCodexPoolAccount(account)));
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
/** Explicit reset bypasses probe pacing, never another owner's lease or quota scope. */
|
|
893
|
-
export function claimManualResetCooldowns(
|
|
894
|
-
config: OcxConfig,
|
|
895
|
-
accountId: string,
|
|
896
|
-
now = Date.now(),
|
|
897
|
-
expectedPoolGeneration?: number,
|
|
898
|
-
): ManualResetCooldownClaim[] {
|
|
899
|
-
if (!manualResetAccountEligible(config, accountId)) return [];
|
|
900
|
-
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
|
|
901
|
-
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return [];
|
|
902
|
-
if (record && expectedPoolGeneration !== undefined && record.generation !== expectedPoolGeneration) return [];
|
|
903
|
-
const claims: ManualResetCooldownClaim[] = [];
|
|
904
|
-
for (const scope of [undefined, "shared"] as const) {
|
|
905
|
-
const health = scope ? scopedHealthFor(accountId, scope) : upstreamHealth.get(accountId);
|
|
906
|
-
if (!health || health.cooldownSource !== "reset-derived" || health.probeLeaseId !== undefined
|
|
907
|
-
|| !Number.isFinite(health.cooldownUntil) || !(health.cooldownUntil! > now)) continue;
|
|
908
|
-
const leaseId = randomUUID();
|
|
909
|
-
const cooldownGeneration = health.cooldownGeneration ?? 0;
|
|
910
|
-
const next = { ...health, probeLeaseId: leaseId, probeLeaseGeneration: cooldownGeneration, lastProbeAt: now };
|
|
911
|
-
if (scope) setScopedHealth(accountId, scope, next);
|
|
912
|
-
else upstreamHealth.set(accountId, next);
|
|
913
|
-
const probe = { accountId, scope, leaseId, cooldownGeneration };
|
|
914
|
-
claims.push(record ? { kind: "pool", probe: {
|
|
915
|
-
...probe, credentialGeneration: record.generation, credentialReplacedAt: record.replacedAt,
|
|
916
|
-
} } : { kind: "main", probe });
|
|
917
|
-
}
|
|
918
|
-
return claims;
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
export type ManualResetRefreshLineage = Readonly<{
|
|
922
|
-
fromGeneration: number;
|
|
923
|
-
toGeneration: number;
|
|
924
|
-
provenance: CodexRefreshProvenance;
|
|
925
|
-
}>;
|
|
926
|
-
|
|
927
|
-
type ManualResetQuotaProof = CodexQuotaRecoveryProbeProof & {
|
|
928
|
-
refreshLineage?: ManualResetRefreshLineage;
|
|
929
|
-
};
|
|
930
|
-
|
|
931
|
-
/** Main proof is checked by the already-owned auth operation, never by a Pool record. */
|
|
932
|
-
export function settleManualResetCooldown(
|
|
933
|
-
config: OcxConfig,
|
|
934
|
-
claim: ManualResetCooldownClaim,
|
|
935
|
-
recovered: boolean,
|
|
936
|
-
proof: ManualResetQuotaProof = {},
|
|
937
|
-
now = Date.now(),
|
|
938
|
-
): boolean {
|
|
939
|
-
if (!recovered) return settleCooldownRecoveryLease(claim.probe, false, now);
|
|
940
|
-
const eligible = manualResetAccountEligible(config, claim.probe.accountId);
|
|
941
|
-
if (claim.kind === "main") return settleCooldownRecoveryLease(claim.probe, eligible, now);
|
|
942
|
-
const lineage = proof.refreshLineage;
|
|
943
|
-
// Equal wall-clock replacement stamps do not establish ancestry. Manual +1
|
|
944
|
-
// recovery additionally needs the actual forced-refresh result for this edge.
|
|
945
|
-
const ownedGeneration = proof.credentialGeneration === claim.probe.credentialGeneration
|
|
946
|
-
|| (proof.credentialGeneration === claim.probe.credentialGeneration + 1
|
|
947
|
-
&& lineage?.fromGeneration === claim.probe.credentialGeneration
|
|
948
|
-
&& lineage.toGeneration === proof.credentialGeneration
|
|
949
|
-
&& (lineage.provenance === "self-refresh" || lineage.provenance === "joined-lineage"));
|
|
950
|
-
return settleCodexQuotaRecoveryProbe(claim.probe, eligible && ownedGeneration, proof, now);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
/** Settle one background recovery claim without mutating account-wide outcome state. */
|
|
954
|
-
export function settleCodexQuotaRecoveryProbe(
|
|
955
|
-
claim: CodexQuotaRecoveryProbeClaim,
|
|
956
|
-
recovered: boolean,
|
|
957
|
-
proof: CodexQuotaRecoveryProbeProof,
|
|
958
|
-
now = Date.now(),
|
|
959
|
-
): boolean {
|
|
960
|
-
const health = claim.scope
|
|
961
|
-
? scopedHealthFor(claim.accountId, claim.scope)
|
|
962
|
-
: upstreamHealth.get(claim.accountId);
|
|
963
|
-
if (!health || health.probeLeaseId !== claim.leaseId) return false;
|
|
964
|
-
const currentRecord = readCodexAccountRecord(claim.accountId);
|
|
965
|
-
const proofGeneration = proof.credentialGeneration;
|
|
966
|
-
// A probe-owned token refresh (getValidCodexToken) advances the credential generation by
|
|
967
|
-
// exactly one while preserving `replacedAt`; an external credential replacement bumps the
|
|
968
|
-
// generation too but stamps a fresh `replacedAt`. Accept the +1 transition only when the
|
|
969
|
-
// claim-time lineage is intact AND the generation the fresh quota was proven under is live.
|
|
970
|
-
const generationFenced = proofGeneration !== undefined
|
|
971
|
-
&& (proofGeneration === claim.credentialGeneration
|
|
972
|
-
? isCodexAccountGenerationLive(claim.accountId, proofGeneration)
|
|
973
|
-
: proofGeneration === claim.credentialGeneration + 1
|
|
974
|
-
&& currentRecord?.replacedAt === claim.credentialReplacedAt
|
|
975
|
-
&& isCodexAccountGenerationLive(claim.accountId, proofGeneration));
|
|
976
|
-
return settleCooldownRecoveryLease(claim, recovered && generationFenced, now);
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
function settleCooldownRecoveryLease(claim: CooldownRecoveryLease, recovered: boolean, now: number): boolean {
|
|
980
|
-
const health = claim.scope ? scopedHealthFor(claim.accountId, claim.scope) : upstreamHealth.get(claim.accountId);
|
|
981
|
-
if (!health || health.probeLeaseId !== claim.leaseId) return false;
|
|
982
|
-
const fenced = (claim.scope === undefined || claim.scope === "shared")
|
|
983
|
-
&& health.cooldownSource === "reset-derived"
|
|
984
|
-
&& (health.cooldownGeneration ?? 0) === claim.cooldownGeneration
|
|
985
|
-
&& (health.probeLeaseGeneration ?? 0) === claim.cooldownGeneration;
|
|
986
|
-
if (!recovered || !fenced) {
|
|
987
|
-
const released = withProbeLeaseReleased(health, now);
|
|
988
|
-
if (claim.scope) setScopedHealth(claim.accountId, claim.scope, released);
|
|
989
|
-
else upstreamHealth.set(claim.accountId, released);
|
|
990
|
-
return false;
|
|
991
|
-
}
|
|
992
|
-
if (claim.scope) {
|
|
993
|
-
deleteScopedHealth(claim.accountId, claim.scope);
|
|
994
|
-
} else {
|
|
995
|
-
const {
|
|
996
|
-
cooldownUntil: _until,
|
|
997
|
-
cooldownSince: _since,
|
|
998
|
-
cooldownSource: _source,
|
|
999
|
-
probeLeaseId: _leaseId,
|
|
1000
|
-
probeLeaseGeneration: _leaseGeneration,
|
|
1001
|
-
// "The quota window moved" is a statement about the whole refusal, so the avoidance it
|
|
1002
|
-
// announced goes with the block it produced. Leaving it would make this escape hatch stop
|
|
1003
|
-
// escaping: the account would still be passed over by every selection it is meant to win.
|
|
1004
|
-
quotaAvoidUntil: _avoid,
|
|
1005
|
-
...rest
|
|
1006
|
-
} = health;
|
|
1007
|
-
upstreamHealth.set(claim.accountId, {
|
|
1008
|
-
...rest,
|
|
1009
|
-
cooldownGeneration: claim.cooldownGeneration + 1,
|
|
1010
|
-
lastProbeAt: now,
|
|
1011
|
-
});
|
|
1012
|
-
}
|
|
1013
|
-
return true;
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
/** Acquire the recovery probe for one confirmed model-specific quota group. */
|
|
1017
|
-
export function tryAcquireCodexQuotaScopeProbeLease(
|
|
1018
|
-
accountId: string,
|
|
1019
|
-
scope: CodexQuotaScope,
|
|
1020
|
-
now = Date.now(),
|
|
1021
|
-
): string | null {
|
|
1022
|
-
const health = scopedHealthFor(accountId, scope);
|
|
1023
|
-
if (!canAcquireQuotaProbeLease(health, now)) return null;
|
|
1024
|
-
const probeLeaseId = randomUUID();
|
|
1025
|
-
setScopedHealth(accountId, scope, {
|
|
1026
|
-
...health!,
|
|
1027
|
-
probeLeaseId,
|
|
1028
|
-
probeLeaseGeneration: health!.cooldownGeneration ?? 0,
|
|
1029
|
-
lastProbeAt: now,
|
|
1030
|
-
});
|
|
1031
|
-
return probeLeaseId;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
/** Side-effect-free check for a confirmed model-specific quota probe. */
|
|
1035
|
-
export function canAcquireCodexQuotaScopeProbeLease(
|
|
1036
|
-
accountId: string,
|
|
1037
|
-
scope: CodexQuotaScope,
|
|
1038
|
-
now = Date.now(),
|
|
1039
|
-
): boolean {
|
|
1040
|
-
return canAcquireQuotaProbeLease(scopedHealthFor(accountId, scope), now);
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
/**
|
|
1044
|
-
* Hand a probe lease back without recording an upstream outcome. Used by paths
|
|
1045
|
-
* that take a lease and then fail before any request reaches upstream.
|
|
1046
|
-
*/
|
|
1047
|
-
export function releaseCodexQuotaProbeLease(accountId: string, leaseId: string, now = Date.now()): void {
|
|
1048
|
-
const health = upstreamHealth.get(accountId);
|
|
1049
|
-
if (!health || health.probeLeaseId !== leaseId) return;
|
|
1050
|
-
upstreamHealth.set(accountId, withProbeLeaseReleased(health, now));
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
/** Release a model-specific quota probe when the request never reaches upstream. */
|
|
1054
|
-
export function releaseCodexQuotaScopeProbeLease(
|
|
1055
|
-
accountId: string,
|
|
1056
|
-
scope: CodexQuotaScope,
|
|
1057
|
-
leaseId: string,
|
|
1058
|
-
now = Date.now(),
|
|
1059
|
-
): void {
|
|
1060
|
-
const health = scopedHealthFor(accountId, scope);
|
|
1061
|
-
if (!health || health.probeLeaseId !== leaseId) return;
|
|
1062
|
-
setScopedHealth(accountId, scope, withProbeLeaseReleased(health, now));
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
/**
|
|
1066
|
-
* True when this outcome belongs to the account's in-flight probe. The
|
|
1067
|
-
* undefined-id guard matters: without it an outcome carrying no lease would match
|
|
1068
|
-
* an account holding no lease and be mistaken for the probe owner.
|
|
1069
|
-
*/
|
|
1070
|
-
function ownsProbeLease(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
|
|
1071
|
-
return meta.probeLeaseId !== undefined && meta.probeLeaseId === health?.probeLeaseId;
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
/**
|
|
1075
|
-
* True when the owning probe may still clear the cooldown. A later 429 bumps the
|
|
1076
|
-
* generation, so a probe that started under an older cooldown must not erase the
|
|
1077
|
-
* newer restriction (which may carry an explicit Retry-After).
|
|
1078
|
-
*/
|
|
1079
|
-
function probeMayClearCooldown(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
|
|
1080
|
-
return ownsProbeLease(health, meta)
|
|
1081
|
-
&& (health!.probeLeaseGeneration ?? 0) === (health!.cooldownGeneration ?? 0);
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
/** Strip the in-flight lease while preserving every hard-cooldown field. */
|
|
1085
|
-
function withProbeLeaseReleased(health: CodexUpstreamHealth, now: number): CodexUpstreamHealth {
|
|
1086
|
-
const { probeLeaseId: _id, probeLeaseGeneration: _gen, ...rest } = health;
|
|
1087
|
-
return { ...rest, lastProbeAt: now };
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
/**
|
|
1091
|
-
* Hard-cooldown bookkeeping that ordinary success/transient transitions rebuild
|
|
1092
|
-
* their health object from. Dropping these would let one late unrelated response
|
|
1093
|
-
* erase a Retry-After source, a cooldown generation, or someone else's live probe.
|
|
1094
|
-
*/
|
|
1095
|
-
function preservedCooldownFields(health: CodexUpstreamHealth | undefined): Partial<CodexUpstreamHealth> {
|
|
1096
|
-
if (!health) return {};
|
|
1097
|
-
// `credentialFailureGeneration` is provenance for ONE credential failure, so it must not survive
|
|
1098
|
-
// into a later transient or quota entry — otherwise that entry inherits the tag and gets spent
|
|
1099
|
-
// when the old credential dies, deleting evidence that was never about it (#2892 gap 4 review).
|
|
1100
|
-
const {
|
|
1101
|
-
consecutiveFailures: _f, consecutiveSuccesses: _s, lastFailureStatus: _st, lastFailureAt: _at,
|
|
1102
|
-
softAvoidUntil: _sa, credentialFailureGeneration: _cg, ...cooldownFields
|
|
1103
|
-
} = health;
|
|
1104
|
-
return cooldownFields;
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
/** Manual selection resets transient routing evidence without bypassing a real 429 cooldown. */
|
|
1108
|
-
export function resetCodexRoutingForManualSelection(accountId: string): void {
|
|
1109
|
-
clearThreadAccountMap();
|
|
1110
|
-
// Manual selection is the operator source of truth — drop any automatic runtime cursor.
|
|
1111
|
-
runtimeActiveCodexAccountId = undefined;
|
|
1112
|
-
// Record the pick as an unspent one-shot on the SHARED scope only. An independent scope
|
|
1113
|
-
// gets no entry on purpose: every write site the guard protects is already skipped for
|
|
1114
|
-
// independent scopes, so an entry there would be state nothing reads — and state nothing
|
|
1115
|
-
// reads is what the next reader mistakes for a rule.
|
|
1116
|
-
//
|
|
1117
|
-
// Seeding happens ONLY here. A pool-driven promote must never create or move a preference,
|
|
1118
|
-
// or the pool would manufacture an operator intent nobody expressed.
|
|
1119
|
-
manualPreference.set(POOL_KEY_CODEX, accountId);
|
|
1120
|
-
// Seed the RR ring so the next unbound new session honors the manually selected account
|
|
1121
|
-
// under round-robin (affinity-cleared threads / null threadId). Fill-first already follows
|
|
1122
|
-
// config.activeCodexAccountId, which the caller persists before invoking this.
|
|
1123
|
-
seedPoolRotationAccount(POOL_KEY_CODEX, accountId);
|
|
1124
|
-
for (const scope of new Set(Object.values(NATIVE_MODEL_QUOTA_SCOPES))) {
|
|
1125
|
-
if (isIndependentCodexQuotaScope(scope)) {
|
|
1126
|
-
seedPoolRotationAccount(codexPoolKeyForScope(scope), accountId);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
// Quota avoidance is a preference, like the soft avoid dropped above, and an operator naming
|
|
1130
|
-
// this account has overruled it. The hard cooldown is the part that survives.
|
|
1131
|
-
const overrule = (health: CodexUpstreamHealth) => {
|
|
1132
|
-
const { quotaAvoidUntil: _avoid, ...retained } = preservedCooldownFields(health);
|
|
1133
|
-
return retained;
|
|
1134
|
-
};
|
|
1135
|
-
const current = upstreamHealth.get(accountId);
|
|
1136
|
-
if (current) {
|
|
1137
|
-
const retained = overrule(current);
|
|
1138
|
-
if (Object.keys(retained).length === 0) upstreamHealth.delete(accountId);
|
|
1139
|
-
else upstreamHealth.set(accountId, { consecutiveFailures: 0, ...retained });
|
|
1140
|
-
}
|
|
1141
|
-
// A reset-derived refusal records its avoidance on the SCOPED map and returns before the
|
|
1142
|
-
// account-wide entry is written, so naming the account has to reach that map too. Stopping
|
|
1143
|
-
// at `upstreamHealth` — and returning early when it holds nothing — overruled nothing in
|
|
1144
|
-
// the case that produces the avoidance this function exists to overrule.
|
|
1145
|
-
for (const [scope, health] of [...(quotaScopedHealth.get(accountId) ?? [])]) {
|
|
1146
|
-
const retained = overrule(health);
|
|
1147
|
-
if (Object.keys(retained).length === 0) deleteScopedHealth(accountId, scope);
|
|
1148
|
-
else setScopedHealth(accountId, scope, { consecutiveFailures: 0, ...retained });
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
export function getCodexAccountCooldownUntil(accountId: string, now = Date.now()): number | null {
|
|
1153
|
-
const cooldownUntil = upstreamHealth.get(accountId)?.cooldownUntil;
|
|
1154
|
-
return typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now ? cooldownUntil : null;
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
/** Read-only cooldown snapshot for shared OAuth health projection (no write side effects). */
|
|
1158
|
-
export function getCodexAccountHealthSnapshot(accountId: string, now = Date.now()): {
|
|
1159
|
-
cooldownUntil?: number;
|
|
1160
|
-
cooldownSource?: CodexCooldownSource;
|
|
1161
|
-
} | null {
|
|
1162
|
-
const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
|
|
1163
|
-
if (cooldownUntil === null) return null;
|
|
1164
|
-
const source = upstreamHealth.get(accountId)?.cooldownSource;
|
|
1165
|
-
return {
|
|
1166
|
-
cooldownUntil,
|
|
1167
|
-
...(source ? { cooldownSource: source } : {}),
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
/**
|
|
1172
|
-
* Read the cooldown relevant to a routed native model. Account-wide cooldowns
|
|
1173
|
-
* (Retry-After/default) always win; reset-derived scoped state applies only to
|
|
1174
|
-
* its confirmed quota group.
|
|
1175
|
-
*/
|
|
1176
|
-
export function getCodexQuotaHealthSnapshot(
|
|
1177
|
-
accountId: string,
|
|
1178
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
1179
|
-
now = Date.now(),
|
|
1180
|
-
): {
|
|
1181
|
-
cooldownUntil?: number;
|
|
1182
|
-
cooldownSource?: CodexCooldownSource;
|
|
1183
|
-
quotaScope?: CodexQuotaScope;
|
|
1184
|
-
} | null {
|
|
1185
|
-
const account = getCodexAccountHealthSnapshot(accountId, now);
|
|
1186
|
-
if (account) return account;
|
|
1187
|
-
if (!quotaScope) return null;
|
|
1188
|
-
const scoped = scopedHealthFor(accountId, quotaScope);
|
|
1189
|
-
const cooldownUntil = scoped?.cooldownUntil;
|
|
1190
|
-
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return null;
|
|
1191
|
-
return {
|
|
1192
|
-
cooldownUntil,
|
|
1193
|
-
...(scoped?.cooldownSource ? { cooldownSource: scoped.cooldownSource } : {}),
|
|
1194
|
-
quotaScope,
|
|
1195
|
-
};
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
export function isCodexAccountInCooldown(accountId: string, now = Date.now()): boolean {
|
|
1199
|
-
return getCodexAccountCooldownUntil(accountId, now) !== null;
|
|
1200
|
-
}
|
|
1201
|
-
|
|
1202
|
-
/**
|
|
1203
|
-
* Manually lift a hard quota cooldown without touching failure history.
|
|
1204
|
-
*
|
|
1205
|
-
* Injected Codex routing makes this proxy the ONLY model path for Codex Desktop, so a
|
|
1206
|
-
* cooldown that outlives the real upstream limit reads to the user as "the whole app is
|
|
1207
|
-
* broken" with no escape but editing config.toml. This is that escape hatch.
|
|
1208
|
-
*
|
|
1209
|
-
* Deliberately narrow:
|
|
1210
|
-
* - Failure counters and softAvoid survive. Clearing a cooldown says "the quota window
|
|
1211
|
-
* moved", not "this account is healthy"; failover must keep its knowledge.
|
|
1212
|
-
* - Dropping `probeLeaseId` is what stops a stale in-flight probe from later "proving"
|
|
1213
|
-
* recovery against a NEWER cooldown: {@link ownsProbeLease} needs the id to match.
|
|
1214
|
-
* `cooldownGeneration` is preserved and bumped as redundancy only — a fresh 429 already
|
|
1215
|
-
* bumps it in {@link recordCodexUpstreamOutcome}, so the bump here is not load-bearing
|
|
1216
|
-
* today and is kept so the invariant survives a future change that retains the lease.
|
|
1217
|
-
*
|
|
1218
|
-
* Returns false when the account carried neither a live cooldown nor a live avoidance window.
|
|
1219
|
-
* The window outlives the cooldown by design — the cooldown caps at fifteen minutes and the
|
|
1220
|
-
* window runs up to six hours — so the moment an operator actually reaches for this escape
|
|
1221
|
-
* hatch is usually after the cooldown lapsed and only the window is still keeping the account
|
|
1222
|
-
* out of rotation. Refusing to look at the window then would leave the hatch shut in the one
|
|
1223
|
-
* case it exists for.
|
|
1224
|
-
*/
|
|
1225
|
-
export function clearCodexAccountCooldown(accountId: string, now = Date.now()): boolean {
|
|
1226
|
-
const clear = (health: CodexUpstreamHealth): CodexUpstreamHealth | null => {
|
|
1227
|
-
const cooldownUntil = health.cooldownUntil;
|
|
1228
|
-
const liveCooldown = typeof cooldownUntil === "number" && Number.isFinite(cooldownUntil) && cooldownUntil > now;
|
|
1229
|
-
const avoidUntil = health.quotaAvoidUntil;
|
|
1230
|
-
const liveAvoidance = typeof avoidUntil === "number" && Number.isFinite(avoidUntil) && avoidUntil > now;
|
|
1231
|
-
if (!liveCooldown && !liveAvoidance) return null;
|
|
1232
|
-
const {
|
|
1233
|
-
cooldownUntil: _until,
|
|
1234
|
-
cooldownSince: _since,
|
|
1235
|
-
cooldownSource: _source,
|
|
1236
|
-
probeLeaseId: _leaseId,
|
|
1237
|
-
probeLeaseGeneration: _leaseGeneration,
|
|
1238
|
-
// Same reasoning as the probe recovery above: "the quota window moved" is a statement
|
|
1239
|
-
// about the whole refusal, so the avoidance it announced goes with the block it
|
|
1240
|
-
// produced. Keeping it would leave this escape hatch not escaping, because selection
|
|
1241
|
-
// would still pass over the account for as long as the announced window runs.
|
|
1242
|
-
quotaAvoidUntil: _avoid,
|
|
1243
|
-
...rest
|
|
1244
|
-
} = health;
|
|
1245
|
-
return {
|
|
1246
|
-
...rest,
|
|
1247
|
-
cooldownGeneration: (health.cooldownGeneration ?? 0) + 1,
|
|
1248
|
-
lastProbeAt: now,
|
|
1249
|
-
};
|
|
1250
|
-
};
|
|
1251
|
-
|
|
1252
|
-
let cleared = false;
|
|
1253
|
-
const accountHealth = upstreamHealth.get(accountId);
|
|
1254
|
-
if (accountHealth) {
|
|
1255
|
-
const next = clear(accountHealth);
|
|
1256
|
-
if (next) {
|
|
1257
|
-
upstreamHealth.set(accountId, next);
|
|
1258
|
-
cleared = true;
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
for (const [scope, health] of quotaScopedHealth.get(accountId) ?? []) {
|
|
1262
|
-
const next = clear(health);
|
|
1263
|
-
if (next) {
|
|
1264
|
-
setScopedHealth(accountId, scope, next);
|
|
1265
|
-
cleared = true;
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
return cleared;
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
export function getCodexAccountSoftAvoidUntil(accountId: string, now = Date.now()): number | null {
|
|
1272
|
-
const softAvoidUntil = upstreamHealth.get(accountId)?.softAvoidUntil;
|
|
1273
|
-
return typeof softAvoidUntil === "number" && Number.isFinite(softAvoidUntil) && softAvoidUntil > now
|
|
1274
|
-
? softAvoidUntil
|
|
1275
|
-
: null;
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
export function isCodexAccountSoftAvoided(accountId: string, now = Date.now()): boolean {
|
|
1279
|
-
return getCodexAccountSoftAvoidUntil(accountId, now) !== null;
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
/**
|
|
1283
|
-
* Plan keys the operator excluded from automatic rotation. Absent or empty means no policy, so an
|
|
1284
|
-
* existing install rotates exactly as before. Compared with `codexPlanKey` because the stored plan
|
|
1285
|
-
* is an unrestricted provider string whose casing this repository does not control.
|
|
1286
|
-
*/
|
|
1287
|
-
function excludedCodexPoolPlanKeys(config: OcxConfig): ReadonlySet<string> | undefined {
|
|
1288
|
-
const configured = config.codexPool?.excludedPlans;
|
|
1289
|
-
if (!configured?.length) return undefined;
|
|
1290
|
-
const keys = configured
|
|
1291
|
-
.map(plan => codexPlanKey(plan))
|
|
1292
|
-
.filter((key): key is string => key !== undefined);
|
|
1293
|
-
return keys.length > 0 ? new Set(keys) : undefined;
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
/**
|
|
1297
|
-
* Whether the operator's plan policy removes this account from automatic selection.
|
|
1298
|
-
*
|
|
1299
|
-
* Modelled on pause rather than usability: an excluded account keeps its credential, quota history,
|
|
1300
|
-
* and affinity, stays visible on the account surface, and is still reachable by explicit account
|
|
1301
|
-
* selection. Only automatic rotation skips it, which is the distinction #4211 asked for.
|
|
1302
|
-
*
|
|
1303
|
-
* It is checked in the same two places pause is checked, and that is not redundancy. The eligible
|
|
1304
|
-
* list is consulted only when routing picks a NEW account; an already-active or already-affined
|
|
1305
|
-
* account is served straight from {@link isCodexAccountSelectable}. A lapsed subscription leaves
|
|
1306
|
-
* behind exactly that account, so a policy that filtered only the eligible list would miss the case
|
|
1307
|
-
* it exists for.
|
|
1308
|
-
*
|
|
1309
|
-
* `__main__` is exempt. {@link getPoolAccountPlanForSelection} withholds the main plan during a
|
|
1310
|
-
* selection-only drain so routing never reads the fenced native credential for it, so a rule that
|
|
1311
|
-
* covered main would disagree with itself between drain and ordinary routing.
|
|
1312
|
-
*/
|
|
1313
|
-
export function isCodexAccountPlanExcluded(
|
|
1314
|
-
config: OcxConfig,
|
|
1315
|
-
accountId: string,
|
|
1316
|
-
precomputed?: ReadonlySet<string>,
|
|
1317
|
-
): boolean {
|
|
1318
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return false;
|
|
1319
|
-
// Callers that test a whole list pass the set once rather than rebuilding it per row.
|
|
1320
|
-
const excluded = precomputed ?? excludedCodexPoolPlanKeys(config);
|
|
1321
|
-
if (!excluded) return false;
|
|
1322
|
-
const plan = codexPlanKey(getPoolAccountPlan(config, accountId));
|
|
1323
|
-
return plan !== undefined && excluded.has(plan);
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
function isCodexAccountSelectable(
|
|
1327
|
-
config: OcxConfig,
|
|
1328
|
-
accountId: string,
|
|
1329
|
-
now: number,
|
|
1330
|
-
quotaScope?: CodexQuotaScope,
|
|
1331
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1332
|
-
): boolean {
|
|
1333
|
-
return !isCodexAccountPaused(config, accountId)
|
|
1334
|
-
&& !isCodexAccountPlanExcluded(config, accountId)
|
|
1335
|
-
&& getCodexQuotaHealthSnapshot(accountId, quotaScope, now) === null
|
|
1336
|
-
&& !isCodexQuotaAvoided(accountId, quotaScope, now)
|
|
1337
|
-
&& !isCodexAccountSoftAvoided(accountId, now)
|
|
1338
|
-
&& isCodexAccountUsable(config, accountId, selectionOptions);
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
|
-
/**
|
|
1342
|
-
* Which guard in {@link isCodexAccountSelectable} refused this account, if any.
|
|
1343
|
-
*
|
|
1344
|
-
* Deliberately the same predicates in the same order as that function, because the point is to
|
|
1345
|
-
* REPORT the guard that actually fired rather than to re-derive a plausible-looking cause. An
|
|
1346
|
-
* earlier version of the release reason checked only a subset and let a paused, plan-excluded,
|
|
1347
|
-
* cooled-down or quota-avoided release fall through to a quota fallback, which named something
|
|
1348
|
-
* routing never used -- a diagnostic that is confidently wrong in exactly the cases an operator
|
|
1349
|
-
* would consult it for (#4598).
|
|
1350
|
-
*/
|
|
1351
|
-
function codexAccountBlockReason(
|
|
1352
|
-
config: OcxConfig,
|
|
1353
|
-
accountId: string,
|
|
1354
|
-
now: number,
|
|
1355
|
-
quotaScope?: CodexQuotaScope,
|
|
1356
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1357
|
-
): CodexAffinityReason | undefined {
|
|
1358
|
-
if (isCodexAccountPaused(config, accountId)) return "paused";
|
|
1359
|
-
if (isCodexAccountPlanExcluded(config, accountId)) return "plan_excluded";
|
|
1360
|
-
if (getCodexQuotaHealthSnapshot(accountId, quotaScope, now) !== null) return "cooldown";
|
|
1361
|
-
if (isCodexQuotaAvoided(accountId, quotaScope, now)) return "quota_avoided";
|
|
1362
|
-
if (isCodexAccountSoftAvoided(accountId, now)) return "transient";
|
|
1363
|
-
if (!isCodexAccountUsable(config, accountId, selectionOptions)) return "unusable";
|
|
1364
|
-
return undefined;
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
function threadAffinityScope(quotaScope?: CodexQuotaScope): BaseThreadAffinityScope {
|
|
1368
|
-
return quotaScope ?? LEGACY_THREAD_AFFINITY_SCOPE;
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
function admissibleAffinityComponent(value: string): boolean {
|
|
1372
|
-
return retainedUtf8Bytes(value) <= MAX_AFFINITY_COMPONENT_BYTES;
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
function modelDetourAffinityScope(
|
|
1376
|
-
modelId: string | undefined,
|
|
1377
|
-
quotaScope?: CodexQuotaScope,
|
|
1378
|
-
): ModelDetourAffinityScope | undefined {
|
|
1379
|
-
const canonicalModelId = modelId?.trim().toLowerCase();
|
|
1380
|
-
if (!canonicalModelId || !admissibleAffinityComponent(canonicalModelId)) return undefined;
|
|
1381
|
-
return `model-detour:${threadAffinityScope(quotaScope)}:${canonicalModelId}`;
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
function getThreadAffinityForScope(
|
|
1385
|
-
threadId: string,
|
|
1386
|
-
scope: ThreadAffinityScope,
|
|
1387
|
-
): ThreadAffinityEntry | undefined {
|
|
1388
|
-
if (!admissibleAffinityComponent(threadId)) return undefined;
|
|
1389
|
-
return threadAccountMap.get(threadId)?.get(scope);
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
function getThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): ThreadAffinityEntry | undefined {
|
|
1393
|
-
return getThreadAffinityForScope(threadId, threadAffinityScope(quotaScope));
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
function getModelDetourAffinity(
|
|
1397
|
-
threadId: string,
|
|
1398
|
-
modelId: string | undefined,
|
|
1399
|
-
quotaScope?: CodexQuotaScope,
|
|
1400
|
-
): ThreadAffinityEntry | undefined {
|
|
1401
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1402
|
-
return scope ? getThreadAffinityForScope(threadId, scope) : undefined;
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
function deleteThreadAffinityForScope(threadId: string, scope: ThreadAffinityScope): void {
|
|
1406
|
-
if (!admissibleAffinityComponent(threadId)) return;
|
|
1407
|
-
const affinities = threadAccountMap.get(threadId);
|
|
1408
|
-
if (!affinities) return;
|
|
1409
|
-
if (affinities.delete(scope)) {
|
|
1410
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1411
|
-
}
|
|
1412
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
function deleteThreadAffinity(threadId: string, quotaScope?: CodexQuotaScope): void {
|
|
1416
|
-
deleteThreadAffinityForScope(threadId, threadAffinityScope(quotaScope));
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
function deleteModelDetourAffinity(
|
|
1420
|
-
threadId: string,
|
|
1421
|
-
modelId: string | undefined,
|
|
1422
|
-
quotaScope?: CodexQuotaScope,
|
|
1423
|
-
): void {
|
|
1424
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1425
|
-
if (scope) deleteThreadAffinityForScope(threadId, scope);
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
/** Remove only the matching failed account's affinities for one thread. */
|
|
1429
|
-
function deleteThreadAffinitiesForAccount(threadId: string, accountId: string): void {
|
|
1430
|
-
if (!admissibleAffinityComponent(threadId) || !admissibleAffinityComponent(accountId)) return;
|
|
1431
|
-
const affinities = threadAccountMap.get(threadId);
|
|
1432
|
-
if (!affinities) return;
|
|
1433
|
-
for (const [scope, entry] of affinities) {
|
|
1434
|
-
if (entry.accountId === accountId && affinities.delete(scope)) {
|
|
1435
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1436
|
-
}
|
|
1437
|
-
}
|
|
1438
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1439
|
-
}
|
|
1440
|
-
|
|
1441
|
-
function threadAffinityEntryCount(): number {
|
|
1442
|
-
return threadAffinityEntryTotal;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
function isThreadAffinityExpired(entry: ThreadAffinityEntry, now: number): boolean {
|
|
1446
|
-
return now - entry.lastUsedAt > CODEX_THREAD_AFFINITY_IDLE_TTL_MS;
|
|
1447
|
-
}
|
|
1448
|
-
|
|
1449
|
-
function isThreadAffinityGenerationLive(entry: ThreadAffinityEntry): boolean {
|
|
1450
|
-
if (entry.accountId === MAIN_CODEX_ACCOUNT_ID) return entry.generation === 0;
|
|
1451
|
-
return isCodexAccountGenerationLive(entry.accountId, entry.generation);
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
/** Generations this account's affinity entries are bound at. Test observability only. */
|
|
1455
|
-
export function debugCodexAffinityGenerations(accountId: string): number[] {
|
|
1456
|
-
const generations: number[] = [];
|
|
1457
|
-
for (const affinities of threadAccountMap.values()) {
|
|
1458
|
-
for (const entry of affinities.values()) {
|
|
1459
|
-
if (entry.accountId === accountId) generations.push(entry.generation);
|
|
1460
|
-
}
|
|
1461
|
-
}
|
|
1462
|
-
return generations;
|
|
1463
|
-
}
|
|
1464
|
-
|
|
1465
|
-
/**
|
|
1466
|
-
* Advance this account's affinity entries from the generation a rejected credential
|
|
1467
|
-
* was bound under to the generation its own refresh produced.
|
|
1468
|
-
*
|
|
1469
|
-
* A 401 refresh-and-replay keeps the request on the same account, but the CAS write
|
|
1470
|
-
* moves the credential from G to G+1, and {@link isThreadAffinityGenerationLive}
|
|
1471
|
-
* demands exact equality — so without this the entry the replay just preserved is
|
|
1472
|
-
* dead on the next request. Not quarantining an account is not the same as keeping
|
|
1473
|
-
* its affinity.
|
|
1474
|
-
*
|
|
1475
|
-
* Lineage is proven by the CALLER, which must pass only a generation its own refresh
|
|
1476
|
-
* produced. Re-deriving it here from `replacedAt` cannot work: the caller reads that
|
|
1477
|
-
* field after the refresh and this function would re-read the same record, so the
|
|
1478
|
-
* comparison is tautological and an external replacement passes it. An external
|
|
1479
|
-
* replacement must retire the affinity, because that credential may belong to a
|
|
1480
|
-
* different upstream identity.
|
|
1481
|
-
*/
|
|
1482
|
-
export function handOffThreadAffinityGeneration(
|
|
1483
|
-
accountId: string,
|
|
1484
|
-
fromGeneration: number,
|
|
1485
|
-
toGeneration: number,
|
|
1486
|
-
): boolean {
|
|
1487
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return false;
|
|
1488
|
-
if (toGeneration !== fromGeneration + 1) return false;
|
|
1489
|
-
const record = readCodexAccountRecord(accountId);
|
|
1490
|
-
if (!record?.credential || record.deletedAt != null) return false;
|
|
1491
|
-
if (record.generation !== toGeneration) return false;
|
|
1492
|
-
let handedOff = false;
|
|
1493
|
-
for (const affinities of threadAccountMap.values()) {
|
|
1494
|
-
for (const entry of affinities.values()) {
|
|
1495
|
-
if (entry.accountId !== accountId || entry.generation !== fromGeneration) continue;
|
|
1496
|
-
entry.generation = toGeneration;
|
|
1497
|
-
handedOff = true;
|
|
1498
|
-
}
|
|
1499
|
-
}
|
|
1500
|
-
return handedOff;
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
function pruneExpiredThreadAffinities(now: number): void {
|
|
1504
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
1505
|
-
for (const [scope, entry] of affinities) {
|
|
1506
|
-
if (isThreadAffinityExpired(entry, now) && affinities.delete(scope)) {
|
|
1507
|
-
threadAffinityEntryTotal = Math.max(0, threadAffinityEntryTotal - 1);
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
if (affinities.size === 0) threadAccountMap.delete(threadId);
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
function pruneLruThreadAffinities(): void {
|
|
1515
|
-
if (threadAffinityEntryCount() <= CODEX_THREAD_AFFINITY_MAX_ENTRIES) return;
|
|
1516
|
-
while (threadAffinityEntryCount() > CODEX_THREAD_AFFINITY_MAX_ENTRIES) {
|
|
1517
|
-
let oldestThreadId: string | null = null;
|
|
1518
|
-
let oldestScope: ThreadAffinityScope | null = null;
|
|
1519
|
-
let oldestLastUsedAt = Number.POSITIVE_INFINITY;
|
|
1520
|
-
let oldestIsDetour = false;
|
|
1521
|
-
for (const [threadId, affinities] of threadAccountMap) {
|
|
1522
|
-
for (const [scope, entry] of affinities) {
|
|
1523
|
-
const candidateIsDetour = isModelDetourAffinityScope(scope);
|
|
1524
|
-
if (
|
|
1525
|
-
(candidateIsDetour && !oldestIsDetour)
|
|
1526
|
-
|| (candidateIsDetour === oldestIsDetour && entry.lastUsedAt < oldestLastUsedAt)
|
|
1527
|
-
) {
|
|
1528
|
-
oldestThreadId = threadId;
|
|
1529
|
-
oldestScope = scope;
|
|
1530
|
-
oldestLastUsedAt = entry.lastUsedAt;
|
|
1531
|
-
oldestIsDetour = candidateIsDetour;
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
if (!oldestThreadId || !oldestScope) return;
|
|
1536
|
-
deleteThreadAffinityForScope(oldestThreadId, oldestScope);
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
|
|
1540
|
-
function bindThreadAffinityForScope(
|
|
1541
|
-
threadId: string,
|
|
1542
|
-
accountId: string,
|
|
1543
|
-
now: number,
|
|
1544
|
-
scope: ThreadAffinityScope,
|
|
1545
|
-
): void {
|
|
1546
|
-
if (!admissibleAffinityComponent(threadId) || !admissibleAffinityComponent(accountId)) return;
|
|
1547
|
-
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
|
|
1548
|
-
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return;
|
|
1549
|
-
pruneExpiredThreadAffinities(now);
|
|
1550
|
-
const affinities = threadAccountMap.get(threadId) ?? new Map<ThreadAffinityScope, ThreadAffinityEntry>();
|
|
1551
|
-
const previous = affinities.get(scope);
|
|
1552
|
-
affinities.set(scope, {
|
|
1553
|
-
accountId,
|
|
1554
|
-
generation: accountId === MAIN_CODEX_ACCOUNT_ID ? 0 : record!.generation,
|
|
1555
|
-
createdAt: previous?.createdAt ?? now,
|
|
1556
|
-
lastUsedAt: now,
|
|
1557
|
-
lastReevalAt: now,
|
|
1558
|
-
});
|
|
1559
|
-
if (!previous) threadAffinityEntryTotal += 1;
|
|
1560
|
-
threadAccountMap.set(threadId, affinities);
|
|
1561
|
-
pruneLruThreadAffinities();
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
function bindThreadAffinity(
|
|
1565
|
-
threadId: string,
|
|
1566
|
-
accountId: string,
|
|
1567
|
-
now: number,
|
|
1568
|
-
quotaScope?: CodexQuotaScope,
|
|
1569
|
-
): void {
|
|
1570
|
-
bindThreadAffinityForScope(threadId, accountId, now, threadAffinityScope(quotaScope));
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
function bindModelDetourAffinity(
|
|
1574
|
-
threadId: string,
|
|
1575
|
-
accountId: string,
|
|
1576
|
-
now: number,
|
|
1577
|
-
modelId: string | undefined,
|
|
1578
|
-
quotaScope?: CodexQuotaScope,
|
|
1579
|
-
): void {
|
|
1580
|
-
const scope = modelDetourAffinityScope(modelId, quotaScope);
|
|
1581
|
-
if (scope) bindThreadAffinityForScope(threadId, accountId, now, scope);
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
|
-
function getEligiblePoolAccounts(
|
|
1585
|
-
config: OcxConfig,
|
|
1586
|
-
excludeId?: string,
|
|
1587
|
-
now = Date.now(),
|
|
1588
|
-
quotaScope?: CodexQuotaScope,
|
|
1589
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1590
|
-
skipFailoverReadyCandidates = false,
|
|
1591
|
-
): readonly string[] {
|
|
1592
|
-
const excludedPlans = excludedCodexPoolPlanKeys(config);
|
|
1593
|
-
const ids = (config.codexAccounts ?? [])
|
|
1594
|
-
.filter(account => isSelectableCodexPoolAccount(account)
|
|
1595
|
-
&& account.id !== excludeId
|
|
1596
|
-
&& !isCodexAccountPaused(config, account.id)
|
|
1597
|
-
&& !isCodexAccountPlanExcluded(config, account.id, excludedPlans)
|
|
1598
|
-
&& !isAccountNeedsReauth(account.id)
|
|
1599
|
-
&& (!skipFailoverReadyCandidates || !shouldFailover(config, account.id, now)))
|
|
1600
|
-
.filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
|
|
1601
|
-
.filter(account => !isCodexAccountSoftAvoided(account.id, now))
|
|
1602
|
-
.filter(account => !isCodexQuotaAvoided(account.id, quotaScope, now))
|
|
1603
|
-
.filter(account => isCodexAccountUsable(config, account.id, selectionOptions))
|
|
1604
|
-
.map(account => account.id);
|
|
1605
|
-
// The main Codex account is not stored in config.codexAccounts; include it as a
|
|
1606
|
-
// first-class rotation candidate when its read-only token is usable (Option A).
|
|
1607
|
-
if (
|
|
1608
|
-
excludeId !== MAIN_CODEX_ACCOUNT_ID
|
|
1609
|
-
&& !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)
|
|
1610
|
-
&& (!isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) || hasMainAccountRefreshGrant())
|
|
1611
|
-
&& getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null
|
|
1612
|
-
&& !isCodexAccountSoftAvoided(MAIN_CODEX_ACCOUNT_ID, now)
|
|
1613
|
-
// The main login is not in `config.codexAccounts`, so it never passes through the
|
|
1614
|
-
// filters above and this is the only place an avoidance window can exclude it. Without
|
|
1615
|
-
// this the window a refusal announced applies to the pool but not to the account that
|
|
1616
|
-
// earned it: the cooldown caps at fifteen minutes, the window runs up to six hours, and
|
|
1617
|
-
// in between the main account returns as a first-class candidate.
|
|
1618
|
-
&& !isCodexQuotaAvoided(MAIN_CODEX_ACCOUNT_ID, quotaScope, now)
|
|
1619
|
-
&& (!skipFailoverReadyCandidates || !shouldFailover(config, MAIN_CODEX_ACCOUNT_ID, now))
|
|
1620
|
-
&& isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions)
|
|
1621
|
-
) {
|
|
1622
|
-
ids.unshift(MAIN_CODEX_ACCOUNT_ID);
|
|
1623
|
-
}
|
|
1624
|
-
// Single choke point for selection order: every strategy, failover, and preview
|
|
1625
|
-
// reaches the pool through here, so tiering applies once rather than per picker.
|
|
1626
|
-
// Eligibility above is unchanged — this only narrows an already-eligible list.
|
|
1627
|
-
return selectPriorityTier(
|
|
1628
|
-
ids,
|
|
1629
|
-
codexAccountPriorityLookup(config),
|
|
1630
|
-
id => hasCodexQuotaHeadroom(config, id, selectionOptions, now),
|
|
1631
|
-
pinnedCodexAccountId(config),
|
|
1632
|
-
);
|
|
1633
|
-
}
|
|
1634
|
-
|
|
1635
|
-
function listEligibleCodexAccountIds(
|
|
1636
|
-
config: OcxConfig,
|
|
1637
|
-
now: number,
|
|
1638
|
-
quotaScope?: CodexQuotaScope,
|
|
1639
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1640
|
-
): readonly string[] {
|
|
1641
|
-
return getEligiblePoolAccounts(config, undefined, now, quotaScope, selectionOptions);
|
|
1642
|
-
}
|
|
1643
|
-
|
|
1644
|
-
/** Shared reset timestamps are not evidence for independent model-quota groups. */
|
|
1645
|
-
function accountPoolStrategyForScope(config: OcxConfig, quotaScope?: CodexQuotaScope) {
|
|
1646
|
-
const strategy = normalizeCodexAccountPoolStrategy(config.accountPoolStrategy);
|
|
1647
|
-
return strategy === "reset-first" && isIndependentCodexQuotaScope(quotaScope) ? "quota" : strategy;
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
function stickyLimitForConfig(config: OcxConfig): number {
|
|
1651
|
-
return normalizeAccountPoolStickyLimit(config.accountPoolStickyLimit);
|
|
1652
|
-
}
|
|
1653
|
-
|
|
1654
|
-
/**
|
|
1655
|
-
* Whether an account still has quota to give under the auto-switch threshold.
|
|
1656
|
-
*
|
|
1657
|
-
* Fill-first and the priority tier filter share this predicate, and share both of
|
|
1658
|
-
* its escape hatches. A disabled threshold means only health, pause, and reauth
|
|
1659
|
-
* may drain an account; unknown usage is a guess, so it must neither force
|
|
1660
|
-
* fill-first off the active account nor drain a tier that was simply never
|
|
1661
|
-
* primed. A genuinely exhausted account 429s into cooldown and leaves
|
|
1662
|
-
* eligibility on its own.
|
|
1663
|
-
*/
|
|
1664
|
-
function hasCodexQuotaHeadroom(
|
|
1665
|
-
config: OcxConfig,
|
|
1666
|
-
accountId: string,
|
|
1667
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1668
|
-
now: number = Date.now(),
|
|
1669
|
-
): boolean {
|
|
1670
|
-
const threshold = config.autoSwitchThreshold ?? 80;
|
|
1671
|
-
if (threshold <= 0) return true;
|
|
1672
|
-
const usage = computeCodexUsageScore(
|
|
1673
|
-
getAccountQuota(accountId),
|
|
1674
|
-
getPoolAccountPlanForSelection(config, accountId, selectionOptions),
|
|
1675
|
-
now,
|
|
1676
|
-
);
|
|
1677
|
-
if (isUnknownUsage(usage)) return true;
|
|
1678
|
-
return usage < threshold;
|
|
1679
|
-
}
|
|
1680
|
-
|
|
1681
|
-
/**
|
|
1682
|
-
* Is a live binding held for its prompt cache?
|
|
1683
|
-
*
|
|
1684
|
-
* Unset means yes. Cache affinity shipped as an opt-in flag (#4292) and then #4546 measured
|
|
1685
|
-
* what the default costs: a pool whose accounts all sit in the 80-99% band hands a bound
|
|
1686
|
-
* conversation from account to account, and because provider prompt caches are account-isolated
|
|
1687
|
-
* every hop re-sends the entire prefix. An install that has never heard of this flag is exactly
|
|
1688
|
-
* the install that gets hurt by it, so the protection cannot be something you have to find.
|
|
1689
|
-
*
|
|
1690
|
-
* `false` restores capacity-first routing byte-for-byte. It is a real choice -- a pinned thread
|
|
1691
|
-
* on a busy account pays latency -- and it stays available; it is just no longer the default.
|
|
1692
|
-
*/
|
|
1693
|
-
function isCacheAffinityEnabled(config: OcxConfig): boolean {
|
|
1694
|
-
return config.pool?.cacheAffinity !== false;
|
|
242
|
+
if (isHealthGenerationReconciled(context.generation)) return 0;
|
|
243
|
+
const removed = pruneHealthAccountsForContext(context.codexAccountIds);
|
|
244
|
+
// Sweep preferences the same way, for the account set this generation actually has. The
|
|
245
|
+
// delete path above is the direct route; this is the one that catches an account removed
|
|
246
|
+
// by an edit the runtime never saw. Deliberately not counted in `removed`, which reports
|
|
247
|
+
// health rows.
|
|
248
|
+
forgetRoutingPreferencesOutside(context.codexAccountIds);
|
|
249
|
+
commitHealthReconcile(context.generation, context.codexAccountIds);
|
|
250
|
+
return removed;
|
|
1695
251
|
}
|
|
1696
|
-
|
|
1697
252
|
/**
|
|
1698
253
|
* Is a transient failure streak the ONLY thing standing between this thread and its account?
|
|
1699
254
|
*
|
|
@@ -1725,13 +280,9 @@ function isTransientOnlyAffinityBlock(
|
|
|
1725
280
|
if (!isCodexAccountUsable(config, entry.accountId, selectionOptions)) return false;
|
|
1726
281
|
if (getCodexQuotaHealthSnapshot(entry.accountId, quotaScope, now) !== null) return false;
|
|
1727
282
|
if (isCodexQuotaAvoided(entry.accountId, quotaScope, now)) return false;
|
|
1728
|
-
return shouldFailover(config, entry.accountId, now)
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
/** Has a held binding waited longer than a transient failure can reasonably explain? */
|
|
1732
|
-
function isTransientHoldExpired(entry: ThreadAffinityEntry, now: number): boolean {
|
|
1733
|
-
return entry.transientHoldSince !== undefined
|
|
1734
|
-
&& now - entry.transientHoldSince > CODEX_TRANSIENT_AFFINITY_HOLD_MS;
|
|
283
|
+
return shouldFailover(config, entry.accountId, now)
|
|
284
|
+
|| isCodexAccountSoftAvoided(entry.accountId, now)
|
|
285
|
+
|| isCodexPoolRefreshCooling(entry.accountId, now);
|
|
1735
286
|
}
|
|
1736
287
|
|
|
1737
288
|
/**
|
|
@@ -1742,7 +293,7 @@ function isTransientHoldExpired(entry: ThreadAffinityEntry, now: number): boolea
|
|
|
1742
293
|
* that chance away.
|
|
1743
294
|
*/
|
|
1744
295
|
function isTransientHoldSpentForAccount(threadId: string, accountId: string, now: number): boolean {
|
|
1745
|
-
const affinities =
|
|
296
|
+
const affinities = getThreadAffinityScopes(threadId);
|
|
1746
297
|
if (!affinities) return false;
|
|
1747
298
|
let matched = false;
|
|
1748
299
|
for (const entry of affinities.values()) {
|
|
@@ -1784,429 +335,76 @@ function transientDetourAccount(
|
|
|
1784
335
|
: pickAlternateCodexAccount(config, entry.accountId, now, quotaScope, selectionOptions);
|
|
1785
336
|
}
|
|
1786
337
|
|
|
1787
|
-
/** Earliest future shared short/weekly reset; missing evidence and ties use usage order. */
|
|
1788
|
-
function pickResetFirstCodexAccount(
|
|
1789
|
-
config: OcxConfig,
|
|
1790
|
-
ids: readonly string[],
|
|
1791
|
-
now: number,
|
|
1792
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1793
|
-
): string | null {
|
|
1794
|
-
const available = ids.filter(id => hasCodexQuotaHeadroom(config, id, selectionOptions, now));
|
|
1795
|
-
if (available.length === 0) return pickLowestUsageAmong(config, ids, selectionOptions, now);
|
|
1796
|
-
let earliest = Number.POSITIVE_INFINITY;
|
|
1797
|
-
let candidates: string[] = [];
|
|
1798
|
-
for (const id of available) {
|
|
1799
|
-
const quota = getAccountQuota(id);
|
|
1800
|
-
const resets = [quota?.shortResetAt, quota?.weeklyResetAt]
|
|
1801
|
-
.filter((reset): reset is number => typeof reset === "number" && Number.isFinite(reset))
|
|
1802
|
-
.map(resetAtToMs)
|
|
1803
|
-
.filter(reset => reset > now);
|
|
1804
|
-
const next = Math.min(...resets);
|
|
1805
|
-
if (next < earliest) {
|
|
1806
|
-
earliest = next;
|
|
1807
|
-
candidates = [id];
|
|
1808
|
-
} else if (next === earliest) candidates.push(id);
|
|
1809
|
-
}
|
|
1810
|
-
return pickLowestUsageAmong(config, candidates, selectionOptions, now);
|
|
1811
|
-
}
|
|
1812
|
-
|
|
1813
|
-
/**
|
|
1814
|
-
* Fill-first: keep selectable active under threshold; otherwise advance to the next
|
|
1815
|
-
* eligible id in stable sorted order after the current active (wrapping).
|
|
1816
|
-
*/
|
|
1817
|
-
function pickFillFirstCodexAccount(
|
|
1818
|
-
config: OcxConfig,
|
|
1819
|
-
now: number,
|
|
1820
|
-
quotaScope?: CodexQuotaScope,
|
|
1821
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1822
|
-
): string | null {
|
|
1823
|
-
const eligible = listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions);
|
|
1824
|
-
if (eligible.length === 0) return null;
|
|
1825
|
-
|
|
1826
|
-
const active = getEffectiveActiveCodexAccountId(config);
|
|
1827
|
-
if (active && eligible.includes(active) && hasCodexQuotaHeadroom(config, active, selectionOptions, now)) {
|
|
1828
|
-
return active;
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
return pickNextFillFirstCodexAccount(config, active ?? null, eligible, now, selectionOptions);
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
|
-
/** Next eligible account in stable order after `afterId` (wrapping). */
|
|
1835
|
-
function pickNextFillFirstCodexAccount(
|
|
1836
|
-
config: OcxConfig,
|
|
1837
|
-
afterId: string | null,
|
|
1838
|
-
eligible: readonly string[] = listEligibleCodexAccountIds(config, Date.now()),
|
|
1839
|
-
now = Date.now(),
|
|
1840
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1841
|
-
): string | null {
|
|
1842
|
-
if (eligible.length === 0) return null;
|
|
1843
|
-
const ordered = [...eligible].sort((a, b) => a.localeCompare(b));
|
|
1844
|
-
if (!afterId) {
|
|
1845
|
-
// Prefer an under-threshold account when starting with no active cursor.
|
|
1846
|
-
for (const id of ordered) {
|
|
1847
|
-
if (hasCodexQuotaHeadroom(config, id, selectionOptions, now)) return id;
|
|
1848
|
-
}
|
|
1849
|
-
return ordered[0] ?? null;
|
|
1850
|
-
}
|
|
1851
|
-
|
|
1852
|
-
const allConfigured = [
|
|
1853
|
-
...(isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions) || afterId === MAIN_CODEX_ACCOUNT_ID
|
|
1854
|
-
? [MAIN_CODEX_ACCOUNT_ID]
|
|
1855
|
-
: []),
|
|
1856
|
-
...(config.codexAccounts ?? []).filter(account => !account.isMain).map(account => account.id),
|
|
1857
|
-
];
|
|
1858
|
-
const stableAll = [...new Set(allConfigured)].sort((a, b) => a.localeCompare(b));
|
|
1859
|
-
const startIdx = stableAll.indexOf(afterId);
|
|
1860
|
-
if (startIdx < 0) {
|
|
1861
|
-
for (const id of ordered) {
|
|
1862
|
-
if (hasCodexQuotaHeadroom(config, id, selectionOptions, now)) return id;
|
|
1863
|
-
}
|
|
1864
|
-
return ordered[0] ?? null;
|
|
1865
|
-
}
|
|
1866
|
-
|
|
1867
|
-
// Skip successors that are also at/above threshold (known drained usage).
|
|
1868
|
-
let fallback: string | null = null;
|
|
1869
|
-
for (let step = 1; step <= stableAll.length; step++) {
|
|
1870
|
-
const candidate = stableAll[(startIdx + step) % stableAll.length]!;
|
|
1871
|
-
if (!eligible.includes(candidate)) continue;
|
|
1872
|
-
if (!fallback) fallback = candidate;
|
|
1873
|
-
if (hasCodexQuotaHeadroom(config, candidate, selectionOptions, now)) return candidate;
|
|
1874
|
-
}
|
|
1875
|
-
return fallback ?? ordered[0] ?? null;
|
|
1876
|
-
}
|
|
1877
|
-
|
|
1878
338
|
/**
|
|
1879
|
-
*
|
|
1880
|
-
* to the legacy quota path (or when the strategy is quota).
|
|
339
|
+
* Which account is ACTUALLY answering for one conversation key right now (#4546, wp8).
|
|
1881
340
|
*
|
|
1882
|
-
*
|
|
1883
|
-
*
|
|
1884
|
-
*
|
|
1885
|
-
*
|
|
341
|
+
* First placement reads this, not the binding alone: a parent parked on a transient detour
|
|
342
|
+
* is being served by the detour, so a new child placed "where the parent lives" would miss
|
|
343
|
+
* the warm account by one hop. A dead binding, an expired hold, and an ineligible serving
|
|
344
|
+
* account all answer null -- the caller then tries a sibling, then falls back to cold
|
|
345
|
+
* placement, which is the correct order because a stale home is worse than no hint.
|
|
1886
346
|
*
|
|
1887
|
-
*
|
|
1888
|
-
*
|
|
1889
|
-
*
|
|
1890
|
-
*
|
|
1891
|
-
* the
|
|
347
|
+
* "Right now" includes the MODEL lane. A parent whose home account is not entitled to this
|
|
348
|
+
* model is being served through a model-scoped detour, which is the same "serving, not stale
|
|
349
|
+
* home" case one level further in: reading only the ordinary binding would hand the child an
|
|
350
|
+
* account this request cannot use, and it would then start cold on the very model whose
|
|
351
|
+
* warm account the family already found. The detour scope embeds the model and the quota
|
|
352
|
+
* scope, so the entry consulted here is compatible by construction.
|
|
1892
353
|
*/
|
|
1893
|
-
function
|
|
1894
|
-
|
|
1895
|
-
threadId: string | null,
|
|
1896
|
-
now: number,
|
|
1897
|
-
commit: boolean,
|
|
1898
|
-
quotaScope?: CodexQuotaScope,
|
|
1899
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1900
|
-
commitSharedActive = commit,
|
|
1901
|
-
commitAffinity = commit,
|
|
1902
|
-
): string | null {
|
|
1903
|
-
const strategy = accountPoolStrategyForScope(config, quotaScope);
|
|
1904
|
-
if (strategy === "quota") return null;
|
|
1905
|
-
const poolKey = codexPoolKeyForScope(quotaScope);
|
|
1906
|
-
|
|
1907
|
-
let picked: string | null = null;
|
|
1908
|
-
if (strategy === "round-robin") {
|
|
1909
|
-
const eligible = listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions);
|
|
1910
|
-
const limit = stickyLimitForConfig(config);
|
|
1911
|
-
if (!commit) {
|
|
1912
|
-
return peekRoundRobinAccount(poolKey, eligible, limit);
|
|
1913
|
-
}
|
|
1914
|
-
picked = pickRoundRobinAccount(poolKey, eligible, limit);
|
|
1915
|
-
if (!picked) return null;
|
|
1916
|
-
if (commitSharedActive) {
|
|
1917
|
-
if (!isIndependentCodexQuotaScope(quotaScope)
|
|
1918
|
-
&& !manualPreferenceBlocks(codexPoolKeyForScope(quotaScope), picked)) {
|
|
1919
|
-
rememberActiveCodexAccount(config, picked);
|
|
1920
|
-
}
|
|
1921
|
-
}
|
|
1922
|
-
if (commitAffinity && threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
1923
|
-
notePoolRotationSuccess(poolKey, picked, limit);
|
|
1924
|
-
return picked;
|
|
1925
|
-
}
|
|
1926
|
-
|
|
1927
|
-
if (strategy === "fill-first" || strategy === "reset-first") {
|
|
1928
|
-
picked = strategy === "reset-first"
|
|
1929
|
-
? pickResetFirstCodexAccount(config, listEligibleCodexAccountIds(config, now, quotaScope, selectionOptions), now, selectionOptions)
|
|
1930
|
-
: pickFillFirstCodexAccount(config, now, quotaScope, selectionOptions);
|
|
1931
|
-
if (!picked) return null;
|
|
1932
|
-
if (commitSharedActive) {
|
|
1933
|
-
if (!isIndependentCodexQuotaScope(quotaScope)
|
|
1934
|
-
&& !manualPreferenceBlocks(codexPoolKeyForScope(quotaScope), picked)) {
|
|
1935
|
-
rememberActiveCodexAccount(config, picked);
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
if (commitAffinity && threadId) bindThreadAffinity(threadId, picked, now, quotaScope);
|
|
1939
|
-
return picked;
|
|
1940
|
-
}
|
|
1941
|
-
|
|
1942
|
-
return null;
|
|
1943
|
-
}
|
|
1944
|
-
|
|
1945
|
-
export function getPoolAccountPlan(config: OcxConfig, accountId: string): string | undefined {
|
|
1946
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID) return getMainAccountPlan();
|
|
1947
|
-
return (config.codexAccounts ?? [])
|
|
1948
|
-
.find(account => isSelectableCodexPoolAccount(account) && account.id === accountId)?.plan;
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
/** Selection-only main routing must not lazily read the fenced native credential for its plan. */
|
|
1952
|
-
function getPoolAccountPlanForSelection(
|
|
354
|
+
function lineageServingAccountId(
|
|
355
|
+
conversationKey: string,
|
|
1953
356
|
config: OcxConfig,
|
|
1954
|
-
accountId: string,
|
|
1955
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1956
|
-
): string | undefined {
|
|
1957
|
-
if (accountId === MAIN_CODEX_ACCOUNT_ID && selectionOptions?.nativeMainSelectionOnly === true) {
|
|
1958
|
-
return undefined;
|
|
1959
|
-
}
|
|
1960
|
-
return getPoolAccountPlan(config, accountId);
|
|
1961
|
-
}
|
|
1962
|
-
|
|
1963
|
-
/** Shared routing state must ignore a request-scoped entitlement roster. */
|
|
1964
|
-
function sharedStateSelectionOptions(
|
|
1965
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1966
|
-
): Pick<
|
|
1967
|
-
CodexAccountUsabilityOptions,
|
|
1968
|
-
"nativeMainSelectionOnly" | "isMainAccountTokenLive"
|
|
1969
|
-
> | undefined {
|
|
1970
|
-
if (!selectionOptions) return undefined;
|
|
1971
|
-
return {
|
|
1972
|
-
...(selectionOptions.nativeMainSelectionOnly !== undefined
|
|
1973
|
-
? { nativeMainSelectionOnly: selectionOptions.nativeMainSelectionOnly }
|
|
1974
|
-
: {}),
|
|
1975
|
-
...(selectionOptions.isMainAccountTokenLive
|
|
1976
|
-
? { isMainAccountTokenLive: selectionOptions.isMainAccountTokenLive }
|
|
1977
|
-
: {}),
|
|
1978
|
-
};
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
function pickLowerUsageAccount(
|
|
1982
|
-
config: OcxConfig,
|
|
1983
|
-
active: string,
|
|
1984
|
-
activeUsage: number,
|
|
1985
357
|
now: number,
|
|
1986
358
|
quotaScope?: CodexQuotaScope,
|
|
1987
359
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
1988
|
-
|
|
1989
|
-
): string {
|
|
1990
|
-
let best = active;
|
|
1991
|
-
let bestUsage = activeUsage;
|
|
1992
|
-
for (const id of getEligiblePoolAccounts(
|
|
1993
|
-
config,
|
|
1994
|
-
active,
|
|
1995
|
-
now,
|
|
1996
|
-
quotaScope,
|
|
1997
|
-
selectionOptions,
|
|
1998
|
-
skipFailoverReadyCandidates,
|
|
1999
|
-
)) {
|
|
2000
|
-
const usage = computeCodexUsageScore(
|
|
2001
|
-
getAccountQuota(id),
|
|
2002
|
-
getPoolAccountPlanForSelection(config, id, selectionOptions),
|
|
2003
|
-
now,
|
|
2004
|
-
);
|
|
2005
|
-
if (usage < bestUsage) {
|
|
2006
|
-
best = id;
|
|
2007
|
-
bestUsage = usage;
|
|
2008
|
-
}
|
|
2009
|
-
}
|
|
2010
|
-
return best;
|
|
2011
|
-
}
|
|
2012
|
-
|
|
2013
|
-
/** Coolest account in an already-selected candidate list; first index wins ties. */
|
|
2014
|
-
function pickLowestUsageAmong(
|
|
2015
|
-
config: OcxConfig,
|
|
2016
|
-
ids: readonly string[],
|
|
2017
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2018
|
-
now: number = Date.now(),
|
|
2019
|
-
): string | null {
|
|
2020
|
-
let best: string | null = null;
|
|
2021
|
-
let bestUsage = Number.POSITIVE_INFINITY;
|
|
2022
|
-
for (const id of ids) {
|
|
2023
|
-
const usage = computeCodexUsageScore(
|
|
2024
|
-
getAccountQuota(id),
|
|
2025
|
-
getPoolAccountPlanForSelection(config, id, selectionOptions),
|
|
2026
|
-
now,
|
|
2027
|
-
);
|
|
2028
|
-
if (usage < bestUsage) {
|
|
2029
|
-
best = id;
|
|
2030
|
-
bestUsage = usage;
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
return best;
|
|
2034
|
-
}
|
|
2035
|
-
|
|
2036
|
-
export function pickLowestUsageCodexAccount(
|
|
2037
|
-
config: OcxConfig,
|
|
2038
|
-
excludeId?: string,
|
|
2039
|
-
now = Date.now(),
|
|
2040
|
-
quotaScope?: CodexQuotaScope,
|
|
2041
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2042
|
-
): string | null {
|
|
2043
|
-
return pickLowestUsageAmong(
|
|
2044
|
-
config,
|
|
2045
|
-
getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions),
|
|
2046
|
-
selectionOptions,
|
|
2047
|
-
now,
|
|
2048
|
-
);
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
/**
|
|
2052
|
-
* Strategy-aware alternate after a cooled/excluded account (same-request 429 retry
|
|
2053
|
-
* and active promotion). Quota keeps lowest-usage; fill-first advances stable order;
|
|
2054
|
-
* round-robin takes the next ring pick (caller should have noted the failure).
|
|
2055
|
-
*/
|
|
2056
|
-
export function pickAlternateCodexAccount(
|
|
2057
|
-
config: OcxConfig,
|
|
2058
|
-
excludeId: string,
|
|
2059
|
-
now = Date.now(),
|
|
2060
|
-
quotaScope?: CodexQuotaScope,
|
|
2061
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
360
|
+
modelId?: string,
|
|
2062
361
|
): string | null {
|
|
2063
|
-
const
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
|
|
2070
|
-
return pickRoundRobinAccount(codexPoolKeyForScope(quotaScope), eligible, stickyLimitForConfig(config));
|
|
2071
|
-
}
|
|
2072
|
-
if (strategy === "fill-first") {
|
|
2073
|
-
const eligible = getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions);
|
|
2074
|
-
return pickNextFillFirstCodexAccount(config, excludeId, eligible, now, selectionOptions);
|
|
2075
|
-
}
|
|
2076
|
-
if (strategy === "reset-first") {
|
|
2077
|
-
return pickResetFirstCodexAccount(config, getEligiblePoolAccounts(config, excludeId, now, quotaScope, selectionOptions), now, selectionOptions);
|
|
362
|
+
const entry = (modelId !== undefined
|
|
363
|
+
? getModelDetourAffinity(conversationKey, modelId, quotaScope)
|
|
364
|
+
: undefined)
|
|
365
|
+
?? getThreadAffinity(conversationKey, quotaScope);
|
|
366
|
+
if (!entry || isThreadAffinityExpired(entry, now) || !isThreadAffinityGenerationLive(entry)) {
|
|
367
|
+
return null;
|
|
2078
368
|
}
|
|
2079
|
-
|
|
369
|
+
const holdLive = entry.transientHoldSince !== undefined && !isTransientHoldExpired(entry, now);
|
|
370
|
+
const serving = holdLive && entry.transientDetourAccountId !== undefined
|
|
371
|
+
? entry.transientDetourAccountId
|
|
372
|
+
: entry.accountId;
|
|
373
|
+
return isCodexAccountSelectable(config, serving, now, quotaScope, selectionOptions)
|
|
374
|
+
&& !hasUnrecoveredCodexQuotaRefusal(serving, quotaScope)
|
|
375
|
+
&& !shouldFailover(config, serving, now)
|
|
376
|
+
&& !isCodexAccountSoftAvoided(serving, now)
|
|
377
|
+
? serving
|
|
378
|
+
: null;
|
|
2080
379
|
}
|
|
2081
380
|
|
|
2082
381
|
/**
|
|
2083
|
-
*
|
|
2084
|
-
*
|
|
2085
|
-
*
|
|
2086
|
-
*
|
|
2087
|
-
* the selection rule that could drift from it.
|
|
2088
|
-
*
|
|
2089
|
-
* This exists because preview and resolve have to agree on the FIRST transient detour, not just
|
|
2090
|
-
* on later ones. Preview feeds subagent model-availability scoring, so a preview that reported
|
|
2091
|
-
* the bound account while resolve was about to serve from a cool sibling could retire a model
|
|
2092
|
-
* over usage the request would never have touched.
|
|
382
|
+
* First placement only: where a child with NO binding of its own should start. Parent's
|
|
383
|
+
* current serving account first, then a compatible sibling's -- "compatible" meaning the
|
|
384
|
+
* same quota-scope slot, since a Reserve sibling says nothing about the shared lane. The
|
|
385
|
+
* child still binds under its own key; this is a hint for turn one, not a root-wide pin.
|
|
2093
386
|
*/
|
|
2094
|
-
function
|
|
387
|
+
function pickLineageServingAccount(
|
|
2095
388
|
config: OcxConfig,
|
|
2096
|
-
|
|
389
|
+
lineage: CodexThreadLineage,
|
|
2097
390
|
now: number,
|
|
2098
391
|
quotaScope?: CodexQuotaScope,
|
|
2099
392
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
}
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
* cannot be detected by comparing values: a pool-driven promote legitimately moves the
|
|
2114
|
-
* persisted active account, and reading that as staleness would silently spend the
|
|
2115
|
-
* operator's one-shot. Invalidation is keyed to the OPERATOR path instead — another manual
|
|
2116
|
-
* selection, the account leaving the pool, or a successful dispatch on it.
|
|
2117
|
-
*/
|
|
2118
|
-
const manualPreference = new Map<string, string>();
|
|
2119
|
-
|
|
2120
|
-
/**
|
|
2121
|
-
* Spend the one-shot for a pool scope once a dispatch on that account actually succeeded.
|
|
2122
|
-
* This is the Codex analogue of `commitAnthropicSelectionRouting`, which Codex lacks.
|
|
2123
|
-
*
|
|
2124
|
-
* Wiring this BEFORE the guard below is not a style choice. Measured: with the guard in
|
|
2125
|
-
* place and no consume site, the first manual selection freezes the automatic cursor
|
|
2126
|
-
* permanently and 15 of 69 rotation tests fail.
|
|
2127
|
-
*/
|
|
2128
|
-
function consumeManualPreference(accountId: string, poolKey: string): void {
|
|
2129
|
-
if (manualPreference.get(poolKey) === accountId) manualPreference.delete(poolKey);
|
|
2130
|
-
}
|
|
2131
|
-
|
|
2132
|
-
/**
|
|
2133
|
-
* Drop an account's preference in every scope. Pause and exclusion do not route through
|
|
2134
|
-
* `resetCodexRoutingForManualSelection`, so without this a preference could outlive the
|
|
2135
|
-
* account it names and keep suppressing the automatic cursor.
|
|
2136
|
-
*/
|
|
2137
|
-
function forgetManualPreference(accountId: string): void {
|
|
2138
|
-
for (const [poolKey, preferred] of manualPreference) {
|
|
2139
|
-
if (preferred === accountId) manualPreference.delete(poolKey);
|
|
2140
|
-
}
|
|
2141
|
-
}
|
|
2142
|
-
|
|
2143
|
-
/**
|
|
2144
|
-
* True while an unspent operator selection for this scope names a DIFFERENT account than
|
|
2145
|
-
* the automatic pick about to be recorded.
|
|
2146
|
-
*
|
|
2147
|
-
* Callers pass their own scope: an independent quota scope keeps its own entry and must
|
|
2148
|
-
* never read the shared one. The failover promote does NOT consult this — see its call
|
|
2149
|
-
* site for why.
|
|
2150
|
-
*/
|
|
2151
|
-
function manualPreferenceBlocks(poolKey: string, accountId: string): boolean {
|
|
2152
|
-
const preferred = manualPreference.get(poolKey);
|
|
2153
|
-
return preferred !== undefined && preferred !== accountId;
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
export function getEffectiveActiveCodexAccountId(config: OcxConfig): string | undefined {
|
|
2157
|
-
return runtimeActiveCodexAccountId ?? config.activeCodexAccountId;
|
|
2158
|
-
}
|
|
2159
|
-
|
|
2160
|
-
/**
|
|
2161
|
-
* Whether the account routing is currently on is there because an operator asked
|
|
2162
|
-
* for it, rather than because a strategy landed on it. Surfaces read this instead
|
|
2163
|
-
* of comparing the stored pin themselves, which would report a pin that a later
|
|
2164
|
-
* automatic pick has already moved past.
|
|
2165
|
-
*/
|
|
2166
|
-
export function isEffectiveCodexAccountPinned(config: OcxConfig): boolean {
|
|
2167
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2168
|
-
return pinned !== undefined && pinned === getEffectiveActiveCodexAccountId(config);
|
|
2169
|
-
}
|
|
2170
|
-
|
|
2171
|
-
/**
|
|
2172
|
-
* Automatic strategy / failover cursor only — never mutates `config.activeCodexAccountId`
|
|
2173
|
-
* so an unrelated `saveConfig` cannot persist transient rotation as operator selection.
|
|
2174
|
-
*/
|
|
2175
|
-
function rememberActiveCodexAccount(_config: OcxConfig, accountId: string): void {
|
|
2176
|
-
runtimeActiveCodexAccountId = accountId;
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
/**
|
|
2180
|
-
* End the manual pin when routing moves to a different account. Returns whether
|
|
2181
|
-
* the pin changed so the caller can fold it into a write it was already making.
|
|
2182
|
-
*/
|
|
2183
|
-
function releaseCodexAccountPinFor(config: OcxConfig, accountId: string): boolean {
|
|
2184
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2185
|
-
if (pinned === undefined || pinned === accountId) return false;
|
|
2186
|
-
clearCodexAccountPin(config);
|
|
2187
|
-
return true;
|
|
2188
|
-
}
|
|
2189
|
-
|
|
2190
|
-
/** Persist operator (or quota-strategy) active selection to config + disk. */
|
|
2191
|
-
function setActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
2192
|
-
runtimeActiveCodexAccountId = undefined;
|
|
2193
|
-
const releasedPin = releaseCodexAccountPinFor(config, accountId);
|
|
2194
|
-
if (config.activeCodexAccountId === accountId && !releasedPin) return;
|
|
2195
|
-
config.activeCodexAccountId = accountId;
|
|
2196
|
-
saveConfigPreservingClaudeCode(config);
|
|
2197
|
-
}
|
|
2198
|
-
|
|
2199
|
-
/** Quota strategy persists; RR/fill-first keep a process-local cursor only. */
|
|
2200
|
-
function promoteActiveCodexAccount(config: OcxConfig, accountId: string): void {
|
|
2201
|
-
if (normalizeCodexAccountPoolStrategy(config.accountPoolStrategy) === "quota") {
|
|
2202
|
-
setActiveCodexAccount(config, accountId);
|
|
2203
|
-
return;
|
|
393
|
+
modelId?: string,
|
|
394
|
+
): { accountId: string; reason: CodexAffinityReason } | null {
|
|
395
|
+
if (lineage.parentConversationKey !== undefined) {
|
|
396
|
+
const parent = lineageServingAccountId(
|
|
397
|
+
lineage.parentConversationKey, config, now, quotaScope, selectionOptions, modelId,
|
|
398
|
+
);
|
|
399
|
+
if (parent) return { accountId: parent, reason: "lineage_parent" };
|
|
400
|
+
for (const siblingKey of lineage.siblingConversationKeys) {
|
|
401
|
+
const sibling = lineageServingAccountId(
|
|
402
|
+
siblingKey, config, now, quotaScope, selectionOptions, modelId,
|
|
403
|
+
);
|
|
404
|
+
if (sibling) return { accountId: sibling, reason: "lineage_sibling" };
|
|
405
|
+
}
|
|
2204
406
|
}
|
|
2205
|
-
|
|
2206
|
-
// saves this release with its own write; a transient failover does not, so the
|
|
2207
|
-
// pin survives a restart that also clears the failure history behind it.
|
|
2208
|
-
releaseCodexAccountPinFor(config, accountId);
|
|
2209
|
-
rememberActiveCodexAccount(config, accountId);
|
|
407
|
+
return null;
|
|
2210
408
|
}
|
|
2211
409
|
|
|
2212
410
|
/**
|
|
@@ -2234,54 +432,12 @@ export function reconcileCodexActiveAfterExclusion(
|
|
|
2234
432
|
clearCodexAccountPin(config, excludedAccountId);
|
|
2235
433
|
if (!wasEffective) return getEffectiveActiveCodexAccountId(config) ?? null;
|
|
2236
434
|
|
|
2237
|
-
|
|
435
|
+
forgetRuntimeActiveCodexAccount();
|
|
2238
436
|
const fallback = pickAlternateCodexAccount(config, excludedAccountId, now);
|
|
2239
437
|
if (fallback) promoteActiveCodexAccount(config, fallback);
|
|
2240
438
|
return fallback;
|
|
2241
439
|
}
|
|
2242
440
|
|
|
2243
|
-
function isUnknownUsage(usage: number): boolean {
|
|
2244
|
-
return usage >= CODEX_UNKNOWN_USAGE_SCORE;
|
|
2245
|
-
}
|
|
2246
|
-
|
|
2247
|
-
/**
|
|
2248
|
-
* Move an unbound request back up when a higher tier regains headroom — the
|
|
2249
|
-
* weekly-reset case. Returns null when nothing should change.
|
|
2250
|
-
*
|
|
2251
|
-
* Downward moves are deliberately left to {@link applyQuotaAutoSwitch}: this only
|
|
2252
|
-
* fires when the tier filter has already excluded `active`, and only toward a
|
|
2253
|
-
* tier that strictly outranks it. Threads bound by affinity never reach here.
|
|
2254
|
-
*/
|
|
2255
|
-
function pickPriorityPreemption(
|
|
2256
|
-
config: OcxConfig,
|
|
2257
|
-
active: string,
|
|
2258
|
-
now: number,
|
|
2259
|
-
quotaScope?: CodexQuotaScope,
|
|
2260
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2261
|
-
): string | null {
|
|
2262
|
-
const eligible = getEligiblePoolAccounts(config, undefined, now, quotaScope, selectionOptions);
|
|
2263
|
-
if (eligible.length === 0 || eligible.includes(active)) return null;
|
|
2264
|
-
const pinned = pinnedCodexAccountId(config);
|
|
2265
|
-
// A live pin already lowered the tier ceiling; never preempt past an explicit
|
|
2266
|
-
// operator choice. Same liveness test the tier filter applies, so preview and
|
|
2267
|
-
// resolve agree even before the pin is garbage-collected.
|
|
2268
|
-
if (
|
|
2269
|
-
pinned !== undefined
|
|
2270
|
-
&& eligible.includes(pinned)
|
|
2271
|
-
&& hasCodexQuotaHeadroom(config, pinned, selectionOptions, now)
|
|
2272
|
-
) return null;
|
|
2273
|
-
const priorityOf = codexAccountPriorityLookup(config);
|
|
2274
|
-
if (priorityOf(eligible[0]!) <= priorityOf(active)) return null;
|
|
2275
|
-
// Members without headroom are in the tier only because a sibling has some;
|
|
2276
|
-
// picking one would hand the request straight back to a drained account.
|
|
2277
|
-
return pickLowestUsageAmong(
|
|
2278
|
-
config,
|
|
2279
|
-
eligible.filter(id => hasCodexQuotaHeadroom(config, id, selectionOptions, now)),
|
|
2280
|
-
selectionOptions,
|
|
2281
|
-
now,
|
|
2282
|
-
);
|
|
2283
|
-
}
|
|
2284
|
-
|
|
2285
441
|
/**
|
|
2286
442
|
* Release a pin whose account is durably drained. "Use this account now" ends
|
|
2287
443
|
* when the account crosses the auto-switch threshold or stops being selectable
|
|
@@ -2316,114 +472,16 @@ function releaseDrainedCodexAccountPin(
|
|
|
2316
472
|
saveConfigPreservingClaudeCode(config);
|
|
2317
473
|
}
|
|
2318
474
|
|
|
2319
|
-
function applyQuotaAutoSwitch(
|
|
2320
|
-
config: OcxConfig,
|
|
2321
|
-
active: string,
|
|
2322
|
-
now: number,
|
|
2323
|
-
quotaScope?: CodexQuotaScope,
|
|
2324
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2325
|
-
commitSharedSelection = true,
|
|
2326
|
-
): string {
|
|
2327
|
-
const threshold = config.autoSwitchThreshold ?? 80;
|
|
2328
|
-
if (threshold <= 0) return active;
|
|
2329
|
-
const quota = getAccountQuota(active);
|
|
2330
|
-
const activeUsage = computeCodexUsageScore(
|
|
2331
|
-
quota,
|
|
2332
|
-
getPoolAccountPlanForSelection(config, active, selectionOptions),
|
|
2333
|
-
now,
|
|
2334
|
-
);
|
|
2335
|
-
// Unknown usage is not evidence that a user's explicit selection crossed the
|
|
2336
|
-
// threshold. Wait for quota priming instead of rotating among guesses.
|
|
2337
|
-
if (isUnknownUsage(activeUsage)) return active;
|
|
2338
|
-
if (activeUsage < threshold) return active;
|
|
2339
|
-
const best = pickLowerUsageAccount(config, active, activeUsage, now, quotaScope, selectionOptions);
|
|
2340
|
-
if (best !== active) {
|
|
2341
|
-
if (commitSharedSelection && !isIndependentCodexQuotaScope(quotaScope)) {
|
|
2342
|
-
setActiveCodexAccount(config, best);
|
|
2343
|
-
}
|
|
2344
|
-
return best;
|
|
2345
|
-
}
|
|
2346
|
-
|
|
2347
|
-
return active;
|
|
2348
|
-
}
|
|
2349
|
-
|
|
2350
|
-
function shouldFailover(config: OcxConfig, accountId: string, now: number): boolean {
|
|
2351
|
-
const threshold = config.upstreamFailoverThreshold ?? 3;
|
|
2352
|
-
if (threshold <= 0) return false;
|
|
2353
|
-
dropSpentCredentialFailure(accountId);
|
|
2354
|
-
const health = upstreamHealth.get(accountId);
|
|
2355
|
-
if (health?.lastFailureAt && now - health.lastFailureAt > CODEX_FAILURE_WINDOW_MS) return false;
|
|
2356
|
-
return !!health && health.consecutiveFailures >= threshold;
|
|
2357
|
-
}
|
|
2358
|
-
|
|
2359
|
-
function isHealthySharedCodexSelection(
|
|
2360
|
-
config: OcxConfig,
|
|
2361
|
-
accountId: string,
|
|
2362
|
-
now: number,
|
|
2363
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
2364
|
-
selectionOptions: CodexAccountUsabilityOptions | undefined,
|
|
2365
|
-
): boolean {
|
|
2366
|
-
return isCodexAccountSelectable(config, accountId, now, quotaScope, selectionOptions)
|
|
2367
|
-
&& hasCodexQuotaHeadroom(config, accountId, selectionOptions, now)
|
|
2368
|
-
&& !shouldFailover(config, accountId, now);
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
function strategySelectionOptionsForModelDetour(
|
|
2372
|
-
config: OcxConfig,
|
|
2373
|
-
now: number,
|
|
2374
|
-
quotaScope: CodexQuotaScope | undefined,
|
|
2375
|
-
selectionOptions: CodexAccountUsabilityOptions | undefined,
|
|
2376
|
-
): CodexAccountUsabilityOptions | undefined {
|
|
2377
|
-
if (selectionOptions?.modelEligibleAccountIds === undefined) return selectionOptions;
|
|
2378
|
-
const sharedSelectionOptions = sharedStateSelectionOptions(selectionOptions) ?? {};
|
|
2379
|
-
return {
|
|
2380
|
-
...selectionOptions,
|
|
2381
|
-
modelEligibleAccountIds: new Set(
|
|
2382
|
-
[...selectionOptions.modelEligibleAccountIds].filter(accountId =>
|
|
2383
|
-
isHealthySharedCodexSelection(
|
|
2384
|
-
config,
|
|
2385
|
-
accountId,
|
|
2386
|
-
now,
|
|
2387
|
-
quotaScope,
|
|
2388
|
-
sharedSelectionOptions,
|
|
2389
|
-
)
|
|
2390
|
-
),
|
|
2391
|
-
),
|
|
2392
|
-
};
|
|
2393
|
-
}
|
|
2394
|
-
|
|
2395
|
-
function applyFailureFailover(
|
|
2396
|
-
config: OcxConfig,
|
|
2397
|
-
active: string,
|
|
2398
|
-
now: number,
|
|
2399
|
-
quotaScope?: CodexQuotaScope,
|
|
2400
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2401
|
-
commitSharedSelection = true,
|
|
2402
|
-
): string {
|
|
2403
|
-
if (!shouldFailover(config, active, now)) return active;
|
|
2404
|
-
const best = pickAlternateCodexAccount(config, active, now, quotaScope, selectionOptions);
|
|
2405
|
-
if (best) {
|
|
2406
|
-
// The scope still routes away from the failing account — that is this request's
|
|
2407
|
-
// own decision — but an independent one must not persist a new shared active
|
|
2408
|
-
// account. recordCodexUpstreamOutcome only suppresses the promotion it makes at
|
|
2409
|
-
// the moment of the failure; the streak outlives the soft avoid, so a later
|
|
2410
|
-
// scoped resolve reaches here with the streak still tripped and would otherwise
|
|
2411
|
-
// move the shared cursor after all.
|
|
2412
|
-
if (commitSharedSelection && !isIndependentCodexQuotaScope(quotaScope)) {
|
|
2413
|
-
promoteActiveCodexAccount(config, best);
|
|
2414
|
-
}
|
|
2415
|
-
return best;
|
|
2416
|
-
}
|
|
2417
|
-
return active;
|
|
2418
|
-
}
|
|
2419
|
-
|
|
2420
475
|
export function resolveCodexAccountForThread(
|
|
2421
476
|
threadId: string | null,
|
|
2422
477
|
config: OcxConfig,
|
|
2423
478
|
now = Date.now(),
|
|
2424
479
|
quotaScope?: CodexQuotaScope,
|
|
480
|
+
lineage?: CodexThreadLineage,
|
|
2425
481
|
): string | null {
|
|
2426
|
-
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now, quotaScope);
|
|
482
|
+
const resolution = resolveCodexAccountForThreadDetailed(threadId, config, now, quotaScope, undefined, undefined, lineage);
|
|
483
|
+
// A WITHHELD dispatch is deliberately not an account here: this wrapper cannot carry a retry
|
|
484
|
+
// time, and answering with the held account is the send the hold prevents. Fails closed.
|
|
2427
485
|
return resolution.status === "selected" ? resolution.accountId : null;
|
|
2428
486
|
}
|
|
2429
487
|
|
|
@@ -2462,7 +520,7 @@ function carriesQuotaRefusal(health: CodexUpstreamHealth | undefined): boolean {
|
|
|
2462
520
|
* quota group, so a spent Spark window still cannot displace the same thread's Terra binding.
|
|
2463
521
|
*/
|
|
2464
522
|
function hasUnrecoveredCodexQuotaRefusal(accountId: string, quotaScope?: CodexQuotaScope): boolean {
|
|
2465
|
-
if (carriesQuotaRefusal(
|
|
523
|
+
if (carriesQuotaRefusal(getAccountHealth(accountId))) return true;
|
|
2466
524
|
return quotaScope !== undefined && carriesQuotaRefusal(scopedHealthFor(accountId, quotaScope));
|
|
2467
525
|
}
|
|
2468
526
|
|
|
@@ -2530,34 +588,6 @@ function previewReusableAffinityAccount(
|
|
|
2530
588
|
return entry.accountId;
|
|
2531
589
|
}
|
|
2532
590
|
|
|
2533
|
-
/**
|
|
2534
|
-
* May a LIVE binding be moved for quota reasons?
|
|
2535
|
-
*
|
|
2536
|
-
* Default: no. The bar is genuine exhaustion, because moving a bound conversation discards
|
|
2537
|
-
* the prompt cache warmed on its account and a threshold crossing is a hint that the account
|
|
2538
|
-
* is getting busy rather than evidence it cannot serve (#4546). Deliberately NOT
|
|
2539
|
-
* `hasCodexQuotaHeadroom`, which reads `usage < autoSwitchThreshold` and would reproduce the
|
|
2540
|
-
* old rule under a new name.
|
|
2541
|
-
*
|
|
2542
|
-
* With `pool.cacheAffinity: false` the historical rule comes back: a crossing of
|
|
2543
|
-
* `autoSwitchThreshold` is enough. That is capacity-first routing, and an operator who wants
|
|
2544
|
-
* it keeps it -- but it is no longer what an install gets by never having heard of the flag.
|
|
2545
|
-
*/
|
|
2546
|
-
function mayRebindAffinityForQuota(
|
|
2547
|
-
config: OcxConfig,
|
|
2548
|
-
accountId: string,
|
|
2549
|
-
usage: number,
|
|
2550
|
-
threshold: number,
|
|
2551
|
-
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2552
|
-
): boolean {
|
|
2553
|
-
const overThreshold = threshold > 0 && !isUnknownUsage(usage) && usage >= threshold;
|
|
2554
|
-
if (!isCacheAffinityEnabled(config)) return overThreshold;
|
|
2555
|
-
// The usable half is already guaranteed by both callers, which gate on
|
|
2556
|
-
// isCodexAccountSelectable; kept explicit so the predicate reads correctly on its own.
|
|
2557
|
-
return !isCodexAccountUsable(config, accountId, selectionOptions)
|
|
2558
|
-
|| (!isUnknownUsage(usage) && usage >= 100);
|
|
2559
|
-
}
|
|
2560
|
-
|
|
2561
591
|
/** Reset ordering may move a binding only under the existing cache-affinity release policy. */
|
|
2562
592
|
function resetFirstAffinityReplacement(
|
|
2563
593
|
entry: ThreadAffinityEntry,
|
|
@@ -2699,6 +729,7 @@ export function previewCodexAccountForRequest(
|
|
|
2699
729
|
quotaScope?: CodexQuotaScope,
|
|
2700
730
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2701
731
|
modelId?: string,
|
|
732
|
+
lineage?: CodexThreadLineage,
|
|
2702
733
|
): string | null {
|
|
2703
734
|
// A request-scoped model detour keeps its own serving-account affinity. Preview
|
|
2704
735
|
// reads it before the ordinary lane, but never repairs or deletes it. Roster
|
|
@@ -2724,6 +755,30 @@ export function previewCodexAccountForRequest(
|
|
|
2724
755
|
);
|
|
2725
756
|
if (ordinaryPreview) return ordinaryPreview;
|
|
2726
757
|
|
|
758
|
+
// A conversation carried across an in-process swap is still bound under the pre-#4546 raw
|
|
759
|
+
// parent key, and resolve adopts that binding rather than rebinding cold. Preview has to name
|
|
760
|
+
// the same account. Read-only, as everything here is: it neither adopts nor retires the entry.
|
|
761
|
+
if (threadId && !entry && lineage?.legacyConversationKey !== undefined) {
|
|
762
|
+
const legacyPreview = previewReusableAffinityAccount(
|
|
763
|
+
getThreadAffinity(lineage.legacyConversationKey, quotaScope),
|
|
764
|
+
config,
|
|
765
|
+
now,
|
|
766
|
+
quotaScope,
|
|
767
|
+
selectionOptions,
|
|
768
|
+
);
|
|
769
|
+
if (legacyPreview) return legacyPreview;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// First placement mirrors resolve: a child with no binding previews the account actually
|
|
773
|
+
// serving its parent (or a compatible sibling), so the subagent fallback does not decide
|
|
774
|
+
// against a cold pick the real request would never make. Read-only: nothing binds here.
|
|
775
|
+
if (threadId && !entry && lineage) {
|
|
776
|
+
const lineagePreview = pickLineageServingAccount(
|
|
777
|
+
config, lineage, now, quotaScope, selectionOptions, modelId,
|
|
778
|
+
);
|
|
779
|
+
if (lineagePreview) return lineagePreview.accountId;
|
|
780
|
+
}
|
|
781
|
+
|
|
2727
782
|
const strategyPick = pickUnboundStrategyAccount(
|
|
2728
783
|
config,
|
|
2729
784
|
threadId,
|
|
@@ -2765,6 +820,9 @@ export function previewCodexAccountForRequest(
|
|
|
2765
820
|
const best = pickLowestUsageCodexAccount(config, active, now, quotaScope, selectionOptions);
|
|
2766
821
|
if (best) active = best;
|
|
2767
822
|
}
|
|
823
|
+
// Same correction resolve applies, for the same reason: preview must name the account the
|
|
824
|
+
// request will actually use, or subagent fallback scores a model against the wrong one.
|
|
825
|
+
active = preferModelEntitledAccount(config, active, now, quotaScope, selectionOptions);
|
|
2768
826
|
if (!isCodexAccountUsable(config, active, selectionOptions)) {
|
|
2769
827
|
return hasConfiguredPoolAccount(config, active, selectionOptions) ? active : null;
|
|
2770
828
|
}
|
|
@@ -2782,6 +840,7 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2782
840
|
quotaScope?: CodexQuotaScope,
|
|
2783
841
|
selectionOptions?: CodexAccountUsabilityOptions,
|
|
2784
842
|
modelId?: string,
|
|
843
|
+
lineage?: CodexThreadLineage,
|
|
2785
844
|
): CodexThreadResolution {
|
|
2786
845
|
// An entitlement roster constrains only this model request. It must not rewrite
|
|
2787
846
|
// the operator's shared active/pin choice or the task's ordinary-model affinity.
|
|
@@ -2809,6 +868,13 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2809
868
|
)
|
|
2810
869
|
);
|
|
2811
870
|
|
|
871
|
+
// A conversation that was live across an in-process code swap is still bound under the
|
|
872
|
+
// pre-#4546 raw parent key. Adopt that binding onto this thread's key BEFORE anything below
|
|
873
|
+
// reads an entry, so the conversation arrives here as an ordinary bound thread instead of a
|
|
874
|
+
// cold one: every branch that follows -- detour reuse, transient hold, quota re-eval --
|
|
875
|
+
// should treat it as the continuing conversation it is. No-op on a fresh process.
|
|
876
|
+
if (threadId) adoptLegacyLineageAffinity(threadId, lineage, now, quotaScope, modelId);
|
|
877
|
+
|
|
2812
878
|
if (threadId && modelScopedSelection) {
|
|
2813
879
|
const detourEntry = getModelDetourAffinity(threadId, modelId, quotaScope);
|
|
2814
880
|
if (detourEntry) {
|
|
@@ -2850,15 +916,12 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2850
916
|
const lane = transientDetourAccount(config, detourEntry, now, quotaScope, selectionOptions);
|
|
2851
917
|
detourEntry.transientHoldSince ??= now;
|
|
2852
918
|
detourEntry.lastUsedAt = now;
|
|
2853
|
-
if (lane !== null && lane !== detourEntry.accountId) {
|
|
2854
|
-
detourEntry.transientDetourAccountId = lane;
|
|
2855
|
-
return { status: "selected", accountId: lane, affinity: { move: "detour", reason: "transient" } };
|
|
2856
|
-
}
|
|
2857
919
|
// A provider-wide outage soft-avoids every sibling, so there is nowhere to detour.
|
|
2858
920
|
// That is a statement about where this request can go, not about who owns the
|
|
2859
921
|
// conversation: dropping the pin here would rebuild the cold prefix elsewhere for
|
|
2860
|
-
// exactly the failure
|
|
2861
|
-
|
|
922
|
+
// exactly the failure the hold exists to survive -- nor a licence to send at the
|
|
923
|
+
// failing account, which is what the dispatch resolver bounds (#4701).
|
|
924
|
+
return resolveTransientHoldDispatch(detourEntry, lane, now);
|
|
2862
925
|
}
|
|
2863
926
|
// Detour expiry or invalidation must not expire the ordinary task. Drop only
|
|
2864
927
|
// this model lane and select from ordinary/shared state below.
|
|
@@ -2932,16 +995,11 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2932
995
|
const detour = transientDetourAccount(config, entry, now, quotaScope, selectionOptions);
|
|
2933
996
|
entry.transientHoldSince ??= now;
|
|
2934
997
|
entry.lastUsedAt = now;
|
|
2935
|
-
if (detour !== null && detour !== entry.accountId) {
|
|
2936
|
-
entry.transientDetourAccountId = detour;
|
|
2937
|
-
// Deliberately no promoteActiveCodexAccount and no rebind: this is one request routing
|
|
2938
|
-
// around a blip, not the pool deciding where the conversation now lives.
|
|
2939
|
-
return { status: "selected", accountId: detour, affinity: { move: "detour", reason: "transient" } };
|
|
2940
|
-
}
|
|
2941
998
|
// No sibling can take it either -- the usual shape of a provider-wide 503. The binding
|
|
2942
999
|
// survives: "cannot send right now" and "forget which account owns this conversation"
|
|
2943
|
-
// are different answers
|
|
2944
|
-
|
|
1000
|
+
// are different answers. So is the third answer this used to give -- "send at the
|
|
1001
|
+
// failing account" -- now a bounded probe or a typed refusal (#4701).
|
|
1002
|
+
return resolveTransientHoldDispatch(entry, detour, now);
|
|
2945
1003
|
}
|
|
2946
1004
|
// A model-only exclusion does not invalidate the shared task binding. Health,
|
|
2947
1005
|
// generation, pause, cooldown, and failure evidence still retire it normally.
|
|
@@ -2989,6 +1047,38 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
2989
1047
|
// arrives) is the reason this request is starting cold, so it outranks having found nothing.
|
|
2990
1048
|
releaseReason ??= peekPendingReleaseReason(threadId);
|
|
2991
1049
|
|
|
1050
|
+
// FIRST PLACEMENT for a child thread (#4546, wp8). A child with no binding of its own used
|
|
1051
|
+
// to bind under the raw parent id -- an entry unrelated to the root's real binding -- or
|
|
1052
|
+
// land cold while its parent was being served warm somewhere. Consult the family's CURRENT
|
|
1053
|
+
// serving account first (detour included), then a compatible sibling's, and only then fall
|
|
1054
|
+
// through to cold placement. The child binds under its OWN key below: this is a warm start,
|
|
1055
|
+
// not a root-wide pin, so a later move of the parent never drags the child with it.
|
|
1056
|
+
//
|
|
1057
|
+
// Guarded on `entry === undefined`, which is strictly narrower than "has no usable binding":
|
|
1058
|
+
// a thread whose binding was just released above still holds its own history and re-decides
|
|
1059
|
+
// through the ordinary path. Only a thread that has never bound takes a family hint. That
|
|
1060
|
+
// also makes `preserveExistingModelScopedAffinity` unreachable here -- it is only ever set
|
|
1061
|
+
// while reusing an existing model-detour entry -- so this binds through the ordinary lane.
|
|
1062
|
+
if (threadId && entry === undefined && lineage) {
|
|
1063
|
+
const lineagePick = pickLineageServingAccount(
|
|
1064
|
+
config, lineage, now, quotaScope, selectionOptions, modelId,
|
|
1065
|
+
);
|
|
1066
|
+
if (lineagePick) {
|
|
1067
|
+
bindThreadAffinity(threadId, lineagePick.accountId, now, quotaScope);
|
|
1068
|
+
// Deliberately no promoteActiveCodexAccount: a family hint places THIS request, it does
|
|
1069
|
+
// not move the operator-visible shared cursor for unrelated new threads.
|
|
1070
|
+
return {
|
|
1071
|
+
status: "selected",
|
|
1072
|
+
accountId: lineagePick.accountId,
|
|
1073
|
+
// A pending release still outranks the hint as the reported reason, and consuming it
|
|
1074
|
+
// here is what stops the next request reporting the same release a second time.
|
|
1075
|
+
affinity: releaseReason === undefined
|
|
1076
|
+
? { move: "new_bind", reason: lineagePick.reason }
|
|
1077
|
+
: affinityAfterRelease(threadId, releaseReason),
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
|
|
2992
1082
|
// A request-scoped roster may still contain unhealthy candidates. Non-quota strategies return
|
|
2993
1083
|
// before the quota/failover helpers below, so prefer only shared-healthy roster members here;
|
|
2994
1084
|
// otherwise RR/fill-first can immediately re-pick a known failing account even when another
|
|
@@ -3123,6 +1213,10 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
3123
1213
|
selectionOptions,
|
|
3124
1214
|
!preserveSharedSelectionForModelDetour,
|
|
3125
1215
|
);
|
|
1216
|
+
// The shared cursor can name an account whose own roster denies this model, and an active
|
|
1217
|
+
// account never passes through the eligible list. Correct it for THIS request only -- nothing
|
|
1218
|
+
// is persisted -- and only toward an account eligibility already admitted (#4768).
|
|
1219
|
+
active = preferModelEntitledAccount(config, active, now, quotaScope, selectionOptions);
|
|
3126
1220
|
if (!isCodexAccountUsable(config, active, selectionOptions)) {
|
|
3127
1221
|
return hasConfiguredPoolAccount(config, active, selectionOptions)
|
|
3128
1222
|
? { status: "selected", accountId: active, affinity: affinityAfterRelease(threadId, releaseReason) }
|
|
@@ -3144,6 +1238,7 @@ export function resolveCodexAccountForThreadDetailed(
|
|
|
3144
1238
|
return { status: "selected", accountId: active, affinity: affinityAfterRelease(threadId, releaseReason) };
|
|
3145
1239
|
}
|
|
3146
1240
|
|
|
1241
|
+
|
|
3147
1242
|
export function recordCodexUpstreamOutcome(
|
|
3148
1243
|
config: OcxConfig,
|
|
3149
1244
|
accountId: string | null,
|
|
@@ -3158,8 +1253,12 @@ export function recordCodexUpstreamOutcome(
|
|
|
3158
1253
|
recordUpstreamHostFailure(meta.hostKey, { code: meta.lastFailureCode, now: meta.now ?? Date.now() });
|
|
3159
1254
|
}
|
|
3160
1255
|
if (!accountId) return;
|
|
1256
|
+
// Conclude the half-open recovery trial BEFORE the admissibility gate below (#4701): an
|
|
1257
|
+
// outcome that gate drops still ended this request, and a lease nobody hands back leaves the
|
|
1258
|
+
// next trial waiting out its deadline. The settle carries its own fences, so this is safe here.
|
|
1259
|
+
settleTransientProbeForOutcome(accountId, meta, classifyCodexUpstreamOutcome(outcome, meta.denial));
|
|
3161
1260
|
const writerGeneration = meta.writerGeneration ?? captureConfigGeneration();
|
|
3162
|
-
if (
|
|
1261
|
+
if (!isHealthAccountAdmissible(accountId, writerGeneration)) return;
|
|
3163
1262
|
const now = meta.now ?? Date.now();
|
|
3164
1263
|
const outcomeClass = classifyCodexUpstreamOutcome(outcome, meta.denial);
|
|
3165
1264
|
// Reject retired quota evidence before stale-credential cleanup or any shared mutation.
|
|
@@ -3204,12 +1303,12 @@ export function recordCodexUpstreamOutcome(
|
|
|
3204
1303
|
if (Object.keys(retained).length > 1) setScopedHealth(accountId, quotaScope, retained);
|
|
3205
1304
|
else deleteScopedHealth(accountId, quotaScope);
|
|
3206
1305
|
}
|
|
3207
|
-
const current =
|
|
1306
|
+
const current = getAccountHealth(accountId);
|
|
3208
1307
|
const cooldownUntil = getCodexAccountCooldownUntil(accountId, now);
|
|
3209
1308
|
// A leased probe that is still on its own cooldown generation proves the
|
|
3210
1309
|
// account recovered: clear the hard cooldown outright (#433).
|
|
3211
1310
|
if (cooldownUntil && probeMayClearCooldown(current, meta)) {
|
|
3212
|
-
|
|
1311
|
+
deleteAccountHealth(accountId);
|
|
3213
1312
|
return;
|
|
3214
1313
|
}
|
|
3215
1314
|
// Owning probe on a stale generation: the lease is done, but a newer 429
|
|
@@ -3221,7 +1320,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3221
1320
|
if (failoverEnabled && current && current.consecutiveFailures >= 2) {
|
|
3222
1321
|
const consecutiveSuccesses = (current.consecutiveSuccesses ?? 0) + 1;
|
|
3223
1322
|
if (consecutiveSuccesses < 2) {
|
|
3224
|
-
|
|
1323
|
+
setAccountHealth(accountId, {
|
|
3225
1324
|
...base!,
|
|
3226
1325
|
...preserved,
|
|
3227
1326
|
consecutiveSuccesses,
|
|
@@ -3231,14 +1330,14 @@ export function recordCodexUpstreamOutcome(
|
|
|
3231
1330
|
}
|
|
3232
1331
|
// Level 1 clears immediately; escalated accounts need two consecutive healthy terminals.
|
|
3233
1332
|
// Hard quota cooldown intentionally survives either recovery path.
|
|
3234
|
-
if (cooldownUntil)
|
|
3235
|
-
else
|
|
1333
|
+
if (cooldownUntil) setAccountHealth(accountId, { consecutiveFailures: 0, ...preserved });
|
|
1334
|
+
else deleteAccountHealth(accountId);
|
|
3236
1335
|
return;
|
|
3237
1336
|
}
|
|
3238
1337
|
if (outcomeClass === "caller") {
|
|
3239
1338
|
// A 4xx does not change account health, but it does conclude an in-flight
|
|
3240
1339
|
// probe — otherwise the lease would never be handed back.
|
|
3241
|
-
const current =
|
|
1340
|
+
const current = getAccountHealth(accountId);
|
|
3242
1341
|
const scopedProbe = meta.probeQuotaScope
|
|
3243
1342
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3244
1343
|
: undefined;
|
|
@@ -3246,7 +1345,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3246
1345
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3247
1346
|
}
|
|
3248
1347
|
if (ownsProbeLease(current, meta)) {
|
|
3249
|
-
|
|
1348
|
+
setAccountHealth(accountId, withProbeLeaseReleased(current!, now));
|
|
3250
1349
|
}
|
|
3251
1350
|
return;
|
|
3252
1351
|
}
|
|
@@ -3257,7 +1356,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3257
1356
|
// it and must not happen (#914). Conclude any owned probe lease, record the
|
|
3258
1357
|
// failure under the (provider, host) ledger when one is named, and leave
|
|
3259
1358
|
// account health, thread affinity, and the active account untouched.
|
|
3260
|
-
const current =
|
|
1359
|
+
const current = getAccountHealth(accountId);
|
|
3261
1360
|
const scopedProbe = meta.probeQuotaScope
|
|
3262
1361
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3263
1362
|
: undefined;
|
|
@@ -3265,7 +1364,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3265
1364
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3266
1365
|
}
|
|
3267
1366
|
if (ownsProbeLease(current, meta)) {
|
|
3268
|
-
|
|
1367
|
+
setAccountHealth(accountId, withProbeLeaseReleased(current!, now));
|
|
3269
1368
|
}
|
|
3270
1369
|
return;
|
|
3271
1370
|
}
|
|
@@ -3276,8 +1375,8 @@ export function recordCodexUpstreamOutcome(
|
|
|
3276
1375
|
// Record the failure so routing stops preferring it, but do not mark it for
|
|
3277
1376
|
// reauthentication and do not sweep its thread affinities: telling the user to
|
|
3278
1377
|
// re-login is wrong advice that cannot fix a workspace grant.
|
|
3279
|
-
|
|
3280
|
-
consecutiveFailures: (
|
|
1378
|
+
setAccountHealth(accountId, {
|
|
1379
|
+
consecutiveFailures: (getAccountHealth(accountId)?.consecutiveFailures ?? 0) + 1,
|
|
3281
1380
|
lastFailureStatus,
|
|
3282
1381
|
lastFailureAt: now,
|
|
3283
1382
|
});
|
|
@@ -3315,7 +1414,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3315
1414
|
* Affinity sweeping needs no tag: an affinity entry already carries a credential generation and
|
|
3316
1415
|
* self-invalidates on the next check, and re-adding swept entries would be a worse bug.
|
|
3317
1416
|
*/
|
|
3318
|
-
|
|
1417
|
+
setAccountHealth(accountId, {
|
|
3319
1418
|
consecutiveFailures: 1,
|
|
3320
1419
|
lastFailureStatus,
|
|
3321
1420
|
lastFailureAt: now,
|
|
@@ -3324,7 +1423,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3324
1423
|
? { credentialFailureGeneration: meta.credentialGeneration }
|
|
3325
1424
|
: {}),
|
|
3326
1425
|
});
|
|
3327
|
-
|
|
1426
|
+
deleteAllScopedHealth(accountId);
|
|
3328
1427
|
// The reauth flag carries the same provenance, so a replacement landing after this call cannot
|
|
3329
1428
|
// inherit a quarantine that was never about it.
|
|
3330
1429
|
markAccountNeedsReauth(accountId, writerGeneration, meta.credentialGeneration);
|
|
@@ -3385,13 +1484,13 @@ export function recordCodexUpstreamOutcome(
|
|
|
3385
1484
|
if (scopedProbe && meta.probeQuotaScope && ownsProbeLease(scopedProbe, meta)) {
|
|
3386
1485
|
setScopedHealth(accountId, meta.probeQuotaScope, withProbeLeaseReleased(scopedProbe, now));
|
|
3387
1486
|
}
|
|
3388
|
-
const prior =
|
|
1487
|
+
const prior = getAccountHealth(accountId);
|
|
3389
1488
|
// Every cooldown write bumps the generation so a probe issued against the
|
|
3390
1489
|
// previous cooldown can no longer clear this one (#433).
|
|
3391
1490
|
const cooldownGeneration = (prior?.cooldownGeneration ?? 0) + 1;
|
|
3392
1491
|
// A failed probe concludes its lease; an unrelated 429 leaves the live probe alone.
|
|
3393
1492
|
const ownsLease = ownsProbeLease(prior, meta);
|
|
3394
|
-
|
|
1493
|
+
setAccountHealth(accountId, {
|
|
3395
1494
|
consecutiveFailures: 0,
|
|
3396
1495
|
lastFailureStatus,
|
|
3397
1496
|
lastFailureAt: now,
|
|
@@ -3431,7 +1530,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3431
1530
|
}
|
|
3432
1531
|
|
|
3433
1532
|
// transient (connect_error / timeout / 5xx)
|
|
3434
|
-
const current =
|
|
1533
|
+
const current = getAccountHealth(accountId);
|
|
3435
1534
|
const scopedProbe = meta.probeQuotaScope
|
|
3436
1535
|
? scopedHealthFor(accountId, meta.probeQuotaScope)
|
|
3437
1536
|
: undefined;
|
|
@@ -3457,7 +1556,7 @@ export function recordCodexUpstreamOutcome(
|
|
|
3457
1556
|
now + escalationMs,
|
|
3458
1557
|
)
|
|
3459
1558
|
: undefined;
|
|
3460
|
-
|
|
1559
|
+
setAccountHealth(accountId, {
|
|
3461
1560
|
...preservedCooldownFields(transientBase),
|
|
3462
1561
|
consecutiveFailures,
|
|
3463
1562
|
lastFailureStatus,
|