claude-multiacc 2.0.18 → 2.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/bin/claude +136 -29
- package/bin/claude-accounts +30 -8
- package/bin/codex +45 -19
- package/bin/codex-accounts +11 -2
- package/docs/ACCOUNT_OPERATIONS.md +2 -0
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/package.json +1 -1
- package/tests/run-tests.sh +85 -10
- package/tests/test_codex_reset.py +66 -1
package/README.md
CHANGED
|
@@ -72,12 +72,12 @@ repo bin/ first on PATH (rc-file block) /root/claude-multiacc/ (addon repo)
|
|
|
72
72
|
4. Among the accounts that remain valid **on this machine** (an OAuth login —
|
|
73
73
|
`.credentials.json`, or a macOS Keychain item this session can open — or a
|
|
74
74
|
`server.token` that passes its first inference preflight) and not limit-excluded,
|
|
75
|
-
|
|
76
|
-
headroom
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
form a **30-percentage-point weekly-headroom band** around the account with the most
|
|
76
|
+
remaining headroom (see below). The pool **rotates away from the account it just handed
|
|
77
|
+
out** and samples the other accounts in that band at random. This prevents one nominally
|
|
78
|
+
emptiest account from taking every launch while near-peers idle, while accounts more than
|
|
79
|
+
30 points behind remain protected. `CLAUDE_MULTIACC_HEADROOM_BAND=0` restores strict
|
|
80
|
+
best-headroom selection; `CLAUDE_SHIM_SELECT=random` uses the whole eligible pool.
|
|
81
81
|
5. If every account is limit-excluded → the highest-headroom *authenticable* account
|
|
82
82
|
anyway + a warning in `selection.log` (degraded beats down: the 100% rule).
|
|
83
83
|
6. If nothing is usable at all → stock passthrough, with the reason in `selection.log`
|
|
@@ -88,11 +88,12 @@ Fable request consumes at once: the 5-hour **session** bucket, the **weekly all-
|
|
|
88
88
|
bucket, and the **weekly Fable** bucket. Anthropic's docs confirm these reset on very
|
|
89
89
|
different horizons — the session bucket refills every ~5 hours, but weekly buckets only
|
|
90
90
|
refill on the account's fixed weekly reset (days away). So the picker ranks primarily on
|
|
91
|
-
**weekly headroom** (the peak of the durable buckets)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
lowest wins.) **Exclusion** still fires on *any* bucket
|
|
91
|
+
**weekly headroom** (the peak of the durable buckets) to form the 30-point band. Accounts
|
|
92
|
+
inside that band are deliberately treated as peers and spread randomly; with a zero-width
|
|
93
|
+
band the self-healing session bucket is the tiebreaker. An account at 10% weekly remains
|
|
94
|
+
protected from one at 70% weekly because the latter is outside the band. (`score = weekly%
|
|
95
|
+
× 1000 + session%` in strict mode, lowest wins.) **Exclusion** still fires on *any* bucket
|
|
96
|
+
≥ 90% — a full session bucket
|
|
96
97
|
really does block right now — but that marker expires when the session resets, not days
|
|
97
98
|
later.
|
|
98
99
|
|
package/bin/claude
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# Every invocation runs under a randomly picked subscription account with limit headroom.
|
|
4
4
|
# Self-contained on purpose: no sourcing, so a broken repo file can never break `claude`.
|
|
5
5
|
# Selection: CLAUDE_CONFIG_DIR/CLAUDE_CODE_OAUTH_TOKEN passthrough > CLAUDE_ACCOUNT pin >
|
|
6
|
-
# random
|
|
6
|
+
# random within the 30-point headroom band > least-utilized fallback (degraded beats down).
|
|
7
7
|
# Accounts whose login is DEAD (expired refresh token, or a `.expired` marker from a
|
|
8
8
|
# failed auth) are never selected — not even as the all-limited fallback — because they
|
|
9
9
|
# fail every call outright; `claude-accounts expired` / `relogin` fix them.
|
|
@@ -271,6 +271,47 @@ scoped_over_tok() { # $1 = acct dir
|
|
|
271
271
|
'
|
|
272
272
|
}
|
|
273
273
|
|
|
274
|
+
CLIENT_RECOVERY_FETCHED=0
|
|
275
|
+
CLIENT_RECOVERY_PEAK=100
|
|
276
|
+
client_marker_recovered() { # $1 account dir, $2 marker
|
|
277
|
+
local fetched peak delay marked mk cutoff ck newer=0
|
|
278
|
+
# A client rejection is stronger than an immediately-following usage read: that
|
|
279
|
+
# endpoint can lag or serve a cached bucket. It is not stronger forever. Once a
|
|
280
|
+
# successful, below-threshold reading was fetched at least five minutes after the
|
|
281
|
+
# marker, it is newer first-hand evidence that the account recovered (or that an
|
|
282
|
+
# old/shared transcript was attributed incorrectly). Keeping such a marker until its
|
|
283
|
+
# claimed reset stranded five 0%-usage accounts in the live pool on 2026-09-03.
|
|
284
|
+
if grep -q 'reason=client-rate-limit' "$2" 2>/dev/null; then
|
|
285
|
+
fetched="$(telem_fetched_at "$1" 2>/dev/null || echo 0)"
|
|
286
|
+
peak="$(cutoff_field "$1" max_percent 2>/dev/null || echo 100)"
|
|
287
|
+
delay="${CLAUDE_MULTIACC_CLIENT_LIMIT_CONFIRM_DELAY:-300}"
|
|
288
|
+
num_ok "$delay" || delay=300
|
|
289
|
+
[ "$delay" -gt 3600 ] && delay=3600
|
|
290
|
+
marked="$(LC_ALL=C sed -n 's/.*marked_at=\([^ ]*\).*/\1/p' "$2" 2>/dev/null | head -1)"
|
|
291
|
+
mk="$(iso_key "$marked")"
|
|
292
|
+
cutoff=0
|
|
293
|
+
if num_ok "$fetched" && [ "$fetched" -ge "$delay" ]; then
|
|
294
|
+
cutoff=$((fetched - delay))
|
|
295
|
+
fi
|
|
296
|
+
ck="$(iso_key "$(iso_of_epoch "$cutoff")")"
|
|
297
|
+
# Sync and atomic copies change mtimes, so prefer the marker's semantic timestamp.
|
|
298
|
+
# Old marker formats fall back to mtime until the next writer upgrades them.
|
|
299
|
+
if num_ok "$mk" && [ "${#mk}" -eq 14 ] && num_ok "$ck"; then
|
|
300
|
+
[ "$mk" -le "$ck" ] && newer=1
|
|
301
|
+
elif num_ok "$fetched" \
|
|
302
|
+
&& [ "$(file_mtime "$2")" -le "$((fetched - delay))" ]; then
|
|
303
|
+
newer=1
|
|
304
|
+
fi
|
|
305
|
+
if [ "$newer" = 1 ] && num_ok "$peak" \
|
|
306
|
+
&& [ "$peak" -lt "${CLAUDE_MULTIACC_THRESHOLD:-90}" ]; then
|
|
307
|
+
CLIENT_RECOVERY_FETCHED="$fetched"
|
|
308
|
+
CLIENT_RECOVERY_PEAK="$peak"
|
|
309
|
+
return 0
|
|
310
|
+
fi
|
|
311
|
+
fi
|
|
312
|
+
return 1
|
|
313
|
+
}
|
|
314
|
+
|
|
274
315
|
marker_active() { # true if $1/.limited is still in force; clears cleanly-expired markers
|
|
275
316
|
local m="$1/.limited" reset="" tok
|
|
276
317
|
[ -f "$m" ] || return 1
|
|
@@ -285,6 +326,14 @@ marker_active() { # true if $1/.limited is still in force; clears cleanly-expire
|
|
|
285
326
|
rm -f "$m" 2>/dev/null
|
|
286
327
|
return 1
|
|
287
328
|
fi
|
|
329
|
+
if client_marker_recovered "$1" "$m"; then
|
|
330
|
+
printf '%s\n' "$CLIENT_RECOVERY_FETCHED" > "$1/.client-limit-cleared.$$" 2>/dev/null \
|
|
331
|
+
&& mv -f "$1/.client-limit-cleared.$$" "$1/.client-limit-cleared" 2>/dev/null \
|
|
332
|
+
|| rm -f "$1/.client-limit-cleared.$$" 2>/dev/null
|
|
333
|
+
rm -f "$m" 2>/dev/null
|
|
334
|
+
sel_log "$(basename "$1") client limit cleared by newer telemetry (${CLIENT_RECOVERY_PEAK}%)"
|
|
335
|
+
return 1
|
|
336
|
+
fi
|
|
288
337
|
# A park scoped to ONE model does not stop a run that will use another one.
|
|
289
338
|
tok="$(scoped_tok_of_marker "$1")"
|
|
290
339
|
if [ -n "$tok" ] && ! scoped_blocks_run "$tok"; then
|
|
@@ -396,6 +445,23 @@ sel_score_of() { # $1 = acct dir
|
|
|
396
445
|
printf '%s\n' $((w * 1000 + s))
|
|
397
446
|
}
|
|
398
447
|
|
|
448
|
+
# The direct shims use the same 30-point headroom band as pool-selection.v2. Strict
|
|
449
|
+
# best-headroom selection burned one account to its limit while equally healthy
|
|
450
|
+
# neighbours idled. A caller can set 0 to restore strict ranking.
|
|
451
|
+
HEADROOM_BAND="${CLAUDE_MULTIACC_HEADROOM_BAND:-30}"
|
|
452
|
+
case "$HEADROOM_BAND" in ''|*[!0-9]*|??????*) HEADROOM_BAND=30 ;; esac
|
|
453
|
+
[ "$HEADROOM_BAND" -gt 100 ] && HEADROOM_BAND=100
|
|
454
|
+
|
|
455
|
+
rank_weekly_of() { # $1 = acct dir -> comparable weekly use, or fail when unknown
|
|
456
|
+
local w
|
|
457
|
+
if w="$(fresh_field "$1" weekly_percent)" || w="$(fresh_field "$1" max_percent)"; then
|
|
458
|
+
printf '%s\n' "$w"
|
|
459
|
+
return 0
|
|
460
|
+
fi
|
|
461
|
+
[ "$SEL_DEGRADED" = 1 ] || return 1
|
|
462
|
+
stale_weekly "$1"
|
|
463
|
+
}
|
|
464
|
+
|
|
399
465
|
# The .limited marker's own fields (line 1: reset epoch; line 2: "bucket=…
|
|
400
466
|
# percent=… … reason=…"). All tolerant: an unreadable or bare marker answers
|
|
401
467
|
# "unknown", never an error — these feed the all-limited fallback only.
|
|
@@ -621,14 +687,14 @@ sess_transcript() { # $1 acct dir, $2 session id
|
|
|
621
687
|
}
|
|
622
688
|
|
|
623
689
|
# Newest still-in-force rejection this account's own sessions recorded.
|
|
624
|
-
# Prints "<reset-epoch> <rateLimitType>"; fails when there is none.
|
|
690
|
+
# Prints "<reset-epoch> <rateLimitType> <rejection-ISO>"; fails when there is none.
|
|
625
691
|
# This runs on EVERY invocation, so it is bounded on purpose: newest session first,
|
|
626
692
|
# stop at the first in-force rejection, and never read more than QUOTA_SCAN_MAX_FILES
|
|
627
693
|
# transcripts. Missing an older rejection costs nothing — a limit that is still in force
|
|
628
694
|
# rejects the very next request too, and that lands in a newer transcript.
|
|
629
695
|
client_limit_scan() { # $1 acct dir
|
|
630
696
|
local idx="$1/.sessions-index" memo="$1/.client-scan" id p line r t read_n=0 i last=""
|
|
631
|
-
local ln claim ts ck ak ttl
|
|
697
|
+
local ln claim ts ck ak ttl cleared sk
|
|
632
698
|
local ids=() claims=()
|
|
633
699
|
[ "${CLAUDE_MULTIACC_CLIENT_LIMITS:-1}" = "0" ] && return 1
|
|
634
700
|
[ -f "$idx" ] || return 1
|
|
@@ -672,12 +738,28 @@ client_limit_scan() { # $1 acct dir
|
|
|
672
738
|
| tail -1)"
|
|
673
739
|
[ -n "$line" ] || continue
|
|
674
740
|
case "$line" in *'"status":"rejected"'*|*'"status": "rejected"'*) ;; *) continue ;; esac
|
|
741
|
+
# A later successful usage read below the threshold supersedes an older rejection.
|
|
742
|
+
# Without this watermark, clearing its marker achieved nothing: the next launch
|
|
743
|
+
# scanned the same transcript and recreated the same days-long park.
|
|
744
|
+
ts="$(printf '%s' "$line" | LC_ALL=C sed -n \
|
|
745
|
+
's/.*"timestamp"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')"
|
|
746
|
+
ck="$(iso_key "$ts")"
|
|
747
|
+
cleared=""
|
|
748
|
+
[ -f "$1/.client-limit-cleared" ] \
|
|
749
|
+
&& { IFS= read -r cleared < "$1/.client-limit-cleared" 2>/dev/null || cleared=""; }
|
|
750
|
+
if num_ok "$cleared"; then
|
|
751
|
+
sk="$(iso_key "$(iso_of_epoch "$cleared")")"
|
|
752
|
+
if num_ok "$ck" && num_ok "$sk"; then
|
|
753
|
+
[ "$ck" -le "$sk" ] && continue
|
|
754
|
+
elif [ "$(file_mtime "$p")" -le "$cleared" ]; then
|
|
755
|
+
continue
|
|
756
|
+
fi
|
|
757
|
+
fi
|
|
675
758
|
# A rejection recorded BEFORE this account took the session over belongs to whoever
|
|
676
759
|
# was running it then, not to us. Undatable => not attributed (fail open).
|
|
677
760
|
if [ -n "$claim" ]; then
|
|
678
|
-
ts="$(printf '%s' "$line" | LC_ALL=C sed -n 's/.*"timestamp"[[:space:]]*:[[:space:]]*"\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[^"]*\)".*/\1/p')"
|
|
679
761
|
[ -n "$ts" ] || continue
|
|
680
|
-
|
|
762
|
+
ak="$(iso_key "$claim")"
|
|
681
763
|
num_ok "$ck" || continue
|
|
682
764
|
num_ok "$ak" || continue
|
|
683
765
|
[ "$ck" -lt "$ak" ] && continue
|
|
@@ -687,7 +769,7 @@ client_limit_scan() { # $1 acct dir
|
|
|
687
769
|
num_ok "$r" || continue
|
|
688
770
|
[ "$r" -gt "$now" ] || continue
|
|
689
771
|
t="$(printf '%s' "$line" | LC_ALL=C sed -n 's/.*"rateLimitType"[[:space:]]*:[[:space:]]*"\([A-Za-z0-9_.-]*\)".*/\1/p')"
|
|
690
|
-
printf '%s %s\n' "$r" "${t:-unknown}"
|
|
772
|
+
printf '%s %s %s\n' "$r" "${t:-unknown}" "${ts:--}"
|
|
691
773
|
return 0
|
|
692
774
|
done
|
|
693
775
|
printf '%s\n' "$now" 2>/dev/null > "$memo.$$" \
|
|
@@ -749,8 +831,8 @@ mark_client_auth_dead() { # $1 acct dir
|
|
|
749
831
|
fi
|
|
750
832
|
}
|
|
751
833
|
|
|
752
|
-
mark_client_limit() { # $1 acct dir, $2 reset epoch, $3 rate limit type
|
|
753
|
-
local m="$1/.limited" cur=""
|
|
834
|
+
mark_client_limit() { # $1 acct dir, $2 reset epoch, $3 rate limit type, $4 rejection ISO
|
|
835
|
+
local m="$1/.limited" cur="" marked="$4" mk
|
|
754
836
|
# Never shorten a marker that already reaches further out (a weekly park must
|
|
755
837
|
# survive a 5h report), and never rewrite the same one on every invocation.
|
|
756
838
|
if [ -f "$m" ]; then
|
|
@@ -758,9 +840,13 @@ mark_client_limit() { # $1 acct dir, $2 reset epoch, $3 rate limit type
|
|
|
758
840
|
num_ok "$cur" || cur=0
|
|
759
841
|
[ "$cur" -ge "$2" ] && return 0
|
|
760
842
|
fi
|
|
843
|
+
mk="$(iso_key "$marked")"
|
|
844
|
+
if ! num_ok "$mk" || [ "${#mk}" -ne 14 ]; then
|
|
845
|
+
marked="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
846
|
+
fi
|
|
761
847
|
{
|
|
762
848
|
echo "$2"
|
|
763
|
-
echo "bucket=client:$3 percent=100 marked_at=$
|
|
849
|
+
echo "bucket=client:$3 percent=100 marked_at=$marked reason=client-rate-limit"
|
|
764
850
|
} 2>/dev/null > "$1/.limited.$$" \
|
|
765
851
|
&& mv -f "$1/.limited.$$" "$m" 2>/dev/null \
|
|
766
852
|
|| rm -f "$1/.limited.$$" 2>/dev/null || true
|
|
@@ -1023,7 +1109,11 @@ for d in "$ACC_ROOT"/acct-*; do
|
|
|
1023
1109
|
# `claude-accounts status`, to a concurrent run in another terminal, and to sync.
|
|
1024
1110
|
# The cost is bounded by the scan's own file budget and its clean-result memo.
|
|
1025
1111
|
if lim="$(client_limit_scan "$d")"; then
|
|
1026
|
-
|
|
1112
|
+
lim_reset="${lim%% *}"
|
|
1113
|
+
lim_rest="${lim#* }"
|
|
1114
|
+
lim_type="${lim_rest%% *}"
|
|
1115
|
+
lim_marked="${lim_rest#* }"
|
|
1116
|
+
mark_client_limit "$d" "$lim_reset" "$lim_type" "$lim_marked"
|
|
1027
1117
|
continue
|
|
1028
1118
|
fi
|
|
1029
1119
|
over_threshold "$d" && continue
|
|
@@ -1092,44 +1182,53 @@ remember_pick() { # $1 acct dir — best effort. stderr is silenced BEFORE the r
|
|
|
1092
1182
|
# user's claude arguments.
|
|
1093
1183
|
PICK_DIR=""
|
|
1094
1184
|
PICK_SCORE=""
|
|
1185
|
+
PICK_BAND_COUNT=0
|
|
1186
|
+
PICK_STRICT=0
|
|
1095
1187
|
pick_best() { # args: candidate dirs
|
|
1096
|
-
local d avoid best="" bestv=1000000 ties=0 i n
|
|
1097
|
-
local cand=() score=()
|
|
1188
|
+
local d w avoid best="" bestv=1000000 bestw=101 ceiling=100 ties=0 i n match
|
|
1189
|
+
local cand=() score=() weekly=() known=() pool=()
|
|
1098
1190
|
avoid="$(last_pick_id)"
|
|
1099
1191
|
for d in "$@"; do
|
|
1100
1192
|
cand+=("$d")
|
|
1101
1193
|
score+=("$(sel_score_of "$d")")
|
|
1194
|
+
if w="$(rank_weekly_of "$d")"; then weekly+=("$w"); known+=(1)
|
|
1195
|
+
else weekly+=(100); known+=(0); fi
|
|
1102
1196
|
done
|
|
1103
1197
|
n=${#cand[@]}
|
|
1104
1198
|
i=0
|
|
1105
1199
|
while [ "$i" -lt "$n" ]; do
|
|
1106
1200
|
[ "${score[$i]}" -lt "$bestv" ] && bestv="${score[$i]}"
|
|
1201
|
+
[ "${known[$i]}" = 1 ] && [ "${weekly[$i]}" -lt "$bestw" ] && bestw="${weekly[$i]}"
|
|
1107
1202
|
i=$((i + 1))
|
|
1108
1203
|
done
|
|
1109
|
-
|
|
1204
|
+
ceiling=$((bestw + HEADROOM_BAND)); [ "$ceiling" -gt 100 ] && ceiling=100
|
|
1110
1205
|
i=0
|
|
1111
1206
|
while [ "$i" -lt "$n" ]; do
|
|
1112
|
-
|
|
1207
|
+
match=0
|
|
1208
|
+
if [ "$PICK_STRICT" = 0 ] && [ "$HEADROOM_BAND" -gt 0 ] && [ "$bestw" -le 100 ]; then
|
|
1209
|
+
[ "${known[$i]}" = 1 ] && [ "${weekly[$i]}" -le "$ceiling" ] && match=1
|
|
1210
|
+
else
|
|
1211
|
+
[ "${score[$i]}" -eq "$bestv" ] && match=1
|
|
1212
|
+
fi
|
|
1213
|
+
[ "$match" = 1 ] && pool+=("$i")
|
|
1214
|
+
i=$((i + 1))
|
|
1215
|
+
done
|
|
1216
|
+
PICK_BAND_COUNT=${#pool[@]}
|
|
1217
|
+
# Reservoir-sample inside the band, skipping the account just handed out.
|
|
1218
|
+
for i in "${pool[@]}"; do
|
|
1219
|
+
if [ "${cand[$i]##*/}" != "$avoid" ]; then
|
|
1113
1220
|
ties=$((ties + 1))
|
|
1114
1221
|
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
1115
1222
|
fi
|
|
1116
|
-
i=$((i + 1))
|
|
1117
1223
|
done
|
|
1118
1224
|
if [ -z "$best" ]; then
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
i=0
|
|
1123
|
-
while [ "$i" -lt "$n" ]; do
|
|
1124
|
-
if [ "${score[$i]}" -eq "$bestv" ]; then
|
|
1125
|
-
ties=$((ties + 1))
|
|
1126
|
-
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
1127
|
-
fi
|
|
1128
|
-
i=$((i + 1))
|
|
1225
|
+
for i in "${pool[@]}"; do
|
|
1226
|
+
ties=$((ties + 1))
|
|
1227
|
+
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
1129
1228
|
done
|
|
1130
1229
|
fi
|
|
1131
1230
|
PICK_DIR="$best"
|
|
1132
|
-
PICK_SCORE="$
|
|
1231
|
+
PICK_SCORE="$(sel_score_of "$best")"
|
|
1133
1232
|
}
|
|
1134
1233
|
|
|
1135
1234
|
# Telemetry going stale is not a per-run detail, it is a pool-wide outage: with no
|
|
@@ -1173,6 +1272,7 @@ if [ "${#eligible[@]}" -gt 0 ]; then
|
|
|
1173
1272
|
[ "$degraded" = 1 ] && SEL_DEGRADED=1
|
|
1174
1273
|
if [ "${CLAUDE_SHIM_SELECT:-headroom}" = "random" ]; then
|
|
1175
1274
|
PICK_DIR="${eligible[$((RANDOM % ${#eligible[@]}))]}"
|
|
1275
|
+
PICK_BAND_COUNT=${#eligible[@]}
|
|
1176
1276
|
else
|
|
1177
1277
|
pick_best "${eligible[@]}"
|
|
1178
1278
|
fi
|
|
@@ -1191,7 +1291,9 @@ else
|
|
|
1191
1291
|
if [ "${#soft[@]}" -gt 0 ]; then
|
|
1192
1292
|
assess_telemetry "${soft[@]}"
|
|
1193
1293
|
[ "$degraded" = 1 ] && SEL_DEGRADED=1
|
|
1294
|
+
PICK_STRICT=1
|
|
1194
1295
|
pick_best "${soft[@]}"
|
|
1296
|
+
PICK_STRICT=0
|
|
1195
1297
|
else
|
|
1196
1298
|
# Every account is exhausted RIGHT NOW: nothing serves, so hand out the one
|
|
1197
1299
|
# that unblocks first — its rejection window is the shortest.
|
|
@@ -1300,9 +1402,12 @@ if [ "$blind" = 1 ]; then
|
|
|
1300
1402
|
# DEGRADED still ranks, on old readings that remain true; BLIND cannot rank at all and
|
|
1301
1403
|
# is a coin flip. Calling both of them "random" would send someone hunting the wrong bug.
|
|
1302
1404
|
if [ "$degraded" = 1 ]; then
|
|
1303
|
-
sel_log "$acct weekly=$(stale_weekly "$pick" || echo '?')% session=?%
|
|
1405
|
+
sel_log "$acct weekly=$(stale_weekly "$pick" || echo '?')% session=?%" \
|
|
1406
|
+
"ranking=DEGRADED telemetry-age=${blind_age}s band=${HEADROOM_BAND}" \
|
|
1407
|
+
"band-count=${PICK_BAND_COUNT} pwd=$PWD"
|
|
1304
1408
|
else
|
|
1305
|
-
sel_log "$acct weekly=?% session=?% ranking=BLIND telemetry-age=${blind_age}s
|
|
1409
|
+
sel_log "$acct weekly=?% session=?% ranking=BLIND telemetry-age=${blind_age}s" \
|
|
1410
|
+
"band=${HEADROOM_BAND} band-count=${PICK_BAND_COUNT} pwd=$PWD"
|
|
1306
1411
|
fi
|
|
1307
1412
|
# Terminal only, at most hourly — a service-spawned `claude -p` must keep its stderr
|
|
1308
1413
|
# byte-clean, and this is advice, never a failure.
|
|
@@ -1324,7 +1429,9 @@ if [ "$blind" = 1 ]; then
|
|
|
1324
1429
|
fi
|
|
1325
1430
|
fi
|
|
1326
1431
|
else
|
|
1327
|
-
sel_log "$acct weekly=$(fresh_field "$pick" weekly_percent || echo '?')%
|
|
1432
|
+
sel_log "$acct weekly=$(fresh_field "$pick" weekly_percent || echo '?')%" \
|
|
1433
|
+
"session=$(fresh_field "$pick" session_percent || echo '?')%" \
|
|
1434
|
+
"band=${HEADROOM_BAND} band-count=${PICK_BAND_COUNT} pwd=$PWD"
|
|
1328
1435
|
fi
|
|
1329
1436
|
|
|
1330
1437
|
export CLAUDE_SHIM_ACTIVE=1
|
package/bin/claude-accounts
CHANGED
|
@@ -1520,6 +1520,14 @@ REFRESH_DENIED_BACKOFF = 21600 # 4xx = grant likely revoked: 6h; re-login neede
|
|
|
1520
1520
|
# exactly which ceremony fixes it.
|
|
1521
1521
|
USAGE_DENIED_BACKOFF = 21600
|
|
1522
1522
|
USAGE_FAIL_BACKOFF = 900 # anything else non-2xx: 15 min, doubling to 30
|
|
1523
|
+
# A real client 429 beats an immediately-following usage response, which may be cached.
|
|
1524
|
+
# A later successful response under the threshold is newer first-hand evidence and must
|
|
1525
|
+
# release the account instead of preserving a false marker until a days-away reset.
|
|
1526
|
+
try:
|
|
1527
|
+
CLIENT_LIMIT_CONFIRM_DELAY = max(
|
|
1528
|
+
0, min(3600, int(os.environ.get('CLAUDE_MULTIACC_CLIENT_LIMIT_CONFIRM_DELAY', '300'))))
|
|
1529
|
+
except ValueError:
|
|
1530
|
+
CLIENT_LIMIT_CONFIRM_DELAY = 300
|
|
1523
1531
|
|
|
1524
1532
|
def say(msg):
|
|
1525
1533
|
if not quiet:
|
|
@@ -2207,23 +2215,37 @@ for acct in manifest.get('accounts', []):
|
|
|
2207
2215
|
# A shim-written marker outlives a clean limits pass while its own window
|
|
2208
2216
|
# is still open:
|
|
2209
2217
|
# error-cooldown — the account failed a real call moments ago.
|
|
2210
|
-
# client-rate-limit — Claude Code itself was REJECTED on this account
|
|
2211
|
-
#
|
|
2212
|
-
#
|
|
2213
|
-
#
|
|
2214
|
-
#
|
|
2215
|
-
# the 429.
|
|
2218
|
+
# client-rate-limit — Claude Code itself was REJECTED on this account.
|
|
2219
|
+
# Preserve that over a possibly-cached response during
|
|
2220
|
+
# a short grace period. A later successful low-usage
|
|
2221
|
+
# response disproves it; otherwise false/shared-session
|
|
2222
|
+
# markers strand recovered accounts for days.
|
|
2216
2223
|
keep = False
|
|
2224
|
+
client_recovered = False
|
|
2217
2225
|
try:
|
|
2218
2226
|
txt = open(mpath).read()
|
|
2219
2227
|
first = txt.splitlines()[0] if txt else ''
|
|
2220
|
-
|
|
2221
|
-
|
|
2228
|
+
active = first.isdigit() and int(first) > now
|
|
2229
|
+
marked_at = next((part[10:] for part in txt.split()
|
|
2230
|
+
if part.startswith('marked_at=')), '')
|
|
2231
|
+
marked_epoch = parse_iso(marked_at)
|
|
2232
|
+
if marked_epoch is None:
|
|
2233
|
+
marked_epoch = os.path.getmtime(mpath)
|
|
2234
|
+
recent_client = now - marked_epoch < CLIENT_LIMIT_CONFIRM_DELAY
|
|
2235
|
+
client_recovered = active and 'reason=client-rate-limit' in txt \
|
|
2236
|
+
and not recent_client
|
|
2237
|
+
if active and ('reason=error-cooldown' in txt
|
|
2238
|
+
or ('reason=client-rate-limit' in txt and recent_client)):
|
|
2222
2239
|
keep = True
|
|
2223
2240
|
except Exception:
|
|
2224
2241
|
pass
|
|
2225
2242
|
if not keep:
|
|
2226
2243
|
os.remove(mpath)
|
|
2244
|
+
if client_recovered:
|
|
2245
|
+
cleared = os.path.join(d, '.client-limit-cleared')
|
|
2246
|
+
with open(cleared + '.tmp', 'w') as f:
|
|
2247
|
+
f.write(f'{int(now)}\n')
|
|
2248
|
+
os.replace(cleared + '.tmp', cleared)
|
|
2227
2249
|
say(f'{aid}: marker cleared (max {maxp}%)')
|
|
2228
2250
|
if not quiet:
|
|
2229
2251
|
detail = ' '.join(f"{b['name']}={b['percent']}%" for b in buckets)
|
package/bin/codex
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
# Every invocation runs under a randomly picked ChatGPT subscription account with limit
|
|
4
4
|
# headroom. Self-contained on purpose: no sourcing, so a broken repo file can never
|
|
5
5
|
# break `codex`.
|
|
6
|
-
# Selection: CODEX_HOME passthrough > CODEX_ACCOUNT pin >
|
|
7
|
-
#
|
|
6
|
+
# Selection: CODEX_HOME passthrough > CODEX_ACCOUNT pin > random among limit-eligible
|
|
7
|
+
# accounts in the 30-point headroom band > least-utilized fallback (degraded beats down).
|
|
8
8
|
# Accounts whose login is DEAD (a `.expired` marker from a failed refresh/verify/run)
|
|
9
9
|
# are never selected — not even as the all-limited fallback — because they fail every
|
|
10
10
|
# call outright; `codex-accounts expired` / `relogin` fix them.
|
|
@@ -154,6 +154,19 @@ sel_score_of() { # $1 = acct dir
|
|
|
154
154
|
printf '%s\n' $((w * 1000 + s))
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
HEADROOM_BAND="${CODEX_MULTIACC_HEADROOM_BAND:-30}"
|
|
158
|
+
case "$HEADROOM_BAND" in ''|*[!0-9]*|??????*) HEADROOM_BAND=30 ;; esac
|
|
159
|
+
[ "$HEADROOM_BAND" -gt 100 ] && HEADROOM_BAND=100
|
|
160
|
+
|
|
161
|
+
rank_weekly_of() { # $1 = acct dir -> comparable weekly use, or fail when unknown
|
|
162
|
+
local w
|
|
163
|
+
if w="$(fresh_field "$1" weekly_percent)" || w="$(fresh_field "$1" max_percent)"; then
|
|
164
|
+
printf '%s\n' "$w"
|
|
165
|
+
return 0
|
|
166
|
+
fi
|
|
167
|
+
return 1
|
|
168
|
+
}
|
|
169
|
+
|
|
157
170
|
# Backstop for a lost/failed marker write: fresh telemetry with ANY bucket at/over the
|
|
158
171
|
# threshold excludes the account even if .limited is missing. Stale/unreadable => not
|
|
159
172
|
# over (fail open — telemetry must never invent exclusions).
|
|
@@ -460,55 +473,66 @@ remember_pick() { # $1 acct dir — best effort. stderr is silenced BEFORE the r
|
|
|
460
473
|
# user's codex arguments.
|
|
461
474
|
PICK_DIR=""
|
|
462
475
|
PICK_SCORE=""
|
|
476
|
+
PICK_BAND_COUNT=0
|
|
477
|
+
PICK_STRICT=0
|
|
463
478
|
pick_best() { # args: candidate dirs
|
|
464
|
-
local d avoid best="" bestv=1000000 ties=0 i n
|
|
465
|
-
local cand=() score=()
|
|
479
|
+
local d w avoid best="" bestv=1000000 bestw=101 ceiling=100 ties=0 i n match
|
|
480
|
+
local cand=() score=() weekly=() known=() pool=()
|
|
466
481
|
avoid="$(last_pick_id)"
|
|
467
482
|
for d in "$@"; do
|
|
468
483
|
cand+=("$d")
|
|
469
484
|
score+=("$(sel_score_of "$d")")
|
|
485
|
+
if w="$(rank_weekly_of "$d")"; then weekly+=("$w"); known+=(1)
|
|
486
|
+
else weekly+=(100); known+=(0); fi
|
|
470
487
|
done
|
|
471
488
|
n=${#cand[@]}
|
|
472
489
|
i=0
|
|
473
490
|
while [ "$i" -lt "$n" ]; do
|
|
474
491
|
[ "${score[$i]}" -lt "$bestv" ] && bestv="${score[$i]}"
|
|
492
|
+
[ "${known[$i]}" = 1 ] && [ "${weekly[$i]}" -lt "$bestw" ] && bestw="${weekly[$i]}"
|
|
475
493
|
i=$((i + 1))
|
|
476
494
|
done
|
|
477
|
-
|
|
495
|
+
ceiling=$((bestw + HEADROOM_BAND)); [ "$ceiling" -gt 100 ] && ceiling=100
|
|
478
496
|
i=0
|
|
479
497
|
while [ "$i" -lt "$n" ]; do
|
|
480
|
-
|
|
498
|
+
match=0
|
|
499
|
+
if [ "$PICK_STRICT" = 0 ] && [ "$HEADROOM_BAND" -gt 0 ] && [ "$bestw" -le 100 ]; then
|
|
500
|
+
[ "${known[$i]}" = 1 ] && [ "${weekly[$i]}" -le "$ceiling" ] && match=1
|
|
501
|
+
else
|
|
502
|
+
[ "${score[$i]}" -eq "$bestv" ] && match=1
|
|
503
|
+
fi
|
|
504
|
+
[ "$match" = 1 ] && pool+=("$i")
|
|
505
|
+
i=$((i + 1))
|
|
506
|
+
done
|
|
507
|
+
PICK_BAND_COUNT=${#pool[@]}
|
|
508
|
+
for i in "${pool[@]}"; do
|
|
509
|
+
if [ "${cand[$i]##*/}" != "$avoid" ]; then
|
|
481
510
|
ties=$((ties + 1))
|
|
482
511
|
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
483
512
|
fi
|
|
484
|
-
i=$((i + 1))
|
|
485
513
|
done
|
|
486
514
|
if [ -z "$best" ]; then
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
i=0
|
|
491
|
-
while [ "$i" -lt "$n" ]; do
|
|
492
|
-
if [ "${score[$i]}" -eq "$bestv" ]; then
|
|
493
|
-
ties=$((ties + 1))
|
|
494
|
-
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
495
|
-
fi
|
|
496
|
-
i=$((i + 1))
|
|
515
|
+
for i in "${pool[@]}"; do
|
|
516
|
+
ties=$((ties + 1))
|
|
517
|
+
[ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
|
|
497
518
|
done
|
|
498
519
|
fi
|
|
499
520
|
PICK_DIR="$best"
|
|
500
|
-
PICK_SCORE="$
|
|
521
|
+
PICK_SCORE="$(sel_score_of "$best")"
|
|
501
522
|
}
|
|
502
523
|
|
|
503
524
|
if [ "${#eligible[@]}" -gt 0 ]; then
|
|
504
525
|
if [ "${CODEX_SHIM_SELECT:-headroom}" = "random" ]; then
|
|
505
526
|
PICK_DIR="${eligible[$((RANDOM % ${#eligible[@]}))]}"
|
|
527
|
+
PICK_BAND_COUNT=${#eligible[@]}
|
|
506
528
|
else
|
|
507
529
|
pick_best "${eligible[@]}"
|
|
508
530
|
fi
|
|
509
531
|
else
|
|
510
532
|
# Every account is limit-marked: degraded service beats a hard failure (100% rule).
|
|
533
|
+
PICK_STRICT=1
|
|
511
534
|
pick_best "${valid[@]}"
|
|
535
|
+
PICK_STRICT=0
|
|
512
536
|
sel_log "all-limited fallback=$(basename "$PICK_DIR") weekly=$(fresh_field "$PICK_DIR" weekly_percent || echo '?')%"
|
|
513
537
|
fi
|
|
514
538
|
pick="$PICK_DIR"
|
|
@@ -538,7 +562,9 @@ if [ "$stale" = 1 ] && [ -x "$SELF_DIR/codex-accounts" ]; then
|
|
|
538
562
|
fi
|
|
539
563
|
|
|
540
564
|
acct="$(basename "$pick")"
|
|
541
|
-
sel_log "$acct weekly=$(fresh_field "$pick" weekly_percent || echo '?')%
|
|
565
|
+
sel_log "$acct weekly=$(fresh_field "$pick" weekly_percent || echo '?')%" \
|
|
566
|
+
"session=$(fresh_field "$pick" session_percent || echo '?')%" \
|
|
567
|
+
"band=${HEADROOM_BAND} band-count=${PICK_BAND_COUNT} pwd=$PWD"
|
|
542
568
|
|
|
543
569
|
export CODEX_SHIM_ACTIVE=1
|
|
544
570
|
|
package/bin/codex-accounts
CHANGED
|
@@ -1492,8 +1492,17 @@ for acct in manifest.get('accounts', []):
|
|
|
1492
1492
|
b for b in buckets if b['percent'] >= threshold]
|
|
1493
1493
|
mpath = os.path.join(d, '.limited')
|
|
1494
1494
|
if offenders:
|
|
1495
|
-
|
|
1496
|
-
|
|
1495
|
+
# ONE bucket, described consistently: the marker's epoch is the reset of
|
|
1496
|
+
# the bucket its detail line names. It used to pair the highest PERCENT
|
|
1497
|
+
# with the LATEST reset over every offender, and the two came from
|
|
1498
|
+
# different buckets — a 100% five-hour window resetting tonight written
|
|
1499
|
+
# with a seven-day epoch — so everything that trusts the marker parked
|
|
1500
|
+
# the account for days over a window that refills in hours. #20 fixed
|
|
1501
|
+
# this for the claude pool and left the codex writer behind; app-robot
|
|
1502
|
+
# reads both with the same rule. Among offenders the longest-lived wins,
|
|
1503
|
+
# since any bucket over the threshold stays there until its own reset.
|
|
1504
|
+
worst = max(offenders, key=lambda b: (int(b['resets_epoch']), b['percent']))
|
|
1505
|
+
reset_epoch = int(worst['resets_epoch'])
|
|
1497
1506
|
# Atomic: a concurrent shim must never read a half-written marker.
|
|
1498
1507
|
with open(mpath + '.tmp', 'w') as f:
|
|
1499
1508
|
f.write(f'{reset_epoch}\n')
|
|
@@ -332,8 +332,10 @@ derived from the pool root. Uninstalling an instance removes only that instance'
|
|
|
332
332
|
| `CLAUDE_ACCOUNT=acct-03` | pin this invocation to one account (wins over markers) |
|
|
333
333
|
| `CLAUDE_CONFIG_DIR=...` | shim passes straight through (scripts can pin the old way) |
|
|
334
334
|
| `CLAUDE_MULTIACC_DISABLE=1` | bypass selection entirely |
|
|
335
|
+
| `CLAUDE_MULTIACC_HEADROOM_BAND=0..100` | weekly points treated as peers; default `30`, `0` is strict |
|
|
335
336
|
| `CLAUDE_MULTIACC_CLIENT_LIMITS=0` | ignore the client's own rate-limit records |
|
|
336
337
|
| `CLAUDE_MULTIACC_CLIENT_SCAN_TTL=<s>` | clean client-limit scan cache; default 20s |
|
|
338
|
+
| `CLAUDE_MULTIACC_CLIENT_LIMIT_CONFIRM_DELAY=<s>` | low-telemetry recovery grace; default 300s |
|
|
337
339
|
| `CLAUDE_SHIM_RETRY=0` | disable the `-p` auto-retry |
|
|
338
340
|
| `CLAUDE_ACCOUNTS_ROOT=...` | relocate the pool; legacy spelling is `CLAUDE_ACCOUNTS_DIR` |
|
|
339
341
|
| `CLAUDE_MULTIACC_SYNC_TARGET=...` | sync target, overriding the manifest; `none` = local-only |
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
package/tests/run-tests.sh
CHANGED
|
@@ -331,6 +331,35 @@ done
|
|
|
331
331
|
[ "$all2" = "1" ] && t_ok "picks the highest weekly-headroom account (weekly 20 over 80)" \
|
|
332
332
|
|| t_fail "headroom selection" "picked the more-utilized account"
|
|
333
333
|
|
|
334
|
+
# The default 30-point band prevents strict headroom ranking from burning one account
|
|
335
|
+
# to zero while its near-peers idle. Both boundaries participate; 31 points does not.
|
|
336
|
+
lj 0 10 10 > "$ACC/acct-01/limits.json"
|
|
337
|
+
lj 30 10 30 > "$ACC/acct-02/limits.json"
|
|
338
|
+
hits1=0; hits2=0
|
|
339
|
+
for _ in $(seq 1 20); do
|
|
340
|
+
case "$(claude 2>&1)" in
|
|
341
|
+
*CFG=acct-01*) hits1=$((hits1+1)) ;;
|
|
342
|
+
*CFG=acct-02*) hits2=$((hits2+1)) ;;
|
|
343
|
+
esac
|
|
344
|
+
done
|
|
345
|
+
{ [ "$hits1" -gt 0 ] && [ "$hits2" -gt 0 ]; } \
|
|
346
|
+
&& t_ok "30-point headroom band spreads direct Claude launches (acct-01=$hits1 acct-02=$hits2)" \
|
|
347
|
+
|| t_fail "Claude headroom band spread" "acct-01=$hits1 acct-02=$hits2"
|
|
348
|
+
lj 31 10 31 > "$ACC/acct-02/limits.json"
|
|
349
|
+
all1=1
|
|
350
|
+
for _ in $(seq 1 10); do
|
|
351
|
+
case "$(claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
352
|
+
done
|
|
353
|
+
[ "$all1" = 1 ] && t_ok "an account 31 points behind stays outside the Claude band" \
|
|
354
|
+
|| t_fail "Claude headroom band boundary" "the 31-point account was selected"
|
|
355
|
+
lj 20 10 20 > "$ACC/acct-02/limits.json"
|
|
356
|
+
all1=1
|
|
357
|
+
for _ in $(seq 1 10); do
|
|
358
|
+
case "$(CLAUDE_MULTIACC_HEADROOM_BAND=0 claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
359
|
+
done
|
|
360
|
+
[ "$all1" = 1 ] && t_ok "Claude headroom band 0 restores strict ranking" \
|
|
361
|
+
|| t_fail "Claude zero headroom band" "the runner-up was selected"
|
|
362
|
+
|
|
334
363
|
# THE KEY CASE from the research: a high (but sub-threshold) SESSION bucket must NOT
|
|
335
364
|
# deprioritize an account whose weekly headroom is better. acct-01: session 85, weekly 10;
|
|
336
365
|
# acct-02: session 20, weekly 70. Both eligible (max<90). acct-01 is the better pick —
|
|
@@ -344,14 +373,14 @@ done
|
|
|
344
373
|
[ "$all1" = "1" ] && t_ok "high session does NOT beat better weekly headroom (10w/85s over 70w/20s)" \
|
|
345
374
|
|| t_fail "weekly-over-session" "ranked the account with less weekly headroom higher"
|
|
346
375
|
|
|
347
|
-
#
|
|
376
|
+
# In strict mode, equal weekly usage still uses session usage as its tiebreaker.
|
|
348
377
|
lj 40 20 40 > "$ACC/acct-01/limits.json"
|
|
349
378
|
lj 40 80 80 > "$ACC/acct-02/limits.json"
|
|
350
379
|
all1=1
|
|
351
380
|
for _ in $(seq 1 15); do
|
|
352
|
-
case "$(claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
381
|
+
case "$(CLAUDE_MULTIACC_HEADROOM_BAND=0 claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
353
382
|
done
|
|
354
|
-
[ "$all1" = "1" ] && t_ok "equal weekly
|
|
383
|
+
[ "$all1" = "1" ] && t_ok "strict mode: equal weekly usage uses the session tiebreak" \
|
|
355
384
|
|| t_fail "session tiebreak" "did not use session to break a weekly tie"
|
|
356
385
|
|
|
357
386
|
# fully equal scores spread load across accounts
|
|
@@ -1036,7 +1065,8 @@ mkclientlimit() { # mkclientlimit <acct dir> <session id> <resetsAt> [rejection
|
|
|
1036
1065
|
}
|
|
1037
1066
|
SID1="81bf8b20-013f-4414-8878-e0289bec9ad0"
|
|
1038
1067
|
RESET1=$(( $(date +%s) + 1800 ))
|
|
1039
|
-
|
|
1068
|
+
REJECTED1="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"
|
|
1069
|
+
mkclientlimit "$ACC/acct-01" "$SID1" "$RESET1" "$REJECTED1"
|
|
1040
1070
|
all2=1
|
|
1041
1071
|
for _ in $(seq 1 12); do
|
|
1042
1072
|
case "$(claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
@@ -1048,6 +1078,9 @@ first="$(head -1 "$ACC/acct-01/.limited" 2>/dev/null)"
|
|
|
1048
1078
|
|| t_fail "client marker reset" "want $RESET1, got '${first:-<none>}'"
|
|
1049
1079
|
grep -q 'reason=client-rate-limit' "$ACC/acct-01/.limited" 2>/dev/null \
|
|
1050
1080
|
&& t_ok "marker is tagged client-rate-limit" || t_fail "client marker reason" "$(cat "$ACC/acct-01/.limited" 2>/dev/null)"
|
|
1081
|
+
grep -q "marked_at=$REJECTED1" "$ACC/acct-01/.limited" 2>/dev/null \
|
|
1082
|
+
&& t_ok "client marker preserves the rejection timestamp" \
|
|
1083
|
+
|| t_fail "client marker timestamp" "$(cat "$ACC/acct-01/.limited" 2>/dev/null)"
|
|
1051
1084
|
out="$(CLAUDE_ACCOUNT=acct-01 claude 2>&1)"
|
|
1052
1085
|
check "explicit pin still wins over a client-reported limit" "CFG=acct-01" "$out"
|
|
1053
1086
|
|
|
@@ -1055,6 +1088,25 @@ check "explicit pin still wins over a client-reported limit" "CFG=acct-01" "$out
|
|
|
1055
1088
|
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-mid.json" claude-accounts limits --quiet
|
|
1056
1089
|
[ -f "$ACC/acct-01/.limited" ] && t_ok "client-rate-limit marker survives a clean limits refresh" \
|
|
1057
1090
|
|| t_fail "client marker vs limits" "marker was cleared while the window was still open"
|
|
1091
|
+
# A later successful usage read under the threshold supersedes the client marker. The
|
|
1092
|
+
# live pool had five accounts at 0-3% hidden behind days-old client markers, leaving one
|
|
1093
|
+
# nominally-empty (but actually rejected) account to receive every direct launch.
|
|
1094
|
+
# A fleet sync refreshes the file's mtime, so recovery must use its semantic timestamp.
|
|
1095
|
+
{ head -1 "$ACC/acct-01/.limited"; \
|
|
1096
|
+
sed 's/marked_at=[^ ]*/marked_at=2020-01-01T00:00:00Z/' "$ACC/acct-01/.limited" | tail -1; \
|
|
1097
|
+
} > "$ACC/acct-01/.limited.tmp"
|
|
1098
|
+
mv "$ACC/acct-01/.limited.tmp" "$ACC/acct-01/.limited"
|
|
1099
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-mid.json" claude-accounts limits --force --quiet
|
|
1100
|
+
[ ! -f "$ACC/acct-01/.limited" ] \
|
|
1101
|
+
&& t_ok "newer low telemetry clears an old client-rate-limit marker" \
|
|
1102
|
+
|| t_fail "old client marker vs limits" "marker survived confirmed recovery"
|
|
1103
|
+
[ -f "$ACC/acct-01/.client-limit-cleared" ] \
|
|
1104
|
+
&& t_ok "confirmed client-limit recovery records a transcript watermark" \
|
|
1105
|
+
|| t_fail "client recovery watermark" "watermark missing"
|
|
1106
|
+
claude >/dev/null 2>&1
|
|
1107
|
+
[ ! -f "$ACC/acct-01/.limited" ] \
|
|
1108
|
+
&& t_ok "an old transcript cannot recreate a telemetry-disproved marker" \
|
|
1109
|
+
|| t_fail "client recovery watermark" "old rejection recreated the marker"
|
|
1058
1110
|
rm -f "$ACC/acct-01/.limited"
|
|
1059
1111
|
|
|
1060
1112
|
# an ALREADY-ELAPSED rejection is history, not an exclusion
|
|
@@ -1386,7 +1438,9 @@ expected="$(cat "$WORK/stdin13")"
|
|
|
1386
1438
|
[ "$out" = "$expected" ] && t_ok "stdin/stdout byte fidelity (-p pipe)" || t_fail "stdin fidelity" "got: $out"
|
|
1387
1439
|
|
|
1388
1440
|
# ---- 14. selection log written ------------------------------------------------
|
|
1389
|
-
|
|
1441
|
+
log_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}T.*acct-0[123] weekly=[0-9?]+% session=[0-9?]+%'
|
|
1442
|
+
log_pattern="$log_pattern band=30 band-count=[0-9]+ pwd="
|
|
1443
|
+
grep -qE "$log_pattern" "$ACC/selection.log" \
|
|
1390
1444
|
&& t_ok "selection.log format" || t_fail "selection.log format" "no matching lines"
|
|
1391
1445
|
grep -qE 'sk-ant-oat|accessToken|refreshToken' "$ACC/selection.log" \
|
|
1392
1446
|
&& t_fail "selection.log has no secrets" "a token leaked into the log" \
|
|
@@ -3302,6 +3356,27 @@ for _ in $(seq 1 15); do
|
|
|
3302
3356
|
done
|
|
3303
3357
|
[ "$all2" = "1" ] && t_ok "codex: picks the highest weekly-headroom account" \
|
|
3304
3358
|
|| t_fail "codex headroom selection" "picked the more-utilized account"
|
|
3359
|
+
# Keep direct-provider behavior aligned with pool-selection.v2: both accounts inside
|
|
3360
|
+
# the default 30-point band receive launches; a caller can set 0 for strict ranking.
|
|
3361
|
+
cxlj 0 10 10 > "$CX/acct-01/limits.json"
|
|
3362
|
+
cxlj 30 10 30 > "$CX/acct-02/limits.json"
|
|
3363
|
+
hits1=0; hits2=0
|
|
3364
|
+
for _ in $(seq 1 20); do
|
|
3365
|
+
case "$(codex 2>&1)" in
|
|
3366
|
+
*CFG=acct-01*) hits1=$((hits1+1)) ;;
|
|
3367
|
+
*CFG=acct-02*) hits2=$((hits2+1)) ;;
|
|
3368
|
+
esac
|
|
3369
|
+
done
|
|
3370
|
+
{ [ "$hits1" -gt 0 ] && [ "$hits2" -gt 0 ]; } \
|
|
3371
|
+
&& t_ok "codex: 30-point headroom band spreads launches (acct-01=$hits1 acct-02=$hits2)" \
|
|
3372
|
+
|| t_fail "codex headroom band spread" "acct-01=$hits1 acct-02=$hits2"
|
|
3373
|
+
cxlj 20 10 20 > "$CX/acct-02/limits.json"
|
|
3374
|
+
all1=1
|
|
3375
|
+
for _ in $(seq 1 10); do
|
|
3376
|
+
case "$(CODEX_MULTIACC_HEADROOM_BAND=0 codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
3377
|
+
done
|
|
3378
|
+
[ "$all1" = 1 ] && t_ok "codex: headroom band 0 restores strict ranking" \
|
|
3379
|
+
|| t_fail "codex zero headroom band" "the runner-up was selected"
|
|
3305
3380
|
# high session must NOT beat better weekly headroom
|
|
3306
3381
|
cxlj 10 85 85 > "$CX/acct-01/limits.json"
|
|
3307
3382
|
cxlj 70 20 70 > "$CX/acct-02/limits.json"
|
|
@@ -3622,13 +3697,13 @@ codex >/dev/null 2>&1
|
|
|
3622
3697
|
|| t_fail "codex soft park" "an elapsed soft park still excluded the account"
|
|
3623
3698
|
|
|
3624
3699
|
# ---- C8. exec auto-retry ------------------------------------------------------------
|
|
3625
|
-
# acct-01
|
|
3626
|
-
#
|
|
3700
|
+
# Strict mode makes acct-01's better score deterministic for these retry-path tests;
|
|
3701
|
+
# the scripted failure then forces the retry onto acct-02.
|
|
3627
3702
|
cx_first_01() { cxlj 5 5 5 > "$CX/acct-01/limits.json"; cxlj 20 20 20 > "$CX/acct-02/limits.json"; }
|
|
3628
3703
|
cx_first_01
|
|
3629
3704
|
# rate limit: retry on the other account + self-expiring cooldown for the failed one
|
|
3630
3705
|
echo "fail:acct-01" > "$FAKE_CTL2"
|
|
3631
|
-
out="$(codex exec "hello" < /dev/null 2>&1)"
|
|
3706
|
+
out="$(CODEX_MULTIACC_HEADROOM_BAND=0 codex exec "hello" < /dev/null 2>&1)"
|
|
3632
3707
|
case "$out" in *CFG=acct-02*) t_ok "codex: exec retries a rate-limited account on another" ;;
|
|
3633
3708
|
*) t_fail "codex retry" "did not land on acct-02: $out" ;; esac
|
|
3634
3709
|
if [ -f "$CX/acct-01/.limited" ]; then
|
|
@@ -3642,7 +3717,7 @@ rm -f "$CX/acct-01/.limited" "$FAKE_CTL2"
|
|
|
3642
3717
|
# auth failure parks (soft) instead of a cooldown
|
|
3643
3718
|
cx_first_01
|
|
3644
3719
|
echo "authfail:acct-01" > "$FAKE_CTL2"
|
|
3645
|
-
codex exec "hello" < /dev/null >/dev/null 2>&1
|
|
3720
|
+
CODEX_MULTIACC_HEADROOM_BAND=0 codex exec "hello" < /dev/null >/dev/null 2>&1
|
|
3646
3721
|
if [ -f "$CX/acct-01/.expired" ]; then
|
|
3647
3722
|
grep -q "reason=auth-error" "$CX/acct-01/.expired" && grep -q "soft_until=" "$CX/acct-01/.expired" \
|
|
3648
3723
|
&& t_ok "codex: auth failure parks the account with a soft stamp" \
|
|
@@ -3654,7 +3729,7 @@ rm -f "$CX/acct-01/.expired" "$FAKE_CTL2"
|
|
|
3654
3729
|
# org-disabled failure parks as org-blocked
|
|
3655
3730
|
cx_first_01
|
|
3656
3731
|
echo "orgfail:acct-01" > "$FAKE_CTL2"
|
|
3657
|
-
codex exec "hello" < /dev/null >/dev/null 2>&1
|
|
3732
|
+
CODEX_MULTIACC_HEADROOM_BAND=0 codex exec "hello" < /dev/null >/dev/null 2>&1
|
|
3658
3733
|
grep -q "reason=org-blocked" "$CX/acct-01/.expired" 2>/dev/null \
|
|
3659
3734
|
&& t_ok "codex: workspace-disabled failure parks as org-blocked" \
|
|
3660
3735
|
|| t_fail "codex org park from run" "marker: $(cat "$CX/acct-01/.expired" 2>/dev/null || echo none)"
|
|
@@ -61,7 +61,9 @@ class ResetHandler(BaseHTTPRequestHandler):
|
|
|
61
61
|
return
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
class
|
|
64
|
+
class CodexPoolSandbox:
|
|
65
|
+
"""A one-account codex pool with a fake usage/credits endpoint."""
|
|
66
|
+
|
|
65
67
|
def setUp(self) -> None:
|
|
66
68
|
self.temp = tempfile.TemporaryDirectory(prefix="multiacc-reset-")
|
|
67
69
|
self.pool = Path(self.temp.name) / "pool"
|
|
@@ -88,6 +90,8 @@ class CodexResetIntegrationTest(unittest.TestCase):
|
|
|
88
90
|
self.thread.join(timeout=5)
|
|
89
91
|
self.temp.cleanup()
|
|
90
92
|
|
|
93
|
+
|
|
94
|
+
class CodexResetIntegrationTest(CodexPoolSandbox, unittest.TestCase):
|
|
91
95
|
def run_limits(self) -> subprocess.CompletedProcess:
|
|
92
96
|
base = f"http://127.0.0.1:{self.server.server_port}/wham"
|
|
93
97
|
env = os.environ.copy()
|
|
@@ -166,5 +170,66 @@ class CodexResetIntegrationTest(unittest.TestCase):
|
|
|
166
170
|
self.assertEqual(self.server.state["credit_gets"], 1)
|
|
167
171
|
|
|
168
172
|
|
|
173
|
+
class CodexMarkerNamesOneBucketTest(CodexPoolSandbox, unittest.TestCase):
|
|
174
|
+
"""The marker's epoch is the reset of the bucket its detail line NAMES.
|
|
175
|
+
|
|
176
|
+
The codex writer used to pair the highest-PERCENT bucket's name with the
|
|
177
|
+
LATEST reset over every offender, so a five-hour window at 100% resetting
|
|
178
|
+
tonight was written under a seven-day epoch — and everything that trusts the
|
|
179
|
+
marker parked the account for a week over a window that refills in hours.
|
|
180
|
+
#20 fixed this for the claude pool and left codex behind.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
def run_limits(self) -> subprocess.CompletedProcess: # no auto-reset: it clears markers
|
|
184
|
+
base = f"http://127.0.0.1:{self.server.server_port}/wham"
|
|
185
|
+
env = os.environ.copy()
|
|
186
|
+
env.update({"CODEX_ACCOUNTS_ROOT": str(self.pool),
|
|
187
|
+
"CODEX_MULTIACC_NO_SYNC": "1", "CODEX_MULTIACC_MIN_FETCH": "0",
|
|
188
|
+
"CODEX_MULTIACC_AUTO_RESET": "0", "PYTHONDONTWRITEBYTECODE": "1",
|
|
189
|
+
"CODEX_MULTIACC_USAGE_URL": f"{base}/usage"})
|
|
190
|
+
return subprocess.run([REPO / "bin/codex-accounts", "limits", "--force"],
|
|
191
|
+
capture_output=True, text=True, env=env, timeout=20, check=False)
|
|
192
|
+
|
|
193
|
+
def _marker(self):
|
|
194
|
+
text = (self.pool / "acct-01/.limited").read_text(encoding="utf-8").splitlines()
|
|
195
|
+
return int(text[0]), text[1]
|
|
196
|
+
|
|
197
|
+
def test_two_offenders_write_one_bucket_with_its_own_reset(self) -> None:
|
|
198
|
+
now = int(time.time())
|
|
199
|
+
session_reset, weekly_reset = now + 3600, now + 6 * 86400
|
|
200
|
+
self.server.state["usage"] = {"plan_type": "pro", "rate_limit": {
|
|
201
|
+
"allowed": True,
|
|
202
|
+
# 100% and back in an hour …
|
|
203
|
+
"primary_window": {"used_percent": 100, "limit_window_seconds": 18000,
|
|
204
|
+
"reset_at": session_reset},
|
|
205
|
+
# … beside 95% that is six days out. Both are over the threshold.
|
|
206
|
+
"secondary_window": {"used_percent": 95, "limit_window_seconds": 604800,
|
|
207
|
+
"reset_at": weekly_reset}}}
|
|
208
|
+
self.assertEqual(self.run_limits().returncode, 0)
|
|
209
|
+
epoch, detail = self._marker()
|
|
210
|
+
# The longest-lived offender is named, and the epoch is ITS reset — the
|
|
211
|
+
# account really is excluded until then, and a reader that takes the
|
|
212
|
+
# detail's percent gets the percent of the very bucket the epoch belongs
|
|
213
|
+
# to rather than a different bucket's.
|
|
214
|
+
self.assertEqual(epoch, weekly_reset)
|
|
215
|
+
self.assertIn("percent=95", detail)
|
|
216
|
+
self.assertIn(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(weekly_reset)), detail)
|
|
217
|
+
self.assertNotIn("percent=100", detail)
|
|
218
|
+
|
|
219
|
+
def test_a_single_offender_still_names_itself(self) -> None:
|
|
220
|
+
now = int(time.time())
|
|
221
|
+
reset = now + 4 * 86400
|
|
222
|
+
self.server.state["usage"] = {"plan_type": "pro", "rate_limit": {
|
|
223
|
+
"allowed": True,
|
|
224
|
+
"primary_window": {"used_percent": 91, "limit_window_seconds": 604800,
|
|
225
|
+
"reset_at": reset},
|
|
226
|
+
"secondary_window": {"used_percent": 12, "limit_window_seconds": 18000,
|
|
227
|
+
"reset_at": now + 900}}}
|
|
228
|
+
self.assertEqual(self.run_limits().returncode, 0)
|
|
229
|
+
epoch, detail = self._marker()
|
|
230
|
+
self.assertEqual(epoch, reset)
|
|
231
|
+
self.assertIn("percent=91", detail)
|
|
232
|
+
|
|
233
|
+
|
|
169
234
|
if __name__ == "__main__":
|
|
170
235
|
unittest.main()
|