claude-multiacc 2.0.20 → 2.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE_ACCS_TASK.md +10 -5
- package/README.md +75 -20
- package/bin/claude +190 -63
- package/bin/claude-accounts +202 -34
- package/bin/codex +258 -46
- package/bin/codex-accounts +206 -22
- package/docs/ACCOUNT_OPERATIONS.md +31 -6
- package/docs/TRACK_PROMPT.md +76 -0
- package/docs/UNIFIED_SELECTOR.md +40 -7
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/lib/report.py +38 -2
- package/lib/selector_policy.py +42 -7
- package/lib/selector_primitives.py +25 -16
- package/package.json +1 -1
- package/tests/run-tests.sh +1517 -37
- package/tests/test_selector.py +104 -2
package/tests/run-tests.sh
CHANGED
|
@@ -317,7 +317,7 @@ check "passthrough with CLAUDE_CODE_OAUTH_TOKEN" "CFG=none" "$out"
|
|
|
317
317
|
out="$(CLAUDE_MULTIACC_DISABLE=1 claude 2>&1)"
|
|
318
318
|
check "passthrough when disabled" "CFG=none" "$out"
|
|
319
319
|
|
|
320
|
-
# ---- 5. headroom selection:
|
|
320
|
+
# ---- 5. headroom selection: the session gate first, then the WEEKLY headroom band ----
|
|
321
321
|
lj() { # lj <weekly> <session> <max> -> a fresh limits.json body
|
|
322
322
|
printf '{"fetched_at":%s,"weekly_percent":%s,"session_percent":%s,"max_percent":%s,"buckets":[]}' "$now" "$1" "$2" "$3"
|
|
323
323
|
}
|
|
@@ -360,28 +360,131 @@ done
|
|
|
360
360
|
[ "$all1" = 1 ] && t_ok "Claude headroom band 0 restores strict ranking" \
|
|
361
361
|
|| t_fail "Claude zero headroom band" "the runner-up was selected"
|
|
362
362
|
|
|
363
|
-
# THE KEY CASE
|
|
364
|
-
#
|
|
365
|
-
#
|
|
366
|
-
#
|
|
363
|
+
# THE KEY CASE, REVERSED on 2026-09-03. It used to assert that a high (but
|
|
364
|
+
# sub-threshold) SESSION bucket must not deprioritize an account with better weekly
|
|
365
|
+
# headroom. The operator asked for the opposite: "among accounts where high session
|
|
366
|
+
# limits it must choose randomly from ones where highest weekly limits" — an account
|
|
367
|
+
# whose 5h bucket is nearly spent is about to be rejected whatever its weekly headroom.
|
|
368
|
+
# So the SESSION GATE (default 50) is the FIRST cut and the 30-point weekly band ranks
|
|
369
|
+
# only what clears it. acct-01: session 85 (past the gate), weekly 10; acct-02: session
|
|
370
|
+
# 20, weekly 70. Both eligible (max<90) — acct-02 wins every time now.
|
|
367
371
|
lj 10 85 85 > "$ACC/acct-01/limits.json"
|
|
368
372
|
lj 70 20 70 > "$ACC/acct-02/limits.json"
|
|
373
|
+
all2=1
|
|
374
|
+
for _ in $(seq 1 15); do
|
|
375
|
+
case "$(claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
376
|
+
done
|
|
377
|
+
[ "$all2" = "1" ] && t_ok "a session bucket past the gate is skipped while a fresher one exists (70w/20s over 10w/85s)" \
|
|
378
|
+
|| t_fail "session gate" "the account with its 5h bucket at 85% was still selected"
|
|
379
|
+
# The pick has to be auditable: the log names the gate and how many cleared it.
|
|
380
|
+
: > "$ACC/selection.log"
|
|
381
|
+
claude >/dev/null 2>&1
|
|
382
|
+
grep -qE 'acct-02 weekly=70% session=20% band=30 band-count=1 session-gate=50 session-ok=1 pwd=' "$ACC/selection.log" \
|
|
383
|
+
&& t_ok "the selection log carries session-gate=50 session-ok=1 when one of two clears" \
|
|
384
|
+
|| t_fail "session gate log" "$(tail -1 "$ACC/selection.log")"
|
|
385
|
+
|
|
386
|
+
# Both session buckets inside the gate: the gate has nothing to say and weekly decides,
|
|
387
|
+
# exactly as before the change.
|
|
388
|
+
lj 10 45 45 > "$ACC/acct-01/limits.json"
|
|
389
|
+
lj 70 20 70 > "$ACC/acct-02/limits.json"
|
|
390
|
+
all1=1
|
|
391
|
+
for _ in $(seq 1 15); do
|
|
392
|
+
case "$(claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
393
|
+
done
|
|
394
|
+
[ "$all1" = "1" ] && t_ok "with both sessions inside the gate weekly headroom decides (10w/45s over 70w/20s)" \
|
|
395
|
+
|| t_fail "session gate no-op" "the gate changed a ranking where every candidate cleared it"
|
|
396
|
+
|
|
397
|
+
# Nobody clears the gate: it COMPARES candidates, it never empties the pool — it steps
|
|
398
|
+
# aside and weekly ranks the whole set (10w wins), and the log says session-ok=0.
|
|
399
|
+
lj 10 85 85 > "$ACC/acct-01/limits.json"
|
|
400
|
+
lj 70 60 70 > "$ACC/acct-02/limits.json"
|
|
401
|
+
: > "$ACC/selection.log"
|
|
369
402
|
all1=1
|
|
370
403
|
for _ in $(seq 1 15); do
|
|
371
404
|
case "$(claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
372
405
|
done
|
|
373
|
-
[ "$all1" = "1" ] && t_ok "
|
|
374
|
-
|| t_fail "
|
|
406
|
+
[ "$all1" = "1" ] && t_ok "with nobody inside the gate it steps aside and weekly ranks (10w/85s over 70w/60s)" \
|
|
407
|
+
|| t_fail "session gate step-aside" "an empty gate emptied the pool instead of stepping aside"
|
|
408
|
+
grep -qE 'acct-01 weekly=10% session=85% band=30 band-count=1 session-gate=50 session-ok=0 pwd=' "$ACC/selection.log" \
|
|
409
|
+
&& t_ok "the selection log carries session-ok=0 when the gate steps aside" \
|
|
410
|
+
|| t_fail "session gate log" "$(tail -1 "$ACC/selection.log")"
|
|
411
|
+
|
|
412
|
+
# The gate is a knob, like the band: 100 turns it off and pure weekly ranking returns.
|
|
413
|
+
lj 10 85 85 > "$ACC/acct-01/limits.json"
|
|
414
|
+
lj 70 20 70 > "$ACC/acct-02/limits.json"
|
|
415
|
+
all1=1
|
|
416
|
+
for _ in $(seq 1 15); do
|
|
417
|
+
case "$(CLAUDE_MULTIACC_SESSION_GATE=100 claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
418
|
+
done
|
|
419
|
+
[ "$all1" = "1" ] && t_ok "CLAUDE_MULTIACC_SESSION_GATE=100 disables the gate" \
|
|
420
|
+
|| t_fail "session gate off" "the gate still fired at 100"
|
|
421
|
+
# ...and garbage in that env var falls back to the 50-point default, never to "off".
|
|
422
|
+
all2=1
|
|
423
|
+
for _ in $(seq 1 15); do
|
|
424
|
+
case "$(CLAUDE_MULTIACC_SESSION_GATE=abc claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
425
|
+
done
|
|
426
|
+
[ "$all2" = "1" ] && t_ok "a non-numeric CLAUDE_MULTIACC_SESSION_GATE falls back to 50" \
|
|
427
|
+
|| t_fail "session gate validation" "a garbage gate value changed the outcome"
|
|
375
428
|
|
|
376
|
-
#
|
|
429
|
+
# Equal weekly usage: the GATE decides, not a tiebreak. Session stopped being a ranking
|
|
430
|
+
# input inside the band on 2026-09-03 — acct-02's 80-point session bucket simply never
|
|
431
|
+
# reaches the weekly comparison, whether the band is strict or the default 30.
|
|
377
432
|
lj 40 20 40 > "$ACC/acct-01/limits.json"
|
|
378
433
|
lj 40 80 80 > "$ACC/acct-02/limits.json"
|
|
379
434
|
all1=1
|
|
380
435
|
for _ in $(seq 1 15); do
|
|
381
436
|
case "$(CLAUDE_MULTIACC_HEADROOM_BAND=0 claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
382
437
|
done
|
|
383
|
-
[ "$all1" = "1" ] && t_ok "strict mode:
|
|
384
|
-
|| t_fail "session
|
|
438
|
+
[ "$all1" = "1" ] && t_ok "strict mode: the session gate decides an exact weekly tie" \
|
|
439
|
+
|| t_fail "session gate tie" "a weekly tie was not resolved by the session gate"
|
|
440
|
+
all1=1
|
|
441
|
+
for _ in $(seq 1 15); do
|
|
442
|
+
case "$(claude 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
443
|
+
done
|
|
444
|
+
[ "$all1" = "1" ] && t_ok "the gate (not the band) removes the session-heavy half of a weekly tie" \
|
|
445
|
+
|| t_fail "session gate tie" "the default band let the 80-point session account back in"
|
|
446
|
+
|
|
447
|
+
# Inside the gate, session is NOT a tiebreaker any more: an exact weekly tie between two
|
|
448
|
+
# gate-clearing accounts is a coin flip even in strict mode (it used to go to the lower
|
|
449
|
+
# session, through the old weekly*1000+session score).
|
|
450
|
+
lj 40 20 40 > "$ACC/acct-01/limits.json"
|
|
451
|
+
lj 40 45 45 > "$ACC/acct-02/limits.json"
|
|
452
|
+
hits1=0; hits2=0
|
|
453
|
+
for _ in $(seq 1 20); do
|
|
454
|
+
case "$(CLAUDE_MULTIACC_HEADROOM_BAND=0 claude 2>&1)" in
|
|
455
|
+
*CFG=acct-01*) hits1=$((hits1+1)) ;;
|
|
456
|
+
*CFG=acct-02*) hits2=$((hits2+1)) ;;
|
|
457
|
+
esac
|
|
458
|
+
done
|
|
459
|
+
{ [ "$hits1" -gt 0 ] && [ "$hits2" -gt 0 ]; } \
|
|
460
|
+
&& t_ok "strict mode: session does not break an exact weekly tie inside the gate (acct-01=$hits1 acct-02=$hits2)" \
|
|
461
|
+
|| t_fail "session tiebreak removed" "acct-01=$hits1 acct-02=$hits2 (want both >0)"
|
|
462
|
+
|
|
463
|
+
# Clearing the gate takes BOTH readings, as in pool-selection.v2: a fresh file with a
|
|
464
|
+
# session reading but no weekly one must not become the sole gate-clearer and win the
|
|
465
|
+
# all-gated tie over an account with a truthful weekly reading — not even with a
|
|
466
|
+
# max_percent to fall back on (the shims used to rank on that; the policy never could).
|
|
467
|
+
# (No writer produces such a file; this pins parity with lib/selector_policy.py.)
|
|
468
|
+
printf '{"fetched_at":%s,"max_percent":10,"session_percent":10,"buckets":[]}' "$now" > "$ACC/acct-01/limits.json"
|
|
469
|
+
lj 20 80 80 > "$ACC/acct-02/limits.json"
|
|
470
|
+
all2=1
|
|
471
|
+
for _ in $(seq 1 15); do
|
|
472
|
+
case "$(claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
473
|
+
done
|
|
474
|
+
[ "$all2" = "1" ] && t_ok "a session reading without a weekly one never clears the gate" \
|
|
475
|
+
|| t_fail "gate needs both readings" "an unknown-weekly account beat a truthful weekly reading"
|
|
476
|
+
|
|
477
|
+
# ...and the converse: a weekly reading without a session one is not "known" either — it
|
|
478
|
+
# neither clears the gate nor ranks once the gate steps aside (10w/?s vs 70w/80s -> the
|
|
479
|
+
# 70w account, as in pool-selection.v2, where quota_known needs both readings).
|
|
480
|
+
printf '{"fetched_at":%s,"weekly_percent":10,"max_percent":10,"buckets":[]}' "$now" > "$ACC/acct-01/limits.json"
|
|
481
|
+
lj 70 80 80 > "$ACC/acct-02/limits.json"
|
|
482
|
+
all2=1
|
|
483
|
+
for _ in $(seq 1 15); do
|
|
484
|
+
case "$(claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
485
|
+
done
|
|
486
|
+
[ "$all2" = "1" ] && t_ok "a weekly reading without a session one is unknown to both cuts" \
|
|
487
|
+
|| t_fail "known needs both readings" "a session-less weekly reading ranked as known"
|
|
385
488
|
|
|
386
489
|
# fully equal scores spread load across accounts
|
|
387
490
|
lj 10 10 10 > "$ACC/acct-01/limits.json"
|
|
@@ -409,6 +512,10 @@ done
|
|
|
409
512
|
{ [ "$hits1" -gt 0 ] && [ "$hits2" -gt 0 ]; } \
|
|
410
513
|
&& t_ok "CLAUDE_SHIM_SELECT=random restores uniform spread" \
|
|
411
514
|
|| t_fail "random opt-out" "acct-01=$hits1 acct-02=$hits2"
|
|
515
|
+
# ...and its log line must not claim a gate that never ran (codex review, 2026-09-04).
|
|
516
|
+
grep -qE 'band=30 band-count=2 session-gate=off session-ok=2 pwd=' "$ACC/selection.log" \
|
|
517
|
+
&& t_ok "random mode logs session-gate=off instead of a cut it never made" \
|
|
518
|
+
|| t_fail "random mode gate log" "$(grep 'SHIM_SELECT\|session-gate' "$ACC/selection.log" | tail -1)"
|
|
412
519
|
|
|
413
520
|
# Unknown telemetry ranks behind every truthful reading, never as neutral or free.
|
|
414
521
|
printf '{"fetched_at":1,"weekly_percent":1,"session_percent":1,"max_percent":1,"buckets":[]}' > "$ACC/acct-01/limits.json"
|
|
@@ -478,17 +585,35 @@ printf '%s\nbucket=session percent=95 reason=limits\n' "$((now-10))" > "$ACC/acc
|
|
|
478
585
|
claude >/dev/null 2>&1
|
|
479
586
|
[ ! -f "$ACC/acct-01/.limited" ] && t_ok "expired marker auto-cleared" || t_fail "expired marker auto-cleared" "marker still present"
|
|
480
587
|
|
|
481
|
-
# ---- 9. all limited ->
|
|
588
|
+
# ---- 9. all limited -> the still-serving accounts go through the same two cuts, strict weekly ----
|
|
482
589
|
printf '%s\nx\n' "$((now+3600))" > "$ACC/acct-01/.limited"
|
|
483
590
|
printf '%s\nx\n' "$((now+3600))" > "$ACC/acct-02/.limited"
|
|
484
|
-
|
|
485
|
-
|
|
591
|
+
lj 97 10 97 > "$ACC/acct-01/limits.json"
|
|
592
|
+
lj 91 10 91 > "$ACC/acct-02/limits.json"
|
|
486
593
|
out="$(claude 2>&1)"
|
|
487
|
-
check "all-limited falls back to
|
|
594
|
+
check "all-limited falls back to the still-serving account with the most weekly headroom" "CFG=acct-02" "$out"
|
|
488
595
|
grep -q "all-limited fallback=acct-02" "$ACC/selection.log" \
|
|
489
596
|
&& t_ok "fallback logged" || t_fail "fallback logged" "no all-limited line in selection.log"
|
|
490
597
|
rm -f "$ACC/acct-01/.limited" "$ACC/acct-02/.limited"
|
|
491
598
|
|
|
599
|
+
# The fallback applies the SAME two cuts: with every account limit-marked but still
|
|
600
|
+
# serving, an account past the session gate (10w/85s) yields to one inside it (70w/20s)
|
|
601
|
+
# even though its weekly headroom is far better — exactly as in ordinary selection.
|
|
602
|
+
# (On the pre-gate rule this picked 10w/85s, the best strict weekly score.)
|
|
603
|
+
printf '%s\nbucket=weekly_all percent=95 marked_at=x reason=limits\n' "$((now+3600))" > "$ACC/acct-01/.limited"
|
|
604
|
+
printf '%s\nbucket=weekly_all percent=95 marked_at=x reason=limits\n' "$((now+3600))" > "$ACC/acct-02/.limited"
|
|
605
|
+
lj 10 85 85 > "$ACC/acct-01/limits.json"
|
|
606
|
+
lj 70 20 70 > "$ACC/acct-02/limits.json"
|
|
607
|
+
all2=1
|
|
608
|
+
for _ in $(seq 1 12); do
|
|
609
|
+
case "$(claude 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
610
|
+
done
|
|
611
|
+
[ "$all2" = "1" ] && t_ok "the all-limited fallback applies the session gate before strict weekly (70w/20s over 10w/85s)" \
|
|
612
|
+
|| t_fail "fallback session gate" "the fallback handed out the account past the session gate"
|
|
613
|
+
grep -q "all-limited fallback=acct-02 weekly=70%" "$ACC/selection.log" \
|
|
614
|
+
&& t_ok "the fallback line names the gated pick" || t_fail "fallback gate log" "$(tail -1 "$ACC/selection.log")"
|
|
615
|
+
rm -f "$ACC/acct-01/.limited" "$ACC/acct-02/.limited"
|
|
616
|
+
|
|
492
617
|
# ---- 9a2. the fallback tells "still serving" from "rejected right now" --------------
|
|
493
618
|
# acct-01: weekly at 99% — worse headroom, but still answering requests.
|
|
494
619
|
# acct-02: session at 100% — far better weekly (7%), but every request bounces until
|
|
@@ -512,8 +637,8 @@ check "all exhausted: the soonest reset is handed out" "CFG=acct-02" "$out"
|
|
|
512
637
|
grep -q "all-exhausted resets_in=" "$ACC/selection.log" \
|
|
513
638
|
&& t_ok "the all-exhausted pick is logged with its reset" || t_fail "all-exhausted log" "no line"
|
|
514
639
|
rm -f "$ACC/acct-01/.limited" "$ACC/acct-02/.limited"
|
|
515
|
-
|
|
516
|
-
|
|
640
|
+
lj 97 10 97 > "$ACC/acct-01/limits.json"
|
|
641
|
+
lj 91 10 91 > "$ACC/acct-02/limits.json"
|
|
517
642
|
|
|
518
643
|
# ---- 9a3. a token park is credential-scoped ----------------------------------------
|
|
519
644
|
# A dead portable token beside a LIVE login: this session runs the account on the
|
|
@@ -717,8 +842,8 @@ rm -f "$ACC/acct-01/server.token"
|
|
|
717
842
|
printf '%s' "$DEAD_CREDS" > "$ACC/acct-01/.credentials.json"
|
|
718
843
|
printf '%s' "$HEALTHY_CREDS" > "$ACC/acct-02/.credentials.json"
|
|
719
844
|
printf 'sk-ant-oat01-REVOKEDREVOKEDREVOKEDREVOKEDREVOKEDREVOKED' > "$ACC/acct-01/server.token"
|
|
720
|
-
|
|
721
|
-
|
|
845
|
+
lj 1 1 1 > "$ACC/acct-01/limits.json" # ranks first, so its preflight is what runs
|
|
846
|
+
lj 50 1 50 > "$ACC/acct-02/limits.json"
|
|
722
847
|
rm -f "$ACC/.last-pick" "$ACC/acct-01/.expired" "$ACC/acct-01/.server-token-verified"
|
|
723
848
|
out="$(claude --resume d6ccbac0-6643-4780-a99e-3afa1683478e 2>&1)"
|
|
724
849
|
check "revoked setup-token fails over before --resume" "CFG=acct-02" "$out"
|
|
@@ -1438,8 +1563,14 @@ expected="$(cat "$WORK/stdin13")"
|
|
|
1438
1563
|
[ "$out" = "$expected" ] && t_ok "stdin/stdout byte fidelity (-p pipe)" || t_fail "stdin fidelity" "got: $out"
|
|
1439
1564
|
|
|
1440
1565
|
# ---- 14. selection log written ------------------------------------------------
|
|
1566
|
+
# One COMPLETE reading first, on purpose: the pool is still carrying the session-only
|
|
1567
|
+
# documents an earlier limits pass wrote, and since 2026-09-04 a pool where no account
|
|
1568
|
+
# has BOTH percentages is BLIND (bin/claude telem_blind) and logs the blind format
|
|
1569
|
+
# instead. The ranked format asserted below only exists when something actually ranked.
|
|
1570
|
+
lj 20 10 20 > "$ACC/acct-01/limits.json"
|
|
1571
|
+
claude >/dev/null 2>&1
|
|
1441
1572
|
log_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}T.*acct-0[123] weekly=[0-9?]+% session=[0-9?]+%'
|
|
1442
|
-
log_pattern="$log_pattern band=30 band-count=[0-9]+ pwd="
|
|
1573
|
+
log_pattern="$log_pattern band=30 band-count=[0-9]+ session-gate=50 session-ok=[0-9]+ pwd="
|
|
1443
1574
|
grep -qE "$log_pattern" "$ACC/selection.log" \
|
|
1444
1575
|
&& t_ok "selection.log format" || t_fail "selection.log format" "no matching lines"
|
|
1445
1576
|
grep -qE 'sk-ant-oat|accessToken|refreshToken' "$ACC/selection.log" \
|
|
@@ -1931,12 +2062,620 @@ assert groups["weekly_scoped:Fable"] == "weekly", groups
|
|
|
1931
2062
|
EOF
|
|
1932
2063
|
[ $? -eq 0 ] && t_ok "limits records weekly_percent(55) + session_percent(88) with correct groups" \
|
|
1933
2064
|
|| t_fail "weekly/session classification" "see limits.json"
|
|
1934
|
-
#
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
2065
|
+
# The two numbers must stay SEPARATE on disk. There is no combined score any more
|
|
2066
|
+
# (2026-09-03): the shim GATES on session_percent and BANDS on weekly_percent, so a
|
|
2067
|
+
# session bucket at 88 must never be folded into — or allowed to poison — the weekly
|
|
2068
|
+
# reading the band ranks on.
|
|
2069
|
+
sc="$(python3 -c "import json;d=json.load(open('$ACC/acct-01/limits.json'));print('%s/%s' % (d['weekly_percent'], d['session_percent']))")"
|
|
2070
|
+
[ "$sc" = "55/88" ] && t_ok "weekly and session are recorded as separate ranking inputs (55 weekly / 88 session)" \
|
|
2071
|
+
|| t_fail "weekly/session ranking inputs" "got $sc"
|
|
1938
2072
|
rm -f "$ACC/acct-01/limits.json" "$ACC/acct-01/.limited"
|
|
1939
2073
|
|
|
2074
|
+
# ---- 16-nodata. a 0% bucket with NO reset window is NO DATA, not an empty account ----
|
|
2075
|
+
# 2026-09-04, my-mini: for acct-13/acct-14 the usage endpoint answered EVERY bucket
|
|
2076
|
+
# `percent: 0, resets_at: null` while Claude Code was being rejected on those same two
|
|
2077
|
+
# accounts with "You've hit your weekly limit · resets Sep 8 at 1am" (epoch 1788818400).
|
|
2078
|
+
# The writer recorded the zeros verbatim, so two provably exhausted accounts became the
|
|
2079
|
+
# leaders of the weekly band and absorbed 31 of the last ~60 picks — and the shim's
|
|
2080
|
+
# telemetry-based recovery then deleted their truthful client:seven_day markers about six
|
|
2081
|
+
# times a day each. A truthful bucket ALWAYS carries the window it resets in, so 0% with
|
|
2082
|
+
# no window is no data. These run in a pool of their own: one no-data account against one
|
|
2083
|
+
# honest one makes "which account ranked" unambiguous.
|
|
2084
|
+
ND="$WORK/nodata-pool"
|
|
2085
|
+
mkdir -p "$ND/acct-01" "$ND/acct-02" "$ND/tmp"
|
|
2086
|
+
: > "$ND/.limits-kick"
|
|
2087
|
+
cat > "$ND/accounts.json" <<'EOF'
|
|
2088
|
+
{"version":1,"server":"none","threshold":90,"accounts":[
|
|
2089
|
+
{"id":"acct-01","email":"nd1@test","home":"mac","added_at":"2026-07-13T00:00:00Z"},
|
|
2090
|
+
{"id":"acct-02","email":"nd2@test","home":"mac","added_at":"2026-07-13T00:00:00Z"}]}
|
|
2091
|
+
EOF
|
|
2092
|
+
for i in 01 02; do
|
|
2093
|
+
printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-nd%s","refreshToken":"r","expiresAt":9999999999999,"refreshTokenExpiresAt":9999999999999}}' \
|
|
2094
|
+
"$i" > "$ND/acct-$i/.credentials.json"
|
|
2095
|
+
done
|
|
2096
|
+
ndl() { # ndl <weekly> <session> <max> -> a truthful, in-window reading on stdout
|
|
2097
|
+
local t; t="$(date +%s)"
|
|
2098
|
+
printf '{"fetched_at":%s,"source":"oauth","weekly_percent":%s,"session_percent":%s,"max_percent":%s,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2099
|
+
"$t" "$1" "$2" "$3" "$((t + 259200))"
|
|
2100
|
+
}
|
|
2101
|
+
ndlimits() { # ndlimits <fixture> [extra args] -> a real refresh over the whole ND pool
|
|
2102
|
+
local f="$1"; shift
|
|
2103
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$f" \
|
|
2104
|
+
claude-accounts limits --force "$@" 2>&1
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
# the incident payload, byte-for-byte in shape: every bucket 0%, every window null
|
|
2108
|
+
cat > "$WORK/usage-allzero.json" <<'EOF'
|
|
2109
|
+
{"limits":[
|
|
2110
|
+
{"kind":"session","group":"session","percent":0,"resets_at":null,"scope":null},
|
|
2111
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":null,"scope":null},
|
|
2112
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2113
|
+
]}
|
|
2114
|
+
EOF
|
|
2115
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2116
|
+
check "an all-zero/no-window payload is reported as no usable telemetry" \
|
|
2117
|
+
"no usable telemetry (account ranks as unknown, not as empty)" "$out"
|
|
2118
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2119
|
+
import json, sys
|
|
2120
|
+
d = json.load(open(sys.argv[1]))
|
|
2121
|
+
assert d.get('no_data') is True, d
|
|
2122
|
+
# A MISSING field is what makes the shim's fresh_field/cutoff_field reads fail, which
|
|
2123
|
+
# is what makes the account unknown to both cuts. Writing 0 here is the bug.
|
|
2124
|
+
for k in ('max_percent', 'weekly_percent', 'session_percent', 'weekly_resets_epoch'):
|
|
2125
|
+
assert k not in d, (k, d)
|
|
2126
|
+
assert len(d['buckets']) == 3, d # the raw buckets stay, for diagnostics
|
|
2127
|
+
assert d['fetched_at'] > 0 and d['source'], d
|
|
2128
|
+
EOF
|
|
2129
|
+
[ $? -eq 0 ] && t_ok "a no-data pass records no_data and NONE of the three percent signals" \
|
|
2130
|
+
|| t_fail "no_data document" "see $ND/acct-01/limits.json"
|
|
2131
|
+
|
|
2132
|
+
# ONE uninformative bucket beside real ones changes nothing (the live acct-16 shape:
|
|
2133
|
+
# weekly_scoped:Fable 0/null next to a real session and a real weekly_all).
|
|
2134
|
+
cat > "$WORK/usage-mixed-nodata.json" <<'EOF'
|
|
2135
|
+
{"limits":[
|
|
2136
|
+
{"kind":"session","group":"session","percent":12,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2137
|
+
{"kind":"weekly_all","group":"weekly","percent":40,"resets_at":"2099-01-05T00:00:00+00:00","scope":null},
|
|
2138
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2139
|
+
]}
|
|
2140
|
+
EOF
|
|
2141
|
+
ndlimits "$WORK/usage-mixed-nodata.json" --quiet >/dev/null
|
|
2142
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2143
|
+
import json, sys, time, calendar
|
|
2144
|
+
d = json.load(open(sys.argv[1]))
|
|
2145
|
+
assert 'no_data' not in d, d
|
|
2146
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (12, 40, 40), d
|
|
2147
|
+
want = calendar.timegm(time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S"))
|
|
2148
|
+
assert d['weekly_resets_epoch'] == want, d
|
|
2149
|
+
assert len(d['buckets']) == 3, d # the quiet bucket is still recorded
|
|
2150
|
+
EOF
|
|
2151
|
+
[ $? -eq 0 ] && t_ok "one uninformative bucket beside real ones leaves the ranking untouched (12/40)" \
|
|
2152
|
+
|| t_fail "mixed no-data payload" "see $ND/acct-01/limits.json"
|
|
2153
|
+
|
|
2154
|
+
# 0% WITH a real window is informative: a genuinely fresh account must still rank empty.
|
|
2155
|
+
cat > "$WORK/usage-zero-real-windows.json" <<'EOF'
|
|
2156
|
+
{"limits":[
|
|
2157
|
+
{"kind":"session","group":"session","percent":0,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2158
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2159
|
+
]}
|
|
2160
|
+
EOF
|
|
2161
|
+
ndlimits "$WORK/usage-zero-real-windows.json" --quiet >/dev/null
|
|
2162
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2163
|
+
import json, sys
|
|
2164
|
+
d = json.load(open(sys.argv[1]))
|
|
2165
|
+
assert 'no_data' not in d, d
|
|
2166
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (0, 0, 0), d
|
|
2167
|
+
EOF
|
|
2168
|
+
[ $? -eq 0 ] && t_ok "0% WITH real reset windows still records a real, empty 0% reading" \
|
|
2169
|
+
|| t_fail "zero-with-windows payload" "see $ND/acct-01/limits.json"
|
|
2170
|
+
|
|
2171
|
+
# ...and the horizon that 0% is valid until comes from the bucket that reported one. A
|
|
2172
|
+
# null-window sibling's synthesized now+1h used to win the min() and shorten it.
|
|
2173
|
+
cat > "$WORK/usage-zero-mixed-window.json" <<'EOF'
|
|
2174
|
+
{"limits":[
|
|
2175
|
+
{"kind":"session","group":"session","percent":0,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2176
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":"2099-01-05T00:00:00+00:00","scope":null},
|
|
2177
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2178
|
+
]}
|
|
2179
|
+
EOF
|
|
2180
|
+
ndlimits "$WORK/usage-zero-mixed-window.json" --quiet >/dev/null
|
|
2181
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2182
|
+
import json, sys, time, calendar
|
|
2183
|
+
d = json.load(open(sys.argv[1]))
|
|
2184
|
+
want = calendar.timegm(time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S"))
|
|
2185
|
+
assert d['weekly_percent'] == 0 and d['weekly_resets_epoch'] == want, d
|
|
2186
|
+
EOF
|
|
2187
|
+
[ $? -eq 0 ] && t_ok "a window-less bucket cannot shorten the horizon a real 0% reading is valid for" \
|
|
2188
|
+
|| t_fail "no-data horizon" "see $ND/acct-01/limits.json"
|
|
2189
|
+
|
|
2190
|
+
# ---- 16-nodata-rank. an unknown account never leads the weekly band ------------------
|
|
2191
|
+
# The whole point of the rule: 0/0 ranked BETTER than a truthful 45% account, so every
|
|
2192
|
+
# pick went to the exhausted one. Unknown must lose to any account with a real reading.
|
|
2193
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null
|
|
2194
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2195
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2196
|
+
: > "$ND/selection.log"
|
|
2197
|
+
nd_hits=0
|
|
2198
|
+
for _ in $(seq 1 10); do
|
|
2199
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_hits=$((nd_hits+1)) ;; esac
|
|
2200
|
+
done
|
|
2201
|
+
[ "$nd_hits" = "0" ] \
|
|
2202
|
+
&& t_ok "a no-data account never outranks an account with real telemetry (0 of 10 picks)" \
|
|
2203
|
+
|| t_fail "no_data ranking" "the fake-zero account took $nd_hits of 10 picks"
|
|
2204
|
+
grep -q "acct-02 weekly=45% session=10% band=30 band-count=1" "$ND/selection.log" \
|
|
2205
|
+
&& t_ok "the pick logs the known account alone in the band" \
|
|
2206
|
+
|| t_fail "no_data band" "selection.log: $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2207
|
+
grep -q "acct-01 weekly=0%" "$ND/selection.log" \
|
|
2208
|
+
&& t_fail "no_data band leader" "the fake-zero account was logged as a 0% pick" \
|
|
2209
|
+
|| t_ok "the fake-zero account is never logged as the band leader"
|
|
2210
|
+
|
|
2211
|
+
# ---- 16-nodata-marker. a client-reported WEEKLY rejection outlives any telemetry -----
|
|
2212
|
+
# The second half of the incident. Claude Code's own rejection wrote
|
|
2213
|
+
# `bucket=client:seven_day percent=100 reason=client-rate-limit` with the server's reset
|
|
2214
|
+
# (Sep 8); the next invocation read the fake 0%, called it newer first-hand evidence and
|
|
2215
|
+
# deleted the marker. A seven-day window cannot fall from a server-proven 100% to under
|
|
2216
|
+
# the threshold before it resets, so no reading may clear it — however fresh and however
|
|
2217
|
+
# informative. Here acct-01's telemetry is a REAL 5%, so only the bucket rule can save it.
|
|
2218
|
+
ndl 5 5 5 > "$ND/acct-01/limits.json"
|
|
2219
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2220
|
+
printf '%s\nbucket=client:seven_day percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2221
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2222
|
+
rm -f "$ND"/acct-0*/.client-limit-cleared "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2223
|
+
: > "$ND/selection.log"
|
|
2224
|
+
w_hits=0
|
|
2225
|
+
for _ in $(seq 1 6); do
|
|
2226
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) w_hits=$((w_hits+1)) ;; esac
|
|
2227
|
+
done
|
|
2228
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$w_hits" = "0" ]; } \
|
|
2229
|
+
&& t_ok "a client:seven_day marker survives fresh below-threshold telemetry" \
|
|
2230
|
+
|| t_fail "weekly client marker" "marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) hits=$w_hits"
|
|
2231
|
+
grep -q "client limit cleared by newer telemetry" "$ND/selection.log" \
|
|
2232
|
+
&& t_fail "weekly client marker" "the shim logged a recovery for a weekly rejection" \
|
|
2233
|
+
|| t_ok "no recovery event is logged for a weekly rejection"
|
|
2234
|
+
[ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2235
|
+
&& t_ok "no recovery watermark is written for a weekly rejection" \
|
|
2236
|
+
|| t_fail "weekly client marker" "a watermark was written"
|
|
2237
|
+
# case-insensitive on the bucket name: seven_day_opus is weekly too
|
|
2238
|
+
printf '%s\nbucket=client:Seven_Day_Opus percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2239
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2240
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2241
|
+
[ -f "$ND/acct-01/.limited" ] && t_ok "a model-scoped weekly bucket (seven_day_opus) is weekly too" \
|
|
2242
|
+
|| t_fail "weekly client marker" "seven_day_opus was cleared"
|
|
2243
|
+
|
|
2244
|
+
# ...but the 5h window self-heals within hours, so #22 (2026-09-03) still holds: a
|
|
2245
|
+
# five_hour marker DOES clear once a below-threshold reading was fetched after it.
|
|
2246
|
+
printf '%s\nbucket=client:five_hour percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2247
|
+
"$(( $(date +%s) + 1800 ))" > "$ND/acct-01/.limited"
|
|
2248
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2249
|
+
: > "$ND/selection.log"
|
|
2250
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2251
|
+
[ ! -f "$ND/acct-01/.limited" ] \
|
|
2252
|
+
&& t_ok "a client:five_hour marker still clears on newer below-threshold telemetry (#22)" \
|
|
2253
|
+
|| t_fail "five_hour client marker" "the 2026-09-03 recovery stopped working"
|
|
2254
|
+
[ -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2255
|
+
&& t_ok "the five_hour recovery still records its watermark" \
|
|
2256
|
+
|| t_fail "five_hour client marker" "watermark missing"
|
|
2257
|
+
grep -q "acct-01 client limit cleared by newer telemetry (5%)" "$ND/selection.log" \
|
|
2258
|
+
&& t_ok "the five_hour recovery still logs the reading it acted on" \
|
|
2259
|
+
|| t_fail "five_hour client marker" "selection.log: $(tail -2 "$ND/selection.log" 2>/dev/null | tr '\n' ' ')"
|
|
2260
|
+
|
|
2261
|
+
# ---- 16-nodata-shorten. an offender pass must never SHORTEN a weekly client marker ---
|
|
2262
|
+
# The writer's offenders branch used to overwrite `.limited` unconditionally: a session
|
|
2263
|
+
# bucket crossing the threshold (+1h reset) replaced a client:seven_day marker four days
|
|
2264
|
+
# out, and after that hour the provably exhausted account was back in the pool (codex
|
|
2265
|
+
# review, 2026-09-04). The client's own reset reaches further and must win.
|
|
2266
|
+
printf '%s\nbucket=client:seven_day percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2267
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2268
|
+
keep_reset_before="$(head -1 "$ND/acct-01/.limited")"
|
|
2269
|
+
python3 - "$WORK/usage-shorten.json" <<'PJ'
|
|
2270
|
+
import json, sys, datetime
|
|
2271
|
+
soon = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat()
|
|
2272
|
+
json.dump({"limits": [
|
|
2273
|
+
{"kind": "session", "percent": 95, "resets_at": soon},
|
|
2274
|
+
{"kind": "seven_day", "percent": 0, "resets_at": None},
|
|
2275
|
+
]}, open(sys.argv[1], 'w'))
|
|
2276
|
+
PJ
|
|
2277
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-shorten.json" \
|
|
2278
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2279
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$(head -1 "$ND/acct-01/.limited")" = "$keep_reset_before" ] \
|
|
2280
|
+
&& grep -q 'reason=client-rate-limit' "$ND/acct-01/.limited"; } \
|
|
2281
|
+
&& t_ok "an over-threshold session pass keeps the further-reaching weekly client marker" \
|
|
2282
|
+
|| t_fail "marker shortened" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2283
|
+
# ...while a LATER reset may still extend the exclusion (more caution is allowed):
|
|
2284
|
+
python3 - "$WORK/usage-extend.json" <<'PJ'
|
|
2285
|
+
import json, sys, datetime
|
|
2286
|
+
far = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=6)).isoformat()
|
|
2287
|
+
json.dump({"limits": [
|
|
2288
|
+
{"kind": "seven_day", "percent": 95, "resets_at": far},
|
|
2289
|
+
]}, open(sys.argv[1], 'w'))
|
|
2290
|
+
PJ
|
|
2291
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-extend.json" \
|
|
2292
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2293
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$(head -1 "$ND/acct-01/.limited")" -gt "$keep_reset_before" ]; } \
|
|
2294
|
+
&& t_ok "a further-out offender may still extend the marker" \
|
|
2295
|
+
|| t_fail "marker extend" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2296
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2297
|
+
# ...and a GARBLED client marker (a first line the shims' num_ok would refuse) must not
|
|
2298
|
+
# be preserved by that guard: to the shims it is an active-forever park, so the writer
|
|
2299
|
+
# replacing it with a valid offender marker is a repair, not a shortening.
|
|
2300
|
+
printf '9999999999999999999\nbucket=client:seven_day reason=client-rate-limit\n' > "$ND/acct-01/.limited"
|
|
2301
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-shorten.json" \
|
|
2302
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2303
|
+
{ [ -f "$ND/acct-01/.limited" ] && grep -q 'reason=limits' "$ND/acct-01/.limited"; } \
|
|
2304
|
+
&& t_ok "a garbled client marker is repaired by the offender write, not preserved" \
|
|
2305
|
+
|| t_fail "garbled marker repair" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2306
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2307
|
+
|
|
2308
|
+
# ---- 16-nodata-noclear. an uninformative reading cannot clear ANY marker -------------
|
|
2309
|
+
# End to end, in the incident's own order: the fake-zero payload goes through the real
|
|
2310
|
+
# writer, then a five_hour marker (the kind that IS allowed to clear) is planted on top.
|
|
2311
|
+
# Pre-fix that pass wrote max_percent 0 and the very next invocation deleted the marker
|
|
2312
|
+
# with "client limit cleared by newer telemetry (0%)". A reading with no percent at all
|
|
2313
|
+
# reads as unknown, and unknown proves nothing.
|
|
2314
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null
|
|
2315
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2316
|
+
printf '%s\nbucket=client:five_hour percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2317
|
+
"$(( $(date +%s) + 1800 ))" > "$ND/acct-01/.limited"
|
|
2318
|
+
rm -f "$ND/acct-01/.client-limit-cleared" "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2319
|
+
: > "$ND/selection.log"
|
|
2320
|
+
for _ in 1 2 3; do CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1; done
|
|
2321
|
+
[ -f "$ND/acct-01/.limited" ] \
|
|
2322
|
+
&& t_ok "a limits.json with no percent fields cannot clear even a five_hour marker" \
|
|
2323
|
+
|| t_fail "no_data marker clearing" "fake-zero telemetry unparked the account"
|
|
2324
|
+
grep -q "client limit cleared" "$ND/selection.log" \
|
|
2325
|
+
&& t_fail "no_data marker clearing" "a recovery was logged from a no-data reading" \
|
|
2326
|
+
|| t_ok "no recovery is logged from a no-data reading"
|
|
2327
|
+
[ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2328
|
+
&& t_ok "no recovery watermark is written from a no-data reading" \
|
|
2329
|
+
|| t_fail "no_data marker clearing" "a watermark was written from a reading with no percent"
|
|
2330
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2331
|
+
|
|
2332
|
+
# ---- 16-nodata-writer. the LIMITS PASS obeys the same marker rule as the shim --------
|
|
2333
|
+
# The shim's rule above is only half a fix. `claude-accounts limits` runs on a 15-minute
|
|
2334
|
+
# schedule and deletes markers itself, and until 2026-09-04 it kept a client rejection
|
|
2335
|
+
# only while it was newer than CLIENT_LIMIT_CONFIRM_DELAY (300s) — so five minutes after
|
|
2336
|
+
# the shim refused to unpark acct-13, the scheduled pass deleted the same client:seven_day
|
|
2337
|
+
# marker anyway, on a payload whose every bucket said `percent 0, resets_at null`. Two
|
|
2338
|
+
# writers with two rules is one rule: the weaker one. This is the claude twin of the codex
|
|
2339
|
+
# C13c block, and the pass is driven for real (fixture endpoint -> writer -> disk).
|
|
2340
|
+
nd_mark() { # nd_mark <acct dir> <bucket> <reset-offset-seconds> <marked_at ISO>
|
|
2341
|
+
printf '%s\nbucket=%s percent=100 marked_at=%s reason=client-rate-limit\n' \
|
|
2342
|
+
"$(( $(date +%s) + $3 ))" "$2" "$4" > "$1/.limited"
|
|
2343
|
+
}
|
|
2344
|
+
nd_acct01_log() { printf '%s' "$1" | grep 'acct-01' | tr '\n' ' '; }
|
|
2345
|
+
# A REAL, informative, below-threshold reading: the kind that IS allowed to clear a
|
|
2346
|
+
# five-hour marker, and the one that must never clear a weekly one.
|
|
2347
|
+
cat > "$WORK/usage-nd-low.json" <<'EOF'
|
|
2348
|
+
{"limits":[
|
|
2349
|
+
{"kind":"session","group":"session","percent":5,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2350
|
+
{"kind":"weekly_all","group":"weekly","percent":5,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2351
|
+
]}
|
|
2352
|
+
EOF
|
|
2353
|
+
|
|
2354
|
+
# (1) A pass that reported nothing proves nothing, so it clears nothing — whatever the
|
|
2355
|
+
# marker says and however old it is. On origin/main this marker is 300s past its confirm
|
|
2356
|
+
# delay and the pass deletes it with "marker cleared (max 0%)".
|
|
2357
|
+
nd_mark "$ND/acct-01" client:seven_day 345600 2020-01-01T00:00:00Z
|
|
2358
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2359
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2360
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2361
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2362
|
+
&& t_ok "a no-data limits pass keeps an aged client:seven_day marker, and logs that it kept it" \
|
|
2363
|
+
|| t_fail "writer marker: no-data pass" \
|
|
2364
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2365
|
+
|
|
2366
|
+
# (2) ...and a pass that DID report something still cannot clear a weekly rejection
|
|
2367
|
+
# before its reset: a seven-day window cannot fall from the server-proven 100% that
|
|
2368
|
+
# wrote the marker to 5% while it is still open. The keep must come from the BUCKET
|
|
2369
|
+
# rule, not from silence, so the "no usable telemetry" line must NOT appear here.
|
|
2370
|
+
nd_mark "$ND/acct-01" client:seven_day 345600 2020-01-01T00:00:00Z
|
|
2371
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2372
|
+
out="$(ndlimits "$WORK/usage-nd-low.json")"
|
|
2373
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2374
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker cleared" \
|
|
2375
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2376
|
+
&& t_ok "an informative 5% pass keeps a client:seven_day marker on the bucket rule alone" \
|
|
2377
|
+
|| t_fail "writer marker: weekly vs informative pass" \
|
|
2378
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2379
|
+
|
|
2380
|
+
# (3) The 5h window self-heals in hours, so #22 (2026-09-03) still holds at the writer:
|
|
2381
|
+
# an aged five_hour rejection DOES clear once a pass has real numbers under the
|
|
2382
|
+
# threshold. This one passes on origin/main too — deliberately: it is the guard against
|
|
2383
|
+
# over-correcting (2) into "no client marker ever clears", which would strand accounts
|
|
2384
|
+
# sitting at 0% usage for days, which is the bug #22 existed to fix.
|
|
2385
|
+
nd_mark "$ND/acct-01" client:five_hour 1800 2020-01-01T00:00:00Z
|
|
2386
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2387
|
+
out="$(ndlimits "$WORK/usage-nd-low.json")"
|
|
2388
|
+
{ [ ! -f "$ND/acct-01/.limited" ] && [ -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2389
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker cleared (max 5%)"; } \
|
|
2390
|
+
&& t_ok "an informative 5% pass still clears an aged client:five_hour marker (#22)" \
|
|
2391
|
+
|| t_fail "writer marker: five_hour recovery" \
|
|
2392
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2393
|
+
|
|
2394
|
+
# (4) ...but the SAME five_hour marker survives a pass that said nothing. "0%" and "no
|
|
2395
|
+
# reading" are the same bytes on origin/main, and that is the whole incident.
|
|
2396
|
+
nd_mark "$ND/acct-01" client:five_hour 1800 2020-01-01T00:00:00Z
|
|
2397
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2398
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2399
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2400
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2401
|
+
&& t_ok "a no-data pass keeps even a client:five_hour marker — the kind it may clear when informative" \
|
|
2402
|
+
|| t_fail "writer marker: five_hour vs no-data pass" \
|
|
2403
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2404
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2405
|
+
|
|
2406
|
+
# ---- 16-nodata-signals. each ranking signal comes from a bucket of its OWN kind ------
|
|
2407
|
+
# 2026-09-04, second defect: weekly_percent fell back to the overall peak and
|
|
2408
|
+
# session_percent to a flat 0. So an account whose weekly buckets said nothing while its
|
|
2409
|
+
# 5h bucket read 40% was recorded as 40% WEEKLY — a number no bucket ever reported, on
|
|
2410
|
+
# the signal the band ranks on — and its mirror image was recorded as session 0%, which
|
|
2411
|
+
# walks straight through the session gate. A signal nobody reported must be ABSENT: the
|
|
2412
|
+
# shim needs BOTH readings to call an account known (pick_best's quota_known rule), so a
|
|
2413
|
+
# missing one costs the account its place in the band and nothing else.
|
|
2414
|
+
cat > "$WORK/usage-session-only.json" <<'EOF'
|
|
2415
|
+
{"limits":[
|
|
2416
|
+
{"kind":"session","group":"session","percent":40,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2417
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":null,"scope":null},
|
|
2418
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2419
|
+
]}
|
|
2420
|
+
EOF
|
|
2421
|
+
ndlimits "$WORK/usage-session-only.json" --quiet >/dev/null
|
|
2422
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2423
|
+
import json, sys
|
|
2424
|
+
d = json.load(open(sys.argv[1]))
|
|
2425
|
+
assert 'no_data' not in d, d # one bucket DID report: this is a reading
|
|
2426
|
+
assert (d['session_percent'], d['max_percent']) == (40, 40), d
|
|
2427
|
+
# The two weekly buckets said `0% / no window`. Recording 40 here (round 1's fallback to
|
|
2428
|
+
# the overall peak) or 0 (origin/main's max over silent weekly buckets) both invent the
|
|
2429
|
+
# only number the weekly band ranks on.
|
|
2430
|
+
assert 'weekly_percent' not in d, d
|
|
2431
|
+
assert 'weekly_resets_epoch' not in d, d # a horizon without a reading means nothing
|
|
2432
|
+
EOF
|
|
2433
|
+
[ $? -eq 0 ] && t_ok "a session-only reading records session+max and NO weekly_percent" \
|
|
2434
|
+
|| t_fail "session-only signals" "see $ND/acct-01/limits.json"
|
|
2435
|
+
|
|
2436
|
+
# ...and the shim reads that as UNKNOWN, so a truthful 30w/10s account takes every pick.
|
|
2437
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2438
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2439
|
+
: > "$ND/selection.log"
|
|
2440
|
+
nd_sess_hits=0
|
|
2441
|
+
for _ in $(seq 1 10); do
|
|
2442
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_sess_hits=$((nd_sess_hits+1)) ;; esac
|
|
2443
|
+
done
|
|
2444
|
+
[ "$nd_sess_hits" = "0" ] \
|
|
2445
|
+
&& t_ok "an account with no weekly reading never enters the band (0 of 10 picks)" \
|
|
2446
|
+
|| t_fail "session-only ranking" "the weekly-less account took $nd_sess_hits of 10 picks"
|
|
2447
|
+
|
|
2448
|
+
# The mirror image: a weekly reading with nothing to say about the session bucket. Here
|
|
2449
|
+
# the rival is WORSE on weekly (80 against 37) and still wins every pick, because it is
|
|
2450
|
+
# the only candidate that can clear the session gate — which is exactly what a fabricated
|
|
2451
|
+
# `session_percent: 0` would have handed the silent account for free.
|
|
2452
|
+
cat > "$WORK/usage-weekly-only.json" <<'EOF'
|
|
2453
|
+
{"limits":[
|
|
2454
|
+
{"kind":"session","group":"session","percent":0,"resets_at":null,"scope":null},
|
|
2455
|
+
{"kind":"weekly_all","group":"weekly","percent":37,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2456
|
+
]}
|
|
2457
|
+
EOF
|
|
2458
|
+
ndlimits "$WORK/usage-weekly-only.json" --quiet >/dev/null
|
|
2459
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2460
|
+
import json, sys, time, calendar
|
|
2461
|
+
d = json.load(open(sys.argv[1]))
|
|
2462
|
+
assert 'no_data' not in d, d
|
|
2463
|
+
assert (d['weekly_percent'], d['max_percent']) == (37, 37), d
|
|
2464
|
+
assert d['weekly_resets_epoch'] == calendar.timegm(
|
|
2465
|
+
time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S")), d
|
|
2466
|
+
assert 'session_percent' not in d, d # the 5h bucket reported nothing at all
|
|
2467
|
+
EOF
|
|
2468
|
+
[ $? -eq 0 ] && t_ok "a weekly-only reading records weekly+max+horizon and NO session_percent" \
|
|
2469
|
+
|| t_fail "weekly-only signals" "see $ND/acct-01/limits.json"
|
|
2470
|
+
ndl 80 10 80 > "$ND/acct-02/limits.json"
|
|
2471
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2472
|
+
: > "$ND/selection.log"
|
|
2473
|
+
nd_wk_hits=0
|
|
2474
|
+
for _ in $(seq 1 6); do
|
|
2475
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_wk_hits=$((nd_wk_hits+1)) ;; esac
|
|
2476
|
+
done
|
|
2477
|
+
[ "$nd_wk_hits" = "0" ] \
|
|
2478
|
+
&& t_ok "an unknown session reading never clears the gate, even with the better weekly (0 of 6)" \
|
|
2479
|
+
|| t_fail "weekly-only gate" "the session-less account cleared the gate $nd_wk_hits of 6 times"
|
|
2480
|
+
grep -q "acct-02 weekly=80% session=10% band=30 band-count=1 session-gate=50 session-ok=1" "$ND/selection.log" \
|
|
2481
|
+
&& t_ok "the log shows one account clearing the gate and one band member" \
|
|
2482
|
+
|| t_fail "weekly-only gate log" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2483
|
+
|
|
2484
|
+
# ---- 16-nodata-blind. a fresh timestamp is not a usable reading ---------------------
|
|
2485
|
+
# The third face of the same defect: blindness was judged on fetched_at alone, so a pool
|
|
2486
|
+
# of freshly-written no_data documents looked FRESH — selection.log carried no
|
|
2487
|
+
# ranking=BLIND line, `status` said nothing was wrong, and every account read as unknown,
|
|
2488
|
+
# which is a pool-wide coin flip. An outage that reports itself as healthy is the eleven
|
|
2489
|
+
# days of 2026-08-11 all over again, this time with a current timestamp on it.
|
|
2490
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null # BOTH accounts: no_data
|
|
2491
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2492
|
+
: > "$ND/selection.log"
|
|
2493
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2494
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s band=30 band-count=2 session-gate=50 session-ok=0 pwd=" \
|
|
2495
|
+
"$ND/selection.log" \
|
|
2496
|
+
&& t_ok "an all-no_data pool logs ranking=BLIND although its telemetry is seconds old" \
|
|
2497
|
+
|| t_fail "no_data blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2498
|
+
out="$(CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts status 2>&1)"
|
|
2499
|
+
check "status calls an all-no_data pool blind" "RANKING IS BLIND" "$out"
|
|
2500
|
+
# ...and says which KIND of blind, because the two take opposite advice. The eleven-day
|
|
2501
|
+
# 2026-08 outage was stale telemetry — fetch again, then log in. This one is current
|
|
2502
|
+
# telemetry that says nothing: the credential is working perfectly, so sending the
|
|
2503
|
+
# operator to `claude-accounts login` is sending them after a fault that does not exist.
|
|
2504
|
+
check "the no_data banner names the endpoint, not the login" \
|
|
2505
|
+
"Those fetches authenticated; a re-login does NOT fix this" "$out"
|
|
2506
|
+
case "$out" in
|
|
2507
|
+
*"claude-accounts login <acct-NN>"*)
|
|
2508
|
+
t_fail "no_data banner fix line" "a current-but-unusable pool was told to re-login" ;;
|
|
2509
|
+
*) t_ok "the no_data banner does not prescribe a re-login" ;;
|
|
2510
|
+
esac
|
|
2511
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/nodata.json" 2>/dev/null
|
|
2512
|
+
python3 - "$ND/nodata.json" <<'EOF'
|
|
2513
|
+
import json, sys
|
|
2514
|
+
d = json.load(open(sys.argv[1]))
|
|
2515
|
+
assert d['summary']['telemetry'] == 'blind', d['summary']
|
|
2516
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2517
|
+
for a in d['accounts']:
|
|
2518
|
+
u = a['usage']
|
|
2519
|
+
# The panel has to be able to tell THIS outage from the eleven-day one: the readings
|
|
2520
|
+
# are current (not stale), they simply carry nothing to rank on.
|
|
2521
|
+
assert u['no_data'] is True, a
|
|
2522
|
+
assert u['stale'] is False, a
|
|
2523
|
+
assert u['weekly_percent'] is None and u['session_percent'] is None, a
|
|
2524
|
+
EOF
|
|
2525
|
+
[ $? -eq 0 ] && t_ok "--json reports ranking_blind with per-account no_data on fresh readings" \
|
|
2526
|
+
|| t_fail "json no_data blindness" "see $ND/nodata.json"
|
|
2527
|
+
|
|
2528
|
+
# One real reading is enough to rank the pool, and it must take the picks. A no_data
|
|
2529
|
+
# neighbour is unknown, not free.
|
|
2530
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2531
|
+
: > "$ND/selection.log"
|
|
2532
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2533
|
+
nd_mix_hits=0
|
|
2534
|
+
for _ in $(seq 1 6); do
|
|
2535
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-02*) nd_mix_hits=$((nd_mix_hits+1)) ;; esac
|
|
2536
|
+
done
|
|
2537
|
+
{ [ "$nd_mix_hits" = "6" ] && ! grep -q "ranking=BLIND" "$ND/selection.log"; } \
|
|
2538
|
+
&& t_ok "one no_data account beside a real one leaves the pool ranking, and the real one wins 6/6" \
|
|
2539
|
+
|| t_fail "mixed no_data pool" "acct-02 took $nd_mix_hits of 6; $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2540
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/nodata-mixed.json" 2>/dev/null
|
|
2541
|
+
python3 - "$ND/nodata-mixed.json" <<'EOF'
|
|
2542
|
+
import json, sys
|
|
2543
|
+
d = json.load(open(sys.argv[1]))
|
|
2544
|
+
assert d['summary']['telemetry'] == 'fresh', d['summary']
|
|
2545
|
+
assert d['summary']['ranking_blind'] is False, d['summary']
|
|
2546
|
+
byid = {a['id']: a for a in d['accounts']}
|
|
2547
|
+
assert byid['acct-01']['usage']['no_data'] is True, byid['acct-01']
|
|
2548
|
+
# true-only: an ordinary reading must carry the shape every consumer already knows
|
|
2549
|
+
assert 'no_data' not in byid['acct-02']['usage'], byid['acct-02']
|
|
2550
|
+
EOF
|
|
2551
|
+
[ $? -eq 0 ] && t_ok "--json calls the mixed pool fresh and flags only the no_data account" \
|
|
2552
|
+
|| t_fail "json mixed no_data" "see $ND/nodata-mixed.json"
|
|
2553
|
+
|
|
2554
|
+
# ...and a no_data document is never DEGRADABLE. Degraded ranking exists for stale
|
|
2555
|
+
# readings that are still true (a weekly bucket only rises until its reset); a document
|
|
2556
|
+
# with no weekly reading and no horizon has nothing to be true. It must drag the pool to
|
|
2557
|
+
# BLIND rather than let one neighbour's stale number rank alone.
|
|
2558
|
+
printf '{"fetched_at":%s,"source":"oauth","no_data":true,"buckets":[]}' "$(date +%s)" \
|
|
2559
|
+
> "$ND/acct-01/limits.json"
|
|
2560
|
+
printf '{"fetched_at":%s,"weekly_percent":4,"session_percent":0,"max_percent":4,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2561
|
+
"$(( $(date +%s) - 950000 ))" "$(( $(date +%s) + 200000 ))" > "$ND/acct-02/limits.json"
|
|
2562
|
+
: > "$ND/selection.log"
|
|
2563
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2564
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2565
|
+
{ grep -q "ranking=BLIND" "$ND/selection.log" && ! grep -q "ranking=DEGRADED" "$ND/selection.log"; } \
|
|
2566
|
+
&& t_ok "an in-window no_data candidate turns degraded ranking off for the whole pool" \
|
|
2567
|
+
|| t_fail "no_data degraded" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2568
|
+
# the same holds once the no_data document itself goes stale (nothing to rank, ever)
|
|
2569
|
+
printf '{"fetched_at":%s,"source":"oauth","no_data":true,"buckets":[]}' "$(( $(date +%s) - 950000 ))" \
|
|
2570
|
+
> "$ND/acct-01/limits.json"
|
|
2571
|
+
: > "$ND/selection.log"
|
|
2572
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2573
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2574
|
+
{ grep -q "ranking=BLIND" "$ND/selection.log" && ! grep -q "ranking=DEGRADED" "$ND/selection.log"; } \
|
|
2575
|
+
&& t_ok "a stale no_data document is not degradable either" \
|
|
2576
|
+
|| t_fail "stale no_data degraded" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2577
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2578
|
+
|
|
2579
|
+
# ---- 16-onefield-blind. HALF a reading is not a reading -----------------------------
|
|
2580
|
+
# The same defect one layer in, and the one the round-2 fix walked past. Blindness was
|
|
2581
|
+
# taught to reject a no_data document — but it accepted one carrying EITHER percentage,
|
|
2582
|
+
# while pick_best has always needed BOTH (the quota_known rule) before it will call an
|
|
2583
|
+
# account known. The writer emits exactly those half documents, per signal, whenever one
|
|
2584
|
+
# group of buckets goes silent (16-nodata-signals above). So a pool whose every reading
|
|
2585
|
+
# was session-only tied every account, picked uniformly at RANDOM, logged no
|
|
2586
|
+
# ranking=BLIND, and had `status` calling the telemetry fresh — the 2026-08 outage's
|
|
2587
|
+
# defining symptom, with a current timestamp on it. Blind is blind however the reading
|
|
2588
|
+
# came up short.
|
|
2589
|
+
ndl_half() { # ndl_half <session pct> -> a fresh, truthful, SESSION-ONLY reading
|
|
2590
|
+
printf '{"fetched_at":%s,"source":"oauth","session_percent":%s,"max_percent":%s,"buckets":[]}' \
|
|
2591
|
+
"$(date +%s)" "$1" "$1"
|
|
2592
|
+
}
|
|
2593
|
+
ndl_half 12 > "$ND/acct-01/limits.json"
|
|
2594
|
+
ndl_half 18 > "$ND/acct-02/limits.json"
|
|
2595
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2596
|
+
: > "$ND/selection.log"
|
|
2597
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2598
|
+
# session-ok=0 is the point, not a detail: both readings carry a session percentage well
|
|
2599
|
+
# under the gate, and NEITHER clears it, because the gate is part of the same "known"
|
|
2600
|
+
# rule. A pool that cannot rank must not look like one that ranked and tied.
|
|
2601
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s band=30 band-count=2 session-gate=50 session-ok=0 pwd=" \
|
|
2602
|
+
"$ND/selection.log" \
|
|
2603
|
+
&& t_ok "a pool of session-only readings logs ranking=BLIND although both are seconds old" \
|
|
2604
|
+
|| t_fail "one-signal blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2605
|
+
out="$(CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts status 2>&1)"
|
|
2606
|
+
check "status calls a one-signal pool blind" "RANKING IS BLIND" "$out"
|
|
2607
|
+
check "the one-signal banner says the readings are incomplete" \
|
|
2608
|
+
"ranking needs BOTH a weekly and a session percentage" "$out"
|
|
2609
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/onefield.json" 2>/dev/null
|
|
2610
|
+
python3 - "$ND/onefield.json" <<'EOF'
|
|
2611
|
+
import json, sys
|
|
2612
|
+
d = json.load(open(sys.argv[1]))
|
|
2613
|
+
assert d['summary']['telemetry'] == 'blind', d['summary']
|
|
2614
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2615
|
+
for a in d['accounts']:
|
|
2616
|
+
u = a['usage']
|
|
2617
|
+
# Current, well-formed, and NOT a no_data document — one real bucket did report.
|
|
2618
|
+
# It simply is not enough to rank on, and the panel has to agree with the shim.
|
|
2619
|
+
assert u['stale'] is False, a
|
|
2620
|
+
assert 'no_data' not in u, a
|
|
2621
|
+
assert u['session_percent'] is not None and u['weekly_percent'] is None, a
|
|
2622
|
+
EOF
|
|
2623
|
+
[ $? -eq 0 ] && t_ok "--json reports ranking_blind for readings that are half present" \
|
|
2624
|
+
|| t_fail "json one-signal blindness" "see $ND/onefield.json"
|
|
2625
|
+
|
|
2626
|
+
# ...and the weekly-only mirror of the same rule: known needs BOTH, whichever half is
|
|
2627
|
+
# missing. DEGRADED must not fire either — these readings are FRESH, and degraded exists
|
|
2628
|
+
# for an outage of age, not for fresh emptiness (codex review, 2026-09-04).
|
|
2629
|
+
ndl_whalf() { # ndl_whalf <weekly pct> -> a fresh, truthful, WEEKLY-ONLY reading
|
|
2630
|
+
printf '{"fetched_at":%s,"source":"oauth","weekly_percent":%s,"max_percent":%s,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2631
|
+
"$(date +%s)" "$1" "$1" "$(( $(date +%s) + 500000 ))"
|
|
2632
|
+
}
|
|
2633
|
+
ndl_whalf 4 > "$ND/acct-01/limits.json"
|
|
2634
|
+
ndl_whalf 80 > "$ND/acct-02/limits.json"
|
|
2635
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2636
|
+
: > "$ND/selection.log"
|
|
2637
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2638
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s .*session-ok=0 pwd=" "$ND/selection.log" \
|
|
2639
|
+
&& t_ok "a pool of weekly-only readings logs ranking=BLIND, not DEGRADED" \
|
|
2640
|
+
|| t_fail "weekly-only blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2641
|
+
! grep -q "ranking=DEGRADED" "$ND/selection.log" \
|
|
2642
|
+
&& t_ok "fresh weekly-only readings never promote the pool to DEGRADED" \
|
|
2643
|
+
|| t_fail "fresh-degraded" "a fresh one-signal pool ranked DEGRADED"
|
|
2644
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/whalf.json" 2>/dev/null
|
|
2645
|
+
python3 - "$ND/whalf.json" <<'PJ'
|
|
2646
|
+
import json, sys
|
|
2647
|
+
d = json.load(open(sys.argv[1]))
|
|
2648
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2649
|
+
PJ
|
|
2650
|
+
[ $? -eq 0 ] && t_ok "--json calls the fresh weekly-only pool blind (not degraded)" \
|
|
2651
|
+
|| t_fail "json one-signal verdict" "see $ND/whalf.json"
|
|
2652
|
+
|
|
2653
|
+
|
|
2654
|
+
# One COMPLETE reading beside them is enough to rank the pool, and it must take every
|
|
2655
|
+
# pick — the half readings are unknown, not free. (This holds on the pre-fix tree too:
|
|
2656
|
+
# it is the guard against over-correcting "half is blind" into "half is excluded".)
|
|
2657
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2658
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2659
|
+
: > "$ND/selection.log"
|
|
2660
|
+
nd_half_hits=0
|
|
2661
|
+
for _ in $(seq 1 6); do
|
|
2662
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-02*) nd_half_hits=$((nd_half_hits+1)) ;; esac
|
|
2663
|
+
done
|
|
2664
|
+
{ [ "$nd_half_hits" = "6" ] && ! grep -q "ranking=BLIND" "$ND/selection.log"; } \
|
|
2665
|
+
&& t_ok "one complete reading beside a session-only one leaves the pool ranking, and wins 6/6" \
|
|
2666
|
+
|| t_fail "mixed one-signal pool" \
|
|
2667
|
+
"acct-02 took $nd_half_hits of 6; $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2668
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/onefield-mixed.json" 2>/dev/null
|
|
2669
|
+
python3 - "$ND/onefield-mixed.json" <<'EOF'
|
|
2670
|
+
import json, sys
|
|
2671
|
+
d = json.load(open(sys.argv[1]))
|
|
2672
|
+
assert d['summary']['telemetry'] == 'fresh', d['summary']
|
|
2673
|
+
assert d['summary']['ranking_blind'] is False, d['summary']
|
|
2674
|
+
EOF
|
|
2675
|
+
[ $? -eq 0 ] && t_ok "--json calls the mixed one-signal pool fresh" \
|
|
2676
|
+
|| t_fail "json mixed one-signal" "see $ND/onefield-mixed.json"
|
|
2677
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2678
|
+
|
|
1940
2679
|
# ---- 16-marker. the marker names ONE bucket and carries THAT bucket's reset ----------
|
|
1941
2680
|
# A 100% session bucket (resets in an hour) beside a 100% Fable-only weekly bucket
|
|
1942
2681
|
# (resets in five days) used to produce "bucket=session … resets_at=<+1h>" on line 2
|
|
@@ -2410,6 +3149,9 @@ EOF
|
|
|
2410
3149
|
grep -q "ranking=BLIND" "$SD/selection.log" \
|
|
2411
3150
|
&& t_ok "selection.log records that ranking ran blind" \
|
|
2412
3151
|
|| t_fail "blind ranking log" "no ranking=BLIND line: $(tail -1 "$SD/selection.log")"
|
|
3152
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]+s band=30 band-count=[0-9]+ session-gate=50 session-ok=0 pwd=" "$SD/selection.log" \
|
|
3153
|
+
&& t_ok "the blind line carries both cuts in the standard field order" \
|
|
3154
|
+
|| t_fail "blind line fields" "$(tail -1 "$SD/selection.log")"
|
|
2413
3155
|
grep -q "telemetry-age=9[0-9]\{5\}s" "$SD/selection.log" \
|
|
2414
3156
|
&& t_ok "the blind line carries the age of the outage" \
|
|
2415
3157
|
|| t_fail "blind ranking age" "$(tail -1 "$SD/selection.log")"
|
|
@@ -2431,17 +3173,20 @@ EOF
|
|
|
2431
3173
|
# ...and telemetry INSIDE the window must still rank. 900s used to be the window,
|
|
2432
3174
|
# which is below the ~3600s floor the endpoint itself enforces (Retry-After: 3600),
|
|
2433
3175
|
# so a healthy pool spent most of every hour ranking neutral for no reason.
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
3176
|
+
# acct-02 is put OUTSIDE the 30-point band (weekly 40 against acct-01's 4) on purpose:
|
|
3177
|
+
# inside the band the two are peers and the pick is a coin flip, which would make this
|
|
3178
|
+
# assertion about the freshness window flaky for reasons that have nothing to do with it.
|
|
3179
|
+
printf '{"fetched_at":%s,"source":"oauth","max_percent":4,"weekly_percent":4,"session_percent":1,"buckets":[]}' \
|
|
3180
|
+
"$((now - 1200))" > "$SD/acct-01/limits.json"
|
|
3181
|
+
printf '{"fetched_at":%s,"source":"oauth","max_percent":40,"weekly_percent":40,"session_percent":1,"buckets":[]}' \
|
|
3182
|
+
"$((now - 1200))" > "$SD/acct-02/limits.json"
|
|
2438
3183
|
: > "$SD/selection.log"
|
|
2439
3184
|
CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1
|
|
2440
3185
|
! grep -q "ranking=BLIND" "$SD/selection.log" \
|
|
2441
3186
|
&& t_ok "20-minute-old telemetry still ranks (window matches the endpoint's own floor)" \
|
|
2442
3187
|
|| t_fail "stale window" "20-minute-old data was treated as blind"
|
|
2443
3188
|
grep -q "acct-01 weekly=4%" "$SD/selection.log" \
|
|
2444
|
-
&& t_ok "the pool ranks on real numbers and picks the account with more headroom" \
|
|
3189
|
+
&& t_ok "the pool ranks on real numbers and picks the account with more headroom (4% over 40%)" \
|
|
2445
3190
|
|| t_fail "headroom ranking" "$(tail -1 "$SD/selection.log")"
|
|
2446
3191
|
|
|
2447
3192
|
# ---- blind does not mean neutral --------------------------------------------
|
|
@@ -2467,6 +3212,10 @@ EOF
|
|
|
2467
3212
|
grep -q "ranking=DEGRADED" "$SD/selection.log" \
|
|
2468
3213
|
&& t_ok "a degraded pick is logged as degraded, not as blind" \
|
|
2469
3214
|
|| t_fail "degraded log" "$(tail -1 "$SD/selection.log")"
|
|
3215
|
+
# Session is unknown in a degraded pool, so nobody clears the gate: session-ok=0.
|
|
3216
|
+
grep -qE "ranking=DEGRADED telemetry-age=[0-9]+s band=30 band-count=[0-9]+ session-gate=50 session-ok=0 pwd=" "$SD/selection.log" \
|
|
3217
|
+
&& t_ok "the degraded line carries both cuts in the standard field order" \
|
|
3218
|
+
|| t_fail "degraded line fields" "$(tail -1 "$SD/selection.log")"
|
|
2470
3219
|
grep -q "acct-02 weekly=4% .*ranking=DEGRADED" "$SD/selection.log" \
|
|
2471
3220
|
&& t_ok "the degraded line reports the stale reading it actually ranked on" \
|
|
2472
3221
|
|| t_fail "degraded weekly" "$(tail -1 "$SD/selection.log")"
|
|
@@ -2712,10 +3461,20 @@ RACER
|
|
|
2712
3461
|
# threshold on its session bucket (max 91%). So it is picked whenever it is eligible,
|
|
2713
3462
|
# and skipped only when the cutoff actually fires — which isolates the cutoff window
|
|
2714
3463
|
# from the ranking window instead of conflating "excluded" with "outranked".
|
|
3464
|
+
# BOTH sessions sit above the 50-point session gate (91 and 55) deliberately: with
|
|
3465
|
+
# acct-02 inside the gate it would be the only gated candidate and win on the gate
|
|
3466
|
+
# alone, and these assertions would then be measuring the gate instead of the cutoff
|
|
3467
|
+
# window. Nobody clearing the gate makes it step aside, so weekly alone ranks here.
|
|
2715
3468
|
mk_cutoff_pool() { # $1 = age of both readings, in seconds
|
|
3469
|
+
# A FRESH clock, not the suite-wide $now captured at startup: the 800s case leaves
|
|
3470
|
+
# only 100s of headroom inside the 900s cutoff window, and the suite takes longer
|
|
3471
|
+
# than that to get here — under load (2026-09-04, parallel review agents) the reading
|
|
3472
|
+
# aged past the window and the assertion flipped. Ages here must mean age AT THE
|
|
3473
|
+
# SHIM'S OWN CLOCK, whenever this test happens to run.
|
|
3474
|
+
local now; now="$(date -u +%s)"
|
|
2716
3475
|
printf '{"fetched_at":%s,"weekly_percent":1,"session_percent":91,"max_percent":91,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2717
3476
|
"$((now - $1))" "$((now + 200000))" > "$SD/acct-01/limits.json"
|
|
2718
|
-
printf '{"fetched_at":%s,"weekly_percent":50,"session_percent":
|
|
3477
|
+
printf '{"fetched_at":%s,"weekly_percent":50,"session_percent":55,"max_percent":55,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2719
3478
|
"$((now - $1))" "$((now + 200000))" > "$SD/acct-02/limits.json"
|
|
2720
3479
|
: > "$SD/selection.log"
|
|
2721
3480
|
rm -f "$SD/.last-pick"
|
|
@@ -2965,6 +3724,153 @@ else
|
|
|
2965
3724
|
t_ok "sync validation tests skipped (Mac-only feature; server refuses sync by design)"
|
|
2966
3725
|
fi
|
|
2967
3726
|
|
|
3727
|
+
# ---- 16d. every limits pass fans telemetry out to the manifest's peers ---------------
|
|
3728
|
+
# Why this has teeth (2026-09-04): the runner Macs mini-3..mini-8 hold only portable
|
|
3729
|
+
# setup tokens, and the usage endpoint refuses those for good (403, no user:profile), so
|
|
3730
|
+
# the ONLY telemetry they can ever rank on is the source machine's. limits_distribute
|
|
3731
|
+
# pushes limits.json + .limited to the manifest's `server` AND to every entry in `peers`
|
|
3732
|
+
# — nothing had ever proven the peer half, and it is the half that decides whether six
|
|
3733
|
+
# machines rank blind. Not Mac-gated like `sync`: a fake rsync on PATH records the argv
|
|
3734
|
+
# and the --files-from list (the caller deletes that list as soon as the last push
|
|
3735
|
+
# returns, so it is read at invocation time, not afterwards).
|
|
3736
|
+
DPOOL="$WORK/distribute-pool"
|
|
3737
|
+
mkdir -p "$DPOOL/acct-01" "$DPOOL/acct-02" "$DPOOL/tmp"
|
|
3738
|
+
: > "$DPOOL/.limits-kick"
|
|
3739
|
+
cat > "$DPOOL/accounts.json" <<'EOF'
|
|
3740
|
+
{"version":1,"server":"root@203.0.113.9","server_root":"/root/.claude-accounts",
|
|
3741
|
+
"server_repo":"/root/claude-multiacc","threshold":90,
|
|
3742
|
+
"peers":[
|
|
3743
|
+
{"target":"gas@mini-3","root":"/Users/gas/.claude-accounts","repo":"/Users/gas/claude-multiacc"},
|
|
3744
|
+
{"target":"gas@mini-4","root":"/Users/gas/.claude-accounts","repo":"/Users/gas/claude-multiacc"}],
|
|
3745
|
+
"accounts":[
|
|
3746
|
+
{"id":"acct-01","email":"dp1@test","home":"mac","added_at":"2026-07-13T00:00:00Z"},
|
|
3747
|
+
{"id":"acct-02","email":"dp2@test","home":"mac","added_at":"2026-07-13T00:00:00Z"}]}
|
|
3748
|
+
EOF
|
|
3749
|
+
for i in 01 02; do
|
|
3750
|
+
printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-dp%s","refreshToken":"r","expiresAt":9999999999999,"refreshTokenExpiresAt":9999999999999}}' \
|
|
3751
|
+
"$i" > "$DPOOL/acct-$i/.credentials.json"
|
|
3752
|
+
done
|
|
3753
|
+
# an exclusion marker rides along with the readings (this one survives a clean pass)
|
|
3754
|
+
printf '%s\nbucket=error-cooldown percent=? reason=error-cooldown\n' "$(( $(date +%s) + 600 ))" \
|
|
3755
|
+
> "$DPOOL/acct-02/.limited"
|
|
3756
|
+
RSFAKE="$WORK/rsyncfake"
|
|
3757
|
+
export RSLOG="$WORK/rsync-push.log"
|
|
3758
|
+
mkdir -p "$RSFAKE"
|
|
3759
|
+
cat > "$RSFAKE/rsync" <<'EOF'
|
|
3760
|
+
#!/usr/bin/env bash
|
|
3761
|
+
printf 'RSYNC %s\n' "$*" >> "${RSLOG:?}"
|
|
3762
|
+
for a in "$@"; do
|
|
3763
|
+
case "$a" in
|
|
3764
|
+
--files-from=*)
|
|
3765
|
+
while IFS= read -r l; do printf 'FILE %s\n' "$l" >> "$RSLOG"; done < "${a#--files-from=}" ;;
|
|
3766
|
+
esac
|
|
3767
|
+
done
|
|
3768
|
+
exit 0
|
|
3769
|
+
EOF
|
|
3770
|
+
chmod +x "$RSFAKE/rsync"
|
|
3771
|
+
: > "$RSLOG"
|
|
3772
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3773
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3774
|
+
# The push is deliberately DETACHED — a sleeping peer must never delay the next refresh —
|
|
3775
|
+
# so wait for it instead of assuming it finished.
|
|
3776
|
+
waited=0
|
|
3777
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3778
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3779
|
+
done
|
|
3780
|
+
# The pushes are serialized by a lock dir, so a slow peer can never stack them up. Let
|
|
3781
|
+
# it drain before the next case, or that case's push would be dropped, not made.
|
|
3782
|
+
dp_drain() {
|
|
3783
|
+
local w=0
|
|
3784
|
+
while [ "$w" -lt 50 ] && [ -d "$DPOOL/tmp/limits-push.lock" ]; do sleep 0.2; w=$((w + 1)); done
|
|
3785
|
+
}
|
|
3786
|
+
dp_drain
|
|
3787
|
+
n_push="$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)"
|
|
3788
|
+
[ "$n_push" = "3" ] && t_ok "a limits pass pushes once per target (server + 2 peers)" \
|
|
3789
|
+
|| t_fail "limits distribute" "expected 3 rsync calls, saw $n_push: $(tr '\n' '|' < "$RSLOG")"
|
|
3790
|
+
grep -q "^RSYNC .*root@203.0.113.9:/root/.claude-accounts/" "$RSLOG" \
|
|
3791
|
+
&& t_ok "telemetry is pushed to the manifest server" \
|
|
3792
|
+
|| t_fail "limits distribute server" "$(grep '^RSYNC' "$RSLOG")"
|
|
3793
|
+
for peer in gas@mini-3 gas@mini-4; do
|
|
3794
|
+
grep -q "^RSYNC .*$peer:/Users/gas/.claude-accounts/" "$RSLOG" \
|
|
3795
|
+
&& t_ok "telemetry is pushed to manifest peer $peer" \
|
|
3796
|
+
|| t_fail "limits distribute peer" "$peer never received a push: $(grep '^RSYNC' "$RSLOG")"
|
|
3797
|
+
done
|
|
3798
|
+
# ...and every target gets the SAME list: each account's reading, plus any marker.
|
|
3799
|
+
for f in "acct-01/limits.json" "acct-02/limits.json" "acct-02/.limited"; do
|
|
3800
|
+
[ "$(grep -c "^FILE $f\$" "$RSLOG")" = "3" ] \
|
|
3801
|
+
&& t_ok "the pushed file list names $f for all three targets" \
|
|
3802
|
+
|| t_fail "limits distribute file list" "$f appears $(grep -c "^FILE $f\$" "$RSLOG")x, want 3"
|
|
3803
|
+
done
|
|
3804
|
+
|
|
3805
|
+
# A REPLICA receives telemetry and must never push it back: two writers racing over one
|
|
3806
|
+
# pool is last-writer-wins chaos, and a manifest carrying `peers` is itself pushed TO the
|
|
3807
|
+
# replicas — so the role marker is a machine-local side file, checked before anything else.
|
|
3808
|
+
printf 'replica\n' > "$DPOOL/sync-role"
|
|
3809
|
+
: > "$RSLOG"
|
|
3810
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3811
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3812
|
+
sleep 1
|
|
3813
|
+
[ ! -s "$RSLOG" ] && t_ok "a replica pool never pushes telemetry (limits_distribute is a no-op)" \
|
|
3814
|
+
|| t_fail "replica distribute" "a replica pushed: $(tr '\n' '|' < "$RSLOG")"
|
|
3815
|
+
# only an EXACT 'replica' suppresses it — same anchor as sync
|
|
3816
|
+
printf 'not-replica\n' > "$DPOOL/sync-role"
|
|
3817
|
+
: > "$RSLOG"
|
|
3818
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3819
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3820
|
+
waited=0
|
|
3821
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3822
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3823
|
+
done
|
|
3824
|
+
dp_drain
|
|
3825
|
+
[ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" = "3" ] \
|
|
3826
|
+
&& t_ok "only an exact 'replica' value suppresses the telemetry push" \
|
|
3827
|
+
|| t_fail "replica anchor (distribute)" "'not-replica' suppressed the push"
|
|
3828
|
+
rm -f "$DPOOL/sync-role"
|
|
3829
|
+
|
|
3830
|
+
# ---- 16d-lock. a distribute lock is honored while it is alive, broken once it is not -
|
|
3831
|
+
# The push serializes on a lock DIRECTORY, and a detached push that is killed (logout,
|
|
3832
|
+
# reboot, pkill) never runs its EXIT trap. `mkdir "$lock" || return 0` can then never
|
|
3833
|
+
# succeed again: on the live pool one stranded lock stopped ALL telemetry distribution
|
|
3834
|
+
# from 2026-09-03 00:29 until it was removed by hand on 2026-09-04 — 32 hours in which
|
|
3835
|
+
# every peer ranked on whatever limits.json it happened to already have, which is the
|
|
3836
|
+
# blindness this push exists to prevent, and nothing anywhere said so. A push is seconds
|
|
3837
|
+
# of rsync under hard timeouts, so a lock older than ten minutes belongs to a process
|
|
3838
|
+
# that is gone.
|
|
3839
|
+
: > "$RSLOG"
|
|
3840
|
+
: > "$DPOOL/sync.log"
|
|
3841
|
+
rm -rf "$DPOOL/tmp/limits-push.lock"
|
|
3842
|
+
mkdir -p "$DPOOL/tmp/limits-push.lock"
|
|
3843
|
+
touch -t 202001010000 "$DPOOL/tmp/limits-push.lock" # abandoned in 2020, not busy
|
|
3844
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3845
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3846
|
+
waited=0
|
|
3847
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3848
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3849
|
+
done
|
|
3850
|
+
dp_drain
|
|
3851
|
+
[ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" = "3" ] \
|
|
3852
|
+
&& t_ok "a stale limits-push lock is broken and the pass distributes anyway" \
|
|
3853
|
+
|| t_fail "stale distribute lock" "expected 3 rsync calls, saw $(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)"
|
|
3854
|
+
grep -q "stale limits-push lock broken" "$DPOOL/sync.log" \
|
|
3855
|
+
&& t_ok "breaking the lock is recorded, so a recurrence is visible instead of silent" \
|
|
3856
|
+
|| t_fail "stale lock log" "sync.log: $(tail -3 "$DPOOL/sync.log" 2>/dev/null | tr '\n' '|')"
|
|
3857
|
+
[ ! -d "$DPOOL/tmp/limits-push.lock" ] \
|
|
3858
|
+
&& t_ok "the retaken lock is released at the end of the push, not leaked again" \
|
|
3859
|
+
|| t_fail "stale lock retake" "the lock dir is still present after the push"
|
|
3860
|
+
|
|
3861
|
+
# ...and a lock that a LIVE push is holding is still absolute: two rsyncs racing into one
|
|
3862
|
+
# peer is exactly what the lock exists to stop, so a fresh one skips this pass entirely.
|
|
3863
|
+
: > "$RSLOG"
|
|
3864
|
+
mkdir -p "$DPOOL/tmp/limits-push.lock" # mtime = now: someone is pushing
|
|
3865
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3866
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3867
|
+
sleep 1
|
|
3868
|
+
{ [ ! -s "$RSLOG" ] && [ -d "$DPOOL/tmp/limits-push.lock" ]; } \
|
|
3869
|
+
&& t_ok "a fresh distribute lock is honored: no push, and the lock is left where it was" \
|
|
3870
|
+
|| t_fail "live distribute lock" "pushes=$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null) lock=$([ -d "$DPOOL/tmp/limits-push.lock" ] && echo held || echo REMOVED)"
|
|
3871
|
+
rm -rf "$DPOOL/tmp/limits-push.lock"
|
|
3872
|
+
unset RSLOG
|
|
3873
|
+
|
|
2968
3874
|
# API keys are never accepted as credentials (subscription-only requirement)
|
|
2969
3875
|
printf 'sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' > "$WORK/apikey.txt"
|
|
2970
3876
|
out="$(claude-accounts import apikey@test --id acct-11 --token-file "$WORK/apikey.txt" --no-sync 2>&1)"
|
|
@@ -3346,7 +4252,7 @@ check "codex: passthrough with CODEX_HOME" "CFG=other" "$out"
|
|
|
3346
4252
|
out="$(CODEX_MULTIACC_DISABLE=1 codex 2>&1)"
|
|
3347
4253
|
check "codex: passthrough when disabled" "CFG=none" "$out"
|
|
3348
4254
|
|
|
3349
|
-
# ---- C3. headroom selection
|
|
4255
|
+
# ---- C3. headroom selection: session gate first, then the WEEKLY headroom band ------
|
|
3350
4256
|
cxlj() { printf '{"fetched_at":%s,"weekly_percent":%s,"session_percent":%s,"max_percent":%s,"buckets":[]}' "$now" "$1" "$2" "$3"; }
|
|
3351
4257
|
cxlj 80 10 80 > "$CX/acct-01/limits.json"
|
|
3352
4258
|
cxlj 20 10 20 > "$CX/acct-02/limits.json"
|
|
@@ -3377,15 +4283,118 @@ for _ in $(seq 1 10); do
|
|
|
3377
4283
|
done
|
|
3378
4284
|
[ "$all1" = 1 ] && t_ok "codex: headroom band 0 restores strict ranking" \
|
|
3379
4285
|
|| t_fail "codex zero headroom band" "the runner-up was selected"
|
|
3380
|
-
#
|
|
4286
|
+
# THE KEY CASE, REVERSED on 2026-09-03, in parity with the claude shim (section 5):
|
|
4287
|
+
# the operator asked for "among accounts where high session limits it must choose
|
|
4288
|
+
# randomly from ones where highest weekly limits", so a nearly-spent 5h bucket is the
|
|
4289
|
+
# FIRST cut and the 30-point weekly band ranks only what clears the gate. acct-01:
|
|
4290
|
+
# session 85 (past the gate), weekly 10; acct-02: session 20, weekly 70 -> acct-02.
|
|
4291
|
+
cxlj 10 85 85 > "$CX/acct-01/limits.json"
|
|
4292
|
+
cxlj 70 20 70 > "$CX/acct-02/limits.json"
|
|
4293
|
+
all2=1
|
|
4294
|
+
for _ in $(seq 1 15); do
|
|
4295
|
+
case "$(codex 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
4296
|
+
done
|
|
4297
|
+
[ "$all2" = "1" ] && t_ok "codex: a session bucket past the gate is skipped while a fresher one exists (70w/20s over 10w/85s)" \
|
|
4298
|
+
|| t_fail "codex session gate" "the account with its 5h bucket at 85% was still selected"
|
|
4299
|
+
: > "$CX/selection.log"
|
|
4300
|
+
codex >/dev/null 2>&1
|
|
4301
|
+
grep -qE 'acct-02 weekly=70% session=20% band=30 band-count=1 session-gate=50 session-ok=1 pwd=' "$CX/selection.log" \
|
|
4302
|
+
&& t_ok "codex: the selection log carries session-gate=50 session-ok=1 when one of two clears" \
|
|
4303
|
+
|| t_fail "codex session gate log" "$(tail -1 "$CX/selection.log")"
|
|
4304
|
+
# Both sessions inside the gate: it has nothing to say and weekly decides as before.
|
|
4305
|
+
cxlj 10 45 45 > "$CX/acct-01/limits.json"
|
|
4306
|
+
cxlj 70 20 70 > "$CX/acct-02/limits.json"
|
|
4307
|
+
all1=1
|
|
4308
|
+
for _ in $(seq 1 15); do
|
|
4309
|
+
case "$(codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
4310
|
+
done
|
|
4311
|
+
[ "$all1" = "1" ] && t_ok "codex: with both sessions inside the gate weekly headroom decides (10w/45s over 70w/20s)" \
|
|
4312
|
+
|| t_fail "codex session gate no-op" "the gate changed a ranking where every candidate cleared it"
|
|
4313
|
+
# Nobody clears it: the gate compares, it never empties the pool — it steps aside.
|
|
4314
|
+
cxlj 10 85 85 > "$CX/acct-01/limits.json"
|
|
4315
|
+
cxlj 70 60 70 > "$CX/acct-02/limits.json"
|
|
4316
|
+
: > "$CX/selection.log"
|
|
4317
|
+
all1=1
|
|
4318
|
+
for _ in $(seq 1 15); do
|
|
4319
|
+
case "$(codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
4320
|
+
done
|
|
4321
|
+
[ "$all1" = "1" ] && t_ok "codex: with nobody inside the gate it steps aside and weekly ranks (10w/85s over 70w/60s)" \
|
|
4322
|
+
|| t_fail "codex session gate step-aside" "an empty gate emptied the pool instead of stepping aside"
|
|
4323
|
+
grep -qE 'acct-01 weekly=10% session=85% band=30 band-count=1 session-gate=50 session-ok=0 pwd=' "$CX/selection.log" \
|
|
4324
|
+
&& t_ok "codex: the selection log carries session-ok=0 when the gate steps aside" \
|
|
4325
|
+
|| t_fail "codex session gate log" "$(tail -1 "$CX/selection.log")"
|
|
4326
|
+
# The gate is a knob, like the band: 100 turns it off; garbage falls back to 50.
|
|
3381
4327
|
cxlj 10 85 85 > "$CX/acct-01/limits.json"
|
|
3382
4328
|
cxlj 70 20 70 > "$CX/acct-02/limits.json"
|
|
3383
4329
|
all1=1
|
|
4330
|
+
for _ in $(seq 1 15); do
|
|
4331
|
+
case "$(CODEX_MULTIACC_SESSION_GATE=100 codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
4332
|
+
done
|
|
4333
|
+
[ "$all1" = "1" ] && t_ok "codex: CODEX_MULTIACC_SESSION_GATE=100 disables the gate" \
|
|
4334
|
+
|| t_fail "codex session gate off" "the gate still fired at 100"
|
|
4335
|
+
all2=1
|
|
4336
|
+
for _ in $(seq 1 15); do
|
|
4337
|
+
case "$(CODEX_MULTIACC_SESSION_GATE=abc codex 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
4338
|
+
done
|
|
4339
|
+
[ "$all2" = "1" ] && t_ok "codex: a non-numeric CODEX_MULTIACC_SESSION_GATE falls back to 50" \
|
|
4340
|
+
|| t_fail "codex session gate validation" "a garbage gate value changed the outcome"
|
|
4341
|
+
# Equal weekly usage: the GATE decides, never a tiebreak — strict band or default 30.
|
|
4342
|
+
cxlj 40 20 40 > "$CX/acct-01/limits.json"
|
|
4343
|
+
cxlj 40 80 80 > "$CX/acct-02/limits.json"
|
|
4344
|
+
all1=1
|
|
4345
|
+
for _ in $(seq 1 15); do
|
|
4346
|
+
case "$(CODEX_MULTIACC_HEADROOM_BAND=0 codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
4347
|
+
done
|
|
4348
|
+
[ "$all1" = "1" ] && t_ok "codex: strict mode: the session gate decides an exact weekly tie" \
|
|
4349
|
+
|| t_fail "codex session gate tie" "a weekly tie was not resolved by the session gate"
|
|
4350
|
+
all1=1
|
|
3384
4351
|
for _ in $(seq 1 15); do
|
|
3385
4352
|
case "$(codex 2>&1)" in *CFG=acct-01*) ;; *) all1=0 ;; esac
|
|
3386
4353
|
done
|
|
3387
|
-
[ "$all1" = "1" ] && t_ok "codex:
|
|
3388
|
-
|| t_fail "codex
|
|
4354
|
+
[ "$all1" = "1" ] && t_ok "codex: the gate (not the band) removes the session-heavy half of a weekly tie" \
|
|
4355
|
+
|| t_fail "codex session gate tie" "the default band let the 80-point session account back in"
|
|
4356
|
+
|
|
4357
|
+
# Inside the gate, session is NOT a tiebreaker any more: an exact weekly tie between two
|
|
4358
|
+
# gate-clearing accounts is a coin flip even in strict mode (it used to go to the lower
|
|
4359
|
+
# session, through the old weekly*1000+session score).
|
|
4360
|
+
cxlj 40 20 40 > "$CX/acct-01/limits.json"
|
|
4361
|
+
cxlj 40 45 45 > "$CX/acct-02/limits.json"
|
|
4362
|
+
hits1=0; hits2=0
|
|
4363
|
+
for _ in $(seq 1 20); do
|
|
4364
|
+
case "$(CODEX_MULTIACC_HEADROOM_BAND=0 codex 2>&1)" in
|
|
4365
|
+
*CFG=acct-01*) hits1=$((hits1+1)) ;;
|
|
4366
|
+
*CFG=acct-02*) hits2=$((hits2+1)) ;;
|
|
4367
|
+
esac
|
|
4368
|
+
done
|
|
4369
|
+
{ [ "$hits1" -gt 0 ] && [ "$hits2" -gt 0 ]; } \
|
|
4370
|
+
&& t_ok "codex: strict mode: session does not break an exact weekly tie inside the gate (acct-01=$hits1 acct-02=$hits2)" \
|
|
4371
|
+
|| t_fail "codex: session tiebreak removed" "acct-01=$hits1 acct-02=$hits2 (want both >0)"
|
|
4372
|
+
|
|
4373
|
+
# Clearing the gate takes BOTH readings, as in pool-selection.v2: a fresh file with a
|
|
4374
|
+
# session reading but no weekly one must not become the sole gate-clearer and win the
|
|
4375
|
+
# all-gated tie over an account with a truthful weekly reading — not even with a
|
|
4376
|
+
# max_percent to fall back on (the shims used to rank on that; the policy never could).
|
|
4377
|
+
# (No writer produces such a file; this pins parity with lib/selector_policy.py.)
|
|
4378
|
+
printf '{"fetched_at":%s,"max_percent":10,"session_percent":10,"buckets":[]}' "$now" > "$CX/acct-01/limits.json"
|
|
4379
|
+
cxlj 20 80 80 > "$CX/acct-02/limits.json"
|
|
4380
|
+
all2=1
|
|
4381
|
+
for _ in $(seq 1 15); do
|
|
4382
|
+
case "$(codex 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
4383
|
+
done
|
|
4384
|
+
[ "$all2" = "1" ] && t_ok "codex: a session reading without a weekly one never clears the gate" \
|
|
4385
|
+
|| t_fail "codex: gate needs both readings" "an unknown-weekly account beat a truthful weekly reading"
|
|
4386
|
+
|
|
4387
|
+
# ...and the converse: a weekly reading without a session one is not "known" either — it
|
|
4388
|
+
# neither clears the gate nor ranks once the gate steps aside (10w/?s vs 70w/80s -> the
|
|
4389
|
+
# 70w account, as in pool-selection.v2, where quota_known needs both readings).
|
|
4390
|
+
printf '{"fetched_at":%s,"weekly_percent":10,"max_percent":10,"buckets":[]}' "$now" > "$CX/acct-01/limits.json"
|
|
4391
|
+
cxlj 70 80 80 > "$CX/acct-02/limits.json"
|
|
4392
|
+
all2=1
|
|
4393
|
+
for _ in $(seq 1 15); do
|
|
4394
|
+
case "$(codex 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
4395
|
+
done
|
|
4396
|
+
[ "$all2" = "1" ] && t_ok "codex: a weekly reading without a session one is unknown to both cuts" \
|
|
4397
|
+
|| t_fail "codex: known needs both readings" "a session-less weekly reading ranked as known"
|
|
3389
4398
|
# equal scores spread load
|
|
3390
4399
|
cxlj 10 10 10 > "$CX/acct-01/limits.json"
|
|
3391
4400
|
cxlj 10 10 10 > "$CX/acct-02/limits.json"
|
|
@@ -3652,17 +4661,65 @@ c_n="$(printf '%s' "$c_distinct" | wc -w | tr -d ' ')"
|
|
|
3652
4661
|
rm -rf "$CX/acct-09"
|
|
3653
4662
|
rm -f "$CX/.last-pick"
|
|
3654
4663
|
|
|
3655
|
-
# ---- C6. all limited ->
|
|
4664
|
+
# ---- C6. all limited -> the still-serving accounts go through the same two cuts, strict weekly --
|
|
3656
4665
|
printf '%s\nbucket=7d percent=95 reason=limits\n' "$((now+3600))" > "$CX/acct-01/.limited"
|
|
3657
4666
|
printf '%s\nbucket=7d percent=99 reason=limits\n' "$((now+3600))" > "$CX/acct-02/.limited"
|
|
3658
4667
|
cxlj 95 10 95 > "$CX/acct-01/limits.json"
|
|
3659
4668
|
cxlj 99 10 99 > "$CX/acct-02/limits.json"
|
|
3660
4669
|
out="$(codex 2>&1)"
|
|
3661
|
-
check "codex: all-limited falls back to
|
|
4670
|
+
check "codex: all-limited falls back to the still-serving account with the most weekly headroom" "CFG=acct-01" "$out"
|
|
3662
4671
|
grep -q "all-limited fallback=acct-01" "$CX/selection.log" \
|
|
3663
4672
|
&& t_ok "codex: all-limited fallback logged" || t_fail "codex fallback log" "no all-limited line"
|
|
3664
4673
|
rm -f "$CX"/acct-*/.limited "$CX"/acct-*/limits.json
|
|
3665
4674
|
|
|
4675
|
+
# The fallback applies the SAME two cuts (parity with the claude shim, section 9): an
|
|
4676
|
+
# account past the session gate yields to one inside it even with far better weekly.
|
|
4677
|
+
printf '%s\nbucket=7d percent=95 reason=limits\n' "$((now+3600))" > "$CX/acct-01/.limited"
|
|
4678
|
+
printf '%s\nbucket=7d percent=95 reason=limits\n' "$((now+3600))" > "$CX/acct-02/.limited"
|
|
4679
|
+
cxlj 10 85 85 > "$CX/acct-01/limits.json"
|
|
4680
|
+
cxlj 70 20 70 > "$CX/acct-02/limits.json"
|
|
4681
|
+
all2=1
|
|
4682
|
+
for _ in $(seq 1 12); do
|
|
4683
|
+
case "$(codex 2>&1)" in *CFG=acct-02*) ;; *) all2=0 ;; esac
|
|
4684
|
+
done
|
|
4685
|
+
[ "$all2" = "1" ] && t_ok "codex: the all-limited fallback applies the session gate before strict weekly (70w/20s over 10w/85s)" \
|
|
4686
|
+
|| t_fail "codex fallback session gate" "the fallback handed out the account past the session gate"
|
|
4687
|
+
grep -q "all-limited fallback=acct-02 weekly=70%" "$CX/selection.log" \
|
|
4688
|
+
&& t_ok "codex: the fallback line names the gated pick" || t_fail "codex fallback gate log" "$(tail -1 "$CX/selection.log")"
|
|
4689
|
+
rm -f "$CX"/acct-*/.limited "$CX"/acct-*/limits.json
|
|
4690
|
+
|
|
4691
|
+
# ---- C6b. the fallback tells "still serving" from "rejected right now" ----------------
|
|
4692
|
+
# Ported from the claude shim (2026-08-29) when the session gate reached the fallback.
|
|
4693
|
+
# acct-01: 7d window at 99% — worse headroom, but still answering. acct-02: 5h window at
|
|
4694
|
+
# 100% — far better weekly (7%), but every request bounces until the reset.
|
|
4695
|
+
printf '%s\nbucket=7d percent=99 marked_at=x reason=limits\n' "$((now+3600))" > "$CX/acct-01/.limited"
|
|
4696
|
+
printf '%s\nbucket=5h percent=100 marked_at=x reason=limits\n' "$((now+600))" > "$CX/acct-02/.limited"
|
|
4697
|
+
cxlj 99 10 99 > "$CX/acct-01/limits.json"
|
|
4698
|
+
cxlj 7 100 100 > "$CX/acct-02/limits.json"
|
|
4699
|
+
out="$(codex 2>&1)"
|
|
4700
|
+
check "codex: a still-serving limited account beats an exhausted one with more headroom" "CFG=acct-01" "$out"
|
|
4701
|
+
# The codex-review case (2026-09-04): the exhausted account is the only one INSIDE the
|
|
4702
|
+
# session gate; the gate must not resurrect it over a still-serving account outside it.
|
|
4703
|
+
printf '%s\nbucket=7d percent=100 marked_at=x reason=limits\n' "$((now+3600))" > "$CX/acct-01/.limited"
|
|
4704
|
+
printf '%s\nbucket=7d percent=95 marked_at=x reason=limits\n' "$((now+3600))" > "$CX/acct-02/.limited"
|
|
4705
|
+
cxlj 100 20 100 > "$CX/acct-01/limits.json"
|
|
4706
|
+
cxlj 95 85 95 > "$CX/acct-02/limits.json"
|
|
4707
|
+
out="$(codex 2>&1)"
|
|
4708
|
+
check "codex: an exhausted account inside the gate never beats a still-serving one outside it" "CFG=acct-02" "$out"
|
|
4709
|
+
# A real client rejection (a 429 the server sent) is exhausted whatever percent says.
|
|
4710
|
+
printf '%s\nbucket=client:5h percent=95 marked_at=x reason=client-rate-limit\n' "$((now+600))" > "$CX/acct-01/.limited"
|
|
4711
|
+
cxlj 95 10 95 > "$CX/acct-01/limits.json"
|
|
4712
|
+
out="$(codex 2>&1)"
|
|
4713
|
+
check "codex: a client-rejected account is not the fallback while another still serves" "CFG=acct-02" "$out"
|
|
4714
|
+
# Every account exhausted RIGHT NOW: hand out the one that unblocks first.
|
|
4715
|
+
printf '%s\nbucket=5h percent=100 marked_at=x reason=limits\n' "$((now+7200))" > "$CX/acct-01/.limited"
|
|
4716
|
+
printf '%s\nbucket=5h percent=100 marked_at=x reason=limits\n' "$((now+600))" > "$CX/acct-02/.limited"
|
|
4717
|
+
out="$(codex 2>&1)"
|
|
4718
|
+
check "codex: all exhausted: the soonest reset is handed out" "CFG=acct-02" "$out"
|
|
4719
|
+
grep -q "all-exhausted resets_in=" "$CX/selection.log" \
|
|
4720
|
+
&& t_ok "codex: the all-exhausted pick is logged with its reset" || t_fail "codex all-exhausted log" "no line"
|
|
4721
|
+
rm -f "$CX"/acct-*/.limited "$CX"/acct-*/limits.json
|
|
4722
|
+
|
|
3666
4723
|
# ---- C7. dead logins: .expired excludes, heals on newer credential ------------------
|
|
3667
4724
|
printf '%s\nreason=refresh-denied-http-400 marked_at=t detail=x\n' "$now" > "$CX/acct-01/.expired"
|
|
3668
4725
|
all2=1
|
|
@@ -3682,6 +4739,9 @@ mk_cx_auth "$CX/acct-01/auth.json" a@cx "$FUTURE_EXP"
|
|
|
3682
4739
|
out="$(CODEX_SHIM_SELECT=random codex 2>&1)"
|
|
3683
4740
|
[ ! -f "$CX/acct-01/.expired" ] && t_ok "codex: newer auth.json clears a credential park" \
|
|
3684
4741
|
|| t_fail "codex park heal" ".expired survived a newer credential"
|
|
4742
|
+
grep -qE 'band=30 band-count=[0-9]+ session-gate=off session-ok=[0-9]+ pwd=' "$CX/selection.log" \
|
|
4743
|
+
&& t_ok "codex: random mode logs session-gate=off instead of a cut it never made" \
|
|
4744
|
+
|| t_fail "codex random mode gate log" "$(tail -1 "$CX/selection.log")"
|
|
3685
4745
|
# an org-blocked park survives a newer credential (policy, not credential)
|
|
3686
4746
|
printf '%s\nreason=org-blocked marked_at=t detail=x\n' "$now" > "$CX/acct-01/.expired"
|
|
3687
4747
|
sleep 1
|
|
@@ -4095,6 +5155,417 @@ out="$(CODEX_MULTIACC_USAGE_URL="file://$WORK/cx-usage-low.json" codex-accounts
|
|
|
4095
5155
|
check "codex: 429 backoff honored" "acct-02: backing off after 429" "$out"
|
|
4096
5156
|
rm -f "$CX"/acct-*/limits.json
|
|
4097
5157
|
|
|
5158
|
+
# ---- C13b. codex parity: a 0% window with NO reported reset is NO DATA ---------------
|
|
5159
|
+
# The claude pool's 2026-09-04 incident (acct-13/acct-14 served every bucket
|
|
5160
|
+
# `percent: 0, resets_at: null`, ranked as the emptiest accounts in the fleet, handed
|
|
5161
|
+
# 31 of the last ~60 picks while the client was being rejected on them at their weekly
|
|
5162
|
+
# limit) is a payload failure, not a claude-specific one — this writer must refuse the
|
|
5163
|
+
# same way. Codex's usage payload carries reset_at INSIDE each window, and this writer
|
|
5164
|
+
# formats resets_at itself, so "the payload reported a window" is the distinction.
|
|
5165
|
+
CD="$WORK/cx-nodata-pool"
|
|
5166
|
+
mkdir -p "$CD/acct-01" "$CD/acct-02" "$CD/tmp"
|
|
5167
|
+
: > "$CD/.limits-kick"
|
|
5168
|
+
cat > "$CD/accounts.json" <<'EOF'
|
|
5169
|
+
{"version":1,"server":"none","threshold":90,"accounts":[
|
|
5170
|
+
{"id":"acct-01","email":"nd1@cx","home":"mac","added_at":"2026-08-21T00:00:00Z"},
|
|
5171
|
+
{"id":"acct-02","email":"nd2@cx","home":"mac","added_at":"2026-08-21T00:00:00Z"}]}
|
|
5172
|
+
EOF
|
|
5173
|
+
mk_cx_auth "$CD/acct-01/auth.json" nd1@cx "$FUTURE_EXP"
|
|
5174
|
+
mk_cx_auth "$CD/acct-02/auth.json" nd2@cx "$FUTURE_EXP"
|
|
5175
|
+
cdl() { # cdl <weekly> <session> <max> -> a truthful, in-window reading on stdout
|
|
5176
|
+
printf '{"fetched_at":%s,"source":"chatgpt","weekly_percent":%s,"session_percent":%s,"max_percent":%s,"plan":"pro","buckets":[]}' \
|
|
5177
|
+
"$(date +%s)" "$1" "$2" "$3"
|
|
5178
|
+
}
|
|
5179
|
+
cdlimits() { # cdlimits <fixture> [extra args] -> a real refresh over the whole CD pool
|
|
5180
|
+
local f="$1"; shift
|
|
5181
|
+
CODEX_ACCOUNTS_ROOT="$CD" CODEX_MULTIACC_USAGE_URL="file://$f" \
|
|
5182
|
+
codex-accounts limits --force "$@" 2>&1
|
|
5183
|
+
}
|
|
5184
|
+
|
|
5185
|
+
# every window 0% with no reset_at at all: nothing here says anything
|
|
5186
|
+
cat > "$WORK/cx-usage-allzero.json" <<'EOF'
|
|
5187
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5188
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5189
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5190
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}},
|
|
5191
|
+
"additional_rate_limits":[]}
|
|
5192
|
+
EOF
|
|
5193
|
+
out="$(cdlimits "$WORK/cx-usage-allzero.json")"
|
|
5194
|
+
check "codex: an all-zero/no-window payload is reported as no usable telemetry" \
|
|
5195
|
+
"no usable telemetry (account ranks as unknown, not as empty)" "$out"
|
|
5196
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5197
|
+
import json, sys
|
|
5198
|
+
d = json.load(open(sys.argv[1]))
|
|
5199
|
+
assert d.get('no_data') is True, d
|
|
5200
|
+
for k in ('max_percent', 'weekly_percent', 'session_percent'):
|
|
5201
|
+
assert k not in d, (k, d)
|
|
5202
|
+
assert len(d['buckets']) == 2 and d['plan'] == 'pro', d # diagnostics survive
|
|
5203
|
+
# the writer's internal "did the payload report this window" flag never reaches disk
|
|
5204
|
+
for b in d['buckets']:
|
|
5205
|
+
assert '_reset_known' not in b, b
|
|
5206
|
+
EOF
|
|
5207
|
+
[ $? -eq 0 ] && t_ok "codex: a no-data pass records no_data and none of the percent signals" \
|
|
5208
|
+
|| t_fail "codex no_data document" "see $CD/acct-01/limits.json"
|
|
5209
|
+
|
|
5210
|
+
# one silent model window beside real ones changes nothing
|
|
5211
|
+
cat > "$WORK/cx-usage-mixed-nodata.json" <<EOF
|
|
5212
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5213
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5214
|
+
"primary_window":{"used_percent":5,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5215
|
+
"secondary_window":{"used_percent":9,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5216
|
+
"additional_rate_limits":[
|
|
5217
|
+
{"limit_name":"GPT-5.3-Codex-Spark","rate_limit":{"allowed":true,"limit_reached":false,
|
|
5218
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5219
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}}}]}
|
|
5220
|
+
EOF
|
|
5221
|
+
cdlimits "$WORK/cx-usage-mixed-nodata.json" --quiet >/dev/null
|
|
5222
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5223
|
+
import json, sys
|
|
5224
|
+
d = json.load(open(sys.argv[1]))
|
|
5225
|
+
assert 'no_data' not in d, d
|
|
5226
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (5, 9, 9), d
|
|
5227
|
+
assert len(d['buckets']) == 4, d # the two silent model windows are still recorded
|
|
5228
|
+
EOF
|
|
5229
|
+
[ $? -eq 0 ] && t_ok "codex: one silent model window beside real ones leaves the ranking untouched (5/9)" \
|
|
5230
|
+
|| t_fail "codex mixed no-data payload" "see $CD/acct-01/limits.json"
|
|
5231
|
+
|
|
5232
|
+
# 0% WITH reported resets is a real, empty reading
|
|
5233
|
+
cat > "$WORK/cx-usage-zero-real-windows.json" <<EOF
|
|
5234
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5235
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5236
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5237
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5238
|
+
"additional_rate_limits":[]}
|
|
5239
|
+
EOF
|
|
5240
|
+
cdlimits "$WORK/cx-usage-zero-real-windows.json" --quiet >/dev/null
|
|
5241
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5242
|
+
import json, sys
|
|
5243
|
+
d = json.load(open(sys.argv[1]))
|
|
5244
|
+
assert 'no_data' not in d, d
|
|
5245
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (0, 0, 0), d
|
|
5246
|
+
EOF
|
|
5247
|
+
[ $? -eq 0 ] && t_ok "codex: 0% WITH reported reset windows still records a real, empty reading" \
|
|
5248
|
+
|| t_fail "codex zero-with-windows payload" "see $CD/acct-01/limits.json"
|
|
5249
|
+
|
|
5250
|
+
# ...and an unknown account loses to any account with a real reading
|
|
5251
|
+
cdlimits "$WORK/cx-usage-allzero.json" --quiet >/dev/null
|
|
5252
|
+
cdl 45 10 45 > "$CD/acct-02/limits.json"
|
|
5253
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick "$CD"/acct-0*/.limited
|
|
5254
|
+
: > "$CD/selection.log"
|
|
5255
|
+
cd_hits=0
|
|
5256
|
+
for _ in $(seq 1 10); do
|
|
5257
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cd_hits=$((cd_hits+1)) ;; esac
|
|
5258
|
+
done
|
|
5259
|
+
[ "$cd_hits" = "0" ] \
|
|
5260
|
+
&& t_ok "codex: a no-data account never outranks one with real telemetry (0 of 10 picks)" \
|
|
5261
|
+
|| t_fail "codex no_data ranking" "the fake-zero account took $cd_hits of 10 picks"
|
|
5262
|
+
grep -q "acct-02 weekly=45% session=10% band=30 band-count=1" "$CD/selection.log" \
|
|
5263
|
+
&& t_ok "codex: the pick logs the known account alone in the band" \
|
|
5264
|
+
|| t_fail "codex no_data band" "selection.log: $(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5265
|
+
|
|
5266
|
+
# ---- C13c. codex parity: ONE marker rule, driven through the REAL rollout path -------
|
|
5267
|
+
# The codex shim has no telemetry-based clearing path at all (a marker leaves it only
|
|
5268
|
+
# when its own reset epoch passes), so the whole rule lives in `codex-accounts limits`.
|
|
5269
|
+
# Until 2026-09-04 that writer kept EVERY active client marker unconditionally, which
|
|
5270
|
+
# looks safe and is half wrong in each direction: a 7d rejection was safe by accident
|
|
5271
|
+
# rather than by rule, and a 5h rejection — a window that refills within hours — parked
|
|
5272
|
+
# the account until its own epoch, the exact stranding #22 (2026-09-03) had to fix on the
|
|
5273
|
+
# claude side.
|
|
5274
|
+
#
|
|
5275
|
+
# These cases no longer HAND the writer a marker. bin/codex writes it, from a rollout
|
|
5276
|
+
# transcript, the way a rejected run does — because the rule the writer applies reads a
|
|
5277
|
+
# token the SHIM chooses, and testing it on a hand-written marker tests a vocabulary the
|
|
5278
|
+
# product never produces. That is exactly what went wrong once already: the guard matched
|
|
5279
|
+
# seven_day/7d/weekly while the scan labelled markers `client:primary`/`client:secondary`
|
|
5280
|
+
# after the rollout's own rate_limits KEY names, which map to no fixed window at all
|
|
5281
|
+
# (live payloads report `primary` as the 10080-minute one). Weekly protection was a no-op
|
|
5282
|
+
# on this provider, and four green assertions said otherwise. The label is now derived
|
|
5283
|
+
# from the record's own `window_minutes` at write time, and these tests drive that
|
|
5284
|
+
# derivation end to end: rollout -> shim -> marker -> limits pass -> selection.
|
|
5285
|
+
# A third, healthy account: with only two, "everything was parked so the pool fell back"
|
|
5286
|
+
# and "the freed account was chosen" produce the same log line.
|
|
5287
|
+
mkdir -p "$CD/acct-03"
|
|
5288
|
+
mk_cx_auth "$CD/acct-03/auth.json" nd3@cx "$FUTURE_EXP"
|
|
5289
|
+
python3 - "$CD/accounts.json" <<'EOF'
|
|
5290
|
+
import json, sys
|
|
5291
|
+
d = json.load(open(sys.argv[1]))
|
|
5292
|
+
if not any(a['id'] == 'acct-03' for a in d['accounts']):
|
|
5293
|
+
d['accounts'].append({"id": "acct-03", "email": "nd3@cx", "home": "mac",
|
|
5294
|
+
"added_at": "2026-08-21T00:00:00Z"})
|
|
5295
|
+
json.dump(d, open(sys.argv[1], 'w'))
|
|
5296
|
+
EOF
|
|
5297
|
+
cx_mark() { # cx_mark <acct dir> <bucket> <reset-offset-seconds> <marked_at ISO>
|
|
5298
|
+
printf '%s\nbucket=%s percent=100 marked_at=%s reason=client-rate-limit\n' \
|
|
5299
|
+
"$(( $(date +%s) + $3 ))" "$2" "$4" > "$1/.limited"
|
|
5300
|
+
}
|
|
5301
|
+
cx_rollout() { # cx_rollout <acct dir> <used_percent> <resets_at epoch> <window_minutes|-> <record ISO>
|
|
5302
|
+
# One rollout, shaped like the real transcript: the codex CLI writes the windows the
|
|
5303
|
+
# server reported into every run's JSONL, and client_limit_scan reads the newest tail.
|
|
5304
|
+
# window_minutes rides in the SAME fragment as used_percent/resets_at — which is why
|
|
5305
|
+
# the shim can label the marker with the window that was actually spent.
|
|
5306
|
+
local day="$1/sessions/2026/09/04" f win=""
|
|
5307
|
+
rm -rf "$1/sessions"
|
|
5308
|
+
mkdir -p "$day"
|
|
5309
|
+
f="$day/rollout-2026-09-04T01-43-21-c13c0001-7fc3-7291-a0fc-7b4e2b035f1a.jsonl"
|
|
5310
|
+
[ "$4" = "-" ] || win="\"window_minutes\":$4,"
|
|
5311
|
+
{
|
|
5312
|
+
printf '{"timestamp":"%s","type":"session_meta","payload":{"session_id":"c13c0001","cwd":"/proj"}}\n' "$5"
|
|
5313
|
+
printf '{"timestamp":"%s","type":"event_msg","payload":{"type":"token_count","info":{"model_context_window":258400},"rate_limits":{"limit_id":"codex","limit_name":null,"primary":{"used_percent":%s,%s"resets_at":%s},"secondary":null,"credits":{"has_credits":false,"unlimited":false}}}}\n' \
|
|
5314
|
+
"$5" "$2" "$win" "$3"
|
|
5315
|
+
} > "$f"
|
|
5316
|
+
rm -f "$1/.client-scan"
|
|
5317
|
+
}
|
|
5318
|
+
cx_bucket() { # the bucket token on line 2 of <acct dir>/.limited, or '<none>'
|
|
5319
|
+
local b=""
|
|
5320
|
+
[ -f "$1/.limited" ] && b="$(sed -n '2s/.*bucket=\([^ ]*\).*/\1/p' "$1/.limited" 2>/dev/null)"
|
|
5321
|
+
printf '%s\n' "${b:-<none>}"
|
|
5322
|
+
}
|
|
5323
|
+
cdlimits0() { # cdlimits with the confirm delay OFF — a marker kept by this pass is kept
|
|
5324
|
+
# by RULE, never merely because the rejection is seconds old. (Real markers
|
|
5325
|
+
# are written by the shim moments before, so there is no "aged" marker to
|
|
5326
|
+
# fabricate; the delay is pinned separately below.)
|
|
5327
|
+
local f="$1"; shift
|
|
5328
|
+
CODEX_ACCOUNTS_ROOT="$CD" CODEX_MULTIACC_USAGE_URL="file://$f" \
|
|
5329
|
+
CODEX_MULTIACC_CLIENT_LIMIT_CONFIRM_DELAY=0 codex-accounts limits --force "$@" 2>&1
|
|
5330
|
+
}
|
|
5331
|
+
cx_shim() { CODEX_ACCOUNTS_ROOT="$CD" codex >/dev/null 2>&1; }
|
|
5332
|
+
rm -f "$CD"/acct-0*/.limited "$CD"/acct-0*/.client-limit-cleared "$CD"/acct-0*/.client-scan
|
|
5333
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5334
|
+
cx_now="$(date +%s)"
|
|
5335
|
+
cx_wreset=$((cx_now + 345600)) # four days out: a weekly window, still open
|
|
5336
|
+
cx_sreset=$((cx_now + 1800)) # half an hour out: a five-hour window
|
|
5337
|
+
# A record STAMPED BEFORE any clear: the watermark tests below turn on this timestamp,
|
|
5338
|
+
# and a rollout the writer's clear did not supersede would prove nothing about it.
|
|
5339
|
+
cx_ts_old="$(python3 -c 'import time; print(time.strftime("%Y-%m-%dT%H:%M:%S.120Z", time.gmtime(time.time() - 600)))')"
|
|
5340
|
+
|
|
5341
|
+
# (1) The marker names the WINDOW the server spent, not the key the payload happened to
|
|
5342
|
+
# use. 10080 minutes is the weekly bucket however the rollout labels it.
|
|
5343
|
+
cx_rollout "$CD/acct-01" "97.4" "$cx_wreset" 10080 "$cx_ts_old"
|
|
5344
|
+
: > "$CD/selection.log"
|
|
5345
|
+
cx_shim
|
|
5346
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5347
|
+
&& grep -q "acct-01 LIMITED by its own run (7d:97, resets $cx_wreset) — client-reported" \
|
|
5348
|
+
"$CD/selection.log"; } \
|
|
5349
|
+
&& t_ok "codex: a 10080-minute rejection is marked client:7d, not after the rollout's key name" \
|
|
5350
|
+
|| t_fail "codex weekly marker label" \
|
|
5351
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5352
|
+
|
|
5353
|
+
# (2) ...and THAT is the token the writer's weekly guard reads. An informative pass with
|
|
5354
|
+
# real numbers far under the threshold, and the confirm delay switched off so nothing but
|
|
5355
|
+
# the bucket rule can be keeping it: the marker has to survive the pass AND the next
|
|
5356
|
+
# launch, because a weekly window cannot fall from the server-proven 97% that wrote it to
|
|
5357
|
+
# 9% while it is still open.
|
|
5358
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5359
|
+
cx_shim
|
|
5360
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5361
|
+
&& [ ! -f "$CD/acct-01/.client-limit-cleared" ] \
|
|
5362
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker cleared"; } \
|
|
5363
|
+
&& t_ok "codex: the client:7d marker outlives an informative 9% pass and the launch after it" \
|
|
5364
|
+
|| t_fail "codex weekly marker vs informative pass" \
|
|
5365
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep acct-01 | tr '\n' ' ')"
|
|
5366
|
+
|
|
5367
|
+
# (3) A pass that reported nothing proves nothing, and has to SAY so — or a 32-hour
|
|
5368
|
+
# telemetry stall reads exactly like a healthy pool.
|
|
5369
|
+
out="$(cdlimits0 "$WORK/cx-usage-allzero.json")"
|
|
5370
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5371
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
5372
|
+
&& t_ok "codex: a no-data pass keeps the client:7d marker, and logs that it kept it" \
|
|
5373
|
+
|| t_fail "codex weekly marker vs no-data pass" \
|
|
5374
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep acct-01 | tr '\n' ' ')"
|
|
5375
|
+
|
|
5376
|
+
# (4) The differential, from the same code path with one number changed: 300 minutes is
|
|
5377
|
+
# the self-healing session window, so the identical rejection on acct-02 is labelled
|
|
5378
|
+
# client:5h and DOES clear on an informative pass — the #22 (2026-09-03) behavior, which
|
|
5379
|
+
# over-correcting (2) into "no client marker ever clears" would have destroyed.
|
|
5380
|
+
cx_rollout "$CD/acct-02" "99.0" "$cx_sreset" 300 "$cx_ts_old"
|
|
5381
|
+
cx_shim
|
|
5382
|
+
cx_5h_bucket="$(cx_bucket "$CD/acct-02")"
|
|
5383
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5384
|
+
{ [ "$cx_5h_bucket" = "client:5h" ] && [ ! -f "$CD/acct-02/.limited" ] \
|
|
5385
|
+
&& printf '%s' "$out" | grep -q "acct-02: marker cleared (max 9%)" \
|
|
5386
|
+
&& [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ]; } \
|
|
5387
|
+
&& t_ok "codex: a 300-minute rejection is marked client:5h and clears, beside a 7d one that does not" \
|
|
5388
|
+
|| t_fail "codex 5h vs 7d marker rule" \
|
|
5389
|
+
"5h-marked=$cx_5h_bucket 5h-now=$(cx_bucket "$CD/acct-02") 7d=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep 'marker' | tr '\n' ' ')"
|
|
5390
|
+
|
|
5391
|
+
# (5) ...and it STAYS cleared. The rollout that reported the spent window is still on
|
|
5392
|
+
# disk and the scan re-reads its tail on every launch, so without the `.client-limit-
|
|
5393
|
+
# cleared` watermark the clear achieves nothing: the very next `codex` rewrites the same
|
|
5394
|
+
# park, once per 15-minute pass, forever. Two launches, because the first would already
|
|
5395
|
+
# have re-marked.
|
|
5396
|
+
{ [ -f "$CD/acct-02/.client-limit-cleared" ] && { cx_shim; cx_shim; true; } \
|
|
5397
|
+
&& [ ! -f "$CD/acct-02/.limited" ]; } \
|
|
5398
|
+
&& t_ok "codex: the cleared 5h marker is not re-written from the same rollout (watermark)" \
|
|
5399
|
+
|| t_fail "codex client-limit watermark" \
|
|
5400
|
+
"watermark=$([ -f "$CD/acct-02/.client-limit-cleared" ] && echo yes || echo MISSING) bucket=$(cx_bucket "$CD/acct-02")"
|
|
5401
|
+
|
|
5402
|
+
# (6) That decision reaches selection: the weekly-parked account stays out while the
|
|
5403
|
+
# freed one comes back and takes every pick (acct-03 sits 36 points outside the band, so
|
|
5404
|
+
# "acct-02 is eligible again" is the only thing that can produce this).
|
|
5405
|
+
cdl 9 5 9 > "$CD/acct-02/limits.json"
|
|
5406
|
+
cdl 45 10 45 > "$CD/acct-03/limits.json"
|
|
5407
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick
|
|
5408
|
+
: > "$CD/selection.log"
|
|
5409
|
+
cw_hits=0; cw2_hits=0
|
|
5410
|
+
for _ in $(seq 1 6); do
|
|
5411
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in
|
|
5412
|
+
*CFG=acct-01*) cw_hits=$((cw_hits+1)) ;;
|
|
5413
|
+
*CFG=acct-02*) cw2_hits=$((cw2_hits+1)) ;;
|
|
5414
|
+
esac
|
|
5415
|
+
done
|
|
5416
|
+
{ [ "$cw_hits" = "0" ] && [ "$cw2_hits" = "6" ] \
|
|
5417
|
+
&& [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ]; } \
|
|
5418
|
+
&& t_ok "codex: the weekly rejection keeps its account out while the cleared 5h one returns (0 vs 6 of 6)" \
|
|
5419
|
+
|| t_fail "codex marker selection" \
|
|
5420
|
+
"acct-01=$cw_hits acct-02=$cw2_hits of 6; 7d marker=$(cx_bucket "$CD/acct-01")"
|
|
5421
|
+
|
|
5422
|
+
# (7) The watermark is a brake, not a mute: a rejection recorded AFTER the clear is news,
|
|
5423
|
+
# and parks the account again. (Stamped five seconds past the watermark the writer
|
|
5424
|
+
# actually wrote, so this cannot pass by clock luck.)
|
|
5425
|
+
cx_wm="$(head -1 "$CD/acct-02/.client-limit-cleared" 2>/dev/null)"
|
|
5426
|
+
cx_ts_new="$(python3 -c 'import sys, time; print(time.strftime("%Y-%m-%dT%H:%M:%S.120Z", time.gmtime(int(sys.argv[1]) + 5)))' "${cx_wm:-$cx_now}")"
|
|
5427
|
+
cx_rollout "$CD/acct-02" "96.0" "$cx_sreset" 300 "$cx_ts_new"
|
|
5428
|
+
cx_shim
|
|
5429
|
+
{ [ "$(cx_bucket "$CD/acct-02")" = "client:5h" ] \
|
|
5430
|
+
&& grep -q 'percent=96' "$CD/acct-02/.limited"; } \
|
|
5431
|
+
&& t_ok "codex: a rejection recorded after the clear parks the account again (a brake, not a mute)" \
|
|
5432
|
+
|| t_fail "codex watermark is not a mute" \
|
|
5433
|
+
"bucket=$(cx_bucket "$CD/acct-02") line: $(sed -n 2p "$CD/acct-02/.limited" 2>/dev/null)"
|
|
5434
|
+
|
|
5435
|
+
# (8) A report that names no window at all keeps the raw key name — unknown window stays
|
|
5436
|
+
# CLEARABLE, the #22 fail-open direction, and its own reset epoch still bounds it. It
|
|
5437
|
+
# must clear on an informative pass and stay cleared like any other 5h-class marker.
|
|
5438
|
+
rm -f "$CD/acct-03/.limited" "$CD/acct-03/.client-limit-cleared"
|
|
5439
|
+
cx_rollout "$CD/acct-03" "97.0" "$cx_sreset" - "$cx_ts_old"
|
|
5440
|
+
cx_shim
|
|
5441
|
+
cx_nw_bucket="$(cx_bucket "$CD/acct-03")"
|
|
5442
|
+
cdlimits0 "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5443
|
+
cx_shim
|
|
5444
|
+
{ [ "$cx_nw_bucket" = "client:primary" ] && [ ! -f "$CD/acct-03/.limited" ] \
|
|
5445
|
+
&& [ -f "$CD/acct-03/.client-limit-cleared" ]; } \
|
|
5446
|
+
&& t_ok "codex: a report with no window_minutes keeps the raw key name and stays clearable" \
|
|
5447
|
+
|| t_fail "codex windowless marker" \
|
|
5448
|
+
"marked=$cx_nw_bucket now=$(cx_bucket "$CD/acct-03") watermark=$([ -f "$CD/acct-03/.client-limit-cleared" ] && echo yes || echo no)"
|
|
5449
|
+
|
|
5450
|
+
# (9) The same holds for a marker written by an OLDER version and still on disk: it names
|
|
5451
|
+
# `primary`/`secondary`, which is no window, so it keeps its pre-2026-09-04 clearable
|
|
5452
|
+
# behavior rather than being promoted to a weekly park by accident. (Passes on the
|
|
5453
|
+
# pre-fix tree too — deliberately: it is the guard against reading the new guard as
|
|
5454
|
+
# "anything ambiguous sticks".)
|
|
5455
|
+
rm -rf "$CD/acct-03/sessions"
|
|
5456
|
+
rm -f "$CD/acct-03/.client-limit-cleared"
|
|
5457
|
+
cx_mark "$CD/acct-03" client:secondary 345600 2020-01-01T00:00:00Z
|
|
5458
|
+
cdlimits0 "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5459
|
+
[ ! -f "$CD/acct-03/.limited" ] \
|
|
5460
|
+
&& t_ok "codex: a legacy client:secondary marker (no window in its name) is still clearable" \
|
|
5461
|
+
|| t_fail "codex legacy marker" "bucket=$(cx_bucket "$CD/acct-03") survived an informative pass"
|
|
5462
|
+
|
|
5463
|
+
# (10) The confirm delay, which is what stops a cached usage response from erasing a
|
|
5464
|
+
# rejection the client was handed seconds ago. The shim writes marked_at=NOW, so the
|
|
5465
|
+
# marker below is genuinely fresh: the default 300s must keep it, and the same pass with
|
|
5466
|
+
# the window closed clears it — same env var, clamp and semantics as the claude writer.
|
|
5467
|
+
rm -f "$CD/acct-02/.limited" "$CD/acct-02/.client-limit-cleared"
|
|
5468
|
+
cx_rollout "$CD/acct-02" "99.0" "$cx_sreset" 300 "$cx_ts_old"
|
|
5469
|
+
cx_shim
|
|
5470
|
+
out="$(cdlimits "$WORK/cx-usage-low.json")"
|
|
5471
|
+
{ [ "$(cx_bucket "$CD/acct-02")" = "client:5h" ] \
|
|
5472
|
+
&& ! printf '%s' "$out" | grep -q "acct-02: marker cleared"; } \
|
|
5473
|
+
&& t_ok "codex: the default 300s confirm delay keeps a rejection handed over seconds ago" \
|
|
5474
|
+
|| t_fail "codex confirm delay" \
|
|
5475
|
+
"bucket=$(cx_bucket "$CD/acct-02") log: $(printf '%s' "$out" | grep acct-02 | tr '\n' ' ')"
|
|
5476
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5477
|
+
{ [ ! -f "$CD/acct-02/.limited" ] \
|
|
5478
|
+
&& printf '%s' "$out" | grep -q "acct-02: marker cleared (max 9%)"; } \
|
|
5479
|
+
&& t_ok "codex: past the confirm delay the same 5h rejection clears on a real reading" \
|
|
5480
|
+
|| t_fail "codex confirm delay expiry" \
|
|
5481
|
+
"bucket=$(cx_bucket "$CD/acct-02") log: $(printf '%s' "$out" | grep acct-02 | tr '\n' ' ')"
|
|
5482
|
+
|
|
5483
|
+
# (11) error-cooldown is untouched by all of the above: the account failed a real call
|
|
5484
|
+
# moments ago, and no usage reading disproves that. (This one holds on origin/main too —
|
|
5485
|
+
# it is the guard against the rewritten branch quietly dropping a case.)
|
|
5486
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5487
|
+
printf '%s\nbucket=error-cooldown percent=? reason=error-cooldown\n' "$(( $(date +%s) + 600 ))" \
|
|
5488
|
+
> "$CD/acct-01/.limited"
|
|
5489
|
+
cdlimits "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5490
|
+
[ -f "$CD/acct-01/.limited" ] \
|
|
5491
|
+
&& t_ok "codex: an error-cooldown marker still survives an informative clean pass" \
|
|
5492
|
+
|| t_fail "codex cooldown vs limits" "the cooldown marker was cleared"
|
|
5493
|
+
rm -f "$CD"/acct-0*/.limited "$CD"/acct-0*/.client-limit-cleared "$CD"/acct-0*/.client-scan
|
|
5494
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5495
|
+
|
|
5496
|
+
# ---- C13d. codex parity: each ranking signal comes from a window of its OWN kind -----
|
|
5497
|
+
# The claude writer's second 2026-09-04 defect, mirrored here: weekly_percent fell back
|
|
5498
|
+
# to the overall peak and session_percent to a flat 0, so a payload where only the 5h
|
|
5499
|
+
# window said anything was recorded as a WEEKLY reading, and one where only the 7d window
|
|
5500
|
+
# spoke walked through the session gate on a zero nobody reported. A signal no window
|
|
5501
|
+
# reported must be ABSENT — the shim needs both readings to call an account known.
|
|
5502
|
+
cat > "$WORK/cx-usage-session-only.json" <<EOF
|
|
5503
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5504
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5505
|
+
"primary_window":{"used_percent":40,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5506
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}},
|
|
5507
|
+
"additional_rate_limits":[]}
|
|
5508
|
+
EOF
|
|
5509
|
+
cdlimits "$WORK/cx-usage-session-only.json" --quiet >/dev/null
|
|
5510
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5511
|
+
import json, sys
|
|
5512
|
+
d = json.load(open(sys.argv[1]))
|
|
5513
|
+
assert 'no_data' not in d, d # one window DID report: this is a reading
|
|
5514
|
+
assert (d['session_percent'], d['max_percent']) == (40, 40), d
|
|
5515
|
+
# The 7d window reported no reset_at and 0%. Recording 40 here (the overall peak) or 0
|
|
5516
|
+
# (max over a silent window) both invent the number the weekly band ranks on.
|
|
5517
|
+
assert 'weekly_percent' not in d, d
|
|
5518
|
+
EOF
|
|
5519
|
+
[ $? -eq 0 ] && t_ok "codex: a session-only reading records session+max and NO weekly_percent" \
|
|
5520
|
+
|| t_fail "codex session-only signals" "see $CD/acct-01/limits.json"
|
|
5521
|
+
cdl 30 10 30 > "$CD/acct-02/limits.json"
|
|
5522
|
+
cdl 30 10 30 > "$CD/acct-03/limits.json"
|
|
5523
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick "$CD"/acct-0*/.limited
|
|
5524
|
+
: > "$CD/selection.log"
|
|
5525
|
+
cs_hits=0
|
|
5526
|
+
for _ in $(seq 1 10); do
|
|
5527
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cs_hits=$((cs_hits+1)) ;; esac
|
|
5528
|
+
done
|
|
5529
|
+
[ "$cs_hits" = "0" ] \
|
|
5530
|
+
&& t_ok "codex: an account with no weekly reading never enters the band (0 of 10 picks)" \
|
|
5531
|
+
|| t_fail "codex session-only ranking" "the weekly-less account took $cs_hits of 10 picks"
|
|
5532
|
+
|
|
5533
|
+
cat > "$WORK/cx-usage-weekly-only.json" <<EOF
|
|
5534
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5535
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5536
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5537
|
+
"secondary_window":{"used_percent":37,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5538
|
+
"additional_rate_limits":[]}
|
|
5539
|
+
EOF
|
|
5540
|
+
cdlimits "$WORK/cx-usage-weekly-only.json" --quiet >/dev/null
|
|
5541
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5542
|
+
import json, sys
|
|
5543
|
+
d = json.load(open(sys.argv[1]))
|
|
5544
|
+
assert 'no_data' not in d, d
|
|
5545
|
+
assert (d['weekly_percent'], d['max_percent']) == (37, 37), d
|
|
5546
|
+
assert 'session_percent' not in d, d # the 5h window reported nothing at all
|
|
5547
|
+
EOF
|
|
5548
|
+
[ $? -eq 0 ] && t_ok "codex: a weekly-only reading records weekly+max and NO session_percent" \
|
|
5549
|
+
|| t_fail "codex weekly-only signals" "see $CD/acct-01/limits.json"
|
|
5550
|
+
# ...and the rivals are WORSE on weekly (80 against 37) and still take every pick,
|
|
5551
|
+
# because they are the only ones that can clear the session gate — which is exactly what
|
|
5552
|
+
# a fabricated `session_percent: 0` would have handed the silent account for free.
|
|
5553
|
+
cdl 80 10 80 > "$CD/acct-02/limits.json"
|
|
5554
|
+
cdl 80 10 80 > "$CD/acct-03/limits.json"
|
|
5555
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick
|
|
5556
|
+
: > "$CD/selection.log"
|
|
5557
|
+
cwk_hits=0
|
|
5558
|
+
for _ in $(seq 1 6); do
|
|
5559
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cwk_hits=$((cwk_hits+1)) ;; esac
|
|
5560
|
+
done
|
|
5561
|
+
[ "$cwk_hits" = "0" ] \
|
|
5562
|
+
&& t_ok "codex: an unknown session reading never clears the gate, even with the better weekly (0 of 6)" \
|
|
5563
|
+
|| t_fail "codex weekly-only gate" "the session-less account cleared the gate $cwk_hits of 6 times"
|
|
5564
|
+
grep -q "band=30 band-count=2 session-gate=50 session-ok=2" "$CD/selection.log" \
|
|
5565
|
+
&& t_ok "codex: the log shows exactly the two accounts that reported a session bucket" \
|
|
5566
|
+
|| t_fail "codex weekly-only gate log" "$(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5567
|
+
rm -f "$CD"/acct-0*/.limited
|
|
5568
|
+
|
|
4098
5569
|
# ---- C14. oauth refresh via the token endpoint --------------------------------------
|
|
4099
5570
|
# expired bearer, missing endpoint: fail open with backoff
|
|
4100
5571
|
mk_cx_auth "$CX/acct-01/auth.json" a@cx 1000
|
|
@@ -4616,13 +6087,22 @@ check "import refuses to land the same email in a second slot" "already register
|
|
|
4616
6087
|
[ "$rc" != "0" ] && t_ok "conflicting-id import exits nonzero" || t_fail "conflicting id" "exited 0"
|
|
4617
6088
|
|
|
4618
6089
|
# ---- 20c. instance isolation: the shim resolves the same root as the CLI ------------
|
|
6090
|
+
# The leak assertion is a BEFORE/AFTER of the default pool, taken around the instance
|
|
6091
|
+
# run. It used to grep $ACC/selection.log for "acct-01" — a line some earlier, unrelated
|
|
6092
|
+
# test had to have left there, and since picks are random that line is not guaranteed:
|
|
6093
|
+
# the check flaked once in five runs while proving nothing about isolation either way.
|
|
6094
|
+
# Nothing but bin/claude writes selection.log, and this section runs no default-pool
|
|
6095
|
+
# shim, so an unchanged line count is exactly "the instance run stayed in its own pool".
|
|
6096
|
+
iso_before="$(wc -l < "$ACC/selection.log" 2>/dev/null || echo 0)"
|
|
4619
6097
|
out="$(CLAUDE_ACCOUNTS_ROOT="$JP2" claude 2>&1)"
|
|
4620
6098
|
check "shim honors CLAUDE_ACCOUNTS_ROOT (instance pool)" "CFG=acct-01" "$out"
|
|
4621
6099
|
[ -f "$JP2/selection.log" ] && t_ok "instance pool records its own selection log" \
|
|
4622
6100
|
|| t_fail "instance selection log" "missing at $JP2/selection.log"
|
|
4623
|
-
|
|
6101
|
+
iso_after="$(wc -l < "$ACC/selection.log" 2>/dev/null || echo 0)"
|
|
6102
|
+
{ [ "$iso_before" = "$iso_after" ] && ! grep -q "portable@test" "$ACC/accounts.json"; } \
|
|
4624
6103
|
&& t_ok "the default pool was untouched by the instance run" \
|
|
4625
|
-
|| t_fail "pool isolation"
|
|
6104
|
+
|| t_fail "pool isolation" \
|
|
6105
|
+
"the instance run leaked into $ACC (selection.log $iso_before -> $iso_after lines)"
|
|
4626
6106
|
|
|
4627
6107
|
# ---- 20d. sync target: overridable, and a local-only mode that pushes nowhere -------
|
|
4628
6108
|
out="$(CLAUDE_ACCOUNTS_ROOT="$JP" CLAUDE_MULTIACC_NO_SYNC=0 claude-accounts sync --no-server 2>&1)"
|