claude-multiacc 2.0.21 → 2.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -3
- package/bin/claude +82 -9
- package/bin/claude-accounts +198 -33
- package/bin/codex +82 -5
- package/bin/codex-accounts +202 -21
- package/docs/ACCOUNT_OPERATIONS.md +26 -4
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/lib/report.py +38 -2
- package/package.json +1 -1
- package/tests/run-tests.sh +1186 -2
package/tests/run-tests.sh
CHANGED
|
@@ -1563,6 +1563,12 @@ expected="$(cat "$WORK/stdin13")"
|
|
|
1563
1563
|
[ "$out" = "$expected" ] && t_ok "stdin/stdout byte fidelity (-p pipe)" || t_fail "stdin fidelity" "got: $out"
|
|
1564
1564
|
|
|
1565
1565
|
# ---- 14. selection log written ------------------------------------------------
|
|
1566
|
+
# One COMPLETE reading first, on purpose: the pool is still carrying the session-only
|
|
1567
|
+
# documents an earlier limits pass wrote, and since 2026-09-04 a pool where no account
|
|
1568
|
+
# has BOTH percentages is BLIND (bin/claude telem_blind) and logs the blind format
|
|
1569
|
+
# instead. The ranked format asserted below only exists when something actually ranked.
|
|
1570
|
+
lj 20 10 20 > "$ACC/acct-01/limits.json"
|
|
1571
|
+
claude >/dev/null 2>&1
|
|
1566
1572
|
log_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}T.*acct-0[123] weekly=[0-9?]+% session=[0-9?]+%'
|
|
1567
1573
|
log_pattern="$log_pattern band=30 band-count=[0-9]+ session-gate=50 session-ok=[0-9]+ pwd="
|
|
1568
1574
|
grep -qE "$log_pattern" "$ACC/selection.log" \
|
|
@@ -2065,6 +2071,611 @@ sc="$(python3 -c "import json;d=json.load(open('$ACC/acct-01/limits.json'));prin
|
|
|
2065
2071
|
|| t_fail "weekly/session ranking inputs" "got $sc"
|
|
2066
2072
|
rm -f "$ACC/acct-01/limits.json" "$ACC/acct-01/.limited"
|
|
2067
2073
|
|
|
2074
|
+
# ---- 16-nodata. a 0% bucket with NO reset window is NO DATA, not an empty account ----
|
|
2075
|
+
# 2026-09-04, my-mini: for acct-13/acct-14 the usage endpoint answered EVERY bucket
|
|
2076
|
+
# `percent: 0, resets_at: null` while Claude Code was being rejected on those same two
|
|
2077
|
+
# accounts with "You've hit your weekly limit · resets Sep 8 at 1am" (epoch 1788818400).
|
|
2078
|
+
# The writer recorded the zeros verbatim, so two provably exhausted accounts became the
|
|
2079
|
+
# leaders of the weekly band and absorbed 31 of the last ~60 picks — and the shim's
|
|
2080
|
+
# telemetry-based recovery then deleted their truthful client:seven_day markers about six
|
|
2081
|
+
# times a day each. A truthful bucket ALWAYS carries the window it resets in, so 0% with
|
|
2082
|
+
# no window is no data. These run in a pool of their own: one no-data account against one
|
|
2083
|
+
# honest one makes "which account ranked" unambiguous.
|
|
2084
|
+
ND="$WORK/nodata-pool"
|
|
2085
|
+
mkdir -p "$ND/acct-01" "$ND/acct-02" "$ND/tmp"
|
|
2086
|
+
: > "$ND/.limits-kick"
|
|
2087
|
+
cat > "$ND/accounts.json" <<'EOF'
|
|
2088
|
+
{"version":1,"server":"none","threshold":90,"accounts":[
|
|
2089
|
+
{"id":"acct-01","email":"nd1@test","home":"mac","added_at":"2026-07-13T00:00:00Z"},
|
|
2090
|
+
{"id":"acct-02","email":"nd2@test","home":"mac","added_at":"2026-07-13T00:00:00Z"}]}
|
|
2091
|
+
EOF
|
|
2092
|
+
for i in 01 02; do
|
|
2093
|
+
printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-nd%s","refreshToken":"r","expiresAt":9999999999999,"refreshTokenExpiresAt":9999999999999}}' \
|
|
2094
|
+
"$i" > "$ND/acct-$i/.credentials.json"
|
|
2095
|
+
done
|
|
2096
|
+
ndl() { # ndl <weekly> <session> <max> -> a truthful, in-window reading on stdout
|
|
2097
|
+
local t; t="$(date +%s)"
|
|
2098
|
+
printf '{"fetched_at":%s,"source":"oauth","weekly_percent":%s,"session_percent":%s,"max_percent":%s,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2099
|
+
"$t" "$1" "$2" "$3" "$((t + 259200))"
|
|
2100
|
+
}
|
|
2101
|
+
ndlimits() { # ndlimits <fixture> [extra args] -> a real refresh over the whole ND pool
|
|
2102
|
+
local f="$1"; shift
|
|
2103
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$f" \
|
|
2104
|
+
claude-accounts limits --force "$@" 2>&1
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
# the incident payload, byte-for-byte in shape: every bucket 0%, every window null
|
|
2108
|
+
cat > "$WORK/usage-allzero.json" <<'EOF'
|
|
2109
|
+
{"limits":[
|
|
2110
|
+
{"kind":"session","group":"session","percent":0,"resets_at":null,"scope":null},
|
|
2111
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":null,"scope":null},
|
|
2112
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2113
|
+
]}
|
|
2114
|
+
EOF
|
|
2115
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2116
|
+
check "an all-zero/no-window payload is reported as no usable telemetry" \
|
|
2117
|
+
"no usable telemetry (account ranks as unknown, not as empty)" "$out"
|
|
2118
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2119
|
+
import json, sys
|
|
2120
|
+
d = json.load(open(sys.argv[1]))
|
|
2121
|
+
assert d.get('no_data') is True, d
|
|
2122
|
+
# A MISSING field is what makes the shim's fresh_field/cutoff_field reads fail, which
|
|
2123
|
+
# is what makes the account unknown to both cuts. Writing 0 here is the bug.
|
|
2124
|
+
for k in ('max_percent', 'weekly_percent', 'session_percent', 'weekly_resets_epoch'):
|
|
2125
|
+
assert k not in d, (k, d)
|
|
2126
|
+
assert len(d['buckets']) == 3, d # the raw buckets stay, for diagnostics
|
|
2127
|
+
assert d['fetched_at'] > 0 and d['source'], d
|
|
2128
|
+
EOF
|
|
2129
|
+
[ $? -eq 0 ] && t_ok "a no-data pass records no_data and NONE of the three percent signals" \
|
|
2130
|
+
|| t_fail "no_data document" "see $ND/acct-01/limits.json"
|
|
2131
|
+
|
|
2132
|
+
# ONE uninformative bucket beside real ones changes nothing (the live acct-16 shape:
|
|
2133
|
+
# weekly_scoped:Fable 0/null next to a real session and a real weekly_all).
|
|
2134
|
+
cat > "$WORK/usage-mixed-nodata.json" <<'EOF'
|
|
2135
|
+
{"limits":[
|
|
2136
|
+
{"kind":"session","group":"session","percent":12,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2137
|
+
{"kind":"weekly_all","group":"weekly","percent":40,"resets_at":"2099-01-05T00:00:00+00:00","scope":null},
|
|
2138
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2139
|
+
]}
|
|
2140
|
+
EOF
|
|
2141
|
+
ndlimits "$WORK/usage-mixed-nodata.json" --quiet >/dev/null
|
|
2142
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2143
|
+
import json, sys, time, calendar
|
|
2144
|
+
d = json.load(open(sys.argv[1]))
|
|
2145
|
+
assert 'no_data' not in d, d
|
|
2146
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (12, 40, 40), d
|
|
2147
|
+
want = calendar.timegm(time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S"))
|
|
2148
|
+
assert d['weekly_resets_epoch'] == want, d
|
|
2149
|
+
assert len(d['buckets']) == 3, d # the quiet bucket is still recorded
|
|
2150
|
+
EOF
|
|
2151
|
+
[ $? -eq 0 ] && t_ok "one uninformative bucket beside real ones leaves the ranking untouched (12/40)" \
|
|
2152
|
+
|| t_fail "mixed no-data payload" "see $ND/acct-01/limits.json"
|
|
2153
|
+
|
|
2154
|
+
# 0% WITH a real window is informative: a genuinely fresh account must still rank empty.
|
|
2155
|
+
cat > "$WORK/usage-zero-real-windows.json" <<'EOF'
|
|
2156
|
+
{"limits":[
|
|
2157
|
+
{"kind":"session","group":"session","percent":0,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2158
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2159
|
+
]}
|
|
2160
|
+
EOF
|
|
2161
|
+
ndlimits "$WORK/usage-zero-real-windows.json" --quiet >/dev/null
|
|
2162
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2163
|
+
import json, sys
|
|
2164
|
+
d = json.load(open(sys.argv[1]))
|
|
2165
|
+
assert 'no_data' not in d, d
|
|
2166
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (0, 0, 0), d
|
|
2167
|
+
EOF
|
|
2168
|
+
[ $? -eq 0 ] && t_ok "0% WITH real reset windows still records a real, empty 0% reading" \
|
|
2169
|
+
|| t_fail "zero-with-windows payload" "see $ND/acct-01/limits.json"
|
|
2170
|
+
|
|
2171
|
+
# ...and the horizon that 0% is valid until comes from the bucket that reported one. A
|
|
2172
|
+
# null-window sibling's synthesized now+1h used to win the min() and shorten it.
|
|
2173
|
+
cat > "$WORK/usage-zero-mixed-window.json" <<'EOF'
|
|
2174
|
+
{"limits":[
|
|
2175
|
+
{"kind":"session","group":"session","percent":0,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2176
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":"2099-01-05T00:00:00+00:00","scope":null},
|
|
2177
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2178
|
+
]}
|
|
2179
|
+
EOF
|
|
2180
|
+
ndlimits "$WORK/usage-zero-mixed-window.json" --quiet >/dev/null
|
|
2181
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2182
|
+
import json, sys, time, calendar
|
|
2183
|
+
d = json.load(open(sys.argv[1]))
|
|
2184
|
+
want = calendar.timegm(time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S"))
|
|
2185
|
+
assert d['weekly_percent'] == 0 and d['weekly_resets_epoch'] == want, d
|
|
2186
|
+
EOF
|
|
2187
|
+
[ $? -eq 0 ] && t_ok "a window-less bucket cannot shorten the horizon a real 0% reading is valid for" \
|
|
2188
|
+
|| t_fail "no-data horizon" "see $ND/acct-01/limits.json"
|
|
2189
|
+
|
|
2190
|
+
# ---- 16-nodata-rank. an unknown account never leads the weekly band ------------------
|
|
2191
|
+
# The whole point of the rule: 0/0 ranked BETTER than a truthful 45% account, so every
|
|
2192
|
+
# pick went to the exhausted one. Unknown must lose to any account with a real reading.
|
|
2193
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null
|
|
2194
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2195
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2196
|
+
: > "$ND/selection.log"
|
|
2197
|
+
nd_hits=0
|
|
2198
|
+
for _ in $(seq 1 10); do
|
|
2199
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_hits=$((nd_hits+1)) ;; esac
|
|
2200
|
+
done
|
|
2201
|
+
[ "$nd_hits" = "0" ] \
|
|
2202
|
+
&& t_ok "a no-data account never outranks an account with real telemetry (0 of 10 picks)" \
|
|
2203
|
+
|| t_fail "no_data ranking" "the fake-zero account took $nd_hits of 10 picks"
|
|
2204
|
+
grep -q "acct-02 weekly=45% session=10% band=30 band-count=1" "$ND/selection.log" \
|
|
2205
|
+
&& t_ok "the pick logs the known account alone in the band" \
|
|
2206
|
+
|| t_fail "no_data band" "selection.log: $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2207
|
+
grep -q "acct-01 weekly=0%" "$ND/selection.log" \
|
|
2208
|
+
&& t_fail "no_data band leader" "the fake-zero account was logged as a 0% pick" \
|
|
2209
|
+
|| t_ok "the fake-zero account is never logged as the band leader"
|
|
2210
|
+
|
|
2211
|
+
# ---- 16-nodata-marker. a client-reported WEEKLY rejection outlives any telemetry -----
|
|
2212
|
+
# The second half of the incident. Claude Code's own rejection wrote
|
|
2213
|
+
# `bucket=client:seven_day percent=100 reason=client-rate-limit` with the server's reset
|
|
2214
|
+
# (Sep 8); the next invocation read the fake 0%, called it newer first-hand evidence and
|
|
2215
|
+
# deleted the marker. A seven-day window cannot fall from a server-proven 100% to under
|
|
2216
|
+
# the threshold before it resets, so no reading may clear it — however fresh and however
|
|
2217
|
+
# informative. Here acct-01's telemetry is a REAL 5%, so only the bucket rule can save it.
|
|
2218
|
+
ndl 5 5 5 > "$ND/acct-01/limits.json"
|
|
2219
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2220
|
+
printf '%s\nbucket=client:seven_day percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2221
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2222
|
+
rm -f "$ND"/acct-0*/.client-limit-cleared "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2223
|
+
: > "$ND/selection.log"
|
|
2224
|
+
w_hits=0
|
|
2225
|
+
for _ in $(seq 1 6); do
|
|
2226
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) w_hits=$((w_hits+1)) ;; esac
|
|
2227
|
+
done
|
|
2228
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$w_hits" = "0" ]; } \
|
|
2229
|
+
&& t_ok "a client:seven_day marker survives fresh below-threshold telemetry" \
|
|
2230
|
+
|| t_fail "weekly client marker" "marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) hits=$w_hits"
|
|
2231
|
+
grep -q "client limit cleared by newer telemetry" "$ND/selection.log" \
|
|
2232
|
+
&& t_fail "weekly client marker" "the shim logged a recovery for a weekly rejection" \
|
|
2233
|
+
|| t_ok "no recovery event is logged for a weekly rejection"
|
|
2234
|
+
[ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2235
|
+
&& t_ok "no recovery watermark is written for a weekly rejection" \
|
|
2236
|
+
|| t_fail "weekly client marker" "a watermark was written"
|
|
2237
|
+
# case-insensitive on the bucket name: seven_day_opus is weekly too
|
|
2238
|
+
printf '%s\nbucket=client:Seven_Day_Opus percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2239
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2240
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2241
|
+
[ -f "$ND/acct-01/.limited" ] && t_ok "a model-scoped weekly bucket (seven_day_opus) is weekly too" \
|
|
2242
|
+
|| t_fail "weekly client marker" "seven_day_opus was cleared"
|
|
2243
|
+
|
|
2244
|
+
# ...but the 5h window self-heals within hours, so #22 (2026-09-03) still holds: a
|
|
2245
|
+
# five_hour marker DOES clear once a below-threshold reading was fetched after it.
|
|
2246
|
+
printf '%s\nbucket=client:five_hour percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2247
|
+
"$(( $(date +%s) + 1800 ))" > "$ND/acct-01/.limited"
|
|
2248
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2249
|
+
: > "$ND/selection.log"
|
|
2250
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2251
|
+
[ ! -f "$ND/acct-01/.limited" ] \
|
|
2252
|
+
&& t_ok "a client:five_hour marker still clears on newer below-threshold telemetry (#22)" \
|
|
2253
|
+
|| t_fail "five_hour client marker" "the 2026-09-03 recovery stopped working"
|
|
2254
|
+
[ -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2255
|
+
&& t_ok "the five_hour recovery still records its watermark" \
|
|
2256
|
+
|| t_fail "five_hour client marker" "watermark missing"
|
|
2257
|
+
grep -q "acct-01 client limit cleared by newer telemetry (5%)" "$ND/selection.log" \
|
|
2258
|
+
&& t_ok "the five_hour recovery still logs the reading it acted on" \
|
|
2259
|
+
|| t_fail "five_hour client marker" "selection.log: $(tail -2 "$ND/selection.log" 2>/dev/null | tr '\n' ' ')"
|
|
2260
|
+
|
|
2261
|
+
# ---- 16-nodata-shorten. an offender pass must never SHORTEN a weekly client marker ---
|
|
2262
|
+
# The writer's offenders branch used to overwrite `.limited` unconditionally: a session
|
|
2263
|
+
# bucket crossing the threshold (+1h reset) replaced a client:seven_day marker four days
|
|
2264
|
+
# out, and after that hour the provably exhausted account was back in the pool (codex
|
|
2265
|
+
# review, 2026-09-04). The client's own reset reaches further and must win.
|
|
2266
|
+
printf '%s\nbucket=client:seven_day percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2267
|
+
"$(( $(date +%s) + 345600 ))" > "$ND/acct-01/.limited"
|
|
2268
|
+
keep_reset_before="$(head -1 "$ND/acct-01/.limited")"
|
|
2269
|
+
python3 - "$WORK/usage-shorten.json" <<'PJ'
|
|
2270
|
+
import json, sys, datetime
|
|
2271
|
+
soon = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat()
|
|
2272
|
+
json.dump({"limits": [
|
|
2273
|
+
{"kind": "session", "percent": 95, "resets_at": soon},
|
|
2274
|
+
{"kind": "seven_day", "percent": 0, "resets_at": None},
|
|
2275
|
+
]}, open(sys.argv[1], 'w'))
|
|
2276
|
+
PJ
|
|
2277
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-shorten.json" \
|
|
2278
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2279
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$(head -1 "$ND/acct-01/.limited")" = "$keep_reset_before" ] \
|
|
2280
|
+
&& grep -q 'reason=client-rate-limit' "$ND/acct-01/.limited"; } \
|
|
2281
|
+
&& t_ok "an over-threshold session pass keeps the further-reaching weekly client marker" \
|
|
2282
|
+
|| t_fail "marker shortened" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2283
|
+
# ...while a LATER reset may still extend the exclusion (more caution is allowed):
|
|
2284
|
+
python3 - "$WORK/usage-extend.json" <<'PJ'
|
|
2285
|
+
import json, sys, datetime
|
|
2286
|
+
far = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=6)).isoformat()
|
|
2287
|
+
json.dump({"limits": [
|
|
2288
|
+
{"kind": "seven_day", "percent": 95, "resets_at": far},
|
|
2289
|
+
]}, open(sys.argv[1], 'w'))
|
|
2290
|
+
PJ
|
|
2291
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-extend.json" \
|
|
2292
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2293
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ "$(head -1 "$ND/acct-01/.limited")" -gt "$keep_reset_before" ]; } \
|
|
2294
|
+
&& t_ok "a further-out offender may still extend the marker" \
|
|
2295
|
+
|| t_fail "marker extend" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2296
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2297
|
+
# ...and a GARBLED client marker (a first line the shims' num_ok would refuse) must not
|
|
2298
|
+
# be preserved by that guard: to the shims it is an active-forever park, so the writer
|
|
2299
|
+
# replacing it with a valid offender marker is a repair, not a shortening.
|
|
2300
|
+
printf '9999999999999999999\nbucket=client:seven_day reason=client-rate-limit\n' > "$ND/acct-01/.limited"
|
|
2301
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-shorten.json" \
|
|
2302
|
+
claude-accounts limits --force --quiet >/dev/null 2>&1
|
|
2303
|
+
{ [ -f "$ND/acct-01/.limited" ] && grep -q 'reason=limits' "$ND/acct-01/.limited"; } \
|
|
2304
|
+
&& t_ok "a garbled client marker is repaired by the offender write, not preserved" \
|
|
2305
|
+
|| t_fail "garbled marker repair" "now: $(head -2 "$ND/acct-01/.limited" 2>/dev/null | tr '\n' ' ')"
|
|
2306
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2307
|
+
|
|
2308
|
+
# ---- 16-nodata-noclear. an uninformative reading cannot clear ANY marker -------------
|
|
2309
|
+
# End to end, in the incident's own order: the fake-zero payload goes through the real
|
|
2310
|
+
# writer, then a five_hour marker (the kind that IS allowed to clear) is planted on top.
|
|
2311
|
+
# Pre-fix that pass wrote max_percent 0 and the very next invocation deleted the marker
|
|
2312
|
+
# with "client limit cleared by newer telemetry (0%)". A reading with no percent at all
|
|
2313
|
+
# reads as unknown, and unknown proves nothing.
|
|
2314
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null
|
|
2315
|
+
ndl 45 10 45 > "$ND/acct-02/limits.json"
|
|
2316
|
+
printf '%s\nbucket=client:five_hour percent=100 marked_at=2020-01-01T00:00:00Z reason=client-rate-limit\n' \
|
|
2317
|
+
"$(( $(date +%s) + 1800 ))" > "$ND/acct-01/.limited"
|
|
2318
|
+
rm -f "$ND/acct-01/.client-limit-cleared" "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2319
|
+
: > "$ND/selection.log"
|
|
2320
|
+
for _ in 1 2 3; do CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1; done
|
|
2321
|
+
[ -f "$ND/acct-01/.limited" ] \
|
|
2322
|
+
&& t_ok "a limits.json with no percent fields cannot clear even a five_hour marker" \
|
|
2323
|
+
|| t_fail "no_data marker clearing" "fake-zero telemetry unparked the account"
|
|
2324
|
+
grep -q "client limit cleared" "$ND/selection.log" \
|
|
2325
|
+
&& t_fail "no_data marker clearing" "a recovery was logged from a no-data reading" \
|
|
2326
|
+
|| t_ok "no recovery is logged from a no-data reading"
|
|
2327
|
+
[ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2328
|
+
&& t_ok "no recovery watermark is written from a no-data reading" \
|
|
2329
|
+
|| t_fail "no_data marker clearing" "a watermark was written from a reading with no percent"
|
|
2330
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2331
|
+
|
|
2332
|
+
# ---- 16-nodata-writer. the LIMITS PASS obeys the same marker rule as the shim --------
|
|
2333
|
+
# The shim's rule above is only half a fix. `claude-accounts limits` runs on a 15-minute
|
|
2334
|
+
# schedule and deletes markers itself, and until 2026-09-04 it kept a client rejection
|
|
2335
|
+
# only while it was newer than CLIENT_LIMIT_CONFIRM_DELAY (300s) — so five minutes after
|
|
2336
|
+
# the shim refused to unpark acct-13, the scheduled pass deleted the same client:seven_day
|
|
2337
|
+
# marker anyway, on a payload whose every bucket said `percent 0, resets_at null`. Two
|
|
2338
|
+
# writers with two rules is one rule: the weaker one. This is the claude twin of the codex
|
|
2339
|
+
# C13c block, and the pass is driven for real (fixture endpoint -> writer -> disk).
|
|
2340
|
+
nd_mark() { # nd_mark <acct dir> <bucket> <reset-offset-seconds> <marked_at ISO>
|
|
2341
|
+
printf '%s\nbucket=%s percent=100 marked_at=%s reason=client-rate-limit\n' \
|
|
2342
|
+
"$(( $(date +%s) + $3 ))" "$2" "$4" > "$1/.limited"
|
|
2343
|
+
}
|
|
2344
|
+
nd_acct01_log() { printf '%s' "$1" | grep 'acct-01' | tr '\n' ' '; }
|
|
2345
|
+
# A REAL, informative, below-threshold reading: the kind that IS allowed to clear a
|
|
2346
|
+
# five-hour marker, and the one that must never clear a weekly one.
|
|
2347
|
+
cat > "$WORK/usage-nd-low.json" <<'EOF'
|
|
2348
|
+
{"limits":[
|
|
2349
|
+
{"kind":"session","group":"session","percent":5,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2350
|
+
{"kind":"weekly_all","group":"weekly","percent":5,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2351
|
+
]}
|
|
2352
|
+
EOF
|
|
2353
|
+
|
|
2354
|
+
# (1) A pass that reported nothing proves nothing, so it clears nothing — whatever the
|
|
2355
|
+
# marker says and however old it is. On origin/main this marker is 300s past its confirm
|
|
2356
|
+
# delay and the pass deletes it with "marker cleared (max 0%)".
|
|
2357
|
+
nd_mark "$ND/acct-01" client:seven_day 345600 2020-01-01T00:00:00Z
|
|
2358
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2359
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2360
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2361
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2362
|
+
&& t_ok "a no-data limits pass keeps an aged client:seven_day marker, and logs that it kept it" \
|
|
2363
|
+
|| t_fail "writer marker: no-data pass" \
|
|
2364
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2365
|
+
|
|
2366
|
+
# (2) ...and a pass that DID report something still cannot clear a weekly rejection
|
|
2367
|
+
# before its reset: a seven-day window cannot fall from the server-proven 100% that
|
|
2368
|
+
# wrote the marker to 5% while it is still open. The keep must come from the BUCKET
|
|
2369
|
+
# rule, not from silence, so the "no usable telemetry" line must NOT appear here.
|
|
2370
|
+
nd_mark "$ND/acct-01" client:seven_day 345600 2020-01-01T00:00:00Z
|
|
2371
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2372
|
+
out="$(ndlimits "$WORK/usage-nd-low.json")"
|
|
2373
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2374
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker cleared" \
|
|
2375
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2376
|
+
&& t_ok "an informative 5% pass keeps a client:seven_day marker on the bucket rule alone" \
|
|
2377
|
+
|| t_fail "writer marker: weekly vs informative pass" \
|
|
2378
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2379
|
+
|
|
2380
|
+
# (3) The 5h window self-heals in hours, so #22 (2026-09-03) still holds at the writer:
|
|
2381
|
+
# an aged five_hour rejection DOES clear once a pass has real numbers under the
|
|
2382
|
+
# threshold. This one passes on origin/main too — deliberately: it is the guard against
|
|
2383
|
+
# over-correcting (2) into "no client marker ever clears", which would strand accounts
|
|
2384
|
+
# sitting at 0% usage for days, which is the bug #22 existed to fix.
|
|
2385
|
+
nd_mark "$ND/acct-01" client:five_hour 1800 2020-01-01T00:00:00Z
|
|
2386
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2387
|
+
out="$(ndlimits "$WORK/usage-nd-low.json")"
|
|
2388
|
+
{ [ ! -f "$ND/acct-01/.limited" ] && [ -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2389
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker cleared (max 5%)"; } \
|
|
2390
|
+
&& t_ok "an informative 5% pass still clears an aged client:five_hour marker (#22)" \
|
|
2391
|
+
|| t_fail "writer marker: five_hour recovery" \
|
|
2392
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2393
|
+
|
|
2394
|
+
# (4) ...but the SAME five_hour marker survives a pass that said nothing. "0%" and "no
|
|
2395
|
+
# reading" are the same bytes on origin/main, and that is the whole incident.
|
|
2396
|
+
nd_mark "$ND/acct-01" client:five_hour 1800 2020-01-01T00:00:00Z
|
|
2397
|
+
rm -f "$ND/acct-01/.client-limit-cleared"
|
|
2398
|
+
out="$(ndlimits "$WORK/usage-allzero.json")"
|
|
2399
|
+
{ [ -f "$ND/acct-01/.limited" ] && [ ! -f "$ND/acct-01/.client-limit-cleared" ] \
|
|
2400
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
2401
|
+
&& t_ok "a no-data pass keeps even a client:five_hour marker — the kind it may clear when informative" \
|
|
2402
|
+
|| t_fail "writer marker: five_hour vs no-data pass" \
|
|
2403
|
+
"marker=$([ -f "$ND/acct-01/.limited" ] && echo kept || echo DELETED) log: $(nd_acct01_log "$out")"
|
|
2404
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2405
|
+
|
|
2406
|
+
# ---- 16-nodata-signals. each ranking signal comes from a bucket of its OWN kind ------
|
|
2407
|
+
# 2026-09-04, second defect: weekly_percent fell back to the overall peak and
|
|
2408
|
+
# session_percent to a flat 0. So an account whose weekly buckets said nothing while its
|
|
2409
|
+
# 5h bucket read 40% was recorded as 40% WEEKLY — a number no bucket ever reported, on
|
|
2410
|
+
# the signal the band ranks on — and its mirror image was recorded as session 0%, which
|
|
2411
|
+
# walks straight through the session gate. A signal nobody reported must be ABSENT: the
|
|
2412
|
+
# shim needs BOTH readings to call an account known (pick_best's quota_known rule), so a
|
|
2413
|
+
# missing one costs the account its place in the band and nothing else.
|
|
2414
|
+
cat > "$WORK/usage-session-only.json" <<'EOF'
|
|
2415
|
+
{"limits":[
|
|
2416
|
+
{"kind":"session","group":"session","percent":40,"resets_at":"2099-01-01T00:00:00+00:00","scope":null},
|
|
2417
|
+
{"kind":"weekly_all","group":"weekly","percent":0,"resets_at":null,"scope":null},
|
|
2418
|
+
{"kind":"weekly_scoped","group":"weekly","percent":0,"resets_at":null,"scope":{"model":{"display_name":"Fable"}}}
|
|
2419
|
+
]}
|
|
2420
|
+
EOF
|
|
2421
|
+
ndlimits "$WORK/usage-session-only.json" --quiet >/dev/null
|
|
2422
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2423
|
+
import json, sys
|
|
2424
|
+
d = json.load(open(sys.argv[1]))
|
|
2425
|
+
assert 'no_data' not in d, d # one bucket DID report: this is a reading
|
|
2426
|
+
assert (d['session_percent'], d['max_percent']) == (40, 40), d
|
|
2427
|
+
# The two weekly buckets said `0% / no window`. Recording 40 here (round 1's fallback to
|
|
2428
|
+
# the overall peak) or 0 (origin/main's max over silent weekly buckets) both invent the
|
|
2429
|
+
# only number the weekly band ranks on.
|
|
2430
|
+
assert 'weekly_percent' not in d, d
|
|
2431
|
+
assert 'weekly_resets_epoch' not in d, d # a horizon without a reading means nothing
|
|
2432
|
+
EOF
|
|
2433
|
+
[ $? -eq 0 ] && t_ok "a session-only reading records session+max and NO weekly_percent" \
|
|
2434
|
+
|| t_fail "session-only signals" "see $ND/acct-01/limits.json"
|
|
2435
|
+
|
|
2436
|
+
# ...and the shim reads that as UNKNOWN, so a truthful 30w/10s account takes every pick.
|
|
2437
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2438
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2439
|
+
: > "$ND/selection.log"
|
|
2440
|
+
nd_sess_hits=0
|
|
2441
|
+
for _ in $(seq 1 10); do
|
|
2442
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_sess_hits=$((nd_sess_hits+1)) ;; esac
|
|
2443
|
+
done
|
|
2444
|
+
[ "$nd_sess_hits" = "0" ] \
|
|
2445
|
+
&& t_ok "an account with no weekly reading never enters the band (0 of 10 picks)" \
|
|
2446
|
+
|| t_fail "session-only ranking" "the weekly-less account took $nd_sess_hits of 10 picks"
|
|
2447
|
+
|
|
2448
|
+
# The mirror image: a weekly reading with nothing to say about the session bucket. Here
|
|
2449
|
+
# the rival is WORSE on weekly (80 against 37) and still wins every pick, because it is
|
|
2450
|
+
# the only candidate that can clear the session gate — which is exactly what a fabricated
|
|
2451
|
+
# `session_percent: 0` would have handed the silent account for free.
|
|
2452
|
+
cat > "$WORK/usage-weekly-only.json" <<'EOF'
|
|
2453
|
+
{"limits":[
|
|
2454
|
+
{"kind":"session","group":"session","percent":0,"resets_at":null,"scope":null},
|
|
2455
|
+
{"kind":"weekly_all","group":"weekly","percent":37,"resets_at":"2099-01-05T00:00:00+00:00","scope":null}
|
|
2456
|
+
]}
|
|
2457
|
+
EOF
|
|
2458
|
+
ndlimits "$WORK/usage-weekly-only.json" --quiet >/dev/null
|
|
2459
|
+
python3 - "$ND/acct-01/limits.json" <<'EOF'
|
|
2460
|
+
import json, sys, time, calendar
|
|
2461
|
+
d = json.load(open(sys.argv[1]))
|
|
2462
|
+
assert 'no_data' not in d, d
|
|
2463
|
+
assert (d['weekly_percent'], d['max_percent']) == (37, 37), d
|
|
2464
|
+
assert d['weekly_resets_epoch'] == calendar.timegm(
|
|
2465
|
+
time.strptime("2099-01-05T00:00:00", "%Y-%m-%dT%H:%M:%S")), d
|
|
2466
|
+
assert 'session_percent' not in d, d # the 5h bucket reported nothing at all
|
|
2467
|
+
EOF
|
|
2468
|
+
[ $? -eq 0 ] && t_ok "a weekly-only reading records weekly+max+horizon and NO session_percent" \
|
|
2469
|
+
|| t_fail "weekly-only signals" "see $ND/acct-01/limits.json"
|
|
2470
|
+
ndl 80 10 80 > "$ND/acct-02/limits.json"
|
|
2471
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2472
|
+
: > "$ND/selection.log"
|
|
2473
|
+
nd_wk_hits=0
|
|
2474
|
+
for _ in $(seq 1 6); do
|
|
2475
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-01*) nd_wk_hits=$((nd_wk_hits+1)) ;; esac
|
|
2476
|
+
done
|
|
2477
|
+
[ "$nd_wk_hits" = "0" ] \
|
|
2478
|
+
&& t_ok "an unknown session reading never clears the gate, even with the better weekly (0 of 6)" \
|
|
2479
|
+
|| t_fail "weekly-only gate" "the session-less account cleared the gate $nd_wk_hits of 6 times"
|
|
2480
|
+
grep -q "acct-02 weekly=80% session=10% band=30 band-count=1 session-gate=50 session-ok=1" "$ND/selection.log" \
|
|
2481
|
+
&& t_ok "the log shows one account clearing the gate and one band member" \
|
|
2482
|
+
|| t_fail "weekly-only gate log" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2483
|
+
|
|
2484
|
+
# ---- 16-nodata-blind. a fresh timestamp is not a usable reading ---------------------
|
|
2485
|
+
# The third face of the same defect: blindness was judged on fetched_at alone, so a pool
|
|
2486
|
+
# of freshly-written no_data documents looked FRESH — selection.log carried no
|
|
2487
|
+
# ranking=BLIND line, `status` said nothing was wrong, and every account read as unknown,
|
|
2488
|
+
# which is a pool-wide coin flip. An outage that reports itself as healthy is the eleven
|
|
2489
|
+
# days of 2026-08-11 all over again, this time with a current timestamp on it.
|
|
2490
|
+
ndlimits "$WORK/usage-allzero.json" --quiet >/dev/null # BOTH accounts: no_data
|
|
2491
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2492
|
+
: > "$ND/selection.log"
|
|
2493
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2494
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s band=30 band-count=2 session-gate=50 session-ok=0 pwd=" \
|
|
2495
|
+
"$ND/selection.log" \
|
|
2496
|
+
&& t_ok "an all-no_data pool logs ranking=BLIND although its telemetry is seconds old" \
|
|
2497
|
+
|| t_fail "no_data blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2498
|
+
out="$(CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts status 2>&1)"
|
|
2499
|
+
check "status calls an all-no_data pool blind" "RANKING IS BLIND" "$out"
|
|
2500
|
+
# ...and says which KIND of blind, because the two take opposite advice. The eleven-day
|
|
2501
|
+
# 2026-08 outage was stale telemetry — fetch again, then log in. This one is current
|
|
2502
|
+
# telemetry that says nothing: the credential is working perfectly, so sending the
|
|
2503
|
+
# operator to `claude-accounts login` is sending them after a fault that does not exist.
|
|
2504
|
+
check "the no_data banner names the endpoint, not the login" \
|
|
2505
|
+
"Those fetches authenticated; a re-login does NOT fix this" "$out"
|
|
2506
|
+
case "$out" in
|
|
2507
|
+
*"claude-accounts login <acct-NN>"*)
|
|
2508
|
+
t_fail "no_data banner fix line" "a current-but-unusable pool was told to re-login" ;;
|
|
2509
|
+
*) t_ok "the no_data banner does not prescribe a re-login" ;;
|
|
2510
|
+
esac
|
|
2511
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/nodata.json" 2>/dev/null
|
|
2512
|
+
python3 - "$ND/nodata.json" <<'EOF'
|
|
2513
|
+
import json, sys
|
|
2514
|
+
d = json.load(open(sys.argv[1]))
|
|
2515
|
+
assert d['summary']['telemetry'] == 'blind', d['summary']
|
|
2516
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2517
|
+
for a in d['accounts']:
|
|
2518
|
+
u = a['usage']
|
|
2519
|
+
# The panel has to be able to tell THIS outage from the eleven-day one: the readings
|
|
2520
|
+
# are current (not stale), they simply carry nothing to rank on.
|
|
2521
|
+
assert u['no_data'] is True, a
|
|
2522
|
+
assert u['stale'] is False, a
|
|
2523
|
+
assert u['weekly_percent'] is None and u['session_percent'] is None, a
|
|
2524
|
+
EOF
|
|
2525
|
+
[ $? -eq 0 ] && t_ok "--json reports ranking_blind with per-account no_data on fresh readings" \
|
|
2526
|
+
|| t_fail "json no_data blindness" "see $ND/nodata.json"
|
|
2527
|
+
|
|
2528
|
+
# One real reading is enough to rank the pool, and it must take the picks. A no_data
|
|
2529
|
+
# neighbour is unknown, not free.
|
|
2530
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2531
|
+
: > "$ND/selection.log"
|
|
2532
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2533
|
+
nd_mix_hits=0
|
|
2534
|
+
for _ in $(seq 1 6); do
|
|
2535
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-02*) nd_mix_hits=$((nd_mix_hits+1)) ;; esac
|
|
2536
|
+
done
|
|
2537
|
+
{ [ "$nd_mix_hits" = "6" ] && ! grep -q "ranking=BLIND" "$ND/selection.log"; } \
|
|
2538
|
+
&& t_ok "one no_data account beside a real one leaves the pool ranking, and the real one wins 6/6" \
|
|
2539
|
+
|| t_fail "mixed no_data pool" "acct-02 took $nd_mix_hits of 6; $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2540
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/nodata-mixed.json" 2>/dev/null
|
|
2541
|
+
python3 - "$ND/nodata-mixed.json" <<'EOF'
|
|
2542
|
+
import json, sys
|
|
2543
|
+
d = json.load(open(sys.argv[1]))
|
|
2544
|
+
assert d['summary']['telemetry'] == 'fresh', d['summary']
|
|
2545
|
+
assert d['summary']['ranking_blind'] is False, d['summary']
|
|
2546
|
+
byid = {a['id']: a for a in d['accounts']}
|
|
2547
|
+
assert byid['acct-01']['usage']['no_data'] is True, byid['acct-01']
|
|
2548
|
+
# true-only: an ordinary reading must carry the shape every consumer already knows
|
|
2549
|
+
assert 'no_data' not in byid['acct-02']['usage'], byid['acct-02']
|
|
2550
|
+
EOF
|
|
2551
|
+
[ $? -eq 0 ] && t_ok "--json calls the mixed pool fresh and flags only the no_data account" \
|
|
2552
|
+
|| t_fail "json mixed no_data" "see $ND/nodata-mixed.json"
|
|
2553
|
+
|
|
2554
|
+
# ...and a no_data document is never DEGRADABLE. Degraded ranking exists for stale
|
|
2555
|
+
# readings that are still true (a weekly bucket only rises until its reset); a document
|
|
2556
|
+
# with no weekly reading and no horizon has nothing to be true. It must drag the pool to
|
|
2557
|
+
# BLIND rather than let one neighbour's stale number rank alone.
|
|
2558
|
+
printf '{"fetched_at":%s,"source":"oauth","no_data":true,"buckets":[]}' "$(date +%s)" \
|
|
2559
|
+
> "$ND/acct-01/limits.json"
|
|
2560
|
+
printf '{"fetched_at":%s,"weekly_percent":4,"session_percent":0,"max_percent":4,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2561
|
+
"$(( $(date +%s) - 950000 ))" "$(( $(date +%s) + 200000 ))" > "$ND/acct-02/limits.json"
|
|
2562
|
+
: > "$ND/selection.log"
|
|
2563
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2564
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2565
|
+
{ grep -q "ranking=BLIND" "$ND/selection.log" && ! grep -q "ranking=DEGRADED" "$ND/selection.log"; } \
|
|
2566
|
+
&& t_ok "an in-window no_data candidate turns degraded ranking off for the whole pool" \
|
|
2567
|
+
|| t_fail "no_data degraded" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2568
|
+
# the same holds once the no_data document itself goes stale (nothing to rank, ever)
|
|
2569
|
+
printf '{"fetched_at":%s,"source":"oauth","no_data":true,"buckets":[]}' "$(( $(date +%s) - 950000 ))" \
|
|
2570
|
+
> "$ND/acct-01/limits.json"
|
|
2571
|
+
: > "$ND/selection.log"
|
|
2572
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2573
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2574
|
+
{ grep -q "ranking=BLIND" "$ND/selection.log" && ! grep -q "ranking=DEGRADED" "$ND/selection.log"; } \
|
|
2575
|
+
&& t_ok "a stale no_data document is not degradable either" \
|
|
2576
|
+
|| t_fail "stale no_data degraded" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2577
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2578
|
+
|
|
2579
|
+
# ---- 16-onefield-blind. HALF a reading is not a reading -----------------------------
|
|
2580
|
+
# The same defect one layer in, and the one the round-2 fix walked past. Blindness was
|
|
2581
|
+
# taught to reject a no_data document — but it accepted one carrying EITHER percentage,
|
|
2582
|
+
# while pick_best has always needed BOTH (the quota_known rule) before it will call an
|
|
2583
|
+
# account known. The writer emits exactly those half documents, per signal, whenever one
|
|
2584
|
+
# group of buckets goes silent (16-nodata-signals above). So a pool whose every reading
|
|
2585
|
+
# was session-only tied every account, picked uniformly at RANDOM, logged no
|
|
2586
|
+
# ranking=BLIND, and had `status` calling the telemetry fresh — the 2026-08 outage's
|
|
2587
|
+
# defining symptom, with a current timestamp on it. Blind is blind however the reading
|
|
2588
|
+
# came up short.
|
|
2589
|
+
ndl_half() { # ndl_half <session pct> -> a fresh, truthful, SESSION-ONLY reading
|
|
2590
|
+
printf '{"fetched_at":%s,"source":"oauth","session_percent":%s,"max_percent":%s,"buckets":[]}' \
|
|
2591
|
+
"$(date +%s)" "$1" "$1"
|
|
2592
|
+
}
|
|
2593
|
+
ndl_half 12 > "$ND/acct-01/limits.json"
|
|
2594
|
+
ndl_half 18 > "$ND/acct-02/limits.json"
|
|
2595
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2596
|
+
: > "$ND/selection.log"
|
|
2597
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2598
|
+
# session-ok=0 is the point, not a detail: both readings carry a session percentage well
|
|
2599
|
+
# under the gate, and NEITHER clears it, because the gate is part of the same "known"
|
|
2600
|
+
# rule. A pool that cannot rank must not look like one that ranked and tied.
|
|
2601
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s band=30 band-count=2 session-gate=50 session-ok=0 pwd=" \
|
|
2602
|
+
"$ND/selection.log" \
|
|
2603
|
+
&& t_ok "a pool of session-only readings logs ranking=BLIND although both are seconds old" \
|
|
2604
|
+
|| t_fail "one-signal blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2605
|
+
out="$(CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts status 2>&1)"
|
|
2606
|
+
check "status calls a one-signal pool blind" "RANKING IS BLIND" "$out"
|
|
2607
|
+
check "the one-signal banner says the readings are incomplete" \
|
|
2608
|
+
"ranking needs BOTH a weekly and a session percentage" "$out"
|
|
2609
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/onefield.json" 2>/dev/null
|
|
2610
|
+
python3 - "$ND/onefield.json" <<'EOF'
|
|
2611
|
+
import json, sys
|
|
2612
|
+
d = json.load(open(sys.argv[1]))
|
|
2613
|
+
assert d['summary']['telemetry'] == 'blind', d['summary']
|
|
2614
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2615
|
+
for a in d['accounts']:
|
|
2616
|
+
u = a['usage']
|
|
2617
|
+
# Current, well-formed, and NOT a no_data document — one real bucket did report.
|
|
2618
|
+
# It simply is not enough to rank on, and the panel has to agree with the shim.
|
|
2619
|
+
assert u['stale'] is False, a
|
|
2620
|
+
assert 'no_data' not in u, a
|
|
2621
|
+
assert u['session_percent'] is not None and u['weekly_percent'] is None, a
|
|
2622
|
+
EOF
|
|
2623
|
+
[ $? -eq 0 ] && t_ok "--json reports ranking_blind for readings that are half present" \
|
|
2624
|
+
|| t_fail "json one-signal blindness" "see $ND/onefield.json"
|
|
2625
|
+
|
|
2626
|
+
# ...and the weekly-only mirror of the same rule: known needs BOTH, whichever half is
|
|
2627
|
+
# missing. DEGRADED must not fire either — these readings are FRESH, and degraded exists
|
|
2628
|
+
# for an outage of age, not for fresh emptiness (codex review, 2026-09-04).
|
|
2629
|
+
ndl_whalf() { # ndl_whalf <weekly pct> -> a fresh, truthful, WEEKLY-ONLY reading
|
|
2630
|
+
printf '{"fetched_at":%s,"source":"oauth","weekly_percent":%s,"max_percent":%s,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2631
|
+
"$(date +%s)" "$1" "$1" "$(( $(date +%s) + 500000 ))"
|
|
2632
|
+
}
|
|
2633
|
+
ndl_whalf 4 > "$ND/acct-01/limits.json"
|
|
2634
|
+
ndl_whalf 80 > "$ND/acct-02/limits.json"
|
|
2635
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick "$ND"/acct-0*/.limited
|
|
2636
|
+
: > "$ND/selection.log"
|
|
2637
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude >/dev/null 2>&1
|
|
2638
|
+
grep -qE "ranking=BLIND telemetry-age=[0-9]{1,2}s .*session-ok=0 pwd=" "$ND/selection.log" \
|
|
2639
|
+
&& t_ok "a pool of weekly-only readings logs ranking=BLIND, not DEGRADED" \
|
|
2640
|
+
|| t_fail "weekly-only blindness" "$(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2641
|
+
! grep -q "ranking=DEGRADED" "$ND/selection.log" \
|
|
2642
|
+
&& t_ok "fresh weekly-only readings never promote the pool to DEGRADED" \
|
|
2643
|
+
|| t_fail "fresh-degraded" "a fresh one-signal pool ranked DEGRADED"
|
|
2644
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/whalf.json" 2>/dev/null
|
|
2645
|
+
python3 - "$ND/whalf.json" <<'PJ'
|
|
2646
|
+
import json, sys
|
|
2647
|
+
d = json.load(open(sys.argv[1]))
|
|
2648
|
+
assert d['summary']['ranking_blind'] is True, d['summary']
|
|
2649
|
+
PJ
|
|
2650
|
+
[ $? -eq 0 ] && t_ok "--json calls the fresh weekly-only pool blind (not degraded)" \
|
|
2651
|
+
|| t_fail "json one-signal verdict" "see $ND/whalf.json"
|
|
2652
|
+
|
|
2653
|
+
|
|
2654
|
+
# One COMPLETE reading beside them is enough to rank the pool, and it must take every
|
|
2655
|
+
# pick — the half readings are unknown, not free. (This holds on the pre-fix tree too:
|
|
2656
|
+
# it is the guard against over-correcting "half is blind" into "half is excluded".)
|
|
2657
|
+
ndl 30 10 30 > "$ND/acct-02/limits.json"
|
|
2658
|
+
rm -f "$ND/.pick-seq" "$ND"/acct-0*/.last-pick
|
|
2659
|
+
: > "$ND/selection.log"
|
|
2660
|
+
nd_half_hits=0
|
|
2661
|
+
for _ in $(seq 1 6); do
|
|
2662
|
+
case "$(CLAUDE_ACCOUNTS_ROOT="$ND" claude 2>&1)" in *CFG=acct-02*) nd_half_hits=$((nd_half_hits+1)) ;; esac
|
|
2663
|
+
done
|
|
2664
|
+
{ [ "$nd_half_hits" = "6" ] && ! grep -q "ranking=BLIND" "$ND/selection.log"; } \
|
|
2665
|
+
&& t_ok "one complete reading beside a session-only one leaves the pool ranking, and wins 6/6" \
|
|
2666
|
+
|| t_fail "mixed one-signal pool" \
|
|
2667
|
+
"acct-02 took $nd_half_hits of 6; $(tail -1 "$ND/selection.log" 2>/dev/null)"
|
|
2668
|
+
CLAUDE_ACCOUNTS_ROOT="$ND" claude-accounts list --json > "$ND/onefield-mixed.json" 2>/dev/null
|
|
2669
|
+
python3 - "$ND/onefield-mixed.json" <<'EOF'
|
|
2670
|
+
import json, sys
|
|
2671
|
+
d = json.load(open(sys.argv[1]))
|
|
2672
|
+
assert d['summary']['telemetry'] == 'fresh', d['summary']
|
|
2673
|
+
assert d['summary']['ranking_blind'] is False, d['summary']
|
|
2674
|
+
EOF
|
|
2675
|
+
[ $? -eq 0 ] && t_ok "--json calls the mixed one-signal pool fresh" \
|
|
2676
|
+
|| t_fail "json mixed one-signal" "see $ND/onefield-mixed.json"
|
|
2677
|
+
rm -f "$ND"/acct-0*/.limited "$ND"/acct-0*/.client-limit-cleared
|
|
2678
|
+
|
|
2068
2679
|
# ---- 16-marker. the marker names ONE bucket and carries THAT bucket's reset ----------
|
|
2069
2680
|
# A 100% session bucket (resets in an hour) beside a 100% Fable-only weekly bucket
|
|
2070
2681
|
# (resets in five days) used to produce "bucket=session … resets_at=<+1h>" on line 2
|
|
@@ -2855,6 +3466,12 @@ RACER
|
|
|
2855
3466
|
# alone, and these assertions would then be measuring the gate instead of the cutoff
|
|
2856
3467
|
# window. Nobody clearing the gate makes it step aside, so weekly alone ranks here.
|
|
2857
3468
|
mk_cutoff_pool() { # $1 = age of both readings, in seconds
|
|
3469
|
+
# A FRESH clock, not the suite-wide $now captured at startup: the 800s case leaves
|
|
3470
|
+
# only 100s of headroom inside the 900s cutoff window, and the suite takes longer
|
|
3471
|
+
# than that to get here — under load (2026-09-04, parallel review agents) the reading
|
|
3472
|
+
# aged past the window and the assertion flipped. Ages here must mean age AT THE
|
|
3473
|
+
# SHIM'S OWN CLOCK, whenever this test happens to run.
|
|
3474
|
+
local now; now="$(date -u +%s)"
|
|
2858
3475
|
printf '{"fetched_at":%s,"weekly_percent":1,"session_percent":91,"max_percent":91,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
2859
3476
|
"$((now - $1))" "$((now + 200000))" > "$SD/acct-01/limits.json"
|
|
2860
3477
|
printf '{"fetched_at":%s,"weekly_percent":50,"session_percent":55,"max_percent":55,"weekly_resets_epoch":%s,"buckets":[]}' \
|
|
@@ -3107,6 +3724,153 @@ else
|
|
|
3107
3724
|
t_ok "sync validation tests skipped (Mac-only feature; server refuses sync by design)"
|
|
3108
3725
|
fi
|
|
3109
3726
|
|
|
3727
|
+
# ---- 16d. every limits pass fans telemetry out to the manifest's peers ---------------
|
|
3728
|
+
# Why this has teeth (2026-09-04): the runner Macs mini-3..mini-8 hold only portable
|
|
3729
|
+
# setup tokens, and the usage endpoint refuses those for good (403, no user:profile), so
|
|
3730
|
+
# the ONLY telemetry they can ever rank on is the source machine's. limits_distribute
|
|
3731
|
+
# pushes limits.json + .limited to the manifest's `server` AND to every entry in `peers`
|
|
3732
|
+
# — nothing had ever proven the peer half, and it is the half that decides whether six
|
|
3733
|
+
# machines rank blind. Not Mac-gated like `sync`: a fake rsync on PATH records the argv
|
|
3734
|
+
# and the --files-from list (the caller deletes that list as soon as the last push
|
|
3735
|
+
# returns, so it is read at invocation time, not afterwards).
|
|
3736
|
+
DPOOL="$WORK/distribute-pool"
|
|
3737
|
+
mkdir -p "$DPOOL/acct-01" "$DPOOL/acct-02" "$DPOOL/tmp"
|
|
3738
|
+
: > "$DPOOL/.limits-kick"
|
|
3739
|
+
cat > "$DPOOL/accounts.json" <<'EOF'
|
|
3740
|
+
{"version":1,"server":"root@203.0.113.9","server_root":"/root/.claude-accounts",
|
|
3741
|
+
"server_repo":"/root/claude-multiacc","threshold":90,
|
|
3742
|
+
"peers":[
|
|
3743
|
+
{"target":"gas@mini-3","root":"/Users/gas/.claude-accounts","repo":"/Users/gas/claude-multiacc"},
|
|
3744
|
+
{"target":"gas@mini-4","root":"/Users/gas/.claude-accounts","repo":"/Users/gas/claude-multiacc"}],
|
|
3745
|
+
"accounts":[
|
|
3746
|
+
{"id":"acct-01","email":"dp1@test","home":"mac","added_at":"2026-07-13T00:00:00Z"},
|
|
3747
|
+
{"id":"acct-02","email":"dp2@test","home":"mac","added_at":"2026-07-13T00:00:00Z"}]}
|
|
3748
|
+
EOF
|
|
3749
|
+
for i in 01 02; do
|
|
3750
|
+
printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-dp%s","refreshToken":"r","expiresAt":9999999999999,"refreshTokenExpiresAt":9999999999999}}' \
|
|
3751
|
+
"$i" > "$DPOOL/acct-$i/.credentials.json"
|
|
3752
|
+
done
|
|
3753
|
+
# an exclusion marker rides along with the readings (this one survives a clean pass)
|
|
3754
|
+
printf '%s\nbucket=error-cooldown percent=? reason=error-cooldown\n' "$(( $(date +%s) + 600 ))" \
|
|
3755
|
+
> "$DPOOL/acct-02/.limited"
|
|
3756
|
+
RSFAKE="$WORK/rsyncfake"
|
|
3757
|
+
export RSLOG="$WORK/rsync-push.log"
|
|
3758
|
+
mkdir -p "$RSFAKE"
|
|
3759
|
+
cat > "$RSFAKE/rsync" <<'EOF'
|
|
3760
|
+
#!/usr/bin/env bash
|
|
3761
|
+
printf 'RSYNC %s\n' "$*" >> "${RSLOG:?}"
|
|
3762
|
+
for a in "$@"; do
|
|
3763
|
+
case "$a" in
|
|
3764
|
+
--files-from=*)
|
|
3765
|
+
while IFS= read -r l; do printf 'FILE %s\n' "$l" >> "$RSLOG"; done < "${a#--files-from=}" ;;
|
|
3766
|
+
esac
|
|
3767
|
+
done
|
|
3768
|
+
exit 0
|
|
3769
|
+
EOF
|
|
3770
|
+
chmod +x "$RSFAKE/rsync"
|
|
3771
|
+
: > "$RSLOG"
|
|
3772
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3773
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3774
|
+
# The push is deliberately DETACHED — a sleeping peer must never delay the next refresh —
|
|
3775
|
+
# so wait for it instead of assuming it finished.
|
|
3776
|
+
waited=0
|
|
3777
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3778
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3779
|
+
done
|
|
3780
|
+
# The pushes are serialized by a lock dir, so a slow peer can never stack them up. Let
|
|
3781
|
+
# it drain before the next case, or that case's push would be dropped, not made.
|
|
3782
|
+
dp_drain() {
|
|
3783
|
+
local w=0
|
|
3784
|
+
while [ "$w" -lt 50 ] && [ -d "$DPOOL/tmp/limits-push.lock" ]; do sleep 0.2; w=$((w + 1)); done
|
|
3785
|
+
}
|
|
3786
|
+
dp_drain
|
|
3787
|
+
n_push="$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)"
|
|
3788
|
+
[ "$n_push" = "3" ] && t_ok "a limits pass pushes once per target (server + 2 peers)" \
|
|
3789
|
+
|| t_fail "limits distribute" "expected 3 rsync calls, saw $n_push: $(tr '\n' '|' < "$RSLOG")"
|
|
3790
|
+
grep -q "^RSYNC .*root@203.0.113.9:/root/.claude-accounts/" "$RSLOG" \
|
|
3791
|
+
&& t_ok "telemetry is pushed to the manifest server" \
|
|
3792
|
+
|| t_fail "limits distribute server" "$(grep '^RSYNC' "$RSLOG")"
|
|
3793
|
+
for peer in gas@mini-3 gas@mini-4; do
|
|
3794
|
+
grep -q "^RSYNC .*$peer:/Users/gas/.claude-accounts/" "$RSLOG" \
|
|
3795
|
+
&& t_ok "telemetry is pushed to manifest peer $peer" \
|
|
3796
|
+
|| t_fail "limits distribute peer" "$peer never received a push: $(grep '^RSYNC' "$RSLOG")"
|
|
3797
|
+
done
|
|
3798
|
+
# ...and every target gets the SAME list: each account's reading, plus any marker.
|
|
3799
|
+
for f in "acct-01/limits.json" "acct-02/limits.json" "acct-02/.limited"; do
|
|
3800
|
+
[ "$(grep -c "^FILE $f\$" "$RSLOG")" = "3" ] \
|
|
3801
|
+
&& t_ok "the pushed file list names $f for all three targets" \
|
|
3802
|
+
|| t_fail "limits distribute file list" "$f appears $(grep -c "^FILE $f\$" "$RSLOG")x, want 3"
|
|
3803
|
+
done
|
|
3804
|
+
|
|
3805
|
+
# A REPLICA receives telemetry and must never push it back: two writers racing over one
|
|
3806
|
+
# pool is last-writer-wins chaos, and a manifest carrying `peers` is itself pushed TO the
|
|
3807
|
+
# replicas — so the role marker is a machine-local side file, checked before anything else.
|
|
3808
|
+
printf 'replica\n' > "$DPOOL/sync-role"
|
|
3809
|
+
: > "$RSLOG"
|
|
3810
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3811
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3812
|
+
sleep 1
|
|
3813
|
+
[ ! -s "$RSLOG" ] && t_ok "a replica pool never pushes telemetry (limits_distribute is a no-op)" \
|
|
3814
|
+
|| t_fail "replica distribute" "a replica pushed: $(tr '\n' '|' < "$RSLOG")"
|
|
3815
|
+
# only an EXACT 'replica' suppresses it — same anchor as sync
|
|
3816
|
+
printf 'not-replica\n' > "$DPOOL/sync-role"
|
|
3817
|
+
: > "$RSLOG"
|
|
3818
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3819
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3820
|
+
waited=0
|
|
3821
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3822
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3823
|
+
done
|
|
3824
|
+
dp_drain
|
|
3825
|
+
[ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" = "3" ] \
|
|
3826
|
+
&& t_ok "only an exact 'replica' value suppresses the telemetry push" \
|
|
3827
|
+
|| t_fail "replica anchor (distribute)" "'not-replica' suppressed the push"
|
|
3828
|
+
rm -f "$DPOOL/sync-role"
|
|
3829
|
+
|
|
3830
|
+
# ---- 16d-lock. a distribute lock is honored while it is alive, broken once it is not -
|
|
3831
|
+
# The push serializes on a lock DIRECTORY, and a detached push that is killed (logout,
|
|
3832
|
+
# reboot, pkill) never runs its EXIT trap. `mkdir "$lock" || return 0` can then never
|
|
3833
|
+
# succeed again: on the live pool one stranded lock stopped ALL telemetry distribution
|
|
3834
|
+
# from 2026-09-03 00:29 until it was removed by hand on 2026-09-04 — 32 hours in which
|
|
3835
|
+
# every peer ranked on whatever limits.json it happened to already have, which is the
|
|
3836
|
+
# blindness this push exists to prevent, and nothing anywhere said so. A push is seconds
|
|
3837
|
+
# of rsync under hard timeouts, so a lock older than ten minutes belongs to a process
|
|
3838
|
+
# that is gone.
|
|
3839
|
+
: > "$RSLOG"
|
|
3840
|
+
: > "$DPOOL/sync.log"
|
|
3841
|
+
rm -rf "$DPOOL/tmp/limits-push.lock"
|
|
3842
|
+
mkdir -p "$DPOOL/tmp/limits-push.lock"
|
|
3843
|
+
touch -t 202001010000 "$DPOOL/tmp/limits-push.lock" # abandoned in 2020, not busy
|
|
3844
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3845
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3846
|
+
waited=0
|
|
3847
|
+
while [ "$waited" -lt 50 ] && [ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" -lt 3 ]; do
|
|
3848
|
+
sleep 0.2; waited=$((waited + 1))
|
|
3849
|
+
done
|
|
3850
|
+
dp_drain
|
|
3851
|
+
[ "$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)" = "3" ] \
|
|
3852
|
+
&& t_ok "a stale limits-push lock is broken and the pass distributes anyway" \
|
|
3853
|
+
|| t_fail "stale distribute lock" "expected 3 rsync calls, saw $(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null)"
|
|
3854
|
+
grep -q "stale limits-push lock broken" "$DPOOL/sync.log" \
|
|
3855
|
+
&& t_ok "breaking the lock is recorded, so a recurrence is visible instead of silent" \
|
|
3856
|
+
|| t_fail "stale lock log" "sync.log: $(tail -3 "$DPOOL/sync.log" 2>/dev/null | tr '\n' '|')"
|
|
3857
|
+
[ ! -d "$DPOOL/tmp/limits-push.lock" ] \
|
|
3858
|
+
&& t_ok "the retaken lock is released at the end of the push, not leaked again" \
|
|
3859
|
+
|| t_fail "stale lock retake" "the lock dir is still present after the push"
|
|
3860
|
+
|
|
3861
|
+
# ...and a lock that a LIVE push is holding is still absolute: two rsyncs racing into one
|
|
3862
|
+
# peer is exactly what the lock exists to stop, so a fresh one skips this pass entirely.
|
|
3863
|
+
: > "$RSLOG"
|
|
3864
|
+
mkdir -p "$DPOOL/tmp/limits-push.lock" # mtime = now: someone is pushing
|
|
3865
|
+
PATH="$RSFAKE:$PATH" CLAUDE_ACCOUNTS_ROOT="$DPOOL" \
|
|
3866
|
+
CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force --quiet
|
|
3867
|
+
sleep 1
|
|
3868
|
+
{ [ ! -s "$RSLOG" ] && [ -d "$DPOOL/tmp/limits-push.lock" ]; } \
|
|
3869
|
+
&& t_ok "a fresh distribute lock is honored: no push, and the lock is left where it was" \
|
|
3870
|
+
|| t_fail "live distribute lock" "pushes=$(grep -c '^RSYNC ' "$RSLOG" 2>/dev/null) lock=$([ -d "$DPOOL/tmp/limits-push.lock" ] && echo held || echo REMOVED)"
|
|
3871
|
+
rm -rf "$DPOOL/tmp/limits-push.lock"
|
|
3872
|
+
unset RSLOG
|
|
3873
|
+
|
|
3110
3874
|
# API keys are never accepted as credentials (subscription-only requirement)
|
|
3111
3875
|
printf 'sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' > "$WORK/apikey.txt"
|
|
3112
3876
|
out="$(claude-accounts import apikey@test --id acct-11 --token-file "$WORK/apikey.txt" --no-sync 2>&1)"
|
|
@@ -4391,6 +5155,417 @@ out="$(CODEX_MULTIACC_USAGE_URL="file://$WORK/cx-usage-low.json" codex-accounts
|
|
|
4391
5155
|
check "codex: 429 backoff honored" "acct-02: backing off after 429" "$out"
|
|
4392
5156
|
rm -f "$CX"/acct-*/limits.json
|
|
4393
5157
|
|
|
5158
|
+
# ---- C13b. codex parity: a 0% window with NO reported reset is NO DATA ---------------
|
|
5159
|
+
# The claude pool's 2026-09-04 incident (acct-13/acct-14 served every bucket
|
|
5160
|
+
# `percent: 0, resets_at: null`, ranked as the emptiest accounts in the fleet, handed
|
|
5161
|
+
# 31 of the last ~60 picks while the client was being rejected on them at their weekly
|
|
5162
|
+
# limit) is a payload failure, not a claude-specific one — this writer must refuse the
|
|
5163
|
+
# same way. Codex's usage payload carries reset_at INSIDE each window, and this writer
|
|
5164
|
+
# formats resets_at itself, so "the payload reported a window" is the distinction.
|
|
5165
|
+
CD="$WORK/cx-nodata-pool"
|
|
5166
|
+
mkdir -p "$CD/acct-01" "$CD/acct-02" "$CD/tmp"
|
|
5167
|
+
: > "$CD/.limits-kick"
|
|
5168
|
+
cat > "$CD/accounts.json" <<'EOF'
|
|
5169
|
+
{"version":1,"server":"none","threshold":90,"accounts":[
|
|
5170
|
+
{"id":"acct-01","email":"nd1@cx","home":"mac","added_at":"2026-08-21T00:00:00Z"},
|
|
5171
|
+
{"id":"acct-02","email":"nd2@cx","home":"mac","added_at":"2026-08-21T00:00:00Z"}]}
|
|
5172
|
+
EOF
|
|
5173
|
+
mk_cx_auth "$CD/acct-01/auth.json" nd1@cx "$FUTURE_EXP"
|
|
5174
|
+
mk_cx_auth "$CD/acct-02/auth.json" nd2@cx "$FUTURE_EXP"
|
|
5175
|
+
cdl() { # cdl <weekly> <session> <max> -> a truthful, in-window reading on stdout
|
|
5176
|
+
printf '{"fetched_at":%s,"source":"chatgpt","weekly_percent":%s,"session_percent":%s,"max_percent":%s,"plan":"pro","buckets":[]}' \
|
|
5177
|
+
"$(date +%s)" "$1" "$2" "$3"
|
|
5178
|
+
}
|
|
5179
|
+
cdlimits() { # cdlimits <fixture> [extra args] -> a real refresh over the whole CD pool
|
|
5180
|
+
local f="$1"; shift
|
|
5181
|
+
CODEX_ACCOUNTS_ROOT="$CD" CODEX_MULTIACC_USAGE_URL="file://$f" \
|
|
5182
|
+
codex-accounts limits --force "$@" 2>&1
|
|
5183
|
+
}
|
|
5184
|
+
|
|
5185
|
+
# every window 0% with no reset_at at all: nothing here says anything
|
|
5186
|
+
cat > "$WORK/cx-usage-allzero.json" <<'EOF'
|
|
5187
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5188
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5189
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5190
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}},
|
|
5191
|
+
"additional_rate_limits":[]}
|
|
5192
|
+
EOF
|
|
5193
|
+
out="$(cdlimits "$WORK/cx-usage-allzero.json")"
|
|
5194
|
+
check "codex: an all-zero/no-window payload is reported as no usable telemetry" \
|
|
5195
|
+
"no usable telemetry (account ranks as unknown, not as empty)" "$out"
|
|
5196
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5197
|
+
import json, sys
|
|
5198
|
+
d = json.load(open(sys.argv[1]))
|
|
5199
|
+
assert d.get('no_data') is True, d
|
|
5200
|
+
for k in ('max_percent', 'weekly_percent', 'session_percent'):
|
|
5201
|
+
assert k not in d, (k, d)
|
|
5202
|
+
assert len(d['buckets']) == 2 and d['plan'] == 'pro', d # diagnostics survive
|
|
5203
|
+
# the writer's internal "did the payload report this window" flag never reaches disk
|
|
5204
|
+
for b in d['buckets']:
|
|
5205
|
+
assert '_reset_known' not in b, b
|
|
5206
|
+
EOF
|
|
5207
|
+
[ $? -eq 0 ] && t_ok "codex: a no-data pass records no_data and none of the percent signals" \
|
|
5208
|
+
|| t_fail "codex no_data document" "see $CD/acct-01/limits.json"
|
|
5209
|
+
|
|
5210
|
+
# one silent model window beside real ones changes nothing
|
|
5211
|
+
cat > "$WORK/cx-usage-mixed-nodata.json" <<EOF
|
|
5212
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5213
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5214
|
+
"primary_window":{"used_percent":5,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5215
|
+
"secondary_window":{"used_percent":9,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5216
|
+
"additional_rate_limits":[
|
|
5217
|
+
{"limit_name":"GPT-5.3-Codex-Spark","rate_limit":{"allowed":true,"limit_reached":false,
|
|
5218
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5219
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}}}]}
|
|
5220
|
+
EOF
|
|
5221
|
+
cdlimits "$WORK/cx-usage-mixed-nodata.json" --quiet >/dev/null
|
|
5222
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5223
|
+
import json, sys
|
|
5224
|
+
d = json.load(open(sys.argv[1]))
|
|
5225
|
+
assert 'no_data' not in d, d
|
|
5226
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (5, 9, 9), d
|
|
5227
|
+
assert len(d['buckets']) == 4, d # the two silent model windows are still recorded
|
|
5228
|
+
EOF
|
|
5229
|
+
[ $? -eq 0 ] && t_ok "codex: one silent model window beside real ones leaves the ranking untouched (5/9)" \
|
|
5230
|
+
|| t_fail "codex mixed no-data payload" "see $CD/acct-01/limits.json"
|
|
5231
|
+
|
|
5232
|
+
# 0% WITH reported resets is a real, empty reading
|
|
5233
|
+
cat > "$WORK/cx-usage-zero-real-windows.json" <<EOF
|
|
5234
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5235
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5236
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5237
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5238
|
+
"additional_rate_limits":[]}
|
|
5239
|
+
EOF
|
|
5240
|
+
cdlimits "$WORK/cx-usage-zero-real-windows.json" --quiet >/dev/null
|
|
5241
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5242
|
+
import json, sys
|
|
5243
|
+
d = json.load(open(sys.argv[1]))
|
|
5244
|
+
assert 'no_data' not in d, d
|
|
5245
|
+
assert (d['session_percent'], d['weekly_percent'], d['max_percent']) == (0, 0, 0), d
|
|
5246
|
+
EOF
|
|
5247
|
+
[ $? -eq 0 ] && t_ok "codex: 0% WITH reported reset windows still records a real, empty reading" \
|
|
5248
|
+
|| t_fail "codex zero-with-windows payload" "see $CD/acct-01/limits.json"
|
|
5249
|
+
|
|
5250
|
+
# ...and an unknown account loses to any account with a real reading
|
|
5251
|
+
cdlimits "$WORK/cx-usage-allzero.json" --quiet >/dev/null
|
|
5252
|
+
cdl 45 10 45 > "$CD/acct-02/limits.json"
|
|
5253
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick "$CD"/acct-0*/.limited
|
|
5254
|
+
: > "$CD/selection.log"
|
|
5255
|
+
cd_hits=0
|
|
5256
|
+
for _ in $(seq 1 10); do
|
|
5257
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cd_hits=$((cd_hits+1)) ;; esac
|
|
5258
|
+
done
|
|
5259
|
+
[ "$cd_hits" = "0" ] \
|
|
5260
|
+
&& t_ok "codex: a no-data account never outranks one with real telemetry (0 of 10 picks)" \
|
|
5261
|
+
|| t_fail "codex no_data ranking" "the fake-zero account took $cd_hits of 10 picks"
|
|
5262
|
+
grep -q "acct-02 weekly=45% session=10% band=30 band-count=1" "$CD/selection.log" \
|
|
5263
|
+
&& t_ok "codex: the pick logs the known account alone in the band" \
|
|
5264
|
+
|| t_fail "codex no_data band" "selection.log: $(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5265
|
+
|
|
5266
|
+
# ---- C13c. codex parity: ONE marker rule, driven through the REAL rollout path -------
|
|
5267
|
+
# The codex shim has no telemetry-based clearing path at all (a marker leaves it only
|
|
5268
|
+
# when its own reset epoch passes), so the whole rule lives in `codex-accounts limits`.
|
|
5269
|
+
# Until 2026-09-04 that writer kept EVERY active client marker unconditionally, which
|
|
5270
|
+
# looks safe and is half wrong in each direction: a 7d rejection was safe by accident
|
|
5271
|
+
# rather than by rule, and a 5h rejection — a window that refills within hours — parked
|
|
5272
|
+
# the account until its own epoch, the exact stranding #22 (2026-09-03) had to fix on the
|
|
5273
|
+
# claude side.
|
|
5274
|
+
#
|
|
5275
|
+
# These cases no longer HAND the writer a marker. bin/codex writes it, from a rollout
|
|
5276
|
+
# transcript, the way a rejected run does — because the rule the writer applies reads a
|
|
5277
|
+
# token the SHIM chooses, and testing it on a hand-written marker tests a vocabulary the
|
|
5278
|
+
# product never produces. That is exactly what went wrong once already: the guard matched
|
|
5279
|
+
# seven_day/7d/weekly while the scan labelled markers `client:primary`/`client:secondary`
|
|
5280
|
+
# after the rollout's own rate_limits KEY names, which map to no fixed window at all
|
|
5281
|
+
# (live payloads report `primary` as the 10080-minute one). Weekly protection was a no-op
|
|
5282
|
+
# on this provider, and four green assertions said otherwise. The label is now derived
|
|
5283
|
+
# from the record's own `window_minutes` at write time, and these tests drive that
|
|
5284
|
+
# derivation end to end: rollout -> shim -> marker -> limits pass -> selection.
|
|
5285
|
+
# A third, healthy account: with only two, "everything was parked so the pool fell back"
|
|
5286
|
+
# and "the freed account was chosen" produce the same log line.
|
|
5287
|
+
mkdir -p "$CD/acct-03"
|
|
5288
|
+
mk_cx_auth "$CD/acct-03/auth.json" nd3@cx "$FUTURE_EXP"
|
|
5289
|
+
python3 - "$CD/accounts.json" <<'EOF'
|
|
5290
|
+
import json, sys
|
|
5291
|
+
d = json.load(open(sys.argv[1]))
|
|
5292
|
+
if not any(a['id'] == 'acct-03' for a in d['accounts']):
|
|
5293
|
+
d['accounts'].append({"id": "acct-03", "email": "nd3@cx", "home": "mac",
|
|
5294
|
+
"added_at": "2026-08-21T00:00:00Z"})
|
|
5295
|
+
json.dump(d, open(sys.argv[1], 'w'))
|
|
5296
|
+
EOF
|
|
5297
|
+
cx_mark() { # cx_mark <acct dir> <bucket> <reset-offset-seconds> <marked_at ISO>
|
|
5298
|
+
printf '%s\nbucket=%s percent=100 marked_at=%s reason=client-rate-limit\n' \
|
|
5299
|
+
"$(( $(date +%s) + $3 ))" "$2" "$4" > "$1/.limited"
|
|
5300
|
+
}
|
|
5301
|
+
cx_rollout() { # cx_rollout <acct dir> <used_percent> <resets_at epoch> <window_minutes|-> <record ISO>
|
|
5302
|
+
# One rollout, shaped like the real transcript: the codex CLI writes the windows the
|
|
5303
|
+
# server reported into every run's JSONL, and client_limit_scan reads the newest tail.
|
|
5304
|
+
# window_minutes rides in the SAME fragment as used_percent/resets_at — which is why
|
|
5305
|
+
# the shim can label the marker with the window that was actually spent.
|
|
5306
|
+
local day="$1/sessions/2026/09/04" f win=""
|
|
5307
|
+
rm -rf "$1/sessions"
|
|
5308
|
+
mkdir -p "$day"
|
|
5309
|
+
f="$day/rollout-2026-09-04T01-43-21-c13c0001-7fc3-7291-a0fc-7b4e2b035f1a.jsonl"
|
|
5310
|
+
[ "$4" = "-" ] || win="\"window_minutes\":$4,"
|
|
5311
|
+
{
|
|
5312
|
+
printf '{"timestamp":"%s","type":"session_meta","payload":{"session_id":"c13c0001","cwd":"/proj"}}\n' "$5"
|
|
5313
|
+
printf '{"timestamp":"%s","type":"event_msg","payload":{"type":"token_count","info":{"model_context_window":258400},"rate_limits":{"limit_id":"codex","limit_name":null,"primary":{"used_percent":%s,%s"resets_at":%s},"secondary":null,"credits":{"has_credits":false,"unlimited":false}}}}\n' \
|
|
5314
|
+
"$5" "$2" "$win" "$3"
|
|
5315
|
+
} > "$f"
|
|
5316
|
+
rm -f "$1/.client-scan"
|
|
5317
|
+
}
|
|
5318
|
+
cx_bucket() { # the bucket token on line 2 of <acct dir>/.limited, or '<none>'
|
|
5319
|
+
local b=""
|
|
5320
|
+
[ -f "$1/.limited" ] && b="$(sed -n '2s/.*bucket=\([^ ]*\).*/\1/p' "$1/.limited" 2>/dev/null)"
|
|
5321
|
+
printf '%s\n' "${b:-<none>}"
|
|
5322
|
+
}
|
|
5323
|
+
cdlimits0() { # cdlimits with the confirm delay OFF — a marker kept by this pass is kept
|
|
5324
|
+
# by RULE, never merely because the rejection is seconds old. (Real markers
|
|
5325
|
+
# are written by the shim moments before, so there is no "aged" marker to
|
|
5326
|
+
# fabricate; the delay is pinned separately below.)
|
|
5327
|
+
local f="$1"; shift
|
|
5328
|
+
CODEX_ACCOUNTS_ROOT="$CD" CODEX_MULTIACC_USAGE_URL="file://$f" \
|
|
5329
|
+
CODEX_MULTIACC_CLIENT_LIMIT_CONFIRM_DELAY=0 codex-accounts limits --force "$@" 2>&1
|
|
5330
|
+
}
|
|
5331
|
+
cx_shim() { CODEX_ACCOUNTS_ROOT="$CD" codex >/dev/null 2>&1; }
|
|
5332
|
+
rm -f "$CD"/acct-0*/.limited "$CD"/acct-0*/.client-limit-cleared "$CD"/acct-0*/.client-scan
|
|
5333
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5334
|
+
cx_now="$(date +%s)"
|
|
5335
|
+
cx_wreset=$((cx_now + 345600)) # four days out: a weekly window, still open
|
|
5336
|
+
cx_sreset=$((cx_now + 1800)) # half an hour out: a five-hour window
|
|
5337
|
+
# A record STAMPED BEFORE any clear: the watermark tests below turn on this timestamp,
|
|
5338
|
+
# and a rollout the writer's clear did not supersede would prove nothing about it.
|
|
5339
|
+
cx_ts_old="$(python3 -c 'import time; print(time.strftime("%Y-%m-%dT%H:%M:%S.120Z", time.gmtime(time.time() - 600)))')"
|
|
5340
|
+
|
|
5341
|
+
# (1) The marker names the WINDOW the server spent, not the key the payload happened to
|
|
5342
|
+
# use. 10080 minutes is the weekly bucket however the rollout labels it.
|
|
5343
|
+
cx_rollout "$CD/acct-01" "97.4" "$cx_wreset" 10080 "$cx_ts_old"
|
|
5344
|
+
: > "$CD/selection.log"
|
|
5345
|
+
cx_shim
|
|
5346
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5347
|
+
&& grep -q "acct-01 LIMITED by its own run (7d:97, resets $cx_wreset) — client-reported" \
|
|
5348
|
+
"$CD/selection.log"; } \
|
|
5349
|
+
&& t_ok "codex: a 10080-minute rejection is marked client:7d, not after the rollout's key name" \
|
|
5350
|
+
|| t_fail "codex weekly marker label" \
|
|
5351
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5352
|
+
|
|
5353
|
+
# (2) ...and THAT is the token the writer's weekly guard reads. An informative pass with
|
|
5354
|
+
# real numbers far under the threshold, and the confirm delay switched off so nothing but
|
|
5355
|
+
# the bucket rule can be keeping it: the marker has to survive the pass AND the next
|
|
5356
|
+
# launch, because a weekly window cannot fall from the server-proven 97% that wrote it to
|
|
5357
|
+
# 9% while it is still open.
|
|
5358
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5359
|
+
cx_shim
|
|
5360
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5361
|
+
&& [ ! -f "$CD/acct-01/.client-limit-cleared" ] \
|
|
5362
|
+
&& ! printf '%s' "$out" | grep -q "acct-01: marker cleared"; } \
|
|
5363
|
+
&& t_ok "codex: the client:7d marker outlives an informative 9% pass and the launch after it" \
|
|
5364
|
+
|| t_fail "codex weekly marker vs informative pass" \
|
|
5365
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep acct-01 | tr '\n' ' ')"
|
|
5366
|
+
|
|
5367
|
+
# (3) A pass that reported nothing proves nothing, and has to SAY so — or a 32-hour
|
|
5368
|
+
# telemetry stall reads exactly like a healthy pool.
|
|
5369
|
+
out="$(cdlimits0 "$WORK/cx-usage-allzero.json")"
|
|
5370
|
+
{ [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ] \
|
|
5371
|
+
&& printf '%s' "$out" | grep -q "acct-01: marker kept (no usable telemetry)"; } \
|
|
5372
|
+
&& t_ok "codex: a no-data pass keeps the client:7d marker, and logs that it kept it" \
|
|
5373
|
+
|| t_fail "codex weekly marker vs no-data pass" \
|
|
5374
|
+
"bucket=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep acct-01 | tr '\n' ' ')"
|
|
5375
|
+
|
|
5376
|
+
# (4) The differential, from the same code path with one number changed: 300 minutes is
|
|
5377
|
+
# the self-healing session window, so the identical rejection on acct-02 is labelled
|
|
5378
|
+
# client:5h and DOES clear on an informative pass — the #22 (2026-09-03) behavior, which
|
|
5379
|
+
# over-correcting (2) into "no client marker ever clears" would have destroyed.
|
|
5380
|
+
cx_rollout "$CD/acct-02" "99.0" "$cx_sreset" 300 "$cx_ts_old"
|
|
5381
|
+
cx_shim
|
|
5382
|
+
cx_5h_bucket="$(cx_bucket "$CD/acct-02")"
|
|
5383
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5384
|
+
{ [ "$cx_5h_bucket" = "client:5h" ] && [ ! -f "$CD/acct-02/.limited" ] \
|
|
5385
|
+
&& printf '%s' "$out" | grep -q "acct-02: marker cleared (max 9%)" \
|
|
5386
|
+
&& [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ]; } \
|
|
5387
|
+
&& t_ok "codex: a 300-minute rejection is marked client:5h and clears, beside a 7d one that does not" \
|
|
5388
|
+
|| t_fail "codex 5h vs 7d marker rule" \
|
|
5389
|
+
"5h-marked=$cx_5h_bucket 5h-now=$(cx_bucket "$CD/acct-02") 7d=$(cx_bucket "$CD/acct-01") log: $(printf '%s' "$out" | grep 'marker' | tr '\n' ' ')"
|
|
5390
|
+
|
|
5391
|
+
# (5) ...and it STAYS cleared. The rollout that reported the spent window is still on
|
|
5392
|
+
# disk and the scan re-reads its tail on every launch, so without the `.client-limit-
|
|
5393
|
+
# cleared` watermark the clear achieves nothing: the very next `codex` rewrites the same
|
|
5394
|
+
# park, once per 15-minute pass, forever. Two launches, because the first would already
|
|
5395
|
+
# have re-marked.
|
|
5396
|
+
{ [ -f "$CD/acct-02/.client-limit-cleared" ] && { cx_shim; cx_shim; true; } \
|
|
5397
|
+
&& [ ! -f "$CD/acct-02/.limited" ]; } \
|
|
5398
|
+
&& t_ok "codex: the cleared 5h marker is not re-written from the same rollout (watermark)" \
|
|
5399
|
+
|| t_fail "codex client-limit watermark" \
|
|
5400
|
+
"watermark=$([ -f "$CD/acct-02/.client-limit-cleared" ] && echo yes || echo MISSING) bucket=$(cx_bucket "$CD/acct-02")"
|
|
5401
|
+
|
|
5402
|
+
# (6) That decision reaches selection: the weekly-parked account stays out while the
|
|
5403
|
+
# freed one comes back and takes every pick (acct-03 sits 36 points outside the band, so
|
|
5404
|
+
# "acct-02 is eligible again" is the only thing that can produce this).
|
|
5405
|
+
cdl 9 5 9 > "$CD/acct-02/limits.json"
|
|
5406
|
+
cdl 45 10 45 > "$CD/acct-03/limits.json"
|
|
5407
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick
|
|
5408
|
+
: > "$CD/selection.log"
|
|
5409
|
+
cw_hits=0; cw2_hits=0
|
|
5410
|
+
for _ in $(seq 1 6); do
|
|
5411
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in
|
|
5412
|
+
*CFG=acct-01*) cw_hits=$((cw_hits+1)) ;;
|
|
5413
|
+
*CFG=acct-02*) cw2_hits=$((cw2_hits+1)) ;;
|
|
5414
|
+
esac
|
|
5415
|
+
done
|
|
5416
|
+
{ [ "$cw_hits" = "0" ] && [ "$cw2_hits" = "6" ] \
|
|
5417
|
+
&& [ "$(cx_bucket "$CD/acct-01")" = "client:7d" ]; } \
|
|
5418
|
+
&& t_ok "codex: the weekly rejection keeps its account out while the cleared 5h one returns (0 vs 6 of 6)" \
|
|
5419
|
+
|| t_fail "codex marker selection" \
|
|
5420
|
+
"acct-01=$cw_hits acct-02=$cw2_hits of 6; 7d marker=$(cx_bucket "$CD/acct-01")"
|
|
5421
|
+
|
|
5422
|
+
# (7) The watermark is a brake, not a mute: a rejection recorded AFTER the clear is news,
|
|
5423
|
+
# and parks the account again. (Stamped five seconds past the watermark the writer
|
|
5424
|
+
# actually wrote, so this cannot pass by clock luck.)
|
|
5425
|
+
cx_wm="$(head -1 "$CD/acct-02/.client-limit-cleared" 2>/dev/null)"
|
|
5426
|
+
cx_ts_new="$(python3 -c 'import sys, time; print(time.strftime("%Y-%m-%dT%H:%M:%S.120Z", time.gmtime(int(sys.argv[1]) + 5)))' "${cx_wm:-$cx_now}")"
|
|
5427
|
+
cx_rollout "$CD/acct-02" "96.0" "$cx_sreset" 300 "$cx_ts_new"
|
|
5428
|
+
cx_shim
|
|
5429
|
+
{ [ "$(cx_bucket "$CD/acct-02")" = "client:5h" ] \
|
|
5430
|
+
&& grep -q 'percent=96' "$CD/acct-02/.limited"; } \
|
|
5431
|
+
&& t_ok "codex: a rejection recorded after the clear parks the account again (a brake, not a mute)" \
|
|
5432
|
+
|| t_fail "codex watermark is not a mute" \
|
|
5433
|
+
"bucket=$(cx_bucket "$CD/acct-02") line: $(sed -n 2p "$CD/acct-02/.limited" 2>/dev/null)"
|
|
5434
|
+
|
|
5435
|
+
# (8) A report that names no window at all keeps the raw key name — unknown window stays
|
|
5436
|
+
# CLEARABLE, the #22 fail-open direction, and its own reset epoch still bounds it. It
|
|
5437
|
+
# must clear on an informative pass and stay cleared like any other 5h-class marker.
|
|
5438
|
+
rm -f "$CD/acct-03/.limited" "$CD/acct-03/.client-limit-cleared"
|
|
5439
|
+
cx_rollout "$CD/acct-03" "97.0" "$cx_sreset" - "$cx_ts_old"
|
|
5440
|
+
cx_shim
|
|
5441
|
+
cx_nw_bucket="$(cx_bucket "$CD/acct-03")"
|
|
5442
|
+
cdlimits0 "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5443
|
+
cx_shim
|
|
5444
|
+
{ [ "$cx_nw_bucket" = "client:primary" ] && [ ! -f "$CD/acct-03/.limited" ] \
|
|
5445
|
+
&& [ -f "$CD/acct-03/.client-limit-cleared" ]; } \
|
|
5446
|
+
&& t_ok "codex: a report with no window_minutes keeps the raw key name and stays clearable" \
|
|
5447
|
+
|| t_fail "codex windowless marker" \
|
|
5448
|
+
"marked=$cx_nw_bucket now=$(cx_bucket "$CD/acct-03") watermark=$([ -f "$CD/acct-03/.client-limit-cleared" ] && echo yes || echo no)"
|
|
5449
|
+
|
|
5450
|
+
# (9) The same holds for a marker written by an OLDER version and still on disk: it names
|
|
5451
|
+
# `primary`/`secondary`, which is no window, so it keeps its pre-2026-09-04 clearable
|
|
5452
|
+
# behavior rather than being promoted to a weekly park by accident. (Passes on the
|
|
5453
|
+
# pre-fix tree too — deliberately: it is the guard against reading the new guard as
|
|
5454
|
+
# "anything ambiguous sticks".)
|
|
5455
|
+
rm -rf "$CD/acct-03/sessions"
|
|
5456
|
+
rm -f "$CD/acct-03/.client-limit-cleared"
|
|
5457
|
+
cx_mark "$CD/acct-03" client:secondary 345600 2020-01-01T00:00:00Z
|
|
5458
|
+
cdlimits0 "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5459
|
+
[ ! -f "$CD/acct-03/.limited" ] \
|
|
5460
|
+
&& t_ok "codex: a legacy client:secondary marker (no window in its name) is still clearable" \
|
|
5461
|
+
|| t_fail "codex legacy marker" "bucket=$(cx_bucket "$CD/acct-03") survived an informative pass"
|
|
5462
|
+
|
|
5463
|
+
# (10) The confirm delay, which is what stops a cached usage response from erasing a
|
|
5464
|
+
# rejection the client was handed seconds ago. The shim writes marked_at=NOW, so the
|
|
5465
|
+
# marker below is genuinely fresh: the default 300s must keep it, and the same pass with
|
|
5466
|
+
# the window closed clears it — same env var, clamp and semantics as the claude writer.
|
|
5467
|
+
rm -f "$CD/acct-02/.limited" "$CD/acct-02/.client-limit-cleared"
|
|
5468
|
+
cx_rollout "$CD/acct-02" "99.0" "$cx_sreset" 300 "$cx_ts_old"
|
|
5469
|
+
cx_shim
|
|
5470
|
+
out="$(cdlimits "$WORK/cx-usage-low.json")"
|
|
5471
|
+
{ [ "$(cx_bucket "$CD/acct-02")" = "client:5h" ] \
|
|
5472
|
+
&& ! printf '%s' "$out" | grep -q "acct-02: marker cleared"; } \
|
|
5473
|
+
&& t_ok "codex: the default 300s confirm delay keeps a rejection handed over seconds ago" \
|
|
5474
|
+
|| t_fail "codex confirm delay" \
|
|
5475
|
+
"bucket=$(cx_bucket "$CD/acct-02") log: $(printf '%s' "$out" | grep acct-02 | tr '\n' ' ')"
|
|
5476
|
+
out="$(cdlimits0 "$WORK/cx-usage-low.json")"
|
|
5477
|
+
{ [ ! -f "$CD/acct-02/.limited" ] \
|
|
5478
|
+
&& printf '%s' "$out" | grep -q "acct-02: marker cleared (max 9%)"; } \
|
|
5479
|
+
&& t_ok "codex: past the confirm delay the same 5h rejection clears on a real reading" \
|
|
5480
|
+
|| t_fail "codex confirm delay expiry" \
|
|
5481
|
+
"bucket=$(cx_bucket "$CD/acct-02") log: $(printf '%s' "$out" | grep acct-02 | tr '\n' ' ')"
|
|
5482
|
+
|
|
5483
|
+
# (11) error-cooldown is untouched by all of the above: the account failed a real call
|
|
5484
|
+
# moments ago, and no usage reading disproves that. (This one holds on origin/main too —
|
|
5485
|
+
# it is the guard against the rewritten branch quietly dropping a case.)
|
|
5486
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5487
|
+
printf '%s\nbucket=error-cooldown percent=? reason=error-cooldown\n' "$(( $(date +%s) + 600 ))" \
|
|
5488
|
+
> "$CD/acct-01/.limited"
|
|
5489
|
+
cdlimits "$WORK/cx-usage-low.json" --quiet >/dev/null
|
|
5490
|
+
[ -f "$CD/acct-01/.limited" ] \
|
|
5491
|
+
&& t_ok "codex: an error-cooldown marker still survives an informative clean pass" \
|
|
5492
|
+
|| t_fail "codex cooldown vs limits" "the cooldown marker was cleared"
|
|
5493
|
+
rm -f "$CD"/acct-0*/.limited "$CD"/acct-0*/.client-limit-cleared "$CD"/acct-0*/.client-scan
|
|
5494
|
+
rm -rf "$CD"/acct-0*/sessions
|
|
5495
|
+
|
|
5496
|
+
# ---- C13d. codex parity: each ranking signal comes from a window of its OWN kind -----
|
|
5497
|
+
# The claude writer's second 2026-09-04 defect, mirrored here: weekly_percent fell back
|
|
5498
|
+
# to the overall peak and session_percent to a flat 0, so a payload where only the 5h
|
|
5499
|
+
# window said anything was recorded as a WEEKLY reading, and one where only the 7d window
|
|
5500
|
+
# spoke walked through the session gate on a zero nobody reported. A signal no window
|
|
5501
|
+
# reported must be ABSENT — the shim needs both readings to call an account known.
|
|
5502
|
+
cat > "$WORK/cx-usage-session-only.json" <<EOF
|
|
5503
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5504
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5505
|
+
"primary_window":{"used_percent":40,"limit_window_seconds":18000,"reset_at":$((now+3600))},
|
|
5506
|
+
"secondary_window":{"used_percent":0,"limit_window_seconds":604800}},
|
|
5507
|
+
"additional_rate_limits":[]}
|
|
5508
|
+
EOF
|
|
5509
|
+
cdlimits "$WORK/cx-usage-session-only.json" --quiet >/dev/null
|
|
5510
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5511
|
+
import json, sys
|
|
5512
|
+
d = json.load(open(sys.argv[1]))
|
|
5513
|
+
assert 'no_data' not in d, d # one window DID report: this is a reading
|
|
5514
|
+
assert (d['session_percent'], d['max_percent']) == (40, 40), d
|
|
5515
|
+
# The 7d window reported no reset_at and 0%. Recording 40 here (the overall peak) or 0
|
|
5516
|
+
# (max over a silent window) both invent the number the weekly band ranks on.
|
|
5517
|
+
assert 'weekly_percent' not in d, d
|
|
5518
|
+
EOF
|
|
5519
|
+
[ $? -eq 0 ] && t_ok "codex: a session-only reading records session+max and NO weekly_percent" \
|
|
5520
|
+
|| t_fail "codex session-only signals" "see $CD/acct-01/limits.json"
|
|
5521
|
+
cdl 30 10 30 > "$CD/acct-02/limits.json"
|
|
5522
|
+
cdl 30 10 30 > "$CD/acct-03/limits.json"
|
|
5523
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick "$CD"/acct-0*/.limited
|
|
5524
|
+
: > "$CD/selection.log"
|
|
5525
|
+
cs_hits=0
|
|
5526
|
+
for _ in $(seq 1 10); do
|
|
5527
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cs_hits=$((cs_hits+1)) ;; esac
|
|
5528
|
+
done
|
|
5529
|
+
[ "$cs_hits" = "0" ] \
|
|
5530
|
+
&& t_ok "codex: an account with no weekly reading never enters the band (0 of 10 picks)" \
|
|
5531
|
+
|| t_fail "codex session-only ranking" "the weekly-less account took $cs_hits of 10 picks"
|
|
5532
|
+
|
|
5533
|
+
cat > "$WORK/cx-usage-weekly-only.json" <<EOF
|
|
5534
|
+
{"email":"nd@cx","plan_type":"pro",
|
|
5535
|
+
"rate_limit":{"allowed":true,"limit_reached":false,
|
|
5536
|
+
"primary_window":{"used_percent":0,"limit_window_seconds":18000},
|
|
5537
|
+
"secondary_window":{"used_percent":37,"limit_window_seconds":604800,"reset_at":$((now+90000))}},
|
|
5538
|
+
"additional_rate_limits":[]}
|
|
5539
|
+
EOF
|
|
5540
|
+
cdlimits "$WORK/cx-usage-weekly-only.json" --quiet >/dev/null
|
|
5541
|
+
python3 - "$CD/acct-01/limits.json" <<'EOF'
|
|
5542
|
+
import json, sys
|
|
5543
|
+
d = json.load(open(sys.argv[1]))
|
|
5544
|
+
assert 'no_data' not in d, d
|
|
5545
|
+
assert (d['weekly_percent'], d['max_percent']) == (37, 37), d
|
|
5546
|
+
assert 'session_percent' not in d, d # the 5h window reported nothing at all
|
|
5547
|
+
EOF
|
|
5548
|
+
[ $? -eq 0 ] && t_ok "codex: a weekly-only reading records weekly+max and NO session_percent" \
|
|
5549
|
+
|| t_fail "codex weekly-only signals" "see $CD/acct-01/limits.json"
|
|
5550
|
+
# ...and the rivals are WORSE on weekly (80 against 37) and still take every pick,
|
|
5551
|
+
# because they are the only ones that can clear the session gate — which is exactly what
|
|
5552
|
+
# a fabricated `session_percent: 0` would have handed the silent account for free.
|
|
5553
|
+
cdl 80 10 80 > "$CD/acct-02/limits.json"
|
|
5554
|
+
cdl 80 10 80 > "$CD/acct-03/limits.json"
|
|
5555
|
+
rm -f "$CD/.pick-seq" "$CD"/acct-0*/.last-pick
|
|
5556
|
+
: > "$CD/selection.log"
|
|
5557
|
+
cwk_hits=0
|
|
5558
|
+
for _ in $(seq 1 6); do
|
|
5559
|
+
case "$(CODEX_ACCOUNTS_ROOT="$CD" codex 2>&1)" in *CFG=acct-01*) cwk_hits=$((cwk_hits+1)) ;; esac
|
|
5560
|
+
done
|
|
5561
|
+
[ "$cwk_hits" = "0" ] \
|
|
5562
|
+
&& t_ok "codex: an unknown session reading never clears the gate, even with the better weekly (0 of 6)" \
|
|
5563
|
+
|| t_fail "codex weekly-only gate" "the session-less account cleared the gate $cwk_hits of 6 times"
|
|
5564
|
+
grep -q "band=30 band-count=2 session-gate=50 session-ok=2" "$CD/selection.log" \
|
|
5565
|
+
&& t_ok "codex: the log shows exactly the two accounts that reported a session bucket" \
|
|
5566
|
+
|| t_fail "codex weekly-only gate log" "$(tail -1 "$CD/selection.log" 2>/dev/null)"
|
|
5567
|
+
rm -f "$CD"/acct-0*/.limited
|
|
5568
|
+
|
|
4394
5569
|
# ---- C14. oauth refresh via the token endpoint --------------------------------------
|
|
4395
5570
|
# expired bearer, missing endpoint: fail open with backoff
|
|
4396
5571
|
mk_cx_auth "$CX/acct-01/auth.json" a@cx 1000
|
|
@@ -4912,13 +6087,22 @@ check "import refuses to land the same email in a second slot" "already register
|
|
|
4912
6087
|
[ "$rc" != "0" ] && t_ok "conflicting-id import exits nonzero" || t_fail "conflicting id" "exited 0"
|
|
4913
6088
|
|
|
4914
6089
|
# ---- 20c. instance isolation: the shim resolves the same root as the CLI ------------
|
|
6090
|
+
# The leak assertion is a BEFORE/AFTER of the default pool, taken around the instance
|
|
6091
|
+
# run. It used to grep $ACC/selection.log for "acct-01" — a line some earlier, unrelated
|
|
6092
|
+
# test had to have left there, and since picks are random that line is not guaranteed:
|
|
6093
|
+
# the check flaked once in five runs while proving nothing about isolation either way.
|
|
6094
|
+
# Nothing but bin/claude writes selection.log, and this section runs no default-pool
|
|
6095
|
+
# shim, so an unchanged line count is exactly "the instance run stayed in its own pool".
|
|
6096
|
+
iso_before="$(wc -l < "$ACC/selection.log" 2>/dev/null || echo 0)"
|
|
4915
6097
|
out="$(CLAUDE_ACCOUNTS_ROOT="$JP2" claude 2>&1)"
|
|
4916
6098
|
check "shim honors CLAUDE_ACCOUNTS_ROOT (instance pool)" "CFG=acct-01" "$out"
|
|
4917
6099
|
[ -f "$JP2/selection.log" ] && t_ok "instance pool records its own selection log" \
|
|
4918
6100
|
|| t_fail "instance selection log" "missing at $JP2/selection.log"
|
|
4919
|
-
|
|
6101
|
+
iso_after="$(wc -l < "$ACC/selection.log" 2>/dev/null || echo 0)"
|
|
6102
|
+
{ [ "$iso_before" = "$iso_after" ] && ! grep -q "portable@test" "$ACC/accounts.json"; } \
|
|
4920
6103
|
&& t_ok "the default pool was untouched by the instance run" \
|
|
4921
|
-
|| t_fail "pool isolation"
|
|
6104
|
+
|| t_fail "pool isolation" \
|
|
6105
|
+
"the instance run leaked into $ACC (selection.log $iso_before -> $iso_after lines)"
|
|
4922
6106
|
|
|
4923
6107
|
# ---- 20d. sync target: overridable, and a local-only mode that pushes nowhere -------
|
|
4924
6108
|
out="$(CLAUDE_ACCOUNTS_ROOT="$JP" CLAUDE_MULTIACC_NO_SYNC=0 claude-accounts sync --no-server 2>&1)"
|