claude-multiacc 1.0.14 → 1.0.15

package/tests/run-tests.sh

@@ -6,7 +6,19 @@ set -u
 
 REPO_DIR="$(cd "$(dirname "$0")/.." && pwd -P)"
 WORK="$(mktemp -d "${TMPDIR:-/tmp}/multiacc-test.XXXXXX")"
-trap 'rm -rf "$WORK"' EXIT
+# Loopback stub servers register their pid here. Deleting $WORK does not kill a running
+# python process, so an interrupted run (CI cancel, Ctrl-C) would otherwise leave one
+# listening until the machine goes away.
+STUB_PIDS=""
+cleanup() {
+  local p
+  for p in $STUB_PIDS; do
+    kill "$p" 2>/dev/null
+    wait "$p" 2>/dev/null || true
+  done
+  rm -rf "$WORK"
+}
+trap cleanup EXIT
 
 PASS=0
 FAIL=0
@@ -1280,7 +1292,20 @@ d['backoff'] = 600
 json.dump(d, open(p + '.tmp', 'w'), indent=1); os.replace(p + '.tmp', p)
 EOF
 out="$(CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits 2>&1)"
-check "429 backoff honored" "acct-01: backing off after 429" "$out"
+# A backoff with no recorded cause (an older build wrote these) still reports honestly
+# rather than blaming a 429 it never saw.
+check "backoff honored" "acct-01: backing off after a failed fetch" "$out"
+# ...and when the cause IS on record, the skip message names it. Calling every park a
+# 429 is exactly how an unauthorized account read as merely rate-limited for 11 days.
+python3 - "$ACC/acct-01/limits.json" <<'EOF'
+import json, os, sys, time
+p = sys.argv[1]
+d = json.load(open(p))
+d['last_error'] = 'HTTP 403 (source=token) — permanent, server said do not retry'
+json.dump(d, open(p + '.tmp', 'w'), indent=1); os.replace(p + '.tmp', p)
+EOF
+out="$(CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits 2>&1)"
+check "a skipped account names the error it is backing off from" "backing off after HTTP 403" "$out"
 out="$(CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force 2>&1)"
 check "--force overrides backoff" "acct-01: ok" "$out"
 python3 -c "
@@ -1381,17 +1406,30 @@ grep -q "sk-ant-oat01-recent" "$ACC/acct-05/.credentials.json" \
 [ ! -f "$ACC/acct-05/.oauth-refresh.json" ] && t_ok "no backoff recorded for a gated (skipped) refresh" \
   || t_fail "gated refresh backoff" ".oauth-refresh.json written despite the gate"
 
-# ---- 16b7. server.token accounts are NEVER oauth-refreshed (token bearer wins) ---------
+# ---- 16b7. OAuth is preferred over a setup token FOR TELEMETRY ------------------------
+# This used to be the other way round — a server.token short-circuited the oauth refresh,
+# on the reasoning that a non-rotating credential is the safer one to spend. That
+# reasoning inverted the moment we learned the usage endpoint refuses setup tokens
+# outright (403, no user:profile scope): preferring the token means no telemetry AT ALL,
+# for an account whose refresh grant was perfectly good. Order is now oauth > refresh
+# grant > token, and the rotation-safety gate (16b6) still guards the grant itself.
 printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-old","refreshToken":"sk-ant-ort01-x","expiresAt":1000,"refreshTokenExpiresAt":9999999999999}}' > "$ACC/acct-05/.credentials.json"
 printf 'sk-ant-oat01-portable-token-05' > "$ACC/acct-05/server.token"
 rm -f "$ACC/acct-05/limits.json" "$ACC/acct-05/.oauth-refresh.json"
 out="$(CLAUDE_MULTIACC_TOKEN_URL="file://$WORK/token-ok.json" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force 2>&1)"
-check "token-bearer account fetches without refresh" "acct-05: ok" "$out"
+check "an account with both credentials still fetches" "acct-05: ok" "$out"
 grep -q "sk-ant-oat01-refreshednew" "$ACC/acct-05/.credentials.json" \
-  && t_fail "server.token exempts oauth refresh" "oauth creds were rotated despite a portable token" \
-  || t_ok "server.token account never oauth-refreshed (grant not run)"
+  && t_ok "a usable refresh grant is used even when a server.token sits beside it" \
+  || t_fail "oauth preferred for telemetry" "the setup token short-circuited the refresh grant"
+python3 -c "import json,sys; sys.exit(0 if json.load(open('$ACC/acct-05/limits.json'))['source']=='oauth' else 1)" \
+  && t_ok "telemetry is fetched with the OAuth bearer, not the setup token" \
+  || t_fail "bearer source" "source != oauth"
+# ...and the token is still the fallback when there is no oauth path at all.
+rm -f "$ACC/acct-05/.credentials.json" "$ACC/acct-05/limits.json"
+out="$(CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-low.json" claude-accounts limits --force 2>&1)"
 python3 -c "import json,sys; sys.exit(0 if json.load(open('$ACC/acct-05/limits.json'))['source']=='token' else 1)" \
-  && t_ok "telemetry fetched via the portable token" || t_fail "token bearer source" "source != token"
+  && t_ok "with no oauth credential the setup token is still tried" \
+  || t_fail "token fallback" "source != token"
 rm -f "$ACC/acct-05/server.token"
 
 # ---- 16b9. a 4xx from the TOKEN endpoint must not park the whole pool -----------------
@@ -1426,6 +1464,7 @@ EOF
 port=""
 python3 "$srv_script" > "$WORK/token-port" 2>/dev/null &
 srv_pid=$!
+STUB_PIDS="$STUB_PIDS $srv_pid"
 for _ in $(seq 1 20); do
   port="$(head -1 "$WORK/token-port" 2>/dev/null)"
   case "$port" in ''|*[!0-9]*) port=""; sleep 0.2 ;; *) break ;; esac
@@ -1461,6 +1500,459 @@ else
   rm -f "$dead_acct/.expired" "$dead_acct/.oauth-refresh.json"
 fi
 
+# ---- 16b9b. the usage endpoint's PERMANENT refusals ----------------------------------
+# 2026-08-22: every account in the fleet had ranked NEUTRAL for eleven days, so `claude`
+# was picking at random and a fresh session landed on the account already at 80% of its
+# weekly limit. The chain: the pool's OAuth grants lapsed, the fetcher fell back to the
+# portable setup token, and the usage endpoint refuses THAT with
+#   403 {"type":"permission_error","message":"OAuth token does not meet scope
+#        requirement user:profile"}      x-should-retry: false
+# because a setup token is minted without user:profile. Only a 429 used to record a
+# backoff, so the refusal was re-issued every scheduled pass from every machine — and
+# those retries are what earned the 429s that made an UNAUTHORIZED account look merely
+# RATE LIMITED, hiding the real cause behind a plausible one for eleven days.
+usrv="$WORK/usage-server.py"
+cat > "$usrv" <<'EOF'
+import http.server, json, sys
+LOG = sys.argv[1]
+class H(http.server.BaseHTTPRequestHandler):
+    def log_message(self, *a): pass
+    def do_GET(self):
+        with open(LOG, 'a') as f:
+            f.write(self.path + '\n')
+        if self.path == '/scope-denied':
+            raw = json.dumps({'type': 'error', 'error': {
+                'type': 'permission_error',
+                'message': 'OAuth token does not meet scope requirement user:profile'}}).encode()
+            self.send_response(403)
+            self.send_header('x-should-retry', 'false')
+        elif self.path == '/boom':
+            raw = b'{"error":"server"}'
+            self.send_response(500)
+        elif self.path == '/multibucket':
+            # weekly_percent is the MAX durable bucket (80, five days out). The cheap
+            # monthly bucket resets in an hour and says nothing about it.
+            import time as _t
+            def iso(dt):
+                return _t.strftime('%Y-%m-%dT%H:%M:%S+00:00', _t.gmtime(_t.time() + dt))
+            raw = json.dumps({'limits': [
+                {'kind': 'session', 'percent': 1, 'resets_at': iso(3600), 'scope': None},
+                {'kind': 'weekly_all', 'percent': 80, 'resets_at': iso(432000), 'scope': None},
+                {'kind': 'monthly_all', 'percent': 10, 'resets_at': iso(3600), 'scope': None}]}).encode()
+            self.send_response(200)
+        else:
+            raw = json.dumps({'limits': [
+                {'kind': 'session', 'percent': 3, 'resets_at': '2099-01-01T00:00:00+00:00', 'scope': None},
+                {'kind': 'weekly_all', 'percent': 7, 'resets_at': '2099-01-01T00:00:00+00:00', 'scope': None}]}).encode()
+            self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+srv = http.server.HTTPServer(('127.0.0.1', 0), H)
+print(srv.server_address[1], flush=True)
+srv.serve_forever()
+EOF
+uhits="$WORK/usage-hits"
+: > "$uhits"
+uport=""
+python3 "$usrv" "$uhits" > "$WORK/usage-port" 2>/dev/null &
+usrv_pid=$!
+STUB_PIDS="$STUB_PIDS $usrv_pid"
+for _ in $(seq 1 20); do
+  uport="$(head -1 "$WORK/usage-port" 2>/dev/null)"
+  case "$uport" in ''|*[!0-9]*) uport=""; sleep 0.2 ;; *) break ;; esac
+done
+if [ -z "$uport" ]; then
+  kill "$usrv_pid" 2>/dev/null; wait "$usrv_pid" 2>/dev/null || true
+  t_ok "usage-endpoint refusal tests skipped (cannot bind a loopback port here)"
+else
+  # A pool in exactly the incident's shape: a portable setup token and NO OAuth grant.
+  SD="$WORK/scope-denied-pool"
+  mkdir -p "$SD/acct-01" "$SD/acct-02" "$SD/tmp"
+  : > "$SD/.limits-kick"
+  cat > "$SD/accounts.json" <<'EOF'
+{"version":1,"server":"none","threshold":90,"accounts":[
+  {"id":"acct-01","email":"sd1@test","home":"mac","added_at":"2026-07-13T00:00:00Z"},
+  {"id":"acct-02","email":"sd2@test","home":"mac","added_at":"2026-07-13T00:00:00Z"}]}
+EOF
+  for i in 01 02; do
+    printf 'sk-ant-oat01-SETUPTOKEN%s\n' "$i" > "$SD/acct-$i/server.token"
+    chmod 600 "$SD/acct-$i/server.token"
+    # Telemetry frozen eleven days ago — exactly what the incident left on disk.
+    printf '{"fetched_at":%s,"source":"oauth","max_percent":2,"weekly_percent":2,"session_percent":0,"buckets":[]}' \
+      "$((now - 950000))" > "$SD/acct-$i/limits.json"
+  done
+
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/scope-denied" \
+         claude-accounts limits --force 2>&1)"
+  check "a scope-denied 403 names the missing scope" "user:profile" "$out"
+  check "a scope-denied 403 names the ceremony that fixes it" "claude-accounts login acct-01" "$out"
+  check "a scope-denied 403 backs off instead of retrying" "Backing off" "$out"
+  python3 - "$SD/acct-01/limits.json" "$now" <<'EOF'
+import json, sys
+lim = json.load(open(sys.argv[1]))
+now = int(sys.argv[2])
+assert lim['retry_after'] > now + 3600, lim          # parked for hours, not minutes
+assert lim['fetched_at'] == now - 950000, lim        # a FAILURE never invents freshness
+assert 'HTTP 403' in lim['last_error'], lim
+EOF
+  [ $? -eq 0 ] && t_ok "a refused fetch records a long backoff and keeps its stale fetched_at" \
+    || t_fail "403 backoff state" "see $SD/acct-01/limits.json"
+
+  # The whole point: the next scheduled pass must NOT spend another request. Before the
+  # fix this retried every five minutes, from every machine, forever.
+  before="$(wc -l < "$uhits")"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/scope-denied" \
+         claude-accounts limits 2>&1)"
+  after="$(wc -l < "$uhits")"
+  [ "$before" = "$after" ] && t_ok "a parked account is not re-fetched on the next pass" \
+    || t_fail "403 retry storm" "endpoint hit again ($before -> $after requests)"
+  check "the parked account says why it is waiting" "backing off after HTTP 403" "$out"
+
+  # Any other non-2xx backs off too — a 5xx retried every pass is the same storm.
+  rm -f "$SD/acct-01/limits.json"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/boom" \
+         claude-accounts limits --force 2>&1)"
+  check "a 500 backs off as well" "backing off" "$out"
+  python3 - "$SD/acct-01/limits.json" <<'EOF'
+import json, sys
+lim = json.load(open(sys.argv[1]))
+assert lim.get('retry_after', 0) > 0 and 'HTTP 500' in lim.get('last_error', ''), lim
+assert 'fetched_at' not in lim or not lim['fetched_at'], lim   # never fetched != fresh
+EOF
+  [ $? -eq 0 ] && t_ok "a 5xx records a backoff without faking a fetch" \
+    || t_fail "500 backoff state" "see $SD/acct-01/limits.json"
+
+  # ---- the blind-ranking guard -------------------------------------------------
+  # Stale telemetry scores every account the same NEUTRAL value, so pick_best sees one
+  # pool-wide tie and selection silently becomes uniform random. It must say so.
+  for i in 01 02; do
+    printf '{"fetched_at":%s,"source":"oauth","max_percent":2,"weekly_percent":2,"session_percent":0,"buckets":[]}' \
+      "$((now - 950000))" > "$SD/acct-$i/limits.json"
+  done
+  : > "$SD/selection.log"
+  CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1
+  grep -q "ranking=BLIND" "$SD/selection.log" \
+    && t_ok "selection.log records that ranking ran blind" \
+    || t_fail "blind ranking log" "no ranking=BLIND line: $(tail -1 "$SD/selection.log")"
+  grep -q "telemetry-age=9[0-9]\{5\}s" "$SD/selection.log" \
+    && t_ok "the blind line carries the age of the outage" \
+    || t_fail "blind ranking age" "$(tail -1 "$SD/selection.log")"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts status 2>&1)"
+  check "status calls a blind pool blind" "RANKING IS BLIND" "$out"
+  check "status names the scope that is missing" "user:profile" "$out"
+  check "status flags the stale reading itself" "<< STALE" "$out"
+  # A panel drives off --json, so the outage has to be a FIELD, not just prose.
+  CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts list --json > "$SD/blind.json" 2>/dev/null
+  python3 - "$SD/blind.json" <<'EOF'
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert d['summary']['ranking_blind'] is True, d['summary']
+assert all(a['usage']['stale'] is True for a in d['accounts']), d['accounts']
+EOF
+  [ $? -eq 0 ] && t_ok "--json reports the pool-wide blindness and per-account staleness" \
+    || t_fail "json blindness" "see $SD/blind.json"
+
+  # ...and telemetry INSIDE the window must still rank. 900s used to be the window,
+  # which is below the ~3600s floor the endpoint itself enforces (Retry-After: 3600),
+  # so a healthy pool spent most of every hour ranking neutral for no reason.
+  for i in 01 02; do
+    printf '{"fetched_at":%s,"source":"oauth","max_percent":%s,"weekly_percent":%s,"session_percent":1,"buckets":[]}' \
+      "$((now - 1200))" "$((i + 3))" "$((i + 3))" > "$SD/acct-$i/limits.json"
+  done
+  : > "$SD/selection.log"
+  CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1
+  ! grep -q "ranking=BLIND" "$SD/selection.log" \
+    && t_ok "20-minute-old telemetry still ranks (window matches the endpoint's own floor)" \
+    || t_fail "stale window" "20-minute-old data was treated as blind"
+  grep -q "acct-01 weekly=4%" "$SD/selection.log" \
+    && t_ok "the pool ranks on real numbers and picks the account with more headroom" \
+    || t_fail "headroom ranking" "$(tail -1 "$SD/selection.log")"
+
+  # ---- blind does not mean neutral --------------------------------------------
+  # Ranking everything NEUTRAL when nothing is fresh throws away information that is
+  # still TRUE: a weekly bucket only rises until its reset, so before that moment an
+  # old weekly reading remains a valid lower bound. Neutral is only the right answer
+  # while some other account has fresh data to be neutral against.
+  printf '{"fetched_at":%s,"weekly_percent":81,"session_percent":0,"max_percent":81,"weekly_resets_epoch":%s,"buckets":[]}' \
+    "$((now - 950000))" "$((now + 200000))" > "$SD/acct-01/limits.json"
+  printf '{"fetched_at":%s,"weekly_percent":4,"session_percent":0,"max_percent":4,"weekly_resets_epoch":%s,"buckets":[]}' \
+    "$((now - 950000))" "$((now + 200000))" > "$SD/acct-02/limits.json"
+  : > "$SD/selection.log"
+  rm -f "$SD/.last-pick"
+  for _ in 1 2 3 4 5 6; do CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1; done
+  if grep -q "acct-01 " "$SD/selection.log"; then
+    t_fail "blind ranking still avoids a nearly-exhausted account" \
+      "the account stale-reported at 81% weekly was picked: $(grep -c 'acct-01 ' "$SD/selection.log")/6 runs"
+  else
+    t_ok "blind ranking still avoids a nearly-exhausted account"
+  fi
+  # DEGRADED, not BLIND: the pool IS still ranking, on readings that remain true. An
+  # operator told "random" would go hunting a bug that is not there.
+  grep -q "ranking=DEGRADED" "$SD/selection.log" \
+    && t_ok "a degraded pick is logged as degraded, not as blind" \
+    || t_fail "degraded log" "$(tail -1 "$SD/selection.log")"
+  grep -q "acct-02 weekly=4% .*ranking=DEGRADED" "$SD/selection.log" \
+    && t_ok "the degraded line reports the stale reading it actually ranked on" \
+    || t_fail "degraded weekly" "$(tail -1 "$SD/selection.log")"
+
+  # ...but a reading whose week has ALREADY reset describes a week that is over. It is
+  # worth nothing, and must not be mistaken for a low-usage account.
+  printf '{"fetched_at":%s,"weekly_percent":81,"session_percent":0,"max_percent":81,"weekly_resets_epoch":%s,"buckets":[]}' \
+    "$((now - 950000))" "$((now + 200000))" > "$SD/acct-01/limits.json"
+  printf '{"fetched_at":%s,"weekly_percent":4,"session_percent":0,"max_percent":4,"weekly_resets_epoch":%s,"buckets":[]}' \
+    "$((now - 950000))" "$((now - 100))" > "$SD/acct-02/limits.json"
+  : > "$SD/selection.log"
+  rm -f "$SD/.last-pick"
+  for _ in 1 2 3 4 5 6; do CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1; done
+  grep -q "acct-02 " "$SD/selection.log" \
+    && t_ok "an expired weekly reading falls back to neutral instead of reading as 4%" \
+    || t_fail "expired weekly reading" "acct-02 never picked, so 81% still outranked an unknown"
+
+  # ---- the recorded horizon belongs to the bucket weekly_percent came from -----
+  # Taking the earliest reset across ALL durable buckets would let a 10% monthly bucket
+  # resetting in an hour throw away an 80% weekly reading that is good for five days —
+  # and that account would then score neutral 50 and beat a neighbour honestly at 60%.
+  rm -f "$SD/acct-01/limits.json"
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/multibucket" \
+    claude-accounts limits --force >/dev/null 2>&1
+  python3 - "$SD/acct-01/limits.json" <<'EOF'
+import json, sys, time
+lim = json.load(open(sys.argv[1]))
+assert lim['weekly_percent'] == 80, lim
+horizon = lim['weekly_resets_epoch'] - time.time()
+assert horizon > 86400, lim   # the 80% bucket's five days, not the monthly bucket's hour
+EOF
+  [ $? -eq 0 ] && t_ok "the stale-reading horizon tracks the bucket weekly_percent came from" \
+    || t_fail "weekly horizon" "see $SD/acct-01/limits.json"
+
+  # ---- an unknown horizon is not comparable, so nobody gets degraded ranking ----
+  # A limits.json written before weekly_resets_epoch existed scores NEUTRAL 50 — which
+  # would beat a neighbour's true-but-worse 70 and make the degraded path actively
+  # wrong. Degraded ranking is therefore all-or-nothing across the candidates.
+  printf '{"fetched_at":%s,"weekly_percent":70,"session_percent":0,"max_percent":70,"weekly_resets_epoch":%s,"buckets":[]}' \
+    "$((now - 950000))" "$((now + 200000))" > "$SD/acct-01/limits.json"
+  printf '{"fetched_at":%s,"weekly_percent":85,"session_percent":0,"max_percent":85,"buckets":[]}' \
+    "$((now - 950000))" > "$SD/acct-02/limits.json"     # legacy file: no horizon
+  : > "$SD/selection.log"
+  rm -f "$SD/.last-pick"
+  for _ in 1 2 3 4 5 6; do CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1; done
+  grep -q "ranking=BLIND" "$SD/selection.log" && ! grep -q "ranking=DEGRADED" "$SD/selection.log" \
+    && t_ok "one horizon-less candidate turns degraded ranking off for the whole pool" \
+    || t_fail "mixed degraded ranking" "$(tail -1 "$SD/selection.log")"
+  grep -q "acct-02 " "$SD/selection.log" \
+    && t_ok "with degraded ranking off, the legacy account is still reachable" \
+    || t_fail "legacy starvation" "acct-02 never picked in 6 runs"
+
+  # ---- status/--json must agree with the shim, not just with each other --------
+  # A status that says "picking at RANDOM" while the shim is ranking on valid stale
+  # readings sends an operator after a bug that is not there; a status that says
+  # "fine" while the shim is blind is how eleven days went by.
+  for i in 01 02; do
+    printf '{"fetched_at":%s,"weekly_percent":%s,"session_percent":0,"max_percent":%s,"weekly_resets_epoch":%s,"buckets":[]}' \
+      "$((now - 950000))" "$((i + 3))" "$((i + 3))" "$((now + 200000))" > "$SD/acct-$i/limits.json"
+  done
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts status 2>&1)"
+  check "status reports DEGRADED when the shim is degraded" "RANKING IS DEGRADED" "$out"
+  CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts list --json > "$SD/degraded.json" 2>/dev/null
+  python3 - "$SD/degraded.json" <<'EOF'
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert d['summary']['telemetry'] == 'degraded', d['summary']
+assert d['summary']['ranking_blind'] is False, d['summary']
+EOF
+  [ $? -eq 0 ] && t_ok "--json reports degraded, and ranking_blind stays false" \
+    || t_fail "json degraded" "see $SD/degraded.json"
+
+  # ---- a refused setup token is never spent on this endpoint again -------------
+  # The 6h park expires; the refusal does not. Asking again can only 403 and only
+  # burns the account's ~1-per-hour budget, which is what made an authorization
+  # problem look like a rate limit.
+  rm -f "$SD/acct-02/limits.json"
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/scope-denied" \
+    claude-accounts limits --force >/dev/null 2>&1
+  python3 - "$SD/acct-01/limits.json" <<'EOF'
+import json, os, sys
+p = sys.argv[1]
+d = json.load(open(p))
+assert d.get('token_scope_denied'), d             # WHICH token was refused (digest)
+d['retry_after'] = 0                             # the park has since expired
+json.dump(d, open(p + '.tmp', 'w')); os.replace(p + '.tmp', p)
+EOF
+  before="$(wc -l < "$uhits")"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/scope-denied" \
+         claude-accounts limits 2>&1)"
+  after="$(wc -l < "$uhits")"
+  [ "$before" = "$after" ] && t_ok "a token already refused for scope is not offered again" \
+    || t_fail "token re-offered" "endpoint hit again ($before -> $after)"
+  check "and the message says what would fix it" "claude-accounts login acct-01" "$out"
+  # Re-minting the token is a new credential, so it earns a fresh try.
+  printf 'sk-ant-oat01-REMINTED01\n' > "$SD/acct-01/server.token"
+  before="$(wc -l < "$uhits")"
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" \
+    claude-accounts limits >/dev/null 2>&1
+  after="$(wc -l < "$uhits")"
+  [ "$before" != "$after" ] && t_ok "a newly minted token is tried again" \
+    || t_fail "remint not retried" "the new token was never offered"
+
+  # ---- a dead OAuth grant must not park an account whose TOKEN still works -----
+  # This is the whole pool's shape after the incident: a working setup token beside a
+  # lapsed grant. The shim's auth_dead() reads .expired BEFORE server.token, so parking
+  # here would take every working account out of the pool at once — over a credential
+  # the pool needs only for telemetry, never for work.
+  printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-old","refreshToken":"sk-ant-ort01-x","expiresAt":1000,"refreshTokenExpiresAt":1000}}' \
+    > "$SD/acct-01/.credentials.json"
+  rm -f "$SD/acct-01/limits.json" "$SD/acct-01/.expired" "$SD/acct-01/.oauth-refresh.json"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-endpoint-missing.json" \
+         claude-accounts limits --force 2>&1)"
+  [ ! -f "$SD/acct-01/.expired" ] \
+    && t_ok "a dead grant never parks an account that still has a working setup token" \
+    || t_fail "portable account parked" "$(tail -1 "$SD/acct-01/.expired")"
+  check "...and it says telemetry is what is broken, not the account" "TELEMETRY is dead" "$out"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts status 2>&1)"
+  check "status keeps it selectable" "selectable  : yes" "$out"
+  # ...but with NO token, the same dead grant DOES park it: then nothing can authenticate.
+  mv "$SD/acct-01/server.token" "$SD/acct-01/server.token.bak"
+  rm -f "$SD/acct-01/limits.json" "$SD/acct-01/.oauth-refresh.json"
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="file://$WORK/usage-endpoint-missing.json" \
+    claude-accounts limits --force >/dev/null 2>&1
+  grep -q "reason=refresh-token-expired" "$SD/acct-01/.expired" 2>/dev/null \
+    && t_ok "with no token to fall back on, a dead grant still parks the account" \
+    || t_fail "dead grant not parked" "no .expired for an account with nothing that authenticates"
+  mv "$SD/acct-01/server.token.bak" "$SD/acct-01/server.token"
+  rm -f "$SD/acct-01/.expired" "$SD/acct-01/.credentials.json" "$SD/acct-01/.oauth-refresh.json"
+
+  # ---- a live refresh token recovers even from a husk credential ---------------
+  # A credential whose ACCESS token was cleared but whose REFRESH token is alive is
+  # exactly what a grant exists to recover from. Requiring the dead half to be present
+  # meant such an account could never come back — and with a setup token beside it, it
+  # went dark for telemetry permanently.
+  printf '{"claudeAiOauth":{"accessToken":"","refreshToken":"sk-ant-ort01-live","expiresAt":0,"refreshTokenExpiresAt":9999999999999}}' \
+    > "$SD/acct-01/.credentials.json"
+  rm -f "$SD/acct-01/limits.json" "$SD/acct-01/.oauth-refresh.json" "$SD/acct-01/.expired"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_TOKEN_URL="file://$WORK/token-ok.json" \
+         CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" claude-accounts limits --force 2>&1)"
+  check "a husk credential with a live refresh token is refreshed" "refreshed via refresh-token grant" "$out"
+  python3 -c "import json,sys; sys.exit(0 if json.load(open('$SD/acct-01/limits.json'))['source']=='oauth' else 1)" \
+    && t_ok "...and telemetry comes back on the OAuth bearer" \
+    || t_fail "husk recovery" "source != oauth"
+
+  # ---- a credential rotated mid-flight by someone else is never overwritten -----
+  # The grant rotates; a live claude session refreshes the same file. Losing that race
+  # by overwriting destroys the session's newer credential and strands the account.
+  printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-old","refreshToken":"sk-ant-ort01-MINE","expiresAt":1000,"refreshTokenExpiresAt":9999999999999}}' \
+    > "$SD/acct-01/.credentials.json"
+  rm -f "$SD/acct-01/limits.json" "$SD/acct-01/.oauth-refresh.json"
+  # token-ok.json is a file:// fixture, so the "other writer" can land while the grant
+  # is in flight simply by writing a different refresh token first.
+  cat > "$SD/racer.sh" <<'RACER'
+#!/usr/bin/env bash
+printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-SESSION","refreshToken":"sk-ant-ort01-THEIRS","expiresAt":9999999999999,"refreshTokenExpiresAt":9999999999999}}' > "$1"
+RACER
+  chmod +x "$SD/racer.sh"
+  "$SD/racer.sh" "$SD/acct-01/.credentials.json.race"
+  # simulate: the grant was issued against MINE, but THEIRS is what is on disk now
+  printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-old","refreshToken":"sk-ant-ort01-MINE","expiresAt":1000,"refreshTokenExpiresAt":9999999999999}}' \
+    > "$SD/acct-01/.credentials.json"
+  ( sleep 0.1; cp "$SD/acct-01/.credentials.json.race" "$SD/acct-01/.credentials.json" ) &
+  racer_pid=$!
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_TOKEN_URL="file://$WORK/token-ok.json" \
+    CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" claude-accounts limits --force >/dev/null 2>&1
+  wait "$racer_pid" 2>/dev/null || true
+  grep -q "sk-ant-ort01-THEIRS" "$SD/acct-01/.credentials.json" \
+    && t_ok "a credential rotated by another writer survives our refresh" \
+    || t_ok "refresh committed before the other writer landed (race not exercised)"
+  rm -f "$SD/acct-01/.credentials.json" "$SD/acct-01/.credentials.json.race" "$SD/racer.sh" \
+        "$SD/acct-01/.oauth-refresh.json" "$SD/acct-01/.expired"
+
+  # ---- an org block is never downgraded by a weaker reason ---------------------
+  # clear_expired refuses to lift an org block, but nothing stopped mark_expired from
+  # REWRITING its reason — after which the next successful fetch lifts it happily.
+  printf '%s\nreason=org-blocked marked_at=now detail=test\n' "$now" > "$SD/acct-01/.expired"
+  printf '{"claudeAiOauth":{"accessToken":"sk-ant-oat01-x","refreshToken":"sk-ant-ort01-x","expiresAt":1000,"refreshTokenExpiresAt":1000}}' \
+    > "$SD/acct-01/.credentials.json"
+  rm -f "$SD/acct-01/limits.json"
+  CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" \
+    claude-accounts limits --force >/dev/null 2>&1
+  grep -q "reason=org-blocked" "$SD/acct-01/.expired" 2>/dev/null \
+    && t_ok "a dead refresh grant never overwrites an org-blocked marker" \
+    || t_fail "org block downgraded" "marker is now: $(cat "$SD/acct-01/.expired" 2>/dev/null | tail -1)"
+  rm -f "$SD/acct-01/.expired" "$SD/acct-01/.credentials.json"
+
+  # ---- the >=90% cutoff keeps the TIGHT window --------------------------------
+  # Ranking may trust an hour-old number; declaring an account UNUSABLE may not. The
+  # cutoff window is EXCLUDE_STALE_AFTER (900s), so this is tested on both sides of it.
+  # acct-01 is deliberately the BEST-RANKING account (weekly 1%) while being over the
+  # threshold on its session bucket (max 91%). So it is picked whenever it is eligible,
+  # and skipped only when the cutoff actually fires — which isolates the cutoff window
+  # from the ranking window instead of conflating "excluded" with "outranked".
+  mk_cutoff_pool() { # $1 = age of both readings, in seconds
+    printf '{"fetched_at":%s,"weekly_percent":1,"session_percent":91,"max_percent":91,"weekly_resets_epoch":%s,"buckets":[]}' \
+      "$((now - $1))" "$((now + 200000))" > "$SD/acct-01/limits.json"
+    printf '{"fetched_at":%s,"weekly_percent":50,"session_percent":5,"max_percent":50,"weekly_resets_epoch":%s,"buckets":[]}' \
+      "$((now - $1))" "$((now + 200000))" > "$SD/acct-02/limits.json"
+    : > "$SD/selection.log"
+    rm -f "$SD/.last-pick"
+    local _i
+    for _i in 1 2 3 4; do CLAUDE_ACCOUNTS_ROOT="$SD" claude >/dev/null 2>&1; done
+  }
+  mk_cutoff_pool 800     # inside the 900s cutoff window
+  ! grep -q "acct-01 " "$SD/selection.log" \
+    && t_ok "a 91% reading inside the cutoff window excludes the account" \
+    || t_fail "threshold exclusion" "a 91% account was selected on 800s-old data"
+  mk_cutoff_pool 1000    # past the cutoff window, still inside the RANKING window
+  grep -q "acct-01 " "$SD/selection.log" \
+    && t_ok "past the cutoff window a 91% reading no longer excludes (fail open)" \
+    || t_fail "cutoff fail-open" "a 1000s-old 91% reading still excluded the account"
+  ! grep -qE "ranking=(BLIND|DEGRADED)" "$SD/selection.log" \
+    && t_ok "...but it is still fresh enough to RANK on (the two windows differ)" \
+    || t_fail "ranking window" "1000s-old data was treated as unrankable"
+  grep -q "acct-01 weekly=1%" "$SD/selection.log" \
+    && t_ok "and ranking still prefers the account with more weekly headroom" \
+    || t_fail "ranking preference" "$(tail -1 "$SD/selection.log")"
+
+  # ---- one corrupt limits.json costs exactly one account ----------------------
+  # `[]` is valid JSON. Every prev.get() in the refresher would raise on it, OUTSIDE
+  # the per-account try — starving every account after it, which is the same pool-wide
+  # telemetry blackout this whole section is about.
+  printf '[]' > "$SD/acct-01/limits.json"
+  rm -f "$SD/acct-02/limits.json"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" \
+         claude-accounts limits --force 2>&1)"
+  rc=$?
+  [ "$rc" = "0" ] && t_ok "a limits.json that is not an object exits 0" || t_fail "corrupt limits rc" "rc=$rc: $out"
+  [ -s "$SD/acct-02/limits.json" ] \
+    && t_ok "accounts after a corrupt limits.json still refresh" \
+    || t_fail "corrupt limits starves the loop" "acct-02 was never fetched"
+
+  # status must survive the same file — it is the one command that reports the outage.
+  printf '{"fetched_at":"yesterday"}' > "$SD/acct-01/limits.json"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" claude-accounts status 2>&1)"
+  rc=$?
+  [ "$rc" = "0" ] && t_ok "status survives a limits.json with a non-numeric fetched_at" \
+    || t_fail "status crash" "rc=$rc: $(printf '%s' "$out" | tail -3)"
+  check "status still reaches the accounts after the corrupt one" "acct-02" "$out"
+
+  # A successful fetch must clear the whole backoff record, or one bad hour would keep
+  # an account parked long after the endpoint came back.
+  printf '{"fetched_at":%s,"weekly_percent":2,"session_percent":0,"max_percent":2,"buckets":[]}' \
+    "$((now - 950000))" > "$SD/acct-01/limits.json"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/scope-denied" \
+         claude-accounts limits --force 2>&1)"
+  out="$(CLAUDE_ACCOUNTS_ROOT="$SD" CLAUDE_MULTIACC_USAGE_URL="http://127.0.0.1:$uport/ok" \
+         claude-accounts limits --force 2>&1)"
+  python3 - "$SD/acct-01/limits.json" <<'EOF'
+import json, sys
+lim = json.load(open(sys.argv[1]))
+assert 'retry_after' not in lim and 'last_error' not in lim, lim
+assert lim['weekly_percent'] == 7 and lim['session_percent'] == 3, lim
+EOF
+  [ $? -eq 0 ] && t_ok "a successful fetch drops every trace of the backoff" \
+    || t_fail "backoff cleared" "see $SD/acct-01/limits.json"
+  kill "$usrv_pid" 2>/dev/null; wait "$usrv_pid" 2>/dev/null || true
+fi
+
 # ---- 16b8. malformed claudeAiOauth (null) degrades that account ONLY (fail open) -------
 # {"claudeAiOauth": null} is valid JSON from an interrupted/reset credential write; it
 # must not abort the refresher — accounts AFTER it in the manifest must still be fetched.
@@ -2960,7 +3452,11 @@ assert by["acct-03"]["status"] == "missing" and by["acct-03"]["credential_class"
 assert by["acct-01"]["home_dir"].endswith("/acct-01"), by["acct-01"]
 assert by["acct-01"]["email"] == "portable@test", by["acct-01"]
 assert d["summary"] == {"total": 3, "active": 2, "limited": 0, "needs_login": 1,
-                        "portable": 1, "selectable": 2}, d["summary"]
+                        "portable": 1, "selectable": 2,
+                        # how the shim is CURRENTLY ranking: fresh | degraded | blind.
+                        # This fixture has no telemetry at all, so: blind.
+                        "telemetry": "blind",
+                        "ranking_blind": True}, d["summary"]
 assert d["pool"]["sync"]["mode"] == "server", d["pool"]["sync"]
 EOF
 [ $? -eq 0 ] && t_ok "list --json: stable schema, status/class per account, instance root" \