claude-multiacc 2.0.20 → 2.0.22

package/bin/claude-accounts

@@ -379,7 +379,14 @@ for a in doc.get('accounts', []):
             # entirely and picks at random, so the reading below is decoration.
             stale = age is None or age > STALE_AFTER
             flag = '  << STALE — NOT USED FOR RANKING' if stale else ''
-            print(f"  limits      : {'  '.join(parts) or '(none)'}  [{shown}, max {lim.get('max_percent')}%]{flag}")
+            # A no_data document (all-zero buckets with no reset windows, 2026-09-04)
+            # carries no percent fields at all. Printing "max None%" would read like a
+            # healthy account sitting at zero — the exact misreading that handed 31 of
+            # ~60 picks to two exhausted accounts — so name the state instead.
+            peak = lim.get('max_percent')
+            usable = isinstance(peak, (int, float)) and not isinstance(peak, bool)
+            head = f'max {peak}%' if usable else 'NO USABLE TELEMETRY — ranks as unknown'
+            print(f"  limits      : {'  '.join(parts) or '(none)'}  [{shown}, {head}]{flag}")
             err = lim.get('last_error')
             if isinstance(err, str) and err:
                 when = lim.get('last_error_at')
@@ -428,14 +435,46 @@ if verdict == 'unknown':
     print(f"RANKING STATE UNKNOWN: could not compute the pool-wide telemetry verdict "
           f"({verdict_err}). Check the per-account ages above by hand.")
 elif verdict == 'blind':
-    print("RANKING IS BLIND: no account has usage telemetry inside the "
-          f"{STALE_AFTER}s window, and the last readings are too old to mean anything, "
-          "so every account scores the same and `claude` picks at RANDOM — including "
-          "accounts that are nearly out of weekly headroom.")
-    print("  why : see the 'telemetry' lines above (a setup token cannot read the usage "
-          "endpoint — it has no user:profile scope; only an OAuth login on this machine can)")
-    print("  fix : claude-accounts limits --force   # then, if it still fails:")
-    print("        claude-accounts login <acct-NN>  # per account, on THIS machine")
+    # Two ways to be blind, and they take OPPOSITE advice. STALE: the fetches stopped,
+    # so fetch again and, if that keeps failing, log in — the eleven-day 2026-08 outage.
+    # CURRENT-BUT-UNUSABLE: the endpoint answered inside the window and said nothing
+    # rankable (a no_data document — all-zero buckets with no reset window, 2026-09-04 —
+    # or a reading carrying only one of the two percentages). There the credential is
+    # working perfectly; telling the operator to re-login sends them after a fault that
+    # does not exist, and --force just re-asks for the same emptiness. The candidate
+    # rule below is telemetry_state's own, so this text can never name a state the
+    # verdict did not come from.
+    usable_rows = [a for a in doc_rows if a.get('status') in ('active', 'limited')]
+    cands = [a for a in usable_rows if a.get('status') == 'active'] or usable_rows
+    current = [a for a in cands if (a.get('usage') or {})
+               and not (a.get('usage') or {}).get('stale')]
+    nodata = [a['id'] for a in current if (a.get('usage') or {}).get('no_data')]
+    if current:
+        print("RANKING IS BLIND: usage telemetry is INSIDE the "
+              f"{STALE_AFTER}s window but carries no reading the shim can rank on, so "
+              "every account scores the same and `claude` picks at RANDOM — including "
+              "accounts that are nearly out of weekly headroom.")
+        if nodata:
+            print("  why : the usage endpoint returned no usable data for "
+                  f"{', '.join(nodata)} — all-zero buckets with no reset window, which "
+                  'the writer records as "no_data": true rather than as 0% usage. '
+                  "Those fetches authenticated; a re-login does NOT fix this.")
+        else:
+            print("  why : the readings are incomplete — ranking needs BOTH a weekly and "
+                  "a session percentage (see the 'limits' lines above); an account with "
+                  "only one of them is unknown to selection, exactly as if it had none.")
+        print("  fix : nothing local to repair — the endpoint has to answer with real "
+              "buckets again. `claude-accounts limits --force` re-asks; while it keeps "
+              "answering this way, selection stays random.")
+    else:
+        print("RANKING IS BLIND: no account has usage telemetry inside the "
+              f"{STALE_AFTER}s window, and the last readings are too old to mean anything, "
+              "so every account scores the same and `claude` picks at RANDOM — including "
+              "accounts that are nearly out of weekly headroom.")
+        print("  why : see the 'telemetry' lines above (a setup token cannot read the usage "
+              "endpoint — it has no user:profile scope; only an OAuth login on this machine can)")
+        print("  fix : claude-accounts limits --force   # then, if it still fails:")
+        print("        claude-accounts login <acct-NN>  # per account, on THIS machine")
 elif verdict == 'degraded':
     print("RANKING IS DEGRADED: no account has telemetry inside the "
           f"{STALE_AFTER}s window, so `claude` is ranking on the last readings whose "
@@ -1402,7 +1441,22 @@ limits_distribute() {
 limits_distribute_now() {
   local lock="$ACC_ROOT/tmp/limits-push.lock"
   mkdir -p "$ACC_ROOT/tmp" 2>/dev/null || return 0
-  mkdir "$lock" 2>/dev/null || return 0
+  if ! mkdir "$lock" 2>/dev/null; then
+    # A DETACHED push that is killed (logout, reboot, pkill) never runs its EXIT trap,
+    # and mkdir can never take a lock dir nobody will remove: one stranded lock
+    # silently stopped ALL telemetry distribution on the live pool from 2026-09-03
+    # 00:29 until it was deleted by hand on 2026-09-04 — 32 hours in which every peer
+    # ranked on whatever limits.json it happened to already have, which is the exact
+    # blindness this push exists to prevent, and nothing anywhere said so. A push is
+    # seconds of rsync under hard timeouts (--timeout=20, ConnectTimeout=10), so a
+    # lock older than ten minutes belongs to a process that is gone: break it and
+    # retake it. If the retake still fails, a live pusher owns it and this pass skips,
+    # exactly as before.
+    [ $(( $(epoch_now) - $(file_mtime "$lock") )) -gt 600 ] || return 0
+    rm -rf "$lock" 2>/dev/null
+    mkdir "$lock" 2>/dev/null || return 0
+    log_to sync.log "stale limits-push lock broken (older than 600s); distributing"
+  fi
   trap 'rmdir "$lock" 2>/dev/null || true' EXIT
   local server sroot list id d
   list="$ACC_ROOT/tmp/limits-push.$$"
@@ -1544,6 +1598,34 @@ def parse_iso(s):
     except Exception:
         return None
 
+# The ONE rule for deleting a `.limited` marker on a clean pass. It is the writer half
+# of the shim's client_marker_recovered and has to be the SAME rule: the scheduled
+# limits pass runs every 15 minutes, so a writer that clears more freely than the shim
+# just undoes the shim's fix on its own timer. Before 2026-09-04 it did exactly that —
+# a truthful client:seven_day marker was deleted 300s after it was written, on a pass
+# whose every bucket said `percent 0, resets_at null`, i.e. on nothing at all.
+#   * a pass with NO informative bucket proves nothing, so it clears nothing;
+#   * a client rejection naming a WEEKLY window outlives every reading until its own
+#     reset — a weekly bucket cannot fall from the server-proven 100% that wrote the
+#     marker to under the threshold while that window is still open, so a reading
+#     that says it did is wrong by construction;
+#   * a client rejection naming a session/5h window still clears once the pass is
+#     informative and at least CLIENT_LIMIT_CONFIRM_DELAY newer than the marker (#22,
+#     2026-09-03: 5h markers stranded accounts sitting at 0% usage for days);
+#   * error-cooldown is untouched: it keeps its own window out, exactly as before.
+def marker_bucket(txt):
+    for part in txt.split():
+        if part.startswith('bucket='):
+            return part[7:]
+    return ''
+
+def weekly_marker(txt):
+    # Matched the way both shims match it (bin/claude ~303, same token list): the
+    # claude client writes client:seven_day / client:seven_day_opus, the codex client
+    # writes client:7d, and any future weekly* name is caught too.
+    b = marker_bucket(txt).lower()
+    return 'seven_day' in b or '7d' in b or 'weekly' in b
+
 # `.expired` — the persistent "this account cannot authenticate" marker the shim
 # honors. Written only for a PROVEN dead grant (expired/absent refresh token, or a
 # 4xx from the refresh endpoint), never for a transient network/5xx/429 hiccup.
@@ -2156,12 +2238,41 @@ for acct in manifest.get('accounts', []):
     #   weekly_percent — peak of the durable (weekly/monthly) buckets; the PRIMARY
     #                    ranking signal, because weekly headroom only returns on the
     #                    account's fixed weekly reset (days away).
-    #   session_percent— peak of the self-healing 5h bucket; a soft tiebreaker only.
-    maxp = max([b['percent'] for b in buckets] or [0])
-    weekly = [b['percent'] for b in buckets if b['group'] != 'session']
-    session = [b['percent'] for b in buckets if b['group'] == 'session']
-    weeklyp = max(weekly) if weekly else maxp
-    sessionp = max(session) if session else 0
+    #   session_percent— peak of the self-healing 5h bucket; the session GATE's input:
+    #                    the shim ranks only accounts at/under CLAUDE_MULTIACC_SESSION_GATE
+    #                    (default 50) while any clear it. (A soft tiebreaker until
+    #                    2026-09-03 — the operator asked for session FIRST, then weekly.)
+    # A bucket only feeds those three signals if it SAID something. An INFORMATIVE
+    # bucket has a percent above 0, or a parseable reset window. 2026-09-04: for
+    # acct-13/acct-14 the usage endpoint answered EVERY bucket `percent: 0,
+    # resets_at: null` while Claude Code was being rejected on those same accounts
+    # with "You've hit your weekly limit · resets Sep 8"; this writer recorded the
+    # zeros verbatim, which made two provably exhausted accounts the leaders of the
+    # weekly band and handed them 31 of the last ~60 picks. A truthful bucket ALWAYS
+    # carries the window it resets in, so 0% with no window is NO DATA, not an empty
+    # account. 0% WITH a real window stays informative — a genuinely fresh account
+    # must still rank as empty — and one uninformative bucket beside real ones (the
+    # acct-16 shape: `weekly_scoped:Fable` 0/null next to a real session and
+    # weekly_all) leaves the real buckets ranking exactly as they do today.
+    def informative(b):
+        return b['percent'] > 0 or parse_iso(b.get('resets_at')) is not None
+
+    live = [b for b in buckets if informative(b)]
+    weekly = [b['percent'] for b in live if b['group'] != 'session']
+    session = [b['percent'] for b in live if b['group'] == 'session']
+    # PER SIGNAL, never borrowed from another one. Each of the three answers a
+    # different question, so each is written only when a bucket of ITS OWN kind said
+    # something. Until 2026-09-04 weekly_percent fell back to the overall peak and
+    # session_percent to 0: an account whose weekly buckets were all uninformative
+    # while its 5h bucket read 40% was recorded as 40% WEEKLY — a number no bucket
+    # ever reported, and the signal the band ranks on — and the mirror image
+    # (informative weekly, silent session) was recorded as session 0%, which walks
+    # straight through the session gate. A signal nobody reported must be ABSENT so
+    # the shim reads it as unknown; inventing one is the same mistake as recording a
+    # fake zero, one layer up.
+    maxp = max([b['percent'] for b in live] or [0])
+    weeklyp = max(weekly) if weekly else None
+    sessionp = max(session) if session else None
     # How long weekly_percent keeps meaning something. A weekly bucket only ever RISES
     # until its reset, so before that moment a stale percent is still a valid lower
     # bound and the shim can rank on it when nothing fresher exists; after it, the
@@ -2170,13 +2281,32 @@ for acct in manifest.get('accounts', []):
     # It must come from the bucket weekly_percent actually CAME FROM: a low monthly
     # bucket resetting in an hour says nothing about an 80% weekly one that resets in
     # five days, and taking the minimum over all of them would throw the 80% away.
-    wresets = [int(b['resets_epoch']) for b in buckets
+    wresets = [int(b['resets_epoch']) for b in live
                if b['group'] != 'session' and b['percent'] == weeklyp
                and isinstance(b.get('resets_epoch'), int)]
-    out = {'fetched_at': int(now), 'source': source, 'max_percent': maxp,
-           'weekly_percent': weeklyp, 'session_percent': sessionp,
-           'weekly_resets_epoch': min(wresets) if wresets else 0,
-           'buckets': buckets}
+    out = {'fetched_at': int(now), 'source': source}
+    if live:
+        out['max_percent'] = maxp
+    if weeklyp is not None:
+        out['weekly_percent'] = weeklyp
+    if sessionp is not None:
+        out['session_percent'] = sessionp
+    if weeklyp is not None:
+        # Written with weekly_percent or not at all: the horizon describes THAT
+        # reading, and the shim's degraded path needs both or neither.
+        out['weekly_resets_epoch'] = min(wresets) if wresets else 0
+    if not live:
+        # Nothing usable in the entire payload. Keep the diagnostics (fetched_at,
+        # source, the raw buckets) and write NONE of the three percent signals: a
+        # missing field makes the shim's fresh_field/cutoff_field reads fail, so the
+        # account is UNKNOWN to both selection cuts — never the weekly band's leader,
+        # never inside the session gate, and never able to clear a client-rate-limit
+        # marker. Unknown is the honest reading; "0%" is what re-admitted two provably
+        # exhausted accounts on 2026-09-04 (see informative() above).
+        out['no_data'] = True
+        say(f'{aid}: usage endpoint returned all-zero buckets with no reset windows — '
+            f'no usable telemetry (account ranks as unknown, not as empty)')
+    out['buckets'] = buckets
     tmp = lpath + '.tmp'
     with open(tmp, 'w') as f:
         json.dump(out, f, indent=1)
@@ -2202,14 +2332,37 @@ for acct in manifest.get('accounts', []):
         worst = max(account_level or offenders,
                     key=lambda b: (int(b['resets_epoch']), b['percent']))
         reset_epoch = int(worst['resets_epoch'])
-        # Atomic: a concurrent shim must never read a half-written marker.
-        with open(mpath + '.tmp', 'w') as f:
-            f.write(f'{reset_epoch}\n')
-            f.write(f"bucket={worst['name']} percent={worst['percent']} "
-                    f"marked_at={time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
-                    f"reason=limits resets_at={worst['resets_at']}\n")
-        os.replace(mpath + '.tmp', mpath)
-        say(f"{aid}: LIMITED {worst['name']} at {worst['percent']}% (resets {worst['resets_at']})")
+        # Never SHORTEN an active client-rate-limit marker. A client rejection is
+        # first-hand server evidence with its own reset; this pass's worst offender
+        # can be a mere session bucket an hour from resetting, and overwriting the
+        # marker with that shorter horizon re-admits a provably exhausted account
+        # early (codex review, 2026-09-04: a client:seven_day four days out replaced
+        # by a 95% session bucket +1h). A LATER reset may still extend the exclusion.
+        keep_client = False
+        try:
+            cur = open(mpath).read()
+            first = cur.splitlines()[0] if cur else ''
+            # The shims' own validity rule (num_ok: digits only, bounded length): a
+            # signed/padded/absurd first line is a GARBLED marker to them, and a
+            # garbled marker must be rewritten here, not preserved.
+            if first.isdigit() and len(first) <= 18:
+                cur_reset = int(first)
+                keep_client = ('reason=client-rate-limit' in cur
+                               and cur_reset > now and cur_reset >= reset_epoch)
+        except Exception:
+            pass
+        if keep_client:
+            say(f"{aid}: keeping the client-reported marker (its reset reaches further "
+                f"than this pass's worst offender)")
+        else:
+            # Atomic: a concurrent shim must never read a half-written marker.
+            with open(mpath + '.tmp', 'w') as f:
+                f.write(f'{reset_epoch}\n')
+                f.write(f"bucket={worst['name']} percent={worst['percent']} "
+                        f"marked_at={time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
+                        f"reason=limits resets_at={worst['resets_at']}\n")
+            os.replace(mpath + '.tmp', mpath)
+            say(f"{aid}: LIMITED {worst['name']} at {worst['percent']}% (resets {worst['resets_at']})")
     else:
         if os.path.exists(mpath):
             # A shim-written marker outlives a clean limits pass while its own window
@@ -2232,11 +2385,17 @@ for acct in manifest.get('accounts', []):
                 if marked_epoch is None:
                     marked_epoch = os.path.getmtime(mpath)
                 recent_client = now - marked_epoch < CLIENT_LIMIT_CONFIRM_DELAY
-                client_recovered = active and 'reason=client-rate-limit' in txt \
-                    and not recent_client
-                if active and ('reason=error-cooldown' in txt
-                               or ('reason=client-rate-limit' in txt and recent_client)):
+                is_client = 'reason=client-rate-limit' in txt
+                # See weekly_marker() above for why these three come first and in
+                # this order; the shim applies the identical test per invocation.
+                if active and not live:
+                    keep = True
+                elif active and is_client and weekly_marker(txt):
+                    keep = True
+                elif active and ('reason=error-cooldown' in txt
+                                 or (is_client and recent_client)):
                     keep = True
+                client_recovered = active and is_client and not keep
             except Exception:
                 pass
             if not keep:
@@ -2246,7 +2405,16 @@ for acct in manifest.get('accounts', []):
                     with open(cleared + '.tmp', 'w') as f:
                         f.write(f'{int(now)}\n')
                     os.replace(cleared + '.tmp', cleared)
-                say(f'{aid}: marker cleared (max {maxp}%)')
+                # On a no-data pass `live` is empty and maxp is 0 only because
+                # nothing was reported (see informative() above). Log that, instead
+                # of a "0%" that reads like a proven-empty account — the exact
+                # misreading behind the 2026-09-04 incident.
+                seen = f'max {maxp}%' if live else 'no usable telemetry'
+                say(f'{aid}: marker cleared ({seen})')
+            elif not live:
+                # One line per account, so a no-data pass is legible in limits.log:
+                # the marker was not re-confirmed here, it was merely not disproved.
+                say(f'{aid}: marker kept (no usable telemetry)')
         if not quiet:
             detail = '  '.join(f"{b['name']}={b['percent']}%" for b in buckets)
             print(f'{aid}: ok  {detail}')