claude-multiacc 1.0.13 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +276 -15
- package/bin/claude +565 -46
- package/bin/claude-accounts +447 -75
- package/bin/codex +254 -26
- package/bin/codex-accounts +157 -52
- package/install.sh +196 -69
- package/lib/common.sh +262 -2
- package/lib/credential.py +336 -0
- package/lib/report.py +434 -0
- package/package.json +1 -1
- package/tests/run-tests.sh +1556 -8
package/bin/claude-accounts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# claude-accounts — manage the claude-multiacc account pool.
|
|
3
3
|
# Subcommands: list status add import remove mint sync verify limits post-sync health
|
|
4
|
-
# expired relogin
|
|
4
|
+
# expired relogin export-credential import-credential
|
|
5
5
|
set -u
|
|
6
6
|
# lib/audit.py is imported by several subcommands; keep the install tree free of
|
|
7
7
|
# __pycache__ (it may be root-owned, read-only, or an npm global prefix).
|
|
@@ -22,8 +22,8 @@ usage() {
|
|
|
22
22
|
claude-accounts — multi-account pool manager for claude-multiacc
|
|
23
23
|
|
|
24
24
|
USAGE
|
|
25
|
-
claude-accounts list
|
|
26
|
-
claude-accounts status
|
|
25
|
+
claude-accounts list [--json] brief account list (--json: machine-readable)
|
|
26
|
+
claude-accounts status [--json] full health: auth, per-bucket limits, markers
|
|
27
27
|
claude-accounts add [email] [--token] [--force]
|
|
28
28
|
login-FIRST: runs the full Claude Code login (`claude auth login` — the normal
|
|
29
29
|
browser sign-in, no long-lived-token step-up), then registers only after the
|
|
@@ -51,17 +51,34 @@ USAGE
|
|
|
51
51
|
claude-accounts adopt <acct-NN> make acct-NN THIS machine's existing
|
|
52
52
|
default ~/.claude login (dir symlink —
|
|
53
53
|
single credential file, no grant fork)
|
|
54
|
+
claude-accounts export-credential <acct-NN> [--out PATH] [--identity-only]
|
|
55
|
+
print a self-contained JSON blob (credential + identity metadata) for a
|
|
56
|
+
PORTABLE account, i.e. one with a setup-token. REFUSES a machine-local OAuth
|
|
57
|
+
credential (exit 3) — those cannot be copied; sign in on the machine that
|
|
58
|
+
needs them, or run 'mint <acct-NN>' once to make the account portable.
|
|
59
|
+
--identity-only exports registry metadata with NO credential material.
|
|
60
|
+
Exit: 0 ok, 3 machine-local, 4 no credential, 5 unusable material.
|
|
61
|
+
claude-accounts import-credential [acct-NN] [--in PATH|-] [--home mac|server]
|
|
62
|
+
[--force] [--no-sync]
|
|
63
|
+
install a blob from export-credential (stdin by default): creates the account
|
|
64
|
+
dir + manifest entry as needed, or refreshes the credential of the account
|
|
65
|
+
that already owns that email. Non-interactive — this is how a daemon
|
|
66
|
+
distributes accounts to a machine.
|
|
54
67
|
claude-accounts remove <acct-NN> [--yes] delete account (propagates to server)
|
|
55
68
|
claude-accounts dedupe [--yes] remove any account registered twice
|
|
56
69
|
(same email), keeping one per email
|
|
57
70
|
claude-accounts mint <acct-NN> mint server token via `claude setup-token`
|
|
58
71
|
--paste paste an already-minted token instead of running setup-token
|
|
59
|
-
claude-accounts sync
|
|
60
|
-
manifest 'peers' (extra machines). Mac
|
|
61
|
-
A pool with a 'sync-role' file saying
|
|
72
|
+
claude-accounts sync [--no-server] push manifest+tokens to the sync target AND
|
|
73
|
+
any manifest 'peers' (extra machines). Mac
|
|
74
|
+
only. A pool with a 'sync-role' file saying
|
|
62
75
|
'replica' never pushes (it receives).
|
|
76
|
+
--no-server (or a pool whose target is
|
|
77
|
+
'none') keeps sync LOCAL: validate + seed +
|
|
78
|
+
fix perms, push nowhere — for pools a panel
|
|
79
|
+
or runner daemon distributes.
|
|
63
80
|
claude-accounts verify [--quick] auth matrix; full mode runs `-p "reply OK"` per account
|
|
64
|
-
claude-accounts limits [--quiet] [--force]
|
|
81
|
+
claude-accounts limits [--quiet] [--force] [--json]
|
|
65
82
|
refresh usage buckets, apply >=90% markers. Auto-refreshes long-expired
|
|
66
83
|
OAuth access tokens via the refresh-token grant (rotated credential is
|
|
67
84
|
persisted), so idle accounts keep fresh telemetry and stay selectable.
|
|
@@ -73,7 +90,12 @@ USAGE
|
|
|
73
90
|
claude-accounts post-sync (server side) seed dirs, fix perms, quick verify
|
|
74
91
|
|
|
75
92
|
ENV
|
|
76
|
-
|
|
93
|
+
CLAUDE_ACCOUNTS_ROOT pool root, overriding ~/.claude-accounts — one isolated pool
|
|
94
|
+
per app-robot instance on a shared machine (legacy spelling
|
|
95
|
+
CLAUDE_ACCOUNTS_DIR still works)
|
|
96
|
+
CLAUDE_MULTIACC_SYNC_TARGET sync target (user@host), overriding the manifest;
|
|
97
|
+
'none' = local-only, nothing is pushed anywhere
|
|
98
|
+
CLAUDE_MULTIACC_SYNC_ROOT / _SYNC_REPO remote pool root / addon repo for it
|
|
77
99
|
CLAUDE_ACCOUNT pin the shim to one account
|
|
78
100
|
CLAUDE_SHIM_RETRY=0 disable -p auto-retry
|
|
79
101
|
CLAUDE_MULTIACC_DISABLE=1 bypass the shim entirely
|
|
@@ -95,17 +117,19 @@ next_id() {
|
|
|
95
117
|
done
|
|
96
118
|
}
|
|
97
119
|
|
|
98
|
-
manifest_add_account() { # id email home
|
|
99
|
-
"$PYBIN" - "$MANIFEST" "$1" "$2" "$3" <<'PYEOF'
|
|
120
|
+
manifest_add_account() { # id email home [added_at]
|
|
121
|
+
"$PYBIN" - "$MANIFEST" "$1" "$2" "$3" "${4:-}" <<'PYEOF'
|
|
100
122
|
import json, sys, time
|
|
101
|
-
path, aid, email, home = sys.argv[1:
|
|
123
|
+
path, aid, email, home, added_at = sys.argv[1:6]
|
|
102
124
|
doc = json.load(open(path))
|
|
103
125
|
accounts = [a for a in doc.get('accounts', []) if a['id'] != aid]
|
|
104
126
|
accounts.append({
|
|
105
127
|
'id': aid,
|
|
106
128
|
'email': email,
|
|
107
129
|
'home': home,
|
|
108
|
-
|
|
130
|
+
# An imported account keeps the added_at it was registered with on the machine it
|
|
131
|
+
# came from, so the same account reads identically across the fleet.
|
|
132
|
+
'added_at': added_at or time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
109
133
|
})
|
|
110
134
|
accounts.sort(key=lambda a: a['id'])
|
|
111
135
|
doc['accounts'] = accounts
|
|
@@ -128,6 +152,17 @@ for a in json.load(open(sys.argv[1])).get('accounts', []):
|
|
|
128
152
|
PYEOF
|
|
129
153
|
}
|
|
130
154
|
|
|
155
|
+
manifest_email_of() { # prints the email registered for <id>, empty if unknown
|
|
156
|
+
[ -f "$MANIFEST" ] || return 0
|
|
157
|
+
"$PYBIN" - "$MANIFEST" "$1" <<'PYEOF' 2>/dev/null
|
|
158
|
+
import json, sys
|
|
159
|
+
for a in json.load(open(sys.argv[1])).get('accounts', []):
|
|
160
|
+
if isinstance(a, dict) and a.get('id') == sys.argv[2]:
|
|
161
|
+
print(a.get('email', ''))
|
|
162
|
+
break
|
|
163
|
+
PYEOF
|
|
164
|
+
}
|
|
165
|
+
|
|
131
166
|
manifest_del_account() { # id
|
|
132
167
|
"$PYBIN" - "$MANIFEST" "$1" <<'PYEOF'
|
|
133
168
|
import json, sys
|
|
@@ -143,8 +178,11 @@ PYEOF
|
|
|
143
178
|
}
|
|
144
179
|
|
|
145
180
|
auto_sync() { # best effort after mutations, Mac only, loud on failure
|
|
146
|
-
[ "$(machine_kind)" = "mac" ] || return 0
|
|
147
181
|
[ "${CLAUDE_MULTIACC_NO_SYNC:-0}" = "1" ] && return 0
|
|
182
|
+
# Local-only pool (target 'none'): there is nothing to push — the panel/runner
|
|
183
|
+
# daemon distributes accounts — so a mutation must not warn about a missing server.
|
|
184
|
+
sync_target_is_local "$(sync_target)" && return 0
|
|
185
|
+
[ "$(machine_kind)" = "mac" ] || return 0
|
|
148
186
|
# A replica pool never pushes (the source machine owns the account set) —
|
|
149
187
|
# silently, so every mutation on a replica does not nag about it.
|
|
150
188
|
sync_is_replica && return 0
|
|
@@ -154,6 +192,14 @@ auto_sync() { # best effort after mutations, Mac only, loud on failure
|
|
|
154
192
|
|
|
155
193
|
cmd_list() {
|
|
156
194
|
require_manifest
|
|
195
|
+
local json=0
|
|
196
|
+
while [ $# -gt 0 ]; do
|
|
197
|
+
case "$1" in
|
|
198
|
+
--json) json=1; shift ;;
|
|
199
|
+
*) die "unknown option: $1 (usage: claude-accounts list [--json])" ;;
|
|
200
|
+
esac
|
|
201
|
+
done
|
|
202
|
+
if [ "$json" = "1" ]; then emit_report_json list; return $?; fi
|
|
157
203
|
"$PYBIN" - "$MANIFEST" "$ACC_ROOT" "$LIB_DIR" "$(machine_kind)" <<'PYEOF'
|
|
158
204
|
import json, os, re, sys
|
|
159
205
|
doc = json.load(open(sys.argv[1]))
|
|
@@ -205,6 +251,14 @@ PYEOF
|
|
|
205
251
|
|
|
206
252
|
cmd_status() {
|
|
207
253
|
require_manifest
|
|
254
|
+
local json=0
|
|
255
|
+
while [ $# -gt 0 ]; do
|
|
256
|
+
case "$1" in
|
|
257
|
+
--json) json=1; shift ;;
|
|
258
|
+
*) die "unknown option: $1 (usage: claude-accounts status [--json])" ;;
|
|
259
|
+
esac
|
|
260
|
+
done
|
|
261
|
+
if [ "$json" = "1" ]; then emit_report_json status; return $?; fi
|
|
208
262
|
"$PYBIN" - "$MANIFEST" "$ACC_ROOT" "$LIB_DIR" "$(machine_kind)" <<'PYEOF'
|
|
209
263
|
import json, os, sys, time
|
|
210
264
|
doc = json.load(open(sys.argv[1]))
|
|
@@ -213,6 +267,25 @@ sys.path = [sys.argv[3]] + [p for p in sys.path if p not in ('', '.')]
|
|
|
213
267
|
from audit import audit_account # noqa: E402 (shared with the shim's rule)
|
|
214
268
|
machine = sys.argv[4]
|
|
215
269
|
now = time.time()
|
|
270
|
+
# Must match the shim's window (bin/claude), or status would call data "fresh" that
|
|
271
|
+
# selection has already stopped ranking on. Same fallback as the shim for a garbled
|
|
272
|
+
# override: a bad env var must never be the thing that stops status from printing.
|
|
273
|
+
try:
|
|
274
|
+
STALE_AFTER = int(os.environ.get('CLAUDE_MULTIACC_STALE_AFTER') or 3600)
|
|
275
|
+
except ValueError:
|
|
276
|
+
STALE_AFTER = 3600
|
|
277
|
+
if STALE_AFTER <= 0:
|
|
278
|
+
STALE_AFTER = 3600
|
|
279
|
+
|
|
280
|
+
def telem_fetched_at(aid):
|
|
281
|
+
"""Epoch of aid's last SUCCESSFUL usage fetch, or 0. Never raises: this file is
|
|
282
|
+
hand-editable and syncs between machines, and one corrupt copy must not stop
|
|
283
|
+
status from printing the pool-wide verdict it exists to show."""
|
|
284
|
+
try:
|
|
285
|
+
v = json.load(open(os.path.join(root, aid, 'limits.json'))).get('fetched_at', 0)
|
|
286
|
+
except Exception:
|
|
287
|
+
return 0
|
|
288
|
+
return v if isinstance(v, (int, float)) and not isinstance(v, bool) and v > 0 else 0
|
|
216
289
|
|
|
217
290
|
def last_pick(aid):
|
|
218
291
|
path = os.path.join(root, 'selection.log')
|
|
@@ -234,12 +307,15 @@ print(f"server : {doc.get('server','-')} (root: {doc.get('server_root','-')}
|
|
|
234
307
|
print(f"threshold : {doc.get('threshold', 90)}% (any bucket at/above => account excluded)")
|
|
235
308
|
print()
|
|
236
309
|
needs_login = []
|
|
310
|
+
selectable_ids = []
|
|
237
311
|
for a in doc.get('accounts', []):
|
|
238
312
|
aid = a['id']
|
|
239
313
|
d = os.path.join(root, aid)
|
|
240
314
|
st = audit_account(root, a, machine=machine)
|
|
241
315
|
if st['state'] in ('expired', 'blocked', 'missing'):
|
|
242
316
|
needs_login.append(aid)
|
|
317
|
+
else:
|
|
318
|
+
selectable_ids.append(aid)
|
|
243
319
|
banner = {'ok': '', 'remote': ' [not logged in here — grant lives elsewhere]',
|
|
244
320
|
'missing': ' ** NO LOGIN — claude-accounts relogin %s **' % aid,
|
|
245
321
|
'expired': ' ** LOGIN EXPIRED — claude-accounts relogin %s **' % aid,
|
|
@@ -271,9 +347,33 @@ for a in doc.get('accounts', []):
|
|
|
271
347
|
if os.path.isfile(lpath):
|
|
272
348
|
try:
|
|
273
349
|
lim = json.load(open(lpath))
|
|
274
|
-
|
|
350
|
+
if not isinstance(lim, dict):
|
|
351
|
+
raise ValueError('not an object')
|
|
352
|
+
fetched = telem_fetched_at(aid)
|
|
353
|
+
age = int(now - fetched) if fetched else None
|
|
275
354
|
parts = [f"{b['name']}={b['percent']}%" for b in lim.get('buckets', [])]
|
|
276
|
-
|
|
355
|
+
if age is None:
|
|
356
|
+
shown = 'never fetched'
|
|
357
|
+
elif age >= 86400:
|
|
358
|
+
shown = f'{age // 86400}d {age % 86400 // 3600}h old'
|
|
359
|
+
elif age >= 3600:
|
|
360
|
+
shown = f'{age // 3600}h {age % 3600 // 60}m old'
|
|
361
|
+
else:
|
|
362
|
+
shown = f'{age}s old'
|
|
363
|
+
# Loud, because stale numbers do not look stale: they look like a healthy
|
|
364
|
+
# account sitting at 2%. Past the ranking window the shim ignores them
|
|
365
|
+
# entirely and picks at random, so the reading below is decoration.
|
|
366
|
+
stale = age is None or age > STALE_AFTER
|
|
367
|
+
flag = ' << STALE — NOT USED FOR RANKING' if stale else ''
|
|
368
|
+
print(f" limits : {' '.join(parts) or '(none)'} [{shown}, max {lim.get('max_percent')}%]{flag}")
|
|
369
|
+
err = lim.get('last_error')
|
|
370
|
+
if isinstance(err, str) and err:
|
|
371
|
+
when = lim.get('last_error_at')
|
|
372
|
+
ago = f' ({int(now - when)}s ago)' if isinstance(when, (int, float)) else ''
|
|
373
|
+
print(f" telemetry : last fetch FAILED{ago}: {err}")
|
|
374
|
+
retry = lim.get('retry_after', 0)
|
|
375
|
+
if isinstance(retry, (int, float)) and retry > now:
|
|
376
|
+
print(f" retrying in {int(retry - now)}s")
|
|
277
377
|
except Exception as e:
|
|
278
378
|
print(f" limits : unreadable ({e})")
|
|
279
379
|
else:
|
|
@@ -295,6 +395,40 @@ for a in doc.get('accounts', []):
|
|
|
295
395
|
if needs_login:
|
|
296
396
|
print(f"{len(needs_login)} account(s) are EXCLUDED from selection: {', '.join(needs_login)}")
|
|
297
397
|
print("What each one needs: claude-accounts expired")
|
|
398
|
+
# Pool-wide verdict. Per-account ages are easy to skim past; "the pool is picking at
|
|
399
|
+
# random" is not. This is the line that would have caught an eleven-day outage on the
|
|
400
|
+
# first day someone ran status.
|
|
401
|
+
# The verdict comes from lib/report.py so this text, `--json`, and the shim's own
|
|
402
|
+
# ranking can never drift apart — a status that disagrees with selection is worse than
|
|
403
|
+
# no status at all.
|
|
404
|
+
try:
|
|
405
|
+
from report import telemetry_state, build # noqa: E402
|
|
406
|
+
doc_rows = build(root, 'claude', machine, 'status')['accounts']
|
|
407
|
+
verdict = telemetry_state(doc_rows, now)
|
|
408
|
+
except Exception as e:
|
|
409
|
+
verdict = 'unknown'
|
|
410
|
+
verdict_err = str(e)[:200]
|
|
411
|
+
if verdict == 'unknown':
|
|
412
|
+
# Silence here would recreate the observability half of the incident: a blind pool
|
|
413
|
+
# that says nothing. Say the verdict could not be computed, and why.
|
|
414
|
+
print(f"RANKING STATE UNKNOWN: could not compute the pool-wide telemetry verdict "
|
|
415
|
+
f"({verdict_err}). Check the per-account ages above by hand.")
|
|
416
|
+
elif verdict == 'blind':
|
|
417
|
+
print("RANKING IS BLIND: no account has usage telemetry inside the "
|
|
418
|
+
f"{STALE_AFTER}s window, and the last readings are too old to mean anything, "
|
|
419
|
+
"so every account scores the same and `claude` picks at RANDOM — including "
|
|
420
|
+
"accounts that are nearly out of weekly headroom.")
|
|
421
|
+
print(" why : see the 'telemetry' lines above (a setup token cannot read the usage "
|
|
422
|
+
"endpoint — it has no user:profile scope; only an OAuth login on this machine can)")
|
|
423
|
+
print(" fix : claude-accounts limits --force # then, if it still fails:")
|
|
424
|
+
print(" claude-accounts login <acct-NN> # per account, on THIS machine")
|
|
425
|
+
elif verdict == 'degraded':
|
|
426
|
+
print("RANKING IS DEGRADED: no account has telemetry inside the "
|
|
427
|
+
f"{STALE_AFTER}s window, so `claude` is ranking on the last readings whose "
|
|
428
|
+
"weekly bucket has not reset yet. Better than random, but it cannot see usage "
|
|
429
|
+
"since those readings were taken.")
|
|
430
|
+
print(" fix : claude-accounts limits --force # then, if it still fails:")
|
|
431
|
+
print(" claude-accounts login <acct-NN> # per account, on THIS machine")
|
|
298
432
|
PYEOF
|
|
299
433
|
}
|
|
300
434
|
|
|
@@ -922,11 +1056,12 @@ cmd_relogin() {
|
|
|
922
1056
|
|
|
923
1057
|
cmd_limits() {
|
|
924
1058
|
require_manifest
|
|
925
|
-
local quiet=0 force=0
|
|
1059
|
+
local quiet=0 force=0 json=0
|
|
926
1060
|
while [ $# -gt 0 ]; do
|
|
927
1061
|
case "$1" in
|
|
928
1062
|
--quiet) quiet=1; shift ;;
|
|
929
1063
|
--force) force=1; shift ;; # ignore freshness/backoff (manual override)
|
|
1064
|
+
--json) json=1; quiet=1; shift ;; # refresh silently, then emit the report
|
|
930
1065
|
*) die "unknown option: $1" ;;
|
|
931
1066
|
esac
|
|
932
1067
|
done
|
|
@@ -935,14 +1070,20 @@ cmd_limits() {
|
|
|
935
1070
|
if ! mkdir "$lock" 2>/dev/null; then
|
|
936
1071
|
local age=$(( $(epoch_now) - $(file_mtime "$lock") ))
|
|
937
1072
|
if [ "$age" -lt 120 ]; then
|
|
1073
|
+
# A --json caller still gets the document (built from the state on disk) —
|
|
1074
|
+
# a machine-readable verb must never answer a concurrent run with silence.
|
|
1075
|
+
if [ "$json" = "1" ]; then emit_report_json limits; return $?; fi
|
|
938
1076
|
[ "$quiet" = "1" ] || echo "another limits refresh is running; skipping"
|
|
939
1077
|
return 0
|
|
940
1078
|
fi
|
|
941
1079
|
rm -rf "$lock"
|
|
942
|
-
mkdir "$lock" 2>/dev/null
|
|
1080
|
+
if ! mkdir "$lock" 2>/dev/null; then
|
|
1081
|
+
if [ "$json" = "1" ]; then emit_report_json limits; return $?; fi
|
|
1082
|
+
return 0
|
|
1083
|
+
fi
|
|
943
1084
|
fi
|
|
944
|
-
|
|
945
|
-
trap
|
|
1085
|
+
LIMITS_LOCK="$lock"
|
|
1086
|
+
trap limits_lock_release EXIT
|
|
946
1087
|
rotate_log limits.log
|
|
947
1088
|
# NB: expired OAuth access tokens are refreshed inside the Python below via the
|
|
948
1089
|
# refresh-token grant. (`claude auth status` was tried for this and does NOT
|
|
@@ -955,12 +1096,12 @@ cmd_limits() {
|
|
|
955
1096
|
[ "$threshold" -gt 90 ] && threshold=90
|
|
956
1097
|
[ "$threshold" -lt 1 ] && threshold=90
|
|
957
1098
|
"$PYBIN" - "$ACC_ROOT" "$threshold" "$quiet" "$USAGE_URL" "$force" <<'PYEOF' 2>>"$ACC_ROOT/limits.log"
|
|
958
|
-
import json, os, sys, time, urllib.request
|
|
1099
|
+
import hashlib, json, os, sys, time, urllib.request
|
|
959
1100
|
|
|
960
1101
|
root, threshold, quiet, url, force = sys.argv[1], int(sys.argv[2]), sys.argv[3] == '1', sys.argv[4], sys.argv[5] == '1'
|
|
961
1102
|
now = time.time()
|
|
962
1103
|
# Don't re-fetch an account whose data is younger than this (endpoint rate-limits).
|
|
963
|
-
MIN_FETCH_INTERVAL = int(os.environ.get('CLAUDE_MULTIACC_MIN_FETCH', '
|
|
1104
|
+
MIN_FETCH_INTERVAL = int(os.environ.get('CLAUDE_MULTIACC_MIN_FETCH', '240'))
|
|
964
1105
|
|
|
965
1106
|
# OAuth refresh-token grant — the same endpoint + public client id Claude Code
|
|
966
1107
|
# itself uses to keep .credentials.json alive. An account that sits idle past its
|
|
@@ -976,6 +1117,13 @@ CLIENT_ID = os.environ.get('CLAUDE_MULTIACC_CLIENT_ID',
|
|
|
976
1117
|
REFRESH_MIN_EXPIRED = 300
|
|
977
1118
|
REFRESH_FAIL_BACKOFF = 600 # transient (network/5xx/429): retry in 10 min
|
|
978
1119
|
REFRESH_DENIED_BACKOFF = 21600 # 4xx = grant likely revoked: 6h; re-login needed anyway
|
|
1120
|
+
# A usage fetch the server says will never succeed (x-should-retry: false — e.g. the
|
|
1121
|
+
# 403 "does not meet scope requirement user:profile" that a setup token ALWAYS gets)
|
|
1122
|
+
# is not a hiccup to retry every pass. Retrying it is what manufactured the 429s that
|
|
1123
|
+
# then hid the real cause for days, so a definitive refusal parks for 6h and says
|
|
1124
|
+
# exactly which ceremony fixes it.
|
|
1125
|
+
USAGE_DENIED_BACKOFF = 21600
|
|
1126
|
+
USAGE_FAIL_BACKOFF = 900 # anything else non-2xx: 15 min, doubling to 30
|
|
979
1127
|
|
|
980
1128
|
def say(msg):
|
|
981
1129
|
if not quiet:
|
|
@@ -997,6 +1145,15 @@ def parse_iso(s):
|
|
|
997
1145
|
# 4xx from the refresh endpoint), never for a transient network/5xx/429 hiccup.
|
|
998
1146
|
def mark_expired(d, slug, detail=''):
|
|
999
1147
|
mpath = os.path.join(d, '.expired')
|
|
1148
|
+
# An org-blocked marker is the strongest statement there is about an account and
|
|
1149
|
+
# only `verify` or a re-login may lift it (see clear_expired). Overwriting its
|
|
1150
|
+
# REASON with a weaker one is how it gets lifted by accident: the next successful
|
|
1151
|
+
# fetch sees a reason clear_expired is willing to drop, and the block vanishes.
|
|
1152
|
+
try:
|
|
1153
|
+
if 'reason=org-blocked' in open(mpath, errors='replace').read():
|
|
1154
|
+
return
|
|
1155
|
+
except OSError:
|
|
1156
|
+
pass
|
|
1000
1157
|
try:
|
|
1001
1158
|
with open(mpath + '.tmp', 'w') as f:
|
|
1002
1159
|
f.write(f'{int(now)}\n')
|
|
@@ -1028,6 +1185,39 @@ try:
|
|
|
1028
1185
|
except Exception as e:
|
|
1029
1186
|
sys.exit(f'cannot read manifest: {e}')
|
|
1030
1187
|
|
|
1188
|
+
def token_digest(path):
|
|
1189
|
+
"""Stable, non-secret identifier for a credential file's CONTENTS. Used to
|
|
1190
|
+
remember which setup token this endpoint refused, so a re-minted one still gets a
|
|
1191
|
+
try while the refused one is never spent again. Truncated: this only has to
|
|
1192
|
+
distinguish credentials, and a full hash of a secret is not something to write
|
|
1193
|
+
into a file that syncs between machines."""
|
|
1194
|
+
try:
|
|
1195
|
+
with open(path, 'rb') as f:
|
|
1196
|
+
return hashlib.sha256(f.read()).hexdigest()[:16]
|
|
1197
|
+
except OSError:
|
|
1198
|
+
return None
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
def park_dead_grant(aid, d, slug, detail):
|
|
1202
|
+
"""Park an account whose OAuth grant is dead — UNLESS it also has a portable setup
|
|
1203
|
+
token, in which case the grant being dead proves nothing about the account: the
|
|
1204
|
+
token still authenticates every real call. Parking it would take a perfectly
|
|
1205
|
+
working account out of the pool over a credential the pool does not need for work,
|
|
1206
|
+
only for telemetry. (The shim's auth_dead() checks the .expired marker BEFORE
|
|
1207
|
+
server.token, so a marker written here really would remove it.)"""
|
|
1208
|
+
try:
|
|
1209
|
+
portable = os.path.getsize(os.path.join(d, 'server.token')) > 0
|
|
1210
|
+
except OSError:
|
|
1211
|
+
portable = False
|
|
1212
|
+
if portable:
|
|
1213
|
+
say(f'{aid}: OAuth grant is dead ({slug}) but its setup token still works — '
|
|
1214
|
+
f'staying in the pool. TELEMETRY is dead for it until it has an OAuth login '
|
|
1215
|
+
f'here: claude-accounts login {aid}')
|
|
1216
|
+
return
|
|
1217
|
+
mark_expired(d, slug, detail)
|
|
1218
|
+
say(f'{aid}: {detail} — re-login needed (claude-accounts relogin {aid})')
|
|
1219
|
+
|
|
1220
|
+
|
|
1031
1221
|
def refresh_oauth(aid, d, cpath):
|
|
1032
1222
|
"""Refresh a long-expired OAuth access token via the refresh-token grant and
|
|
1033
1223
|
persist the ROTATED credential atomically (0600). Returns the new bearer, or
|
|
@@ -1044,20 +1234,22 @@ def refresh_oauth(aid, d, cpath):
|
|
|
1044
1234
|
return None
|
|
1045
1235
|
except Exception:
|
|
1046
1236
|
return None
|
|
1047
|
-
|
|
1048
|
-
|
|
1237
|
+
# NB: no "must have an accessToken" gate. A credential whose access token was
|
|
1238
|
+
# cleared but whose REFRESH token is alive is exactly the shape a grant is supposed
|
|
1239
|
+
# to recover from; requiring the dead half to be present meant such an account could
|
|
1240
|
+
# never come back, and (with a setup token beside it) went dark for telemetry
|
|
1241
|
+
# forever. The expiresAt gate below is what protects a live session's credential.
|
|
1049
1242
|
if not o.get('refreshToken'):
|
|
1050
|
-
|
|
1051
|
-
|
|
1243
|
+
park_dead_grant(aid, d, 'no-refresh-token',
|
|
1244
|
+
'credential has no refresh token and its access token expired')
|
|
1052
1245
|
return None
|
|
1053
1246
|
if o.get('expiresAt', 0) / 1000.0 > now - REFRESH_MIN_EXPIRED:
|
|
1054
1247
|
return None # not expired long enough to prove no live session owns it
|
|
1055
1248
|
if o.get('refreshTokenExpiresAt', 0) / 1000.0 <= now:
|
|
1056
1249
|
# Nothing can revive this account: park it so the shim stops selecting it
|
|
1057
1250
|
# (every run under it would fail with "OAuth session expired").
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
say(f'{aid}: refresh token expired — re-login needed (claude-accounts relogin {aid})')
|
|
1251
|
+
park_dead_grant(aid, d, 'refresh-token-expired',
|
|
1252
|
+
'the refresh token itself expired; only a re-login can fix it')
|
|
1061
1253
|
return None
|
|
1062
1254
|
if not force:
|
|
1063
1255
|
try:
|
|
@@ -1077,6 +1269,7 @@ def refresh_oauth(aid, d, cpath):
|
|
|
1077
1269
|
pass
|
|
1078
1270
|
say(f'{aid}: oauth refresh failed ({why}); backing off {wait}s; limits left as-is')
|
|
1079
1271
|
|
|
1272
|
+
started_with = o['refreshToken']
|
|
1080
1273
|
body = json.dumps({'grant_type': 'refresh_token',
|
|
1081
1274
|
'refresh_token': o['refreshToken'],
|
|
1082
1275
|
'client_id': CLIENT_ID}).encode()
|
|
@@ -1105,11 +1298,11 @@ def refresh_oauth(aid, d, cpath):
|
|
|
1105
1298
|
except Exception:
|
|
1106
1299
|
pass
|
|
1107
1300
|
if 'invalid_grant' in body:
|
|
1108
|
-
|
|
1109
|
-
|
|
1301
|
+
park_dead_grant(aid, d, f'refresh-denied-http-{e.code}',
|
|
1302
|
+
'the refresh grant was refused as invalid_grant (revoked or rotated away)')
|
|
1110
1303
|
elif denials >= 3:
|
|
1111
|
-
|
|
1112
|
-
|
|
1304
|
+
park_dead_grant(aid, d, f'refresh-denied-http-{e.code}',
|
|
1305
|
+
f'the refresh grant was refused {denials} times in a row')
|
|
1113
1306
|
back_off(REFRESH_DENIED_BACKOFF,
|
|
1114
1307
|
f'HTTP {e.code} — refresh token may be revoked; re-login needed',
|
|
1115
1308
|
denials=denials)
|
|
@@ -1137,10 +1330,26 @@ def refresh_oauth(aid, d, cpath):
|
|
|
1137
1330
|
if data.get('refresh_token_expires_in'):
|
|
1138
1331
|
o['refreshTokenExpiresAt'] = int((now + float(data['refresh_token_expires_in'])) * 1000)
|
|
1139
1332
|
doc['claudeAiOauth'] = o
|
|
1333
|
+
# CHECK-AND-SET. The grant rotates, and a live claude session refreshes the same
|
|
1334
|
+
# file. REFRESH_MIN_EXPIRED makes that unlikely, not impossible — and losing the
|
|
1335
|
+
# race by overwriting means the session's newer credential is destroyed. If the
|
|
1336
|
+
# on-disk refresh token is no longer the one this grant was issued against, the
|
|
1337
|
+
# other writer won: keep its result, discard ours.
|
|
1338
|
+
try:
|
|
1339
|
+
with open(cpath) as f:
|
|
1340
|
+
disk = json.load(f).get('claudeAiOauth', {})
|
|
1341
|
+
if isinstance(disk, dict) and disk.get('refreshToken') != started_with:
|
|
1342
|
+
say(f'{aid}: credential was refreshed by something else mid-flight — '
|
|
1343
|
+
f'keeping the newer one on disk')
|
|
1344
|
+
return None
|
|
1345
|
+
except Exception:
|
|
1346
|
+
pass
|
|
1140
1347
|
try:
|
|
1141
1348
|
fd = os.open(cpath + '.tmp', os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
1142
1349
|
with os.fdopen(fd, 'w') as f:
|
|
1143
1350
|
json.dump(doc, f)
|
|
1351
|
+
f.flush()
|
|
1352
|
+
os.fsync(f.fileno())
|
|
1144
1353
|
os.replace(cpath + '.tmp', cpath)
|
|
1145
1354
|
except Exception as e:
|
|
1146
1355
|
say(f'{aid}: token refreshed but credentials NOT persisted ({e}) — re-login may be needed')
|
|
@@ -1172,18 +1381,43 @@ for acct in manifest.get('accounts', []):
|
|
|
1172
1381
|
prev = json.load(open(lpath))
|
|
1173
1382
|
except Exception:
|
|
1174
1383
|
prev = {}
|
|
1384
|
+
# Valid JSON is not the same as a usable document: `[]`, `null` and `"broken"` all
|
|
1385
|
+
# parse, and every prev.get() below would then raise OUTSIDE the try — killing the
|
|
1386
|
+
# loop and starving every account after this one of telemetry. One corrupt file
|
|
1387
|
+
# must cost exactly one account.
|
|
1388
|
+
if not isinstance(prev, dict):
|
|
1389
|
+
prev = {}
|
|
1390
|
+
|
|
1391
|
+
def num(key, default=0):
|
|
1392
|
+
"""A field of the wrong type is the same as an absent one. limits.json is
|
|
1393
|
+
hand-editable, syncs between machines, and is written by more than one
|
|
1394
|
+
version of this tool at once."""
|
|
1395
|
+
v = prev.get(key, default)
|
|
1396
|
+
return v if isinstance(v, (int, float)) and not isinstance(v, bool) else default
|
|
1397
|
+
|
|
1175
1398
|
if not force:
|
|
1176
|
-
age = now -
|
|
1399
|
+
age = now - num('fetched_at')
|
|
1177
1400
|
if age < MIN_FETCH_INTERVAL:
|
|
1178
1401
|
continue
|
|
1179
|
-
retry_at =
|
|
1402
|
+
retry_at = num('retry_after')
|
|
1180
1403
|
if retry_at > now:
|
|
1181
|
-
|
|
1404
|
+
# Name the ACTUAL error. Reporting every park as a 429 is what let an
|
|
1405
|
+
# unauthorized account read as merely rate-limited for eleven days.
|
|
1406
|
+
why = prev.get('last_error')
|
|
1407
|
+
why = why if isinstance(why, str) and why else 'a failed fetch'
|
|
1408
|
+
say(f'{aid}: backing off after {why} ({int(retry_at - now)}s left); limits left as-is')
|
|
1182
1409
|
continue
|
|
1183
1410
|
|
|
1411
|
+
# BEARER ORDER MATTERS, and it is not the obvious one. A setup token authenticates
|
|
1412
|
+
# inference forever but is minted WITHOUT the user:profile scope this endpoint
|
|
1413
|
+
# requires, so it can only ever produce a 403 here. Trying it before the OAuth
|
|
1414
|
+
# refresh grant — which is what this did — killed telemetry for accounts whose
|
|
1415
|
+
# refresh token was still perfectly good, purely because a server.token sat beside
|
|
1416
|
+
# it. OAuth first, refresh second, token only as a genuine last resort.
|
|
1184
1417
|
bearer = None
|
|
1185
1418
|
source = None
|
|
1186
1419
|
cpath = os.path.join(d, '.credentials.json')
|
|
1420
|
+
tpath = os.path.join(d, 'server.token')
|
|
1187
1421
|
if os.path.isfile(cpath):
|
|
1188
1422
|
try:
|
|
1189
1423
|
c = json.load(open(cpath)).get('claudeAiOauth', {})
|
|
@@ -1191,11 +1425,6 @@ for acct in manifest.get('accounts', []):
|
|
|
1191
1425
|
bearer, source = c['accessToken'], 'oauth'
|
|
1192
1426
|
except Exception:
|
|
1193
1427
|
pass
|
|
1194
|
-
tpath = os.path.join(d, 'server.token')
|
|
1195
|
-
if not bearer and os.path.isfile(tpath):
|
|
1196
|
-
t = open(tpath).read().strip()
|
|
1197
|
-
if t:
|
|
1198
|
-
bearer, source = t, 'token'
|
|
1199
1428
|
if not bearer and os.path.isfile(cpath):
|
|
1200
1429
|
# Hard fail-open guard: NOTHING a single account's refresh does may abort
|
|
1201
1430
|
# the loop — every account after it would silently starve of telemetry.
|
|
@@ -1206,6 +1435,25 @@ for acct in manifest.get('accounts', []):
|
|
|
1206
1435
|
tok = None
|
|
1207
1436
|
if tok:
|
|
1208
1437
|
bearer, source = tok, 'oauth'
|
|
1438
|
+
if not bearer and os.path.isfile(tpath):
|
|
1439
|
+
# Once this endpoint has refused THIS token file for lacking a scope, asking
|
|
1440
|
+
# again is guaranteed to fail and only spends the account's hourly budget —
|
|
1441
|
+
# which is how a permanent authorization problem disguised itself as a rate
|
|
1442
|
+
# limit. Keyed on the file's mtime, so re-minting the token retries it.
|
|
1443
|
+
# Keyed on a non-secret digest of the token itself, not its mtime: `sync`
|
|
1444
|
+
# pushes tokens with rsync -a (mtimes preserved) and two different tokens can
|
|
1445
|
+
# land on the same whole second, either of which would skip a credential that
|
|
1446
|
+
# was never actually refused.
|
|
1447
|
+
denied_digest = prev.get('token_scope_denied')
|
|
1448
|
+
tdigest = token_digest(tpath)
|
|
1449
|
+
if not force and tdigest and denied_digest == tdigest:
|
|
1450
|
+
say(f'{aid}: setup token cannot read usage (no user:profile scope) and there '
|
|
1451
|
+
f'is no OAuth login here — telemetry stays dark until: '
|
|
1452
|
+
f'claude-accounts login {aid}')
|
|
1453
|
+
continue
|
|
1454
|
+
t = open(tpath).read().strip()
|
|
1455
|
+
if t:
|
|
1456
|
+
bearer, source = t, 'token'
|
|
1209
1457
|
if not bearer:
|
|
1210
1458
|
# Fail OPEN: no usable bearer => leave existing state; never block work on telemetry.
|
|
1211
1459
|
say(f'{aid}: no fresh bearer (expired oauth and/or no token); limits left as-is')
|
|
@@ -1221,6 +1469,24 @@ for acct in manifest.get('accounts', []):
|
|
|
1221
1469
|
resp = urllib.request.urlopen(req, timeout=15)
|
|
1222
1470
|
data = json.loads(resp.read().decode())
|
|
1223
1471
|
except urllib.error.HTTPError as e:
|
|
1472
|
+
body = ''
|
|
1473
|
+
try:
|
|
1474
|
+
body = e.read().decode('utf-8', 'replace')[:400]
|
|
1475
|
+
except Exception:
|
|
1476
|
+
pass
|
|
1477
|
+
|
|
1478
|
+
def park(wait, note):
|
|
1479
|
+
"""Record a failed fetch WITHOUT inventing freshness: fetched_at is left
|
|
1480
|
+
exactly as it was, so a parked account still reads as stale everywhere."""
|
|
1481
|
+
prev['retry_after'] = int(now + wait)
|
|
1482
|
+
prev['backoff'] = wait
|
|
1483
|
+
prev['last_error'] = note
|
|
1484
|
+
prev['last_error_at'] = int(now)
|
|
1485
|
+
tmp = lpath + '.tmp'
|
|
1486
|
+
with open(tmp, 'w') as f:
|
|
1487
|
+
json.dump(prev, f, indent=1)
|
|
1488
|
+
os.replace(tmp, lpath)
|
|
1489
|
+
|
|
1224
1490
|
if e.code == 429:
|
|
1225
1491
|
# Respect Retry-After; otherwise exponential backoff capped at 30 min.
|
|
1226
1492
|
try:
|
|
@@ -1228,19 +1494,69 @@ for acct in manifest.get('accounts', []):
|
|
|
1228
1494
|
except (TypeError, ValueError):
|
|
1229
1495
|
wait = 0
|
|
1230
1496
|
if wait <= 0:
|
|
1231
|
-
wait = min(1800, max(120, int(
|
|
1497
|
+
wait = min(1800, max(120, int(num('backoff', 60)) * 2))
|
|
1498
|
+
park(wait, f'HTTP 429 (rate limited, source={source})')
|
|
1499
|
+
say(f'{aid}: rate limited (429); backing off {wait}s; failing open')
|
|
1500
|
+
else:
|
|
1501
|
+
# EVERY non-2xx now backs off. Before this only a 429 did, so a permanent
|
|
1502
|
+
# refusal was re-issued on every scheduled pass from every machine in the
|
|
1503
|
+
# fleet — and those retries are what earned the 429s that made an
|
|
1504
|
+
# unauthorized account look merely rate-limited. Telemetry then sat frozen
|
|
1505
|
+
# while the shim ranked the whole pool NEUTRAL, i.e. picked at random.
|
|
1506
|
+
denied = (str(e.headers.get('x-should-retry') or '').lower() == 'false'
|
|
1507
|
+
or 'scope requirement' in body)
|
|
1508
|
+
# A setup token can NEVER grow the user:profile scope, so that refusal is
|
|
1509
|
+
# permanent and parks for hours. Any other "do not retry" may well be a
|
|
1510
|
+
# policy or endpoint issue someone fixes in minutes — park it for an hour,
|
|
1511
|
+
# not six, so recovery does not wait on a human running --force.
|
|
1512
|
+
scope_denied = denied and 'scope requirement' in body
|
|
1513
|
+
if scope_denied:
|
|
1514
|
+
wait = USAGE_DENIED_BACKOFF
|
|
1515
|
+
elif denied:
|
|
1516
|
+
wait = USAGE_DENIED_BACKOFF // 6
|
|
1517
|
+
else:
|
|
1518
|
+
wait = min(1800, max(USAGE_FAIL_BACKOFF,
|
|
1519
|
+
int(num('backoff', USAGE_FAIL_BACKOFF // 2)) * 2))
|
|
1520
|
+
if scope_denied and source == 'token':
|
|
1521
|
+
# Remember WHICH token was refused, so this credential is never spent
|
|
1522
|
+
# on this endpoint again — but a freshly minted one still gets a try.
|
|
1523
|
+
tdigest = token_digest(tpath)
|
|
1524
|
+
if tdigest:
|
|
1525
|
+
prev['token_scope_denied'] = tdigest
|
|
1526
|
+
park(wait, f'HTTP {e.code} (source={source})'
|
|
1527
|
+
+ (' — permanent, server said do not retry' if denied else ''))
|
|
1528
|
+
if scope_denied and source == 'token':
|
|
1529
|
+
# The exact shape of this outage: a setup token is minted WITHOUT the
|
|
1530
|
+
# user:profile scope the usage endpoint requires, so an account whose
|
|
1531
|
+
# OAuth grant lapsed keeps working for inference and goes permanently
|
|
1532
|
+
# dark for telemetry. Only a sign-in ON THIS MACHINE restores it.
|
|
1533
|
+
say(f'{aid}: usage endpoint refuses the setup token (HTTP {e.code} — a '
|
|
1534
|
+
f'setup token has no user:profile scope). Telemetry is DEAD for this '
|
|
1535
|
+
f'account until it has an OAuth login here: claude-accounts login {aid}. '
|
|
1536
|
+
f'Backing off {wait}s.')
|
|
1537
|
+
elif denied:
|
|
1538
|
+
say(f'{aid}: usage fetch refused for good (HTTP {e.code}, source={source}); '
|
|
1539
|
+
f'backing off {wait}s — re-login needed: claude-accounts login {aid}')
|
|
1540
|
+
else:
|
|
1541
|
+
say(f'{aid}: usage fetch failed (HTTP {e.code}); backing off {wait}s; failing open')
|
|
1542
|
+
continue
|
|
1543
|
+
except Exception as e:
|
|
1544
|
+
# Network/parse trouble is transient by nature, but it still must not be retried
|
|
1545
|
+
# every 5 minutes forever — that is how a fleet talks itself into a 429.
|
|
1546
|
+
wait = min(1800, max(USAGE_FAIL_BACKOFF,
|
|
1547
|
+
int(num('backoff', USAGE_FAIL_BACKOFF // 2)) * 2))
|
|
1548
|
+
try:
|
|
1232
1549
|
prev['retry_after'] = int(now + wait)
|
|
1233
1550
|
prev['backoff'] = wait
|
|
1551
|
+
prev['last_error'] = str(e)[:200]
|
|
1552
|
+
prev['last_error_at'] = int(now)
|
|
1234
1553
|
tmp = lpath + '.tmp'
|
|
1235
1554
|
with open(tmp, 'w') as f:
|
|
1236
1555
|
json.dump(prev, f, indent=1)
|
|
1237
1556
|
os.replace(tmp, lpath)
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
continue
|
|
1242
|
-
except Exception as e:
|
|
1243
|
-
say(f'{aid}: usage fetch failed ({e}); failing open')
|
|
1557
|
+
except Exception:
|
|
1558
|
+
pass
|
|
1559
|
+
say(f'{aid}: usage fetch failed ({e}); backing off {wait}s; failing open')
|
|
1244
1560
|
continue
|
|
1245
1561
|
|
|
1246
1562
|
def pct_of(v):
|
|
@@ -1326,8 +1642,21 @@ for acct in manifest.get('accounts', []):
|
|
|
1326
1642
|
session = [b['percent'] for b in buckets if b['group'] == 'session']
|
|
1327
1643
|
weeklyp = max(weekly) if weekly else maxp
|
|
1328
1644
|
sessionp = max(session) if session else 0
|
|
1645
|
+
# How long weekly_percent keeps meaning something. A weekly bucket only ever RISES
|
|
1646
|
+
# until its reset, so before that moment a stale percent is still a valid lower
|
|
1647
|
+
# bound and the shim can rank on it when nothing fresher exists; after it, the
|
|
1648
|
+
# number describes a week that is over and says nothing at all. Recording the
|
|
1649
|
+
# horizon here keeps the shim from having to parse buckets[] on every invocation.
|
|
1650
|
+
# It must come from the bucket weekly_percent actually CAME FROM: a low monthly
|
|
1651
|
+
# bucket resetting in an hour says nothing about an 80% weekly one that resets in
|
|
1652
|
+
# five days, and taking the minimum over all of them would throw the 80% away.
|
|
1653
|
+
wresets = [int(b['resets_epoch']) for b in buckets
|
|
1654
|
+
if b['group'] != 'session' and b['percent'] == weeklyp
|
|
1655
|
+
and isinstance(b.get('resets_epoch'), int)]
|
|
1329
1656
|
out = {'fetched_at': int(now), 'source': source, 'max_percent': maxp,
|
|
1330
|
-
'weekly_percent': weeklyp, 'session_percent': sessionp,
|
|
1657
|
+
'weekly_percent': weeklyp, 'session_percent': sessionp,
|
|
1658
|
+
'weekly_resets_epoch': min(wresets) if wresets else 0,
|
|
1659
|
+
'buckets': buckets}
|
|
1331
1660
|
tmp = lpath + '.tmp'
|
|
1332
1661
|
with open(tmp, 'w') as f:
|
|
1333
1662
|
json.dump(out, f, indent=1)
|
|
@@ -1350,13 +1679,21 @@ for acct in manifest.get('accounts', []):
|
|
|
1350
1679
|
say(f"{aid}: LIMITED {worst['name']} at {worst['percent']}% (resets {worst['resets_at']})")
|
|
1351
1680
|
else:
|
|
1352
1681
|
if os.path.exists(mpath):
|
|
1353
|
-
# A shim-written
|
|
1354
|
-
#
|
|
1682
|
+
# A shim-written marker outlives a clean limits pass while its own window
|
|
1683
|
+
# is still open:
|
|
1684
|
+
# error-cooldown — the account failed a real call moments ago.
|
|
1685
|
+
# client-rate-limit — Claude Code itself was REJECTED on this account and
|
|
1686
|
+
# recorded the reset the API handed it. That is
|
|
1687
|
+
# first-hand evidence; a usage payload that disagrees
|
|
1688
|
+
# (different bucket set, cached edge) must not unpark
|
|
1689
|
+
# the account early and send work straight back into
|
|
1690
|
+
# the 429.
|
|
1355
1691
|
keep = False
|
|
1356
1692
|
try:
|
|
1357
1693
|
txt = open(mpath).read()
|
|
1358
1694
|
first = txt.splitlines()[0] if txt else ''
|
|
1359
|
-
if 'reason=error-cooldown' in txt
|
|
1695
|
+
if ('reason=error-cooldown' in txt or 'reason=client-rate-limit' in txt) \
|
|
1696
|
+
and first.isdigit() and int(first) > now:
|
|
1360
1697
|
keep = True
|
|
1361
1698
|
except Exception:
|
|
1362
1699
|
pass
|
|
@@ -1367,6 +1704,18 @@ for acct in manifest.get('accounts', []):
|
|
|
1367
1704
|
detail = ' '.join(f"{b['name']}={b['percent']}%" for b in buckets)
|
|
1368
1705
|
print(f'{aid}: ok {detail}')
|
|
1369
1706
|
PYEOF
|
|
1707
|
+
local refresh_rc=$?
|
|
1708
|
+
# Release the lock BEFORE the report: a --json caller must not hold the refresh
|
|
1709
|
+
# lock while a consumer reads its output.
|
|
1710
|
+
limits_lock_release
|
|
1711
|
+
trap - EXIT
|
|
1712
|
+
# The refresher fails open per account; a NON-zero status means the pass itself
|
|
1713
|
+
# broke (unreadable manifest, dead python). Report the state anyway — stale data
|
|
1714
|
+
# beats silence — but hand the caller the failure, exactly as before --json existed.
|
|
1715
|
+
if [ "$json" = "1" ]; then
|
|
1716
|
+
emit_report_json limits || return $?
|
|
1717
|
+
fi
|
|
1718
|
+
return "$refresh_rc"
|
|
1370
1719
|
}
|
|
1371
1720
|
|
|
1372
1721
|
cmd_verify() {
|
|
@@ -1402,6 +1751,14 @@ def mark_expired(d, slug, detail=''):
|
|
|
1402
1751
|
"""Park an account the shim must stop selecting. Verify is the strongest signal
|
|
1403
1752
|
there is — a real inference call that came back 'not authenticated'."""
|
|
1404
1753
|
mpath = os.path.join(d, '.expired')
|
|
1754
|
+
# Same rule as the limits path: an org block outranks every other reason and only
|
|
1755
|
+
# a PASSING verify or a re-login may lift it. Rewriting its reason with a weaker
|
|
1756
|
+
# one is how it gets lifted by accident later.
|
|
1757
|
+
try:
|
|
1758
|
+
if 'reason=org-blocked' in open(mpath, errors='replace').read():
|
|
1759
|
+
return
|
|
1760
|
+
except OSError:
|
|
1761
|
+
pass
|
|
1405
1762
|
try:
|
|
1406
1763
|
with open(mpath + '.tmp', 'w') as f:
|
|
1407
1764
|
f.write(f'{int(time.time())}\n')
|
|
@@ -1606,17 +1963,45 @@ sync_is_replica() {
|
|
|
1606
1963
|
|
|
1607
1964
|
cmd_sync() {
|
|
1608
1965
|
require_manifest
|
|
1966
|
+
local allow_empty=0 no_server=0
|
|
1967
|
+
while [ $# -gt 0 ]; do
|
|
1968
|
+
case "$1" in
|
|
1969
|
+
--allow-empty) allow_empty=1; shift ;;
|
|
1970
|
+
--no-server) no_server=1; shift ;; # this run pushes nowhere, whatever the manifest says
|
|
1971
|
+
*) die "unknown option: $1 (usage: claude-accounts sync [--no-server] [--allow-empty])" ;;
|
|
1972
|
+
esac
|
|
1973
|
+
done
|
|
1974
|
+
local server sroot srepo
|
|
1975
|
+
server="$(sync_target)"
|
|
1976
|
+
sroot="$(sync_target_root)"
|
|
1977
|
+
srepo="$(sync_target_repo)"
|
|
1978
|
+
# LOCAL-ONLY: no ssh target at all, because a panel/runner daemon distributes this
|
|
1979
|
+
# pool. Validate + fix up locally and stop — the same verb keeps working, it simply
|
|
1980
|
+
# has nowhere to push. (Not Mac-gated: a local pool is legitimate on any host.)
|
|
1981
|
+
# This mode NARROWS the replica rule, it never widens it: a replica pushes nothing,
|
|
1982
|
+
# and a local-only pool pushes nothing whether or not it is a replica. The marker is
|
|
1983
|
+
# still honored and still reported, so a replica can never start pushing by having
|
|
1984
|
+
# its sync target changed.
|
|
1985
|
+
if [ "$no_server" = "1" ] || sync_target_is_local "$server"; then
|
|
1986
|
+
rotate_log sync.log
|
|
1987
|
+
manifest_well_formed || { log_to sync.log "FAIL: manifest malformed (local sync)"; \
|
|
1988
|
+
die "manifest is not valid JSON or has no well-formed accounts — fix $MANIFEST"; }
|
|
1989
|
+
local role="source"
|
|
1990
|
+
sync_is_replica && role="replica"
|
|
1991
|
+
log_to sync.log "sync (local-only, role=$role): no server target${no_server:+ (--no-server)}"
|
|
1992
|
+
local_pool_fixup
|
|
1993
|
+
if [ "$role" = "replica" ]; then
|
|
1994
|
+
echo "sync ok (local-only, and this pool is a sync replica — nothing pushed either way)"
|
|
1995
|
+
else
|
|
1996
|
+
echo "sync ok (local-only: pool at $ACC_ROOT validated and re-seeded; nothing pushed)"
|
|
1997
|
+
fi
|
|
1998
|
+
return 0
|
|
1999
|
+
fi
|
|
1609
2000
|
[ "$(machine_kind)" = "mac" ] || die "sync runs on the Mac (source of truth), not the server"
|
|
1610
2001
|
if sync_is_replica; then
|
|
1611
2002
|
echo "this pool is a sync replica — the source machine pushes here; nothing sent"
|
|
1612
2003
|
return 0
|
|
1613
2004
|
fi
|
|
1614
|
-
local allow_empty=0
|
|
1615
|
-
[ "${1:-}" = "--allow-empty" ] && allow_empty=1
|
|
1616
|
-
local server sroot srepo
|
|
1617
|
-
server="$(manifest_get server "$DEFAULT_SERVER")"
|
|
1618
|
-
sroot="$(manifest_get server_root "$DEFAULT_SERVER_ROOT")"
|
|
1619
|
-
srepo="$(manifest_get server_repo "$DEFAULT_SERVER_REPO")"
|
|
1620
2005
|
# These land inside remote shell commands — anything but a plain target/path is a
|
|
1621
2006
|
# command-injection vector from a corrupted or hand-edited manifest. EVERY target
|
|
1622
2007
|
# (primary and peers) is validated before anything is pushed anywhere.
|
|
@@ -1648,17 +2033,8 @@ EOF
|
|
|
1648
2033
|
# never be pushed (it would blank the target pools), and must never make the removal
|
|
1649
2034
|
# propagation wipe a target's credentials. Emptying the pool on purpose is possible
|
|
1650
2035
|
# via `sync --allow-empty`, so this can never happen by accident.
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
doc = json.load(open(sys.argv[1]))
|
|
1654
|
-
accounts = doc.get('accounts')
|
|
1655
|
-
if not isinstance(accounts, list):
|
|
1656
|
-
sys.exit(1)
|
|
1657
|
-
for a in accounts:
|
|
1658
|
-
if not isinstance(a, dict) or not re.fullmatch(r'acct-\d{2}', str(a.get('id', ''))) \
|
|
1659
|
-
or not str(a.get('email', '')).strip():
|
|
1660
|
-
sys.exit(1)
|
|
1661
|
-
PYEOF
|
|
2036
|
+
manifest_well_formed \
|
|
2037
|
+
|| fail "manifest is not valid JSON or has no well-formed accounts — refusing to sync"
|
|
1662
2038
|
if [ -z "$(account_ids)" ] && [ "$allow_empty" != "1" ]; then
|
|
1663
2039
|
fail "manifest has zero accounts — refusing to blank the target pools (use 'sync --allow-empty' if that is really intended)"
|
|
1664
2040
|
fi
|
|
@@ -1685,13 +2061,7 @@ EOF
|
|
|
1685
2061
|
|
|
1686
2062
|
cmd_post_sync() {
|
|
1687
2063
|
require_manifest
|
|
1688
|
-
|
|
1689
|
-
for id in $(account_ids); do
|
|
1690
|
-
d="$ACC_ROOT/$id"
|
|
1691
|
-
seed_account_dir "$d"
|
|
1692
|
-
[ -f "$d/server.token" ] && chmod 600 "$d/server.token" 2>/dev/null
|
|
1693
|
-
[ -f "$d/.credentials.json" ] && chmod 600 "$d/.credentials.json" 2>/dev/null
|
|
1694
|
-
done
|
|
2064
|
+
local_pool_fixup
|
|
1695
2065
|
log_to sync.log "post-sync: seeded $(account_ids | wc -l | tr -d ' ') account dirs"
|
|
1696
2066
|
( cmd_limits --quiet ) || true # subshell: release the limits lock before verify
|
|
1697
2067
|
cmd_verify --quick
|
|
@@ -1761,6 +2131,8 @@ case "${1:-help}" in
|
|
|
1761
2131
|
status) shift; cmd_status "$@" ;;
|
|
1762
2132
|
add) shift; cmd_add "$@" ;;
|
|
1763
2133
|
import) shift; cmd_import "$@" ;;
|
|
2134
|
+
export-credential|export-cred) shift; cmd_export_credential "$@" ;;
|
|
2135
|
+
import-credential|import-cred) shift; cmd_import_credential "$@" ;;
|
|
1764
2136
|
adopt) shift; cmd_adopt "$@" ;;
|
|
1765
2137
|
dedupe) shift; cmd_dedupe "$@" ;;
|
|
1766
2138
|
remove) shift; cmd_remove "$@" ;;
|