@crvouga/dev-fleet 0.2.0 → 0.4.0

package/cliproxy/scripts/accounts.py

@@ -36,3 +36,219 @@ def pending(auths):
         if wanted and data.get('prefix') != wanted:
             result.append((path.name, wanted))
     return result
+
+
+# ── Account state ────────────────────────────────────────────────────────────
+#
+# One state per credential, from its file (never a token, only whether one is
+# there and when the access token expires) and from the proxy's management
+# API, which knows about refreshes, cooldowns and disabled credentials.
+
+STATES = ('usable', 'cooling-down', 'needs-login', 'refresh-failing', 'removed')
+
+# A refresh the provider refused for good: only a new sign-in helps.
+REJECTED = re.compile(r'invalid_grant|invalid_token|unauthorized|revoked|refresh token (?:is )?(?:invalid|expired|rejected|reused)', re.I)
+
+
+def _instant(value):
+    """Seconds since the epoch from an ISO 8601 string or a number, or None."""
+    import datetime
+    if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0:
+        return float(value) / (1000 if value > 1e11 else 1)
+    if isinstance(value, str) and value and not value.startswith('0001-'):
+        try:
+            parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00'))
+        except ValueError:
+            return None
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=datetime.timezone.utc)
+        return parsed.timestamp()
+    return None
+
+
+def iso(seconds):
+    import datetime
+    return datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def classify(credential, entry, now):
+    """(state, until): `credential` is the auth file, `entry` what the
+    management API says about it (None when it does not list it). `until` is
+    when a cooldown clears, as epoch seconds, or None."""
+    entry = entry or {}
+    if entry.get('disabled') is True or credential.get('disabled') is True:
+        return 'removed', None
+    message = str(entry.get('status_message') or entry.get('last_error') or '')
+    expires = _instant(credential.get('expired') or credential.get('expiry'))
+    expired = expires is not None and expires <= now
+    has_refresh = bool(credential.get('refresh_token'))
+    if REJECTED.search(message) or (expired and not has_refresh):
+        return 'needs-login', None
+    if entry.get('status') == 'error' and 'refresh' in message.lower():
+        return ('needs-login' if expired else 'refresh-failing'), None
+    quota = entry.get('quota') if isinstance(entry.get('quota'), dict) else {}
+    until = max(
+        [t for t in (_instant(entry.get('next_retry_after')),
+                     _instant(quota.get('next_recover_at'))) if t is not None and t > now]
+        or [0]
+    )
+    if until or entry.get('unavailable') is True or quota.get('exceeded') is True:
+        return 'cooling-down', (until or None)
+    return 'usable', None
+
+
+def credentials(auths):
+    """[(file name, credential)] for every readable credential, in a stable
+    order: by provider, then sign-in email, then file name. Each provider's
+    accounts are numbered in this order, the same order every node lists
+    them in (src/agents/accounts.ts), so an index means one account
+    everywhere."""
+    found = []
+    for path in pathlib.Path(auths).glob('*.json'):
+        try:
+            data = json.loads(path.read_text())
+        except (OSError, ValueError):
+            continue
+        if isinstance(data, dict) and isinstance(data.get('type'), str):
+            found.append((path.name, data))
+    email = lambda data: data.get('email') if isinstance(data.get('email'), str) else ''
+    return sorted(found, key=lambda item: (item[1]['type'], email(item[1]), item[0]))
+
+
+def roster(auths, entries=(), now=None):
+    """Every provider account: provider, opaque index (1, 2, … per provider),
+    label, state and, while cooling down, when it clears. `entries` is the
+    management API's list of auth files. Never a token."""
+    import time
+    now = time.time() if now is None else now
+    by_name = {e.get('name'): e for e in entries if isinstance(e, dict)}
+    counters = {}
+    result = []
+    for name, credential in credentials(auths):
+        provider = credential['type']
+        counters[provider] = counters.get(provider, 0) + 1
+        state, until = classify(credential, by_name.get(name), now)
+        email = credential.get('email')
+        result.append({
+            'provider': provider,
+            'index': counters[provider],
+            'label': email if isinstance(email, str) else provider + '-' + str(counters[provider]),
+            'prefix': prefix(provider, email),
+            'state': state,
+            'until': iso(until) if until else None,
+            'file': name,
+        })
+    return result
+
+
+def version(accounts):
+    """The roster version: a digest of the non-secret metadata of the accounts
+    every node offers (their routing prefixes). Every node computes the same
+    digest from the proxy's model list (src/accounts/roster.ts)."""
+    import hashlib
+    prefixes = sorted({a['prefix'] for a in accounts if a.get('prefix') and a.get('state') != 'removed'})
+    return hashlib.sha256('\n'.join(prefixes).encode()).hexdigest()[:16]
+
+
+def public(accounts):
+    """The roster as reported: no file name."""
+    return [{k: v for k, v in a.items() if k != 'file'} for a in accounts]
+
+
+def find(accounts, provider, index):
+    """The one account `provider` `index` names, or None."""
+    for account in accounts:
+        if account['provider'] == provider and account['index'] == index:
+            return account
+    return None
+
+
+# ── Sign-in, re-login and removal ────────────────────────────────────────────
+#
+# A login writes into a staging directory, never into the auth directory. Only
+# a complete credential (readable JSON with a type) is moved in, with one
+# rename, so an interrupted login leaves no half-written auth file.
+
+def staging(auths):
+    return pathlib.Path(auths).parent / 'auths-staging'
+
+
+def begin(auths):
+    """A fresh, empty staging directory; leftovers of an interrupted login are
+    discarded."""
+    import shutil
+    stage = staging(auths)
+    shutil.rmtree(stage, ignore_errors=True)
+    stage.mkdir(mode=0o700, parents=True)
+    return stage
+
+
+def adopt(stage, auths, replace=None):
+    """Moves every complete credential from `stage` into `auths` and returns
+    their file names. `replace` (a file name) is the credential a re-login
+    signs in again: it is replaced by the new one in the same rename. The
+    staging directory is removed whatever happens."""
+    import os
+    import shutil
+    stage, auths = pathlib.Path(stage), pathlib.Path(auths)
+    adopted = []
+    try:
+        complete = []
+        for path in sorted(stage.glob('*.json')):
+            try:
+                data = json.loads(path.read_text())
+            except (OSError, ValueError):
+                continue
+            if isinstance(data, dict) and isinstance(data.get('type'), str):
+                complete.append(path)
+        if replace and len(complete) != 1:
+            raise ValueError('A re-login must produce exactly one credential')
+        for path in complete:
+            target = auths / (replace or path.name)
+            os.chmod(path, 0o600)
+            os.replace(path, target)
+            adopted.append(target.name)
+        if replace and adopted and adopted[0] != replace:
+            (auths / replace).unlink(missing_ok=True)
+        return adopted
+    finally:
+        shutil.rmtree(stage, ignore_errors=True)
+
+
+def remove(auths, provider, index):
+    """Deletes the credential of one account; returns its file name."""
+    account = find(roster(auths), provider, index)
+    if account is None:
+        raise ValueError('No such account')
+    (pathlib.Path(auths) / account['file']).unlink()
+    return account['file']
+
+
+# ── Refresh tokens across checkpoints ────────────────────────────────────────
+#
+# Providers rotate refresh tokens. Each checkpoint records a short digest of
+# every credential's refresh token (never the token). After a restore, a
+# credential whose digest differs from the newest checkpoint's carries a
+# token the provider has already replaced: that account needs a new login.
+
+def token_digests(auths):
+    import hashlib
+    digests = {}
+    for name, credential in credentials(auths):
+        token = credential.get('refresh_token')
+        if isinstance(token, str) and token:
+            digests[name] = hashlib.sha256(token.encode()).hexdigest()[:16]
+    return digests
+
+
+def stale_after_restore(auths, newest):
+    """[(provider, index)] of the restored accounts whose refresh token is
+    older than the one `newest` (the token digests of the newest checkpoint)
+    records."""
+    current = token_digests(auths)
+    stale = []
+    for account in roster(auths):
+        name = account['file']
+        if name in newest and name in current and current[name] != newest[name]:
+            stale.append((account['provider'], account['index']))
+    return stale

package/cliproxy/scripts/checkpoint.py

@@ -68,7 +68,7 @@ def validate(root):
         raise ValueError("Provider cache directory missing")
     for path in (state / "runtime/auths").iterdir():
         if path.name == "kv.db" and path.is_file() and path.stat().st_size > 0:
-            continue  # Opaque upstream bbolt state, captured while the process is stopped.
+            continue  # Opaque upstream bbolt state, copied while the proxy runs (see capture).
         # .json: provider credential; .cds: upstream per-account cooldown
         # records (JSON object, no credential), kept so limits survive moves.
         if path.suffix not in (".json", ".cds") or not path.is_file():
@@ -127,11 +127,39 @@ def copy_state(root, target):
             shutil.copyfile(source, destination)
     shutil.copyfile(root / "INSTALLATION.json", target / "INSTALLATION.json")
     (target / "CHECKPOINT.json").write_text(
-        json.dumps({"version": 1, "files": hashes(target)}) + "\n"
+        json.dumps(
+            {
+                "version": 1,
+                "files": hashes(target),
+                "tokens": _accounts().token_digests(target / "state/runtime/auths"),
+            }
+        )
+        + "\n"
     )
     validate(target)
 
 
+def _accounts():
+    """accounts.py beside this file, however this module was loaded."""
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location(
+        "accounts", pathlib.Path(__file__).resolve().parent / "accounts.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def stale_accounts(restored, newest):
+    """[(provider, index)] of the accounts in the restored checkpoint
+    `restored` whose refresh token the provider has since replaced, judged
+    against the manifest of the newest checkpoint `newest`. Restoring the
+    newest checkpoint itself finds none."""
+    tokens = json.loads((newest / "CHECKPOINT.json").read_text()).get("tokens") or {}
+    return _accounts().stale_after_restore(restored / "state/runtime/auths", tokens)
+
+
 def capture(root, target, attempts=6, sleep=time.sleep):
     """Copies the state of the running stack without ever stopping it: agents
     use the proxy around the clock, and a stopped connector fails every one of

package/cliproxy/scripts/cli.py

@@ -21,7 +21,7 @@ root = cliproxy_host.default_root().resolve()
 args = sys.argv[1:]
 if not args or args[0] in ["help", "--help", "-h"]:
     print(
-        "Usage: devfleet cliproxy COMMAND\nCommands: install [--revision REV|--rollback|--dry-run|--prepare-only], audit, public-endpoint [enable|start|disable|verify], status, start, stop, restart, health, models, logs, validate, apply FILE, add-account PROVIDER, inference PROTOCOL MODEL, model-health [--profile liveness|readiness|light|nightly|manual] [--suite core|cursor|all] [--target live|mock], model-suite (same flags; defaults to nightly), backup, gc, snapshots, check, check-full, restore SNAPSHOT EMPTY_DIRECTORY, restore-cache --previous-node-stopped, vault-login, recover, recover-node NEW_DIRECTORY [--vault-token-file FILE], verify-recovery, portable-login, activate, drain"
+        "Usage: devfleet cliproxy COMMAND\nCommands: install [--revision REV|--rollback|--dry-run|--prepare-only], audit, public-endpoint [enable|start|disable|verify], status, start, stop, restart, health, models, logs, validate, apply FILE, add-account PROVIDER, relogin PROVIDER INDEX, remove-account PROVIDER INDEX, accounts [--json], client-keys, issue-key|rotate-key|retire-key|revoke-key NODE, inference PROTOCOL MODEL, model-health [--profile liveness|readiness|light|nightly|manual] [--suite core|cursor|all] [--target live|mock], model-suite (same flags; defaults to nightly), backup, gc, snapshots, check, check-full, restore SNAPSHOT EMPTY_DIRECTORY, restore-cache --previous-node-stopped, vault-login, recover, recover-node NEW_DIRECTORY [--vault-token-file FILE], verify-recovery, portable-login, activate, drain"
     )
     sys.exit(0)
 command, *rest = args

package/cliproxy/scripts/clientkeys.py

@@ -0,0 +1,99 @@
+"""One proxy client key per node, rotated with overlap and revoked with the node.
+
+The key ring is a private JSON document in the proxy's secrets
+(`state/secrets/client-keys.json`): per node, the current key and, during a
+rotation, the previous one. The proxy accepts every key in the ring
+(`api_keys`), so a rotation adds the new key, every node moves to it, and
+only then is the old key retired: no request fails in between. Revoking a
+node removes its keys and leaves every other node working. Reports name keys
+only by a short fingerprint, never by value.
+"""
+import hashlib
+import json
+import os
+import pathlib
+import re
+import secrets
+
+NODE = re.compile(r'^[a-z][a-z0-9-]{0,62}$')
+
+
+def fingerprint(key):
+    return hashlib.sha256(key.encode()).hexdigest()[:12]
+
+
+def load(path):
+    try:
+        ring = json.loads(pathlib.Path(path).read_text())
+    except (OSError, ValueError):
+        return {'version': 1, 'nodes': {}}
+    if not isinstance(ring, dict) or ring.get('version') != 1:
+        raise ValueError('Unsupported client key ring')
+    return ring
+
+
+def save(path, ring):
+    path = pathlib.Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    staged = path.with_suffix('.new')
+    staged.write_text(json.dumps(ring, indent=2, sort_keys=True) + '\n')
+    os.chmod(staged, 0o600)
+    os.replace(staged, path)
+
+
+def _node(ring, node):
+    if not NODE.match(node):
+        raise ValueError('Invalid node id')
+    return ring['nodes'].setdefault(node, {'current': None, 'previous': None})
+
+
+def issue(ring, node, generate=lambda: secrets.token_urlsafe(32)):
+    """Gives `node` a key when it has none; returns its current key."""
+    entry = _node(ring, node)
+    if not entry['current']:
+        entry['current'] = generate()
+    return entry['current']
+
+
+def rotate(ring, node, generate=lambda: secrets.token_urlsafe(32)):
+    """Starts a rotation: a new current key, the old one kept as previous
+    until `retire`. Both are accepted meanwhile."""
+    entry = _node(ring, node)
+    if entry['previous']:
+        raise ValueError('A rotation of this node is already in progress; retire it first')
+    entry['previous'], entry['current'] = entry['current'], generate()
+    return entry['current']
+
+
+def retire(ring, node):
+    """Ends a rotation once the node uses its new key."""
+    _node(ring, node)['previous'] = None
+
+
+def revoke(ring, node):
+    """Removes every key of `node`."""
+    ring['nodes'].pop(node, None)
+
+
+def api_keys(ring, shared=()):
+    """Every key the proxy accepts: the ring's, plus `shared` keys still in
+    use (the fleet-wide key, until each node has its own)."""
+    keys = list(shared)
+    for node in sorted(ring['nodes']):
+        entry = ring['nodes'][node]
+        for key in (entry['current'], entry['previous']):
+            if key and key not in keys:
+                keys.append(key)
+    return keys
+
+
+def report(ring):
+    """Per node: the current key's fingerprint and whether a rotation is
+    still waiting to be retired. No key value."""
+    return {
+        node: {
+            'current': fingerprint(entry['current']) if entry['current'] else None,
+            'rotating': bool(entry['previous']),
+        }
+        for node, entry in sorted(ring['nodes'].items())
+    }

package/cliproxy/scripts/lease.py

@@ -0,0 +1,80 @@
+"""A fencing lease on the proxy state: only its holder is the writer.
+
+Providers rotate refresh tokens, so two proxies started from the same
+checkpoint spend each other's tokens. The lease is one small JSON document in
+a store both would see (the checkpoint bucket in production, a directory in
+tests): the holder, a fencing epoch that only ever grows, and when it
+expires. A proxy that cannot take the lease refuses to start as writer.
+"""
+import json
+import os
+import pathlib
+import time
+
+TTL = 120
+
+
+class Held(Exception):
+    """Another proxy holds the lease."""
+
+    def __init__(self, holder, expires):
+        super().__init__('The proxy state lease is held by another proxy')
+        self.holder = holder
+        self.expires = expires
+
+
+class DirectoryStore:
+    """A lease document in a directory, replaced with one rename. `put`
+    writes only when the document is still the one `get` returned."""
+
+    def __init__(self, directory):
+        self.path = pathlib.Path(directory) / 'proxy-lease.json'
+
+    def get(self):
+        try:
+            return json.loads(self.path.read_text())
+        except (OSError, ValueError):
+            return None
+
+    def put(self, document, expected):
+        import fcntl
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.path.with_suffix('.lock'), 'w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            if self.get() != expected:
+                return False
+            staged = self.path.with_suffix('.new')
+            staged.write_text(json.dumps(document) + '\n')
+            os.replace(staged, self.path)
+            return True
+
+
+def acquire(store, holder, now=None, ttl=TTL):
+    """Takes or renews the lease for `holder`; returns its fencing epoch.
+    Raises Held while another holder's lease has not expired. Taking over an
+    expired lease increases the epoch, so the old holder's writes are fenced."""
+    now = time.time() if now is None else now
+    current = store.get()
+    if current and current.get('holder') != holder and current.get('expires', 0) > now:
+        raise Held(current.get('holder'), current.get('expires'))
+    epoch = (current or {}).get('epoch', 0)
+    if not current or current.get('holder') != holder:
+        epoch += 1
+    document = {'holder': holder, 'epoch': epoch, 'expires': now + ttl}
+    if not store.put(document, current):
+        raise Held(None, None)
+    return epoch
+
+
+def valid(store, holder, epoch, now=None):
+    """Whether `holder` still holds the lease at `epoch`: checked before a
+    token refresh is written."""
+    now = time.time() if now is None else now
+    current = store.get() or {}
+    return current.get('holder') == holder and current.get('epoch') == epoch and current.get('expires', 0) > now
+
+
+def release(store, holder):
+    current = store.get()
+    if current and current.get('holder') == holder:
+        store.put({**current, 'expires': 0}, current)

package/cliproxy/scripts/ops

@@ -8,6 +8,7 @@ import pathlib as _pathlib
 sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
 import accounts
 import cliproxy_host
+import clientkeys
 
 import fcntl, json, os, pathlib, shutil, subprocess, sys, tempfile, time, urllib.request, urllib.error
 
@@ -205,6 +206,92 @@ def apply_prefixes(prefixes):
     sys.exit("The proxy did not record the declared account prefixes.")
 
 
+LOGIN_FLAGS = {
+    "codex": "-codex-device-login",
+    "claude": "-claude-login",
+    "antigravity": "-antigravity-login",
+    "kimi": "-kimi-login",
+    "xai": "-xai-login",
+    "meta": "-meta-login",
+    "devin": "-devin-login",
+}
+
+
+def login(provider, replace=None):
+    """Runs the proxy's own login into a staging directory, then moves the
+    complete credential in with one rename: an interrupted login leaves no
+    half-written auth file. `replace` is the credential a re-login renews."""
+    import yaml
+
+    if provider not in LOGIN_FLAGS:
+        sys.exit(
+            "This release has no login flag for that provider. Use the official dashboard OAuth/API key configuration."
+        )
+    auths = root / "state/runtime/auths"
+    stage = accounts.begin(auths)
+    config = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    config.setdefault("oauth", {})["auth-dir"] = "/state/" + stage.name
+    login_config = root / "state/runtime/login-config.yaml"
+    login_config.write_text(yaml.safe_dump(config))
+    try:
+        compose(
+            "exec",
+            "proxy",
+            "/CLIProxyAPI/CLIProxyAPI",
+            "-config",
+            "/state/" + login_config.name,
+            LOGIN_FLAGS[provider],
+            "-no-browser",
+        )
+        if not accounts.adopt(stage, auths, replace=replace):
+            sys.exit("The login wrote no credential; nothing was changed.")
+    finally:
+        login_config.unlink(missing_ok=True)
+        shutil.rmtree(stage, ignore_errors=True)
+
+
+def account_roster():
+    """The roster with each account's state, from the credential files and
+    the management API (which knows cooldowns and refresh failures)."""
+    entries = []
+    try:
+        admin = (root / "state/secrets/management-key").read_text().strip()
+        code, body = request("/v0/management/auth-files", admin)
+        if code == 200:
+            entries = json.loads(body).get("files") or []
+    except Exception:
+        entries = []
+    roster = accounts.roster(root / "state/runtime/auths", entries)
+    return {
+        "version": 1,
+        "rosterVersion": accounts.version(roster),
+        "managementApi": bool(entries),
+        "accounts": accounts.public(roster),
+    }
+
+
+def key_ring():
+    return clientkeys.load(root / "state/secrets/client-keys.json")
+
+
+def apply_keys(ring):
+    """The proxy accepts exactly the shared key and the ring's keys."""
+    import yaml
+
+    live = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    shared = (root / "state/secrets/client-key").read_text().strip()
+    live.setdefault("access", {})["api-keys"] = clientkeys.api_keys(ring, [shared])
+    with tempfile.NamedTemporaryFile(
+        "w", dir=root / "state/runtime", suffix=".yaml", delete=False
+    ) as file:
+        file.write(yaml.safe_dump(live, sort_keys=False))
+    try:
+        os.chmod(file.name, 0o600)
+        apply_config(pathlib.Path(file.name), restart=False)
+    finally:
+        pathlib.Path(file.name).unlink(missing_ok=True)
+
+
 def lock():
     f = open(root / "state/status/operation.lock", "w")
     if os.geteuid() == 0:
@@ -414,33 +501,58 @@ try:
         sys.exit(
             "Install a reviewed Git revision with bun run cliproxy install --revision REV; use --rollback for the recorded previous revision."
         )
-    elif command == "add-account":
+    elif command in ["add-account", "relogin"]:
+        # `add-account PROVIDER` signs in a new account; `relogin PROVIDER
+        # INDEX` signs that one account in again. Prefixes follow at once.
         operation_lock = lock()
         provider = sys.argv[2]
-        flags = {
-            "codex": "-codex-device-login",
-            "claude": "-claude-login",
-            "gemini": "-login",
-            "antigravity": "-antigravity-login",
-            "kimi": "-kimi-login",
-            "xai": "-xai-login",
-            "meta": "-meta-login",
-            "devin": "-devin-login",
-        }
-        if provider == "gemini":
-            sys.exit(
-                "This release has no Gemini CLI login flag. Use official dashboard OAuth/API key configuration."
+        replace = None
+        if command == "relogin":
+            account = accounts.find(
+                accounts.roster(root / "state/runtime/auths"), provider, int(sys.argv[3])
             )
-        flag = flags[provider]
-        compose(
-            "exec",
-            "proxy",
-            "/CLIProxyAPI/CLIProxyAPI",
-            "-config",
-            "/state/config.yaml",
-            flag,
-            "-no-browser",
-        )
+            if account is None:
+                sys.exit("No such account. List them with: devfleet fleet-accounts")
+            replace = account["file"]
+        login(provider, replace)
+        prefixes = pending_prefixes()
+        if prefixes:
+            apply_prefixes(prefixes)
+    elif command == "remove-account":
+        # Deletes the credential; the proxy stops serving it, and the next
+        # account propagation removes its instances from every node.
+        operation_lock = lock()
+        accounts.remove(root / "state/runtime/auths", sys.argv[2], int(sys.argv[3]))
+    elif command == "accounts":
+        report = account_roster()
+        if "--json" in sys.argv[2:]:
+            print(json.dumps(report))
+        else:
+            print("Roster version:", report["version"])
+            for account in report["accounts"]:
+                print(
+                    account["provider"],
+                    account["index"],
+                    account["state"],
+                    account["until"] or "",
+                )
+    elif command == "client-keys":
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(key_ring())}))
+    elif command in ["issue-key", "rotate-key", "retire-key", "revoke-key"]:
+        # Per-node client keys. A rotation keeps the old key accepted until
+        # retire-key, so no running agent fails while nodes move over.
+        operation_lock = lock()
+        ring = key_ring()
+        node = sys.argv[2]
+        {
+            "issue-key": clientkeys.issue,
+            "rotate-key": clientkeys.rotate,
+            "retire-key": clientkeys.retire,
+            "revoke-key": clientkeys.revoke,
+        }[command](ring, node)
+        clientkeys.save(root / "state/secrets/client-keys.json", ring)
+        apply_keys(ring)
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(ring)}))
     elif command == "inference":
         protocol, model = sys.argv[2:4]
         key = (root / "state/secrets/client-key").read_text().strip()
@@ -519,7 +631,7 @@ try:
         )
     else:
         sys.exit(
-            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER inference PROTOCOL MODEL portable-login activate drain"
+            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER relogin PROVIDER INDEX remove-account PROVIDER INDEX accounts [--json] client-keys issue-key|rotate-key|retire-key|revoke-key NODE inference PROTOCOL MODEL portable-login activate drain"
         )
 except Exception:
     sys.exit(

package/cliproxy/scripts/restore-verify

@@ -58,3 +58,18 @@ print(
     "credential files validated offline.",
 )
 print("No restored credentials or code were activated.")
+# `--newest DIR`: the newest checkpoint. Refresh tokens rotate, so an older
+# checkpoint can hold tokens the provider has already replaced; name exactly
+# the accounts that need a new login (provider and index, never an account).
+if "--newest" in sys.argv[2:]:
+    import checkpoint
+
+    newest = pathlib.Path(sys.argv[sys.argv.index("--newest") + 1]).resolve()
+    if not (newest / "CHECKPOINT.json").is_file():
+        newest = checkpoint.locate(newest)
+    stale = checkpoint.stale_accounts(root, newest)
+    for provider, index in stale:
+        print("needs-login:", provider, index)
+    print("Accounts restored with a current refresh token:", "all" if not stale else "all others")
+    if stale:
+        sys.exit(4)