@crvouga/dev-fleet 0.1.0 → 0.3.0

package/cliproxy/scripts/clientkeys.py

@@ -0,0 +1,99 @@
+"""One proxy client key per node, rotated with overlap and revoked with the node.
+
+The key ring is a private JSON document in the proxy's secrets
+(`state/secrets/client-keys.json`): per node, the current key and, during a
+rotation, the previous one. The proxy accepts every key in the ring
+(`api_keys`), so a rotation adds the new key, every node moves to it, and
+only then is the old key retired: no request fails in between. Revoking a
+node removes its keys and leaves every other node working. Reports name keys
+only by a short fingerprint, never by value.
+"""
+import hashlib
+import json
+import os
+import pathlib
+import re
+import secrets
+
+NODE = re.compile(r'^[a-z][a-z0-9-]{0,62}$')
+
+
+def fingerprint(key):
+    return hashlib.sha256(key.encode()).hexdigest()[:12]
+
+
+def load(path):
+    try:
+        ring = json.loads(pathlib.Path(path).read_text())
+    except (OSError, ValueError):
+        return {'version': 1, 'nodes': {}}
+    if not isinstance(ring, dict) or ring.get('version') != 1:
+        raise ValueError('Unsupported client key ring')
+    return ring
+
+
+def save(path, ring):
+    path = pathlib.Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    staged = path.with_suffix('.new')
+    staged.write_text(json.dumps(ring, indent=2, sort_keys=True) + '\n')
+    os.chmod(staged, 0o600)
+    os.replace(staged, path)
+
+
+def _node(ring, node):
+    if not NODE.match(node):
+        raise ValueError('Invalid node id')
+    return ring['nodes'].setdefault(node, {'current': None, 'previous': None})
+
+
+def issue(ring, node, generate=lambda: secrets.token_urlsafe(32)):
+    """Gives `node` a key when it has none; returns its current key."""
+    entry = _node(ring, node)
+    if not entry['current']:
+        entry['current'] = generate()
+    return entry['current']
+
+
+def rotate(ring, node, generate=lambda: secrets.token_urlsafe(32)):
+    """Starts a rotation: a new current key, the old one kept as previous
+    until `retire`. Both are accepted meanwhile."""
+    entry = _node(ring, node)
+    if entry['previous']:
+        raise ValueError('A rotation of this node is already in progress; retire it first')
+    entry['previous'], entry['current'] = entry['current'], generate()
+    return entry['current']
+
+
+def retire(ring, node):
+    """Ends a rotation once the node uses its new key."""
+    _node(ring, node)['previous'] = None
+
+
+def revoke(ring, node):
+    """Removes every key of `node`."""
+    ring['nodes'].pop(node, None)
+
+
+def api_keys(ring, shared=()):
+    """Every key the proxy accepts: the ring's, plus `shared` keys still in
+    use (the fleet-wide key, until each node has its own)."""
+    keys = list(shared)
+    for node in sorted(ring['nodes']):
+        entry = ring['nodes'][node]
+        for key in (entry['current'], entry['previous']):
+            if key and key not in keys:
+                keys.append(key)
+    return keys
+
+
+def report(ring):
+    """Per node: the current key's fingerprint and whether a rotation is
+    still waiting to be retired. No key value."""
+    return {
+        node: {
+            'current': fingerprint(entry['current']) if entry['current'] else None,
+            'rotating': bool(entry['previous']),
+        }
+        for node, entry in sorted(ring['nodes'].items())
+    }

package/cliproxy/scripts/lease.py

@@ -0,0 +1,80 @@
+"""A fencing lease on the proxy state: only its holder is the writer.
+
+Providers rotate refresh tokens, so two proxies started from the same
+checkpoint spend each other's tokens. The lease is one small JSON document in
+a store both would see (the checkpoint bucket in production, a directory in
+tests): the holder, a fencing epoch that only ever grows, and when it
+expires. A proxy that cannot take the lease refuses to start as writer.
+"""
+import json
+import os
+import pathlib
+import time
+
+TTL = 120
+
+
+class Held(Exception):
+    """Another proxy holds the lease."""
+
+    def __init__(self, holder, expires):
+        super().__init__('The proxy state lease is held by another proxy')
+        self.holder = holder
+        self.expires = expires
+
+
+class DirectoryStore:
+    """A lease document in a directory, replaced with one rename. `put`
+    writes only when the document is still the one `get` returned."""
+
+    def __init__(self, directory):
+        self.path = pathlib.Path(directory) / 'proxy-lease.json'
+
+    def get(self):
+        try:
+            return json.loads(self.path.read_text())
+        except (OSError, ValueError):
+            return None
+
+    def put(self, document, expected):
+        import fcntl
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.path.with_suffix('.lock'), 'w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            if self.get() != expected:
+                return False
+            staged = self.path.with_suffix('.new')
+            staged.write_text(json.dumps(document) + '\n')
+            os.replace(staged, self.path)
+            return True
+
+
+def acquire(store, holder, now=None, ttl=TTL):
+    """Takes or renews the lease for `holder`; returns its fencing epoch.
+    Raises Held while another holder's lease has not expired. Taking over an
+    expired lease increases the epoch, so the old holder's writes are fenced."""
+    now = time.time() if now is None else now
+    current = store.get()
+    if current and current.get('holder') != holder and current.get('expires', 0) > now:
+        raise Held(current.get('holder'), current.get('expires'))
+    epoch = (current or {}).get('epoch', 0)
+    if not current or current.get('holder') != holder:
+        epoch += 1
+    document = {'holder': holder, 'epoch': epoch, 'expires': now + ttl}
+    if not store.put(document, current):
+        raise Held(None, None)
+    return epoch
+
+
+def valid(store, holder, epoch, now=None):
+    """Whether `holder` still holds the lease at `epoch`: checked before a
+    token refresh is written."""
+    now = time.time() if now is None else now
+    current = store.get() or {}
+    return current.get('holder') == holder and current.get('epoch') == epoch and current.get('expires', 0) > now
+
+
+def release(store, holder):
+    current = store.get()
+    if current and current.get('holder') == holder:
+        store.put({**current, 'expires': 0}, current)

package/cliproxy/scripts/ops

@@ -8,6 +8,7 @@ import pathlib as _pathlib
 sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
 import accounts
 import cliproxy_host
+import clientkeys
 
 import fcntl, json, os, pathlib, shutil, subprocess, sys, tempfile, time, urllib.request, urllib.error
 
@@ -205,6 +206,92 @@ def apply_prefixes(prefixes):
     sys.exit("The proxy did not record the declared account prefixes.")
 
 
+LOGIN_FLAGS = {
+    "codex": "-codex-device-login",
+    "claude": "-claude-login",
+    "antigravity": "-antigravity-login",
+    "kimi": "-kimi-login",
+    "xai": "-xai-login",
+    "meta": "-meta-login",
+    "devin": "-devin-login",
+}
+
+
+def login(provider, replace=None):
+    """Runs the proxy's own login into a staging directory, then moves the
+    complete credential in with one rename: an interrupted login leaves no
+    half-written auth file. `replace` is the credential a re-login renews."""
+    import yaml
+
+    if provider not in LOGIN_FLAGS:
+        sys.exit(
+            "This release has no login flag for that provider. Use the official dashboard OAuth/API key configuration."
+        )
+    auths = root / "state/runtime/auths"
+    stage = accounts.begin(auths)
+    config = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    config.setdefault("oauth", {})["auth-dir"] = "/state/" + stage.name
+    login_config = root / "state/runtime/login-config.yaml"
+    login_config.write_text(yaml.safe_dump(config))
+    try:
+        compose(
+            "exec",
+            "proxy",
+            "/CLIProxyAPI/CLIProxyAPI",
+            "-config",
+            "/state/" + login_config.name,
+            LOGIN_FLAGS[provider],
+            "-no-browser",
+        )
+        if not accounts.adopt(stage, auths, replace=replace):
+            sys.exit("The login wrote no credential; nothing was changed.")
+    finally:
+        login_config.unlink(missing_ok=True)
+        shutil.rmtree(stage, ignore_errors=True)
+
+
+def account_roster():
+    """The roster with each account's state, from the credential files and
+    the management API (which knows cooldowns and refresh failures)."""
+    entries = []
+    try:
+        admin = (root / "state/secrets/management-key").read_text().strip()
+        code, body = request("/v0/management/auth-files", admin)
+        if code == 200:
+            entries = json.loads(body).get("files") or []
+    except Exception:
+        entries = []
+    roster = accounts.roster(root / "state/runtime/auths", entries)
+    return {
+        "version": 1,
+        "rosterVersion": accounts.version(roster),
+        "managementApi": bool(entries),
+        "accounts": accounts.public(roster),
+    }
+
+
+def key_ring():
+    return clientkeys.load(root / "state/secrets/client-keys.json")
+
+
+def apply_keys(ring):
+    """The proxy accepts exactly the shared key and the ring's keys."""
+    import yaml
+
+    live = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    shared = (root / "state/secrets/client-key").read_text().strip()
+    live.setdefault("access", {})["api-keys"] = clientkeys.api_keys(ring, [shared])
+    with tempfile.NamedTemporaryFile(
+        "w", dir=root / "state/runtime", suffix=".yaml", delete=False
+    ) as file:
+        file.write(yaml.safe_dump(live, sort_keys=False))
+    try:
+        os.chmod(file.name, 0o600)
+        apply_config(pathlib.Path(file.name), restart=False)
+    finally:
+        pathlib.Path(file.name).unlink(missing_ok=True)
+
+
 def lock():
     f = open(root / "state/status/operation.lock", "w")
     if os.geteuid() == 0:
@@ -414,33 +501,58 @@ try:
         sys.exit(
             "Install a reviewed Git revision with bun run cliproxy install --revision REV; use --rollback for the recorded previous revision."
         )
-    elif command == "add-account":
+    elif command in ["add-account", "relogin"]:
+        # `add-account PROVIDER` signs in a new account; `relogin PROVIDER
+        # INDEX` signs that one account in again. Prefixes follow at once.
         operation_lock = lock()
         provider = sys.argv[2]
-        flags = {
-            "codex": "-codex-device-login",
-            "claude": "-claude-login",
-            "gemini": "-login",
-            "antigravity": "-antigravity-login",
-            "kimi": "-kimi-login",
-            "xai": "-xai-login",
-            "meta": "-meta-login",
-            "devin": "-devin-login",
-        }
-        if provider == "gemini":
-            sys.exit(
-                "This release has no Gemini CLI login flag. Use official dashboard OAuth/API key configuration."
+        replace = None
+        if command == "relogin":
+            account = accounts.find(
+                accounts.roster(root / "state/runtime/auths"), provider, int(sys.argv[3])
             )
-        flag = flags[provider]
-        compose(
-            "exec",
-            "proxy",
-            "/CLIProxyAPI/CLIProxyAPI",
-            "-config",
-            "/state/config.yaml",
-            flag,
-            "-no-browser",
-        )
+            if account is None:
+                sys.exit("No such account. List them with: devfleet fleet-accounts")
+            replace = account["file"]
+        login(provider, replace)
+        prefixes = pending_prefixes()
+        if prefixes:
+            apply_prefixes(prefixes)
+    elif command == "remove-account":
+        # Deletes the credential; the proxy stops serving it, and the next
+        # account propagation removes its instances from every node.
+        operation_lock = lock()
+        accounts.remove(root / "state/runtime/auths", sys.argv[2], int(sys.argv[3]))
+    elif command == "accounts":
+        report = account_roster()
+        if "--json" in sys.argv[2:]:
+            print(json.dumps(report))
+        else:
+            print("Roster version:", report["version"])
+            for account in report["accounts"]:
+                print(
+                    account["provider"],
+                    account["index"],
+                    account["state"],
+                    account["until"] or "",
+                )
+    elif command == "client-keys":
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(key_ring())}))
+    elif command in ["issue-key", "rotate-key", "retire-key", "revoke-key"]:
+        # Per-node client keys. A rotation keeps the old key accepted until
+        # retire-key, so no running agent fails while nodes move over.
+        operation_lock = lock()
+        ring = key_ring()
+        node = sys.argv[2]
+        {
+            "issue-key": clientkeys.issue,
+            "rotate-key": clientkeys.rotate,
+            "retire-key": clientkeys.retire,
+            "revoke-key": clientkeys.revoke,
+        }[command](ring, node)
+        clientkeys.save(root / "state/secrets/client-keys.json", ring)
+        apply_keys(ring)
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(ring)}))
     elif command == "inference":
         protocol, model = sys.argv[2:4]
         key = (root / "state/secrets/client-key").read_text().strip()
@@ -519,7 +631,7 @@ try:
         )
     else:
         sys.exit(
-            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER inference PROTOCOL MODEL portable-login activate drain"
+            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER relogin PROVIDER INDEX remove-account PROVIDER INDEX accounts [--json] client-keys issue-key|rotate-key|retire-key|revoke-key NODE inference PROTOCOL MODEL portable-login activate drain"
         )
 except Exception:
     sys.exit(

package/cliproxy/scripts/restore-verify

@@ -58,3 +58,18 @@ print(
     "credential files validated offline.",
 )
 print("No restored credentials or code were activated.")
+# `--newest DIR`: the newest checkpoint. Refresh tokens rotate, so an older
+# checkpoint can hold tokens the provider has already replaced; name exactly
+# the accounts that need a new login (provider and index, never an account).
+if "--newest" in sys.argv[2:]:
+    import checkpoint
+
+    newest = pathlib.Path(sys.argv[sys.argv.index("--newest") + 1]).resolve()
+    if not (newest / "CHECKPOINT.json").is_file():
+        newest = checkpoint.locate(newest)
+    stale = checkpoint.stale_accounts(root, newest)
+    for provider, index in stale:
+        print("needs-login:", provider, index)
+    print("Accounts restored with a current refresh token:", "all" if not stale else "all others")
+    if stale:
+        sys.exit(4)

package/docs/accounts.md

@@ -0,0 +1,154 @@
+# Provider accounts
+
+**The rule.** Provider accounts exist only as the CLI proxy's credential files
+on the manager. Every node and IDE gets a view derived from them, and nothing
+on a node holds a provider login of its own. T3 offers one instance per
+account, named `<Provider> <account email>`, and every launcher reaches the
+provider through the proxy.
+
+## The roster
+
+```sh
+npx devfleet fleet-accounts            # provider, index, label, state, when a cooldown clears
+npx devfleet fleet-accounts --diff     # and what this machine offers against it
+```
+
+Each account has one state, derived from its credential file and the proxy's
+management API:
+
+| State             | Meaning                                                                    |
+| ----------------- | -------------------------------------------------------------------------- |
+| `usable`          | Serves requests                                                            |
+| `cooling-down`    | The provider limited it; `until` says when it clears, when the proxy knows |
+| `refresh-failing` | Its token refresh fails for now, and its access token is still valid       |
+| `needs-login`     | The provider refused its refresh token: only signing in again helps        |
+| `removed`         | Disabled on the proxy                                                      |
+
+The index of an account is its position among its provider's accounts, sorted
+by sign-in email. Checks name accounts by provider and index only.
+
+The **roster version** is a digest of the routing prefixes of the accounts.
+The manager computes it from the credential files and every node from the
+proxy's model list, so the two agree exactly when a node offers what the
+proxy holds.
+
+## Checks
+
+| Check                              | Fails when                                                                                             |
+| ---------------------------------- | ------------------------------------------------------------------------------------------------------ |
+| `proxy:account:<provider>`         | The provider has no usable account; it warns below `services.proxy.accounts.minimumUsable` (default 2) |
+| `accounts:roster-current`          | The node offers another roster version than the proxy's                                                |
+| `accounts:native-login`            | (warns) An agent CLI on the node keeps a provider login of its own                                     |
+| `proxy:reachable-from`             | The node gets no answer from the proxy; it warns above two seconds                                     |
+| `agent:<tool>:account-<n>:session` | That account's instance does not run a session; it warns while the account is cooling down             |
+| `agent:<tool>:removed-<n>:session` | The node still offers an instance for an account the proxy no longer holds                             |
+
+Session results are cached for six hours under a key that holds the roster
+version and the version of the client key, so any account change or key
+rotation re-verifies at once.
+
+## Adding, signing in again, removing
+
+```sh
+npx devfleet fleet-accounts --add claude
+npx devfleet fleet-accounts --relogin codex 2
+npx devfleet fleet-accounts --remove codex 2
+```
+
+A sign-in writes into a staging directory beside the credentials; only a
+complete credential is moved in, with one rename, so an interrupted sign-in
+leaves nothing behind. Routing prefixes are applied in the same command. Every
+change is then propagated to every node.
+
+## Propagation without a full sync
+
+```sh
+npx devfleet fleet-sync --only accounts
+```
+
+On the manager and every worker at once, each node re-reads the proxy's
+roster and rewrites only its per-account T3 instances. T3 watches its
+settings, so nothing restarts and no running turn ends. Instance ids derive
+from the account's routing prefix and stay the same. A node that cannot be
+reached catches up on the next propagation or sync; until then
+`accounts:roster-current` reports the version it has.
+
+## Choosing an account
+
+Unpinned requests go through a loopback relay on every node. When the fleet
+declares a policy for a provider, the relay picks the account of each request:
+
+```json
+"services": {
+  "proxy": {
+    "endpoint": "https://cli-proxy.example.com",
+    "accounts": {
+      "minimumUsable": 2,
+      "selection": {
+        "codex": { "strategy": "fill-first", "order": ["a@example.com"] }
+      }
+    }
+  }
+}
+```
+
+| Strategy                 | Order                                            |
+| ------------------------ | ------------------------------------------------ |
+| `fill-first`             | The same order for every request                 |
+| `round-robin`            | Each request starts one account further along    |
+| `least-recently-limited` | The account limited longest ago (or never) first |
+
+The order is the accounts `order` names, in that order, then the others in
+the order of their routing prefix. A quota or authentication refusal sends
+the request on to the next account, and the refused one cools down for its
+`Retry-After` (a minute without one). Without a policy the proxy's own
+fill-first pool chooses. An instance pinned to one account answers "This
+account is cooling down until …" and names the other instances of its
+provider.
+
+## Proxy outages
+
+Every relayed launch (Claude, Codex, Gemini and Antigravity, pinned or not)
+retries a proxy that does not answer for about a minute. If it stays down,
+the agent gets one distinct, retryable error: "The CLI proxy has been
+unreachable since …".
+
+## Checkpoints and the proxy state
+
+Each checkpoint records a short digest of every credential's refresh token,
+never the token. After restoring an older checkpoint, `restore-verify
+RESTORED --newest NEWEST` names exactly the accounts whose refresh token the
+provider has since replaced (`needs-login: <provider> <index>`).
+
+`cliproxy/scripts/lease.py` is a fencing lease on the proxy state: a second
+proxy cannot take it while the first holds it, and taking over an expired
+lease fences the old holder.
+
+## Client keys
+
+`devfleet cliproxy issue-key|rotate-key|retire-key|revoke-key NODE` keep one
+proxy client key per node in the manager's key ring. A rotation keeps the old
+key accepted until `retire-key`, so no running agent fails while nodes move
+over. Revoking one node's key leaves the others working.
+`devfleet cliproxy client-keys` reports each node's key by fingerprint only.
+
+## Not yet done
+
+- The proxy does not notify anyone when an account turns `needs-login`, and
+  no watcher on the manager triggers propagation or a checkpoint when a
+  credential file changes. Run `fleet-sync --only accounts` or
+  `fleet-accounts` instead.
+- Nothing forces a refresh of every account after a restore. The digest
+  comparison above is offline.
+- The lease is not yet taken by the proxy service at start, and its
+  production store (the checkpoint bucket) is not wired.
+- Per-node client keys are not yet distributed to nodes through the secrets
+  provider, so nodes still present the shared key. No
+  `proxy:client-key-current` check exists yet.
+- OpenCode and Grok read the proxy's address from rendered configuration and
+  are not relayed yet. There is no warm standby proxy.
+- The selection policy is applied by the client relay; the proxy itself stays
+  fill-first.
+- A sign-in uses the provider's device code where the proxy offers one.
+  Other providers print a callback URL, and no loopback forward is managed
+  for them.