@crvouga/dev-fleet 0.2.0 → 0.3.0

package/cliproxy/scripts/ops

@@ -8,6 +8,7 @@ import pathlib as _pathlib
 sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
 import accounts
 import cliproxy_host
+import clientkeys
 
 import fcntl, json, os, pathlib, shutil, subprocess, sys, tempfile, time, urllib.request, urllib.error
 
@@ -205,6 +206,92 @@ def apply_prefixes(prefixes):
     sys.exit("The proxy did not record the declared account prefixes.")
 
 
+LOGIN_FLAGS = {
+    "codex": "-codex-device-login",
+    "claude": "-claude-login",
+    "antigravity": "-antigravity-login",
+    "kimi": "-kimi-login",
+    "xai": "-xai-login",
+    "meta": "-meta-login",
+    "devin": "-devin-login",
+}
+
+
+def login(provider, replace=None):
+    """Runs the proxy's own login into a staging directory, then moves the
+    complete credential in with one rename: an interrupted login leaves no
+    half-written auth file. `replace` is the credential a re-login renews."""
+    import yaml
+
+    if provider not in LOGIN_FLAGS:
+        sys.exit(
+            "This release has no login flag for that provider. Use the official dashboard OAuth/API key configuration."
+        )
+    auths = root / "state/runtime/auths"
+    stage = accounts.begin(auths)
+    config = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    config.setdefault("oauth", {})["auth-dir"] = "/state/" + stage.name
+    login_config = root / "state/runtime/login-config.yaml"
+    login_config.write_text(yaml.safe_dump(config))
+    try:
+        compose(
+            "exec",
+            "proxy",
+            "/CLIProxyAPI/CLIProxyAPI",
+            "-config",
+            "/state/" + login_config.name,
+            LOGIN_FLAGS[provider],
+            "-no-browser",
+        )
+        if not accounts.adopt(stage, auths, replace=replace):
+            sys.exit("The login wrote no credential; nothing was changed.")
+    finally:
+        login_config.unlink(missing_ok=True)
+        shutil.rmtree(stage, ignore_errors=True)
+
+
+def account_roster():
+    """The roster with each account's state, from the credential files and
+    the management API (which knows cooldowns and refresh failures)."""
+    entries = []
+    try:
+        admin = (root / "state/secrets/management-key").read_text().strip()
+        code, body = request("/v0/management/auth-files", admin)
+        if code == 200:
+            entries = json.loads(body).get("files") or []
+    except Exception:
+        entries = []
+    roster = accounts.roster(root / "state/runtime/auths", entries)
+    return {
+        "version": 1,
+        "rosterVersion": accounts.version(roster),
+        "managementApi": bool(entries),
+        "accounts": accounts.public(roster),
+    }
+
+
+def key_ring():
+    return clientkeys.load(root / "state/secrets/client-keys.json")
+
+
+def apply_keys(ring):
+    """The proxy accepts exactly the shared key and the ring's keys."""
+    import yaml
+
+    live = yaml.safe_load((root / "state/runtime/config.yaml").read_text())
+    shared = (root / "state/secrets/client-key").read_text().strip()
+    live.setdefault("access", {})["api-keys"] = clientkeys.api_keys(ring, [shared])
+    with tempfile.NamedTemporaryFile(
+        "w", dir=root / "state/runtime", suffix=".yaml", delete=False
+    ) as file:
+        file.write(yaml.safe_dump(live, sort_keys=False))
+    try:
+        os.chmod(file.name, 0o600)
+        apply_config(pathlib.Path(file.name), restart=False)
+    finally:
+        pathlib.Path(file.name).unlink(missing_ok=True)
+
+
 def lock():
     f = open(root / "state/status/operation.lock", "w")
     if os.geteuid() == 0:
@@ -414,33 +501,58 @@ try:
         sys.exit(
             "Install a reviewed Git revision with bun run cliproxy install --revision REV; use --rollback for the recorded previous revision."
         )
-    elif command == "add-account":
+    elif command in ["add-account", "relogin"]:
+        # `add-account PROVIDER` signs in a new account; `relogin PROVIDER
+        # INDEX` signs that one account in again. Prefixes follow at once.
         operation_lock = lock()
         provider = sys.argv[2]
-        flags = {
-            "codex": "-codex-device-login",
-            "claude": "-claude-login",
-            "gemini": "-login",
-            "antigravity": "-antigravity-login",
-            "kimi": "-kimi-login",
-            "xai": "-xai-login",
-            "meta": "-meta-login",
-            "devin": "-devin-login",
-        }
-        if provider == "gemini":
-            sys.exit(
-                "This release has no Gemini CLI login flag. Use official dashboard OAuth/API key configuration."
+        replace = None
+        if command == "relogin":
+            account = accounts.find(
+                accounts.roster(root / "state/runtime/auths"), provider, int(sys.argv[3])
             )
-        flag = flags[provider]
-        compose(
-            "exec",
-            "proxy",
-            "/CLIProxyAPI/CLIProxyAPI",
-            "-config",
-            "/state/config.yaml",
-            flag,
-            "-no-browser",
-        )
+            if account is None:
+                sys.exit("No such account. List them with: devfleet fleet-accounts")
+            replace = account["file"]
+        login(provider, replace)
+        prefixes = pending_prefixes()
+        if prefixes:
+            apply_prefixes(prefixes)
+    elif command == "remove-account":
+        # Deletes the credential; the proxy stops serving it, and the next
+        # account propagation removes its instances from every node.
+        operation_lock = lock()
+        accounts.remove(root / "state/runtime/auths", sys.argv[2], int(sys.argv[3]))
+    elif command == "accounts":
+        report = account_roster()
+        if "--json" in sys.argv[2:]:
+            print(json.dumps(report))
+        else:
+            print("Roster version:", report["version"])
+            for account in report["accounts"]:
+                print(
+                    account["provider"],
+                    account["index"],
+                    account["state"],
+                    account["until"] or "",
+                )
+    elif command == "client-keys":
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(key_ring())}))
+    elif command in ["issue-key", "rotate-key", "retire-key", "revoke-key"]:
+        # Per-node client keys. A rotation keeps the old key accepted until
+        # retire-key, so no running agent fails while nodes move over.
+        operation_lock = lock()
+        ring = key_ring()
+        node = sys.argv[2]
+        {
+            "issue-key": clientkeys.issue,
+            "rotate-key": clientkeys.rotate,
+            "retire-key": clientkeys.retire,
+            "revoke-key": clientkeys.revoke,
+        }[command](ring, node)
+        clientkeys.save(root / "state/secrets/client-keys.json", ring)
+        apply_keys(ring)
+        print(json.dumps({"version": 1, "nodes": clientkeys.report(ring)}))
     elif command == "inference":
         protocol, model = sys.argv[2:4]
         key = (root / "state/secrets/client-key").read_text().strip()
@@ -519,7 +631,7 @@ try:
         )
     else:
         sys.exit(
-            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER inference PROTOCOL MODEL portable-login activate drain"
+            "Commands: start stop restart status health models logs validate apply FILE declared [--apply] add-account PROVIDER relogin PROVIDER INDEX remove-account PROVIDER INDEX accounts [--json] client-keys issue-key|rotate-key|retire-key|revoke-key NODE inference PROTOCOL MODEL portable-login activate drain"
         )
 except Exception:
     sys.exit(

package/cliproxy/scripts/restore-verify

@@ -58,3 +58,18 @@ print(
     "credential files validated offline.",
 )
 print("No restored credentials or code were activated.")
+# `--newest DIR`: the newest checkpoint. Refresh tokens rotate, so an older
+# checkpoint can hold tokens the provider has already replaced; name exactly
+# the accounts that need a new login (provider and index, never an account).
+if "--newest" in sys.argv[2:]:
+    import checkpoint
+
+    newest = pathlib.Path(sys.argv[sys.argv.index("--newest") + 1]).resolve()
+    if not (newest / "CHECKPOINT.json").is_file():
+        newest = checkpoint.locate(newest)
+    stale = checkpoint.stale_accounts(root, newest)
+    for provider, index in stale:
+        print("needs-login:", provider, index)
+    print("Accounts restored with a current refresh token:", "all" if not stale else "all others")
+    if stale:
+        sys.exit(4)

package/docs/accounts.md

@@ -0,0 +1,154 @@
+# Provider accounts
+
+**The rule.** Provider accounts exist only as the CLI proxy's credential files
+on the manager. Every node and IDE gets a view derived from them, and nothing
+on a node holds a provider login of its own. T3 offers one instance per
+account, named `<Provider> <account email>`, and every launcher reaches the
+provider through the proxy.
+
+## The roster
+
+```sh
+npx devfleet fleet-accounts            # provider, index, label, state, when a cooldown clears
+npx devfleet fleet-accounts --diff     # and what this machine offers against it
+```
+
+Each account has one state, derived from its credential file and the proxy's
+management API:
+
+| State             | Meaning                                                                    |
+| ----------------- | -------------------------------------------------------------------------- |
+| `usable`          | Serves requests                                                            |
+| `cooling-down`    | The provider limited it; `until` says when it clears, when the proxy knows |
+| `refresh-failing` | Its token refresh fails for now, and its access token is still valid       |
+| `needs-login`     | The provider refused its refresh token: only signing in again helps        |
+| `removed`         | Disabled on the proxy                                                      |
+
+The index of an account is its position among its provider's accounts, sorted
+by sign-in email. Checks name accounts by provider and index only.
+
+The **roster version** is a digest of the routing prefixes of the accounts.
+The manager computes it from the credential files and every node from the
+proxy's model list, so the two agree exactly when a node offers what the
+proxy holds.
+
+## Checks
+
+| Check                              | Fails when                                                                                             |
+| ---------------------------------- | ------------------------------------------------------------------------------------------------------ |
+| `proxy:account:<provider>`         | The provider has no usable account; it warns below `services.proxy.accounts.minimumUsable` (default 2) |
+| `accounts:roster-current`          | The node offers another roster version than the proxy's                                                |
+| `accounts:native-login`            | (warns) An agent CLI on the node keeps a provider login of its own                                     |
+| `proxy:reachable-from`             | The node gets no answer from the proxy; it warns above two seconds                                     |
+| `agent:<tool>:account-<n>:session` | That account's instance does not run a session; it warns while the account is cooling down             |
+| `agent:<tool>:removed-<n>:session` | The node still offers an instance for an account the proxy no longer holds                             |
+
+Session results are cached for six hours under a key that holds the roster
+version and the version of the client key, so any account change or key
+rotation re-verifies at once.
+
+## Adding, signing in again, removing
+
+```sh
+npx devfleet fleet-accounts --add claude
+npx devfleet fleet-accounts --relogin codex 2
+npx devfleet fleet-accounts --remove codex 2
+```
+
+A sign-in writes into a staging directory beside the credentials; only a
+complete credential is moved in, with one rename, so an interrupted sign-in
+leaves nothing behind. Routing prefixes are applied in the same command. Every
+change is then propagated to every node.
+
+## Propagation without a full sync
+
+```sh
+npx devfleet fleet-sync --only accounts
+```
+
+On the manager and every worker at once, each node re-reads the proxy's
+roster and rewrites only its per-account T3 instances. T3 watches its
+settings, so nothing restarts and no running turn ends. Instance ids derive
+from the account's routing prefix and stay the same. A node that cannot be
+reached catches up on the next propagation or sync; until then
+`accounts:roster-current` reports the version it has.
+
+## Choosing an account
+
+Unpinned requests go through a loopback relay on every node. When the fleet
+declares a policy for a provider, the relay picks the account of each request:
+
+```json
+"services": {
+  "proxy": {
+    "endpoint": "https://cli-proxy.example.com",
+    "accounts": {
+      "minimumUsable": 2,
+      "selection": {
+        "codex": { "strategy": "fill-first", "order": ["a@example.com"] }
+      }
+    }
+  }
+}
+```
+
+| Strategy                 | Order                                            |
+| ------------------------ | ------------------------------------------------ |
+| `fill-first`             | The same order for every request                 |
+| `round-robin`            | Each request starts one account further along    |
+| `least-recently-limited` | The account limited longest ago (or never) first |
+
+The order is the accounts `order` names, in that order, then the others in
+the order of their routing prefix. A quota or authentication refusal sends
+the request on to the next account, and the refused one cools down for its
+`Retry-After` (a minute without one). Without a policy the proxy's own
+fill-first pool chooses. An instance pinned to one account answers "This
+account is cooling down until …" and names the other instances of its
+provider.
+
+## Proxy outages
+
+Every relayed launch (Claude, Codex, Gemini and Antigravity, pinned or not)
+retries a proxy that does not answer for about a minute. If it stays down,
+the agent gets one distinct, retryable error: "The CLI proxy has been
+unreachable since …".
+
+## Checkpoints and the proxy state
+
+Each checkpoint records a short digest of every credential's refresh token,
+never the token. After restoring an older checkpoint, `restore-verify
+RESTORED --newest NEWEST` names exactly the accounts whose refresh token the
+provider has since replaced (`needs-login: <provider> <index>`).
+
+`cliproxy/scripts/lease.py` is a fencing lease on the proxy state: a second
+proxy cannot take it while the first holds it, and taking over an expired
+lease fences the old holder.
+
+## Client keys
+
+`devfleet cliproxy issue-key|rotate-key|retire-key|revoke-key NODE` keep one
+proxy client key per node in the manager's key ring. A rotation keeps the old
+key accepted until `retire-key`, so no running agent fails while nodes move
+over. Revoking one node's key leaves the others working.
+`devfleet cliproxy client-keys` reports each node's key by fingerprint only.
+
+## Not yet done
+
+- The proxy does not notify anyone when an account turns `needs-login`, and
+  no watcher on the manager triggers propagation or a checkpoint when a
+  credential file changes. Run `fleet-sync --only accounts` or
+  `fleet-accounts` instead.
+- Nothing forces a refresh of every account after a restore. The digest
+  comparison above is offline.
+- The lease is not yet taken by the proxy service at start, and its
+  production store (the checkpoint bucket) is not wired.
+- Per-node client keys are not yet distributed to nodes through the secrets
+  provider, so nodes still present the shared key. No
+  `proxy:client-key-current` check exists yet.
+- OpenCode and Grok read the proxy's address from rendered configuration and
+  are not relayed yet. There is no warm standby proxy.
+- The selection policy is applied by the client relay; the proxy itself stays
+  fill-first.
+- A sign-in uses the provider's device code where the proxy offers one.
+  Other providers print a callback URL, and no loopback forward is managed
+  for them.

package/docs/declared.md

@@ -18,7 +18,9 @@ may only shrink.
 | Tools: languages, CLIs, coding agents         | `mise.toml`                                                                                         | `mise.lock`: version and checksum for each platform                  | `mise install --locked`                                   | Tool and `agent:verification` checks         |
 | mise itself                                   | `min_version` and `mise_*_sha256` in `mise.toml`                                                    | The digest of its release archive, held here                         | `bootstrap/bootstrap.sh`                                  | Its version, at every toolchain setup        |
 | JavaScript dependencies                       | `package.json`                                                                                      | `bun.lock`                                                           | `bun install --frozen-lockfile`                           | The install fails on a difference            |
-| T3 Code, server and desktop app               | `t3_*` under `[vars]` in `mise.toml`                                                                | SHA-256 of each artifact                                             | `bootstrap/native-t3.ts`; `ansible/ubuntu/t3-desktop.yml` | T3 checks                                    |
+| T3 Code, server and desktop app               | `t3_*` under `[vars]` in `mise.toml`                                                                | SHA-256 of each artifact                                             | `bootstrap/native-t3.ts`; `ansible/ubuntu/t3-desktop.yml` | T3 checks; `t3:version` on every machine     |
+| T3 policy on the manager (updates off)        | `ansible/ubuntu/t3/policy.json`, shared with the workers                                            | The commit                                                           | `ansible/macos/t3-policy.yml`, once, as an administrator  | `t3:updates`                                 |
+| Worker work-in-progress guard                 | `ansible/ubuntu/wip-guard.py`, `ansible/ubuntu/tasks/recovery.yml`                                  | The commit                                                           | `fleet-sync`                                              | `features/ide/work-in-progress.feature`      |
 | Ubuntu packages                               | `packages` and `snapshot` in `ansible/ubuntu/policy.json`                                           | Exact versions from a dated archive snapshot                         | `ansible/ubuntu/tasks/packages.yml`                       | `package:*`                                  |
 | Tailscale on workers                          | `tailscale` in `ansible/ubuntu/policy.json`                                                         | Exact version; digest of the signing key                             | `ansible/ubuntu/tailscale.yml`                            | `package:tailscale`                          |
 | Worker policy: power, SSH, sudo, memory, disk | `ansible/ubuntu/*.yml`, values shared with checks in `policy.json`                                  | The commit                                                           | `fleet-sync`                                              | Host and resource checks                     |

package/docs/durability.md

@@ -0,0 +1,26 @@
+# What survives the loss of a machine
+
+This is the one place that says what dev-fleet keeps when a machine is lost.
+
+| Lost        | Kept                                                                                                                                                                                                                                                                      | Lost with it                                                                                             |
+| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
+| A worker    | Everything `fleet-sync` rebuilds from Git and secrets. Work in agent worktrees (`~/.t3/worktrees`) up to the last run of the work-in-progress guard, every 15 minutes, in `refs/recovery/<node>/<branch>` of each worktree's remote. The T3 environment ID in the roster. | Up to 15 minutes of uncommitted work; T3 threads and their transcripts on that worker.                   |
+| The manager | The fleet state, from the bucket ([state](state.md)). The T3 profile, provider instances and policy, from `fleet.json` and Git.                                                                                                                                           | T3's own data on the manager: its linked environments and its threads. They are not in a backup set yet. |
+
+## Recovering work from a lost worker
+
+The guard pushes the files of each worktree as they were (changed and new
+files, not ignored ones) as one commit on top of its `HEAD`, without touching
+the worktree, its index or its branches:
+
+```sh
+git fetch origin 'refs/recovery/*:refs/recovery/*'
+git log refs/recovery/<node>/<branch>
+git switch -c recovered refs/recovery/<node>/<branch>
+```
+
+A worktree whose work is already on a branch of its remote pushes nothing.
+The guard is `ansible/ubuntu/wip-guard.py`, run by the user timer
+`personal-worker-wip-guard.timer` that `ansible/ubuntu/tasks/recovery.yml`
+installs. It pushes with the worker's own Git credentials, so a worktree
+whose remote the worker cannot write to is reported and skipped.

package/docs/machine-setup.md

@@ -352,9 +352,10 @@ only running T3 is the T3 Code desktop app, installed from the same release's
 upstream `SHA256SUMS` omits packages) and started by the `t3code-desktop.service`
 user unit in the GDM auto-login session. It shares `~/.t3` with the former
 headless service, so environment identity and pairing are kept. A conflicting
-launcher version or a still-running headless service requires reviewed
-maintenance; routine fleet-sync never restarts active turns to promote a new
-pin. The disposable Ubuntu VM test still exercises the former headless service
+launcher version requires reviewed maintenance; the recovery timer stops a
+still-running headless service. Routine fleet-sync never ends a running turn
+to load a new pin or policy: the app restarts into them at the first idle
+check (`t3:running-version`, `t3:policy-loaded`). The disposable Ubuntu VM test still exercises the former headless service
 and does not cover the desktop session. Private routing and paired execution
 require separate evidence.
 

package/docs/topology.md

@@ -53,6 +53,8 @@ your project's `fleet.json` (version 3, checked by [the schema](../schema/fleet.
   the manager.
 - Anything on a worker is disposable: T3 sessions and worktrees, caches,
   checkouts. Durable work lives in Git remotes; credentials come from Vault.
+  Work in progress in agent worktrees is pushed to a recovery ref every 15
+  minutes; [durability](durability.md) says what a lost machine takes with it.
 - The first declared worker is the rollout canary for a revision new to the
   fleet: a failed canary stops promotion to the other workers. Repeating a
   revision that already converged everywhere runs all workers at once.

package/docs/ubuntu-sync.md

@@ -179,21 +179,21 @@ independently. Bespoke TypeScript remains only where a native CLI or API has no
 declarative owner: T3 service lifecycle, Beszel enrollment tokens, provider
 verification and the CLIProxy singleton.
 
-| Policy                      | Workers (Ubuntu 26.04, stateless)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Manager (macOS, singleton)                                                                                                                                                                                                                                                                                                                                                                                |
-| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Never sleep                 | systemd sleep targets masked; logind lid/idle policy; GNOME power plugin set to `nothing`                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | never sleeps on AC power (`pmset -c sleep 0`, `macos/manager.yml`, verified as `manager:power`); battery sleep is left to the human                                                                                                                                                                                                                                                                       |
-| No automatic logout or lock | GNOME `lock-enabled false`, `idle-delay 0`; designated-user lingering                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | not managed (the human uses it)                                                                                                                                                                                                                                                                                                                                                                           |
-| Always-on services          | drop-ins `Restart=always`, `StartLimitIntervalSec=0` for ssh, tailscaled, docker, beszel-agent                                                                                                                                                                                                                                                                                                                                                                                                                                                                | singleton services under launchd and Docker Desktop (see the last row)                                                                                                                                                                                                                                                                                                                                    |
-| T3 health and connection    | `fleet-sync` and `fleet-status --workers` require `worker:t3-health` (local API answers), `t3:manager-endpoint` (manager HTTPS over Tailscale) and `t3:paired-access` (manager T3 link reachable)                                                                                                                                                                                                                                                                                                                                                             | the human's T3 desktop holds the links to every worker (`t3:paired-access`)                                                                                                                                                                                                                                                                                                                               |
-| Memory safety               | `systemd-oomd` (swap 90%, pressure 60%); `user.slice` `MemoryMax=90%`; each `t3code-*.scope` agent `MemoryMax=30%`, `OOMPolicy=continue`; swap file when none                                                                                                                                                                                                                                                                                                                                                                                                 | not managed                                                                                                                                                                                                                                                                                                                                                                                               |
-| Disk safety                 | `personal-fleet-disk-guard` kills only the top user-cgroup writer below 3 GiB free, never deletes files; hourly `personal-fleet-disk-gc.timer` deletes git-ignored build output (`node_modules`, `.cache`, `dist`, ...) in checkouts idle 7d (12h under 15 GiB free) with no process inside, plus unused Docker images and build cache, never source, worktrees or volumes; journal 1G; `/tmp` 25% of RAM, aged 2d hourly, a measured write probe (`TMP_QUOTA_EXHAUSTED`), and T3 agents use disk-backed `/var/tmp` (`TMPDIR`, aged 14d); Docker log rotation | not managed                                                                                                                                                                                                                                                                                                                                                                                               |
-| Tailscale                   | pinned APT package, `tailscaled` enabled, native login required                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | Tailscale app, human login                                                                                                                                                                                                                                                                                                                                                                                |
-| Dashboard agent             | pinned native binary, hardened `beszel-agent` systemd unit as user `beszel`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | not installed                                                                                                                                                                                                                                                                                                                                                                                             |
-| Credential renewal          | user timer `personal-worker-auth` renews the node's own Vault token and GitHub login every 10min                                                                                                                                                                                                                                                                                                                                                                                                                                                              | the human's own Vault and GitHub sessions                                                                                                                                                                                                                                                                                                                                                                 |
-| Noninteractive sudo         | validated `/etc/sudoers.d/personal-fleet-USER`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | never; the human runs `manager.yml` once with `-K`                                                                                                                                                                                                                                                                                                                                                        |
-| T3 worker app               | pinned desktop `.deb` is the only T3 process (UI plus embedded server); GDM auto-login, user unit `t3code-desktop.service` bound to the graphical session with `Restart=always`; `personal-worker-recovery` checks every minute, starts a stopped app and restarts one whose local API missed three checks; administrator policy. A still-running headless `t3code.service` reports `T3_DESKTOP_MIGRATION_REQUIRED` (stopping it ends active turns)                                                                                                           | not a worker                                                                                                                                                                                                                                                                                                                                                                                              |
-| T3 providers                | the same managed instances (`fleet/agents/t3.json`) through locked T3 administrator policy                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | reconciled into the human's T3 settings by `fleet-sync` (`t3:providers`) with the ownership-safe `t3-apply` engine, so every T3 offers the identical set; unrelated and unmanaged instances are preserved. `fleet-sync` also applies the same proxy agent profiles (`configuration:apply`) and verifies the shared Vault gateway credential end to end with `provider:*:inference`, exactly as on workers |
-| Singleton services          | never; clients of the manager's services only                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 | every singleton runs here: CLI proxy (`cliproxy`), Beszel hub (`workstation/monitoring`)                                                                                                                                                                                                                                                                                                                  |
+| Policy                      | Workers (Ubuntu 26.04, stateless)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | Manager (macOS, singleton)                                                                                                                                                                                                                                                                                                                                                                                |
+| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Never sleep                 | systemd sleep targets masked; logind lid/idle policy; GNOME power plugin set to `nothing`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | never sleeps on AC power (`pmset -c sleep 0`, `macos/manager.yml`, verified as `manager:power`); battery sleep is left to the human                                                                                                                                                                                                                                                                       |
+| No automatic logout or lock | GNOME `lock-enabled false`, `idle-delay 0`; designated-user lingering                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | not managed (the human uses it)                                                                                                                                                                                                                                                                                                                                                                           |
+| Always-on services          | drop-ins `Restart=always`, `StartLimitIntervalSec=0` for ssh, tailscaled, docker, beszel-agent                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | singleton services under launchd and Docker Desktop (see the last row)                                                                                                                                                                                                                                                                                                                                    |
+| T3 health and connection    | `fleet-sync` and `fleet-status --workers` require `worker:t3-health` (local API answers), `t3:manager-endpoint` (manager HTTPS over Tailscale) and `t3:paired-access` (manager T3 link reachable)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | the human's T3 desktop holds the links to every worker (`t3:paired-access`)                                                                                                                                                                                                                                                                                                                               |
+| Memory safety               | `systemd-oomd` (swap 90%, pressure 60%); `user.slice` `MemoryMax=90%`; each `t3code-*.scope` agent `MemoryMax=30%`, `OOMPolicy=continue`; swap file when none                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | not managed                                                                                                                                                                                                                                                                                                                                                                                               |
+| Disk safety                 | `personal-fleet-disk-guard` kills only the top user-cgroup writer below 3 GiB free, never deletes files; hourly `personal-fleet-disk-gc.timer` deletes git-ignored build output (`node_modules`, `.cache`, `dist`, ...) in checkouts idle 7d (12h under 15 GiB free) with no process inside, plus unused Docker images and build cache, never source, worktrees or volumes; journal 1G; `/tmp` 25% of RAM, aged 2d hourly, a measured write probe (`TMP_QUOTA_EXHAUSTED`), and T3 agents use disk-backed `/var/tmp` (`TMPDIR`, aged 14d); Docker log rotation                                                                                                                                                                       | not managed                                                                                                                                                                                                                                                                                                                                                                                               |
+| Tailscale                   | pinned APT package, `tailscaled` enabled, native login required                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | Tailscale app, human login                                                                                                                                                                                                                                                                                                                                                                                |
+| Dashboard agent             | pinned native binary, hardened `beszel-agent` systemd unit as user `beszel`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | not installed                                                                                                                                                                                                                                                                                                                                                                                             |
+| Credential renewal          | user timer `personal-worker-auth` renews the node's own Vault token and GitHub login every 10min                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | the human's own Vault and GitHub sessions                                                                                                                                                                                                                                                                                                                                                                 |
+| Noninteractive sudo         | validated `/etc/sudoers.d/personal-fleet-USER`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | never; the human runs `manager.yml` once with `-K`                                                                                                                                                                                                                                                                                                                                                        |
+| T3 worker app               | pinned desktop `.deb` is the only T3 process (UI plus embedded server); GDM auto-login, user unit `t3code-desktop.service` bound to the graphical session with `Restart=always`; `personal-worker-recovery` runs a copy of `t3-health.sh` installed in `/usr/local/lib/personal-fleet` every minute: it stops and disables a still-running headless `t3code.service`, reports `T3_NO_GRAPHICAL_SESSION` and restarts the display manager (three times at most) when there is no session, starts a stopped app after `reset-failed`, restarts one whose local API missed three checks, and restarts one on another release than the installed one or with a deferred policy restart once it runs no agent turn; administrator policy | not a worker                                                                                                                                                                                                                                                                                                                                                                                              |
+| T3 providers                | the same managed instances (`fleet/agents/t3.json`) through locked T3 administrator policy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | reconciled into the human's T3 settings by `fleet-sync` (`t3:providers`) with the ownership-safe `t3-apply` engine, so every T3 offers the identical set; unrelated and unmanaged instances are preserved. `fleet-sync` also applies the same proxy agent profiles (`configuration:apply`) and verifies the shared Vault gateway credential end to end with `provider:*:inference`, exactly as on workers |
+| Singleton services          | never; clients of the manager's services only                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | every singleton runs here: CLI proxy (`cliproxy`), Beszel hub (`workstation/monitoring`)                                                                                                                                                                                                                                                                                                                  |
 
 Credential sync is deliberately indirect: no secret is copied between machines.
 OpenTofu issues each enrolled worker its own renewable periodic token
@@ -257,8 +257,11 @@ the existing LaunchAgent and requires a user session. Unattended manager boot
 acceptance and log-retention configuration remain gates. No node pull scheduler
 is installed or necessary.
 
-Sync never reboots a box. It does restart a worker's T3 when a new provider
-policy is staged, which ends that T3's running turns. Explicit
+Sync never reboots a box and never ends a running agent turn. A worker's T3
+that must restart for a new provider policy or release is restarted only while
+it runs no agent turn; otherwise the restart is pending
+(`T3_POLICY_RESTART_PENDING`, `T3_RESTART_PENDING`) and the recovery timer
+applies it at the first idle check. Explicit
 maintenance reboot/reconnect orchestration remains an acceptance gate. Kernel,
 driver, firmware, hardware and power failures are outside any “never freezes”
 promise. Inspect disk/inodes, swap/memory pressure, `systemctl --failed`, and prior
@@ -303,17 +306,20 @@ launch; nothing is copied between nodes.
 
 **Ansible applies, TypeScript measures.** On a worker the T3 lifecycle is
 declared in playbooks: `t3-policy.yml` stages the provider policy and restarts
-a running T3 that has not loaded it, `t3-desktop.yml` installs the app, retires
+a running T3 that has not loaded it once it runs no agent turn, `t3-desktop.yml` installs the app, retires
 the former headless server and starts it, and `t3-route.yml` publishes the
 private tailnet route when port 443 has no owner. The TypeScript around them
 only refuses conflicts and reports what it observes afterwards. Each sync also
 prunes old revision checkouts on the worker, keeping the new one, the one T3
 launches from and the two most recent others.
 
-**Every sync applies everything.** `fleet-sync` never waits for a quiet
-moment: a worker whose T3 has not loaded the staged provider policy is
-restarted (ending its running turns), and a pending proxy update is installed
-even when it recreates the gateway. The manager finishes before the workers
+**Every sync applies everything, except a T3 restart that would end a turn.**
+A worker whose T3 has not loaded the staged provider policy, or runs an older
+release than the one installed, is restarted only while it runs no agent turn;
+otherwise sync reports the pending restart and the recovery timer applies it
+at the first idle check, failing the check after
+`t3RestartMaxDeferralHours` (`ansible/ubuntu/policy.json`). A pending proxy
+update is installed even when it recreates the gateway. The manager finishes before the workers
 start, so they are verified against the updated proxy. `--dry-run` previews.
 
 **Accounts are never listed in this repository.** The proxy's credential files

package/llms.txt

@@ -22,6 +22,7 @@ Rules for an agent working in a project that uses dev-fleet:
 - [Fleet state](https://github.com/crvouga/dev-fleet/blob/main/docs/state.md): the bucket, replication, replacing the manager, what the bucket does not hold yet
 - [Topology](https://github.com/crvouga/dev-fleet/blob/main/docs/topology.md): the manager and worker roles and the canary
 - [What is declared, and where](https://github.com/crvouga/dev-fleet/blob/main/docs/declared.md): what is pinned on a machine and what is not
+- [Provider accounts](https://github.com/crvouga/dev-fleet/blob/main/docs/accounts.md): the account roster, its states and checks, adding or removing an account, propagation to every node, account selection
 - [CLI proxy](https://github.com/crvouga/dev-fleet/blob/main/cliproxy/README.md): the CLIProxyAPI service on the manager that every coding agent uses
 - [Monitoring](https://github.com/crvouga/dev-fleet/blob/main/ansible/monitoring/README.md): the Beszel hub on the manager and the agent on every machine
 

package/mise.toml

@@ -88,10 +88,12 @@ mise_linux_x64_sha256 = "9a6c9c55f5c6372d0af70a9979fe6d742aea3793267a929e1738022
 mise_linux_arm64_sha256 = "8c8f5d7f748bf3aef431dc50f0ad776270f36025c788aa21c74e2fabdbbd6d3e"
 mise_macos_x64_sha256 = "83a36355bf6ee7c1660300e12dd94464de5c672dab1b8cc33ec4d19356e7ddac"
 mise_macos_arm64_sha256 = "aa2fc59242db108f97ea9fb2d2e2cb42ff1467fcafe3c91e48d51c841ba67a65"
-# Official native Ubuntu server distribution; bootstrap/native-t3.ts owns installation.
+# Official native server distribution (Ubuntu workers, and the manager's
+# pinned CLI); bootstrap/native-t3.ts owns installation. From the release's SHA256SUMS.
 t3_version = "0.1.8-nightly.20261106.4262"
 t3_linux_x64_sha256 = "5df94ca15c4f68f749fa5540fbb9d5ca879ca8750ac43dd2704f6a18899ed6d5"
 t3_linux_arm64_sha256 = "0649615c4a294eee40bc753c8668e4a83c5c3620794f2163758f8dc48a39b880"
+t3_darwin_arm64_sha256 = "40c81f1d696e7056c0c43e29bdfd2a154ea6f6bd9129826034256751418de633"
 # Official T3 Code desktop .deb (same release). Workers run this GUI app as their
 # only T3 process; it embeds the server. Upstream SHA256SUMS omits .deb files,
 # so these digests were computed from the release assets.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/dev-fleet",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Keeps a set of development machines in sync: one manager and many workers",
   "license": "MIT",
   "repository": "github:crvouga/dev-fleet",
@@ -72,5 +72,5 @@
   "engines": {
     "node": ">=22"
   },
-  "gitHead": "6dee8455d48203125eb5c0546c3c4446034fca85"
+  "gitHead": "0c583e40a0c32c0db355e9c0bca6903894945310"
 }