@crvouga/dev-fleet 0.6.0 → 0.7.0

package/README.md

@@ -170,16 +170,10 @@ says where the bucket is and the command is passed its credentials:
 npx devfleet fleet-state             # where the state is, and how it stands with the bucket
 ```
 
-The copy on the manager is disposable. **To replace the manager**, clone your
-project on the new machine and run the same two commands:
-
-```sh
-npm ci
-npx devfleet fleet-sync              # with the two bucket credentials in the environment
-```
-
-The roster and the approved revision come back from the bucket, and the
-machine is rebuilt from Git.
+The copy on the manager is disposable: the roster and the approved revision
+come back from the bucket, and the machine is rebuilt from Git.
+[Replacing the manager](docs/manager-recovery.md) is the runbook for a new or
+lost manager.
 
 Some runs are started by a machine rather than by you: a worker that hands
 `fleet-sync` to the manager, for example. They get no environment, so save the

package/ansible/macos/manager.yml

@@ -1,7 +1,8 @@
 ---
 # Manager power policy. Workers depend on the manager's singleton services
-# (CLI proxy, Beszel hub), so the manager never sleeps on AC power. Sleep on
-# battery is left to the human. The settings are the `pmset` object in
+# (CLI proxy, Beszel hub), so the manager never sleeps on AC power and starts
+# again by itself when power returns (autorestart). Sleep on battery is left
+# to the human. The settings are the `pmset` object in
 # policy.json; fleet-sync measures the same object as manager:power. Changing
 # them needs an administrator: ansible-playbook -K manager.yml
 - name: Keep the fleet manager awake on AC power
@@ -29,5 +30,41 @@
       when: observed != item.value
       changed_when: true
 
+# Docker Desktop hosts the CLI proxy and the monitoring hub, so it starts at
+# login: after a power cut the manager comes back without a person. The
+# setting is the `dockerDesktop` object in policy.json; fleet-sync measures it
+# as part of manager:boot-recovery. Only that key of the user's settings file
+# is changed.
+- name: Start Docker Desktop at login
+  hosts: localhost
+  connection: local
+  become: false
+  gather_facts: false
+  vars_files:
+    - policy.json
+  vars:
+    docker_settings: "{{ lookup('ansible.builtin.env', 'HOME') }}/{{ dockerDesktop.settings }}"
+  tasks:
+    - name: Find the Docker Desktop settings
+      ansible.builtin.stat:
+        path: '{{ docker_settings }}'
+      register: docker_settings_file
+    - name: Read the Docker Desktop settings
+      ansible.builtin.slurp:
+        src: '{{ docker_settings }}'
+      register: docker_settings_text
+      when: docker_settings_file.stat.exists
+    - name: Turn on start at login, keeping every other setting
+      ansible.builtin.copy:
+        dest: '{{ docker_settings }}'
+        content: "{{ current | combine({dockerDesktop.key: true}) | to_nice_json(indent=2) }}\n"
+        mode: '0644'
+      vars:
+        current: '{{ docker_settings_text.content | b64decode | from_json }}'
+      # Docker Desktop writes the file the first time it runs.
+      when:
+        - docker_settings_file.stat.exists
+        - (current[dockerDesktop.key] | default(false)) != true
+
 - name: Stage the fleet's T3 policy
   ansible.builtin.import_playbook: t3-policy.yml

package/ansible/macos/policy.json

@@ -4,7 +4,8 @@
   "architectures": ["arm64", "x86_64"],
   "pmset": {
     "sleep": "0",
-    "womp": "1"
+    "womp": "1",
+    "autorestart": "1"
   },
   "beszelDaemon": "dev.devfleet.beszel-agent",
   "disk": {
@@ -16,5 +17,9 @@
     "unhealthyChecks": 3,
     "maxRestartsPerHour": 3,
     "intervalSeconds": 60
+  },
+  "dockerDesktop": {
+    "settings": "Library/Group Containers/group.com.docker/settings-store.json",
+    "key": "AutoStart"
   }
 }

package/ansible/monitoring/agent.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: true
+  # fleet-repair re-applies this play alone for a failing beszel agent check.
+  tags: [monitoring]
   vars:
     beszel_lock: "{{ lookup('ansible.builtin.file', 'lock.json') | from_json }}"
     beszel_arch: "{{ {'x86_64': 'amd64', 'aarch64': 'arm64'}.get(ansible_facts.architecture, 'unsupported') }}"

package/ansible/ubuntu/baseline.yml

@@ -8,24 +8,30 @@
     - policy.json
   pre_tasks:
     - ansible.builtin.import_tasks: tasks/release.yml
+      tags: [always]
     - name: Require an explicit non-root user
       ansible.builtin.assert:
         that:
           - development_user is match('^[a-z_][a-z0-9_-]{0,31}$')
           - development_user != 'root'
         fail_msg: Unsupported user; name the non-root developer account.
+      tags: [always]
     - name: Read the designated existing account
       ansible.builtin.getent:
         database: passwd
         key: '{{ development_user }}'
+      tags: [always]
     - name: Refuse a different user identity or home
       ansible.builtin.assert:
         that:
           - ansible_facts.getent_passwd[development_user][1] | int == development_uid | int
           - ansible_facts.getent_passwd[development_user][4] == development_home
+      tags: [always]
     - ansible.builtin.import_tasks: tasks/dpkg.yml
     # Memory and disk safety must land even when the disk is nearly full.
+    # fleet-repair re-applies them alone (--tags disk) for worker:capacity:*.
     - ansible.builtin.import_tasks: tasks/resources.yml
+      tags: [disk]
     - name: Inspect capacity before package and service changes
       ansible.builtin.command:
         argv:

package/ansible/ubuntu/recovery.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: false
+  # fleet-repair re-applies this play alone for worker:service-recovery.
+  tags: [recovery]
   tasks:
     - name: Verify the desktop unit runs before adding any recovery trigger
       ansible.builtin.command:

package/ansible/ubuntu/tailscale.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: true
+  # fleet-repair re-applies this play alone for a failing tailscale:* check.
+  tags: [network]
   # The approved version and the signing key's digest: `tailscale` in policy.json.
   vars_files:
     - policy.json

package/docs/consumer-setup.md

@@ -382,10 +382,8 @@ and codes; leave out everything private.
 - **Upgrade the fleet.** `npm install --save-dev --save-exact @crvouga/dev-fleet@latest`,
   commit, `npx devfleet fleet-sync`. The canary proves the new revision first.
   Install an older version and sync to roll back.
-- **Replace the manager.** On the new Mac: clone the project, `npm ci`, put
-  the bucket credentials in the environment, `npx devfleet fleet-sync`. The
-  roster and the approved revision come back from the bucket. See
-  [state](state.md) for what the bucket does not hold yet.
+- **Replace the manager.** Follow
+  [replacing the manager](manager-recovery.md).
 
 ## Moving from a pinned commit and a wrapper script
 

package/docs/manager-recovery.md

@@ -0,0 +1,78 @@
+# Replacing the manager
+
+This is the one runbook for a manager that is replaced or lost: stolen, a dead
+disk, or a new machine. Other documents link here and do not describe the
+procedure themselves.
+
+## What must exist off the machine
+
+Nothing on the old manager is needed. Before you need this runbook, make sure
+these exist elsewhere:
+
+| What                                | Where                                                                    |
+| ----------------------------------- | ------------------------------------------------------------------------ |
+| Your project: `fleet.json`, the pin | Its Git remote                                                           |
+| The fleet state                     | The bucket `fleet.json` names under `state`                              |
+| The bucket's two credentials        | Your password manager (never the manager alone)                          |
+| Secrets                             | Your `secrets` provider; `file` and `env` secrets must be kept elsewhere |
+| The CLI proxy's encrypted snapshots | Its restic repository (`vault` secrets provider only)                    |
+| Your Tailscale and Docker accounts  | Yours: signing in is the only step only you can do                       |
+
+## Restore
+
+On the new machine, with Git, Node and Docker installed:
+
+1. Give it the manager's local host name: `manager.id` in `fleet.json`
+   (`sudo scutil --set LocalHostName ID` on macOS). `fleet-sync` refuses to
+   run on any other machine.
+2. Sign in to Tailscale and approve the device. Sign in to Docker Desktop if
+   your plan needs it.
+3. Fetch the project and check every prerequisite. Nothing is changed:
+
+   ```sh
+   git clone YOUR-PROJECT && cd YOUR-PROJECT && npm ci
+   export DEVFLEET_BUCKET_ACCESS_KEY_ID=… DEVFLEET_BUCKET_SECRET_ACCESS_KEY=…
+   npx devfleet fleet-restore --dry-run
+   ```
+
+   It fails on anything missing (`restore:*` checks, each with its remedy)
+   and lists what will be restored, in order.
+
+4. Restore, then converge:
+
+   ```sh
+   npx devfleet fleet-restore              # the fleet state, from the bucket
+   npx devfleet fleet-sync --take-over     # the manager, then every worker
+   npx devfleet fleet-state --init         # credentials for runs the manager starts itself
+   npx devfleet cliproxy install           # the proxy, from its snapshots
+   ```
+
+   The proxy's own steps are in [its migration runbook](../cliproxy/docs/migration.md).
+   `--take-over` is needed while the old manager's lease runs (seven days
+   after its last change). Use it only once the old machine is gone: an old
+   manager that wakes up afterwards is refused with `FLEET_MANAGER_LEASE_HELD`.
+
+5. Adopt the T3 instances again: `npx devfleet t3-apply --import ID`.
+
+Workers need nothing: they reach the manager by its Tailscale name, so they
+follow it once the new machine holds that name. If the Tailscale name
+changed, change `manager.network.tailscaleName` in `fleet.json` and run
+`fleet-sync`, which re-points every worker.
+
+## What is lost
+
+The monitoring hub's metric history, alerts and settings made by hand. Its
+accounts and systems are recreated. With `env` or `file` secrets, whatever
+was only on the old manager.
+
+## Coming back by itself
+
+A manager restarts after a power cut only when it is set up to
+(`manager:boot-recovery` in `fleet-status`): `autorestart` and Docker Desktop
+at login are applied by `ansible/macos/manager.yml`. The per-user services
+need a user session. **With FileVault on, a Mac waits at the login window
+after every restart until a person signs in**, so nothing comes back
+unattended. That is the trade-off: FileVault protects the proxy's logins and
+the bucket credentials on a stolen Mac; automatic login without FileVault
+lets the fleet recover from a power cut alone. `fleet-status` reports
+FileVault as a warning with that consequence; the choice is yours.

package/docs/operations.md

@@ -304,6 +304,14 @@ The public CLI exposes `devfleet fleet-repair --list` (static catalog) and
 in this package only executes allowlisted existing owner commands. Native
 account logins and unknown/manual checks always remain operator steps.
 
+Every check ID has an owner in `src/check-owners.ts`: a recipe, or the reason
+only a person can repair it, which the plan prints beside the check
+(`manual`). Machine repairs are one tag of a playbook the worker already
+applies: `recovery` (`worker:service-recovery`), `disk`
+(`worker:capacity:*`), `monitoring` (the Beszel agent) and `network` (the
+Tailscale daemon). `state:*` checks take the fleet state from the bucket
+again. A feature fails when a check is reported that has no owner.
+
 Use `--apply --yes` only after review; `--plan-hash` binds application to the
 previewed steps, targets and checkout/shared-input fingerprints. The remote
 protocol independently requires explicit apply, a supported repair action and

package/docs/state.md

@@ -64,9 +64,9 @@ arguments and never written to a report or a log.
 
 **A new fleet.** Nothing more: the first `fleet-sync` seeds the bucket.
 
-**A replacement manager, or a second machine.** Clone your project, `npm ci`,
-and run `fleet-sync` with the credentials. The bucket already holds the fleet,
-so its state is downloaded before anything is changed.
+**A replacement manager.** The bucket already holds the fleet, so its state is
+downloaded before anything is changed. Follow
+[replacing the manager](manager-recovery.md).
 
 **Runs the manager starts by itself.** A worker that hands `fleet-sync` to the
 manager starts a command there that gets no environment and no project. Save
@@ -172,6 +172,7 @@ it records what it applied. Its report carries the result as the
 | `local-ahead`  | This machine has changes the bucket has not received   | The next change, or `--push`, uploads them                 |
 | `bucket-ahead` | Another manager changed the bucket; this copy is stale | The next change or `fleet-sync` takes the bucket's version |
 | `unreachable`  | The bucket did not answer, or did not take an upload   | Changes still apply locally and are uploaded later         |
+| `corrupt`      | The bucket answered with a version that cannot be read | Changes apply locally; `--restore --timestamp` repairs it  |
 | `diverged`     | Both sides changed                                     | Refused. `--restore --yes` takes the bucket's version      |
 
 A stale copy never overwrites the bucket. A machine that was offline while
@@ -210,6 +211,7 @@ once `fleet-sync` has fetched it back.
 | `DEVFLEET_CONFIG`                                                    | A configuration file, instead of the project's `fleet.json`  |
 | `DEVFLEET_STATE`                                                     | Another state directory. Earlier files are not read for it   |
 | `DEVFLEET_BUCKET`                                                    | Another `bucket.json`                                        |
+| `DEVFLEET_MANAGER_HOLDER`                                            | How the manager lease names this machine                     |
 
 How a command reaches the bucket:
 
@@ -254,7 +256,37 @@ mise exec litestream -- litestream restore -no-expand-env \
   ~/.local/share/dev-fleet/state/fleet.db
 ```
 
+`devfleet fleet-state --restore --timestamp 2026-11-01T00:00:00Z --yes` does
+the same and then uploads that version as the bucket's current one. It is the
+remedy for a `corrupt` replica: the bucket answers, but its latest version
+cannot be read.
+
 Litestream is pinned in `mise.toml` and installed the first time it is used.
-The manager is the only machine that runs it. There is one manager, so there is
-one writer; two machines changing the state at the same moment are not
-supported, only detected afterwards.
+The manager is the only machine that runs it.
+
+## One manager at a time
+
+The fleet state carries a **manager lease**: which machine is the manager,
+and until when. Every change takes or renews it, and it reaches the bucket
+with the change. A second machine (a replacement while the old manager still
+runs, or an old manager that wakes up) reads it from the bucket before it
+writes anything and stops with `FLEET_MANAGER_LEASE_HELD`. `fleet-state
+--init` still restores the state on such a machine; it only writes nothing.
+
+The lease runs for seven days after the last change. After that, any machine
+may take it. Before that, `--take-over` (on `fleet-sync`, `fleet-state --init`
+or `fleet-state --import`) makes this machine the manager; use it only once
+the other one is gone ([replacing the manager](manager-recovery.md)).
+
+A machine is named in the lease by a digest of its machine ID
+(`/etc/machine-id` on Linux, the hardware UUID on macOS), never by its host
+name, which a replacement shares. `DEVFLEET_MANAGER_HOLDER` names it instead.
+
+## Schema version
+
+The database records its schema version in `PRAGMA user_version`. The schema
+is a series of numbered migrations (`src/state/migrations/0001-….sql`, …),
+and a command brings an older database forward before it writes. A database a
+newer dev-fleet wrote is refused with `FLEET_STATE_NEWER_THAN_CLI`, read or
+written, and nothing in it changes: install the dev-fleet version the fleet
+runs.

package/docs/topology.md

@@ -41,7 +41,8 @@ your project's `fleet.json` (version 3, checked by [the schema](../schema/fleet.
   manager is asleep, closed or offline, workers have no CLI proxy and no
   dashboard; nothing fails over, because a singleton never runs twice.
 - Its state (proxy OAuth/runtime state, the hub database) is backed up off the
-  machine as each service documents, so a replacement manager restores it.
+  machine as each service documents, so a replacement manager restores it
+  ([replacing the manager](manager-recovery.md)).
 
 ## Workers
 
@@ -63,7 +64,5 @@ your project's `fleet.json` (version 3, checked by [the schema](../schema/fleet.
 
 - **Add or replace a worker:** add it to `workers`, enroll it with
   `fleet-sync`, done. Remove it by deleting the entry.
-- **Replace the manager:** edit `manager`, then move each singleton service with
-  its documented fence-and-restore runbook (never two copies at once):
-  [CLI proxy](../cliproxy/docs/migration.md),
-  [Beszel hub](../ansible/monitoring/README.md).
+- **Replace the manager:** follow
+  [replacing the manager](manager-recovery.md).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/dev-fleet",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Keeps a set of development machines in sync: one manager and many workers",
   "license": "MIT",
   "repository": "github:crvouga/dev-fleet",
@@ -72,5 +72,5 @@
   "engines": {
     "node": ">=22"
   },
-  "gitHead": "9f9718f2853ed7840c846c8dfc8750903c782b87"
+  "gitHead": "77efce7ad99a3d521169a5e773ad6b97ca21bc75"
 }

package/schema/check-ids.json

@@ -218,6 +218,10 @@
       "id": "login:*",
       "meaning": "A native per-machine login is in place."
     },
+    {
+      "id": "manager:boot-recovery",
+      "meaning": "The manager comes back by itself after a power cut or reboot."
+    },
     {
       "id": "manager:docker",
       "meaning": "The manager runs the pinned Docker Desktop."

package/src/check-owners.ts

@@ -0,0 +1,163 @@
+/** Who repairs a check: a repair recipe (`repair`), or the reason only a
+ * person can (`manual`). Every check ID dev-fleet reports matches one entry,
+ * by its exact ID or by a prefix ending in `*` (`beszel:*`). The most
+ * specific match wins. `login:*` is the sign-in recipe named after the rest
+ * of an `auth:` check's ID.
+ */
+export interface CheckOwner {
+  pattern: string;
+  repair?: string;
+  manual?: string;
+}
+
+const IDENTITY =
+  'It needs a person: a sign-in, an approval or a choice only the operator can make.';
+const EVIDENCE =
+  'It reports how a command or a run went, not the state of a machine: repeat the command once its remedy is followed.';
+const DECLARATION =
+  'The fleet configuration (fleet.json) or the roster is wrong: only the operator edits them.';
+const HARDWARE =
+  'It is a property of the hardware, the firmware or the operating system release, which no command changes.';
+
+export const CHECK_OWNERS: CheckOwner[] = [
+  // Toolchain and workstation: automatic.
+  { pattern: 'tool:*', repair: 'workspace' },
+  { pattern: 'tool:docker-*', repair: 'docker' },
+  { pattern: 'tool:github-auth*', repair: 'login:github' },
+  { pattern: 'dependencies', repair: 'workspace' },
+  { pattern: 'auth:toolchain', repair: 'workspace' },
+  { pattern: 'workstation:*', repair: 'workstation' },
+  { pattern: 'auth:*', repair: 'login:*' },
+  { pattern: 'auth:vscode', manual: IDENTITY },
+  { pattern: 'auth:workstation', manual: IDENTITY },
+  { pattern: 't3:settings', repair: 't3' },
+  { pattern: 't3:api', repair: 't3' },
+  // The checks that fail when a machine breaks.
+  { pattern: 'worker:service-recovery', repair: 'recovery' },
+  { pattern: 'worker:capacity', repair: 'disk' },
+  { pattern: 'worker:capacity:*', repair: 'disk' },
+  { pattern: 'beszel:agent', repair: 'monitoring' },
+  { pattern: 'beszel:service', repair: 'monitoring' },
+  { pattern: 'beszel:installation', repair: 'monitoring' },
+  { pattern: 'beszel:protocol', repair: 'monitoring' },
+  { pattern: 'tailscale:configuration', repair: 'network' },
+  { pattern: 'tailscale:online', repair: 'network' },
+  { pattern: 'tailscale:reachability', repair: 'network' },
+  { pattern: 'state:replica', repair: 'state' },
+  { pattern: 'state:*', repair: 'state' },
+  // What only a person can do.
+  {
+    pattern: 'beszel:*',
+    manual:
+      'The monitoring hub and its accounts are converged by fleet-sync on the manager; follow the remedy, then run fleet-sync.',
+  },
+  {
+    pattern: 'tailscale:*',
+    manual:
+      'Signing a machine in to the tailnet and approving it is a person’s; the daemon itself is repaired by the network recipe.',
+  },
+  {
+    pattern: 'state:bucket',
+    manual:
+      'The bucket and its credentials are the operator’s: name them in fleet.json and pass the credentials.',
+  },
+  { pattern: 'state:definition', manual: DECLARATION },
+  {
+    pattern: 'worker:*',
+    manual:
+      'Re-converge the worker with fleet-sync: it applies the whole worker policy at the approved revision.',
+  },
+  { pattern: 'worker:power-loss-recovery', manual: HARDWARE },
+  { pattern: 'worker:github-principal', manual: IDENTITY },
+  { pattern: 'worker:vault-renewal', manual: IDENTITY },
+  {
+    pattern: 'manager:*',
+    manual:
+      'The manager converges itself in fleet-sync; a setting that needs an administrator is applied with ansible-playbook -K ansible/macos/manager.yml.',
+  },
+  { pattern: 'manager:identity', manual: DECLARATION },
+  {
+    pattern: 'manager:docker',
+    manual:
+      'Docker Desktop is installed and updated by a person; no playbook installs it.',
+  },
+  { pattern: 'macos:*', manual: HARDWARE },
+  { pattern: 'ubuntu:release', manual: HARDWARE },
+  {
+    pattern: 'ubuntu:*',
+    manual:
+      'Re-converge the worker with fleet-sync: it applies the Ubuntu baseline.',
+  },
+  { pattern: 'platform:*', manual: HARDWARE },
+  {
+    pattern: 'ssh:*',
+    manual:
+      'SSH trust (host keys, keys in the agent, the alias) is set up by a person once per machine; dev-fleet never accepts a host key for you.',
+  },
+  { pattern: 'login:*', manual: IDENTITY },
+  { pattern: 'provider:*', manual: IDENTITY },
+  {
+    pattern: 'agent:*',
+    manual:
+      'Re-converge the machine with fleet-sync; an agent that needs a sign-in is a person’s.',
+  },
+  {
+    pattern: 't3:*',
+    manual:
+      'T3 pairing and its private route are approved by a person; fleet-sync applies the rest.',
+  },
+  { pattern: 'vscode:*', manual: IDENTITY },
+  {
+    pattern: 'git:*',
+    manual:
+      'Uncommitted or unpushed work belongs to a person; dev-fleet never resets a checkout.',
+  },
+  {
+    pattern: 'git:configuration',
+    manual:
+      'fleet-sync writes the Git identity and credential helper with ansible/github/git.yml; run it from the manager.',
+  },
+  {
+    pattern: 'github:*',
+    manual:
+      'fleet-sync signs every node in to GitHub from the stored github-token (a worker retries every ten minutes); a token that is missing, expired or short of a scope or organisation needs a person to issue a new one.',
+  },
+  {
+    pattern: 'github:git-credential',
+    manual:
+      'fleet-sync writes the GitHub credential helper with ansible/github/git.yml when the git block is declared; run it from the manager.',
+  },
+  { pattern: 'github:rate-limit', manual: EVIDENCE },
+  { pattern: 'github:workflow', manual: EVIDENCE },
+  { pattern: 'fleet:*', manual: DECLARATION },
+  { pattern: 'enrollment:*', manual: DECLARATION },
+  { pattern: 'notify:*', manual: DECLARATION },
+  {
+    pattern: 'proxy:*',
+    manual:
+      'The CLI proxy is installed with devfleet cliproxy install and its accounts are signed in by a person.',
+  },
+  {
+    pattern: 'accounts:roster-current',
+    manual:
+      'The roster reaches every node with devfleet fleet-sync --only accounts, run from the manager.',
+  },
+  { pattern: 'accounts:native-login', manual: IDENTITY },
+  { pattern: 'configuration:*', manual: EVIDENCE },
+  { pattern: 'source:*', manual: EVIDENCE },
+  { pattern: 'bootstrap:*', manual: EVIDENCE },
+  { pattern: 'rollout:*', manual: EVIDENCE },
+  { pattern: 'repair:*', manual: EVIDENCE },
+];
+
+const matches = (pattern: string, id: string): boolean =>
+  pattern.endsWith('*') ? id.startsWith(pattern.slice(0, -1)) : id === pattern;
+
+/** The entry that owns a check, or null for an ID nobody registered. */
+export function checkOwner(id: string): CheckOwner | null {
+  return (
+    CHECK_OWNERS.filter((owner) => matches(owner.pattern, id)).sort(
+      (a, b) => b.pattern.length - a.pattern.length
+    )[0] ?? null
+  );
+}