@crvouga/dev-fleet 0.5.0 → 0.7.0

package/README.md

@@ -170,16 +170,10 @@ says where the bucket is and the command is passed its credentials:
 npx devfleet fleet-state             # where the state is, and how it stands with the bucket
 ```
 
-The copy on the manager is disposable. **To replace the manager**, clone your
-project on the new machine and run the same two commands:
-
-```sh
-npm ci
-npx devfleet fleet-sync              # with the two bucket credentials in the environment
-```
-
-The roster and the approved revision come back from the bucket, and the
-machine is rebuilt from Git.
+The copy on the manager is disposable: the roster and the approved revision
+come back from the bucket, and the machine is rebuilt from Git.
+[Replacing the manager](docs/manager-recovery.md) is the runbook for a new or
+lost manager.
 
 Some runs are started by a machine rather than by you: a worker that hands
 `fleet-sync` to the manager, for example. They get no environment, so save the

package/ansible/macos/manager.yml

@@ -1,7 +1,8 @@
 ---
 # Manager power policy. Workers depend on the manager's singleton services
-# (CLI proxy, Beszel hub), so the manager never sleeps on AC power. Sleep on
-# battery is left to the human. The settings are the `pmset` object in
+# (CLI proxy, Beszel hub), so the manager never sleeps on AC power and starts
+# again by itself when power returns (autorestart). Sleep on battery is left
+# to the human. The settings are the `pmset` object in
 # policy.json; fleet-sync measures the same object as manager:power. Changing
 # them needs an administrator: ansible-playbook -K manager.yml
 - name: Keep the fleet manager awake on AC power
@@ -29,5 +30,41 @@
       when: observed != item.value
       changed_when: true
 
+# Docker Desktop hosts the CLI proxy and the monitoring hub, so it starts at
+# login: after a power cut the manager comes back without a person. The
+# setting is the `dockerDesktop` object in policy.json; fleet-sync measures it
+# as part of manager:boot-recovery. Only that key of the user's settings file
+# is changed.
+- name: Start Docker Desktop at login
+  hosts: localhost
+  connection: local
+  become: false
+  gather_facts: false
+  vars_files:
+    - policy.json
+  vars:
+    docker_settings: "{{ lookup('ansible.builtin.env', 'HOME') }}/{{ dockerDesktop.settings }}"
+  tasks:
+    - name: Find the Docker Desktop settings
+      ansible.builtin.stat:
+        path: '{{ docker_settings }}'
+      register: docker_settings_file
+    - name: Read the Docker Desktop settings
+      ansible.builtin.slurp:
+        src: '{{ docker_settings }}'
+      register: docker_settings_text
+      when: docker_settings_file.stat.exists
+    - name: Turn on start at login, keeping every other setting
+      ansible.builtin.copy:
+        dest: '{{ docker_settings }}'
+        content: "{{ current | combine({dockerDesktop.key: true}) | to_nice_json(indent=2) }}\n"
+        mode: '0644'
+      vars:
+        current: '{{ docker_settings_text.content | b64decode | from_json }}'
+      # Docker Desktop writes the file the first time it runs.
+      when:
+        - docker_settings_file.stat.exists
+        - (current[dockerDesktop.key] | default(false)) != true
+
 - name: Stage the fleet's T3 policy
   ansible.builtin.import_playbook: t3-policy.yml

package/ansible/macos/policy.json

@@ -4,7 +4,8 @@
   "architectures": ["arm64", "x86_64"],
   "pmset": {
     "sleep": "0",
-    "womp": "1"
+    "womp": "1",
+    "autorestart": "1"
   },
   "beszelDaemon": "dev.devfleet.beszel-agent",
   "disk": {
@@ -16,5 +17,9 @@
     "unhealthyChecks": 3,
     "maxRestartsPerHour": 3,
     "intervalSeconds": 60
+  },
+  "dockerDesktop": {
+    "settings": "Library/Group Containers/group.com.docker/settings-store.json",
+    "key": "AutoStart"
   }
 }

package/ansible/monitoring/agent.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: true
+  # fleet-repair re-applies this play alone for a failing beszel agent check.
+  tags: [monitoring]
   vars:
     beszel_lock: "{{ lookup('ansible.builtin.file', 'lock.json') | from_json }}"
     beszel_arch: "{{ {'x86_64': 'amd64', 'aarch64': 'arm64'}.get(ansible_facts.architecture, 'unsupported') }}"

package/ansible/ubuntu/baseline.yml

@@ -8,24 +8,30 @@
     - policy.json
   pre_tasks:
     - ansible.builtin.import_tasks: tasks/release.yml
+      tags: [always]
     - name: Require an explicit non-root user
       ansible.builtin.assert:
         that:
           - development_user is match('^[a-z_][a-z0-9_-]{0,31}$')
           - development_user != 'root'
         fail_msg: Unsupported user; name the non-root developer account.
+      tags: [always]
     - name: Read the designated existing account
       ansible.builtin.getent:
         database: passwd
         key: '{{ development_user }}'
+      tags: [always]
     - name: Refuse a different user identity or home
       ansible.builtin.assert:
         that:
           - ansible_facts.getent_passwd[development_user][1] | int == development_uid | int
           - ansible_facts.getent_passwd[development_user][4] == development_home
+      tags: [always]
     - ansible.builtin.import_tasks: tasks/dpkg.yml
     # Memory and disk safety must land even when the disk is nearly full.
+    # fleet-repair re-applies them alone (--tags disk) for worker:capacity:*.
     - ansible.builtin.import_tasks: tasks/resources.yml
+      tags: [disk]
     - name: Inspect capacity before package and service changes
       ansible.builtin.command:
         argv:
@@ -65,3 +71,5 @@
     - ansible.builtin.import_tasks: tasks/ssh.yml
     - ansible.builtin.import_tasks: tasks/credentials.yml
       when: development_source is defined
+    - ansible.builtin.import_tasks: tasks/github.yml
+      when: development_source is defined

package/ansible/ubuntu/recovery.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: false
+  # fleet-repair re-applies this play alone for worker:service-recovery.
+  tags: [recovery]
   tasks:
     - name: Verify the desktop unit runs before adding any recovery trigger
       ansible.builtin.command:

package/ansible/ubuntu/tailscale.yml

@@ -4,6 +4,8 @@
   connection: local
   become: true
   gather_facts: true
+  # fleet-repair re-applies this play alone for a failing tailscale:* check.
+  tags: [network]
   # The approved version and the signing key's digest: `tailscale` in policy.json.
   vars_files:
     - policy.json

package/ansible/ubuntu/tasks/github.yml

@@ -0,0 +1,50 @@
+---
+# This worker's GitHub sign-in, apart from its Vault renewal: a failure of one
+# never hides or stops the other. The job signs in from the stored credential
+# and adopts a rotated token while the old one still works.
+- name: Declare the GitHub sign-in job
+  vars:
+    # Every step the job runs (the secret read and four gh calls) may take
+    # its full timeout (GITHUB_STEP_TIMEOUT_MS), plus the runtime's start.
+    github_step_timeout_seconds: 30
+    github_job_steps: 5
+    github_job_start_seconds: 30
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-github.service'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      [Unit]
+      Description=Sign this worker in to GitHub from its stored credential
+      [Service]
+      Type=oneshot
+      WorkingDirectory={{ development_source }}
+      ExecStart={{ development_home }}/.local/bin/mise exec -- bun bootstrap/worker-github.ts
+      Environment="PATH={{ development_home }}/.local/share/mise/shims:{{ development_home }}/.local/bin:/usr/local/bin:/usr/bin:/bin"
+      UMask=0077
+      TimeoutStartSec={{ github_step_timeout_seconds * github_job_steps + github_job_start_seconds }}
+- name: Declare the GitHub sign-in schedule
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-github.timer'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      [Unit]
+      Description=Keep this worker signed in to GitHub with the stored credential
+      [Timer]
+      OnBootSec=2min
+      OnUnitActiveSec=10min
+      Persistent=true
+      [Install]
+      WantedBy=timers.target
+- name: Enable this user's GitHub sign-in timer
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: personal-worker-github.timer
+    daemon_reload: true
+    enabled: true
+    state: started

package/bootstrap/worker-auth.ts

@@ -1,5 +1,7 @@
-import { syncWorkerGithub } from './worker-github';
-/** Private scheduled job. The worker retains its own token; raw auth output is never logged. */
+/** Private scheduled job: renews this worker's own Vault token, and nothing
+ * else. GitHub sign-in is its own job (worker-github.ts), so a failure of
+ * one never stops or hides the other. Raw auth output is never logged.
+ */
 import { spawnSync } from 'node:child_process';
 import { resolveVaultConfig, usesVault } from '../src/vault-config';
 
@@ -49,14 +51,11 @@ if (import.meta.main) {
     jobId: 'vault-self-renewal',
     operation: 'vault.token.renew',
   });
-  const vault = renewWorkerVault();
-  const github = vault && syncWorkerGithub();
-  const ok = vault && github;
+  const ok = renewWorkerVault();
   log.info('job_finished', {
     jobId: 'vault-self-renewal',
     operation: 'vault.token.renew',
-    vaultAuthenticated: vault,
-    githubAuthenticated: github,
+    vaultAuthenticated: ok,
     outcome: ok ? 'success' : 'authentication_required',
     durationMs: Date.now() - started,
     retryDelayMs: 600000,

package/bootstrap/worker-github.ts

@@ -1,47 +1,58 @@
-/** Acquire declared service credentials from this worker's Vault session; never copy a login directory. */
-import { spawnSync } from 'node:child_process';
-import { resolveVaultConfig, usesVault } from '../src/vault-config';
-import { fleet as definition } from '../src/fleet-config';
+/** GitHub sign-in of this node from the stored credential; never copies a
+ * login directory. Runs from the timer (`personal-worker-github`) and from
+ * every sync. One JSON line, never a token or raw output.
+ */
+import { hostname } from 'node:os';
+import { fleet } from '../src/fleet-config';
+import { secretsProvider } from '../src/secrets';
+import { runCommand } from '../src/runner';
+import { hostId } from '../src/topology';
+import { githubCredential } from '../src/github/credential';
+import { githubNode } from '../src/github/policy';
+import { syncGithub, type GithubSyncResult } from '../src/github/sync';
 
-export type AuthRunner = (
-  args: string[],
-  options?: { input?: string; env?: NodeJS.ProcessEnv }
-) => { status: number | null; stdout: string | null };
-const execute: AuthRunner = (args, options = {}) =>
-  spawnSync(args[0]!, args.slice(1), {
-    encoding: 'utf8',
-    timeout: 30_000,
-    ...options,
-  });
-export function syncWorkerGithub(run: AuthRunner = execute): boolean {
-  const expected = definition.worker!.githubLogin;
-  const current = run(['gh', 'api', 'user', '--jq', '.login']);
-  if (current.status === 0 && current.stdout?.trim() === expected) return true;
-  // The service token is kept in the shared Vault document only.
-  if (!usesVault()) return false;
-  const { addr, mount, project, config } = resolveVaultConfig();
-  const secret = run([
-    'vault',
-    'kv',
-    'get',
-    `-address=${addr}`,
-    '-field=GITHUB_TOKEN',
-    `${mount}/${project}/${config}`,
-  ]);
-  const token = secret.stdout?.trim() ?? '';
-  if (secret.status !== 0 || !/^[\x21-\x7e]{16,4096}$/.test(token))
-    return false;
-  const validated = run(['gh', 'api', 'user', '--jq', '.login'], {
-    env: { ...process.env, GH_TOKEN: token, GITHUB_TOKEN: undefined },
-  });
-  if (validated.status !== 0 || validated.stdout?.trim() !== expected)
-    return false;
-  const login = run(
-    ['gh', 'auth', 'login', '--hostname', 'github.com', '--with-token'],
-    {
-      input: token + '\n',
-      env: { ...process.env, GH_TOKEN: undefined, GITHUB_TOKEN: undefined },
-    }
+/** The exit status of each outcome, so the unit's result names it too. */
+export const EXIT_CODES: Record<GithubSyncResult['code'], number> = {
+  GITHUB_OK: 0,
+  GITHUB_TOKEN_MISSING: 10,
+  GITHUB_TOKEN_REJECTED: 11,
+  GITHUB_LOGIN_MISMATCH: 12,
+  GITHUB_UNREACHABLE: 13,
+  GITHUB_SIGN_IN_FAILED: 14,
+};
+
+export const GITHUB_JOB = 'dev-fleet-github';
+
+if (import.meta.main) {
+  const started = Date.now();
+  // The manager is never a declared worker; `--node` names a worker whose
+  // host name is not its id.
+  const flag = process.argv.indexOf('--node');
+  const id = flag > 0 ? process.argv[flag + 1] : hostId(hostname());
+  const target = githubNode(fleet, id);
+  const result: GithubSyncResult =
+    target.login === null
+      ? { code: 'GITHUB_LOGIN_MISMATCH', adopted: false }
+      : await syncGithub({
+          run: runCommand,
+          token: githubCredential(
+            fleet,
+            target.node,
+            secretsProvider(fleet.secrets)
+          ),
+          login: target.login,
+          // A person's own login on the manager is measured, never replaced.
+          apply: target.managed,
+        });
+  process.stdout.write(
+    JSON.stringify({
+      name: GITHUB_JOB,
+      event: 'job_finished',
+      role: target.role,
+      code: result.code,
+      adopted: result.adopted,
+      durationMs: Date.now() - started,
+    }) + '\n'
   );
-  return login.status === 0;
+  process.exitCode = EXIT_CODES[result.code];
 }

package/docs/authentication.md

@@ -54,6 +54,7 @@ Every failure code is in [the runbooks](runbooks.md).
 | `monitoring-account-password`  | yes                          | `DEVFLEET_SECRET_MONITORING_ACCOUNT_PASSWORD`  | `~/.config/dev-fleet/secrets/monitoring-account-password`  |
 | `monitoring-tunnel-token`      | yes                          | `DEVFLEET_SECRET_MONITORING_TUNNEL_TOKEN`      | `~/.config/dev-fleet/secrets/monitoring-tunnel-token`      |
 | `proxy-tunnel-token`           | yes                          | `DEVFLEET_SECRET_PROXY_TUNNEL_TOKEN`           | `~/.config/dev-fleet/secrets/proxy-tunnel-token`           |
+| `github-token`                 | when its feature is declared | `DEVFLEET_SECRET_GITHUB_TOKEN`                 | `~/.config/dev-fleet/secrets/github-token`                 |
 | `monitoring-backup-access-key` | when its feature is declared | `DEVFLEET_SECRET_MONITORING_BACKUP_ACCESS_KEY` | `~/.config/dev-fleet/secrets/monitoring-backup-access-key` |
 | `monitoring-backup-secret-key` | when its feature is declared | `DEVFLEET_SECRET_MONITORING_BACKUP_SECRET_KEY` | `~/.config/dev-fleet/secrets/monitoring-backup-secret-key` |
 | `notify-webhook-url`           | when its feature is declared | `DEVFLEET_SECRET_NOTIFY_WEBHOOK_URL`           | `~/.config/dev-fleet/secrets/notify-webhook-url`           |
@@ -67,10 +68,11 @@ Every failure code is in [the runbooks](runbooks.md).
 
 | Check                         | Measured in                                |
 | ----------------------------- | ------------------------------------------ |
+| `auth:github`                 | `src/github/checks.ts`                     |
 | `auth:vault`                  | `src/cli/fleet-sync-vault.ts`              |
 | `worker:git-identity`         | `src/git-identity.ts`                      |
 | `worker:git-push-auth`        | `src/definition.ts`, `src/git-identity.ts` |
-| `worker:github-principal`     | `src/worker-access-checks.ts`              |
+| `worker:github-sync`          | `src/worker-access-checks.ts`              |
 | `worker:proxy-authentication` | `src/worker-access-checks.ts`              |
 | `worker:vault-renewal`        | `src/worker-access-checks.ts`              |
 
@@ -84,6 +86,7 @@ Every failure code is in [the runbooks](runbooks.md).
 | -------------------------------- | -------------------------------------- | ------------------------------------- |
 | `personal-fleet-disk-gc.timer`   | `ansible/ubuntu/tasks/resources.yml`   | OnBootSec=10min, OnCalendar=hourly    |
 | `personal-worker-auth.timer`     | `ansible/ubuntu/tasks/credentials.yml` | OnBootSec=2min, OnUnitActiveSec=10min |
+| `personal-worker-github.timer`   | `ansible/ubuntu/tasks/github.yml`      | OnBootSec=2min, OnUnitActiveSec=10min |
 | `personal-worker-recovery.timer` | `ansible/ubuntu/tasks/recovery.yml`    | OnBootSec=1min, OnUnitActiveSec=1min  |
 | `personal-worker-recovery.timer` | `ansible/ubuntu/tasks/recovery.yml`    | OnBootSec=5min, OnUnitActiveSec=15min |
 

package/docs/consumer-setup.md

@@ -179,6 +179,7 @@ Then replace every placeholder. Each field, from
 | `ubuntu`                                              | `{ "release": "26.04", "architectures": ["amd64", "arm64"] }`: a release of the [support matrix](support-matrix.md). |
 | `worker.requiredCapabilities`                         | Normally all of `github`, `vault`, `t3`, `toolchain`, `workstation`, `worker`.                                       |
 | `worker.githubLogin`                                  | The GitHub login every worker must be signed in as.                                                                  |
+| `github` (optional)                                   | Who the manager signs in to GitHub as, per-node tokens, required scopes; see [GitHub](github.md).                    |
 | `worker.vaultUser`                                    | The Vault user each worker's token is issued for.                                                                    |
 | `worker.toolchain`, `agentProfile`                    | `"mise.lock"`, `"proxy"` (fixed).                                                                                    |
 | `worker.unattendedSudo`, `persistentT3`, `neverSleep` | `true` (fixed policy).                                                                                               |
@@ -206,6 +207,10 @@ dev-fleet asks for four secrets by purpose: `proxy-client-key`,
 | `file`   | `{ "provider": "file" }` (optional `"directory"`)                                                                                                    | One private file per name, default `~/.config/dev-fleet/secrets/NAME`.  |
 | `env`    | `{ "provider": "env" }`                                                                                                                              | `DEVFLEET_SECRET_PROXY_CLIENT_KEY` and so on (upper case, `_` for `-`). |
 
+A fifth, `github-token`, is the GitHub token of `worker.githubLogin`: every node
+is signed in with it ([GitHub](github.md)). With `vault` it needs no location
+of its own: it defaults to the field `GITHUB_TOKEN` beside `proxy-client-key`.
+
 Only `vault` gives each worker its own login and renewal and gives the CLI
 proxy its encrypted backup; with the other three those two features are not
 available yet. You never write a secret value into the project; the human puts
@@ -377,10 +382,8 @@ and codes; leave out everything private.
 - **Upgrade the fleet.** `npm install --save-dev --save-exact @crvouga/dev-fleet@latest`,
   commit, `npx devfleet fleet-sync`. The canary proves the new revision first.
   Install an older version and sync to roll back.
-- **Replace the manager.** On the new Mac: clone the project, `npm ci`, put
-  the bucket credentials in the environment, `npx devfleet fleet-sync`. The
-  roster and the approved revision come back from the bucket. See
-  [state](state.md) for what the bucket does not hold yet.
+- **Replace the manager.** Follow
+  [replacing the manager](manager-recovery.md).
 
 ## Moving from a pinned commit and a wrapper script
 

package/docs/github.md

@@ -0,0 +1,107 @@
+# GitHub on every node
+
+Every node, the manager and each worker, signs in to GitHub from a stored
+credential. Nobody runs `gh auth login` on a node, and no login directory is
+copied between machines.
+
+## Where the token is
+
+The token is the declared secret `github-token`, read through the secrets
+provider your `fleet.json` names, like every other secret:
+
+| Provider | Shared token                                                              | Per-node token (`"credential": "per-node"`) |
+| -------- | ------------------------------------------------------------------------- | ------------------------------------------- |
+| `env`    | `DEVFLEET_SECRET_GITHUB_TOKEN`                                            | `DEVFLEET_SECRET_GITHUB_TOKEN_<NODE>`       |
+| `file`   | `github-token`                                                            | `github-token-<node>`                       |
+| `sops`   | key `github-token`                                                        | key `github-token-<node>`                   |
+| `vault`  | `locations["github-token"]`, or field `GITHUB_TOKEN` beside the proxy key | the same location, field `<FIELD>_<NODE>`   |
+
+The login it must belong to is `worker.githubLogin`.
+
+## The `github` block
+
+All of it is optional:
+
+```json
+"github": {
+  "manager": "fleet",
+  "credential": "shared",
+  "scopes": ["repo", "workflow"],
+  "repositories": ["example-org/app"],
+  "organizations": ["example-org"],
+  "sandbox": "example-org/sandbox"
+}
+```
+
+- `manager`: `"fleet"` (the default) signs the manager in as the fleet login,
+  like a worker. `{ "login": "…" }` names a person's own login: it is checked
+  and never replaced.
+- `credential`: `"shared"` (one token) or `"per-node"` (one per node, so one
+  can be revoked alone and nodes do not share a rate limit).
+- `scopes`, `repositories`, `organizations`: what `github:permissions` and
+  `github:org-access` hold the token to.
+- `sandbox`: the repository `fleet-github-verify` may use.
+
+## How a node is signed in
+
+`bootstrap/worker-github.ts` runs at every `fleet-sync` and, on a worker, every
+ten minutes from the user timer `personal-worker-github`. It is apart from the
+Vault renewal (`personal-worker-auth`): one failing never stops or hides the
+other. It signs gh in with the stored token when the node is not signed in, or
+when the stored token differs from the one gh holds and is valid for the
+declared login. git asks gh for the token at every use
+(`credential.https://github.com…helper = !gh auth git-credential`, set by
+`ansible/github/git.yml` with the identity of the `git` block on every node),
+so a replaced token reaches git with no change and no second copy on disk.
+
+A `fleet-sync` fails with `github:sync` when the node cannot be signed in.
+
+## Rotating the token
+
+1. Issue a new token for the fleet login and put it in the store as
+   `github-token`.
+2. Run `fleet-sync` (the canary worker first, as always), or wait up to ten
+   minutes for every node's timer. Each node adopts the new token while the old
+   one still works.
+3. When `github:token-current` passes on every node, revoke the old token.
+
+## Checks
+
+| Check                   | What it says                                                                                                                                                            |
+| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `auth:github`           | gh answers `auth status` in this context.                                                                                                                               |
+| `github:principal`      | The node is signed in as its declared login (the fleet login, or the manager's personal one).                                                                           |
+| `github:token-current`  | The node uses the token the store holds now (`GITHUB_TOKEN_STALE` otherwise). Not reported for a personal login.                                                        |
+| `github:permissions`    | The token has every declared scope and can push to every declared repository (`GITHUB_PERMISSION_MISSING: …`). Warns for a fine-grained token, which reports no scopes. |
+| `github:expiry`         | Warns 14 days before the token expires, fails 3 days before.                                                                                                            |
+| `github:rate-limit`     | Warns below 20 % of the rate limit left, with the reset time.                                                                                                           |
+| `github:org-access`     | Single sign-on has authorised the token for each declared organisation.                                                                                                 |
+| `github:git-credential` | git asks gh for its GitHub credential.                                                                                                                                  |
+| `worker:github-sync`    | Why a worker is not signed in: `GITHUB_TOKEN_MISSING`, `GITHUB_TOKEN_REJECTED`, `GITHUB_LOGIN_MISMATCH`, `GITHUB_UNREACHABLE`.                                          |
+| `auth:copilot`          | Copilot CLI uses the node's GitHub sign-in; fails `not-signed-in` without one and `COPILOT_TOKEN_UNSUPPORTED` for a classic token.                                      |
+| `github:workflow`       | From `fleet-github-verify`: the first step of clone, branch, commit, push, pull request, checks and comment that failed.                                                |
+
+## Copilot CLI
+
+Copilot CLI takes the token gh holds when it has none of its own, so the
+node's GitHub sign-in is its sign-in. It accepts OAuth, GitHub App user and
+fine-grained tokens with the Copilot Requests permission, not classic
+personal access tokens. Whether the account is entitled to Copilot is not
+measured; no prompt is sent.
+
+## Verifying the whole workflow
+
+`devfleet fleet-github-verify` runs, from the machine it is started on, what an
+agent does on the sandbox repository: clone, a uniquely named branch, an empty
+commit with the configured identity, push, a draft pull request, a read of its
+checks, a comment. It then closes the pull request and deletes the branch,
+whatever failed, and reports anything it could not remove.
+
+## Not yet
+
+- Short-lived GitHub App installation tokens minted by the manager.
+- A container registry, package registry and tool installer adapter: today
+  only git takes its credential from gh.
+- A launchd timer for the manager: it is signed in at every `fleet-sync`.
+- `fleet-github-verify` runs on the machine it is started on; `--node` and a
+  scheduled run are not there yet.

package/docs/manager-recovery.md

@@ -0,0 +1,78 @@
+# Replacing the manager
+
+This is the one runbook for a manager that is replaced or lost: stolen, a dead
+disk, or a new machine. Other documents link here and do not describe the
+procedure themselves.
+
+## What must exist off the machine
+
+Nothing on the old manager is needed. Before you need this runbook, make sure
+these exist elsewhere:
+
+| What                                | Where                                                                    |
+| ----------------------------------- | ------------------------------------------------------------------------ |
+| Your project: `fleet.json`, the pin | Its Git remote                                                           |
+| The fleet state                     | The bucket `fleet.json` names under `state`                              |
+| The bucket's two credentials        | Your password manager (never the manager alone)                          |
+| Secrets                             | Your `secrets` provider; `file` and `env` secrets must be kept elsewhere |
+| The CLI proxy's encrypted snapshots | Its restic repository (`vault` secrets provider only)                    |
+| Your Tailscale and Docker accounts  | Yours: signing in is the only step only you can do                       |
+
+## Restore
+
+On the new machine, with Git, Node and Docker installed:
+
+1. Give it the manager's local host name: `manager.id` in `fleet.json`
+   (`sudo scutil --set LocalHostName ID` on macOS). `fleet-sync` refuses to
+   run on any other machine.
+2. Sign in to Tailscale and approve the device. Sign in to Docker Desktop if
+   your plan needs it.
+3. Fetch the project and check every prerequisite. Nothing is changed:
+
+   ```sh
+   git clone YOUR-PROJECT && cd YOUR-PROJECT && npm ci
+   export DEVFLEET_BUCKET_ACCESS_KEY_ID=… DEVFLEET_BUCKET_SECRET_ACCESS_KEY=…
+   npx devfleet fleet-restore --dry-run
+   ```
+
+   It fails on anything missing (`restore:*` checks, each with its remedy)
+   and lists what will be restored, in order.
+
+4. Restore, then converge:
+
+   ```sh
+   npx devfleet fleet-restore              # the fleet state, from the bucket
+   npx devfleet fleet-sync --take-over     # the manager, then every worker
+   npx devfleet fleet-state --init         # credentials for runs the manager starts itself
+   npx devfleet cliproxy install           # the proxy, from its snapshots
+   ```
+
+   The proxy's own steps are in [its migration runbook](../cliproxy/docs/migration.md).
+   `--take-over` is needed while the old manager's lease runs (seven days
+   after its last change). Use it only once the old machine is gone: an old
+   manager that wakes up afterwards is refused with `FLEET_MANAGER_LEASE_HELD`.
+
+5. Adopt the T3 instances again: `npx devfleet t3-apply --import ID`.
+
+Workers need nothing: they reach the manager by its Tailscale name, so they
+follow it once the new machine holds that name. If the Tailscale name
+changed, change `manager.network.tailscaleName` in `fleet.json` and run
+`fleet-sync`, which re-points every worker.
+
+## What is lost
+
+The monitoring hub's metric history, alerts and settings made by hand. Its
+accounts and systems are recreated. With `env` or `file` secrets, whatever
+was only on the old manager.
+
+## Coming back by itself
+
+A manager restarts after a power cut only when it is set up to
+(`manager:boot-recovery` in `fleet-status`): `autorestart` and Docker Desktop
+at login are applied by `ansible/macos/manager.yml`. The per-user services
+need a user session. **With FileVault on, a Mac waits at the login window
+after every restart until a person signs in**, so nothing comes back
+unattended. That is the trade-off: FileVault protects the proxy's logins and
+the bucket credentials on a stolen Mac; automatic login without FileVault
+lets the fleet recover from a power cut alone. `fleet-status` reports
+FileVault as a warning with that consequence; the choice is yours.

package/docs/operations.md

@@ -107,7 +107,7 @@ remote nodes' required lists. Run `fleet-nodes --schema` for editor/agent valida
 | `ssh:agent`                                                                        | Manager agent exposes an identity; missing agent warns because IdentityFile keys may still work.                                                                      |
 | `ssh:session`                                                                      | Fresh batch SSH session actually runs the Node/Bun probe. No reused multiplexed connection or agent forwarding.                                                       |
 | `ssh:host-key`, `ssh:dns`, `ssh:auth`, `ssh:network`, `ssh:runtime`, `ssh:timeout` | Classified failure and targeted remedy, without raw logs.                                                                                                             |
-| `auth:github`                                                                      | gh auth status --active succeeds on this machine; not proof of Git SSH or every repository permission.                                                                |
+| `auth:github`                                                                      | gh auth status --active succeeds; the `github:*` checks ([GitHub](github.md)) say which login, token, permissions and expiry.                                         |
 | `auth:vault`                                                                       | Server-side vault token lookup succeeds in this context; not proof of access to every secret path.                                                                    |
 | `auth:vscode`                                                                      | code CLI finds the local Remote-SSH extension. Other profiles/Cursor/GUI PATH/agents can differ.                                                                      |
 | `t3:settings`                                                                      | Connect explicitly enabled and not revoked in t3 status --json; settings are not connectivity.                                                                        |
@@ -304,6 +304,14 @@ The public CLI exposes `devfleet fleet-repair --list` (static catalog) and
 in this package only executes allowlisted existing owner commands. Native
 account logins and unknown/manual checks always remain operator steps.
 
+Every check ID has an owner in `src/check-owners.ts`: a recipe, or the reason
+only a person can repair it, which the plan prints beside the check
+(`manual`). Machine repairs are one tag of a playbook the worker already
+applies: `recovery` (`worker:service-recovery`), `disk`
+(`worker:capacity:*`), `monitoring` (the Beszel agent) and `network` (the
+Tailscale daemon). `state:*` checks take the fleet state from the bucket
+again. A feature fails when a check is reported that has no owner.
+
 Use `--apply --yes` only after review; `--plan-hash` binds application to the
 previewed steps, targets and checkout/shared-input fingerprints. The remote
 protocol independently requires explicit apply, a supported repair action and