@crvouga/dev-fleet 0.3.0 → 0.5.0

package/ansible/ubuntu/tasks/dpkg.yml

@@ -0,0 +1,46 @@
+---
+# A package database left inconsistent by an interrupted run is repaired
+# before any package task runs, or stops the play with APT_STATE_BROKEN.
+- name: Wait for legitimate package transactions before inspecting dpkg
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/python3
+      - '-c'
+      - |
+        import fcntl
+        files = [open(path, 'a') for path in ['/var/lib/dpkg/lock-frontend', '/var/lib/dpkg/lock']]
+        for handle in files:
+            fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  register: apt_idle
+  changed_when: false
+  retries: 24
+  delay: 5
+  until: apt_idle.rc == 0
+- name: Inspect package database consistency
+  ansible.builtin.command: dpkg --audit
+  changed_when: false
+  register: dpkg_audit
+# Power lost during a package run leaves dpkg half-configured. Finishing
+# that run is the standard safe repair: it is tried once, bounded, after
+# the lock wait above, and only a database still inconsistent afterwards
+# needs a person.
+- name: Repair an interrupted package run once
+  when: dpkg_audit.stdout | trim != ''
+  block:
+    - name: Finish configuring every unpacked package, bounded
+      ansible.builtin.command:
+        argv: [timeout, '600', dpkg, --configure, -a]
+      environment:
+        DEBIAN_FRONTEND: noninteractive
+    - name: Inspect package database consistency after the repair
+      ansible.builtin.command: dpkg --audit
+      changed_when: false
+      register: dpkg_reaudit
+  rescue:
+    - name: Report a package run the repair could not finish
+      ansible.builtin.fail:
+        msg: APT_STATE_BROKEN; dpkg --configure -a did not finish. Inspect dpkg --audit and use reviewed package recovery. Never remove package lock files.
+- name: Require a consistent package database before modifying packages
+  ansible.builtin.assert:
+    that: (dpkg_reaudit.stdout | default(dpkg_audit.stdout)) | trim == ''
+    fail_msg: APT_STATE_BROKEN; dpkg --configure -a did not clear dpkg --audit. Inspect it and use reviewed package recovery. Never remove package lock files.

package/ansible/ubuntu/tasks/install.yml

@@ -0,0 +1,38 @@
+---
+# The one way a pinned Ubuntu package is installed: from the snapshot,
+# waiting for the APT lock, and never downgrading. A pin is a minimum: a
+# package Ubuntu's security updates moved past its pin is left as it is and
+# reported as a warning (docs/worker-contract.md). `install_pins` maps each
+# package to its pinned version.
+- ansible.builtin.import_tasks: snapshot.yml
+- name: Read the installed package versions
+  ansible.builtin.package_facts:
+    manager: apt
+- name: Compare each installed pinned package with its pin
+  ansible.builtin.command:
+    argv:
+      - dpkg
+      - --compare-versions
+      - '{{ ansible_facts.packages[item][0].version }}'
+      - ge
+      - '{{ install_pins[item] }}'
+  loop: "{{ install_pins.keys() | select('in', ansible_facts.packages) | list }}"
+  register: pin_comparison
+  changed_when: false
+  failed_when: pin_comparison.rc not in [0, 1]
+  check_mode: false
+- name: Select the pinned packages that are absent or older than their pin
+  ansible.builtin.set_fact:
+    below_pin: "{{ (install_pins.keys() | reject('in', ansible_facts.packages) | list) + (pin_comparison.results | selectattr('rc', 'equalto', 1) | map(attribute='item') | list) }}"
+- name: Install the pinned version of each package below its pin, without restarting services
+  ansible.builtin.apt:
+    name: '{{ item }}={{ install_pins[item] }}'
+    state: present
+    lock_timeout: '{{ aptLockTimeoutSeconds }}'
+    allow_downgrade: false
+    allow_unauthenticated: false
+    fail_on_autoremove: true
+    policy_rc_d: 101
+  environment:
+    APT_CONFIG: /etc/personal-fleet/apt.conf
+  loop: '{{ below_pin }}'

package/ansible/ubuntu/tasks/packages.yml

@@ -1,56 +1,5 @@
 ---
-- name: Create owned APT configuration directory
-  ansible.builtin.file:
-    path: /etc/personal-fleet
-    state: directory
-    owner: root
-    group: root
-    mode: '0755'
-- name: Declare snapshot sources used only for managed package transactions
-  ansible.builtin.copy:
-    dest: /etc/personal-fleet/ubuntu.sources
-    owner: root
-    group: root
-    mode: '0644'
-    backup: true
-    content: |
-      Types: deb
-      URIs: https://snapshot.ubuntu.com/ubuntu/{{ snapshot }}/
-      Suites: resolute resolute-updates resolute-security
-      Components: main universe restricted multiverse
-      Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
-  register: snapshot_sources
-- name: Isolate snapshot selection from unrelated APT operations
-  ansible.builtin.copy:
-    dest: /etc/personal-fleet/apt.conf
-    owner: root
-    group: root
-    mode: '0644'
-    content: |
-      Dir::Etc::sourcelist "/etc/personal-fleet/ubuntu.sources";
-      Dir::Etc::sourceparts "-";
-      DPkg::Lock::Timeout "{{ aptLockTimeoutSeconds }}";
-- name: Inspect immutable snapshot package index
-  ansible.builtin.stat:
-    path: '/var/lib/apt/lists/snapshot.ubuntu.com_ubuntu_{{ snapshot }}_dists_resolute_InRelease'
-  register: snapshot_index
-- name: Fetch signed snapshot indexes after source change or interrupted setup
-  ansible.builtin.apt:
-    update_cache: true
-    cache_valid_time: 0
-    lock_timeout: '{{ aptLockTimeoutSeconds }}'
-  environment:
-    APT_CONFIG: /etc/personal-fleet/apt.conf
-  when: snapshot_sources.changed or not snapshot_index.stat.exists
-- name: Install exact managed packages without downgrading or restarting services
-  ansible.builtin.apt:
-    name: '{{ item.key }}={{ item.value }}'
-    state: present
-    lock_timeout: '{{ aptLockTimeoutSeconds }}'
-    allow_downgrade: false
-    allow_unauthenticated: false
-    fail_on_autoremove: true
-    policy_rc_d: 101
-  environment:
-    APT_CONFIG: /etc/personal-fleet/apt.conf
-  loop: '{{ packages | dict2items }}'
+- ansible.builtin.include_tasks: install.yml
+  vars:
+    install_pins: '{{ packages }}'
+- ansible.builtin.import_tasks: unattended-upgrades.yml

package/ansible/ubuntu/tasks/release.yml

@@ -0,0 +1,25 @@
+---
+# The accepted releases are named once, in policy.json. This selects the
+# entry of the running release and refuses everything else.
+- name: Select the accepted release this machine runs
+  ansible.builtin.set_fact:
+    ubuntu_release: "{{ releases | selectattr('release', 'equalto', ansible_facts.distribution_version) | first | default({}) }}"
+- name: Require an accepted release, its codename and one of its architectures
+  ansible.builtin.assert:
+    that:
+      - ansible_facts.distribution == 'Ubuntu'
+      - ubuntu_release.release is defined
+      - ansible_facts.distribution_release == ubuntu_release.codename
+      - "(ansible_facts.architecture | replace('x86_64', 'amd64') | replace('aarch64', 'arm64')) in ubuntu_release.architectures"
+    fail_msg: UBUNTU_BASELINE_MISMATCH; workers run a release docs/support-matrix.md lists. No distribution upgrade is permitted.
+- name: Pin the packages of this release, its own pins over the common ones
+  ansible.builtin.set_fact:
+    packages: '{{ packages | combine(ubuntu_release.packages | default({})) }}'
+- name: Read the kernel release to tell WSL2 apart from a machine of its own
+  ansible.builtin.slurp:
+    src: /proc/sys/kernel/osrelease
+  register: kernel_release
+- name: Refuse WSL2 by name
+  ansible.builtin.assert:
+    that: not (kernel_release.content | b64decode | lower is search('microsoft|wsl'))
+    fail_msg: WORKER_ENVIRONMENT_UNSUPPORTED; WSL2 is not a worker. Run the worker on a physical or virtual machine of its own (docs/support-matrix.md).

package/ansible/ubuntu/tasks/resources.yml

@@ -49,10 +49,11 @@
     line: /swap.fleet none swap sw 0 0
   when: active_swap.stdout | trim == ''
 
-- name: Install the systemd userspace OOM killer
-  ansible.builtin.apt:
-    name: systemd-oomd
-    state: present
+- name: Install the systemd userspace OOM killer at its pin
+  ansible.builtin.include_tasks: install.yml
+  vars:
+    install_pins:
+      systemd-oomd: "{{ packages['systemd-oomd'] }}"
 - name: Create memory and disk policy directories
   ansible.builtin.file:
     path: '{{ item }}'
@@ -224,7 +225,7 @@
       StartLimitIntervalSec=0
       [Service]
       Type=simple
-      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
+      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --inode-floor-percent {{ resourceSafety.inodeFailFreePercent }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
       Restart=always
       RestartSec=5
       OOMScoreAdjust=-1000
@@ -276,7 +277,7 @@
         After=docker.service
         [Service]
         Type=oneshot
-        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
+        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-inode-percent {{ resourceSafety.inodeWarnFreePercent }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}{% for root in resourceSafety.diskGcWorktreeRoots %} --worktree-root {{ root }}{% endfor %} --worktree-idle-days {{ resourceSafety.diskGcWorktreeIdleDays }}{% for cache, mib in resourceSafety.diskGcCacheCapsMiB.items() %} --cache-cap {{ cache }}={{ mib }}{% endfor %} --keep-newest /var/cache/personal-fleet={{ resourceSafety.diskGcPackagesKept }} --mise {{ development_home }}/.local/bin/mise --user {{ development_user }}
 
         TimeoutStartSec=2h
         Nice=19

package/ansible/ubuntu/tasks/services.yml

@@ -29,3 +29,16 @@
   ansible.builtin.systemd_service:
     daemon_reload: true
   when: recovery_dropins.changed
+# A skewed clock breaks TLS, tokens, apt signatures and Vault logins with
+# errors that never mention time. Ubuntu ships one time service (chrony on
+# newer releases, systemd-timesyncd before); whichever is installed stays
+# enabled. A container takes its clock from its host. Measured as worker:clock.
+- name: Inspect the time services this worker has
+  ansible.builtin.service_facts:
+- name: Keep the clock synchronised with the time service Ubuntu ships
+  ansible.builtin.systemd_service:
+    name: '{{ item }}'
+    enabled: true
+    state: started
+  loop: "{{ (['chrony.service', 'systemd-timesyncd.service'] | select('in', ansible_facts.services) | list)[:1] }}"
+  when: ansible_facts.virtualization_type not in ['docker', 'container', 'podman', 'lxc', 'systemd-nspawn']

package/ansible/ubuntu/tasks/snapshot.yml

@@ -0,0 +1,50 @@
+---
+# The snapshot every managed package comes from, separate from the system's
+# own sources. Included before any pinned package is installed; a second
+# inclusion in one run changes nothing.
+- name: Create owned APT configuration directory
+  ansible.builtin.file:
+    path: /etc/personal-fleet
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+- name: Declare snapshot sources used only for managed package transactions
+  ansible.builtin.copy:
+    dest: /etc/personal-fleet/ubuntu.sources
+    owner: root
+    group: root
+    mode: '0644'
+    backup: true
+    content: |
+      Types: deb
+      URIs: https://snapshot.ubuntu.com/ubuntu/{{ snapshot }}/
+      Suites: {{ ubuntu_release.codename }} {{ ubuntu_release.codename }}-updates {{ ubuntu_release.codename }}-security
+      Components: main universe restricted multiverse
+      Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+  register: snapshot_sources
+# Fetching the snapshot indexes must leave the indexes of the system's own
+# sources in place: the system's APT and unattended upgrades still use them.
+- name: Isolate snapshot selection from unrelated APT operations
+  ansible.builtin.copy:
+    dest: /etc/personal-fleet/apt.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      Dir::Etc::sourcelist "/etc/personal-fleet/ubuntu.sources";
+      Dir::Etc::sourceparts "-";
+      APT::Get::List-Cleanup "false";
+      DPkg::Lock::Timeout "{{ aptLockTimeoutSeconds }}";
+- name: Inspect immutable snapshot package index
+  ansible.builtin.stat:
+    path: '/var/lib/apt/lists/snapshot.ubuntu.com_ubuntu_{{ snapshot }}_dists_{{ ubuntu_release.codename }}_InRelease'
+  register: snapshot_index
+- name: Fetch signed snapshot indexes after source change or interrupted setup
+  ansible.builtin.apt:
+    update_cache: true
+    cache_valid_time: 0
+    lock_timeout: '{{ aptLockTimeoutSeconds }}'
+  environment:
+    APT_CONFIG: /etc/personal-fleet/apt.conf
+  when: snapshot_sources.changed or not snapshot_index.stat.exists

package/ansible/ubuntu/tasks/unattended-upgrades.yml

@@ -0,0 +1,24 @@
+---
+# Ubuntu installs security updates by itself, and they keep flowing: a pin is
+# a minimum, never a hold. What runs unattended is declared here so it is
+# known: security updates only, never a reboot, never a removal. The fleet's
+# own file sorts after the package's 50unattended-upgrades and replaces its
+# origins.
+- name: Declare which updates Ubuntu installs unattended
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/52personal-fleet-unattended-upgrades
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      // Managed by personal Fleet: unattended upgrades
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Unattended-Upgrade "1";
+      #clear Unattended-Upgrade::Allowed-Origins;
+      #clear Unattended-Upgrade::Origins-Pattern;
+      Unattended-Upgrade::Allowed-Origins {
+        "${distro_id}:${distro_codename}-security";
+      };
+      Unattended-Upgrade::Automatic-Reboot "false";
+      Unattended-Upgrade::Remove-Unused-Dependencies "false";
+      Unattended-Upgrade::Remove-New-Unused-Dependencies "false";

package/bootstrap/doctor.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  NO_SELECTION,
+  parseSelectionArguments,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 /** Read-only toolchain checks; no logins, installations or changes to Docker. */
 import { spawnSync } from 'node:child_process';
 import { readFileSync } from 'node:fs';
@@ -15,7 +20,7 @@ import {
 type Tool = string | { version: string; lazy?: boolean; os?: string[] };
 type Config = {
   tools: Record<string, Tool>;
-  vars?: { worker_tools?: unknown };
+  vars?: Record<string, unknown>;
 };
 export type Check = {
   id: string;
@@ -42,6 +47,11 @@ const VERSION_ARGS: Record<string, string[]> = {
   kubectl: ['version', '--client=true', '--output=json'],
   helm: ['version', '--short'],
 };
+/** The version a tool prints, where it differs from its mise pin. */
+const PRINTED: Record<string, (pin: string) => string> = {
+  // temurin-25.0.5+8.0.LTS prints as 25.0.5.
+  java: (pin) => pin.replace(/^[a-z]+-/, '').replace(/\+.*$/, ''),
+};
 const run: Runner = (command) => {
   const result = spawnSync(command[0]!, command.slice(1), {
     encoding: 'utf8',
@@ -80,7 +90,8 @@ export function toolChecks(
     if (typeof tool !== 'string' && tool.lazy) continue;
     // A tool pinned only for other platforms has no artifact to verify here.
     if (unavailable(id, tool, platform)) continue;
-    const version = typeof tool === 'string' ? tool : tool.version;
+    const pin = typeof tool === 'string' ? tool : tool.version;
+    const version = PRINTED[id]?.(pin) ?? pin;
     const command = COMMANDS[id] ?? id;
     const result = execute([command, ...(VERSION_ARGS[id] ?? ['--version'])]);
     const exact = new RegExp(
@@ -195,7 +206,21 @@ export function globalConfigCheck(
   };
 }
 
-export function checkMachine(requireDocker = false, worker = false): Check[] {
+/** A worker installs every tool it selects, lazy or not, so each is checked. */
+function installed(tools: Record<string, Tool>): Record<string, Tool> {
+  return Object.fromEntries(
+    Object.entries(tools).map(([id, tool]) => [
+      id,
+      typeof tool === 'string' ? tool : { ...tool, lazy: false },
+    ])
+  );
+}
+
+export function checkMachine(
+  requireDocker = false,
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
+): Check[] {
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Config;
@@ -203,7 +228,7 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
   return [
     ...toolChecks(
       worker
-        ? workerTools(config.tools, config.vars?.worker_tools)
+        ? installed(workerToolset(config.tools, config.vars ?? {}, selection))
         : config.tools
     ),
     ...dockerChecks(requireDocker),
@@ -222,7 +247,8 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
 if (import.meta.main) {
   const checks = checkMachine(
     process.argv.includes('--docker'),
-    process.argv.includes('--worker')
+    process.argv.includes('--worker'),
+    parseSelectionArguments(process.argv)
   );
   const ok = checks.every((check) => check.status !== 'fail');
   if (process.argv.includes('--json'))

package/bootstrap/fleet-remote-bootstrap.sh

@@ -2,14 +2,17 @@
 # Executed over strict SSH from the selected verified Git revision. stdin is
 # one line holding the base64 fleet configuration (empty when there is none),
 # then the base64 Git bundle (nothing in cached mode). All executable content
-# remains in this tracked source.
+# remains in this tracked source. The Ubuntu releases workers may run are
+# named once, in ansible/ubuntu/policy.json; the manager passes them in.
 set -eu
 umask 077
 revision=$1
 bundle_sha=$2
 node=$3
 source_mode=${4:-transfer}
+releases=${5:-}
 case "$source_mode" in cached|transfer) ;; *) exit 2;; esac
+case "$releases" in *[!0-9.\ ]*|'') exit 2;; esac
 case "$revision" in *[!a-f0-9]*|'') exit 2;; esac
 [ "${#revision}" = 40 ] || exit 2
 case "$bundle_sha" in *[!a-f0-9]*|'') exit 2;; esac
@@ -26,7 +29,8 @@ remedy() {
   case "$1" in
     FLEET_REMOTE_LOCKED) echo 'Another fleet operation holds this machine. Wait for it to finish; rerun fleet-sync afterwards. Never remove the lock while its holder runs.';;
     ROOT_DEVELOPMENT_USER_REJECTED) echo 'Connect as the non-root developer account the fleet configuration names.';;
-    UBUNTU_BASELINE_MISMATCH|UBUNTU_ARCHITECTURE_UNSUPPORTED|MACOS_BASELINE_MISMATCH|MACOS_ARCHITECTURE_UNSUPPORTED|PLATFORM_UNSUPPORTED) echo 'Workers run Ubuntu 26.04 on x86_64 or aarch64. Reinstall the machine; fleet-sync never upgrades an operating system.';;
+    UBUNTU_BASELINE_MISMATCH|UBUNTU_ARCHITECTURE_UNSUPPORTED|PLATFORM_UNSUPPORTED) echo 'Workers run an Ubuntu release docs/support-matrix.md lists, on x86_64 or aarch64. Reinstall the machine; fleet-sync never upgrades an operating system.';;
+    WORKER_ENVIRONMENT_UNSUPPORTED) echo 'WSL2 is not a worker. Run the worker on a physical or virtual machine of its own (docs/support-matrix.md).';;
     FLEET_STATE_PATH_UNSAFE) echo 'A fleet state directory under ~/.local/share/personal is a symbolic link. Replace it with a real directory.';;
     BOOTSTRAP_CAPACITY_UNVERIFIED|BOOTSTRAP_DISK_CAPACITY_REQUIRED) echo 'Free at least 2 GiB and 50000 inodes on the home file system, then rerun fleet-sync.';;
     SOURCE_DIGEST_MISMATCH) echo 'The bundle changed in transit. Rerun fleet-sync; if it repeats, check the SSH route.';;
@@ -37,11 +41,9 @@ remedy() {
     APT_DISK_FULL) echo 'Package installation ran out of disk space. Free space under /var and /usr, then rerun fleet-sync.';;
     DPKG_INTERRUPTED) echo 'An earlier package installation that fleet-sync did not start was interrupted. Run sudo dpkg --configure -a once no package process is running, then rerun fleet-sync.';;
     APT_UPDATE_FAILED|APT_INSTALL_FAILED) echo 'Run sudo apt-get update and install the bootstrap packages by hand to see the error, then rerun fleet-sync.';;
-    XCODE_COMMAND_LINE_TOOLS_REQUIRED) echo 'Install the Xcode Command Line Tools (xcode-select --install), then rerun fleet-sync.';;
-    BOOTSTRAP_LOCK_TOOL_REQUIRED) echo 'Install python3 so the operation lock can be taken, then rerun fleet-sync.';;
     SOURCE_CACHE_CHANGED|SOURCE_STATUS_FAILED|SOURCE_CHECKOUT_CONFLICT) echo 'The fleet-owned checkout of this revision holds changes fleet-sync did not make. Inspect and remove that revisions directory, then rerun fleet-sync.';;
     CONFIGURATION_INSTALL_FAILED) echo 'The fleet configuration could not be written under ~/.config/dev-fleet. Check that directory, then rerun fleet-sync.';;
-    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet doctor on the worker.';;
+    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet fleet-status --node for this worker.';;
     *) echo 'Inspect the remote operation lock and rerun the approved revision. No success receipt advanced.';;
   esac
 }
@@ -64,22 +66,14 @@ trap '' HUP
 trap 'exit 1' INT TERM
 # No shell startup assumptions, and no implicit root development user.
 [ "$(id -u)" != 0 ] || die ROOT_DEVELOPMENT_USER_REJECTED
-# Ubuntu 26.04 is the accepted worker platform; macOS 14+ follows the same
-# phases best effort with launchd instead of systemd.
-platform=$(uname -s)
-case "$platform" in
-  Linux)
-    # shellcheck source=/dev/null
-    . /etc/os-release
-    [ "$ID" = ubuntu ] && [ "$VERSION_ID" = 26.04 ] || die UBUNTU_BASELINE_MISMATCH
-    case "$(uname -m)" in x86_64|aarch64) ;; *) die UBUNTU_ARCHITECTURE_UNSUPPORTED;; esac ;;
-  Darwin)
-    major=$(sw_vers -productVersion | cut -d. -f1)
-    case "$major" in ''|*[!0-9]*) die MACOS_BASELINE_MISMATCH;; esac
-    [ "$major" -ge 14 ] || die MACOS_BASELINE_MISMATCH
-    case "$(uname -m)" in arm64|x86_64) ;; *) die MACOS_ARCHITECTURE_UNSUPPORTED;; esac ;;
-  *) die PLATFORM_UNSUPPORTED;;
-esac
+# Workers are the Ubuntu releases the policy accepts (docs/support-matrix.md).
+[ "$(uname -s)" = Linux ] || die PLATFORM_UNSUPPORTED
+# shellcheck source=/dev/null
+. /etc/os-release
+[ "$ID" = ubuntu ] || die UBUNTU_BASELINE_MISMATCH
+case " $releases " in *" $VERSION_ID "*) ;; *) die UBUNTU_BASELINE_MISMATCH;; esac
+case "$(uname -m)" in x86_64|aarch64) ;; *) die UBUNTU_ARCHITECTURE_UNSUPPORTED;; esac
+! grep -qiE 'microsoft|wsl' /proc/sys/kernel/osrelease 2>/dev/null || die WORKER_ENVIRONMENT_UNSUPPORTED
 root="$HOME/.local/share/personal/fleet"
 # Refuse symlinked state roots rather than allowing writes through them.
 for path in "$HOME/.local" "$HOME/.local/share" "$HOME/.local/share/personal" "$root"; do
@@ -89,15 +83,20 @@ mkdir -p "$root"
 phase=lock
 [ ! -L "$root/remote-operation.lock" ] || exit 1
 exec 9>"$root/remote-operation.lock"
-# macOS has no flock(1); lock the same descriptor through fcntl. The lock
-# follows this shell's open descriptor and is released when it exits.
-if command -v flock >/dev/null 2>&1; then
-  flock -n 9 || die FLEET_REMOTE_LOCKED
-else
-  command -v python3 >/dev/null 2>&1 || die BOOTSTRAP_LOCK_TOOL_REQUIRED
-  python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || die FLEET_REMOTE_LOCKED
-fi
-printf '{"version":1,"pid":%s,"revision":"%s","node":"%s"}\n' "$$" "$revision" "$node" >"$root/remote-operation.json"
+# The lock follows this shell's open descriptor and is released when it exits.
+flock -n 9 || die FLEET_REMOTE_LOCKED
+# The operation record: a manager that lost its session reads it to follow
+# this operation to its end instead of starting another (see
+# src/remote-operation.ts). The result of the last operation is beside it.
+started=$(date +%s)
+result="$root/remote-operation-result.json"
+[ ! -L "$root/remote-operation.json" ] && [ ! -L "$result" ] || die FLEET_STATE_PATH_UNSAFE
+rm -f -- "$result"
+operation() {
+  printf '{"version":1,"pid":%s,"revision":"%s","node":"%s","started":%s,"phase":"%s"%s}\n' "$$" "$revision" "$node" "$started" "$1" "${2:-}" >"$root/remote-operation.json.new" &&
+    mv -f -- "$root/remote-operation.json.new" "$root/remote-operation.json"
+}
+operation lock
 # The fleet configuration is installed under the lock, so two overlapping
 # runs never write it at once. It holds no secret.
 phase='configuration-install'
@@ -109,6 +108,7 @@ if [ -n "$configuration" ]; then
   mv "$config_dir/fleet.json.new" "$config_dir/fleet.json" || die CONFIGURATION_INSTALL_FAILED
 fi
 phase=resources
+operation resources
 # A bootstrap floor protects source transfer and minimal prerequisites. The
 # Workstation baseline subsequently enforces its stricter declared capacity.
 free_kib=$(df -Pk "$root" | awk 'END {print $4}')
@@ -120,14 +120,9 @@ if [ "$source_mode" = transfer ]; then
 bundle=$(mktemp "$root/source.XXXXXX")
 # Leave no executable payload copies; this temporary file is only Git data.
 base64 -d >"$bundle"
-if command -v sha256sum >/dev/null 2>&1; then
-  printf '%s  %s\n' "$bundle_sha" "$bundle" | sha256sum -c - >/dev/null 2>&1 || die SOURCE_DIGEST_MISMATCH
-else
-  [ "$(shasum -a 256 "$bundle" | cut -d' ' -f1)" = "$bundle_sha" ] || die SOURCE_DIGEST_MISMATCH
-fi
+printf '%s  %s\n' "$bundle_sha" "$bundle" | sha256sum -c - >/dev/null 2>&1 || die SOURCE_DIGEST_MISMATCH
 fi
 phase=prerequisites
-if [ "$platform" = Linux ]; then
 missing=''
 for package in git curl ca-certificates unzip xz-utils build-essential python3 ansible-core; do
   dpkg-query -W -f='${db:Status-Status}' "$package" 2>/dev/null | grep -qx installed || missing="$missing $package"
@@ -159,12 +154,6 @@ if [ -n "$missing" ]; then
   rm -f -- "$pending" "$apt_log"
   apt_log=''
 fi
-else
-  # Apple Command Line Tools provide git and python3; Ansible runs through the
-  # pinned uv tool runner installed by the shared mise toolchain below.
-  xcode-select -p >/dev/null 2>&1 || die XCODE_COMMAND_LINE_TOOLS_REQUIRED
-  command -v git >/dev/null 2>&1 || die XCODE_COMMAND_LINE_TOOLS_REQUIRED
-fi
 phase=source-verification
 deployment="$root/revisions/$revision"
 [ ! -L "$root/revisions" ] && [ ! -L "$deployment" ] || exit 1
@@ -200,6 +189,7 @@ bundle=''
 clean=$(git -C "$deployment" status --porcelain) || die SOURCE_STATUS_FAILED
 [ -z "$clean" ] || die SOURCE_CHECKOUT_CONFLICT
 phase=toolchain
+operation toolchain
 cd "$deployment"
 export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
 status_file="$root/toolchain-status.json"
@@ -211,15 +201,20 @@ ready_file="$root/toolchain-ready"
 # are still checked (and reinstalled on drift) by node convergence.
 toolchain_key=$(git rev-parse HEAD:mise.toml HEAD:mise.lock HEAD:.mise HEAD:bun.lock HEAD:package.json HEAD:bootstrap 2>/dev/null | tr '\n' ' ')
 [ -n "$toolchain_key" ] || toolchain_key=$revision
+# The tool profiles of the fleet configuration choose what is installed, so a
+# changed configuration runs the setup again.
+if [ -f "$HOME/.config/dev-fleet/fleet.json" ]; then
+  toolchain_key="$toolchain_key$(cksum <"$HOME/.config/dev-fleet/fleet.json" | cut -d' ' -f1)"
+fi
 if [ "$(cat "$ready_file" 2>/dev/null)" = "$toolchain_key" ] && [ -n "$(find "$ready_file" -mmin -1440 2>/dev/null)" ]; then
   toolchain_warm=1
 else
   toolchain_warm=''
   rm -f -- "$status_file" "$ready_file"
 fi
-if command -v timeout >/dev/null 2>&1; then bounded='timeout 1800'; else bounded=''; fi
+bounded='timeout 1800'
 # shellcheck disable=SC2086
-if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
+if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --node "$node" --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
   tool_phase=$(sed -n 's/.*"phase":"\([a-z0-9-]*\)".*/\1/p' "$status_file" 2>/dev/null) || tool_phase=''
   case "$tool_phase" in
     managed-config) die TOOLCHAIN_CONFIG_CONFLICT;;
@@ -254,5 +249,14 @@ for old in $(ls -1t "$root/revisions"); do
   rm -rf -- "${root:?}/revisions/$old" || true
 done
 phase=configuration
+operation configuration
 trap - EXIT INT TERM
-exec mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision"
+# Node convergence writes its evidence to the result file, not to the SSH
+# session: the session may be gone by the time it ends. The evidence is then
+# printed for the manager that is still connected, under the same lock.
+code=0
+mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision" >"$result.partial" 2>"$root/remote-operation.log" || code=$?
+mv -f -- "$result.partial" "$result"
+operation finished ",\"finished\":$(date +%s),\"exit\":$code"
+cat -- "$result" || true
+exit "$code"

package/bootstrap/setup.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  installedSelection,
+  NO_SELECTION,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 /** Used by both OS bootstraps, dev-fleet tools sync and dev-fleet update. No workspace dependencies. */
@@ -26,7 +31,8 @@ function run(args: string[]): void {
 
 export async function syncTools(
   observe: (phase: SetupPhase) => void = () => {},
-  worker = false
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
 ): Promise<void> {
   observe('managed-config');
   const dir = globalConfigDir();
@@ -34,9 +40,9 @@ export async function syncTools(
   observe('mise');
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
-  ) as { tools: Record<string, unknown>; vars: { worker_tools: unknown } };
+  ) as { tools: Record<string, unknown>; vars: Record<string, unknown> };
   const selected = worker
-    ? Object.keys(workerTools(config.tools, config.vars.worker_tools))
+    ? Object.keys(workerToolset(config.tools, config.vars, selection))
     : [];
   run(['mise', 'install', '--locked', ...selected]);
   observe('aws');
@@ -54,11 +60,20 @@ if (import.meta.main) {
   const flag = process.argv.indexOf('--status-file');
   const report = setupStatus(flag < 0 ? undefined : process.argv[flag + 1]);
   let phase: SetupPhase = 'managed-config';
+  const worker = process.argv.includes('--worker');
+  const node = process.argv.indexOf('--node');
   try {
-    await syncTools((current) => {
-      phase = current;
-      report(phase, 'running');
-    }, process.argv.includes('--worker'));
+    await syncTools(
+      (current) => {
+        phase = current;
+        report(phase, 'running');
+      },
+      worker,
+      // The tool profiles of the fleet configuration fleet-sync installed.
+      worker && node >= 0
+        ? installedSelection(process.argv[node + 1])
+        : NO_SELECTION
+    );
     if (!process.argv.includes('--tools-only')) {
       phase = 'dependencies';
       report(phase, 'running');