@crvouga/dev-fleet 0.4.0 → 0.5.0

package/README.md

@@ -42,7 +42,9 @@ dev-fleet is opinionated. These are part of it, not options:
 - The **manager** is a macOS machine. You run `fleet-sync` on it, and it hosts
   the shared services (the CLI proxy and the monitoring hub) in Docker Desktop.
 - **Workers** are Ubuntu 26.04 machines reached over SSH. They are stateless:
-  everything on them is rebuilt from Git and your secrets.
+  everything on them is rebuilt from Git and your secrets. The
+  [support matrix](docs/support-matrix.md) lists every platform and what
+  happens to it.
 - The fleet state is kept in an **S3-compatible bucket** you own.
 - Machines reach each other over **Tailscale**.
 - Coding agents (Claude Code, Codex, Grok, OpenCode, Antigravity) run through

package/ansible/macos/policy.json

@@ -1,6 +1,7 @@
 {
   "version": 1,
   "minimumMajorVersion": 14,
+  "architectures": ["arm64", "x86_64"],
   "pmset": {
     "sleep": "0",
     "womp": "1"

package/ansible/platforms.json

@@ -0,0 +1,111 @@
+{
+  "version": 1,
+  "rows": [
+    {
+      "role": "manager",
+      "platform": "any other",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "The fleet configuration accepts only a macOS manager."
+    },
+    {
+      "role": "worker",
+      "platform": "Ubuntu",
+      "versions": "any other release",
+      "architectures": "any",
+      "status": "refused",
+      "note": "UBUNTU_BASELINE_MISMATCH"
+    },
+    {
+      "role": "worker",
+      "platform": "Ubuntu",
+      "versions": "an accepted release",
+      "architectures": "any other",
+      "status": "refused",
+      "note": "UBUNTU_BASELINE_MISMATCH"
+    },
+    {
+      "role": "worker",
+      "platform": "Debian",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "UBUNTU_BASELINE_MISMATCH"
+    },
+    {
+      "role": "worker",
+      "platform": "macOS",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "The fleet configuration accepts only Linux workers."
+    },
+    {
+      "role": "worker",
+      "platform": "Windows",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "The fleet configuration accepts only Linux workers."
+    },
+    {
+      "role": "worker",
+      "platform": "WSL2",
+      "environment": "wsl",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "WORKER_ENVIRONMENT_UNSUPPORTED"
+    },
+    {
+      "role": "worker",
+      "platform": "container",
+      "environment": "container",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "WORKER_ENVIRONMENT_UNSUPPORTED"
+    },
+    {
+      "role": "toolchain only",
+      "platform": "macOS",
+      "versions": "any",
+      "architectures": "arm64, x86_64",
+      "status": "best effort",
+      "note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
+    },
+    {
+      "role": "toolchain only",
+      "platform": "Linux",
+      "versions": "any",
+      "architectures": "x86_64, aarch64",
+      "status": "best effort",
+      "note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
+    },
+    {
+      "role": "toolchain only",
+      "platform": "WSL2",
+      "versions": "any",
+      "architectures": "x86_64",
+      "status": "best effort",
+      "note": "Treated as Linux: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
+    },
+    {
+      "role": "toolchain only",
+      "platform": "Windows",
+      "versions": "any",
+      "architectures": "x64",
+      "status": "best effort",
+      "note": "No fleet role: bootstrap/bootstrap.ps1 installs the pinned tools (docs/machine-setup.md)."
+    },
+    {
+      "role": "toolchain only",
+      "platform": "Alpine",
+      "versions": "any",
+      "architectures": "any",
+      "status": "refused",
+      "note": "musl: the pinned mise build needs glibc."
+    }
+  ]
+}

package/ansible/ubuntu/authorize-sudo.sh

@@ -3,14 +3,11 @@
 # SSH terminal so sudo reads the operator's password itself; never capture it.
 set -eu
 umask 077
-case "$(uname -s)" in
-  Linux)
-    . /etc/os-release
-    [ "$ID" = ubuntu ] && [ "$VERSION_ID" = 26.04 ] || exit 1 ;;
-  Darwin)
-    [ "$(sw_vers -productVersion | cut -d. -f1)" -ge 14 ] || exit 1 ;;
-  *) exit 1 ;;
-esac
+# Workers are Ubuntu. Which releases is checked by the bootstrap that
+# follows, against ansible/ubuntu/policy.json; this only grants sudo.
+[ "$(uname -s)" = Linux ] || exit 1
+. /etc/os-release
+[ "$ID" = ubuntu ] || exit 1
 user=$(id -un)
 case "$user" in root|''|*[!a-z0-9_-]*) exit 1;; esac
 command -v sudo >/dev/null 2>&1 || exit 1
@@ -23,11 +20,7 @@ mkdir -p "$root"
 chmod 700 "$root"
 [ ! -L "$root/remote-operation.lock" ] || exit 1
 exec 9>"$root/remote-operation.lock"
-if command -v flock >/dev/null 2>&1; then
-  flock -n 9 || exit 1
-else
-  python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || exit 1
-fi
+flock -n 9 || exit 1
 sudo -v
 file=$(mktemp)
 trap 'rm -f "$file"' EXIT
@@ -45,7 +38,5 @@ if sudo -n test -e "$target"; then
   }
   sudo -n cp -p -- "$target" "$target.before-fleet-enrollment"
 fi
-group=root
-[ "$(uname -s)" != Darwin ] || group=wheel
-sudo -n install -o root -g "$group" -m 0440 -- "$file" "$target"
+sudo -n install -o root -g root -m 0440 -- "$file" "$target"
 sudo -n true

package/ansible/ubuntu/baseline.yml

@@ -7,15 +7,13 @@
   vars_files:
     - policy.json
   pre_tasks:
-    - name: Require the supported common release and an explicit non-root user
+    - ansible.builtin.import_tasks: tasks/release.yml
+    - name: Require an explicit non-root user
       ansible.builtin.assert:
         that:
-          - ansible_facts.distribution == 'Ubuntu'
-          - ansible_facts.distribution_version == release
-          - ansible_facts.architecture in ['x86_64', 'aarch64']
           - development_user is match('^[a-z_][a-z0-9_-]{0,31}$')
           - development_user != 'root'
-        fail_msg: Unsupported platform or user; no distribution upgrade is permitted.
+        fail_msg: Unsupported user; name the non-root developer account.
     - name: Read the designated existing account
       ansible.builtin.getent:
         database: passwd
@@ -25,6 +23,7 @@
         that:
           - ansible_facts.getent_passwd[development_user][1] | int == development_uid | int
           - ansible_facts.getent_passwd[development_user][4] == development_home
+    - ansible.builtin.import_tasks: tasks/dpkg.yml
     # Memory and disk safety must land even when the disk is nearly full.
     - ansible.builtin.import_tasks: tasks/resources.yml
     - name: Inspect capacity before package and service changes
@@ -55,29 +54,6 @@
         fail_msg: DOCKER_OWNER_CONFLICT; migrate the existing Docker owner explicitly before applying the Ubuntu-native baseline.
     - name: Inspect existing service ownership
       ansible.builtin.service_facts:
-    - name: Wait for legitimate package transactions before inspecting dpkg
-      ansible.builtin.command:
-        argv:
-          - /usr/bin/python3
-          - '-c'
-          - |
-            import fcntl
-            files = [open(path, 'a') for path in ['/var/lib/dpkg/lock-frontend', '/var/lib/dpkg/lock']]
-            for handle in files:
-                fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
-      register: apt_idle
-      changed_when: false
-      retries: 24
-      delay: 5
-      until: apt_idle.rc == 0
-    - name: Inspect package database consistency
-      ansible.builtin.command: dpkg --audit
-      changed_when: false
-      register: dpkg_audit
-    - name: Require package database recovery before modifying packages
-      ansible.builtin.assert:
-        that: dpkg_audit.stdout | trim == ''
-        fail_msg: APT_STATE_BROKEN; inspect dpkg --audit and use reviewed package recovery. Never remove package lock files.
   tasks:
     - ansible.builtin.import_tasks: tasks/packages.yml
     - ansible.builtin.import_tasks: tasks/docker.yml

package/ansible/ubuntu/disk-gc.py

@@ -11,19 +11,31 @@ What it deletes:
     process is running inside it;
   * Docker images and build cache that no container uses and that are older
     than --docker-age-hours (never containers or volumes);
-  * the apt package cache.
+  * the apt package cache;
+  * agent worktrees under each --worktree-root that nobody has used for
+    --worktree-idle-days, only when Git says they hold no uncommitted,
+    untracked or unpushed work and no process runs in them (the branch is
+    kept);
+  * the oldest files of each --cache-cap DIR=MIB cache beyond its size;
+  * all but the newest N files of each --keep-newest DIR=N directory (the
+    downloaded T3 packages: the current one and the previous one);
+  * tool versions no mise configuration uses (--mise, run as --user).
 When a watched filesystem has less than --pressure-mib free, or less than
 --pressure-inode-percent of its inodes free, idle checkouts
 qualify after --pressure-idle-hours, Docker leftovers after
 --pressure-docker-age-hours, and aged entries of each --pressure-cache
 directory are removed too.
 
-It never deletes a checkout, a tracked file, uncommitted work, a container or
-a volume. Every decision is one JSON line on stdout.
+It never deletes a main checkout, a tracked file, uncommitted or unpushed
+work, a container or a volume. Every decision is one JSON line on stdout, and
+the last one says how many MiB were reclaimed.
 
 Usage: disk-gc.py --home H [--path P ...] [--idle-days N] [--pressure-mib N]
                   [--pressure-inode-percent N] [--pressure-idle-hours N] [--docker-age-hours N]
                   [--pressure-docker-age-hours N] [--pressure-cache DIR ...]
+                  [--worktree-root DIR ...] [--worktree-idle-days N]
+                  [--cache-cap DIR=MIB ...] [--keep-newest DIR=N ...]
+                  [--mise PATH] [--user NAME]
                   [--max-depth N] [--proc DIR] [--home-only] [--dry-run]
 
 --home-only skips the machine-wide Docker and apt steps.
@@ -297,6 +309,150 @@ def collect_cache(directory, age_seconds, now, dry_run):
     return freed
 
 
+def linked_worktrees(root, max_depth=3):
+    """Linked worktrees (a .git file, never a repository) under root."""
+    found = []
+
+    def walk(directory, depth):
+        dot_git = os.path.join(directory, '.git')
+        if os.path.lexists(dot_git):
+            if not is_real_dir(dot_git) and not os.path.islink(dot_git):
+                found.append(directory)
+            return
+        if depth >= max_depth:
+            return
+        try:
+            entries = list(os.scandir(directory))
+        except OSError:
+            return
+        for entry in entries:
+            if entry.is_dir(follow_symlinks=False) and entry.name not in REGENERABLE:
+                walk(entry.path, depth + 1)
+
+    walk(root, 0)
+    return sorted(found)
+
+
+def unsaved_work(worktree):
+    """Why Git cannot vouch for a worktree, or None when everything in it is
+    committed and every commit is on a remote."""
+    status = run_git(worktree, ['status', '--porcelain=v1', '-z', '--untracked-files=all'])
+    if status is None or status.returncode != 0:
+        return 'git_status_failed'
+    if status.stdout.strip(b'\0'):
+        return 'uncommitted'
+    unpushed = run_git(worktree, ['rev-list', '-n', '1', 'HEAD', '--not', '--remotes'])
+    if unpushed is None or unpushed.returncode != 0:
+        return 'git_rev_list_failed'
+    if unpushed.stdout.strip():
+        return 'unpushed'
+    return None
+
+
+def collect_worktrees(root, idle_seconds, proc, now, dry_run):
+    roots = busy_roots(proc)
+    freed = 0
+    for worktree in linked_worktrees(root):
+        gdir = git_dir(worktree)
+        if gdir is None or in_use(worktree, roots):
+            continue
+        if now - git_activity(worktree, gdir) < idle_seconds:
+            continue
+        latest = last_activity(worktree, gdir, [])
+        if latest is None or now - latest < idle_seconds:
+            continue
+        reason = unsaved_work(worktree)
+        if reason:
+            log(event='worktree_kept', worktree=worktree, reason=reason)
+            continue
+        size = size_of(worktree)
+        if not dry_run:
+            common = run_git(worktree, ['rev-parse', '--path-format=absolute', '--git-common-dir'])
+            if common is None or common.returncode != 0:
+                continue
+            out = subprocess.run(
+                GIT + ['--git-dir', common.stdout.decode().strip(), 'worktree', 'remove', '--force', worktree],
+                capture_output=True, timeout=600)
+            if out.returncode != 0 or os.path.lexists(worktree):
+                log(event='worktree_kept', worktree=worktree, reason='git_worktree_remove_failed')
+                continue
+        freed += size
+        log(event='worktree_removed', worktree=worktree, mib=size // MIB, dry_run=dry_run)
+    return freed
+
+
+def files_under(directory):
+    """Every regular file under directory, never through a link."""
+    found = []
+    for current, dirs, files in os.walk(directory):
+        dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(current, d))]
+        for name in files:
+            path = os.path.join(current, name)
+            try:
+                st = os.lstat(path)
+            except OSError:
+                continue
+            if stat.S_ISREG(st.st_mode):
+                found.append((st.st_mtime, st.st_blocks * 512, path))
+    return found
+
+
+def unlink(path, dry_run):
+    if dry_run:
+        return True
+    try:
+        os.unlink(path)
+        return True
+    except OSError:
+        return False
+
+
+def cap_cache(directory, cap_bytes, dry_run):
+    """Removes the oldest files of a cache until it fits its cap."""
+    entries = sorted(files_under(directory))
+    total = sum(size for _, size, _ in entries)
+    freed = 0
+    for _, size, path in entries:
+        if total - freed <= cap_bytes:
+            break
+        if unlink(path, dry_run):
+            freed += size
+    if not dry_run:
+        for current, dirs, files in os.walk(directory, topdown=False):
+            if current != directory and not dirs and not files:
+                try:
+                    os.rmdir(current)
+                except OSError:
+                    pass
+    if freed:
+        log(event='cache_capped', cache=directory, mib=freed // MIB,
+            cap_mib=cap_bytes // MIB, dry_run=dry_run)
+    return freed
+
+
+def keep_newest(directory, count, dry_run):
+    """Removes all but the newest count files directly in directory."""
+    try:
+        entries = [e for e in os.scandir(directory) if e.is_file(follow_symlinks=False)]
+    except OSError:
+        return 0
+    entries.sort(key=lambda e: e.stat(follow_symlinks=False).st_mtime, reverse=True)
+    freed = 0
+    for entry in entries[count:]:
+        size = entry.stat(follow_symlinks=False).st_blocks * 512
+        if unlink(entry.path, dry_run):
+            freed += size
+            log(event='superseded_removed', path=entry.path, mib=size // MIB, dry_run=dry_run)
+    return freed
+
+
+def pair(text, kind):
+    name, _, value = text.rpartition('=')
+    if not name:
+        raise argparse.ArgumentTypeError(f'expected DIR={kind}')
+    return name, int(value)
+
+
 def command(argv, dry_run, name):
     if shutil.which(argv[0]) is None:
         return
@@ -321,6 +477,14 @@ def main(argv):
     parser.add_argument('--docker-age-hours', type=int, default=168)
     parser.add_argument('--pressure-docker-age-hours', type=int, default=24)
     parser.add_argument('--pressure-cache', action='append', default=[])
+    parser.add_argument('--worktree-root', action='append', default=[])
+    parser.add_argument('--worktree-idle-days', type=float, default=30)
+    parser.add_argument('--cache-cap', action='append', default=[],
+                        type=lambda text: pair(text, 'MIB'))
+    parser.add_argument('--keep-newest', action='append', default=[],
+                        type=lambda text: pair(text, 'N'))
+    parser.add_argument('--mise')
+    parser.add_argument('--user')
     parser.add_argument('--max-depth', type=int, default=4)
     parser.add_argument('--proc', default='/proc')
     parser.add_argument('--home-only', action='store_true')
@@ -342,6 +506,23 @@ def main(argv):
     if pressure:
         for cache in args.pressure_cache:
             freed += collect_cache(os.path.join(home, cache), idle, now, args.dry_run)
+    reclaimed = freed
+    for root in args.worktree_root:
+        reclaimed += collect_worktrees(os.path.join(home, root),
+                                       args.worktree_idle_days * 24 * HOUR,
+                                       args.proc, now, args.dry_run)
+    for cache, cap_mib in args.cache_cap:
+        reclaimed += cap_cache(os.path.join(home, cache), cap_mib * MIB, args.dry_run)
+    for directory, count in args.keep_newest:
+        reclaimed += keep_newest(os.path.join(home, directory), count, args.dry_run)
+    if args.mise:
+        installs = os.path.join(home, '.local', 'share', 'mise', 'installs')
+        before_mise = size_of(installs)
+        prune = [args.mise, 'prune', '--yes']
+        if args.user:
+            prune = ['runuser', '-u', args.user, '--', 'env', f'HOME={home}'] + prune
+        command(prune, args.dry_run, 'mise_prune')
+        reclaimed += max(0, before_mise - size_of(installs))
     if not args.home_only:
         command(['docker', 'image', 'prune', '--all', '--force', '--filter',
                  f'until={docker_age}h'], args.dry_run, 'docker_images')
@@ -350,7 +531,8 @@ def main(argv):
         command(['apt-get', 'clean'], args.dry_run, 'apt_cache')
 
     after = {p: free_bytes(p) for p in paths}
-    log(event='finished', checkout_mib=freed // MIB, dry_run=args.dry_run,
+    log(event='finished', checkout_mib=freed // MIB,
+        reclaimed_mib=reclaimed // MIB, dry_run=args.dry_run,
         free_mib={p: v // MIB for p, v in after.items()})
 
 

package/ansible/ubuntu/policy.json

@@ -1,7 +1,12 @@
 {
   "version": 1,
-  "release": "26.04",
-  "architectures": ["amd64", "arm64"],
+  "releases": [
+    {
+      "release": "26.04",
+      "codename": "resolute",
+      "architectures": ["amd64", "arm64"]
+    }
+  ],
   "snapshot": "20261101T000000Z",
   "packages": {
     "sudo": "1.9.17p2-1ubuntu3.2",
@@ -17,7 +22,8 @@
     "python3": "3.14.3-0ubuntu2",
     "unzip": "6.0-29ubuntu1",
     "xz-utils": "5.8.3-1ubuntu0.1",
-    "ansible-core": "2.20.1-1"
+    "ansible-core": "2.20.1-1",
+    "systemd-oomd": "259.9-0ubuntu0.1"
   },
   "tailscale": {
     "version": "1.104.5",
@@ -47,6 +53,15 @@
     "diskGcDockerAgeHours": 168,
     "diskGcPressureDockerAgeHours": 24,
     "diskGcPressureCaches": [".bun/install/cache"],
+    "diskGcWorktreeRoots": [".t3/worktrees"],
+    "diskGcWorktreeIdleDays": 30,
+    "diskGcCacheCapsMiB": {
+      ".bun/install/cache": 4096,
+      ".npm/_cacache": 2048,
+      ".cache/uv": 4096,
+      ".cache/pip": 1024
+    },
+    "diskGcPackagesKept": 2,
     "dockerLogMaxSize": "50m",
     "dockerLogMaxFiles": "3",
     "inodeWarnFreePercent": 10,

package/ansible/ubuntu/t3-desktop.yml

@@ -4,8 +4,8 @@
 # unit starts the app with the graphical session, so the UI is on screen and the
 # server is up after every boot. Starting or migrating is left to fleet-sync.
 #
-# The upstream .deb declares libxss1, which Ubuntu 26.04 no longer ships and the
-# binary does not link. Instead of patching the package or leaving apt with an
+# The upstream .deb declares libxss1, which current Ubuntu releases no longer
+# ship and the binary does not link. Instead of patching the package or leaving apt with an
 # unmet dependency, the verified package payload is unpacked to its own path and
 # its real runtime libraries and bundled AppArmor profile are installed.
 - name: Install the pinned T3 Code desktop app and start it with the session

package/ansible/ubuntu/tailscale.yml

@@ -8,17 +8,28 @@
   vars_files:
     - policy.json
   tasks:
-    - name: Require supported Ubuntu
-      ansible.builtin.assert:
-        that:
-          - ansible_facts.distribution == 'Ubuntu'
-          - ansible_facts.distribution_version == release
+    - ansible.builtin.import_tasks: tasks/release.yml
     - name: Inspect installed packages before possible network interruption
       ansible.builtin.package_facts:
         manager: apt
-    - name: Require maintenance for an installed version change
+    # The pin is a minimum: a newer Tailscale is kept and reported as a
+    # warning by the worker checks; an older one needs a maintenance window.
+    - name: Compare the installed Tailscale with its pin
+      ansible.builtin.command:
+        argv:
+          - dpkg
+          - --compare-versions
+          - '{{ ansible_facts.packages.tailscale[0].version }}'
+          - ge
+          - '{{ tailscale.version }}'
+      register: tailscale_pin
+      changed_when: false
+      failed_when: tailscale_pin.rc not in [0, 1]
+      check_mode: false
+      when: "'tailscale' in ansible_facts.packages"
+    - name: Require maintenance for an upgrade to the pin
       ansible.builtin.assert:
-        that: "'tailscale' not in ansible_facts.packages or ansible_facts.packages.tailscale[0].version == tailscale.version"
+        that: "'tailscale' not in ansible_facts.packages or tailscale_pin.rc == 0"
         fail_msg: TAILSCALE_UPGRADE_PENDING; upgrading an active VPN can disconnect SSH. Review an explicit maintenance operation.
     - name: Inspect existing package-source ownership before writing
       ansible.builtin.find:
@@ -35,7 +46,7 @@
         fail_msg: TAILSCALE_SOURCE_OWNER_CONFLICT; review the existing signed source and its migration before creating another repository entry.
     - name: Install pinned upstream signing key
       ansible.builtin.get_url:
-        url: https://pkgs.tailscale.com/stable/ubuntu/resolute.noarmor.gpg
+        url: https://pkgs.tailscale.com/stable/ubuntu/{{ ubuntu_release.codename }}.noarmor.gpg
         dest: /usr/share/keyrings/tailscale-archive-keyring.gpg
         checksum: 'sha256:{{ tailscale.keySha256 }}'
         owner: root
@@ -44,7 +55,7 @@
         timeout: 30
     - name: Declare the signed upstream package source
       ansible.builtin.apt_repository:
-        repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu resolute main
+        repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu {{ ubuntu_release.codename }} main
         filename: tailscale
         state: present
         update_cache: false

package/ansible/ubuntu/tasks/dpkg.yml

@@ -0,0 +1,46 @@
+---
+# A package database left inconsistent by an interrupted run is repaired
+# before any package task runs, or stops the play with APT_STATE_BROKEN.
+- name: Wait for legitimate package transactions before inspecting dpkg
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/python3
+      - '-c'
+      - |
+        import fcntl
+        files = [open(path, 'a') for path in ['/var/lib/dpkg/lock-frontend', '/var/lib/dpkg/lock']]
+        for handle in files:
+            fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  register: apt_idle
+  changed_when: false
+  retries: 24
+  delay: 5
+  until: apt_idle.rc == 0
+- name: Inspect package database consistency
+  ansible.builtin.command: dpkg --audit
+  changed_when: false
+  register: dpkg_audit
+# Power lost during a package run leaves dpkg half-configured. Finishing
+# that run is the standard safe repair: it is tried once, bounded, after
+# the lock wait above, and only a database still inconsistent afterwards
+# needs a person.
+- name: Repair an interrupted package run once
+  when: dpkg_audit.stdout | trim != ''
+  block:
+    - name: Finish configuring every unpacked package, bounded
+      ansible.builtin.command:
+        argv: [timeout, '600', dpkg, --configure, -a]
+      environment:
+        DEBIAN_FRONTEND: noninteractive
+    - name: Inspect package database consistency after the repair
+      ansible.builtin.command: dpkg --audit
+      changed_when: false
+      register: dpkg_reaudit
+  rescue:
+    - name: Report a package run the repair could not finish
+      ansible.builtin.fail:
+        msg: APT_STATE_BROKEN; dpkg --configure -a did not finish. Inspect dpkg --audit and use reviewed package recovery. Never remove package lock files.
+- name: Require a consistent package database before modifying packages
+  ansible.builtin.assert:
+    that: (dpkg_reaudit.stdout | default(dpkg_audit.stdout)) | trim == ''
+    fail_msg: APT_STATE_BROKEN; dpkg --configure -a did not clear dpkg --audit. Inspect it and use reviewed package recovery. Never remove package lock files.

package/ansible/ubuntu/tasks/install.yml

@@ -0,0 +1,38 @@
+---
+# The one way a pinned Ubuntu package is installed: from the snapshot,
+# waiting for the APT lock, and never downgrading. A pin is a minimum: a
+# package Ubuntu's security updates moved past its pin is left as it is and
+# reported as a warning (docs/worker-contract.md). `install_pins` maps each
+# package to its pinned version.
+- ansible.builtin.import_tasks: snapshot.yml
+- name: Read the installed package versions
+  ansible.builtin.package_facts:
+    manager: apt
+- name: Compare each installed pinned package with its pin
+  ansible.builtin.command:
+    argv:
+      - dpkg
+      - --compare-versions
+      - '{{ ansible_facts.packages[item][0].version }}'
+      - ge
+      - '{{ install_pins[item] }}'
+  loop: "{{ install_pins.keys() | select('in', ansible_facts.packages) | list }}"
+  register: pin_comparison
+  changed_when: false
+  failed_when: pin_comparison.rc not in [0, 1]
+  check_mode: false
+- name: Select the pinned packages that are absent or older than their pin
+  ansible.builtin.set_fact:
+    below_pin: "{{ (install_pins.keys() | reject('in', ansible_facts.packages) | list) + (pin_comparison.results | selectattr('rc', 'equalto', 1) | map(attribute='item') | list) }}"
+- name: Install the pinned version of each package below its pin, without restarting services
+  ansible.builtin.apt:
+    name: '{{ item }}={{ install_pins[item] }}'
+    state: present
+    lock_timeout: '{{ aptLockTimeoutSeconds }}'
+    allow_downgrade: false
+    allow_unauthenticated: false
+    fail_on_autoremove: true
+    policy_rc_d: 101
+  environment:
+    APT_CONFIG: /etc/personal-fleet/apt.conf
+  loop: '{{ below_pin }}'