@crvouga/dev-fleet 0.3.0 → 0.4.0

package/README.md

@@ -98,7 +98,11 @@ Ansible policy and verifies the result. It prints one table and writes a full
 JSON report. The first run on a machine installs everything, mise included, so
 Git is the only tool it needs beforehand.
 
-`--dry-run` changes no machine. On the manager it checks the release out into
+`--dry-run` changes no machine. It first brings the manager's copy of the
+fleet state up to the bucket (never uploading), then prints, and adds to
+`--json` as `plan`, what a sync would change: the revision, each tool pin that
+moves, and what restarts because of it (a T3 restart ends its running agent
+turns). On the manager it checks the release out into
 dev-fleet's cache and installs the Bun that release pins beside any others, so
 that it can run; it never installs the toolchain, publishes the global mise
 configuration, edits shell profiles or touches a worker.
@@ -316,9 +320,11 @@ fleet.json contents.
   `fleet-sync` fetches that copy again when it is deleted, but the proxy then
   needs `devfleet cliproxy install` once more.
 - The manager is a Mac host. There is no container image of the manager.
-- `fleet-sync --schedule enable` (unattended runs on a timer) is not available
-  from the published package yet, so nothing re-applies the fleet until you
-  run `fleet-sync`.
+- `fleet-sync --schedule enable --apply --yes` (unattended runs every 30
+  minutes) is installed by that command on the manager, not yet by the
+  manager's playbook, and its interval is not yet part of `fleet.json`.
+  `fleet-sync --schedule status` lists its last runs, and `fleet-status` fails
+  `manager:schedule` when it has not converged the fleet for two intervals.
 - Not everything on a machine is pinned. [What is declared, and
   where](docs/declared.md) lists what is and what is not: most of a worker's
   own packages, the manager's applications and every login are outside it.
@@ -327,18 +333,18 @@ fleet.json contents.
 
 ## Repository layout
 
-| Path         | What                                                              |
-| ------------ | ----------------------------------------------------------------- |
-| `src/`       | Inventory, orchestration, verification, the report; `src/cli` CLI |
-| `src/state/` | The fleet state database and its replication to the bucket        |
-| `ansible/`   | Machine policy: `ubuntu/`, `macos/`, `monitoring/`                |
-| `agents/`    | Agent launchers T3 runs, and their registry                       |
-| `cliproxy/`  | The CLIProxyAPI service: install, audit, backup, model health     |
-| `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts      |
-| `schema/`    | JSON schemas for `fleet.json` and the private roster              |
-| `examples/`  | A complete example configuration                                  |
-| `features/`  | What dev-fleet does, as Gherkin features: the tests               |
-| `bin/`       | The entry point of the published package                          |
+| Path         | What                                                                    |
+| ------------ | ----------------------------------------------------------------------- |
+| `src/`       | Inventory, orchestration, verification, the report; `src/cli` CLI       |
+| `src/state/` | The fleet state database and its replication to the bucket              |
+| `ansible/`   | Machine policy: `ubuntu/`, `macos/`, `monitoring/`                      |
+| `agents/`    | Agent launchers T3 runs, and their registry                             |
+| `cliproxy/`  | The CLIProxyAPI service: install, audit, backup, model health           |
+| `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts            |
+| `schema/`    | JSON schemas for `fleet.json`, the roster and command output; check IDs |
+| `examples/`  | A complete example configuration                                        |
+| `features/`  | What dev-fleet does, as Gherkin features: the tests                     |
+| `bin/`       | The entry point of the published package                                |
 
 ## Develop
 

package/ansible/github/git.yml

@@ -0,0 +1,66 @@
+---
+# The Git identity and GitHub credentials every node commits and pushes with,
+# the manager included, as the developer: no become. Written in Git's own
+# per-user file (~/.config/git/config), inside a marked block, so a person's
+# ~/.gitconfig is never touched. Variables arrive through a private 0600
+# variables file, never argv.
+- name: Converge the Git identity and GitHub credential helper
+  hosts: localhost
+  connection: local
+  become: false
+  gather_facts: false
+  vars:
+    git_config: '{{ git_home }}/.config/git/config'
+    # gh answers for GitHub through the pinned toolchain's shim, so any
+    # session's PATH works.
+    git_credential_helper: "!'{{ git_home }}/.local/share/mise/shims/gh' auth git-credential"
+  tasks:
+    - name: Require a declared identity
+      ansible.builtin.assert:
+        that:
+          - git_name | length > 0
+          - git_email is match('^[^@ ]+@[^@ ]+$')
+          - git_home is match('^/')
+        fail_msg: GIT_IDENTITY_UNDECLARED; declare git.name and git.email in fleet.json.
+    - name: Create the per-user Git configuration directory
+      ansible.builtin.file:
+        path: '{{ git_home }}/.config/git'
+        state: directory
+        mode: '0700'
+    - name: Declare the identity and the GitHub credential helper
+      ansible.builtin.blockinfile:
+        path: '{{ git_config }}'
+        create: true
+        mode: '0600'
+        marker: '# {mark} dev-fleet: Git identity and GitHub credentials'
+        block: |
+          [user]
+          	name = "{{ git_name }}"
+          	email = {{ git_email }}
+          [credential "https://github.com"]
+          	helper =
+          	helper = {{ git_credential_helper }}
+          [credential "https://gist.github.com"]
+          	helper =
+          	helper = {{ git_credential_helper }}
+    - name: Read the effective identity and GitHub credential helper
+      ansible.builtin.command:
+        argv: [git, config, --global, --get-all, '{{ item }}']
+      environment:
+        HOME: '{{ git_home }}'
+      loop:
+        - user.name
+        - user.email
+        - credential.https://github.com.helper
+      register: git_effective
+      check_mode: false
+      changed_when: false
+      failed_when: false
+    - name: Require the declared identity and helper to be the effective ones (worker:git-identity)
+      ansible.builtin.assert:
+        that:
+          - git_effective.results[0].stdout_lines | default([]) | last | default('') == git_name
+          - git_effective.results[1].stdout_lines | default([]) | last | default('') == git_email
+          - git_effective.results[2].stdout_lines | default([]) | last | default('') == git_credential_helper
+        fail_msg: GIT_IDENTITY_OVERRIDDEN; another Git configuration (often ~/.gitconfig) sets user.name, user.email or the GitHub credential helper after dev-fleet's. Remove those lines.
+      when: not ansible_check_mode

package/ansible/macos/policy.json

@@ -5,5 +5,15 @@
     "sleep": "0",
     "womp": "1"
   },
-  "beszelDaemon": "dev.devfleet.beszel-agent"
+  "beszelDaemon": "dev.devfleet.beszel-agent",
+  "disk": {
+    "warnFreeMiB": 20480,
+    "failFreeMiB": 5120,
+    "dockerWarnGiB": 64
+  },
+  "containerWatchdog": {
+    "unhealthyChecks": 3,
+    "maxRestartsPerHour": 3,
+    "intervalSeconds": 60
+  }
 }

package/ansible/monitoring/compose.yml

@@ -5,6 +5,16 @@ services:
   hub:
     image: henrygd/beszel:0.21.1@sha256:8514a69d9bbbcb74f371a3205561ee231fc0289739cea64e8131f341d695b770
     restart: unless-stopped
+    # Docker restarts a hub that exits; the container watchdog on the manager
+    # restarts one that stays unhealthy (container-watchdog.py).
+    labels:
+      dev.devfleet.watchdog: restart
+    healthcheck:
+      test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     ports:
       - '127.0.0.1:8090:8090'
     environment:

package/ansible/monitoring/container-watchdog.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+"""Restart a fleet container that stays unhealthy, a bounded number of times.
+
+Managed by fleet-sync (ansible/monitoring/hub-darwin.yml) as the per-user
+launchd agent dev.devfleet.container-watchdog on the manager. Docker restarts
+a container that exits, never one that hangs while it still holds its port.
+Every run reads the health of each container labelled
+dev.devfleet.watchdog=restart. One that was unhealthy for --unhealthy-checks
+runs in a row is restarted, at most --max-restarts-per-hour times an hour.
+Past that cap it is left alone and recorded as capped: a restart storm is a
+failing check (manager:container-watchdog), not a loop.
+
+Every decision is one JSON line on stdout; nothing a container printed is
+repeated. The state file holds counters only and is safe to delete.
+
+Usage: container-watchdog.py --state FILE [--unhealthy-checks N]
+                             [--max-restarts-per-hour N] [--dry-run]
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+
+HOUR = 3600
+LABEL = 'dev.devfleet.watchdog=restart'
+
+
+def log(**event):
+    print(json.dumps(event, sort_keys=True), flush=True)
+
+
+def decide(memory, health, now, limit, cap):
+    """The next memory, and the containers to restart now with the reason.
+
+    memory: {name: {"unhealthy": n, "restarts": [t, ...], "capped": t|None}}
+    health: {name: "healthy" | "unhealthy" | "starting" | "none"}
+    """
+    after, restart = {}, []
+    for name, status in sorted(health.items()):
+        before = memory.get(name, {})
+        restarts = [t for t in before.get('restarts', []) if now - t < HOUR]
+        unhealthy = before.get('unhealthy', 0) + 1 if status == 'unhealthy' else 0
+        capped = before.get('capped')
+        if capped is not None and now - capped >= HOUR:
+            capped = None
+        if unhealthy >= limit:
+            if len(restarts) < cap:
+                restarts.append(now)
+                restart.append((name, f'unhealthy for {unhealthy} checks'))
+                unhealthy = 0
+            else:
+                capped = now
+        after[name] = {'unhealthy': unhealthy, 'restarts': restarts,
+                       'capped': capped}
+    return after, restart
+
+
+def docker(*args):
+    return subprocess.run(['docker', *args], capture_output=True, text=True,
+                          timeout=60)
+
+
+def observe():
+    listed = docker('ps', '--filter', f'label={LABEL}', '--format', '{{.Names}}')
+    if listed.returncode:
+        return None
+    health = {}
+    for name in listed.stdout.split():
+        state = docker('inspect', '--format',
+                       '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}',
+                       name)
+        health[name] = state.stdout.strip() if state.returncode == 0 else 'none'
+    return health
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--state', required=True)
+    parser.add_argument('--unhealthy-checks', type=int, default=3)
+    parser.add_argument('--max-restarts-per-hour', type=int, default=3)
+    parser.add_argument('--dry-run', action='store_true')
+    args = parser.parse_args(argv)
+    try:
+        with open(args.state) as file:
+            memory = json.load(file)
+        if not isinstance(memory, dict):
+            memory = {}
+    except (OSError, ValueError):
+        memory = {}
+    health = observe()
+    if health is None:
+        log(event='docker_unavailable')
+        return 0
+    now = time.time()
+    memory, restart = decide(memory, health, now, args.unhealthy_checks,
+                             args.max_restarts_per_hour)
+    for name, reason in restart:
+        ok = True
+        if not args.dry_run:
+            ok = docker('restart', name).returncode == 0
+        log(event='restarted', container=name, reason=reason, ok=ok,
+            dry_run=args.dry_run)
+    for name, entry in memory.items():
+        if entry['capped'] == now:
+            log(event='capped', container=name,
+                restarts=len(entry['restarts']))
+    os.makedirs(os.path.dirname(os.path.abspath(args.state)), exist_ok=True)
+    temporary = args.state + '.tmp'
+    with open(temporary, 'w') as file:
+        json.dump(memory, file, sort_keys=True)
+    os.replace(temporary, args.state)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

package/ansible/monitoring/hub-darwin.yml

@@ -14,6 +14,12 @@
       - compose
       - '-f'
       - '{{ playbook_dir }}/compose.yml'
+    watchdog_policy: "{{ (lookup('ansible.builtin.file', playbook_dir ~ '/../macos/policy.json') | from_json).containerWatchdog }}"
+    watchdog_label: dev.devfleet.container-watchdog
+    watchdog_root: '{{ ansible_facts.env.HOME }}/Library/Application Support/dev-fleet/container-watchdog'
+    watchdog_state: '{{ ansible_facts.env.HOME }}/Library/Caches/dev-fleet/container-watchdog.json'
+    watchdog_plist: '{{ ansible_facts.env.HOME }}/Library/LaunchAgents/{{ watchdog_label }}.plist'
+    watchdog_domain: 'gui/{{ ansible_facts.user_uid }}'
     beszel_environment:
       BESZEL_HUB_URL: '{{ beszel_hub_url }}'
       BESZEL_APP_URL: '{{ beszel_public_url | default(beszel_hub_url) }}'
@@ -100,3 +106,143 @@
       delay: 2
       register: hub_health
       until: hub_health.status == 200
+    # Scheduled off-machine backups, as the fleet configuration declares them
+    # (services.monitoring.backups), through the hub's own superuser API. Only
+    # what differs is changed; the storage keys never leave the variables file.
+    - name: Converge the hub's scheduled off-machine backups
+      when:
+        - beszel_backup_schedule is defined
+        - beszel_fleet_password | default('') | length > 0
+      block:
+        - name: Sign in to the hub as its superuser
+          ansible.builtin.uri:
+            url: http://127.0.0.1:8090/api/collections/_superusers/auth-with-password
+            method: POST
+            body_format: json
+            body:
+              identity: '{{ beszel_fleet_identity }}'
+              password: '{{ beszel_fleet_password }}'
+            return_content: true
+          check_mode: false
+          no_log: true
+          register: hub_superuser
+        - name: Read the hub's backup settings
+          ansible.builtin.uri:
+            url: http://127.0.0.1:8090/api/settings
+            headers:
+              Authorization: '{{ hub_superuser.json.token }}'
+            return_content: true
+          check_mode: false
+          no_log: true
+          register: hub_settings
+        - name: Schedule backups to the declared off-machine storage (BESZEL_BACKUP_CONVERGENCE_FAILED)
+          ansible.builtin.uri:
+            url: http://127.0.0.1:8090/api/settings
+            method: PATCH
+            headers:
+              Authorization: '{{ hub_superuser.json.token }}'
+            body_format: json
+            body:
+              backups:
+                cron: '{{ beszel_backup_schedule }}'
+                cronMaxKeep: '{{ beszel_backup_keep | int }}'
+                s3:
+                  enabled: true
+                  bucket: '{{ beszel_backup_bucket }}'
+                  region: '{{ beszel_backup_region }}'
+                  endpoint: '{{ beszel_backup_endpoint }}'
+                  accessKey: '{{ beszel_backup_access_key }}'
+                  secret: '{{ beszel_backup_secret_key }}'
+                  forcePathStyle: '{{ beszel_backup_force_path_style | default(false) | bool }}'
+          no_log: true
+          changed_when: true
+          when: >-
+            (hub_settings.json.backups | default({})).cron | default('') != beszel_backup_schedule
+            or (hub_settings.json.backups | default({})).cronMaxKeep | default(0) | int != beszel_backup_keep | int
+            or not ((hub_settings.json.backups | default({})).s3 | default({})).enabled | default(false)
+            or ((hub_settings.json.backups | default({})).s3 | default({})).bucket | default('') != beszel_backup_bucket
+            or ((hub_settings.json.backups | default({})).s3 | default({})).region | default('') != beszel_backup_region
+            or ((hub_settings.json.backups | default({})).s3 | default({})).endpoint | default('') != beszel_backup_endpoint
+            or ((hub_settings.json.backups | default({})).s3 | default({})).accessKey | default('') != beszel_backup_access_key
+            or ((hub_settings.json.backups | default({})).s3 | default({})).forcePathStyle | default(false) != beszel_backup_force_path_style | default(false) | bool
+    # Docker restarts a container that exits, never one that hangs while it
+    # holds its port. The watchdog restarts a labelled container (the hub, the
+    # CLI proxy) that stays unhealthy, a bounded number of times an hour; the
+    # bounds are containerWatchdog in ansible/macos/policy.json, and
+    # manager:container-watchdog reports a container past them.
+    - name: Create the container watchdog's directories
+      ansible.builtin.file:
+        path: '{{ item }}'
+        state: directory
+        mode: '0700'
+      loop:
+        - '{{ watchdog_root }}'
+        - '{{ watchdog_state | dirname }}'
+        - '{{ ansible_facts.env.HOME }}/Library/LaunchAgents'
+    - name: Install the container watchdog
+      ansible.builtin.copy:
+        src: '{{ playbook_dir }}/container-watchdog.py'
+        dest: '{{ watchdog_root }}/container-watchdog.py'
+        mode: '0700'
+    - name: Install the container watchdog's launch agent
+      ansible.builtin.copy:
+        dest: '{{ watchdog_plist }}'
+        mode: '0644'
+        content: |
+          <?xml version="1.0" encoding="UTF-8"?>
+          <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+          <!-- Managed by personal Fleet: container watchdog -->
+          <plist version="1.0"><dict>
+            <key>Label</key><string>{{ watchdog_label }}</string>
+            <key>ProgramArguments</key><array>
+              <string>/usr/bin/python3</string>
+              <string>-I</string>
+              <string>-B</string>
+              <string>{{ watchdog_root }}/container-watchdog.py</string>
+              <string>--state</string>
+              <string>{{ watchdog_state }}</string>
+              <string>--unhealthy-checks</string>
+              <string>{{ watchdog_policy.unhealthyChecks }}</string>
+              <string>--max-restarts-per-hour</string>
+              <string>{{ watchdog_policy.maxRestartsPerHour }}</string>
+            </array>
+            <key>EnvironmentVariables</key><dict>
+              <key>PATH</key><string>/usr/local/bin:/usr/bin:/bin:/Applications/Docker.app/Contents/Resources/bin</string>
+            </dict>
+            <key>StartInterval</key><integer>{{ watchdog_policy.intervalSeconds }}</integer>
+            <key>ProcessType</key><string>Background</string>
+            <key>Umask</key><integer>63</integer>
+            <key>StandardOutPath</key><string>{{ watchdog_root }}/watchdog.log</string>
+            <key>StandardErrorPath</key><string>{{ watchdog_root }}/watchdog.log</string>
+          </dict></plist>
+      register: watchdog_agent
+    - name: Inspect whether the container watchdog is loaded
+      ansible.builtin.command:
+        argv:
+          [/bin/launchctl, print, '{{ watchdog_domain }}/{{ watchdog_label }}']
+      changed_when: false
+      failed_when: false
+      register: watchdog_loaded
+    # launchd caches a loaded definition: a changed plist needs a reload.
+    - name: Unload a stale container watchdog definition
+      ansible.builtin.command:
+        argv:
+          [
+            /bin/launchctl,
+            bootout,
+            '{{ watchdog_domain }}/{{ watchdog_label }}',
+          ]
+      when: watchdog_agent.changed and watchdog_loaded.rc == 0
+      failed_when: false
+      changed_when: true
+    - name: Load the container watchdog
+      ansible.builtin.command:
+        argv:
+          [
+            /bin/launchctl,
+            bootstrap,
+            '{{ watchdog_domain }}',
+            '{{ watchdog_plist }}',
+          ]
+      when: watchdog_agent.changed or watchdog_loaded.rc != 0
+      changed_when: true

package/ansible/net/remote-probe.sh

@@ -8,6 +8,11 @@ if [ "$(uname)" = Darwin ]; then
 else
   echo "ncpu=$(nproc)"
   echo "load=$(cut -d' ' -f1 /proc/loadavg)"
+  # Free inodes as a percentage: bytes can be free while every write fails.
+  df -Pi "$HOME" | awk 'NR==2 && $2>0{print "inodefree=" int($4*100/$2)}'
+  # Clock synchronisation, as systemd sees it: 1 or 0.
+  timedatectl show -p NTPSynchronized -p NTP 2>/dev/null | awk -F= '{v=($2=="yes")?1:0} $1=="NTP"{print "ntp=" v} $1=="NTPSynchronized"{print "ntpsync=" v}'
   awk '/MemTotal/{t=$2} /MemAvailable/{a=$2} /SwapTotal/{st=$2} /SwapFree/{sf=$2} END{print "memfree=" int(a*100/t); if (st>0) print "swap=" int((st-sf)*100/st); else print "swap=0"}' /proc/meminfo
 fi
 echo "diskmb=$(df -Pk "$HOME" | awk 'NR==2{print int($4/1024)}')"
+echo "epoch=$(date +%s)"

package/ansible/ubuntu/baseline.yml

@@ -81,6 +81,8 @@
   tasks:
     - ansible.builtin.import_tasks: tasks/packages.yml
     - ansible.builtin.import_tasks: tasks/docker.yml
+    - ansible.builtin.import_tasks: tasks/browser.yml
+      when: "'web' in worker_tool_profiles | default([])"
     - ansible.builtin.import_tasks: tasks/services.yml
     - ansible.builtin.import_tasks: tasks/power.yml
     - ansible.builtin.import_tasks: tasks/user.yml

package/ansible/ubuntu/disk-gc.py

@@ -12,7 +12,8 @@ What it deletes:
   * Docker images and build cache that no container uses and that are older
     than --docker-age-hours (never containers or volumes);
   * the apt package cache.
-When a watched filesystem has less than --pressure-mib free, idle checkouts
+When a watched filesystem has less than --pressure-mib free, or less than
+--pressure-inode-percent of its inodes free, idle checkouts
 qualify after --pressure-idle-hours, Docker leftovers after
 --pressure-docker-age-hours, and aged entries of each --pressure-cache
 directory are removed too.
@@ -21,7 +22,7 @@ It never deletes a checkout, a tracked file, uncommitted work, a container or
 a volume. Every decision is one JSON line on stdout.
 
 Usage: disk-gc.py --home H [--path P ...] [--idle-days N] [--pressure-mib N]
-                  [--pressure-idle-hours N] [--docker-age-hours N]
+                  [--pressure-inode-percent N] [--pressure-idle-hours N] [--docker-age-hours N]
                   [--pressure-docker-age-hours N] [--pressure-cache DIR ...]
                   [--max-depth N] [--proc DIR] [--home-only] [--dry-run]
 
@@ -61,6 +62,25 @@ def free_bytes(path):
     return st.f_bavail * st.f_frsize
 
 
+def free_inodes_percent(path):
+    """Free inodes as a percentage; None on a filesystem without a fixed count."""
+    st = os.statvfs(path)
+    if st.f_files <= 0:
+        return None
+    return st.f_favail * 100 / st.f_files
+
+
+def under_pressure(paths, pressure_bytes, inode_percent):
+    """Any watched filesystem short of free bytes or of free inodes."""
+    for path in paths:
+        if free_bytes(path) < pressure_bytes:
+            return True
+        inodes = free_inodes_percent(path)
+        if inodes is not None and inodes < inode_percent:
+            return True
+    return False
+
+
 def mtime(path):
     try:
         return os.lstat(path).st_mtime
@@ -296,6 +316,7 @@ def main(argv):
     parser.add_argument('--path', action='append', default=[])
     parser.add_argument('--idle-days', type=float, default=7)
     parser.add_argument('--pressure-mib', type=int, default=15360)
+    parser.add_argument('--pressure-inode-percent', type=float, default=0)
     parser.add_argument('--pressure-idle-hours', type=float, default=12)
     parser.add_argument('--docker-age-hours', type=int, default=168)
     parser.add_argument('--pressure-docker-age-hours', type=int, default=24)
@@ -309,7 +330,8 @@ def main(argv):
     paths = args.path or ['/', home]
     now = time.time()
     before = {p: free_bytes(p) for p in paths}
-    pressure = min(before.values()) < args.pressure_mib * MIB
+    pressure = under_pressure(paths, args.pressure_mib * MIB,
+                              args.pressure_inode_percent)
     idle = args.pressure_idle_hours * HOUR if pressure else args.idle_days * 24 * HOUR
     docker_age = args.pressure_docker_age_hours if pressure else args.docker_age_hours
     log(event='started', pressure=pressure, idle_hours=idle / HOUR,

package/ansible/ubuntu/disk-guard.py

@@ -3,14 +3,15 @@
 
 Managed by personal fleet-sync (tasks/resources.yml) as
 personal-fleet-disk-guard.service. Every interval it measures free space on the
-watched filesystems and the bytes each leaf cgroup under user.slice wrote. When
-any watched filesystem is below the floor, it kills only the single cgroup that
+watched filesystems (bytes and inodes) and the bytes each leaf cgroup under user.slice wrote. When
+any watched filesystem is below the floor of free bytes or of free inodes, it
+kills only the single cgroup that
 wrote the most during the last interval, and only if that is at least the write
 threshold. It never deletes files, never touches system services and never
 kills idle work. Free space must be recovered by a person.
 
-Usage: disk-guard.py --path / --path /home/me [--floor-mib N] [--write-mib N]
-                     [--interval S] [--cgroup-root DIR] [--dry-run]
+Usage: disk-guard.py --path / --path /home/me [--floor-mib N]
+                     [--inode-floor-percent N] [--write-mib N] [--interval S] [--cgroup-root DIR] [--dry-run]
 """
 import argparse
 import json
@@ -61,6 +62,21 @@ def free_bytes(path):
     return st.f_bavail * st.f_frsize
 
 
+def free_inodes_percent(path):
+    """Free inodes as a percentage; None on a filesystem without a fixed count."""
+    st = os.statvfs(path)
+    if st.f_files <= 0:
+        return None
+    return st.f_favail * 100 / st.f_files
+
+
+def low_paths(free, inodes, floor, inode_floor):
+    """The watched paths below either floor: bytes, or free inodes in percent."""
+    return [path for path in free
+            if free[path] < floor
+            or (inodes.get(path) is not None and inodes[path] < inode_floor)]
+
+
 def log(**event):
     print(json.dumps(event, sort_keys=True), flush=True)
 
@@ -69,6 +85,7 @@ def main(argv):
     parser = argparse.ArgumentParser()
     parser.add_argument('--path', action='append', required=True)
     parser.add_argument('--floor-mib', type=int, default=3072)
+    parser.add_argument('--inode-floor-percent', type=float, default=2)
     parser.add_argument('--write-mib', type=int, default=32)
     parser.add_argument('--interval', type=float, default=5.0)
     parser.add_argument('--cgroup-root', default='/sys/fs/cgroup/user.slice')
@@ -76,6 +93,7 @@ def main(argv):
     args = parser.parse_args(argv)
     floor, threshold = args.floor_mib * MIB, args.write_mib * MIB
     log(event='started', paths=args.path, floor_mib=args.floor_mib,
+        inode_floor_percent=args.inode_floor_percent,
         write_mib=args.write_mib, interval=args.interval, dry_run=args.dry_run)
     previous = leaf_writes(args.cgroup_root)
     while True:
@@ -83,7 +101,8 @@ def main(argv):
         try:
             current = leaf_writes(args.cgroup_root)
             free = {path: free_bytes(path) for path in args.path}
-            low = [path for path, value in free.items() if value < floor]
+            inodes = {path: free_inodes_percent(path) for path in args.path}
+            low = low_paths(free, inodes, floor, args.inode_floor_percent)
             victim = choose_victim(previous, current, bool(low), threshold)
             previous = current
             if victim is None:

package/ansible/ubuntu/policy.json

@@ -48,6 +48,55 @@
     "diskGcPressureDockerAgeHours": 24,
     "diskGcPressureCaches": [".bun/install/cache"],
     "dockerLogMaxSize": "50m",
-    "dockerLogMaxFiles": "3"
+    "dockerLogMaxFiles": "3",
+    "inodeWarnFreePercent": 10,
+    "inodeFailFreePercent": 2
+  },
+  "clock": {
+    "warnOffsetSeconds": 2,
+    "failOffsetSeconds": 30
+  },
+  "rebootPendingFailDays": 14,
+  "dockerSmokeTest": {
+    "image": "busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec",
+    "timeoutSeconds": 60
+  },
+  "browser": {
+    "packages": {
+      "fonts-liberation": "1:2.1.5-3build1",
+      "libasound2t64": "1.2.15.3-1ubuntu1.1",
+      "libatk-bridge2.0-0t64": "2.60.4-0ubuntu0.1",
+      "libatk1.0-0t64": "2.60.4-0ubuntu0.1",
+      "libatspi2.0-0t64": "2.60.4-0ubuntu0.1",
+      "libcairo2": "1.18.4-3",
+      "libcups2t64": "2.4.16-1ubuntu1.3",
+      "libdbus-1-3": "1.16.2-2ubuntu4",
+      "libdrm2": "2.4.131-1",
+      "libgbm1": "26.0.8-1ubuntu0.3",
+      "libglib2.0-0t64": "2.88.0-1ubuntu0.1",
+      "libnspr4": "2:4.38.2-1ubuntu1",
+      "libnss3": "2:3.120-1ubuntu2.2",
+      "libpango-1.0-0": "1.57.0-1",
+      "libx11-6": "2:1.8.13-1ubuntu0.1",
+      "libxcb1": "1.17.0-2ubuntu1",
+      "libxcomposite1": "1:0.4.6-1build1",
+      "libxdamage1": "1:1.1.7-1",
+      "libxext6": "2:1.3.4-1build3",
+      "libxfixes3": "1:6.0.0-2build2",
+      "libxkbcommon0": "1.13.1-1",
+      "libxrandr2": "2:1.5.4-1build1"
+    },
+    "headlessShell": {
+      "version": "158.0.8127.18",
+      "amd64": {
+        "archive": "linux64/chrome-headless-shell-linux64",
+        "sha256": "18352d22cb0cdf520cbff2ad8c41391f0e02a72886cb53b0fde4bdfc4bd075d6"
+      },
+      "arm64": {
+        "archive": "linux-arm64/chrome-headless-shell-linux-arm64",
+        "sha256": "b136462898353d94c7fab50f9cdaa6d8f0d4c190153916936aa0a03353a26bf2"
+      }
+    },
+    "launchTimeoutSeconds": 60
   }
 }