@crvouga/dev-fleet 0.1.0 → 0.3.0

package/ansible/ubuntu/policy.json

@@ -16,13 +16,17 @@
     "git": "1:2.53.0-1ubuntu1.1",
     "python3": "3.14.3-0ubuntu2",
     "unzip": "6.0-29ubuntu1",
-    "xz-utils": "5.8.3-1ubuntu0.1"
+    "xz-utils": "5.8.3-1ubuntu0.1",
+    "ansible-core": "2.20.1-1"
+  },
+  "tailscale": {
+    "version": "1.104.5",
+    "keySha256": "3e03dacf222698c60b8e2f990b809ca1b3e104de127767864284e6c228f1fb39"
   },
   "aptLockTimeoutSeconds": 120,
   "minimumFreeDiskMiB": 4096,
   "minimumFreeInodes": 10000,
-  "serviceStartLimitBurst": 5,
-  "serviceStartLimitIntervalSeconds": 300,
+  "t3RestartMaxDeferralHours": 24,
   "alwaysOnServices": ["ssh", "tailscaled", "docker", "beszel-agent"],
   "resourceSafety": {
     "swapFileMiB": 8192,
@@ -37,6 +41,12 @@
     "diskGuardFloorMiB": 3072,
     "diskGuardWriteMiB": 32,
     "diskGuardIntervalSeconds": 5,
+    "diskGcIdleDays": 7,
+    "diskGcPressureMiB": 15360,
+    "diskGcPressureIdleHours": 12,
+    "diskGcDockerAgeHours": 168,
+    "diskGcPressureDockerAgeHours": 24,
+    "diskGcPressureCaches": [".bun/install/cache"],
     "dockerLogMaxSize": "50m",
     "dockerLogMaxFiles": "3"
   }

package/ansible/ubuntu/t3-desktop.yml

@@ -53,6 +53,16 @@
       register: t3_desktop_installed
       changed_when: false
       failed_when: false
+    - name: Check whether the desktop app is running before an update
+      become: true
+      become_user: '{{ development_user }}'
+      environment:
+        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+      ansible.builtin.command:
+        argv: [systemctl, --user, is-active, --quiet, t3code-desktop.service]
+      register: t3_desktop_was_active
+      changed_when: false
+      failed_when: false
     - name: Unpack the pinned app when its version differs
       when: t3_desktop_installed.stdout | default('') != t3_desktop_name
       block:
@@ -94,6 +104,7 @@
               - -T
               - '/opt/.t3code-staging{{ t3_desktop_dir }}'
               - '{{ t3_desktop_dir }}'
+          register: t3_desktop_updated
           changed_when: true
         - name: Add the launcher entry and icon
           ansible.builtin.copy:
@@ -218,3 +229,11 @@
         name: t3code-desktop.service
         state: started
       failed_when: false
+    # A running app keeps the files of its release: restart it into the one
+    # just installed, only while it runs no agent turn. Otherwise
+    # personal-worker-recovery restarts it at the first idle check, and
+    # t3:running-version reports T3_RESTART_PENDING until then.
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
+      when:
+        - t3_desktop_was_active.rc == 0
+        - t3_desktop_updated is changed

package/ansible/ubuntu/t3-health.sh

@@ -1,33 +1,104 @@
 #!/bin/sh
-# Keep the worker T3 desktop app (UI and server) up and responsive. Run every minute by
-# personal-worker-recovery.timer (managed by fleet-sync, tasks/recovery.yml).
-# Stopped or failed: start it. Running but its local API has not answered for
-# three checks in a row: restart it, since a hung server serves no turns.
+# Keep the worker T3 desktop app (UI and server) up, responsive and on the
+# installed release. Run every minute by personal-worker-recovery.timer
+# (managed by fleet-sync, tasks/recovery.yml), from the stable copy the role
+# installs beside t3-idle.py.
+# - The former headless server is stopped and disabled: one T3 per worker.
+# - No graphical session: T3_NO_GRAPHICAL_SESSION; after three checks in a row
+#   the display manager is restarted so auto-login starts one, three times at
+#   most until a session is seen again.
+# - Stopped or failed: reset its start limit and start it.
+# - Running but its local API has not answered for three checks in a row:
+#   restart it, since a hung server serves no turns.
+# - Running another release than the installed one, or a policy restart that
+#   was deferred: restart it once it runs no agent turn (T3_RESTART_PENDING).
 set -u
 url=http://127.0.0.1:3773/.well-known/t3/environment
-state="${XDG_RUNTIME_DIR:?}/personal-t3-health-failures"
+unit=t3code-desktop.service
+runtime="${XDG_RUNTIME_DIR:?}"
+failures="$runtime/personal-t3-health-failures"
+no_session="$runtime/personal-t3-no-session"
+display_restarts="$runtime/personal-t3-display-restarts"
+pending="$runtime/personal-t3-restart-pending"
+restarted="$runtime/personal-t3-restarted-for"
+installed="${T3_DESKTOP_DIR:-/opt/T3 Code (Nightly)}/.personal-fleet-version"
+here=$(dirname "$0")
 limit=3
 
-# The former headless server still runs until its approved migration; never
-# start a second T3 next to it.
-if systemctl --user is-active --quiet t3code.service; then
-  echo "Headless T3 still runs; desktop migration pending."
-  exit 0
+count() {
+  n=$(($(cat "$1" 2>/dev/null || echo 0) + 1))
+  echo "$n" > "$1"
+  echo "$n"
+}
+
+if systemctl --user is-active --quiet t3code.service ||
+  systemctl --user is-enabled --quiet t3code.service; then
+  echo "Retiring the former headless T3 server."
+  if ! systemctl --user disable --now t3code.service; then
+    echo "T3_LEGACY_SERVER_RUNNING"
+    exit 1
+  fi
+fi
+
+if ! systemctl --user is-active --quiet graphical-session.target; then
+  rm -f "$failures"
+  seen=$(count "$no_session")
+  echo "T3_NO_GRAPHICAL_SESSION ($seen/$limit)."
+  if [ "$seen" -ge "$limit" ]; then
+    rm -f "$no_session"
+    restarts=$(count "$display_restarts")
+    if [ "$restarts" -le "$limit" ]; then
+      echo "Restarting the display manager so auto-login starts a session ($restarts/$limit)."
+      sudo -n systemctl restart display-manager.service
+    else
+      echo "The display manager was restarted $limit times; a person must log in."
+    fi
+  fi
+  exit 1
 fi
-if ! systemctl --user is-active --quiet t3code-desktop.service; then
-  rm -f "$state"
+rm -f "$no_session" "$display_restarts"
+
+if ! systemctl --user is-active --quiet "$unit"; then
+  rm -f "$failures"
   echo "T3 is not running; starting it."
-  exec systemctl --user start t3code-desktop.service
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user start "$unit"
 fi
-if curl --fail --silent --max-time 10 --output /dev/null "$url"; then
-  rm -f "$state"
+if ! environment=$(curl --fail --silent --max-time 10 "$url"); then
+  failed=$(count "$failures")
+  echo "T3 local API did not answer ($failed/$limit)."
+  if [ "$failed" -ge "$limit" ]; then
+    rm -f "$failures"
+    echo "T3 is unresponsive; restarting it."
+    systemctl --user reset-failed "$unit" 2>/dev/null
+    exec systemctl --user restart "$unit"
+  fi
   exit 0
 fi
-failures=$(($(cat "$state" 2>/dev/null || echo 0) + 1))
-echo "$failures" > "$state"
-echo "T3 local API did not answer ($failures/$limit)."
-if [ "$failures" -ge "$limit" ]; then
-  rm -f "$state"
-  echo "T3 is unresponsive; restarting it."
-  exec systemctl --user restart t3code-desktop.service
+rm -f "$failures"
+
+running=$(printf '%s' "$environment" | sed -n 's/.*"serverVersion" *: *"\([^"]*\)".*/\1/p')
+release=$(sed -n 's/^T3-Code-\(.*\)-[a-z0-9]*\.deb$/\1/p' "$installed" 2>/dev/null)
+reason=
+[ -f "$pending" ] && reason="the staged policy"
+if [ -n "$release" ] && [ "$running" != "$release" ]; then
+  # One restart per release: a server that still reports another version
+  # after it is never restarted in a loop.
+  if [ "$(cat "$restarted" 2>/dev/null)" = "$release" ]; then
+    echo "T3_RUNNING_VERSION_MISMATCH: restarted for $release, still reports ${running:-nothing}."
+    exit 1
+  fi
+  reason="release $release"
+else
+  rm -f "$restarted"
+fi
+[ -z "$reason" ] && exit 0
+if python3 -I -B "$here/t3-idle.py" --state "${T3CODE_HOME:-$HOME/.t3}/userdata/statev2.sqlite"; then
+  rm -f "$pending"
+  [ "$reason" = "release $release" ] && echo "$release" > "$restarted"
+  echo "T3 runs no agent turn; restarting it to load $reason."
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user restart "$unit"
 fi
+echo "T3_RESTART_PENDING: agent turns are running; $reason loads at the first idle check."
+exit 0

package/ansible/ubuntu/t3-idle.py

@@ -0,0 +1,42 @@
+"""Say whether the worker's T3 runs an agent turn: exit 0 when it runs none,
+1 when it runs one and 2 when it cannot tell.
+
+T3 offers no command that lists the running turns of every project, so this
+reads the runs T3 records in its own state database, read-only. Anything it
+cannot read counts as busy: a restart never ends a turn on a guess.
+
+Usage: t3-idle.py --state ~/.t3/userdata/statev2.sqlite
+"""
+import argparse
+import sqlite3
+import sys
+
+IDLE, BUSY, UNKNOWN = 0, 1, 2
+
+
+def running_turns(state):
+    connection = sqlite3.connect(f'file:{state}?mode=ro', uri=True, timeout=5)
+    try:
+        (count,) = connection.execute(
+            "SELECT count(*) FROM orchestration_v2_projection_runs WHERE status = 'running'"
+        ).fetchone()
+        return count
+    finally:
+        connection.close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--state', required=True)
+    args = parser.parse_args()
+    try:
+        count = running_turns(args.state)
+    except sqlite3.Error:
+        print('T3_IDLE_UNKNOWN')
+        return UNKNOWN
+    print(f'{count} running turn(s)')
+    return BUSY if count else IDLE
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/ansible/ubuntu/t3-policy.yml

@@ -41,8 +41,9 @@
         mode: '0644'
         backup: true
     # T3 reads its policy once, at startup. A running app that has not loaded
-    # the staged policy is restarted (this ends its running turns) so every
-    # node offers the same providers after every sync.
+    # the staged policy is restarted so every node offers the same providers,
+    # but only while it runs no agent turn (tasks/t3-restart.yml): a busy app
+    # keeps its turns and loads the policy at the first idle check.
     - name: Check whether the desktop app is running
       become: true
       become_user: '{{ development_user }}'
@@ -62,15 +63,7 @@
       changed_when: false
       failed_when: false
       when: t3_desktop_active.rc == 0
-    - name: Restart T3 so it loads the staged policy
-      become: true
-      become_user: '{{ development_user }}'
-      environment:
-        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
-      ansible.builtin.systemd_service:
-        scope: user
-        name: t3code-desktop.service
-        state: restarted
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
       when:
         - t3_desktop_active.rc == 0
         - t3_policy_loaded.stdout is not search('loaded this policy\.\s*$')

package/ansible/ubuntu/tailscale.yml

@@ -4,18 +4,21 @@
   connection: local
   become: true
   gather_facts: true
+  # The approved version and the signing key's digest: `tailscale` in policy.json.
+  vars_files:
+    - policy.json
   tasks:
     - name: Require supported Ubuntu
       ansible.builtin.assert:
         that:
           - ansible_facts.distribution == 'Ubuntu'
-          - ansible_facts.distribution_version == '26.04'
+          - ansible_facts.distribution_version == release
     - name: Inspect installed packages before possible network interruption
       ansible.builtin.package_facts:
         manager: apt
     - name: Require maintenance for an installed version change
       ansible.builtin.assert:
-        that: "'tailscale' not in ansible_facts.packages or ansible_facts.packages.tailscale[0].version == '1.104.5'"
+        that: "'tailscale' not in ansible_facts.packages or ansible_facts.packages.tailscale[0].version == tailscale.version"
         fail_msg: TAILSCALE_UPGRADE_PENDING; upgrading an active VPN can disconnect SSH. Review an explicit maintenance operation.
     - name: Inspect existing package-source ownership before writing
       ansible.builtin.find:
@@ -34,7 +37,7 @@
       ansible.builtin.get_url:
         url: https://pkgs.tailscale.com/stable/ubuntu/resolute.noarmor.gpg
         dest: /usr/share/keyrings/tailscale-archive-keyring.gpg
-        checksum: sha256:3e03dacf222698c60b8e2f990b809ca1b3e104de127767864284e6c228f1fb39
+        checksum: 'sha256:{{ tailscale.keySha256 }}'
         owner: root
         group: root
         mode: '0644'
@@ -48,11 +51,11 @@
       register: tailscale_source
     - name: Install the approved version only when absent
       ansible.builtin.apt:
-        name: tailscale=1.104.5
+        name: 'tailscale={{ tailscale.version }}'
         state: present
         update_cache: true
         cache_valid_time: 0
-        lock_timeout: 120
+        lock_timeout: '{{ aptLockTimeoutSeconds }}'
         allow_downgrade: false
         allow_unauthenticated: false
       when: "'tailscale' not in ansible_facts.packages"

package/ansible/ubuntu/tasks/recovery.yml

@@ -1,4 +1,25 @@
 ---
+# The unit runs a copy installed outside every revision: a synced revision is
+# pruned two syncs later, and the unit must not name a file that is gone.
+- name: Create the directory of the installed recovery scripts
+  become: true
+  ansible.builtin.file:
+    path: /usr/local/lib/personal-fleet
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+- name: Install the T3 recovery scripts outside the revision
+  become: true
+  ansible.builtin.copy:
+    src: '{{ playbook_dir }}/{{ item }}'
+    dest: '/usr/local/lib/personal-fleet/{{ item }}'
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - t3-health.sh
+    - t3-idle.py
 - name: Declare bounded recovery of the worker T3 desktop app
   ansible.builtin.copy:
     dest: '{{ development_home }}/.config/systemd/user/personal-worker-recovery.service'
@@ -10,7 +31,7 @@
       Description=Keep the worker T3 desktop app running and responsive
       [Service]
       Type=oneshot
-      ExecStart=/bin/sh {{ development_source }}/ansible/ubuntu/t3-health.sh
+      ExecStart=/bin/sh /usr/local/lib/personal-fleet/t3-health.sh
       TimeoutStartSec=90
 - name: Declare persistent service recovery checks
   ansible.builtin.copy:
@@ -37,3 +58,41 @@
     daemon_reload: true
     enabled: true
     state: started
+- name: Declare the work-in-progress guard of agent worktrees
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.service'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      # Managed by personal fleet-sync: ansible/ubuntu/wip-guard.py
+      [Unit]
+      Description=Push work in progress of agent worktrees to recovery refs
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/bin/python3 {{ development_source }}/ansible/ubuntu/wip-guard.py --root {{ development_home }}/.t3/worktrees --node %l
+      TimeoutStartSec=600
+- name: Declare how often work in progress is saved
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.timer'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      [Unit]
+      Description=Save work in progress of agent worktrees every 15 minutes
+      [Timer]
+      OnBootSec=5min
+      OnUnitActiveSec=15min
+      Persistent=true
+      [Install]
+      WantedBy=timers.target
+- name: Enable the work-in-progress guard
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: personal-worker-wip-guard.timer
+    daemon_reload: true
+    enabled: true
+    state: started

package/ansible/ubuntu/tasks/resources.yml

@@ -2,7 +2,9 @@
 # Memory and disk safety: a runaway workload is stopped on its own before the
 # machine, SSH, Tailscale or the T3 server run out of memory or disk. These
 # tasks run before the capacity preflight because they matter most when a disk
-# is nearly full; they only write small files and never delete user data.
+# is nearly full; they only write small files and never delete user data. The
+# disk GC timer they install deletes only build output and caches a build or
+# install recreates (ansible/ubuntu/disk-gc.py).
 - name: Inspect active swap
   ansible.builtin.command:
     argv: [swapon, --show=NAME, --noheadings]
@@ -250,3 +252,61 @@
     enabled: true
     state: '{{ "restarted" if disk_guard_unit.changed else "started" }}'
   when: development_source is defined and disk_guard_source.stat.exists
+
+- name: Read the tracked disk GC
+  ansible.builtin.stat:
+    path: '{{ development_source }}/ansible/ubuntu/disk-gc.py'
+    checksum_algorithm: sha256
+  register: disk_gc_source
+  when: development_source is defined
+- name: Declare the disk GC service and its hourly timer
+  ansible.builtin.copy:
+    dest: '{{ item.dest }}'
+    owner: root
+    group: root
+    mode: '0644'
+    content: '{{ item.content }}'
+  loop:
+    - dest: /etc/systemd/system/personal-fleet-disk-gc.service
+      content: |
+        # Managed by personal fleet-sync: ansible/ubuntu/disk-gc.py
+        # Source sha256 {{ disk_gc_source.stat.checksum }}
+        [Unit]
+        Description=Personal Fleet disk GC (deletes build output and caches a build recreates)
+        After=docker.service
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
+
+        TimeoutStartSec=2h
+        Nice=19
+        IOSchedulingClass=idle
+        CPUQuota=50%
+        MemoryMax=512M
+        NoNewPrivileges=true
+        PrivateTmp=true
+        LogRateLimitIntervalSec=60
+        LogRateLimitBurst=200
+    - dest: /etc/systemd/system/personal-fleet-disk-gc.timer
+      content: |
+        # Managed by personal fleet-sync: ansible/ubuntu/disk-gc.py
+        [Unit]
+        Description=Run the Personal Fleet disk GC hourly
+        [Timer]
+        OnBootSec=10min
+        OnCalendar=hourly
+        RandomizedDelaySec=5min
+        Persistent=true
+        [Install]
+        WantedBy=timers.target
+  loop_control:
+    label: '{{ item.dest }}'
+  when: development_source is defined and disk_gc_source.stat.exists
+  register: disk_gc_units
+- name: Schedule the disk GC
+  ansible.builtin.systemd_service:
+    name: personal-fleet-disk-gc.timer
+    daemon_reload: '{{ disk_gc_units.changed }}'
+    enabled: true
+    state: started
+  when: development_source is defined and disk_gc_source.stat.exists

package/ansible/ubuntu/tasks/t3-restart.yml

@@ -0,0 +1,37 @@
+---
+# Restart a running T3 only while it runs no agent turn: a restart ends every
+# running turn. A busy T3 keeps its turns; the restart is recorded as pending
+# (with the time it was first deferred) and personal-worker-recovery applies it
+# at the first idle check (t3-health.sh). t3_restart_busy restarts it anyway.
+- name: Ask T3 whether an agent turn is running
+  become: true
+  become_user: '{{ development_user }}'
+  ansible.builtin.command:
+    argv:
+      - python3
+      - -I
+      - -B
+      - '{{ playbook_dir }}/t3-idle.py'
+      - --state
+      - '{{ development_home }}/.t3/userdata/statev2.sqlite'
+  register: t3_idle
+  changed_when: false
+  failed_when: false
+- name: Restart T3 while it runs no agent turn
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: t3code-desktop.service
+    state: restarted
+  when: t3_idle is not skipped and (t3_idle.rc == 0 or t3_restart_busy | default(false) | bool)
+- name: Record the restart as pending while agent turns run
+  ansible.builtin.copy:
+    dest: '/run/user/{{ development_uid }}/personal-t3-restart-pending'
+    content: "{{ now(utc=true).timestamp() | int }}\n"
+    force: false
+    owner: '{{ development_user }}'
+    mode: '0600'
+  when: t3_idle is not skipped and t3_idle.rc != 0 and not (t3_restart_busy | default(false) | bool)

package/ansible/ubuntu/wip-guard.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+"""Push work in progress of every agent worktree to a recovery ref.
+
+Managed by personal fleet-sync (tasks/recovery.yml) as
+personal-worker-wip-guard.service, started by personal-worker-wip-guard.timer.
+
+Workers are disposable: a worker lost mid-task must lose at most one interval
+of uncommitted work. For each Git worktree under --root, the files as they are
+now (tracked changes and untracked files that are not ignored) are written as
+one commit on top of HEAD, built in a private index so the worktree, its index
+and its branches are never touched, and pushed to
+
+    refs/recovery/<node>/<branch>
+
+of the worktree's own remote. A clean worktree whose HEAD is already on a
+remote branch has nothing to recover and pushes nothing. Recover with
+`git fetch origin 'refs/recovery/*:refs/recovery/*'`.
+
+Every decision is one JSON line on stdout.
+
+Usage: wip-guard.py --root DIR [--root DIR ...] [--node ID] [--remote NAME]
+                    [--max-depth N] [--dry-run]
+"""
+import argparse
+import json
+import os
+import re
+import socket
+import subprocess
+import tempfile
+from pathlib import Path
+
+GIT = ['git', '-c', 'safe.directory=*']
+AUTHOR = {
+    'GIT_AUTHOR_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_AUTHOR_EMAIL': 'wip-guard@example.invalid',
+    'GIT_COMMITTER_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_COMMITTER_EMAIL': 'wip-guard@example.invalid',
+}
+
+
+def log(**event):
+    print(json.dumps(event, sort_keys=True), flush=True)
+
+
+def git(path, *args, env=None):
+    return subprocess.run(
+        [*GIT, '-C', str(path), *args], capture_output=True, text=True,
+        env={**os.environ, **(env or {})}, timeout=120)
+
+
+def worktrees(root, depth):
+    if depth < 0 or not root.is_dir() or root.is_symlink():
+        return
+    if (root / '.git').exists():
+        yield root
+        return
+    for child in sorted(root.iterdir()):
+        if child.is_dir() and not child.is_symlink():
+            yield from worktrees(child, depth - 1)
+
+
+def ref_part(value):
+    part = re.sub(r'[^A-Za-z0-9._/-]+', '-', value).strip('./-')
+    part = re.sub(r'/+', '/', part).replace('..', '.')
+    return part or 'unnamed'
+
+
+def snapshot(path):
+    """The worktree's files as one commit on HEAD, or HEAD when clean."""
+    head = git(path, 'rev-parse', '--verify', 'HEAD').stdout.strip()
+    if not head:
+        return None, None
+    with tempfile.TemporaryDirectory() as private:
+        index = {'GIT_INDEX_FILE': os.path.join(private, 'index')}
+        if git(path, 'read-tree', 'HEAD', env=index).returncode != 0:
+            return head, None
+        if git(path, 'add', '--all', '--', '.', env=index).returncode != 0:
+            return head, None
+        tree = git(path, 'write-tree', env=index).stdout.strip()
+    if tree == git(path, 'rev-parse', 'HEAD^{tree}').stdout.strip():
+        return head, head
+    commit = git(path, 'commit-tree', tree, '-p', head, '-m',
+                 'Work in progress saved by the dev-fleet guard', env=AUTHOR)
+    return head, commit.stdout.strip() or None
+
+
+def guard(path, node, remote, dry_run):
+    if git(path, 'remote', 'get-url', remote).returncode != 0:
+        return log(worktree=str(path), action='skip', reason='no-remote')
+    head, commit = snapshot(path)
+    if not commit:
+        return log(worktree=str(path), action='skip', reason='no-commit')
+    if commit == head and git(path, 'branch', '--remotes', '--contains', head).stdout.strip():
+        return log(worktree=str(path), action='skip', reason='clean')
+    branch = git(path, 'symbolic-ref', '--quiet', '--short', 'HEAD').stdout.strip()
+    ref = 'refs/recovery/%s/%s' % (ref_part(node), ref_part(branch or 'detached-' + head[:12]))
+    if dry_run:
+        return log(worktree=str(path), action='would-push', ref=ref)
+    pushed = git(path, 'push', '--force', '--no-verify', '--quiet', remote, '%s:%s' % (commit, ref))
+    log(worktree=str(path), action='pushed' if pushed.returncode == 0 else 'failed', ref=ref)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', action='append', required=True)
+    parser.add_argument('--node', default=socket.gethostname().split('.')[0])
+    parser.add_argument('--remote', default='origin')
+    parser.add_argument('--max-depth', type=int, default=3)
+    parser.add_argument('--dry-run', action='store_true')
+    options = parser.parse_args()
+    for root in options.root:
+        for path in worktrees(Path(root).expanduser(), options.max_depth):
+            try:
+                guard(path, options.node, options.remote, options.dry_run)
+            except (OSError, subprocess.SubprocessError):
+                log(worktree=str(path), action='failed', reason='git')
+
+
+if __name__ == '__main__':
+    main()