@crvouga/dev-fleet 0.2.0 → 0.4.0

package/ansible/ubuntu/disk-guard.py

@@ -3,14 +3,15 @@
 
 Managed by personal fleet-sync (tasks/resources.yml) as
 personal-fleet-disk-guard.service. Every interval it measures free space on the
-watched filesystems and the bytes each leaf cgroup under user.slice wrote. When
-any watched filesystem is below the floor, it kills only the single cgroup that
+watched filesystems (bytes and inodes) and the bytes each leaf cgroup under user.slice wrote. When
+any watched filesystem is below the floor of free bytes or of free inodes, it
+kills only the single cgroup that
 wrote the most during the last interval, and only if that is at least the write
 threshold. It never deletes files, never touches system services and never
 kills idle work. Free space must be recovered by a person.
 
-Usage: disk-guard.py --path / --path /home/me [--floor-mib N] [--write-mib N]
-                     [--interval S] [--cgroup-root DIR] [--dry-run]
+Usage: disk-guard.py --path / --path /home/me [--floor-mib N]
+                     [--inode-floor-percent N] [--write-mib N] [--interval S] [--cgroup-root DIR] [--dry-run]
 """
 import argparse
 import json
@@ -61,6 +62,21 @@ def free_bytes(path):
     return st.f_bavail * st.f_frsize
 
 
+def free_inodes_percent(path):
+    """Free inodes as a percentage; None on a filesystem without a fixed count."""
+    st = os.statvfs(path)
+    if st.f_files <= 0:
+        return None
+    return st.f_favail * 100 / st.f_files
+
+
+def low_paths(free, inodes, floor, inode_floor):
+    """The watched paths below either floor: bytes, or free inodes in percent."""
+    return [path for path in free
+            if free[path] < floor
+            or (inodes.get(path) is not None and inodes[path] < inode_floor)]
+
+
 def log(**event):
     print(json.dumps(event, sort_keys=True), flush=True)
 
@@ -69,6 +85,7 @@ def main(argv):
     parser = argparse.ArgumentParser()
     parser.add_argument('--path', action='append', required=True)
     parser.add_argument('--floor-mib', type=int, default=3072)
+    parser.add_argument('--inode-floor-percent', type=float, default=2)
     parser.add_argument('--write-mib', type=int, default=32)
     parser.add_argument('--interval', type=float, default=5.0)
     parser.add_argument('--cgroup-root', default='/sys/fs/cgroup/user.slice')
@@ -76,6 +93,7 @@ def main(argv):
     args = parser.parse_args(argv)
     floor, threshold = args.floor_mib * MIB, args.write_mib * MIB
     log(event='started', paths=args.path, floor_mib=args.floor_mib,
+        inode_floor_percent=args.inode_floor_percent,
         write_mib=args.write_mib, interval=args.interval, dry_run=args.dry_run)
     previous = leaf_writes(args.cgroup_root)
     while True:
@@ -83,7 +101,8 @@ def main(argv):
         try:
             current = leaf_writes(args.cgroup_root)
             free = {path: free_bytes(path) for path in args.path}
-            low = [path for path, value in free.items() if value < floor]
+            inodes = {path: free_inodes_percent(path) for path in args.path}
+            low = low_paths(free, inodes, floor, args.inode_floor_percent)
             victim = choose_victim(previous, current, bool(low), threshold)
             previous = current
             if victim is None:

package/ansible/ubuntu/policy.json

@@ -26,6 +26,7 @@
   "aptLockTimeoutSeconds": 120,
   "minimumFreeDiskMiB": 4096,
   "minimumFreeInodes": 10000,
+  "t3RestartMaxDeferralHours": 24,
   "alwaysOnServices": ["ssh", "tailscaled", "docker", "beszel-agent"],
   "resourceSafety": {
     "swapFileMiB": 8192,
@@ -47,6 +48,55 @@
     "diskGcPressureDockerAgeHours": 24,
     "diskGcPressureCaches": [".bun/install/cache"],
     "dockerLogMaxSize": "50m",
-    "dockerLogMaxFiles": "3"
+    "dockerLogMaxFiles": "3",
+    "inodeWarnFreePercent": 10,
+    "inodeFailFreePercent": 2
+  },
+  "clock": {
+    "warnOffsetSeconds": 2,
+    "failOffsetSeconds": 30
+  },
+  "rebootPendingFailDays": 14,
+  "dockerSmokeTest": {
+    "image": "busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec",
+    "timeoutSeconds": 60
+  },
+  "browser": {
+    "packages": {
+      "fonts-liberation": "1:2.1.5-3build1",
+      "libasound2t64": "1.2.15.3-1ubuntu1.1",
+      "libatk-bridge2.0-0t64": "2.60.4-0ubuntu0.1",
+      "libatk1.0-0t64": "2.60.4-0ubuntu0.1",
+      "libatspi2.0-0t64": "2.60.4-0ubuntu0.1",
+      "libcairo2": "1.18.4-3",
+      "libcups2t64": "2.4.16-1ubuntu1.3",
+      "libdbus-1-3": "1.16.2-2ubuntu4",
+      "libdrm2": "2.4.131-1",
+      "libgbm1": "26.0.8-1ubuntu0.3",
+      "libglib2.0-0t64": "2.88.0-1ubuntu0.1",
+      "libnspr4": "2:4.38.2-1ubuntu1",
+      "libnss3": "2:3.120-1ubuntu2.2",
+      "libpango-1.0-0": "1.57.0-1",
+      "libx11-6": "2:1.8.13-1ubuntu0.1",
+      "libxcb1": "1.17.0-2ubuntu1",
+      "libxcomposite1": "1:0.4.6-1build1",
+      "libxdamage1": "1:1.1.7-1",
+      "libxext6": "2:1.3.4-1build3",
+      "libxfixes3": "1:6.0.0-2build2",
+      "libxkbcommon0": "1.13.1-1",
+      "libxrandr2": "2:1.5.4-1build1"
+    },
+    "headlessShell": {
+      "version": "158.0.8127.18",
+      "amd64": {
+        "archive": "linux64/chrome-headless-shell-linux64",
+        "sha256": "18352d22cb0cdf520cbff2ad8c41391f0e02a72886cb53b0fde4bdfc4bd075d6"
+      },
+      "arm64": {
+        "archive": "linux-arm64/chrome-headless-shell-linux-arm64",
+        "sha256": "b136462898353d94c7fab50f9cdaa6d8f0d4c190153916936aa0a03353a26bf2"
+      }
+    },
+    "launchTimeoutSeconds": 60
   }
 }

package/ansible/ubuntu/t3-desktop.yml

@@ -53,6 +53,16 @@
       register: t3_desktop_installed
       changed_when: false
       failed_when: false
+    - name: Check whether the desktop app is running before an update
+      become: true
+      become_user: '{{ development_user }}'
+      environment:
+        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+      ansible.builtin.command:
+        argv: [systemctl, --user, is-active, --quiet, t3code-desktop.service]
+      register: t3_desktop_was_active
+      changed_when: false
+      failed_when: false
     - name: Unpack the pinned app when its version differs
       when: t3_desktop_installed.stdout | default('') != t3_desktop_name
       block:
@@ -94,6 +104,7 @@
               - -T
               - '/opt/.t3code-staging{{ t3_desktop_dir }}'
               - '{{ t3_desktop_dir }}'
+          register: t3_desktop_updated
           changed_when: true
         - name: Add the launcher entry and icon
           ansible.builtin.copy:
@@ -218,3 +229,11 @@
         name: t3code-desktop.service
         state: started
       failed_when: false
+    # A running app keeps the files of its release: restart it into the one
+    # just installed, only while it runs no agent turn. Otherwise
+    # personal-worker-recovery restarts it at the first idle check, and
+    # t3:running-version reports T3_RESTART_PENDING until then.
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
+      when:
+        - t3_desktop_was_active.rc == 0
+        - t3_desktop_updated is changed

package/ansible/ubuntu/t3-health.sh

@@ -1,33 +1,104 @@
 #!/bin/sh
-# Keep the worker T3 desktop app (UI and server) up and responsive. Run every minute by
-# personal-worker-recovery.timer (managed by fleet-sync, tasks/recovery.yml).
-# Stopped or failed: start it. Running but its local API has not answered for
-# three checks in a row: restart it, since a hung server serves no turns.
+# Keep the worker T3 desktop app (UI and server) up, responsive and on the
+# installed release. Run every minute by personal-worker-recovery.timer
+# (managed by fleet-sync, tasks/recovery.yml), from the stable copy the role
+# installs beside t3-idle.py.
+# - The former headless server is stopped and disabled: one T3 per worker.
+# - No graphical session: T3_NO_GRAPHICAL_SESSION; after three checks in a row
+#   the display manager is restarted so auto-login starts one, three times at
+#   most until a session is seen again.
+# - Stopped or failed: reset its start limit and start it.
+# - Running but its local API has not answered for three checks in a row:
+#   restart it, since a hung server serves no turns.
+# - Running another release than the installed one, or a policy restart that
+#   was deferred: restart it once it runs no agent turn (T3_RESTART_PENDING).
 set -u
 url=http://127.0.0.1:3773/.well-known/t3/environment
-state="${XDG_RUNTIME_DIR:?}/personal-t3-health-failures"
+unit=t3code-desktop.service
+runtime="${XDG_RUNTIME_DIR:?}"
+failures="$runtime/personal-t3-health-failures"
+no_session="$runtime/personal-t3-no-session"
+display_restarts="$runtime/personal-t3-display-restarts"
+pending="$runtime/personal-t3-restart-pending"
+restarted="$runtime/personal-t3-restarted-for"
+installed="${T3_DESKTOP_DIR:-/opt/T3 Code (Nightly)}/.personal-fleet-version"
+here=$(dirname "$0")
 limit=3
 
-# The former headless server still runs until its approved migration; never
-# start a second T3 next to it.
-if systemctl --user is-active --quiet t3code.service; then
-  echo "Headless T3 still runs; desktop migration pending."
-  exit 0
+count() {
+  n=$(($(cat "$1" 2>/dev/null || echo 0) + 1))
+  echo "$n" > "$1"
+  echo "$n"
+}
+
+if systemctl --user is-active --quiet t3code.service ||
+  systemctl --user is-enabled --quiet t3code.service; then
+  echo "Retiring the former headless T3 server."
+  if ! systemctl --user disable --now t3code.service; then
+    echo "T3_LEGACY_SERVER_RUNNING"
+    exit 1
+  fi
+fi
+
+if ! systemctl --user is-active --quiet graphical-session.target; then
+  rm -f "$failures"
+  seen=$(count "$no_session")
+  echo "T3_NO_GRAPHICAL_SESSION ($seen/$limit)."
+  if [ "$seen" -ge "$limit" ]; then
+    rm -f "$no_session"
+    restarts=$(count "$display_restarts")
+    if [ "$restarts" -le "$limit" ]; then
+      echo "Restarting the display manager so auto-login starts a session ($restarts/$limit)."
+      sudo -n systemctl restart display-manager.service
+    else
+      echo "The display manager was restarted $limit times; a person must log in."
+    fi
+  fi
+  exit 1
 fi
-if ! systemctl --user is-active --quiet t3code-desktop.service; then
-  rm -f "$state"
+rm -f "$no_session" "$display_restarts"
+
+if ! systemctl --user is-active --quiet "$unit"; then
+  rm -f "$failures"
   echo "T3 is not running; starting it."
-  exec systemctl --user start t3code-desktop.service
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user start "$unit"
 fi
-if curl --fail --silent --max-time 10 --output /dev/null "$url"; then
-  rm -f "$state"
+if ! environment=$(curl --fail --silent --max-time 10 "$url"); then
+  failed=$(count "$failures")
+  echo "T3 local API did not answer ($failed/$limit)."
+  if [ "$failed" -ge "$limit" ]; then
+    rm -f "$failures"
+    echo "T3 is unresponsive; restarting it."
+    systemctl --user reset-failed "$unit" 2>/dev/null
+    exec systemctl --user restart "$unit"
+  fi
   exit 0
 fi
-failures=$(($(cat "$state" 2>/dev/null || echo 0) + 1))
-echo "$failures" > "$state"
-echo "T3 local API did not answer ($failures/$limit)."
-if [ "$failures" -ge "$limit" ]; then
-  rm -f "$state"
-  echo "T3 is unresponsive; restarting it."
-  exec systemctl --user restart t3code-desktop.service
+rm -f "$failures"
+
+running=$(printf '%s' "$environment" | sed -n 's/.*"serverVersion" *: *"\([^"]*\)".*/\1/p')
+release=$(sed -n 's/^T3-Code-\(.*\)-[a-z0-9]*\.deb$/\1/p' "$installed" 2>/dev/null)
+reason=
+[ -f "$pending" ] && reason="the staged policy"
+if [ -n "$release" ] && [ "$running" != "$release" ]; then
+  # One restart per release: a server that still reports another version
+  # after it is never restarted in a loop.
+  if [ "$(cat "$restarted" 2>/dev/null)" = "$release" ]; then
+    echo "T3_RUNNING_VERSION_MISMATCH: restarted for $release, still reports ${running:-nothing}."
+    exit 1
+  fi
+  reason="release $release"
+else
+  rm -f "$restarted"
+fi
+[ -z "$reason" ] && exit 0
+if python3 -I -B "$here/t3-idle.py" --state "${T3CODE_HOME:-$HOME/.t3}/userdata/statev2.sqlite"; then
+  rm -f "$pending"
+  [ "$reason" = "release $release" ] && echo "$release" > "$restarted"
+  echo "T3 runs no agent turn; restarting it to load $reason."
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user restart "$unit"
 fi
+echo "T3_RESTART_PENDING: agent turns are running; $reason loads at the first idle check."
+exit 0

package/ansible/ubuntu/t3-idle.py

@@ -0,0 +1,42 @@
+"""Say whether the worker's T3 runs an agent turn: exit 0 when it runs none,
+1 when it runs one and 2 when it cannot tell.
+
+T3 offers no command that lists the running turns of every project, so this
+reads the runs T3 records in its own state database, read-only. Anything it
+cannot read counts as busy: a restart never ends a turn on a guess.
+
+Usage: t3-idle.py --state ~/.t3/userdata/statev2.sqlite
+"""
+import argparse
+import sqlite3
+import sys
+
+IDLE, BUSY, UNKNOWN = 0, 1, 2
+
+
+def running_turns(state):
+    connection = sqlite3.connect(f'file:{state}?mode=ro', uri=True, timeout=5)
+    try:
+        (count,) = connection.execute(
+            "SELECT count(*) FROM orchestration_v2_projection_runs WHERE status = 'running'"
+        ).fetchone()
+        return count
+    finally:
+        connection.close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--state', required=True)
+    args = parser.parse_args()
+    try:
+        count = running_turns(args.state)
+    except sqlite3.Error:
+        print('T3_IDLE_UNKNOWN')
+        return UNKNOWN
+    print(f'{count} running turn(s)')
+    return BUSY if count else IDLE
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/ansible/ubuntu/t3-policy.yml

@@ -41,8 +41,9 @@
         mode: '0644'
         backup: true
     # T3 reads its policy once, at startup. A running app that has not loaded
-    # the staged policy is restarted (this ends its running turns) so every
-    # node offers the same providers after every sync.
+    # the staged policy is restarted so every node offers the same providers,
+    # but only while it runs no agent turn (tasks/t3-restart.yml): a busy app
+    # keeps its turns and loads the policy at the first idle check.
     - name: Check whether the desktop app is running
       become: true
       become_user: '{{ development_user }}'
@@ -62,15 +63,7 @@
       changed_when: false
       failed_when: false
       when: t3_desktop_active.rc == 0
-    - name: Restart T3 so it loads the staged policy
-      become: true
-      become_user: '{{ development_user }}'
-      environment:
-        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
-      ansible.builtin.systemd_service:
-        scope: user
-        name: t3code-desktop.service
-        state: restarted
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
       when:
         - t3_desktop_active.rc == 0
         - t3_policy_loaded.stdout is not search('loaded this policy\.\s*$')

package/ansible/ubuntu/tasks/browser.yml

@@ -0,0 +1,56 @@
+---
+# Headless Chromium for browser tests (Playwright, Storybook, end-to-end
+# suites): the system libraries it loads, at the snapshot's exact versions,
+# and one pinned Chrome Headless Shell build. Applied to workers that select
+# the web tool profile; the launch below backs worker:browser.
+- name: Install the exact libraries headless Chromium loads
+  ansible.builtin.apt:
+    name: '{{ item.key }}={{ item.value }}'
+    state: present
+    lock_timeout: '{{ aptLockTimeoutSeconds }}'
+    allow_downgrade: false
+    allow_unauthenticated: false
+    fail_on_autoremove: true
+    policy_rc_d: 101
+  environment:
+    APT_CONFIG: /etc/personal-fleet/apt.conf
+  loop: '{{ browser.packages | dict2items }}'
+- name: Name the headless shell build of this architecture
+  ansible.builtin.set_fact:
+    browser_build: "{{ browser.headlessShell['arm64' if ansible_facts.architecture == 'aarch64' else 'amd64'] }}"
+    browser_root: '/opt/personal-fleet/browser/{{ browser.headlessShell.version }}'
+- name: Create the package cache and the browser directory
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - /var/cache/personal-fleet
+    - '{{ browser_root }}'
+- name: Download the pinned headless shell
+  ansible.builtin.get_url:
+    url: 'https://cdn.playwright.dev/builds/cft/{{ browser.headlessShell.version }}/{{ browser_build.archive }}.zip'
+    dest: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
+    checksum: 'sha256:{{ browser_build.sha256 }}'
+    mode: '0644'
+    timeout: 300
+- name: Unpack the verified headless shell
+  ansible.builtin.unarchive:
+    src: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
+    dest: '{{ browser_root }}'
+    remote_src: true
+    creates: '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
+- name: Launch headless as the developer, load a blank page and exit (BROWSER_LAUNCH_FAILED)
+  ansible.builtin.command:
+    argv:
+      - '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
+      - --headless
+      - --no-sandbox
+      - --disable-gpu
+      - --dump-dom
+      - about:blank
+  become_user: '{{ development_user }}'
+  timeout: '{{ browser.launchTimeoutSeconds }}'
+  changed_when: false

package/ansible/ubuntu/tasks/docker.yml

@@ -29,3 +29,55 @@
     - [docker, compose, version]
     - [docker, buildx, version]
   changed_when: false
+# The engine answering is not proof that an agent can use it: run a pinned
+# image and build a one-line image as the developer, in a fresh session that
+# reads the docker group from scratch. The image is cached here so the same
+# test works offline in worker:docker-run.
+- name: Look for the cached smoke-test image
+  ansible.builtin.command:
+    argv:
+      [
+        docker,
+        image,
+        inspect,
+        --format,
+        '{{ "{{.Id}}" }}',
+        '{{ dockerSmokeTest.image }}',
+      ]
+  become_user: '{{ development_user }}'
+  check_mode: false
+  changed_when: false
+  failed_when: false
+  register: docker_smoke_image
+- name: Cache the pinned smoke-test image
+  ansible.builtin.command:
+    argv: [docker, pull, --quiet, '{{ dockerSmokeTest.image }}']
+  become_user: '{{ development_user }}'
+  when: docker_smoke_image.rc | default(0) != 0
+  changed_when: true
+- name: Run a container as the developer, offline (DOCKER_RUN_FAILED)
+  ansible.builtin.command:
+    argv:
+      [
+        docker,
+        run,
+        --rm,
+        --network,
+        none,
+        --pull,
+        never,
+        '{{ dockerSmokeTest.image }}',
+        'true',
+      ]
+  become_user: '{{ development_user }}'
+  timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
+  changed_when: false
+- name: Build a one-line image as the developer, offline (DOCKER_BUILD_FAILED)
+  ansible.builtin.command:
+    argv: [docker, build, --quiet, --pull=false, --network, none, '-']
+    stdin: |
+      FROM {{ dockerSmokeTest.image }}
+      RUN true
+  become_user: '{{ development_user }}'
+  timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
+  changed_when: false

package/ansible/ubuntu/tasks/recovery.yml

@@ -1,4 +1,25 @@
 ---
+# The unit runs a copy installed outside every revision: a synced revision is
+# pruned two syncs later, and the unit must not name a file that is gone.
+- name: Create the directory of the installed recovery scripts
+  become: true
+  ansible.builtin.file:
+    path: /usr/local/lib/personal-fleet
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+- name: Install the T3 recovery scripts outside the revision
+  become: true
+  ansible.builtin.copy:
+    src: '{{ playbook_dir }}/{{ item }}'
+    dest: '/usr/local/lib/personal-fleet/{{ item }}'
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - t3-health.sh
+    - t3-idle.py
 - name: Declare bounded recovery of the worker T3 desktop app
   ansible.builtin.copy:
     dest: '{{ development_home }}/.config/systemd/user/personal-worker-recovery.service'
@@ -10,7 +31,7 @@
       Description=Keep the worker T3 desktop app running and responsive
       [Service]
       Type=oneshot
-      ExecStart=/bin/sh {{ development_source }}/ansible/ubuntu/t3-health.sh
+      ExecStart=/bin/sh /usr/local/lib/personal-fleet/t3-health.sh
       TimeoutStartSec=90
 - name: Declare persistent service recovery checks
   ansible.builtin.copy:
@@ -37,3 +58,41 @@
     daemon_reload: true
     enabled: true
     state: started
+- name: Declare the work-in-progress guard of agent worktrees
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.service'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      # Managed by personal fleet-sync: ansible/ubuntu/wip-guard.py
+      [Unit]
+      Description=Push work in progress of agent worktrees to recovery refs
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/bin/python3 {{ development_source }}/ansible/ubuntu/wip-guard.py --root {{ development_home }}/.t3/worktrees --node %l
+      TimeoutStartSec=600
+- name: Declare how often work in progress is saved
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.timer'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      [Unit]
+      Description=Save work in progress of agent worktrees every 15 minutes
+      [Timer]
+      OnBootSec=5min
+      OnUnitActiveSec=15min
+      Persistent=true
+      [Install]
+      WantedBy=timers.target
+- name: Enable the work-in-progress guard
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: personal-worker-wip-guard.timer
+    daemon_reload: true
+    enabled: true
+    state: started

package/ansible/ubuntu/tasks/resources.yml

@@ -224,7 +224,7 @@
       StartLimitIntervalSec=0
       [Service]
       Type=simple
-      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
+      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --inode-floor-percent {{ resourceSafety.inodeFailFreePercent }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
       Restart=always
       RestartSec=5
       OOMScoreAdjust=-1000
@@ -276,7 +276,7 @@
         After=docker.service
         [Service]
         Type=oneshot
-        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
+        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-inode-percent {{ resourceSafety.inodeWarnFreePercent }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
 
         TimeoutStartSec=2h
         Nice=19

package/ansible/ubuntu/tasks/services.yml

@@ -29,3 +29,16 @@
   ansible.builtin.systemd_service:
     daemon_reload: true
   when: recovery_dropins.changed
+# A skewed clock breaks TLS, tokens, apt signatures and Vault logins with
+# errors that never mention time. Ubuntu ships one time service (chrony on
+# newer releases, systemd-timesyncd before); whichever is installed stays
+# enabled. A container takes its clock from its host. Measured as worker:clock.
+- name: Inspect the time services this worker has
+  ansible.builtin.service_facts:
+- name: Keep the clock synchronised with the time service Ubuntu ships
+  ansible.builtin.systemd_service:
+    name: '{{ item }}'
+    enabled: true
+    state: started
+  loop: "{{ (['chrony.service', 'systemd-timesyncd.service'] | select('in', ansible_facts.services) | list)[:1] }}"
+  when: ansible_facts.virtualization_type not in ['docker', 'container', 'podman', 'lxc', 'systemd-nspawn']

package/ansible/ubuntu/tasks/t3-restart.yml

@@ -0,0 +1,37 @@
+---
+# Restart a running T3 only while it runs no agent turn: a restart ends every
+# running turn. A busy T3 keeps its turns; the restart is recorded as pending
+# (with the time it was first deferred) and personal-worker-recovery applies it
+# at the first idle check (t3-health.sh). t3_restart_busy restarts it anyway.
+- name: Ask T3 whether an agent turn is running
+  become: true
+  become_user: '{{ development_user }}'
+  ansible.builtin.command:
+    argv:
+      - python3
+      - -I
+      - -B
+      - '{{ playbook_dir }}/t3-idle.py'
+      - --state
+      - '{{ development_home }}/.t3/userdata/statev2.sqlite'
+  register: t3_idle
+  changed_when: false
+  failed_when: false
+- name: Restart T3 while it runs no agent turn
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: t3code-desktop.service
+    state: restarted
+  when: t3_idle is not skipped and (t3_idle.rc == 0 or t3_restart_busy | default(false) | bool)
+- name: Record the restart as pending while agent turns run
+  ansible.builtin.copy:
+    dest: '/run/user/{{ development_uid }}/personal-t3-restart-pending'
+    content: "{{ now(utc=true).timestamp() | int }}\n"
+    force: false
+    owner: '{{ development_user }}'
+    mode: '0600'
+  when: t3_idle is not skipped and t3_idle.rc != 0 and not (t3_restart_busy | default(false) | bool)