@crvouga/dev-fleet 0.2.0 → 0.3.0

package/README.md

@@ -189,6 +189,14 @@ warns that the manager holds the only one. [docs/state.md](docs/state.md)
 explains how replication stays correct, how to move an existing fleet to a
 project, and which state is not in the bucket yet.
 
+## Provider accounts
+
+Provider accounts live only on the CLI proxy. `npx devfleet fleet-accounts`
+lists them with their state; `--add`, `--relogin` and `--remove` change one
+and propagate the result to every node, and `fleet-sync --only accounts`
+propagates them without a full sync.
+[docs/accounts.md](docs/accounts.md) explains how.
+
 ## Add or remove a machine
 
 1. Set up SSH from the manager to the new Ubuntu box (a non-root user with
@@ -233,6 +241,26 @@ package carries: `fleet-state --example` prints a complete example that points
 | `workers`  | The Ubuntu workers                                                  |
 | `services` | The proxy's public origin and the monitoring dashboard and accounts |
 | `worker`   | Policy every worker must meet                                       |
+| `ides`     | The IDEs every machine runs; `["t3"]` when absent                   |
+| `t3`       | T3 settings every machine holds (`profile`) and `requiredModels`    |
+
+### Supported IDEs
+
+Every supported IDE is reached through one interface (`src/ide/ide.ts`): its
+settings on each machine, the manager's own copy (pinned version and policy)
+and what the manager measures of each worker. Enabling one is `ides` in
+`fleet.json`.
+
+| IDE     | `ides` | Manager                                               | Workers                                    |
+| ------- | ------ | ----------------------------------------------------- | ------------------------------------------ |
+| T3 Code | `t3`   | Pinned CLI, policy with updates off, version measured | Pinned desktop app, locked policy, pairing |
+
+Under `t3.profile`, `fleet.json` names top-level T3 settings every machine
+holds the same, such as tool servers. A setting not named there stays the
+machine's own, and one edited on a machine since `fleet-sync` last wrote it
+is reported as `T3_PROFILE_CONFLICT` by the check `t3:profile`, never
+overwritten. The check `t3:models` reports a model T3 offers that the proxy
+does not serve, and a model in `t3.requiredModels` that is missing.
 
 Secrets are asked for by purpose (`proxy-client-key`,
 `monitoring-account-password`, `monitoring-tunnel-token`,

package/ansible/macos/manager.yml

@@ -28,3 +28,6 @@
         observed: "{{ ac_section | regex_findall('^\\s*' ~ item.key ~ '\\s+(\\S+)', multiline=True) | first | default('') }}"
       when: observed != item.value
       changed_when: true
+
+- name: Stage the fleet's T3 policy
+  ansible.builtin.import_playbook: t3-policy.yml

package/ansible/macos/t3-policy.yml

@@ -0,0 +1,49 @@
+---
+# T3 Code on the manager reads the same administrator policy as on every
+# worker (ansible/ubuntu/t3/policy.json): updates off, so the manager's T3
+# changes only when the fleet's pin does. Provider instances are reconciled
+# by fleet-sync on every run and stay unlocked here: locking them needs an
+# administrator for each change of the account roster. fleet-sync measures
+# the result as t3:updates. Needs an administrator: ansible-playbook -K manager.yml
+- name: Stage the fleet's T3 policy on the manager
+  hosts: localhost
+  connection: local
+  become: true
+  gather_facts: false
+  vars:
+    t3_policy_directory: /Library/Application Support/T3Code
+  tasks:
+    - name: Inspect existing administrator policy
+      ansible.builtin.stat:
+        path: '{{ t3_policy_directory }}/policy.json'
+      register: policy_file
+    - name: Inspect Fleet ownership marker
+      ansible.builtin.stat:
+        path: '{{ t3_policy_directory }}/.personal-fleet-policy'
+      register: policy_owner
+    - name: Refuse to overwrite an unrelated administrator policy
+      ansible.builtin.assert:
+        that: not policy_file.stat.exists or policy_owner.stat.exists
+        fail_msg: T3_POLICY_OWNER_CONFLICT; review and merge the existing administrator policy in Git before adoption.
+    - name: Create administrator-owned policy directory
+      ansible.builtin.file:
+        path: '{{ t3_policy_directory }}'
+        state: directory
+        owner: root
+        group: wheel
+        mode: '0755'
+    - name: Record policy ownership before installation
+      ansible.builtin.copy:
+        dest: '{{ t3_policy_directory }}/.personal-fleet-policy'
+        content: "ansible/ubuntu/t3/policy.json\n"
+        owner: root
+        group: wheel
+        mode: '0644'
+    - name: Install the fleet's policy with backup
+      ansible.builtin.copy:
+        content: "{{ lookup('ansible.builtin.file', '../ubuntu/t3/policy.json') | from_json | to_nice_json }}"
+        dest: '{{ t3_policy_directory }}/policy.json'
+        owner: root
+        group: wheel
+        mode: '0644'
+        backup: true

package/ansible/ubuntu/policy.json

@@ -26,6 +26,7 @@
   "aptLockTimeoutSeconds": 120,
   "minimumFreeDiskMiB": 4096,
   "minimumFreeInodes": 10000,
+  "t3RestartMaxDeferralHours": 24,
   "alwaysOnServices": ["ssh", "tailscaled", "docker", "beszel-agent"],
   "resourceSafety": {
     "swapFileMiB": 8192,

package/ansible/ubuntu/t3-desktop.yml

@@ -53,6 +53,16 @@
       register: t3_desktop_installed
       changed_when: false
       failed_when: false
+    - name: Check whether the desktop app is running before an update
+      become: true
+      become_user: '{{ development_user }}'
+      environment:
+        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+      ansible.builtin.command:
+        argv: [systemctl, --user, is-active, --quiet, t3code-desktop.service]
+      register: t3_desktop_was_active
+      changed_when: false
+      failed_when: false
     - name: Unpack the pinned app when its version differs
       when: t3_desktop_installed.stdout | default('') != t3_desktop_name
       block:
@@ -94,6 +104,7 @@
               - -T
               - '/opt/.t3code-staging{{ t3_desktop_dir }}'
               - '{{ t3_desktop_dir }}'
+          register: t3_desktop_updated
           changed_when: true
         - name: Add the launcher entry and icon
           ansible.builtin.copy:
@@ -218,3 +229,11 @@
         name: t3code-desktop.service
         state: started
       failed_when: false
+    # A running app keeps the files of its release: restart it into the one
+    # just installed, only while it runs no agent turn. Otherwise
+    # personal-worker-recovery restarts it at the first idle check, and
+    # t3:running-version reports T3_RESTART_PENDING until then.
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
+      when:
+        - t3_desktop_was_active.rc == 0
+        - t3_desktop_updated is changed

package/ansible/ubuntu/t3-health.sh

@@ -1,33 +1,104 @@
 #!/bin/sh
-# Keep the worker T3 desktop app (UI and server) up and responsive. Run every minute by
-# personal-worker-recovery.timer (managed by fleet-sync, tasks/recovery.yml).
-# Stopped or failed: start it. Running but its local API has not answered for
-# three checks in a row: restart it, since a hung server serves no turns.
+# Keep the worker T3 desktop app (UI and server) up, responsive and on the
+# installed release. Run every minute by personal-worker-recovery.timer
+# (managed by fleet-sync, tasks/recovery.yml), from the stable copy the role
+# installs beside t3-idle.py.
+# - The former headless server is stopped and disabled: one T3 per worker.
+# - No graphical session: T3_NO_GRAPHICAL_SESSION; after three checks in a row
+#   the display manager is restarted so auto-login starts one, three times at
+#   most until a session is seen again.
+# - Stopped or failed: reset its start limit and start it.
+# - Running but its local API has not answered for three checks in a row:
+#   restart it, since a hung server serves no turns.
+# - Running another release than the installed one, or a policy restart that
+#   was deferred: restart it once it runs no agent turn (T3_RESTART_PENDING).
 set -u
 url=http://127.0.0.1:3773/.well-known/t3/environment
-state="${XDG_RUNTIME_DIR:?}/personal-t3-health-failures"
+unit=t3code-desktop.service
+runtime="${XDG_RUNTIME_DIR:?}"
+failures="$runtime/personal-t3-health-failures"
+no_session="$runtime/personal-t3-no-session"
+display_restarts="$runtime/personal-t3-display-restarts"
+pending="$runtime/personal-t3-restart-pending"
+restarted="$runtime/personal-t3-restarted-for"
+installed="${T3_DESKTOP_DIR:-/opt/T3 Code (Nightly)}/.personal-fleet-version"
+here=$(dirname "$0")
 limit=3
 
-# The former headless server still runs until its approved migration; never
-# start a second T3 next to it.
-if systemctl --user is-active --quiet t3code.service; then
-  echo "Headless T3 still runs; desktop migration pending."
-  exit 0
+count() {
+  n=$(($(cat "$1" 2>/dev/null || echo 0) + 1))
+  echo "$n" > "$1"
+  echo "$n"
+}
+
+if systemctl --user is-active --quiet t3code.service ||
+  systemctl --user is-enabled --quiet t3code.service; then
+  echo "Retiring the former headless T3 server."
+  if ! systemctl --user disable --now t3code.service; then
+    echo "T3_LEGACY_SERVER_RUNNING"
+    exit 1
+  fi
+fi
+
+if ! systemctl --user is-active --quiet graphical-session.target; then
+  rm -f "$failures"
+  seen=$(count "$no_session")
+  echo "T3_NO_GRAPHICAL_SESSION ($seen/$limit)."
+  if [ "$seen" -ge "$limit" ]; then
+    rm -f "$no_session"
+    restarts=$(count "$display_restarts")
+    if [ "$restarts" -le "$limit" ]; then
+      echo "Restarting the display manager so auto-login starts a session ($restarts/$limit)."
+      sudo -n systemctl restart display-manager.service
+    else
+      echo "The display manager was restarted $limit times; a person must log in."
+    fi
+  fi
+  exit 1
 fi
-if ! systemctl --user is-active --quiet t3code-desktop.service; then
-  rm -f "$state"
+rm -f "$no_session" "$display_restarts"
+
+if ! systemctl --user is-active --quiet "$unit"; then
+  rm -f "$failures"
   echo "T3 is not running; starting it."
-  exec systemctl --user start t3code-desktop.service
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user start "$unit"
 fi
-if curl --fail --silent --max-time 10 --output /dev/null "$url"; then
-  rm -f "$state"
+if ! environment=$(curl --fail --silent --max-time 10 "$url"); then
+  failed=$(count "$failures")
+  echo "T3 local API did not answer ($failed/$limit)."
+  if [ "$failed" -ge "$limit" ]; then
+    rm -f "$failures"
+    echo "T3 is unresponsive; restarting it."
+    systemctl --user reset-failed "$unit" 2>/dev/null
+    exec systemctl --user restart "$unit"
+  fi
   exit 0
 fi
-failures=$(($(cat "$state" 2>/dev/null || echo 0) + 1))
-echo "$failures" > "$state"
-echo "T3 local API did not answer ($failures/$limit)."
-if [ "$failures" -ge "$limit" ]; then
-  rm -f "$state"
-  echo "T3 is unresponsive; restarting it."
-  exec systemctl --user restart t3code-desktop.service
+rm -f "$failures"
+
+running=$(printf '%s' "$environment" | sed -n 's/.*"serverVersion" *: *"\([^"]*\)".*/\1/p')
+release=$(sed -n 's/^T3-Code-\(.*\)-[a-z0-9]*\.deb$/\1/p' "$installed" 2>/dev/null)
+reason=
+[ -f "$pending" ] && reason="the staged policy"
+if [ -n "$release" ] && [ "$running" != "$release" ]; then
+  # One restart per release: a server that still reports another version
+  # after it is never restarted in a loop.
+  if [ "$(cat "$restarted" 2>/dev/null)" = "$release" ]; then
+    echo "T3_RUNNING_VERSION_MISMATCH: restarted for $release, still reports ${running:-nothing}."
+    exit 1
+  fi
+  reason="release $release"
+else
+  rm -f "$restarted"
+fi
+[ -z "$reason" ] && exit 0
+if python3 -I -B "$here/t3-idle.py" --state "${T3CODE_HOME:-$HOME/.t3}/userdata/statev2.sqlite"; then
+  rm -f "$pending"
+  [ "$reason" = "release $release" ] && echo "$release" > "$restarted"
+  echo "T3 runs no agent turn; restarting it to load $reason."
+  systemctl --user reset-failed "$unit" 2>/dev/null
+  exec systemctl --user restart "$unit"
 fi
+echo "T3_RESTART_PENDING: agent turns are running; $reason loads at the first idle check."
+exit 0

package/ansible/ubuntu/t3-idle.py

@@ -0,0 +1,42 @@
+"""Say whether the worker's T3 runs an agent turn: exit 0 when it runs none,
+1 when it runs one and 2 when it cannot tell.
+
+T3 offers no command that lists the running turns of every project, so this
+reads the runs T3 records in its own state database, read-only. Anything it
+cannot read counts as busy: a restart never ends a turn on a guess.
+
+Usage: t3-idle.py --state ~/.t3/userdata/statev2.sqlite
+"""
+import argparse
+import sqlite3
+import sys
+
+IDLE, BUSY, UNKNOWN = 0, 1, 2
+
+
+def running_turns(state):
+    connection = sqlite3.connect(f'file:{state}?mode=ro', uri=True, timeout=5)
+    try:
+        (count,) = connection.execute(
+            "SELECT count(*) FROM orchestration_v2_projection_runs WHERE status = 'running'"
+        ).fetchone()
+        return count
+    finally:
+        connection.close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--state', required=True)
+    args = parser.parse_args()
+    try:
+        count = running_turns(args.state)
+    except sqlite3.Error:
+        print('T3_IDLE_UNKNOWN')
+        return UNKNOWN
+    print(f'{count} running turn(s)')
+    return BUSY if count else IDLE
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/ansible/ubuntu/t3-policy.yml

@@ -41,8 +41,9 @@
         mode: '0644'
         backup: true
     # T3 reads its policy once, at startup. A running app that has not loaded
-    # the staged policy is restarted (this ends its running turns) so every
-    # node offers the same providers after every sync.
+    # the staged policy is restarted so every node offers the same providers,
+    # but only while it runs no agent turn (tasks/t3-restart.yml): a busy app
+    # keeps its turns and loads the policy at the first idle check.
     - name: Check whether the desktop app is running
       become: true
       become_user: '{{ development_user }}'
@@ -62,15 +63,7 @@
       changed_when: false
       failed_when: false
       when: t3_desktop_active.rc == 0
-    - name: Restart T3 so it loads the staged policy
-      become: true
-      become_user: '{{ development_user }}'
-      environment:
-        XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
-      ansible.builtin.systemd_service:
-        scope: user
-        name: t3code-desktop.service
-        state: restarted
+    - ansible.builtin.import_tasks: tasks/t3-restart.yml
       when:
         - t3_desktop_active.rc == 0
         - t3_policy_loaded.stdout is not search('loaded this policy\.\s*$')

package/ansible/ubuntu/tasks/recovery.yml

@@ -1,4 +1,25 @@
 ---
+# The unit runs a copy installed outside every revision: a synced revision is
+# pruned two syncs later, and the unit must not name a file that is gone.
+- name: Create the directory of the installed recovery scripts
+  become: true
+  ansible.builtin.file:
+    path: /usr/local/lib/personal-fleet
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+- name: Install the T3 recovery scripts outside the revision
+  become: true
+  ansible.builtin.copy:
+    src: '{{ playbook_dir }}/{{ item }}'
+    dest: '/usr/local/lib/personal-fleet/{{ item }}'
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - t3-health.sh
+    - t3-idle.py
 - name: Declare bounded recovery of the worker T3 desktop app
   ansible.builtin.copy:
     dest: '{{ development_home }}/.config/systemd/user/personal-worker-recovery.service'
@@ -10,7 +31,7 @@
       Description=Keep the worker T3 desktop app running and responsive
       [Service]
       Type=oneshot
-      ExecStart=/bin/sh {{ development_source }}/ansible/ubuntu/t3-health.sh
+      ExecStart=/bin/sh /usr/local/lib/personal-fleet/t3-health.sh
       TimeoutStartSec=90
 - name: Declare persistent service recovery checks
   ansible.builtin.copy:
@@ -37,3 +58,41 @@
     daemon_reload: true
     enabled: true
     state: started
+- name: Declare the work-in-progress guard of agent worktrees
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.service'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      # Managed by personal fleet-sync: ansible/ubuntu/wip-guard.py
+      [Unit]
+      Description=Push work in progress of agent worktrees to recovery refs
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/bin/python3 {{ development_source }}/ansible/ubuntu/wip-guard.py --root {{ development_home }}/.t3/worktrees --node %l
+      TimeoutStartSec=600
+- name: Declare how often work in progress is saved
+  ansible.builtin.copy:
+    dest: '{{ development_home }}/.config/systemd/user/personal-worker-wip-guard.timer'
+    owner: '{{ development_user }}'
+    mode: '0600'
+    content: |
+      [Unit]
+      Description=Save work in progress of agent worktrees every 15 minutes
+      [Timer]
+      OnBootSec=5min
+      OnUnitActiveSec=15min
+      Persistent=true
+      [Install]
+      WantedBy=timers.target
+- name: Enable the work-in-progress guard
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: personal-worker-wip-guard.timer
+    daemon_reload: true
+    enabled: true
+    state: started

package/ansible/ubuntu/tasks/t3-restart.yml

@@ -0,0 +1,37 @@
+---
+# Restart a running T3 only while it runs no agent turn: a restart ends every
+# running turn. A busy T3 keeps its turns; the restart is recorded as pending
+# (with the time it was first deferred) and personal-worker-recovery applies it
+# at the first idle check (t3-health.sh). t3_restart_busy restarts it anyway.
+- name: Ask T3 whether an agent turn is running
+  become: true
+  become_user: '{{ development_user }}'
+  ansible.builtin.command:
+    argv:
+      - python3
+      - -I
+      - -B
+      - '{{ playbook_dir }}/t3-idle.py'
+      - --state
+      - '{{ development_home }}/.t3/userdata/statev2.sqlite'
+  register: t3_idle
+  changed_when: false
+  failed_when: false
+- name: Restart T3 while it runs no agent turn
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    name: t3code-desktop.service
+    state: restarted
+  when: t3_idle is not skipped and (t3_idle.rc == 0 or t3_restart_busy | default(false) | bool)
+- name: Record the restart as pending while agent turns run
+  ansible.builtin.copy:
+    dest: '/run/user/{{ development_uid }}/personal-t3-restart-pending'
+    content: "{{ now(utc=true).timestamp() | int }}\n"
+    force: false
+    owner: '{{ development_user }}'
+    mode: '0600'
+  when: t3_idle is not skipped and t3_idle.rc != 0 and not (t3_restart_busy | default(false) | bool)

package/ansible/ubuntu/wip-guard.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+"""Push work in progress of every agent worktree to a recovery ref.
+
+Managed by personal fleet-sync (tasks/recovery.yml) as
+personal-worker-wip-guard.service, started by personal-worker-wip-guard.timer.
+
+Workers are disposable: a worker lost mid-task must lose at most one interval
+of uncommitted work. For each Git worktree under --root, the files as they are
+now (tracked changes and untracked files that are not ignored) are written as
+one commit on top of HEAD, built in a private index so the worktree, its index
+and its branches are never touched, and pushed to
+
+    refs/recovery/<node>/<branch>
+
+of the worktree's own remote. A clean worktree whose HEAD is already on a
+remote branch has nothing to recover and pushes nothing. Recover with
+`git fetch origin 'refs/recovery/*:refs/recovery/*'`.
+
+Every decision is one JSON line on stdout.
+
+Usage: wip-guard.py --root DIR [--root DIR ...] [--node ID] [--remote NAME]
+                    [--max-depth N] [--dry-run]
+"""
+import argparse
+import json
+import os
+import re
+import socket
+import subprocess
+import tempfile
+from pathlib import Path
+
+GIT = ['git', '-c', 'safe.directory=*']
+AUTHOR = {
+    'GIT_AUTHOR_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_AUTHOR_EMAIL': 'wip-guard@example.invalid',
+    'GIT_COMMITTER_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_COMMITTER_EMAIL': 'wip-guard@example.invalid',
+}
+
+
+def log(**event):
+    print(json.dumps(event, sort_keys=True), flush=True)
+
+
+def git(path, *args, env=None):
+    return subprocess.run(
+        [*GIT, '-C', str(path), *args], capture_output=True, text=True,
+        env={**os.environ, **(env or {})}, timeout=120)
+
+
+def worktrees(root, depth):
+    if depth < 0 or not root.is_dir() or root.is_symlink():
+        return
+    if (root / '.git').exists():
+        yield root
+        return
+    for child in sorted(root.iterdir()):
+        if child.is_dir() and not child.is_symlink():
+            yield from worktrees(child, depth - 1)
+
+
+def ref_part(value):
+    part = re.sub(r'[^A-Za-z0-9._/-]+', '-', value).strip('./-')
+    part = re.sub(r'/+', '/', part).replace('..', '.')
+    return part or 'unnamed'
+
+
+def snapshot(path):
+    """The worktree's files as one commit on HEAD, or HEAD when clean."""
+    head = git(path, 'rev-parse', '--verify', 'HEAD').stdout.strip()
+    if not head:
+        return None, None
+    with tempfile.TemporaryDirectory() as private:
+        index = {'GIT_INDEX_FILE': os.path.join(private, 'index')}
+        if git(path, 'read-tree', 'HEAD', env=index).returncode != 0:
+            return head, None
+        if git(path, 'add', '--all', '--', '.', env=index).returncode != 0:
+            return head, None
+        tree = git(path, 'write-tree', env=index).stdout.strip()
+    if tree == git(path, 'rev-parse', 'HEAD^{tree}').stdout.strip():
+        return head, head
+    commit = git(path, 'commit-tree', tree, '-p', head, '-m',
+                 'Work in progress saved by the dev-fleet guard', env=AUTHOR)
+    return head, commit.stdout.strip() or None
+
+
+def guard(path, node, remote, dry_run):
+    if git(path, 'remote', 'get-url', remote).returncode != 0:
+        return log(worktree=str(path), action='skip', reason='no-remote')
+    head, commit = snapshot(path)
+    if not commit:
+        return log(worktree=str(path), action='skip', reason='no-commit')
+    if commit == head and git(path, 'branch', '--remotes', '--contains', head).stdout.strip():
+        return log(worktree=str(path), action='skip', reason='clean')
+    branch = git(path, 'symbolic-ref', '--quiet', '--short', 'HEAD').stdout.strip()
+    ref = 'refs/recovery/%s/%s' % (ref_part(node), ref_part(branch or 'detached-' + head[:12]))
+    if dry_run:
+        return log(worktree=str(path), action='would-push', ref=ref)
+    pushed = git(path, 'push', '--force', '--no-verify', '--quiet', remote, '%s:%s' % (commit, ref))
+    log(worktree=str(path), action='pushed' if pushed.returncode == 0 else 'failed', ref=ref)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', action='append', required=True)
+    parser.add_argument('--node', default=socket.gethostname().split('.')[0])
+    parser.add_argument('--remote', default='origin')
+    parser.add_argument('--max-depth', type=int, default=3)
+    parser.add_argument('--dry-run', action='store_true')
+    options = parser.parse_args()
+    for root in options.root:
+        for path in worktrees(Path(root).expanduser(), options.max_depth):
+            try:
+                guard(path, options.node, options.remote, options.dry_run)
+            except (OSError, subprocess.SubprocessError):
+                log(worktree=str(path), action='failed', reason='git')
+
+
+if __name__ == '__main__':
+    main()

package/bootstrap/native-t3.ts

@@ -19,6 +19,7 @@ interface Configuration {
     t3_version: string;
     t3_linux_x64_sha256: string;
     t3_linux_arm64_sha256: string;
+    t3_darwin_arm64_sha256?: string;
     t3_desktop_amd64_sha256: string;
     t3_desktop_arm64_sha256: string;
   };
@@ -52,20 +53,27 @@ export function managedT3Desktop() {
   ) as Configuration;
   return desktopT3Artifact(config, process.arch);
 }
-export function nativeT3Artifact(config: Configuration, architecture: string) {
-  if (!['x64', 'arm64'].includes(architecture))
-    throw new Error('Unsupported T3 architecture.');
+/** The pinned server archive: Ubuntu workers on either architecture, and
+ * the manager (macOS on Apple silicon, the only macOS build upstream ships).
+ */
+export function nativeT3Artifact(
+  config: Configuration,
+  architecture: string,
+  platform: string = 'linux'
+) {
+  const sha256 = (<Record<string, string | undefined>>{
+    'linux-x64': config.vars.t3_linux_x64_sha256,
+    'linux-arm64': config.vars.t3_linux_arm64_sha256,
+    'darwin-arm64': config.vars.t3_darwin_arm64_sha256,
+  })[`${platform}-${architecture}`];
+  if (sha256 === undefined) throw new Error('Unsupported T3 architecture.');
   const version = config.vars.t3_version;
-  const sha256 =
-    architecture === 'x64'
-      ? config.vars.t3_linux_x64_sha256
-      : config.vars.t3_linux_arm64_sha256;
   if (
     !/^\d+\.\d+\.\d+(?:-nightly\.\d{8}\.\d+)?$/.test(version) ||
     !/^[a-f0-9]{64}$/.test(sha256)
   )
     throw new Error('Invalid T3 pin.');
-  const name = `t3-${version}-linux-${architecture}`;
+  const name = `t3-${version}-${platform}-${architecture}`;
   return {
     version,
     sha256,
@@ -77,18 +85,21 @@ export function managedT3(): { path: string; version: string } {
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Configuration;
-  const artifact = nativeT3Artifact(config, process.arch);
+  const artifact = nativeT3Artifact(config, process.arch, process.platform);
   return {
     path: join(homedir(), '.local/share/personal/tools', artifact.name, 't3'),
     version: artifact.version,
   };
 }
+/** Installs the pinned, checksum-verified server archive on a worker and on
+ * the manager, so both run the T3 version the fleet pins.
+ */
 export async function installNativeT3(): Promise<void> {
-  if (process.platform !== 'linux') return;
+  if (!['linux', 'darwin'].includes(process.platform)) return;
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Configuration;
-  const artifact = nativeT3Artifact(config, process.arch);
+  const artifact = nativeT3Artifact(config, process.arch, process.platform);
   const root = join(homedir(), '.local/share/personal/tools');
   const archive = join(root, `${artifact.name}.tar.gz`);
   mkdirSync(root, { recursive: true, mode: 0o700 });