@crvouga/dev-fleet 0.3.0 → 0.4.0

package/ansible/ubuntu/tasks/browser.yml

@@ -0,0 +1,56 @@
+---
+# Headless Chromium for browser tests (Playwright, Storybook, end-to-end
+# suites): the system libraries it loads, at the snapshot's exact versions,
+# and one pinned Chrome Headless Shell build. Applied to workers that select
+# the web tool profile; the launch below backs worker:browser.
+- name: Install the exact libraries headless Chromium loads
+  ansible.builtin.apt:
+    name: '{{ item.key }}={{ item.value }}'
+    state: present
+    lock_timeout: '{{ aptLockTimeoutSeconds }}'
+    allow_downgrade: false
+    allow_unauthenticated: false
+    fail_on_autoremove: true
+    policy_rc_d: 101
+  environment:
+    APT_CONFIG: /etc/personal-fleet/apt.conf
+  loop: '{{ browser.packages | dict2items }}'
+- name: Name the headless shell build of this architecture
+  ansible.builtin.set_fact:
+    browser_build: "{{ browser.headlessShell['arm64' if ansible_facts.architecture == 'aarch64' else 'amd64'] }}"
+    browser_root: '/opt/personal-fleet/browser/{{ browser.headlessShell.version }}'
+- name: Create the package cache and the browser directory
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - /var/cache/personal-fleet
+    - '{{ browser_root }}'
+- name: Download the pinned headless shell
+  ansible.builtin.get_url:
+    url: 'https://cdn.playwright.dev/builds/cft/{{ browser.headlessShell.version }}/{{ browser_build.archive }}.zip'
+    dest: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
+    checksum: 'sha256:{{ browser_build.sha256 }}'
+    mode: '0644'
+    timeout: 300
+- name: Unpack the verified headless shell
+  ansible.builtin.unarchive:
+    src: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
+    dest: '{{ browser_root }}'
+    remote_src: true
+    creates: '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
+- name: Launch headless as the developer, load a blank page and exit (BROWSER_LAUNCH_FAILED)
+  ansible.builtin.command:
+    argv:
+      - '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
+      - --headless
+      - --no-sandbox
+      - --disable-gpu
+      - --dump-dom
+      - about:blank
+  become_user: '{{ development_user }}'
+  timeout: '{{ browser.launchTimeoutSeconds }}'
+  changed_when: false

package/ansible/ubuntu/tasks/docker.yml

@@ -29,3 +29,55 @@
     - [docker, compose, version]
     - [docker, buildx, version]
   changed_when: false
+# The engine answering is not proof that an agent can use it: run a pinned
+# image and build a one-line image as the developer, in a fresh session that
+# reads the docker group from scratch. The image is cached here so the same
+# test works offline in worker:docker-run.
+- name: Look for the cached smoke-test image
+  ansible.builtin.command:
+    argv:
+      [
+        docker,
+        image,
+        inspect,
+        --format,
+        '{{ "{{.Id}}" }}',
+        '{{ dockerSmokeTest.image }}',
+      ]
+  become_user: '{{ development_user }}'
+  check_mode: false
+  changed_when: false
+  failed_when: false
+  register: docker_smoke_image
+- name: Cache the pinned smoke-test image
+  ansible.builtin.command:
+    argv: [docker, pull, --quiet, '{{ dockerSmokeTest.image }}']
+  become_user: '{{ development_user }}'
+  when: docker_smoke_image.rc | default(0) != 0
+  changed_when: true
+- name: Run a container as the developer, offline (DOCKER_RUN_FAILED)
+  ansible.builtin.command:
+    argv:
+      [
+        docker,
+        run,
+        --rm,
+        --network,
+        none,
+        --pull,
+        never,
+        '{{ dockerSmokeTest.image }}',
+        'true',
+      ]
+  become_user: '{{ development_user }}'
+  timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
+  changed_when: false
+- name: Build a one-line image as the developer, offline (DOCKER_BUILD_FAILED)
+  ansible.builtin.command:
+    argv: [docker, build, --quiet, --pull=false, --network, none, '-']
+    stdin: |
+      FROM {{ dockerSmokeTest.image }}
+      RUN true
+  become_user: '{{ development_user }}'
+  timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
+  changed_when: false

package/ansible/ubuntu/tasks/resources.yml

@@ -224,7 +224,7 @@
       StartLimitIntervalSec=0
       [Service]
       Type=simple
-      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
+      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --inode-floor-percent {{ resourceSafety.inodeFailFreePercent }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
       Restart=always
       RestartSec=5
       OOMScoreAdjust=-1000
@@ -276,7 +276,7 @@
         After=docker.service
         [Service]
         Type=oneshot
-        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
+        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-inode-percent {{ resourceSafety.inodeWarnFreePercent }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
 
         TimeoutStartSec=2h
         Nice=19

package/ansible/ubuntu/tasks/services.yml

@@ -29,3 +29,16 @@
   ansible.builtin.systemd_service:
     daemon_reload: true
   when: recovery_dropins.changed
+# A skewed clock breaks TLS, tokens, apt signatures and Vault logins with
+# errors that never mention time. Ubuntu ships one time service (chrony on
+# newer releases, systemd-timesyncd before); whichever is installed stays
+# enabled. A container takes its clock from its host. Measured as worker:clock.
+- name: Inspect the time services this worker has
+  ansible.builtin.service_facts:
+- name: Keep the clock synchronised with the time service Ubuntu ships
+  ansible.builtin.systemd_service:
+    name: '{{ item }}'
+    enabled: true
+    state: started
+  loop: "{{ (['chrony.service', 'systemd-timesyncd.service'] | select('in', ansible_facts.services) | list)[:1] }}"
+  when: ansible_facts.virtualization_type not in ['docker', 'container', 'podman', 'lxc', 'systemd-nspawn']

package/bootstrap/doctor.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  NO_SELECTION,
+  parseSelectionArguments,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 /** Read-only toolchain checks; no logins, installations or changes to Docker. */
 import { spawnSync } from 'node:child_process';
 import { readFileSync } from 'node:fs';
@@ -15,7 +20,7 @@ import {
 type Tool = string | { version: string; lazy?: boolean; os?: string[] };
 type Config = {
   tools: Record<string, Tool>;
-  vars?: { worker_tools?: unknown };
+  vars?: Record<string, unknown>;
 };
 export type Check = {
   id: string;
@@ -42,6 +47,11 @@ const VERSION_ARGS: Record<string, string[]> = {
   kubectl: ['version', '--client=true', '--output=json'],
   helm: ['version', '--short'],
 };
+/** The version a tool prints, where it differs from its mise pin. */
+const PRINTED: Record<string, (pin: string) => string> = {
+  // temurin-25.0.5+8.0.LTS prints as 25.0.5.
+  java: (pin) => pin.replace(/^[a-z]+-/, '').replace(/\+.*$/, ''),
+};
 const run: Runner = (command) => {
   const result = spawnSync(command[0]!, command.slice(1), {
     encoding: 'utf8',
@@ -80,7 +90,8 @@ export function toolChecks(
     if (typeof tool !== 'string' && tool.lazy) continue;
     // A tool pinned only for other platforms has no artifact to verify here.
     if (unavailable(id, tool, platform)) continue;
-    const version = typeof tool === 'string' ? tool : tool.version;
+    const pin = typeof tool === 'string' ? tool : tool.version;
+    const version = PRINTED[id]?.(pin) ?? pin;
     const command = COMMANDS[id] ?? id;
     const result = execute([command, ...(VERSION_ARGS[id] ?? ['--version'])]);
     const exact = new RegExp(
@@ -195,7 +206,21 @@ export function globalConfigCheck(
   };
 }
 
-export function checkMachine(requireDocker = false, worker = false): Check[] {
+/** A worker installs every tool it selects, lazy or not, so each is checked. */
+function installed(tools: Record<string, Tool>): Record<string, Tool> {
+  return Object.fromEntries(
+    Object.entries(tools).map(([id, tool]) => [
+      id,
+      typeof tool === 'string' ? tool : { ...tool, lazy: false },
+    ])
+  );
+}
+
+export function checkMachine(
+  requireDocker = false,
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
+): Check[] {
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Config;
@@ -203,7 +228,7 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
   return [
     ...toolChecks(
       worker
-        ? workerTools(config.tools, config.vars?.worker_tools)
+        ? installed(workerToolset(config.tools, config.vars ?? {}, selection))
         : config.tools
     ),
     ...dockerChecks(requireDocker),
@@ -222,7 +247,8 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
 if (import.meta.main) {
   const checks = checkMachine(
     process.argv.includes('--docker'),
-    process.argv.includes('--worker')
+    process.argv.includes('--worker'),
+    parseSelectionArguments(process.argv)
   );
   const ok = checks.every((check) => check.status !== 'fail');
   if (process.argv.includes('--json'))

package/bootstrap/fleet-remote-bootstrap.sh

@@ -41,7 +41,7 @@ remedy() {
     BOOTSTRAP_LOCK_TOOL_REQUIRED) echo 'Install python3 so the operation lock can be taken, then rerun fleet-sync.';;
     SOURCE_CACHE_CHANGED|SOURCE_STATUS_FAILED|SOURCE_CHECKOUT_CONFLICT) echo 'The fleet-owned checkout of this revision holds changes fleet-sync did not make. Inspect and remove that revisions directory, then rerun fleet-sync.';;
     CONFIGURATION_INSTALL_FAILED) echo 'The fleet configuration could not be written under ~/.config/dev-fleet. Check that directory, then rerun fleet-sync.';;
-    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet doctor on the worker.';;
+    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet fleet-status --node for this worker.';;
     *) echo 'Inspect the remote operation lock and rerun the approved revision. No success receipt advanced.';;
   esac
 }
@@ -97,7 +97,18 @@ else
   command -v python3 >/dev/null 2>&1 || die BOOTSTRAP_LOCK_TOOL_REQUIRED
   python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || die FLEET_REMOTE_LOCKED
 fi
-printf '{"version":1,"pid":%s,"revision":"%s","node":"%s"}\n' "$$" "$revision" "$node" >"$root/remote-operation.json"
+# The operation record: a manager that lost its session reads it to follow
+# this operation to its end instead of starting another (see
+# src/remote-operation.ts). The result of the last operation is beside it.
+started=$(date +%s)
+result="$root/remote-operation-result.json"
+[ ! -L "$root/remote-operation.json" ] && [ ! -L "$result" ] || die FLEET_STATE_PATH_UNSAFE
+rm -f -- "$result"
+operation() {
+  printf '{"version":1,"pid":%s,"revision":"%s","node":"%s","started":%s,"phase":"%s"%s}\n' "$$" "$revision" "$node" "$started" "$1" "${2:-}" >"$root/remote-operation.json.new" &&
+    mv -f -- "$root/remote-operation.json.new" "$root/remote-operation.json"
+}
+operation lock
 # The fleet configuration is installed under the lock, so two overlapping
 # runs never write it at once. It holds no secret.
 phase='configuration-install'
@@ -109,6 +120,7 @@ if [ -n "$configuration" ]; then
   mv "$config_dir/fleet.json.new" "$config_dir/fleet.json" || die CONFIGURATION_INSTALL_FAILED
 fi
 phase=resources
+operation resources
 # A bootstrap floor protects source transfer and minimal prerequisites. The
 # Workstation baseline subsequently enforces its stricter declared capacity.
 free_kib=$(df -Pk "$root" | awk 'END {print $4}')
@@ -200,6 +212,7 @@ bundle=''
 clean=$(git -C "$deployment" status --porcelain) || die SOURCE_STATUS_FAILED
 [ -z "$clean" ] || die SOURCE_CHECKOUT_CONFLICT
 phase=toolchain
+operation toolchain
 cd "$deployment"
 export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
 status_file="$root/toolchain-status.json"
@@ -211,6 +224,11 @@ ready_file="$root/toolchain-ready"
 # are still checked (and reinstalled on drift) by node convergence.
 toolchain_key=$(git rev-parse HEAD:mise.toml HEAD:mise.lock HEAD:.mise HEAD:bun.lock HEAD:package.json HEAD:bootstrap 2>/dev/null | tr '\n' ' ')
 [ -n "$toolchain_key" ] || toolchain_key=$revision
+# The tool profiles of the fleet configuration choose what is installed, so a
+# changed configuration runs the setup again.
+if [ -f "$HOME/.config/dev-fleet/fleet.json" ]; then
+  toolchain_key="$toolchain_key$(cksum <"$HOME/.config/dev-fleet/fleet.json" | cut -d' ' -f1)"
+fi
 if [ "$(cat "$ready_file" 2>/dev/null)" = "$toolchain_key" ] && [ -n "$(find "$ready_file" -mmin -1440 2>/dev/null)" ]; then
   toolchain_warm=1
 else
@@ -219,7 +237,7 @@ else
 fi
 if command -v timeout >/dev/null 2>&1; then bounded='timeout 1800'; else bounded=''; fi
 # shellcheck disable=SC2086
-if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
+if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --node "$node" --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
   tool_phase=$(sed -n 's/.*"phase":"\([a-z0-9-]*\)".*/\1/p' "$status_file" 2>/dev/null) || tool_phase=''
   case "$tool_phase" in
     managed-config) die TOOLCHAIN_CONFIG_CONFLICT;;
@@ -254,5 +272,14 @@ for old in $(ls -1t "$root/revisions"); do
   rm -rf -- "${root:?}/revisions/$old" || true
 done
 phase=configuration
+operation configuration
 trap - EXIT INT TERM
-exec mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision"
+# Node convergence writes its evidence to the result file, not to the SSH
+# session: the session may be gone by the time it ends. The evidence is then
+# printed for the manager that is still connected, under the same lock.
+code=0
+mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision" >"$result.partial" 2>"$root/remote-operation.log" || code=$?
+mv -f -- "$result.partial" "$result"
+operation finished ",\"finished\":$(date +%s),\"exit\":$code"
+cat -- "$result" || true
+exit "$code"

package/bootstrap/setup.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  installedSelection,
+  NO_SELECTION,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 /** Used by both OS bootstraps, dev-fleet tools sync and dev-fleet update. No workspace dependencies. */
@@ -26,7 +31,8 @@ function run(args: string[]): void {
 
 export async function syncTools(
   observe: (phase: SetupPhase) => void = () => {},
-  worker = false
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
 ): Promise<void> {
   observe('managed-config');
   const dir = globalConfigDir();
@@ -34,9 +40,9 @@ export async function syncTools(
   observe('mise');
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
-  ) as { tools: Record<string, unknown>; vars: { worker_tools: unknown } };
+  ) as { tools: Record<string, unknown>; vars: Record<string, unknown> };
   const selected = worker
-    ? Object.keys(workerTools(config.tools, config.vars.worker_tools))
+    ? Object.keys(workerToolset(config.tools, config.vars, selection))
     : [];
   run(['mise', 'install', '--locked', ...selected]);
   observe('aws');
@@ -54,11 +60,20 @@ if (import.meta.main) {
   const flag = process.argv.indexOf('--status-file');
   const report = setupStatus(flag < 0 ? undefined : process.argv[flag + 1]);
   let phase: SetupPhase = 'managed-config';
+  const worker = process.argv.includes('--worker');
+  const node = process.argv.indexOf('--node');
   try {
-    await syncTools((current) => {
-      phase = current;
-      report(phase, 'running');
-    }, process.argv.includes('--worker'));
+    await syncTools(
+      (current) => {
+        phase = current;
+        report(phase, 'running');
+      },
+      worker,
+      // The tool profiles of the fleet configuration fleet-sync installed.
+      worker && node >= 0
+        ? installedSelection(process.argv[node + 1])
+        : NO_SELECTION
+    );
     if (!process.argv.includes('--tools-only')) {
       phase = 'dependencies';
       report(phase, 'running');

package/bootstrap/worker-tools.ts

@@ -1,3 +1,7 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
 /** A worker installs only this declared subset; every version still comes from mise.lock. */
 export function workerTools<T>(
   tools: Record<string, T>,
@@ -22,3 +26,117 @@ export function workerTools<T>(
     selected.map((key: string) => [key, tools[key]!])
   ) as Record<string, T>;
 }
+
+/** Named tool sets a fleet adds to its workers on top of the base set
+ * (`worker_tools`). Each is `worker_profile_<name>` under [vars] in
+ * mise.toml; every version still comes from mise.lock.
+ */
+export const TOOL_PROFILES = ['web', 'go', 'rust', 'jvm'] as const;
+export type ToolProfile = (typeof TOOL_PROFILES)[number];
+
+/** What a fleet configuration adds to one worker's base tools: profiles and
+ * single tools, each already pinned in mise.toml.
+ */
+export interface ToolSelection {
+  profiles: string[];
+  tools: string[];
+}
+export const NO_SELECTION: ToolSelection = { profiles: [], tools: [] };
+
+const list = (value: unknown): string[] =>
+  Array.isArray(value)
+    ? value.filter((item): item is string => typeof item === 'string')
+    : [];
+const record = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+/** The selection a fleet configuration makes for one worker: the fleet-wide
+ * `worker.toolProfiles` and `worker.tools`, plus that worker's own
+ * `toolProfiles`. Read from the plain document, so the bootstrap needs no
+ * fleet code.
+ */
+export function workerSelection(
+  definition: unknown,
+  node?: string
+): ToolSelection {
+  if (!record(definition)) return NO_SELECTION;
+  const policy = record(definition['worker']) ? definition['worker'] : {};
+  const workers: unknown[] = Array.isArray(definition['workers'])
+    ? definition['workers']
+    : [];
+  const own = workers
+    .filter(record)
+    .filter((entry) => node !== undefined && entry['id'] === node)
+    .flatMap((entry) => list(entry['toolProfiles']));
+  return {
+    profiles: [...new Set([...list(policy['toolProfiles']), ...own])],
+    tools: [...new Set(list(policy['tools']))],
+  };
+}
+
+/** The selection of the fleet configuration fleet-sync installed on this
+ * machine; none when there is no configuration yet.
+ */
+export function installedSelection(
+  node: string | undefined,
+  home: string = homedir()
+): ToolSelection {
+  const path = join(home, '.config/dev-fleet/fleet.json');
+  if (!existsSync(path)) return NO_SELECTION;
+  try {
+    return workerSelection(JSON.parse(readFileSync(path, 'utf8')), node);
+  } catch {
+    return NO_SELECTION;
+  }
+}
+
+/** The tools of one worker: the base set, then each selected profile's, then
+ * the single tools, in that order and each once. A profile mise.toml does not
+ * declare, or a tool it does not pin, is refused.
+ */
+export function workerToolset<T>(
+  tools: Record<string, T>,
+  vars: Record<string, unknown>,
+  selection: ToolSelection = NO_SELECTION
+): Record<string, T> {
+  const names = (value: unknown) =>
+    typeof value === 'string' ? value.split(',').filter(Boolean) : [];
+  const base = names(vars['worker_tools']);
+  const added = selection.profiles.flatMap((profile) => {
+    const declared = vars[`worker_profile_${profile}`];
+    if (typeof declared !== 'string' || !declared)
+      throw new Error(
+        `Unknown worker tool profile "${profile}". Choose from ${TOOL_PROFILES.join(', ')}.`
+      );
+    return names(declared);
+  });
+  for (const tool of selection.tools)
+    if (!Object.hasOwn(tools, tool))
+      throw new Error(
+        `The worker tool "${tool}" has no pin in dev-fleet's mise.toml. Ask for it in an issue.`
+      );
+  return workerTools(tools, [
+    ...new Set([...base, ...added, ...selection.tools]),
+  ]);
+}
+
+/** `--profiles web,go --tools fd` for the doctor and the setup. */
+export function selectionArguments(selection: ToolSelection): string[] {
+  return [
+    ...(selection.profiles.length
+      ? ['--profiles', selection.profiles.join(',')]
+      : []),
+    ...(selection.tools.length ? ['--tools', selection.tools.join(',')] : []),
+  ];
+}
+
+/** The selection named by `--profiles` and `--tools`. */
+export function parseSelectionArguments(argv: string[]): ToolSelection {
+  const value = (flag: string) => {
+    const index = argv.indexOf(flag);
+    return index < 0
+      ? []
+      : (argv[index + 1] ?? '').split(',').filter((item) => item.length > 0);
+  };
+  return { profiles: value('--profiles'), tools: value('--tools') };
+}

package/cliproxy/README.md

@@ -150,7 +150,7 @@ devfleet cliproxy inference messages EXACT_CLAUDE_MODEL_ID
 
 **Account routing.** Credential files are the only list of accounts. `ops declared` (run by every fleet-sync) gives each Claude, Codex and Antigravity credential the model prefix `<type>-<email>` through the proxy's management API (`scripts/accounts.py`; a credential file is never edited directly). A request for `<type>-<email>/<model>` is served only by that account, and `/v1/models` lists those ids, which is how every fleet node discovers the accounts to offer in T3. Nothing in Git names an account.
 
-Unprefixed routing is **fill-first** (`routing.strategy: fill-first`): one account serves until its quota is exhausted and it cools down, then the next account, with session affinity keeping a session on its account while available and bounded retries across eligible accounts for the requested model. `cliproxy apply` requires this policy and the host monitor reports drift; see [Multi-account routing](docs/model-health.md#multi-account-routing-fill-first) for applying it to a running proxy. Automatic preview-model switching is disabled. No model aliases are configured: clients use the upstream model ids exactly as `/v1/models` lists them. [`models.json`](models.json) declares the model ids that are health-checked (`claude-opus-5-5`, `claude-sonnet-5-5`, `gpt-6-astra`, `gpt-6.1-sol`), and the checks fail loudly when one is not served. Usage statistics are enabled but are not a durable billing ledger. Available quota details depend on the provider.
+Unprefixed routing is **fill-first** (`routing.strategy: fill-first`): one account serves until its quota is exhausted and it cools down, then the next account, with session affinity keeping a session on its account while available and bounded retries across eligible accounts for the requested model. `cliproxy apply` requires this policy and the host monitor reports drift; see [Multi-account routing](docs/model-health.md#multi-account-routing-fill-first) for applying it to a running proxy. Automatic preview-model switching is disabled. The only model aliases are the ones `config.template.yaml` declares under `oauth.model-alias` (`ops` applies them and waits until `/v1/models` serves them); every other model id is the upstream id exactly as `/v1/models` lists it. [`models.json`](models.json) declares the model ids that are health-checked (`claude-opus-5-5`, `claude-sonnet-5-5`, `gpt-6-astra`, `gpt-6.1-sol`), and the checks fail loudly when one is not served. Usage statistics are enabled but are not a durable billing ledger. Available quota details depend on the provider.
 
 ## Fleet clients: macOS, Ubuntu and Windows/WSL
 

package/cliproxy/compose.yaml

@@ -3,6 +3,10 @@ services:
   proxy:
     image: ${CLI_PROXY_IMAGE:?image must be pinned}
     restart: unless-stopped
+    # Docker restarts a proxy that exits; the container watchdog on the
+    # manager restarts one that stays unhealthy.
+    labels:
+      dev.devfleet.watchdog: restart
     user: '${CLIPROXY_UID:-1000}:${CLIPROXY_GID:-1000}'
     working_dir: /state
     command: ['/CLIProxyAPI/CLIProxyAPI', '-config', '/state/config.yaml']

package/cliproxy/scripts/checkpoint.py

@@ -68,7 +68,7 @@ def validate(root):
         raise ValueError("Provider cache directory missing")
     for path in (state / "runtime/auths").iterdir():
         if path.name == "kv.db" and path.is_file() and path.stat().st_size > 0:
-            continue  # Opaque upstream bbolt state, captured while the process is stopped.
+            continue  # Opaque upstream bbolt state, copied while the proxy runs (see capture).
         # .json: provider credential; .cds: upstream per-account cooldown
         # records (JSON object, no credential), kept so limits survive moves.
         if path.suffix not in (".json", ".cds") or not path.is_file():

package/docs/authentication.md

@@ -0,0 +1,90 @@
+# Authentication and accounts
+
+Where each credential dev-fleet uses lives, who can read it, how it reaches
+each node, how it rotates and what happens when it fails. The three tables
+are generated from the code; `features/rules/authentication.feature` fails
+when one is out of date (`DEVFLEET_UPDATE_DOCS=1 bun test features/rules`
+rewrites them).
+
+## Where credentials live
+
+| Credential                          | Lives in                                                                                                                      | Read by                                          |
+| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
+| The fleet's secrets (table below)   | The provider `secrets` names in `fleet.json`                                                                                  | The node that needs it, through `src/secrets.ts` |
+| The bucket's two keys               | The environment of the `devfleet` command; `~/.config/dev-fleet/bucket.json` (0600) on the manager after `fleet-state --init` | The manager only                                 |
+| A node's Vault token                | That node's own Vault session (`~/.vault-token`)                                                                              | That node only; never copied                     |
+| A node's GitHub login               | `gh`'s own store on that node                                                                                                 | `gh`, and git through its credential helper      |
+| Provider accounts (Claude, Codex …) | The CLI proxy's store on the manager                                                                                          | The proxy; nodes hold only the proxy client key  |
+| Tailscale and T3 pairing            | Each machine, signed in by a person                                                                                           | That machine                                     |
+
+## How they reach each node
+
+- **Vault.** A person logs each machine in once (`fleet-sync` asks on the
+  manager and passes the password to the worker over SSH on standard input).
+  From then on the worker's `personal-worker-auth` timer renews its own token
+  (`bootstrap/worker-auth.ts`). Nothing renders provider configuration from
+  Vault on a worker: agent profiles come from `fleet-sync`.
+- **GitHub.** The same job restores the declared account (`worker.githubLogin`)
+  with the GitHub service token it reads from its own Vault session
+  (`bootstrap/worker-github.ts`, on standard input, never argv). `fleet-sync`
+  makes `gh` git's credential helper and writes the declared identity
+  (`git` in `fleet.json`, `ansible/github/git.yml`) on every node.
+- **The proxy client key** is read by each node from its own secrets provider
+  whenever a launcher starts an agent.
+- **The monitoring hub's backup storage keys** are read on the manager by
+  `fleet-sync` and handed to the hub's playbook in a private variables file.
+
+## Rotation and failure
+
+Rotate a secret where the provider keeps it; the next read uses it, nothing
+caches it. A Vault token that is not renewed expires: `worker:vault-renewal`
+fails and the fix is one login of that machine. A GitHub token that stops
+working fails `auth:github` and `worker:git-push-auth`; store the new one in
+Vault and the worker restores itself within ten minutes. During a proxy
+outage agents on workers wait: no worker is configured for native clients.
+Every failure code is in [the runbooks](runbooks.md).
+
+## The fleet's secrets
+
+<!-- generated: secrets -->
+
+| Secret                         | Required                     | `env` provider variable                        | `file` provider file                                       |
+| ------------------------------ | ---------------------------- | ---------------------------------------------- | ---------------------------------------------------------- |
+| `proxy-client-key`             | yes                          | `DEVFLEET_SECRET_PROXY_CLIENT_KEY`             | `~/.config/dev-fleet/secrets/proxy-client-key`             |
+| `monitoring-account-password`  | yes                          | `DEVFLEET_SECRET_MONITORING_ACCOUNT_PASSWORD`  | `~/.config/dev-fleet/secrets/monitoring-account-password`  |
+| `monitoring-tunnel-token`      | yes                          | `DEVFLEET_SECRET_MONITORING_TUNNEL_TOKEN`      | `~/.config/dev-fleet/secrets/monitoring-tunnel-token`      |
+| `proxy-tunnel-token`           | yes                          | `DEVFLEET_SECRET_PROXY_TUNNEL_TOKEN`           | `~/.config/dev-fleet/secrets/proxy-tunnel-token`           |
+| `monitoring-backup-access-key` | when its feature is declared | `DEVFLEET_SECRET_MONITORING_BACKUP_ACCESS_KEY` | `~/.config/dev-fleet/secrets/monitoring-backup-access-key` |
+| `monitoring-backup-secret-key` | when its feature is declared | `DEVFLEET_SECRET_MONITORING_BACKUP_SECRET_KEY` | `~/.config/dev-fleet/secrets/monitoring-backup-secret-key` |
+| `notify-webhook-url`           | when its feature is declared | `DEVFLEET_SECRET_NOTIFY_WEBHOOK_URL`           | `~/.config/dev-fleet/secrets/notify-webhook-url`           |
+| `notify-ntfy-url`              | when its feature is declared | `DEVFLEET_SECRET_NOTIFY_NTFY_URL`              | `~/.config/dev-fleet/secrets/notify-ntfy-url`              |
+
+<!-- end: secrets -->
+
+## Checks of credentials
+
+<!-- generated: checks -->
+
+| Check                         | Measured in                                |
+| ----------------------------- | ------------------------------------------ |
+| `auth:vault`                  | `src/cli/fleet-sync-vault.ts`              |
+| `worker:git-identity`         | `src/git-identity.ts`                      |
+| `worker:git-push-auth`        | `src/definition.ts`, `src/git-identity.ts` |
+| `worker:github-principal`     | `src/worker-access-checks.ts`              |
+| `worker:proxy-authentication` | `src/worker-access-checks.ts`              |
+| `worker:vault-renewal`        | `src/worker-access-checks.ts`              |
+
+<!-- end: checks -->
+
+## Timers
+
+<!-- generated: timers -->
+
+| Timer                            | Declared in                            | Schedule                              |
+| -------------------------------- | -------------------------------------- | ------------------------------------- |
+| `personal-fleet-disk-gc.timer`   | `ansible/ubuntu/tasks/resources.yml`   | OnBootSec=10min, OnCalendar=hourly    |
+| `personal-worker-auth.timer`     | `ansible/ubuntu/tasks/credentials.yml` | OnBootSec=2min, OnUnitActiveSec=10min |
+| `personal-worker-recovery.timer` | `ansible/ubuntu/tasks/recovery.yml`    | OnBootSec=1min, OnUnitActiveSec=1min  |
+| `personal-worker-recovery.timer` | `ansible/ubuntu/tasks/recovery.yml`    | OnBootSec=5min, OnUnitActiveSec=15min |
+
+<!-- end: timers -->

package/docs/declared.md

@@ -31,6 +31,8 @@ may only shrink.
 | Ansible on a worker                           | `ansible-core` in `ansible/ubuntu/policy.json`                                                      | Exact version                                                        | `ansible/ubuntu/tasks/packages.yml`                       | `package:ansible-core`                       |
 | The CLI proxy                                 | `cliproxy/compose.yaml`, `image.lock.json`, `requirements.lock`, `launchd/`, `config.template.yaml` | Image digests; hashed Python dependencies                            | `devfleet cliproxy install`, and `fleet-sync` after it    | The proxy's audit                            |
 | Agent profiles and T3 providers               | `agents/registry.json`, `agents/t3.json`                                                            | The commit                                                           | `fleet-sync`                                              | Profile receipts; the plan after every apply |
+| Worker time service                           | `ansible/ubuntu/tasks/services.yml`, offsets in `clock` in `ansible/ubuntu/policy.json`             | The commit                                                           | `fleet-sync`                                              | `worker:clock`                               |
+| The manager's container watchdog              | `ansible/monitoring/container-watchdog.py`, `containerWatchdog` in `ansible/macos/policy.json`      | The commit                                                           | `fleet-sync` (`hub-darwin.yml`)                           | `manager:container-watchdog`                 |
 | Docker Desktop on the manager                 | `docker_desktop_*` under `[vars]` in `mise.toml`                                                    | Version and build                                                    | You                                                       | `manager:docker` (warns)                     |
 | CI                                            | `.github/workflows/`                                                                                | Each action at a commit; tools from `mise.lock`                      | GitHub                                                    | `features/rules/`                            |