@crvouga/dev-fleet 0.2.0 → 0.4.0

package/ansible/ubuntu/wip-guard.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+"""Push work in progress of every agent worktree to a recovery ref.
+
+Managed by personal fleet-sync (tasks/recovery.yml) as
+personal-worker-wip-guard.service, started by personal-worker-wip-guard.timer.
+
+Workers are disposable: a worker lost mid-task must lose at most one interval
+of uncommitted work. For each Git worktree under --root, the files as they are
+now (tracked changes and untracked files that are not ignored) are written as
+one commit on top of HEAD, built in a private index so the worktree, its index
+and its branches are never touched, and pushed to
+
+    refs/recovery/<node>/<branch>
+
+of the worktree's own remote. A clean worktree whose HEAD is already on a
+remote branch has nothing to recover and pushes nothing. Recover with
+`git fetch origin 'refs/recovery/*:refs/recovery/*'`.
+
+Every decision is one JSON line on stdout.
+
+Usage: wip-guard.py --root DIR [--root DIR ...] [--node ID] [--remote NAME]
+                    [--max-depth N] [--dry-run]
+"""
+import argparse
+import json
+import os
+import re
+import socket
+import subprocess
+import tempfile
+from pathlib import Path
+
+GIT = ['git', '-c', 'safe.directory=*']
+AUTHOR = {
+    'GIT_AUTHOR_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_AUTHOR_EMAIL': 'wip-guard@example.invalid',
+    'GIT_COMMITTER_NAME': 'dev-fleet work-in-progress guard',
+    'GIT_COMMITTER_EMAIL': 'wip-guard@example.invalid',
+}
+
+
+def log(**event):
+    print(json.dumps(event, sort_keys=True), flush=True)
+
+
+def git(path, *args, env=None):
+    return subprocess.run(
+        [*GIT, '-C', str(path), *args], capture_output=True, text=True,
+        env={**os.environ, **(env or {})}, timeout=120)
+
+
+def worktrees(root, depth):
+    if depth < 0 or not root.is_dir() or root.is_symlink():
+        return
+    if (root / '.git').exists():
+        yield root
+        return
+    for child in sorted(root.iterdir()):
+        if child.is_dir() and not child.is_symlink():
+            yield from worktrees(child, depth - 1)
+
+
+def ref_part(value):
+    part = re.sub(r'[^A-Za-z0-9._/-]+', '-', value).strip('./-')
+    part = re.sub(r'/+', '/', part).replace('..', '.')
+    return part or 'unnamed'
+
+
+def snapshot(path):
+    """The worktree's files as one commit on HEAD, or HEAD when clean."""
+    head = git(path, 'rev-parse', '--verify', 'HEAD').stdout.strip()
+    if not head:
+        return None, None
+    with tempfile.TemporaryDirectory() as private:
+        index = {'GIT_INDEX_FILE': os.path.join(private, 'index')}
+        if git(path, 'read-tree', 'HEAD', env=index).returncode != 0:
+            return head, None
+        if git(path, 'add', '--all', '--', '.', env=index).returncode != 0:
+            return head, None
+        tree = git(path, 'write-tree', env=index).stdout.strip()
+    if tree == git(path, 'rev-parse', 'HEAD^{tree}').stdout.strip():
+        return head, head
+    commit = git(path, 'commit-tree', tree, '-p', head, '-m',
+                 'Work in progress saved by the dev-fleet guard', env=AUTHOR)
+    return head, commit.stdout.strip() or None
+
+
+def guard(path, node, remote, dry_run):
+    if git(path, 'remote', 'get-url', remote).returncode != 0:
+        return log(worktree=str(path), action='skip', reason='no-remote')
+    head, commit = snapshot(path)
+    if not commit:
+        return log(worktree=str(path), action='skip', reason='no-commit')
+    if commit == head and git(path, 'branch', '--remotes', '--contains', head).stdout.strip():
+        return log(worktree=str(path), action='skip', reason='clean')
+    branch = git(path, 'symbolic-ref', '--quiet', '--short', 'HEAD').stdout.strip()
+    ref = 'refs/recovery/%s/%s' % (ref_part(node), ref_part(branch or 'detached-' + head[:12]))
+    if dry_run:
+        return log(worktree=str(path), action='would-push', ref=ref)
+    pushed = git(path, 'push', '--force', '--no-verify', '--quiet', remote, '%s:%s' % (commit, ref))
+    log(worktree=str(path), action='pushed' if pushed.returncode == 0 else 'failed', ref=ref)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', action='append', required=True)
+    parser.add_argument('--node', default=socket.gethostname().split('.')[0])
+    parser.add_argument('--remote', default='origin')
+    parser.add_argument('--max-depth', type=int, default=3)
+    parser.add_argument('--dry-run', action='store_true')
+    options = parser.parse_args()
+    for root in options.root:
+        for path in worktrees(Path(root).expanduser(), options.max_depth):
+            try:
+                guard(path, options.node, options.remote, options.dry_run)
+            except (OSError, subprocess.SubprocessError):
+                log(worktree=str(path), action='failed', reason='git')
+
+
+if __name__ == '__main__':
+    main()

package/bootstrap/doctor.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  NO_SELECTION,
+  parseSelectionArguments,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 /** Read-only toolchain checks; no logins, installations or changes to Docker. */
 import { spawnSync } from 'node:child_process';
 import { readFileSync } from 'node:fs';
@@ -15,7 +20,7 @@ import {
 type Tool = string | { version: string; lazy?: boolean; os?: string[] };
 type Config = {
   tools: Record<string, Tool>;
-  vars?: { worker_tools?: unknown };
+  vars?: Record<string, unknown>;
 };
 export type Check = {
   id: string;
@@ -42,6 +47,11 @@ const VERSION_ARGS: Record<string, string[]> = {
   kubectl: ['version', '--client=true', '--output=json'],
   helm: ['version', '--short'],
 };
+/** The version a tool prints, where it differs from its mise pin. */
+const PRINTED: Record<string, (pin: string) => string> = {
+  // temurin-25.0.5+8.0.LTS prints as 25.0.5.
+  java: (pin) => pin.replace(/^[a-z]+-/, '').replace(/\+.*$/, ''),
+};
 const run: Runner = (command) => {
   const result = spawnSync(command[0]!, command.slice(1), {
     encoding: 'utf8',
@@ -80,7 +90,8 @@ export function toolChecks(
     if (typeof tool !== 'string' && tool.lazy) continue;
     // A tool pinned only for other platforms has no artifact to verify here.
     if (unavailable(id, tool, platform)) continue;
-    const version = typeof tool === 'string' ? tool : tool.version;
+    const pin = typeof tool === 'string' ? tool : tool.version;
+    const version = PRINTED[id]?.(pin) ?? pin;
     const command = COMMANDS[id] ?? id;
     const result = execute([command, ...(VERSION_ARGS[id] ?? ['--version'])]);
     const exact = new RegExp(
@@ -195,7 +206,21 @@ export function globalConfigCheck(
   };
 }
 
-export function checkMachine(requireDocker = false, worker = false): Check[] {
+/** A worker installs every tool it selects, lazy or not, so each is checked. */
+function installed(tools: Record<string, Tool>): Record<string, Tool> {
+  return Object.fromEntries(
+    Object.entries(tools).map(([id, tool]) => [
+      id,
+      typeof tool === 'string' ? tool : { ...tool, lazy: false },
+    ])
+  );
+}
+
+export function checkMachine(
+  requireDocker = false,
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
+): Check[] {
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Config;
@@ -203,7 +228,7 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
   return [
     ...toolChecks(
       worker
-        ? workerTools(config.tools, config.vars?.worker_tools)
+        ? installed(workerToolset(config.tools, config.vars ?? {}, selection))
         : config.tools
     ),
     ...dockerChecks(requireDocker),
@@ -222,7 +247,8 @@ export function checkMachine(requireDocker = false, worker = false): Check[] {
 if (import.meta.main) {
   const checks = checkMachine(
     process.argv.includes('--docker'),
-    process.argv.includes('--worker')
+    process.argv.includes('--worker'),
+    parseSelectionArguments(process.argv)
   );
   const ok = checks.every((check) => check.status !== 'fail');
   if (process.argv.includes('--json'))

package/bootstrap/fleet-remote-bootstrap.sh

@@ -41,7 +41,7 @@ remedy() {
     BOOTSTRAP_LOCK_TOOL_REQUIRED) echo 'Install python3 so the operation lock can be taken, then rerun fleet-sync.';;
     SOURCE_CACHE_CHANGED|SOURCE_STATUS_FAILED|SOURCE_CHECKOUT_CONFLICT) echo 'The fleet-owned checkout of this revision holds changes fleet-sync did not make. Inspect and remove that revisions directory, then rerun fleet-sync.';;
     CONFIGURATION_INSTALL_FAILED) echo 'The fleet configuration could not be written under ~/.config/dev-fleet. Check that directory, then rerun fleet-sync.';;
-    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet doctor on the worker.';;
+    TOOLCHAIN_*) echo 'The pinned toolchain did not converge. Rerun fleet-sync; it resumes where it stopped. If it repeats, run devfleet fleet-status --node for this worker.';;
     *) echo 'Inspect the remote operation lock and rerun the approved revision. No success receipt advanced.';;
   esac
 }
@@ -97,7 +97,18 @@ else
   command -v python3 >/dev/null 2>&1 || die BOOTSTRAP_LOCK_TOOL_REQUIRED
   python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || die FLEET_REMOTE_LOCKED
 fi
-printf '{"version":1,"pid":%s,"revision":"%s","node":"%s"}\n' "$$" "$revision" "$node" >"$root/remote-operation.json"
+# The operation record: a manager that lost its session reads it to follow
+# this operation to its end instead of starting another (see
+# src/remote-operation.ts). The result of the last operation is beside it.
+started=$(date +%s)
+result="$root/remote-operation-result.json"
+[ ! -L "$root/remote-operation.json" ] && [ ! -L "$result" ] || die FLEET_STATE_PATH_UNSAFE
+rm -f -- "$result"
+operation() {
+  printf '{"version":1,"pid":%s,"revision":"%s","node":"%s","started":%s,"phase":"%s"%s}\n' "$$" "$revision" "$node" "$started" "$1" "${2:-}" >"$root/remote-operation.json.new" &&
+    mv -f -- "$root/remote-operation.json.new" "$root/remote-operation.json"
+}
+operation lock
 # The fleet configuration is installed under the lock, so two overlapping
 # runs never write it at once. It holds no secret.
 phase='configuration-install'
@@ -109,6 +120,7 @@ if [ -n "$configuration" ]; then
   mv "$config_dir/fleet.json.new" "$config_dir/fleet.json" || die CONFIGURATION_INSTALL_FAILED
 fi
 phase=resources
+operation resources
 # A bootstrap floor protects source transfer and minimal prerequisites. The
 # Workstation baseline subsequently enforces its stricter declared capacity.
 free_kib=$(df -Pk "$root" | awk 'END {print $4}')
@@ -200,6 +212,7 @@ bundle=''
 clean=$(git -C "$deployment" status --porcelain) || die SOURCE_STATUS_FAILED
 [ -z "$clean" ] || die SOURCE_CHECKOUT_CONFLICT
 phase=toolchain
+operation toolchain
 cd "$deployment"
 export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
 status_file="$root/toolchain-status.json"
@@ -211,6 +224,11 @@ ready_file="$root/toolchain-ready"
 # are still checked (and reinstalled on drift) by node convergence.
 toolchain_key=$(git rev-parse HEAD:mise.toml HEAD:mise.lock HEAD:.mise HEAD:bun.lock HEAD:package.json HEAD:bootstrap 2>/dev/null | tr '\n' ' ')
 [ -n "$toolchain_key" ] || toolchain_key=$revision
+# The tool profiles of the fleet configuration choose what is installed, so a
+# changed configuration runs the setup again.
+if [ -f "$HOME/.config/dev-fleet/fleet.json" ]; then
+  toolchain_key="$toolchain_key$(cksum <"$HOME/.config/dev-fleet/fleet.json" | cut -d' ' -f1)"
+fi
 if [ "$(cat "$ready_file" 2>/dev/null)" = "$toolchain_key" ] && [ -n "$(find "$ready_file" -mmin -1440 2>/dev/null)" ]; then
   toolchain_warm=1
 else
@@ -219,7 +237,7 @@ else
 fi
 if command -v timeout >/dev/null 2>&1; then bounded='timeout 1800'; else bounded=''; fi
 # shellcheck disable=SC2086
-if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
+if [ -z "$toolchain_warm" ] && ! $bounded bash bootstrap/bootstrap.sh --worker --node "$node" --non-interactive --status-file "$status_file" >/dev/null 2>&1; then
   tool_phase=$(sed -n 's/.*"phase":"\([a-z0-9-]*\)".*/\1/p' "$status_file" 2>/dev/null) || tool_phase=''
   case "$tool_phase" in
     managed-config) die TOOLCHAIN_CONFIG_CONFLICT;;
@@ -254,5 +272,14 @@ for old in $(ls -1t "$root/revisions"); do
   rm -rf -- "${root:?}/revisions/$old" || true
 done
 phase=configuration
+operation configuration
 trap - EXIT INT TERM
-exec mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision"
+# Node convergence writes its evidence to the result file, not to the SSH
+# session: the session may be gone by the time it ends. The evidence is then
+# printed for the manager that is still connected, under the same lock.
+code=0
+mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision" >"$result.partial" 2>"$root/remote-operation.log" || code=$?
+mv -f -- "$result.partial" "$result"
+operation finished ",\"finished\":$(date +%s),\"exit\":$code"
+cat -- "$result" || true
+exit "$code"

package/bootstrap/native-t3.ts

@@ -19,6 +19,7 @@ interface Configuration {
     t3_version: string;
     t3_linux_x64_sha256: string;
     t3_linux_arm64_sha256: string;
+    t3_darwin_arm64_sha256?: string;
     t3_desktop_amd64_sha256: string;
     t3_desktop_arm64_sha256: string;
   };
@@ -52,20 +53,27 @@ export function managedT3Desktop() {
   ) as Configuration;
   return desktopT3Artifact(config, process.arch);
 }
-export function nativeT3Artifact(config: Configuration, architecture: string) {
-  if (!['x64', 'arm64'].includes(architecture))
-    throw new Error('Unsupported T3 architecture.');
+/** The pinned server archive: Ubuntu workers on either architecture, and
+ * the manager (macOS on Apple silicon, the only macOS build upstream ships).
+ */
+export function nativeT3Artifact(
+  config: Configuration,
+  architecture: string,
+  platform: string = 'linux'
+) {
+  const sha256 = (<Record<string, string | undefined>>{
+    'linux-x64': config.vars.t3_linux_x64_sha256,
+    'linux-arm64': config.vars.t3_linux_arm64_sha256,
+    'darwin-arm64': config.vars.t3_darwin_arm64_sha256,
+  })[`${platform}-${architecture}`];
+  if (sha256 === undefined) throw new Error('Unsupported T3 architecture.');
   const version = config.vars.t3_version;
-  const sha256 =
-    architecture === 'x64'
-      ? config.vars.t3_linux_x64_sha256
-      : config.vars.t3_linux_arm64_sha256;
   if (
     !/^\d+\.\d+\.\d+(?:-nightly\.\d{8}\.\d+)?$/.test(version) ||
     !/^[a-f0-9]{64}$/.test(sha256)
   )
     throw new Error('Invalid T3 pin.');
-  const name = `t3-${version}-linux-${architecture}`;
+  const name = `t3-${version}-${platform}-${architecture}`;
   return {
     version,
     sha256,
@@ -77,18 +85,21 @@ export function managedT3(): { path: string; version: string } {
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Configuration;
-  const artifact = nativeT3Artifact(config, process.arch);
+  const artifact = nativeT3Artifact(config, process.arch, process.platform);
   return {
     path: join(homedir(), '.local/share/personal/tools', artifact.name, 't3'),
     version: artifact.version,
   };
 }
+/** Installs the pinned, checksum-verified server archive on a worker and on
+ * the manager, so both run the T3 version the fleet pins.
+ */
 export async function installNativeT3(): Promise<void> {
-  if (process.platform !== 'linux') return;
+  if (!['linux', 'darwin'].includes(process.platform)) return;
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
   ) as Configuration;
-  const artifact = nativeT3Artifact(config, process.arch);
+  const artifact = nativeT3Artifact(config, process.arch, process.platform);
   const root = join(homedir(), '.local/share/personal/tools');
   const archive = join(root, `${artifact.name}.tar.gz`);
   mkdirSync(root, { recursive: true, mode: 0o700 });

package/bootstrap/setup.ts

@@ -1,4 +1,9 @@
-import { workerTools } from './worker-tools';
+import {
+  installedSelection,
+  NO_SELECTION,
+  workerToolset,
+  type ToolSelection,
+} from './worker-tools';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 /** Used by both OS bootstraps, dev-fleet tools sync and dev-fleet update. No workspace dependencies. */
@@ -26,7 +31,8 @@ function run(args: string[]): void {
 
 export async function syncTools(
   observe: (phase: SetupPhase) => void = () => {},
-  worker = false
+  worker = false,
+  selection: ToolSelection = NO_SELECTION
 ): Promise<void> {
   observe('managed-config');
   const dir = globalConfigDir();
@@ -34,9 +40,9 @@ export async function syncTools(
   observe('mise');
   const config = Bun.TOML.parse(
     readFileSync(join(workspaceRoot, 'mise.toml'), 'utf8')
-  ) as { tools: Record<string, unknown>; vars: { worker_tools: unknown } };
+  ) as { tools: Record<string, unknown>; vars: Record<string, unknown> };
   const selected = worker
-    ? Object.keys(workerTools(config.tools, config.vars.worker_tools))
+    ? Object.keys(workerToolset(config.tools, config.vars, selection))
     : [];
   run(['mise', 'install', '--locked', ...selected]);
   observe('aws');
@@ -54,11 +60,20 @@ if (import.meta.main) {
   const flag = process.argv.indexOf('--status-file');
   const report = setupStatus(flag < 0 ? undefined : process.argv[flag + 1]);
   let phase: SetupPhase = 'managed-config';
+  const worker = process.argv.includes('--worker');
+  const node = process.argv.indexOf('--node');
   try {
-    await syncTools((current) => {
-      phase = current;
-      report(phase, 'running');
-    }, process.argv.includes('--worker'));
+    await syncTools(
+      (current) => {
+        phase = current;
+        report(phase, 'running');
+      },
+      worker,
+      // The tool profiles of the fleet configuration fleet-sync installed.
+      worker && node >= 0
+        ? installedSelection(process.argv[node + 1])
+        : NO_SELECTION
+    );
     if (!process.argv.includes('--tools-only')) {
       phase = 'dependencies';
       report(phase, 'running');

package/bootstrap/worker-tools.ts

@@ -1,3 +1,7 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
 /** A worker installs only this declared subset; every version still comes from mise.lock. */
 export function workerTools<T>(
   tools: Record<string, T>,
@@ -22,3 +26,117 @@ export function workerTools<T>(
     selected.map((key: string) => [key, tools[key]!])
   ) as Record<string, T>;
 }
+
+/** Named tool sets a fleet adds to its workers on top of the base set
+ * (`worker_tools`). Each is `worker_profile_<name>` under [vars] in
+ * mise.toml; every version still comes from mise.lock.
+ */
+export const TOOL_PROFILES = ['web', 'go', 'rust', 'jvm'] as const;
+export type ToolProfile = (typeof TOOL_PROFILES)[number];
+
+/** What a fleet configuration adds to one worker's base tools: profiles and
+ * single tools, each already pinned in mise.toml.
+ */
+export interface ToolSelection {
+  profiles: string[];
+  tools: string[];
+}
+export const NO_SELECTION: ToolSelection = { profiles: [], tools: [] };
+
+const list = (value: unknown): string[] =>
+  Array.isArray(value)
+    ? value.filter((item): item is string => typeof item === 'string')
+    : [];
+const record = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+/** The selection a fleet configuration makes for one worker: the fleet-wide
+ * `worker.toolProfiles` and `worker.tools`, plus that worker's own
+ * `toolProfiles`. Read from the plain document, so the bootstrap needs no
+ * fleet code.
+ */
+export function workerSelection(
+  definition: unknown,
+  node?: string
+): ToolSelection {
+  if (!record(definition)) return NO_SELECTION;
+  const policy = record(definition['worker']) ? definition['worker'] : {};
+  const workers: unknown[] = Array.isArray(definition['workers'])
+    ? definition['workers']
+    : [];
+  const own = workers
+    .filter(record)
+    .filter((entry) => node !== undefined && entry['id'] === node)
+    .flatMap((entry) => list(entry['toolProfiles']));
+  return {
+    profiles: [...new Set([...list(policy['toolProfiles']), ...own])],
+    tools: [...new Set(list(policy['tools']))],
+  };
+}
+
+/** The selection of the fleet configuration fleet-sync installed on this
+ * machine; none when there is no configuration yet.
+ */
+export function installedSelection(
+  node: string | undefined,
+  home: string = homedir()
+): ToolSelection {
+  const path = join(home, '.config/dev-fleet/fleet.json');
+  if (!existsSync(path)) return NO_SELECTION;
+  try {
+    return workerSelection(JSON.parse(readFileSync(path, 'utf8')), node);
+  } catch {
+    return NO_SELECTION;
+  }
+}
+
+/** The tools of one worker: the base set, then each selected profile's, then
+ * the single tools, in that order and each once. A profile mise.toml does not
+ * declare, or a tool it does not pin, is refused.
+ */
+export function workerToolset<T>(
+  tools: Record<string, T>,
+  vars: Record<string, unknown>,
+  selection: ToolSelection = NO_SELECTION
+): Record<string, T> {
+  const names = (value: unknown) =>
+    typeof value === 'string' ? value.split(',').filter(Boolean) : [];
+  const base = names(vars['worker_tools']);
+  const added = selection.profiles.flatMap((profile) => {
+    const declared = vars[`worker_profile_${profile}`];
+    if (typeof declared !== 'string' || !declared)
+      throw new Error(
+        `Unknown worker tool profile "${profile}". Choose from ${TOOL_PROFILES.join(', ')}.`
+      );
+    return names(declared);
+  });
+  for (const tool of selection.tools)
+    if (!Object.hasOwn(tools, tool))
+      throw new Error(
+        `The worker tool "${tool}" has no pin in dev-fleet's mise.toml. Ask for it in an issue.`
+      );
+  return workerTools(tools, [
+    ...new Set([...base, ...added, ...selection.tools]),
+  ]);
+}
+
+/** `--profiles web,go --tools fd` for the doctor and the setup. */
+export function selectionArguments(selection: ToolSelection): string[] {
+  return [
+    ...(selection.profiles.length
+      ? ['--profiles', selection.profiles.join(',')]
+      : []),
+    ...(selection.tools.length ? ['--tools', selection.tools.join(',')] : []),
+  ];
+}
+
+/** The selection named by `--profiles` and `--tools`. */
+export function parseSelectionArguments(argv: string[]): ToolSelection {
+  const value = (flag: string) => {
+    const index = argv.indexOf(flag);
+    return index < 0
+      ? []
+      : (argv[index + 1] ?? '').split(',').filter((item) => item.length > 0);
+  };
+  return { profiles: value('--profiles'), tools: value('--tools') };
+}

package/cliproxy/README.md

@@ -146,11 +146,11 @@ devfleet cliproxy inference responses EXACT_CODEX_MODEL_ID
 devfleet cliproxy inference messages EXACT_CLAUDE_MODEL_ID
 ```
 
-Codex uses the upstream device-code flow. Remote callback flows may require the documented dashboard callback flow or temporary SSH loopback forwarding; callback ports are not exposed by default. This pinned version has no generic Gemini CLI login flag; use its supported dashboard/API-key integration. Provider rules still govern permissible account use.
+`devfleet fleet-accounts --add|--relogin|--remove` wraps these and propagates the result to every node; a login writes into a staging directory and only a complete credential is moved in. Codex uses the upstream device-code flow. Remote callback flows may require the documented dashboard callback flow or temporary SSH loopback forwarding; callback ports are not exposed by default. This pinned version has no generic Gemini CLI login flag; use its supported dashboard/API-key integration. Provider rules still govern permissible account use.
 
 **Account routing.** Credential files are the only list of accounts. `ops declared` (run by every fleet-sync) gives each Claude, Codex and Antigravity credential the model prefix `<type>-<email>` through the proxy's management API (`scripts/accounts.py`; a credential file is never edited directly). A request for `<type>-<email>/<model>` is served only by that account, and `/v1/models` lists those ids, which is how every fleet node discovers the accounts to offer in T3. Nothing in Git names an account.
 
-Unprefixed routing is **fill-first** (`routing.strategy: fill-first`): one account serves until its quota is exhausted and it cools down, then the next account, with session affinity keeping a session on its account while available and bounded retries across eligible accounts for the requested model. `cliproxy apply` requires this policy and the host monitor reports drift; see [Multi-account routing](docs/model-health.md#multi-account-routing-fill-first) for applying it to a running proxy. Automatic preview-model switching is disabled. No model aliases are configured: clients use the upstream model ids exactly as `/v1/models` lists them. [`models.json`](models.json) declares the model ids that are health-checked (`claude-opus-5-5`, `claude-sonnet-5-5`, `gpt-6-astra`, `gpt-6.1-sol`), and the checks fail loudly when one is not served. Usage statistics are enabled but are not a durable billing ledger. Available quota details depend on the provider.
+Unprefixed routing is **fill-first** (`routing.strategy: fill-first`): one account serves until its quota is exhausted and it cools down, then the next account, with session affinity keeping a session on its account while available and bounded retries across eligible accounts for the requested model. `cliproxy apply` requires this policy and the host monitor reports drift; see [Multi-account routing](docs/model-health.md#multi-account-routing-fill-first) for applying it to a running proxy. Automatic preview-model switching is disabled. The only model aliases are the ones `config.template.yaml` declares under `oauth.model-alias` (`ops` applies them and waits until `/v1/models` serves them); every other model id is the upstream id exactly as `/v1/models` lists it. [`models.json`](models.json) declares the model ids that are health-checked (`claude-opus-5-5`, `claude-sonnet-5-5`, `gpt-6-astra`, `gpt-6.1-sol`), and the checks fail loudly when one is not served. Usage statistics are enabled but are not a durable billing ledger. Available quota details depend on the provider.
 
 ## Fleet clients: macOS, Ubuntu and Windows/WSL
 
@@ -167,7 +167,7 @@ python3 cliproxy/clients/bootstrap.py --validate
 
 Enter the inference key at the hidden prompt. It is available in Vault `secret/personal/prd:CLIPROXY_FLEET_CLIENT_API_KEY`; dashboard login uses `CLIPROXY_FLEET_MANAGEMENT_KEY`. Secrets never appear in command arguments. Keys are saved mode 0600 and loaded only into child environments. Give runners separate revocable inference keys where needed, never provider OAuth credentials or the admin key.
 
-Codex uses custom-provider CLI overrides and Responses HTTP/SSE. Claude uses its supported base URL/auth-token variables and an explicit Claude model. OpenCode uses its config-file override, with separate Responses and Anthropic SDK providers. Once Claude is authenticated, rerun with `--claude-model MODEL_ID`; otherwise the Claude launcher fails with an explicit setup requirement. Native client config files are preserved. Use plain native clients during a proxy outage.
+Codex uses custom-provider CLI overrides and Responses HTTP/SSE. Claude uses its supported base URL/auth-token variables and an explicit Claude model. OpenCode uses its config-file override, with separate Responses and Anthropic SDK providers. Once Claude is authenticated, rerun with `--claude-model MODEL_ID`; otherwise the Claude launcher fails with an explicit setup requirement. Native client config files are preserved. Fleet nodes never fall back to native clients: provider accounts exist only on the proxy (see [docs/accounts.md](../docs/accounts.md)), and during a proxy outage every relayed launcher retries for about a minute, then reports "proxy unreachable".
 
 Run bootstrap from a repository checkout containing `clients/launcher.py`; detached downloaded bootstrap files are unsupported. Bootstrap is idempotent and refuses unrelated launcher collisions. `--uninstall` removes launcher links while retaining private settings for deliberate cleanup. On Windows run inside WSL; native PowerShell launchers are not provided. Validate from each actual machine and target repository because OpenCode merges project settings.
 
@@ -187,7 +187,7 @@ Vault login is interactive with hidden input; its token is restricted locally an
 
 Encrypted cloud checkpoints run every 15 minutes (legacy systemd adds up to 30 seconds jitter). A checkpoint never stops, pauses or restarts the stack: agents use the proxy around the clock, and a stopped connector fails every request. The capture copies the live state and keeps the copy only if no state file changed while it was taken (retrying a few times); a stack that never holds still fails that backup. A failed capture, restart, upload or retention operation records failure and never advances the success marker. Retention keeps the latest 96 snapshots plus 7 daily, 5 weekly and 12 monthly. Weekly integrity checks read 10% of stored data; `check-full` reads all data. Monitoring fails when the last successful checkpoint is older than 45 minutes. Failures are visible in `cliproxy status` (and `launchctl print gui/$UID/LABEL` or systemd); off-machine checks also run through the CLI proxy health workflow described below.
 
-R2 checkpoints are the durable recovery source for config, provider OAuth/account state and the dedicated Tailscale identity. Local state is a working cache with asynchronous durability: normally up to approximately 15 minutes plus upload duration can be lost, and an outage can extend that indefinitely. Run a successful manual checkpoint after important login/config changes or before moving machines. Provider refresh-token invalidation can still require a fresh provider login after recovering an older checkpoint. This is a single-writer deployment, with operator fencing of the previous node; there is no distributed lease or active-active support.
+R2 checkpoints are the durable recovery source for config, provider OAuth/account state and the dedicated Tailscale identity. Local state is a working cache with asynchronous durability: normally up to approximately 15 minutes plus upload duration can be lost, and an outage can extend that indefinitely. Run a successful manual checkpoint after important login/config changes or before moving machines. Provider refresh-token invalidation can still require a fresh provider login after recovering an older checkpoint. Each checkpoint records a digest of every refresh token, so `restore-verify RESTORED --newest NEWEST` names exactly the accounts that need one. This is a single-writer deployment, with operator fencing of the previous node; there is no distributed lease or active-active support.
 
 Checkpoints include only allowlisted runtime state, endpoint intent, repository identity and source revision metadata. GitHub supplies code; Vault supplies client/admin keys, the encryption password and current cloud credentials. Vault/tunnel tokens, local secret files, logs, prior config copies, status files and executable source are excluded. Config and OAuth files themselves contain secrets and remain encrypted in R2. Local capture/restore directories are private plaintext caches, deleted after normal completion. Abandoned generated `checkpoint-*` and `restore-*` directories under `recovery/` expire after 24 hours. Cleanup runs under the operation lock before each backup operation, so active local captures/restores cannot be collected. Unknown/manual rollback directories are preserved. Restic runs without a persistent local metadata cache.
 

package/cliproxy/compose.yaml

@@ -3,6 +3,10 @@ services:
   proxy:
     image: ${CLI_PROXY_IMAGE:?image must be pinned}
     restart: unless-stopped
+    # Docker restarts a proxy that exits; the container watchdog on the
+    # manager restarts one that stays unhealthy.
+    labels:
+      dev.devfleet.watchdog: restart
     user: '${CLIPROXY_UID:-1000}:${CLIPROXY_GID:-1000}'
     working_dir: /state
     command: ['/CLIProxyAPI/CLIProxyAPI', '-config', '/state/config.yaml']