@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/remote-bootstrap.ts

@@ -24,6 +24,7 @@ import { record } from './config';
 import { installedDefinition, type FleetDefinition } from './definition';
 import { cleanCheck, validCheck } from './protocol';
 import { UBUNTU_RELEASES } from './platform/ubuntu-release';
+import { T3_RESTART_MODES } from './platform/t3-service';
 
 export interface BootstrapPayload {
   revision: string;
@@ -33,7 +34,12 @@ export interface BootstrapPayload {
   authorizeSudo?: string;
   /** The fleet configuration every node keeps at ~/.config/dev-fleet/fleet.json. */
   config?: string;
+  /** `fleet-sync --restart-busy`: a worker's T3 restarts for a new release or
+   * policy even while agent turns run. By default it waits for idle.
+   */
+  restartBusy?: boolean;
 }
+
 /** No manager tool installation; Git/source preparation is part of apply approval. */
 export async function prepareBootstrap(
   root: string,
@@ -107,16 +113,21 @@ async function bundleRevision(
     await unlink(path).catch(() => undefined);
   }
 }
+/** `tailnetKey` is a single-use enrolment key for a worker not on the tailnet
+ * yet. It travels on the session's stdin, after the configuration, and only
+ * to a bootstrap script that reads it: never as an argument.
+ */
 export async function bootstrapRemote(
   node: FleetNode,
   payload: BootstrapPayload,
   run: Runner,
-  follow: Follow = {}
+  options: Follow & { tailnetKey?: string } = {}
 ): Promise<NodeReport> {
+  const { tailnetKey, ...follow } = options;
   const clock = follow.clock ?? realClock;
-  const cached = await cachedRevision(node, payload.revision, run, clock);
-  if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
-  const transfer = cached
+  const held = await cachedRevision(node, payload.revision, run, clock);
+  if ('id' in held) return { node: node.id, checks: [held] };
+  const transfer = held.cached
     ? { mode: 'cached', bundle: '' }
     : { mode: 'transfer', bundle: payload.bundle };
   const command = [
@@ -129,14 +140,22 @@ export async function bootstrapRemote(
     node.id,
     transfer.mode,
     UBUNTU_RELEASES.map((entry) => entry.release).join(' '),
+    T3_RESTART_MODES[payload.restartBusy ? 1 : 0],
   ]
     .map(shellQuote)
     .join(' ');
   const checks = await bootstrapSessions(
     node,
     payload,
-    { command, input: bootstrapInput(payload.config, transfer.bundle) },
-    { run, follow: { ...follow, clock } }
+    {
+      command,
+      input: bootstrapInput(
+        payload.config,
+        transfer.bundle,
+        readsTailnetKey(payload.script) ? (tailnetKey ?? '') : null
+      ),
+    },
+    { run, follow: { ...follow, clock, earlier: held.earlier } }
   );
   return { node: node.id, checks };
 }
@@ -198,23 +217,37 @@ function locked(checks: Check[]): boolean {
 
 /** What the bootstrap reads on stdin: one line with the fleet configuration
  * (installed under the remote lock, so overlapping runs never both write it),
- * then the bundle when the worker lacks the revision.
+ * one line with the tailnet enrolment key (empty when there is none; absent
+ * for a script of an earlier revision, which reads no such line), then the
+ * bundle when the worker lacks the revision.
  */
 export function bootstrapInput(
   config: string | undefined,
-  bundle: string
+  bundle: string,
+  tailnetKey: string | null = null
 ): string {
   const line = config ? Buffer.from(config).toString('base64') : '';
-  return `${line}\n${bundle}`;
+  return tailnetKey === null
+    ? `${line}\n${bundle}`
+    : `${line}\n${tailnetKey}\n${bundle}`;
 }
 
+/** A bootstrap script reads the key line since the revision that added it. */
+export function readsTailnetKey(script: string): boolean {
+  return script.includes('read -r tailnet_key');
+}
+
+/** Whether the worker holds a clean checkout of the revision, and when the
+ * operation its record says had finished there started: that one ended
+ * before this run, so its result is never taken for this run's.
+ */
 async function cachedRevision(
   node: FleetNode,
   revision: string,
   run: Runner,
   clock: Clock
-): Promise<boolean | Check> {
-  const command = `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
+): Promise<{ cached: boolean; earlier: number | null } | Check> {
+  const command = `r="$HOME/.local/share/personal/fleet"; sed -n 's/^{.*"started":\\([0-9][0-9]*\\),.*"finished":[0-9].*$/finished=\\1/p' "$r/remote-operation.json" 2>/dev/null | head -n 1; d="$r/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
   const result = await retrying(
     () => run([...sshArgs(node), command], { timeoutMs: 20_000 }),
     clock
@@ -223,7 +256,12 @@ async function cachedRevision(
     const failure = sshFailure(result);
     if (result.reason !== 'exit' || failure.id !== 'ssh:probe') return failure;
   }
-  return result.ok && result.stdout === 'ready';
+  const lines = result.stdout.split('\n');
+  const finished = /^finished=(\d{1,12})$/.exec(lines[0] ?? '');
+  return {
+    cached: result.ok && lines.at(-1) === 'ready',
+    earlier: finished ? Number(finished[1]) : null,
+  };
 }
 
 function parseEvidence(result: {

package/src/remote-operation.ts

@@ -148,16 +148,30 @@ function followable(operation: Operation, revision: string): boolean {
   );
 }
 
-function finishedWith(operation: Operation, revision: string): Check[] | null {
+/** The result of a finished operation of this revision, unless it is the
+ * operation that had already finished before this run opened a session: a
+ * re-sync of the same revision finds the record of the last run still there,
+ * and its result is not this run's.
+ */
+function finishedWith(
+  operation: Operation,
+  revision: string,
+  earlier: number | null | undefined
+): Check[] | null {
   return !operation.held &&
     operation.finished &&
-    operation.revision === revision
+    operation.revision === revision &&
+    !(typeof earlier === 'number' && operation.started === earlier)
     ? operation.result
     : null;
 }
 
 export interface Follow {
   clock?: Clock;
+  /** When the operation that had finished on the worker before this run
+   * began had started there (the worker's clock, in seconds), when known.
+   */
+  earlier?: number | null;
   /** Told the phase and how long the operation has run, at every look. */
   progress?: (phase: string) => void;
 }
@@ -176,7 +190,7 @@ export async function followOperation(
   const deadline = clock.now() + FOLLOW_AT_MOST_MS;
   let operation = await inspectOperation(node, run, clock);
   if (!operation) return null;
-  const done = finishedWith(operation, revision);
+  const done = finishedWith(operation, revision, options.earlier);
   if (done) return [attached(operation, 'finished'), ...done];
   if (!followable(operation, revision)) return null;
   let last: Operation = operation;
@@ -186,7 +200,7 @@ export async function followOperation(
     operation = await inspectOperation(node, run, clock);
     if (!operation) continue;
     last = operation;
-    const result = finishedWith(operation, revision);
+    const result = finishedWith(operation, revision, options.earlier);
     if (result) return [attached(operation, 'followed'), ...result];
     if (!followable(operation, revision)) return null;
   }

package/src/repair-plan.ts

@@ -1,5 +1,5 @@
 import { checkOwner } from './check-owners';
-import { AUTOMATED_REPAIRS, recipe } from './repair-catalog';
+import { AUTOMATED_REPAIRS, REPAIR_RECIPES, recipe } from './repair-catalog';
 import type { RepairRecipe } from './repair-catalog';
 import type { Check, FleetNode, NodeReport, Fingerprint } from './types';
 
@@ -18,22 +18,33 @@ export interface RepairStep extends RepairRecipe {
   manual?: { check: string; reason: string }[];
 }
 
+const UNOWNED =
+  'No repair recipe is registered for this check: follow its remedy.';
+
+/** What repairs a check: the recipe its owner names, or why only a person
+ * can. An `auth:` check is repaired by the login recipe of its name; one
+ * with no such recipe is a person's, for the reason `login:*` gives. */
+export function checkRepair(
+  id: string
+): { recipe: string } | { manual: string } {
+  const owner = checkOwner(id);
+  if (!owner?.repair) return { manual: owner?.manual ?? UNOWNED };
+  if (owner.repair !== 'login:*') return { recipe: owner.repair };
+  const login = `login:${id.slice('auth:'.length)}`;
+  return REPAIR_RECIPES.some((entry) => entry.id === login)
+    ? { recipe: login }
+    : { manual: checkOwner(login)?.manual ?? UNOWNED };
+}
+
 export function repairId(check: Check): string {
-  const owner = checkOwner(check.id);
-  if (!owner?.repair) return 'operator';
-  return owner.repair === 'login:*'
-    ? `login:${check.id.slice('auth:'.length)}`
-    : owner.repair;
+  const repair = checkRepair(check.id);
+  return 'recipe' in repair ? repair.recipe : 'operator';
 }
 
 /** Why a check is left to a person, or null when a recipe repairs it. */
 export function manualReason(check: Check): string | null {
-  const owner = checkOwner(check.id);
-  if (owner?.repair) return null;
-  return (
-    owner?.manual ??
-    'No repair recipe is registered for this check: follow its remedy.'
-  );
+  const repair = checkRepair(check.id);
+  return 'manual' in repair ? repair.manual : null;
 }
 
 /** Every nonpassing check is accounted for; unknown checks remain operator work. */

package/src/report.ts

@@ -6,6 +6,7 @@ export function makeReport(command: string, nodes: NodeReport[]): Report {
     warn: 0,
     fail: 0,
     skip: 0,
+    'not-applicable': 0,
   };
   for (const node of nodes)
     for (const check of node.checks) summary[check.status] += 1;
@@ -46,7 +47,7 @@ export function renderReport(report: Report): string {
     }
   }
   lines.push(
-    `\n${report.summary.pass} pass, ${report.summary.warn} warn, ${report.summary.fail} fail, ${report.summary.skip} skip`
+    `\n${report.summary.pass} pass, ${report.summary.warn} warn, ${report.summary.fail} fail, ${report.summary.skip} skip${report.summary['not-applicable'] ? `, ${report.summary['not-applicable']} not applicable` : ''}`
   );
   return lines.join('\n');
 }

package/src/schedule-check.ts

@@ -1,8 +1,27 @@
+import macos from '../ansible/macos/policy.json';
+import type { ManagerDefinition } from './definition';
 import type { History } from './state/history';
 import type { Check } from './types';
 
-/** How often the manager's schedule runs fleet-sync. */
-export const SCHEDULE_INTERVAL_SECONDS = 1800;
+/** How often the manager's schedule runs fleet-sync by default. */
+export const SCHEDULE_INTERVAL_SECONDS = macos.fleetSync.intervalMinutes * 60;
+
+/** How often the manager's scheduled jobs run: `manager.schedule` in
+ * fleet.json, or the defaults of `ansible/macos/policy.json`.
+ */
+export function scheduleIntervals(
+  manager: Pick<ManagerDefinition, 'schedule'>
+): {
+  syncSeconds: number;
+  watchMinutes: number;
+} {
+  return {
+    syncSeconds:
+      (manager.schedule?.syncMinutes ?? macos.fleetSync.intervalMinutes) * 60,
+    watchMinutes:
+      manager.schedule?.watchMinutes ?? macos.fleetWatch.intervalMinutes,
+  };
+}
 
 /** `manager:schedule`: a schedule that is on must have succeeded within two
  * of its intervals. A schedule that was never turned on is no check at all.

package/src/secrets.ts

@@ -29,6 +29,10 @@ export const OPTIONAL_SECRET_NAMES = [
   // A short-lived, single-use, tagged Tailscale auth key a new worker joins
   // the tailnet with (fleet-node-seed).
   'tailnet-auth-key',
+  // An API credential of the tailnet's control plane (tailnet in fleet.json)
+  // that creates one single-use key per new worker. It stays on the manager:
+  // no worker ever receives it.
+  'tailnet-admin',
 ] as const;
 export type SecretName =
   (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];

package/src/service-auth.ts

@@ -43,68 +43,75 @@ export async function codingAuth(
   ];
 }
 
-/** CLIs with no sign-in of their own to check, and why. Each says where
- * its access is proven instead; none is a silent skip.
+/** Registered agent CLIs with no sign-in of their own to check, and why.
+ * Each is `not-applicable` on the proxy route and says where its access is
+ * proven instead; none is a silent skip.
  */
 export const PROXY_ROUTED: Record<string, string> = {
   gemini:
     'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
   opencode:
     'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
+  grok: 'grok reads the proxy key from its environment and has no grok login; agent:grok:session proves it from this node.',
+  antigravity:
+    'antigravity signs in through the CLI proxy; agent:antigravity:session proves it from this node.',
 };
 
+/** The services a computer signs in to, each checked as `auth:<id>`. */
+const SERVICES: Service[] = [
+  {
+    id: 'railway',
+    command: ['railway', 'whoami', '--json'],
+    action: 'Run railway login on this computer.',
+    valid: (result) => typeof json(result)?.['id'] === 'string',
+  },
+  {
+    id: 'vercel',
+    command: ['vercel', 'whoami'],
+    action: 'Run vercel login on this computer.',
+    valid: (result) =>
+      /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,100}$/.test(result.stdout.trim()),
+  },
+  {
+    id: 'cloudflare',
+    command: ['wrangler', 'whoami', '--json'],
+    action: 'Run wrangler login on this computer.',
+    valid: (result) => json(result)?.['loggedIn'] === true,
+  },
+  {
+    id: 'aws',
+    command: [
+      'aws',
+      'sts',
+      'get-caller-identity',
+      '--output',
+      'json',
+      '--no-cli-pager',
+    ],
+    action:
+      'Select and authenticate the intended AWS profile on this computer.',
+    valid: (result) =>
+      typeof json(result)?.['Account'] === 'string' &&
+      typeof json(result)?.['Arn'] === 'string',
+  },
+  {
+    id: 'npm',
+    command: ['npm', 'whoami', '--registry=https://registry.npmjs.org/'],
+    action: 'Run npm login only if this computer should publish packages.',
+    valid: (result) =>
+      /^[a-z0-9][a-z0-9._-]{0,213}$/.test(result.stdout.trim()),
+  },
+];
+export const SERVICE_IDS = SERVICES.map((service) => service.id);
+
 export async function serviceAuth(run: Runner): Promise<Check[]> {
-  const services: Service[] = [
-    {
-      id: 'railway',
-      command: ['railway', 'whoami', '--json'],
-      action: 'Run railway login on this computer.',
-      valid: (result) => typeof json(result)?.['id'] === 'string',
-    },
-    {
-      id: 'vercel',
-      command: ['vercel', 'whoami'],
-      action: 'Run vercel login on this computer.',
-      valid: (result) =>
-        /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,100}$/.test(result.stdout.trim()),
-    },
-    {
-      id: 'cloudflare',
-      command: ['wrangler', 'whoami', '--json'],
-      action: 'Run wrangler login on this computer.',
-      valid: (result) => json(result)?.['loggedIn'] === true,
-    },
-    {
-      id: 'aws',
-      command: [
-        'aws',
-        'sts',
-        'get-caller-identity',
-        '--output',
-        'json',
-        '--no-cli-pager',
-      ],
-      action:
-        'Select and authenticate the intended AWS profile on this computer.',
-      valid: (result) =>
-        typeof json(result)?.['Account'] === 'string' &&
-        typeof json(result)?.['Arn'] === 'string',
-    },
-    {
-      id: 'npm',
-      command: ['npm', 'whoami', '--registry=https://registry.npmjs.org/'],
-      action: 'Run npm login only if this computer should publish packages.',
-      valid: (result) =>
-        /^[a-z0-9][a-z0-9._-]{0,213}$/.test(result.stdout.trim()),
-    },
-  ];
   const checks: Check[] = [];
-  for (const service of services) checks.push(await serviceCheck(service, run));
+  for (const service of SERVICES) checks.push(await serviceCheck(service, run));
   checks.push(await copilotCheck(run));
   for (const [id, reason] of Object.entries(PROXY_ROUTED))
     checks.push({
       id: `auth:${id}`,
-      status: 'skip',
+      status: 'not-applicable',
       required: false,
       summary: `AUTH_NOT_SUPPORTED_ON_ROUTE: ${reason}`,
     });

package/src/ssh-connection.ts

@@ -0,0 +1,141 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+import type { FleetDefinition, MachineDefinition } from './definition';
+import { stateRoot } from './state/paths';
+
+/** How the manager reaches a worker, written once: every direct SSH session
+ * (src/ssh.ts) and every Ansible run over SSH (fleet-validate's inventory,
+ * with ansible/ansible.cfg) takes its options from here, so the two cannot
+ * drift. Host keys are checked strictly and never learned on first use. A
+ * few attempts and keepalives let a short network blip pass without ending
+ * a long operation.
+ */
+export const SSH_TRANSPORT: readonly (readonly [string, string])[] = [
+  ['BatchMode', 'yes'],
+  ['StrictHostKeyChecking', 'yes'],
+  ['UpdateHostKeys', 'no'],
+  ['ConnectTimeout', '10'],
+  ['ConnectionAttempts', '3'],
+  ['ServerAliveInterval', '15'],
+  ['ServerAliveCountMax', '4'],
+];
+
+/** A direct session also forwards nothing and shares no connection. Ansible
+ * multiplexes its own (ControlPersist in ansible/ansible.cfg).
+ */
+export const SSH_SESSION: readonly (readonly [string, string])[] = [
+  ['ClearAllForwardings', 'yes'],
+  ['PermitLocalCommand', 'no'],
+  ['ControlMaster', 'no'],
+  ['ControlPath', 'none'],
+];
+
+/** The name a worker's host key is pinned under, whatever its address. */
+export const hostKeyAlias = (id: string): string => `devfleet-${id}`;
+
+/** The fleet-owned known_hosts file: a cache rendered from the pinned keys
+ * (declared in a worker record, or in the fleet state). The operator's own
+ * ~/.ssh files are neither read for it nor changed.
+ */
+export function knownHostsPath(env = process.env, home = homedir()): string {
+  return join(dirname(stateRoot(env, home)), 'ssh', 'known_hosts');
+}
+
+/** A worker without an SSH alias is reached from its declared network
+ * block: its tailnet name, user and port.
+ */
+export function declaredConnection(
+  worker: MachineDefinition | undefined
+):
+  | (MachineDefinition & { network: NonNullable<MachineDefinition['network']> })
+  | null {
+  return worker && worker.ssh === undefined && worker.network
+    ? (worker as MachineDefinition & {
+        network: NonNullable<MachineDefinition['network']>;
+      })
+    : null;
+}
+
+/** Host keys pinned in the fleet state, by worker ID. Only the devfleet CLI
+ * reads the state (it registers the reader); launchers and the worker probe
+ * run under Node, which cannot open the state database.
+ */
+type PinReader = () => Record<string, string>;
+let pinReader: PinReader | null = null;
+export function readPinsWith(reader: PinReader | null): void {
+  pinReader = reader;
+}
+
+/** The key each worker reached from declared data is pinned to: the one its
+ * record or fleet.json declares, else the one the fleet state pins.
+ */
+export function pinnedKeys(
+  definition: FleetDefinition,
+  pins: Record<string, string> = pinReader?.() ?? {}
+): Record<string, string> {
+  return Object.fromEntries(
+    definition.workers.flatMap((worker) => {
+      const declared = declaredConnection(worker);
+      const key = declared?.network.hostKey ?? pins[worker.id];
+      return declared && key ? [[worker.id, key]] : [];
+    })
+  );
+}
+
+export function knownHostsText(keys: Record<string, string>): string {
+  return [
+    '# Managed by dev-fleet: rendered from the fleet state and worker records. Do not edit.',
+    ...Object.entries(keys)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([id, key]) => `${hostKeyAlias(id)} ${key}`),
+    '',
+  ].join('\n');
+}
+
+/** Writes the fleet-owned known_hosts when it differs; returns its path. */
+export function renderKnownHosts(
+  definition: FleetDefinition,
+  env = process.env,
+  pins?: Record<string, string>
+): string {
+  const path = knownHostsPath(env);
+  const text = knownHostsText(pinnedKeys(definition, pins));
+  if (existsSync(path) && readFileSync(path, 'utf8') === text) return path;
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeFileSync(`${path}.new`, text, { mode: 0o600 });
+  renameSync(`${path}.new`, path);
+  return path;
+}
+
+const option = ([name, value]: readonly [string, string]): string =>
+  `${name}=${value}`;
+
+/** The options that reach one worker from declared data. */
+export function connectionOptions(
+  worker: MachineDefinition & {
+    network: NonNullable<MachineDefinition['network']>;
+  },
+  knownHosts: string
+): [string, string][] {
+  return [
+    ['HostKeyAlias', hostKeyAlias(worker.id)],
+    ['UserKnownHostsFile', knownHosts],
+    ['GlobalKnownHostsFile', '/dev/null'],
+    ['User', worker.network.user],
+    ['Port', String(worker.network.sshPort)],
+  ];
+}
+
+/** `-o NAME=VALUE` for each option, as ssh takes them. */
+export function sshOptionArgs(
+  options: readonly (readonly [string, string])[]
+): string[] {
+  return options.flatMap((entry) => ['-o', option(entry)]);
+}

package/src/ssh.ts

@@ -1,40 +1,79 @@
 import type { CommandResult, FleetNode, Check } from './types';
+import { fleet } from './fleet-config';
+import {
+  connectionOptions,
+  declaredConnection,
+  renderKnownHosts,
+  SSH_SESSION,
+  SSH_TRANSPORT,
+  sshOptionArgs,
+} from './ssh-connection';
 
 export function shellQuote(value: string): string {
   return `'${value.replaceAll("'", "'\\''")}'`;
 }
 
-export function sshArgs(node: FleetNode): string[] {
+/** The worker fleet.json declares for a roster entry that is reached from
+ * declared data, if it is one. No configuration (a fresh machine) is none.
+ */
+function declaredWorker(node: Pick<FleetNode, 'id' | 'ssh'>) {
+  try {
+    const worker = declaredConnection(
+      fleet.workers.find((entry) => entry.id === node.id)
+    );
+    return worker && worker.network.tailscaleName === node.ssh ? worker : null;
+  } catch {
+    return null;
+  }
+}
+
+/** One SSH session to a node: unattended, strict about the host key,
+ * forwarding nothing. A worker without an alias is reached from its declared
+ * user, port and tailnet name, against the fleet-owned known_hosts.
+ */
+export function sshArgs(node: Pick<FleetNode, 'id' | 'ssh'>): string[] {
+  const declared = declaredWorker(node);
   return [
     'ssh',
     '-T',
     '-a',
-    '-o',
-    'BatchMode=yes',
-    '-o',
-    'StrictHostKeyChecking=yes',
-    '-o',
-    'UpdateHostKeys=no',
-    '-o',
-    'ConnectTimeout=8',
-    '-o',
-    'ConnectionAttempts=1',
-    '-o',
-    'ServerAliveInterval=15',
-    '-o',
-    'ServerAliveCountMax=4',
-    '-o',
-    'ClearAllForwardings=yes',
-    '-o',
-    'PermitLocalCommand=no',
-    '-o',
-    'ControlMaster=no',
-    '-o',
-    'ControlPath=none',
+    ...sshOptionArgs(SSH_TRANSPORT),
+    ...(declared
+      ? sshOptionArgs(connectionOptions(declared, renderKnownHosts(fleet)))
+      : []),
+    ...sshOptionArgs(SSH_SESSION),
     node.ssh,
   ];
 }
 
+/** How a package playbook that addresses a worker over SSH itself
+ * (ansible/ubuntu/worker-account.yml, ansible/ubuntu/reboot.yml) reaches it:
+ * the options every direct session takes, as one `ansible_ssh_common_args`
+ * value, and the account a worker reached from declared data logs in as. The
+ * play names the login itself (root, or the declared account), so the
+ * declared user is not among the options.
+ */
+export function playbookConnection(node: Pick<FleetNode, 'id' | 'ssh'>): {
+  options: string;
+  user: string | null;
+} {
+  const declared = declaredWorker(node);
+  const options = [
+    ...SSH_TRANSPORT,
+    ...(declared
+      ? connectionOptions(declared, renderKnownHosts(fleet)).filter(
+          ([name]) => name !== 'User'
+        )
+      : []),
+  ];
+  return {
+    options: sshOptionArgs(options)
+      .map((arg) => (arg === '-o' ? arg : shellQuote(arg)))
+      .join(' '),
+    user: declared?.network.user ?? null,
+  };
+}
+
 export function remoteCommand(node: FleetNode, request: object): string[] {
   const encoded = Buffer.from(JSON.stringify(request)).toString('base64');
   const command = `cd -- ${shellQuote(node.workspace)} && export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH" && exec ${shellQuote(node.runtime ?? 'node')} --input-type=module - ${shellQuote(encoded)}`;
@@ -48,7 +87,7 @@ const FAILURES = [
     id: 'ssh:host-key',
     summary: 'SSH host key is unknown or changed.',
     action:
-      'Verify the host fingerprint out of band, then update known_hosts manually. Never disable host-key checking.',
+      'The worker presented a host key other than the pinned one. Verify its fingerprint out of band, then pin the new key: devfleet fleet-node-replace --id ID --host-key "TYPE BASE64" --apply --yes, or change hostKey in its worker record. A worker reached through your own SSH alias is checked against your known_hosts instead. Never disable host-key checking.',
   },
   {
     pattern: