@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/cli/fleet-sync-enrollment.ts

@@ -1,21 +1,58 @@
 import { mkdir, rm } from 'node:fs/promises';
 import { loadRoster } from './roster';
-import { enrolledRoster } from '../definition';
+import { enrolledRoster, sshTarget, type FleetDefinition } from '../definition';
 import { FleetUsageError } from '../errors';
 import { dirname } from 'node:path';
-import { fleet as definition } from '../fleet-config';
+import { fleet } from '../fleet-config';
 import { discoverTailnet } from '../tailscale';
 import { matchComputers } from '../tailnet-matching';
 import { sshArgs, sshFailure } from '../ssh';
 import { ensurePrivateRoot } from '../machine/state';
-import type { FleetNode, Runner } from '../types';
+import { createWorkerAccount } from '../worker-account';
+import { packagePlaybook, type PackagePlaybook } from '../platform/ansible';
+import type { Check, FleetNode, Runner } from '../types';
 import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+import { readParked } from '../state/workers';
 
-/** Interactive enrollment/resume only. Existing paths and explicit identities
- * are preserved. Tailnet candidates never enter the declared membership.
+/** What an enrollment did: the workers whose roster entry it added or
+ * completed, and the checks that say why another one was not enrolled (or
+ * what was done to reach it).
  */
-export async function enrollManager(path: string, run: Runner): Promise<void> {
-  const canonical = definition;
+export interface EnrollmentResult {
+  enrolled: string[];
+  checks: { node: string; check: Check }[];
+}
+
+export interface EnrollmentOptions {
+  /** A person is at the terminal and sees the list of what is enrolled. */
+  interactive: boolean;
+  /** A worker not on the tailnet yet joins it during this run: the fleet
+   * declares a tailnet control plane that issues single-use keys.
+   */
+  joinable?: boolean;
+  definition?: FleetDefinition;
+  /** Applies a playbook of the package from the manager. */
+  playbook?: PackagePlaybook;
+  /** When each parked worker was parked; default the fleet state's. */
+  parked?: Record<string, string>;
+}
+
+/** Enrolls declared workers, and resumes earlier enrollments. Existing paths
+ * and explicit identities are preserved. Tailnet candidates never enter the
+ * declared membership. An interactive run enrolls what a fresh
+ * host-key-verified SSH session reaches. A run without a person enrolls a
+ * worker only when every fact is verified: its pinned host key matched (the
+ * session is strict), `sudo -n` works, and exactly one tailnet device matches
+ * it, or none when this run joins it to the tailnet itself. Anything
+ * ambiguous stays out, with the code that says why; nothing prompts.
+ */
+export async function enrollManager(
+  path: string,
+  run: Runner,
+  options: EnrollmentOptions = { interactive: true }
+): Promise<EnrollmentResult> {
+  const canonical = options.definition ?? fleet;
+  const result: EnrollmentResult = { enrolled: [], checks: [] };
   const original = await existingEnrollment(path);
   const saved =
     original === null
@@ -23,54 +60,178 @@ export async function enrollManager(path: string, run: Runner): Promise<void> {
       : enrolledRoster(canonical, await loadRoster(path)).nodes;
   const nodes: FleetNode[] = canonical.workers.map((node) => ({
     id: node.id,
-    ssh: node.ssh,
+    ssh: sshTarget(node),
     workspace: '/',
     required: ['t3', 'toolchain', 'workstation'],
   }));
+  // A parked worker is off on purpose: it is enrolled once it is resumed.
+  const parked = options.parked ?? readParked();
   const pending = nodes.filter(
-    (node) => !saved.find((entry) => entry.id === node.id)?.tailscaleId
+    (node) =>
+      !parked[node.id] &&
+      !saved.find((entry) => entry.id === node.id)?.tailscaleId
   );
-  if (!pending.length) return;
+  if (!pending.length) return result;
   const tailnet = await discoverTailnet(run);
   const matches = await matchComputers(nodes, tailnet.computers, run);
   const enrolled: FleetNode[] = [];
   for (const node of pending) {
-    const peers = tailnet.computers.filter((peer) =>
-      matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
-    );
-    if (
-      ambiguous(
-        peers.map((peer) => peer.id),
-        matches
-      )
-    )
-      continue;
-    const connection = await inspectConnection(node, peers[0]?.id, run);
+    // A worker already in the roster reports its own state in the run.
+    const known = saved.some((entry) => entry.id === node.id);
+    const connection = await enrollOne(node, {
+      run,
+      options,
+      peers: tailnet.computers
+        .filter((peer) =>
+          matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
+        )
+        .map((peer) => peer.id),
+      matches,
+      user: canonical.workers.find((worker) => worker.id === node.id)?.network
+        ?.user,
+      note: (check) => {
+        if (!known) result.checks.push({ node: node.id, check });
+      },
+    });
     if (connection) enrolled.push(connection);
   }
   const merged = mergeEnrollment(saved, enrolled);
-  if (JSON.stringify(merged) === JSON.stringify(saved)) return;
+  if (JSON.stringify(merged) === JSON.stringify(saved)) return result;
   // Only canonical nodes reached over host-key-verified SSH get here, so
   // enrollment needs no extra confirmation; the list is still shown.
-  process.stderr.write(
-    `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => `  ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
-  );
+  if (options.interactive)
+    process.stderr.write(
+      `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => `  ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
+    );
   await persistEnrollment(path, original, merged);
+  result.enrolled = merged
+    .filter(
+      (node) =>
+        JSON.stringify(node) !==
+        JSON.stringify(saved.find((entry) => entry.id === node.id))
+    )
+    .map((node) => node.id);
+  return result;
 }
 
+/** One worker's connection, when every fact this run needs is verified. */
+async function enrollOne(
+  node: FleetNode,
+  context: {
+    run: Runner;
+    options: EnrollmentOptions;
+    /** The tailnet devices that match this worker. */
+    peers: string[];
+    matches: Map<string, FleetNode[]>;
+    user: string | undefined;
+    note: (check: Check) => void;
+  }
+): Promise<FleetNode | null> {
+  const { run, options, peers, note } = context;
+  if (ambiguous(peers, context.matches)) {
+    if (!options.interactive) note(IDENTITY_CONFLICT);
+    return null;
+  }
+  if (!options.interactive && !peers.length && !options.joinable) {
+    note(NOT_ON_TAILNET);
+    return null;
+  }
+  const connection = await inspectConnection(node, peers[0], {
+    run,
+    options,
+    user: context.user,
+    note,
+    quiet: !options.interactive,
+  });
+  if (!connection || options.interactive) return connection;
+  const privilege = await passwordlessSudo(connection, run);
+  if (privilege) note(privilege);
+  return privilege ? null : connection;
+}
+
+const IDENTITY_CONFLICT: Check = {
+  id: 'tailscale:identity',
+  status: 'fail',
+  summary: 'TAILSCALE_IDENTITY_CONFLICT',
+  remedy: {
+    action:
+      'More than one tailnet device matches this worker, so it was not enrolled. Remove the stale device from the tailnet, then rerun fleet-sync.',
+  },
+};
+
+const NOT_ON_TAILNET: Check = {
+  id: 'tailscale:enrolled-identity',
+  status: 'fail',
+  summary: 'TAILSCALE_ENROLLMENT_REQUIRED',
+  remedy: {
+    action:
+      'No tailnet device matches this worker and fleet.json declares no tailnet control plane to join it with, so a run without a person did not enroll it. Declare tailnet with the secret tailnet-admin, join the machine yourself, or run fleet-sync once in a terminal.',
+  },
+};
+
+/** A run without a person never prompts for sudo: the account has it, or the
+ * worker is not enrolled.
+ */
+async function passwordlessSudo(
+  node: FleetNode,
+  run: Runner
+): Promise<Check | null> {
+  const result = await run([...sshArgs(node), 'sudo -n true'], {
+    timeoutMs: 15_000,
+  });
+  if (result.ok) return null;
+  if (result.reason !== 'exit' || !/sudo:/i.test(result.stderr))
+    return sshFailure(result);
+  return {
+    id: 'bootstrap:privilege',
+    status: 'fail',
+    summary: 'SUDO_AUTHORIZATION_REQUIRED',
+    remedy: {
+      action:
+        'The worker account has no passwordless sudo, so a run without a person did not enroll it. Grant it (a fleet-node-seed seed does), or run fleet-sync once in a terminal to authorize it.',
+    },
+  };
+}
+
+/** The account's home over a fresh strict session. A machine whose image
+ * lets only root in refuses the declared account: it is created once over
+ * the root login, and the session is opened again.
+ */
 async function inspectConnection(
   node: FleetNode,
   peerId: string | undefined,
-  run: Runner
+  context: {
+    run: Runner;
+    options: EnrollmentOptions;
+    user: string | undefined;
+    note: (check: Check) => void;
+    quiet: boolean;
+  }
 ): Promise<FleetNode | null> {
-  const home = await run([...sshArgs(node), 'printf "%s" "$HOME"'], {
-    timeoutMs: 20_000,
-  });
+  const { run } = context;
+  const probe = () =>
+    run([...sshArgs(node), 'printf "%s" "$HOME"'], { timeoutMs: 20_000 });
+  let home = await probe();
+  if (!home.ok && sshFailure(home).id === 'ssh:auth' && context.user) {
+    const account = await createWorkerAccount(
+      { id: node.id, ssh: node.ssh, login: 'root' },
+      context.user,
+      context.options.playbook ?? packagePlaybook(run)
+    );
+    // Only an account that was created is reported here: a root login that
+    // is refused as well stays the SSH failure below.
+    if (account.status === 'pass') {
+      context.note(account);
+      home = await probe();
+    }
+  }
   if (!home.ok) {
     const check = sshFailure(home);
-    process.stderr.write(
-      `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
-    );
+    if (context.quiet) context.note(check);
+    else
+      process.stderr.write(
+        `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
+      );
     return null;
   }
   if (!/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(home.stdout)) return null;

package/src/cli/fleet-sync-forward.ts

@@ -104,14 +104,20 @@ export function forwardedFlags(
 ): string[] {
   const flags = revision ? ['--revision', revision] : [];
   for (const node of options.node ?? []) flags.push('--node', node);
+  if (options.canary) flags.push('--canary', options.canary);
   if (options.schedule) flags.push('--schedule', options.schedule);
   const switches: [boolean | undefined, string][] = [
     [options.all, '--all'],
+    [options.vaultPasswordLogin, '--vault-password-login'],
+    [options.restartBusy, '--restart-busy'],
     [options.dryRun, '--dry-run'],
     [options.apply, '--apply'],
     [options.yes, '--yes'],
     [options.json, '--json'],
     [options.nonInteractive, '--non-interactive'],
+    [options.maintenanceReboot, '--maintenance-reboot'],
+    [options.force, '--force'],
+    [options.takeOver, '--take-over'],
   ];
   for (const [on, flag] of switches) if (on) flags.push(flag);
   return flags;

package/src/cli/fleet-sync-manager.ts

@@ -14,11 +14,15 @@ import { enabled, managerChecks } from '../ide/orchestrate';
 import { idesOf } from '../ide/definition';
 import { convergeGitIdentity, gitChecks } from '../git-identity';
 import { ansibleCommand } from '../platform/ansible';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import macos from '../../ansible/macos/policy.json';
+import { scheduleIntervals } from '../schedule-check';
 import { convergeGithub } from '../github/converge';
 import { PACKAGE_ROOT } from '../package-info';
 import { managerHealth } from '../manager-health';
+import { holdManagerVault } from './fleet-sync-vault';
 import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
+import { homedir, userInfo } from 'node:os';
 import {
   MACOS_MANAGER,
   managerHost,
@@ -47,9 +51,22 @@ export async function convergeManager(
   if (identity.status === 'pass') {
     // Launchers and scheduled jobs on the manager read the installed copy.
     if (input.apply) installConfig();
-    checks.push(await managerPower(run, input.workspace, host));
+    checks.push(
+      input.apply
+        ? await convergeManagerPolicy(run, definition, host)
+        : await managerPower(run, input.workspace, host)
+    );
+    if (input.apply)
+      checks.push(await managerPower(run, input.workspace, host));
     checks.push(await managerBootRecovery(run, input.workspace, host));
     checks.push(await managerDocker(run));
+    // The renewal job is a launch agent: macOS managers only.
+    if (host.platform === 'macOS')
+      checks.push(
+        ...(await attempt('manager:vault-renewal', () =>
+          holdManagerVault(run, { apply: input.apply, root: PACKAGE_ROOT })
+        ))
+      );
     // Each enabled IDE: its settings, pinned version and policy.
     checks.push(
       ...(await managerChecks(enabled(idesOf(definition), IDES), {
@@ -202,7 +219,8 @@ export async function managerAgentProfiles(
 export async function managerIdentity(
   id: string,
   run: Runner,
-  host: ManagerHost = MACOS_MANAGER
+  host: ManagerHost = MACOS_MANAGER,
+  platform: NodeJS.Platform = process.platform
 ): Promise<Check> {
   const local = (await host.localHostName(run)) ?? '';
   return local === id
@@ -216,26 +234,34 @@ export async function managerIdentity(
         status: 'fail',
         summary: 'FLEET_NOT_MANAGER',
         remedy: {
-          action: `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`,
+          action:
+            platform === 'darwin'
+              ? `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`
+              : `This machine runs ${platform}, and the manager is always a macOS machine. Run fleet-sync on the manager (${id}), or from a worker that can SSH to it.`,
         },
       };
 }
 
 /** Workers depend on the manager's services, so it never sleeps on AC power.
  * Measures every setting `ansible/macos/manager.yml` declares, from the same
- * policy file the playbook applies.
+ * policy file the playbook applies. A setting pmset does not list on AC power
+ * is one this Mac does not have (a virtual Mac has no womp): the playbook
+ * skips it, so it is not applicable here either.
  */
 export async function managerPower(
   run: Runner,
   workspace = '.',
   host: ManagerHost = MACOS_MANAGER
 ): Promise<Check> {
-  const drifted = await host.powerDrift(run);
+  const { drifted, notApplicable } = await host.powerDrift(run);
   return drifted.length === 0
     ? {
         id: 'manager:power',
         status: 'pass',
-        summary: 'The manager never sleeps on AC power.',
+        summary:
+          notApplicable.length === 0
+            ? 'The manager never sleeps on AC power.'
+            : `The manager never sleeps on AC power; not applicable on this Mac: ${notApplicable.join(' ')}.`,
       }
     : {
         id: 'manager:power',
@@ -244,7 +270,7 @@ export async function managerPower(
           ? 'MANAGER_SLEEPS_ON_POWER'
           : `MANAGER_POWER_POLICY_DRIFT ${drifted.join(' ')}`,
         remedy: {
-          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once, as an administrator: cd ${workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`,
+          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Grant the manager its power settings once, as an administrator; every fleet-sync then applies the policy with no prompt: cd ${workspace} && ${ansibleCommand().join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml`,
         },
       };
 }
@@ -283,6 +309,95 @@ export async function managerBootRecovery(
   };
 }
 
+/** The receipt of the manager's own playbook, beside the workers' ones. */
+export const MANAGER_RECEIPT = 'macos-manager';
+
+/** On every applying sync the manager applies `ansible/macos/manager.yml`
+ * like a worker applies its baseline: in check mode first, then for real,
+ * with a receipt that skips an unchanged playbook while its postconditions
+ * hold, and a full re-apply at least once a day. It never asks for a
+ * password: what needs root goes through the grant manager-privilege.yml
+ * installs once, and the playbook skips it without the grant.
+ */
+export async function convergeManagerPolicy(
+  run: Runner,
+  definition: Pick<FleetDefinition, 'manager'>,
+  host: ManagerHost = MACOS_MANAGER,
+  {
+    root = PACKAGE_ROOT,
+    scratch,
+    receipts,
+    uid = userInfo().uid,
+    now,
+  }: {
+    root?: string;
+    scratch?: string;
+    receipts?: string;
+    uid?: number;
+    now?: number;
+  } = {}
+): Promise<Check> {
+  if (host.platform !== 'macOS')
+    return {
+      id: 'manager:playbook',
+      status: 'not-applicable',
+      summary: 'The manager playbook is for a macOS manager.',
+    };
+  const result = await runAnsibleWithReceipt(
+    run,
+    join(root, 'ansible/macos/manager.yml'),
+    {
+      cwd: join(root, 'ansible/macos'),
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        fleet_watch_minutes: scheduleIntervals(definition.manager).watchMinutes,
+      }),
+    },
+    {
+      name: MANAGER_RECEIPT,
+      sources: [join(root, 'ansible/macos')],
+      checkFirst: true,
+      verify: async () =>
+        (await host.powerDrift(run)).drifted.length === 0 &&
+        (await watchLoaded(run, uid)),
+      ...(receipts ? { directory: receipts } : {}),
+      ...(scratch ? { scratch } : {}),
+      ...(now === undefined ? {} : { now }),
+    }
+  );
+  if (result.ok)
+    return {
+      id: 'manager:playbook',
+      status: 'pass',
+      summary: result.cached
+        ? 'The manager playbook is unchanged since it was last applied, and its postconditions hold.'
+        : 'The manager playbook was checked, then applied.',
+    };
+  return {
+    id: 'manager:playbook',
+    status: 'fail',
+    summary:
+      result.stage === 'check'
+        ? 'MANAGER_PLAYBOOK_CHECK_FAILED'
+        : 'MANAGER_PLAYBOOK_FAILED',
+    remedy: {
+      action: `Run the playbook in check mode on the manager and read the failing task: ${ansibleCommand().join(' ')} --check --diff ${join(root, 'ansible/macos/manager.yml')}. Nothing in it needs a password; what needs root comes from the one grant (manager-privilege.yml).`,
+    },
+  };
+}
+
+/** Whether launchd holds the fleet-watch job manager.yml installs. Reads only. */
+export async function watchLoaded(run: Runner, uid: number): Promise<boolean> {
+  return (
+    await run(
+      ['/bin/launchctl', 'print', `gui/${uid}/${macos.fleetWatch.label}`],
+      {
+        timeoutMs: 10_000,
+      }
+    )
+  ).ok;
+}
+
 /** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
 export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
   const { vars } = Bun.TOML.parse(

package/src/cli/fleet-sync-output.ts

@@ -51,17 +51,19 @@ const COLUMNS: { title: string; matches: (id: string) => boolean }[] = [
   },
 ];
 
-const ORDER: Status[] = ['fail', 'warn', 'skip', 'pass'];
+const ORDER: Status[] = ['fail', 'warn', 'skip', 'not-applicable', 'pass'];
 const WORD: Record<Status, string> = {
   fail: 'FAIL',
   warn: 'WARN',
   skip: 'SKIP',
+  'not-applicable': 'N/A',
   pass: 'PASS',
 };
 const SYMBOL: Record<Status, string> = {
   fail: '✖',
   warn: '▲',
   skip: '○',
+  'not-applicable': '–',
   pass: '✔',
 };
 
@@ -104,7 +106,7 @@ export function managerText(
   const lines: string[] = [header(c, report, failing, options)];
   lines.push('', ...grid(c, report.nodes, mark, options.color === true));
   lines.push('', totals(c, counts, options.color === true));
-  for (const status of ['fail', 'warn', 'skip'] as const) {
+  for (const status of ['fail', 'warn', 'skip', 'not-applicable'] as const) {
     const problems = groupProblems(
       all.filter((entry) => entry.check.status === status)
     );
@@ -120,6 +122,8 @@ export function managerText(
     lines.push(
       `${c.dim('report   ')}  ${options.reportPath} ${c.dim('(full JSON; or rerun with --json)')}`
     );
+  if (report.enrolled?.length)
+    lines.push(`${c.dim('enrolled ')}  ${report.enrolled.join(', ')}`);
   if (report.candidates)
     lines.push(
       `${c.dim('tailnet  ')}  ${report.candidates} unmanaged device(s) seen; discovery never enrolls them`
@@ -138,10 +142,11 @@ export function stableUrls(text: string): string {
   return text.replace(TAILNET_URL, 'its private tailnet address');
 }
 
-const SECTION: Record<'fail' | 'warn' | 'skip', string> = {
+const SECTION: Record<'fail' | 'warn' | 'skip' | 'not-applicable', string> = {
   fail: 'Failing — fix these',
   warn: 'Warnings — outside the fleet; sync still succeeds',
   skip: 'Not observable on this hardware',
+  'not-applicable': 'Not applicable to this machine — with the reason',
 };
 
 function header(
@@ -268,22 +273,32 @@ function totals(
   counts: Record<Status, number>,
   color: boolean
 ): string {
-  return ORDER.map((status) => {
-    const text = `${color ? `${SYMBOL[status]} ` : ''}${counts[status]} ${LABEL[status]}`;
-    // A zero count is not a signal: never show red/yellow for "0 failing".
-    return counts[status] ? paint(c, status, text) : c.dim(text);
-  }).join(c.dim('   '));
+  // Not-applicable checks are listed only on machines that have some.
+  return ORDER.filter((status) => status !== 'not-applicable' || counts[status])
+    .map((status) => {
+      const text = `${color ? `${SYMBOL[status]} ` : ''}${counts[status]} ${LABEL[status]}`;
+      // A zero count is not a signal: never show red/yellow for "0 failing".
+      return counts[status] ? paint(c, status, text) : c.dim(text);
+    })
+    .join(c.dim('   '));
 }
 
 const LABEL: Record<Status, string> = {
   fail: 'failing',
   warn: 'warnings',
   skip: 'skipped',
+  'not-applicable': 'not applicable',
   pass: 'passing',
 };
 
 function count(checks: Check[]): Record<Status, number> {
-  const counts: Record<Status, number> = { pass: 0, warn: 0, fail: 0, skip: 0 };
+  const counts: Record<Status, number> = {
+    pass: 0,
+    warn: 0,
+    fail: 0,
+    skip: 0,
+    'not-applicable': 0,
+  };
   for (const check of checks)
     counts[
       blocking(check) || check.status !== 'fail' ? check.status : 'warn'
@@ -295,7 +310,12 @@ function worst(checks: Check[]): Status | null {
   if (!checks.length) return null;
   if (checks.some(blocking)) return 'fail';
   if (
-    checks.some((check) => check.status !== 'pass' && check.status !== 'skip')
+    checks.some(
+      (check) =>
+        check.status !== 'pass' &&
+        check.status !== 'skip' &&
+        check.status !== 'not-applicable'
+    )
   )
     return 'warn';
   return checks.every((check) => check.status === 'skip') ? 'skip' : 'pass';
@@ -306,7 +326,7 @@ function paint(c: ChalkInstance, status: Status, text: string): string {
     ? c.red(text)
     : status === 'warn'
       ? c.yellow(text)
-      : status === 'skip'
+      : status === 'skip' || status === 'not-applicable'
         ? c.dim(text)
         : c.green(text);
 }