@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/cli/fleet-sync-forward.ts

@@ -109,11 +109,15 @@ export function forwardedFlags(
   const switches: [boolean | undefined, string][] = [
     [options.all, '--all'],
     [options.vaultPasswordLogin, '--vault-password-login'],
+    [options.restartBusy, '--restart-busy'],
     [options.dryRun, '--dry-run'],
     [options.apply, '--apply'],
     [options.yes, '--yes'],
     [options.json, '--json'],
     [options.nonInteractive, '--non-interactive'],
+    [options.maintenanceReboot, '--maintenance-reboot'],
+    [options.force, '--force'],
+    [options.takeOver, '--take-over'],
   ];
   for (const [on, flag] of switches) if (on) flags.push(flag);
   return flags;

package/src/cli/fleet-sync-manager.ts

@@ -13,12 +13,16 @@ import { IDES } from '../ide';
 import { enabled, managerChecks } from '../ide/orchestrate';
 import { idesOf } from '../ide/definition';
 import { convergeGitIdentity, gitChecks } from '../git-identity';
-import { ansibleCommand, runAnsible } from '../platform/ansible';
+import { ansibleCommand } from '../platform/ansible';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import macos from '../../ansible/macos/policy.json';
+import { scheduleIntervals } from '../schedule-check';
 import { convergeGithub } from '../github/converge';
 import { PACKAGE_ROOT } from '../package-info';
 import { managerHealth } from '../manager-health';
+import { holdManagerVault } from './fleet-sync-vault';
 import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
+import { homedir, userInfo } from 'node:os';
 import {
   MACOS_MANAGER,
   managerHost,
@@ -49,11 +53,20 @@ export async function convergeManager(
     if (input.apply) installConfig();
     checks.push(
       input.apply
-        ? await holdManagerPower(run, input.workspace, host)
+        ? await convergeManagerPolicy(run, definition, host)
         : await managerPower(run, input.workspace, host)
     );
+    if (input.apply)
+      checks.push(await managerPower(run, input.workspace, host));
     checks.push(await managerBootRecovery(run, input.workspace, host));
     checks.push(await managerDocker(run));
+    // The renewal job is a launch agent: macOS managers only.
+    if (host.platform === 'macOS')
+      checks.push(
+        ...(await attempt('manager:vault-renewal', () =>
+          holdManagerVault(run, { apply: input.apply, root: PACKAGE_ROOT })
+        ))
+      );
     // Each enabled IDE: its settings, pinned version and policy.
     checks.push(
       ...(await managerChecks(enabled(idesOf(definition), IDES), {
@@ -231,19 +244,24 @@ export async function managerIdentity(
 
 /** Workers depend on the manager's services, so it never sleeps on AC power.
  * Measures every setting `ansible/macos/manager.yml` declares, from the same
- * policy file the playbook applies.
+ * policy file the playbook applies. A setting pmset does not list on AC power
+ * is one this Mac does not have (a virtual Mac has no womp): the playbook
+ * skips it, so it is not applicable here either.
  */
 export async function managerPower(
   run: Runner,
   workspace = '.',
   host: ManagerHost = MACOS_MANAGER
 ): Promise<Check> {
-  const drifted = await host.powerDrift(run);
+  const { drifted, notApplicable } = await host.powerDrift(run);
   return drifted.length === 0
     ? {
         id: 'manager:power',
         status: 'pass',
-        summary: 'The manager never sleeps on AC power.',
+        summary:
+          notApplicable.length === 0
+            ? 'The manager never sleeps on AC power.'
+            : `The manager never sleeps on AC power; not applicable on this Mac: ${notApplicable.join(' ')}.`,
       }
     : {
         id: 'manager:power',
@@ -291,41 +309,93 @@ export async function managerBootRecovery(
   };
 }
 
-/** Whether the one grant manager-privilege.yml installs is in place: this
- * user may run `pmset -c` without a password. Lists, never runs.
- */
-async function powerGranted(run: Runner): Promise<boolean> {
-  return (
-    await run(['sudo', '-n', '-l', '/usr/bin/pmset', '-c', 'sleep', '0'], {
-      timeoutMs: 10_000,
-    })
-  ).ok;
-}
+/** The receipt of the manager's own playbook, beside the workers' ones. */
+export const MANAGER_RECEIPT = 'macos-manager';
 
-/** On every applying sync: a drifted power policy is applied with
- * manager.yml when the grant is in place, then measured again. Without the
- * grant the check says how to give it, once.
+/** On every applying sync the manager applies `ansible/macos/manager.yml`
+ * like a worker applies its baseline: in check mode first, then for real,
+ * with a receipt that skips an unchanged playbook while its postconditions
+ * hold, and a full re-apply at least once a day. It never asks for a
+ * password: what needs root goes through the grant manager-privilege.yml
+ * installs once, and the playbook skips it without the grant.
  */
-export async function holdManagerPower(
+export async function convergeManagerPolicy(
   run: Runner,
-  workspace = '.',
+  definition: Pick<FleetDefinition, 'manager'>,
   host: ManagerHost = MACOS_MANAGER,
-  { root = PACKAGE_ROOT, scratch }: { root?: string; scratch?: string } = {}
+  {
+    root = PACKAGE_ROOT,
+    scratch,
+    receipts,
+    uid = userInfo().uid,
+    now,
+  }: {
+    root?: string;
+    scratch?: string;
+    receipts?: string;
+    uid?: number;
+    now?: number;
+  } = {}
 ): Promise<Check> {
-  const before = await managerPower(run, workspace, host);
-  if (
-    before.status === 'pass' ||
-    host.platform !== 'macOS' ||
-    !(await powerGranted(run))
-  )
-    return before;
-  await runAnsible(
+  if (host.platform !== 'macOS')
+    return {
+      id: 'manager:playbook',
+      status: 'not-applicable',
+      summary: 'The manager playbook is for a macOS manager.',
+    };
+  const result = await runAnsibleWithReceipt(
     run,
     join(root, 'ansible/macos/manager.yml'),
-    { input: '{}', timeoutMs: 120_000 },
-    scratch
+    {
+      cwd: join(root, 'ansible/macos'),
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        fleet_watch_minutes: scheduleIntervals(definition.manager).watchMinutes,
+      }),
+    },
+    {
+      name: MANAGER_RECEIPT,
+      sources: [join(root, 'ansible/macos')],
+      checkFirst: true,
+      verify: async () =>
+        (await host.powerDrift(run)).drifted.length === 0 &&
+        (await watchLoaded(run, uid)),
+      ...(receipts ? { directory: receipts } : {}),
+      ...(scratch ? { scratch } : {}),
+      ...(now === undefined ? {} : { now }),
+    }
   );
-  return managerPower(run, workspace, host);
+  if (result.ok)
+    return {
+      id: 'manager:playbook',
+      status: 'pass',
+      summary: result.cached
+        ? 'The manager playbook is unchanged since it was last applied, and its postconditions hold.'
+        : 'The manager playbook was checked, then applied.',
+    };
+  return {
+    id: 'manager:playbook',
+    status: 'fail',
+    summary:
+      result.stage === 'check'
+        ? 'MANAGER_PLAYBOOK_CHECK_FAILED'
+        : 'MANAGER_PLAYBOOK_FAILED',
+    remedy: {
+      action: `Run the playbook in check mode on the manager and read the failing task: ${ansibleCommand().join(' ')} --check --diff ${join(root, 'ansible/macos/manager.yml')}. Nothing in it needs a password; what needs root comes from the one grant (manager-privilege.yml).`,
+    },
+  };
+}
+
+/** Whether launchd holds the fleet-watch job manager.yml installs. Reads only. */
+export async function watchLoaded(run: Runner, uid: number): Promise<boolean> {
+  return (
+    await run(
+      ['/bin/launchctl', 'print', `gui/${uid}/${macos.fleetWatch.label}`],
+      {
+        timeoutMs: 10_000,
+      }
+    )
+  ).ok;
 }
 
 /** The Docker Desktop release `mise.toml` pins, as the engine names itself. */

package/src/cli/fleet-sync-output.ts

@@ -51,17 +51,19 @@ const COLUMNS: { title: string; matches: (id: string) => boolean }[] = [
   },
 ];
 
-const ORDER: Status[] = ['fail', 'warn', 'skip', 'pass'];
+const ORDER: Status[] = ['fail', 'warn', 'skip', 'not-applicable', 'pass'];
 const WORD: Record<Status, string> = {
   fail: 'FAIL',
   warn: 'WARN',
   skip: 'SKIP',
+  'not-applicable': 'N/A',
   pass: 'PASS',
 };
 const SYMBOL: Record<Status, string> = {
   fail: '✖',
   warn: '▲',
   skip: '○',
+  'not-applicable': '–',
   pass: '✔',
 };
 
@@ -104,7 +106,7 @@ export function managerText(
   const lines: string[] = [header(c, report, failing, options)];
   lines.push('', ...grid(c, report.nodes, mark, options.color === true));
   lines.push('', totals(c, counts, options.color === true));
-  for (const status of ['fail', 'warn', 'skip'] as const) {
+  for (const status of ['fail', 'warn', 'skip', 'not-applicable'] as const) {
     const problems = groupProblems(
       all.filter((entry) => entry.check.status === status)
     );
@@ -120,6 +122,8 @@ export function managerText(
     lines.push(
       `${c.dim('report   ')}  ${options.reportPath} ${c.dim('(full JSON; or rerun with --json)')}`
     );
+  if (report.enrolled?.length)
+    lines.push(`${c.dim('enrolled ')}  ${report.enrolled.join(', ')}`);
   if (report.candidates)
     lines.push(
       `${c.dim('tailnet  ')}  ${report.candidates} unmanaged device(s) seen; discovery never enrolls them`
@@ -138,10 +142,11 @@ export function stableUrls(text: string): string {
   return text.replace(TAILNET_URL, 'its private tailnet address');
 }
 
-const SECTION: Record<'fail' | 'warn' | 'skip', string> = {
+const SECTION: Record<'fail' | 'warn' | 'skip' | 'not-applicable', string> = {
   fail: 'Failing — fix these',
   warn: 'Warnings — outside the fleet; sync still succeeds',
   skip: 'Not observable on this hardware',
+  'not-applicable': 'Not applicable to this machine — with the reason',
 };
 
 function header(
@@ -268,22 +273,32 @@ function totals(
   counts: Record<Status, number>,
   color: boolean
 ): string {
-  return ORDER.map((status) => {
-    const text = `${color ? `${SYMBOL[status]} ` : ''}${counts[status]} ${LABEL[status]}`;
-    // A zero count is not a signal: never show red/yellow for "0 failing".
-    return counts[status] ? paint(c, status, text) : c.dim(text);
-  }).join(c.dim('   '));
+  // Not-applicable checks are listed only on machines that have some.
+  return ORDER.filter((status) => status !== 'not-applicable' || counts[status])
+    .map((status) => {
+      const text = `${color ? `${SYMBOL[status]} ` : ''}${counts[status]} ${LABEL[status]}`;
+      // A zero count is not a signal: never show red/yellow for "0 failing".
+      return counts[status] ? paint(c, status, text) : c.dim(text);
+    })
+    .join(c.dim('   '));
 }
 
 const LABEL: Record<Status, string> = {
   fail: 'failing',
   warn: 'warnings',
   skip: 'skipped',
+  'not-applicable': 'not applicable',
   pass: 'passing',
 };
 
 function count(checks: Check[]): Record<Status, number> {
-  const counts: Record<Status, number> = { pass: 0, warn: 0, fail: 0, skip: 0 };
+  const counts: Record<Status, number> = {
+    pass: 0,
+    warn: 0,
+    fail: 0,
+    skip: 0,
+    'not-applicable': 0,
+  };
   for (const check of checks)
     counts[
       blocking(check) || check.status !== 'fail' ? check.status : 'warn'
@@ -295,7 +310,12 @@ function worst(checks: Check[]): Status | null {
   if (!checks.length) return null;
   if (checks.some(blocking)) return 'fail';
   if (
-    checks.some((check) => check.status !== 'pass' && check.status !== 'skip')
+    checks.some(
+      (check) =>
+        check.status !== 'pass' &&
+        check.status !== 'skip' &&
+        check.status !== 'not-applicable'
+    )
   )
     return 'warn';
   return checks.every((check) => check.status === 'skip') ? 'skip' : 'pass';
@@ -306,7 +326,7 @@ function paint(c: ChalkInstance, status: Status, text: string): string {
     ? c.red(text)
     : status === 'warn'
       ? c.yellow(text)
-      : status === 'skip'
+      : status === 'skip' || status === 'not-applicable'
         ? c.dim(text)
         : c.green(text);
 }

package/src/cli/fleet-sync-plan.ts

@@ -1,6 +1,7 @@
 import { join } from 'node:path';
 import type { Standing } from '../state/sync';
 import type { Runner } from '../types';
+import { featureRemovals, workerFeatures } from '../worker-features';
 
 /** A tool whose pin changes between the applied and the selected revision. */
 export interface ToolChange {
@@ -31,6 +32,9 @@ export interface SyncPlan {
   tools: ToolChange[];
   restarts: Restart[];
   nodes: { node: string; from: string | null; to: string; changes: boolean }[];
+  /** What a sync removes from a worker because the fleet configuration
+   * turned one of its features off since it was last applied. */
+  removals: { node: string; removes: string[] }[];
 }
 
 /** The coding agents' CLIs: a new version is used by the next session, and a
@@ -84,16 +88,24 @@ export function toolChanges(
 }
 
 /** What restarts follow from the changes, as the playbooks and the toolchain
- * setup act on them.
+ * setup act on them. A worker's T3 restarts only while it runs no agent turn
+ * (`ansible/ubuntu/tasks/t3-restart.yml`) unless `--restart-busy` asks for it
+ * anyway: only then do running turns end.
  */
-export function restarts(tools: ToolChange[], files: string[]): Restart[] {
+export function restarts(
+  tools: ToolChange[],
+  files: string[],
+  restartBusy = false
+): Restart[] {
   const found: Restart[] = [];
+  const when = restartBusy
+    ? 'at once (--restart-busy): its running agent turns end.'
+    : 'once it runs no agent turn: running turns keep going and the restart is pending until then.';
   if (tools.some((change) => change.tool === 't3'))
     found.push({
       service: 't3',
-      reason:
-        'T3 is upgraded and restarted on every machine; its running agent turns end.',
-      endsRunningTurns: true,
+      reason: `T3 is upgraded on every worker and restarted ${when}`,
+      endsRunningTurns: restartBusy,
     });
   else if (
     files.some((file) =>
@@ -102,9 +114,8 @@ export function restarts(tools: ToolChange[], files: string[]): Restart[] {
   )
     found.push({
       service: 't3',
-      reason:
-        'The managed T3 policy may change: a T3 that has not loaded it is restarted, and its running agent turns end.',
-      endsRunningTurns: true,
+      reason: `The managed T3 policy may change: a T3 that has not loaded it is restarted ${when}`,
+      endsRunningTurns: restartBusy,
     });
   const agents = tools.filter((change) => AGENT_TOOLS.includes(change.tool));
   if (agents.length)
@@ -136,6 +147,11 @@ export async function syncPlan(
     state: Standing;
     nodes: string[];
     sources: string[];
+    /** `--restart-busy`: a T3 restart does not wait for idle. */
+    restartBusy?: boolean;
+    /** The fleet configuration last applied (from the fleet state), when
+     * there is one, and the one this sync applies. */
+    definitions?: { applied: unknown; selected: unknown };
   },
   run: Runner
 ): Promise<SyncPlan> {
@@ -163,16 +179,54 @@ export async function syncPlan(
     revision: { from: input.from, to: input.to },
     known: changes !== null,
     tools: changes?.tools ?? [],
-    restarts: changes ? restarts(changes.tools, changes.files) : [],
+    restarts: changes
+      ? restarts(changes.tools, changes.files, input.restartBusy === true)
+      : [],
     nodes: input.nodes.map((node) => ({
       node,
       from: input.from,
       to: input.to,
       changes: differs,
     })),
+    removals: input.definitions ? removals(input.nodes, input.definitions) : [],
   };
 }
 
+/** Per worker, what turning its features off removes. Nothing is known to
+ * be removed from a worker the applied configuration does not name. */
+function removals(
+  nodes: string[],
+  definitions: { applied: unknown; selected: unknown }
+): SyncPlan['removals'] {
+  const workers = (definition: unknown): Record<string, unknown>[] => {
+    const list =
+      typeof definition === 'object' && definition !== null
+        ? (definition as Record<string, unknown>)['workers']
+        : undefined;
+    return Array.isArray(list)
+      ? list.filter(
+          (entry): entry is Record<string, unknown> =>
+            typeof entry === 'object' && entry !== null
+        )
+      : [];
+  };
+  return nodes.flatMap((node) => {
+    const before = workers(definitions.applied).find(
+      (entry) => entry['id'] === node
+    );
+    if (!before) return [];
+    const network = workers(definitions.selected).find(
+      (entry) => entry['id'] === node
+    )?.['network'] as { user?: unknown } | undefined;
+    const removes = featureRemovals(
+      workerFeatures(definitions.applied, node),
+      workerFeatures(definitions.selected, node),
+      typeof network?.user === 'string' ? network.user : 'the worker user'
+    );
+    return removes.length ? [{ node, removes }] : [];
+  });
+}
+
 export const planSources = (workspace: string, root: string): string[] => [
   workspace,
   join(root, 'source'),
@@ -196,5 +250,7 @@ export function planText(plan: SyncPlan): string {
     );
   for (const restart of plan.restarts)
     lines.push(`  Restarts ${restart.service}: ${restart.reason}`);
+  for (const removal of plan.removals ?? [])
+    lines.push(`  Removes from ${removal.node}: ${removal.removes.join(', ')}`);
   return lines.join('\n');
 }

package/src/cli/fleet-sync-records.ts

@@ -0,0 +1,113 @@
+import type { FleetDefinition, MachineDefinition } from '../definition';
+import { readRosterText } from '../state/roster';
+import { changeWorkers, readWorkerStates } from '../state/workers';
+import { parseConfig } from '../config';
+import type { Check, FleetNode, Runner } from '../types';
+import {
+  lifecycleServices,
+  retireWorker,
+  type NodeLifecycleServices,
+} from './fleet-node-lifecycle';
+
+/** The workers of the fleet as worker records describe them come and go
+ * with the records: a record that appears is a declared worker, enrolled by
+ * the next sync like any other; one that disappears is retired by the same
+ * path as fleet-node-retire. fleet-sync calls this before it reads the
+ * roster. It records which workers came from records, so that only those
+ * are ever retired by themselves; a worker removed from fleet.json by hand
+ * must still be retired first.
+ */
+export async function reconcileRecords(
+  definition: FleetDefinition,
+  roster: string,
+  run: Runner,
+  {
+    apply,
+    services = lifecycleServices,
+  }: {
+    apply: boolean;
+    services?: (roster: string) => NodeLifecycleServices;
+  }
+): Promise<Check[]> {
+  const states = readWorkerStates().workers;
+  const declared = new Set(definition.workers.map((worker) => worker.id));
+  const vanished = Object.entries(states)
+    .filter(([id, entry]) => entry.origin === 'records' && !declared.has(id))
+    .map(([id]) => id);
+  const entries = await rosterEntries(roster);
+  const lifecycle = { ...services(roster), run };
+  const checks: Check[] = [];
+  for (const id of vanished) {
+    const retired = await retireWorker(
+      gone(id, entries.get(id)),
+      entries.get(id),
+      lifecycle,
+      apply
+    );
+    const steps = (retired.data as { nodes: { checks: Check[] }[] }).nodes[0]!
+      .checks;
+    const failed = steps.filter((step) => step.status === 'fail');
+    checks.push(
+      failed.length
+        ? {
+            id: 'enrollment:record',
+            status: 'fail',
+            summary: 'WORKER_RECORD_RETIREMENT_FAILED',
+            remedy: {
+              action: `${id}'s worker record disappeared, and retiring it failed at ${failed.map((step) => step.id).join(', ')}. Follow that remedy, then run fleet-sync again.`,
+            },
+          }
+        : {
+            id: 'enrollment:record',
+            status: apply ? 'pass' : 'skip',
+            summary: `${apply ? '' : 'Planned: '}${id}'s worker record disappeared, so it is retired: ${steps.map((step) => step.id).join(', ')}.`,
+          }
+    );
+  }
+  const fresh = definition.workers.filter(
+    (worker) => worker.origin === 'records' && !states[worker.id]?.origin
+  );
+  if (apply && fresh.length)
+    await changeWorkers(lifecycle.state(), (current) =>
+      Object.fromEntries(
+        fresh.map((worker) => [
+          worker.id,
+          { ...current[worker.id], origin: 'records' as const },
+        ])
+      )
+    );
+  return checks;
+}
+
+async function rosterEntries(path: string): Promise<Map<string, FleetNode>> {
+  const text = await readRosterText(path).catch(() => null);
+  if (text === null) return new Map();
+  try {
+    return new Map(
+      parseConfig(JSON.parse(text) as unknown).nodes.map((node) => [
+        node.id,
+        node,
+      ])
+    );
+  } catch {
+    return new Map();
+  }
+}
+
+/** A worker whose record is gone, as far as retiring it needs: reached from
+ * declared data (so its pin is the fleet state's), under the name the roster
+ * last knew. Its user and ports are never used to retire it.
+ */
+function gone(id: string, entry: FleetNode | undefined): MachineDefinition {
+  return {
+    id,
+    os: 'linux',
+    origin: 'records',
+    network: {
+      tailscaleName: entry?.ssh ?? id,
+      user: 'root',
+      sshPort: 22,
+      t3Port: 3773,
+    },
+  };
+}