@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/cli/fleet-sync.ts

@@ -1,3 +1,4 @@
+import type { ManagerReport } from '../manager-sync';
 import { localCheckoutRevision } from '../machine/local-source';
 import {
   definitionSource,
@@ -7,7 +8,7 @@ import {
 import { managerFailure } from '../manager-failure';
 import { managerText, stdoutColor } from './fleet-sync-output';
 import { managerSchedule, type ScheduleAction } from './fleet-sync-schedule';
-import { enrollManager } from './fleet-sync-enrollment';
+import { enrollManager, type EnrollmentResult } from './fleet-sync-enrollment';
 import { authorizeManager } from './fleet-sync-vault';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
@@ -19,14 +20,14 @@ import {
   type Enrollment,
 } from '../machine/state';
 import { loadEnrollment } from '../state/enrollment';
-import { legacyHome } from '../state/paths';
+import { legacyHome, stateRoot } from '../state/paths';
+import { stateDefinition } from '../state/operator';
 import {
   appendRun,
   summarizeRun,
   type SyncRun,
   type Trigger,
 } from '../state/history';
-import { capScheduleLogs } from './schedule-logs';
 import {
   DIVERGED_REMEDY,
   mutateState,
@@ -57,12 +58,22 @@ import {
 
 export interface FleetSyncOptions {
   revision?: string;
+  /** Converge to the revision applied before the current one. */
+  rollback?: boolean;
   dryRun?: boolean;
   apply?: boolean;
   yes?: boolean;
   json?: boolean;
   nonInteractive?: boolean;
   node?: string[];
+  /** The worker to prove a revision new to the fleet on first, when it is
+   * reachable; overrides `canary` in fleet.json.
+   */
+  canary?: string;
+  /** Opt-in: a worker without a token of its own may be logged in with the
+   * fleet's shared Vault user, typed on its own terminal.
+   */
+  vaultPasswordLogin?: boolean;
   all?: boolean;
   schedule?: ScheduleAction;
   /** `accounts`: only the account-derived configuration, on every node. */
@@ -76,6 +87,23 @@ export interface FleetSyncOptions {
   expectedPlanFingerprint?: string;
   /** Take the manager lease from another manager before it expired. */
   takeOver?: boolean;
+  /** Restart a worker's T3 for a new release or policy even while agent
+   * turns run, which ends them; by default the restart waits for idle.
+   */
+  restartBusy?: boolean;
+  /** Opt-in: reboot each worker whose installed updates wait for a reboot,
+   * one at a time, canary first, only while it runs no agent turn, and
+   * verify it is ready before the next.
+   */
+  maintenanceReboot?: boolean;
+  /** With --maintenance-reboot: reboot even while agent turns run. */
+  force?: boolean;
+  /** Internal: what this run's enrollment did, for the report. */
+  enrollment?: EnrollmentResult;
+  /** Internal: the workers that converged in an earlier run. One that never
+   * did may be rebooted once after its first converge.
+   */
+  readyBefore?: string[];
 }
 export function machineRoot(): string {
   const root = join(homedir(), '.local/share/personal/fleet/manager');
@@ -105,7 +133,6 @@ export async function fleetSync(
     });
   if (options.dryRun || options.schedule) return sync();
   const trigger = runTrigger();
-  if (trigger === 'schedule') capScheduleLogs(join(machineRoot(), 'logs'));
   const startedAt = new Date();
   const outcome = await sync();
   await recordRun(
@@ -177,17 +204,8 @@ export async function managerSync(
 ): Promise<CommandOutcome> {
   await ensurePrivateRoot(root);
   const state = await loadEnrollment(legacyHome() ? root : null);
-  const revision = await selectedRevision(
-    options.revision,
-    state?.approvedRevision,
-    settings,
-    run
-  );
-  if (!isRevision(revision))
-    return actionRequired(
-      'FLEET_REVISION_REQUIRED',
-      'No revision to converge to. Run the published dev-fleet package (it names the commit it was built from), or select a full commit SHA with --revision.'
-    );
+  const revision = await managerRevision(options, state, settings, run);
+  if (typeof revision !== 'string') return revision;
   if (options.schedule)
     return scheduleRequest(options, revision, settings, run);
   const rehomed = await rehome(
@@ -229,6 +247,33 @@ export async function managerSync(
   });
 }
 
+/** The revision the manager converges to, or why there is none. */
+async function managerRevision(
+  options: FleetSyncOptions,
+  state: Enrollment | null,
+  settings: Settings,
+  run: Runner
+): Promise<string | CommandOutcome> {
+  const explicit = requestedRevision(options, state);
+  if (explicit === null)
+    return actionRequired(
+      'FLEET_ROLLBACK_UNAVAILABLE',
+      'Nothing was changed: the fleet state records no revision applied before the approved one. Install the dev-fleet version to go back to and run fleet-sync.'
+    );
+  const revision = await selectedRevision(
+    explicit,
+    state?.approvedRevision,
+    settings,
+    run
+  );
+  if (!isRevision(revision))
+    return actionRequired(
+      'FLEET_REVISION_REQUIRED',
+      'No revision to converge to. Run the published dev-fleet package (it names the commit it was built from), or select a full commit SHA with --revision.'
+    );
+  return revision;
+}
+
 /** The preview: the read-only report of every machine, and the plan of what
  * a sync would change on them.
  */
@@ -253,6 +298,11 @@ async function preview(
         .map((node) => node.node)
         .filter((node): node is string => typeof node === 'string'),
       sources: planSources(settings.workspace, context.root),
+      restartBusy: options.restartBusy === true,
+      definitions: {
+        applied: appliedDefinition(),
+        selected: canonicalDefinition,
+      },
     },
     context.run
   );
@@ -263,6 +313,16 @@ async function preview(
   };
 }
 
+/** The fleet configuration the fleet state records as last applied. */
+function appliedDefinition(): unknown {
+  try {
+    const body = stateDefinition(stateRoot());
+    return body === null ? null : (JSON.parse(body) as unknown);
+  } catch {
+    return null;
+  }
+}
+
 interface ApplyContext {
   settings: Settings;
   run: Runner;
@@ -279,15 +339,28 @@ async function authorizeAndApply(
   context: ApplyContext
 ): Promise<CommandOutcome> {
   const { settings, run, revision } = context;
-  const request = { ...options, revision, apply: true, yes: true };
-  if (!interactive(options)) return applyLocked(root, state, request, context);
-  await enrollManager(settings.config, run);
+  const request = {
+    ...options,
+    revision,
+    apply: true,
+    yes: true,
+    readyBefore: readyBefore(state),
+  };
+  const person = interactive(options);
+  // A run without a person enrolls a worker only when every fact it needs
+  // is verified; it never prompts.
+  const enrollment = await enrollManager(settings.config, run, {
+    interactive: person,
+    joinable: Boolean(canonicalDefinition.tailnet),
+  });
+  if (!person)
+    return applyLocked(root, state, { ...request, enrollment }, context);
   // Needed to hand each node its own Vault token; nodes report any gap.
   await authorizeManager({ interactive: true }, run);
   return applyLocked(
     root,
     state,
-    { ...request, interactiveAuthorization: true },
+    { ...request, enrollment, interactiveAuthorization: true },
     context
   );
 }
@@ -308,6 +381,7 @@ function applyLocked(
     ...(definitionSource().kind === 'file'
       ? { definition: JSON.stringify(canonicalDefinition, null, 2) + '\n' }
       : {}),
+    scoped: Boolean(request.node?.length),
   });
   return withMachineLock(root, async () => {
     record.approve();
@@ -316,7 +390,7 @@ function applyLocked(
         { ...request, rolledOut: state?.appliedRevision === revision },
         settings,
         run,
-        (report) => record.settle(report.ok)
+        (report) => record.settle(report.ok, appliedRevisions(report))
       );
     } finally {
       await record.abandon();
@@ -349,6 +423,12 @@ async function offManager(
   forward: ForwardSpawn | undefined
 ): Promise<CommandOutcome> {
   const definition = canonicalDefinition;
+  // Only the manager's fleet state knows the revision applied before.
+  if (options.rollback)
+    return actionRequired(
+      'FLEET_NOT_MANAGER',
+      `fleet-sync --rollback runs only on the manager (${definition.manager.id}); nothing was changed.`
+    );
   const revision =
     options.revision ??
     packageRevision() ??
@@ -356,7 +436,7 @@ async function offManager(
   if (forwardingDisabled())
     return actionRequired(
       'FLEET_NOT_MANAGER',
-      `fleet-sync runs only on the manager (${definition.manager.id}); nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
+      `fleet-sync runs only on the manager (${definition.manager.id}), which is always a macOS machine; nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
       revision ?? null
     );
   const runScript = revision
@@ -405,11 +485,42 @@ function validate(options: FleetSyncOptions): void {
       '--revision requires an exact lowercase 40-character commit SHA.'
     );
   validateOnly(options);
+  validateRollback(options);
+  if (options.restartBusy && (options.only || options.schedule))
+    throw new FleetUsageError(
+      '--restart-busy applies to one full sync: it takes neither --only nor --schedule.'
+    );
+  validateMaintenance(options);
   if (options.dryRun && (options.apply || options.yes))
     throw new FleetUsageError(
       '--dry-run cannot be combined with apply confirmation.'
     );
 }
+function validateMaintenance(options: FleetSyncOptions): void {
+  if (options.force && !options.maintenanceReboot)
+    throw new FleetUsageError(
+      '--force only goes with --maintenance-reboot: it reboots a worker even while agent turns run.'
+    );
+  if (
+    options.maintenanceReboot &&
+    (options.dryRun || options.schedule || options.only)
+  )
+    throw new FleetUsageError(
+      '--maintenance-reboot reboots workers; it takes neither --dry-run, --schedule nor --only.'
+    );
+}
+function validateRollback(options: FleetSyncOptions): void {
+  if (!options.rollback) return;
+  if (options.revision !== undefined || options.schedule)
+    throw new FleetUsageError(
+      '--rollback chooses the revision itself: drop --revision and --schedule.'
+    );
+  if (options.only)
+    throw new FleetUsageError(
+      '--rollback converges the whole fleet; it takes no --only.'
+    );
+}
+
 function validateOnly(options: FleetSyncOptions): void {
   if (options.only !== undefined && options.only !== 'accounts')
     throw new FleetUsageError('--only takes one scope: accounts.');
@@ -441,6 +552,17 @@ async function onlyAccounts(
   return syncAccounts(options, settings, { revision, run });
 }
 
+/** The workers that converged before this run. A fleet state written before
+ * it recorded each worker's revision counts every worker of a fleet that
+ * was applied as converged.
+ */
+function readyBefore(state: Enrollment | null): string[] {
+  if (state?.workers) return Object.keys(state.workers);
+  return state?.appliedRevision
+    ? canonicalDefinition.workers.map((worker) => worker.id)
+    : [];
+}
+
 function interactive(options: FleetSyncOptions): boolean {
   return (
     ![
@@ -507,6 +629,24 @@ function validateSchedule(schedule: unknown): void {
     throw new FleetUsageError('Unknown manager schedule action.');
 }
 
+/** The revision a request names: --revision, or for --rollback the one
+ * applied before the approved one (null when the fleet state records none).
+ * After an approved revision that never became the applied one (its canary
+ * failed), that is the applied revision itself.
+ */
+function requestedRevision(
+  options: FleetSyncOptions,
+  state: Enrollment | null
+): string | undefined | null {
+  if (!options.rollback) return options.revision;
+  if (
+    state?.appliedRevision &&
+    state.appliedRevision !== state.approvedRevision
+  )
+    return state.appliedRevision;
+  return state?.previousRevision ?? null;
+}
+
 async function selectedRevision(
   explicit: string | undefined,
   approved: string | undefined,
@@ -520,3 +660,14 @@ async function selectedRevision(
     approved
   );
 }
+
+/** The revision each worker holds after this run, for the fleet state: a
+ * worker that converged keeps its progress even when the fleet is not ready.
+ */
+function appliedRevisions(report: ManagerReport): Record<string, string> {
+  return Object.fromEntries(
+    report.nodes
+      .filter((node) => node.role !== 'manager' && node.appliedRevision)
+      .map((node) => [node.node, node.appliedRevision!])
+  );
+}

package/src/cli/fleet-validate.ts

@@ -0,0 +1,277 @@
+import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import {
+  DefinitionError,
+  parseDefinition,
+  sshTarget,
+  type DefinitionProblem,
+  type FleetDefinition,
+} from '../definition';
+import { FleetUsageError } from '../errors';
+import { PACKAGE_ROOT } from '../package-info';
+import { ansibleCommand, ansibleConfig } from '../platform/ansible';
+import { shellQuote } from '../ssh';
+import {
+  connectionOptions,
+  declaredConnection,
+  knownHostsPath,
+  SSH_TRANSPORT,
+  sshOptionArgs,
+} from '../ssh-connection';
+import {
+  expandRecords,
+  runRecordCommand,
+  type RecordCommand,
+} from '../worker-records';
+import { runCommand } from '../runner';
+import type { Check, Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetValidateOptions {
+  file?: string;
+  ansible?: boolean;
+  json?: boolean;
+}
+
+export interface FleetValidateServices {
+  run: Runner;
+  env: NodeJS.ProcessEnv;
+  platform: NodeJS.Platform;
+  /** The package's Ansible playbooks, relative to `root`. */
+  root: string;
+  /** Runs a command `workerRecords` names. */
+  records?: RecordCommand;
+}
+
+const SERVICES: FleetValidateServices = {
+  run: runCommand,
+  env: process.env,
+  platform: process.platform,
+  root: PACKAGE_ROOT,
+};
+
+/** The playbooks a definition must be able to drive, in the package. */
+const PLAYBOOK_DIRECTORIES = ['ansible/ubuntu', 'ansible/macos'];
+
+/** The Ansible inventory a definition describes: the manager, and every
+ * worker in the `workers` group in the order it is declared. Host names are
+ * the fleet ids; a machine is reached as fleet-sync reaches it.
+ */
+export function fleetInventory(
+  definition: FleetDefinition,
+  knownHosts = knownHostsPath()
+): unknown {
+  const workers = Object.fromEntries(
+    definition.workers.map((worker) => {
+      const declared = declaredConnection(worker);
+      // The same options a direct session takes (src/ssh-connection.ts).
+      const options = [
+        ...SSH_TRANSPORT,
+        ...(declared ? connectionOptions(declared, knownHosts) : []),
+      ];
+      return [
+        worker.id,
+        {
+          ansible_host: sshTarget(worker),
+          ...(worker.network
+            ? {
+                ansible_user: worker.network.user,
+                ansible_port: worker.network.sshPort,
+              }
+            : {}),
+          ansible_ssh_common_args: sshOptionArgs(options)
+            .map((arg) => (arg === '-o' ? arg : shellQuote(arg)))
+            .join(' '),
+        },
+      ];
+    })
+  );
+  return {
+    all: {
+      children: {
+        manager: {
+          hosts: {
+            [definition.manager.id]: {
+              ansible_connection: 'local',
+              ansible_user: definition.manager.network.user,
+            },
+          },
+        },
+        workers: { hosts: workers },
+      },
+    },
+  };
+}
+
+/** Reads and judges one fleet.json. Nothing is contacted: with `--ansible`
+ * the pinned Ansible renders the inventory and checks the playbooks' syntax
+ * on this machine.
+ */
+export async function fleetValidate(
+  options: FleetValidateOptions,
+  services: FleetValidateServices = SERVICES
+): Promise<CommandOutcome> {
+  const path = options.file ?? services.env['DEVFLEET_CONFIG'];
+  if (!path)
+    throw new FleetUsageError(
+      'No fleet configuration to validate. Run devfleet from the project that holds your fleet.json, or pass --file PATH.'
+    );
+  let definition: FleetDefinition;
+  try {
+    definition = parseDefinition(
+      expandRecords(
+        await readJson(path),
+        dirname(path),
+        services.records ?? runRecordCommand
+      )
+    );
+  } catch (error) {
+    if (error instanceof DefinitionError) return invalid(path, error.problem);
+    throw error;
+  }
+  const checks: Check[] = [
+    {
+      id: 'definition:schema',
+      status: 'pass',
+      summary: `${path} is a valid fleet definition: a manager and ${definition.workers.length} worker(s).`,
+    },
+  ];
+  if (options.ansible)
+    checks.push(...(await ansibleChecks(definition, services)));
+  const ok = checks.every((check) => check.status !== 'fail');
+  return {
+    exitCode: ok ? 0 : 1,
+    data: { version: 1, command: 'fleet-validate', ok, path, checks },
+    text: checks
+      .map(
+        (check) => `${check.status.toUpperCase()} ${check.id}: ${check.summary}`
+      )
+      .join('\n'),
+  };
+}
+
+async function readJson(path: string): Promise<unknown> {
+  let text: string;
+  try {
+    text = await readFile(path, 'utf8');
+  } catch {
+    throw new DefinitionError({
+      code: 'DEFINITION_UNREADABLE',
+      path: '$',
+      expected: 'a readable file',
+      actual: 'no file',
+    });
+  }
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    throw new DefinitionError({
+      code: 'DEFINITION_UNREADABLE',
+      path: '$',
+      expected: 'JSON',
+      actual: 'text that is not JSON',
+    });
+  }
+}
+
+function invalid(path: string, problem: DefinitionProblem): CommandOutcome {
+  const data = {
+    version: 1,
+    command: 'fleet-validate',
+    ok: false,
+    path,
+    problem,
+  };
+  return {
+    exitCode: 1,
+    data,
+    text: `${problem.code} at ${problem.path}: expected ${problem.expected}, found ${problem.actual}.`,
+  };
+}
+
+/** Renders the inventory, lists it with `ansible-inventory` and checks the
+ * syntax of every playbook against it.
+ */
+async function ansibleChecks(
+  definition: FleetDefinition,
+  services: FleetValidateServices
+): Promise<Check[]> {
+  const directory = await mkdtemp(join(tmpdir(), 'devfleet-validate-'));
+  const inventory = join(directory, 'inventory.json');
+  try {
+    await writeFile(
+      inventory,
+      JSON.stringify(fleetInventory(definition, knownHostsPath(services.env))),
+      { mode: 0o600 }
+    );
+    const playbook = ansibleCommand(services.platform, services.root);
+    const tool = (name: string) => [...playbook.slice(0, -1), name];
+    const listed = await services.run(
+      [...tool('ansible-inventory'), '-i', inventory, '--list'],
+      { timeoutMs: 120_000, env: ansibleConfig(services.root) }
+    );
+    const checks: Check[] = [
+      listed.ok
+        ? {
+            id: 'definition:inventory',
+            status: 'pass',
+            summary: 'Ansible reads the inventory the definition describes.',
+          }
+        : {
+            id: 'definition:inventory',
+            status: 'fail',
+            summary: 'DEFINITION_INVENTORY_REJECTED',
+            remedy: {
+              action:
+                'Run devfleet fleet-validate --ansible on the manager after fleet-sync installed the pinned Ansible; report the failure as an issue if it persists.',
+            },
+          },
+    ];
+    const failed: string[] = [];
+    for (const file of await playbooks(services.root)) {
+      const result = await services.run(
+        [
+          ...playbook,
+          '-i',
+          inventory,
+          '--syntax-check',
+          join(services.root, file),
+        ],
+        { timeoutMs: 120_000, env: ansibleConfig(services.root) }
+      );
+      if (!result.ok) failed.push(file);
+    }
+    checks.push(
+      failed.length
+        ? {
+            id: 'definition:playbooks',
+            status: 'fail',
+            summary: `DEFINITION_PLAYBOOK_SYNTAX: ${failed.join(', ')}`,
+            remedy: {
+              action:
+                'The installed dev-fleet package carries a playbook Ansible cannot parse; report it as an issue.',
+            },
+          }
+        : {
+            id: 'definition:playbooks',
+            status: 'pass',
+            summary: 'Every playbook passes ansible-playbook --syntax-check.',
+          }
+    );
+    return checks;
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+
+/** Every top-level playbook the fleet applies, sorted. */
+async function playbooks(root: string): Promise<string[]> {
+  const found: string[] = [];
+  for (const directory of PLAYBOOK_DIRECTORIES) {
+    const entries = await readdir(join(root, directory)).catch(() => []);
+    for (const entry of entries)
+      if (entry.endsWith('.yml')) found.push(join(directory, entry));
+  }
+  return found.sort();
+}

package/src/cli/fleet-watch.ts

@@ -35,6 +35,10 @@ export interface FleetWatchServices {
   secrets: SecretsProvider;
   send: typeof fetch;
   now: () => Date;
+  /** Parks the workers `worker:idle` warns about; only when fleet.json turns
+   * parking.autoPark on. One check per worker it parked or could not.
+   */
+  autoPark?: (reports: NodeReport[]) => Promise<Check[]>;
 }
 
 /** Compares the fleet's health with the previous run and notifies on
@@ -61,7 +65,8 @@ export async function fleetWatch(
   const previous = services.store
     ? parseMemory(await services.store.read().catch(() => null))
     : null;
-  const observed = observe(await services.status());
+  const reports = await services.status();
+  const observed = observe(reports);
   const { notifications, memory } = transitions(
     previous,
     observed,
@@ -80,6 +85,7 @@ export async function fleetWatch(
         ))
       );
   if (!notify) checks.push(unconfigured('warn'));
+  if (services.autoPark) checks.push(...(await services.autoPark(reports)));
   checks.push(await heartbeat(services.store, JSON.stringify(memory)));
   return outcome(checks, notifications);
 }