@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/cli/fleet-sync.ts

@@ -8,7 +8,7 @@ import {
 import { managerFailure } from '../manager-failure';
 import { managerText, stdoutColor } from './fleet-sync-output';
 import { managerSchedule, type ScheduleAction } from './fleet-sync-schedule';
-import { enrollManager } from './fleet-sync-enrollment';
+import { enrollManager, type EnrollmentResult } from './fleet-sync-enrollment';
 import { authorizeManager } from './fleet-sync-vault';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
@@ -20,14 +20,14 @@ import {
   type Enrollment,
 } from '../machine/state';
 import { loadEnrollment } from '../state/enrollment';
-import { legacyHome } from '../state/paths';
+import { legacyHome, stateRoot } from '../state/paths';
+import { stateDefinition } from '../state/operator';
 import {
   appendRun,
   summarizeRun,
   type SyncRun,
   type Trigger,
 } from '../state/history';
-import { capScheduleLogs } from './schedule-logs';
 import {
   DIVERGED_REMEDY,
   mutateState,
@@ -58,6 +58,8 @@ import {
 
 export interface FleetSyncOptions {
   revision?: string;
+  /** Converge to the revision applied before the current one. */
+  rollback?: boolean;
   dryRun?: boolean;
   apply?: boolean;
   yes?: boolean;
@@ -85,6 +87,23 @@ export interface FleetSyncOptions {
   expectedPlanFingerprint?: string;
   /** Take the manager lease from another manager before it expired. */
   takeOver?: boolean;
+  /** Restart a worker's T3 for a new release or policy even while agent
+   * turns run, which ends them; by default the restart waits for idle.
+   */
+  restartBusy?: boolean;
+  /** Opt-in: reboot each worker whose installed updates wait for a reboot,
+   * one at a time, canary first, only while it runs no agent turn, and
+   * verify it is ready before the next.
+   */
+  maintenanceReboot?: boolean;
+  /** With --maintenance-reboot: reboot even while agent turns run. */
+  force?: boolean;
+  /** Internal: what this run's enrollment did, for the report. */
+  enrollment?: EnrollmentResult;
+  /** Internal: the workers that converged in an earlier run. One that never
+   * did may be rebooted once after its first converge.
+   */
+  readyBefore?: string[];
 }
 export function machineRoot(): string {
   const root = join(homedir(), '.local/share/personal/fleet/manager');
@@ -114,7 +133,6 @@ export async function fleetSync(
     });
   if (options.dryRun || options.schedule) return sync();
   const trigger = runTrigger();
-  if (trigger === 'schedule') capScheduleLogs(join(machineRoot(), 'logs'));
   const startedAt = new Date();
   const outcome = await sync();
   await recordRun(
@@ -186,17 +204,8 @@ export async function managerSync(
 ): Promise<CommandOutcome> {
   await ensurePrivateRoot(root);
   const state = await loadEnrollment(legacyHome() ? root : null);
-  const revision = await selectedRevision(
-    options.revision,
-    state?.approvedRevision,
-    settings,
-    run
-  );
-  if (!isRevision(revision))
-    return actionRequired(
-      'FLEET_REVISION_REQUIRED',
-      'No revision to converge to. Run the published dev-fleet package (it names the commit it was built from), or select a full commit SHA with --revision.'
-    );
+  const revision = await managerRevision(options, state, settings, run);
+  if (typeof revision !== 'string') return revision;
   if (options.schedule)
     return scheduleRequest(options, revision, settings, run);
   const rehomed = await rehome(
@@ -238,6 +247,33 @@ export async function managerSync(
   });
 }
 
+/** The revision the manager converges to, or why there is none. */
+async function managerRevision(
+  options: FleetSyncOptions,
+  state: Enrollment | null,
+  settings: Settings,
+  run: Runner
+): Promise<string | CommandOutcome> {
+  const explicit = requestedRevision(options, state);
+  if (explicit === null)
+    return actionRequired(
+      'FLEET_ROLLBACK_UNAVAILABLE',
+      'Nothing was changed: the fleet state records no revision applied before the approved one. Install the dev-fleet version to go back to and run fleet-sync.'
+    );
+  const revision = await selectedRevision(
+    explicit,
+    state?.approvedRevision,
+    settings,
+    run
+  );
+  if (!isRevision(revision))
+    return actionRequired(
+      'FLEET_REVISION_REQUIRED',
+      'No revision to converge to. Run the published dev-fleet package (it names the commit it was built from), or select a full commit SHA with --revision.'
+    );
+  return revision;
+}
+
 /** The preview: the read-only report of every machine, and the plan of what
  * a sync would change on them.
  */
@@ -262,6 +298,11 @@ async function preview(
         .map((node) => node.node)
         .filter((node): node is string => typeof node === 'string'),
       sources: planSources(settings.workspace, context.root),
+      restartBusy: options.restartBusy === true,
+      definitions: {
+        applied: appliedDefinition(),
+        selected: canonicalDefinition,
+      },
     },
     context.run
   );
@@ -272,6 +313,16 @@ async function preview(
   };
 }
 
+/** The fleet configuration the fleet state records as last applied. */
+function appliedDefinition(): unknown {
+  try {
+    const body = stateDefinition(stateRoot());
+    return body === null ? null : (JSON.parse(body) as unknown);
+  } catch {
+    return null;
+  }
+}
+
 interface ApplyContext {
   settings: Settings;
   run: Runner;
@@ -288,15 +339,28 @@ async function authorizeAndApply(
   context: ApplyContext
 ): Promise<CommandOutcome> {
   const { settings, run, revision } = context;
-  const request = { ...options, revision, apply: true, yes: true };
-  if (!interactive(options)) return applyLocked(root, state, request, context);
-  await enrollManager(settings.config, run);
+  const request = {
+    ...options,
+    revision,
+    apply: true,
+    yes: true,
+    readyBefore: readyBefore(state),
+  };
+  const person = interactive(options);
+  // A run without a person enrolls a worker only when every fact it needs
+  // is verified; it never prompts.
+  const enrollment = await enrollManager(settings.config, run, {
+    interactive: person,
+    joinable: Boolean(canonicalDefinition.tailnet),
+  });
+  if (!person)
+    return applyLocked(root, state, { ...request, enrollment }, context);
   // Needed to hand each node its own Vault token; nodes report any gap.
   await authorizeManager({ interactive: true }, run);
   return applyLocked(
     root,
     state,
-    { ...request, interactiveAuthorization: true },
+    { ...request, enrollment, interactiveAuthorization: true },
     context
   );
 }
@@ -317,6 +381,7 @@ function applyLocked(
     ...(definitionSource().kind === 'file'
       ? { definition: JSON.stringify(canonicalDefinition, null, 2) + '\n' }
       : {}),
+    scoped: Boolean(request.node?.length),
   });
   return withMachineLock(root, async () => {
     record.approve();
@@ -358,6 +423,12 @@ async function offManager(
   forward: ForwardSpawn | undefined
 ): Promise<CommandOutcome> {
   const definition = canonicalDefinition;
+  // Only the manager's fleet state knows the revision applied before.
+  if (options.rollback)
+    return actionRequired(
+      'FLEET_NOT_MANAGER',
+      `fleet-sync --rollback runs only on the manager (${definition.manager.id}); nothing was changed.`
+    );
   const revision =
     options.revision ??
     packageRevision() ??
@@ -414,11 +485,42 @@ function validate(options: FleetSyncOptions): void {
       '--revision requires an exact lowercase 40-character commit SHA.'
     );
   validateOnly(options);
+  validateRollback(options);
+  if (options.restartBusy && (options.only || options.schedule))
+    throw new FleetUsageError(
+      '--restart-busy applies to one full sync: it takes neither --only nor --schedule.'
+    );
+  validateMaintenance(options);
   if (options.dryRun && (options.apply || options.yes))
     throw new FleetUsageError(
       '--dry-run cannot be combined with apply confirmation.'
     );
 }
+function validateMaintenance(options: FleetSyncOptions): void {
+  if (options.force && !options.maintenanceReboot)
+    throw new FleetUsageError(
+      '--force only goes with --maintenance-reboot: it reboots a worker even while agent turns run.'
+    );
+  if (
+    options.maintenanceReboot &&
+    (options.dryRun || options.schedule || options.only)
+  )
+    throw new FleetUsageError(
+      '--maintenance-reboot reboots workers; it takes neither --dry-run, --schedule nor --only.'
+    );
+}
+function validateRollback(options: FleetSyncOptions): void {
+  if (!options.rollback) return;
+  if (options.revision !== undefined || options.schedule)
+    throw new FleetUsageError(
+      '--rollback chooses the revision itself: drop --revision and --schedule.'
+    );
+  if (options.only)
+    throw new FleetUsageError(
+      '--rollback converges the whole fleet; it takes no --only.'
+    );
+}
+
 function validateOnly(options: FleetSyncOptions): void {
   if (options.only !== undefined && options.only !== 'accounts')
     throw new FleetUsageError('--only takes one scope: accounts.');
@@ -450,6 +552,17 @@ async function onlyAccounts(
   return syncAccounts(options, settings, { revision, run });
 }
 
+/** The workers that converged before this run. A fleet state written before
+ * it recorded each worker's revision counts every worker of a fleet that
+ * was applied as converged.
+ */
+function readyBefore(state: Enrollment | null): string[] {
+  if (state?.workers) return Object.keys(state.workers);
+  return state?.appliedRevision
+    ? canonicalDefinition.workers.map((worker) => worker.id)
+    : [];
+}
+
 function interactive(options: FleetSyncOptions): boolean {
   return (
     ![
@@ -516,6 +629,24 @@ function validateSchedule(schedule: unknown): void {
     throw new FleetUsageError('Unknown manager schedule action.');
 }
 
+/** The revision a request names: --revision, or for --rollback the one
+ * applied before the approved one (null when the fleet state records none).
+ * After an approved revision that never became the applied one (its canary
+ * failed), that is the applied revision itself.
+ */
+function requestedRevision(
+  options: FleetSyncOptions,
+  state: Enrollment | null
+): string | undefined | null {
+  if (!options.rollback) return options.revision;
+  if (
+    state?.appliedRevision &&
+    state.appliedRevision !== state.approvedRevision
+  )
+    return state.appliedRevision;
+  return state?.previousRevision ?? null;
+}
+
 async function selectedRevision(
   explicit: string | undefined,
   approved: string | undefined,

package/src/cli/fleet-validate.ts

@@ -1,15 +1,29 @@
 import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
 import {
   DefinitionError,
   parseDefinition,
+  sshTarget,
   type DefinitionProblem,
   type FleetDefinition,
 } from '../definition';
 import { FleetUsageError } from '../errors';
 import { PACKAGE_ROOT } from '../package-info';
-import { ansibleCommand } from '../platform/ansible';
+import { ansibleCommand, ansibleConfig } from '../platform/ansible';
+import { shellQuote } from '../ssh';
+import {
+  connectionOptions,
+  declaredConnection,
+  knownHostsPath,
+  SSH_TRANSPORT,
+  sshOptionArgs,
+} from '../ssh-connection';
+import {
+  expandRecords,
+  runRecordCommand,
+  type RecordCommand,
+} from '../worker-records';
 import { runCommand } from '../runner';
 import type { Check, Runner } from '../types';
 import type { CommandOutcome } from './contracts';
@@ -26,6 +40,8 @@ export interface FleetValidateServices {
   platform: NodeJS.Platform;
   /** The package's Ansible playbooks, relative to `root`. */
   root: string;
+  /** Runs a command `workerRecords` names. */
+  records?: RecordCommand;
 }
 
 const SERVICES: FleetValidateServices = {
@@ -42,20 +58,34 @@ const PLAYBOOK_DIRECTORIES = ['ansible/ubuntu', 'ansible/macos'];
  * worker in the `workers` group in the order it is declared. Host names are
  * the fleet ids; a machine is reached as fleet-sync reaches it.
  */
-export function fleetInventory(definition: FleetDefinition): unknown {
+export function fleetInventory(
+  definition: FleetDefinition,
+  knownHosts = knownHostsPath()
+): unknown {
   const workers = Object.fromEntries(
-    definition.workers.map((worker) => [
-      worker.id,
-      {
-        ansible_host: worker.ssh,
-        ...(worker.network
-          ? {
-              ansible_user: worker.network.user,
-              ansible_port: worker.network.sshPort,
-            }
-          : {}),
-      },
-    ])
+    definition.workers.map((worker) => {
+      const declared = declaredConnection(worker);
+      // The same options a direct session takes (src/ssh-connection.ts).
+      const options = [
+        ...SSH_TRANSPORT,
+        ...(declared ? connectionOptions(declared, knownHosts) : []),
+      ];
+      return [
+        worker.id,
+        {
+          ansible_host: sshTarget(worker),
+          ...(worker.network
+            ? {
+                ansible_user: worker.network.user,
+                ansible_port: worker.network.sshPort,
+              }
+            : {}),
+          ansible_ssh_common_args: sshOptionArgs(options)
+            .map((arg) => (arg === '-o' ? arg : shellQuote(arg)))
+            .join(' '),
+        },
+      ];
+    })
   );
   return {
     all: {
@@ -89,7 +119,13 @@ export async function fleetValidate(
     );
   let definition: FleetDefinition;
   try {
-    definition = parseDefinition(await readJson(path));
+    definition = parseDefinition(
+      expandRecords(
+        await readJson(path),
+        dirname(path),
+        services.records ?? runRecordCommand
+      )
+    );
   } catch (error) {
     if (error instanceof DefinitionError) return invalid(path, error.problem);
     throw error;
@@ -164,14 +200,16 @@ async function ansibleChecks(
   const directory = await mkdtemp(join(tmpdir(), 'devfleet-validate-'));
   const inventory = join(directory, 'inventory.json');
   try {
-    await writeFile(inventory, JSON.stringify(fleetInventory(definition)), {
-      mode: 0o600,
-    });
+    await writeFile(
+      inventory,
+      JSON.stringify(fleetInventory(definition, knownHostsPath(services.env))),
+      { mode: 0o600 }
+    );
     const playbook = ansibleCommand(services.platform, services.root);
     const tool = (name: string) => [...playbook.slice(0, -1), name];
     const listed = await services.run(
       [...tool('ansible-inventory'), '-i', inventory, '--list'],
-      { timeoutMs: 120_000 }
+      { timeoutMs: 120_000, env: ansibleConfig(services.root) }
     );
     const checks: Check[] = [
       listed.ok
@@ -200,7 +238,7 @@ async function ansibleChecks(
           '--syntax-check',
           join(services.root, file),
         ],
-        { timeoutMs: 120_000 }
+        { timeoutMs: 120_000, env: ansibleConfig(services.root) }
       );
       if (!result.ok) failed.push(file);
     }

package/src/cli/fleet-watch.ts

@@ -35,6 +35,10 @@ export interface FleetWatchServices {
   secrets: SecretsProvider;
   send: typeof fetch;
   now: () => Date;
+  /** Parks the workers `worker:idle` warns about; only when fleet.json turns
+   * parking.autoPark on. One check per worker it parked or could not.
+   */
+  autoPark?: (reports: NodeReport[]) => Promise<Check[]>;
 }
 
 /** Compares the fleet's health with the previous run and notifies on
@@ -61,7 +65,8 @@ export async function fleetWatch(
   const previous = services.store
     ? parseMemory(await services.store.read().catch(() => null))
     : null;
-  const observed = observe(await services.status());
+  const reports = await services.status();
+  const observed = observe(reports);
   const { notifications, memory } = transitions(
     previous,
     observed,
@@ -80,6 +85,7 @@ export async function fleetWatch(
         ))
       );
   if (!notify) checks.push(unconfigured('warn'));
+  if (services.autoPark) checks.push(...(await services.autoPark(reports)));
   checks.push(await heartbeat(services.store, JSON.stringify(memory)));
   return outcome(checks, notifications);
 }

package/src/cli/fleet.ts

@@ -16,13 +16,37 @@ import { probeSource } from './probe-source';
 import { executeRepairCommand } from './repair';
 import { scheduleCheck } from '../schedule-check';
 import { readHistory } from '../state/history';
+import { withUsage } from '../worker-activity';
+import { activeNodes, appendUsage } from './worker-usage-report';
+import { fleet, secret } from '../fleet-config';
+import { appendTailnetLeftovers } from '../tailnet-cleanup';
+import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
+import { packageInfo } from '../package-info';
+import { managerRuns } from './fleet-sync-checkout';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
 
 export interface FleetServices {
   run: Runner;
   source: () => Promise<string>;
+  /** The tailnet control plane fleet.json declares, asked for devices
+   * tagged as workers that no declared worker accounts for. Absent, or null,
+   * fleet-status says nothing about them.
+   */
+  tailnet?: () => Promise<TailnetControl | null>;
+  /** The commit the running package names; absent or null in a checkout. */
+  revision?: string | null;
+  /** Where the manager keeps its fleet-owned checkout of each revision. */
+  runs?: string;
 }
 
-const SERVICES: FleetServices = { run: runCommand, source: probeSource };
+const SERVICES: FleetServices = {
+  run: runCommand,
+  source: probeSource,
+  tailnet: () => fleetTailnetControl(fleet.tailnet, secret),
+  revision: packageInfo().revision,
+  runs: managerRuns(),
+};
 
 export async function executeFleet(
   command: string,
@@ -114,11 +138,14 @@ async function diagnose(
   settings: Settings & { missingRoster: boolean },
   services: FleetServices
 ): Promise<CommandOutcome> {
-  const source = nodes.length ? await services.source() : undefined;
+  // A parked worker is not contacted; it is listed as parked.
+  const active = activeNodes(nodes);
+  const source = active.length ? await services.source() : undefined;
+  const root = statusRoot(settings, services);
   const reports = await doctor(
-    nodes,
+    active,
     {
-      root: settings.workspace,
+      root,
       local: options.local === true,
       required: capabilities(options.require),
       timeoutMs: timeout(options),
@@ -126,11 +153,23 @@ async function diagnose(
     },
     services.run
   );
+  explainMissingCheckout(reports, root);
   if (!options.local && settings.missingRoster) reports.push(missingRoster());
   const schedule = scheduleCheck(readHistory(), new Date());
   if (schedule) reports[0]?.checks.push(schedule);
+  // What each worker is doing, unless only this manager is checked.
+  const usage = await usageOf(
+    reports,
+    active,
+    services.run,
+    skipsFleet(options, settings)
+  );
+  // Devices tagged as workers that are no declared worker (#123).
+  if (!skipsFleet(options, settings))
+    await appendTailnetLeftovers(reports, fleet, services.tailnet, nodes);
   const report = {
     ...makeReport('doctor', reports),
+    ...(usage ?? {}),
     coverage: {
       scope: options.local
         ? 'local'
@@ -146,10 +185,53 @@ async function diagnose(
   return {
     exitCode: reportExitCode(report, options.strict === true),
     data: report,
-    text: renderReport(report),
+    text: withUsage(renderReport(report), usage),
+  };
+}
+
+/** What the manager and the workers are measured against. The published
+ * package holds no Git history and no bun.lock, and whatever repository
+ * surrounds it is the consumer's project: from a package it is the
+ * fleet-owned checkout fleet-sync made of the commit the package names. A
+ * checkout (development) is its own reference.
+ */
+function statusRoot(settings: Settings, services: FleetServices): string {
+  const { revision, runs } = services;
+  return revision && runs ? join(runs, revision) : settings.workspace;
+}
+
+/** The manager has not run fleet-sync at this package's commit yet (a
+ * checkout is its own reference, so it always exists).
+ */
+function explainMissingCheckout(reports: NodeReport[], root: string): void {
+  if (existsSync(root)) return;
+  const check = reports[0]?.checks.find((item) => item.id === 'workspace');
+  if (!check) return;
+  check.summary =
+    'This manager has no fleet-owned checkout of the commit this package names; it is made by fleet-sync.';
+  check.remedy = {
+    action:
+      'Run devfleet fleet-sync from the project that holds your fleet.json, then fleet-status again.',
   };
 }
 
+/** Only this manager is checked: what is said of the fleet is skipped. */
+function skipsFleet(
+  options: CliOptions,
+  settings: { missingRoster: boolean }
+): boolean {
+  return Boolean(options.local || settings.missingRoster);
+}
+
+function usageOf(
+  reports: NodeReport[],
+  nodes: FleetNode[],
+  run: Runner,
+  skip: boolean
+) {
+  return skip ? Promise.resolve(null) : appendUsage(reports, nodes, { run });
+}
+
 function missingRoster(): NodeReport {
   return {
     node: 'fleet',

package/src/cli/managed-health.ts

@@ -3,6 +3,7 @@ import { verifyT3Endpoint } from '../t3-endpoint';
 import type { Check, NodeReport, Runner } from '../types';
 import { fleet as definition } from '../fleet-config';
 import { enrolledRoster } from '../definition';
+import { readParked } from '../state/workers';
 import type { Tailnet } from '../tailscale';
 import type { FleetNode } from '../types';
 import {
@@ -12,6 +13,7 @@ import {
 } from '../monitoring/beszel';
 import { hubAccess } from '../monitoring/hub-access';
 import { hubUrl } from '../topology';
+import { leftoverSystems, listSystems } from '../monitoring/systems';
 
 export interface DashboardAccess {
   (hub: string): Promise<{ request: BeszelRequest }>;
@@ -76,6 +78,7 @@ async function nodeEvidence(
         ) ?? dashboard;
       report.checks.push(...evidence.checks);
     }
+    dashboard.checks.push(...(await leftovers(request)));
   } catch (error) {
     dashboard.checks.push({
       id: 'beszel:access',
@@ -92,6 +95,21 @@ async function nodeEvidence(
   }
 }
 
+/** Systems the hub watches that no declared worker (parked ones included)
+ * and not the manager is named after. A hub whose list cannot be read says
+ * nothing here; its access is reported above.
+ */
+async function leftovers(request: BeszelRequest): Promise<Check[]> {
+  try {
+    return leftoverSystems(await listSystems(request), [
+      definition.manager.id,
+      ...definition.workers.map((worker) => worker.id),
+    ]);
+  } catch {
+    return [];
+  }
+}
+
 async function hubReachable(
   hub: string,
   fetcher: typeof fetch
@@ -155,7 +173,13 @@ export function managedHealthScope(
   roster: FleetNode[],
   includeManager = true
 ) {
-  const canonical = definition;
+  // A parked worker is off on purpose: it is not audited, not missing and
+  // its device is not expected online (fleet-status lists it as parked).
+  const parked = readParked();
+  const canonical = {
+    ...definition,
+    workers: definition.workers.filter((entry) => !parked[entry.id]),
+  };
   const declared = roster.filter((node) =>
     canonical.workers.some((entry) => entry.id === node.id)
   );