@crvouga/dev-fleet 0.8.0 → 0.9.1

package/src/cli/fleet-manifest.ts

@@ -1,5 +1,7 @@
 import { readFile } from 'node:fs/promises';
 import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import { declaredFor, workerFeatures } from '../worker-features';
 import { runCommand } from '../runner';
 import type { Runner } from '../types';
 import {
@@ -62,7 +64,11 @@ export async function fleetManifest(
   const fresh = options.reference
     ? await reference(options.reference)
     : undefined;
-  const policy = await ubuntuPolicy(services.root);
+  // A worker is measured against what its own features install.
+  const policy = declaredFor(
+    await ubuntuPolicy(services.root),
+    workerFeatures(fleet, node.id)
+  );
   const measured = await measureManifest(node, services.run, policy);
   if ('check' in measured)
     return {
@@ -75,9 +81,9 @@ export async function fleetManifest(
         checks: [measured.check],
       },
     };
-  const { manifest } = measured;
+  const { manifest, release } = measured;
   const differences = [
-    ...declaredDifferences(manifest, policy),
+    ...declaredDifferences(manifest, policy, release),
     ...(fresh ? diffManifests(fresh, manifest) : []),
   ];
   const check = manifestCheck(differences);

package/src/cli/fleet-node-lifecycle.ts

@@ -1,9 +1,28 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { parseConfig } from '../config';
-import type { FleetDefinition, MachineDefinition } from '../definition';
+import {
+  HOST_KEY,
+  type FleetDefinition,
+  type MachineDefinition,
+} from '../definition';
+import { hubAccess } from '../monitoring/hub-access';
+import type { HubRequest } from '../monitoring/enroll';
+import { moveSystem, removeSystem } from '../monitoring/systems';
+import {
+  declaredConnection,
+  hostKeyAlias,
+  renderKnownHosts,
+} from '../ssh-connection';
+import { changeWorkers, readPins } from '../state/workers';
+import { hubUrl } from '../topology';
 import { FleetUsageError } from '../errors';
-import { fleet } from '../fleet-config';
+import { fleet, secret } from '../fleet-config';
+import {
+  deviceLeftToPerson,
+  tailnetCleanup,
+  type TailnetCleanup,
+} from '../tailnet-cleanup';
 import type { Enrollment } from '../machine/state';
 import { PACKAGE_ROOT } from '../package-info';
 import { runAnsible } from '../platform/ansible';
@@ -13,8 +32,10 @@ import { loadEnrollment, storeEnrollment } from '../state/enrollment';
 import { STATE_ROSTER, writeRosterText } from '../state/roster';
 import { mutateState, stateContext, type StateContext } from '../state/sync';
 import { discoverTailnet, type TailnetComputer } from '../tailscale';
-import type { Check, FleetConfig, Runner } from '../types';
+import type { Check, FleetConfig, FleetNode, Runner } from '../types';
 import { usesVault } from '../vault-config';
+import { managerT3, type ManagerT3 } from '../t3/peer-link';
+import { pairAndRecord } from '../t3/pairing';
 import {
   issueWorkerToken,
   revokeWorkerToken,
@@ -45,8 +66,22 @@ export interface NodeLifecycleServices {
   /** Applies a playbook of the package with these variables. */
   playbook: (name: string, variables: object) => Promise<boolean>;
   now: () => Date;
+  /** The monitoring hub's API, signed in as the fleet account. Absent, the
+   * hub is left as it is and the step says so.
+   */
+  monitoring?: () => Promise<HubRequest>;
+  /** Removes a worker's old device from the tailnet, so a new machine gets
+   * the declared name: the control plane fleet.json declares, asked with the
+   * `tailnet-admin` secret. Absent, no device is removed and the step is
+   * reported as skipped, with what a person does instead.
+   */
+  tailnet?: TailnetCleanup;
+  /** The manager's own T3, which pairs with a replacement's environment. */
+  t3: () => ManagerT3;
 }
 
+export type { TailnetCleanup };
+
 export function lifecycleServices(roster: string): NodeLifecycleServices {
   return {
     run: runCommand,
@@ -63,6 +98,12 @@ export function lifecycleServices(roster: string): NodeLifecycleServices {
         })
       ).ok,
     now: () => new Date(),
+    monitoring: async () => (await hubAccess(hubUrl(fleet))).request,
+    // Only a fleet that declares a tailnet control plane removes devices.
+    ...(fleet.tailnet
+      ? { tailnet: tailnetCleanup(fleet.tailnet, secret) }
+      : {}),
+    t3: () => managerT3(runCommand),
   };
 }
 
@@ -85,6 +126,8 @@ export async function keyName(
   worker: MachineDefinition,
   run: Runner
 ): Promise<string> {
+  // A worker reached from declared data is pinned under its fleet ID.
+  if (!worker.ssh) return hostKeyAlias(worker.id);
   const resolved = await run(['ssh', '-G', worker.ssh], { timeoutMs: 10_000 });
   const option = (name: string) =>
     resolved.ok
@@ -137,9 +180,11 @@ const planned = (id: string, summary: string): Check => ({
 const preview = (options: NodeLifecycleOptions): boolean =>
   !(options.apply && options.yes);
 
-/** Takes a worker out of the fleet for good: revokes its Vault token, forgets
- * its pinned host key, removes its roster entry and leaves a tombstone in the
- * fleet state. Preview until --apply --yes. Running it again changes nothing.
+/** Takes a worker out of the fleet for good: revokes its Vault token,
+ * forgets its pinned host key, deletes its monitoring system, removes the
+ * manager's T3 link to it and its tailnet device (when the tailnet control is
+ * wired), removes its roster entry and leaves a tombstone in the fleet state.
+ * Preview until --apply --yes. Running it again changes nothing.
  */
 export async function fleetNodeRetire(
   options: NodeLifecycleOptions,
@@ -147,47 +192,42 @@ export async function fleetNodeRetire(
 ): Promise<CommandOutcome> {
   const worker = declared(services.definition(), options.id);
   const roster = await readRoster(services.roster);
-  const enrolled = roster?.nodes.some((node) => node.id === worker.id) ?? false;
-  if (enrolled && roster!.nodes.length === 1)
+  const entry = roster?.nodes.find((node) => node.id === worker.id);
+  if (entry && roster!.nodes.length === 1)
     throw new FleetUsageError(
       'The roster must keep at least one worker. Add the replacement with fleet-node-add first.'
     );
+  return retireWorker(worker, entry, services, !preview(options));
+}
+
+/** Retires one worker; with `apply` false it only plans. Shared with the
+ * retirement of a worker whose record disappeared (fleet-sync).
+ */
+export async function retireWorker(
+  worker: MachineDefinition,
+  entry: FleetNode | undefined,
+  services: NodeLifecycleServices,
+  apply: boolean
+): Promise<CommandOutcome> {
   const host = await keyName(worker, services.run);
-  if (preview(options))
+  if (!apply)
     return outcome(
       'fleet-node-retire',
       worker.id,
-      [
-        ...(services.vault()
-          ? [planned('auth:worker-token', `revoke ${worker.id}'s Vault token`)]
-          : []),
-        planned('ssh:host-key', `forget the host key of ${host}`),
-        planned(
-          'enrollment:connection',
-          enrolled
-            ? `remove ${worker.id} from the roster`
-            : `${worker.id} is not in the roster`
-        ),
-        planned('state:retired', `record that ${worker.id} is retired`),
-      ],
+      planRetirement(worker, entry, host, services.vault()),
       false
     );
   const checks: Check[] = [];
   if (services.vault())
     checks.push(await revokeWorkerToken(worker.id, services.run));
+  checks.push(await pinHostKey(worker, host, null, services));
   checks.push(
-    (await services.playbook('macos/worker-host-key.yml', {
-      host,
-      pin: 'absent',
-      known_hosts: services.knownHosts,
-    }))
-      ? {
-          id: 'ssh:host-key',
-          status: 'pass',
-          summary: `The manager no longer trusts any host key of ${host}.`,
-        }
-      : hostKeyFailure()
+    await onHub(services, (request) => removeSystem(request, worker.id))
   );
+  if (entry?.t3EnvironmentId)
+    checks.push(await unlinkT3(entry.t3EnvironmentId, services.run));
+  if (entry?.tailscaleId)
+    checks.push(await removeDevice(worker, entry.tailscaleId, services));
   await changeRoster(services, (current) =>
     current && current.nodes.some((node) => node.id === worker.id)
       ? {
@@ -201,10 +241,161 @@ export async function fleetNodeRetire(
     status: 'pass',
     summary: `${worker.id} is not in the roster.`,
   });
+  // Nothing of the worker stays in the fleet state but its tombstone.
+  await changeWorkers(services.state(), () => ({ [worker.id]: null }));
   checks.push(await tombstone(worker.id, services));
   return outcome('fleet-node-retire', worker.id, checks, true);
 }
 
+function planRetirement(
+  worker: MachineDefinition,
+  entry: FleetNode | undefined,
+  host: string,
+  vault: boolean
+): Check[] {
+  return [
+    ...(vault
+      ? [planned('auth:worker-token', `revoke ${worker.id}'s Vault token`)]
+      : []),
+    planned('ssh:host-key', `forget the host key of ${host}`),
+    planned('beszel:system', `delete the monitoring system ${worker.id}`),
+    ...(entry?.t3EnvironmentId
+      ? [
+          planned(
+            't3:link',
+            `remove the manager's T3 link to ${entry.t3EnvironmentId}`
+          ),
+        ]
+      : []),
+    ...(entry?.tailscaleId
+      ? [
+          planned(
+            'tailscale:device',
+            `remove the tailnet device ${entry.tailscaleId}`
+          ),
+        ]
+      : []),
+    planned(
+      'enrollment:connection',
+      entry
+        ? `remove ${worker.id} from the roster`
+        : `${worker.id} is not in the roster`
+    ),
+    planned('state:retired', `record that ${worker.id} is retired`),
+  ];
+}
+
+/** Pins `key` as a worker's host key, or forgets it (null). A worker reached
+ * from declared data is pinned in the fleet state, and the fleet-owned
+ * known_hosts is rendered again; one reached through the operator's own SSH
+ * alias is pinned in their known_hosts by the pinned Ansible.
+ */
+export async function pinHostKey(
+  worker: MachineDefinition,
+  host: string,
+  key: string | null,
+  services: Pick<
+    NodeLifecycleServices,
+    'state' | 'playbook' | 'knownHosts' | 'now' | 'definition'
+  >
+): Promise<Check> {
+  if (!declaredConnection(worker)) {
+    const pinned = await services.playbook('macos/worker-host-key.yml', {
+      host,
+      ...(key ? { pin: 'present', key } : { pin: 'absent' }),
+      known_hosts: services.knownHosts,
+    });
+    if (!pinned) return hostKeyFailure();
+    return {
+      id: 'ssh:host-key',
+      status: 'pass',
+      summary: key
+        ? `The manager trusts exactly the given host key of ${host}.`
+        : `The manager no longer trusts any host key of ${host}.`,
+    };
+  }
+  const at = services.now().toISOString();
+  await changeWorkers(services.state(), (current) => {
+    const {
+      hostKey: _key,
+      hostKeyPinnedAt: _at,
+      ...rest
+    } = current[worker.id] ?? {};
+    return {
+      [worker.id]: key ? { ...rest, hostKey: key, hostKeyPinnedAt: at } : rest,
+    };
+  });
+  renderKnownHosts(services.definition(), process.env, readPins());
+  return {
+    id: 'ssh:host-key',
+    status: 'pass',
+    summary: key
+      ? `The fleet state pins exactly the given host key of ${worker.id}.`
+      : `The fleet state pins no host key of ${worker.id}.`,
+  };
+}
+
+/** One change on the monitoring hub. A hub that cannot be reached is a
+ * warning, never a reason to stop.
+ */
+async function onHub(
+  services: NodeLifecycleServices,
+  change: (request: HubRequest) => Promise<Check>
+): Promise<Check> {
+  if (!services.monitoring)
+    return {
+      id: 'beszel:system',
+      status: 'skip',
+      summary: 'The monitoring hub was not asked: there is no hub access here.',
+    };
+  try {
+    return await change(await services.monitoring());
+  } catch (error) {
+    return {
+      id: 'beszel:system',
+      status: 'warn',
+      summary:
+        error instanceof Error && /^BESZEL_[A-Z_]+$/.test(error.message)
+          ? error.message
+          : 'BESZEL_API_UNAVAILABLE',
+      remedy: {
+        action:
+          'The hub was not changed. Run fleet-sync on the manager so the hub answers, then run this command again; fleet-status --workers lists systems that match no declared worker.',
+      },
+    };
+  }
+}
+
+/** Forgets the manager's T3 link to an environment that is gone. */
+async function unlinkT3(environment: string, run: Runner): Promise<Check> {
+  const result = await run(['t3', 'environment', 'unlink', environment], {
+    timeoutMs: 30_000,
+  });
+  return result.ok
+    ? {
+        id: 't3:link',
+        status: 'pass',
+        summary: `The manager no longer links to the T3 environment ${environment}.`,
+      }
+    : {
+        id: 't3:link',
+        status: 'warn',
+        summary: 'T3_LINK_UNLINK_FAILED',
+        remedy: {
+          action: `Remove the link to ${environment} in the manager's T3 (Settings, Connections), then run the command again.`,
+        },
+      };
+}
+
+async function removeDevice(
+  worker: MachineDefinition,
+  device: string,
+  services: NodeLifecycleServices
+): Promise<Check> {
+  if (services.tailnet) return services.tailnet.removeDevice(worker, device);
+  return deviceLeftToPerson(worker, device);
+}
+
 async function tombstone(
   id: string,
   services: NodeLifecycleServices
@@ -241,10 +432,14 @@ async function tombstone(
 
 /** Puts a fresh machine in place of a worker under the same id: verifies the
  * new device on the tailnet, pins its host key, binds the roster entry to the
- * new device and issues it a token of its own (revoking the old one). Preview
- * until --apply --yes; then `fleet-sync --node ID` brings it to ready. When
- * nothing changed (the same device, the same key, a token of its own) it
- * changes nothing.
+ * new device, points its monitoring system at it, removes the manager's T3
+ * link to the old environment, pairs the manager's T3 with the new one (now,
+ * or in the next fleet-sync once it runs), removes the old tailnet device
+ * (when the tailnet control is wired) and issues it a token of its own
+ * (revoking the old one).
+ * Preview until --apply --yes; then `fleet-sync --node ID` brings it to
+ * ready. When nothing changed (the same device, the same key, a token of its
+ * own) it changes nothing.
  */
 export async function fleetNodeReplace(
   options: NodeLifecycleOptions,
@@ -255,13 +450,19 @@ export async function fleetNodeReplace(
   const device = await newDevice(worker, services.run);
   if ('check' in device)
     return outcome('fleet-node-replace', worker.id, [device.check], false);
+  const replaced = entry.tailscaleId !== device.computer.id;
   const plan: Replacement = {
     worker,
+    node: { ...entry, tailscaleId: device.computer.id },
     device: device.computer.id,
     host: await keyName(worker, services.run),
     hostKey: options.hostKey,
-    replaced: entry.tailscaleId !== device.computer.id,
+    replaced,
     needsToken: false,
+    ...(replaced && entry.tailscaleId ? { previous: entry.tailscaleId } : {}),
+    ...(replaced && entry.t3EnvironmentId
+      ? { environment: entry.t3EnvironmentId }
+      : {}),
   };
   plan.needsToken =
     services.vault() &&
@@ -288,11 +489,17 @@ export async function fleetNodeReplace(
 
 interface Replacement {
   worker: MachineDefinition;
+  /** The roster entry, bound to the new device. */
+  node: FleetNode;
   device: string;
   host: string;
   hostKey: string | undefined;
   replaced: boolean;
   needsToken: boolean;
+  /** The device the worker ran as before, when it changed. */
+  previous?: string;
+  /** The T3 environment the manager linked to before, when it changed. */
+  environment?: string;
 }
 
 async function replaceable(
@@ -300,15 +507,14 @@ async function replaceable(
   options: NodeLifecycleOptions,
   services: NodeLifecycleServices
 ) {
-  if (
-    options.hostKey !== undefined &&
-    !/^(ssh-ed25519|ecdsa-sha2-nistp256|ssh-rsa) [A-Za-z0-9+/=]{16,}$/.test(
-      options.hostKey
-    )
-  )
+  if (options.hostKey !== undefined && !HOST_KEY.test(options.hostKey))
     throw new FleetUsageError(
       '--host-key expects one public key, "TYPE BASE64", as ssh-keyscan or the seed prints it.'
     );
+  if (options.hostKey !== undefined && worker.network?.hostKey)
+    throw new FleetUsageError(
+      `${worker.id}'s host key is declared by its worker record. Change hostKey in the record instead; it is pinned on the next command.`
+    );
   const roster = await readRoster(services.roster);
   const entry = roster?.nodes.find((node) => node.id === worker.id);
   if (!entry)
@@ -327,8 +533,27 @@ function planReplacement(plan: Replacement): Check[] {
       planned(
         'enrollment:connection',
         `bind ${plan.worker.id} to ${plan.device}`
+      ),
+      planned(
+        'beszel:system',
+        `point the monitoring system ${plan.worker.id} at the new machine`
       )
     );
+  if (plan.environment)
+    steps.push(
+      planned('t3:link', `remove the manager's T3 link to ${plan.environment}`)
+    );
+  if (plan.replaced)
+    steps.push(
+      planned(
+        't3:link',
+        `pair the manager's T3 with the new T3 environment of ${plan.worker.id}`
+      )
+    );
+  if (plan.previous)
+    steps.push(
+      planned('tailscale:device', `remove the tailnet device ${plan.previous}`)
+    );
   if (plan.needsToken)
     steps.push(
       planned('auth:worker-token', `issue ${plan.worker.id} a token of its own`)
@@ -343,30 +568,19 @@ async function carryOut(
   const { worker } = plan;
   const steps: Check[] = [];
   if (plan.hostKey)
-    steps.push(
-      (await services.playbook('macos/worker-host-key.yml', {
-        host: plan.host,
-        pin: 'present',
-        key: plan.hostKey,
-        known_hosts: services.knownHosts,
-      }))
-        ? {
-            id: 'ssh:host-key',
-            status: 'pass',
-            summary: `The manager trusts exactly the given host key of ${plan.host}.`,
-          }
-        : hostKeyFailure()
-    );
+    steps.push(await pinHostKey(worker, plan.host, plan.hostKey, services));
   if (plan.replaced) {
     await changeRoster(services, (current) =>
       current
         ? {
             ...current,
-            nodes: current.nodes.map((node) =>
-              node.id === worker.id
-                ? { ...node, tailscaleId: plan.device }
-                : node
-            ),
+            nodes: current.nodes.map((node) => {
+              if (node.id !== worker.id) return node;
+              // The old environment's identity goes with the old machine:
+              // the next sync records the new one once it is linked.
+              const { t3EnvironmentId: _old, ...rest } = node;
+              return { ...rest, tailscaleId: plan.device };
+            }),
           }
         : null
     );
@@ -375,7 +589,21 @@ async function carryOut(
       status: 'pass',
       summary: `The roster binds ${worker.id} to the device ${plan.device}.`,
     });
+    steps.push(
+      await onHub(services, (request) =>
+        moveSystem(
+          request,
+          worker.id,
+          worker.network?.tailscaleName ?? plan.host
+        )
+      )
+    );
   }
+  if (plan.environment)
+    steps.push(await unlinkT3(plan.environment, services.run));
+  if (plan.replaced) steps.push(await pairReplacement(plan.node, services));
+  if (plan.previous)
+    steps.push(await removeDevice(worker, plan.previous, services));
   if (plan.needsToken) {
     const revoked = await revokeWorkerToken(worker.id, services.run);
     steps.push(
@@ -392,6 +620,23 @@ async function carryOut(
   return steps;
 }
 
+/** Pairs the manager's T3 with a replacement's environment and records it
+ * in the roster. A machine whose T3 does not run yet (or refuses) is left to
+ * the next fleet-sync, which pairs it once it runs.
+ */
+function pairReplacement(
+  node: FleetNode,
+  services: NodeLifecycleServices
+): Promise<Check> {
+  return pairAndRecord(node, {
+    roster: services.roster,
+    run: services.run,
+    manager: services.t3(),
+    context: services.state,
+    now: services.now,
+  });
+}
+
 async function ownToken(id: string, run: Runner): Promise<boolean> {
   const location = workerTokenLocation(id);
   return (
@@ -415,7 +660,11 @@ async function newDevice(
   run: Runner
 ): Promise<{ computer: TailnetComputer } | { check: Check }> {
   const tailnet = await discoverTailnet(run);
-  const name = (worker.network?.tailscaleName ?? worker.ssh).toLowerCase();
+  const name = (
+    worker.network?.tailscaleName ??
+    worker.ssh ??
+    ''
+  ).toLowerCase();
   const short = name.split('.')[0]!;
   const found = tailnet.computers.filter(
     (computer) =>