@crvouga/dev-fleet 0.6.0 → 0.8.0

package/src/cli/fleet-manifest.ts

@@ -0,0 +1,102 @@
+import { readFile } from 'node:fs/promises';
+import { FleetUsageError } from '../errors';
+import { runCommand } from '../runner';
+import type { Runner } from '../types';
+import {
+  declaredDifferences,
+  diffManifests,
+  manifestCheck,
+  manifestDigest,
+  measureManifest,
+  ubuntuPolicy,
+  type WorkerManifest,
+} from '../worker-manifest';
+import type { CommandOutcome } from './contracts';
+import { readRoster } from './roster';
+
+export interface FleetManifestOptions {
+  id?: string;
+  /** A manifest from a fresh build of the same revision (fleet-manifest
+   * --json of that build) to compare the worker with.
+   */
+  reference?: string;
+}
+
+export interface FleetManifestServices {
+  run: Runner;
+  roster: string;
+  root?: string;
+}
+
+export const manifestServices = (roster: string): FleetManifestServices => ({
+  run: runCommand,
+  roster,
+});
+
+async function reference(path: string): Promise<WorkerManifest> {
+  try {
+    const value = JSON.parse(await readFile(path, 'utf8')) as {
+      manifest?: WorkerManifest;
+    } & Partial<WorkerManifest>;
+    const manifest = value.manifest ?? (value as WorkerManifest);
+    if (manifest.version !== 1 || !manifest.packages) throw new Error();
+    return manifest;
+  } catch {
+    throw new FleetUsageError(
+      '--reference must name a manifest written by devfleet fleet-manifest --json.'
+    );
+  }
+}
+
+/** Read-only: measures what an enrolled worker is made of, and the check
+ * worker:manifest against the declared pins and, with --reference, a fresh
+ * build of the same revision.
+ */
+export async function fleetManifest(
+  options: FleetManifestOptions,
+  services: FleetManifestServices
+): Promise<CommandOutcome> {
+  const roster = await readRoster(services.roster);
+  const node = roster?.nodes.find((entry) => entry.id === options.id);
+  if (!node) throw new FleetUsageError('--id must name an enrolled worker.');
+  const fresh = options.reference
+    ? await reference(options.reference)
+    : undefined;
+  const policy = await ubuntuPolicy(services.root);
+  const measured = await measureManifest(node, services.run, policy);
+  if ('check' in measured)
+    return {
+      exitCode: 1,
+      data: {
+        version: 1,
+        command: 'fleet-manifest',
+        ok: false,
+        node: node.id,
+        checks: [measured.check],
+      },
+    };
+  const { manifest } = measured;
+  const differences = [
+    ...declaredDifferences(manifest, policy),
+    ...(fresh ? diffManifests(fresh, manifest) : []),
+  ];
+  const check = manifestCheck(differences);
+  const digest = manifestDigest(manifest);
+  return {
+    exitCode: check.status === 'fail' ? 1 : 0,
+    data: {
+      version: 1,
+      command: 'fleet-manifest',
+      ok: check.status !== 'fail',
+      node: node.id,
+      digest,
+      manifest,
+      differences,
+      checks: [check],
+    },
+    text: [
+      `${node.id}: ${Object.keys(manifest.packages).length} packages, ${Object.keys(manifest.tools).length} tools, digest ${digest}`,
+      `${check.status.toUpperCase()} ${check.id}: ${check.summary}`,
+    ].join('\n'),
+  };
+}

package/src/cli/fleet-node-lifecycle.ts

@@ -0,0 +1,455 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { parseConfig } from '../config';
+import type { FleetDefinition, MachineDefinition } from '../definition';
+import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import type { Enrollment } from '../machine/state';
+import { PACKAGE_ROOT } from '../package-info';
+import { runAnsible } from '../platform/ansible';
+import { makeReport, renderReport } from '../report';
+import { runCommand } from '../runner';
+import { loadEnrollment, storeEnrollment } from '../state/enrollment';
+import { STATE_ROSTER, writeRosterText } from '../state/roster';
+import { mutateState, stateContext, type StateContext } from '../state/sync';
+import { discoverTailnet, type TailnetComputer } from '../tailscale';
+import type { Check, FleetConfig, Runner } from '../types';
+import { usesVault } from '../vault-config';
+import {
+  issueWorkerToken,
+  revokeWorkerToken,
+  workerTokenLocation,
+} from '../worker-credentials';
+import type { CommandOutcome } from './contracts';
+import { readRoster } from './roster';
+
+export interface NodeLifecycleOptions {
+  id?: string;
+  /** The replacement's SSH host key, `TYPE BASE64`, verified out of band or
+   * written by the seed that created it.
+   */
+  hostKey?: string;
+  apply?: boolean;
+  yes?: boolean;
+}
+
+export interface NodeLifecycleServices {
+  run: Runner;
+  /** The roster: the fleet state's, or a file `--config` names. */
+  roster: string;
+  definition: () => FleetDefinition;
+  vault: () => boolean;
+  /** The manager's known_hosts file. */
+  knownHosts: string;
+  state: () => StateContext;
+  /** Applies a playbook of the package with these variables. */
+  playbook: (name: string, variables: object) => Promise<boolean>;
+  now: () => Date;
+}
+
+export function lifecycleServices(roster: string): NodeLifecycleServices {
+  return {
+    run: runCommand,
+    roster,
+    definition: () => fleet,
+    vault: usesVault,
+    knownHosts: join(homedir(), '.ssh/known_hosts'),
+    state: () => stateContext(runCommand),
+    playbook: async (name, variables) =>
+      (
+        await runAnsible(runCommand, join(PACKAGE_ROOT, 'ansible', name), {
+          input: JSON.stringify(variables),
+          timeoutMs: 300_000,
+        })
+      ).ok,
+    now: () => new Date(),
+  };
+}
+
+function declared(
+  definition: FleetDefinition,
+  id: string | undefined
+): MachineDefinition {
+  const worker = definition.workers.find((entry) => entry.id === id);
+  if (!worker)
+    throw new FleetUsageError(
+      '--id must name a worker fleet.json declares. To retire a worker, retire it first and then remove it from fleet.json.'
+    );
+  return worker;
+}
+
+/** The name ssh checks the host key of: the HostName the alias resolves to,
+ * as `[host]:port` when the port is not 22.
+ */
+export async function keyName(
+  worker: MachineDefinition,
+  run: Runner
+): Promise<string> {
+  const resolved = await run(['ssh', '-G', worker.ssh], { timeoutMs: 10_000 });
+  const option = (name: string) =>
+    resolved.ok
+      ? new RegExp(`^${name} (\\S+)$`, 'm').exec(resolved.stdout)?.[1]
+      : undefined;
+  const host =
+    option('hostname') ?? worker.network?.tailscaleName ?? worker.ssh;
+  const port = option('port') ?? String(worker.network?.sshPort ?? 22);
+  return port === '22' ? host : `[${host}]:${port}`;
+}
+
+/** Changes the roster in the fleet state (or the file `--config` names). */
+async function changeRoster(
+  services: NodeLifecycleServices,
+  change: (roster: FleetConfig | null) => FleetConfig | null
+): Promise<void> {
+  const save = async () => {
+    const next = change(await readRoster(services.roster));
+    if (next)
+      await writeRosterText(
+        services.roster,
+        `${JSON.stringify(parseConfig(next), null, 2)}\n`
+      );
+  };
+  if (services.roster === STATE_ROSTER)
+    await mutateState(services.state(), save);
+  else await save();
+}
+
+function outcome(
+  command: string,
+  id: string,
+  checks: Check[],
+  applied: boolean
+): CommandOutcome {
+  const report = { ...makeReport(command, [{ node: id, checks }]), applied };
+  return {
+    exitCode: checks.some((check) => check.status === 'fail') ? 1 : 0,
+    data: report,
+    text: renderReport(report),
+  };
+}
+
+const planned = (id: string, summary: string): Check => ({
+  id,
+  status: 'skip',
+  summary: `Planned: ${summary}`,
+});
+
+const preview = (options: NodeLifecycleOptions): boolean =>
+  !(options.apply && options.yes);
+
+/** Takes a worker out of the fleet for good: revokes its Vault token, forgets
+ * its pinned host key, removes its roster entry and leaves a tombstone in the
+ * fleet state. Preview until --apply --yes. Running it again changes nothing.
+ */
+export async function fleetNodeRetire(
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+): Promise<CommandOutcome> {
+  const worker = declared(services.definition(), options.id);
+  const roster = await readRoster(services.roster);
+  const enrolled = roster?.nodes.some((node) => node.id === worker.id) ?? false;
+  if (enrolled && roster!.nodes.length === 1)
+    throw new FleetUsageError(
+      'The roster must keep at least one worker. Add the replacement with fleet-node-add first.'
+    );
+  const host = await keyName(worker, services.run);
+  if (preview(options))
+    return outcome(
+      'fleet-node-retire',
+      worker.id,
+      [
+        ...(services.vault()
+          ? [planned('auth:worker-token', `revoke ${worker.id}'s Vault token`)]
+          : []),
+        planned('ssh:host-key', `forget the host key of ${host}`),
+        planned(
+          'enrollment:connection',
+          enrolled
+            ? `remove ${worker.id} from the roster`
+            : `${worker.id} is not in the roster`
+        ),
+        planned('state:retired', `record that ${worker.id} is retired`),
+      ],
+      false
+    );
+  const checks: Check[] = [];
+  if (services.vault())
+    checks.push(await revokeWorkerToken(worker.id, services.run));
+  checks.push(
+    (await services.playbook('macos/worker-host-key.yml', {
+      host,
+      pin: 'absent',
+      known_hosts: services.knownHosts,
+    }))
+      ? {
+          id: 'ssh:host-key',
+          status: 'pass',
+          summary: `The manager no longer trusts any host key of ${host}.`,
+        }
+      : hostKeyFailure()
+  );
+  await changeRoster(services, (current) =>
+    current && current.nodes.some((node) => node.id === worker.id)
+      ? {
+          ...current,
+          nodes: current.nodes.filter((node) => node.id !== worker.id),
+        }
+      : null
+  );
+  checks.push({
+    id: 'enrollment:connection',
+    status: 'pass',
+    summary: `${worker.id} is not in the roster.`,
+  });
+  checks.push(await tombstone(worker.id, services));
+  return outcome('fleet-node-retire', worker.id, checks, true);
+}
+
+async function tombstone(
+  id: string,
+  services: NodeLifecycleServices
+): Promise<Check> {
+  const record = async (): Promise<Enrollment | null> => {
+    const enrollment = await loadEnrollment(null);
+    if (!enrollment) return null;
+    const { [id]: _applied, ...workers } = enrollment.workers ?? {};
+    const retired = {
+      ...enrollment,
+      workers,
+      retired: {
+        ...enrollment.retired,
+        [id]: enrollment.retired?.[id] ?? services.now().toISOString(),
+      },
+    };
+    storeEnrollment(retired);
+    return retired;
+  };
+  const stored = (await mutateState(services.state(), record)).value;
+  return stored
+    ? {
+        id: 'state:retired',
+        status: 'pass',
+        summary: `The fleet state records that ${id} was retired at ${stored.retired![id]}.`,
+      }
+    : {
+        id: 'state:retired',
+        status: 'skip',
+        summary:
+          'No revision was ever applied to this fleet, so there is nothing to record.',
+      };
+}
+
+/** Puts a fresh machine in place of a worker under the same id: verifies the
+ * new device on the tailnet, pins its host key, binds the roster entry to the
+ * new device and issues it a token of its own (revoking the old one). Preview
+ * until --apply --yes; then `fleet-sync --node ID` brings it to ready. When
+ * nothing changed (the same device, the same key, a token of its own) it
+ * changes nothing.
+ */
+export async function fleetNodeReplace(
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+): Promise<CommandOutcome> {
+  const worker = declared(services.definition(), options.id);
+  const entry = await replaceable(worker, options, services);
+  const device = await newDevice(worker, services.run);
+  if ('check' in device)
+    return outcome('fleet-node-replace', worker.id, [device.check], false);
+  const plan: Replacement = {
+    worker,
+    device: device.computer.id,
+    host: await keyName(worker, services.run),
+    hostKey: options.hostKey,
+    replaced: entry.tailscaleId !== device.computer.id,
+    needsToken: false,
+  };
+  plan.needsToken =
+    services.vault() &&
+    (plan.replaced || !(await ownToken(worker.id, services.run)));
+  const identity: Check = {
+    id: 'tailscale:identity',
+    status: 'pass',
+    summary: `${worker.id} is the online device ${plan.device}${plan.replaced ? `, replacing ${entry.tailscaleId ?? 'none'}` : ''}.`,
+  };
+  if (preview(options))
+    return outcome(
+      'fleet-node-replace',
+      worker.id,
+      [identity, ...planReplacement(plan)],
+      false
+    );
+  return outcome(
+    'fleet-node-replace',
+    worker.id,
+    [identity, ...(await carryOut(plan, services))],
+    true
+  );
+}
+
+interface Replacement {
+  worker: MachineDefinition;
+  device: string;
+  host: string;
+  hostKey: string | undefined;
+  replaced: boolean;
+  needsToken: boolean;
+}
+
+async function replaceable(
+  worker: MachineDefinition,
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+) {
+  if (
+    options.hostKey !== undefined &&
+    !/^(ssh-ed25519|ecdsa-sha2-nistp256|ssh-rsa) [A-Za-z0-9+/=]{16,}$/.test(
+      options.hostKey
+    )
+  )
+    throw new FleetUsageError(
+      '--host-key expects one public key, "TYPE BASE64", as ssh-keyscan or the seed prints it.'
+    );
+  const roster = await readRoster(services.roster);
+  const entry = roster?.nodes.find((node) => node.id === worker.id);
+  if (!entry)
+    throw new FleetUsageError(
+      `${worker.id} is not in the roster. Add it with fleet-node-add first.`
+    );
+  return entry;
+}
+
+function planReplacement(plan: Replacement): Check[] {
+  const steps: Check[] = [];
+  if (plan.hostKey)
+    steps.push(planned('ssh:host-key', `pin the host key of ${plan.host}`));
+  if (plan.replaced)
+    steps.push(
+      planned(
+        'enrollment:connection',
+        `bind ${plan.worker.id} to ${plan.device}`
+      )
+    );
+  if (plan.needsToken)
+    steps.push(
+      planned('auth:worker-token', `issue ${plan.worker.id} a token of its own`)
+    );
+  return steps;
+}
+
+async function carryOut(
+  plan: Replacement,
+  services: NodeLifecycleServices
+): Promise<Check[]> {
+  const { worker } = plan;
+  const steps: Check[] = [];
+  if (plan.hostKey)
+    steps.push(
+      (await services.playbook('macos/worker-host-key.yml', {
+        host: plan.host,
+        pin: 'present',
+        key: plan.hostKey,
+        known_hosts: services.knownHosts,
+      }))
+        ? {
+            id: 'ssh:host-key',
+            status: 'pass',
+            summary: `The manager trusts exactly the given host key of ${plan.host}.`,
+          }
+        : hostKeyFailure()
+    );
+  if (plan.replaced) {
+    await changeRoster(services, (current) =>
+      current
+        ? {
+            ...current,
+            nodes: current.nodes.map((node) =>
+              node.id === worker.id
+                ? { ...node, tailscaleId: plan.device }
+                : node
+            ),
+          }
+        : null
+    );
+    steps.push({
+      id: 'enrollment:connection',
+      status: 'pass',
+      summary: `The roster binds ${worker.id} to the device ${plan.device}.`,
+    });
+  }
+  if (plan.needsToken) {
+    const revoked = await revokeWorkerToken(worker.id, services.run);
+    steps.push(
+      revoked.status === 'fail'
+        ? revoked
+        : await issueWorkerToken(worker.id, services.run)
+    );
+  }
+  steps.push({
+    id: 'rollout:next',
+    status: 'pass',
+    summary: `Run devfleet fleet-sync --node ${worker.id} to converge the new machine.`,
+  });
+  return steps;
+}
+
+async function ownToken(id: string, run: Runner): Promise<boolean> {
+  const location = workerTokenLocation(id);
+  return (
+    await run(
+      [
+        'vault',
+        'kv',
+        'get',
+        `-mount=${location.mount}`,
+        '-field=accessor',
+        location.path,
+      ],
+      { timeoutMs: 20_000 }
+    )
+  ).ok;
+}
+
+/** The one online computer on the tailnet that carries the worker's name. */
+async function newDevice(
+  worker: MachineDefinition,
+  run: Runner
+): Promise<{ computer: TailnetComputer } | { check: Check }> {
+  const tailnet = await discoverTailnet(run);
+  const name = (worker.network?.tailscaleName ?? worker.ssh).toLowerCase();
+  const short = name.split('.')[0]!;
+  const found = tailnet.computers.filter(
+    (computer) =>
+      !computer.self &&
+      computer.online &&
+      [computer.dns.toLowerCase(), computer.name.toLowerCase()].some(
+        (candidate) => candidate === name || candidate.split('.')[0] === short
+      )
+  );
+  if (found.length === 1) return { computer: found[0]! };
+  return {
+    check: {
+      id: 'tailscale:identity',
+      status: 'fail',
+      summary: found.length
+        ? 'TAILSCALE_IDENTITY_CONFLICT'
+        : 'TAILSCALE_NODE_MISSING',
+      remedy: {
+        action: found.length
+          ? `More than one online device is named like ${worker.id}. Remove the old device from the tailnet, then run the command again.`
+          : `No online device is named like ${worker.id}. Start the new machine, join it to the tailnet under that name, then run the command again.`,
+      },
+    },
+  };
+}
+
+function hostKeyFailure(): Check {
+  return {
+    id: 'ssh:host-key',
+    status: 'fail',
+    summary: 'SSH_HOST_KEY_NOT_CHANGED',
+    remedy: {
+      action:
+        'The pinned Ansible could not change known_hosts on this manager. Run devfleet fleet-validate --ansible, then this command again.',
+    },
+  };
+}

package/src/cli/fleet-node-seed.ts

@@ -0,0 +1,151 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { FleetDefinition } from '../definition';
+import { ubuntuRelease } from '../platform/ubuntu-release';
+import { FleetUsageError } from '../errors';
+import { fleet, secret } from '../fleet-config';
+import type { SecretsProvider } from '../secrets';
+import type { Check } from '../types';
+import {
+  generateHostKey,
+  renderSeed,
+  seedUserData,
+  tailscalePins,
+} from '../worker-seed';
+import type { CommandOutcome } from './contracts';
+import {
+  keyName,
+  lifecycleServices,
+  type NodeLifecycleServices,
+} from './fleet-node-lifecycle';
+
+export interface NodeSeedOptions {
+  id?: string;
+  output?: string;
+  publicKey?: string;
+  autoinstall?: boolean;
+  apply?: boolean;
+  yes?: boolean;
+}
+
+export interface NodeSeedServices extends Pick<
+  NodeLifecycleServices,
+  'run' | 'playbook' | 'knownHosts'
+> {
+  definition: () => FleetDefinition;
+  secret: SecretsProvider;
+  home: string;
+}
+
+export function seedServices(roster: string): NodeSeedServices {
+  const { run, playbook, knownHosts } = lifecycleServices(roster);
+  return {
+    run,
+    playbook,
+    knownHosts,
+    definition: () => fleet,
+    secret,
+    home: homedir(),
+  };
+}
+
+/** Writes the first-boot seed of a declared worker to a private file: the
+ * seed carries a single-use auth key, so it is never printed. With --apply
+ * --yes the manager pins the host key the seed installs, so the new machine
+ * is trusted without a first-use step.
+ */
+export async function fleetNodeSeed(
+  options: NodeSeedOptions,
+  services: NodeSeedServices
+): Promise<CommandOutcome> {
+  const worker = services
+    .definition()
+    .workers.find((entry) => entry.id === options.id);
+  if (!worker)
+    throw new FleetUsageError('--id must name a worker fleet.json declares.');
+  if (!options.output)
+    throw new FleetUsageError(
+      '--output PATH is required: the seed holds a single-use auth key and is never printed.'
+    );
+  const keyFile =
+    options.publicKey ?? join(services.home, '.ssh/id_ed25519.pub');
+  const managerKey = await readFile(keyFile, 'utf8').catch(() => {
+    throw new FleetUsageError(
+      `No manager public key at ${keyFile}; name one with --public-key PATH.`
+    );
+  });
+  const authKey = await services.secret('tailnet-auth-key');
+  if (!authKey)
+    throw new FleetUsageError(
+      'The secrets provider holds no tailnet-auth-key. Store a short-lived, single-use, tagged Tailscale auth key there first.'
+    );
+  const release = ubuntuRelease(services.definition().ubuntu.release);
+  if (!release)
+    throw new FleetUsageError(
+      'fleet.json names an Ubuntu release the support matrix does not list.'
+    );
+  const hostKey = await generateHostKey(worker.id, services.run);
+  const seed = renderSeed(
+    await seedUserData({
+      worker,
+      managerKey,
+      authKey,
+      hostKey,
+      tailscale: await tailscalePins(),
+      codename: release.codename,
+    }),
+    Boolean(options.autoinstall)
+  );
+  await writeFile(options.output, seed, { mode: 0o600 });
+  const host = await keyName(worker, services.run);
+  const publicKey = hostKey.public.split(' ').slice(0, 2).join(' ');
+  const pin: Check =
+    options.apply && options.yes
+      ? (await services.playbook('macos/worker-host-key.yml', {
+          host,
+          pin: 'present',
+          key: publicKey,
+          known_hosts: services.knownHosts,
+        }))
+        ? {
+            id: 'ssh:host-key',
+            status: 'pass',
+            summary: `The manager trusts the seed's host key for ${host}.`,
+          }
+        : {
+            id: 'ssh:host-key',
+            status: 'fail',
+            summary: 'SSH_HOST_KEY_NOT_CHANGED',
+            remedy: {
+              action:
+                'The pinned Ansible could not change known_hosts; run the command again.',
+            },
+          }
+      : {
+          id: 'ssh:host-key',
+          status: 'skip',
+          summary: `Planned: pin the seed's host key for ${host}; run again with --apply --yes.`,
+        };
+  return {
+    exitCode: pin.status === 'fail' ? 1 : 0,
+    data: {
+      version: 1,
+      command: 'fleet-node-seed',
+      ok: pin.status !== 'fail',
+      worker: worker.id,
+      output: options.output,
+      format: options.autoinstall ? 'autoinstall' : 'cloud-init',
+      hostKey: publicKey,
+      checks: [pin],
+    },
+    text: [
+      `Wrote the ${options.autoinstall ? 'autoinstall' : 'cloud-init'} seed of ${worker.id} to ${options.output} (private; it holds a single-use auth key).`,
+      `Host key: ${publicKey}`,
+      `${pin.status.toUpperCase()} ${pin.id}: ${pin.summary}`,
+      'Boot a stock Ubuntu image with it; once the machine is on the tailnet, add it with fleet-node-add and run devfleet fleet-sync --node ' +
+        worker.id +
+        '.',
+    ].join('\n'),
+  };
+}