@crvouga/dev-fleet 0.7.0 → 0.8.0

package/schema/check-ids.json

@@ -66,6 +66,14 @@
       "id": "auth:vault",
       "meaning": "This machine has its own Vault session."
     },
+    {
+      "id": "auth:vault-isolation",
+      "meaning": "A worker reads its Vault token from a document of its own, not from the fleet's shared one."
+    },
+    {
+      "id": "auth:worker-token",
+      "meaning": "A worker has a Vault token of its own that only it and the manager can read, or a retired worker's token is revoked."
+    },
     {
       "id": "beszel:*",
       "meaning": "A monitoring (Beszel) check of this machine."
@@ -150,6 +158,18 @@
       "id": "configuration:operation",
       "meaning": "The remote configuration operation completed."
     },
+    {
+      "id": "definition:inventory",
+      "meaning": "Ansible reads the inventory the fleet definition describes."
+    },
+    {
+      "id": "definition:playbooks",
+      "meaning": "Every playbook passes ansible-playbook --syntax-check."
+    },
+    {
+      "id": "definition:schema",
+      "meaning": "fleet.json follows every rule of the fleet definition."
+    },
     {
       "id": "enrollment:connection",
       "meaning": "The manager has an enrolled connection to this worker."
@@ -306,6 +326,10 @@
       "id": "rollout:evidence",
       "meaning": "A worker's rollout produced verification evidence."
     },
+    {
+      "id": "rollout:next",
+      "meaning": "Names the command that converges a worker after it was replaced."
+    },
     {
       "id": "rollout:operation",
       "meaning": "The remote rollout operation completed."
@@ -390,6 +414,10 @@
       "id": "state:replica",
       "meaning": "The bucket holds this manager's fleet state."
     },
+    {
+      "id": "state:retired",
+      "meaning": "The fleet state records that a retired worker was retired."
+    },
     {
       "id": "t3:api",
       "meaning": "The local T3 API answers."
@@ -538,6 +566,10 @@
       "id": "worker:package:*",
       "meaning": "The pinned Ubuntu package is installed at or past its pin."
     },
+    {
+      "id": "worker:manifest",
+      "meaning": "A worker is made of what the declared pins and a fresh build of the same revision are made of."
+    },
     {
       "id": "worker:power-loss-recovery",
       "meaning": "The worker powers on after AC loss."

package/schema/fleet.schema.json

@@ -58,7 +58,6 @@
     },
     "workers": {
       "type": "array",
-      "minItems": 1,
       "maxItems": 64,
       "items": {
         "type": "object",
@@ -68,7 +67,7 @@
           "id": {
             "type": "string",
             "pattern": "^[a-z][a-z0-9-]{0,62}$",
-            "description": "A short stable id for the worker. The first worker listed is the canary: a revision new to the fleet must pass there before the others receive it."
+            "description": "A short stable id for the worker. A revision new to the fleet must pass on one worker, the canary, before the others receive it: the one named under canary, or else the first reachable worker in this order."
           },
           "ssh": {
             "type": "string",
@@ -119,6 +118,11 @@
         }
       }
     },
+    "canary": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9-]{0,62}$",
+      "description": "The id of the worker that proves a revision new to the fleet first. When it is unreachable, the next reachable worker proves it instead. Default: the first reachable worker."
+    },
     "services": {
       "type": "object",
       "additionalProperties": false,
@@ -626,6 +630,22 @@
                       "pattern": "^[A-Za-z0-9_]+$"
                     }
                   }
+                },
+                "tailnet-auth-key": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["path", "field"],
+                  "properties": {
+                    "path": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9._/-]+$"
+                    },
+                    "field": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9_]+$"
+                    }
+                  },
+                  "description": "Optional: a short-lived, single-use, tagged Tailscale auth key a new worker joins the tailnet with (fleet-node-seed)."
                 }
               },
               "description": "Required for every secret except github-token, which defaults to the field GITHUB_TOKEN beside proxy-client-key."

package/src/check-owners.ts

@@ -130,6 +130,7 @@ export const CHECK_OWNERS: CheckOwner[] = [
   { pattern: 'github:rate-limit', manual: EVIDENCE },
   { pattern: 'github:workflow', manual: EVIDENCE },
   { pattern: 'fleet:*', manual: DECLARATION },
+  { pattern: 'definition:*', manual: DECLARATION },
   { pattern: 'enrollment:*', manual: DECLARATION },
   { pattern: 'notify:*', manual: DECLARATION },
   {

package/src/cli/fleet-manifest.ts

@@ -0,0 +1,102 @@
+import { readFile } from 'node:fs/promises';
+import { FleetUsageError } from '../errors';
+import { runCommand } from '../runner';
+import type { Runner } from '../types';
+import {
+  declaredDifferences,
+  diffManifests,
+  manifestCheck,
+  manifestDigest,
+  measureManifest,
+  ubuntuPolicy,
+  type WorkerManifest,
+} from '../worker-manifest';
+import type { CommandOutcome } from './contracts';
+import { readRoster } from './roster';
+
+export interface FleetManifestOptions {
+  id?: string;
+  /** A manifest from a fresh build of the same revision (fleet-manifest
+   * --json of that build) to compare the worker with.
+   */
+  reference?: string;
+}
+
+export interface FleetManifestServices {
+  run: Runner;
+  roster: string;
+  root?: string;
+}
+
+export const manifestServices = (roster: string): FleetManifestServices => ({
+  run: runCommand,
+  roster,
+});
+
+async function reference(path: string): Promise<WorkerManifest> {
+  try {
+    const value = JSON.parse(await readFile(path, 'utf8')) as {
+      manifest?: WorkerManifest;
+    } & Partial<WorkerManifest>;
+    const manifest = value.manifest ?? (value as WorkerManifest);
+    if (manifest.version !== 1 || !manifest.packages) throw new Error();
+    return manifest;
+  } catch {
+    throw new FleetUsageError(
+      '--reference must name a manifest written by devfleet fleet-manifest --json.'
+    );
+  }
+}
+
+/** Read-only: measures what an enrolled worker is made of, and the check
+ * worker:manifest against the declared pins and, with --reference, a fresh
+ * build of the same revision.
+ */
+export async function fleetManifest(
+  options: FleetManifestOptions,
+  services: FleetManifestServices
+): Promise<CommandOutcome> {
+  const roster = await readRoster(services.roster);
+  const node = roster?.nodes.find((entry) => entry.id === options.id);
+  if (!node) throw new FleetUsageError('--id must name an enrolled worker.');
+  const fresh = options.reference
+    ? await reference(options.reference)
+    : undefined;
+  const policy = await ubuntuPolicy(services.root);
+  const measured = await measureManifest(node, services.run, policy);
+  if ('check' in measured)
+    return {
+      exitCode: 1,
+      data: {
+        version: 1,
+        command: 'fleet-manifest',
+        ok: false,
+        node: node.id,
+        checks: [measured.check],
+      },
+    };
+  const { manifest } = measured;
+  const differences = [
+    ...declaredDifferences(manifest, policy),
+    ...(fresh ? diffManifests(fresh, manifest) : []),
+  ];
+  const check = manifestCheck(differences);
+  const digest = manifestDigest(manifest);
+  return {
+    exitCode: check.status === 'fail' ? 1 : 0,
+    data: {
+      version: 1,
+      command: 'fleet-manifest',
+      ok: check.status !== 'fail',
+      node: node.id,
+      digest,
+      manifest,
+      differences,
+      checks: [check],
+    },
+    text: [
+      `${node.id}: ${Object.keys(manifest.packages).length} packages, ${Object.keys(manifest.tools).length} tools, digest ${digest}`,
+      `${check.status.toUpperCase()} ${check.id}: ${check.summary}`,
+    ].join('\n'),
+  };
+}

package/src/cli/fleet-node-lifecycle.ts

@@ -0,0 +1,455 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { parseConfig } from '../config';
+import type { FleetDefinition, MachineDefinition } from '../definition';
+import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import type { Enrollment } from '../machine/state';
+import { PACKAGE_ROOT } from '../package-info';
+import { runAnsible } from '../platform/ansible';
+import { makeReport, renderReport } from '../report';
+import { runCommand } from '../runner';
+import { loadEnrollment, storeEnrollment } from '../state/enrollment';
+import { STATE_ROSTER, writeRosterText } from '../state/roster';
+import { mutateState, stateContext, type StateContext } from '../state/sync';
+import { discoverTailnet, type TailnetComputer } from '../tailscale';
+import type { Check, FleetConfig, Runner } from '../types';
+import { usesVault } from '../vault-config';
+import {
+  issueWorkerToken,
+  revokeWorkerToken,
+  workerTokenLocation,
+} from '../worker-credentials';
+import type { CommandOutcome } from './contracts';
+import { readRoster } from './roster';
+
+export interface NodeLifecycleOptions {
+  id?: string;
+  /** The replacement's SSH host key, `TYPE BASE64`, verified out of band or
+   * written by the seed that created it.
+   */
+  hostKey?: string;
+  apply?: boolean;
+  yes?: boolean;
+}
+
+export interface NodeLifecycleServices {
+  run: Runner;
+  /** The roster: the fleet state's, or a file `--config` names. */
+  roster: string;
+  definition: () => FleetDefinition;
+  vault: () => boolean;
+  /** The manager's known_hosts file. */
+  knownHosts: string;
+  state: () => StateContext;
+  /** Applies a playbook of the package with these variables. */
+  playbook: (name: string, variables: object) => Promise<boolean>;
+  now: () => Date;
+}
+
+export function lifecycleServices(roster: string): NodeLifecycleServices {
+  return {
+    run: runCommand,
+    roster,
+    definition: () => fleet,
+    vault: usesVault,
+    knownHosts: join(homedir(), '.ssh/known_hosts'),
+    state: () => stateContext(runCommand),
+    playbook: async (name, variables) =>
+      (
+        await runAnsible(runCommand, join(PACKAGE_ROOT, 'ansible', name), {
+          input: JSON.stringify(variables),
+          timeoutMs: 300_000,
+        })
+      ).ok,
+    now: () => new Date(),
+  };
+}
+
+function declared(
+  definition: FleetDefinition,
+  id: string | undefined
+): MachineDefinition {
+  const worker = definition.workers.find((entry) => entry.id === id);
+  if (!worker)
+    throw new FleetUsageError(
+      '--id must name a worker fleet.json declares. To retire a worker, retire it first and then remove it from fleet.json.'
+    );
+  return worker;
+}
+
+/** The name ssh checks the host key of: the HostName the alias resolves to,
+ * as `[host]:port` when the port is not 22.
+ */
+export async function keyName(
+  worker: MachineDefinition,
+  run: Runner
+): Promise<string> {
+  const resolved = await run(['ssh', '-G', worker.ssh], { timeoutMs: 10_000 });
+  const option = (name: string) =>
+    resolved.ok
+      ? new RegExp(`^${name} (\\S+)$`, 'm').exec(resolved.stdout)?.[1]
+      : undefined;
+  const host =
+    option('hostname') ?? worker.network?.tailscaleName ?? worker.ssh;
+  const port = option('port') ?? String(worker.network?.sshPort ?? 22);
+  return port === '22' ? host : `[${host}]:${port}`;
+}
+
+/** Changes the roster in the fleet state (or the file `--config` names). */
+async function changeRoster(
+  services: NodeLifecycleServices,
+  change: (roster: FleetConfig | null) => FleetConfig | null
+): Promise<void> {
+  const save = async () => {
+    const next = change(await readRoster(services.roster));
+    if (next)
+      await writeRosterText(
+        services.roster,
+        `${JSON.stringify(parseConfig(next), null, 2)}\n`
+      );
+  };
+  if (services.roster === STATE_ROSTER)
+    await mutateState(services.state(), save);
+  else await save();
+}
+
+function outcome(
+  command: string,
+  id: string,
+  checks: Check[],
+  applied: boolean
+): CommandOutcome {
+  const report = { ...makeReport(command, [{ node: id, checks }]), applied };
+  return {
+    exitCode: checks.some((check) => check.status === 'fail') ? 1 : 0,
+    data: report,
+    text: renderReport(report),
+  };
+}
+
+const planned = (id: string, summary: string): Check => ({
+  id,
+  status: 'skip',
+  summary: `Planned: ${summary}`,
+});
+
+const preview = (options: NodeLifecycleOptions): boolean =>
+  !(options.apply && options.yes);
+
+/** Takes a worker out of the fleet for good: revokes its Vault token, forgets
+ * its pinned host key, removes its roster entry and leaves a tombstone in the
+ * fleet state. Preview until --apply --yes. Running it again changes nothing.
+ */
+export async function fleetNodeRetire(
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+): Promise<CommandOutcome> {
+  const worker = declared(services.definition(), options.id);
+  const roster = await readRoster(services.roster);
+  const enrolled = roster?.nodes.some((node) => node.id === worker.id) ?? false;
+  if (enrolled && roster!.nodes.length === 1)
+    throw new FleetUsageError(
+      'The roster must keep at least one worker. Add the replacement with fleet-node-add first.'
+    );
+  const host = await keyName(worker, services.run);
+  if (preview(options))
+    return outcome(
+      'fleet-node-retire',
+      worker.id,
+      [
+        ...(services.vault()
+          ? [planned('auth:worker-token', `revoke ${worker.id}'s Vault token`)]
+          : []),
+        planned('ssh:host-key', `forget the host key of ${host}`),
+        planned(
+          'enrollment:connection',
+          enrolled
+            ? `remove ${worker.id} from the roster`
+            : `${worker.id} is not in the roster`
+        ),
+        planned('state:retired', `record that ${worker.id} is retired`),
+      ],
+      false
+    );
+  const checks: Check[] = [];
+  if (services.vault())
+    checks.push(await revokeWorkerToken(worker.id, services.run));
+  checks.push(
+    (await services.playbook('macos/worker-host-key.yml', {
+      host,
+      pin: 'absent',
+      known_hosts: services.knownHosts,
+    }))
+      ? {
+          id: 'ssh:host-key',
+          status: 'pass',
+          summary: `The manager no longer trusts any host key of ${host}.`,
+        }
+      : hostKeyFailure()
+  );
+  await changeRoster(services, (current) =>
+    current && current.nodes.some((node) => node.id === worker.id)
+      ? {
+          ...current,
+          nodes: current.nodes.filter((node) => node.id !== worker.id),
+        }
+      : null
+  );
+  checks.push({
+    id: 'enrollment:connection',
+    status: 'pass',
+    summary: `${worker.id} is not in the roster.`,
+  });
+  checks.push(await tombstone(worker.id, services));
+  return outcome('fleet-node-retire', worker.id, checks, true);
+}
+
+async function tombstone(
+  id: string,
+  services: NodeLifecycleServices
+): Promise<Check> {
+  const record = async (): Promise<Enrollment | null> => {
+    const enrollment = await loadEnrollment(null);
+    if (!enrollment) return null;
+    const { [id]: _applied, ...workers } = enrollment.workers ?? {};
+    const retired = {
+      ...enrollment,
+      workers,
+      retired: {
+        ...enrollment.retired,
+        [id]: enrollment.retired?.[id] ?? services.now().toISOString(),
+      },
+    };
+    storeEnrollment(retired);
+    return retired;
+  };
+  const stored = (await mutateState(services.state(), record)).value;
+  return stored
+    ? {
+        id: 'state:retired',
+        status: 'pass',
+        summary: `The fleet state records that ${id} was retired at ${stored.retired![id]}.`,
+      }
+    : {
+        id: 'state:retired',
+        status: 'skip',
+        summary:
+          'No revision was ever applied to this fleet, so there is nothing to record.',
+      };
+}
+
+/** Puts a fresh machine in place of a worker under the same id: verifies the
+ * new device on the tailnet, pins its host key, binds the roster entry to the
+ * new device and issues it a token of its own (revoking the old one). Preview
+ * until --apply --yes; then `fleet-sync --node ID` brings it to ready. When
+ * nothing changed (the same device, the same key, a token of its own) it
+ * changes nothing.
+ */
+export async function fleetNodeReplace(
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+): Promise<CommandOutcome> {
+  const worker = declared(services.definition(), options.id);
+  const entry = await replaceable(worker, options, services);
+  const device = await newDevice(worker, services.run);
+  if ('check' in device)
+    return outcome('fleet-node-replace', worker.id, [device.check], false);
+  const plan: Replacement = {
+    worker,
+    device: device.computer.id,
+    host: await keyName(worker, services.run),
+    hostKey: options.hostKey,
+    replaced: entry.tailscaleId !== device.computer.id,
+    needsToken: false,
+  };
+  plan.needsToken =
+    services.vault() &&
+    (plan.replaced || !(await ownToken(worker.id, services.run)));
+  const identity: Check = {
+    id: 'tailscale:identity',
+    status: 'pass',
+    summary: `${worker.id} is the online device ${plan.device}${plan.replaced ? `, replacing ${entry.tailscaleId ?? 'none'}` : ''}.`,
+  };
+  if (preview(options))
+    return outcome(
+      'fleet-node-replace',
+      worker.id,
+      [identity, ...planReplacement(plan)],
+      false
+    );
+  return outcome(
+    'fleet-node-replace',
+    worker.id,
+    [identity, ...(await carryOut(plan, services))],
+    true
+  );
+}
+
+interface Replacement {
+  worker: MachineDefinition;
+  device: string;
+  host: string;
+  hostKey: string | undefined;
+  replaced: boolean;
+  needsToken: boolean;
+}
+
+async function replaceable(
+  worker: MachineDefinition,
+  options: NodeLifecycleOptions,
+  services: NodeLifecycleServices
+) {
+  if (
+    options.hostKey !== undefined &&
+    !/^(ssh-ed25519|ecdsa-sha2-nistp256|ssh-rsa) [A-Za-z0-9+/=]{16,}$/.test(
+      options.hostKey
+    )
+  )
+    throw new FleetUsageError(
+      '--host-key expects one public key, "TYPE BASE64", as ssh-keyscan or the seed prints it.'
+    );
+  const roster = await readRoster(services.roster);
+  const entry = roster?.nodes.find((node) => node.id === worker.id);
+  if (!entry)
+    throw new FleetUsageError(
+      `${worker.id} is not in the roster. Add it with fleet-node-add first.`
+    );
+  return entry;
+}
+
+function planReplacement(plan: Replacement): Check[] {
+  const steps: Check[] = [];
+  if (plan.hostKey)
+    steps.push(planned('ssh:host-key', `pin the host key of ${plan.host}`));
+  if (plan.replaced)
+    steps.push(
+      planned(
+        'enrollment:connection',
+        `bind ${plan.worker.id} to ${plan.device}`
+      )
+    );
+  if (plan.needsToken)
+    steps.push(
+      planned('auth:worker-token', `issue ${plan.worker.id} a token of its own`)
+    );
+  return steps;
+}
+
+async function carryOut(
+  plan: Replacement,
+  services: NodeLifecycleServices
+): Promise<Check[]> {
+  const { worker } = plan;
+  const steps: Check[] = [];
+  if (plan.hostKey)
+    steps.push(
+      (await services.playbook('macos/worker-host-key.yml', {
+        host: plan.host,
+        pin: 'present',
+        key: plan.hostKey,
+        known_hosts: services.knownHosts,
+      }))
+        ? {
+            id: 'ssh:host-key',
+            status: 'pass',
+            summary: `The manager trusts exactly the given host key of ${plan.host}.`,
+          }
+        : hostKeyFailure()
+    );
+  if (plan.replaced) {
+    await changeRoster(services, (current) =>
+      current
+        ? {
+            ...current,
+            nodes: current.nodes.map((node) =>
+              node.id === worker.id
+                ? { ...node, tailscaleId: plan.device }
+                : node
+            ),
+          }
+        : null
+    );
+    steps.push({
+      id: 'enrollment:connection',
+      status: 'pass',
+      summary: `The roster binds ${worker.id} to the device ${plan.device}.`,
+    });
+  }
+  if (plan.needsToken) {
+    const revoked = await revokeWorkerToken(worker.id, services.run);
+    steps.push(
+      revoked.status === 'fail'
+        ? revoked
+        : await issueWorkerToken(worker.id, services.run)
+    );
+  }
+  steps.push({
+    id: 'rollout:next',
+    status: 'pass',
+    summary: `Run devfleet fleet-sync --node ${worker.id} to converge the new machine.`,
+  });
+  return steps;
+}
+
+async function ownToken(id: string, run: Runner): Promise<boolean> {
+  const location = workerTokenLocation(id);
+  return (
+    await run(
+      [
+        'vault',
+        'kv',
+        'get',
+        `-mount=${location.mount}`,
+        '-field=accessor',
+        location.path,
+      ],
+      { timeoutMs: 20_000 }
+    )
+  ).ok;
+}
+
+/** The one online computer on the tailnet that carries the worker's name. */
+async function newDevice(
+  worker: MachineDefinition,
+  run: Runner
+): Promise<{ computer: TailnetComputer } | { check: Check }> {
+  const tailnet = await discoverTailnet(run);
+  const name = (worker.network?.tailscaleName ?? worker.ssh).toLowerCase();
+  const short = name.split('.')[0]!;
+  const found = tailnet.computers.filter(
+    (computer) =>
+      !computer.self &&
+      computer.online &&
+      [computer.dns.toLowerCase(), computer.name.toLowerCase()].some(
+        (candidate) => candidate === name || candidate.split('.')[0] === short
+      )
+  );
+  if (found.length === 1) return { computer: found[0]! };
+  return {
+    check: {
+      id: 'tailscale:identity',
+      status: 'fail',
+      summary: found.length
+        ? 'TAILSCALE_IDENTITY_CONFLICT'
+        : 'TAILSCALE_NODE_MISSING',
+      remedy: {
+        action: found.length
+          ? `More than one online device is named like ${worker.id}. Remove the old device from the tailnet, then run the command again.`
+          : `No online device is named like ${worker.id}. Start the new machine, join it to the tailnet under that name, then run the command again.`,
+      },
+    },
+  };
+}
+
+function hostKeyFailure(): Check {
+  return {
+    id: 'ssh:host-key',
+    status: 'fail',
+    summary: 'SSH_HOST_KEY_NOT_CHANGED',
+    remedy: {
+      action:
+        'The pinned Ansible could not change known_hosts on this manager. Run devfleet fleet-validate --ansible, then this command again.',
+    },
+  };
+}