@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/tailnet-cleanup.ts

@@ -0,0 +1,185 @@
+import type { FleetDefinition, MachineDefinition } from './definition';
+import type { SecretsProvider } from './secrets';
+import {
+  TailnetControlError,
+  controlRemedy,
+  fleetTailnetControl,
+  type ControlHttp,
+  type TailnetConfig,
+  type TailnetControl,
+} from './tailnet-control';
+import type { Check } from './types';
+
+/** Removes the tailnet device a replaced or retired worker ran as, so the
+ * next machine under its id gets the declared name. Asked through the
+ * control plane fleet.json declares (`tailnet`), with the `tailnet-admin`
+ * credential; the credential is read when a device is removed, never kept.
+ */
+export interface TailnetCleanup {
+  /** Removes the device `deviceId` that `worker` ran as; a check
+   * `tailscale:device` that says whether it is gone.
+   */
+  removeDevice(worker: MachineDefinition, deviceId: string): Promise<Check>;
+}
+
+/** The device left to a person: no control plane is declared, or no
+ * credential to ask it with is stored.
+ */
+export function deviceLeftToPerson(
+  worker: MachineDefinition,
+  device: string,
+  why = 'no tailnet control is configured'
+): Check {
+  return {
+    id: 'tailscale:device',
+    status: 'skip',
+    summary: `The tailnet device ${device} was not removed: ${why}.`,
+    remedy: {
+      action: `Remove the device ${device} in the tailnet's admin console, so that a new machine gets the name ${worker.network?.tailscaleName ?? worker.id}. To have it removed by itself, declare tailnet in fleet.json and store the secret tailnet-admin.`,
+    },
+  };
+}
+
+/** The cleanup through the tailnet control plane a fleet declares. */
+export function tailnetCleanup(
+  config: TailnetConfig,
+  secret: SecretsProvider,
+  http?: ControlHttp
+): TailnetCleanup {
+  return {
+    async removeDevice(worker, device) {
+      let control: TailnetControl | null;
+      try {
+        control = await fleetTailnetControl(config, secret, http);
+      } catch (error) {
+        if (
+          error instanceof TailnetControlError &&
+          error.code === 'TAILNET_ADMIN_REQUIRED'
+        )
+          return deviceLeftToPerson(
+            worker,
+            device,
+            'the secrets provider holds no tailnet-admin'
+          );
+        throw error;
+      }
+      if (!control) return deviceLeftToPerson(worker, device);
+      try {
+        await control.deleteDevice(device);
+      } catch (error) {
+        if (!(error instanceof TailnetControlError)) throw error;
+        // A device that is already gone is what was asked for.
+        if (!(await stillListed(control, config.tag, device)))
+          return removed(device);
+        return {
+          id: 'tailscale:device',
+          status: 'warn',
+          summary: error.code,
+          remedy: {
+            action: `${controlRemedy(error.code)} Or remove the device ${device} in the tailnet's admin console.`,
+          },
+        };
+      }
+      return removed(device);
+    },
+  };
+}
+
+const removed = (device: string): Check => ({
+  id: 'tailscale:device',
+  status: 'pass',
+  summary: `The tailnet no longer holds the device ${device}.`,
+});
+
+async function stillListed(
+  control: TailnetControl,
+  tag: string,
+  device: string
+): Promise<boolean> {
+  try {
+    return (await control.listDevices(tag)).some(
+      (entry) => entry.id === device
+    );
+  } catch {
+    return true;
+  }
+}
+
+/** A device name as the control plane gives it, for a report: one DNS-like
+ * name, or nothing. Never anything else the control plane answered.
+ */
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
+const label = (name: string): string =>
+  name.toLowerCase().replace(/\.$/, '').split('.')[0]!;
+
+/** Devices that carry the fleet's worker tag and are no declared worker
+ * (parked ones included) nor a device the roster binds: a machine left over
+ * from a rebuild, or one joined by hand. Each is a warning `tailscale:leftover`
+ * that names the device by its name only. A fleet without a control plane,
+ * or without the credential to ask it, says nothing here.
+ */
+export async function tailnetLeftovers(
+  definition: FleetDefinition,
+  control: () => Promise<TailnetControl | null>,
+  bound: string[] = []
+): Promise<Check[]> {
+  const tag = definition.tailnet?.tag;
+  if (!tag) return [];
+  let devices;
+  try {
+    const found = await control();
+    if (!found) return [];
+    devices = await found.listDevices(tag);
+  } catch (error) {
+    if (!(error instanceof TailnetControlError)) throw error;
+    if (error.code === 'TAILNET_ADMIN_REQUIRED') return [];
+    return [
+      {
+        id: 'tailscale:leftover',
+        status: 'skip',
+        summary: error.code,
+        remedy: { action: controlRemedy(error.code) },
+      },
+    ];
+  }
+  const declared = new Set(
+    [definition.manager, ...definition.workers].flatMap((machine) => [
+      machine.id,
+      ...(machine.network ? [label(machine.network.tailscaleName)] : []),
+    ])
+  );
+  return devices
+    .filter(
+      (device) =>
+        !bound.includes(device.id) && !declared.has(label(device.name))
+    )
+    .map((device) => {
+      const name = NAME.test(device.name) ? device.name : 'without a name';
+      return {
+        id: 'tailscale:leftover',
+        status: 'warn' as const,
+        summary: `The tailnet holds the device ${name}, tagged ${tag}, which is no declared worker.`,
+        remedy: {
+          action: `If the device ${name} left the fleet, remove it in the tailnet's admin console. If it is a worker, declare it in fleet.json (or its worker record).`,
+        },
+      };
+    });
+}
+
+/** Adds the leftover devices to a fleet-status report, as their own
+ * `tailnet` entry; nothing when there are none.
+ */
+export async function appendTailnetLeftovers(
+  reports: { node: string; checks: Check[] }[],
+  definition: FleetDefinition,
+  control: (() => Promise<TailnetControl | null>) | undefined,
+  nodes: { tailscaleId?: string }[]
+): Promise<void> {
+  if (!control) return;
+  const checks = await tailnetLeftovers(
+    definition,
+    control,
+    nodes.flatMap((node) => (node.tailscaleId ? [node.tailscaleId] : []))
+  );
+  if (checks.length) reports.push({ node: 'tailnet', checks });
+}

package/src/tailnet-control.ts

@@ -0,0 +1,346 @@
+import { record } from './config';
+import type { SecretsProvider } from './secrets';
+
+/** The tailnet control plane a fleet enrols its workers with: the hosted
+ * Tailscale (API v2) or a self-hosted Headscale (its REST API). The contract
+ * is "a control plane that can issue pre-authorised keys": for each new
+ * worker dev-fleet asks it for one key that is single-use, pre-authorised,
+ * tagged and expires within minutes. The credential that asks is the secret
+ * `tailnet-admin`; it stays on the manager and never reaches a worker.
+ */
+export type TailnetConfig =
+  | {
+      api: 'tailscale';
+      /** The API origin; https://api.tailscale.com when not named. */
+      url?: string;
+      tag: string;
+    }
+  | {
+      api: 'headscale';
+      /** The Headscale server: its API, and the login server workers use. */
+      url: string;
+      tag: string;
+      /** The Headscale user the keys belong to (its ID from Headscale 0.26). */
+      user: string;
+    };
+
+const ORIGIN = /^https:\/\/[A-Za-z0-9.-]{1,253}(:[0-9]{1,5})?$/;
+const TAG = /^tag:[a-z0-9][a-z0-9-]{0,62}$/;
+const USER = /^[A-Za-z0-9._@-]{1,64}$/;
+const TAILSCALE_API = 'https://api.tailscale.com';
+
+/** The `tailnet` block of a fleet configuration, or null when it is invalid. */
+export function parseTailnetConfig(value: unknown): TailnetConfig | null {
+  if (!record(value)) return null;
+  const { api, url, tag, user } = value;
+  const allowed =
+    api === 'headscale' ? ['api', 'url', 'tag', 'user'] : ['api', 'url', 'tag'];
+  if (Object.keys(value).some((key) => !allowed.includes(key))) return null;
+  if (!matches(tag, TAG) || (url !== undefined && !matches(url, ORIGIN)))
+    return null;
+  if (api === 'tailscale')
+    return { api, tag, ...(typeof url === 'string' ? { url } : {}) };
+  return api === 'headscale' && typeof url === 'string' && matches(user, USER)
+    ? { api, url, tag, user }
+    : null;
+}
+
+function matches(value: unknown, pattern: RegExp): value is string {
+  return typeof value === 'string' && pattern.test(value);
+}
+
+/** The server a worker logs in to; null for the hosted Tailscale, whose
+ * client already knows it.
+ */
+export function loginServer(config: TailnetConfig | undefined): string | null {
+  return config?.api === 'headscale' ? config.url : null;
+}
+
+/** One request to the control plane, and its answer. Injected in scenarios. */
+export interface ControlRequest {
+  method: 'GET' | 'POST' | 'DELETE';
+  url: string;
+  headers: Record<string, string>;
+  body?: string;
+}
+export interface ControlResponse {
+  status: number;
+  /** The parsed JSON body; null when there is none. */
+  body: unknown;
+}
+export type ControlHttp = (request: ControlRequest) => Promise<ControlResponse>;
+
+/** Why the control plane did not do what was asked. Never carries what it
+ * answered: a body may echo the credential or a key.
+ */
+export class TailnetControlError extends Error {
+  constructor(
+    readonly code:
+      | 'TAILNET_ADMIN_REQUIRED'
+      | 'TAILNET_CONTROL_DENIED'
+      | 'TAILNET_CONTROL_REJECTED'
+      | 'TAILNET_CONTROL_UNAVAILABLE'
+      | 'TAILNET_CONTROL_RESPONSE_INVALID'
+  ) {
+    super(code);
+  }
+}
+
+export interface AuthKey {
+  /** Delivered to one worker only: in its seed, or on the bootstrap's stdin. */
+  key: string;
+  expires: string;
+}
+export interface TailnetDevice {
+  /** The stable node ID: what `tailscale status --json` calls ID. */
+  id: string;
+  name: string;
+  tags: string[];
+}
+export interface TailnetControl {
+  /** One key: single-use, pre-authorised, carrying the fleet's tag and
+   * expiring after `expirySeconds` (ten minutes when not named).
+   */
+  createAuthKey(request: {
+    worker: string;
+    expirySeconds?: number;
+  }): Promise<AuthKey>;
+  /** Every device that carries the tag. */
+  listDevices(tag: string): Promise<TailnetDevice[]>;
+  deleteDevice(id: string): Promise<void>;
+  /** A worker's node key never expires, so nobody renews it. Tagged devices
+   * already have key expiry disabled on the hosted Tailscale; this disables
+   * it for a device that has it. Headscale gives a node registered with a
+   * pre-authorised key no expiry, so there it changes nothing.
+   */
+  disableKeyExpiry(id: string): Promise<void>;
+}
+
+const KEY = /^[A-Za-z0-9_-]{16,256}$/;
+const ID = /^[A-Za-z0-9_-]{1,128}$/;
+const DEFAULT_EXPIRY_SECONDS = 600;
+
+/** The control plane `config` names, asked with `credential`. */
+export function tailnetControl(
+  config: TailnetConfig,
+  credential: string,
+  http: ControlHttp = fetchHttp
+): TailnetControl {
+  const origin = config.url ?? TAILSCALE_API;
+  const ask = async (
+    method: ControlRequest['method'],
+    path: string,
+    body?: object
+  ): Promise<unknown> => {
+    let response: ControlResponse;
+    try {
+      response = await http({
+        method,
+        url: `${origin}${path}`,
+        headers: {
+          authorization: `Bearer ${credential}`,
+          accept: 'application/json',
+          ...(body ? { 'content-type': 'application/json' } : {}),
+        },
+        ...(body ? { body: JSON.stringify(body) } : {}),
+      });
+    } catch {
+      throw new TailnetControlError('TAILNET_CONTROL_UNAVAILABLE');
+    }
+    if (response.status === 401 || response.status === 403)
+      throw new TailnetControlError('TAILNET_CONTROL_DENIED');
+    if (response.status >= 500 || response.status === 429)
+      throw new TailnetControlError('TAILNET_CONTROL_UNAVAILABLE');
+    if (response.status < 200 || response.status > 299)
+      throw new TailnetControlError('TAILNET_CONTROL_REJECTED');
+    return response.body;
+  };
+  const device = (id: string) => {
+    if (!ID.test(id)) throw new TailnetControlError('TAILNET_CONTROL_REJECTED');
+    return encodeURIComponent(id);
+  };
+  return config.api === 'tailscale'
+    ? tailscaleApi(config, ask, device)
+    : headscaleApi(config, ask, device);
+}
+
+type Ask = (
+  method: ControlRequest['method'],
+  path: string,
+  body?: object
+) => Promise<unknown>;
+
+function seconds(request: { expirySeconds?: number }): number {
+  const value = request.expirySeconds ?? DEFAULT_EXPIRY_SECONDS;
+  // Minutes, never hours: a key that leaks is useless soon after.
+  return Math.min(Math.max(Math.floor(value), 60), 3600);
+}
+
+function authKey(key: unknown, expires: unknown): AuthKey {
+  if (typeof key !== 'string' || !KEY.test(key) || typeof expires !== 'string')
+    throw new TailnetControlError('TAILNET_CONTROL_RESPONSE_INVALID');
+  return { key, expires };
+}
+
+function strings(value: unknown): string[] {
+  return Array.isArray(value)
+    ? value.filter((item): item is string => typeof item === 'string')
+    : [];
+}
+
+/** Tailscale API v2: https://tailscale.com/api. `-` is the tailnet of the
+ * credential.
+ */
+function tailscaleApi(
+  config: TailnetConfig,
+  ask: Ask,
+  device: (id: string) => string
+): TailnetControl {
+  return {
+    async createAuthKey(request) {
+      const answer = await ask('POST', '/api/v2/tailnet/-/keys', {
+        capabilities: {
+          devices: {
+            create: {
+              reusable: false,
+              ephemeral: false,
+              preauthorized: true,
+              tags: [config.tag],
+            },
+          },
+        },
+        expirySeconds: seconds(request),
+        description: `devfleet ${request.worker}`.slice(0, 50),
+      });
+      if (!record(answer))
+        throw new TailnetControlError('TAILNET_CONTROL_RESPONSE_INVALID');
+      return authKey(answer['key'], answer['expires']);
+    },
+    async listDevices(tag) {
+      const answer = await ask('GET', '/api/v2/tailnet/-/devices');
+      if (!record(answer) || !Array.isArray(answer['devices']))
+        throw new TailnetControlError('TAILNET_CONTROL_RESPONSE_INVALID');
+      return answer['devices'].filter(record).flatMap((entry) => {
+        const id = entry['nodeId'] ?? entry['id'];
+        const tags = strings(entry['tags']);
+        return typeof id === 'string' && tags.includes(tag)
+          ? [{ id, name: String(entry['hostname'] ?? entry['name']), tags }]
+          : [];
+      });
+    },
+    async deleteDevice(id) {
+      await ask('DELETE', `/api/v2/device/${device(id)}`);
+    },
+    async disableKeyExpiry(id) {
+      await ask('POST', `/api/v2/device/${device(id)}/key`, {
+        keyExpiryDisabled: true,
+      });
+    },
+  };
+}
+
+/** Headscale's REST API (/api/v1), asked with one of its API keys. */
+function headscaleApi(
+  config: Extract<TailnetConfig, { api: 'headscale' }>,
+  ask: Ask,
+  device: (id: string) => string
+): TailnetControl {
+  return {
+    async createAuthKey(request) {
+      const expiration = new Date(
+        Date.now() + seconds(request) * 1000
+      ).toISOString();
+      const answer = await ask('POST', '/api/v1/preauthkey', {
+        user: config.user,
+        reusable: false,
+        ephemeral: false,
+        expiration,
+        aclTags: [config.tag],
+      });
+      const key = record(answer) ? answer['preAuthKey'] : null;
+      if (!record(key))
+        throw new TailnetControlError('TAILNET_CONTROL_RESPONSE_INVALID');
+      return authKey(key['key'], key['expiration'] ?? expiration);
+    },
+    async listDevices(tag) {
+      const answer = await ask('GET', '/api/v1/node');
+      if (!record(answer) || !Array.isArray(answer['nodes']))
+        throw new TailnetControlError('TAILNET_CONTROL_RESPONSE_INVALID');
+      return answer['nodes'].filter(record).flatMap((entry) => {
+        const id = entry['id'];
+        // Headscale has named a node's tags differently across releases.
+        const tags = [
+          ...new Set([
+            ...strings(entry['tags']),
+            ...strings(entry['forcedTags']),
+            ...strings(entry['validTags']),
+          ]),
+        ];
+        return (typeof id === 'string' || typeof id === 'number') &&
+          tags.includes(tag)
+          ? [
+              {
+                id: String(id),
+                name: String(entry['givenName'] ?? entry['name']),
+                tags,
+              },
+            ]
+          : [];
+      });
+    },
+    async deleteDevice(id) {
+      await ask('DELETE', `/api/v1/node/${device(id)}`);
+    },
+    async disableKeyExpiry(id) {
+      device(id);
+    },
+  };
+}
+
+/** The fleet's control plane, or null when it declares none. A declared one
+ * without its credential is refused: nothing is enrolled on a guess.
+ */
+export async function fleetTailnetControl(
+  config: TailnetConfig | undefined,
+  secret: SecretsProvider,
+  http?: ControlHttp
+): Promise<TailnetControl | null> {
+  if (!config) return null;
+  const credential = await secret('tailnet-admin');
+  if (!credential) throw new TailnetControlError('TAILNET_ADMIN_REQUIRED');
+  return tailnetControl(config, credential, http);
+}
+
+/** What each code asks the operator to do. Never what the server answered. */
+export function controlRemedy(code: TailnetControlError['code']): string {
+  return {
+    TAILNET_ADMIN_REQUIRED:
+      'fleet.json declares a tailnet control plane, but the secrets provider holds no tailnet-admin. Store an API credential that can create auth keys there (a Tailscale API access token or OAuth client secret with the auth_keys and devices scopes, or a Headscale API key), then rerun.',
+    TAILNET_CONTROL_DENIED:
+      'The control plane refused the tailnet-admin credential. Replace it with one that is valid and may create tagged auth keys for the fleet tag, then rerun.',
+    TAILNET_CONTROL_REJECTED:
+      'The control plane refused the request. Check that the tag in fleet.json (tailnet.tag) exists and that the credential may use it, and for Headscale that tailnet.user names an existing user, then rerun.',
+    TAILNET_CONTROL_UNAVAILABLE:
+      'The control plane could not be reached or is busy. Check tailnet.url and the network, then rerun.',
+    TAILNET_CONTROL_RESPONSE_INVALID:
+      'The control plane answered with something that is not an auth key or a device list. Check that tailnet.api names the right kind of server, then rerun.',
+  }[code];
+}
+
+const fetchHttp: ControlHttp = async (request) => {
+  const response = await fetch(request.url, {
+    method: request.method,
+    headers: request.headers,
+    ...(request.body ? { body: request.body } : {}),
+    redirect: 'error',
+    signal: AbortSignal.timeout(15_000),
+  });
+  const text = await response.text();
+  let body: unknown = null;
+  try {
+    body = text ? (JSON.parse(text) as unknown) : null;
+  } catch {
+    body = null;
+  }
+  return { status: response.status, body };
+};

package/src/tailscale.ts

@@ -167,7 +167,7 @@ export function keyExpiryCheck(
   const remedy = {
     action: computer.self
       ? 'Renew the node key of the manager before it expires: tailscale up --force-reauth on the manager, or disable key expiry for it in the Tailscale admin console.'
-      : 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
+      : 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). With tailnet and the secret tailnet-admin declared, fleet-sync enrols workers tagged and disables expiry for an enrolled worker that still has one. After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
   };
   if (days < 0)
     return { id, status: 'fail', summary: 'TAILSCALE_KEY_EXPIRED', remedy };

package/src/toolchain-checks.ts

@@ -40,12 +40,21 @@ export async function toolchainChecks(
     return [
       ...(!result.ok || !data['ok'] ? [fallback] : []),
       ...(checks as Check[]),
+      ...(worker?.containers === false ? [dockerCliNotApplicable] : []),
     ];
   } catch {
     return [fallback];
   }
 }
 
+/** The `containers` feature is off: the base tools leave the Docker CLI out. */
+const dockerCliNotApplicable: Check = {
+  id: 'tool:docker-cli',
+  status: 'not-applicable',
+  summary:
+    'Not applicable: containers are off for this worker (features.containers), so the Docker CLI is not installed.',
+};
+
 /** Checks of the doctor that are not a tool version, with their own words. */
 const OWN: Record<string, { pass: string; fail: string; action: string }> = {
   'global-mise-config': {
@@ -107,7 +116,8 @@ function doctorArguments(root: string, worker?: ToolSelection): string[] {
     'bun',
     join(root, 'bootstrap/doctor.ts'),
     '--json',
-    '--docker',
+    // A worker without containers has no Docker engine to require.
+    ...(worker?.containers === false ? [] : ['--docker']),
     ...(worker ? ['--worker', ...selectionArguments(worker)] : []),
   ];
 }

package/src/types.ts

@@ -1,4 +1,4 @@
-export type Status = 'pass' | 'warn' | 'fail' | 'skip';
+export type Status = 'pass' | 'warn' | 'fail' | 'skip' | 'not-applicable';
 export type Capability =
   | 'github'
   | 'vault'
@@ -40,7 +40,9 @@ export interface Check {
 
 /** Only a required failure blocks readiness. Warnings (conditions outside the
  * fleet, such as a provider quota) and skips (not observable here) are shown
- * but never fail a run.
+ * but never fail a run. A not-applicable check (the summary says why: a
+ * virtual machine has no firmware power setting) never fails, not even under
+ * --strict.
  */
 export function blocking(check: Check): boolean {
   return check.status === 'fail' && check.required !== false;

package/src/worker-account.ts

@@ -0,0 +1,63 @@
+import type { PackagePlaybook } from './platform/ansible';
+import { playbookConnection } from './ssh';
+import type { Check } from './types';
+
+/** The codes ansible/ubuntu/worker-account.yml stops with. */
+const CODES = [
+  'WORKER_ACCOUNT_UNDECLARED',
+  'WORKER_ACCOUNT_ROOT_REQUIRED',
+  'SUDOERS_OWNERSHIP_CONFLICT',
+];
+
+/** The one command that creates the account by hand, for the remedy. */
+export function accountCommand(user: string): string {
+  return `useradd --create-home --shell /bin/bash --groups sudo ${user}`;
+}
+
+/** Creates the declared worker account over a root login, through
+ * ansible/ubuntu/worker-account.yml run from the manager: an image that lets
+ * only root in gets the account the fleet converges, with the key root was
+ * reached with and the fleet's sudo rule. `login` is `root` when the alias
+ * names another user that cannot log in yet, and empty when the alias logs
+ * in as root itself.
+ */
+export async function createWorkerAccount(
+  target: { id: string; ssh: string; login: 'root' | '' },
+  user: string | undefined,
+  playbook: PackagePlaybook
+): Promise<Check> {
+  const failure = (summary: string, action: string): Check => ({
+    id: 'worker:account',
+    status: 'fail',
+    summary,
+    remedy: { action },
+  });
+  if (!user || user === 'root')
+    return failure(
+      'WORKER_ACCOUNT_UNDECLARED',
+      'The worker lets only root in, and fleet.json names no non-root account for it. Declare workers[].network.user; fleet-sync then creates that account and continues as it.'
+    );
+  const result = await playbook(
+    'ubuntu/worker-account.yml',
+    {
+      target: target.ssh,
+      login: target.login,
+      user,
+      ssh_options: playbookConnection(target).options,
+    },
+    300_000
+  );
+  if (result.ok)
+    return {
+      id: 'worker:account',
+      status: 'pass',
+      summary: `Created the declared account ${user} over the root login, with the key root was reached with and the fleet's sudo rule.`,
+    };
+  const code =
+    CODES.find((candidate) => result.stdout.includes(candidate)) ??
+    'WORKER_ACCOUNT_CREATION_FAILED';
+  return failure(
+    code,
+    `The declared account ${user} could not be created over the root login. Create it on the worker as root (${accountCommand(user)}), authorise the manager's key for it and grant it passwordless sudo, then rerun fleet-sync.`
+  );
+}