@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/ssh-connection.ts

@@ -0,0 +1,141 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+import type { FleetDefinition, MachineDefinition } from './definition';
+import { stateRoot } from './state/paths';
+
+/** How the manager reaches a worker, written once: every direct SSH session
+ * (src/ssh.ts) and every Ansible run over SSH (fleet-validate's inventory,
+ * with ansible/ansible.cfg) takes its options from here, so the two cannot
+ * drift. Host keys are checked strictly and never learned on first use. A
+ * few attempts and keepalives let a short network blip pass without ending
+ * a long operation.
+ */
+export const SSH_TRANSPORT: readonly (readonly [string, string])[] = [
+  ['BatchMode', 'yes'],
+  ['StrictHostKeyChecking', 'yes'],
+  ['UpdateHostKeys', 'no'],
+  ['ConnectTimeout', '10'],
+  ['ConnectionAttempts', '3'],
+  ['ServerAliveInterval', '15'],
+  ['ServerAliveCountMax', '4'],
+];
+
+/** A direct session also forwards nothing and shares no connection. Ansible
+ * multiplexes its own (ControlPersist in ansible/ansible.cfg).
+ */
+export const SSH_SESSION: readonly (readonly [string, string])[] = [
+  ['ClearAllForwardings', 'yes'],
+  ['PermitLocalCommand', 'no'],
+  ['ControlMaster', 'no'],
+  ['ControlPath', 'none'],
+];
+
+/** The name a worker's host key is pinned under, whatever its address. */
+export const hostKeyAlias = (id: string): string => `devfleet-${id}`;
+
+/** The fleet-owned known_hosts file: a cache rendered from the pinned keys
+ * (declared in a worker record, or in the fleet state). The operator's own
+ * ~/.ssh files are neither read for it nor changed.
+ */
+export function knownHostsPath(env = process.env, home = homedir()): string {
+  return join(dirname(stateRoot(env, home)), 'ssh', 'known_hosts');
+}
+
+/** A worker without an SSH alias is reached from its declared network
+ * block: its tailnet name, user and port.
+ */
+export function declaredConnection(
+  worker: MachineDefinition | undefined
+):
+  | (MachineDefinition & { network: NonNullable<MachineDefinition['network']> })
+  | null {
+  return worker && worker.ssh === undefined && worker.network
+    ? (worker as MachineDefinition & {
+        network: NonNullable<MachineDefinition['network']>;
+      })
+    : null;
+}
+
+/** Host keys pinned in the fleet state, by worker ID. Only the devfleet CLI
+ * reads the state (it registers the reader); launchers and the worker probe
+ * run under Node, which cannot open the state database.
+ */
+type PinReader = () => Record<string, string>;
+let pinReader: PinReader | null = null;
+export function readPinsWith(reader: PinReader | null): void {
+  pinReader = reader;
+}
+
+/** The key each worker reached from declared data is pinned to: the one its
+ * record or fleet.json declares, else the one the fleet state pins.
+ */
+export function pinnedKeys(
+  definition: FleetDefinition,
+  pins: Record<string, string> = pinReader?.() ?? {}
+): Record<string, string> {
+  return Object.fromEntries(
+    definition.workers.flatMap((worker) => {
+      const declared = declaredConnection(worker);
+      const key = declared?.network.hostKey ?? pins[worker.id];
+      return declared && key ? [[worker.id, key]] : [];
+    })
+  );
+}
+
+export function knownHostsText(keys: Record<string, string>): string {
+  return [
+    '# Managed by dev-fleet: rendered from the fleet state and worker records. Do not edit.',
+    ...Object.entries(keys)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([id, key]) => `${hostKeyAlias(id)} ${key}`),
+    '',
+  ].join('\n');
+}
+
+/** Writes the fleet-owned known_hosts when it differs; returns its path. */
+export function renderKnownHosts(
+  definition: FleetDefinition,
+  env = process.env,
+  pins?: Record<string, string>
+): string {
+  const path = knownHostsPath(env);
+  const text = knownHostsText(pinnedKeys(definition, pins));
+  if (existsSync(path) && readFileSync(path, 'utf8') === text) return path;
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeFileSync(`${path}.new`, text, { mode: 0o600 });
+  renameSync(`${path}.new`, path);
+  return path;
+}
+
+const option = ([name, value]: readonly [string, string]): string =>
+  `${name}=${value}`;
+
+/** The options that reach one worker from declared data. */
+export function connectionOptions(
+  worker: MachineDefinition & {
+    network: NonNullable<MachineDefinition['network']>;
+  },
+  knownHosts: string
+): [string, string][] {
+  return [
+    ['HostKeyAlias', hostKeyAlias(worker.id)],
+    ['UserKnownHostsFile', knownHosts],
+    ['GlobalKnownHostsFile', '/dev/null'],
+    ['User', worker.network.user],
+    ['Port', String(worker.network.sshPort)],
+  ];
+}
+
+/** `-o NAME=VALUE` for each option, as ssh takes them. */
+export function sshOptionArgs(
+  options: readonly (readonly [string, string])[]
+): string[] {
+  return options.flatMap((entry) => ['-o', option(entry)]);
+}

package/src/ssh.ts

@@ -1,40 +1,79 @@
 import type { CommandResult, FleetNode, Check } from './types';
+import { fleet } from './fleet-config';
+import {
+  connectionOptions,
+  declaredConnection,
+  renderKnownHosts,
+  SSH_SESSION,
+  SSH_TRANSPORT,
+  sshOptionArgs,
+} from './ssh-connection';
 
 export function shellQuote(value: string): string {
   return `'${value.replaceAll("'", "'\\''")}'`;
 }
 
-export function sshArgs(node: FleetNode): string[] {
+/** The worker fleet.json declares for a roster entry that is reached from
+ * declared data, if it is one. No configuration (a fresh machine) is none.
+ */
+function declaredWorker(node: Pick<FleetNode, 'id' | 'ssh'>) {
+  try {
+    const worker = declaredConnection(
+      fleet.workers.find((entry) => entry.id === node.id)
+    );
+    return worker && worker.network.tailscaleName === node.ssh ? worker : null;
+  } catch {
+    return null;
+  }
+}
+
+/** One SSH session to a node: unattended, strict about the host key,
+ * forwarding nothing. A worker without an alias is reached from its declared
+ * user, port and tailnet name, against the fleet-owned known_hosts.
+ */
+export function sshArgs(node: Pick<FleetNode, 'id' | 'ssh'>): string[] {
+  const declared = declaredWorker(node);
   return [
     'ssh',
     '-T',
     '-a',
-    '-o',
-    'BatchMode=yes',
-    '-o',
-    'StrictHostKeyChecking=yes',
-    '-o',
-    'UpdateHostKeys=no',
-    '-o',
-    'ConnectTimeout=8',
-    '-o',
-    'ConnectionAttempts=1',
-    '-o',
-    'ServerAliveInterval=15',
-    '-o',
-    'ServerAliveCountMax=4',
-    '-o',
-    'ClearAllForwardings=yes',
-    '-o',
-    'PermitLocalCommand=no',
-    '-o',
-    'ControlMaster=no',
-    '-o',
-    'ControlPath=none',
+    ...sshOptionArgs(SSH_TRANSPORT),
+    ...(declared
+      ? sshOptionArgs(connectionOptions(declared, renderKnownHosts(fleet)))
+      : []),
+    ...sshOptionArgs(SSH_SESSION),
     node.ssh,
   ];
 }
 
+/** How a package playbook that addresses a worker over SSH itself
+ * (ansible/ubuntu/worker-account.yml, ansible/ubuntu/reboot.yml) reaches it:
+ * the options every direct session takes, as one `ansible_ssh_common_args`
+ * value, and the account a worker reached from declared data logs in as. The
+ * play names the login itself (root, or the declared account), so the
+ * declared user is not among the options.
+ */
+export function playbookConnection(node: Pick<FleetNode, 'id' | 'ssh'>): {
+  options: string;
+  user: string | null;
+} {
+  const declared = declaredWorker(node);
+  const options = [
+    ...SSH_TRANSPORT,
+    ...(declared
+      ? connectionOptions(declared, renderKnownHosts(fleet)).filter(
+          ([name]) => name !== 'User'
+        )
+      : []),
+  ];
+  return {
+    options: sshOptionArgs(options)
+      .map((arg) => (arg === '-o' ? arg : shellQuote(arg)))
+      .join(' '),
+    user: declared?.network.user ?? null,
+  };
+}
+
 export function remoteCommand(node: FleetNode, request: object): string[] {
   const encoded = Buffer.from(JSON.stringify(request)).toString('base64');
   const command = `cd -- ${shellQuote(node.workspace)} && export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH" && exec ${shellQuote(node.runtime ?? 'node')} --input-type=module - ${shellQuote(encoded)}`;
@@ -48,7 +87,7 @@ const FAILURES = [
     id: 'ssh:host-key',
     summary: 'SSH host key is unknown or changed.',
     action:
-      'Verify the host fingerprint out of band, then update known_hosts manually. Never disable host-key checking.',
+      'The worker presented a host key other than the pinned one. Verify its fingerprint out of band, then pin the new key: devfleet fleet-node-replace --id ID --host-key "TYPE BASE64" --apply --yes, or change hostKey in its worker record. A worker reached through your own SSH alias is checked against your known_hosts instead. Never disable host-key checking.',
   },
   {
     pattern:

package/src/state/store.ts

@@ -17,9 +17,11 @@ import { FleetUsageError } from '../errors';
  */
 export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
 /** Kept when there is something to keep; a fleet without them is complete:
- * `history` is the record of recent fleet-sync runs and of the schedule.
+ * `history` is the record of recent fleet-sync runs and of the schedule;
+ * `workers` what the manager knows of each worker beyond fleet.json: its
+ * pinned SSH host key and whether it is parked (src/state/workers.ts).
  */
-export const OPTIONAL_DOCUMENTS = ['history'] as const;
+export const OPTIONAL_DOCUMENTS = ['history', 'workers'] as const;
 export type DocumentName = (typeof DOCUMENTS)[number];
 /** Every document the fleet state may hold. */
 export type StoredName = DocumentName | (typeof OPTIONAL_DOCUMENTS)[number];

package/src/state/sync.ts

@@ -349,9 +349,14 @@ export function pull(context: StateContext): Promise<Standing> {
 }
 
 /** Replaces this copy with the version the bucket held at `timestamp`, then
- * uploads it as the bucket's current version: the way back from a replica
- * whose latest version is damaged. `unreachable` when the bucket holds no
- * readable version at that time.
+ * uploads it as the bucket's current version, which takes the manager lease:
+ * the way back from a replica whose latest version is damaged. `unreachable`
+ * when the bucket holds no readable version at that time.
+ *
+ * Refused, before anything changes, while another manager holds the lease in
+ * the bucket's current version, unless told to take it over. When that
+ * version cannot be read the lease of this machine's copy, the last one it
+ * saw, decides instead.
  */
 export function pullAt(
   context: StateContext,
@@ -367,6 +372,17 @@ export function pullAt(
       'No bucket is configured, so there is no earlier version to restore.'
     );
   return locked(context.root, async () => {
+    const database = databasePath(context.root);
+    const current = await fetchRemote({ ...context, bucket });
+    try {
+      const held = leaseIn(
+        context,
+        current.status === 'present' ? current.file : database
+      );
+      if (held) throw new LeaseHeldError(leaseHeldMessage(held));
+    } finally {
+      if (current.status === 'present') discard(current.file);
+    }
     const remote = await fetchRemote(
       { ...context, bucket },
       true,
@@ -374,10 +390,17 @@ export function pullAt(
     );
     if (remote.status !== 'present')
       return remote.status === 'absent' ? 'empty' : remote.status;
-    const database = databasePath(context.root);
     discard(database);
     renameSync(remote.file, database);
     chmodSync(database, 0o600);
+    // The earlier version names whoever held the lease then.
+    if (context.holder) {
+      const now = (context.now ?? (() => new Date()))();
+      writeLease(database, {
+        holder: context.holder,
+        expiresAt: new Date(now.getTime() + LEASE_MS).toISOString(),
+      });
+    }
     return upload({ ...context, bucket }, true);
   });
 }
@@ -408,11 +431,17 @@ export class LeaseHeldError extends FleetUsageError {}
  * and this machine was not told to take it over. Reads local files only: call
  * it on the bucket's version, after a refresh.
  */
-export function leaseConflict(
+export function leaseConflict(context: StateContext, now?: Date): Lease | null {
+  return leaseIn(context, databasePath(context.root), now);
+}
+
+/** The lease another manager holds over the database at `file`. */
+function leaseIn(
   context: StateContext,
+  file: string,
   now: Date = (context.now ?? (() => new Date()))()
 ): Lease | null {
-  const lease = readLease(databasePath(context.root));
+  const lease = readLease(file);
   if (!context.holder || !lease || lease.holder === context.holder) return null;
   if (Date.parse(lease.expiresAt) <= now.getTime() || context.takeOver)
     return null;

package/src/state/workers.ts

@@ -0,0 +1,140 @@
+import { record } from '../config';
+import { HOST_KEY } from '../definition';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+import { mutateState, type StateContext, type Standing } from './sync';
+
+/** What the manager knows of one worker that fleet.json does not say. */
+export interface WorkerState {
+  /** The pinned SSH host key, "TYPE BASE64", and when it was pinned. */
+  hostKey?: string;
+  hostKeyPinnedAt?: string;
+  /** When the worker was parked: stopped or destroyed on purpose. Absent,
+   * the worker is active.
+   */
+  parkedAt?: string;
+  /** `records`: the worker was enrolled from a worker record, so it is
+   * retired by itself once its record disappears.
+   */
+  origin?: 'records';
+  /** When fleet-node-replace put a new machine in the worker's place: the
+   * manager pairs its T3 with the new machine's T3 environment as soon as
+   * it runs, then forgets this. A worker enrolled for the first time is
+   * paired by a person.
+   */
+  t3PairingRequestedAt?: string;
+}
+
+/** The `workers` document of the fleet state. */
+export interface WorkerStates {
+  version: 1;
+  workers: Record<string, WorkerState>;
+}
+
+const ID = /^[a-z][a-z0-9-]{0,62}$/;
+const isTime = (value: unknown): value is string =>
+  typeof value === 'string' && !Number.isNaN(Date.parse(value));
+
+const RULES: Record<keyof WorkerState, (value: unknown) => boolean> = {
+  hostKey: (value) => typeof value === 'string' && HOST_KEY.test(value),
+  hostKeyPinnedAt: isTime,
+  parkedAt: isTime,
+  origin: (value) => value === 'records',
+  t3PairingRequestedAt: isTime,
+};
+
+function workerState(value: unknown): WorkerState | null {
+  if (!record(value)) return null;
+  const broken = Object.entries(RULES).some(
+    ([field, valid]) => value[field] !== undefined && !valid(value[field])
+  );
+  if (broken) return null;
+  const { hostKey, hostKeyPinnedAt, parkedAt, origin, t3PairingRequestedAt } =
+    value;
+  return {
+    ...(typeof hostKey === 'string' ? { hostKey } : {}),
+    ...(isTime(hostKeyPinnedAt) ? { hostKeyPinnedAt } : {}),
+    ...(isTime(parkedAt) ? { parkedAt } : {}),
+    ...(origin === 'records' ? { origin } : {}),
+    ...(isTime(t3PairingRequestedAt) ? { t3PairingRequestedAt } : {}),
+  };
+}
+
+/** The document, or an empty one. An entry that cannot be read is dropped:
+ * an unreadable pin pins nothing, so the worker is refused, never trusted.
+ */
+export function parseWorkerStates(text: string | null): WorkerStates {
+  let value: unknown = null;
+  try {
+    value = text === null ? null : JSON.parse(text);
+  } catch {
+    value = null;
+  }
+  const workers: Record<string, WorkerState> = {};
+  if (record(value) && record(value['workers']))
+    for (const [id, entry] of Object.entries(value['workers'])) {
+      const parsed = workerState(entry);
+      if (ID.test(id) && parsed) workers[id] = parsed;
+    }
+  return { version: 1, workers };
+}
+
+export function readWorkerStates(env = process.env): WorkerStates {
+  return parseWorkerStates(
+    readDocument(databasePath(stateRoot(env)), 'workers')
+  );
+}
+
+/** The host keys pinned in the fleet state, by worker ID. */
+export function readPins(env = process.env): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(readWorkerStates(env).workers).flatMap(([id, entry]) =>
+      entry.hostKey ? [[id, entry.hostKey]] : []
+    )
+  );
+}
+
+/** When each parked worker was parked, by worker ID. */
+export function readParked(env = process.env): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(readWorkerStates(env).workers).flatMap(([id, entry]) =>
+      entry.parkedAt ? [[id, entry.parkedAt]] : []
+    )
+  );
+}
+
+function writeWorkerStates(states: WorkerStates, env = process.env): void {
+  const workers = Object.fromEntries(
+    Object.entries(states.workers)
+      .filter(([, entry]) => Object.keys(entry).length > 0)
+      .sort(([a], [b]) => a.localeCompare(b))
+  );
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'workers',
+    `${JSON.stringify({ version: 1, workers }, null, 2)}\n`
+  );
+}
+
+/** Changes the entries of some workers on top of the bucket's version and
+ * uploads the result. `change` returns the new entry of each worker it
+ * changes, or null to forget the worker.
+ */
+export async function changeWorkers(
+  context: StateContext,
+  change: (
+    current: Record<string, WorkerState>
+  ) => Record<string, WorkerState | null>
+): Promise<{ workers: Record<string, WorkerState>; standing: Standing }> {
+  const { value, standing } = await mutateState(context, () => {
+    const states = readWorkerStates();
+    const updates = change(states.workers);
+    const workers = { ...states.workers };
+    for (const [id, entry] of Object.entries(updates))
+      if (entry === null) delete workers[id];
+      else workers[id] = entry;
+    writeWorkerStates({ version: 1, workers });
+    return workers;
+  });
+  return { workers: value, standing };
+}

package/src/t3/pairing.ts

@@ -1,6 +1,9 @@
 import { parseConfig } from '../config';
 import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
 import { mutateState, stateContext, type StateContext } from '../state/sync';
+import { changeWorkers, readWorkerStates } from '../state/workers';
+import type { Check, FleetNode, Runner } from '../types';
+import { pairWorker, pairingCheck, type ManagerT3 } from './peer-link';
 
 /** Records the T3 environment a worker runs in the roster, the first time a
  * verified link from the manager reaches it. From then on the link check
@@ -29,3 +32,59 @@ export async function rememberEnvironment(
   if (path !== STATE_ROSTER) return save();
   return (await mutateState(context(), save)).value;
 }
+
+/** What pairing a worker needs: where the roster is, how commands run, the
+ * manager's T3 and the fleet state. */
+export interface PairingServices {
+  roster: string;
+  run: Runner;
+  manager: ManagerT3;
+  context: () => StateContext;
+  now: () => Date;
+}
+
+/** Pairs the manager's T3 with the environment a worker runs now, without a
+ * person (src/t3/peer-link.ts), and records it in the roster. Until it is
+ * paired, the fleet state keeps the request, so the next fleet-sync pairs
+ * it once the worker's T3 runs; once paired, the request is forgotten.
+ */
+export async function pairAndRecord(
+  node: FleetNode,
+  services: PairingServices
+): Promise<Check> {
+  const pairing = await pairWorker(node, services.run, services.manager);
+  if ('paired' in pairing)
+    await rememberEnvironment(
+      services.roster,
+      node.id,
+      pairing.paired,
+      services.context
+    );
+  const requested = services.now().toISOString();
+  await changeWorkers(services.context(), (current) => {
+    const { t3PairingRequestedAt: earlier, ...rest } = current[node.id] ?? {};
+    return {
+      [node.id]:
+        'paired' in pairing
+          ? rest
+          : { ...rest, t3PairingRequestedAt: earlier ?? requested },
+    };
+  });
+  return pairingCheck(node.id, pairing);
+}
+
+/** fleet-sync's part: pairs a worker only when fleet-node-replace asked for
+ * it. A worker enrolled for the first time is paired by a person.
+ */
+export async function pairIfRequested(
+  node: FleetNode,
+  services: Omit<PairingServices, 'context' | 'now'> &
+    Partial<Pick<PairingServices, 'context'>>
+): Promise<Check | undefined> {
+  if (!readWorkerStates().workers[node.id]?.t3PairingRequestedAt) return;
+  return pairAndRecord(node, {
+    context: () => stateContext(services.run),
+    ...services,
+    now: () => new Date(),
+  });
+}