@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/cli/fleet-sync-vault.ts

@@ -1,7 +1,11 @@
 import { spawnSync } from 'node:child_process';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
 import { fleet as definition } from '../fleet-config';
 import { sshArgs, shellQuote } from '../ssh';
 import { resolveVaultConfig, usesVault } from '../vault-config';
+import { workerTokenLocation } from '../worker-credentials';
+import { runAnsible } from '../platform/ansible';
 import type { Check, FleetNode, Runner } from '../types';
 
 const TOOL_PATH =
@@ -15,7 +19,11 @@ const terminal: TerminalRunner = (args) =>
 
 type VaultState = 'authenticated' | 'unauthenticated' | 'unavailable';
 
-/** Vault field holding the issued token for one worker node. */
+/** The field of the fleet's shared document that held a worker's token
+ * before each worker had a document of its own (src/worker-credentials.ts).
+ * Read only when the worker's own document holds none, and reported: every
+ * worker that can read the shared document can read that field.
+ */
 export function workerTokenField(node: string): string {
   return `FLEET_WORKER_TOKEN_${node.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
 }
@@ -49,7 +57,13 @@ export async function authorizeManager(
  */
 export async function authorizeVault(
   node: FleetNode,
-  options: { interactive: boolean },
+  options: {
+    interactive: boolean;
+    /** Opt-in: the fleet's shared Vault user, typed on the worker's own
+     * terminal, when no token of its own was issued.
+     */
+    passwordLogin?: boolean;
+  },
   run: Runner,
   attach: TerminalRunner = terminal
 ): Promise<Check[]> {
@@ -59,8 +73,10 @@ export async function authorizeVault(
   if (before === 'authenticated') return [pass(node)];
   // Fresh node: the locked toolchain (including vault) is not installed yet.
   if (before === 'unavailable') return [];
-  if (await deliverWorkerToken(node, run)) return [pass(node)];
-  if (!options.interactive) return [required(node)];
+  const delivered = await deliverWorkerToken(node, run);
+  if (delivered === 'own') return [pass(node)];
+  if (delivered === 'shared') return [pass(node), shared(node)];
+  if (!options.interactive || !options.passwordLogin) return [required(node)];
   process.stderr.write(
     `[${node.id}] No issued worker token found. Enter the Vault password to log this machine in.\n`
   );
@@ -72,30 +88,54 @@ export async function authorizeVault(
   ];
 }
 
+/** Delivers the worker's token, from its own document or else from the
+ * field it used to have in the shared one; says which, or that none was.
+ */
 async function deliverWorkerToken(
   node: FleetNode,
   run: Runner
-): Promise<boolean> {
-  const { addr, mount, project, config } = resolveVaultConfig();
-  const secret = await run(
-    [
-      'vault',
-      'kv',
-      'get',
-      `-address=${addr}`,
-      `-mount=${mount}`,
-      `-field=${workerTokenField(node.id)}`,
-      `${project}/${config}`,
-    ],
-    { timeoutMs: 20_000 }
-  );
-  const token = secret.ok ? secret.stdout.trim() : '';
-  if (!/^[\x21-\x7e]{16,4096}$/.test(token)) return false;
-  const delivered = await run(
-    remote(node, ['vault', 'login', `-address=${addr}`, '-no-print', '-']),
-    { input: token + '\n', timeoutMs: 30_000 }
-  );
-  return delivered.ok && (await vaultState(node, run)) === 'authenticated';
+): Promise<'own' | 'shared' | null> {
+  const vault = resolveVaultConfig();
+  const own = workerTokenLocation(node.id, vault);
+  const sources = [
+    { from: 'own' as const, ...own },
+    {
+      from: 'shared' as const,
+      mount: vault.mount,
+      path: `${vault.project}/${vault.config}`,
+      field: workerTokenField(node.id),
+    },
+  ];
+  for (const source of sources) {
+    const secret = await run(
+      [
+        'vault',
+        'kv',
+        'get',
+        `-address=${vault.addr}`,
+        `-mount=${source.mount}`,
+        `-field=${source.field}`,
+        source.path,
+      ],
+      { timeoutMs: 20_000 }
+    );
+    const token = secret.ok ? secret.stdout.trim() : '';
+    if (!/^[\x21-\x7e]{16,4096}$/.test(token)) continue;
+    const delivered = await run(
+      remote(node, [
+        'vault',
+        'login',
+        `-address=${vault.addr}`,
+        '-no-print',
+        '-',
+      ]),
+      { input: token + '\n', timeoutMs: 30_000 }
+    );
+    return delivered.ok && (await vaultState(node, run)) === 'authenticated'
+      ? source.from
+      : null;
+  }
+  return null;
 }
 
 async function vaultState(node: FleetNode, run: Runner): Promise<VaultState> {
@@ -153,6 +193,18 @@ function pass(node: FleetNode): Check {
   };
 }
 
+function shared(node: FleetNode): Check {
+  const own = workerTokenLocation(node.id);
+  return {
+    id: 'auth:vault-isolation',
+    status: 'warn',
+    summary: 'VAULT_WORKER_TOKEN_SHARED',
+    remedy: {
+      action: `${node.id}'s token was read from the fleet's shared document, which every worker can read. Issue it a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes (kept at ${own.mount}/${own.path}), then remove ${workerTokenField(node.id)} from the shared document.`,
+    },
+  };
+}
+
 function required(node: FleetNode): Check {
   const { addr } = resolveVaultConfig();
   return {
@@ -160,7 +212,81 @@ function required(node: FleetNode): Check {
     status: 'fail',
     summary: 'VAULT_LOGIN_REQUIRED',
     remedy: {
-      action: `Run devfleet fleet-sync in a terminal, or store this node's token in the shared Vault document as ${workerTokenField(node.id)}. Manual fallback on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
+      action: `Issue ${node.id} a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes. Opt-in fallback: devfleet fleet-sync --vault-password-login in a terminal, or on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
     },
   };
 }
+
+/**
+ * The manager's own Vault session renews itself every ten minutes, like each
+ * worker's: `ansible/macos/vault-renew.yml`, applied on every applying sync.
+ * Measured as the session answering and the job being loaded; a lapsed
+ * session needs one login by a person, and until then agents on the manager
+ * use the proxy client key they last read (src/agents/secrets.ts).
+ */
+export async function holdManagerVault(
+  run: Runner,
+  options: {
+    apply: boolean;
+    root: string;
+    home?: string;
+    uid?: number;
+    scratch?: string;
+  }
+): Promise<Check[]> {
+  if (!usesVault()) return [];
+  const { addr } = resolveVaultConfig();
+  const home = options.home ?? homedir();
+  if (options.apply)
+    await runAnsible(
+      run,
+      join(options.root, 'ansible/macos/vault-renew.yml'),
+      {
+        input: JSON.stringify({
+          vault_address: addr,
+          mise: join(home, '.local/bin/mise'),
+        }),
+        timeoutMs: 120_000,
+      },
+      options.scratch
+    );
+  const loaded = (
+    await run(
+      [
+        'launchctl',
+        'print',
+        `gui/${options.uid ?? process.getuid?.() ?? 0}/${MANAGER_VAULT_RENEWAL}`,
+      ],
+      { timeoutMs: 10_000 }
+    )
+  ).ok;
+  const session = (await run(lookup(), { timeoutMs: 20_000 })).ok;
+  return [vaultRenewalCheck(session, loaded, addr)];
+}
+
+function vaultRenewalCheck(
+  session: boolean,
+  loaded: boolean,
+  addr: string
+): Check {
+  if (session && loaded)
+    return {
+      id: 'manager:vault-renewal',
+      status: 'pass',
+      summary:
+        'The manager is logged in to Vault and renews its session every ten minutes.',
+    };
+  return {
+    id: 'manager:vault-renewal',
+    status: 'fail',
+    summary: session ? 'VAULT_RENEWAL_MISSING' : 'VAULT_LOGIN_REQUIRED',
+    remedy: {
+      action: session
+        ? `The manager's Vault session is not renewed and lapses at its TTL. Run devfleet fleet-sync on the manager; it loads ${MANAGER_VAULT_RENEWAL}.`
+        : `Log the manager in once, in your own terminal: vault login -address=${addr} -method=userpass username=${definition.worker?.vaultUser ?? 'YOUR_USERNAME'}. Then run devfleet fleet-sync. Until then the manager's agents use the proxy client key they last read, and new worker tokens cannot be issued.`,
+    },
+  };
+}
+
+/** The launchd label of the manager's Vault renewal job (vault-renew.yml). */
+export const MANAGER_VAULT_RENEWAL = 'dev.devfleet.vault-renew';

package/src/cli/fleet-sync-worker.ts

@@ -0,0 +1,214 @@
+import type { FleetDefinition } from '../definition';
+import {
+  controlRemedy,
+  TailnetControlError,
+  type TailnetControl,
+} from '../tailnet-control';
+import type { TailnetComputer } from '../tailscale';
+import type { PackagePlaybook } from '../platform/ansible';
+import type { Check, FleetNode, NodeReport } from '../types';
+import { createWorkerAccount } from '../worker-account';
+import {
+  rebootNeeded,
+  rebootWorker,
+  updatesPending,
+  type RebootReason,
+} from '../worker-reboot';
+
+/** What one worker's deployment needs around its converge: the converge
+ * itself (bootstrap, verification, monitoring), the tailnet control plane
+ * when the fleet declares one, the tailnet as the manager sees it, and what
+ * the run was asked to do.
+ */
+export interface WorkerDeployment {
+  definition: FleetDefinition;
+  /** Applies a playbook of the package from the manager. */
+  playbook: PackagePlaybook;
+  /** One converge of the worker; `tailnetKey` is handed to its bootstrap. */
+  converge: (
+    node: FleetNode,
+    revision: string,
+    tailnetKey?: string
+  ) => Promise<NodeReport>;
+  /** Null when the fleet declares no tailnet control plane. */
+  control: () => Promise<TailnetControl | null>;
+  /** The devices of the tailnet, as the manager's Tailscale lists them. */
+  computers: () => Promise<TailnetComputer[]>;
+  /** The workers that converged in an earlier run. */
+  readyBefore: string[];
+  maintenanceReboot?: boolean;
+  force?: boolean;
+  /** Whether the bootstrap of this revision reads a tailnet key. */
+  acceptsTailnetKey: boolean;
+}
+
+/** Deploys one worker and, around its converge: hands a worker that is not
+ * on the tailnet a single-use key, keeps an enrolled worker's node key from
+ * expiring, creates the declared account on an image that lets only root
+ * in, and reboots the worker once when its first converge needs it (or for
+ * maintenance, when asked), converging it again to verify it.
+ */
+export async function deployWorker(
+  node: FleetNode,
+  revision: string,
+  deployment: WorkerDeployment
+): Promise<NodeReport> {
+  const before: Check[] = [];
+  const key = await enrolmentKey(node, deployment, before);
+  before.push(...(await keyExpiry(node, deployment)));
+  let target = node;
+  let result = await deployment.converge(target, revision, key);
+  const account = await rootOnly(node, result, deployment);
+  if (account) {
+    before.push(account.check);
+    if (account.node) {
+      target = account.node;
+      result = await deployment.converge(target, revision, key);
+    }
+  }
+  const reason = rebootReason(node.id, result.checks, deployment);
+  if (reason) {
+    const reboot = await rebootWorker(
+      target,
+      {
+        revision,
+        reason,
+        ...(reason === 'maintenance' && deployment.force
+          ? { force: true }
+          : {}),
+      },
+      deployment.playbook
+    );
+    if (reboot.status === 'pass')
+      result = await deployment.converge(target, revision);
+    result.checks.unshift(reboot);
+  }
+  result.checks.unshift(...before);
+  return result;
+}
+
+/** Why this worker is rebooted now, if it is. A worker that converged
+ * before is never rebooted for its first converge; only an explicit
+ * maintenance run reboots it.
+ */
+function rebootReason(
+  id: string,
+  checks: Check[],
+  deployment: WorkerDeployment
+): RebootReason | null {
+  if (deployment.maintenanceReboot && updatesPending(checks))
+    return 'maintenance';
+  if (!deployment.readyBefore.includes(id) && rebootNeeded(checks))
+    return 'first-converge';
+  return null;
+}
+
+/** A single-use, pre-authorised, tagged key for a worker the roster has no
+ * tailnet identity for yet. The key is never reported.
+ */
+async function enrolmentKey(
+  node: FleetNode,
+  deployment: WorkerDeployment,
+  checks: Check[]
+): Promise<string | undefined> {
+  if (node.tailscaleId || !deployment.acceptsTailnetKey) return undefined;
+  try {
+    const control = await deployment.control();
+    if (!control) return undefined;
+    const minted = await control.createAuthKey({ worker: node.id });
+    checks.push({
+      id: 'tailscale:enrolment-key',
+      status: 'pass',
+      summary: `A single-use, pre-authorised key tagged ${deployment.definition.tailnet!.tag}, expiring within minutes, was handed to the worker on the bootstrap's stdin.`,
+    });
+    return minted.key;
+  } catch (error) {
+    if (!(error instanceof TailnetControlError)) throw error;
+    checks.push(controlFailure('tailscale:enrolment-key', error));
+    return undefined;
+  }
+}
+
+/** An enrolled worker's node key never expires: the control plane disables
+ * expiry for a worker whose key has one (tagged workers have none on the
+ * hosted Tailscale). The tailscale:key-expiry check still measures it.
+ */
+async function keyExpiry(
+  node: FleetNode,
+  deployment: WorkerDeployment
+): Promise<Check[]> {
+  if (!node.tailscaleId) return [];
+  try {
+    const control = await deployment.control();
+    if (!control) return [];
+    const peer = (await deployment.computers()).find(
+      (computer) => computer.id === node.tailscaleId
+    );
+    if (peer?.keyExpiry === undefined) return [];
+    await control.disableKeyExpiry(node.tailscaleId);
+    return [
+      {
+        id: 'tailscale:key-expiry-policy',
+        status: 'pass',
+        summary:
+          'Key expiry of this worker was disabled through the tailnet control plane; tailscale:key-expiry measures it again on the next run.',
+      },
+    ];
+  } catch (error) {
+    if (!(error instanceof TailnetControlError)) throw error;
+    return [
+      {
+        ...controlFailure('tailscale:key-expiry-policy', error),
+        status: 'warn',
+      },
+    ];
+  }
+}
+
+function controlFailure(id: string, error: TailnetControlError): Check {
+  return {
+    id,
+    status: 'fail',
+    summary: error.code,
+    remedy: { action: controlRemedy(error.code) },
+  };
+}
+
+/** The bootstrap refused to run as root: the alias logs in as root. With a
+ * declared account, it is created over that login and the run continues as
+ * it (`user@alias`); the next runs need the alias to name it, since the SSH
+ * policy turns root login off.
+ */
+async function rootOnly(
+  node: FleetNode,
+  result: NodeReport,
+  deployment: WorkerDeployment
+): Promise<{ check: Check; node?: FleetNode } | null> {
+  if (
+    !result.checks.some(
+      (check) => check.summary === 'ROOT_DEVELOPMENT_USER_REJECTED'
+    )
+  )
+    return null;
+  const user = deployment.definition.workers.find(
+    (worker) => worker.id === node.id
+  )?.network?.user;
+  const check = await createWorkerAccount(
+    { id: node.id, ssh: node.ssh, login: '' },
+    user,
+    deployment.playbook
+  );
+  if (check.status !== 'pass') return { check };
+  return {
+    check: {
+      ...check,
+      status: 'warn',
+      summary: `${check.summary} This run continued as ${user}@${node.ssh}; set User ${user} for the SSH alias ${node.ssh}, since root login is turned off.`,
+    },
+    node: {
+      ...node,
+      ssh: `${user}@${node.ssh}`,
+      workspace: `/home/${user}/.local/share/personal/fleet`,
+    },
+  };
+}