@crvouga/dev-fleet 0.7.0 → 0.8.0

package/src/cli/fleet-node-seed.ts

@@ -0,0 +1,151 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { FleetDefinition } from '../definition';
+import { ubuntuRelease } from '../platform/ubuntu-release';
+import { FleetUsageError } from '../errors';
+import { fleet, secret } from '../fleet-config';
+import type { SecretsProvider } from '../secrets';
+import type { Check } from '../types';
+import {
+  generateHostKey,
+  renderSeed,
+  seedUserData,
+  tailscalePins,
+} from '../worker-seed';
+import type { CommandOutcome } from './contracts';
+import {
+  keyName,
+  lifecycleServices,
+  type NodeLifecycleServices,
+} from './fleet-node-lifecycle';
+
+export interface NodeSeedOptions {
+  id?: string;
+  output?: string;
+  publicKey?: string;
+  autoinstall?: boolean;
+  apply?: boolean;
+  yes?: boolean;
+}
+
+export interface NodeSeedServices extends Pick<
+  NodeLifecycleServices,
+  'run' | 'playbook' | 'knownHosts'
+> {
+  definition: () => FleetDefinition;
+  secret: SecretsProvider;
+  home: string;
+}
+
+export function seedServices(roster: string): NodeSeedServices {
+  const { run, playbook, knownHosts } = lifecycleServices(roster);
+  return {
+    run,
+    playbook,
+    knownHosts,
+    definition: () => fleet,
+    secret,
+    home: homedir(),
+  };
+}
+
+/** Writes the first-boot seed of a declared worker to a private file: the
+ * seed carries a single-use auth key, so it is never printed. With --apply
+ * --yes the manager pins the host key the seed installs, so the new machine
+ * is trusted without a first-use step.
+ */
+export async function fleetNodeSeed(
+  options: NodeSeedOptions,
+  services: NodeSeedServices
+): Promise<CommandOutcome> {
+  const worker = services
+    .definition()
+    .workers.find((entry) => entry.id === options.id);
+  if (!worker)
+    throw new FleetUsageError('--id must name a worker fleet.json declares.');
+  if (!options.output)
+    throw new FleetUsageError(
+      '--output PATH is required: the seed holds a single-use auth key and is never printed.'
+    );
+  const keyFile =
+    options.publicKey ?? join(services.home, '.ssh/id_ed25519.pub');
+  const managerKey = await readFile(keyFile, 'utf8').catch(() => {
+    throw new FleetUsageError(
+      `No manager public key at ${keyFile}; name one with --public-key PATH.`
+    );
+  });
+  const authKey = await services.secret('tailnet-auth-key');
+  if (!authKey)
+    throw new FleetUsageError(
+      'The secrets provider holds no tailnet-auth-key. Store a short-lived, single-use, tagged Tailscale auth key there first.'
+    );
+  const release = ubuntuRelease(services.definition().ubuntu.release);
+  if (!release)
+    throw new FleetUsageError(
+      'fleet.json names an Ubuntu release the support matrix does not list.'
+    );
+  const hostKey = await generateHostKey(worker.id, services.run);
+  const seed = renderSeed(
+    await seedUserData({
+      worker,
+      managerKey,
+      authKey,
+      hostKey,
+      tailscale: await tailscalePins(),
+      codename: release.codename,
+    }),
+    Boolean(options.autoinstall)
+  );
+  await writeFile(options.output, seed, { mode: 0o600 });
+  const host = await keyName(worker, services.run);
+  const publicKey = hostKey.public.split(' ').slice(0, 2).join(' ');
+  const pin: Check =
+    options.apply && options.yes
+      ? (await services.playbook('macos/worker-host-key.yml', {
+          host,
+          pin: 'present',
+          key: publicKey,
+          known_hosts: services.knownHosts,
+        }))
+        ? {
+            id: 'ssh:host-key',
+            status: 'pass',
+            summary: `The manager trusts the seed's host key for ${host}.`,
+          }
+        : {
+            id: 'ssh:host-key',
+            status: 'fail',
+            summary: 'SSH_HOST_KEY_NOT_CHANGED',
+            remedy: {
+              action:
+                'The pinned Ansible could not change known_hosts; run the command again.',
+            },
+          }
+      : {
+          id: 'ssh:host-key',
+          status: 'skip',
+          summary: `Planned: pin the seed's host key for ${host}; run again with --apply --yes.`,
+        };
+  return {
+    exitCode: pin.status === 'fail' ? 1 : 0,
+    data: {
+      version: 1,
+      command: 'fleet-node-seed',
+      ok: pin.status !== 'fail',
+      worker: worker.id,
+      output: options.output,
+      format: options.autoinstall ? 'autoinstall' : 'cloud-init',
+      hostKey: publicKey,
+      checks: [pin],
+    },
+    text: [
+      `Wrote the ${options.autoinstall ? 'autoinstall' : 'cloud-init'} seed of ${worker.id} to ${options.output} (private; it holds a single-use auth key).`,
+      `Host key: ${publicKey}`,
+      `${pin.status.toUpperCase()} ${pin.id}: ${pin.summary}`,
+      'Boot a stock Ubuntu image with it; once the machine is on the tailnet, add it with fleet-node-add and run devfleet fleet-sync --node ' +
+        worker.id +
+        '.',
+    ].join('\n'),
+  };
+}

package/src/cli/fleet-sync-forward.ts

@@ -104,9 +104,11 @@ export function forwardedFlags(
 ): string[] {
   const flags = revision ? ['--revision', revision] : [];
   for (const node of options.node ?? []) flags.push('--node', node);
+  if (options.canary) flags.push('--canary', options.canary);
   if (options.schedule) flags.push('--schedule', options.schedule);
   const switches: [boolean | undefined, string][] = [
     [options.all, '--all'],
+    [options.vaultPasswordLogin, '--vault-password-login'],
     [options.dryRun, '--dry-run'],
     [options.apply, '--apply'],
     [options.yes, '--yes'],

package/src/cli/fleet-sync-manager.ts

@@ -13,7 +13,7 @@ import { IDES } from '../ide';
 import { enabled, managerChecks } from '../ide/orchestrate';
 import { idesOf } from '../ide/definition';
 import { convergeGitIdentity, gitChecks } from '../git-identity';
-import { ansibleCommand } from '../platform/ansible';
+import { ansibleCommand, runAnsible } from '../platform/ansible';
 import { convergeGithub } from '../github/converge';
 import { PACKAGE_ROOT } from '../package-info';
 import { managerHealth } from '../manager-health';
@@ -47,7 +47,11 @@ export async function convergeManager(
   if (identity.status === 'pass') {
     // Launchers and scheduled jobs on the manager read the installed copy.
     if (input.apply) installConfig();
-    checks.push(await managerPower(run, input.workspace, host));
+    checks.push(
+      input.apply
+        ? await holdManagerPower(run, input.workspace, host)
+        : await managerPower(run, input.workspace, host)
+    );
     checks.push(await managerBootRecovery(run, input.workspace, host));
     checks.push(await managerDocker(run));
     // Each enabled IDE: its settings, pinned version and policy.
@@ -202,7 +206,8 @@ export async function managerAgentProfiles(
 export async function managerIdentity(
   id: string,
   run: Runner,
-  host: ManagerHost = MACOS_MANAGER
+  host: ManagerHost = MACOS_MANAGER,
+  platform: NodeJS.Platform = process.platform
 ): Promise<Check> {
   const local = (await host.localHostName(run)) ?? '';
   return local === id
@@ -216,7 +221,10 @@ export async function managerIdentity(
         status: 'fail',
         summary: 'FLEET_NOT_MANAGER',
         remedy: {
-          action: `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`,
+          action:
+            platform === 'darwin'
+              ? `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`
+              : `This machine runs ${platform}, and the manager is always a macOS machine. Run fleet-sync on the manager (${id}), or from a worker that can SSH to it.`,
         },
       };
 }
@@ -244,7 +252,7 @@ export async function managerPower(
           ? 'MANAGER_SLEEPS_ON_POWER'
           : `MANAGER_POWER_POLICY_DRIFT ${drifted.join(' ')}`,
         remedy: {
-          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once, as an administrator: cd ${workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`,
+          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Grant the manager its power settings once, as an administrator; every fleet-sync then applies the policy with no prompt: cd ${workspace} && ${ansibleCommand().join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml`,
         },
       };
 }
@@ -283,6 +291,43 @@ export async function managerBootRecovery(
   };
 }
 
+/** Whether the one grant manager-privilege.yml installs is in place: this
+ * user may run `pmset -c` without a password. Lists, never runs.
+ */
+async function powerGranted(run: Runner): Promise<boolean> {
+  return (
+    await run(['sudo', '-n', '-l', '/usr/bin/pmset', '-c', 'sleep', '0'], {
+      timeoutMs: 10_000,
+    })
+  ).ok;
+}
+
+/** On every applying sync: a drifted power policy is applied with
+ * manager.yml when the grant is in place, then measured again. Without the
+ * grant the check says how to give it, once.
+ */
+export async function holdManagerPower(
+  run: Runner,
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER,
+  { root = PACKAGE_ROOT, scratch }: { root?: string; scratch?: string } = {}
+): Promise<Check> {
+  const before = await managerPower(run, workspace, host);
+  if (
+    before.status === 'pass' ||
+    host.platform !== 'macOS' ||
+    !(await powerGranted(run))
+  )
+    return before;
+  await runAnsible(
+    run,
+    join(root, 'ansible/macos/manager.yml'),
+    { input: '{}', timeoutMs: 120_000 },
+    scratch
+  );
+  return managerPower(run, workspace, host);
+}
+
 /** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
 export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
   const { vars } = Bun.TOML.parse(

package/src/cli/fleet-sync-remote.ts

@@ -103,16 +103,26 @@ export async function selected(
   return outcome(report, planFingerprint, started);
 }
 
-/** A revision new to the fleet is proven on the first declared worker before
- * the rest; repeating one that already converged everywhere needs no canary,
- * so every worker converges at once.
+/** A revision new to the fleet is proven on one worker before the rest: the
+ * one `--canary` or fleet.json names, else the first declared worker, and
+ * the next reachable one when that cannot be reached. Repeating a revision
+ * that already converged everywhere needs no canary, so every worker
+ * converges at once.
  */
 function canary(
   canonical: FleetDefinition,
   options: FleetSyncOptions
 ): { canary?: string } {
-  const first = canonical.workers[0];
-  return first && !options.rolledOut ? { canary: first.id } : {};
+  if (
+    options.canary &&
+    !canonical.workers.some((worker) => worker.id === options.canary)
+  )
+    throw new FleetUsageError(
+      '--canary names a worker fleet.json does not declare.'
+    );
+  const preferred =
+    options.canary ?? canonical.canary ?? canonical.workers[0]?.id;
+  return preferred && !options.rolledOut ? { canary: preferred } : {};
 }
 
 /** Saves the full JSON report, then renders the short human summary. */
@@ -165,6 +175,7 @@ function deployWithMonitoring(
     /** The manager's checkout, and the roster an identity is recorded in. */
     workspace: string;
     roster: string;
+    passwordLogin: boolean;
   }
 ): DeployNode {
   return async (node, revision) => {
@@ -189,7 +200,10 @@ function deployWithMonitoring(
       const checks = await authorizePrivilege(node, payload.authorizeSudo, run);
       if (checks.length) return { node: node.id, checks };
     }
-    const vault = { interactive: ui.interactiveAuthorization };
+    const vault = {
+      interactive: ui.interactiveAuthorization,
+      passwordLogin: ui.passwordLogin,
+    };
     if (ui.interactiveAuthorization) ui.progress.stop();
     const before = await authorizeVault(node, vault, run);
     ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
@@ -242,6 +256,7 @@ function scopeDefinition(canonical: FleetDefinition, ids?: string[]) {
 function interfaceOptions(options: FleetSyncOptions) {
   return {
     progress: syncProgress(!options.json),
+    passwordLogin: Boolean(options.vaultPasswordLogin),
     interactiveAuthorization: Boolean(
       options.interactiveAuthorization &&
       !options.json &&

package/src/cli/fleet-sync-state.ts

@@ -21,8 +21,11 @@ import type { Check } from '../types';
 export interface RunRecord {
   /** Before any machine changes: this revision is the approved one. */
   approve(): void;
-  /** Once the result is known. The check says whether the bucket holds it. */
-  settle(converged: boolean): Promise<Check>;
+  /** Once the result is known. The check says whether the bucket holds it.
+   * `applied` is the revision each worker that converged now holds: it is
+   * kept even when the fleet as a whole is not ready.
+   */
+  settle(converged: boolean, applied?: Record<string, string>): Promise<Check>;
   /** For a run that stopped before its result: its approval is still uploaded. */
   abandon(): Promise<void>;
 }
@@ -70,12 +73,18 @@ export function runRecord(
       }
       claimLease(state.context);
     },
-    async settle(converged) {
+    async settle(converged, applied = {}) {
       settled = true;
+      const workers = Object.keys(applied).length
+        ? { workers: { ...previous?.workers, ...applied } }
+        : {};
+      if (!converged && workers.workers)
+        storeEnrollment({ ...approval, ...workers }, env);
       if (converged)
         storeEnrollment(
           {
             ...approval,
+            ...workers,
             appliedRevision: revision,
             ...(previous?.appliedRevision &&
             previous.appliedRevision !== revision

package/src/cli/fleet-sync-vault.ts

@@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process';
 import { fleet as definition } from '../fleet-config';
 import { sshArgs, shellQuote } from '../ssh';
 import { resolveVaultConfig, usesVault } from '../vault-config';
+import { workerTokenLocation } from '../worker-credentials';
 import type { Check, FleetNode, Runner } from '../types';
 
 const TOOL_PATH =
@@ -15,7 +16,11 @@ const terminal: TerminalRunner = (args) =>
 
 type VaultState = 'authenticated' | 'unauthenticated' | 'unavailable';
 
-/** Vault field holding the issued token for one worker node. */
+/** The field of the fleet's shared document that held a worker's token
+ * before each worker had a document of its own (src/worker-credentials.ts).
+ * Read only when the worker's own document holds none, and reported: every
+ * worker that can read the shared document can read that field.
+ */
 export function workerTokenField(node: string): string {
   return `FLEET_WORKER_TOKEN_${node.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
 }
@@ -49,7 +54,13 @@ export async function authorizeManager(
  */
 export async function authorizeVault(
   node: FleetNode,
-  options: { interactive: boolean },
+  options: {
+    interactive: boolean;
+    /** Opt-in: the fleet's shared Vault user, typed on the worker's own
+     * terminal, when no token of its own was issued.
+     */
+    passwordLogin?: boolean;
+  },
   run: Runner,
   attach: TerminalRunner = terminal
 ): Promise<Check[]> {
@@ -59,8 +70,10 @@ export async function authorizeVault(
   if (before === 'authenticated') return [pass(node)];
   // Fresh node: the locked toolchain (including vault) is not installed yet.
   if (before === 'unavailable') return [];
-  if (await deliverWorkerToken(node, run)) return [pass(node)];
-  if (!options.interactive) return [required(node)];
+  const delivered = await deliverWorkerToken(node, run);
+  if (delivered === 'own') return [pass(node)];
+  if (delivered === 'shared') return [pass(node), shared(node)];
+  if (!options.interactive || !options.passwordLogin) return [required(node)];
   process.stderr.write(
     `[${node.id}] No issued worker token found. Enter the Vault password to log this machine in.\n`
   );
@@ -72,30 +85,54 @@ export async function authorizeVault(
   ];
 }
 
+/** Delivers the worker's token, from its own document or else from the
+ * field it used to have in the shared one; says which, or that none was.
+ */
 async function deliverWorkerToken(
   node: FleetNode,
   run: Runner
-): Promise<boolean> {
-  const { addr, mount, project, config } = resolveVaultConfig();
-  const secret = await run(
-    [
-      'vault',
-      'kv',
-      'get',
-      `-address=${addr}`,
-      `-mount=${mount}`,
-      `-field=${workerTokenField(node.id)}`,
-      `${project}/${config}`,
-    ],
-    { timeoutMs: 20_000 }
-  );
-  const token = secret.ok ? secret.stdout.trim() : '';
-  if (!/^[\x21-\x7e]{16,4096}$/.test(token)) return false;
-  const delivered = await run(
-    remote(node, ['vault', 'login', `-address=${addr}`, '-no-print', '-']),
-    { input: token + '\n', timeoutMs: 30_000 }
-  );
-  return delivered.ok && (await vaultState(node, run)) === 'authenticated';
+): Promise<'own' | 'shared' | null> {
+  const vault = resolveVaultConfig();
+  const own = workerTokenLocation(node.id, vault);
+  const sources = [
+    { from: 'own' as const, ...own },
+    {
+      from: 'shared' as const,
+      mount: vault.mount,
+      path: `${vault.project}/${vault.config}`,
+      field: workerTokenField(node.id),
+    },
+  ];
+  for (const source of sources) {
+    const secret = await run(
+      [
+        'vault',
+        'kv',
+        'get',
+        `-address=${vault.addr}`,
+        `-mount=${source.mount}`,
+        `-field=${source.field}`,
+        source.path,
+      ],
+      { timeoutMs: 20_000 }
+    );
+    const token = secret.ok ? secret.stdout.trim() : '';
+    if (!/^[\x21-\x7e]{16,4096}$/.test(token)) continue;
+    const delivered = await run(
+      remote(node, [
+        'vault',
+        'login',
+        `-address=${vault.addr}`,
+        '-no-print',
+        '-',
+      ]),
+      { input: token + '\n', timeoutMs: 30_000 }
+    );
+    return delivered.ok && (await vaultState(node, run)) === 'authenticated'
+      ? source.from
+      : null;
+  }
+  return null;
 }
 
 async function vaultState(node: FleetNode, run: Runner): Promise<VaultState> {
@@ -153,6 +190,18 @@ function pass(node: FleetNode): Check {
   };
 }
 
+function shared(node: FleetNode): Check {
+  const own = workerTokenLocation(node.id);
+  return {
+    id: 'auth:vault-isolation',
+    status: 'warn',
+    summary: 'VAULT_WORKER_TOKEN_SHARED',
+    remedy: {
+      action: `${node.id}'s token was read from the fleet's shared document, which every worker can read. Issue it a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes (kept at ${own.mount}/${own.path}), then remove ${workerTokenField(node.id)} from the shared document.`,
+    },
+  };
+}
+
 function required(node: FleetNode): Check {
   const { addr } = resolveVaultConfig();
   return {
@@ -160,7 +209,7 @@ function required(node: FleetNode): Check {
     status: 'fail',
     summary: 'VAULT_LOGIN_REQUIRED',
     remedy: {
-      action: `Run devfleet fleet-sync in a terminal, or store this node's token in the shared Vault document as ${workerTokenField(node.id)}. Manual fallback on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
+      action: `Issue ${node.id} a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes. Opt-in fallback: devfleet fleet-sync --vault-password-login in a terminal, or on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
     },
   };
 }

package/src/cli/fleet-sync.ts

@@ -1,3 +1,4 @@
+import type { ManagerReport } from '../manager-sync';
 import { localCheckoutRevision } from '../machine/local-source';
 import {
   definitionSource,
@@ -63,6 +64,14 @@ export interface FleetSyncOptions {
   json?: boolean;
   nonInteractive?: boolean;
   node?: string[];
+  /** The worker to prove a revision new to the fleet on first, when it is
+   * reachable; overrides `canary` in fleet.json.
+   */
+  canary?: string;
+  /** Opt-in: a worker without a token of its own may be logged in with the
+   * fleet's shared Vault user, typed on its own terminal.
+   */
+  vaultPasswordLogin?: boolean;
   all?: boolean;
   schedule?: ScheduleAction;
   /** `accounts`: only the account-derived configuration, on every node. */
@@ -316,7 +325,7 @@ function applyLocked(
         { ...request, rolledOut: state?.appliedRevision === revision },
         settings,
         run,
-        (report) => record.settle(report.ok)
+        (report) => record.settle(report.ok, appliedRevisions(report))
       );
     } finally {
       await record.abandon();
@@ -356,7 +365,7 @@ async function offManager(
   if (forwardingDisabled())
     return actionRequired(
       'FLEET_NOT_MANAGER',
-      `fleet-sync runs only on the manager (${definition.manager.id}); nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
+      `fleet-sync runs only on the manager (${definition.manager.id}), which is always a macOS machine; nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
       revision ?? null
     );
   const runScript = revision
@@ -520,3 +529,14 @@ async function selectedRevision(
     approved
   );
 }
+
+/** The revision each worker holds after this run, for the fleet state: a
+ * worker that converged keeps its progress even when the fleet is not ready.
+ */
+function appliedRevisions(report: ManagerReport): Record<string, string> {
+  return Object.fromEntries(
+    report.nodes
+      .filter((node) => node.role !== 'manager' && node.appliedRevision)
+      .map((node) => [node.node, node.appliedRevision!])
+  );
+}