@crvouga/dev-fleet 0.6.0 → 0.8.0

package/src/cli/fleet-sync-vault.ts

@@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process';
 import { fleet as definition } from '../fleet-config';
 import { sshArgs, shellQuote } from '../ssh';
 import { resolveVaultConfig, usesVault } from '../vault-config';
+import { workerTokenLocation } from '../worker-credentials';
 import type { Check, FleetNode, Runner } from '../types';
 
 const TOOL_PATH =
@@ -15,7 +16,11 @@ const terminal: TerminalRunner = (args) =>
 
 type VaultState = 'authenticated' | 'unauthenticated' | 'unavailable';
 
-/** Vault field holding the issued token for one worker node. */
+/** The field of the fleet's shared document that held a worker's token
+ * before each worker had a document of its own (src/worker-credentials.ts).
+ * Read only when the worker's own document holds none, and reported: every
+ * worker that can read the shared document can read that field.
+ */
 export function workerTokenField(node: string): string {
   return `FLEET_WORKER_TOKEN_${node.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
 }
@@ -49,7 +54,13 @@ export async function authorizeManager(
  */
 export async function authorizeVault(
   node: FleetNode,
-  options: { interactive: boolean },
+  options: {
+    interactive: boolean;
+    /** Opt-in: the fleet's shared Vault user, typed on the worker's own
+     * terminal, when no token of its own was issued.
+     */
+    passwordLogin?: boolean;
+  },
   run: Runner,
   attach: TerminalRunner = terminal
 ): Promise<Check[]> {
@@ -59,8 +70,10 @@ export async function authorizeVault(
   if (before === 'authenticated') return [pass(node)];
   // Fresh node: the locked toolchain (including vault) is not installed yet.
   if (before === 'unavailable') return [];
-  if (await deliverWorkerToken(node, run)) return [pass(node)];
-  if (!options.interactive) return [required(node)];
+  const delivered = await deliverWorkerToken(node, run);
+  if (delivered === 'own') return [pass(node)];
+  if (delivered === 'shared') return [pass(node), shared(node)];
+  if (!options.interactive || !options.passwordLogin) return [required(node)];
   process.stderr.write(
     `[${node.id}] No issued worker token found. Enter the Vault password to log this machine in.\n`
   );
@@ -72,30 +85,54 @@ export async function authorizeVault(
   ];
 }
 
+/** Delivers the worker's token, from its own document or else from the
+ * field it used to have in the shared one; says which, or that none was.
+ */
 async function deliverWorkerToken(
   node: FleetNode,
   run: Runner
-): Promise<boolean> {
-  const { addr, mount, project, config } = resolveVaultConfig();
-  const secret = await run(
-    [
-      'vault',
-      'kv',
-      'get',
-      `-address=${addr}`,
-      `-mount=${mount}`,
-      `-field=${workerTokenField(node.id)}`,
-      `${project}/${config}`,
-    ],
-    { timeoutMs: 20_000 }
-  );
-  const token = secret.ok ? secret.stdout.trim() : '';
-  if (!/^[\x21-\x7e]{16,4096}$/.test(token)) return false;
-  const delivered = await run(
-    remote(node, ['vault', 'login', `-address=${addr}`, '-no-print', '-']),
-    { input: token + '\n', timeoutMs: 30_000 }
-  );
-  return delivered.ok && (await vaultState(node, run)) === 'authenticated';
+): Promise<'own' | 'shared' | null> {
+  const vault = resolveVaultConfig();
+  const own = workerTokenLocation(node.id, vault);
+  const sources = [
+    { from: 'own' as const, ...own },
+    {
+      from: 'shared' as const,
+      mount: vault.mount,
+      path: `${vault.project}/${vault.config}`,
+      field: workerTokenField(node.id),
+    },
+  ];
+  for (const source of sources) {
+    const secret = await run(
+      [
+        'vault',
+        'kv',
+        'get',
+        `-address=${vault.addr}`,
+        `-mount=${source.mount}`,
+        `-field=${source.field}`,
+        source.path,
+      ],
+      { timeoutMs: 20_000 }
+    );
+    const token = secret.ok ? secret.stdout.trim() : '';
+    if (!/^[\x21-\x7e]{16,4096}$/.test(token)) continue;
+    const delivered = await run(
+      remote(node, [
+        'vault',
+        'login',
+        `-address=${vault.addr}`,
+        '-no-print',
+        '-',
+      ]),
+      { input: token + '\n', timeoutMs: 30_000 }
+    );
+    return delivered.ok && (await vaultState(node, run)) === 'authenticated'
+      ? source.from
+      : null;
+  }
+  return null;
 }
 
 async function vaultState(node: FleetNode, run: Runner): Promise<VaultState> {
@@ -153,6 +190,18 @@ function pass(node: FleetNode): Check {
   };
 }
 
+function shared(node: FleetNode): Check {
+  const own = workerTokenLocation(node.id);
+  return {
+    id: 'auth:vault-isolation',
+    status: 'warn',
+    summary: 'VAULT_WORKER_TOKEN_SHARED',
+    remedy: {
+      action: `${node.id}'s token was read from the fleet's shared document, which every worker can read. Issue it a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes (kept at ${own.mount}/${own.path}), then remove ${workerTokenField(node.id)} from the shared document.`,
+    },
+  };
+}
+
 function required(node: FleetNode): Check {
   const { addr } = resolveVaultConfig();
   return {
@@ -160,7 +209,7 @@ function required(node: FleetNode): Check {
     status: 'fail',
     summary: 'VAULT_LOGIN_REQUIRED',
     remedy: {
-      action: `Run devfleet fleet-sync in a terminal, or store this node's token in the shared Vault document as ${workerTokenField(node.id)}. Manual fallback on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
+      action: `Issue ${node.id} a token of its own with devfleet fleet-node-replace --id ${node.id} --apply --yes. Opt-in fallback: devfleet fleet-sync --vault-password-login in a terminal, or on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
     },
   };
 }

package/src/cli/fleet-sync.ts

@@ -1,3 +1,4 @@
+import type { ManagerReport } from '../manager-sync';
 import { localCheckoutRevision } from '../machine/local-source';
 import {
   definitionSource,
@@ -31,6 +32,8 @@ import {
   DIVERGED_REMEDY,
   mutateState,
   refresh,
+  leaseConflict,
+  leaseHeldMessage,
   stateContext,
   type Standing,
 } from '../state/sync';
@@ -61,6 +64,14 @@ export interface FleetSyncOptions {
   json?: boolean;
   nonInteractive?: boolean;
   node?: string[];
+  /** The worker to prove a revision new to the fleet on first, when it is
+   * reachable; overrides `canary` in fleet.json.
+   */
+  canary?: string;
+  /** Opt-in: a worker without a token of its own may be logged in with the
+   * fleet's shared Vault user, typed on its own terminal.
+   */
+  vaultPasswordLogin?: boolean;
   all?: boolean;
   schedule?: ScheduleAction;
   /** `accounts`: only the account-derived configuration, on every node. */
@@ -72,6 +83,8 @@ export interface FleetSyncOptions {
   /** Internal: set for interactive terminal runs (sudo enrollment prompts). */
   interactiveAuthorization?: boolean;
   expectedPlanFingerprint?: string;
+  /** Take the manager lease from another manager before it expired. */
+  takeOver?: boolean;
 }
 export function machineRoot(): string {
   const root = join(homedir(), '.local/share/personal/fleet/manager');
@@ -140,6 +153,31 @@ export interface ManagerSync {
   restart: () => Promise<CommandOutcome>;
 }
 
+/** Why the fleet state stops this run before any machine or the state is
+ * changed, if it does.
+ */
+function stateRefusal(
+  standing: Standing,
+  options: FleetSyncOptions,
+  run: Runner,
+  revision: string
+): CommandOutcome | null {
+  if (standing === 'diverged')
+    return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
+  // One manager at a time: another one's lease stops this run.
+  const held = leaseConflict({
+    ...stateContext(run),
+    takeOver: options.takeOver === true,
+  });
+  return held
+    ? actionRequired(
+        'FLEET_MANAGER_LEASE_HELD',
+        leaseHeldMessage(held),
+        revision
+      )
+    : null;
+}
+
 /** fleet-sync on the machine already known to be the manager. */
 export async function managerSync(
   options: FleetSyncOptions,
@@ -177,8 +215,8 @@ export async function managerSync(
   // version of the configuration, the roster and the approval, not from a
   // stale local copy. Refreshing never uploads.
   const fresh = await refresh(stateContext(run));
-  if (fresh.standing === 'diverged')
-    return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
+  const refused = stateRefusal(fresh.standing, options, run, revision);
+  if (refused) return refused;
   if (fresh.replaced) {
     // Everything read so far came from the stale copy, the manager's own
     // identity included.
@@ -274,7 +312,7 @@ function applyLocked(
 ): Promise<CommandOutcome> {
   const { settings, run, revision } = context;
   const record = runRecord(state, revision, {
-    context: stateContext(run),
+    context: { ...stateContext(run), takeOver: request.takeOver === true },
     standing: context.standing,
     ...(definitionSource().kind === 'file'
       ? { definition: JSON.stringify(canonicalDefinition, null, 2) + '\n' }
@@ -287,7 +325,7 @@ function applyLocked(
         { ...request, rolledOut: state?.appliedRevision === revision },
         settings,
         run,
-        (report) => record.settle(report.ok)
+        (report) => record.settle(report.ok, appliedRevisions(report))
       );
     } finally {
       await record.abandon();
@@ -327,7 +365,7 @@ async function offManager(
   if (forwardingDisabled())
     return actionRequired(
       'FLEET_NOT_MANAGER',
-      `fleet-sync runs only on the manager (${definition.manager.id}); nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
+      `fleet-sync runs only on the manager (${definition.manager.id}), which is always a macOS machine; nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
       revision ?? null
     );
   const runScript = revision
@@ -491,3 +529,14 @@ async function selectedRevision(
     approved
   );
 }
+
+/** The revision each worker holds after this run, for the fleet state: a
+ * worker that converged keeps its progress even when the fleet is not ready.
+ */
+function appliedRevisions(report: ManagerReport): Record<string, string> {
+  return Object.fromEntries(
+    report.nodes
+      .filter((node) => node.role !== 'manager' && node.appliedRevision)
+      .map((node) => [node.node, node.appliedRevision!])
+  );
+}

package/src/cli/fleet-validate.ts

@@ -0,0 +1,239 @@
+import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  DefinitionError,
+  parseDefinition,
+  type DefinitionProblem,
+  type FleetDefinition,
+} from '../definition';
+import { FleetUsageError } from '../errors';
+import { PACKAGE_ROOT } from '../package-info';
+import { ansibleCommand } from '../platform/ansible';
+import { runCommand } from '../runner';
+import type { Check, Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetValidateOptions {
+  file?: string;
+  ansible?: boolean;
+  json?: boolean;
+}
+
+export interface FleetValidateServices {
+  run: Runner;
+  env: NodeJS.ProcessEnv;
+  platform: NodeJS.Platform;
+  /** The package's Ansible playbooks, relative to `root`. */
+  root: string;
+}
+
+const SERVICES: FleetValidateServices = {
+  run: runCommand,
+  env: process.env,
+  platform: process.platform,
+  root: PACKAGE_ROOT,
+};
+
+/** The playbooks a definition must be able to drive, in the package. */
+const PLAYBOOK_DIRECTORIES = ['ansible/ubuntu', 'ansible/macos'];
+
+/** The Ansible inventory a definition describes: the manager, and every
+ * worker in the `workers` group in the order it is declared. Host names are
+ * the fleet ids; a machine is reached as fleet-sync reaches it.
+ */
+export function fleetInventory(definition: FleetDefinition): unknown {
+  const workers = Object.fromEntries(
+    definition.workers.map((worker) => [
+      worker.id,
+      {
+        ansible_host: worker.ssh,
+        ...(worker.network
+          ? {
+              ansible_user: worker.network.user,
+              ansible_port: worker.network.sshPort,
+            }
+          : {}),
+      },
+    ])
+  );
+  return {
+    all: {
+      children: {
+        manager: {
+          hosts: {
+            [definition.manager.id]: {
+              ansible_connection: 'local',
+              ansible_user: definition.manager.network.user,
+            },
+          },
+        },
+        workers: { hosts: workers },
+      },
+    },
+  };
+}
+
+/** Reads and judges one fleet.json. Nothing is contacted: with `--ansible`
+ * the pinned Ansible renders the inventory and checks the playbooks' syntax
+ * on this machine.
+ */
+export async function fleetValidate(
+  options: FleetValidateOptions,
+  services: FleetValidateServices = SERVICES
+): Promise<CommandOutcome> {
+  const path = options.file ?? services.env['DEVFLEET_CONFIG'];
+  if (!path)
+    throw new FleetUsageError(
+      'No fleet configuration to validate. Run devfleet from the project that holds your fleet.json, or pass --file PATH.'
+    );
+  let definition: FleetDefinition;
+  try {
+    definition = parseDefinition(await readJson(path));
+  } catch (error) {
+    if (error instanceof DefinitionError) return invalid(path, error.problem);
+    throw error;
+  }
+  const checks: Check[] = [
+    {
+      id: 'definition:schema',
+      status: 'pass',
+      summary: `${path} is a valid fleet definition: a manager and ${definition.workers.length} worker(s).`,
+    },
+  ];
+  if (options.ansible)
+    checks.push(...(await ansibleChecks(definition, services)));
+  const ok = checks.every((check) => check.status !== 'fail');
+  return {
+    exitCode: ok ? 0 : 1,
+    data: { version: 1, command: 'fleet-validate', ok, path, checks },
+    text: checks
+      .map(
+        (check) => `${check.status.toUpperCase()} ${check.id}: ${check.summary}`
+      )
+      .join('\n'),
+  };
+}
+
+async function readJson(path: string): Promise<unknown> {
+  let text: string;
+  try {
+    text = await readFile(path, 'utf8');
+  } catch {
+    throw new DefinitionError({
+      code: 'DEFINITION_UNREADABLE',
+      path: '$',
+      expected: 'a readable file',
+      actual: 'no file',
+    });
+  }
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    throw new DefinitionError({
+      code: 'DEFINITION_UNREADABLE',
+      path: '$',
+      expected: 'JSON',
+      actual: 'text that is not JSON',
+    });
+  }
+}
+
+function invalid(path: string, problem: DefinitionProblem): CommandOutcome {
+  const data = {
+    version: 1,
+    command: 'fleet-validate',
+    ok: false,
+    path,
+    problem,
+  };
+  return {
+    exitCode: 1,
+    data,
+    text: `${problem.code} at ${problem.path}: expected ${problem.expected}, found ${problem.actual}.`,
+  };
+}
+
+/** Renders the inventory, lists it with `ansible-inventory` and checks the
+ * syntax of every playbook against it.
+ */
+async function ansibleChecks(
+  definition: FleetDefinition,
+  services: FleetValidateServices
+): Promise<Check[]> {
+  const directory = await mkdtemp(join(tmpdir(), 'devfleet-validate-'));
+  const inventory = join(directory, 'inventory.json');
+  try {
+    await writeFile(inventory, JSON.stringify(fleetInventory(definition)), {
+      mode: 0o600,
+    });
+    const playbook = ansibleCommand(services.platform, services.root);
+    const tool = (name: string) => [...playbook.slice(0, -1), name];
+    const listed = await services.run(
+      [...tool('ansible-inventory'), '-i', inventory, '--list'],
+      { timeoutMs: 120_000 }
+    );
+    const checks: Check[] = [
+      listed.ok
+        ? {
+            id: 'definition:inventory',
+            status: 'pass',
+            summary: 'Ansible reads the inventory the definition describes.',
+          }
+        : {
+            id: 'definition:inventory',
+            status: 'fail',
+            summary: 'DEFINITION_INVENTORY_REJECTED',
+            remedy: {
+              action:
+                'Run devfleet fleet-validate --ansible on the manager after fleet-sync installed the pinned Ansible; report the failure as an issue if it persists.',
+            },
+          },
+    ];
+    const failed: string[] = [];
+    for (const file of await playbooks(services.root)) {
+      const result = await services.run(
+        [
+          ...playbook,
+          '-i',
+          inventory,
+          '--syntax-check',
+          join(services.root, file),
+        ],
+        { timeoutMs: 120_000 }
+      );
+      if (!result.ok) failed.push(file);
+    }
+    checks.push(
+      failed.length
+        ? {
+            id: 'definition:playbooks',
+            status: 'fail',
+            summary: `DEFINITION_PLAYBOOK_SYNTAX: ${failed.join(', ')}`,
+            remedy: {
+              action:
+                'The installed dev-fleet package carries a playbook Ansible cannot parse; report it as an issue.',
+            },
+          }
+        : {
+            id: 'definition:playbooks',
+            status: 'pass',
+            summary: 'Every playbook passes ansible-playbook --syntax-check.',
+          }
+    );
+    return checks;
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+
+/** Every top-level playbook the fleet applies, sorted. */
+async function playbooks(root: string): Promise<string[]> {
+  const found: string[] = [];
+  for (const directory of PLAYBOOK_DIRECTORIES) {
+    const entries = await readdir(join(root, directory)).catch(() => []);
+    for (const entry of entries)
+      if (entry.endsWith('.yml')) found.push(join(directory, entry));
+  }
+  return found.sort();
+}