@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/cli/fleet-sync.ts

@@ -0,0 +1,309 @@
+import { localCheckoutRevision } from '../machine/local-source';
+import { fleet as canonicalDefinition } from '../fleet-config';
+import { managerFailure } from '../manager-failure';
+import { managerText, stdoutColor } from './fleet-sync-output';
+import { managerSchedule, type ScheduleAction } from './fleet-sync-schedule';
+import { enrollManager } from './fleet-sync-enrollment';
+import { authorizeManager } from './fleet-sync-vault';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { existsSync, renameSync } from 'node:fs';
+import {
+  readEnrollment,
+  isRevision,
+  saveEnrollment,
+  withMachineLock,
+  ensurePrivateRoot,
+} from '../machine/state';
+import { runCommand } from '../runner';
+import type { Runner } from '../types';
+import type { Settings } from './settings';
+import type { CommandOutcome } from './contracts';
+import { FleetUsageError } from '../errors';
+import { FleetLockError } from '../machine/lock-error';
+import { selected } from './fleet-sync-remote';
+import { managerIdentity } from './fleet-sync-manager';
+import { trackedScript } from '../remote-bootstrap';
+import { rehome } from './fleet-sync-checkout';
+import {
+  forwardingDisabled,
+  forwardToManager,
+  type ForwardSpawn,
+} from './fleet-sync-forward';
+
+declare const PERSONAL_SOURCE_REVISION: string | null;
+export interface FleetSyncOptions {
+  revision?: string;
+  dryRun?: boolean;
+  apply?: boolean;
+  yes?: boolean;
+  json?: boolean;
+  nonInteractive?: boolean;
+  node?: string[];
+  all?: boolean;
+  schedule?: ScheduleAction;
+  /** Internal: this revision already converged the whole fleet, so no canary
+   * has to prove it first and every worker converges at once.
+   */
+  rolledOut?: boolean;
+  /** Internal: set for interactive terminal runs (sudo enrollment prompts). */
+  interactiveAuthorization?: boolean;
+  expectedPlanFingerprint?: string;
+}
+export function machineRoot(): string {
+  const root = join(homedir(), '.local/share/personal/fleet/manager');
+  // One-time move from the pre-topology name; keeps the approved revision.
+  const legacy = join(homedir(), '.local/share/personal/fleet/controller');
+  if (!existsSync(root) && existsSync(legacy)) renameSync(legacy, root);
+  return root;
+}
+
+/** Public contract v3: the manager converges itself, then every enrolled worker. */
+export async function fleetSync(
+  options: FleetSyncOptions,
+  settings: Settings,
+  run: Runner = runCommand,
+  forward?: ForwardSpawn
+): Promise<CommandOutcome> {
+  validate(options);
+  if (!(await onManager(run)))
+    return offManager(options, settings, run, forward);
+  if (options.schedule === 'status') return scheduleStatus(settings, run);
+  const root = machineRoot();
+  await ensurePrivateRoot(root);
+  const state = await readEnrollment(root);
+  const revision = await selectedRevision(
+    options.revision,
+    state?.approvedRevision,
+    settings,
+    run
+  );
+  if (!isRevision(revision))
+    return actionRequired(
+      'FLEET_REVISION_REQUIRED',
+      'Select an approved full SHA with --revision, or use a published package containing clean source provenance.'
+    );
+  if (options.schedule)
+    return scheduleRequest(options, revision, settings, run);
+  const rehomed = await rehome(
+    options,
+    { revision, workspace: settings.workspace },
+    run
+  );
+  if (rehomed) return rehomed;
+  const request = { ...options, revision };
+  if (request.dryRun) return selected(request, settings, run);
+  // Applies without a confirmation prompt; --dry-run is the preview.
+  request.apply = true;
+  request.yes = true;
+  if (interactive(options)) {
+    await enrollManager(settings.config, run);
+    // Needed to hand each node its own Vault token; nodes report any gap.
+    await authorizeManager({ interactive: true }, run);
+    request.interactiveAuthorization = true;
+  }
+  return applyLocked(root, state, request, { settings, run });
+}
+
+/** Records the approved revision, converges, then records what was applied. */
+function applyLocked(
+  root: string,
+  state: Awaited<ReturnType<typeof readEnrollment>>,
+  request: FleetSyncOptions & { revision: string },
+  context: { settings: Settings; run: Runner }
+): Promise<CommandOutcome> {
+  const { revision } = request;
+  const { settings, run } = context;
+  return withMachineLock(root, async () => {
+    const approval = {
+      ...state,
+      version: 1 as const,
+      id: 'manager',
+      approvedRevision: revision,
+    };
+    await saveEnrollment(root, approval);
+    const result = await selected(
+      { ...request, rolledOut: state?.appliedRevision === revision },
+      settings,
+      run
+    );
+    if (result.exitCode === 0)
+      await saveEnrollment(root, {
+        ...approval,
+        appliedRevision: revision,
+        ...(state?.appliedRevision && state.appliedRevision !== revision
+          ? { previousRevision: state.appliedRevision }
+          : {}),
+      });
+    return result;
+  }).catch((error: unknown) => {
+    if (error instanceof FleetLockError)
+      return actionRequired(
+        'FLEET_MANAGER_LOCKED',
+        'Another manager run or interrupted lock exists. Inspect manager and remote processes before removing an abandoned sync.lock; never steal active package-manager locks.'
+      );
+    throw error;
+  });
+}
+/** Every run (previews and schedules too) happens on the manager: a worker can
+ * neither converge the manager's singletons nor observe its T3 links, so a
+ * partial run there only produces misleading failures.
+ */
+async function onManager(run: Runner): Promise<boolean> {
+  const { manager } = canonicalDefinition;
+  return (await managerIdentity(manager.id, run)).status === 'pass';
+}
+
+/** Hands the request to the manager; refuses (without touching any worker)
+ * when hand-off is disabled, e.g. on the manager side of a hand-off.
+ */
+async function offManager(
+  options: FleetSyncOptions,
+  settings: Settings,
+  run: Runner,
+  forward: ForwardSpawn | undefined
+): Promise<CommandOutcome> {
+  const definition = canonicalDefinition;
+  const revision =
+    options.revision ??
+    packageRevision() ??
+    (await localCheckoutRevision(settings.workspace, run));
+  if (forwardingDisabled())
+    return actionRequired(
+      'FLEET_NOT_MANAGER',
+      `fleet-sync runs only on the manager (${definition.manager.id}); nothing was changed. Run it there, or from a worker that can SSH to the manager.`,
+      revision ?? null
+    );
+  const runScript = revision
+    ? await trackedScript(
+        settings.workspace,
+        revision,
+        'bootstrap/fleet-manager-run.sh',
+        run
+      )
+    : null;
+  return forwardToManager(definition, options, revision, {
+    spawn: forward,
+    script: runScript,
+  });
+}
+
+function scheduleStatus(
+  settings: Settings,
+  run: Runner
+): Promise<CommandOutcome> {
+  return managerSchedule(
+    {
+      action: 'status',
+      revision: '',
+      config: settings.config,
+      authorized: false,
+    },
+    run
+  );
+}
+
+function actionRequired(
+  code: string,
+  remedy: string,
+  revision: string | null = null
+): CommandOutcome {
+  const report = managerFailure(canonicalDefinition, revision, code, remedy);
+  return {
+    exitCode: 1,
+    data: report,
+    text: managerText(report, { color: stdoutColor() }),
+  };
+}
+
+function validate(options: FleetSyncOptions): void {
+  validateSchedule(options.schedule);
+  if (options.all && options.node?.length)
+    throw new FleetUsageError('Choose --all or --node, not both.');
+  if (options.revision !== undefined && !isRevision(options.revision))
+    throw new FleetUsageError(
+      '--revision requires an exact lowercase 40-character commit SHA.'
+    );
+  if (options.dryRun && (options.apply || options.yes))
+    throw new FleetUsageError(
+      '--dry-run cannot be combined with apply confirmation.'
+    );
+}
+function interactive(options: FleetSyncOptions): boolean {
+  return (
+    ![
+      options.dryRun,
+      options.apply,
+      options.yes,
+      options.json,
+      options.nonInteractive,
+      process.env['CI'],
+    ].some(Boolean) && Boolean(process.stdin.isTTY && process.stderr.isTTY)
+  );
+}
+
+function packageRevision(): string | undefined {
+  return typeof PERSONAL_SOURCE_REVISION === 'string'
+    ? PERSONAL_SOURCE_REVISION
+    : undefined;
+}
+
+async function scheduleRequest(
+  options: FleetSyncOptions,
+  revision: string,
+  settings: Settings,
+  run: Runner
+): Promise<CommandOutcome> {
+  const root = machineRoot();
+
+  if (options.dryRun)
+    return actionRequired(
+      'FLEET_SCHEDULE_PREVIEW',
+      'No schedule changed. Inspect --schedule status; scheduling writes require --apply --yes.'
+    );
+  if (!options.apply || !options.yes)
+    return managerSchedule(
+      {
+        action: options.schedule!,
+        revision,
+        config: settings.config,
+        authorized: false,
+      },
+      run
+    );
+  return withMachineLock(root, () =>
+    managerSchedule(
+      {
+        action: options.schedule!,
+        revision,
+        config: settings.config,
+        authorized: Boolean(options.apply && options.yes),
+      },
+      run
+    )
+  );
+}
+
+function validateSchedule(schedule: unknown): void {
+  if (
+    schedule !== undefined &&
+    !['enable', 'pause', 'resume', 'disable', 'status'].includes(
+      String(schedule)
+    )
+  )
+    throw new FleetUsageError('Unknown manager schedule action.');
+}
+
+async function selectedRevision(
+  explicit: string | undefined,
+  approved: string | undefined,
+  settings: Settings,
+  run: Runner
+): Promise<string | undefined> {
+  return (
+    explicit ??
+    packageRevision() ??
+    (await localCheckoutRevision(settings.workspace, run)) ??
+    approved
+  );
+}

package/src/cli/fleet.ts

@@ -0,0 +1,163 @@
+import { selectNodes } from '../config';
+import { doctor } from '../manager';
+import { FleetUsageError } from '../errors';
+import { onboarding } from '../onboard';
+import { makeReport, renderReport, reportExitCode } from '../report';
+import { runCommand } from '../runner';
+import schema from '../../schema/roster.schema.json';
+import type { FleetConfig, FleetNode, NodeReport, Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+import type { Settings } from './settings';
+import { capabilities, timeout, validateOptions } from './options';
+import type { CliOptions } from './options';
+import { addNode, nodeFromOptions, readRoster } from './roster';
+import { executeTailnet } from './tailnet';
+import { probeSource } from './probe-source';
+import { executeRepairCommand } from './repair';
+
+export interface FleetServices {
+  run: Runner;
+  source: () => Promise<string>;
+}
+
+const SERVICES: FleetServices = { run: runCommand, source: probeSource };
+
+export async function executeFleet(
+  command: string,
+  options: CliOptions,
+  settings: Settings,
+  services: FleetServices = SERVICES
+): Promise<CommandOutcome> {
+  validateOptions(command, options);
+  if (['repair', 'repairs'].includes(command))
+    return executeRepairCommand(command, options, settings, services);
+  if (['discover', 'health'].includes(command))
+    return executeTailnet(command, options, settings, services);
+  if (command === 'schema') return result(schema);
+  if (command === 'add') return register(settings, options);
+  const roster = options.local ? null : await readRoster(settings.config);
+  const nodes = resolveNodes(command, options, roster);
+  if (['nodes', 'onboard'].includes(command))
+    return rosterResult(command, nodes, roster !== null);
+  return diagnose(
+    nodes,
+    options,
+    { ...settings, missingRoster: !roster },
+    services
+  );
+}
+
+function rosterResult(
+  command: string,
+  nodes: FleetNode[],
+  configured: boolean
+): CommandOutcome {
+  return result({
+    version: 1,
+    command,
+    ok: true,
+    ...(command === 'nodes' ? { configured } : {}),
+    nodes:
+      command === 'onboard'
+        ? nodes.map((node) => onboarding(node, 'devfleet'))
+        : nodes,
+  });
+}
+
+function resolveNodes(
+  command: string,
+  options: CliOptions,
+  roster: FleetConfig | null
+): FleetNode[] {
+  if (!roster && !options.local && !['doctor', 'nodes'].includes(command))
+    throw new FleetUsageError(
+      'No fleet roster. Use fleet-node-add or the interactive Register action; use fleet-status --local for this machine.'
+    );
+  if (options.node?.length && !roster)
+    throw new FleetUsageError(
+      'Cannot select nodes without a roster. Use fleet-node-add first.'
+    );
+  return roster ? selectNodes(roster, options.node ?? []) : [];
+}
+
+function result(data: unknown, text?: string): CommandOutcome {
+  return { exitCode: 0, data, ...(text ? { text } : {}) };
+}
+
+async function register(
+  settings: Settings,
+  options: CliOptions
+): Promise<CommandOutcome> {
+  const node = nodeFromOptions(options);
+  const config = await addNode(settings.config, node, options.apply === true);
+  return result({
+    version: 1,
+    command: 'add',
+    ok: true,
+    applied: options.apply === true,
+    path: settings.config,
+    node,
+    nodes: config.nodes,
+    next: options.apply
+      ? 'Run fleet-nodes --onboard, then fleet-status --json.'
+      : 'Review this preview, then repeat with --apply --yes.',
+  });
+}
+
+async function diagnose(
+  nodes: FleetNode[],
+  options: CliOptions,
+  settings: Settings & { missingRoster: boolean },
+  services: FleetServices
+): Promise<CommandOutcome> {
+  const source = nodes.length ? await services.source() : undefined;
+  const reports = await doctor(
+    nodes,
+    {
+      root: settings.workspace,
+      local: options.local === true,
+      required: capabilities(options.require),
+      timeoutMs: timeout(options),
+      ...(source ? { source } : {}),
+    },
+    services.run
+  );
+  if (!options.local && settings.missingRoster) reports.push(missingRoster());
+  const report = {
+    ...makeReport('doctor', reports),
+    coverage: {
+      scope: options.local
+        ? 'local'
+        : options.node?.length
+          ? 'selected'
+          : 'all-configured',
+      manager: true,
+      configuredNodesTested: nodes.map((node) => node.id),
+      rosterAvailable: !settings.missingRoster,
+      note: 'Directed SSH checks from this manager only. T3 relay/peer and VS Code forwarding require operator tests.',
+    },
+  };
+  return {
+    exitCode: reportExitCode(report, options.strict === true),
+    data: report,
+    text: renderReport(report),
+  };
+}
+
+function missingRoster(): NodeReport {
+  return {
+    node: 'fleet',
+    checks: [
+      {
+        id: 'fleet:roster',
+        status: 'fail',
+        summary:
+          'No developer fleet roster is configured. Only this manager could be checked; fleet-wide health is unknown.',
+        remedy: {
+          action:
+            'Open personal --interactive and choose Register a developer computer, or run devfleet fleet-node-add --help. Then run fleet-status --json.',
+        },
+      },
+    ],
+  };
+}

package/src/cli/managed-health.ts

@@ -0,0 +1,219 @@
+import { verifyT3Linked } from '../t3-linked-check';
+import { verifyT3Endpoint } from '../t3-endpoint';
+import type { Check, NodeReport, Runner } from '../types';
+import { fleet as definition } from '../fleet-config';
+import { enrolledRoster } from '../definition';
+import type { Tailnet } from '../tailscale';
+import type { FleetNode } from '../types';
+import {
+  PUBLIC_HUB_URL,
+  verifyBeszel,
+  type BeszelRequest,
+} from '../monitoring/beszel';
+import { hubAccess } from '../monitoring/hub-access';
+import { hubUrl } from '../topology';
+
+export interface DashboardAccess {
+  (hub: string): Promise<{ request: BeszelRequest }>;
+}
+
+/** The hub always runs on the manager, reached by its MagicDNS name. */
+export function dashboardUrl(tailnet: Tailnet): string | null {
+  const canonical = definition;
+  const visible = tailnet.computers.some(
+    (computer) =>
+      computer.dns.replace(/\.$/, '') ===
+      canonical.manager.network.tailscaleName
+  );
+  return visible ? hubUrl(canonical) : null;
+}
+
+/** Centralized dashboard evidence for every canonical worker: the hub must answer
+ * privately, and each node must be registered, connected and reporting fresh
+ * samples. A missing manager Vault session is reported, never silently passed.
+ */
+export async function appendDashboardEvidence(
+  reports: NodeReport[],
+  tailnet: Tailnet,
+  options: { access?: DashboardAccess; fetcher?: typeof fetch } = {}
+): Promise<string | null> {
+  const hub = dashboardUrl(tailnet);
+  const dashboard: NodeReport = { node: 'dashboard', checks: [] };
+  reports.push(dashboard);
+  if (!hub) {
+    dashboard.checks.push(
+      fail(
+        'beszel:hub',
+        'BESZEL_HUB_SELECTION_REQUIRED',
+        'The hub runs on the manager, which is not visible in Tailscale. Wake the manager and reconnect it to the tailnet.'
+      )
+    );
+    return null;
+  }
+  const fetcher = options.fetcher ?? fetch;
+  dashboard.checks.push(await hubReachable(hub, fetcher));
+  dashboard.checks.push(await publicEndpoint(fetcher));
+  await nodeEvidence(reports, dashboard, hub, options.access ?? hubAccess);
+  return hub;
+}
+
+async function nodeEvidence(
+  reports: NodeReport[],
+  dashboard: NodeReport,
+  hub: string,
+  access: DashboardAccess
+): Promise<void> {
+  const canonical = definition;
+  try {
+    const { request } = await access(hub);
+    for (const machine of canonical.workers) {
+      const evidence = await verifyBeszel(hub, machine.id, request);
+      const report =
+        reports.find(
+          (entry) =>
+            entry.node === machine.network?.tailscaleName ||
+            entry.node === machine.id
+        ) ?? dashboard;
+      report.checks.push(...evidence.checks);
+    }
+  } catch (error) {
+    dashboard.checks.push({
+      id: 'beszel:access',
+      status: 'skip',
+      summary:
+        error instanceof Error && error.message.startsWith('BESZEL_')
+          ? error.message
+          : 'BESZEL_ACCOUNT_REQUIRED',
+      remedy: {
+        action:
+          'Authenticate this manager to Vault so BESZEL_FLEET_AUTH can be read, then rerun fleet-status --workers. Node registration and metric freshness are unverified until then.',
+      },
+    });
+  }
+}
+
+async function hubReachable(
+  hub: string,
+  fetcher: typeof fetch
+): Promise<Check> {
+  try {
+    const response = await fetcher(`${hub}/api/health`, {
+      redirect: 'error',
+      signal: AbortSignal.timeout(10_000),
+    });
+    if (response.ok)
+      return {
+        id: 'beszel:hub',
+        status: 'pass',
+        summary: `Dashboard answers privately at ${hub}.`,
+      };
+  } catch {
+    /* Connection details are not evidence; fall through. */
+  }
+  return fail(
+    'beszel:hub',
+    'BESZEL_HUB_UNREACHABLE',
+    `The private dashboard did not answer at ${hub}/api/health. The hub runs on the manager: keep it awake with Docker Desktop running, then run devfleet fleet-sync there.`
+  );
+}
+
+/** The Cloudflare hostname is convenience access; the private hub stays canonical,
+ * so an offline connector warns instead of failing the audit.
+ */
+async function publicEndpoint(fetcher: typeof fetch): Promise<Check> {
+  try {
+    const response = await fetcher(`${PUBLIC_HUB_URL}/api/health`, {
+      redirect: 'error',
+      signal: AbortSignal.timeout(10_000),
+    });
+    if (response.ok)
+      return {
+        id: 'beszel:public-endpoint',
+        status: 'pass',
+        summary: `Dashboard answers publicly at ${PUBLIC_HUB_URL}.`,
+      };
+  } catch {
+    /* Connection details are not evidence; fall through. */
+  }
+  return {
+    id: 'beszel:public-endpoint',
+    status: 'warn',
+    summary: 'BESZEL_PUBLIC_ENDPOINT_OFFLINE',
+    remedy: {
+      action: `${PUBLIC_HUB_URL} did not answer. Create the Cloudflare tunnel and store its token in the fleet's secrets, then run fleet-sync on the manager so the hub starts its Cloudflare connector. The private dashboard is unaffected.`,
+    },
+  };
+}
+
+function fail(id: string, summary: string, action: string): Check {
+  return { id, status: 'fail', summary, remedy: { action } };
+}
+
+/** Explicit developer scope; the ordinary health command still audits all peers. */
+export function managedHealthScope(
+  tailnet: Tailnet,
+  roster: FleetNode[],
+  includeManager = true
+) {
+  const canonical = definition;
+  const declared = roster.filter((node) =>
+    canonical.workers.some((entry) => entry.id === node.id)
+  );
+  const nodes = declared.length
+    ? enrolledRoster(canonical, { version: 1, nodes: declared }).nodes
+    : [];
+  const computers = tailnet.computers.filter(
+    (peer) =>
+      (includeManager && peer.self) ||
+      canonical.workers.some(
+        (node) =>
+          peer.dns === node.network?.tailscaleName ||
+          peer.dns.split('.')[0] === node.ssh ||
+          nodes.some(
+            (entry) => entry.id === node.id && entry.tailscaleId === peer.id
+          )
+      )
+  );
+  const missing = canonical.workers.filter(
+    (node) => !nodes.some((entry) => entry.id === node.id)
+  );
+  return {
+    nodes,
+    tailnet: {
+      ...tailnet,
+      computers,
+      checks: [
+        ...tailnet.checks,
+        ...missing.map((node) => ({
+          id: `fleet:enrollment:${node.id}`,
+          status: 'fail' as const,
+          summary:
+            'Canonical developer node lacks a verified SSH/check-out registration.',
+          remedy: {
+            action: `Enroll ${node.id} with devfleet fleet-node-add, or use interactive fleet-sync enrollment.`,
+          },
+        })),
+      ],
+    },
+    excluded: tailnet.computers
+      .filter((peer) => !computers.includes(peer))
+      .map((peer) => peer.dns),
+  };
+}
+
+export async function appendManagerLinks(
+  reports: NodeReport[],
+  registered: FleetNode[],
+  run: Runner
+): Promise<void> {
+  for (const node of registered) {
+    const machine = definition.workers.find((entry) => entry.id === node.id);
+    const report = reports.find(
+      (entry) =>
+        entry.node === machine?.network?.tailscaleName || entry.node === node.id
+    );
+    if (!report) continue;
+    report.checks.push(...(await verifyT3Endpoint(node, run)));
+    report.checks.push(await verifyT3Linked(node, run));
+  }
+}