@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/manager-sync.ts

@@ -0,0 +1,405 @@
+import { diagnostic, type Diagnostic } from './diagnostics';
+import {
+  blocking,
+  type Check,
+  type FleetNode,
+  type NodeReport,
+  type Runner,
+} from './types';
+import { discoverTailnet, type TailnetComputer } from './tailscale';
+import { matchComputers } from './tailnet-matching';
+import { sshArgs, sshFailure } from './ssh';
+import type { FleetDefinition, MachineDefinition } from './definition';
+import { publicHubUrl } from './monitoring/beszel';
+import {
+  convergeable,
+  evaluatePlatform,
+  PLATFORM_PROBE,
+  type MachinePlatform,
+} from './machine/platform';
+
+export interface ManagerNode extends NodeReport {
+  /** The singleton manager converges locally; workers over SSH. */
+  role?: 'manager' | 'worker';
+  reachability: 'online' | 'offline' | 'missing' | 'conflict' | 'unknown';
+  desiredRevision: string | null;
+  appliedRevision: string | null;
+  durationMs: number;
+  diagnostics?: Diagnostic[];
+}
+export interface ManagerReport {
+  version: 3;
+  command: 'fleet-sync';
+  execution: 'manager-push';
+  ok: boolean;
+  applied: boolean;
+  desiredRevision: string | null;
+  checks: Check[];
+  nodes: ManagerNode[];
+  candidates: number;
+  dashboardUrl: string | null;
+}
+export type DeployNode = (
+  node: FleetNode,
+  revision: string
+) => Promise<NodeReport>;
+
+/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git. */
+export async function inspectBootstrap(
+  node: FleetNode,
+  run: Runner,
+  os: MachinePlatform = 'linux'
+): Promise<Check[]> {
+  const result = await run([...sshArgs(node), PLATFORM_PROBE], {
+    timeoutMs: 20_000,
+  });
+  if (!result.ok) return [sshFailure(result)];
+  const baseline = evaluatePlatform(os, result.stdout);
+  return [
+    {
+      id: 'ssh:fresh-session',
+      status: 'pass',
+      summary:
+        'Fresh host-key-verified SSH session succeeded without remote developer prerequisites.',
+    },
+    {
+      id: baseline.id,
+      status: baseline.supported ? 'pass' : 'fail',
+      summary: baseline.summary,
+      remedy: {
+        action:
+          'Review the release/architecture migration in source. No operating-system upgrade or downgrade is performed.',
+      },
+    },
+  ];
+}
+
+/** One invocation, one immutable revision. Serial application bounds pressure and
+ * lets a failed declared canary stop promotion; diagnostics still cover all nodes.
+ */
+export async function synchronizeManager(
+  input: {
+    definition: FleetDefinition;
+    nodes: FleetNode[];
+    revision: string;
+    apply: boolean;
+    canary?: string;
+    progress?: (node: string, phase: string) => void;
+    /** Called once per node as soon as its result is final. */
+    finished?: (node: ManagerNode) => void;
+    /** Converges the manager locally; omitted in tests. */
+    manager?: () => Promise<ManagerNode>;
+  },
+  run: Runner,
+  deploy: DeployNode
+): Promise<ManagerReport> {
+  const tailnet = await discoverTailnet(run);
+  const matches = await matchComputers(input.nodes, tailnet.computers, run);
+  const report: ManagerReport = {
+    version: 3,
+    command: 'fleet-sync',
+    execution: 'manager-push',
+    ok: false,
+    applied: false,
+    desiredRevision: input.revision,
+    checks: tailnet.checks,
+    // Printed for people: the stable public name, never a tailnet address.
+    dashboardUrl: publicHubUrl(),
+    nodes: [],
+    candidates: tailnet.computers.filter(
+      (peer) => !peer.self && !matches.get(peer.id)?.length
+    ).length,
+  };
+  const managerRun = input.manager ? convergeManagerNode(input) : null;
+  // The manager finishes first: updating it may restart the proxy every
+  // worker is then verified against.
+  if (managerRun) await managerRun;
+  const workers = await Promise.all(
+    input.definition.workers.filter(convergeable).map((machine) => {
+      input.progress?.(
+        machine.id,
+        'Inspecting enrolled identity and fresh SSH access'
+      );
+      return inspectManagerNode(
+        machine,
+        {
+          nodes: input.nodes,
+          revision: input.revision,
+          computers: tailnet.computers,
+          matches,
+        },
+        run
+      );
+    })
+  );
+  report.nodes.push(...workers);
+  if (input.apply) await applyNodes(report, input, deploy);
+  else workers.forEach((node) => input.finished?.(node));
+  if (managerRun) report.nodes.unshift(await managerRun);
+  report.ok = ready(report);
+  for (const node of report.nodes)
+    node.diagnostics = node.checks.map((check) => diagnostic(node.node, check));
+  return report;
+}
+async function convergeManagerNode(input: {
+  definition: FleetDefinition;
+  progress?: (node: string, phase: string) => void;
+  finished?: (node: ManagerNode) => void;
+  manager?: () => Promise<ManagerNode>;
+}): Promise<ManagerNode> {
+  input.progress?.(
+    input.definition.manager.id,
+    'Converging the manager and its singleton services locally'
+  );
+  const manager = await input.manager!();
+  input.finished?.(manager);
+  return manager;
+}
+
+/** Ready when nothing required failed; warnings and skips do not count. */
+function ready(report: ManagerReport): boolean {
+  return (
+    !report.checks.some(blocking) &&
+    report.nodes.length > 0 &&
+    report.nodes.every(
+      (node) => node.checks.length > 0 && !node.checks.some(blocking)
+    )
+  );
+}
+
+async function applyNodes(
+  report: ManagerReport,
+  input: {
+    nodes: FleetNode[];
+    revision: string;
+    canary?: string;
+    progress?: (node: string, phase: string) => void;
+    finished?: (node: ManagerNode) => void;
+  },
+  deploy: DeployNode
+): Promise<void> {
+  const workers = report.nodes.filter((node) => node.role !== 'manager');
+  // A revision new to the fleet is proven on one worker first. When the run
+  // selects only later workers, the first selected one proves it instead:
+  // narrowing a run never skips the canary.
+  const canary = input.canary
+    ? (workers.find((node) => node.node === input.canary) ?? workers[0])
+    : undefined;
+  let blocked = report.checks.some(blocking);
+  const step = async (result: ManagerNode) => {
+    const deployable = mayDeploy(result, input.nodes);
+    if (await rollOut(result, { deployable, blocked }, input, deploy))
+      report.applied = true;
+    input.finished?.(result);
+    return deployable;
+  };
+  // The canary goes first; a failed canary stops promotion. The remaining
+  // workers are independent, so they converge concurrently.
+  if (canary) {
+    const deployable = await step(canary);
+    if (!deployable || canary.checks.some(blocksPromotion)) blocked = true;
+  }
+  await Promise.all(workers.filter((node) => node !== canary).map(step));
+}
+
+/** One node's rollout step; true when its remote operation ran. */
+async function rollOut(
+  result: ManagerNode,
+  { deployable, blocked }: { deployable: boolean; blocked: boolean },
+  input: {
+    nodes: FleetNode[];
+    revision: string;
+    progress?: (node: string, phase: string) => void;
+  },
+  deploy: DeployNode
+): Promise<boolean> {
+  if (!deployable) return false;
+  if (blocked) {
+    result.checks.push({
+      id: 'rollout:promotion',
+      status: 'fail',
+      summary: 'FLEET_PROMOTION_BLOCKED',
+      remedy: {
+        action:
+          'Resolve the discovery or canary failure above, then rerun the same approved revision.',
+      },
+    });
+    return false;
+  }
+  input.progress?.(
+    result.node,
+    'Applying the approved revision and verifying postconditions'
+  );
+  return deployNode(result, input, deploy);
+}
+
+/** Deploys one node and records its evidence; true when the remote ran. */
+async function deployNode(
+  result: ManagerNode,
+  input: { nodes: FleetNode[]; revision: string },
+  deploy: DeployNode
+): Promise<boolean> {
+  const started = Date.now();
+  let ran = false;
+  try {
+    const outcome = await deploy(
+      input.nodes.find((node) => node.id === result.node)!,
+      input.revision
+    );
+    ran = true;
+    result.checks.push(...outcome.checks);
+    if (!outcome.checks.length)
+      result.checks.push({
+        id: 'rollout:evidence',
+        status: 'fail',
+        summary: 'FLEET_VERIFICATION_MISSING',
+      });
+  } catch {
+    result.checks.push({
+      id: 'rollout:operation',
+      status: 'fail',
+      summary: 'FLEET_REMOTE_OPERATION_FAILED',
+      remedy: {
+        action:
+          'Inspect the remote operation lock before retrying. A disconnected manager does not establish that installation stopped.',
+      },
+    });
+  }
+  result.durationMs += Date.now() - started;
+  if (!result.checks.some(blocking)) result.appliedRevision = input.revision;
+  return ran;
+}
+
+function connectionState(
+  conflict: boolean,
+  peer: TailnetComputer | undefined
+): ManagerNode['reachability'] {
+  if (conflict) return 'conflict';
+  if (!peer) return 'missing';
+  return peer.online ? 'online' : 'offline';
+}
+async function inspectManagerNode(
+  machine: MachineDefinition,
+  context: {
+    nodes: FleetNode[];
+    revision: string;
+    computers: TailnetComputer[];
+    matches: Map<string, FleetNode[]>;
+  },
+  run: Runner
+): Promise<ManagerNode> {
+  const { nodes, revision, computers, matches } = context;
+  const id = machine.id;
+  const started = Date.now();
+  const node = nodes.find((entry) => entry.id === id);
+  const peers = computers.filter((peer) =>
+    matches.get(peer.id)?.some((entry) => entry.id === id)
+  );
+  const peer = peers[0];
+  const conflict =
+    peers.length > 1 || (peer && matches.get(peer.id)!.length > 1);
+  const checks: Check[] = [];
+  if (!node)
+    checks.push({
+      id: 'enrollment:connection',
+      status: 'fail',
+      summary: 'FLEET_CONNECTION_REQUIRED',
+      remedy: {
+        action:
+          'Add this canonical node’s reviewed SSH alias and managed workspace path to the private manager overrides. Do not substitute a discovered peer.',
+      },
+    });
+  else if (conflict)
+    checks.push({
+      id: 'tailscale:identity',
+      status: 'fail',
+      summary: 'TAILSCALE_IDENTITY_CONFLICT',
+      remedy: {
+        action:
+          'Resolve duplicate alias/device mappings before enrollment or apply.',
+      },
+    });
+  else {
+    checks.push(...(await inspectBootstrap(node, run, machine.os)));
+    checks.push(...identityChecks(node, peer));
+  }
+  if (!peer || !peer.online)
+    checks.push({
+      id: 'tailscale:reachability',
+      status: 'fail',
+      summary: peer ? 'TAILSCALE_NODE_OFFLINE' : 'TAILSCALE_NODE_MISSING',
+      remedy: {
+        action:
+          'Restore the intended node’s private connection. A trusted LAN SSH route can be inspected, but discovery never enrolls a replacement.',
+      },
+    });
+  return {
+    node: id,
+    role: 'worker',
+    checks,
+    desiredRevision: revision,
+    appliedRevision: null,
+    durationMs: Date.now() - started,
+    reachability: connectionState(Boolean(conflict), peer),
+  };
+}
+
+function identityChecks(
+  node: FleetNode,
+  peer: TailnetComputer | undefined
+): Check[] {
+  const checks: Check[] = [];
+  if (!node.tailscaleId || (peer && peer.id !== node.tailscaleId))
+    checks.push({
+      id: 'tailscale:enrolled-identity',
+      status: 'fail',
+      summary: node.tailscaleId
+        ? 'TAILSCALE_DEVICE_REPLACED'
+        : 'TAILSCALE_ENROLLMENT_REQUIRED',
+      remedy: {
+        action:
+          'Verify the device identity and SSH host key, then record its stable tailscaleId in the reviewed private connection override. Names alone cannot authorize replacement.',
+      },
+    });
+  return checks;
+}
+
+function mayDeploy(result: ManagerNode, nodes: FleetNode[]): boolean {
+  const allowBootstrap =
+    nodes.find((node) => node.id === result.node)?.bootstrapViaSsh === true;
+  return result.checks.every(
+    (check) =>
+      !blocking(check) ||
+      (allowBootstrap &&
+        [
+          'TAILSCALE_NODE_OFFLINE',
+          'TAILSCALE_NODE_MISSING',
+          'TAILSCALE_ENROLLMENT_REQUIRED',
+        ].includes(check.summary))
+  );
+}
+
+/** Service/account acceptance remains required for readiness, but a missing
+ * external account, dashboard or client pairing must not stop independent OS
+ * configuration on other nodes. Actual deployment failures stop promotion.
+ */
+export function blocksPromotion(check: Check): boolean {
+  if (!blocking(check)) return false;
+  if (check.id === 't3:service' && T3_MAINTENANCE.has(check.summary))
+    return false;
+  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|ubuntu:boot-policy$)/.test(
+    check.id
+  );
+}
+
+/** Native T3 outcomes that need an operator maintenance window (an active
+ * listener or running release must never be killed by routine sync). They are
+ * reported as failures but do not block independent OS policy on other nodes.
+ */
+const T3_MAINTENANCE = new Set([
+  'T3_LISTENER_CONFLICT',
+  'T3_ACTIVE_LISTENER_MIGRATION_REQUIRED',
+  'T3_MAINTENANCE_REQUIRED',
+  'T3_POLICY_RESTART_PENDING',
+  'T3_DESKTOP_MIGRATION_REQUIRED',
+]);

package/src/manager.ts

@@ -0,0 +1,62 @@
+import { compareFingerprints, fingerprint } from './fingerprint';
+import { probe } from './probe';
+import { mapBounded } from './report';
+import { bundleProbe, inspectRemote } from './transport';
+import type { Capability, FleetNode, NodeReport, Runner } from './types';
+
+export interface DoctorOptions {
+  root: string;
+  timeoutMs: number;
+  local: boolean;
+  required: Capability[];
+  source?: string;
+}
+
+export async function doctor(
+  nodes: FleetNode[],
+  options: DoctorOptions,
+  run: Runner
+): Promise<NodeReport[]> {
+  const local = await probe(options.root, options.required, run);
+  const agent = await run(['ssh-add', '-l']);
+  const reports: NodeReport[] = [
+    {
+      node: 'manager',
+      snapshot: local,
+      checks: [
+        ...local.checks,
+        {
+          id: 'ssh:agent',
+          status: agent.ok ? 'pass' : 'warn',
+          summary: agent.ok
+            ? 'SSH agent exposes at least one identity.'
+            : 'No accessible SSH agent identities; explicit IdentityFile keys may still work.',
+          ...(!agent.ok
+            ? {
+                remedy: {
+                  action:
+                    'Unlock/load the intended key in the manager session. Check SSH_AUTH_SOCK and GUI/editor agent visibility; do not copy private keys.',
+                },
+              }
+            : {}),
+        },
+      ],
+    },
+  ];
+  if (options.local || nodes.length === 0) return reports;
+  const source = options.source ?? (await bundleProbe());
+  const reference = await fingerprint(options.root, run);
+  const remote = await mapBounded(nodes, async (node) => {
+    const report = await inspectRemote(
+      node,
+      { action: 'probe', source, timeoutMs: options.timeoutMs },
+      run
+    );
+    if (report.snapshot)
+      report.checks.push(
+        ...compareFingerprints(reference, report.snapshot.fingerprint)
+      );
+    return report;
+  });
+  return [...reports, ...remote];
+}

package/src/monitoring/agent-evidence.ts

@@ -0,0 +1,146 @@
+import type { Check, Runner } from '../types';
+
+/** Where installing the agent writes: the pinned build and its state. */
+export const AGENT_PATHS = ['/usr/local/lib', '/var/lib'] as const;
+/** Room for the archive, the extracted build and the agent's state. */
+export const AGENT_FREE_KIB = 256 * 1024;
+
+/** Measures, before anything is installed, that each path the agent writes
+ * to has room. A full disk is named as such instead of surfacing later as an
+ * incomplete installation.
+ */
+export async function agentDisk(run: Runner): Promise<Check> {
+  const result = await run(['df', '-Pk', ...AGENT_PATHS], {
+    timeoutMs: 10_000,
+  });
+  const free = result.ok ? available(result.stdout) : null;
+  if (!free || free.length !== AGENT_PATHS.length)
+    return {
+      id: 'beszel:disk',
+      status: 'fail',
+      summary: 'BESZEL_DISK_UNVERIFIED',
+      remedy: {
+        action: `Free space under ${AGENT_PATHS.join(' and ')} could not be measured. Check that df works for the fleet user, then rerun fleet-sync. Nothing was installed.`,
+      },
+    };
+  const least = Math.min(...free);
+  if (least < AGENT_FREE_KIB)
+    return {
+      id: 'beszel:disk',
+      status: 'fail',
+      summary: 'BESZEL_DISK_FULL',
+      remedy: {
+        action: `The monitoring agent needs ${AGENT_FREE_KIB / 1024} MiB free under each of ${AGENT_PATHS.join(' and ')}; the fullest has ${Math.floor(least / 1024)} MiB. Free space on the worker, then rerun fleet-sync. Nothing was installed and the running agent was left as it was.`,
+      },
+    };
+  return {
+    id: 'beszel:disk',
+    status: 'pass',
+    summary: `At least ${Math.floor(least / 1024)} MiB free where the agent is installed.`,
+  };
+}
+
+/** The "Available" column of POSIX df output, one value per path. */
+function available(output: string): number[] | null {
+  const rows = output.trim().split('\n').slice(1);
+  const values = rows.map((row) => Number(row.trim().split(/\s+/)[3]));
+  return values.length && values.every(Number.isSafeInteger) ? values : null;
+}
+
+/** What the pinned agent (Beszel 0.21.1) logs about its connection to the
+ * hub, from its source:
+ * - `agent/connection_manager.go` logs "WebSocket connected" once the hub
+ *   proved its key;
+ * - the hub answers an unknown token with 401 (`internal/hub/agent_connect.go`),
+ *   which the WebSocket library reports as "unexpected status code: 401";
+ * - a hub whose key is not the one the agent was given fails
+ *   `verifySignature`: "invalid signature - check KEY value";
+ * - a hub that is down behind its private route answers 502, 503 or 504.
+ * Only these fixed texts are matched; nothing of the log is reported.
+ */
+const OUTCOMES: { pattern: RegExp; check: Omit<Check, 'id'> }[] = [
+  {
+    pattern: /WebSocket connected/,
+    check: {
+      status: 'pass',
+      summary: 'The agent is connected to the hub, which proved its key.',
+    },
+  },
+  {
+    pattern: /unexpected status code: 401\b/,
+    check: {
+      status: 'fail',
+      summary: 'BESZEL_AGENT_TOKEN_REJECTED',
+      remedy: {
+        action:
+          "The hub refused this worker's agent token. Rerun fleet-sync from the manager so the worker's enrollment is resumed and its token delivered again. Never register a second record for the worker.",
+      },
+    },
+  },
+  {
+    pattern: /invalid signature - check KEY value/,
+    check: {
+      status: 'fail',
+      summary: 'BESZEL_AGENT_KEY_REJECTED',
+      remedy: {
+        action:
+          "The hub the agent reached did not prove the key it was given. Make sure the worker's agent points at the manager's hub, then rerun fleet-sync from the manager so the hub's key is delivered again.",
+      },
+    },
+  },
+  {
+    pattern: /unexpected status code: 50[234]\b/,
+    check: {
+      status: 'warn',
+      summary: 'BESZEL_HUB_UNAVAILABLE_TO_AGENT',
+      remedy: {
+        action:
+          'The private route answered but the hub behind it did not. Keep the manager awake with Docker Desktop running and rerun fleet-sync there; the agent reconnects by itself.',
+      },
+    },
+  },
+];
+
+/** The agent's newest connection outcome, read from its own log. The log
+ * holds host names, so only the outcome is reported.
+ */
+export async function agentConnection(run: Runner): Promise<Check> {
+  const result = await run(
+    [
+      'sudo',
+      '-n',
+      'journalctl',
+      '--unit=beszel-agent.service',
+      '--output=cat',
+      '--no-pager',
+      '--lines=200',
+    ],
+    { timeoutMs: 10_000 }
+  );
+  if (!result.ok)
+    return {
+      id: 'beszel:hub-connection',
+      status: 'skip',
+      summary: 'BESZEL_AGENT_LOG_UNAVAILABLE',
+      remedy: {
+        action:
+          "The agent's log could not be read, so how the hub answered it is unknown. The hub's own evidence in fleet-status --workers still applies.",
+      },
+    };
+  return { id: 'beszel:hub-connection', ...newestOutcome(result.stdout) };
+}
+
+function newestOutcome(log: string): Omit<Check, 'id'> {
+  const lines = log.split('\n').reverse();
+  for (const line of lines)
+    for (const outcome of OUTCOMES)
+      if (outcome.pattern.test(line)) return outcome.check;
+  return {
+    status: 'skip',
+    summary: 'BESZEL_AGENT_CONNECTION_PENDING',
+    remedy: {
+      action:
+        'The agent has not reached the hub yet. The next fleet-sync, or fleet-status --workers, reports whether it did.',
+    },
+  };
+}