@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/manager-sync.ts

@@ -0,0 +1,400 @@
+import { diagnostic, type Diagnostic } from './diagnostics';
+import {
+  blocking,
+  type Check,
+  type FleetNode,
+  type NodeReport,
+  type Runner,
+} from './types';
+import { discoverTailnet, type TailnetComputer } from './tailscale';
+import { matchComputers } from './tailnet-matching';
+import { sshArgs, sshFailure } from './ssh';
+import type { FleetDefinition, MachineDefinition } from './definition';
+import { PUBLIC_HUB_URL } from './monitoring/beszel';
+import {
+  convergeable,
+  evaluatePlatform,
+  PLATFORM_PROBE,
+  type MachinePlatform,
+} from './machine/platform';
+
+export interface ManagerNode extends NodeReport {
+  /** The singleton manager converges locally; workers over SSH. */
+  role?: 'manager' | 'worker';
+  reachability: 'online' | 'offline' | 'missing' | 'conflict' | 'unknown';
+  desiredRevision: string | null;
+  appliedRevision: string | null;
+  durationMs: number;
+  diagnostics?: Diagnostic[];
+}
+export interface ManagerReport {
+  version: 3;
+  command: 'fleet-sync';
+  execution: 'manager-push';
+  ok: boolean;
+  applied: boolean;
+  desiredRevision: string | null;
+  checks: Check[];
+  nodes: ManagerNode[];
+  candidates: number;
+  dashboardUrl: string | null;
+}
+export type DeployNode = (
+  node: FleetNode,
+  revision: string
+) => Promise<NodeReport>;
+
+/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git. */
+export async function inspectBootstrap(
+  node: FleetNode,
+  run: Runner,
+  os: MachinePlatform = 'linux'
+): Promise<Check[]> {
+  const result = await run([...sshArgs(node), PLATFORM_PROBE], {
+    timeoutMs: 20_000,
+  });
+  if (!result.ok) return [sshFailure(result)];
+  const baseline = evaluatePlatform(os, result.stdout);
+  return [
+    {
+      id: 'ssh:fresh-session',
+      status: 'pass',
+      summary:
+        'Fresh host-key-verified SSH session succeeded without remote developer prerequisites.',
+    },
+    {
+      id: baseline.id,
+      status: baseline.supported ? 'pass' : 'fail',
+      summary: baseline.summary,
+      remedy: {
+        action:
+          'Review the release/architecture migration in source. No operating-system upgrade or downgrade is performed.',
+      },
+    },
+  ];
+}
+
+/** One invocation, one immutable revision. Serial application bounds pressure and
+ * lets a failed declared canary stop promotion; diagnostics still cover all nodes.
+ */
+export async function synchronizeManager(
+  input: {
+    definition: FleetDefinition;
+    nodes: FleetNode[];
+    revision: string;
+    apply: boolean;
+    canary?: string;
+    progress?: (node: string, phase: string) => void;
+    /** Called once per node as soon as its result is final. */
+    finished?: (node: ManagerNode) => void;
+    /** Converges the manager locally; omitted in tests. */
+    manager?: () => Promise<ManagerNode>;
+  },
+  run: Runner,
+  deploy: DeployNode
+): Promise<ManagerReport> {
+  const tailnet = await discoverTailnet(run);
+  const matches = await matchComputers(input.nodes, tailnet.computers, run);
+  const report: ManagerReport = {
+    version: 3,
+    command: 'fleet-sync',
+    execution: 'manager-push',
+    ok: false,
+    applied: false,
+    desiredRevision: input.revision,
+    checks: tailnet.checks,
+    // Printed for people: the stable public name, never a tailnet address.
+    dashboardUrl: PUBLIC_HUB_URL,
+    nodes: [],
+    candidates: tailnet.computers.filter(
+      (peer) => !peer.self && !matches.get(peer.id)?.length
+    ).length,
+  };
+  const managerRun = input.manager ? convergeManagerNode(input) : null;
+  // The manager finishes first: updating it may restart the proxy every
+  // worker is then verified against.
+  if (managerRun) await managerRun;
+  const workers = await Promise.all(
+    input.definition.workers.filter(convergeable).map((machine) => {
+      input.progress?.(
+        machine.id,
+        'Inspecting enrolled identity and fresh SSH access'
+      );
+      return inspectManagerNode(
+        machine,
+        {
+          nodes: input.nodes,
+          revision: input.revision,
+          computers: tailnet.computers,
+          matches,
+        },
+        run
+      );
+    })
+  );
+  report.nodes.push(...workers);
+  if (input.apply) await applyNodes(report, input, deploy);
+  else workers.forEach((node) => input.finished?.(node));
+  if (managerRun) report.nodes.unshift(await managerRun);
+  report.ok = ready(report);
+  for (const node of report.nodes)
+    node.diagnostics = node.checks.map((check) => diagnostic(node.node, check));
+  return report;
+}
+async function convergeManagerNode(input: {
+  definition: FleetDefinition;
+  progress?: (node: string, phase: string) => void;
+  finished?: (node: ManagerNode) => void;
+  manager?: () => Promise<ManagerNode>;
+}): Promise<ManagerNode> {
+  input.progress?.(
+    input.definition.manager.id,
+    'Converging the manager and its singleton services locally'
+  );
+  const manager = await input.manager!();
+  input.finished?.(manager);
+  return manager;
+}
+
+/** Ready when nothing required failed; warnings and skips do not count. */
+function ready(report: ManagerReport): boolean {
+  return (
+    !report.checks.some(blocking) &&
+    report.nodes.length > 0 &&
+    report.nodes.every(
+      (node) => node.checks.length > 0 && !node.checks.some(blocking)
+    )
+  );
+}
+
+async function applyNodes(
+  report: ManagerReport,
+  input: {
+    nodes: FleetNode[];
+    revision: string;
+    canary?: string;
+    progress?: (node: string, phase: string) => void;
+    finished?: (node: ManagerNode) => void;
+  },
+  deploy: DeployNode
+): Promise<void> {
+  const workers = report.nodes.filter((node) => node.role !== 'manager');
+  const canary = workers.find((node) => node.node === input.canary);
+  let blocked = report.checks.some(blocking);
+  const step = async (result: ManagerNode) => {
+    const deployable = mayDeploy(result, input.nodes);
+    if (await rollOut(result, { deployable, blocked }, input, deploy))
+      report.applied = true;
+    input.finished?.(result);
+    return deployable;
+  };
+  // The canary goes first; a failed canary stops promotion. The remaining
+  // workers are independent, so they converge concurrently.
+  if (canary) {
+    const deployable = await step(canary);
+    if (!deployable || canary.checks.some(blocksPromotion)) blocked = true;
+  }
+  await Promise.all(workers.filter((node) => node !== canary).map(step));
+}
+
+/** One node's rollout step; true when its remote operation ran. */
+async function rollOut(
+  result: ManagerNode,
+  { deployable, blocked }: { deployable: boolean; blocked: boolean },
+  input: {
+    nodes: FleetNode[];
+    revision: string;
+    progress?: (node: string, phase: string) => void;
+  },
+  deploy: DeployNode
+): Promise<boolean> {
+  if (!deployable) return false;
+  if (blocked) {
+    result.checks.push({
+      id: 'rollout:promotion',
+      status: 'fail',
+      summary: 'FLEET_PROMOTION_BLOCKED',
+      remedy: {
+        action:
+          'Resolve the discovery or canary failure above, then rerun the same approved revision.',
+      },
+    });
+    return false;
+  }
+  input.progress?.(
+    result.node,
+    'Applying the approved revision and verifying postconditions'
+  );
+  return deployNode(result, input, deploy);
+}
+
+/** Deploys one node and records its evidence; true when the remote ran. */
+async function deployNode(
+  result: ManagerNode,
+  input: { nodes: FleetNode[]; revision: string },
+  deploy: DeployNode
+): Promise<boolean> {
+  const started = Date.now();
+  let ran = false;
+  try {
+    const outcome = await deploy(
+      input.nodes.find((node) => node.id === result.node)!,
+      input.revision
+    );
+    ran = true;
+    result.checks.push(...outcome.checks);
+    if (!outcome.checks.length)
+      result.checks.push({
+        id: 'rollout:evidence',
+        status: 'fail',
+        summary: 'FLEET_VERIFICATION_MISSING',
+      });
+  } catch {
+    result.checks.push({
+      id: 'rollout:operation',
+      status: 'fail',
+      summary: 'FLEET_REMOTE_OPERATION_FAILED',
+      remedy: {
+        action:
+          'Inspect the remote operation lock before retrying. A disconnected manager does not establish that installation stopped.',
+      },
+    });
+  }
+  result.durationMs += Date.now() - started;
+  if (!result.checks.some(blocking)) result.appliedRevision = input.revision;
+  return ran;
+}
+
+function connectionState(
+  conflict: boolean,
+  peer: TailnetComputer | undefined
+): ManagerNode['reachability'] {
+  if (conflict) return 'conflict';
+  if (!peer) return 'missing';
+  return peer.online ? 'online' : 'offline';
+}
+async function inspectManagerNode(
+  machine: MachineDefinition,
+  context: {
+    nodes: FleetNode[];
+    revision: string;
+    computers: TailnetComputer[];
+    matches: Map<string, FleetNode[]>;
+  },
+  run: Runner
+): Promise<ManagerNode> {
+  const { nodes, revision, computers, matches } = context;
+  const id = machine.id;
+  const started = Date.now();
+  const node = nodes.find((entry) => entry.id === id);
+  const peers = computers.filter((peer) =>
+    matches.get(peer.id)?.some((entry) => entry.id === id)
+  );
+  const peer = peers[0];
+  const conflict =
+    peers.length > 1 || (peer && matches.get(peer.id)!.length > 1);
+  const checks: Check[] = [];
+  if (!node)
+    checks.push({
+      id: 'enrollment:connection',
+      status: 'fail',
+      summary: 'FLEET_CONNECTION_REQUIRED',
+      remedy: {
+        action:
+          'Add this canonical node’s reviewed SSH alias and managed workspace path to the private manager overrides. Do not substitute a discovered peer.',
+      },
+    });
+  else if (conflict)
+    checks.push({
+      id: 'tailscale:identity',
+      status: 'fail',
+      summary: 'TAILSCALE_IDENTITY_CONFLICT',
+      remedy: {
+        action:
+          'Resolve duplicate alias/device mappings before enrollment or apply.',
+      },
+    });
+  else {
+    checks.push(...(await inspectBootstrap(node, run, machine.os)));
+    checks.push(...identityChecks(node, peer));
+  }
+  if (!peer || !peer.online)
+    checks.push({
+      id: 'tailscale:reachability',
+      status: 'fail',
+      summary: peer ? 'TAILSCALE_NODE_OFFLINE' : 'TAILSCALE_NODE_MISSING',
+      remedy: {
+        action:
+          'Restore the intended node’s private connection. A trusted LAN SSH route can be inspected, but discovery never enrolls a replacement.',
+      },
+    });
+  return {
+    node: id,
+    role: 'worker',
+    checks,
+    desiredRevision: revision,
+    appliedRevision: null,
+    durationMs: Date.now() - started,
+    reachability: connectionState(Boolean(conflict), peer),
+  };
+}
+
+function identityChecks(
+  node: FleetNode,
+  peer: TailnetComputer | undefined
+): Check[] {
+  const checks: Check[] = [];
+  if (!node.tailscaleId || (peer && peer.id !== node.tailscaleId))
+    checks.push({
+      id: 'tailscale:enrolled-identity',
+      status: 'fail',
+      summary: node.tailscaleId
+        ? 'TAILSCALE_DEVICE_REPLACED'
+        : 'TAILSCALE_ENROLLMENT_REQUIRED',
+      remedy: {
+        action:
+          'Verify the device identity and SSH host key, then record its stable tailscaleId in the reviewed private connection override. Names alone cannot authorize replacement.',
+      },
+    });
+  return checks;
+}
+
+function mayDeploy(result: ManagerNode, nodes: FleetNode[]): boolean {
+  const allowBootstrap =
+    nodes.find((node) => node.id === result.node)?.bootstrapViaSsh === true;
+  return result.checks.every(
+    (check) =>
+      !blocking(check) ||
+      (allowBootstrap &&
+        [
+          'TAILSCALE_NODE_OFFLINE',
+          'TAILSCALE_NODE_MISSING',
+          'TAILSCALE_ENROLLMENT_REQUIRED',
+        ].includes(check.summary))
+  );
+}
+
+/** Service/account acceptance remains required for readiness, but a missing
+ * external account, dashboard or client pairing must not stop independent OS
+ * configuration on other nodes. Actual deployment failures stop promotion.
+ */
+export function blocksPromotion(check: Check): boolean {
+  if (!blocking(check)) return false;
+  if (check.id === 't3:service' && T3_MAINTENANCE.has(check.summary))
+    return false;
+  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|ubuntu:boot-policy$)/.test(
+    check.id
+  );
+}
+
+/** Native T3 outcomes that need an operator maintenance window (an active
+ * listener or running release must never be killed by routine sync). They are
+ * reported as failures but do not block independent OS policy on other nodes.
+ */
+const T3_MAINTENANCE = new Set([
+  'T3_LISTENER_CONFLICT',
+  'T3_ACTIVE_LISTENER_MIGRATION_REQUIRED',
+  'T3_MAINTENANCE_REQUIRED',
+  'T3_POLICY_RESTART_PENDING',
+  'T3_DESKTOP_MIGRATION_REQUIRED',
+]);

package/src/manager.ts

@@ -0,0 +1,62 @@
+import { compareFingerprints, fingerprint } from './fingerprint';
+import { probe } from './probe';
+import { mapBounded } from './report';
+import { bundleProbe, inspectRemote } from './transport';
+import type { Capability, FleetNode, NodeReport, Runner } from './types';
+
+export interface DoctorOptions {
+  root: string;
+  timeoutMs: number;
+  local: boolean;
+  required: Capability[];
+  source?: string;
+}
+
+export async function doctor(
+  nodes: FleetNode[],
+  options: DoctorOptions,
+  run: Runner
+): Promise<NodeReport[]> {
+  const local = await probe(options.root, options.required, run);
+  const agent = await run(['ssh-add', '-l']);
+  const reports: NodeReport[] = [
+    {
+      node: 'manager',
+      snapshot: local,
+      checks: [
+        ...local.checks,
+        {
+          id: 'ssh:agent',
+          status: agent.ok ? 'pass' : 'warn',
+          summary: agent.ok
+            ? 'SSH agent exposes at least one identity.'
+            : 'No accessible SSH agent identities; explicit IdentityFile keys may still work.',
+          ...(!agent.ok
+            ? {
+                remedy: {
+                  action:
+                    'Unlock/load the intended key in the manager session. Check SSH_AUTH_SOCK and GUI/editor agent visibility; do not copy private keys.',
+                },
+              }
+            : {}),
+        },
+      ],
+    },
+  ];
+  if (options.local || nodes.length === 0) return reports;
+  const source = options.source ?? (await bundleProbe());
+  const reference = await fingerprint(options.root, run);
+  const remote = await mapBounded(nodes, async (node) => {
+    const report = await inspectRemote(
+      node,
+      { action: 'probe', source, timeoutMs: options.timeoutMs },
+      run
+    );
+    if (report.snapshot)
+      report.checks.push(
+        ...compareFingerprints(reference, report.snapshot.fingerprint)
+      );
+    return report;
+  });
+  return [...reports, ...remote];
+}

package/src/monitoring/apply.ts

@@ -0,0 +1,171 @@
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { convergeManagerHub } from './manager-hub';
+import { resolve } from 'node:path';
+import { fleet } from '../fleet-config';
+import { hubUrl, roleOf } from '../topology';
+import { record } from '../config';
+import { isRevision } from '../machine/state';
+import { privateHub } from './beszel';
+import { runLockedCommand as runCommand } from '../runner';
+import type { Check, Runner } from '../types';
+
+export async function applyAgent(
+  value: unknown,
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const request = validateRequest(value);
+  const definition = fleet;
+  await validateSource(request, root, run);
+  const role = roleOf(definition, request.node);
+  if (role === null) throw new Error('BESZEL_NOT_ENROLLED');
+  if (request.mode === 'hub') {
+    // The hub runs only on the manager; workers only ever run the agent.
+    if (role !== 'manager') throw new Error('BESZEL_NOT_HUB');
+    return convergeManagerHub(root, definition, run);
+  }
+  if (
+    typeof request.publicKey !== 'string' ||
+    typeof request.token !== 'string'
+  )
+    throw new Error('BESZEL_CREDENTIALS_REQUIRED');
+  // Agents connect only to the manager's private hub URL.
+  if (privateHub(request.hubUrl) !== privateHub(hubUrl(definition)))
+    throw new Error('BESZEL_HUB_MISMATCH');
+  return installAgent(request, root, run);
+}
+
+/** Workers (Linux) install a hardened systemd unit. The macOS manager's own
+ * agent is a per-user launch agent installed locally by fleet-sync
+ * (installManagerAgent), never through this remote entry point.
+ */
+async function installAgent(
+  request: Request,
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'agent.yml',
+    {
+      cwd: monitoring,
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        beszel_node: request.node,
+        beszel_hub_url: privateHub(request.hubUrl),
+        beszel_public_key: request.publicKey,
+        beszel_agent_token: request.token,
+      }),
+    },
+    {
+      name: 'beszel-agent',
+      sources: [monitoring],
+      verify: () => agentActive(run),
+    }
+  );
+  if (!result.ok)
+    return [
+      {
+        id: 'beszel:installation',
+        status: 'fail',
+        summary: 'BESZEL_INSTALLATION_INCOMPLETE',
+        remedy: {
+          action:
+            'Inspect sudo, signed artifact access, free disk and existing service ownership. Preserve the native agent DATA_DIR and rerun the manager.',
+        },
+      },
+    ];
+  const ready = await agentActive(run);
+  return [
+    {
+      id: 'beszel:service',
+      status: ready ? 'pass' : 'fail',
+      summary: ready
+        ? 'Native Beszel service is enabled and active; hub metrics remain a separate gate.'
+        : 'BESZEL_SERVICE_STOPPED',
+    },
+  ];
+}
+async function agentActive(run: Runner): Promise<boolean> {
+  return (
+    (await run(['systemctl', 'is-enabled', 'beszel-agent.service'])).ok &&
+    (await run(['systemctl', 'is-active', 'beszel-agent.service'])).ok
+  );
+}
+interface Request {
+  node: string;
+  revision: string;
+  hubUrl: string;
+  mode?: unknown;
+  publicKey?: unknown;
+  token?: unknown;
+}
+function validateRequest(value: unknown): Request {
+  if (
+    !record(value) ||
+    typeof value['node'] !== 'string' ||
+    !isRevision(value['revision']) ||
+    typeof value['hubUrl'] !== 'string'
+  )
+    throw new Error('BESZEL_INPUT_INVALID');
+  return value as unknown as Request;
+}
+async function validateSource(
+  request: Request,
+  root: string,
+  run: Runner
+): Promise<void> {
+  for (const [args, expected] of [
+    [['git', 'rev-parse', 'HEAD'], request.revision],
+    [['git', 'rev-parse', '--show-toplevel'], resolve(root)],
+    [['git', 'status', '--porcelain'], ''],
+  ] as const) {
+    const observed = await run([...args], { cwd: root });
+    if (!observed.ok || observed.stdout.trim() !== expected)
+      throw new Error('BESZEL_SOURCE_MISMATCH');
+  }
+}
+async function main(): Promise<Check[]> {
+  let raw = '';
+  for await (const chunk of process.stdin) {
+    raw += String(chunk);
+    if (raw.length > 16_384) throw new Error('BESZEL_INPUT_LIMIT');
+  }
+  return applyAgent(JSON.parse(raw), process.cwd(), runCommand);
+}
+/** Turns a known refusal into structured evidence; anything else is redacted. */
+export function refusal(error: unknown): Check[] {
+  const code = error instanceof Error ? error.message : '';
+  const action = REFUSALS[code];
+  return [
+    action
+      ? {
+          id: 'beszel:operation',
+          status: 'fail',
+          summary: code,
+          remedy: { action },
+        }
+      : {
+          id: 'beszel:operation',
+          status: 'fail',
+          summary: 'BESZEL_OPERATION_INCOMPLETE',
+        },
+  ];
+}
+const REFUSALS: Record<string, string> = {
+  BESZEL_NOT_HUB:
+    'The Beszel hub runs only on the manager. Workers run only the agent; converge the hub from the manager with fleet-sync.',
+  BESZEL_NOT_ENROLLED:
+    'Declare this machine in fleet.json (the manager or a worker) before enrolling monitoring.',
+  BESZEL_SOURCE_MISMATCH:
+    'The checkout is not the clean, expected revision. Rerun fleet-sync so the reviewed source is installed first.',
+  BESZEL_HUB_MISMATCH:
+    "Agents connect only to the manager's private hub URL from fleet.json. Rerun fleet-sync from an up-to-date checkout.",
+  BESZEL_CREDENTIALS_REQUIRED:
+    'Agent enrollment credentials were not delivered. Rerun fleet-sync from the manager.',
+};
+if (import.meta.main) {
+  const checks = await main().catch(refusal);
+  process.stdout.write(JSON.stringify({ version: 1, checks }));
+}