@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/monitoring/manager-agent.ts

@@ -0,0 +1,73 @@
+import { userInfo } from 'node:os';
+import { resolve } from 'node:path';
+import macos from '../../ansible/macos/policy.json';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { privateHub } from './beszel';
+import type { AgentEnrollment } from './enroll';
+import type { Check, Runner } from '../types';
+
+/** Installs the manager's own Beszel agent locally (never over SSH) as a
+ * per-user launchd agent, so the manager is always a system on its own hub.
+ * No sudo: fleet-sync on the manager stays unattended.
+ */
+export async function installManagerAgent(
+  root: string,
+  { node, enrollment }: { node: string; enrollment: AgentEnrollment },
+  run: Runner,
+  { uid = userInfo().uid, directory }: { uid?: number; directory?: string } = {}
+): Promise<Check[]> {
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'agent-darwin.yml',
+    {
+      cwd: monitoring,
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        beszel_node: node,
+        beszel_hub_url: privateHub(enrollment.hubUrl),
+        beszel_public_key: enrollment.publicKey,
+        beszel_agent_token: enrollment.token,
+      }),
+    },
+    {
+      name: 'beszel-manager-agent',
+      sources: [monitoring, resolve(root, 'ansible/macos')],
+      verify: async () => agentRunning(run, uid),
+      ...(directory ? { directory } : {}),
+    }
+  );
+  if (!result.ok)
+    return [
+      {
+        id: 'beszel:installation',
+        status: 'fail',
+        summary: 'BESZEL_INSTALLATION_INCOMPLETE',
+        remedy: {
+          action:
+            'Inspect signed artifact access, free disk and ~/Library/LaunchAgents on the manager. A leftover root daemon must be removed once (sudo launchctl bootout system/' +
+            macos.beszelDaemon +
+            '). Agent data under ~/Library/Application Support/personal-beszel-agent is preserved.',
+        },
+      },
+    ];
+  const running = await agentRunning(run, uid);
+  return [
+    {
+      id: 'beszel:service',
+      status: running ? 'pass' : 'fail',
+      summary: running
+        ? 'The manager runs its own Beszel agent as an always-on launch agent; hub metrics remain a separate gate.'
+        : 'BESZEL_SERVICE_STOPPED',
+    },
+  ];
+}
+
+async function agentRunning(run: Runner, uid: number): Promise<boolean> {
+  const state = await run([
+    '/bin/launchctl',
+    'print',
+    `gui/${uid}/${macos.beszelDaemon}`,
+  ]);
+  return state.ok && /state = running/.test(state.stdout);
+}

package/src/monitoring/manager-hub.ts

@@ -0,0 +1,232 @@
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { delimiter, join, resolve } from 'node:path';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { failure, inspectRoute, publicEndpointCheck, tunnelToken } from './hub';
+import { BESZEL_PUBLIC_VIEWER_IDENTITY, fleetAccount } from './hub-access';
+import { ensurePublicViewer, type HubLogin } from './hub-viewer';
+import type { FleetDefinition } from '../definition';
+import { hostId, hubUrl } from '../topology';
+import { PUBLIC_HUB_URL, privateHub } from './beszel';
+import type { Check, Runner } from '../types';
+
+/** Bundled CLI of the Tailscale macOS app when no `tailscale` is on PATH. */
+export const TAILSCALE_APP_CLI =
+  '/Applications/Tailscale.app/Contents/MacOS/Tailscale';
+const LOCAL_HUB = 'http://127.0.0.1:8090';
+
+/** Per-user hub paths on the manager. Data stays in the human's home (and its
+ * normal backups); it is never copied to a worker.
+ */
+export function managerHubPaths(home = homedir()): {
+  data: string;
+  tunnel: string;
+} {
+  const support = join(home, 'Library', 'Application Support');
+  return {
+    data: join(support, 'personal-beszel'),
+    tunnel: join(support, 'personal-beszel-tunnel'),
+  };
+}
+
+/** `tailscale` on PATH (Homebrew or the app's installed CLI), else the app
+ * bundle's executable; null when neither exists.
+ */
+export function tailscaleCommand(
+  path = process.env['PATH'] ?? '',
+  exists: (candidate: string) => boolean = existsSync
+): string | null {
+  for (const directory of path.split(delimiter).filter(Boolean)) {
+    const candidate = join(directory, 'tailscale');
+    if (exists(candidate)) return candidate;
+  }
+  return exists(TAILSCALE_APP_CLI) ? TAILSCALE_APP_CLI : null;
+}
+
+export interface ManagerHubOptions {
+  home?: string;
+  tailscale?: string | null;
+  /** Fleet login for a fresh hub; defaults to the manager's Vault session. */
+  account?: () => Promise<HubLogin | null>;
+  /** Receipt directory for skipping an unchanged hub apply. */
+  receipts?: string;
+  /** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
+  viewer?: (login: HubLogin | null) => Promise<Check>;
+}
+
+/** Converges the singleton Beszel hub locally on the manager: pinned Compose
+ * under Docker Desktop, the connector token from the manager's own Vault
+ * session (private variables file, never argv or output), a private Tailscale
+ * Serve route, then independent health checks locally and over the tailnet.
+ */
+export async function convergeManagerHub(
+  root: string,
+  definition: FleetDefinition,
+  run: Runner,
+  options: ManagerHubOptions = {}
+): Promise<Check[]> {
+  const hub = privateHub(hubUrl(definition));
+  const refused = await preconditions(definition, run);
+  if (refused) return refused;
+  const paths = managerHubPaths(options.home);
+  const tunnel = await tunnelToken(run);
+  const account = await (options.account ?? fleetAccount)();
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'hub-darwin.yml',
+    {
+      cwd: monitoring,
+      input: JSON.stringify(hubVariables(hub, paths, tunnel, account)),
+      timeoutMs: 300_000,
+    },
+    {
+      name: 'beszel-hub',
+      sources: [monitoring],
+      // Health is verified independently below on every run.
+      verify: async () =>
+        (
+          await run([
+            'curl',
+            '-fsS',
+            '--max-time',
+            '5',
+            '-o',
+            '/dev/null',
+            `${LOCAL_HUB}/api/health`,
+          ])
+        ).ok,
+      ...(options.receipts ? { directory: options.receipts } : {}),
+    }
+  );
+  if (!result.ok)
+    return failure(
+      'BESZEL_HUB_INSTALLATION_INCOMPLETE',
+      'Inspect Docker Desktop, the Compose image, file sharing for ~/Library/Application Support and disk space. Existing data is never deleted.'
+    );
+  const route = await servePrivateRoute(
+    hub,
+    run,
+    options.tailscale === undefined ? tailscaleCommand() : options.tailscale
+  );
+  if (route) return route;
+  const health = await verifyHealth(hub, run);
+  if (health) return health;
+  return [
+    {
+      id: 'beszel:hub',
+      status: 'pass',
+      summary: `Pinned hub on the manager is healthy locally and at ${hub}; authentication and agent metrics are separate checks.`,
+    },
+    publicEndpointCheck(tunnel !== null),
+    await ensureViewer(options, account),
+  ];
+}
+
+function hubVariables(
+  hub: string,
+  paths: { data: string; tunnel: string },
+  tunnel: string | null,
+  account: HubLogin | null
+): Record<string, string> {
+  return {
+    beszel_hub_url: hub,
+    beszel_public_url: PUBLIC_HUB_URL,
+    beszel_auto_login: BESZEL_PUBLIC_VIEWER_IDENTITY,
+    beszel_data_dir: paths.data,
+    beszel_tunnel_dir: paths.tunnel,
+    ...(tunnel ? { beszel_tunnel_token: tunnel } : {}),
+    ...(account
+      ? {
+          beszel_fleet_identity: account.identity,
+          beszel_fleet_password: account.password,
+        }
+      : {}),
+  };
+}
+
+function ensureViewer(
+  options: ManagerHubOptions,
+  account: HubLogin | null
+): Promise<Check> {
+  return (options.viewer ?? ensurePublicViewer)(account);
+}
+
+/** Runs only on the manager, and only with Docker Desktop's engine up. */
+async function preconditions(
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Check[] | null> {
+  const local = await run(['/usr/sbin/scutil', '--get', 'LocalHostName']);
+  if (!local.ok || hostId(local.stdout.trim()) !== definition.manager.id)
+    return failure(
+      'BESZEL_NOT_MANAGER',
+      `The Beszel hub runs only on the manager (${definition.manager.id}). Run fleet-sync from the manager; workers only run the agent.`
+    );
+  const docker = await run(
+    ['docker', 'info', '--format', '{{.ServerVersion}}'],
+    { timeoutMs: 20_000 }
+  );
+  if (!docker.ok || docker.stdout.trim() === '')
+    return failure(
+      'BESZEL_DOCKER_DESKTOP_REQUIRED',
+      'Start Docker Desktop on the manager (and enable "Start Docker Desktop when you sign in"), then rerun fleet-sync. Existing hub data is never deleted.'
+    );
+  return null;
+}
+
+async function servePrivateRoute(
+  hub: string,
+  run: Runner,
+  cli: string | null
+): Promise<Check[] | null> {
+  if (!cli)
+    return failure(
+      'BESZEL_TAILSCALE_REQUIRED',
+      `Install and sign in to Tailscale on the manager (Homebrew CLI or ${TAILSCALE_APP_CLI}).`
+    );
+  const status = await run([cli, 'serve', 'status', '--json']);
+  if (!status.ok)
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'Inspect Tailscale Serve permission and existing private routes before retry.'
+    );
+  const route = inspectRoute(status.stdout, hub);
+  if (Array.isArray(route)) return route;
+  if (!route) {
+    const serve = [cli, 'serve', '--bg', '--https=8443', LOCAL_HUB] as const;
+    const applied =
+      (await run([...serve], { timeoutMs: 20_000 })).ok ||
+      (await run(['sudo', '-n', ...serve], { timeoutMs: 20_000 })).ok;
+    if (!applied)
+      return failure(
+        'BESZEL_PRIVATE_ROUTE_REQUIRED',
+        'Allow Tailscale Serve (HTTPS certificates) for this tailnet and approve it in the Tailscale app if asked; no tailnet policy or public ingress is changed.'
+      );
+  }
+  const verified = await run([cli, 'serve', 'status', '--json']);
+  if (!verified.ok || inspectRoute(verified.stdout, hub) !== true)
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'The requested route was not independently observed after application. Inspect Serve permissions and configuration; installer success is insufficient.'
+    );
+  return null;
+}
+
+async function verifyHealth(hub: string, run: Runner): Promise<Check[] | null> {
+  const probe = (url: string) =>
+    run(['curl', '-fsS', '--max-time', '10', '-o', '/dev/null', url], {
+      timeoutMs: 15_000,
+    });
+  if (!(await probe(`${LOCAL_HUB}/api/health`)).ok)
+    return failure(
+      'BESZEL_HUB_UNHEALTHY',
+      'The hub container did not answer /api/health locally. Inspect Docker Desktop and the personal-beszel Compose project; preserve its data.'
+    );
+  if (!(await probe(`${hub}/api/health`)).ok)
+    return failure(
+      'BESZEL_HUB_UNREACHABLE',
+      `The hub is healthy locally but not at ${hub}. Inspect Tailscale on the manager and its Serve route.`
+    );
+  return null;
+}

package/src/monitoring/remote.ts

@@ -0,0 +1,65 @@
+import type { Check, FleetNode, Runner } from '../types';
+import { shellQuote, sshArgs, sshFailure } from '../ssh';
+import { validCheck, cleanCheck } from '../protocol';
+import { record } from '../config';
+import { isRevision } from '../machine/state';
+import type { AgentEnrollment } from './enroll';
+import { remoteLockPrelude } from '../remote-lock';
+
+/** Installs the Beszel agent on a worker. Only node-specific agent credentials
+ * cross SSH; hub account credentials stay on the manager. Workers never host
+ * the hub (it runs only on the manager), so there is no remote hub mode.
+ * The installed source and remote operation lock are reused.
+ */
+export async function installRemoteAgent(
+  node: FleetNode,
+  revision: string,
+  enrollment: AgentEnrollment,
+  run: Runner
+): Promise<Check[]> {
+  if (!isRevision(revision)) throw new Error('BESZEL_REVISION');
+  const root = `"$HOME/.local/share/personal/fleet/revisions/${revision}"`;
+  const locked = shellQuote(
+    JSON.stringify({
+      version: 1,
+      checks: [
+        {
+          id: 'beszel:lock',
+          status: 'fail',
+          summary: 'FLEET_REMOTE_LOCKED',
+          remedy: {
+            action:
+              'Another remote operation owns the lock. Inspect its state and retry after it finishes; never delete the lock or start a competing installer.',
+          },
+        },
+      ],
+    })
+  );
+  const command = `cd ${root} && { ${remoteLockPrelude()} || { printf '%s\\n' ${locked}; exit 0; }; } && trap '' HUP && exec "$HOME/.local/bin/mise" exec --locked bun -- bun ${shellQuote('src/monitoring/apply.ts')}`;
+  const result = await run([...sshArgs(node), command], {
+    input: JSON.stringify({ ...enrollment, node: node.id, revision }),
+    timeoutMs: 360_000,
+  });
+  if (!result.ok) return [sshFailure(result)];
+  try {
+    const value: unknown = JSON.parse(result.stdout);
+    if (
+      record(value) &&
+      value['version'] === 1 &&
+      Array.isArray(value['checks']) &&
+      value['checks'].length > 0 &&
+      value['checks'].length <= 20 &&
+      value['checks'].every(validCheck)
+    )
+      return value['checks'].map(cleanCheck);
+  } catch {
+    /* Never expose native output or credentials. */
+  }
+  return [
+    {
+      id: 'beszel:protocol',
+      status: 'fail',
+      summary: 'BESZEL_REMOTE_EVIDENCE_INVALID',
+    },
+  ];
+}

package/src/network-definition.ts

@@ -0,0 +1,83 @@
+import { type FleetDefinition, type MachineDefinition } from './definition';
+import { record } from './config';
+import { FleetUsageError } from './errors';
+
+export interface NetworkHost {
+  alias: string;
+  tailscaleName: string;
+  sshPort: number;
+  user: string;
+  identityFile: string;
+  os: 'linux' | 'macOS' | 'windows';
+  t3Port: number;
+}
+export function networkProjection(
+  definition: FleetDefinition,
+  identities: Record<string, string>
+): NetworkHost[] {
+  return definition.workers.flatMap((node) =>
+    node.network
+      ? [
+          {
+            alias: node.ssh,
+            ...node.network,
+            os: node.os,
+            identityFile: identities[node.id] ?? '',
+          },
+        ]
+      : []
+  );
+}
+function compatible(
+  node: MachineDefinition | undefined,
+  host: Record<string, unknown>
+): node is MachineDefinition {
+  return Boolean(
+    node?.network &&
+    Object.entries(node.network).every(([key, value]) => host[key] === value) &&
+    node.os === host['os']
+  );
+}
+/** Immutable migration input supplies old manager paths only, never desired hosts. */
+export function legacyIdentities(
+  definition: FleetDefinition,
+  raw: unknown
+): Record<string, string> {
+  if (!record(raw) || !Array.isArray(raw['hosts']))
+    throw new FleetUsageError('Invalid legacy Workstation inventory.');
+  const result: Record<string, string> = {};
+  for (const host of raw['hosts']) {
+    if (!record(host))
+      throw new FleetUsageError('Invalid legacy network host.');
+    const node = definition.workers.find((node) => node.ssh === host['alias']);
+    // A retired host has no manager identity to migrate into this fleet.
+    if (!node) continue;
+    if (!compatible(node, host))
+      throw new FleetUsageError(
+        'Legacy Workstation and canonical Fleet definitions conflict. Review IDs, aliases and network settings before migration.'
+      );
+    const identity = host['identityFile'];
+    if (
+      typeof identity !== 'string' ||
+      !/^(~\/|\/)[^\x00-\x1f\x7f]+$/.test(identity)
+    )
+      throw new FleetUsageError('Legacy identity path is invalid.');
+    if (result[node.id] !== undefined)
+      throw new FleetUsageError('Duplicate legacy machine assignment.');
+    result[node.id] = identity;
+  }
+  return result;
+}
+/** `legacy` is the consumer's earlier inventory, when it has one: it only
+ * supplies identity paths the fleet configuration does not hold.
+ */
+export function loadNetworkDefinition(
+  definition: FleetDefinition,
+  legacy: unknown,
+  overrides?: Record<string, string>
+): NetworkHost[] {
+  return networkProjection(
+    definition,
+    overrides ?? legacyIdentities(definition, legacy)
+  );
+}