@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/monitoring/hub-backup.ts

@@ -0,0 +1,121 @@
+import { record } from '../config';
+import type { Check } from '../types';
+import {
+  LOCAL_HUB,
+  api,
+  superuserToken,
+  type HubLogin,
+  type Session,
+} from './hub-viewer';
+
+/** A scheduled backup older than this means the schedule is not running. */
+export const BACKUP_MAX_AGE_MS = 48 * 60 * 60 * 1000;
+
+function backupCheck(
+  status: Check['status'],
+  summary: string,
+  action?: string
+): Check {
+  return {
+    id: 'beszel:backup',
+    status,
+    summary,
+    ...(action ? { remedy: { action } } : {}),
+  };
+}
+
+const SETTINGS =
+  'Open the hub as its superuser (/_/ on the manager), Settings > Backups.';
+
+/** Reads, never changes, the hub's own backup settings and its list of
+ * backups through the stock PocketBase API of the pinned release (superuser
+ * only: GET /api/settings and GET /api/backups). Backups are configured in
+ * the hub's UI; this says whether they are scheduled, kept off the manager
+ * and recent. Storage names, keys and endpoints are never reported.
+ */
+export async function verifyHubBackups(
+  login: HubLogin | null,
+  fetcher: typeof fetch = fetch,
+  base: string = LOCAL_HUB,
+  now: number = Date.now()
+): Promise<Check> {
+  if (!login)
+    return backupCheck(
+      'skip',
+      'BESZEL_BACKUP_UNVERIFIED',
+      "The fleet admin login is unavailable, so the hub's backups were not read. Log the manager's Vault session in and rerun fleet-sync."
+    );
+  let settings: unknown;
+  let backups: unknown;
+  try {
+    const anonymous: Session = { fetcher, base, token: '' };
+    const session = {
+      ...anonymous,
+      token: await superuserToken(anonymous, login),
+    };
+    settings = await api(session, 'GET', '/api/settings');
+    backups = await api(session, 'GET', '/api/backups');
+  } catch {
+    return backupCheck(
+      'warn',
+      'BESZEL_BACKUP_UNVERIFIED',
+      `The hub's backup settings could not be read with the fleet admin login. ${SETTINGS} Details are withheld.`
+    );
+  }
+  const policy = backupPolicy(settings);
+  if (!policy || !Array.isArray(backups))
+    return backupCheck(
+      'warn',
+      'BESZEL_BACKUP_UNVERIFIED',
+      `The hub answered in a form this release does not expect. ${SETTINGS}`
+    );
+  if (!policy.cron)
+    return backupCheck(
+      'warn',
+      'BESZEL_BACKUP_UNSCHEDULED',
+      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS} Set a schedule and how many to keep.`
+    );
+  if (!policy.offHost)
+    return backupCheck(
+      'warn',
+      'BESZEL_BACKUP_OFFHOST_REQUIRED',
+      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Store them in existing encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
+    );
+  const newest = newestBackup(backups);
+  if (newest === null || now - newest > BACKUP_MAX_AGE_MS)
+    return backupCheck(
+      'warn',
+      'BESZEL_BACKUP_STALE',
+      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. ${SETTINGS} Check the schedule and the storage's credentials, then create a backup by hand.`
+    );
+  return backupCheck(
+    'pass',
+    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(newest).toISOString()}.`
+  );
+}
+
+function backupPolicy(
+  settings: unknown
+): { cron: string; keep: number; offHost: boolean } | null {
+  if (!record(settings) || !record(settings['backups'])) return null;
+  const backups = settings['backups'];
+  const s3 = record(backups['s3']) ? backups['s3'] : {};
+  return {
+    cron: typeof backups['cron'] === 'string' ? backups['cron'].trim() : '',
+    keep:
+      typeof backups['cronMaxKeep'] === 'number' ? backups['cronMaxKeep'] : 0,
+    offHost: s3['enabled'] === true,
+  };
+}
+
+/** PocketBase lists `modified` as "2006-01-02 15:04:05.000Z". */
+function newestBackup(backups: unknown[]): number | null {
+  const times = backups
+    .map((entry) =>
+      record(entry) && typeof entry['modified'] === 'string'
+        ? Date.parse(entry['modified'].replace(' ', 'T'))
+        : NaN
+    )
+    .filter(Number.isFinite);
+  return times.length ? Math.max(...times) : null;
+}

package/src/monitoring/hub-viewer.ts

@@ -0,0 +1,142 @@
+import { randomBytes } from 'node:crypto';
+import { record } from '../config';
+import type { Check } from '../types';
+import { publicViewerIdentity } from './hub-access';
+
+/** Loopback address of the hub container on the manager. */
+export const LOCAL_HUB = 'http://127.0.0.1:8090';
+
+type Fetcher = typeof fetch;
+export interface HubLogin {
+  identity: string;
+  password: string;
+}
+
+function viewerCheck(
+  status: Check['status'],
+  summary: string,
+  action?: string
+): Check {
+  return {
+    id: 'beszel:public-viewer',
+    status,
+    summary,
+    ...(action ? { remedy: { action } } : {}),
+  };
+}
+
+export interface Session {
+  fetcher: Fetcher;
+  base: string;
+  token: string;
+}
+
+export async function api(
+  session: Session,
+  method: string,
+  path: string,
+  body?: object
+): Promise<unknown> {
+  const response = await session.fetcher(`${session.base}${path}`, {
+    method,
+    headers: {
+      Authorization: session.token,
+      'Content-Type': 'application/json',
+    },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+    redirect: 'error',
+    signal: AbortSignal.timeout(10_000),
+  });
+  // Bodies can echo emails or tokens: only the status is ever surfaced.
+  if (!response.ok) throw new Error(`HTTP_${response.status}`);
+  return response.json() as Promise<unknown>;
+}
+
+export async function superuserToken(
+  session: Session,
+  login: HubLogin
+): Promise<string> {
+  const auth = await api(
+    session,
+    'POST',
+    '/api/collections/_superusers/auth-with-password',
+    login
+  );
+  if (!record(auth) || typeof auth['token'] !== 'string')
+    throw new Error('BESZEL_PROTOCOL');
+  return auth['token'];
+}
+
+async function findViewer(
+  session: Session
+): Promise<{ id: string; role: unknown } | null> {
+  const filter = encodeURIComponent(`email="${publicViewerIdentity()}"`);
+  const list = await api(
+    session,
+    'GET',
+    `/api/collections/users/records?filter=${filter}&perPage=2`
+  );
+  const items =
+    record(list) && Array.isArray(list['items']) ? list['items'] : [];
+  const first: unknown = items[0];
+  if (!record(first) || typeof first['id'] !== 'string') return null;
+  return { id: first['id'], role: first['role'] };
+}
+
+async function createViewer(session: Session): Promise<void> {
+  const password = randomBytes(24).toString('base64url');
+  await api(session, 'POST', '/api/collections/users/records', {
+    email: publicViewerIdentity(),
+    username: 'public-viewer',
+    password,
+    passwordConfirm: password,
+    verified: true,
+    role: 'readonly',
+  });
+}
+
+/** Idempotently makes sure the read-only viewer exists, then re-reads it to
+ * prove its role. AUTO_LOGIN signs every visitor in as this account, so an
+ * existing account with any other role is demoted rather than trusted. The
+ * password is random, never stored or printed, and is not needed to sign in.
+ */
+export async function ensurePublicViewer(
+  login: HubLogin | null,
+  fetcher: Fetcher = fetch,
+  base: string = LOCAL_HUB
+): Promise<Check> {
+  if (!login)
+    return viewerCheck(
+      'warn',
+      'BESZEL_VIEWER_UNPROVISIONED',
+      "The fleet admin login is unavailable. Log the manager's Vault session in with read access to BESZEL_FLEET_AUTH and rerun fleet-sync; until then the public dashboard shows the login page."
+    );
+  try {
+    const anonymous: Session = { fetcher, base, token: '' };
+    const session = {
+      ...anonymous,
+      token: await superuserToken(anonymous, login),
+    };
+    const existing = await findViewer(session);
+    if (!existing) await createViewer(session);
+    else if (existing.role !== 'readonly')
+      await api(
+        session,
+        'PATCH',
+        `/api/collections/users/records/${existing.id}`,
+        { role: 'readonly' }
+      );
+    const verified = await findViewer(session);
+    if (verified?.role !== 'readonly') throw new Error('BESZEL_VIEWER_ROLE');
+    return viewerCheck(
+      'pass',
+      `Visitors are signed in automatically as ${publicViewerIdentity()} (read-only).`
+    );
+  } catch {
+    return viewerCheck(
+      'warn',
+      'BESZEL_VIEWER_UNVERIFIED',
+      `Could not create or verify the read-only ${publicViewerIdentity()} user with the fleet admin login. Create it in the hub (Settings > Users, role Read-only); never make it an admin. Details are withheld.`
+    );
+  }
+}

package/src/monitoring/hub.ts

@@ -0,0 +1,93 @@
+import { record } from '../config';
+import type { Check, Runner } from '../types';
+import { publicHubUrl } from './beszel';
+import { secretVia } from '../fleet-config';
+
+/** Shared Beszel hub helpers. The hub runs only on the manager; see
+ * manager-hub.ts. Workers never host it.
+ */
+export function publicEndpointCheck(connected: boolean): Check {
+  return connected
+    ? {
+        id: 'beszel:public-endpoint',
+        status: 'pass',
+        summary: `Cloudflare connector for ${publicHubUrl()} was started from the Vault credential.`,
+      }
+    : {
+        id: 'beszel:public-endpoint',
+        status: 'warn',
+        summary: 'BESZEL_TUNNEL_CREDENTIAL_UNAVAILABLE',
+        remedy: {
+          action: `Create the Cloudflare tunnel and store its token in the fleet's secrets, then log the manager's Vault session in with read access to secret/beszel/tunnel and rerun fleet-sync. The private dashboard is unaffected; ${publicHubUrl()} stays offline until then.`,
+        },
+      };
+}
+
+/** The dashboard connector token from this fleet's secrets provider. Null
+ * when it is unavailable. Never printed, logged or placed in argv.
+ */
+export function tunnelToken(run: Runner): Promise<string | null> {
+  return secretVia(run)('monitoring-tunnel-token');
+}
+export function failure(code: string, action: string): Check[] {
+  return [
+    { id: 'beszel:hub', status: 'fail', summary: code, remedy: { action } },
+  ];
+}
+
+export function inspectRoute(
+  output: string,
+  hub: string,
+  target = { port: '8443', proxy: 'http://127.0.0.1:8090' }
+): boolean | Check[] {
+  let configured = false;
+  try {
+    const value: unknown = JSON.parse(output);
+    if (!record(value)) throw new Error();
+    if (publicOrConflictingTcp(value, hub, target.port))
+      return failure(
+        'BESZEL_PRIVATE_ROUTE_CONFLICT',
+        'Existing Funnel/public or TCP routing conflicts with private Beszel access. Review the source-owned route; do not reset Serve.'
+      );
+    const web = value['Web'];
+    if (record(web)) {
+      const entry = web[new URL(hub).hostname + ':' + target.port];
+      if (entry !== undefined) {
+        if (!expectedHandler(entry, target.proxy))
+          return failure(
+            'BESZEL_PRIVATE_ROUTE_CONFLICT',
+            'Port 8443 is owned by a different Tailscale Serve handler. Review a source-controlled route migration; do not reset Serve.'
+          );
+        configured = true;
+      }
+    }
+  } catch {
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'Review Tailscale Serve status without exposing raw configuration.'
+    );
+  }
+  return configured;
+}
+
+function publicOrConflictingTcp(
+  value: Record<string, unknown>,
+  hub: string,
+  port: string
+): boolean {
+  const funnel = value['AllowFunnel'];
+  if (record(funnel) && funnel[new URL(hub).hostname + ':' + port] === true)
+    return true;
+  const tcp = value['TCP'];
+  if (!record(tcp) || tcp[port] === undefined) return false;
+  return !record(tcp[port]) || tcp[port]['HTTPS'] !== true;
+}
+
+function expectedHandler(entry: unknown, proxy: string): boolean {
+  return (
+    record(entry) &&
+    record(entry['Handlers']) &&
+    record(entry['Handlers']['/']) &&
+    entry['Handlers']['/']['Proxy'] === proxy
+  );
+}

package/src/monitoring/manager-agent.ts

@@ -0,0 +1,73 @@
+import { userInfo } from 'node:os';
+import { resolve } from 'node:path';
+import macos from '../../ansible/macos/policy.json';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { privateHub } from './beszel';
+import type { AgentEnrollment } from './enroll';
+import type { Check, Runner } from '../types';
+
+/** Installs the manager's own Beszel agent locally (never over SSH) as a
+ * per-user launchd agent, so the manager is always a system on its own hub.
+ * No sudo: fleet-sync on the manager stays unattended.
+ */
+export async function installManagerAgent(
+  root: string,
+  { node, enrollment }: { node: string; enrollment: AgentEnrollment },
+  run: Runner,
+  { uid = userInfo().uid, directory }: { uid?: number; directory?: string } = {}
+): Promise<Check[]> {
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'agent-darwin.yml',
+    {
+      cwd: monitoring,
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        beszel_node: node,
+        beszel_hub_url: privateHub(enrollment.hubUrl),
+        beszel_public_key: enrollment.publicKey,
+        beszel_agent_token: enrollment.token,
+      }),
+    },
+    {
+      name: 'beszel-manager-agent',
+      sources: [monitoring, resolve(root, 'ansible/macos')],
+      verify: async () => agentRunning(run, uid),
+      ...(directory ? { directory } : {}),
+    }
+  );
+  if (!result.ok)
+    return [
+      {
+        id: 'beszel:installation',
+        status: 'fail',
+        summary: 'BESZEL_INSTALLATION_INCOMPLETE',
+        remedy: {
+          action:
+            'Inspect signed artifact access, free disk and ~/Library/LaunchAgents on the manager. A leftover root daemon must be removed once (sudo launchctl bootout system/' +
+            macos.beszelDaemon +
+            '). Agent data under ~/Library/Application Support/personal-beszel-agent is preserved.',
+        },
+      },
+    ];
+  const running = await agentRunning(run, uid);
+  return [
+    {
+      id: 'beszel:service',
+      status: running ? 'pass' : 'fail',
+      summary: running
+        ? 'The manager runs its own Beszel agent as an always-on launch agent; hub metrics remain a separate gate.'
+        : 'BESZEL_SERVICE_STOPPED',
+    },
+  ];
+}
+
+async function agentRunning(run: Runner, uid: number): Promise<boolean> {
+  const state = await run([
+    '/bin/launchctl',
+    'print',
+    `gui/${uid}/${macos.beszelDaemon}`,
+  ]);
+  return state.ok && /state = running/.test(state.stdout);
+}

package/src/monitoring/manager-hub.ts

@@ -0,0 +1,248 @@
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { delimiter, join, resolve } from 'node:path';
+import { runAnsible } from '../platform/ansible';
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { failure, inspectRoute, publicEndpointCheck, tunnelToken } from './hub';
+import { publicViewerIdentity, fleetAccount } from './hub-access';
+import { ensurePublicViewer, type HubLogin } from './hub-viewer';
+import { verifyHubBackups } from './hub-backup';
+import type { FleetDefinition } from '../definition';
+import { hostId, hubUrl } from '../topology';
+import { publicHubUrl, privateHub } from './beszel';
+import type { Check, Runner } from '../types';
+
+/** Bundled CLI of the Tailscale macOS app when no `tailscale` is on PATH. */
+export const TAILSCALE_APP_CLI =
+  '/Applications/Tailscale.app/Contents/MacOS/Tailscale';
+const LOCAL_HUB = 'http://127.0.0.1:8090';
+/** The one place the route is declared; `hub-route-darwin.yml` receives it. */
+export const HUB_ROUTE = { port: '8443', proxy: LOCAL_HUB };
+
+/** Per-user hub paths on the manager. Data stays in the human's home (and its
+ * normal backups); it is never copied to a worker.
+ */
+export function managerHubPaths(home = homedir()): {
+  data: string;
+  tunnel: string;
+} {
+  const support = join(home, 'Library', 'Application Support');
+  return {
+    data: join(support, 'personal-beszel'),
+    tunnel: join(support, 'personal-beszel-tunnel'),
+  };
+}
+
+/** `tailscale` on PATH (Homebrew or the app's installed CLI), else the app
+ * bundle's executable; null when neither exists.
+ */
+export function tailscaleCommand(
+  path = process.env['PATH'] ?? '',
+  exists: (candidate: string) => boolean = existsSync
+): string | null {
+  for (const directory of path.split(delimiter).filter(Boolean)) {
+    const candidate = join(directory, 'tailscale');
+    if (exists(candidate)) return candidate;
+  }
+  return exists(TAILSCALE_APP_CLI) ? TAILSCALE_APP_CLI : null;
+}
+
+export interface ManagerHubOptions {
+  home?: string;
+  tailscale?: string | null;
+  /** Fleet login for a fresh hub; defaults to the manager's Vault session. */
+  account?: () => Promise<HubLogin | null>;
+  /** Receipt directory for skipping an unchanged hub apply. */
+  receipts?: string;
+  /** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
+  viewer?: (login: HubLogin | null) => Promise<Check>;
+  /** Reads the hub's backup settings; defaults to the loopback hub API. */
+  backups?: (login: HubLogin | null) => Promise<Check>;
+}
+
+/** Converges the singleton Beszel hub locally on the manager: pinned Compose
+ * under Docker Desktop, the connector token from the manager's own Vault
+ * session (private variables file, never argv or output), a private Tailscale
+ * Serve route (its own playbook), then independent health checks locally and
+ * over the tailnet.
+ */
+export async function convergeManagerHub(
+  root: string,
+  definition: FleetDefinition,
+  run: Runner,
+  options: ManagerHubOptions = {}
+): Promise<Check[]> {
+  const hub = privateHub(hubUrl(definition));
+  const refused = await preconditions(definition, run);
+  if (refused) return refused;
+  const paths = managerHubPaths(options.home);
+  const tunnel = await tunnelToken(run);
+  const account = await (options.account ?? fleetAccount)();
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'hub-darwin.yml',
+    {
+      cwd: monitoring,
+      input: JSON.stringify(hubVariables(hub, paths, tunnel, account)),
+      timeoutMs: 300_000,
+    },
+    {
+      name: 'beszel-hub',
+      sources: [monitoring],
+      // Health is verified independently below on every run.
+      verify: async () =>
+        (
+          await run([
+            'curl',
+            '-fsS',
+            '--max-time',
+            '5',
+            '-o',
+            '/dev/null',
+            `${LOCAL_HUB}/api/health`,
+          ])
+        ).ok,
+      ...(options.receipts ? { directory: options.receipts } : {}),
+    }
+  );
+  if (!result.ok)
+    return failure(
+      'BESZEL_HUB_INSTALLATION_INCOMPLETE',
+      'Inspect Docker Desktop, the Compose image, file sharing for ~/Library/Application Support and disk space. Existing data is never deleted.'
+    );
+  const route = await servePrivateRoute(
+    { hub, monitoring },
+    run,
+    options.tailscale === undefined ? tailscaleCommand() : options.tailscale
+  );
+  if (route) return route;
+  const health = await verifyHealth(hub, run);
+  if (health) return health;
+  return [
+    {
+      id: 'beszel:hub',
+      status: 'pass',
+      summary: `Pinned hub on the manager is healthy locally and at ${hub}; authentication and agent metrics are separate checks.`,
+    },
+    publicEndpointCheck(tunnel !== null),
+    await ensureViewer(options, account),
+    await (options.backups ?? verifyHubBackups)(account),
+  ];
+}
+
+function hubVariables(
+  hub: string,
+  paths: { data: string; tunnel: string },
+  tunnel: string | null,
+  account: HubLogin | null
+): Record<string, string> {
+  return {
+    beszel_hub_url: hub,
+    beszel_public_url: publicHubUrl(),
+    beszel_auto_login: publicViewerIdentity(),
+    beszel_data_dir: paths.data,
+    beszel_tunnel_dir: paths.tunnel,
+    ...(tunnel ? { beszel_tunnel_token: tunnel } : {}),
+    ...(account
+      ? {
+          beszel_fleet_identity: account.identity,
+          beszel_fleet_password: account.password,
+        }
+      : {}),
+  };
+}
+
+function ensureViewer(
+  options: ManagerHubOptions,
+  account: HubLogin | null
+): Promise<Check> {
+  return (options.viewer ?? ensurePublicViewer)(account);
+}
+
+/** Runs only on the manager, and only with Docker Desktop's engine up. */
+async function preconditions(
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Check[] | null> {
+  const local = await run(['/usr/sbin/scutil', '--get', 'LocalHostName']);
+  if (!local.ok || hostId(local.stdout.trim()) !== definition.manager.id)
+    return failure(
+      'BESZEL_NOT_MANAGER',
+      `The Beszel hub runs only on the manager (${definition.manager.id}). Run fleet-sync from the manager; workers only run the agent.`
+    );
+  const docker = await run(
+    ['docker', 'info', '--format', '{{.ServerVersion}}'],
+    { timeoutMs: 20_000 }
+  );
+  if (!docker.ok || docker.stdout.trim() === '')
+    return failure(
+      'BESZEL_DOCKER_DESKTOP_REQUIRED',
+      'Start Docker Desktop on the manager (and enable "Start Docker Desktop when you sign in"), then rerun fleet-sync. Existing hub data is never deleted.'
+    );
+  return null;
+}
+
+/** `hub-route-darwin.yml` publishes the route when its port has no owner; this
+ * refuses a conflicting owner beforehand and measures the result.
+ */
+async function servePrivateRoute(
+  { hub, monitoring }: { hub: string; monitoring: string },
+  run: Runner,
+  cli: string | null
+): Promise<Check[] | null> {
+  if (!cli)
+    return failure(
+      'BESZEL_TAILSCALE_REQUIRED',
+      `Install and sign in to Tailscale on the manager (Homebrew CLI or ${TAILSCALE_APP_CLI}).`
+    );
+  const status = await run([cli, 'serve', 'status', '--json']);
+  if (!status.ok)
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'Inspect Tailscale Serve permission and existing private routes before retry.'
+    );
+  const route = inspectRoute(status.stdout, hub, HUB_ROUTE);
+  if (Array.isArray(route)) return route;
+  if (!route) {
+    const published = await runAnsible(run, 'hub-route-darwin.yml', {
+      cwd: monitoring,
+      input: JSON.stringify({
+        beszel_tailscale_cli: cli,
+        beszel_route_port: HUB_ROUTE.port,
+        beszel_route_target: HUB_ROUTE.proxy,
+      }),
+      timeoutMs: 120_000,
+    });
+    if (!published.ok)
+      return failure(
+        'BESZEL_PRIVATE_ROUTE_REQUIRED',
+        'Allow Tailscale Serve (HTTPS certificates) for this tailnet and approve it in the Tailscale app if asked; no tailnet policy or public ingress is changed.'
+      );
+  }
+  const verified = await run([cli, 'serve', 'status', '--json']);
+  if (!verified.ok || inspectRoute(verified.stdout, hub, HUB_ROUTE) !== true)
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'The requested route was not independently observed after application. Inspect Serve permissions and configuration; installer success is insufficient.'
+    );
+  return null;
+}
+
+async function verifyHealth(hub: string, run: Runner): Promise<Check[] | null> {
+  const probe = (url: string) =>
+    run(['curl', '-fsS', '--max-time', '10', '-o', '/dev/null', url], {
+      timeoutMs: 15_000,
+    });
+  if (!(await probe(`${LOCAL_HUB}/api/health`)).ok)
+    return failure(
+      'BESZEL_HUB_UNHEALTHY',
+      'The hub container did not answer /api/health locally. Inspect Docker Desktop and the personal-beszel Compose project; preserve its data.'
+    );
+  if (!(await probe(`${hub}/api/health`)).ok)
+    return failure(
+      'BESZEL_HUB_UNREACHABLE',
+      `The hub is healthy locally but not at ${hub}. Inspect Tailscale on the manager and its Serve route.`
+    );
+  return null;
+}