@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/monitoring/apply.ts

@@ -0,0 +1,183 @@
+import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
+import { convergeManagerHub } from './manager-hub';
+import { resolve } from 'node:path';
+import { fleet } from '../fleet-config';
+import { hubUrl, roleOf } from '../topology';
+import { record } from '../config';
+import { isRevision } from '../machine/state';
+import { privateHub } from './beszel';
+import { agentConnection, agentDisk } from './agent-evidence';
+import { runLockedCommand as runCommand } from '../runner';
+import type { Check, Runner } from '../types';
+
+export async function applyAgent(
+  value: unknown,
+  root: string,
+  run: Runner,
+  options: { receipts?: string } = {}
+): Promise<Check[]> {
+  const request = validateRequest(value);
+  const definition = fleet;
+  await validateSource(request, root, run);
+  const role = roleOf(definition, request.node);
+  if (role === null) throw new Error('BESZEL_NOT_ENROLLED');
+  if (request.mode === 'hub') {
+    // The hub runs only on the manager; workers only ever run the agent.
+    if (role !== 'manager') throw new Error('BESZEL_NOT_HUB');
+    return convergeManagerHub(root, definition, run);
+  }
+  if (
+    typeof request.publicKey !== 'string' ||
+    typeof request.token !== 'string'
+  )
+    throw new Error('BESZEL_CREDENTIALS_REQUIRED');
+  // Agents connect only to the manager's private hub URL.
+  if (privateHub(request.hubUrl) !== privateHub(hubUrl(definition)))
+    throw new Error('BESZEL_HUB_MISMATCH');
+  return installAgent(request, root, run, options.receipts);
+}
+
+/** Workers (Linux) install a hardened systemd unit. The macOS manager's own
+ * agent is a per-user launch agent installed locally by fleet-sync
+ * (installManagerAgent), never through this remote entry point.
+ */
+async function installAgent(
+  request: Request,
+  root: string,
+  run: Runner,
+  receipts: string | undefined
+): Promise<Check[]> {
+  // Measured before anything changes: a full disk is named, not guessed at.
+  const disk = await agentDisk(run);
+  if (disk.status !== 'pass') return [disk];
+  const monitoring = resolve(root, 'ansible/monitoring');
+  const result = await runAnsibleWithReceipt(
+    run,
+    'agent.yml',
+    {
+      cwd: monitoring,
+      timeoutMs: 300_000,
+      input: JSON.stringify({
+        beszel_node: request.node,
+        beszel_hub_url: privateHub(request.hubUrl),
+        beszel_public_key: request.publicKey,
+        beszel_agent_token: request.token,
+      }),
+    },
+    {
+      name: 'beszel-agent',
+      sources: [monitoring],
+      verify: () => agentActive(run),
+      ...(receipts ? { directory: receipts } : {}),
+    }
+  );
+  if (!result.ok)
+    return [
+      disk,
+      {
+        id: 'beszel:installation',
+        status: 'fail',
+        summary: 'BESZEL_INSTALLATION_INCOMPLETE',
+        remedy: {
+          action:
+            'Inspect sudo, signed artifact access, free disk and existing service ownership. Preserve the native agent DATA_DIR and rerun the manager.',
+        },
+      },
+    ];
+  const ready = await agentActive(run);
+  return [
+    disk,
+    {
+      id: 'beszel:service',
+      status: ready ? 'pass' : 'fail',
+      summary: ready
+        ? 'Native Beszel service is enabled and active; hub metrics remain a separate gate.'
+        : 'BESZEL_SERVICE_STOPPED',
+    },
+    // How the hub answered the agent, from the agent's side: a rejected token
+    // or key is told apart from a hub that is merely down.
+    ...(ready ? [await agentConnection(run)] : []),
+  ];
+}
+async function agentActive(run: Runner): Promise<boolean> {
+  return (
+    (await run(['systemctl', 'is-enabled', 'beszel-agent.service'])).ok &&
+    (await run(['systemctl', 'is-active', 'beszel-agent.service'])).ok
+  );
+}
+interface Request {
+  node: string;
+  revision: string;
+  hubUrl: string;
+  mode?: unknown;
+  publicKey?: unknown;
+  token?: unknown;
+}
+function validateRequest(value: unknown): Request {
+  if (
+    !record(value) ||
+    typeof value['node'] !== 'string' ||
+    !isRevision(value['revision']) ||
+    typeof value['hubUrl'] !== 'string'
+  )
+    throw new Error('BESZEL_INPUT_INVALID');
+  return value as unknown as Request;
+}
+async function validateSource(
+  request: Request,
+  root: string,
+  run: Runner
+): Promise<void> {
+  for (const [args, expected] of [
+    [['git', 'rev-parse', 'HEAD'], request.revision],
+    [['git', 'rev-parse', '--show-toplevel'], resolve(root)],
+    [['git', 'status', '--porcelain'], ''],
+  ] as const) {
+    const observed = await run([...args], { cwd: root });
+    if (!observed.ok || observed.stdout.trim() !== expected)
+      throw new Error('BESZEL_SOURCE_MISMATCH');
+  }
+}
+async function main(): Promise<Check[]> {
+  let raw = '';
+  for await (const chunk of process.stdin) {
+    raw += String(chunk);
+    if (raw.length > 16_384) throw new Error('BESZEL_INPUT_LIMIT');
+  }
+  return applyAgent(JSON.parse(raw), process.cwd(), runCommand);
+}
+/** Turns a known refusal into structured evidence; anything else is redacted. */
+export function refusal(error: unknown): Check[] {
+  const code = error instanceof Error ? error.message : '';
+  const action = REFUSALS[code];
+  return [
+    action
+      ? {
+          id: 'beszel:operation',
+          status: 'fail',
+          summary: code,
+          remedy: { action },
+        }
+      : {
+          id: 'beszel:operation',
+          status: 'fail',
+          summary: 'BESZEL_OPERATION_INCOMPLETE',
+        },
+  ];
+}
+const REFUSALS: Record<string, string> = {
+  BESZEL_NOT_HUB:
+    'The Beszel hub runs only on the manager. Workers run only the agent; converge the hub from the manager with fleet-sync.',
+  BESZEL_NOT_ENROLLED:
+    'Declare this machine in fleet.json (the manager or a worker) before enrolling monitoring.',
+  BESZEL_SOURCE_MISMATCH:
+    'The checkout is not the clean, expected revision. Rerun fleet-sync so the reviewed source is installed first.',
+  BESZEL_HUB_MISMATCH:
+    "Agents connect only to the manager's private hub URL from fleet.json. Rerun fleet-sync from an up-to-date checkout.",
+  BESZEL_CREDENTIALS_REQUIRED:
+    'Agent enrollment credentials were not delivered. Rerun fleet-sync from the manager.',
+};
+if (import.meta.main) {
+  const checks = await main().catch(refusal);
+  process.stdout.write(JSON.stringify({ version: 1, checks }));
+}

package/src/monitoring/beszel.ts

@@ -0,0 +1,198 @@
+import { fleet } from '../fleet-config';
+import lock from '../../ansible/monitoring/lock.json';
+import type { Check } from '../types';
+
+export interface BeszelEvidence {
+  dashboardUrl: string;
+  checks: Check[];
+  receivedAt: string | null;
+}
+export type BeszelRequest = (path: string) => Promise<unknown>;
+type Row = Record<string, unknown>;
+function object(value: unknown): value is Row {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function items(value: unknown): Row[] {
+  if (
+    !object(value) ||
+    !Array.isArray(value['items']) ||
+    !value['items'].every(object)
+  )
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'];
+}
+function check(
+  id: string,
+  status: Check['status'],
+  summary: string,
+  action: string
+): Check {
+  return { id: `beszel:${id}`, status, summary, remedy: { action } };
+}
+/** Public dashboard hostname served through the Cloudflare
+ * tunnel `beszel`. Agents and the manager keep using the private hub URL.
+ */
+export const publicHubUrl = (): string => fleet.services.monitoring.publicUrl;
+
+export function privateHub(value: string): string {
+  const url = new URL(value);
+  if (
+    url.protocol !== 'https:' ||
+    !url.hostname.endsWith('.ts.net') ||
+    url.port !== '8443' ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash ||
+    url.pathname !== '/'
+  )
+    throw new Error('BESZEL_PRIVATE_URL');
+  return url.origin;
+}
+
+/** Read-only PocketBase endpoints, verified against the upstream Beszel
+ * release `ansible/monitoring/lock.json` pins.
+ * Never return raw records: they may contain tokens, hosts or user information.
+ */
+export async function verifyBeszel(
+  hub: string,
+  node: string,
+  request: BeszelRequest,
+  now = Date.now()
+): Promise<BeszelEvidence> {
+  const dashboardUrl = privateHub(hub);
+  if (!/^[a-z][a-z0-9-]{0,62}$/.test(node)) throw new Error('BESZEL_NODE_ID');
+  const result: BeszelEvidence = { dashboardUrl, checks: [], receivedAt: null };
+  try {
+    const info = await request('/api/beszel/info');
+    if (!object(info) || info['v'] !== lock.version) {
+      result.checks.push(
+        check(
+          'version',
+          'fail',
+          'BESZEL_VERSION_MISMATCH',
+          'Apply the pinned hub release before using its versioned API contract.'
+        )
+      );
+      return result;
+    }
+    result.checks.push(
+      check(
+        'version',
+        'pass',
+        `Authenticated hub reports ${lock.version}.`,
+        'None.'
+      )
+    );
+    const filter = encodeURIComponent(`name="${node}"`);
+    const records = items(
+      await request(
+        `/api/collections/systems/records?filter=${filter}&perPage=2`
+      )
+    );
+    if (records.length !== 1) {
+      result.checks.push(
+        check(
+          'registration',
+          'fail',
+          records.length ? 'BESZEL_DUPLICATE_NODE' : 'BESZEL_NODE_MISSING',
+          'Review the canonical node in the hub. Preserve its fingerprint and history; do not create a replacement identity.'
+        )
+      );
+      return result;
+    }
+    await verifySample(records[0]!, result, request, now);
+  } catch {
+    result.checks.push(
+      check(
+        'api',
+        'fail',
+        'BESZEL_API_UNVERIFIED',
+        'Verify private reachability and a valid hub account with access to this node. Raw HTTP bodies are withheld.'
+      )
+    );
+  }
+  return result;
+}
+
+/** No redirects, browser, token minting, retries or response logging. */
+export function beszelReader(
+  hub: string,
+  token: string,
+  fetcher: typeof fetch = fetch
+): BeszelRequest {
+  const origin = privateHub(hub);
+  return async (path) => {
+    if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
+    const response = await fetcher(`${origin}${path}`, {
+      headers: { Authorization: token },
+      redirect: 'error',
+      signal: AbortSignal.timeout(10_000),
+    });
+    if (!response.ok) throw new Error('BESZEL_API_UNVERIFIED');
+    const text = await response.text();
+    if (text.length > 256_000) throw new Error('BESZEL_RESPONSE_LIMIT');
+    return JSON.parse(text) as unknown;
+  };
+}
+
+async function verifySample(
+  record: Row,
+  result: BeszelEvidence,
+  request: BeszelRequest,
+  now: number
+): Promise<void> {
+  const id = record['id'];
+  if (typeof id !== 'string' || !/^[a-z0-9]{15}$/.test(id))
+    throw new Error('BESZEL_PROTOCOL');
+  result.checks.push(
+    check(
+      'registration',
+      'pass',
+      'Exactly one canonical node record is visible to this account.',
+      'None.'
+    )
+  );
+  result.checks.push(
+    check(
+      'connection',
+      record['status'] === 'up' ? 'pass' : 'fail',
+      record['status'] === 'up'
+        ? 'Hub reports node connected.'
+        : 'BESZEL_AGENT_DISCONNECTED',
+      'Inspect agent service, private hub connectivity and node-specific token; preserve its DATA_DIR.'
+    )
+  );
+  await checkFreshness(id, result, request, now);
+}
+async function checkFreshness(
+  id: string,
+  result: BeszelEvidence,
+  request: BeszelRequest,
+  now: number
+): Promise<void> {
+  const stats = items(
+    await request(
+      `/api/collections/system_stats/records?filter=${encodeURIComponent(`system="${id}" && type="1m"`)}&sort=-created&perPage=1&fields=created`
+    )
+  );
+  const created = stats[0]?.['created'];
+  const timestamp = typeof created === 'string' ? Date.parse(created) : NaN;
+  const fresh =
+    Number.isFinite(timestamp) &&
+    now - timestamp <= 180_000 &&
+    now >= timestamp - 30_000;
+  result.receivedAt = Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : null;
+  result.checks.push(
+    check(
+      'freshness',
+      fresh ? 'pass' : 'fail',
+      fresh
+        ? `Hub received sample ${result.receivedAt}; within 180 seconds.`
+        : 'BESZEL_METRICS_STALE',
+      'Check hub and node clocks, agent connection and disk capacity. Rerun after the next sampling interval.'
+    )
+  );
+}

package/src/monitoring/enroll.ts

@@ -0,0 +1,111 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { record } from '../config';
+import { privateHub } from './beszel';
+import lock from '../../ansible/monitoring/lock.json';
+
+export type HubRequest = (path: string, body?: object) => Promise<unknown>;
+export interface AgentEnrollment {
+  systemId: string;
+  hubUrl: string;
+  publicKey: string;
+  token: string;
+}
+function rows(value: unknown): Record<string, unknown>[] {
+  if (
+    !record(value) ||
+    !Array.isArray(value['items']) ||
+    !value['items'].every(record)
+  )
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'];
+}
+function identifier(value: unknown): string {
+  if (typeof value !== 'string' || !/^[a-z0-9]{15}$/.test(value))
+    throw new Error('BESZEL_PROTOCOL');
+  return value;
+}
+/** Mutating API adapter: call only after fleet approval and under manager lock.
+ * Uses the same systems/fingerprints API as upstream Add System, never its DB.
+ * The returned token is secret material: pass only through private SSH stdin.
+ */
+export async function enrollBeszel(
+  input: { node: string; hub: string; userId: string; host: string },
+  request: HubRequest
+): Promise<AgentEnrollment> {
+  const hubUrl = privateHub(input.hub);
+  if (
+    !/^[a-z][a-z0-9-]{0,62}$/.test(input.node) ||
+    !/^[a-z0-9.-]+\.ts\.net$/.test(input.host)
+  )
+    throw new Error('BESZEL_NODE_INVALID');
+  const info = await hubInfo(request);
+  const systemId = await ensureSystem(input, request);
+  const fingerprints = rows(
+    await request(
+      `/api/collections/fingerprints/records?filter=${encodeURIComponent(`system="${systemId}"`)}&perPage=2`
+    )
+  );
+  if (fingerprints.length > 1) throw new Error('BESZEL_DUPLICATE_FINGERPRINT');
+  let token = fingerprints[0]?.['token'];
+  if (fingerprints.length === 0) {
+    token = randomUUID();
+    await request('/api/collections/fingerprints/records', {
+      id: createHash('sha256')
+        .update(`personal:fingerprint:${systemId}`)
+        .digest('hex')
+        .slice(0, 15),
+      system: systemId,
+      token,
+    });
+  }
+  if (
+    typeof token !== 'string' ||
+    !/^[a-zA-Z0-9_-][a-zA-Z0-9_-]{15,63}$/.test(token)
+  )
+    throw new Error('BESZEL_TOKEN_INVALID');
+  return { systemId, hubUrl, publicKey: info['key'], token };
+}
+async function ensureSystem(
+  input: { node: string; userId: string; host: string },
+  request: HubRequest
+): Promise<string> {
+  const systems = rows(
+    await request(
+      `/api/collections/systems/records?filter=${encodeURIComponent(`name="${input.node}"`)}&perPage=2`
+    )
+  );
+  if (systems.length > 1) throw new Error('BESZEL_DUPLICATE_NODE');
+  if (systems[0]) {
+    if (systems[0]['host'] !== input.host)
+      throw new Error('BESZEL_HOST_CONFLICT');
+    return identifier(systems[0]['id']);
+  }
+  const id = createHash('sha256')
+    .update(`personal:system:${input.node}`)
+    .digest('hex')
+    .slice(0, 15);
+  const created = await request('/api/collections/systems/records', {
+    id,
+    name: input.node,
+    host: input.host,
+    port: '45876',
+    users: [identifier(input.userId)],
+  });
+  if (!record(created) || created['id'] !== id)
+    throw new Error('BESZEL_PROTOCOL');
+  return id;
+}
+
+async function hubInfo(
+  request: HubRequest
+): Promise<Record<string, unknown> & { key: string }> {
+  const info = await request('/api/beszel/info');
+  if (
+    !record(info) ||
+    info['v'] !== lock.version ||
+    typeof info['key'] !== 'string' ||
+    !info['key'].startsWith('ssh-ed25519 ')
+  )
+    throw new Error('BESZEL_VERSION_OR_KEY');
+  return info as Record<string, unknown> & { key: string };
+}

package/src/monitoring/hub-access.ts

@@ -0,0 +1,114 @@
+import { privateHub } from './beszel';
+import type { HubRequest } from './enroll';
+import { record } from '../config';
+import { fleet, secret } from '../fleet-config';
+
+/** The normal (non-superuser) Beszel account the manager enrolls nodes with.
+ * Its password is the fleet's monitoring-account-password secret. A fresh hub on the
+ * manager creates the account itself on first start (hub-darwin.yml).
+ */
+export const fleetIdentity = (): string =>
+  fleet.services.monitoring.managerAccount;
+
+/** Read-only Beszel user the hub signs every visitor in as (AUTO_LOGIN), so the
+ * dashboard needs no login. Create it once in the hub (Settings > Users, role
+ * "Read-only"); it holds no secret and must never be an admin. If the account
+ * does not exist the hub simply shows its normal login page.
+ */
+export const publicViewerIdentity = (): string =>
+  fleet.services.monitoring.viewerAccount;
+
+/** Accept the plain password, or a legacy JSON identity/password pair. */
+export function fleetCredentials(secret: string): {
+  identity: string;
+  password: string;
+} {
+  const value = secret.trim();
+  if (!value) throw new Error('BESZEL_ACCOUNT_INVALID');
+  if (value.startsWith('{')) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(value);
+    } catch {
+      throw new Error('BESZEL_ACCOUNT_INVALID');
+    }
+    if (
+      !record(parsed) ||
+      typeof parsed['identity'] !== 'string' ||
+      typeof parsed['password'] !== 'string'
+    )
+      throw new Error('BESZEL_ACCOUNT_INVALID');
+    return { identity: parsed['identity'], password: parsed['password'] };
+  }
+  return { identity: fleetIdentity(), password: value };
+}
+
+/** The fleet login from the manager's Vault session, or null when absent.
+ * Used to create the account on a hub's first start; never printed.
+ */
+export async function fleetAccount(): Promise<{
+  identity: string;
+  password: string;
+} | null> {
+  const password = await secret('monitoring-account-password');
+  try {
+    return password ? fleetCredentials(password) : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Reads the existing account password only. This never creates credentials. The normal Beszel account must have fleet-node access.
+ */
+export async function hubAccess(
+  hub: string
+): Promise<{ userId: string; request: HubRequest }> {
+  const url = privateHub(hub);
+  const password = await secret('monitoring-account-password');
+  if (!password) throw new Error('BESZEL_ACCOUNT_REQUIRED');
+  const credentials = fleetCredentials(password);
+  const login = await call(
+    url,
+    '/api/collections/users/auth-with-password',
+    '',
+    credentials
+  );
+  if (
+    !record(login) ||
+    typeof login['token'] !== 'string' ||
+    !record(login['record']) ||
+    typeof login['record']['id'] !== 'string'
+  )
+    throw new Error('BESZEL_AUTH_REQUIRED');
+  const token = login['token'];
+  return {
+    userId: login['record']['id'],
+    request: (path, body) => call(url, path, token, body),
+  };
+}
+async function call(
+  url: string,
+  path: string,
+  token: string,
+  body?: object
+): Promise<unknown> {
+  if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
+  const response = await fetch(`${url}${path}`, {
+    method: body ? 'POST' : 'GET',
+    headers: { Authorization: token, 'Content-Type': 'application/json' },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+    redirect: 'error',
+    signal: AbortSignal.timeout(10_000),
+  });
+  if (!response.ok) throw new Error(failure(response.status, path));
+  return response.json() as Promise<unknown>;
+}
+
+/** PocketBase answers a rejected password login with 400, not 401. */
+export function failure(status: number, path: string): string {
+  if (status === 400 && path.endsWith('/auth-with-password'))
+    return 'BESZEL_ACCOUNT_REQUIRED';
+  return status === 401 || status === 403
+    ? 'BESZEL_AUTH_REQUIRED'
+    : 'BESZEL_API_UNAVAILABLE';
+}