@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/monitoring/beszel.ts

@@ -0,0 +1,191 @@
+import { fleet } from '../fleet-config';
+import type { Check } from '../types';
+
+export interface BeszelEvidence {
+  dashboardUrl: string;
+  checks: Check[];
+  receivedAt: string | null;
+}
+export type BeszelRequest = (path: string) => Promise<unknown>;
+type Row = Record<string, unknown>;
+function object(value: unknown): value is Row {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function items(value: unknown): Row[] {
+  if (
+    !object(value) ||
+    !Array.isArray(value['items']) ||
+    !value['items'].every(object)
+  )
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'];
+}
+function check(
+  id: string,
+  status: Check['status'],
+  summary: string,
+  action: string
+): Check {
+  return { id: `beszel:${id}`, status, summary, remedy: { action } };
+}
+/** Public dashboard hostname served through the Cloudflare
+ * tunnel `beszel`. Agents and the manager keep using the private hub URL.
+ */
+export const PUBLIC_HUB_URL = fleet.services.monitoring.publicUrl;
+
+export function privateHub(value: string): string {
+  const url = new URL(value);
+  if (
+    url.protocol !== 'https:' ||
+    !url.hostname.endsWith('.ts.net') ||
+    url.port !== '8443' ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash ||
+    url.pathname !== '/'
+  )
+    throw new Error('BESZEL_PRIVATE_URL');
+  return url.origin;
+}
+
+/** Read-only PocketBase endpoints, verified against upstream Beszel v0.21.1.
+ * Never return raw records: they may contain tokens, hosts or user information.
+ */
+export async function verifyBeszel(
+  hub: string,
+  node: string,
+  request: BeszelRequest,
+  now = Date.now()
+): Promise<BeszelEvidence> {
+  const dashboardUrl = privateHub(hub);
+  if (!/^[a-z][a-z0-9-]{0,62}$/.test(node)) throw new Error('BESZEL_NODE_ID');
+  const result: BeszelEvidence = { dashboardUrl, checks: [], receivedAt: null };
+  try {
+    const info = await request('/api/beszel/info');
+    if (!object(info) || info['v'] !== '0.21.1') {
+      result.checks.push(
+        check(
+          'version',
+          'fail',
+          'BESZEL_VERSION_MISMATCH',
+          'Apply the pinned hub release before using its versioned API contract.'
+        )
+      );
+      return result;
+    }
+    result.checks.push(
+      check('version', 'pass', 'Authenticated hub reports 0.21.1.', 'None.')
+    );
+    const filter = encodeURIComponent(`name="${node}"`);
+    const records = items(
+      await request(
+        `/api/collections/systems/records?filter=${filter}&perPage=2`
+      )
+    );
+    if (records.length !== 1) {
+      result.checks.push(
+        check(
+          'registration',
+          'fail',
+          records.length ? 'BESZEL_DUPLICATE_NODE' : 'BESZEL_NODE_MISSING',
+          'Review the canonical node in the hub. Preserve its fingerprint and history; do not create a replacement identity.'
+        )
+      );
+      return result;
+    }
+    await verifySample(records[0]!, result, request, now);
+  } catch {
+    result.checks.push(
+      check(
+        'api',
+        'fail',
+        'BESZEL_API_UNVERIFIED',
+        'Verify private reachability and a valid hub account with access to this node. Raw HTTP bodies are withheld.'
+      )
+    );
+  }
+  return result;
+}
+
+/** No redirects, browser, token minting, retries or response logging. */
+export function beszelReader(
+  hub: string,
+  token: string,
+  fetcher: typeof fetch = fetch
+): BeszelRequest {
+  const origin = privateHub(hub);
+  return async (path) => {
+    if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
+    const response = await fetcher(`${origin}${path}`, {
+      headers: { Authorization: token },
+      redirect: 'error',
+      signal: AbortSignal.timeout(10_000),
+    });
+    if (!response.ok) throw new Error('BESZEL_API_UNVERIFIED');
+    const text = await response.text();
+    if (text.length > 256_000) throw new Error('BESZEL_RESPONSE_LIMIT');
+    return JSON.parse(text) as unknown;
+  };
+}
+
+async function verifySample(
+  record: Row,
+  result: BeszelEvidence,
+  request: BeszelRequest,
+  now: number
+): Promise<void> {
+  const id = record['id'];
+  if (typeof id !== 'string' || !/^[a-z0-9]{15}$/.test(id))
+    throw new Error('BESZEL_PROTOCOL');
+  result.checks.push(
+    check(
+      'registration',
+      'pass',
+      'Exactly one canonical node record is visible to this account.',
+      'None.'
+    )
+  );
+  result.checks.push(
+    check(
+      'connection',
+      record['status'] === 'up' ? 'pass' : 'fail',
+      record['status'] === 'up'
+        ? 'Hub reports node connected.'
+        : 'BESZEL_AGENT_DISCONNECTED',
+      'Inspect agent service, private hub connectivity and node-specific token; preserve its DATA_DIR.'
+    )
+  );
+  await checkFreshness(id, result, request, now);
+}
+async function checkFreshness(
+  id: string,
+  result: BeszelEvidence,
+  request: BeszelRequest,
+  now: number
+): Promise<void> {
+  const stats = items(
+    await request(
+      `/api/collections/system_stats/records?filter=${encodeURIComponent(`system="${id}" && type="1m"`)}&sort=-created&perPage=1&fields=created`
+    )
+  );
+  const created = stats[0]?.['created'];
+  const timestamp = typeof created === 'string' ? Date.parse(created) : NaN;
+  const fresh =
+    Number.isFinite(timestamp) &&
+    now - timestamp <= 180_000 &&
+    now >= timestamp - 30_000;
+  result.receivedAt = Number.isFinite(timestamp)
+    ? new Date(timestamp).toISOString()
+    : null;
+  result.checks.push(
+    check(
+      'freshness',
+      fresh ? 'pass' : 'fail',
+      fresh
+        ? `Hub received sample ${result.receivedAt}; within 180 seconds.`
+        : 'BESZEL_METRICS_STALE',
+      'Check hub and node clocks, agent connection and disk capacity. Rerun after the next sampling interval.'
+    )
+  );
+}

package/src/monitoring/enroll.ts

@@ -0,0 +1,110 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { record } from '../config';
+import { privateHub } from './beszel';
+
+export type HubRequest = (path: string, body?: object) => Promise<unknown>;
+export interface AgentEnrollment {
+  systemId: string;
+  hubUrl: string;
+  publicKey: string;
+  token: string;
+}
+function rows(value: unknown): Record<string, unknown>[] {
+  if (
+    !record(value) ||
+    !Array.isArray(value['items']) ||
+    !value['items'].every(record)
+  )
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'];
+}
+function identifier(value: unknown): string {
+  if (typeof value !== 'string' || !/^[a-z0-9]{15}$/.test(value))
+    throw new Error('BESZEL_PROTOCOL');
+  return value;
+}
+/** Mutating API adapter: call only after fleet approval and under manager lock.
+ * Uses the same systems/fingerprints API as upstream Add System, never its DB.
+ * The returned token is secret material: pass only through private SSH stdin.
+ */
+export async function enrollBeszel(
+  input: { node: string; hub: string; userId: string; host: string },
+  request: HubRequest
+): Promise<AgentEnrollment> {
+  const hubUrl = privateHub(input.hub);
+  if (
+    !/^[a-z][a-z0-9-]{0,62}$/.test(input.node) ||
+    !/^[a-z0-9.-]+\.ts\.net$/.test(input.host)
+  )
+    throw new Error('BESZEL_NODE_INVALID');
+  const info = await hubInfo(request);
+  const systemId = await ensureSystem(input, request);
+  const fingerprints = rows(
+    await request(
+      `/api/collections/fingerprints/records?filter=${encodeURIComponent(`system="${systemId}"`)}&perPage=2`
+    )
+  );
+  if (fingerprints.length > 1) throw new Error('BESZEL_DUPLICATE_FINGERPRINT');
+  let token = fingerprints[0]?.['token'];
+  if (fingerprints.length === 0) {
+    token = randomUUID();
+    await request('/api/collections/fingerprints/records', {
+      id: createHash('sha256')
+        .update(`personal:fingerprint:${systemId}`)
+        .digest('hex')
+        .slice(0, 15),
+      system: systemId,
+      token,
+    });
+  }
+  if (
+    typeof token !== 'string' ||
+    !/^[a-zA-Z0-9_-][a-zA-Z0-9_-]{15,63}$/.test(token)
+  )
+    throw new Error('BESZEL_TOKEN_INVALID');
+  return { systemId, hubUrl, publicKey: info['key'], token };
+}
+async function ensureSystem(
+  input: { node: string; userId: string; host: string },
+  request: HubRequest
+): Promise<string> {
+  const systems = rows(
+    await request(
+      `/api/collections/systems/records?filter=${encodeURIComponent(`name="${input.node}"`)}&perPage=2`
+    )
+  );
+  if (systems.length > 1) throw new Error('BESZEL_DUPLICATE_NODE');
+  if (systems[0]) {
+    if (systems[0]['host'] !== input.host)
+      throw new Error('BESZEL_HOST_CONFLICT');
+    return identifier(systems[0]['id']);
+  }
+  const id = createHash('sha256')
+    .update(`personal:system:${input.node}`)
+    .digest('hex')
+    .slice(0, 15);
+  const created = await request('/api/collections/systems/records', {
+    id,
+    name: input.node,
+    host: input.host,
+    port: '45876',
+    users: [identifier(input.userId)],
+  });
+  if (!record(created) || created['id'] !== id)
+    throw new Error('BESZEL_PROTOCOL');
+  return id;
+}
+
+async function hubInfo(
+  request: HubRequest
+): Promise<Record<string, unknown> & { key: string }> {
+  const info = await request('/api/beszel/info');
+  if (
+    !record(info) ||
+    info['v'] !== '0.21.1' ||
+    typeof info['key'] !== 'string' ||
+    !info['key'].startsWith('ssh-ed25519 ')
+  )
+    throw new Error('BESZEL_VERSION_OR_KEY');
+  return info as Record<string, unknown> & { key: string };
+}

package/src/monitoring/hub-access.ts

@@ -0,0 +1,113 @@
+import { privateHub } from './beszel';
+import type { HubRequest } from './enroll';
+import { record } from '../config';
+import { fleet, secret } from '../fleet-config';
+
+/** The normal (non-superuser) Beszel account the manager enrolls nodes with.
+ * Its password is the fleet's monitoring-account-password secret. A fresh hub on the
+ * manager creates the account itself on first start (hub-darwin.yml).
+ */
+export const BESZEL_FLEET_IDENTITY = fleet.services.monitoring.managerAccount;
+
+/** Read-only Beszel user the hub signs every visitor in as (AUTO_LOGIN), so the
+ * dashboard needs no login. Create it once in the hub (Settings > Users, role
+ * "Read-only"); it holds no secret and must never be an admin. If the account
+ * does not exist the hub simply shows its normal login page.
+ */
+export const BESZEL_PUBLIC_VIEWER_IDENTITY =
+  fleet.services.monitoring.viewerAccount;
+
+/** Accept the plain password, or a legacy JSON identity/password pair. */
+export function fleetCredentials(secret: string): {
+  identity: string;
+  password: string;
+} {
+  const value = secret.trim();
+  if (!value) throw new Error('BESZEL_ACCOUNT_INVALID');
+  if (value.startsWith('{')) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(value);
+    } catch {
+      throw new Error('BESZEL_ACCOUNT_INVALID');
+    }
+    if (
+      !record(parsed) ||
+      typeof parsed['identity'] !== 'string' ||
+      typeof parsed['password'] !== 'string'
+    )
+      throw new Error('BESZEL_ACCOUNT_INVALID');
+    return { identity: parsed['identity'], password: parsed['password'] };
+  }
+  return { identity: BESZEL_FLEET_IDENTITY, password: value };
+}
+
+/** The fleet login from the manager's Vault session, or null when absent.
+ * Used to create the account on a hub's first start; never printed.
+ */
+export async function fleetAccount(): Promise<{
+  identity: string;
+  password: string;
+} | null> {
+  const password = await secret('monitoring-account-password');
+  try {
+    return password ? fleetCredentials(password) : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Reads the existing account password only. This never creates credentials. The normal Beszel account must have fleet-node access.
+ */
+export async function hubAccess(
+  hub: string
+): Promise<{ userId: string; request: HubRequest }> {
+  const url = privateHub(hub);
+  const password = await secret('monitoring-account-password');
+  if (!password) throw new Error('BESZEL_ACCOUNT_REQUIRED');
+  const credentials = fleetCredentials(password);
+  const login = await call(
+    url,
+    '/api/collections/users/auth-with-password',
+    '',
+    credentials
+  );
+  if (
+    !record(login) ||
+    typeof login['token'] !== 'string' ||
+    !record(login['record']) ||
+    typeof login['record']['id'] !== 'string'
+  )
+    throw new Error('BESZEL_AUTH_REQUIRED');
+  const token = login['token'];
+  return {
+    userId: login['record']['id'],
+    request: (path, body) => call(url, path, token, body),
+  };
+}
+async function call(
+  url: string,
+  path: string,
+  token: string,
+  body?: object
+): Promise<unknown> {
+  if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
+  const response = await fetch(`${url}${path}`, {
+    method: body ? 'POST' : 'GET',
+    headers: { Authorization: token, 'Content-Type': 'application/json' },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+    redirect: 'error',
+    signal: AbortSignal.timeout(10_000),
+  });
+  if (!response.ok) throw new Error(failure(response.status, path));
+  return response.json() as Promise<unknown>;
+}
+
+/** PocketBase answers a rejected password login with 400, not 401. */
+export function failure(status: number, path: string): string {
+  if (status === 400 && path.endsWith('/auth-with-password'))
+    return 'BESZEL_ACCOUNT_REQUIRED';
+  return status === 401 || status === 403
+    ? 'BESZEL_AUTH_REQUIRED'
+    : 'BESZEL_API_UNAVAILABLE';
+}

package/src/monitoring/hub-viewer.ts

@@ -0,0 +1,142 @@
+import { randomBytes } from 'node:crypto';
+import { record } from '../config';
+import type { Check } from '../types';
+import { BESZEL_PUBLIC_VIEWER_IDENTITY } from './hub-access';
+
+/** Loopback address of the hub container on the manager. */
+export const LOCAL_HUB = 'http://127.0.0.1:8090';
+
+type Fetcher = typeof fetch;
+export interface HubLogin {
+  identity: string;
+  password: string;
+}
+
+function viewerCheck(
+  status: Check['status'],
+  summary: string,
+  action?: string
+): Check {
+  return {
+    id: 'beszel:public-viewer',
+    status,
+    summary,
+    ...(action ? { remedy: { action } } : {}),
+  };
+}
+
+interface Session {
+  fetcher: Fetcher;
+  base: string;
+  token: string;
+}
+
+async function api(
+  session: Session,
+  method: string,
+  path: string,
+  body?: object
+): Promise<unknown> {
+  const response = await session.fetcher(`${session.base}${path}`, {
+    method,
+    headers: {
+      Authorization: session.token,
+      'Content-Type': 'application/json',
+    },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+    redirect: 'error',
+    signal: AbortSignal.timeout(10_000),
+  });
+  // Bodies can echo emails or tokens: only the status is ever surfaced.
+  if (!response.ok) throw new Error(`HTTP_${response.status}`);
+  return response.json() as Promise<unknown>;
+}
+
+async function superuserToken(
+  session: Session,
+  login: HubLogin
+): Promise<string> {
+  const auth = await api(
+    session,
+    'POST',
+    '/api/collections/_superusers/auth-with-password',
+    login
+  );
+  if (!record(auth) || typeof auth['token'] !== 'string')
+    throw new Error('BESZEL_PROTOCOL');
+  return auth['token'];
+}
+
+async function findViewer(
+  session: Session
+): Promise<{ id: string; role: unknown } | null> {
+  const filter = encodeURIComponent(`email="${BESZEL_PUBLIC_VIEWER_IDENTITY}"`);
+  const list = await api(
+    session,
+    'GET',
+    `/api/collections/users/records?filter=${filter}&perPage=2`
+  );
+  const items =
+    record(list) && Array.isArray(list['items']) ? list['items'] : [];
+  const first: unknown = items[0];
+  if (!record(first) || typeof first['id'] !== 'string') return null;
+  return { id: first['id'], role: first['role'] };
+}
+
+async function createViewer(session: Session): Promise<void> {
+  const password = randomBytes(24).toString('base64url');
+  await api(session, 'POST', '/api/collections/users/records', {
+    email: BESZEL_PUBLIC_VIEWER_IDENTITY,
+    username: 'public-viewer',
+    password,
+    passwordConfirm: password,
+    verified: true,
+    role: 'readonly',
+  });
+}
+
+/** Idempotently makes sure the read-only viewer exists, then re-reads it to
+ * prove its role. AUTO_LOGIN signs every visitor in as this account, so an
+ * existing account with any other role is demoted rather than trusted. The
+ * password is random, never stored or printed, and is not needed to sign in.
+ */
+export async function ensurePublicViewer(
+  login: HubLogin | null,
+  fetcher: Fetcher = fetch,
+  base: string = LOCAL_HUB
+): Promise<Check> {
+  if (!login)
+    return viewerCheck(
+      'warn',
+      'BESZEL_VIEWER_UNPROVISIONED',
+      "The fleet admin login is unavailable. Log the manager's Vault session in with read access to BESZEL_FLEET_AUTH and rerun fleet-sync; until then the public dashboard shows the login page."
+    );
+  try {
+    const anonymous: Session = { fetcher, base, token: '' };
+    const session = {
+      ...anonymous,
+      token: await superuserToken(anonymous, login),
+    };
+    const existing = await findViewer(session);
+    if (!existing) await createViewer(session);
+    else if (existing.role !== 'readonly')
+      await api(
+        session,
+        'PATCH',
+        `/api/collections/users/records/${existing.id}`,
+        { role: 'readonly' }
+      );
+    const verified = await findViewer(session);
+    if (verified?.role !== 'readonly') throw new Error('BESZEL_VIEWER_ROLE');
+    return viewerCheck(
+      'pass',
+      `Visitors are signed in automatically as ${BESZEL_PUBLIC_VIEWER_IDENTITY} (read-only).`
+    );
+  } catch {
+    return viewerCheck(
+      'warn',
+      'BESZEL_VIEWER_UNVERIFIED',
+      `Could not create or verify the read-only ${BESZEL_PUBLIC_VIEWER_IDENTITY} user with the fleet admin login. Create it in the hub (Settings > Users, role Read-only); never make it an admin. Details are withheld.`
+    );
+  }
+}

package/src/monitoring/hub.ts

@@ -0,0 +1,93 @@
+import { record } from '../config';
+import type { Check, Runner } from '../types';
+import { PUBLIC_HUB_URL } from './beszel';
+import { secretVia } from '../fleet-config';
+
+/** Shared Beszel hub helpers. The hub runs only on the manager; see
+ * manager-hub.ts. Workers never host it.
+ */
+export function publicEndpointCheck(connected: boolean): Check {
+  return connected
+    ? {
+        id: 'beszel:public-endpoint',
+        status: 'pass',
+        summary: `Cloudflare connector for ${PUBLIC_HUB_URL} was started from the Vault credential.`,
+      }
+    : {
+        id: 'beszel:public-endpoint',
+        status: 'warn',
+        summary: 'BESZEL_TUNNEL_CREDENTIAL_UNAVAILABLE',
+        remedy: {
+          action: `Create the Cloudflare tunnel and store its token in the fleet's secrets, then log the manager's Vault session in with read access to secret/beszel/tunnel and rerun fleet-sync. The private dashboard is unaffected; ${PUBLIC_HUB_URL} stays offline until then.`,
+        },
+      };
+}
+
+/** The dashboard connector token from this fleet's secrets provider. Null
+ * when it is unavailable. Never printed, logged or placed in argv.
+ */
+export function tunnelToken(run: Runner): Promise<string | null> {
+  return secretVia(run)('monitoring-tunnel-token');
+}
+export function failure(code: string, action: string): Check[] {
+  return [
+    { id: 'beszel:hub', status: 'fail', summary: code, remedy: { action } },
+  ];
+}
+
+export function inspectRoute(
+  output: string,
+  hub: string,
+  target = { port: '8443', proxy: 'http://127.0.0.1:8090' }
+): boolean | Check[] {
+  let configured = false;
+  try {
+    const value: unknown = JSON.parse(output);
+    if (!record(value)) throw new Error();
+    if (publicOrConflictingTcp(value, hub, target.port))
+      return failure(
+        'BESZEL_PRIVATE_ROUTE_CONFLICT',
+        'Existing Funnel/public or TCP routing conflicts with private Beszel access. Review the source-owned route; do not reset Serve.'
+      );
+    const web = value['Web'];
+    if (record(web)) {
+      const entry = web[new URL(hub).hostname + ':' + target.port];
+      if (entry !== undefined) {
+        if (!expectedHandler(entry, target.proxy))
+          return failure(
+            'BESZEL_PRIVATE_ROUTE_CONFLICT',
+            'Port 8443 is owned by a different Tailscale Serve handler. Review a source-controlled route migration; do not reset Serve.'
+          );
+        configured = true;
+      }
+    }
+  } catch {
+    return failure(
+      'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
+      'Review Tailscale Serve status without exposing raw configuration.'
+    );
+  }
+  return configured;
+}
+
+function publicOrConflictingTcp(
+  value: Record<string, unknown>,
+  hub: string,
+  port: string
+): boolean {
+  const funnel = value['AllowFunnel'];
+  if (record(funnel) && funnel[new URL(hub).hostname + ':' + port] === true)
+    return true;
+  const tcp = value['TCP'];
+  if (!record(tcp) || tcp[port] === undefined) return false;
+  return !record(tcp[port]) || tcp[port]['HTTPS'] !== true;
+}
+
+function expectedHandler(entry: unknown, proxy: string): boolean {
+  return (
+    record(entry) &&
+    record(entry['Handlers']) &&
+    record(entry['Handlers']['/']) &&
+    entry['Handlers']['/']['Proxy'] === proxy
+  );
+}