@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/cli/fleet-sync-monitoring.ts

@@ -0,0 +1,185 @@
+import type { FleetNode, Runner, Check } from '../types';
+import type { FleetDefinition } from '../definition';
+import { hubUrl, roleOf } from '../topology';
+import { discoverTailnet } from '../tailscale';
+import { fleetIdentity, hubAccess } from '../monitoring/hub-access';
+import { enrollBeszel } from '../monitoring/enroll';
+import { verifyBeszel } from '../monitoring/beszel';
+import { installRemoteAgent } from '../monitoring/remote';
+import { installManagerAgent } from '../monitoring/manager-agent';
+
+export interface MonitoringDependencies {
+  access?: typeof hubAccess;
+  sampleDeadlineMs?: number;
+}
+
+/** Enrolls one worker's Beszel agent against the manager's hub. Workers never
+ * host the hub; it is converged on the manager by convergeManagerHub. Called
+ * after source bootstrap, only under approved manager application.
+ */
+export function synchronizeMonitoringWith(
+  dependencies: MonitoringDependencies = {}
+) {
+  return async function synchronize(
+    node: FleetNode,
+    definition: FleetDefinition,
+    revision: string,
+    run: Runner
+  ): Promise<Check[]> {
+    try {
+      return await enrollWorker(node, definition, {
+        revision,
+        run,
+        ...dependencies,
+      });
+    } catch (error) {
+      return unavailable(errorCode(error));
+    }
+  };
+}
+export const synchronizeMonitoring = synchronizeMonitoringWith();
+
+/** The manager is always a monitored system on its own hub: after the hub
+ * converges, fleet-sync enrolls the manager like a worker, but installs its
+ * agent locally as a per-user launch agent (no SSH, no sudo).
+ */
+export async function enrollManagerMonitoring(
+  definition: FleetDefinition,
+  workspace: string,
+  run: Runner,
+  dependencies: MonitoringDependencies & {
+    install?: typeof installManagerAgent;
+  } = {}
+): Promise<Check[]> {
+  const { manager } = definition;
+  try {
+    const hub = hubUrl(definition);
+    if (!(await hubReachable(hub, run)))
+      return unavailable('BESZEL_HUB_UNREACHABLE');
+    const access = await (dependencies.access ?? hubAccess)(hub);
+    const enrollment = await enrollBeszel(
+      {
+        node: manager.id,
+        host: manager.network.tailscaleName,
+        hub,
+        userId: access.userId,
+      },
+      access.request
+    );
+    const checks = await (dependencies.install ?? installManagerAgent)(
+      workspace,
+      { node: manager.id, enrollment },
+      run
+    );
+    if (checks.some((check) => check.status === 'fail')) return checks;
+    const evidence = await waitForSample(
+      hub,
+      manager.id,
+      access.request,
+      dependencies.sampleDeadlineMs ?? 90_000
+    );
+    return [...checks, ...evidence.checks];
+  } catch (error) {
+    return unavailable(errorCode(error));
+  }
+}
+
+async function enrollWorker(
+  node: FleetNode,
+  definition: FleetDefinition,
+  context: MonitoringDependencies & { revision: string; run: Runner }
+): Promise<Check[]> {
+  const { revision, run } = context;
+  if (roleOf(definition, node.id) !== 'worker')
+    return unavailable('BESZEL_NOT_ENROLLED');
+  const hub = hubUrl(definition);
+  if (!(await hubReachable(hub, run)))
+    return unavailable('BESZEL_HUB_UNREACHABLE');
+  const tailnet = await discoverTailnet(run);
+  const target = tailnet.computers.find((peer) => peer.id === node.tailscaleId);
+  if (!target) return unavailable('BESZEL_NODE_TAILNET_REQUIRED');
+  const access = await (context.access ?? hubAccess)(hub);
+  const enrollment = await enrollBeszel(
+    { node: node.id, host: target.dns, hub, userId: access.userId },
+    access.request
+  );
+  const checks = await installRemoteAgent(node, revision, enrollment, run);
+  if (checks.some((check) => check.status === 'fail')) return checks;
+  const evidence = await waitForSample(
+    hub,
+    node.id,
+    access.request,
+    context.sampleDeadlineMs ?? 90_000
+  );
+  return [...checks, ...evidence.checks];
+}
+
+const PASSTHROUGH = [
+  'BESZEL_ACCOUNT_REQUIRED',
+  'BESZEL_ACCOUNT_INVALID',
+  'BESZEL_AUTH_REQUIRED',
+  'BESZEL_API_UNAVAILABLE',
+];
+function errorCode(error: unknown): string {
+  return error instanceof Error && PASSTHROUGH.includes(error.message)
+    ? error.message
+    : 'BESZEL_ENROLLMENT_INCOMPLETE';
+}
+
+async function hubReachable(hub: string, run: Runner): Promise<boolean> {
+  const probe = await run(
+    [
+      'curl',
+      '-fsS',
+      '--max-time',
+      '10',
+      '-o',
+      '/dev/null',
+      `${hub}/api/health`,
+    ],
+    { timeoutMs: 15_000 }
+  );
+  return probe.ok;
+}
+
+const remedies = (): Record<string, string> => ({
+  BESZEL_ACCOUNT_REQUIRED: `The hub rejected the fleet login ${fleetIdentity()}. A new hub on the manager creates it on first start from BESZEL_FLEET_AUTH; an older hub needs it once, by its superuser, as a verified users record with that Vault password. Never paste the password on a command line.`,
+  BESZEL_HUB_UNREACHABLE:
+    'The hub runs on the manager; keep it awake and Docker Desktop running, then rerun fleet-sync from the manager so it converges the hub first.',
+  BESZEL_NODE_TAILNET_REQUIRED:
+    'The worker is not visible on the tailnet. Sign it in to Tailscale and record its tailscaleId in the local roster, then rerun fleet-sync.',
+  BESZEL_NOT_ENROLLED:
+    'Only workers declared in fleet.json run the agent over SSH; fleet-sync installs the manager’s own agent locally after its hub.',
+});
+function unavailable(code: string): Check[] {
+  return [
+    {
+      id: 'beszel:enrollment',
+      status: 'fail',
+      summary: code,
+      remedy: {
+        action:
+          remedies()[code] ??
+          "Verify the manager's private hub is running and the manager has BESZEL_FLEET_AUTH through existing Vault authorization. The normal hub account must access these systems. Inspect duplicate registration/host conflicts before retry; no token or raw response is printed.",
+      },
+    },
+  ];
+}
+
+async function waitForSample(
+  hub: string,
+  node: string,
+  request: Parameters<typeof verifyBeszel>[2],
+  deadlineMs: number
+) {
+  const deadline = Date.now() + deadlineMs;
+  let evidence = await verifyBeszel(hub, node, request);
+  while (
+    Date.now() < deadline &&
+    evidence.checks.some((check) => check.summary === 'BESZEL_METRICS_STALE')
+  ) {
+    await new Promise((resolve) => setTimeout(resolve, 10_000));
+    evidence = await verifyBeszel(hub, node, request);
+  }
+  return evidence;
+}

package/src/cli/fleet-sync-output.ts

@@ -0,0 +1,317 @@
+import { Chalk, type ChalkInstance } from 'chalk';
+import type { ManagerNode, ManagerReport } from '../manager-sync';
+import { blocking, type Check, type Status } from '../types';
+
+/** Report columns: each owns the checks whose id matches. A check matches the
+ * first column that claims it; anything unclaimed still counts in RESULT.
+ */
+const COLUMNS: { title: string; matches: (id: string) => boolean }[] = [
+  { title: 'NET', matches: (id) => id.startsWith('tailscale:') },
+  { title: 'SSH', matches: (id) => id.startsWith('ssh:') },
+  {
+    title: 'OS',
+    matches: (id) =>
+      ['ubuntu:baseline', 'configuration:apply'].includes(id) ||
+      id.startsWith('manager:'),
+  },
+  {
+    title: 'T3',
+    matches: (id) =>
+      [
+        't3:service',
+        't3:policy',
+        't3:providers',
+        't3:antigravity-key',
+        't3:private-route',
+        'worker:t3-recovery',
+        'worker:t3-health',
+        'worker:service-recovery',
+      ].includes(id),
+  },
+  {
+    title: 'LINK',
+    matches: (id) =>
+      ['t3:manager-endpoint', 't3:manager-origin', 't3:paired-access'].includes(
+        id
+      ),
+  },
+  {
+    title: 'PROXY',
+    matches: (id) =>
+      id.startsWith('provider:') ||
+      id.startsWith('proxy:') ||
+      id === 'worker:proxy-singleton',
+  },
+  { title: 'AGENTS', matches: (id) => id.startsWith('agent:') },
+  {
+    title: 'MON',
+    matches: (id) =>
+      id.startsWith('beszel:') || id === 'worker:service:beszel-agent',
+  },
+];
+
+const ORDER: Status[] = ['fail', 'warn', 'skip', 'pass'];
+const WORD: Record<Status, string> = {
+  fail: 'FAIL',
+  warn: 'WARN',
+  skip: 'SKIP',
+  pass: 'PASS',
+};
+const SYMBOL: Record<Status, string> = {
+  fail: '✖',
+  warn: '▲',
+  skip: '○',
+  pass: '✔',
+};
+
+export interface OutputOptions {
+  /** Colors and symbols; plain words otherwise (pipes, NO_COLOR, agents). */
+  color?: boolean;
+  durationMs?: number;
+  /** Where the full JSON report of this run was saved. */
+  reportPath?: string;
+}
+
+/** Colors only for an interactive stdout that allows them. */
+export function stdoutColor(): boolean {
+  if (process.env['NO_COLOR'] !== undefined) return false;
+  if (process.env['FORCE_COLOR'] !== undefined) return true;
+  return Boolean(process.stdout.isTTY) && process.env['TERM'] !== 'dumb';
+}
+
+/** Short, stable report: one header line, a node grid, then each problem once
+ * (deduplicated across nodes) with its code and fix. Every line is greppable:
+ * `FAIL <check> <CODE> [nodes]` in plain mode, the same with symbols in color.
+ */
+export function managerText(
+  report: ManagerReport,
+  options: OutputOptions = {}
+): string {
+  const c = new Chalk({ level: options.color ? 1 : 0 });
+  const mark = (status: Status | null): string =>
+    status === null
+      ? c.dim('·')
+      : paint(c, status, options.color ? SYMBOL[status] : WORD[status]);
+  const all = [
+    ...report.checks.map((check) => ({ node: 'fleet', check })),
+    ...report.nodes.flatMap((node) =>
+      node.checks.map((check) => ({ node: node.node, check }))
+    ),
+  ];
+  const counts = count(all.map((entry) => entry.check));
+  const failing = all.some((entry) => blocking(entry.check));
+  const lines: string[] = [header(c, report, failing, options)];
+  lines.push('', ...grid(c, report.nodes, mark, options.color === true));
+  lines.push('', totals(c, counts, options.color === true));
+  for (const status of ['fail', 'warn', 'skip'] as const) {
+    const problems = groupProblems(
+      all.filter((entry) => entry.check.status === status)
+    );
+    if (!problems.length) continue;
+    lines.push('', c.bold(SECTION[status]));
+    for (const problem of problems)
+      lines.push(...problemLines(c, problem, options.color === true));
+  }
+  lines.push('');
+  if (report.dashboardUrl)
+    lines.push(`${c.dim('dashboard')}  ${c.cyan(report.dashboardUrl)}`);
+  if (options.reportPath)
+    lines.push(
+      `${c.dim('report   ')}  ${options.reportPath} ${c.dim('(full JSON; or rerun with --json)')}`
+    );
+  if (report.candidates)
+    lines.push(
+      `${c.dim('tailnet  ')}  ${report.candidates} unmanaged device(s) seen; discovery never enrolls them`
+    );
+  if (!report.applied) lines.push(c.dim('No node deployment was performed.'));
+  return stableUrls(lines.join('\n'));
+}
+
+const TAILNET_URL =
+  /\bhttps?:\/\/[a-z0-9.-]+\.ts\.net(?::\d+)?(?:\/[^\s'",;)]*[^\s'",;).])?/gi;
+
+/** Printed output names only stable public URLs. A tailnet
+ * MagicDNS address in any summary or remedy is replaced, never shown.
+ */
+export function stableUrls(text: string): string {
+  return text.replace(TAILNET_URL, 'its private tailnet address');
+}
+
+const SECTION: Record<'fail' | 'warn' | 'skip', string> = {
+  fail: 'Failing — fix these',
+  warn: 'Warnings — outside the fleet; sync still succeeds',
+  skip: 'Not observable on this hardware',
+};
+
+function header(
+  c: ChalkInstance,
+  report: ManagerReport,
+  failing: boolean,
+  options: OutputOptions
+): string {
+  const verdict = failing
+    ? c.red.bold(`${options.color ? '✖ ' : ''}ACTION REQUIRED`)
+    : report.applied
+      ? c.green.bold(`${options.color ? '✔ ' : ''}CONVERGED`)
+      : c.green.bold(`${options.color ? '✔ ' : ''}PREVIEW OK`);
+  const parts = [
+    c.bold('fleet-sync'),
+    verdict,
+    `${c.dim('revision')} ${(report.desiredRevision ?? 'none').slice(0, 12)}`,
+  ];
+  if (options.durationMs !== undefined)
+    parts.push(`${c.dim('in')} ${duration(options.durationMs)}`);
+  return parts.join('  ');
+}
+
+function grid(
+  c: ChalkInstance,
+  nodes: ManagerNode[],
+  mark: (status: Status | null) => string,
+  color: boolean
+): string[] {
+  const width = Math.max(4, ...nodes.map((node) => node.node.length));
+  const cell = (text: string, visible: number, size: number) =>
+    text + ' '.repeat(Math.max(1, size - visible + 1));
+  const head = [
+    'NODE'.padEnd(width + 1),
+    'ROLE'.padEnd(8),
+    ...COLUMNS.map((column) => column.title.padEnd(6)),
+    'RESULT',
+  ].join(' ');
+  const rows = nodes.map((node) => {
+    const statuses = COLUMNS.map((column) =>
+      worst(node.checks.filter((check) => column.matches(check.id)))
+    );
+    const failing = node.checks.filter(blocking).length;
+    const result =
+      node.reachability !== 'online' && node.role !== 'manager'
+        ? c.red(node.reachability.toUpperCase())
+        : failing
+          ? c.red(`${failing} failing`)
+          : node.appliedRevision
+            ? c.green('ready')
+            : c.yellow('not applied');
+    return [
+      cell(c.bold(node.node), node.node.length, width),
+      cell(c.dim(node.role ?? 'worker'), (node.role ?? 'worker').length, 7),
+      ...statuses.map((status) =>
+        cell(
+          mark(status),
+          status === null ? 1 : color ? 1 : WORD[status].length,
+          5
+        )
+      ),
+      result,
+    ].join(' ');
+  });
+  return [c.dim(head), ...rows];
+}
+
+interface Problem {
+  check: Check;
+  nodes: string[];
+}
+
+function groupProblems(entries: { node: string; check: Check }[]): Problem[] {
+  const groups = new Map<string, Problem>();
+  for (const { node, check } of entries) {
+    const key = [
+      check.id,
+      check.status,
+      check.summary,
+      check.remedy?.action ?? '',
+    ].join('\u0000');
+    const group = groups.get(key);
+    if (group) group.nodes.push(node);
+    else groups.set(key, { check, nodes: [node] });
+  }
+  return [...groups.values()];
+}
+
+function problemLines(
+  c: ChalkInstance,
+  { check, nodes }: Problem,
+  color: boolean
+): string[] {
+  const { code, detail } = splitSummary(check.summary);
+  const lead = paint(
+    c,
+    check.status,
+    color ? SYMBOL[check.status] : WORD[check.status]
+  );
+  const lines = [
+    `${lead} ${c.bold(check.id)}  ${code ? paint(c, check.status, code) : ''}${code && detail ? ' ' : ''}${detail}`.trimEnd(),
+    `    ${c.dim('on')} ${nodes.join(', ')}`,
+  ];
+  if (check.remedy?.action)
+    lines.push(`    ${c.cyan('→')} ${check.remedy.action}`);
+  if (check.remedy?.command)
+    lines.push(`    ${c.cyan('$')} ${check.remedy.command}`);
+  return lines;
+}
+
+/** Summaries start with an UPPER_SNAKE code when there is one. */
+export function splitSummary(summary: string): {
+  code: string;
+  detail: string;
+} {
+  const match = /^([A-Z][A-Z0-9_]{2,})(?:\s+(.*))?$/s.exec(summary.trim());
+  return match
+    ? { code: match[1]!, detail: match[2] ?? '' }
+    : { code: '', detail: summary };
+}
+
+function totals(
+  c: ChalkInstance,
+  counts: Record<Status, number>,
+  color: boolean
+): string {
+  return ORDER.map((status) => {
+    const text = `${color ? `${SYMBOL[status]} ` : ''}${counts[status]} ${LABEL[status]}`;
+    // A zero count is not a signal: never show red/yellow for "0 failing".
+    return counts[status] ? paint(c, status, text) : c.dim(text);
+  }).join(c.dim('   '));
+}
+
+const LABEL: Record<Status, string> = {
+  fail: 'failing',
+  warn: 'warnings',
+  skip: 'skipped',
+  pass: 'passing',
+};
+
+function count(checks: Check[]): Record<Status, number> {
+  const counts: Record<Status, number> = { pass: 0, warn: 0, fail: 0, skip: 0 };
+  for (const check of checks)
+    counts[
+      blocking(check) || check.status !== 'fail' ? check.status : 'warn'
+    ]++;
+  return counts;
+}
+
+function worst(checks: Check[]): Status | null {
+  if (!checks.length) return null;
+  if (checks.some(blocking)) return 'fail';
+  if (
+    checks.some((check) => check.status !== 'pass' && check.status !== 'skip')
+  )
+    return 'warn';
+  return checks.every((check) => check.status === 'skip') ? 'skip' : 'pass';
+}
+
+function paint(c: ChalkInstance, status: Status, text: string): string {
+  return status === 'fail'
+    ? c.red(text)
+    : status === 'warn'
+      ? c.yellow(text)
+      : status === 'skip'
+        ? c.dim(text)
+        : c.green(text);
+}
+
+export function duration(ms: number): string {
+  const seconds = Math.round(ms / 1000);
+  if (seconds < 60) return `${seconds}s`;
+  return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, '0')}s`;
+}

package/src/cli/fleet-sync-privilege.ts

@@ -0,0 +1,27 @@
+import type { Check, FleetNode, Runner } from '../types';
+import { sshArgs, sshFailure } from '../ssh';
+
+/** Initial enrollment must establish sudo once. Routine convergence never asks for passwords. */
+export async function authorizePrivilege(
+  node: FleetNode,
+  _script: string | undefined,
+  run: Runner
+): Promise<Check[]> {
+  const existing = await run([...sshArgs(node), 'sudo -n true'], {
+    timeoutMs: 15_000,
+  });
+  if (existing.ok) return [];
+  if (existing.reason !== 'exit' || !/sudo:/i.test(existing.stderr))
+    return [sshFailure(existing)];
+  return [
+    {
+      id: 'bootstrap:privilege',
+      status: 'fail',
+      summary: 'SUDO_AUTHORIZATION_REQUIRED',
+      remedy: {
+        action:
+          'Use the source-owned ansible/ubuntu/authorize-sudo.sh once during initial administrator enrollment. Runtime convergence requires sudo -n and never prompts for or stores a password.',
+      },
+    },
+  ];
+}

package/src/cli/fleet-sync-progress.ts

@@ -0,0 +1,109 @@
+import { Chalk } from 'chalk';
+import ora, { type Ora } from 'ora';
+import type { ManagerNode } from '../manager-sync';
+import { blocking } from '../types';
+import { duration } from './fleet-sync-output';
+
+/** Live progress on stderr. A terminal gets one spinner with an elapsed timer
+ * and a persisted line per finished step; anything else (pipes, CI, agents)
+ * gets timestamped lines plus a heartbeat each minute, never color.
+ */
+export interface SyncProgress {
+  step(node: string, phase: string): void;
+  finished(node: ManagerNode): void;
+  stop(): void;
+}
+
+export function syncProgress(
+  enabled: boolean,
+  stream: NodeJS.WriteStream = process.stderr
+): SyncProgress {
+  if (!enabled) return { step() {}, finished() {}, stop() {} };
+  const live =
+    Boolean(stream.isTTY) &&
+    process.env['NO_COLOR'] === undefined &&
+    process.env['TERM'] !== 'dumb' &&
+    !process.env['CI'];
+  return live ? spinnerProgress(stream) : lineProgress(stream);
+}
+
+function spinnerProgress(stream: NodeJS.WriteStream): SyncProgress {
+  const c = new Chalk({ level: 1 });
+  let spinner: Ora | null = null;
+  let label = '';
+  let started = 0;
+  let timer: ReturnType<typeof setInterval> | null = null;
+  const text = () => `${label}  ${c.dim(duration(Date.now() - started))}`;
+  const settle = () => {
+    if (timer) clearInterval(timer);
+    timer = null;
+    if (spinner?.isSpinning)
+      spinner.stopAndPersist({
+        symbol: c.dim('›'),
+        text: c.dim(`${label}  ${duration(Date.now() - started)}`),
+      });
+    spinner = null;
+  };
+  return {
+    step(node, phase) {
+      settle();
+      label = `${c.bold(node)}  ${phase}`;
+      started = Date.now();
+      spinner = ora({ text: text(), stream, discardStdin: false }).start();
+      timer = setInterval(() => {
+        if (spinner) spinner.text = text();
+      }, 1000);
+    },
+    finished(node) {
+      settle();
+      const failing = node.checks.filter(blocking).length;
+      const warnings = node.checks.filter((c) => c.status === 'warn').length;
+      const detail = [
+        failing ? c.red(`${failing} failing`) : '',
+        warnings ? c.yellow(`${warnings} warnings`) : '',
+      ]
+        .filter(Boolean)
+        .join(c.dim(', '));
+      const symbol = failing
+        ? c.red('✖')
+        : warnings
+          ? c.yellow('▲')
+          : c.green('✔');
+      stream.write(
+        `${symbol} ${c.bold(node.node)}  ${detail || c.green('ready')}  ${c.dim(duration(node.durationMs))}\n`
+      );
+    },
+    stop: settle,
+  };
+}
+
+function lineProgress(stream: NodeJS.WriteStream): SyncProgress {
+  let timer: ReturnType<typeof setInterval> | null = null;
+  const stamp = () => new Date().toISOString().slice(11, 19);
+  const clear = () => {
+    if (timer) clearInterval(timer);
+    timer = null;
+  };
+  return {
+    step(node, phase) {
+      clear();
+      const started = Date.now();
+      stream.write(`${stamp()} ${node} ${phase}\n`);
+      timer = setInterval(
+        () =>
+          stream.write(
+            `${stamp()} ${node} still ${phase.toLowerCase()} (${duration(Date.now() - started)})\n`
+          ),
+        60_000
+      );
+    },
+    finished(node) {
+      clear();
+      const failing = node.checks.filter(blocking).length;
+      stream.write(
+        `${stamp()} ${node.node} ${failing ? `FAIL ${failing} failing` : 'DONE'} (${duration(node.durationMs)})\n`
+      );
+    },
+    stop: clear,
+  };
+}