@crvouga/dev-fleet 0.3.0 → 0.4.0

package/src/git-identity.ts

@@ -0,0 +1,160 @@
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import { runAnsible } from './platform/ansible';
+import type { GitIdentity } from './definition';
+import type { Check, Runner } from './types';
+
+/** `ansible/github/git.yml` writes the declared identity and the GitHub
+ * credential helper in the node's own Git configuration; the two checks
+ * below measure them afterwards. The same on the manager and every worker.
+ */
+export async function convergeGitIdentity(
+  root: string,
+  git: GitIdentity | undefined,
+  run: Runner,
+  home: string = homedir()
+): Promise<Check[]> {
+  if (!git) return [];
+  const result = await runAnsible(
+    run,
+    resolve(root, 'ansible/github/git.yml'),
+    {
+      cwd: resolve(root, 'ansible/github'),
+      timeoutMs: 120_000,
+      input: JSON.stringify({
+        git_name: git.name,
+        git_email: git.email,
+        git_home: home,
+      }),
+    }
+  );
+  return [
+    {
+      id: 'git:configuration',
+      status: result.ok ? 'pass' : 'fail',
+      summary: result.ok
+        ? 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
+        : 'GIT_CONFIGURATION_INCOMPLETE',
+      ...(result.ok
+        ? {}
+        : {
+            remedy: {
+              action:
+                'Inspect ~/.config/git/config and ~/.gitconfig on this node: a ~/.gitconfig that sets user.name, user.email or a GitHub credential helper wins over the fleet. Remove those lines and rerun fleet-sync.',
+            },
+          }),
+    },
+  ];
+}
+
+/** Git with no prompt: a missing credential fails instead of waiting. */
+const QUIET = {
+  GIT_TERMINAL_PROMPT: '0',
+  GCM_INTERACTIVE: 'never',
+  GIT_ASKPASS: '',
+  SSH_ASKPASS: '',
+};
+
+async function effective(run: Runner, key: string): Promise<string> {
+  const result = await run(['git', 'config', '--global', '--get-all', key], {
+    timeoutMs: 10_000,
+  });
+  return result.ok ? (result.stdout.trim().split('\n').pop() ?? '') : '';
+}
+
+/** Commits carry the declared author: the effective `user.name` and
+ * `user.email` outside any repository are the fleet's.
+ */
+export async function gitIdentityCheck(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check> {
+  const id = 'worker:git-identity';
+  if (!git)
+    return {
+      id,
+      status: 'warn',
+      summary: 'GIT_IDENTITY_UNDECLARED',
+      remedy: {
+        action:
+          'Declare git.name and git.email in fleet.json and rerun fleet-sync; until then commits on this node carry whatever author it happens to have.',
+      },
+    };
+  const name = await effective(run, 'user.name');
+  const email = await effective(run, 'user.email');
+  if (name === git.name && email === git.email)
+    return {
+      id,
+      status: 'pass',
+      summary: `Commits on this node are authored as ${git.name} <${git.email}>.`,
+    };
+  return {
+    id,
+    status: 'fail',
+    summary: !name || !email ? 'GIT_IDENTITY_MISSING' : 'GIT_IDENTITY_MISMATCH',
+    remedy: {
+      action:
+        'Rerun fleet-sync: it writes the identity in ~/.config/git/config. A ~/.gitconfig that sets another user.name or user.email wins over it; remove those lines.',
+    },
+  };
+}
+
+/** A push is authorized through git itself, never through gh: Git's
+ * credential helper must answer for github.com, and the declared repository
+ * must answer an authenticated listing. The credential is read and dropped;
+ * it is never kept, printed or reported.
+ */
+export async function gitPushAuthCheck(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check> {
+  const id = 'worker:git-push-auth';
+  const fill = await run(['git', 'credential', 'fill'], {
+    input: 'protocol=https\nhost=github.com\n\n',
+    env: QUIET,
+    timeoutMs: 20_000,
+  });
+  if (!fill.ok || !/^password=./m.test(fill.stdout))
+    return {
+      id,
+      status: 'fail',
+      summary: 'GIT_CREDENTIAL_HELPER_MISSING',
+      remedy: {
+        action:
+          'Git has no credential for github.com, so an agent cannot push over HTTPS. Rerun fleet-sync, which declares gh as the credential helper; if gh is signed out, the worker signs in again from its own Vault session (auth:github).',
+      },
+    };
+  if (git?.verifyRepository) {
+    const listed = await run(
+      ['git', 'ls-remote', '--heads', git.verifyRepository],
+      { env: QUIET, timeoutMs: 30_000 }
+    );
+    if (!listed.ok)
+      return {
+        id,
+        status: 'fail',
+        summary:
+          listed.reason === 'timeout'
+            ? 'GIT_REMOTE_UNREACHABLE'
+            : 'GIT_REMOTE_REFUSED',
+        remedy: {
+          action:
+            "The declared repository (git.verifyRepository) did not answer git with this node's credentials. Check that the fleet's GitHub account can push to it and that github.com is reachable.",
+        },
+      };
+  }
+  return {
+    id,
+    status: 'pass',
+    summary: git?.verifyRepository
+      ? "Git's credential helper answers for github.com and the declared repository answers through git."
+      : "Git's credential helper answers for github.com; declare git.verifyRepository to prove a repository too.",
+  };
+}
+
+export async function gitChecks(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check[]> {
+  return [await gitIdentityCheck(git, run), await gitPushAuthCheck(git, run)];
+}

package/src/machine-health.ts

@@ -1,3 +1,4 @@
+import policy from '../ansible/ubuntu/policy.json';
 import type { DoctorCheck } from './doctor-check';
 
 /** Machine health sample parsed from `net/remote-probe.sh` output. */
@@ -7,6 +8,14 @@ export type HealthSample = {
   readonly memFreePct?: number;
   readonly swapPct?: number;
   readonly diskMb?: number;
+  /** Free inodes on the home filesystem, as a percentage of all of them. */
+  readonly inodesFreePct?: number;
+  /** 1 when the clock is synchronised to a time source, 0 when not. */
+  readonly clockSynced?: number;
+  /** 1 when a time service is enabled, 0 when not. */
+  readonly timeService?: number;
+  /** The machine's clock, in seconds since the epoch. */
+  readonly epoch?: number;
 };
 
 const PROBE_KEYS = {
@@ -15,6 +24,10 @@ const PROBE_KEYS = {
   memfree: 'memFreePct',
   swap: 'swapPct',
   diskmb: 'diskMb',
+  inodefree: 'inodesFreePct',
+  ntpsync: 'clockSynced',
+  ntp: 'timeService',
+  epoch: 'epoch',
 } as const;
 
 function isProbeKey(key: string): key is keyof typeof PROBE_KEYS {
@@ -33,7 +46,12 @@ export function parseProbe(output: string): HealthSample {
   return sample;
 }
 
-const LABELS = { load: 'CPU load', memory: 'Memory', disk: 'Disk space' };
+const LABELS = {
+  load: 'CPU load',
+  memory: 'Memory',
+  disk: 'Disk space',
+  inodes: 'Free inodes',
+};
 
 function check(
   id: keyof typeof LABELS,
@@ -88,8 +106,37 @@ export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
   return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
 }
 
+/** Free inodes, with the floors `policy.json` gives the disk guard: a
+ * filesystem with free bytes and no free inodes fails every write.
+ */
+export function inodesCheck(
+  inodesFreePct: number | undefined,
+  floors = INODE_FLOORS
+): DoctorCheck | null {
+  if (inodesFreePct === undefined) return null;
+  const what = `${inodesFreePct}% of inodes free`;
+  if (inodesFreePct < floors.failPercent)
+    return check(
+      'inodes',
+      'fail',
+      `INODES_EXHAUSTED, ${what}. Every new file fails although bytes are free.`,
+      'Remove idle dependency trees and worktrees: the hourly disk collector does so under pressure, or delete them by hand.'
+    );
+  if (inodesFreePct < floors.warnPercent)
+    return check('inodes', 'warn', `low, ${what}`);
+  return check('inodes', 'pass', what);
+}
+
+const INODE_FLOORS = {
+  warnPercent: policy.resourceSafety.inodeWarnFreePercent,
+  failPercent: policy.resourceSafety.inodeFailFreePercent,
+};
+
 export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
-  return [loadCheck(s, alias), memoryCheck(s), diskCheck(s.diskMb)].filter(
-    (c): c is DoctorCheck => c !== null
-  );
+  return [
+    loadCheck(s, alias),
+    memoryCheck(s),
+    diskCheck(s.diskMb),
+    inodesCheck(s.inodesFreePct),
+  ].filter((c): c is DoctorCheck => c !== null);
 }

package/src/manager-health.ts

@@ -0,0 +1,205 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import macos from '../ansible/macos/policy.json';
+import type { Check, Runner } from './types';
+
+/** Read-only measurements of the manager's operating system and disk, from
+ * the floors `ansible/macos/policy.json` declares. Nothing a command printed
+ * is reported: only what was concluded from it.
+ */
+export async function managerHealth(
+  run: Runner,
+  home = homedir()
+): Promise<Check[]> {
+  return [
+    await managerRelease(run),
+    await managerUpdates(run),
+    await managerDisk(run),
+    await containerWatchdog(run, home),
+  ];
+}
+
+/** The manager runs at least the macOS release the policy names. */
+export async function managerRelease(run: Runner): Promise<Check> {
+  const id = 'manager:release';
+  const result = await run(['sw_vers', '-productVersion']);
+  const major = /^(\d+)(?:\.\d+)*$/.exec(result.stdout.trim())?.[1];
+  if (!result.ok || major === undefined)
+    return { id, status: 'skip', summary: 'MANAGER_RELEASE_UNOBSERVED' };
+  if (Number(major) >= macos.minimumMajorVersion)
+    return {
+      id,
+      status: 'pass',
+      summary: `The manager runs macOS ${major}, at least the required ${macos.minimumMajorVersion}.`,
+    };
+  return {
+    id,
+    status: 'fail',
+    summary: `MANAGER_RELEASE_UNSUPPORTED macOS ${major}, below ${macos.minimumMajorVersion}`,
+    remedy: {
+      action: `Update the manager to macOS ${macos.minimumMajorVersion} or later in System Settings > General > Software Update, then rerun fleet-sync.`,
+    },
+  };
+}
+
+/** Pending macOS updates warn, and so does one that waits for a restart.
+ * `--no-scan` reads the list macOS already has: nothing is downloaded.
+ */
+export async function managerUpdates(run: Runner): Promise<Check> {
+  const id = 'manager:updates';
+  const result = await run(['softwareupdate', '--list', '--no-scan'], {
+    timeoutMs: 60_000,
+  });
+  const output = `${result.stdout}\n${result.stderr}`;
+  if (!result.ok)
+    return { id, status: 'skip', summary: 'MANAGER_UPDATES_UNOBSERVED' };
+  const pending = (output.match(/^\s*\* Label:/gm) ?? []).length;
+  const restart = /Action:\s*restart/i.test(output);
+  if (pending === 0)
+    return { id, status: 'pass', summary: 'No macOS update is pending.' };
+  return {
+    id,
+    status: 'warn',
+    summary: restart
+      ? `MANAGER_RESTART_PENDING ${pending} updates, at least one waits for a restart`
+      : `MANAGER_UPDATES_PENDING ${pending} updates`,
+    remedy: {
+      action:
+        'Install the pending updates in System Settings > General > Software Update at a time no worker depends on the manager, then rerun fleet-sync.',
+    },
+  };
+}
+
+const MIB = 1024;
+const GIB = 1024 ** 3;
+
+/** Free space on the data volume, and the space Docker Desktop holds. */
+export async function managerDisk(run: Runner): Promise<Check> {
+  const id = 'manager:disk';
+  const df = await run(['df', '-Pk', '/System/Volumes/Data']);
+  const kib = Number(df.stdout.split('\n')[1]?.trim().split(/\s+/)[3]);
+  if (!df.ok || !Number.isFinite(kib))
+    return { id, status: 'skip', summary: 'MANAGER_DISK_UNOBSERVED' };
+  const free = Math.floor(kib / MIB);
+  const docker = await dockerBytes(run);
+  const dockerGiB = docker === null ? null : docker / GIB;
+  const held =
+    dockerGiB === null ? '' : `; Docker holds ${dockerGiB.toFixed(1)} GiB`;
+  const remedy = {
+    action:
+      'Free space on the manager: docker system prune removes unused images and build cache; empty large caches and downloads.',
+  };
+  if (free < macos.disk.failFreeMiB)
+    return {
+      id,
+      status: 'fail',
+      summary: `MANAGER_DISK_FULL ${free} MiB free${held}`,
+      remedy,
+    };
+  if (free < macos.disk.warnFreeMiB)
+    return {
+      id,
+      status: 'warn',
+      summary: `MANAGER_DISK_LOW ${free} MiB free${held}`,
+      remedy,
+    };
+  if (dockerGiB !== null && dockerGiB > macos.disk.dockerWarnGiB)
+    return {
+      id,
+      status: 'warn',
+      summary: `MANAGER_DOCKER_DISK_LARGE ${free} MiB free${held}`,
+      remedy,
+    };
+  return {
+    id,
+    status: 'pass',
+    summary: `${Math.floor(free / MIB)} GiB free on the data volume${held}.`,
+  };
+}
+
+const UNITS: Record<string, number> = {
+  B: 1,
+  kB: 1e3,
+  KB: 1e3,
+  MB: 1e6,
+  GB: 1e9,
+  TB: 1e12,
+};
+
+/** The bytes of images, containers, volumes and build cache, or null when
+ * Docker does not answer.
+ */
+async function dockerBytes(run: Runner): Promise<number | null> {
+  const result = await run(
+    ['docker', 'system', 'df', '--format', '{{.Size}}'],
+    { timeoutMs: 30_000 }
+  );
+  if (!result.ok) return null;
+  let total = 0;
+  for (const line of result.stdout.split('\n')) {
+    const match = /^([\d.]+)\s*([kKMGT]?B)$/.exec(line.trim());
+    if (match) total += Number(match[1]) * (UNITS[match[2]!] ?? 0);
+  }
+  return total;
+}
+
+/** What the container watchdog remembers (ansible/monitoring/container-watchdog.py). */
+type WatchdogMemory = Record<
+  string,
+  { restarts?: number[]; capped?: number | null }
+>;
+
+/** A container the watchdog stopped restarting is a restart storm: it fails,
+ * and one it restarted within the hour warns.
+ */
+export async function containerWatchdog(
+  run: Runner,
+  home: string,
+  now = Date.now()
+): Promise<Check> {
+  const id = 'manager:container-watchdog';
+  const result = await run([
+    'cat',
+    join(home, 'Library/Caches/dev-fleet/container-watchdog.json'),
+  ]);
+  let memory: WatchdogMemory;
+  try {
+    memory = JSON.parse(result.stdout) as WatchdogMemory;
+  } catch {
+    memory = {};
+  }
+  if (!result.ok || typeof memory !== 'object' || memory === null)
+    return { id, status: 'skip', summary: 'CONTAINER_WATCHDOG_UNOBSERVED' };
+  const hour = (time: unknown) =>
+    typeof time === 'number' && now / 1000 - time < 3600;
+  const names = Object.keys(memory).filter((name) =>
+    /^[A-Za-z0-9_.-]{1,128}$/.test(name)
+  );
+  const capped = names.filter((name) => hour(memory[name]?.capped));
+  const restarted = names.filter((name) =>
+    (memory[name]?.restarts ?? []).some(hour)
+  );
+  const remedy = {
+    action:
+      'Read the container watchdog log (~/Library/Application Support/dev-fleet/container-watchdog/watchdog.log) and the container logs, fix the cause, then docker restart the container.',
+  };
+  if (capped.length)
+    return {
+      id,
+      status: 'fail',
+      summary: `CONTAINER_RESTART_STORM ${capped.join(' ')}`,
+      remedy,
+    };
+  if (restarted.length)
+    return {
+      id,
+      status: 'warn',
+      summary: `CONTAINER_RESTARTED_UNHEALTHY ${restarted.join(' ')}`,
+      remedy,
+    };
+  return {
+    id,
+    status: 'pass',
+    summary: 'No container stayed unhealthy in the last hour.',
+  };
+}

package/src/manager-sync.ts

@@ -6,7 +6,11 @@ import {
   type NodeReport,
   type Runner,
 } from './types';
-import { discoverTailnet, type TailnetComputer } from './tailscale';
+import {
+  discoverTailnet,
+  keyExpiryCheck,
+  type TailnetComputer,
+} from './tailscale';
 import { matchComputers } from './tailnet-matching';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
@@ -102,7 +106,12 @@ export async function synchronizeManager(
     ok: false,
     applied: false,
     desiredRevision: input.revision,
-    checks: tailnet.checks,
+    checks: [
+      ...tailnet.checks,
+      ...(tailnet.computers.some((peer) => peer.self)
+        ? [keyExpiryCheck(tailnet.computers.find((peer) => peer.self))]
+        : []),
+    ],
     // Printed for people: the stable public name, never a tailnet address.
     dashboardUrl: publicHubUrl(),
     nodes: [],
@@ -322,6 +331,7 @@ async function inspectManagerNode(
   else {
     checks.push(...(await inspectBootstrap(node, run, machine.os)));
     checks.push(...identityChecks(node, peer));
+    if (peer) checks.push(keyExpiryCheck(peer));
   }
   if (!peer || !peer.online)
     checks.push({

package/src/monitoring/hub-backup.ts

@@ -1,4 +1,5 @@
 import { record } from '../config';
+import type { HubBackups } from '../definition';
 import type { Check } from '../types';
 import {
   LOCAL_HUB,
@@ -10,6 +11,10 @@ import {
 
 /** A scheduled backup older than this means the schedule is not running. */
 export const BACKUP_MAX_AGE_MS = 48 * 60 * 60 * 1000;
+/** Declared backups that have not reached off-machine storage for this long
+ * fail: the grace covers a fresh hub and a storage outage of a few days.
+ */
+export const BACKUP_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
 
 function backupCheck(
   status: Check['status'],
@@ -24,21 +29,26 @@ function backupCheck(
   };
 }
 
-const SETTINGS =
-  'Open the hub as its superuser (/_/ on the manager), Settings > Backups.';
+const UNDECLARED =
+  'Declare services.monitoring.backups in fleet.json (a schedule, how many to keep and S3-compatible storage off the manager) and the secrets monitoring-backup-access-key and monitoring-backup-secret-key; fleet-sync then sets them through the hub.';
+const DECLARED =
+  "fleet-sync sets the backups declared under services.monitoring.backups. Check that the secrets monitoring-backup-access-key and monitoring-backup-secret-key are readable on the manager and that the manager's Vault session is logged in, then rerun fleet-sync.";
 
-/** Reads, never changes, the hub's own backup settings and its list of
- * backups through the stock PocketBase API of the pinned release (superuser
- * only: GET /api/settings and GET /api/backups). Backups are configured in
- * the hub's UI; this says whether they are scheduled, kept off the manager
- * and recent. Storage names, keys and endpoints are never reported.
+/** Reads the hub's own backup settings and its list of backups through the
+ * stock PocketBase API of the pinned release (superuser only: GET
+ * /api/settings and GET /api/backups): whether they are scheduled, kept off
+ * the manager and recent. fleet-sync sets them from the fleet configuration
+ * (`hub-darwin.yml`); with them declared, anything but that is a failure, a
+ * stale backup after a grace period. Storage names, keys and endpoints are
+ * never reported.
  */
 export async function verifyHubBackups(
   login: HubLogin | null,
   fetcher: typeof fetch = fetch,
   base: string = LOCAL_HUB,
-  now: number = Date.now()
+  options: { now?: number; declared?: HubBackups | undefined } = {}
 ): Promise<Check> {
+  const SETTINGS = options.declared ? DECLARED : UNDECLARED;
   if (!login)
     return backupCheck(
       'skip',
@@ -69,31 +79,67 @@ export async function verifyHubBackups(
       'BESZEL_BACKUP_UNVERIFIED',
       `The hub answered in a form this release does not expect. ${SETTINGS}`
     );
+  return judgeBackups(
+    policy,
+    newestBackup(backups),
+    options.now ?? Date.now(),
+    options.declared
+  );
+}
+
+/** Undeclared backups only warn; declared ones fail when they are not what
+ * the fleet configuration says, or have not reached storage for longer than
+ * the grace period.
+ */
+function judgeBackups(
+  policy: { cron: string; keep: number; offHost: boolean },
+  newest: number | null,
+  now: number,
+  declared: HubBackups | undefined
+): Check {
+  const SETTINGS = declared ? DECLARED : UNDECLARED;
+  const missing: Check['status'] = declared ? 'fail' : 'warn';
   if (!policy.cron)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_UNSCHEDULED',
-      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS} Set a schedule and how many to keep.`
+      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS}`
     );
   if (!policy.offHost)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_OFFHOST_REQUIRED',
-      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Store them in existing encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
+      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Use encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
     );
-  const newest = newestBackup(backups);
-  if (newest === null || now - newest > BACKUP_MAX_AGE_MS)
+  if (
+    declared &&
+    (policy.cron !== declared.schedule || policy.keep !== declared.keep)
+  )
     return backupCheck(
-      'warn',
+      'fail',
+      'BESZEL_BACKUP_NOT_CONVERGED',
+      `The hub's backup schedule is not the one the fleet configuration declares. ${SETTINGS}`
+    );
+  const age = newest === null ? null : now - newest;
+  if (age === null || age > BACKUP_MAX_AGE_MS)
+    return backupCheck(
+      staleStatus(age, declared !== undefined),
       'BESZEL_BACKUP_STALE',
-      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. ${SETTINGS} Check the schedule and the storage's credentials, then create a backup by hand.`
+      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. Check the storage's credentials and that the hub can reach it, then create a backup by hand (the hub's /_/ page, Settings > Backups). ${SETTINGS}`
     );
   return backupCheck(
     'pass',
-    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(newest).toISOString()}.`
+    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(now - age).toISOString()}.`
   );
 }
 
+/** A fresh hub has no backup yet; declared backups that stopped reaching
+ * storage fail once the grace period is over.
+ */
+function staleStatus(age: number | null, declared: boolean): Check['status'] {
+  return declared && age !== null && age > BACKUP_GRACE_MS ? 'fail' : 'warn';
+}
+
 function backupPolicy(
   settings: unknown
 ): { cron: string; keep: number; offHost: boolean } | null {