@crvouga/dev-fleet 0.3.0 → 0.5.0

package/src/manager-sync.ts

@@ -6,16 +6,19 @@ import {
   type NodeReport,
   type Runner,
 } from './types';
-import { discoverTailnet, type TailnetComputer } from './tailscale';
+import {
+  discoverTailnet,
+  keyExpiryCheck,
+  type TailnetComputer,
+} from './tailscale';
 import { matchComputers } from './tailnet-matching';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
 import { publicHubUrl } from './monitoring/beszel';
 import {
-  convergeable,
+  evaluateEnvironment,
   evaluatePlatform,
   PLATFORM_PROBE,
-  type MachinePlatform,
 } from './machine/platform';
 
 export interface ManagerNode extends NodeReport {
@@ -47,14 +50,14 @@ export type DeployNode = (
 /** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git. */
 export async function inspectBootstrap(
   node: FleetNode,
-  run: Runner,
-  os: MachinePlatform = 'linux'
+  run: Runner
 ): Promise<Check[]> {
   const result = await run([...sshArgs(node), PLATFORM_PROBE], {
     timeoutMs: 20_000,
   });
   if (!result.ok) return [sshFailure(result)];
-  const baseline = evaluatePlatform(os, result.stdout);
+  const baseline = evaluatePlatform(result.stdout);
+  const environment = evaluateEnvironment(result.stdout);
   return [
     {
       id: 'ssh:fresh-session',
@@ -71,6 +74,15 @@ export async function inspectBootstrap(
           'Review the release/architecture migration in source. No operating-system upgrade or downgrade is performed.',
       },
     },
+    {
+      id: environment.id,
+      status: environment.supported ? 'pass' : 'fail',
+      summary: environment.summary,
+      remedy: {
+        action:
+          'Run the worker on a physical or virtual machine of its own; docs/support-matrix.md lists what is supported.',
+      },
+    },
   ];
 }
 
@@ -102,7 +114,12 @@ export async function synchronizeManager(
     ok: false,
     applied: false,
     desiredRevision: input.revision,
-    checks: tailnet.checks,
+    checks: [
+      ...tailnet.checks,
+      ...(tailnet.computers.some((peer) => peer.self)
+        ? [keyExpiryCheck(tailnet.computers.find((peer) => peer.self))]
+        : []),
+    ],
     // Printed for people: the stable public name, never a tailnet address.
     dashboardUrl: publicHubUrl(),
     nodes: [],
@@ -115,7 +132,7 @@ export async function synchronizeManager(
   // worker is then verified against.
   if (managerRun) await managerRun;
   const workers = await Promise.all(
-    input.definition.workers.filter(convergeable).map((machine) => {
+    input.definition.workers.map((machine) => {
       input.progress?.(
         machine.id,
         'Inspecting enrolled identity and fresh SSH access'
@@ -320,8 +337,9 @@ async function inspectManagerNode(
       },
     });
   else {
-    checks.push(...(await inspectBootstrap(node, run, machine.os)));
+    checks.push(...(await inspectBootstrap(node, run)));
     checks.push(...identityChecks(node, peer));
+    if (peer) checks.push(keyExpiryCheck(peer));
   }
   if (!peer || !peer.online)
     checks.push({

package/src/monitoring/hub-backup.ts

@@ -1,4 +1,5 @@
 import { record } from '../config';
+import type { HubBackups } from '../definition';
 import type { Check } from '../types';
 import {
   LOCAL_HUB,
@@ -10,6 +11,10 @@ import {
 
 /** A scheduled backup older than this means the schedule is not running. */
 export const BACKUP_MAX_AGE_MS = 48 * 60 * 60 * 1000;
+/** Declared backups that have not reached off-machine storage for this long
+ * fail: the grace covers a fresh hub and a storage outage of a few days.
+ */
+export const BACKUP_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
 
 function backupCheck(
   status: Check['status'],
@@ -24,21 +29,26 @@ function backupCheck(
   };
 }
 
-const SETTINGS =
-  'Open the hub as its superuser (/_/ on the manager), Settings > Backups.';
+const UNDECLARED =
+  'Declare services.monitoring.backups in fleet.json (a schedule, how many to keep and S3-compatible storage off the manager) and the secrets monitoring-backup-access-key and monitoring-backup-secret-key; fleet-sync then sets them through the hub.';
+const DECLARED =
+  "fleet-sync sets the backups declared under services.monitoring.backups. Check that the secrets monitoring-backup-access-key and monitoring-backup-secret-key are readable on the manager and that the manager's Vault session is logged in, then rerun fleet-sync.";
 
-/** Reads, never changes, the hub's own backup settings and its list of
- * backups through the stock PocketBase API of the pinned release (superuser
- * only: GET /api/settings and GET /api/backups). Backups are configured in
- * the hub's UI; this says whether they are scheduled, kept off the manager
- * and recent. Storage names, keys and endpoints are never reported.
+/** Reads the hub's own backup settings and its list of backups through the
+ * stock PocketBase API of the pinned release (superuser only: GET
+ * /api/settings and GET /api/backups): whether they are scheduled, kept off
+ * the manager and recent. fleet-sync sets them from the fleet configuration
+ * (`hub-darwin.yml`); with them declared, anything but that is a failure, a
+ * stale backup after a grace period. Storage names, keys and endpoints are
+ * never reported.
  */
 export async function verifyHubBackups(
   login: HubLogin | null,
   fetcher: typeof fetch = fetch,
   base: string = LOCAL_HUB,
-  now: number = Date.now()
+  options: { now?: number; declared?: HubBackups | undefined } = {}
 ): Promise<Check> {
+  const SETTINGS = options.declared ? DECLARED : UNDECLARED;
   if (!login)
     return backupCheck(
       'skip',
@@ -69,31 +79,67 @@ export async function verifyHubBackups(
       'BESZEL_BACKUP_UNVERIFIED',
       `The hub answered in a form this release does not expect. ${SETTINGS}`
     );
+  return judgeBackups(
+    policy,
+    newestBackup(backups),
+    options.now ?? Date.now(),
+    options.declared
+  );
+}
+
+/** Undeclared backups only warn; declared ones fail when they are not what
+ * the fleet configuration says, or have not reached storage for longer than
+ * the grace period.
+ */
+function judgeBackups(
+  policy: { cron: string; keep: number; offHost: boolean },
+  newest: number | null,
+  now: number,
+  declared: HubBackups | undefined
+): Check {
+  const SETTINGS = declared ? DECLARED : UNDECLARED;
+  const missing: Check['status'] = declared ? 'fail' : 'warn';
   if (!policy.cron)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_UNSCHEDULED',
-      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS} Set a schedule and how many to keep.`
+      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS}`
     );
   if (!policy.offHost)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_OFFHOST_REQUIRED',
-      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Store them in existing encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
+      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Use encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
     );
-  const newest = newestBackup(backups);
-  if (newest === null || now - newest > BACKUP_MAX_AGE_MS)
+  if (
+    declared &&
+    (policy.cron !== declared.schedule || policy.keep !== declared.keep)
+  )
     return backupCheck(
-      'warn',
+      'fail',
+      'BESZEL_BACKUP_NOT_CONVERGED',
+      `The hub's backup schedule is not the one the fleet configuration declares. ${SETTINGS}`
+    );
+  const age = newest === null ? null : now - newest;
+  if (age === null || age > BACKUP_MAX_AGE_MS)
+    return backupCheck(
+      staleStatus(age, declared !== undefined),
       'BESZEL_BACKUP_STALE',
-      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. ${SETTINGS} Check the schedule and the storage's credentials, then create a backup by hand.`
+      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. Check the storage's credentials and that the hub can reach it, then create a backup by hand (the hub's /_/ page, Settings > Backups). ${SETTINGS}`
     );
   return backupCheck(
     'pass',
-    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(newest).toISOString()}.`
+    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(now - age).toISOString()}.`
   );
 }
 
+/** A fresh hub has no backup yet; declared backups that stopped reaching
+ * storage fail once the grace period is over.
+ */
+function staleStatus(age: number | null, declared: boolean): Check['status'] {
+  return declared && age !== null && age > BACKUP_GRACE_MS ? 'fail' : 'warn';
+}
+
 function backupPolicy(
   settings: unknown
 ): { cron: string; keep: number; offHost: boolean } | null {

package/src/monitoring/manager-hub.ts

@@ -8,6 +8,7 @@ import { publicViewerIdentity, fleetAccount } from './hub-access';
 import { ensurePublicViewer, type HubLogin } from './hub-viewer';
 import { verifyHubBackups } from './hub-backup';
 import type { FleetDefinition } from '../definition';
+import { secretVia } from '../fleet-config';
 import { hostId, hubUrl } from '../topology';
 import { publicHubUrl, privateHub } from './beszel';
 import type { Check, Runner } from '../types';
@@ -56,6 +57,9 @@ export interface ManagerHubOptions {
   receipts?: string;
   /** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
   viewer?: (login: HubLogin | null) => Promise<Check>;
+  /** The declared backups for the hub's playbook; defaults to the fleet
+   * configuration and its secrets. */
+  backupStorage?: () => Promise<Record<string, string | number | boolean>>;
   /** Reads the hub's backup settings; defaults to the loopback hub API. */
   backups?: (login: HubLogin | null) => Promise<Check>;
 }
@@ -78,13 +82,19 @@ export async function convergeManagerHub(
   const paths = managerHubPaths(options.home);
   const tunnel = await tunnelToken(run);
   const account = await (options.account ?? fleetAccount)();
+  const backups = await (
+    options.backupStorage ?? (() => backupVariables(definition, run))
+  )();
   const monitoring = resolve(root, 'ansible/monitoring');
   const result = await runAnsibleWithReceipt(
     run,
     'hub-darwin.yml',
     {
       cwd: monitoring,
-      input: JSON.stringify(hubVariables(hub, paths, tunnel, account)),
+      input: JSON.stringify({
+        ...hubVariables(hub, paths, tunnel, account),
+        ...backups,
+      }),
       timeoutMs: 300_000,
     },
     {
@@ -127,7 +137,13 @@ export async function convergeManagerHub(
     },
     publicEndpointCheck(tunnel !== null),
     await ensureViewer(options, account),
-    await (options.backups ?? verifyHubBackups)(account),
+    await (
+      options.backups ??
+      ((login) =>
+        verifyHubBackups(login, fetch, LOCAL_HUB, {
+          declared: definition.services.monitoring.backups,
+        }))
+    )(account),
   ];
 }
 
@@ -153,6 +169,32 @@ function hubVariables(
   };
 }
 
+/** The declared backup schedule and storage, with the storage's keys from
+ * the fleet's secrets; nothing when no backups are declared or a key is
+ * missing (beszel:backup then says so).
+ */
+export async function backupVariables(
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Record<string, string | number | boolean>> {
+  const declared = definition.services.monitoring.backups;
+  if (!declared) return {};
+  const secret = secretVia(run);
+  const accessKey = await secret('monitoring-backup-access-key');
+  const secretKey = await secret('monitoring-backup-secret-key');
+  if (!accessKey || !secretKey) return {};
+  return {
+    beszel_backup_schedule: declared.schedule,
+    beszel_backup_keep: declared.keep,
+    beszel_backup_bucket: declared.storage.bucket,
+    beszel_backup_endpoint: declared.storage.endpoint,
+    beszel_backup_region: declared.storage.region,
+    beszel_backup_force_path_style: declared.storage.forcePathStyle ?? false,
+    beszel_backup_access_key: accessKey,
+    beszel_backup_secret_key: secretKey,
+  };
+}
+
 function ensureViewer(
   options: ManagerHubOptions,
   account: HubLogin | null

package/src/network-definition.ts

@@ -8,7 +8,7 @@ export interface NetworkHost {
   sshPort: number;
   user: string;
   identityFile: string;
-  os: 'linux' | 'macOS' | 'windows';
+  os: 'linux';
   t3Port: number;
 }
 export function networkProjection(

package/src/node-converge-linux.ts

@@ -22,6 +22,8 @@ import {
 import { userInfo } from 'node:os';
 import { resolve } from 'node:path';
 import { auditUbuntu } from './platform/ubuntu-audit';
+import { convergeGitIdentity } from './git-identity';
+import { workerSelection } from '../bootstrap/worker-tools';
 import {
   applyAgentProfiles,
   attempt,
@@ -59,7 +61,7 @@ export async function convergeLinux(
       await applyTailscale(root, run),
     ]))
   );
-  const { baseline, verified } = await baselineStep(root, run);
+  const { baseline, verified } = await baselineStep(root, run, node);
   checks.push(...baseline, ...verified);
   checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
   checks.push(
@@ -67,6 +69,11 @@ export async function convergeLinux(
       configureWorkstation(root, run)
     ))
   );
+  checks.push(
+    ...(await attempt('git:configuration', () =>
+      convergeGitIdentity(root, fleet.git, run)
+    ))
+  );
   checks.push(
     ...(await attempt('configuration:apply', () =>
       applyAgentProfiles(root, node, revision, run)
@@ -110,7 +117,8 @@ export async function convergeLinux(
  */
 async function baselineStep(
   root: string,
-  run: Runner
+  run: Runner,
+  node: string
 ): Promise<{ baseline: Check[]; verified: Check[] }> {
   const measure = async () => [
     ...(await attempt('ubuntu:boot-policy', () =>
@@ -122,14 +130,14 @@ async function baselineStep(
   ];
   let cached = false;
   let baseline = await attempt('ubuntu:baseline', async () => {
-    const applied = await applyBaseline(root, run, false);
+    const applied = await applyBaseline(root, run, false, node);
     cached = applied.cached;
     return applied.checks;
   });
   let verified = await measure();
   if (cached && verified.some((check) => check.status === 'fail')) {
     baseline = await attempt('ubuntu:baseline', async () => {
-      return (await applyBaseline(root, run, true)).checks;
+      return (await applyBaseline(root, run, true, node)).checks;
     });
     verified = await measure();
   }
@@ -139,7 +147,8 @@ async function baselineStep(
 async function applyBaseline(
   root: string,
   run: Runner,
-  force: boolean
+  force: boolean,
+  node: string
 ): Promise<{ cached: boolean; checks: Check[] }> {
   const user = userInfo();
   const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -154,6 +163,8 @@ async function applyBaseline(
         development_user: user.username,
         development_uid: user.uid,
         development_home: user.homedir,
+        // The web profile brings headless Chromium (tasks/browser.yml).
+        worker_tool_profiles: workerSelection(fleet, node).profiles,
       }),
     },
     {

package/src/node-converge.ts

@@ -23,7 +23,7 @@ async function main(): Promise<Check[]> {
       summary: 'PLATFORM_UNSUPPORTED',
       remedy: {
         action:
-          'Workers are Ubuntu 26.04. The macOS manager converges itself locally in fleet-sync.',
+          'Workers run an Ubuntu release docs/support-matrix.md lists. The macOS manager converges itself locally in fleet-sync.',
       },
     },
   ];

package/src/notify.ts

@@ -0,0 +1,239 @@
+import { record } from './config';
+import type { SecretName, SecretsProvider } from './secrets';
+import type { Check, NodeReport } from './types';
+
+/** Where fleet-watch sends a notification. The address of each target is a
+ * secret, read by purpose: a webhook or ntfy URL often carries its own token.
+ */
+export const NOTIFY_TARGETS = {
+  webhook: 'notify-webhook-url',
+  ntfy: 'notify-ntfy-url',
+} as const satisfies Record<string, SecretName>;
+export type NotifyTarget = keyof typeof NOTIFY_TARGETS;
+
+export interface NotifyConfig {
+  targets: NotifyTarget[];
+  /** Notify again about a machine that stays unhealthy this often; 0 never. */
+  repeatHours: number;
+}
+
+const DEFAULT_REPEAT_HOURS = 24;
+
+/** The `notify` block of a fleet configuration, or null when it is invalid. */
+export function parseNotify(value: unknown): NotifyConfig | null {
+  if (!record(value)) return null;
+  if (
+    Object.keys(value).some((key) => !['targets', 'repeatHours'].includes(key))
+  )
+    return null;
+  const { targets, repeatHours = DEFAULT_REPEAT_HOURS } = value;
+  if (
+    !Array.isArray(targets) ||
+    targets.length === 0 ||
+    new Set(targets).size !== targets.length ||
+    !targets.every(
+      (target) =>
+        typeof target === 'string' && Object.hasOwn(NOTIFY_TARGETS, target)
+    ) ||
+    typeof repeatHours !== 'number' ||
+    !Number.isInteger(repeatHours) ||
+    repeatHours < 0 ||
+    repeatHours > 720
+  )
+    return null;
+  return { targets: targets as NotifyTarget[], repeatHours };
+}
+
+/** What the watcher remembers of each machine between two runs. */
+export interface WatchMemory {
+  version: 1;
+  /** When the watcher last ran: the heartbeat an off-machine check reads. */
+  heartbeatAt: string;
+  nodes: Record<
+    string,
+    { state: 'ready' | 'degraded'; since: string; notifiedAt: string | null }
+  >;
+}
+
+/** One machine as fleet-status saw it: the IDs of its failing checks. */
+export interface Observation {
+  node: string;
+  failing: string[];
+}
+
+export interface Notification {
+  event: 'degraded' | 'recovered' | 'still-degraded' | 'test';
+  node: string;
+  checks: string[];
+}
+
+/** Only what a report concluded: the required checks that failed. */
+export function observe(nodes: NodeReport[]): Observation[] {
+  return nodes.map((node) => ({
+    node: node.node,
+    failing: node.checks
+      .filter((check) => check.status === 'fail' && check.required !== false)
+      .map((check) => check.id),
+  }));
+}
+
+/** Notifications on transitions only (ready to degraded and back), and a
+ * reminder for a machine that stays degraded longer than the repeat interval.
+ * A machine never seen before counts as ready.
+ */
+export function transitions(
+  previous: WatchMemory | null,
+  observed: Observation[],
+  now: Date,
+  repeatHours: number
+): { notifications: Notification[]; memory: WatchMemory } {
+  const at = now.toISOString();
+  const notifications: Notification[] = [];
+  const nodes: WatchMemory['nodes'] = {};
+  for (const { node, failing } of observed) {
+    const before = previous?.nodes[node];
+    const state = failing.length ? 'degraded' : 'ready';
+    const checks = failing.filter(safe);
+    if (state === (before?.state ?? 'ready')) {
+      const due = state === 'degraded' && repeatDue(before, now, repeatHours);
+      if (due) notifications.push({ event: 'still-degraded', node, checks });
+      nodes[node] = before
+        ? { ...before, ...(due ? { notifiedAt: at } : {}) }
+        : { state, since: at, notifiedAt: null };
+      continue;
+    }
+    notifications.push({
+      event: state === 'degraded' ? 'degraded' : 'recovered',
+      node,
+      checks,
+    });
+    nodes[node] = { state, since: at, notifiedAt: at };
+  }
+  return { notifications, memory: { version: 1, heartbeatAt: at, nodes } };
+}
+
+function repeatDue(
+  before: WatchMemory['nodes'][string] | undefined,
+  now: Date,
+  repeatHours: number
+): boolean {
+  if (repeatHours <= 0 || !before?.notifiedAt) return false;
+  return (
+    now.getTime() - Date.parse(before.notifiedAt) >= repeatHours * 3_600_000
+  );
+}
+
+/** The memory stored last time, or null when there is none to trust. */
+export function parseMemory(text: string | null): WatchMemory | null {
+  if (!text) return null;
+  try {
+    const value: unknown = JSON.parse(text);
+    if (
+      !record(value) ||
+      value['version'] !== 1 ||
+      typeof value['heartbeatAt'] !== 'string' ||
+      !record(value['nodes'])
+    )
+      return null;
+    return value as unknown as WatchMemory;
+  } catch {
+    return null;
+  }
+}
+
+/** Node and check IDs are the only words a notification carries. */
+const safe = (id: string): boolean => /^[A-Za-z0-9:._@-]{1,128}$/.test(id);
+
+const WORDS: Record<Notification['event'], string> = {
+  degraded: 'is degraded',
+  recovered: 'is ready again',
+  'still-degraded': 'is still degraded',
+  test: 'sent a test notification',
+};
+
+export function notificationText(notification: Notification): string {
+  const node = safe(notification.node) ? notification.node : 'a machine';
+  const checks = notification.checks.filter(safe);
+  return `dev-fleet: ${node} ${WORDS[notification.event]}${
+    checks.length ? `. Failing checks: ${checks.join(', ')}` : ''
+  }`;
+}
+
+/** Sends one notification to every target. The result names the target and
+ * whether it took the notification: never its address or its answer.
+ */
+export async function deliver(
+  config: NotifyConfig,
+  notification: Notification,
+  secrets: SecretsProvider,
+  send: typeof fetch = fetch
+): Promise<Check[]> {
+  const text = notificationText(notification);
+  const checks: Check[] = [];
+  for (const target of config.targets) {
+    const id = `notify:${target}`;
+    const url = await secrets(NOTIFY_TARGETS[target]);
+    if (!url || !/^https?:\/\//.test(url)) {
+      checks.push({
+        id,
+        status: 'fail',
+        summary: 'NOTIFY_TARGET_UNAVAILABLE',
+        remedy: {
+          action: `Store the address of the ${target} target as the secret "${NOTIFY_TARGETS[target]}" with the secret provider fleet.json names.`,
+        },
+      });
+      continue;
+    }
+    const ok = await send(url, request(target, notification, text))
+      .then((response) => response.ok)
+      .catch(() => false);
+    checks.push(
+      ok
+        ? {
+            id,
+            status: 'pass',
+            summary: `The ${target} target took the notification.`,
+          }
+        : {
+            id,
+            status: 'fail',
+            summary: 'NOTIFY_DELIVERY_FAILED',
+            remedy: {
+              action: `Check that the ${target} target answers, then run devfleet fleet-watch --test.`,
+            },
+          }
+    );
+  }
+  return checks;
+}
+
+function request(
+  target: NotifyTarget,
+  notification: Notification,
+  text: string
+): RequestInit {
+  const signal = AbortSignal.timeout(15_000);
+  if (target === 'ntfy')
+    return {
+      method: 'POST',
+      body: text,
+      headers: {
+        Title: `dev-fleet: ${notification.event}`,
+        Tags:
+          notification.event === 'recovered' ? 'white_check_mark' : 'warning',
+      },
+      signal,
+    };
+  return {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      source: 'dev-fleet',
+      event: notification.event,
+      node: safe(notification.node) ? notification.node : null,
+      checks: notification.checks.filter(safe),
+      text,
+    }),
+    signal,
+  };
+}