@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/ide/t3.ts

@@ -0,0 +1,111 @@
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import type { Check, FleetNode } from '../types';
+import type { Ide, MachineContext, WorkerContext } from './ide';
+import { fleet } from '../fleet-config';
+import { machineProviders } from '../t3/providers';
+import { providerNames } from '../t3/names-check';
+import { profileCheck } from '../t3/profile';
+import { modelsCheck } from '../t3/models';
+import { proxyAccounts, type Roster } from '../t3/accounts';
+import {
+  MACOS_POLICY,
+  managerVersion,
+  updatesCheck,
+  workerVersion,
+} from '../t3/version';
+import { verifyT3Endpoint } from '../t3-endpoint';
+import { observedIdentity, verifyT3Linked } from '../t3-linked-check';
+import { T3_RESTARTED_SUMMARY } from '../platform/t3-policy';
+import { ansibleCommand } from '../platform/ansible';
+
+/** Where T3 keeps its data on this machine. */
+const t3Home = (): string =>
+  resolve(process.env['T3CODE_HOME'] ?? join(homedir(), '.t3'));
+
+/** What the T3 checks read besides the machine: injected by scenarios. */
+export interface T3Inputs {
+  t3Home?: string;
+  roster?: Roster;
+  profile?: Record<string, unknown>;
+  requiredModels?: Record<string, string[]>;
+  policyPath?: string;
+}
+
+/** T3 Code, the first supported IDE. Its install, service and policy on
+ * workers are the `ansible/ubuntu/t3*` playbooks and on the manager
+ * `ansible/macos/t3-policy.yml`; everything else is reached from here.
+ */
+export function t3Ide(inputs: T3Inputs = {}): Ide {
+  const home = () => inputs.t3Home ?? t3Home();
+  const settings = async (context: MachineContext): Promise<Check[]> => {
+    const required = inputs.requiredModels ?? fleet.t3?.requiredModels;
+    const options = {
+      t3Home: home(),
+      ...(inputs.roster ? { roster: inputs.roster } : {}),
+    };
+    return [
+      await machineProviders(
+        context.id,
+        context.workspace,
+        context.apply,
+        options
+      ),
+      await providerNames(
+        context.id,
+        context.workspace,
+        context.apply,
+        options
+      ),
+      await profileCheck(
+        join(home(), 'userdata/settings.json'),
+        inputs.profile ?? fleet.t3?.profile ?? {},
+        { apply: context.apply }
+      ),
+      await modelsCheck({
+        t3Home: home(),
+        roster: inputs.roster ?? proxyAccounts,
+        ...(required ? { required } : {}),
+      }),
+    ];
+  };
+  return {
+    id: 't3',
+    settings,
+    async manager(context) {
+      return [
+        ...(await settings(context)),
+        await managerVersion(home(), context.workspace),
+        await updatesCheck(
+          inputs.policyPath ?? MACOS_POLICY,
+          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`
+        ),
+      ];
+    },
+    async worker(node: FleetNode, context: WorkerContext) {
+      const checks = [...(await verifyT3Endpoint(node, context.run))];
+      // A worker T3 restarted by this sync (idle policy heal) needs a minute
+      // or two before the manager's link reconnects.
+      const restarted = context.earlier.some(
+        (check) => check.summary === T3_RESTARTED_SUMMARY
+      );
+      const linked = await verifyT3Linked(node, context.run, {
+        reconnectMs: restarted ? 180_000 : 0,
+      });
+      checks.push(linked);
+      if (
+        linked.status !== 'fail' &&
+        !node.t3EnvironmentId &&
+        context.remember
+      ) {
+        const identity = await observedIdentity(node, context.run).catch(
+          () => undefined
+        );
+        if (identity)
+          await context.remember(node.id, identity).catch(() => undefined);
+      }
+      checks.push(await workerVersion(node, context.run, context.workspace));
+      return checks;
+    },
+  };
+}

package/src/machine-health.ts

@@ -1,3 +1,4 @@
+import policy from '../ansible/ubuntu/policy.json';
 import type { DoctorCheck } from './doctor-check';
 
 /** Machine health sample parsed from `net/remote-probe.sh` output. */
@@ -7,6 +8,14 @@ export type HealthSample = {
   readonly memFreePct?: number;
   readonly swapPct?: number;
   readonly diskMb?: number;
+  /** Free inodes on the home filesystem, as a percentage of all of them. */
+  readonly inodesFreePct?: number;
+  /** 1 when the clock is synchronised to a time source, 0 when not. */
+  readonly clockSynced?: number;
+  /** 1 when a time service is enabled, 0 when not. */
+  readonly timeService?: number;
+  /** The machine's clock, in seconds since the epoch. */
+  readonly epoch?: number;
 };
 
 const PROBE_KEYS = {
@@ -15,6 +24,10 @@ const PROBE_KEYS = {
   memfree: 'memFreePct',
   swap: 'swapPct',
   diskmb: 'diskMb',
+  inodefree: 'inodesFreePct',
+  ntpsync: 'clockSynced',
+  ntp: 'timeService',
+  epoch: 'epoch',
 } as const;
 
 function isProbeKey(key: string): key is keyof typeof PROBE_KEYS {
@@ -33,7 +46,12 @@ export function parseProbe(output: string): HealthSample {
   return sample;
 }
 
-const LABELS = { load: 'CPU load', memory: 'Memory', disk: 'Disk space' };
+const LABELS = {
+  load: 'CPU load',
+  memory: 'Memory',
+  disk: 'Disk space',
+  inodes: 'Free inodes',
+};
 
 function check(
   id: keyof typeof LABELS,
@@ -88,8 +106,37 @@ export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
   return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
 }
 
+/** Free inodes, with the floors `policy.json` gives the disk guard: a
+ * filesystem with free bytes and no free inodes fails every write.
+ */
+export function inodesCheck(
+  inodesFreePct: number | undefined,
+  floors = INODE_FLOORS
+): DoctorCheck | null {
+  if (inodesFreePct === undefined) return null;
+  const what = `${inodesFreePct}% of inodes free`;
+  if (inodesFreePct < floors.failPercent)
+    return check(
+      'inodes',
+      'fail',
+      `INODES_EXHAUSTED, ${what}. Every new file fails although bytes are free.`,
+      'Remove idle dependency trees and worktrees: the hourly disk collector does so under pressure, or delete them by hand.'
+    );
+  if (inodesFreePct < floors.warnPercent)
+    return check('inodes', 'warn', `low, ${what}`);
+  return check('inodes', 'pass', what);
+}
+
+const INODE_FLOORS = {
+  warnPercent: policy.resourceSafety.inodeWarnFreePercent,
+  failPercent: policy.resourceSafety.inodeFailFreePercent,
+};
+
 export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
-  return [loadCheck(s, alias), memoryCheck(s), diskCheck(s.diskMb)].filter(
-    (c): c is DoctorCheck => c !== null
-  );
+  return [
+    loadCheck(s, alias),
+    memoryCheck(s),
+    diskCheck(s.diskMb),
+    inodesCheck(s.inodesFreePct),
+  ].filter((c): c is DoctorCheck => c !== null);
 }

package/src/manager-health.ts

@@ -0,0 +1,205 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import macos from '../ansible/macos/policy.json';
+import type { Check, Runner } from './types';
+
+/** Read-only measurements of the manager's operating system and disk, from
+ * the floors `ansible/macos/policy.json` declares. Nothing a command printed
+ * is reported: only what was concluded from it.
+ */
+export async function managerHealth(
+  run: Runner,
+  home = homedir()
+): Promise<Check[]> {
+  return [
+    await managerRelease(run),
+    await managerUpdates(run),
+    await managerDisk(run),
+    await containerWatchdog(run, home),
+  ];
+}
+
+/** The manager runs at least the macOS release the policy names. */
+export async function managerRelease(run: Runner): Promise<Check> {
+  const id = 'manager:release';
+  const result = await run(['sw_vers', '-productVersion']);
+  const major = /^(\d+)(?:\.\d+)*$/.exec(result.stdout.trim())?.[1];
+  if (!result.ok || major === undefined)
+    return { id, status: 'skip', summary: 'MANAGER_RELEASE_UNOBSERVED' };
+  if (Number(major) >= macos.minimumMajorVersion)
+    return {
+      id,
+      status: 'pass',
+      summary: `The manager runs macOS ${major}, at least the required ${macos.minimumMajorVersion}.`,
+    };
+  return {
+    id,
+    status: 'fail',
+    summary: `MANAGER_RELEASE_UNSUPPORTED macOS ${major}, below ${macos.minimumMajorVersion}`,
+    remedy: {
+      action: `Update the manager to macOS ${macos.minimumMajorVersion} or later in System Settings > General > Software Update, then rerun fleet-sync.`,
+    },
+  };
+}
+
+/** Pending macOS updates warn, and so does one that waits for a restart.
+ * `--no-scan` reads the list macOS already has: nothing is downloaded.
+ */
+export async function managerUpdates(run: Runner): Promise<Check> {
+  const id = 'manager:updates';
+  const result = await run(['softwareupdate', '--list', '--no-scan'], {
+    timeoutMs: 60_000,
+  });
+  const output = `${result.stdout}\n${result.stderr}`;
+  if (!result.ok)
+    return { id, status: 'skip', summary: 'MANAGER_UPDATES_UNOBSERVED' };
+  const pending = (output.match(/^\s*\* Label:/gm) ?? []).length;
+  const restart = /Action:\s*restart/i.test(output);
+  if (pending === 0)
+    return { id, status: 'pass', summary: 'No macOS update is pending.' };
+  return {
+    id,
+    status: 'warn',
+    summary: restart
+      ? `MANAGER_RESTART_PENDING ${pending} updates, at least one waits for a restart`
+      : `MANAGER_UPDATES_PENDING ${pending} updates`,
+    remedy: {
+      action:
+        'Install the pending updates in System Settings > General > Software Update at a time no worker depends on the manager, then rerun fleet-sync.',
+    },
+  };
+}
+
+const MIB = 1024;
+const GIB = 1024 ** 3;
+
+/** Free space on the data volume, and the space Docker Desktop holds. */
+export async function managerDisk(run: Runner): Promise<Check> {
+  const id = 'manager:disk';
+  const df = await run(['df', '-Pk', '/System/Volumes/Data']);
+  const kib = Number(df.stdout.split('\n')[1]?.trim().split(/\s+/)[3]);
+  if (!df.ok || !Number.isFinite(kib))
+    return { id, status: 'skip', summary: 'MANAGER_DISK_UNOBSERVED' };
+  const free = Math.floor(kib / MIB);
+  const docker = await dockerBytes(run);
+  const dockerGiB = docker === null ? null : docker / GIB;
+  const held =
+    dockerGiB === null ? '' : `; Docker holds ${dockerGiB.toFixed(1)} GiB`;
+  const remedy = {
+    action:
+      'Free space on the manager: docker system prune removes unused images and build cache; empty large caches and downloads.',
+  };
+  if (free < macos.disk.failFreeMiB)
+    return {
+      id,
+      status: 'fail',
+      summary: `MANAGER_DISK_FULL ${free} MiB free${held}`,
+      remedy,
+    };
+  if (free < macos.disk.warnFreeMiB)
+    return {
+      id,
+      status: 'warn',
+      summary: `MANAGER_DISK_LOW ${free} MiB free${held}`,
+      remedy,
+    };
+  if (dockerGiB !== null && dockerGiB > macos.disk.dockerWarnGiB)
+    return {
+      id,
+      status: 'warn',
+      summary: `MANAGER_DOCKER_DISK_LARGE ${free} MiB free${held}`,
+      remedy,
+    };
+  return {
+    id,
+    status: 'pass',
+    summary: `${Math.floor(free / MIB)} GiB free on the data volume${held}.`,
+  };
+}
+
+const UNITS: Record<string, number> = {
+  B: 1,
+  kB: 1e3,
+  KB: 1e3,
+  MB: 1e6,
+  GB: 1e9,
+  TB: 1e12,
+};
+
+/** The bytes of images, containers, volumes and build cache, or null when
+ * Docker does not answer.
+ */
+async function dockerBytes(run: Runner): Promise<number | null> {
+  const result = await run(
+    ['docker', 'system', 'df', '--format', '{{.Size}}'],
+    { timeoutMs: 30_000 }
+  );
+  if (!result.ok) return null;
+  let total = 0;
+  for (const line of result.stdout.split('\n')) {
+    const match = /^([\d.]+)\s*([kKMGT]?B)$/.exec(line.trim());
+    if (match) total += Number(match[1]) * (UNITS[match[2]!] ?? 0);
+  }
+  return total;
+}
+
+/** What the container watchdog remembers (ansible/monitoring/container-watchdog.py). */
+type WatchdogMemory = Record<
+  string,
+  { restarts?: number[]; capped?: number | null }
+>;
+
+/** A container the watchdog stopped restarting is a restart storm: it fails,
+ * and one it restarted within the hour warns.
+ */
+export async function containerWatchdog(
+  run: Runner,
+  home: string,
+  now = Date.now()
+): Promise<Check> {
+  const id = 'manager:container-watchdog';
+  const result = await run([
+    'cat',
+    join(home, 'Library/Caches/dev-fleet/container-watchdog.json'),
+  ]);
+  let memory: WatchdogMemory;
+  try {
+    memory = JSON.parse(result.stdout) as WatchdogMemory;
+  } catch {
+    memory = {};
+  }
+  if (!result.ok || typeof memory !== 'object' || memory === null)
+    return { id, status: 'skip', summary: 'CONTAINER_WATCHDOG_UNOBSERVED' };
+  const hour = (time: unknown) =>
+    typeof time === 'number' && now / 1000 - time < 3600;
+  const names = Object.keys(memory).filter((name) =>
+    /^[A-Za-z0-9_.-]{1,128}$/.test(name)
+  );
+  const capped = names.filter((name) => hour(memory[name]?.capped));
+  const restarted = names.filter((name) =>
+    (memory[name]?.restarts ?? []).some(hour)
+  );
+  const remedy = {
+    action:
+      'Read the container watchdog log (~/Library/Application Support/dev-fleet/container-watchdog/watchdog.log) and the container logs, fix the cause, then docker restart the container.',
+  };
+  if (capped.length)
+    return {
+      id,
+      status: 'fail',
+      summary: `CONTAINER_RESTART_STORM ${capped.join(' ')}`,
+      remedy,
+    };
+  if (restarted.length)
+    return {
+      id,
+      status: 'warn',
+      summary: `CONTAINER_RESTARTED_UNHEALTHY ${restarted.join(' ')}`,
+      remedy,
+    };
+  return {
+    id,
+    status: 'pass',
+    summary: 'No container stayed unhealthy in the last hour.',
+  };
+}

package/src/manager-sync.ts

@@ -6,7 +6,11 @@ import {
   type NodeReport,
   type Runner,
 } from './types';
-import { discoverTailnet, type TailnetComputer } from './tailscale';
+import {
+  discoverTailnet,
+  keyExpiryCheck,
+  type TailnetComputer,
+} from './tailscale';
 import { matchComputers } from './tailnet-matching';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
@@ -102,7 +106,12 @@ export async function synchronizeManager(
     ok: false,
     applied: false,
     desiredRevision: input.revision,
-    checks: tailnet.checks,
+    checks: [
+      ...tailnet.checks,
+      ...(tailnet.computers.some((peer) => peer.self)
+        ? [keyExpiryCheck(tailnet.computers.find((peer) => peer.self))]
+        : []),
+    ],
     // Printed for people: the stable public name, never a tailnet address.
     dashboardUrl: publicHubUrl(),
     nodes: [],
@@ -322,6 +331,7 @@ async function inspectManagerNode(
   else {
     checks.push(...(await inspectBootstrap(node, run, machine.os)));
     checks.push(...identityChecks(node, peer));
+    if (peer) checks.push(keyExpiryCheck(peer));
   }
   if (!peer || !peer.online)
     checks.push({
@@ -385,21 +395,7 @@ function mayDeploy(result: ManagerNode, nodes: FleetNode[]): boolean {
  */
 export function blocksPromotion(check: Check): boolean {
   if (!blocking(check)) return false;
-  if (check.id === 't3:service' && T3_MAINTENANCE.has(check.summary))
-    return false;
-  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|ubuntu:boot-policy$)/.test(
+  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|t3:running-version$|ubuntu:boot-policy$)/.test(
     check.id
   );
 }
-
-/** Native T3 outcomes that need an operator maintenance window (an active
- * listener or running release must never be killed by routine sync). They are
- * reported as failures but do not block independent OS policy on other nodes.
- */
-const T3_MAINTENANCE = new Set([
-  'T3_LISTENER_CONFLICT',
-  'T3_ACTIVE_LISTENER_MIGRATION_REQUIRED',
-  'T3_MAINTENANCE_REQUIRED',
-  'T3_POLICY_RESTART_PENDING',
-  'T3_DESKTOP_MIGRATION_REQUIRED',
-]);

package/src/monitoring/hub-backup.ts

@@ -1,4 +1,5 @@
 import { record } from '../config';
+import type { HubBackups } from '../definition';
 import type { Check } from '../types';
 import {
   LOCAL_HUB,
@@ -10,6 +11,10 @@ import {
 
 /** A scheduled backup older than this means the schedule is not running. */
 export const BACKUP_MAX_AGE_MS = 48 * 60 * 60 * 1000;
+/** Declared backups that have not reached off-machine storage for this long
+ * fail: the grace covers a fresh hub and a storage outage of a few days.
+ */
+export const BACKUP_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
 
 function backupCheck(
   status: Check['status'],
@@ -24,21 +29,26 @@ function backupCheck(
   };
 }
 
-const SETTINGS =
-  'Open the hub as its superuser (/_/ on the manager), Settings > Backups.';
+const UNDECLARED =
+  'Declare services.monitoring.backups in fleet.json (a schedule, how many to keep and S3-compatible storage off the manager) and the secrets monitoring-backup-access-key and monitoring-backup-secret-key; fleet-sync then sets them through the hub.';
+const DECLARED =
+  "fleet-sync sets the backups declared under services.monitoring.backups. Check that the secrets monitoring-backup-access-key and monitoring-backup-secret-key are readable on the manager and that the manager's Vault session is logged in, then rerun fleet-sync.";
 
-/** Reads, never changes, the hub's own backup settings and its list of
- * backups through the stock PocketBase API of the pinned release (superuser
- * only: GET /api/settings and GET /api/backups). Backups are configured in
- * the hub's UI; this says whether they are scheduled, kept off the manager
- * and recent. Storage names, keys and endpoints are never reported.
+/** Reads the hub's own backup settings and its list of backups through the
+ * stock PocketBase API of the pinned release (superuser only: GET
+ * /api/settings and GET /api/backups): whether they are scheduled, kept off
+ * the manager and recent. fleet-sync sets them from the fleet configuration
+ * (`hub-darwin.yml`); with them declared, anything but that is a failure, a
+ * stale backup after a grace period. Storage names, keys and endpoints are
+ * never reported.
  */
 export async function verifyHubBackups(
   login: HubLogin | null,
   fetcher: typeof fetch = fetch,
   base: string = LOCAL_HUB,
-  now: number = Date.now()
+  options: { now?: number; declared?: HubBackups | undefined } = {}
 ): Promise<Check> {
+  const SETTINGS = options.declared ? DECLARED : UNDECLARED;
   if (!login)
     return backupCheck(
       'skip',
@@ -69,31 +79,67 @@ export async function verifyHubBackups(
       'BESZEL_BACKUP_UNVERIFIED',
       `The hub answered in a form this release does not expect. ${SETTINGS}`
     );
+  return judgeBackups(
+    policy,
+    newestBackup(backups),
+    options.now ?? Date.now(),
+    options.declared
+  );
+}
+
+/** Undeclared backups only warn; declared ones fail when they are not what
+ * the fleet configuration says, or have not reached storage for longer than
+ * the grace period.
+ */
+function judgeBackups(
+  policy: { cron: string; keep: number; offHost: boolean },
+  newest: number | null,
+  now: number,
+  declared: HubBackups | undefined
+): Check {
+  const SETTINGS = declared ? DECLARED : UNDECLARED;
+  const missing: Check['status'] = declared ? 'fail' : 'warn';
   if (!policy.cron)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_UNSCHEDULED',
-      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS} Set a schedule and how many to keep.`
+      `The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS}`
     );
   if (!policy.offHost)
     return backupCheck(
-      'warn',
+      missing,
       'BESZEL_BACKUP_OFFHOST_REQUIRED',
-      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Store them in existing encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
+      `Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Use encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
     );
-  const newest = newestBackup(backups);
-  if (newest === null || now - newest > BACKUP_MAX_AGE_MS)
+  if (
+    declared &&
+    (policy.cron !== declared.schedule || policy.keep !== declared.keep)
+  )
     return backupCheck(
-      'warn',
+      'fail',
+      'BESZEL_BACKUP_NOT_CONVERGED',
+      `The hub's backup schedule is not the one the fleet configuration declares. ${SETTINGS}`
+    );
+  const age = newest === null ? null : now - newest;
+  if (age === null || age > BACKUP_MAX_AGE_MS)
+    return backupCheck(
+      staleStatus(age, declared !== undefined),
       'BESZEL_BACKUP_STALE',
-      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. ${SETTINGS} Check the schedule and the storage's credentials, then create a backup by hand.`
+      `No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. Check the storage's credentials and that the hub can reach it, then create a backup by hand (the hub's /_/ page, Settings > Backups). ${SETTINGS}`
     );
   return backupCheck(
     'pass',
-    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(newest).toISOString()}.`
+    `Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(now - age).toISOString()}.`
   );
 }
 
+/** A fresh hub has no backup yet; declared backups that stopped reaching
+ * storage fail once the grace period is over.
+ */
+function staleStatus(age: number | null, declared: boolean): Check['status'] {
+  return declared && age !== null && age > BACKUP_GRACE_MS ? 'fail' : 'warn';
+}
+
 function backupPolicy(
   settings: unknown
 ): { cron: string; keep: number; offHost: boolean } | null {

package/src/monitoring/manager-hub.ts

@@ -8,6 +8,7 @@ import { publicViewerIdentity, fleetAccount } from './hub-access';
 import { ensurePublicViewer, type HubLogin } from './hub-viewer';
 import { verifyHubBackups } from './hub-backup';
 import type { FleetDefinition } from '../definition';
+import { secretVia } from '../fleet-config';
 import { hostId, hubUrl } from '../topology';
 import { publicHubUrl, privateHub } from './beszel';
 import type { Check, Runner } from '../types';
@@ -56,6 +57,9 @@ export interface ManagerHubOptions {
   receipts?: string;
   /** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
   viewer?: (login: HubLogin | null) => Promise<Check>;
+  /** The declared backups for the hub's playbook; defaults to the fleet
+   * configuration and its secrets. */
+  backupStorage?: () => Promise<Record<string, string | number | boolean>>;
   /** Reads the hub's backup settings; defaults to the loopback hub API. */
   backups?: (login: HubLogin | null) => Promise<Check>;
 }
@@ -78,13 +82,19 @@ export async function convergeManagerHub(
   const paths = managerHubPaths(options.home);
   const tunnel = await tunnelToken(run);
   const account = await (options.account ?? fleetAccount)();
+  const backups = await (
+    options.backupStorage ?? (() => backupVariables(definition, run))
+  )();
   const monitoring = resolve(root, 'ansible/monitoring');
   const result = await runAnsibleWithReceipt(
     run,
     'hub-darwin.yml',
     {
       cwd: monitoring,
-      input: JSON.stringify(hubVariables(hub, paths, tunnel, account)),
+      input: JSON.stringify({
+        ...hubVariables(hub, paths, tunnel, account),
+        ...backups,
+      }),
       timeoutMs: 300_000,
     },
     {
@@ -127,7 +137,13 @@ export async function convergeManagerHub(
     },
     publicEndpointCheck(tunnel !== null),
     await ensureViewer(options, account),
-    await (options.backups ?? verifyHubBackups)(account),
+    await (
+      options.backups ??
+      ((login) =>
+        verifyHubBackups(login, fetch, LOCAL_HUB, {
+          declared: definition.services.monitoring.backups,
+        }))
+    )(account),
   ];
 }
 
@@ -153,6 +169,32 @@ function hubVariables(
   };
 }
 
+/** The declared backup schedule and storage, with the storage's keys from
+ * the fleet's secrets; nothing when no backups are declared or a key is
+ * missing (beszel:backup then says so).
+ */
+export async function backupVariables(
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Record<string, string | number | boolean>> {
+  const declared = definition.services.monitoring.backups;
+  if (!declared) return {};
+  const secret = secretVia(run);
+  const accessKey = await secret('monitoring-backup-access-key');
+  const secretKey = await secret('monitoring-backup-secret-key');
+  if (!accessKey || !secretKey) return {};
+  return {
+    beszel_backup_schedule: declared.schedule,
+    beszel_backup_keep: declared.keep,
+    beszel_backup_bucket: declared.storage.bucket,
+    beszel_backup_endpoint: declared.storage.endpoint,
+    beszel_backup_region: declared.storage.region,
+    beszel_backup_force_path_style: declared.storage.forcePathStyle ?? false,
+    beszel_backup_access_key: accessKey,
+    beszel_backup_secret_key: secretKey,
+  };
+}
+
 function ensureViewer(
   options: ManagerHubOptions,
   account: HubLogin | null