@crvouga/dev-fleet 0.3.0 → 0.4.0

package/src/cli/fleet-watch.ts

@@ -0,0 +1,198 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fleet, secret } from '../fleet-config';
+import {
+  deliver,
+  observe,
+  parseMemory,
+  transitions,
+  type Notification,
+  type NotifyConfig,
+} from '../notify';
+import { connection } from '../state/connection';
+import type { Bucket } from '../state/bucket';
+import type { SecretsProvider } from '../secrets';
+import type { Check, NodeReport } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetWatchOptions {
+  test?: boolean;
+}
+
+/** What the watcher remembers between runs, kept in the bucket beside the
+ * fleet state. Its time is the heartbeat an off-machine check reads.
+ */
+export interface WatchStore {
+  read: () => Promise<string | null>;
+  write: (text: string) => Promise<void>;
+}
+
+export interface FleetWatchServices {
+  notify: NotifyConfig | undefined;
+  /** The machines as `fleet-status --workers` sees them now. */
+  status: () => Promise<NodeReport[]>;
+  store: WatchStore | null;
+  secrets: SecretsProvider;
+  send: typeof fetch;
+  now: () => Date;
+}
+
+/** Compares the fleet's health with the previous run and notifies on
+ * transitions only. `--test` sends one test notification and nothing else.
+ */
+export async function fleetWatch(
+  options: FleetWatchOptions,
+  services: FleetWatchServices
+): Promise<CommandOutcome> {
+  const { notify } = services;
+  if (options.test) {
+    const checks = notify
+      ? await deliver(
+          notify,
+          { event: 'test', node: 'fleet-watch', checks: [] },
+          services.secrets,
+          services.send
+        )
+      : [unconfigured('fail')];
+    return outcome(checks, [
+      { event: 'test', node: 'fleet-watch', checks: [] },
+    ]);
+  }
+  const previous = services.store
+    ? parseMemory(await services.store.read().catch(() => null))
+    : null;
+  const observed = observe(await services.status());
+  const { notifications, memory } = transitions(
+    previous,
+    observed,
+    services.now(),
+    notify?.repeatHours ?? 0
+  );
+  const checks: Check[] = [];
+  for (const notification of notifications)
+    if (notify)
+      checks.push(
+        ...(await deliver(
+          notify,
+          notification,
+          services.secrets,
+          services.send
+        ))
+      );
+  if (!notify) checks.push(unconfigured('warn'));
+  checks.push(await heartbeat(services.store, JSON.stringify(memory)));
+  return outcome(checks, notifications);
+}
+
+function unconfigured(status: 'warn' | 'fail'): Check {
+  return {
+    id: 'notify:configured',
+    status,
+    summary: 'NOTIFY_NOT_CONFIGURED',
+    remedy: {
+      action:
+        'Add a "notify" block to fleet.json with its targets, and store each target address as its secret (notify-webhook-url, notify-ntfy-url).',
+    },
+  };
+}
+
+async function heartbeat(
+  store: WatchStore | null,
+  text: string
+): Promise<Check> {
+  const id = 'watch:heartbeat';
+  if (!store)
+    return {
+      id,
+      status: 'warn',
+      summary: 'WATCH_HEARTBEAT_NO_BUCKET',
+      remedy: {
+        action:
+          'Name the bucket under "state" in fleet.json: the heartbeat and what the watcher remembers are kept there.',
+      },
+    };
+  return store.write(text).then(
+    () => ({
+      id,
+      status: 'pass' as const,
+      summary: 'The heartbeat and the health of each machine were stored.',
+    }),
+    () => ({
+      id,
+      status: 'fail' as const,
+      summary: 'WATCH_HEARTBEAT_UNWRITTEN',
+      remedy: {
+        action:
+          'Check the bucket credentials this run was given (devfleet fleet-state shows the connection).',
+      },
+    })
+  );
+}
+
+function outcome(
+  checks: Check[],
+  notifications: Notification[]
+): CommandOutcome {
+  const ok = !checks.some((check) => check.status === 'fail');
+  const lines = [
+    ...notifications.map(
+      (entry) =>
+        `${entry.event} ${entry.node}${entry.checks.length ? `: ${entry.checks.join(', ')}` : ''}`
+    ),
+    ...checks.map((check) => `${check.status} ${check.id} ${check.summary}`),
+  ];
+  return {
+    exitCode: ok ? 0 : 1,
+    data: { version: 1, command: 'fleet-watch', ok, notifications, checks },
+    text: lines.join('\n') || 'Nothing changed.',
+  };
+}
+
+/** The watcher's object in the bucket, beside the fleet state. */
+export const WATCH_OBJECT = 'watch.json';
+
+export function bucketStore(bucket: Bucket | null): WatchStore | null {
+  if (!bucket) return null;
+  if (bucket.kind === 'directory') {
+    const path = join(bucket.path, WATCH_OBJECT);
+    return {
+      read: () => readFile(path, 'utf8').catch(() => null),
+      write: async (text) => {
+        await mkdir(dirname(path), { recursive: true });
+        await writeFile(path, text, { mode: 0o600 });
+      },
+    };
+  }
+  const client = new Bun.S3Client({
+    bucket: bucket.bucket,
+    ...(bucket.endpoint ? { endpoint: bucket.endpoint } : {}),
+    ...(bucket.region ? { region: bucket.region } : {}),
+    ...(bucket.accessKeyId && bucket.secretAccessKey
+      ? {
+          accessKeyId: bucket.accessKeyId,
+          secretAccessKey: bucket.secretAccessKey,
+        }
+      : {}),
+  });
+  const file = client.file(`${bucket.prefix}/${WATCH_OBJECT}`);
+  return {
+    read: async () => ((await file.exists()) ? file.text() : null),
+    write: async (text) => {
+      await file.write(text, { type: 'application/json' });
+    },
+  };
+}
+
+/** The services of a real run, on the manager. */
+export function watchServices(
+  status: () => Promise<NodeReport[]>
+): FleetWatchServices {
+  return {
+    notify: fleet.notify,
+    status,
+    store: bucketStore(connection().bucket),
+    secrets: secret,
+    send: fetch,
+    now: () => new Date(),
+  };
+}

package/src/cli/fleet.ts

@@ -14,6 +14,8 @@ import { addNode, nodeFromOptions, readRoster } from './roster';
 import { executeTailnet } from './tailnet';
 import { probeSource } from './probe-source';
 import { executeRepairCommand } from './repair';
+import { scheduleCheck } from '../schedule-check';
+import { readHistory } from '../state/history';
 
 export interface FleetServices {
   run: Runner;
@@ -125,6 +127,8 @@ async function diagnose(
     services.run
   );
   if (!options.local && settings.missingRoster) reports.push(missingRoster());
+  const schedule = scheduleCheck(readHistory(), new Date());
+  if (schedule) reports[0]?.checks.push(schedule);
   const report = {
     ...makeReport('doctor', reports),
     coverage: {

package/src/cli/register-fleet.ts

@@ -8,6 +8,9 @@ import { fleetSync, type FleetSyncOptions } from './fleet-sync';
 import { syncNode, type NodeSyncOptions } from './node-sync';
 import { fleetState, type FleetStateOptions } from './fleet-state';
 import { fleetAccounts, type FleetAccountsOptions } from './fleet-accounts';
+import { fleetReport, type FleetReportOptions } from './fleet-report';
+import { fleetWatch, watchServices } from './fleet-watch';
+import type { NodeReport } from '../types';
 
 type Complete = (outcome: CommandOutcome) => void;
 
@@ -282,6 +285,52 @@ function registerState({ program, complete }: Registration) {
     });
 }
 
+function registerReport({ program, complete }: Registration) {
+  program
+    .command('fleet-report')
+    .description(
+      'Report how reliably fleet-sync ran: success rate, duration, failing checks and time not ready'
+    )
+    .option('--since <duration>', 'How far back: 30d (default), 12h or 90m')
+    .action((_options, command: Command) => {
+      complete(fleetReport(command.optsWithGlobals<FleetReportOptions>()));
+    });
+}
+
+function registerWatch({ program, defaults, complete }: Registration) {
+  program
+    .command('fleet-watch')
+    .description(
+      "Check the workers and notify the fleet's targets when one becomes unhealthy or recovers"
+    )
+    .option('--test', 'Send one test notification to every target and stop')
+    .addHelpText(
+      'after',
+      '\nRuns fleet-status --workers, compares it with the previous run kept in the bucket and notifies only on a change (and again every repeatHours while a worker stays unhealthy). Targets come from "notify" in fleet.json; their addresses are secrets. Every run stores a heartbeat in the bucket for an off-machine check.'
+    )
+    .action(async (_options, command: Command) => {
+      const options = command.optsWithGlobals<
+        CliOptions & { test?: boolean }
+      >();
+      const settings = await withSettings(defaults, options);
+      const status = async (): Promise<NodeReport[]> => {
+        const report = await executeFleet(
+          'health',
+          { ...options, workers: true, managed: true },
+          settings
+        );
+        const data = report.data as { nodes?: NodeReport[] };
+        return data.nodes ?? [];
+      };
+      complete(
+        await fleetWatch(
+          { ...(options.test ? { test: true } : {}) },
+          watchServices(status)
+        )
+      );
+    });
+}
+
 export function registerFleet(
   program: Command,
   defaults: Settings,
@@ -296,6 +345,8 @@ export function registerFleet(
   );
   registerSync(registration);
   registerState(registration);
+  registerReport(registration);
+  registerWatch(registration);
   registerRepair(registration);
   registerNodes(registration);
   registerAccounts(registration);

package/src/cli/registry.ts

@@ -50,6 +50,13 @@ export const COMMANDS: CommandSpec[] = [
       'Read-only canonical worker readiness, dependencies, services and authentication; lists every remaining gap.',
       ['workers', 'readiness', 'health', 'canonical'],
     ],
+    [
+      'fleet-watch',
+      'Notify when a worker becomes unhealthy or recovers',
+      'Runs fleet-status --workers, compares it with the previous run kept in the bucket and notifies the targets fleet.json names under "notify" on a change only. Stores a heartbeat for an off-machine check. --test sends one test notification.',
+      ['notify', 'alert', 'webhook', 'ntfy', 'heartbeat', 'watch', 'monitor'],
+      true,
+    ],
     [
       'fleet-discover',
       'Discover Tailscale computers',
@@ -90,6 +97,12 @@ export const COMMANDS: CommandSpec[] = [
       ['state', 'bucket', 'restore', 'recover', 'disposable'],
       true,
     ],
+    [
+      'fleet-report',
+      'Report how reliably fleet-sync runs',
+      'Read-only: success rate, mean duration, the most frequent failing check IDs and how long each machine was not ready, from the history of fleet-sync runs kept in the fleet state. --since 30d (default), 12h or 90m.',
+      ['history', 'reliability', 'report', 'failures', 'sync'],
+    ],
     [
       'fleet-node-sync',
       'Sync this node',

package/src/cli/schedule-logs.ts

@@ -0,0 +1,36 @@
+import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** The most a scheduled job's log may hold; a longer one keeps its newest
+ * quarter.
+ */
+export const SCHEDULE_LOG_BYTES = 1024 * 1024;
+
+/** Caps the schedule's own logs (`<label>.out.log`, `<label>.err.log`) at the
+ * start of every scheduled run. launchd and systemd append to them, so the
+ * newest lines are what is kept. Nothing else in the directory is touched.
+ */
+export function capScheduleLogs(
+  directory: string,
+  limit = SCHEDULE_LOG_BYTES
+): void {
+  let names: string[];
+  try {
+    names = readdirSync(directory);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    if (!/^[a-z0-9-]+\.(out|err)\.log$/.test(name)) continue;
+    const path = join(directory, name);
+    try {
+      const stat = statSync(path);
+      if (!stat.isFile() || stat.size <= limit) continue;
+      const kept = readFileSync(path).subarray(stat.size - limit / 4);
+      const start = kept.indexOf(10) + 1;
+      writeFileSync(path, kept.subarray(start), { mode: 0o600 });
+    } catch {
+      // A log that cannot be capped is capped by the next run.
+    }
+  }
+}

package/src/clock.ts

@@ -0,0 +1,63 @@
+import policy from '../ansible/ubuntu/policy.json';
+import type { HealthSample } from './machine-health';
+import type { Check } from './types';
+
+const { warnOffsetSeconds, failOffsetSeconds } = policy.clock;
+
+/** A skewed clock breaks TLS, token validation, apt signatures and Vault
+ * logins with errors that never mention time. The clock passes when it is
+ * synchronised to a time source and, where a reference time was measured,
+ * within the offset floors of `policy.json`.
+ *
+ * `offsetSeconds` is the difference to the reference (the manager's clock),
+ * when one was measured.
+ */
+export function clockCheck(
+  id: string,
+  sample: Pick<HealthSample, 'clockSynced' | 'timeService'>,
+  offsetSeconds?: number
+): Check {
+  const offset =
+    offsetSeconds === undefined ? undefined : Math.abs(offsetSeconds);
+  const measured =
+    offset === undefined ? '' : `, ${offset.toFixed(1)} s from the manager`;
+  const remedy = clockRemedy(sample.timeService);
+  if (sample.clockSynced === undefined && offset === undefined)
+    return {
+      id,
+      status: 'skip',
+      summary: 'CLOCK_UNOBSERVED: the clock synchronisation could not be read.',
+    };
+  if (offset !== undefined && offset > failOffsetSeconds)
+    return { id, status: 'fail', summary: `CLOCK_SKEWED${measured}`, remedy };
+  if (sample.clockSynced === 0 && sample.timeService === 0)
+    return {
+      id,
+      status: 'fail',
+      summary: `CLOCK_NOT_SYNCHRONISED: no time service is enabled${measured}`,
+      remedy,
+    };
+  if (sample.clockSynced === 0)
+    return {
+      id,
+      status: 'warn',
+      summary: `CLOCK_NOT_SYNCHRONISED${measured}`,
+      remedy,
+    };
+  if (offset !== undefined && offset > warnOffsetSeconds)
+    return { id, status: 'warn', summary: `CLOCK_DRIFTING${measured}`, remedy };
+  return {
+    id,
+    status: 'pass',
+    summary: `The clock is synchronised to a time source${measured}.`,
+  };
+}
+
+function clockRemedy(timeService: number | undefined): { action: string } {
+  return {
+    action:
+      timeService === 0
+        ? 'Enable the time service: apply the approved fleet-sync revision, which keeps chrony or systemd-timesyncd enabled on workers (timedatectl set-ntp true is the same setting).'
+        : 'Check that the time service reaches a time server (timedatectl timesync-status) and that UDP port 123 is not blocked.',
+  };
+}

package/src/definition.ts

@@ -1,4 +1,9 @@
-import { parseWorkerPolicy, type WorkerPolicy } from './worker-policy';
+import {
+  parseWorkerPolicy,
+  toolProfiles,
+  type WorkerPolicy,
+} from './worker-policy';
+import type { ToolProfile } from '../bootstrap/worker-tools';
 import { readFileSync } from 'node:fs';
 import { FleetUsageError } from './errors';
 import { parseConfig, record } from './config';
@@ -11,11 +16,14 @@ import {
   type IdeId,
   type T3Definition,
 } from './ide/definition';
+import { parseNotify, type NotifyConfig } from './notify';
 
 export interface MachineDefinition {
   id: string;
   ssh: string;
   os: 'linux' | 'macOS' | 'windows';
+  /** Tool profiles this worker adds to the fleet-wide ones. */
+  toolProfiles?: ToolProfile[];
   network?: {
     tailscaleName: string;
     user: string;
@@ -55,6 +63,8 @@ export interface FleetDefinition {
       managerAccount: string;
       /** Read-only Beszel account visitors are signed in as. */
       viewerAccount: string;
+      /** Scheduled off-machine backups fleet-sync sets on the hub. */
+      backups?: HubBackups;
     };
   };
   worker?: WorkerPolicy;
@@ -62,6 +72,12 @@ export interface FleetDefinition {
   ides?: IdeId[];
   /** What every machine's T3 holds beyond its provider instances. */
   t3?: T3Definition;
+  /** The Git identity and GitHub credential helper every node, manager
+   * included, commits and pushes with.
+   */
+  git?: GitIdentity;
+  /** Where fleet-watch sends notifications; absent, nobody is notified. */
+  notify?: NotifyConfig;
   ubuntu: { release: '26.04'; architectures: ['amd64', 'arm64'] };
 }
 /** How provider accounts are watched and chosen (src/accounts/). */
@@ -80,6 +96,29 @@ export interface AccountSelection {
    */
   order?: string[];
 }
+export interface HubBackups {
+  /** Cron expression of the hub's backup schedule. */
+  schedule: string;
+  /** How many scheduled backups the hub keeps. */
+  keep: number;
+  /** S3-compatible storage off the manager; its keys are secrets. */
+  storage: {
+    bucket: string;
+    endpoint: string;
+    region: string;
+    forcePathStyle?: boolean;
+  };
+}
+export interface GitIdentity {
+  /** `user.name` of every commit. */
+  name: string;
+  /** `user.email` of every commit. */
+  email: string;
+  /** A repository each node must reach through git with its own
+   * credentials, proving it can push (`worker:git-push-auth`).
+   */
+  verifyRepository?: string;
+}
 const ID = /^[a-z][a-z0-9-]{0,62}$/;
 const HOST = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$/;
 function fail(): never {
@@ -116,7 +155,10 @@ function validPort(value: unknown): boolean {
   );
 }
 function worker(value: unknown): MachineDefinition {
-  if (!record(value) || !keys(value, ['id', 'ssh', 'os', 'network']))
+  if (
+    !record(value) ||
+    !keys(value, ['id', 'ssh', 'os', 'network', 'toolProfiles'])
+  )
     return fail();
   if (typeof value['id'] !== 'string' || !ID.test(value['id'])) return fail();
   if (typeof value['ssh'] !== 'string' || !HOST.test(value['ssh']))
@@ -128,6 +170,9 @@ function worker(value: unknown): MachineDefinition {
     ssh: value['ssh'],
     os: 'linux',
     ...(value['network'] ? { network: network(value['network']) } : {}),
+    ...(value['toolProfiles'] === undefined
+      ? {}
+      : { toolProfiles: toolProfiles(value['toolProfiles']) ?? fail() }),
   };
 }
 function manager(value: unknown): ManagerDefinition {
@@ -176,6 +221,8 @@ export function parseDefinition(value: unknown): FleetDefinition {
       'worker',
       'ides',
       't3',
+      'git',
+      'notify',
     ])
   )
     return fail();
@@ -194,12 +241,25 @@ export function parseDefinition(value: unknown): FleetDefinition {
     workers,
     services: services(value['services']),
     ubuntu: baseline(value['ubuntu']),
+    ...optionalSections(value),
+  };
+}
+/** The sections a fleet may leave out, kept only when named: an earlier
+ * revision must still read the copy.
+ */
+function optionalSections(
+  value: Record<string, unknown>
+): Pick<FleetDefinition, 'worker' | 'ides' | 't3' | 'git' | 'notify'> {
+  return {
     ...(value['worker'] === undefined
       ? {}
       : { worker: parseWorkerPolicy(value['worker']) }),
-    // Kept only when named: an earlier revision must still read the copy.
     ...(value['ides'] === undefined ? {} : { ides: parseIdes(value['ides']) }),
     ...(value['t3'] === undefined ? {} : { t3: parseT3(value['t3']) }),
+    ...(value['git'] === undefined ? {} : { git: gitIdentity(value['git']) }),
+    ...(value['notify'] === undefined
+      ? {}
+      : { notify: parseNotify(value['notify']) ?? fail() }),
   };
 }
 /** Singleton services name no host: they always run on the manager. */
@@ -271,19 +331,96 @@ function accountSelection(entry: unknown): AccountSelection {
 function monitoringService(
   value: unknown
 ): FleetDefinition['services']['monitoring'] {
-  const fields = ['hubPort', 'publicUrl', 'managerAccount', 'viewerAccount'];
+  const fields = [
+    'hubPort',
+    'publicUrl',
+    'managerAccount',
+    'viewerAccount',
+    'backups',
+  ];
   if (!record(value) || !keys(value, fields) || value['hubPort'] !== 8443)
     return fail();
   const { publicUrl, managerAccount, viewerAccount } = value;
   if (!origin(publicUrl) || !account(managerAccount) || !account(viewerAccount))
     return fail();
-  return { hubPort: 8443, publicUrl, managerAccount, viewerAccount };
+  return {
+    hubPort: 8443,
+    publicUrl,
+    managerAccount,
+    viewerAccount,
+    ...(value['backups'] === undefined
+      ? {}
+      : { backups: hubBackups(value['backups']) }),
+  };
+}
+const text = (value: unknown, pattern: RegExp): value is string =>
+  typeof value === 'string' && pattern.test(value);
+function hubBackups(value: unknown): HubBackups {
+  if (!record(value) || !keys(value, ['schedule', 'keep', 'storage']))
+    return fail();
+  const { schedule, keep } = value;
+  if (
+    !text(schedule, /^[0-9*/,-]+( [0-9*/,-]+){4}$/) ||
+    !Number.isInteger(keep) ||
+    Number(keep) < 1 ||
+    Number(keep) > 365
+  )
+    return fail();
+  return {
+    schedule,
+    keep: Number(keep),
+    storage: backupStorage(value['storage']),
+  };
+}
+function backupStorage(value: unknown): HubBackups['storage'] {
+  if (
+    !record(value) ||
+    !keys(value, ['bucket', 'endpoint', 'region', 'forcePathStyle'])
+  )
+    return fail();
+  const { bucket, endpoint, region, forcePathStyle } = value;
+  if (
+    !text(bucket, /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/) ||
+    !text(endpoint, /^https:\/\/[A-Za-z0-9.-]{1,253}(:[0-9]{1,5})?$/) ||
+    !text(region, /^[a-z0-9-]{1,32}$/) ||
+    (forcePathStyle !== undefined && typeof forcePathStyle !== 'boolean')
+  )
+    return fail();
+  return {
+    bucket,
+    endpoint,
+    region,
+    ...(forcePathStyle === undefined ? {} : { forcePathStyle }),
+  };
 }
 const origin = (value: unknown): value is string =>
   typeof value === 'string' && /^https:\/\/[A-Za-z0-9.-]{1,253}$/.test(value);
 const account = (value: unknown): value is string =>
   typeof value === 'string' &&
   /^[^\s@]{1,64}@[A-Za-z0-9.-]{1,253}$/.test(value);
+function gitIdentity(value: unknown): GitIdentity {
+  if (!record(value) || !keys(value, ['name', 'email', 'verifyRepository']))
+    return fail();
+  const { name, email, verifyRepository } = value;
+  // Written into a quoted Git configuration value: no quote, backslash or
+  // line break.
+  if (typeof name !== 'string' || !/^[^"\\\n\r]{1,100}$/.test(name))
+    return fail();
+  if (!account(email)) return fail();
+  if (
+    verifyRepository !== undefined &&
+    (typeof verifyRepository !== 'string' ||
+      !/^https:\/\/github\.com\/[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/.test(
+        verifyRepository
+      ))
+  )
+    return fail();
+  return {
+    name,
+    email,
+    ...(verifyRepository === undefined ? {} : { verifyRepository }),
+  };
+}
 function secrets(value: unknown): SecretsConfig {
   return parseSecretsConfig(value) ?? fail();
 }