@crvouga/dev-fleet 0.3.0 → 0.4.0

package/src/monitoring/manager-hub.ts

@@ -8,6 +8,7 @@ import { publicViewerIdentity, fleetAccount } from './hub-access';
 import { ensurePublicViewer, type HubLogin } from './hub-viewer';
 import { verifyHubBackups } from './hub-backup';
 import type { FleetDefinition } from '../definition';
+import { secretVia } from '../fleet-config';
 import { hostId, hubUrl } from '../topology';
 import { publicHubUrl, privateHub } from './beszel';
 import type { Check, Runner } from '../types';
@@ -56,6 +57,9 @@ export interface ManagerHubOptions {
   receipts?: string;
   /** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
   viewer?: (login: HubLogin | null) => Promise<Check>;
+  /** The declared backups for the hub's playbook; defaults to the fleet
+   * configuration and its secrets. */
+  backupStorage?: () => Promise<Record<string, string | number | boolean>>;
   /** Reads the hub's backup settings; defaults to the loopback hub API. */
   backups?: (login: HubLogin | null) => Promise<Check>;
 }
@@ -78,13 +82,19 @@ export async function convergeManagerHub(
   const paths = managerHubPaths(options.home);
   const tunnel = await tunnelToken(run);
   const account = await (options.account ?? fleetAccount)();
+  const backups = await (
+    options.backupStorage ?? (() => backupVariables(definition, run))
+  )();
   const monitoring = resolve(root, 'ansible/monitoring');
   const result = await runAnsibleWithReceipt(
     run,
     'hub-darwin.yml',
     {
       cwd: monitoring,
-      input: JSON.stringify(hubVariables(hub, paths, tunnel, account)),
+      input: JSON.stringify({
+        ...hubVariables(hub, paths, tunnel, account),
+        ...backups,
+      }),
       timeoutMs: 300_000,
     },
     {
@@ -127,7 +137,13 @@ export async function convergeManagerHub(
     },
     publicEndpointCheck(tunnel !== null),
     await ensureViewer(options, account),
-    await (options.backups ?? verifyHubBackups)(account),
+    await (
+      options.backups ??
+      ((login) =>
+        verifyHubBackups(login, fetch, LOCAL_HUB, {
+          declared: definition.services.monitoring.backups,
+        }))
+    )(account),
   ];
 }
 
@@ -153,6 +169,32 @@ function hubVariables(
   };
 }
 
+/** The declared backup schedule and storage, with the storage's keys from
+ * the fleet's secrets; nothing when no backups are declared or a key is
+ * missing (beszel:backup then says so).
+ */
+export async function backupVariables(
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Record<string, string | number | boolean>> {
+  const declared = definition.services.monitoring.backups;
+  if (!declared) return {};
+  const secret = secretVia(run);
+  const accessKey = await secret('monitoring-backup-access-key');
+  const secretKey = await secret('monitoring-backup-secret-key');
+  if (!accessKey || !secretKey) return {};
+  return {
+    beszel_backup_schedule: declared.schedule,
+    beszel_backup_keep: declared.keep,
+    beszel_backup_bucket: declared.storage.bucket,
+    beszel_backup_endpoint: declared.storage.endpoint,
+    beszel_backup_region: declared.storage.region,
+    beszel_backup_force_path_style: declared.storage.forcePathStyle ?? false,
+    beszel_backup_access_key: accessKey,
+    beszel_backup_secret_key: secretKey,
+  };
+}
+
 function ensureViewer(
   options: ManagerHubOptions,
   account: HubLogin | null

package/src/node-converge-linux.ts

@@ -22,6 +22,8 @@ import {
 import { userInfo } from 'node:os';
 import { resolve } from 'node:path';
 import { auditUbuntu } from './platform/ubuntu-audit';
+import { convergeGitIdentity } from './git-identity';
+import { workerSelection } from '../bootstrap/worker-tools';
 import {
   applyAgentProfiles,
   attempt,
@@ -59,7 +61,7 @@ export async function convergeLinux(
       await applyTailscale(root, run),
     ]))
   );
-  const { baseline, verified } = await baselineStep(root, run);
+  const { baseline, verified } = await baselineStep(root, run, node);
   checks.push(...baseline, ...verified);
   checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
   checks.push(
@@ -67,6 +69,11 @@ export async function convergeLinux(
       configureWorkstation(root, run)
     ))
   );
+  checks.push(
+    ...(await attempt('git:configuration', () =>
+      convergeGitIdentity(root, fleet.git, run)
+    ))
+  );
   checks.push(
     ...(await attempt('configuration:apply', () =>
       applyAgentProfiles(root, node, revision, run)
@@ -110,7 +117,8 @@ export async function convergeLinux(
  */
 async function baselineStep(
   root: string,
-  run: Runner
+  run: Runner,
+  node: string
 ): Promise<{ baseline: Check[]; verified: Check[] }> {
   const measure = async () => [
     ...(await attempt('ubuntu:boot-policy', () =>
@@ -122,14 +130,14 @@ async function baselineStep(
   ];
   let cached = false;
   let baseline = await attempt('ubuntu:baseline', async () => {
-    const applied = await applyBaseline(root, run, false);
+    const applied = await applyBaseline(root, run, false, node);
     cached = applied.cached;
     return applied.checks;
   });
   let verified = await measure();
   if (cached && verified.some((check) => check.status === 'fail')) {
     baseline = await attempt('ubuntu:baseline', async () => {
-      return (await applyBaseline(root, run, true)).checks;
+      return (await applyBaseline(root, run, true, node)).checks;
     });
     verified = await measure();
   }
@@ -139,7 +147,8 @@ async function baselineStep(
 async function applyBaseline(
   root: string,
   run: Runner,
-  force: boolean
+  force: boolean,
+  node: string
 ): Promise<{ cached: boolean; checks: Check[] }> {
   const user = userInfo();
   const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -154,6 +163,8 @@ async function applyBaseline(
         development_user: user.username,
         development_uid: user.uid,
         development_home: user.homedir,
+        // The web profile brings headless Chromium (tasks/browser.yml).
+        worker_tool_profiles: workerSelection(fleet, node).profiles,
       }),
     },
     {

package/src/notify.ts

@@ -0,0 +1,239 @@
+import { record } from './config';
+import type { SecretName, SecretsProvider } from './secrets';
+import type { Check, NodeReport } from './types';
+
+/** Where fleet-watch sends a notification. The address of each target is a
+ * secret, read by purpose: a webhook or ntfy URL often carries its own token.
+ */
+export const NOTIFY_TARGETS = {
+  webhook: 'notify-webhook-url',
+  ntfy: 'notify-ntfy-url',
+} as const satisfies Record<string, SecretName>;
+export type NotifyTarget = keyof typeof NOTIFY_TARGETS;
+
+export interface NotifyConfig {
+  targets: NotifyTarget[];
+  /** Notify again about a machine that stays unhealthy this often; 0 never. */
+  repeatHours: number;
+}
+
+const DEFAULT_REPEAT_HOURS = 24;
+
+/** The `notify` block of a fleet configuration, or null when it is invalid. */
+export function parseNotify(value: unknown): NotifyConfig | null {
+  if (!record(value)) return null;
+  if (
+    Object.keys(value).some((key) => !['targets', 'repeatHours'].includes(key))
+  )
+    return null;
+  const { targets, repeatHours = DEFAULT_REPEAT_HOURS } = value;
+  if (
+    !Array.isArray(targets) ||
+    targets.length === 0 ||
+    new Set(targets).size !== targets.length ||
+    !targets.every(
+      (target) =>
+        typeof target === 'string' && Object.hasOwn(NOTIFY_TARGETS, target)
+    ) ||
+    typeof repeatHours !== 'number' ||
+    !Number.isInteger(repeatHours) ||
+    repeatHours < 0 ||
+    repeatHours > 720
+  )
+    return null;
+  return { targets: targets as NotifyTarget[], repeatHours };
+}
+
+/** What the watcher remembers of each machine between two runs. */
+export interface WatchMemory {
+  version: 1;
+  /** When the watcher last ran: the heartbeat an off-machine check reads. */
+  heartbeatAt: string;
+  nodes: Record<
+    string,
+    { state: 'ready' | 'degraded'; since: string; notifiedAt: string | null }
+  >;
+}
+
+/** One machine as fleet-status saw it: the IDs of its failing checks. */
+export interface Observation {
+  node: string;
+  failing: string[];
+}
+
+export interface Notification {
+  event: 'degraded' | 'recovered' | 'still-degraded' | 'test';
+  node: string;
+  checks: string[];
+}
+
+/** Only what a report concluded: the required checks that failed. */
+export function observe(nodes: NodeReport[]): Observation[] {
+  return nodes.map((node) => ({
+    node: node.node,
+    failing: node.checks
+      .filter((check) => check.status === 'fail' && check.required !== false)
+      .map((check) => check.id),
+  }));
+}
+
+/** Notifications on transitions only (ready to degraded and back), and a
+ * reminder for a machine that stays degraded longer than the repeat interval.
+ * A machine never seen before counts as ready.
+ */
+export function transitions(
+  previous: WatchMemory | null,
+  observed: Observation[],
+  now: Date,
+  repeatHours: number
+): { notifications: Notification[]; memory: WatchMemory } {
+  const at = now.toISOString();
+  const notifications: Notification[] = [];
+  const nodes: WatchMemory['nodes'] = {};
+  for (const { node, failing } of observed) {
+    const before = previous?.nodes[node];
+    const state = failing.length ? 'degraded' : 'ready';
+    const checks = failing.filter(safe);
+    if (state === (before?.state ?? 'ready')) {
+      const due = state === 'degraded' && repeatDue(before, now, repeatHours);
+      if (due) notifications.push({ event: 'still-degraded', node, checks });
+      nodes[node] = before
+        ? { ...before, ...(due ? { notifiedAt: at } : {}) }
+        : { state, since: at, notifiedAt: null };
+      continue;
+    }
+    notifications.push({
+      event: state === 'degraded' ? 'degraded' : 'recovered',
+      node,
+      checks,
+    });
+    nodes[node] = { state, since: at, notifiedAt: at };
+  }
+  return { notifications, memory: { version: 1, heartbeatAt: at, nodes } };
+}
+
+function repeatDue(
+  before: WatchMemory['nodes'][string] | undefined,
+  now: Date,
+  repeatHours: number
+): boolean {
+  if (repeatHours <= 0 || !before?.notifiedAt) return false;
+  return (
+    now.getTime() - Date.parse(before.notifiedAt) >= repeatHours * 3_600_000
+  );
+}
+
+/** The memory stored last time, or null when there is none to trust. */
+export function parseMemory(text: string | null): WatchMemory | null {
+  if (!text) return null;
+  try {
+    const value: unknown = JSON.parse(text);
+    if (
+      !record(value) ||
+      value['version'] !== 1 ||
+      typeof value['heartbeatAt'] !== 'string' ||
+      !record(value['nodes'])
+    )
+      return null;
+    return value as unknown as WatchMemory;
+  } catch {
+    return null;
+  }
+}
+
+/** Node and check IDs are the only words a notification carries. */
+const safe = (id: string): boolean => /^[A-Za-z0-9:._@-]{1,128}$/.test(id);
+
+const WORDS: Record<Notification['event'], string> = {
+  degraded: 'is degraded',
+  recovered: 'is ready again',
+  'still-degraded': 'is still degraded',
+  test: 'sent a test notification',
+};
+
+export function notificationText(notification: Notification): string {
+  const node = safe(notification.node) ? notification.node : 'a machine';
+  const checks = notification.checks.filter(safe);
+  return `dev-fleet: ${node} ${WORDS[notification.event]}${
+    checks.length ? `. Failing checks: ${checks.join(', ')}` : ''
+  }`;
+}
+
+/** Sends one notification to every target. The result names the target and
+ * whether it took the notification: never its address or its answer.
+ */
+export async function deliver(
+  config: NotifyConfig,
+  notification: Notification,
+  secrets: SecretsProvider,
+  send: typeof fetch = fetch
+): Promise<Check[]> {
+  const text = notificationText(notification);
+  const checks: Check[] = [];
+  for (const target of config.targets) {
+    const id = `notify:${target}`;
+    const url = await secrets(NOTIFY_TARGETS[target]);
+    if (!url || !/^https?:\/\//.test(url)) {
+      checks.push({
+        id,
+        status: 'fail',
+        summary: 'NOTIFY_TARGET_UNAVAILABLE',
+        remedy: {
+          action: `Store the address of the ${target} target as the secret "${NOTIFY_TARGETS[target]}" with the secret provider fleet.json names.`,
+        },
+      });
+      continue;
+    }
+    const ok = await send(url, request(target, notification, text))
+      .then((response) => response.ok)
+      .catch(() => false);
+    checks.push(
+      ok
+        ? {
+            id,
+            status: 'pass',
+            summary: `The ${target} target took the notification.`,
+          }
+        : {
+            id,
+            status: 'fail',
+            summary: 'NOTIFY_DELIVERY_FAILED',
+            remedy: {
+              action: `Check that the ${target} target answers, then run devfleet fleet-watch --test.`,
+            },
+          }
+    );
+  }
+  return checks;
+}
+
+function request(
+  target: NotifyTarget,
+  notification: Notification,
+  text: string
+): RequestInit {
+  const signal = AbortSignal.timeout(15_000);
+  if (target === 'ntfy')
+    return {
+      method: 'POST',
+      body: text,
+      headers: {
+        Title: `dev-fleet: ${notification.event}`,
+        Tags:
+          notification.event === 'recovered' ? 'white_check_mark' : 'warning',
+      },
+      signal,
+    };
+  return {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      source: 'dev-fleet',
+      event: notification.event,
+      node: safe(notification.node) ? notification.node : null,
+      checks: notification.checks.filter(safe),
+      text,
+    }),
+    signal,
+  };
+}

package/src/remote-bootstrap.ts

@@ -4,7 +4,22 @@ import { join } from 'node:path';
 import { prepareSource } from './machine/source';
 import { isRevision } from './machine/state';
 import { sshArgs, shellQuote, sshFailure } from './ssh';
-import type { FleetNode, NodeReport, Runner, Check } from './types';
+import type {
+  FleetNode,
+  NodeReport,
+  Runner,
+  Check,
+  CommandResult,
+} from './types';
+import {
+  followOperation,
+  realClock,
+  retrying,
+  RETRY_PAUSES_MS,
+  transient,
+  type Clock,
+  type Follow,
+} from './remote-operation';
 import { record } from './config';
 import { installedDefinition, type FleetDefinition } from './definition';
 import { cleanCheck, validCheck } from './protocol';
@@ -94,9 +109,11 @@ async function bundleRevision(
 export async function bootstrapRemote(
   node: FleetNode,
   payload: BootstrapPayload,
-  run: Runner
+  run: Runner,
+  follow: Follow = {}
 ): Promise<NodeReport> {
-  const cached = await cachedRevision(node, payload.revision, run);
+  const clock = follow.clock ?? realClock;
+  const cached = await cachedRevision(node, payload.revision, run, clock);
   if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
   const transfer = cached
     ? { mode: 'cached', bundle: '' }
@@ -113,27 +130,68 @@ export async function bootstrapRemote(
   ]
     .map(shellQuote)
     .join(' ');
-  const result = await run([...sshArgs(node), command], {
-    input: bootstrapInput(payload.config, transfer.bundle),
-    timeoutMs: 5_400_000,
-  });
-  const evidence = parseEvidence(result);
-  if (evidence) return { node: node.id, checks: evidence };
-  if (!result.ok) return { node: node.id, checks: [sshFailure(result)] };
-  return {
-    node: node.id,
-    checks: [
-      {
-        id: 'bootstrap:protocol',
-        status: 'fail',
-        summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
-        remedy: {
-          action:
-            'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
-        },
+  const checks = await bootstrapSessions(
+    node,
+    payload,
+    { command, input: bootstrapInput(payload.config, transfer.bundle) },
+    { run, follow: { ...follow, clock } }
+  );
+  return { node: node.id, checks };
+}
+
+/** Opens the bootstrap session, and again after a lost one unless the
+ * worker's own operation can be followed to its end instead.
+ */
+async function bootstrapSessions(
+  node: FleetNode,
+  payload: BootstrapPayload,
+  session: { command: string; input: string },
+  context: { run: Runner; follow: Follow & { clock: Clock } }
+): Promise<Check[]> {
+  const { run, follow } = context;
+  let result: CommandResult | undefined;
+  for (const pause of [0, ...RETRY_PAUSES_MS]) {
+    if (result) {
+      // The session was lost, not the operation: the worker keeps running
+      // it, so the manager follows it rather than starting another.
+      await follow.clock.pause(pause);
+      const followed = await followOperation(
+        node,
+        payload.revision,
+        run,
+        follow
+      );
+      if (followed) return followed;
+    }
+    result = await run([...sshArgs(node), session.command], {
+      input: session.input,
+      timeoutMs: 5_400_000,
+    });
+    const evidence = parseEvidence(result);
+    if (evidence && locked(evidence))
+      return (
+        (await followOperation(node, payload.revision, run, follow)) ?? evidence
+      );
+    if (evidence) return evidence;
+    if (result.reason !== 'exit' || !transient(result)) break;
+  }
+  if (!result!.ok) return [sshFailure(result!)];
+  return [
+    {
+      id: 'bootstrap:protocol',
+      status: 'fail',
+      summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
+      remedy: {
+        action:
+          'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
       },
-    ],
-  };
+    },
+  ];
+}
+
+/** Another operation holds the worker: its lock was the first thing tried. */
+function locked(checks: Check[]): boolean {
+  return checks.some((check) => check.summary === 'FLEET_REMOTE_LOCKED');
 }
 
 /** What the bootstrap reads on stdin: one line with the fleet configuration
@@ -151,10 +209,14 @@ export function bootstrapInput(
 async function cachedRevision(
   node: FleetNode,
   revision: string,
-  run: Runner
+  run: Runner,
+  clock: Clock
 ): Promise<boolean | Check> {
   const command = `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
-  const result = await run([...sshArgs(node), command], { timeoutMs: 20_000 });
+  const result = await retrying(
+    () => run([...sshArgs(node), command], { timeoutMs: 20_000 }),
+    clock
+  );
   if (!result.ok) {
     const failure = sshFailure(result);
     if (result.reason !== 'exit' || failure.id !== 'ssh:probe') return failure;