@celilo/cli 0.14.1 → 0.14.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.14.1",
+  "version": "0.14.4",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -57,7 +57,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1024.0",
-    "@celilo/capabilities": "^0.9.0",
+    "@celilo/capabilities": "^0.9.1",
     "@celilo/cli-display": "^0.1.9",
     "@celilo/core": "^0.1.0",
     "@celilo/event-bus": "^0.1.8",

package/src/capabilities/public-web-publish.test.ts

@@ -516,7 +516,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
     };
   }
 
-  test('registers the public A record at externalIp for a NEW hostname', async () => {
+  test('registers the public A record for a NEW hostname without supplying an IP', async () => {
     const { ops } = makeRouteOps();
     const { registrar, calls } = makeRegistrar();
     const cap = createPublicWeb({
@@ -530,7 +530,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      externalIp: '100.100.0.100',
     });
 
     await cap.register_route({
@@ -539,7 +538,9 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       hostname: 'nexus.example.com',
     });
 
-    expect(calls).toEqual([{ fqdn: 'nexus.example.com', ip: '100.100.0.100' }]);
+    // No `ip` — the registrar publishes the source address of the update
+    // request, which is by construction the address the internet must dial.
+    expect(calls).toEqual([{ fqdn: 'nexus.example.com', ip: undefined }]);
   });
 
   test('skips the default hostname (caddy already registered it at install)', async () => {
@@ -556,7 +557,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      externalIp: '100.100.0.100',
     });
 
     await cap.register_route({ type: 'static', path: '/', hostname: 'www.example.com' });
@@ -568,7 +568,13 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
   // hostname that can't be wired FAILS the deploy instead of silently
   // reporting success (the "served but unreachable" anti-pattern).
 
-  test('fails loudly when externalIp is unknown for a NEW hostname', async () => {
+  // The regression guard for #464. This case used to THROW "caddy has no known
+  // external IP" — and it threw on every real fleet, because the stored copy it
+  // demanded was never populated: caddy produced a `public_ip` hook output that
+  // its manifest never declared, so the framework discarded it. A correct fleet
+  // with correct public DNS could not register a new hostname. Registration must
+  // not depend on knowing the external IP at all.
+  test('registers a NEW hostname even though no external IP is known anywhere', async () => {
     const { ops } = makeRouteOps();
     const { registrar, calls } = makeRegistrar();
     const cap = createPublicWeb({
@@ -582,13 +588,11 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      // externalIp omitted
     });
 
-    await expect(
-      cap.register_route({ type: 'static', path: '/', hostname: 'nexus.example.com' }),
-    ).rejects.toThrow('no known external IP');
-    expect(calls).toHaveLength(0);
+    await cap.register_route({ type: 'static', path: '/', hostname: 'nexus.example.com' });
+
+    expect(calls).toEqual([{ fqdn: 'nexus.example.com', ip: undefined }]);
   });
 
   test('fails loudly when no dns_registrar is available for a NEW hostname', async () => {
@@ -602,8 +606,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       hostnames: ['www.example.com'],
       caddyModuleId: 'caddy',
       dnsManagedDomains: ['example.com'],
-      // dnsRegistrar + externalIp both omitted
-      externalIp: '100.100.0.100',
+      // dnsRegistrar omitted
     });
 
     await expect(
@@ -629,7 +632,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      externalIp: '100.100.0.100',
     });
 
     await expect(
@@ -656,7 +658,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      externalIp: '100.100.0.100',
       // biome-ignore lint/suspicious/noExplicitAny: minimal dns_internal stub
       dnsInternal: dnsInternal as any,
     });
@@ -687,7 +688,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
       dnsManagedDomains: ['example.com'],
       // biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
       dnsRegistrar: registrar as any,
-      externalIp: '100.100.0.100',
       firewallNatIp: '192.168.0.253',
       // biome-ignore lint/suspicious/noExplicitAny: minimal dns_internal stub
       dnsInternal: dnsInternal as any,

package/src/cli/commands/alerts-sweep.ts

@@ -152,5 +152,35 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
     `${report.unsuppressed} unsuppressed`,
     `${report.notified} notified`,
   ];
-  return { success: true, message: `alert sweep: ${parts.join(', ')}` };
+  // Only shown when non-zero: a quiet sweep should stay quiet. But a delivery
+  // that failed, was deferred, or was declined must never render as `0 notified`
+  // and nothing else — that is indistinguishable from "nothing needed sending",
+  // which is exactly how a transport that has stopped paging looks like a quiet
+  // night (#450).
+  if (report.deferred > 0) parts.push(`${report.deferred} deferred`);
+  if (report.deferredDelivered > 0) parts.push(`${report.deferredDelivered} deferred-delivered`);
+  if (report.failed > 0) parts.push(`${report.failed} FAILED`);
+  if (report.noPolicy > 0) parts.push(`${report.noPolicy} no-policy`);
+
+  const lines = [`alert sweep: ${parts.join(', ')}`];
+
+  // The reason escalation declined is the single most useful fact when someone
+  // asks "why was I not paged", so name it rather than aggregating it away.
+  const skipped = Object.entries(report.skipped).sort(([, a], [, b]) => b - a);
+  if (skipped.length > 0) {
+    lines.push(`  not delivered: ${skipped.map(([r, n]) => `${r}×${n}`).join(', ')}`);
+  }
+  // The error itself, not just a count: the transport is loaded lazily inside
+  // the send, so a capability that will not load produces no other record
+  // anywhere — nothing ever reaches the transport's own logs.
+  for (const failure of report.failures) {
+    lines.push(`  FAILED ${failure}`);
+  }
+  if (report.noPolicy > 0) {
+    lines.push(
+      `  ${report.noPolicy} live alert(s) have no escalation policy — assign one with:\n    celilo escalation-policy assign <policy> <monitor>`,
+    );
+  }
+
+  return { success: true, message: lines.join('\n') };
 }

package/src/cli/commands/system-audit.ts

@@ -52,7 +52,7 @@ import type { TerraformPlanRunner } from '../../services/audit/terraform-plan';
 import { getServiceCredentials, listContainerServices } from '../../services/container-service';
 import { collectFirewallReach } from '../../services/firewall-reach';
 import { runAllHealthChecks } from '../../services/health-runner';
-import { listMachines } from '../../services/machine-pool';
+import { probeMachines } from '../../services/machine-probe';
 import { parseStoredConfigValue } from '../../services/module-config';
 import { buildTerraformEnvForModule } from '../../services/terraform-env';
 import { hasFlag } from '../parser';
@@ -353,43 +353,11 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
     }),
   );
 
-  // Machines-reachable: SSH probe each pool machine in parallel.
-  // BatchMode=yes prevents password prompt hangs; ConnectTimeout=5
-  // bounds wait on unresponsive hosts. The audit's collapse logic
-  // turns "all unreachable" into a single host/network finding.
-  const allMachines = await listMachines();
-  const machineReachableResults: MachineReachableResult[] = await Promise.all(
-    allMachines.map(async (m): Promise<MachineReachableResult> => {
-      try {
-        await execFileAsync(
-          'ssh',
-          [
-            '-o',
-            'BatchMode=yes',
-            '-o',
-            'ConnectTimeout=5',
-            '-o',
-            'StrictHostKeyChecking=no',
-            '-o',
-            'UserKnownHostsFile=/dev/null',
-            `${m.sshUser}@${m.ipAddress}`,
-            'true',
-          ],
-          { timeout: 8000 },
-        );
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
-      } catch (err) {
-        const e = err as { stderr?: string; message?: string };
-        return {
-          id: m.id,
-          hostname: m.hostname,
-          ipAddress: m.ipAddress,
-          reachable: false,
-          message: (e.stderr || e.message || 'unknown error').slice(0, 200),
-        };
-      }
-    }),
-  );
+  // Machines-reachable: probe each pool machine. Shared with the monitor sweep
+  // (services/machine-probe.ts) — this block used to be a second copy of the
+  // same SSH logic, and both copies reported the local management box as
+  // unreachable because celilo does not hold an SSH key for itself.
+  const machineReachableResults: MachineReachableResult[] = await probeMachines();
 
   const migrationsFolder = findMigrationsFolderSafe();
 

package/src/hooks/capability-loader.ts

@@ -361,17 +361,6 @@ export async function loadCapabilityFunctions(
       debugLog(`public_web: using firewall natIp ${firewallNatIp} for internal DNS`);
     }
 
-    // Caddy's external (WAN) IP for the D1 public A record (M1 / #328). Caddy
-    // learns it from its own firewall.exposeService at install and persists it
-    // as the `public_ip` hook output (stored as a caddy secret). register_route
-    // points a new hostname's PUBLIC record here, so it's reachable from one
-    // deploy — no per-hostname DNAT (shared :443 ingress).
-    const caddyExternalIp =
-      typeof providerSecrets.public_ip === 'string' ? providerSecrets.public_ip : undefined;
-    if (caddyExternalIp) {
-      debugLog(`public_web: using caddy externalIp ${caddyExternalIp} for public DNS`);
-    }
-
     // Caddy's configured hostnames — public_web rejects routes for any
     // hostname not in this list, throwing a structured error that runs
     // caddy's `managed_hostname` ensure interview. Empty list means
@@ -484,7 +473,6 @@ export async function loadCapabilityFunctions(
         dnsInternal: result.dns_internal as DnsInternalCapability | undefined,
         firewallNatIp,
         dnsRegistrar: result.dns_registrar as DnsRegistrarCapability | undefined,
-        externalIp: caddyExternalIp,
         hostnames: caddyHostnames,
         caddyModuleId: provider.moduleId,
         dnsManagedDomains,

package/src/services/alerting/builtin-source.ts

@@ -12,14 +12,9 @@
  * as "nothing is wrong".
  */
 
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
 import { auditMachinesReachable } from '../audit/machines-reachable';
-import type { MachineReachableResult } from '../audit/machines-reachable';
 import type { DriftCategory, DriftFinding } from '../audit/types';
-import { listMachines } from '../machine-pool';
-
-const execFileAsync = promisify(execFile);
+import { probeMachines } from '../machine-probe';
 
 /** Categories a monitor can currently schedule. */
 export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = ['machines_reachable'];
@@ -28,49 +23,6 @@ export function isSchedulableBuiltin(category: string): category is DriftCategor
   return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
 }
 
-/**
- * SSH-probe every pool machine.
- *
- * `BatchMode=yes` prevents a password prompt from hanging the probe forever,
- * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
- * condition this check exists to detect must not be the one that wedges it.
- */
-async function probeMachines(): Promise<MachineReachableResult[]> {
-  const machines = await listMachines();
-  return Promise.all(
-    machines.map(async (m): Promise<MachineReachableResult> => {
-      try {
-        await execFileAsync(
-          'ssh',
-          [
-            '-o',
-            'BatchMode=yes',
-            '-o',
-            'ConnectTimeout=5',
-            '-o',
-            'StrictHostKeyChecking=no',
-            '-o',
-            'UserKnownHostsFile=/dev/null',
-            `${m.sshUser}@${m.ipAddress}`,
-            'true',
-          ],
-          { timeout: 8000 },
-        );
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
-      } catch (err) {
-        const e = err as { stderr?: string; message?: string };
-        return {
-          id: m.id,
-          hostname: m.hostname,
-          ipAddress: m.ipAddress,
-          reachable: false,
-          message: e.stderr?.trim() || e.message || 'SSH probe failed',
-        };
-      }
-    }),
-  );
-}
-
 export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
   if (category === 'machines_reachable') {
     return auditMachinesReachable({ results: await probeMachines() });

package/src/services/alerting/sweep-runner.test.ts

@@ -226,4 +226,94 @@ describe('runSweep', () => {
     expect(db.select().from(monitors).get()?.lastRunAt).toEqual(NOW);
     expect(monitor.lastRunAt).toBeNull();
   });
+
+  // A delivery that never happened must be distinguishable from one that was
+  // never needed. Both used to render as `notified: 0` and nothing else, which
+  // is how a firing-but-undelivered alert became undebuggable (#450).
+  describe('undelivered alerts are accounted for, not silently dropped', () => {
+    test('an alert with no escalation policy is counted, not skipped in silence', async () => {
+      // `notifyDepsFor` returning null IS "nobody is configured to be told" —
+      // the default in every other test here, which is why this went unnoticed.
+      const report = await runSweep(db, currentMonitors(), deps());
+
+      expect(liveAlerts()).toHaveLength(1);
+      expect(report.notified).toBe(0);
+      expect(report.noPolicy).toBe(1);
+    });
+
+    test('escalation declining to notify records WHICH reason', async () => {
+      // A fresh alert is inside its grace window, so escalation declines with
+      // `within_grace` — a real skip reason reached through the real code path
+      // rather than a stubbed outcome.
+      const notifyDeps = {
+        steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }],
+        routes: new Map([['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }]]),
+        routeDetails: new Map([
+          [
+            'route-1',
+            { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never,
+          ],
+        ]),
+        quietHoursByPerson: new Map(),
+        bypassQuietHours: false,
+        transportFor: () => {
+          throw new Error('transport must not be reached for a skipped delivery');
+        },
+        mintToken: () => 'tok',
+        now: NOW,
+      } as never;
+
+      const report = await runSweep(
+        db,
+        currentMonitors(),
+        deps({ notifyDepsFor: () => notifyDeps }),
+      );
+
+      expect(report.notified).toBe(0);
+      expect(report.noPolicy).toBe(0);
+      expect(report.skipped.within_grace).toBe(1);
+    });
+
+    test('a transport that cannot be loaded records the error, not just a count', async () => {
+      // The transport is resolved lazily INSIDE the send, so a capability that
+      // will not load never reaches the transport's own logs. If the sweep does
+      // not carry the message, it exists nowhere.
+      const notifyDeps = (now: Date) =>
+        ({
+          steps: [{ stepIndex: 0, routeId: 'route-1', delayMinutes: 0 }],
+          routes: new Map([
+            ['route-1', { id: 'route-1', severityFloor: 'warning', enabled: true }],
+          ]),
+          routeDetails: new Map([
+            [
+              'route-1',
+              { id: 'route-1', personId: 'p1', address: '+15550000000', canAck: false } as never,
+            ],
+          ]),
+          quietHoursByPerson: new Map(),
+          bypassQuietHours: false,
+          transportFor: () => {
+            throw new Error('does not provide the notification capability');
+          },
+          mintToken: () => 'tok',
+          now,
+        }) as never;
+
+      // First sweep creates the alert; the second is past the grace window, so
+      // escalation actually reaches the transport.
+      await runSweep(db, currentMonitors(), deps());
+      const at = later(20);
+      const report = await runSweep(
+        db,
+        currentMonitors(),
+        deps({ notifyDepsFor: () => notifyDeps(at) }, failing, at),
+      );
+
+      expect(report.notified).toBe(0);
+      expect(report.failed).toBe(1);
+      expect(report.failures).toHaveLength(1);
+      expect(report.failures[0]).toContain(PORT_CHECK);
+      expect(report.failures[0]).toContain('does not provide the notification capability');
+    });
+  });
 });

package/src/services/alerting/sweep-runner.ts

@@ -61,6 +61,34 @@ export interface SweepReport {
   /** Messages held over quiet hours and delivered now that the window ended. */
   deferredDelivered: number;
   failed: number;
+  /**
+   * Live alerts nobody is configured to be told about — no escalation policy on
+   * the monitor, so `notifyDepsFor` returns null.
+   *
+   * Counted rather than skipped in silence: "nothing needed sending" and "an
+   * alert is firing and no policy points at anyone" are opposite situations that
+   * previously rendered identically as `0 notified`.
+   */
+  noPolicy: number;
+  /**
+   * Deliveries escalation declined, keyed by its reason (`within_grace`,
+   * `no_eligible_route`, …).
+   *
+   * `notifyAlert` returns the reason precisely so the caller can record it — its
+   * own contract says a silent skip is indistinguishable from a bug. Dropping it
+   * here is what made a firing-but-undelivered alert undebuggable (#450).
+   */
+  skipped: Record<string, number>;
+  /**
+   * Why each failed delivery failed, as `<alert key>: <error>`.
+   *
+   * A count alone does not answer the only question that matters after a page
+   * did not arrive. The transport is loaded lazily *inside* `notifyAlert`'s try
+   * block, so "the capability would not load" and "Signal rejected the message"
+   * both surface here and nowhere else — there is no daemon-side log for the
+   * former, because nothing ever reached the daemon.
+   */
+  failures: string[];
 }
 
 /**
@@ -85,6 +113,9 @@ export async function runSweep(
     deferred: 0,
     deferredDelivered: 0,
     failed: 0,
+    noPolicy: 0,
+    skipped: {},
+    failures: [],
   };
 
   // 1. Run due monitors.
@@ -162,24 +193,36 @@ export async function runSweep(
     try {
       const outcome = await deliverDeferred(alert, route, notifyDeps);
       if (outcome.result === 'sent') report.deferredDelivered++;
-      else if (outcome.result === 'failed') report.failed++;
-    } catch {
+      else if (outcome.result === 'failed') {
+        report.failed++;
+        report.failures.push(`${alert.key} (deferred): ${outcome.error}`);
+      }
+    } catch (error) {
       report.failed++;
+      report.failures.push(
+        `${alert.key} (deferred): ${error instanceof Error ? error.message : String(error)}`,
+      );
     }
   }
 
   // 5. Notify. Re-read: the steps above changed state under us.
   for (const alert of loadAllLiveAlerts(db)) {
     const notifyDeps = deps.notifyDepsFor(alert);
-    if (!notifyDeps) continue;
+    if (!notifyDeps) {
+      report.noPolicy++;
+      continue;
+    }
 
     let outcome: NotifyOutcome;
     try {
       outcome = await notifyAlert(alert, notifyDeps);
-    } catch {
+    } catch (error) {
       // notifyAlert already converts transport errors into a `failed` outcome;
       // reaching here means something above the transport broke.
       report.failed++;
+      report.failures.push(
+        `${alert.key}: ${error instanceof Error ? error.message : String(error)}`,
+      );
       continue;
     }
 
@@ -197,6 +240,9 @@ export async function runSweep(
       report.deferred++;
     } else if (outcome.result === 'failed') {
       report.failed++;
+      report.failures.push(`${alert.key}: ${outcome.error}`);
+    } else if (outcome.result === 'skipped') {
+      report.skipped[outcome.reason] = (report.skipped[outcome.reason] ?? 0) + 1;
     }
   }
 

package/src/services/machine-probe.test.ts

@@ -0,0 +1,50 @@
+/**
+ * The local management box must never be reported unreachable.
+ *
+ * celilo runs there as the `celilo` user and deliberately holds no SSH key for
+ * itself, so probing `root@127.0.0.1` fails with `Permission denied (publickey)`
+ * on a completely healthy host. That produced a permanently firing
+ * `machines_reachable` alert against celilo-mgr — and that monitor is
+ * unsuppressible by design, so nothing could explain it away.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { auditMachinesReachable } from './audit/machines-reachable';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+describe('local machine reachability', () => {
+  test('LOCAL_MACHINE_IP is the loopback address the machine pool records', () => {
+    // The probe's skip is keyed on this exact value, so pin it.
+    expect(LOCAL_MACHINE_IP).toBe('127.0.0.1');
+  });
+
+  test('a reachable local box produces no finding', async () => {
+    const findings = await auditMachinesReachable({
+      results: [
+        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { id: 'briq', hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
+      ],
+    });
+
+    expect(findings).toEqual([]);
+  });
+
+  test('an unreachable remote machine still produces a finding', async () => {
+    // The skip must not blunt the check for the machines it exists to watch.
+    const findings = await auditMachinesReachable({
+      results: [
+        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        {
+          id: 'briq',
+          hostname: 'briq',
+          ipAddress: '192.168.0.254',
+          reachable: false,
+          message: 'connection refused',
+        },
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].message).toContain('briq');
+  });
+});

package/src/services/machine-probe.ts

@@ -0,0 +1,73 @@
+/**
+ * Reachability probe for the machine pool.
+ *
+ * One implementation, deliberately. This SSH probe previously existed twice —
+ * once in `alerting/builtin-source.ts` for the monitor sweep and once in
+ * `cli/commands/system-audit.ts` for `celilo system audit` — as identical
+ * copy-pasted blocks. Both carried the same bug, and fixing one would have left
+ * the other reporting the management server as unreachable forever.
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import type { MachineReachableResult } from './audit/machines-reachable';
+import { listMachines } from './machine-pool';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Probe every machine in the pool.
+ *
+ * `BatchMode=yes` prevents a password prompt from hanging the probe forever,
+ * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
+ * condition this check exists to detect must not be the one that wedges it.
+ *
+ * The local management box is reported reachable WITHOUT probing it. celilo
+ * runs there as the `celilo` user and deliberately does not materialize an SSH
+ * key for itself, so `ssh root@127.0.0.1` fails with `Permission denied
+ * (publickey)` on a perfectly healthy host. Left unhandled that produced a
+ * permanently firing `machines_reachable` alert against celilo-mgr — and since
+ * that monitor is unsuppressible by design, nothing could explain it away.
+ *
+ * The check is also meaningless there: if the box running this code were
+ * unreachable, this code would not be running.
+ */
+export async function probeMachines(): Promise<MachineReachableResult[]> {
+  const machines = await listMachines();
+  return Promise.all(
+    machines.map(async (m): Promise<MachineReachableResult> => {
+      if (m.ipAddress === LOCAL_MACHINE_IP) {
+        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+      }
+      try {
+        await execFileAsync(
+          'ssh',
+          [
+            '-o',
+            'BatchMode=yes',
+            '-o',
+            'ConnectTimeout=5',
+            '-o',
+            'StrictHostKeyChecking=no',
+            '-o',
+            'UserKnownHostsFile=/dev/null',
+            `${m.sshUser}@${m.ipAddress}`,
+            'true',
+          ],
+          { timeout: 8000 },
+        );
+        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+      } catch (err) {
+        const e = err as { stderr?: string; message?: string };
+        return {
+          id: m.id,
+          hostname: m.hostname,
+          ipAddress: m.ipAddress,
+          reachable: false,
+          message: e.stderr?.trim() || e.message || 'SSH probe failed',
+        };
+      }
+    }),
+  );
+}