@celilo/cli 2.2.1 → 3.0.0

package/src/services/capability-compat.ts

@@ -0,0 +1,128 @@
+/**
+ * Capability compatibility between a module's declared requirements and the
+ * providers actually deployed in the fleet.
+ *
+ * The pure half mirrors the runtime lookup's semantics (see
+ * `deploy-preflight.ts` section 2): a requirement is served when ANY deployed
+ * provider satisfies it under `compareConsumerToProvider`. The reason
+ * `deploy-preflight` and the upgrade path must agree is the failure
+ * celilo#1361 recorded: celilo-website was upgraded against a caddy whose
+ * bundled provider rejected its publish calls, and every attempt burned a
+ * release. The upgrade path now asks this question BEFORE updating, and
+ * defers instead of deploying into a wall.
+ */
+
+import { compareConsumerToProvider } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { capabilities } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import { isPrivilegedCapability } from '../manifest/validate';
+
+export interface CapabilityRequirement {
+  name: string;
+  version: string;
+}
+
+export interface DeployedCapabilityProvider {
+  capabilityName: string;
+  moduleId: string;
+  version: string;
+}
+
+export interface UnservedCapability {
+  capability: string;
+  /** The version the module's manifest requires. */
+  required: string;
+  /** Example provider that does not serve it (the first mismatch found). */
+  providerModuleId: string | null;
+  providerVersion: string | null;
+  reason:
+    | 'no_provider'
+    | 'major_mismatch_higher'
+    | 'major_mismatch_lower'
+    | 'minor_mismatch_higher'
+    | 'caller_minor_too_old';
+  message: string;
+}
+
+/**
+ * Which of a module's capability requirements no deployed provider serves.
+ * Pure (Rule 10): the I/O — reading the capabilities table — happens in
+ * `capabilityBlockersForManifest`.
+ */
+export function unservedCapabilityRequirements(
+  requirements: CapabilityRequirement[] | undefined,
+  providers: DeployedCapabilityProvider[],
+): UnservedCapability[] {
+  const unserved: UnservedCapability[] = [];
+  for (const cap of requirements ?? []) {
+    // Framework-granted privileges (e.g. cross_module_read) are not
+    // provider-backed — the same skip `deploy-preflight` makes.
+    if (isPrivilegedCapability(cap.name)) continue;
+
+    const installed = providers.filter((p) => p.capabilityName === cap.name);
+    if (installed.length === 0) {
+      unserved.push({
+        capability: cap.name,
+        required: cap.version,
+        providerModuleId: null,
+        providerVersion: null,
+        reason: 'no_provider',
+        message: `no deployed provider for '${cap.name}'`,
+      });
+      continue;
+    }
+    let anyCompatible = false;
+    let example: DeployedCapabilityProvider | undefined;
+    let reason: UnservedCapability['reason'] = 'major_mismatch_higher';
+    for (const p of installed) {
+      const result = compareConsumerToProvider(cap.version, p.version);
+      if (result.compatible) {
+        anyCompatible = true;
+        break;
+      }
+      if (!example) {
+        example = p;
+        reason = result.reason;
+      }
+    }
+    if (anyCompatible || !example) continue;
+    unserved.push({
+      capability: cap.name,
+      required: cap.version,
+      providerModuleId: example.moduleId,
+      providerVersion: example.version,
+      reason,
+      message: `requires ${cap.name}@${cap.version} but deployed provider '${example.moduleId}' provides ${cap.name}@${example.version} (${reason})`,
+    });
+  }
+  return unserved;
+}
+
+/** Every capability provider currently registered in the fleet. */
+export function readDeployedProviders(db: DbClient): DeployedCapabilityProvider[] {
+  return db
+    .select({
+      capabilityName: capabilities.capabilityName,
+      moduleId: capabilities.moduleId,
+      version: capabilities.version,
+    })
+    .from(capabilities)
+    .all();
+}
+
+/**
+ * The upgrade-path entry point: which of a target manifest's capability
+ * requirements the deployed fleet cannot serve. A non-empty result means the
+ * upgrade would deploy into a wall the preflight would then reject — defer
+ * instead of burning the attempt.
+ */
+export function capabilityBlockersForManifest(
+  db: DbClient,
+  manifest: Pick<ModuleManifest, 'requires' | 'id'>,
+): UnservedCapability[] {
+  return unservedCapabilityRequirements(
+    manifest.requires?.capabilities as CapabilityRequirement[] | undefined,
+    readDeployedProviders(db),
+  );
+}

package/src/services/celilo-events.test.ts

@@ -29,7 +29,7 @@ describe('celilo lifecycle events', () => {
     process.env.EVENT_BUS_DB = dbPath;
   });
   afterEach(() => {
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/celilo-mgmt-hooks.test.ts

@@ -11,6 +11,14 @@ import type { HookContext } from '@celilo/capabilities';
 // runnable from the apps/celilo working dir.
 const HOOK_DIR = join(import.meta.dirname, '../../../../modules/celilo-mgmt/scripts');
 
+// on_backup reads its machine-pool snapshot from the context input the
+// framework stages (backup-create.ts, from listMachines()). These tests
+// inject a JSON string so the snapshot is deterministic on every host. The
+// pre-staging version of this hook shelled `celilo machine list --json` and
+// these tests had to inject a runner stub to avoid depending on the installed
+// CLI's age (celilo#1333); the staged input made that seam unnecessary.
+const STUB_MACHINE_POOL = [{ hostname: 'stub-machine', zone: 'dmz' }];
+
 interface BackupHookOutput {
   artifact_count: number;
   size_bytes: number;
@@ -116,6 +124,7 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
         backup_dir: backupDir,
         cross_module_root: crossModuleRoot,
         system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
       }),
     )) as BackupHookOutput;
 
@@ -135,7 +144,6 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
     expect(result.artifact_count).toBeGreaterThan(0);
     expect(result.size_bytes).toBeGreaterThan(0);
   });
-
   it('refuses when the framework did not stage celilo state', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
 
@@ -148,24 +156,34 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
     ).rejects.toThrow(/system_state_root was not provided/);
   });
 
-  it('machine-pool.json is valid JSON (array)', async () => {
+  it('machine-pool.json is the staged snapshot (valid JSON array)', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
     await hook(
       buildContext({
         backup_dir: backupDir,
         cross_module_root: crossModuleRoot,
         system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
       }),
     );
 
-    const machinePool = JSON.parse(readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'));
-    expect(Array.isArray(machinePool)).toBe(true);
+    // Content, not just shape. A bare Array.isArray also passes when the
+    // hook swallowed a snapshot failure and wrote an empty pool, which is
+    // exactly the failure mode this stub exists to rule out.
+    const machinePool: unknown = JSON.parse(
+      readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'),
+    );
+    expect(machinePool).toEqual(STUB_MACHINE_POOL);
   });
 
   it('proceeds (with a warning) when cross_module_root is missing', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
     const result = (await hook(
-      buildContext({ backup_dir: backupDir, system_state_root: systemStateRoot }),
+      buildContext({
+        backup_dir: backupDir,
+        system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
+      }),
     )) as BackupHookOutput;
 
     expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);

package/src/services/container-service.test.ts

@@ -55,7 +55,7 @@ describe('container-service', () => {
 
     // Clear environment variables
     resetTestDbPath();
-    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    delete process.env.CELILO_MASTER_KEY_PATH;
   });
 
   describe('addContainerService', () => {

package/src/services/cross-module-read.test.ts

@@ -72,7 +72,7 @@ describe('materializeCrossModuleRoot', () => {
   afterEach(() => {
     closeDb();
     resetTestDbPath();
-    process.env.CELILO_DATA_DIR = undefined;
+    delete process.env.CELILO_DATA_DIR;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -180,7 +180,7 @@ describe('applyCrossModuleWriteRoot', () => {
   });
 
   afterEach(() => {
-    process.env.CELILO_DATA_DIR = undefined;
+    delete process.env.CELILO_DATA_DIR;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/deploy-preflight.ts

@@ -27,6 +27,7 @@ import type { DbClient } from '../db/client';
 import { getDb } from '../db/client';
 import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
+import { isPrivilegedCapability } from '../manifest/validate';
 import { buildResolutionContext } from '../variables/context';
 import { E2E_CONFLICT_FIX, runningE2eContainers } from './e2e-guard';
 import { describeCapabilityProblem, findBrokenCapabilityDerivations } from './fleet-checks';
@@ -113,6 +114,12 @@ export async function runPreflight(
   //    template-generation against an ABI the provider can't fulfil.
   if (manifest.requires?.capabilities) {
     for (const cap of manifest.requires.capabilities) {
+      // Framework-granted privileges (e.g. cross_module_read) aren't
+      // provider-backed — they're gated by the allow-list at import time,
+      // not satisfied by deploying another module. Skip them here.
+      if (isPrivilegedCapability(cap.name)) {
+        continue;
+      }
       const providers = db
         .select()
         .from(capabilities)

package/src/services/deploy-terraform.ts

@@ -2,7 +2,7 @@ import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { log } from '../cli/prompts';
 import { executeBuildWithProgress } from './build-stream';
-import { validateTerraformPlanSafety } from './terraform-safety';
+import { planAfterImportVerdict, validateTerraformPlanSafety } from './terraform-safety';
 
 export interface TerraformResult {
   success: boolean;
@@ -173,6 +173,43 @@ export async function executeTerraform(
 
       if (importResult.success) {
         log.success('Resource imported into Terraform state');
+
+        // The import REPLACED the plan validated above. That plan described a
+        // create; the plan terraform computes now can describe a REPLACE,
+        // because `ostemplate` and `rootfs[0].storage` do not round-trip
+        // through an import and each forces one. Re-plan and re-check, or the
+        // retry applies a destroy the safety gate exists to forbid — which is
+        // how a redeploy destroyed signal's LXC on a converged fleet
+        // (celilo#1374).
+        const rePlan = await executeBuildWithProgress({
+          command: 'terraform',
+          args: ['plan', '-detailed-exitcode', '-no-color'],
+          cwd: terraformDir,
+          title: 'Re-planning after import',
+          env: terraformEnv,
+          noInteractive,
+        });
+
+        const verdict = planAfterImportVerdict(rePlan.exitCode, rePlan.output);
+
+        if (verdict.action === 'done') {
+          // The import alone reconciled state with reality: adopt and stop.
+          if (phases) phases.terraformApply = true;
+          log.success('Infrastructure adopted by import (no changes needed)');
+          return { success: true, output: rePlan.output, exitCode: 0 };
+        }
+
+        if (verdict.action === 'refuse') {
+          if (phases) phases.terraformApply = false;
+          log.error('Refusing to apply the plan computed after import');
+          return {
+            success: false,
+            output: rePlan.output,
+            error: verdict.error,
+            exitCode: 2,
+          };
+        }
+
         log.message('  Retrying deployment...');
 
         // Retry apply after successful import

package/src/services/deployed-systems.ts

@@ -242,7 +242,7 @@ export function upsertDeployedSystem(
     .run();
 }
 
-function asZone(value: string | null | undefined): NetworkZone | null {
+export function asZone(value: string | null | undefined): NetworkZone | null {
   return value && (NETWORK_ZONES as readonly string[]).includes(value)
     ? (value as NetworkZone)
     : null;

package/src/services/events-daemon.test.ts

@@ -10,6 +10,7 @@ import {
   readInstalledUnit,
   renderLaunchdPlist,
   renderSystemdUnit,
+  resolveDaemonPathEnv,
   resolveRestartScope,
   resolveRunAsUser,
   restartDaemon,
@@ -27,11 +28,15 @@ describe('renderSystemdUnit', () => {
       concurrency: 4,
       home: '/home/op',
       scope: 'user',
+      pathEnv: '/bun/bin:/usr/bin:/bin',
     });
     expect(out).toContain(
       'ExecStart=/usr/local/bin/celilo events run --poll-ms 1000 --concurrency 4',
     );
     expect(out).toContain('Environment=EVENT_BUS_DB=/var/lib/celilo/events.db');
+    // celilo#1373 — the unit's environment must resolve bun; the global celilo
+    // wrapper is a bash script whose first act is `command -v bun`.
+    expect(out).toContain('Environment=PATH=/bun/bin:/usr/bin:/bin');
     expect(out).toContain('Restart=on-failure');
     expect(out).toContain('WantedBy=default.target');
     expect(out).not.toContain('User=');
@@ -45,6 +50,7 @@ describe('renderSystemdUnit', () => {
       concurrency: 8,
       home: '/h',
       scope: 'user',
+      pathEnv: '/bun/bin:/usr/bin',
     });
     expect(out).toContain('--poll-ms 250 --concurrency 8');
   });
@@ -57,6 +63,7 @@ describe('renderSystemdUnit', () => {
       concurrency: 4,
       home: '/root',
       scope: 'system',
+      pathEnv: '/bun/bin:/usr/bin',
       runAsUser: 'celilo',
     });
     expect(out).toContain('User=celilo');
@@ -75,6 +82,7 @@ describe('renderLaunchdPlist', () => {
       concurrency: 4,
       home: '/Users/op',
       scope: 'user',
+      pathEnv: '/bun/bin:/usr/bin:/bin',
     });
     expect(out).toContain('<string>/Users/op/Library/Logs/celilo-events.out.log</string>');
     expect(out).toContain('<string>/Users/op/Library/Logs/celilo-events.err.log</string>');
@@ -85,6 +93,29 @@ describe('renderLaunchdPlist', () => {
     expect(out).not.toContain('<key>UserName</key>');
   });
 
+  // celilo#1373 — launchd starts jobs with its own default PATH (no bun),
+  // and the global celilo wrapper is a bash script whose first act is
+  // `command -v bun`. The rendered plist MUST carry a PATH that resolves
+  // bun, inside EnvironmentVariables, or the unit crash-loops forever.
+  it('carries a PATH in EnvironmentVariables that resolves bun', () => {
+    const out = renderLaunchdPlist({
+      celiloPath: '/Users/op/.bun/bin/celilo',
+      busDbPath: '/db',
+      pollMs: 1000,
+      concurrency: 4,
+      home: '/Users/op',
+      scope: 'user',
+      pathEnv: '/bun/bin:/usr/local/bin:/usr/bin:/bin',
+    });
+    expect(out).toContain('<key>PATH</key>');
+    expect(out).toContain('<string>/bun/bin:/usr/local/bin:/usr/bin:/bin</string>');
+    // PATH lives INSIDE the EnvironmentVariables dict, after EVENT_BUS_DB.
+    const envIdx = out.indexOf('<key>EnvironmentVariables</key>');
+    const pathIdx = out.indexOf('<key>PATH</key>');
+    expect(envIdx).toBeGreaterThan(-1);
+    expect(pathIdx).toBeGreaterThan(envIdx);
+  });
+
   it('system scope sets UserName and logs under /Library/Logs', () => {
     const out = renderLaunchdPlist({
       celiloPath: '/c',
@@ -93,6 +124,7 @@ describe('renderLaunchdPlist', () => {
       concurrency: 4,
       home: '/Users/op',
       scope: 'system',
+      pathEnv: '/bun/bin:/usr/bin',
       runAsUser: 'celilo',
     });
     expect(out).toContain('<key>UserName</key>');
@@ -103,6 +135,31 @@ describe('renderLaunchdPlist', () => {
   });
 });
 
+describe('resolveDaemonPathEnv', () => {
+  // celilo#1373 — the whole point of the PATH key is that the bun the
+  // wrapper needs is actually in the directory we wrote. First colon
+  // segment, and a bun executable must exist in it.
+  it('starts with a directory that contains a bun executable', () => {
+    const firstDir = resolveDaemonPathEnv().split(':')[0] ?? '';
+    expect(firstDir).not.toBe('');
+    expect(existsSync(join(firstDir, 'bun'))).toBe(true);
+  });
+
+  it('planDaemonInstall pins the PATH into the rendered unit', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'celilo-pathenv-'));
+    const celiloPath = join(dir, 'fake-celilo');
+    writeFileSync(celiloPath, '#!/bin/sh\n', { mode: 0o755 });
+    const plan = planDaemonInstall({
+      platform: 'darwin',
+      scope: 'user',
+      home: join(dir, 'home'),
+      celiloPath,
+      pathEnv: '/bun/bin:/usr/bin:/bin',
+    });
+    expect(plan.unitContent).toContain('<string>/bun/bin:/usr/bin:/bin</string>');
+  });
+});
+
 describe('getDaemonUnitPath', () => {
   it('returns the systemd user path on linux', () => {
     expect(getDaemonUnitPath('linux', '/home/op')).toBe(

package/src/services/events-daemon.ts

@@ -48,6 +48,12 @@ export interface InstallDaemonOptions {
   platform?: SupervisorPlatform;
   /** Override the bus DB path. Defaults to celilo's getEventBusPath(). */
   busDbPath?: string;
+  /**
+   * Override the PATH written into the unit's environment. Defaults to
+   * resolveDaemonPathEnv() captured at render time. Test seam — pin it so
+   * assertions don't depend on the machine's bun install.
+   */
+  pathEnv?: string;
   /** Override the home directory used to compute install paths. */
   home?: string;
   /** Prefix for system-scope paths. Test seam — see getDaemonUnitPath. */
@@ -159,6 +165,47 @@ export function getDaemonUnitPath(
     : join(home, 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`);
 }
 
+export interface LaunchdUnitStatus {
+  /** The pid launchd currently runs the unit under, or null when it isn't running. */
+  pid: number | null;
+  /**
+   * The unit's last exit status in wait4 form (256 = exited 1, 15 = killed by
+   * SIGTERM), or null when launchd has no record of one.
+   */
+  lastExitStatus: number | null;
+}
+
+/**
+ * Probe launchd for the unit's health. Uses `launchctl list <label>` —
+ * on current macOS the richer `launchctl print gui/<uid>/<label>` answers
+ * "Domain does not support specified action" for legacy-loaded jobs, while
+ * `list <label>` prints a legacy-style dict carrying `PID` (absent when not
+ * running) and `LastExitStatus`.
+ *
+ * Returns null when launchd doesn't answer at all (not loaded, linux, the
+ * binary missing) — null is "don't know", never "bad". Only the user scope
+ * is probed: the system domain needs root, and a mac running a system-scope
+ * celilo daemon is not a supported shape (see supervisorCommands).
+ */
+export function launchdUnitStatus(scope: SupervisorScope = 'user'): LaunchdUnitStatus | null {
+  if (scope !== 'user' || detectPlatform() !== 'darwin') return null;
+  let out: string;
+  try {
+    out = execFileSync('launchctl', ['list', LAUNCHD_LABEL], {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    return null;
+  }
+  const pidMatch = out.match(/"PID" = (\d+)/);
+  const exitMatch = out.match(/"LastExitStatus" = (-?\d+)/);
+  return {
+    pid: pidMatch ? Number(pidMatch[1]) : null,
+    lastExitStatus: exitMatch ? Number(exitMatch[1]) : null,
+  };
+}
+
 /**
  * Find the celilo executable. Priority:
  *   1. Explicit override
@@ -185,6 +232,25 @@ export function resolveCeliloPath(override?: string): string {
   return found;
 }
 
+/**
+ * A PATH that resolves bun, for a supervisor unit's environment.
+ *
+ * launchd starts jobs with its own default PATH, and systemd runs
+ * ExecStart with a similarly minimal one. Neither includes bun, and the
+ * global `celilo` entry is a bash wrapper whose first act is
+ * `command -v bun` — so a unit without a PATH key crash-loops forever on
+ * "Error: celilo requires bun" (celilo#1373: 22680 respawns on one Mac).
+ * The PATH is captured at RENDER time — the moment the unit is written —
+ * from Bun.which('bun'), falling back to process.execPath, which IS the
+ * bun binary under the bun runtime. Standard dirs follow so the
+ * dispatcher's own subprocesses keep working.
+ */
+export function resolveDaemonPathEnv(): string {
+  const bun =
+    (Bun as unknown as { which: (cmd: string) => string | null }).which('bun') ?? process.execPath;
+  return `${dirname(bun)}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin`;
+}
+
 /**
  * Run-as user for a system-scope unit: the owner of the celilo state
  * dir (the bus DB's directory). Falls back to the current user when
@@ -211,6 +277,11 @@ interface UnitInputs {
   concurrency: number;
   home: string;
   scope: SupervisorScope;
+  /**
+   * PATH injected into the unit's environment (celilo#1373). Required:
+   * a unit without it cannot resolve bun and crash-loops.
+   */
+  pathEnv: string;
   /** Required for system scope; ignored for user scope. */
   runAsUser?: string;
 }
@@ -232,6 +303,7 @@ ${userLine}ExecStart=${input.celiloPath} events run --poll-ms ${input.pollMs} --
 Restart=on-failure
 RestartSec=10s
 Environment=EVENT_BUS_DB=${input.busDbPath}
+Environment=PATH=${input.pathEnv}
 # stdout/stderr are captured by ${journalHint}.
 StandardOutput=journal
 StandardError=journal
@@ -271,6 +343,8 @@ ${userNameBlock}  <key>ProgramArguments</key>
   <dict>
     <key>EVENT_BUS_DB</key>
     <string>${input.busDbPath}</string>
+    <key>PATH</key>
+    <string>${input.pathEnv}</string>
   </dict>
   <key>RunAtLoad</key>
   <true/>
@@ -334,6 +408,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
   const home = opts.home ?? homedir();
   const celiloPath = resolveCeliloPath(opts.celiloPath);
   const busDbPath = opts.busDbPath ?? getEventBusPath();
+  const pathEnv = opts.pathEnv ?? resolveDaemonPathEnv();
   const pollMs = opts.pollMs ?? 1000;
   const concurrency = opts.concurrency ?? 4;
   const runAsUser = scope === 'system' ? resolveRunAsUser(busDbPath, opts.runAsUser) : undefined;
@@ -346,6 +421,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
     concurrency,
     home,
     scope,
+    pathEnv,
     runAsUser,
   };
   const unitContent =

package/src/services/firewall-reach.ts

@@ -57,27 +57,40 @@ export function loadZoneSubnets(db: DbClient): string[] {
 
 /**
  * Read each firewall's live ruleset and pair it with celilo's composed trusted
- * set. A firewall that can't be reached contributes nothing rather than a false
- * "no unowned reach" — an unreadable box is unknown, not clean.
+ * set. Firewalls whose ruleset cannot be read are reported separately, as
+ * `unreachable` — the audit turns each into an `unmeasured` finding, because
+ * an unreadable box is unknown, not clean (D7).
  */
-export function collectFirewallReach(db: DbClient, run: Runner = execRunner): FirewallReachState[] {
+export interface FirewallReachCollection {
+  firewalls: FirewallReachState[];
+  unreachableFirewalls: string[];
+}
+
+export function collectFirewallReach(
+  db: DbClient,
+  run: Runner = execRunner,
+): FirewallReachCollection {
   const zoneSubnets = loadZoneSubnets(db);
-  if (zoneSubnets.length === 0) return [];
+  if (zoneSubnets.length === 0) return { firewalls: [], unreachableFirewalls: [] };
 
-  const states: FirewallReachState[] = [];
+  const firewalls: FirewallReachState[] = [];
+  const unreachableFirewalls: string[] = [];
   for (const firewallIp of listFirewallIps(db)) {
     // escape-hatch: reads the running filter table (`iptables-save`), which is
     // the only place an out-of-band rule appears. Read-only, no capability.
     const result = runAppCommand({ ipv4_address: firewallIp }, 'iptables-save -t filter', run, {
       timeoutMs: IPTABLES_SAVE_TIMEOUT_MS,
     });
-    if (!result.ok) continue;
+    if (!result.ok) {
+      unreachableFirewalls.push(firewallIp);
+      continue;
+    }
 
-    states.push({
+    firewalls.push({
       firewallIp,
       live: parseLiveReachRules(result.stdout, zoneSubnets),
       known: loadTrustedSubnets(db, firewallIp).map((e) => e.subnet),
     });
   }
-  return states;
+  return { firewalls, unreachableFirewalls };
 }