@celilo/cli 2.0.0 → 2.1.0

package/src/services/dns-discovery.ts

@@ -0,0 +1,92 @@
+/**
+ * Read the upstream DNS resolvers this box uses.
+ *
+ * This sits beside `network-discovery.ts` and for the same reason: celilo
+ * records facts about the management box itself, and a module is not the right
+ * place to read them. The network half moved first
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md),
+ * the fleet key followed (openspec/changes/hook-process-boundary, design D9b),
+ * and DNS is the last of the three. It came from
+ * `modules/celilo-mgmt/scripts/discovery.ts`, whose hook could only reach
+ * celilo by spawning the CLI — which a jailed hook cannot do at all
+ * (celilo#1225).
+ *
+ * Reading it here is not merely tidier, it is the only place it can be
+ * correct. celilo-mgmt is never deployed to a remote box (ruled 2026-09-02),
+ * so the host whose resolvers we want is always the host celilo runs on. A
+ * hook reading `/etc/resolv.conf` reads that same file, one process further
+ * out, for no benefit.
+ *
+ * Pure parsers are separated from the IO wrapper so they test without a host
+ * (Rule 2.3).
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+
+/**
+ * Where a host publishes its real upstream resolvers, most trustworthy first.
+ *
+ * systemd-resolved's file comes first deliberately: on such a host
+ * `/etc/resolv.conf` names the local 127.0.0.53 stub rather than the upstreams
+ * behind it, so reading only the second file discovers a loopback address and
+ * writes it into system config as the fleet's DNS.
+ */
+const RESOLVER_FILES = ['/run/systemd/resolve/resolv.conf', '/etc/resolv.conf'] as const;
+
+/** Where we land when the host offers no usable upstream of its own. */
+const PUBLIC_FALLBACK = '1.1.1.1';
+
+export interface DnsServers {
+  primary: string;
+  fallback: string;
+}
+
+/** Extract `nameserver` IPs from resolv.conf-format text, in file order. */
+export function parseNameservers(resolvConf: string): string[] {
+  return resolvConf
+    .split('\n')
+    .map((l) => l.trim())
+    .filter((l) => l.startsWith('nameserver '))
+    .map((l) => l.split(/\s+/)[1])
+    .filter(Boolean);
+}
+
+/**
+ * Choose primary and fallback from discovered nameservers.
+ *
+ * Loopback entries are dropped rather than used. A stub resolver's address is
+ * a real answer to "what does this host query" and the wrong answer to "what
+ * should the fleet query", because nothing else on the network can reach it.
+ */
+export function chooseDns(nameservers: string[]): DnsServers {
+  const upstream = nameservers.filter((ns) => !ns.startsWith('127.') && ns !== '::1');
+  return {
+    primary: upstream[0] ?? PUBLIC_FALLBACK,
+    fallback: upstream[1] ?? PUBLIC_FALLBACK,
+  };
+}
+
+/** Reads a resolver file, or returns null when it is not present. */
+export type ResolverFileReader = (path: string) => string | null;
+
+const readResolverFile: ResolverFileReader = (path) =>
+  existsSync(path) ? readFileSync(path, 'utf-8') : null;
+
+/**
+ * Read this box's upstream resolvers.
+ *
+ * Takes the systemd file only when it yielded a real server. `/etc/resolv.conf`
+ * is the last resort and whatever it gives is accepted, including the fallback,
+ * because there is nothing further to consult.
+ */
+export function discoverDns(read: ResolverFileReader = readResolverFile): DnsServers {
+  for (const path of RESOLVER_FILES) {
+    const contents = read(path);
+    if (contents === null) continue;
+    const servers = chooseDns(parseNameservers(contents));
+    if (servers.primary !== PUBLIC_FALLBACK || path === '/etc/resolv.conf') {
+      return servers;
+    }
+  }
+  return { primary: PUBLIC_FALLBACK, fallback: PUBLIC_FALLBACK };
+}

package/src/services/fleet-checks.ts

@@ -29,7 +29,11 @@ import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capabilit
 import type { ModuleManifest } from '../manifest/schema';
 
 import { getServiceCredentials, listContainerServices } from './container-service';
-import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
+import {
+  CONTROL_PLANE_MODULE_ID,
+  getModuleSystems,
+  listAllModuleSystems,
+} from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -43,7 +47,7 @@ import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
 /** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+const CONTROL_PLANE_MODULE = CONTROL_PLANE_MODULE_ID;
 
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other

package/src/services/health-runner.ts

@@ -17,7 +17,8 @@ import type { HookLogger, HookResult } from '../hooks/types';
 import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
-import { getModuleSystems } from './deployed-systems';
+import { controlPlaneHealthChecks } from './control-plane-health';
+import { CONTROL_PLANE_MODULE_ID, getModuleSystems } from './deployed-systems';
 import { remoteAccessPolicy } from './remote-access';
 
 export interface HealthCheckItem {
@@ -152,6 +153,19 @@ export async function runModuleHealthCheck(
     return { moduleId, status: 'error', checks: [], error: 'Module not found' };
   }
 
+  // The control plane checks itself, in celilo's own process (celilo#1225).
+  //
+  // This was a health_check hook and it failed under the jail in the worst
+  // possible way: it asked `existsSync(db_path)`, the jail deliberately does
+  // not bind celilo's data directory, and so it reported a healthy fleet's
+  // database MISSING and failed the deploy over a file that was right there.
+  // `runFleetChecks` is what `celilo system doctor` already runs and asks
+  // eight questions where the hook asked two.
+  if (moduleId === CONTROL_PLANE_MODULE_ID) {
+    const checks = await controlPlaneHealthChecks(db);
+    return finishHealthCheck(moduleId, module.state, checks, db, options);
+  }
+
   const manifest = module.manifestData as ModuleManifest;
   const hookDef = manifest.hooks?.health_check;
 
@@ -222,6 +236,23 @@ export async function runModuleHealthCheck(
   };
 
   const checks = outputs.checks || [];
+  return finishHealthCheck(moduleId, module.state, checks, db, options, hookResult.artifactPaths);
+}
+
+/**
+ * Derive the verdict from the named checks and move the module's state.
+ *
+ * Shared by the hook path and the control plane's own path so the two cannot
+ * drift on what `degraded` means or on when a state transition is suppressed.
+ */
+function finishHealthCheck(
+  moduleId: string,
+  moduleState: ModuleState,
+  checks: HealthCheckItem[],
+  db: DbClient,
+  options: HealthCheckOptions,
+  artifactPaths?: string[],
+): HealthCheckResult {
   const hasFail = checks.some((c) => c.status === 'fail');
   const hasWarn = checks.some((c) => c.status === 'warn');
 
@@ -231,12 +262,12 @@ export async function runModuleHealthCheck(
       ? 'degraded'
       : 'healthy';
 
-  const nextState = nextModuleState(module.state, status, options.unattended ?? false);
+  const nextState = nextModuleState(moduleState, status, options.unattended ?? false);
   if (nextState) {
     db.update(modules).set({ state: nextState }).where(eq(modules.id, moduleId)).run();
   }
 
-  return { moduleId, status, checks, artifactPaths: hookResult.artifactPaths };
+  return { moduleId, status, checks, ...(artifactPaths ? { artifactPaths } : {}) };
 }
 
 /**

package/src/services/module-deploy.ts

@@ -30,11 +30,13 @@ import {
   interviewForMissingSecrets,
 } from './config-interview';
 import { getContainerService } from './container-service';
+import { bootstrapControlPlane } from './control-plane-bootstrap';
 import { executeAnsible } from './deploy-ansible';
 import { planDeployment } from './deploy-planner';
 import { waitForSSH } from './deploy-ssh';
 import { executeTerraform, parseTerraformOutputs } from './deploy-terraform';
 import { validateAndPrepareDeployment } from './deploy-validation';
+import { CONTROL_PLANE_MODULE_ID } from './deployed-systems';
 import { getModuleSystems } from './deployed-systems';
 import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
@@ -1351,6 +1353,55 @@ async function deployModuleImpl(
         // container ID/IP got assigned per module in the DB.
       }
 
+      // celilo initialises its own box, in its own process.
+      //
+      // This was `celilo-mgmt`'s `on_install` hook until celilo#1225. The hook
+      // reached every one of these operations by spawning the `celilo` CLI,
+      // which a jailed hook cannot do — the mount set binds no `/usr/bin`, no
+      // `/usr/local/bin` and no shell — so the deploy ran Ansible to completion
+      // and then died in its own install hook on any host with a jail backend.
+      //
+      // It runs at the point the hook used to, for the reason the hook ran
+      // there: the Ansible role has just installed celilo and started the
+      // dispatcher, and none of this can be read before that.
+      if (moduleId === CONTROL_PLANE_MODULE_ID) {
+        log.success('Initializing celilo management state');
+        try {
+          const bootstrap = await bootstrapControlPlane({ db });
+
+          log.info(`  DNS: primary ${bootstrap.dns.primary}, fallback ${bootstrap.dns.fallback}`);
+          for (const applied of bootstrap.network.applied) {
+            log.info(`  Network: ${applied}`);
+          }
+          if (bootstrap.network.skipped) {
+            log.warn(`  Network not recorded: ${bootstrap.network.skipped}`);
+          }
+          // The operator authorises this key on every machine celilo manages,
+          // so it is printed rather than merely stored. It was the loudest line
+          // the hook produced and it stays the loudest line here.
+          log.success(
+            `celilo fleet SSH key — add this public key to machines celilo will manage:\n${bootstrap.fleetKey.publicKey}`,
+          );
+
+          if (bootstrap.dispatcher.status === 'fail') {
+            // A management plane with no dispatcher is broken: event-driven
+            // reconciles never deliver. Fail the install, do not whisper.
+            log.error(`Dispatcher check failed: ${bootstrap.dispatcher.summary}`);
+            return {
+              success: false,
+              phases,
+              error:
+                `celilo-mgmt: ${bootstrap.dispatcher.summary}. ${bootstrap.dispatcher.remediation ?? ''}`.trim(),
+            };
+          }
+          log.success(`Event dispatcher is live (${bootstrap.dispatcher.summary})`);
+        } catch (err) {
+          const detail = err instanceof Error ? err.message : String(err);
+          log.error(`Initializing celilo management state failed: ${detail}`);
+          return { success: false, phases, error: detail };
+        }
+      }
+
       // Run on_install hook (post-deploy actions like port forwarding, DNS registration)
       if (manifest.hooks?.on_install) {
         const onInstallDef = manifest.hooks.on_install;

package/src/services/remote-access.test.ts

@@ -7,16 +7,27 @@
  *   2. an explicit non-root user is allowed — the fleet key's authority is
  *      root on fleet systems, and a non-root account trusts it only where an
  *      operator installed it (the cPanel bootstrap);
- *   3. otherwise the target must be a system the module or one of its
- *      instances provisioned, and a refusal names the module, the target, and
- *      the capability route.
+ *   3. the target is a system the module or one of its instances provisioned;
+ *   4. or it is a machine in celilo's pool that this module's own config names
+ *      — the case a config-only manager module needs, since it provisions
+ *      nothing and so can never satisfy rule 3 (celilo#1225);
+ *
+ *   and otherwise a refusal names the module, the target, and the capability
+ *   route.
  */
 
 import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
 import { existsSync } from 'node:fs';
 import { rm } from 'node:fs/promises';
 import { type DbClient, createDbClient } from '../db/client';
-import { capabilities, moduleInstances, moduleSystems, modules } from '../db/schema';
+import {
+  capabilities,
+  machines,
+  moduleConfigs,
+  moduleInstances,
+  moduleSystems,
+  modules,
+} from '../db/schema';
 import { remoteAccessPolicy } from './remote-access';
 
 const TEST_DB_PATH = './test-remote-access.db';
@@ -48,6 +59,26 @@ function seedSystem(db: DbClient, moduleId: string, ipv4Address: string, hostnam
     .run();
 }
 
+/** A box the operator admitted to celilo's pool. */
+function seedPoolMachine(db: DbClient, ipAddress: string, hostname: string): void {
+  db.insert(machines)
+    .values({
+      id: `machine-${hostname}`,
+      hostname,
+      zone: 'internal',
+      ipAddress,
+      sshUser: 'root',
+      sshKeyEncrypted: 'x',
+      hardware: { cpu_cores: 1, memory_mb: 512, disk_gb: 8 },
+    })
+    .run();
+}
+
+/** A config value the operator set on a module. */
+function seedConfig(db: DbClient, moduleId: string, key: string, value: string): void {
+  db.insert(moduleConfigs).values({ moduleId, key, value }).run();
+}
+
 describe('remoteAccessPolicy', () => {
   let db: DbClient;
 
@@ -136,4 +167,57 @@ describe('remoteAccessPolicy', () => {
     );
     expect(verdict.allowed).toBe(false);
   });
+
+  describe('a config-only module managing a pool machine (celilo#1225)', () => {
+    // `iptables` is the archetype: it configures a pre-existing firewall and is
+    // deployed onto nothing, so `module_systems` holds no row for it EVER. Rule
+    // 3 is unsatisfiable for it by construction, not by timing.
+    const FIREWALL = '10.226.1.254';
+
+    beforeEach(() => {
+      seedModule(db, 'iptables');
+    });
+
+    test("allows a pool machine this module's config names", () => {
+      seedPoolMachine(db, FIREWALL, 'fw-main');
+      seedConfig(db, 'iptables', 'firewall_ip', FIREWALL);
+
+      expect(
+        remoteAccessPolicy('iptables', db).checkTarget({ ipv4_address: FIREWALL }, false),
+      ).toEqual({ allowed: true });
+    });
+
+    test('pool membership ALONE is not authorization', () => {
+      // Otherwise any module could reach every box celilo knows about, which is
+      // far wider than the rule this policy states.
+      seedPoolMachine(db, FIREWALL, 'fw-main');
+
+      const result = remoteAccessPolicy('iptables', db).checkTarget(
+        { ipv4_address: FIREWALL },
+        false,
+      );
+      expect(result.allowed).toBe(false);
+    });
+
+    test('config ALONE is not authorization', () => {
+      // A module names its own config keys. If naming an address were enough, a
+      // module could point itself at any box in the world.
+      seedConfig(db, 'iptables', 'firewall_ip', FIREWALL);
+
+      const result = remoteAccessPolicy('iptables', db).checkTarget(
+        { ipv4_address: FIREWALL },
+        false,
+      );
+      expect(result.allowed).toBe(false);
+    });
+
+    test("one module's config does not unlock another module's box", () => {
+      seedPoolMachine(db, FIREWALL, 'fw-main');
+      seedConfig(db, 'iptables', 'firewall_ip', FIREWALL);
+
+      // caddy has its own systems and no config naming the firewall.
+      const result = remoteAccessPolicy('caddy', db).checkTarget({ ipv4_address: FIREWALL }, false);
+      expect(result.allowed).toBe(false);
+    });
+  });
 });

package/src/services/remote-access.ts

@@ -40,7 +40,7 @@
 
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities, moduleSystems } from '../db/schema';
+import { capabilities, machines, moduleConfigs, moduleSystems } from '../db/schema';
 import type { RemoteAccessPolicy } from '../hooks/remote-broker';
 import { getModuleSystems } from './deployed-systems';
 import { ownedSystemModuleIds } from './module-instances';
@@ -60,11 +60,62 @@ export function remoteAccessPolicy(moduleId: string, db: DbClient): RemoteAccess
         return { allowed: true };
       }
 
+      if (managesPoolMachine(moduleId, target.ipv4_address, db)) {
+        return { allowed: true };
+      }
+
       return { allowed: false, message: refusalMessage(moduleId, target.ipv4_address, db) };
     },
   };
 }
 
+/**
+ * A machine in celilo's pool that this module's own config names.
+ *
+ * ## The case this exists for
+ *
+ * A module that MANAGES a pre-existing box rather than being deployed onto one
+ * — `iptables` configuring the firewall is the archetype — provisions no system
+ * and therefore never owns one. `module_systems` records nothing for a
+ * config-only module at any point in its lifecycle, so the owned-systems branch
+ * above cannot ever pass for it. Without this, such a module has no sanctioned
+ * path to the one box it exists to configure, and no amount of waiting fixes it
+ * (celilo#1225).
+ *
+ * ## Why this is not a module granting itself reach
+ *
+ * Both halves are the OPERATOR's acts, and a module can supply neither:
+ *
+ *   - The address is in the machine pool. `celilo machine add <ip> --ssh-key…`
+ *     is the operator saying celilo may use its credential on that box. The
+ *     fleet key's authority is exactly the set of boxes celilo was given, and
+ *     this is that set — the same reasoning as the non-root carve-out above.
+ *   - The module's own config names that address. `firewall_ip` is
+ *     `source: user, required: true`; an operator typed it.
+ *
+ * A module declares which config KEYS exist, never their values, so it cannot
+ * point itself at a box the operator did not both admit to the pool and assign
+ * to it. Two independent operator decisions have to agree.
+ *
+ * ## What it deliberately does not do
+ *
+ * It does not allow every pool machine. That would let any module reach any box
+ * celilo knows, which is far wider than the rule this file states. The
+ * intersection is the point: pool membership alone is not authorization, and
+ * config alone is not either.
+ */
+function managesPoolMachine(moduleId: string, ipv4Address: string, db: DbClient): boolean {
+  const inPool = db.select().from(machines).where(eq(machines.ipAddress, ipv4Address)).get();
+  if (!inPool) return false;
+
+  return db
+    .select({ value: moduleConfigs.value })
+    .from(moduleConfigs)
+    .where(eq(moduleConfigs.moduleId, moduleId))
+    .all()
+    .some((row) => row.value === ipv4Address);
+}
+
 /**
  * Name the module, the target, and the capability route (task 5.2). The owner
  * lookup is best-effort colour: the refusal stands whether or not celilo can