@celilo/cli 1.14.0 → 2.1.0

package/src/services/control-plane-health.test.ts

@@ -0,0 +1,66 @@
+/**
+ * The control plane's health check (celilo#1225).
+ *
+ * The mapping is what these pin. The failure that made this code exist was not
+ * a wrong assertion — the hook's `existsSync(db_path)` was a perfectly correct
+ * question — it was a correct question asked from inside a jail that does not
+ * bind the directory, so a present database read as absent. Nothing in a unit
+ * test can catch that shape; what a unit test CAN hold is that every finding
+ * reaches the operator with its status and its remediation intact, because the
+ * unattended path is the one nobody is watching.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { findingAsHealthCheck } from './control-plane-health';
+import type { FleetFinding } from './fleet-checks';
+
+function finding(over: Partial<FleetFinding> = {}): FleetFinding {
+  return {
+    id: 'dispatcher',
+    title: 'Event dispatcher',
+    status: 'ok',
+    summary: 'a dispatcher is live',
+    detail: [],
+    remediation: null,
+    autoFixable: false,
+    ...over,
+  };
+}
+
+describe('findingAsHealthCheck', () => {
+  test("ok becomes pass — celilo's vocabulary is not the health path's", () => {
+    expect(findingAsHealthCheck(finding({ status: 'ok' })).status).toBe('pass');
+    expect(findingAsHealthCheck(finding({ status: 'warn' })).status).toBe('warn');
+    expect(findingAsHealthCheck(finding({ status: 'fail' })).status).toBe('fail');
+  });
+
+  test('the check is named by the finding id, so an operator can grep for it', () => {
+    const item = findingAsHealthCheck(finding({ id: 'schema_drift' }));
+    expect(item.name).toBe('schema_drift');
+    expect(item.message).toBe('a dispatcher is live');
+  });
+
+  test('the remediation survives the mapping', () => {
+    // This is the line that tells an operator what to DO, and the health path
+    // is the one that runs unattended every fifteen minutes. Dropping it here
+    // would leave a failing check with no next step, quietly.
+    const item = findingAsHealthCheck(
+      finding({
+        status: 'fail',
+        detail: ['no live dispatcher — heartbeat absent'],
+        remediation: 'systemctl enable --now celilo-events.service',
+      }),
+    );
+    expect(item.details).toContain('no live dispatcher — heartbeat absent');
+    expect(item.details).toContain('Fix: systemctl enable --now celilo-events.service');
+  });
+
+  test('a healthy finding carries no empty details field', () => {
+    expect(findingAsHealthCheck(finding()).details).toBeUndefined();
+  });
+
+  test('detail lines survive without a remediation', () => {
+    const item = findingAsHealthCheck(finding({ status: 'warn', detail: ['two dispatchers'] }));
+    expect(item.details).toBe('two dispatchers');
+  });
+});

package/src/services/control-plane-health.ts

@@ -0,0 +1,67 @@
+/**
+ * The control plane's health check, run by celilo rather than by a hook.
+ *
+ * ## Why this is not a hook
+ *
+ * It was `modules/celilo-mgmt/scripts/health-check.ts` until celilo#1225, and
+ * it failed under the hook jail in the way that is worst: **it reported a
+ * healthy fleet's database missing.** The hook asked `existsSync(db_path)`,
+ * the jail does not bind celilo's data directory (deliberately — that is where
+ * `master.key` and `celilo.db` live), so the answer was `false` and the deploy
+ * failed on a file that was sitting right there. Its other two checks shelled
+ * out to the CLI, which a jailed hook cannot reach at all.
+ *
+ * Every one of those was celilo inspecting itself. Deleting `on_install` for
+ * that reason and leaving this behind would have fixed one hook and left the
+ * deploy failing on the next.
+ *
+ * ## What replaced it is stronger, not merely relocated
+ *
+ * The hook asked two questions: does a file exist, and is this hostname in
+ * `celilo machine list`. `runFleetChecks` is what celilo already runs for
+ * `celilo system doctor`, and it asks eight, including the four-part dispatcher
+ * probe whose whole point is that "a process is up" reports green on a
+ * dispatcher that is unsupervised, on stale code, and delivering nothing.
+ *
+ * So the module gets a better check by having none of its own. That is the
+ * outcome to keep in mind before anyone adds a hook back: a health_check hook
+ * here can only re-ask, worse, questions celilo can answer directly.
+ */
+
+import type { DbClient } from '../db/client';
+import { withCeliloBusAsync } from './control-plane-bootstrap';
+import { type FleetFinding, runFleetChecks } from './fleet-checks';
+import type { HealthCheckItem } from './health-runner';
+
+/** `FleetFinding`'s three statuses in `HealthCheckItem`'s vocabulary. */
+const STATUS: Record<FleetFinding['status'], HealthCheckItem['status']> = {
+  ok: 'pass',
+  warn: 'warn',
+  fail: 'fail',
+};
+
+/**
+ * One finding as a health item.
+ *
+ * `detail` and `remediation` are folded into `details` rather than dropped:
+ * the health path is what runs unattended every fifteen minutes, so the line
+ * telling an operator what to DO has to survive the mapping.
+ */
+export function findingAsHealthCheck(finding: FleetFinding): HealthCheckItem {
+  const details = [
+    ...finding.detail,
+    ...(finding.remediation ? [`Fix: ${finding.remediation}`] : []),
+  ];
+  return {
+    name: finding.id,
+    status: STATUS[finding.status],
+    message: finding.summary,
+    ...(details.length > 0 ? { details: details.join('\n') } : {}),
+  };
+}
+
+/** Run celilo's own fleet checks and report them as the control plane's health. */
+export async function controlPlaneHealthChecks(db: DbClient): Promise<HealthCheckItem[]> {
+  const findings = await withCeliloBusAsync((bus) => runFleetChecks(bus, db));
+  return findings.map(findingAsHealthCheck);
+}

package/src/services/deploy-ansible.ts

@@ -138,7 +138,7 @@ function parseAnsibleLine(line: string): string | null {
  */
 export async function executeAnsible(
   generatedPath: string,
-  options?: { noInteractive?: boolean; check?: boolean },
+  options?: { noInteractive?: boolean; check?: boolean; tags?: string[] },
 ): Promise<AnsibleResult> {
   const ansibleDir = join(generatedPath, 'ansible');
   const inventoryPath = join(ansibleDir, 'inventory', 'hosts.ini');
@@ -166,6 +166,14 @@ export async function executeAnsible(
         // stored record of the host (celilo#902 design D6). One argument, not a
         // second execution path — everything else about the run is identical.
         ...(options?.check ? ['--check'] : []),
+        // `--tags` scopes the run to the tagged tasks. The static-content
+        // converge uses it (capability-owned-tables D10): the full playbook
+        // re-templates the bootstrap Caddyfile, which would overwrite the
+        // REAL config the provider's reconcile wrote to disk and reload caddy
+        // into serving nothing. A publish converge must touch /srv/www and
+        // nothing else. An unscoped run (the deploy) still executes every
+        // untagged task alongside the tagged ones.
+        ...(options?.tags?.length ? ['--tags', options.tags.join(',')] : []),
         shellEscape(playbookPath),
       ],
       cwd: ansibleDir,

package/src/services/deployed-systems.ts

@@ -48,6 +48,18 @@ function rowToSystem(row: typeof moduleSystems.$inferSelect): DeployedSystem {
  * Returns [] for API-only modules (e.g. namecheap) — that is a modeled state,
  * not an error.
  */
+/**
+ * The module that IS celilo's control plane.
+ *
+ * celilo knows this module by name in several places — its privileged
+ * capability allow-list, the subnet it trusts, the fleet checks that ask where
+ * the control plane runs, and the deploy step that initialises the box's own
+ * state. The id lives here because every one of those already reaches for
+ * `getModuleSystems`, and three separate private copies of the string is how
+ * they drift.
+ */
+export const CONTROL_PLANE_MODULE_ID = 'celilo-mgmt';
+
 export function getModuleSystems(moduleId: string, db: DbClient): DeployedSystem[] {
   const rows = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
   return rows.map(rowToSystem).sort((a, b) => a.name.localeCompare(b.name));

package/src/services/dns-discovery.test.ts

@@ -0,0 +1,93 @@
+/**
+ * Moved from `modules/celilo-mgmt/scripts/discovery.test.ts` with celilo#1225.
+ *
+ * The parser cases came across unchanged. The `discoverDns` cases are new and
+ * could not have been written there: the module's version read the filesystem
+ * directly, so the file-preference rule — the part with the actual failure mode
+ * — had no seam to test through.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { type ResolverFileReader, chooseDns, discoverDns, parseNameservers } from './dns-discovery';
+
+const SYSTEMD = '/run/systemd/resolve/resolv.conf';
+const ETC = '/etc/resolv.conf';
+
+/** A host offering exactly the files given, and nothing else. */
+function host(files: Record<string, string>): ResolverFileReader {
+  return (path) => files[path] ?? null;
+}
+
+describe('parseNameservers', () => {
+  test('extracts nameserver IPs in order', () => {
+    const resolv = ['# comment', 'nameserver 9.9.9.9', 'nameserver 8.8.4.4', 'search lan'].join(
+      '\n',
+    );
+    expect(parseNameservers(resolv)).toEqual(['9.9.9.9', '8.8.4.4']);
+  });
+
+  test('returns empty for no nameservers', () => {
+    expect(parseNameservers('search lan\noptions edns0')).toEqual([]);
+  });
+});
+
+describe('chooseDns', () => {
+  test('drops loopback stubs (systemd-resolved 127.0.0.53)', () => {
+    expect(chooseDns(['127.0.0.53'])).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+  });
+
+  test('uses the first two real upstreams', () => {
+    expect(chooseDns(['9.9.9.9', '8.8.4.4', '1.0.0.1'])).toEqual({
+      primary: '9.9.9.9',
+      fallback: '8.8.4.4',
+    });
+  });
+
+  test('falls back to 1.1.1.1 for the second when only one upstream', () => {
+    expect(chooseDns(['9.9.9.9'])).toEqual({ primary: '9.9.9.9', fallback: '1.1.1.1' });
+  });
+});
+
+describe('discoverDns', () => {
+  test('prefers the systemd upstream file over the stub in /etc/resolv.conf', () => {
+    // The case the file order exists for. Reading only /etc/resolv.conf here
+    // discovers 127.0.0.53, which no other box on the network can reach.
+    const servers = discoverDns(
+      host({
+        [SYSTEMD]: 'nameserver 9.9.9.9\nnameserver 8.8.4.4\n',
+        [ETC]: 'nameserver 127.0.0.53\n',
+      }),
+    );
+    expect(servers).toEqual({ primary: '9.9.9.9', fallback: '8.8.4.4' });
+  });
+
+  test('falls through to /etc/resolv.conf when the systemd file yields nothing real', () => {
+    const servers = discoverDns(
+      host({
+        [SYSTEMD]: 'nameserver 127.0.0.53\n',
+        [ETC]: 'nameserver 192.0.2.10\nnameserver 192.0.2.11\n',
+      }),
+    );
+    expect(servers).toEqual({ primary: '192.0.2.10', fallback: '192.0.2.11' });
+  });
+
+  test('reads /etc/resolv.conf on a host with no systemd resolver file', () => {
+    expect(discoverDns(host({ [ETC]: 'nameserver 192.0.2.10\n' }))).toEqual({
+      primary: '192.0.2.10',
+      fallback: '1.1.1.1',
+    });
+  });
+
+  test('accepts a loopback-only /etc/resolv.conf as the public fallback, not as an error', () => {
+    // Last resort: whatever it gives is taken, and a stub-only host yields the
+    // public fallback rather than an unreachable address.
+    expect(discoverDns(host({ [ETC]: 'nameserver 127.0.0.53\n' }))).toEqual({
+      primary: '1.1.1.1',
+      fallback: '1.1.1.1',
+    });
+  });
+
+  test('returns the public fallback on a host with no resolver files at all', () => {
+    expect(discoverDns(host({}))).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+  });
+});

package/src/services/dns-discovery.ts

@@ -0,0 +1,92 @@
+/**
+ * Read the upstream DNS resolvers this box uses.
+ *
+ * This sits beside `network-discovery.ts` and for the same reason: celilo
+ * records facts about the management box itself, and a module is not the right
+ * place to read them. The network half moved first
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md),
+ * the fleet key followed (openspec/changes/hook-process-boundary, design D9b),
+ * and DNS is the last of the three. It came from
+ * `modules/celilo-mgmt/scripts/discovery.ts`, whose hook could only reach
+ * celilo by spawning the CLI — which a jailed hook cannot do at all
+ * (celilo#1225).
+ *
+ * Reading it here is not merely tidier, it is the only place it can be
+ * correct. celilo-mgmt is never deployed to a remote box (ruled 2026-09-02),
+ * so the host whose resolvers we want is always the host celilo runs on. A
+ * hook reading `/etc/resolv.conf` reads that same file, one process further
+ * out, for no benefit.
+ *
+ * Pure parsers are separated from the IO wrapper so they test without a host
+ * (Rule 2.3).
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+
+/**
+ * Where a host publishes its real upstream resolvers, most trustworthy first.
+ *
+ * systemd-resolved's file comes first deliberately: on such a host
+ * `/etc/resolv.conf` names the local 127.0.0.53 stub rather than the upstreams
+ * behind it, so reading only the second file discovers a loopback address and
+ * writes it into system config as the fleet's DNS.
+ */
+const RESOLVER_FILES = ['/run/systemd/resolve/resolv.conf', '/etc/resolv.conf'] as const;
+
+/** Where we land when the host offers no usable upstream of its own. */
+const PUBLIC_FALLBACK = '1.1.1.1';
+
+export interface DnsServers {
+  primary: string;
+  fallback: string;
+}
+
+/** Extract `nameserver` IPs from resolv.conf-format text, in file order. */
+export function parseNameservers(resolvConf: string): string[] {
+  return resolvConf
+    .split('\n')
+    .map((l) => l.trim())
+    .filter((l) => l.startsWith('nameserver '))
+    .map((l) => l.split(/\s+/)[1])
+    .filter(Boolean);
+}
+
+/**
+ * Choose primary and fallback from discovered nameservers.
+ *
+ * Loopback entries are dropped rather than used. A stub resolver's address is
+ * a real answer to "what does this host query" and the wrong answer to "what
+ * should the fleet query", because nothing else on the network can reach it.
+ */
+export function chooseDns(nameservers: string[]): DnsServers {
+  const upstream = nameservers.filter((ns) => !ns.startsWith('127.') && ns !== '::1');
+  return {
+    primary: upstream[0] ?? PUBLIC_FALLBACK,
+    fallback: upstream[1] ?? PUBLIC_FALLBACK,
+  };
+}
+
+/** Reads a resolver file, or returns null when it is not present. */
+export type ResolverFileReader = (path: string) => string | null;
+
+const readResolverFile: ResolverFileReader = (path) =>
+  existsSync(path) ? readFileSync(path, 'utf-8') : null;
+
+/**
+ * Read this box's upstream resolvers.
+ *
+ * Takes the systemd file only when it yielded a real server. `/etc/resolv.conf`
+ * is the last resort and whatever it gives is accepted, including the fallback,
+ * because there is nothing further to consult.
+ */
+export function discoverDns(read: ResolverFileReader = readResolverFile): DnsServers {
+  for (const path of RESOLVER_FILES) {
+    const contents = read(path);
+    if (contents === null) continue;
+    const servers = chooseDns(parseNameservers(contents));
+    if (servers.primary !== PUBLIC_FALLBACK || path === '/etc/resolv.conf') {
+      return servers;
+    }
+  }
+  return { primary: PUBLIC_FALLBACK, fallback: PUBLIC_FALLBACK };
+}

package/src/services/fleet-checks.ts

@@ -29,7 +29,11 @@ import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capabilit
 import type { ModuleManifest } from '../manifest/schema';
 
 import { getServiceCredentials, listContainerServices } from './container-service';
-import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
+import {
+  CONTROL_PLANE_MODULE_ID,
+  getModuleSystems,
+  listAllModuleSystems,
+} from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -43,7 +47,7 @@ import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
 /** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+const CONTROL_PLANE_MODULE = CONTROL_PLANE_MODULE_ID;
 
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other

package/src/services/health-runner.ts

@@ -17,7 +17,9 @@ import type { HookLogger, HookResult } from '../hooks/types';
 import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
-import { getModuleSystems } from './deployed-systems';
+import { controlPlaneHealthChecks } from './control-plane-health';
+import { CONTROL_PLANE_MODULE_ID, getModuleSystems } from './deployed-systems';
+import { remoteAccessPolicy } from './remote-access';
 
 export interface HealthCheckItem {
   name: string;
@@ -151,6 +153,19 @@ export async function runModuleHealthCheck(
     return { moduleId, status: 'error', checks: [], error: 'Module not found' };
   }
 
+  // The control plane checks itself, in celilo's own process (celilo#1225).
+  //
+  // This was a health_check hook and it failed under the jail in the worst
+  // possible way: it asked `existsSync(db_path)`, the jail deliberately does
+  // not bind celilo's data directory, and so it reported a healthy fleet's
+  // database MISSING and failed the deploy over a file that was right there.
+  // `runFleetChecks` is what `celilo system doctor` already runs and asks
+  // eight questions where the hook asked two.
+  if (moduleId === CONTROL_PLANE_MODULE_ID) {
+    const checks = await controlPlaneHealthChecks(db);
+    return finishHealthCheck(moduleId, module.state, checks, db, options);
+  }
+
   const manifest = module.manifestData as ModuleManifest;
   const hookDef = manifest.hooks?.health_check;
 
@@ -196,6 +211,7 @@ export async function runModuleHealthCheck(
       capabilities: capabilityFunctions,
       requiredCapabilities,
       systems: getModuleSystems(moduleId, db),
+      remoteAccess: remoteAccessPolicy(moduleId, db),
     },
   );
   presentation.finish(hookResult.success);
@@ -220,6 +236,23 @@ export async function runModuleHealthCheck(
   };
 
   const checks = outputs.checks || [];
+  return finishHealthCheck(moduleId, module.state, checks, db, options, hookResult.artifactPaths);
+}
+
+/**
+ * Derive the verdict from the named checks and move the module's state.
+ *
+ * Shared by the hook path and the control plane's own path so the two cannot
+ * drift on what `degraded` means or on when a state transition is suppressed.
+ */
+function finishHealthCheck(
+  moduleId: string,
+  moduleState: ModuleState,
+  checks: HealthCheckItem[],
+  db: DbClient,
+  options: HealthCheckOptions,
+  artifactPaths?: string[],
+): HealthCheckResult {
   const hasFail = checks.some((c) => c.status === 'fail');
   const hasWarn = checks.some((c) => c.status === 'warn');
 
@@ -229,12 +262,12 @@ export async function runModuleHealthCheck(
       ? 'degraded'
       : 'healthy';
 
-  const nextState = nextModuleState(module.state, status, options.unattended ?? false);
+  const nextState = nextModuleState(moduleState, status, options.unattended ?? false);
   if (nextState) {
     db.update(modules).set({ state: nextState }).where(eq(modules.id, moduleId)).run();
   }
 
-  return { moduleId, status, checks, artifactPaths: hookResult.artifactPaths };
+  return { moduleId, status, checks, ...(artifactPaths ? { artifactPaths } : {}) };
 }
 
 /**

package/src/services/module-build.test.ts

@@ -1,8 +1,7 @@
-import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
 import { existsSync } from 'node:fs';
 import { mkdir, rm, writeFile } from 'node:fs/promises';
 import { eq } from 'drizzle-orm';
-import { log } from '../cli/prompts';
 import { type DbClient, createDbClient } from '../db/client';
 import { moduleBuilds, modules } from '../db/schema';
 import { skipIntegration } from '../test-utils/integration-guard';
@@ -40,7 +39,6 @@ describe('Module Build Service', () => {
         version TEXT NOT NULL,
         built_at INTEGER NOT NULL DEFAULT (unixepoch()),
         artifacts TEXT NOT NULL,
-        environment TEXT,
         status TEXT NOT NULL,
         build_log TEXT,
         FOREIGN KEY (module_id) REFERENCES modules(id) ON DELETE CASCADE
@@ -98,65 +96,6 @@ describe('Module Build Service', () => {
       expect(result.error).toContain('does not have a build section');
     });
 
-    test('should detect Nix environment from flake.nix', async () => {
-      // Create test module directory with flake.nix
-      await mkdir(TEST_MODULE_DIR, { recursive: true });
-      await mkdir(`${TEST_MODULE_DIR}/build`, { recursive: true });
-      await writeFile(`${TEST_MODULE_DIR}/flake.nix`, '# Nix flake');
-      await writeFile(
-        `${TEST_MODULE_DIR}/build/playbook.yml`,
-        `---
-- name: Test build
-  hosts: localhost
-  gather_facts: false
-  tasks:
-    - name: Debug message
-      ansible.builtin.debug:
-        msg: "Nix build test"
-`,
-      );
-
-      db.insert(modules)
-        .values({
-          id: 'nix-module',
-          name: 'Nix Module',
-          version: '1.0.0',
-          sourcePath: TEST_MODULE_DIR,
-          manifestData: {
-            id: 'nix-module',
-            name: 'Nix Module',
-            version: '1.0.0',
-            build: {
-              script: 'build/playbook.yml',
-            },
-          },
-        })
-        .run();
-
-      // Mock log.info / log.warn to capture output (module-build.ts now
-      // routes through these instead of console.log so output flows through
-      // the active ProgressDisplay during a real deploy).
-      const logInfoSpy = spyOn(log, 'info').mockImplementation(() => {});
-      const logWarnSpy = spyOn(log, 'warn').mockImplementation(() => {});
-
-      const _result = await buildModuleFromSource('nix-module', db);
-
-      // Verify Nix detection or fallback message was logged
-      // If nix is available: "Entering Nix environment"
-      // If nix not available: "flake.nix detected but nix command not available"
-      const calls = [...logInfoSpy.mock.calls, ...logWarnSpy.mock.calls].map((c) => c[0]);
-      const hasNixMessage = calls.some(
-        (msg) => msg.includes('Nix environment') || msg.includes('flake.nix detected'),
-      );
-      expect(hasNixMessage).toBe(true);
-
-      logInfoSpy.mockRestore();
-      logWarnSpy.mockRestore();
-
-      // Build result depends on whether Nix is installed
-      // We're just testing that detection happens (message logged)
-    }, 10000); // 10 second timeout
-
     test.skipIf(skipIntegration({ tools: ['ansible'] }))(
       'should record build metadata in database',
       async () => {
@@ -208,7 +147,6 @@ describe('Module Build Service', () => {
         expect(buildRecord?.moduleId).toBe('record-test');
         expect(buildRecord?.version).toBe('1.0.0');
         expect(buildRecord?.status).toBe('success');
-        expect(buildRecord?.environment).toBe('system'); // No flake.nix
       },
       10000,
     ); // 10 second timeout for ansible execution
@@ -282,7 +220,6 @@ describe('Module Build Service', () => {
           artifacts: ['/path/to/artifact'],
           status: 'success',
           buildLog: 'Build completed',
-          environment: 'nix',
         })
         .run();