@celilo/cli 0.4.0-alpha.1 → 0.4.1

package/src/services/dns-provider-backfill.test.ts

@@ -0,0 +1,150 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DnsRecordRequest, HookLogger } from '@celilo/capabilities';
+import { closeDb, getDb } from '../db/client';
+import { runMigrations } from '../db/migrate';
+import { capabilities, moduleConfigs, modules, webRoutes } from '../db/schema';
+import { backfillWebRouteDns } from './dns-provider-backfill';
+
+const silentLogger: HookLogger = {
+  info() {},
+  warn() {},
+  error() {},
+  debug() {},
+} as unknown as HookLogger;
+
+describe('backfillWebRouteDns (ISS-0029)', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-webroute-backfill-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function seedModule(id: string) {
+    getDb()
+      .insert(modules)
+      .values({
+        id,
+        name: id,
+        version: '1.0.0',
+        manifestData: { id, name: id, version: '1.0.0', celilo_contract: '1.0' },
+        sourcePath: `/tmp/${id}`,
+      })
+      .run();
+  }
+
+  function seedFirewall(natIp: string) {
+    seedModule('iptables');
+    getDb()
+      .insert(capabilities)
+      .values({ moduleId: 'iptables', capabilityName: 'firewall', version: '1.0.0', data: {} })
+      .run();
+    getDb()
+      .insert(moduleConfigs)
+      .values({
+        moduleId: 'iptables',
+        key: 'nat_ip',
+        value: natIp,
+        valueJson: JSON.stringify(natIp),
+      })
+      .run();
+  }
+
+  function seedRoute(hostname: string, path = '/') {
+    // web_routes.module_id is a FK → modules; ensure the owner exists.
+    getDb()
+      .insert(modules)
+      .values({
+        id: 'caddy',
+        name: 'caddy',
+        version: '1.0.0',
+        manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '1.0' },
+        sourcePath: '/tmp/caddy',
+      })
+      .onConflictDoNothing()
+      .run();
+    getDb()
+      .insert(webRoutes)
+      .values({
+        slug: `${hostname}${path}`,
+        moduleId: 'caddy',
+        type: 'reverse_proxy',
+        path,
+        hostname,
+      })
+      .run();
+  }
+
+  /** Stub capability loader returning a dns_internal that records its calls. */
+  function stubLoader(calls: DnsRecordRequest[]) {
+    return async () => ({
+      dns_internal: {
+        async registerRecord(req: DnsRecordRequest) {
+          calls.push(req);
+        },
+        async deleteRecord() {},
+      },
+    });
+  }
+
+  it('registers each DISTINCT web-route hostname at the firewall nat_ip', async () => {
+    seedFirewall('100.64.0.1');
+    seedRoute('apt.celilo.computer', '/');
+    seedRoute('apt.celilo.computer', '/-/publish'); // same host, different path → deduped
+    seedRoute('registry.lunacycle.net', '/');
+
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+
+    expect(calls.map((c) => c.host).sort()).toEqual([
+      'apt.celilo.computer',
+      'registry.lunacycle.net',
+    ]);
+    expect(calls.every((c) => c.value === '100.64.0.1' && c.type === 'A')).toBe(true);
+  });
+
+  it('skips without throwing when no firewall nat_ip is available', async () => {
+    seedRoute('apt.celilo.computer');
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+    expect(calls).toHaveLength(0);
+  });
+
+  it('is a no-op when there are no web routes', async () => {
+    seedFirewall('100.64.0.1');
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+    expect(calls).toHaveLength(0);
+  });
+
+  it('aggregates per-host failures into one error', async () => {
+    seedFirewall('100.64.0.1');
+    seedRoute('a.celilo.computer');
+    seedRoute('b.celilo.computer');
+    const failing = async () => ({
+      dns_internal: {
+        async registerRecord(req: DnsRecordRequest) {
+          if (req.host === 'a.celilo.computer') throw new Error('boom');
+        },
+        async deleteRecord() {},
+      },
+    });
+    await expect(backfillWebRouteDns('technitium', getDb(), silentLogger, failing)).rejects.toThrow(
+      /failed for 1 host/,
+    );
+  });
+});

package/src/services/dns-provider-backfill.ts

@@ -14,10 +14,11 @@
  * D5 division of labour: celilo owns host inventory, the module owns DNS.
  */
 
-import type { HookLogger } from '@celilo/capabilities';
+import type { DnsInternalCapability, HookLogger } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities as capabilitiesTable, modules } from '../db/schema';
+import { capabilities as capabilitiesTable, modules, webRoutes } from '../db/schema';
+import { loadCapabilityFunctions, resolveFirewallNatIp } from '../hooks/capability-loader';
 import { runNamedHook } from '../hooks/run-named-hook';
 import type { HookName } from '../hooks/types';
 import { getModuleSystems } from './deployed-systems';
@@ -73,3 +74,72 @@ export async function backfillProviderDns(
     throw new Error(`DNS backfill failed for ${failures.length} host(s): ${failures.join('; ')}`);
   }
 }
+
+/**
+ * Backfill split-horizon records for every PUBLISHED web-route hostname
+ * (ISS-0029). The per-system backfill above covers each deployed host by its
+ * bare hostname; this covers the FQDNs modules publish via public_web (e.g.
+ * `apt.celilo.computer`), which a late-deploying provider would otherwise never
+ * learn. Each hostname is registered at the firewall NAT IP — the same value
+ * public_web uses for the live registration — so backfill and live agree.
+ *
+ * Unlike the per-system path, this does NOT go through `on_system_event` (which
+ * concatenates `<hostname>.<zone>` and would corrupt an already-FQDN host).
+ * It calls the provider's `registerRecord` directly with the FQDN; the
+ * provider creates the split-horizon zone on demand (Phase 1). Attempts every
+ * hostname, then throws an aggregate error if any failed.
+ */
+export async function backfillWebRouteDns(
+  moduleId: string,
+  db: DbClient,
+  logger: HookLogger,
+  // Injectable so unit tests don't dynamically import a real provider module.
+  loadCaps: typeof loadCapabilityFunctions = loadCapabilityFunctions,
+): Promise<void> {
+  const hostnames = [
+    ...new Set(
+      db
+        .select({ hostname: webRoutes.hostname })
+        .from(webRoutes)
+        .all()
+        .map((r) => r.hostname),
+    ),
+  ];
+  if (hostnames.length === 0) return;
+
+  const natIp = await resolveFirewallNatIp(db);
+  if (!natIp) {
+    // No firewall NAT IP to point at — the live public_web path falls back to
+    // Caddy's IP per route; we don't replicate that here. Records land when the
+    // route is (re)published. Surface it rather than silently doing nothing.
+    logger.warn(
+      `web-route DNS backfill for '${moduleId}' skipped: no firewall nat_ip available (${hostnames.length} hostname(s) deferred to live registration)`,
+    );
+    return;
+  }
+
+  const caps = await loadCaps(moduleId, db, logger);
+  const dnsInternal = caps.dns_internal as DnsInternalCapability | undefined;
+  if (!dnsInternal) {
+    logger.warn(`web-route DNS backfill: dns_internal capability not loadable for '${moduleId}'`);
+    return;
+  }
+
+  logger.info(
+    `Backfilling ${hostnames.length} web-route hostname(s) into '${moduleId}' at ${natIp}`,
+  );
+  const failures: string[] = [];
+  for (const host of hostnames) {
+    try {
+      await dnsInternal.registerRecord({ host, type: 'A', value: natIp });
+    } catch (err) {
+      failures.push(`${host}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  if (failures.length > 0) {
+    throw new Error(
+      `web-route DNS backfill failed for ${failures.length} host(s): ${failures.join('; ')}`,
+    );
+  }
+}

package/src/services/e2e-guard.test.ts

@@ -0,0 +1,38 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { E2E_CONFLICT_FIX, E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
+
+describe('e2e-guard', () => {
+  let prev: string | undefined;
+  beforeEach(() => {
+    prev = process.env.CELILO_DB_PATH;
+  });
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = prev;
+  });
+
+  it('skips the docker check when running against a temp test DB', () => {
+    // Integration tests point CELILO_DB_PATH at os.tmpdir() and must not trip
+    // over a developer's unrelated e2e stack.
+    process.env.CELILO_DB_PATH = join(tmpdir(), 'celilo-test.db');
+    expect(runningE2eContainers()).toEqual([]);
+  });
+
+  it('returns an array and never throws when docker is checked', () => {
+    // A non-temp path defeats the short-circuit, exercising the real docker
+    // path. Result depends on the host's docker state; the contract is "an
+    // array of celilo-e2e-* names, never an exception".
+    process.env.CELILO_DB_PATH = '/opt/celilo/celilo.db';
+    const result = runningE2eContainers();
+    expect(Array.isArray(result)).toBe(true);
+    for (const name of result) {
+      expect(name.startsWith('celilo-e2e-')).toBe(true);
+    }
+  });
+
+  it('exposes a deploy message that includes the remediation hint', () => {
+    expect(E2E_CONFLICT_MESSAGE).toContain(E2E_CONFLICT_FIX);
+    expect(E2E_CONFLICT_MESSAGE).toContain('mutually exclusive');
+  });
+});

package/src/services/e2e-guard.ts

@@ -0,0 +1,43 @@
+/**
+ * Live/e2e mutual-exclusion guard.
+ *
+ * A live deploy and the e2e simulator can't run at the same time: the e2e
+ * stack binds the same docker networks and hostnames a live deploy touches,
+ * so a deploy fired while an e2e network is up (e.g. a leftover `--keep` run)
+ * would collide. Both the deploy itself and the fast pre-flight check use this
+ * helper to refuse before touching any infrastructure.
+ */
+
+import { execSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+
+/** One-line remediation, shared by the deploy error and the preflight error. */
+export const E2E_CONFLICT_FIX = 'Stop e2e tests first: cele2e down';
+
+/** Flat error string the deploy returns when an e2e stack is up. */
+export const E2E_CONFLICT_MESSAGE = `Cannot deploy: e2e test containers are running.
+Live and e2e environments are mutually exclusive.
+${E2E_CONFLICT_FIX}`;
+
+/**
+ * Names of running `celilo-e2e-*` containers (empty = clear to deploy).
+ *
+ * Skipped (returns `[]`) when CELILO_DB_PATH points at a temp dir, since
+ * integration tests run against throwaway DBs and never touch docker — they
+ * must not trip over a developer's unrelated e2e stack. A docker failure
+ * (not installed / not running) is treated as "clear": there's nothing to
+ * conflict with.
+ */
+export function runningE2eContainers(): string[] {
+  if (process.env.CELILO_DB_PATH?.startsWith(tmpdir())) return [];
+  try {
+    const running = execSync('docker ps --format "{{.Names}}" 2>/dev/null', {
+      encoding: 'utf-8',
+      timeout: 5000,
+    });
+    return running.split('\n').filter((name) => name.startsWith('celilo-e2e-'));
+  } catch {
+    // docker not installed / not running — nothing to conflict with.
+    return [];
+  }
+}

package/src/services/module-deploy.ts

@@ -35,6 +35,7 @@ import { waitForSSH } from './deploy-ssh';
 import { executeTerraform, parseTerraformOutputs } from './deploy-terraform';
 import { validateAndPrepareDeployment } from './deploy-validation';
 import { getModuleSystems } from './deployed-systems';
+import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
@@ -341,31 +342,12 @@ async function deployModuleImpl(
     : null;
 
   try {
-    // Check for e2e test containers — live and e2e environments are mutually exclusive.
-    // Skip when using a test database (integration tests use os.tmpdir() paths).
-    const { tmpdir } = await import('node:os');
-    const usingTestDb = process.env.CELILO_DB_PATH?.startsWith(tmpdir());
-    if (!usingTestDb) {
-      try {
-        const { execSync } = await import('node:child_process');
-        const running = execSync('docker ps --format "{{.Names}}" 2>/dev/null', {
-          encoding: 'utf-8',
-          timeout: 5000,
-        });
-        const e2eContainers = running.split('\n').filter((n) => n.startsWith('celilo-e2e-'));
-        if (e2eContainers.length > 0) {
-          return {
-            success: false,
-            error:
-              'Cannot deploy: e2e test containers are running.\n' +
-              'Live and e2e environments are mutually exclusive.\n' +
-              'Stop e2e tests first: cele2e down',
-            phases,
-          };
-        }
-      } catch {
-        // docker ps failed — Docker may not be installed or running, that's fine
-      }
+    // Live and e2e environments are mutually exclusive — refuse if an e2e
+    // stack is up (shared docker networks/hostnames). See e2e-guard.ts.
+    // Pre-flight checks the same condition; this is the defense-in-depth
+    // guard for callers that skip pre-flight.
+    if (runningE2eContainers().length > 0) {
+      return { success: false, error: E2E_CONFLICT_MESSAGE, phases };
     }
 
     const validation = await validateAndPrepareDeployment(moduleId, db);
@@ -1292,7 +1274,7 @@ async function deployModuleImpl(
       // provider (deliveries bind at emit time). Non-providers skip this
       // entirely; their registration rides the system.created event below.
       // v2/EVENT_DRIVEN_HOOK_SUBSCRIPTIONS.md.
-      const { isDnsInternalProvider, backfillProviderDns } = await import(
+      const { isDnsInternalProvider, backfillProviderDns, backfillWebRouteDns } = await import(
         './dns-provider-backfill'
       );
       if (isDnsInternalProvider(moduleId, db)) {
@@ -1304,7 +1286,11 @@ async function deployModuleImpl(
         dnsGauge.start();
         try {
           const dnsLogger = createGaugeLogger(dnsGauge, moduleId, 'dns_backfill');
+          // Per-system records (bare hostnames) ...
           await backfillProviderDns(moduleId, db, dnsLogger);
+          // ... and published web-route FQDNs (apt.celilo.computer), which the
+          // per-system path doesn't cover (ISS-0029).
+          await backfillWebRouteDns(moduleId, db, dnsLogger);
           dnsGauge.stop(true);
         } catch (error) {
           dnsGauge.stop(false);

package/src/services/responder-probe.test.ts

@@ -0,0 +1,87 @@
+/**
+ * Recurrence gate for ISS-0025: a headless deploy interview must fail fast
+ * instead of hanging forever when no responder is listening. The deploy path
+ * routes every interview query through `busInterviewGuarded`, which calls
+ * `ensureResponderForInterview` — so this guard is the choke point to protect.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defineEvents, openBus } from '@celilo/event-bus';
+import { ensureResponderForInterview } from './responder-probe';
+
+const NO_SCHEMAS = defineEvents({});
+
+function setTTY(value: boolean | undefined): void {
+  Object.defineProperty(process.stdin, 'isTTY', { value, configurable: true });
+}
+
+describe('ensureResponderForInterview (ISS-0025)', () => {
+  let dir: string;
+  let dbPath: string;
+  let origEnv: string | undefined;
+  let origTTY: boolean | undefined;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'responder-guard-'));
+    dbPath = join(dir, 'events.db');
+    origEnv = process.env.EVENT_BUS_DB;
+    process.env.EVENT_BUS_DB = dbPath;
+    origTTY = process.stdin.isTTY;
+  });
+  afterEach(() => {
+    if (origEnv === undefined) process.env.EVENT_BUS_DB = undefined;
+    else process.env.EVENT_BUS_DB = origEnv;
+    setTTY(origTTY);
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('throws an actionable error when non-TTY and no responder is listening', async () => {
+    setTTY(false);
+    // No responder on the bus → the probe times out → fail fast, not a hang.
+    await expect(ensureResponderForInterview('config.required.foo.bar')).rejects.toThrow(
+      /No responder is listening/,
+    );
+  });
+
+  it('names the blocked prompt and the remediations in the error', async () => {
+    setTTY(false);
+    let message = '';
+    try {
+      await ensureResponderForInterview('secret.required.caddy.api_token');
+    } catch (err) {
+      message = err instanceof Error ? err.message : String(err);
+    }
+    expect(message).toContain('secret.required.caddy.api_token');
+    expect(message).toContain('celilo events respond');
+  });
+
+  it('resolves when a responder answers the probe', async () => {
+    setTTY(false);
+    const responder = openBus({ dbPath, events: NO_SCHEMAS });
+    responder.watch('responder.probe', (event) => {
+      responder.emitRaw(
+        `${event.type}.reply`,
+        { kind: 'programmatic', emittedBy: 'test' },
+        { replyFor: event.id, emittedBy: 'test' },
+      );
+    });
+
+    await expect(ensureResponderForInterview('config.required.foo.bar')).resolves.toBeUndefined();
+
+    responder.close();
+  });
+
+  it('skips the probe entirely on a TTY (the terminal-responder will answer)', async () => {
+    setTTY(true);
+    // No responder running, but a TTY short-circuits before probing — the
+    // built-in terminal-responder is the responder.
+    await expect(ensureResponderForInterview('config.required.foo.bar')).resolves.toBeUndefined();
+  });
+});

package/src/services/responder-probe.ts

@@ -13,6 +13,7 @@
  */
 
 import { defineEvents, openBus } from '@celilo/event-bus';
+import { getEventBusPath } from '../config/paths';
 
 const NO_SCHEMAS = defineEvents({});
 
@@ -43,3 +44,31 @@ export async function probeForResponder(busDbPath: string, timeoutMs = 1500): Pr
     bus.close();
   }
 }
+
+/**
+ * Fail fast instead of hanging forever on a deploy interview (ISS-0025).
+ *
+ * `busInterview` waits indefinitely (`timeoutMs: 0`) for a responder's reply.
+ * On a TTY the deploy registers a terminal-responder, so a prompt will be
+ * answered — we skip the probe. Headless (non-TTY) with no responder listening,
+ * the prompt would hang forever; we probe once and throw an actionable error so
+ * a `module generate`-style fail-fast applies to deploys too. Call this
+ * immediately before emitting an interview query (see `busInterviewGuarded`).
+ *
+ * @param queryType the interview event type about to be emitted (e.g.
+ *   `config.required.<m>.<k>`), echoed in the error so the operator sees which
+ *   prompt is blocked.
+ */
+export async function ensureResponderForInterview(queryType: string): Promise<void> {
+  if (process.stdin.isTTY) return;
+  const available = await probeForResponder(getEventBusPath());
+  if (available) return;
+  throw new Error(
+    `No responder is listening and stdin isn't a TTY, so this interview prompt can't be answered (${queryType}).
+
+Either:
+  1. Run it in a terminal — the built-in prompt will ask, or
+  2. Start a responder in another shell:  celilo events respond
+     (or pre-stage answers:  celilo events respond --values <file>)`,
+  );
+}

package/src/services/restore-from-file.test.ts

@@ -8,6 +8,7 @@
  * via this service) is an e2e concern, not exercised at the unit level.
  */
 
+import { Database } from 'bun:sqlite';
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import {
   existsSync,
@@ -38,11 +39,14 @@ describe('applyStagedSystemFiles', () => {
     keyPath = join(dir, 'master.key');
     process.env.CELILO_DB_PATH = livePath;
     process.env.CELILO_MASTER_KEY_PATH = keyPath;
+    process.env.CELILO_DATA_DIR = dir; // getModuleStoragePath() = dir/modules
   });
 
   afterEach(() => {
+    closeDb();
     process.env.CELILO_DB_PATH = undefined;
     process.env.CELILO_MASTER_KEY_PATH = undefined;
+    process.env.CELILO_DATA_DIR = undefined;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -124,6 +128,48 @@ describe('applyStagedSystemFiles', () => {
     expect(result.dbApplied).toBe(false);
     expect(result.keyApplied).toBe(false);
   });
+
+  it('lays down staged module source dirs at the modules storage path', () => {
+    const stagedSrc = join(stagingDir, 'module_src');
+    mkdirSync(join(stagedSrc, 'caddy', 'scripts'), { recursive: true });
+    writeFileSync(join(stagedSrc, 'caddy', 'manifest.yml'), 'id: caddy');
+    writeFileSync(join(stagedSrc, 'caddy', 'scripts', 'hook.ts'), '// hook');
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.moduleSourcesApplied).toBe(1);
+    // getModuleStoragePath() = <CELILO_DATA_DIR>/modules = dir/modules.
+    const laid = join(dir, 'modules', 'caddy');
+    expect(readFileSync(join(laid, 'manifest.yml'), 'utf-8')).toBe('id: caddy');
+    expect(readFileSync(join(laid, 'scripts', 'hook.ts'), 'utf-8')).toBe('// hook');
+  });
+
+  it("rewrites every module's source_path to this box on the staged DB before swap", () => {
+    // A real staged SQLite DB whose module points at a FOREIGN (macOS) path —
+    // exactly the macOS→Linux cross-host case ISS-0051 broke on. Build a minimal
+    // self-contained DB (no runMigrations, which would hold a connection and
+    // lock the journal-mode switch the rewrite needs).
+    const stagedDbPath = join(stagingDir, 'celilo.db');
+    const seed = new Database(stagedDbPath);
+    seed.run(
+      'CREATE TABLE modules (id TEXT PRIMARY KEY, name TEXT, version TEXT, manifest_data TEXT, source_path TEXT)',
+    );
+    seed.run(
+      "INSERT INTO modules (id, name, version, manifest_data, source_path) VALUES ('caddy', 'caddy', '2.0.0', '{}', '/Users/someone/Library/Application Support/celilo/modules/caddy')",
+    );
+    seed.close();
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(true);
+
+    // The swapped-in live DB must now point at THIS box's modules dir, not the
+    // source box's dead macOS path.
+    const live = new Database(livePath);
+    const row = live.query("SELECT source_path AS sp FROM modules WHERE id = 'caddy'").get() as {
+      sp: string;
+    };
+    live.close();
+    expect(row.sp).toBe(join(dir, 'modules', 'caddy'));
+  });
 });
 
 describe('restoreFromArtifactFile error paths', () => {