@celilo/cli 0.1.5 → 0.1.7

package/src/hooks/capability-loader.ts

@@ -17,20 +17,13 @@ import {
   isCompiledCapabilityFactory,
   wrapWithLogging,
 } from '@celilo/capabilities';
-import type { HookLogger, RouteOps } from '@celilo/capabilities';
+import type { DnsInternalCapability, HookLogger, RouteOps } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import {
-  capabilities,
-  machines,
-  moduleConfigs,
-  moduleInfrastructure,
-  modules,
-  secrets,
-  webRoutes,
-} from '../db/schema';
+import { capabilities, modules, secrets, webRoutes } from '../db/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
+import { loadHookConfigMap } from './load-hook-config';
 
 /**
  * Mapping of capability names to their function module entry points.
@@ -110,41 +103,9 @@ export async function loadCapabilityFunctions(
     `Loading capabilities. Registry has: public_web (framework), ${Object.keys(CAPABILITY_MODULE_MAP).join(', ')}`,
   );
 
-  // Handle public_web via framework implementation (no dynamic import needed)
-  const publicWebProviders = db
-    .select()
-    .from(capabilities)
-    .where(eq(capabilities.capabilityName, 'public_web'))
-    .all();
-
-  if (publicWebProviders.length > 0) {
-    const provider = publicWebProviders[0];
-    debugLog(`public_web: found provider module ${provider.moduleId}`);
-    const providerConfig = await loadModuleConfig(provider.moduleId, db);
-    const providerSecrets = await loadModuleSecrets(provider.moduleId, masterKey, db);
-    const routeOps = buildRouteOps(db);
-
-    try {
-      // createPublicWeb internally applies wrapWithLogging using the
-      // logger we pass in here, so the consumer doesn't need to wrap
-      // anything at this layer.
-      result.public_web = createPublicWeb({
-        moduleId: consumingModuleId,
-        logger,
-        config: providerConfig,
-        secrets: providerSecrets,
-        routeOps,
-      });
-      debugLog(`public_web: loaded via framework implementation for ${consumingModuleId}`);
-    } catch (error) {
-      debugLog(
-        `public_web: FAILED to load: ${error instanceof Error ? error.message : String(error)}`,
-      );
-    }
-  } else {
-    debugLog('public_web: not registered in DB, skipping');
-  }
-
+  // Load all non-public_web capabilities first so dns_internal (and others)
+  // are available when we construct the framework-owned public_web capability.
+  // public_web optionally threads dns_internal through for split-horizon DNS.
   for (const [capName, moduleInfo] of Object.entries(CAPABILITY_MODULE_MAP)) {
     // Check if this capability is registered (a provider module is deployed)
     // For zone-scoped capabilities (like firewall), there may be multiple providers
@@ -251,6 +212,115 @@ export async function loadCapabilityFunctions(
     }
   }
 
+  // Handle public_web via framework implementation (no dynamic import needed).
+  // Loaded after the capability loop so dns_internal is already available to
+  // thread through for split-horizon DNS record registration.
+  const publicWebProviders = db
+    .select()
+    .from(capabilities)
+    .where(eq(capabilities.capabilityName, 'public_web'))
+    .all();
+
+  if (publicWebProviders.length > 0) {
+    const provider = publicWebProviders[0];
+    debugLog(`public_web: found provider module ${provider.moduleId}`);
+    const providerConfig = await loadModuleConfig(provider.moduleId, db);
+    const providerSecrets = await loadModuleSecrets(provider.moduleId, masterKey, db);
+    const routeOps = buildRouteOps(db);
+
+    // Find the firewall NAT IP so split-horizon records point to the iptables
+    // internal interface rather than Caddy's unreachable DMZ container IP.
+    const firewallProviders = db
+      .select()
+      .from(capabilities)
+      .where(eq(capabilities.capabilityName, 'firewall'))
+      .all();
+    let firewallNatIp: string | undefined;
+    for (const fp of firewallProviders) {
+      const fpConfig = await loadModuleConfig(fp.moduleId, db);
+      if (typeof fpConfig.nat_ip === 'string' && fpConfig.nat_ip) {
+        firewallNatIp = fpConfig.nat_ip;
+        debugLog(
+          `public_web: using firewall natIp ${firewallNatIp} from ${fp.moduleId} for internal DNS`,
+        );
+        break;
+      }
+    }
+
+    // Caddy's configured hostnames — public_web rejects routes for any
+    // hostname not in this list, throwing a structured error that runs
+    // caddy's `managed_hostname` ensure interview. Empty list means
+    // caddy isn't configured yet — also a structured error path
+    // (caller will see it on the first register_route).
+    const caddyHostnames: string[] = [];
+    if (Array.isArray(providerConfig.hostnames)) {
+      for (const h of providerConfig.hostnames) {
+        if (typeof h === 'string' && h) caddyHostnames.push(h);
+      }
+    }
+
+    // Domains managed by some dns_registrar (primary + additional).
+    // Each entry in caddy's hostnames must also be in this list — else
+    // DNS won't resolve and TLS will eventually break. The registrar's
+    // module id drives the second ensure interview when a hostname
+    // isn't covered.
+    const dnsRegistrarProviders = db
+      .select()
+      .from(capabilities)
+      .where(eq(capabilities.capabilityName, 'dns_registrar'))
+      .all();
+    const dnsManagedDomains: string[] = [];
+    let dnsRegistrarModuleId: string | undefined;
+    for (const drp of dnsRegistrarProviders) {
+      // First registrar wins — multi-registrar setups can't be auto-extended
+      // since we'd have to pick which one to add the new domain to.
+      if (!dnsRegistrarModuleId) dnsRegistrarModuleId = drp.moduleId;
+      const drConfig = await loadModuleConfig(drp.moduleId, db);
+      // dns_registrar 4.0.0+ exposes a single `domains` array (the
+      // primary_domain/additional_domains split was collapsed — see
+      // apps/celilo/designs/DNS_REGISTRAR_MULTI_DOMAIN.md). Older
+      // registrars that haven't been migrated still use the split shape;
+      // we keep that fallback so a stale config can still be read.
+      if (Array.isArray(drConfig.domains)) {
+        for (const d of drConfig.domains) {
+          if (typeof d === 'string' && d) dnsManagedDomains.push(d);
+        }
+      } else {
+        if (typeof drConfig.primary_domain === 'string' && drConfig.primary_domain) {
+          dnsManagedDomains.push(drConfig.primary_domain);
+        }
+        if (Array.isArray(drConfig.additional_domains)) {
+          for (const d of drConfig.additional_domains) {
+            if (typeof d === 'string' && d) dnsManagedDomains.push(d);
+          }
+        }
+      }
+    }
+
+    try {
+      result.public_web = createPublicWeb({
+        moduleId: consumingModuleId,
+        logger,
+        config: providerConfig,
+        secrets: providerSecrets,
+        routeOps,
+        dnsInternal: result.dns_internal as DnsInternalCapability | undefined,
+        firewallNatIp,
+        hostnames: caddyHostnames,
+        caddyModuleId: provider.moduleId,
+        dnsManagedDomains,
+        dnsRegistrarModuleId,
+      });
+      debugLog(`public_web: loaded via framework implementation for ${consumingModuleId}`);
+    } catch (error) {
+      debugLog(
+        `public_web: FAILED to load: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  } else {
+    debugLog('public_web: not registered in DB, skipping');
+  }
+
   return result;
 }
 
@@ -294,33 +364,15 @@ function buildCapabilityInterface(
 /**
  * Load all config values for a module.
  *
- * Injects `target_ip` from the deployment machine when the module was deployed
- * to an existing machine (IPAM writes `target_ip` to moduleConfigs for
- * container deploys, but machine deploys skip that path).
+ * Thin wrapper around `loadHookConfigMap` — kept private here so
+ * existing callers in this file don't need an import-path churn. The
+ * shared helper handles the `target_ip` / `ip.primary` machine-IP
+ * fallback for machine deploys (IPAM writes those keys to
+ * `module_configs` for container deploys, but machine deploys skip
+ * that write).
  */
 async function loadModuleConfig(moduleId: string, db: DbClient): Promise<Record<string, unknown>> {
-  const configs = db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, moduleId)).all();
-
-  const result: Record<string, unknown> = {};
-  for (const c of configs) {
-    result[c.key] = c.valueJson ? JSON.parse(c.valueJson) : c.value;
-  }
-
-  if (!result.target_ip) {
-    const infra = db
-      .select()
-      .from(moduleInfrastructure)
-      .where(eq(moduleInfrastructure.moduleId, moduleId))
-      .get();
-    if (infra?.machineId) {
-      const machine = db.select().from(machines).where(eq(machines.id, infra.machineId)).get();
-      if (machine) {
-        result.target_ip = machine.ipAddress;
-      }
-    }
-  }
-
-  return result;
+  return loadHookConfigMap(moduleId, db);
 }
 
 /**
@@ -490,9 +542,17 @@ function buildRouteOps(db: DbClient): RouteOps {
       return db.select().from(webRoutes).all();
     },
     upsertRoute(route) {
-      // Delete existing route for this path + module, then insert
+      // Path uniqueness is scoped to (hostname, path). Same module +
+      // same (hostname, path) is treated as an upsert, so delete first
+      // to keep the operation atomic in the eyes of the unique index.
       db.delete(webRoutes)
-        .where(and(eq(webRoutes.path, route.path), eq(webRoutes.moduleId, route.moduleId)))
+        .where(
+          and(
+            eq(webRoutes.hostname, route.hostname),
+            eq(webRoutes.path, route.path),
+            eq(webRoutes.moduleId, route.moduleId),
+          ),
+        )
         .run();
       db.insert(webRoutes)
         .values({
@@ -500,9 +560,9 @@ function buildRouteOps(db: DbClient): RouteOps {
           moduleId: route.moduleId,
           type: route.type,
           path: route.path,
+          hostname: route.hostname,
           targetHost: route.targetHost ?? null,
           targetPort: route.targetPort ?? null,
-          subdomain: route.subdomain ?? null,
           websocket: route.websocket ?? false,
           contentHash: route.contentHash ?? null,
         })

package/src/hooks/define-hook.test.ts

@@ -55,6 +55,7 @@ function makeContext(overrides: Partial<HookContext> = {}): HookContext {
 // Minimal fakes for the capability interfaces — used in both consumer and
 // provider tests below.
 const fakePublicWeb: PublicWebCapability = {
+  defaultHostname: 'www.example.com',
   async publishStaticSite() {
     return { success: true, path: '/test', filesUploaded: 0, contentHash: 'fake' };
   },
@@ -70,6 +71,9 @@ const fakePublicWeb: PublicWebCapability = {
   async unregister_routes() {
     return undefined;
   },
+  async getServerIp() {
+    return '10.0.10.10';
+  },
 };
 
 const fakeIdp: IdpCapability = {
@@ -261,6 +265,7 @@ describe('defineCapabilityFunction', () => {
     const factory = defineCapabilityFunction({
       capability: 'public_web',
       handler: () => ({
+        defaultHostname: 'www.example.com',
         async publishStaticSite() {
           return { success: true, path: '/x', filesUploaded: 0, contentHash: 'x' };
         },
@@ -276,6 +281,9 @@ describe('defineCapabilityFunction', () => {
         async unregister_routes() {
           return undefined;
         },
+        async getServerIp() {
+          return '10.0.10.10';
+        },
       }),
     });
 
@@ -340,7 +348,7 @@ describe('defineCapabilityFunction', () => {
       logger: makeLogger(),
     });
 
-    const result = await methods.registerHost({ host: 'www', ip: '1.2.3.4' });
+    const result = await methods.registerHost({ fqdn: 'www.example.com', ip: '1.2.3.4' });
     expect(result.success).toBe(true);
     expect(result.outputs.registered_ip).toBe('1.2.3.4');
   });

package/src/hooks/executor.ts

@@ -12,7 +12,7 @@
 
 import { existsSync, mkdirSync, readdirSync, statSync } from 'node:fs';
 import { join, resolve } from 'node:path';
-import { isCompiledHook } from '@celilo/capabilities';
+import { isCompiledHook, isMissingProviderInputError } from '@celilo/capabilities';
 import {
   type ContractHookSignature,
   resolveContract,
@@ -458,6 +458,27 @@ export async function invokeHook(
       logger.info(`Screenshot saved: ${screenshotPath}`);
     }
 
+    // Recognise the cross-module structured error so the orchestrator can
+    // run the ensure interview and retry the hook. The check goes through
+    // a duck-typed guard because @celilo/capabilities can be loaded from
+    // either the workspace package or a module's bundled copy, and
+    // `instanceof` fails across those identity boundaries.
+    if (isMissingProviderInputError(error)) {
+      return {
+        success: false,
+        outputs: {},
+        error: message,
+        screenshotPath,
+        duration: Date.now() - startTime,
+        missingProviderInput: {
+          providerModuleId: error.providerModuleId,
+          ensureId: error.ensureId,
+          value: error.value,
+          humanContext: error.humanContext,
+        },
+      };
+    }
+
     return {
       success: false,
       outputs: {},

package/src/hooks/load-hook-config.test.ts

@@ -0,0 +1,165 @@
+/**
+ * Unit tests for `loadHookConfigMap`.
+ *
+ * The helper centralises the "build the config map a hook script will
+ * see" logic across `capability-loader`, `run-named-hook`, and
+ * `health-runner`. The behaviour under test:
+ *
+ *   1. `module_configs` rows are loaded as-is, parsed via `valueJson`
+ *      when present.
+ *   2. If `target_ip` is already in the rows, the machine fallback is
+ *      skipped entirely (deploy already wrote it).
+ *   3. If `target_ip` isn't set AND the module has a `module_infrastructure`
+ *      row pointing at a machine, fill BOTH `target_ip` and `ip.primary`
+ *      from `machines.ipAddress`.
+ *   4. If neither is true (no infra row, or infra has no machineId,
+ *      or the machine record is gone), neither key gets filled.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import type { DbClient } from '../db/client';
+import { machines, moduleConfigs, moduleInfrastructure, modules } from '../db/schema';
+import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
+import { loadHookConfigMap } from './load-hook-config';
+
+describe('loadHookConfigMap', () => {
+  let db: DbClient;
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+    db.insert(modules)
+      .values({
+        id: 'mod',
+        name: 'test',
+        version: '1.0.0',
+        sourcePath: '/tmp/x',
+        manifestData: {},
+      })
+      .run();
+  });
+
+  afterEach(async () => {
+    await cleanupTestDatabase(db);
+  });
+
+  test('returns empty config map when no rows + no infrastructure', async () => {
+    const result = await loadHookConfigMap('mod', db);
+    expect(result).toEqual({});
+  });
+
+  test('returns scalar string config values from module_configs', async () => {
+    db.insert(moduleConfigs)
+      .values([
+        { moduleId: 'mod', key: 'hostname', value: 'caddy', valueJson: null },
+        { moduleId: 'mod', key: 'acme_email', value: 'admin@example.com', valueJson: null },
+      ])
+      .run();
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result).toEqual({ hostname: 'caddy', acme_email: 'admin@example.com' });
+  });
+
+  test('parses valueJson for complex types (arrays, objects)', async () => {
+    db.insert(moduleConfigs)
+      .values([
+        {
+          moduleId: 'mod',
+          key: 'hostnames',
+          value: '',
+          valueJson: '["www.example.com","example.com"]',
+        },
+        { moduleId: 'mod', key: 'plain', value: 'string-val', valueJson: null },
+      ])
+      .run();
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result.hostnames).toEqual(['www.example.com', 'example.com']);
+    expect(result.plain).toBe('string-val');
+  });
+
+  test('preserves an explicit target_ip in module_configs (skips machine fallback)', async () => {
+    db.insert(moduleConfigs)
+      .values({ moduleId: 'mod', key: 'target_ip', value: '10.0.10.10', valueJson: null })
+      .run();
+    // A machine record exists but should NOT override the explicit value.
+    db.insert(machines)
+      .values({
+        id: 'machine-1',
+        hostname: 'm1',
+        zone: 'dmz',
+        ipAddress: '192.168.0.99',
+        sshUser: 'root',
+        sshKeyEncrypted: 'x',
+        hardware: { cpu_cores: 1, memory_mb: 512, disk_gb: 10 },
+      })
+      .run();
+    db.insert(moduleInfrastructure)
+      .values({
+        id: `infra-${Math.random().toString(36).slice(2)}`,
+        moduleId: 'mod',
+        infrastructureType: 'machine',
+        machineId: 'machine-1',
+      })
+      .run();
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result.target_ip).toBe('10.0.10.10');
+    // ip.primary is only filled by the fallback path; explicit
+    // target_ip means the helper short-circuits and never looks at the
+    // machine.
+    expect(result['ip.primary']).toBeUndefined();
+  });
+
+  test('fills BOTH target_ip and ip.primary from machine.ipAddress when target_ip is unset', async () => {
+    db.insert(machines)
+      .values({
+        id: 'machine-1',
+        hostname: 'caddy-host',
+        zone: 'dmz',
+        ipAddress: '10.0.10.10',
+        sshUser: 'root',
+        sshKeyEncrypted: 'x',
+        hardware: { cpu_cores: 1, memory_mb: 512, disk_gb: 10 },
+      })
+      .run();
+    db.insert(moduleInfrastructure)
+      .values({
+        id: `infra-${Math.random().toString(36).slice(2)}`,
+        moduleId: 'mod',
+        infrastructureType: 'machine',
+        machineId: 'machine-1',
+      })
+      .run();
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result.target_ip).toBe('10.0.10.10');
+    expect(result['ip.primary']).toBe('10.0.10.10');
+  });
+
+  test('leaves target_ip unset when there is no infrastructure record', async () => {
+    db.insert(moduleConfigs)
+      .values({ moduleId: 'mod', key: 'hostname', value: 'caddy', valueJson: null })
+      .run();
+    // no moduleInfrastructure row inserted
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result.hostname).toBe('caddy');
+    expect(result.target_ip).toBeUndefined();
+    expect(result['ip.primary']).toBeUndefined();
+  });
+
+  test('leaves target_ip unset when infrastructure row has no machineId', async () => {
+    db.insert(moduleInfrastructure)
+      .values({
+        id: `infra-${Math.random().toString(36).slice(2)}`,
+        moduleId: 'mod',
+        infrastructureType: 'container_service',
+        machineId: null,
+      })
+      .run();
+
+    const result = await loadHookConfigMap('mod', db);
+    expect(result.target_ip).toBeUndefined();
+    expect(result['ip.primary']).toBeUndefined();
+  });
+});

package/src/hooks/load-hook-config.ts

@@ -0,0 +1,60 @@
+/**
+ * Build the canonical config map a hook script sees.
+ *
+ * Every code path that runs a hook (deploy / health_check / on_uninstall /
+ * `module run-hook`) needs to assemble the same config shape; without a
+ * single source of truth the implementations drift and hooks see
+ * subtly-different state depending on which command invoked them. The
+ * concrete bug that motivated this helper: caddy's `on_uninstall` hook
+ * skipped its DNAT cleanup because `runNamedHook` (a recently-extracted
+ * helper) didn't apply the `target_ip` fallback that
+ * `capability-loader.ts:loadModuleConfig` had — same shape, different
+ * code, easy to miss.
+ *
+ * The shape:
+ *   - Every row from `module_configs`, parsed from `valueJson` when set.
+ *   - If `target_ip` isn't in the row set, look up the deployment
+ *     machine and inject both `target_ip` AND `ip.primary` from
+ *     `machines.ipAddress`. Two keys because consumers historically used
+ *     either name (e.g. caddy's `setup-network.ts` checks
+ *     `target_ip || ip.primary`); fixing the drift means filling both.
+ *
+ * Container deploys write `target_ip` into `module_configs` explicitly
+ * during generate/deploy, so the fallback only fires for machine
+ * deploys (existing iron, no terraform — what every e2e test uses).
+ */
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { machines, moduleConfigs, moduleInfrastructure } from '../db/schema';
+
+export async function loadHookConfigMap(
+  moduleId: string,
+  db: DbClient,
+): Promise<Record<string, unknown>> {
+  const configRecords = db
+    .select()
+    .from(moduleConfigs)
+    .where(eq(moduleConfigs.moduleId, moduleId))
+    .all();
+
+  const configMap: Record<string, unknown> = {};
+  for (const c of configRecords) {
+    configMap[c.key] = c.valueJson ? JSON.parse(c.valueJson) : c.value;
+  }
+
+  if (configMap.target_ip) return configMap;
+
+  const infra = db
+    .select()
+    .from(moduleInfrastructure)
+    .where(eq(moduleInfrastructure.moduleId, moduleId))
+    .get();
+  if (!infra?.machineId) return configMap;
+
+  const machine = db.select().from(machines).where(eq(machines.id, infra.machineId)).get();
+  if (!machine) return configMap;
+
+  configMap.target_ip = machine.ipAddress;
+  configMap['ip.primary'] = machine.ipAddress;
+  return configMap;
+}

package/src/hooks/logger.ts

@@ -6,8 +6,15 @@
  */
 
 import type { FuelGauge } from '../cli/fuel-gauge';
+import { getActiveDisplay } from '../cli/prompts';
 import type { HookLogger } from './types';
 
+const DIM = '\x1b[2m';
+const RESET = '\x1b[0m';
+const GREEN = '\x1b[32m';
+const YELLOW = '\x1b[33m';
+const RED = '\x1b[31m';
+
 /**
  * Create a hook logger that outputs to a FuelGauge progress indicator
  *
@@ -22,20 +29,43 @@ export function createGaugeLogger(
   hookName: string,
 ): HookLogger {
   const prefix = `[${moduleId}:${hookName}]`;
+  const nonInteractive = !process.stdout.isTTY;
+
+  // When a ProgressDisplay is active, the running step header
+  // (e.g. "… celilo-website: on_install") already names the hook,
+  // so the per-line "[moduleId:hookName]" prefix becomes noise.
+  // Dim the message and skip the stdout double-write — display
+  // owns the output channel and forwards via gauge.addOutput →
+  // display.subEvent. When no display is active, fall back to
+  // the prefixed gauge output.
+  function emit(level: 'info' | 'warn' | 'error' | 'success', message: string) {
+    if (getActiveDisplay()) {
+      const icon =
+        level === 'warn'
+          ? `${YELLOW}⚠${RESET} `
+          : level === 'error'
+            ? `${RED}✗${RESET} `
+            : level === 'success'
+              ? `${GREEN}✓${RESET} `
+              : '';
+      gauge.addOutput(`${icon}${DIM}${message}${RESET}`);
+      return;
+    }
+
+    const icon =
+      level === 'warn' ? '⚠ ' : level === 'error' ? '✗ ' : level === 'success' ? '✓ ' : '';
+    const line = `${prefix} ${icon}${message}`;
+    gauge.addOutput(line);
+    if (nonInteractive) {
+      process.stdout.write(`${line}\n`);
+    }
+  }
 
   return {
-    info(message: string) {
-      gauge.addOutput(`${prefix} ${message}`);
-    },
-    warn(message: string) {
-      gauge.addOutput(`${prefix} ⚠ ${message}`);
-    },
-    error(message: string) {
-      gauge.addOutput(`${prefix} ✗ ${message}`);
-    },
-    success(message: string) {
-      gauge.addOutput(`${prefix} ✓ ${message}`);
-    },
+    info: (message: string) => emit('info', message),
+    warn: (message: string) => emit('warn', message),
+    error: (message: string) => emit('error', message),
+    success: (message: string) => emit('success', message),
   };
 }