@celilo/e2e 0.5.1 → 0.6.1

package/src/docker-compose-generator.ts

@@ -1,4 +1,5 @@
 import { stringify } from 'yaml';
+import { SIMULATOR_IPS } from './simulator-ips';
 import type { MachineSpec, NetworkConfig, TopologyPreset } from './types';
 import { ZONE_GATEWAYS } from './types';
 
@@ -46,6 +47,7 @@ function baseService(opts: {
   environment?: Record<string, string>;
   tmpfs?: string[];
   privileged?: boolean;
+  dns?: string[];
 }) {
   // Auto-assign an image tag from the Dockerfile name so Docker can
   // skip builds when pre-built images exist.
@@ -162,7 +164,7 @@ export function generateSharedInfraYaml(): string {
 
   services['root-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.authoritative-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.53' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.ROOT_DNS } },
     cap_add: ['NET_ADMIN'],
     volumes: [
       './config/dns/knot-root.conf:/config/knot.conf:ro',
@@ -172,22 +174,31 @@ export function generateSharedInfraYaml(): string {
 
   services['tld-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.authoritative-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.54' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.TLD_DNS } },
     cap_add: ['NET_ADMIN'],
     volumes: [
       './config/dns/knot-tld.conf:/config/knot.conf:ro',
       './config/dns/com.zone:/config/com.zone:ro',
       './config/dns/org.zone:/config/org.zone:ro',
+      './config/dns/net.zone:/config/net.zone:ro',
     ],
   });
 
   services['namecheap-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.namecheap-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.55' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.NAMECHEAP_DNS } },
     cap_add: ['NET_ADMIN'],
+    // Per-domain DDNS passwords for the cross-domain test. Fallback
+    // `test123` still applies to any domain not listed.
+    environment: {
+      DDNS_PASSWORDS:
+        '{"iamtheinternet.org":"test123","celilo.computer":"test456","lunacycle.net":"test789"}',
+    },
     volumes: [
       './config/dns/knot-namecheap.conf:/config/knot.conf:ro',
       './config/dns/iamtheinternet.org.zone:/config/iamtheinternet.org.zone',
+      './config/dns/celilo.computer.zone:/config/celilo.computer.zone',
+      './config/dns/lunacycle.net.zone:/config/lunacycle.net.zone',
       './config/dns/park-your-domain.com.zone:/config/park-your-domain.com.zone',
     ],
   });
@@ -204,7 +215,7 @@ export function generateSharedInfraYaml(): string {
 
   services.letsencrypt = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.pebble' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.100' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PEBBLE } },
     cap_add: ['NET_ADMIN'],
     command: '-config /config/pebble-config.json -dnsserver 100.100.0.1:53',
     volumes: [
@@ -214,6 +225,28 @@ export function generateSharedInfraYaml(): string {
     ],
   });
 
+  // Registry stand-in. Context widened to `..` (the `packages/` dir) so the
+  // Dockerfile can COPY from packages/registry-server (single source of truth
+  // shared with the celilo-registry module).
+  services.registry = baseService({
+    build: { context: '..', dockerfile: 'e2e/docker/Dockerfile.registry' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.REGISTRY } },
+    cap_add: ['NET_ADMIN'],
+    volumes: ['../../modules:/modules:ro'],
+  });
+
+  // External website prober simulator — mimics isitup.org's /api.json endpoint.
+  // Sits on internet-external so probe requests exercise the real public-facing
+  // path (target domain → DNS → ISP → firewall → Caddy). Uses comcast-resolver
+  // at 100.100.0.1 as DNS so it can recursively resolve the simulated public
+  // zones (e.g. iamtheinternet.org).
+  services.isitup = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.isitup' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.ISITUP } },
+    cap_add: ['NET_ADMIN'],
+    dns: ['100.100.0.1'],
+  });
+
   const compose = { networks, volumes, services };
   return stringify(compose, { lineWidth: 120 });
 }
@@ -252,7 +285,8 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
   const services: Record<string, unknown> = {};
 
   // --- Management (always present) ---
-  const managementDefaultGw = config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
+  const managementDefaultGw =
+    config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
 
   services.management = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.management' },
@@ -264,7 +298,28 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
       ...(config.managementVolumes ?? []),
     ],
     environment: {
-      DDNS_ENDPOINT: 'http://100.64.0.55:8080/update',
+      DDNS_ENDPOINT: `http://${SIMULATOR_IPS.NAMECHEAP_DNS}:8080/update`,
+      DEFAULT_GATEWAY: managementDefaultGw,
+      CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
+    },
+  });
+
+  // --- Internal split-horizon DNS resolver ---
+  // Resolves the test domain to caddy's internal DMZ IP so management can
+  // reach it via a direct route rather than through the external interface
+  // (hairpin NAT would be needed otherwise). Management already has
+  // nameserver 192.168.0.10 as its primary resolver in /etc/resolv.conf.
+  const caddyIp =
+    config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??
+    config.dmzMachines[0]?.ip ??
+    '10.0.10.10';
+  services['dns-int'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.internal-resolver' },
+    networks: { internal: { ipv4_address: '192.168.0.10' } },
+    cap_add: ['NET_ADMIN'],
+    environment: {
+      CADDY_IP: caddyIp,
+      DOMAIN: config.domain ?? 'iamtheinternet.org',
       DEFAULT_GATEWAY: managementDefaultGw,
     },
   });
@@ -306,13 +361,18 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
     const zoneNetwork = machine.zone;
     const gateway = ZONE_GATEWAYS[machine.zone];
 
-    const isAppZone = machine.zone === 'app';
-    const dockerfile = isAppZone
-      ? 'docker/Dockerfile.target-machine-docker'
-      : 'docker/Dockerfile.target-machine';
+    const needsDocker = machine.zone === 'app' || machine.docker === true;
+    const dockerfile =
+      machine.dockerfile ||
+      (needsDocker
+        ? 'docker/Dockerfile.target-machine-docker'
+        : 'docker/Dockerfile.target-machine');
 
     const machineVolumes = ['ssh-keys:/ssh-keys:ro'];
-    if (isAppZone) {
+
+    // Mount the host image cache into Docker-capable machines so the
+    // docker-image-preload service can load them at startup without pulling.
+    if (needsDocker) {
       machineVolumes.push('./docker-image-cache:/docker-image-cache:ro');
     }
 
@@ -364,7 +424,8 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
 
   const services: Record<string, unknown> = {};
 
-  const managementDefaultGw = config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
+  const managementDefaultGw =
+    config.topology === 'direct-internet' ? '192.168.0.254' : '192.168.0.1';
 
   services.management = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.management' },
@@ -376,14 +437,30 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
       ...(config.managementVolumes ?? []),
     ],
     environment: {
-      DDNS_ENDPOINT: 'http://100.64.0.55:8080/update',
+      DDNS_ENDPOINT: `http://${SIMULATOR_IPS.NAMECHEAP_DNS}:8080/update`,
       DEFAULT_GATEWAY: managementDefaultGw,
+      CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
     },
   });
 
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
   Object.assign(services, topologyFn());
 
+  const legacyCaddyIp =
+    config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??
+    config.dmzMachines[0]?.ip ??
+    '10.0.10.10';
+  services['dns-int'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.internal-resolver' },
+    networks: { internal: { ipv4_address: '192.168.0.10' } },
+    cap_add: ['NET_ADMIN'],
+    environment: {
+      CADDY_IP: legacyCaddyIp,
+      DOMAIN: config.domain ?? 'iamtheinternet.org',
+      DEFAULT_GATEWAY: managementDefaultGw,
+    },
+  });
+
   services['fw-ext'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.router' },
     networks: {
@@ -406,7 +483,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
 
   services['root-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.authoritative-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.53' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.ROOT_DNS } },
     cap_add: ['NET_ADMIN'],
     volumes: [
       './config/dns/knot-root.conf:/config/knot.conf:ro',
@@ -416,22 +493,31 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
 
   services['tld-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.authoritative-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.54' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.TLD_DNS } },
     cap_add: ['NET_ADMIN'],
     volumes: [
       './config/dns/knot-tld.conf:/config/knot.conf:ro',
       './config/dns/com.zone:/config/com.zone:ro',
       './config/dns/org.zone:/config/org.zone:ro',
+      './config/dns/net.zone:/config/net.zone:ro',
     ],
   });
 
   services['namecheap-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.namecheap-dns' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.55' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.NAMECHEAP_DNS } },
     cap_add: ['NET_ADMIN'],
+    // Per-domain DDNS passwords for the cross-domain test. Fallback
+    // `test123` still applies to any domain not listed.
+    environment: {
+      DDNS_PASSWORDS:
+        '{"iamtheinternet.org":"test123","celilo.computer":"test456","lunacycle.net":"test789"}',
+    },
     volumes: [
       './config/dns/knot-namecheap.conf:/config/knot.conf:ro',
       './config/dns/iamtheinternet.org.zone:/config/iamtheinternet.org.zone',
+      './config/dns/celilo.computer.zone:/config/celilo.computer.zone',
+      './config/dns/lunacycle.net.zone:/config/lunacycle.net.zone',
       './config/dns/park-your-domain.com.zone:/config/park-your-domain.com.zone',
     ],
   });
@@ -448,7 +534,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
 
   services.letsencrypt = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.pebble' },
-    networks: { 'internet-external': { ipv4_address: '100.64.0.100' } },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PEBBLE } },
     cap_add: ['NET_ADMIN'],
     command: '-config /config/pebble-config.json -dnsserver 100.100.0.1:53',
     volumes: [
@@ -458,6 +544,16 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     ],
   });
 
+  // Registry stand-in. Context widened to `..` (the `packages/` dir) so the
+  // Dockerfile can COPY from packages/registry-server (single source of truth
+  // shared with the celilo-registry module).
+  services.registry = baseService({
+    build: { context: '..', dockerfile: 'e2e/docker/Dockerfile.registry' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.REGISTRY } },
+    cap_add: ['NET_ADMIN'],
+    volumes: ['../../modules:/modules:ro'],
+  });
+
   const allMachines = [
     ...config.dmzMachines,
     ...config.appMachines,
@@ -469,13 +565,16 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     const zoneNetwork = machine.zone;
     const gateway = ZONE_GATEWAYS[machine.zone];
 
-    const isAppZone = machine.zone === 'app';
-    const dockerfile = isAppZone
-      ? 'docker/Dockerfile.target-machine-docker'
-      : 'docker/Dockerfile.target-machine';
+    const needsDocker = machine.zone === 'app' || machine.docker === true;
+    const dockerfile =
+      machine.dockerfile ||
+      (needsDocker
+        ? 'docker/Dockerfile.target-machine-docker'
+        : 'docker/Dockerfile.target-machine');
 
     const machineVolumes = ['ssh-keys:/ssh-keys:ro'];
-    if (isAppZone) {
+
+    if (needsDocker) {
       machineVolumes.push('./docker-image-cache:/docker-image-cache:ro');
     }
 
@@ -502,5 +601,10 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
 }
 
 export function getAllMachines(config: NetworkConfig): MachineSpec[] {
-  return [...config.dmzMachines, ...config.appMachines, ...config.secureMachines, ...config.internalMachines];
+  return [
+    ...config.dmzMachines,
+    ...config.appMachines,
+    ...config.secureMachines,
+    ...config.internalMachines,
+  ];
 }

package/src/network-builder.ts

@@ -78,6 +78,27 @@ export class NetworkBuilder {
     return this;
   }
 
+  /** Add internal-zone machines that need the Docker-capable image (e.g., Technitium) */
+  internalDocker(machines: Record<string, string>): this {
+    for (const [name, ip] of Object.entries(machines)) {
+      this.config.internalMachines.push({ name, ip, zone: 'internal', docker: true });
+    }
+    return this;
+  }
+
+  /** Add a Technitium DNS machine using the dedicated pre-installed image */
+  technitium(machines: Record<string, string>): this {
+    for (const [name, ip] of Object.entries(machines)) {
+      this.config.internalMachines.push({
+        name,
+        ip,
+        zone: 'internal',
+        dockerfile: 'docker/Dockerfile.target-technitium',
+      });
+    }
+    return this;
+  }
+
   withDhcpClient(): this {
     this.config.dhcpClient = true;
     return this;

package/src/parse-line.ts

@@ -0,0 +1,147 @@
+/**
+ * Line parser for the e2e test runner.
+ *
+ * Translates celilo CLI output lines into progress display calls.
+ * Extracted as a separate module so it can be unit-tested without
+ * spinning up Docker or a full ProgressDisplay.
+ */
+
+const dim = '\x1b[2m';
+const green = '\x1b[32m';
+const red = '\x1b[31m';
+const reset = '\x1b[0m';
+
+export interface DisplaySink {
+  startStep(doing: string, done: string): void;
+  doneStep(override?: string): void;
+  failStep(message: string): void;
+  instantEvent(message: string): void;
+  subEvent(message: string): void;
+}
+
+export function parseLine(line: string, display: DisplaySink): void {
+  // Protocol: [progress:start] doing | done
+  const startMatch = line.match(/\[progress:start\] (.*?) \| (.*)/);
+  if (startMatch) {
+    display.startStep(startMatch[1], startMatch[2]);
+    return;
+  }
+
+  // Protocol: [progress:done] optional message
+  const doneMatch = line.match(/\[progress:done\](.*)/);
+  if (doneMatch) {
+    const msg = doneMatch[1].trim();
+    display.doneStep(msg || undefined);
+    return;
+  }
+
+  // Protocol: [progress:fail] message
+  const failMatch = line.match(/\[progress:fail\] (.*)/);
+  if (failMatch) {
+    display.failStep(failMatch[1]);
+    return;
+  }
+
+  // Protocol: [progress:sub] message (sub-event under the current step)
+  const subMatch = line.match(/\[progress:sub\] (.*)/);
+  if (subMatch) {
+    display.subEvent(subMatch[1]);
+    return;
+  }
+
+  // Legacy: [progress] message (treat as instant event)
+  const legacyMatch = line.match(/\[progress\] (.*)/);
+  if (legacyMatch) {
+    display.instantEvent(legacyMatch[1]);
+    return;
+  }
+
+  const deployMatch = line.match(/Module '([^']+)' deployed successfully/);
+  if (deployMatch) {
+    display.subEvent(`${green}✔${reset} ${deployMatch[1]} deployed`);
+    return;
+  }
+
+  const importMatch = line.match(/Successfully imported module: (\S+)/);
+  if (importMatch) {
+    display.subEvent(`${green}✔${reset} ${importMatch[1]} imported`);
+    return;
+  }
+
+  if (line.includes('Deploying software')) {
+    display.subEvent('running Ansible');
+    return;
+  }
+  if (line.includes('Running on_install hook') || line.includes('on_install hook')) {
+    display.subEvent('running on_install hook');
+    return;
+  }
+  if (line.includes('Software deployed') || line.includes('Software deployed successfully')) {
+    display.subEvent(`${green}✔${reset} Ansible done`);
+    return;
+  }
+  if (line.includes('Infrastructure selected')) {
+    display.subEvent('infrastructure selected');
+    return;
+  }
+  if (line.includes('Terraform skipped')) {
+    display.subEvent('terraform skipped (existing machine)');
+    return;
+  }
+  if (line.includes('terraform init') || line.includes('Terraform init')) {
+    display.subEvent('running terraform init');
+    return;
+  }
+  if (line.includes('terraform apply') || line.includes('Terraform apply')) {
+    display.subEvent('running terraform apply');
+    return;
+  }
+
+  const retryMatch = line.match(/RETRYING.*?(\d+) retries left/);
+  if (retryMatch) {
+    display.subEvent(`health check (${retryMatch[1]} retries left)`);
+    return;
+  }
+
+  // Ansible progress markers emitted via process.stderr in deploy-ansible.ts
+  if (line.startsWith('[ansible:play] ')) {
+    display.subEvent(`${dim}▶${reset} ${line.slice(15)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:task] ')) {
+    display.subEvent(`  • ${line.slice(15)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:handler] ')) {
+    display.subEvent(`  ↺ ${line.slice(18)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:recap] ')) {
+    const recap = line.slice(16);
+    const failed = recap.match(/failed=(\d+)/)?.[1];
+    const icon = failed && failed !== '0' ? `${red}✗${reset}` : `${green}✔${reset}`;
+    display.subEvent(`${icon} ${recap}`);
+    return;
+  }
+
+  // Hook log output: "[moduleId:hookName] message"
+  const hookLogMatch = line.match(/^\[[\w-]+:[\w_-]+\] (.*)/);
+  if (hookLogMatch) {
+    display.subEvent(`${dim}hook:${reset} ${hookLogMatch[1]}`);
+    return;
+  }
+
+  // Error lines. Require the identifier ending in "Error:" to be either
+  // standalone ("Error:") or PascalCase ("CeliloCommandError:") — NOT a
+  // camelCase variable like "stageError:" that bun prints as source context
+  // for failing tests.
+  const errorMatch = line.match(/\b(?:[A-Z]\w*)?Error: (.*)/);
+  if (
+    errorMatch &&
+    !line.includes('bun test') &&
+    !line.includes('ipv4: Address already')
+  ) {
+    const err = errorMatch[1].slice(0, 70);
+    display.subEvent(`${red}✗${reset} ${err}`);
+  }
+}