@celilo/e2e 0.11.2 → 0.12.0

package/config/resolver/unbound.conf

@@ -38,6 +38,10 @@ server:
     # releases, so the signal module's deploy-time download resolves inside
     # the sealed network. See SIMULATOR_IPS.SIGNAL_RELEASE.
     local-data: "signal-release.lab. IN A 100.64.0.62"
+    # IP echo simulator — the rig's api.ipify.org. celilo's `public_dns` check
+    # asks it the one thing the fleet cannot ask itself: what address it is
+    # currently reachable at. See SIMULATOR_IPS.IP_ECHO.
+    local-data: "ip-echo.lab. IN A 100.64.0.65"
     # External website prober simulator — without this override, isitup.org
     # falls through the "forward-zone: ." below and resolves to the REAL
     # Namecheap-parked domain on the public internet.

package/config/routing/public-resolver-routes.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+# The OFF-FLEET public resolver — the rig's stand-in for 1.1.1.1.
+#
+# It exists because the fleet's own resolver cannot answer the question the
+# `public_dns` check asks. `system init` points celilo at comcast-resolver
+# (the ISP's), and a deployed fleet at the internal split-horizon resolver;
+# both answer with whatever is right for a client INSIDE. celilo refuses to use
+# either as evidence about the public internet, so the simulation has to
+# contain a second, genuinely independent public resolver — exactly the
+# real-world distinction between your ISP's resolver and 1.1.1.1.
+#
+# Unlike comcast-resolver this sits ON internet-external (the public internet
+# network) alongside the authoritative servers, so its stub-zones reach
+# namecheap-dns on the same wire with no route needed.
+#
+# Replying to the CUSTOMER is a different matter, and getting this wrong is why
+# the first run of the public_dns e2e stage saw nothing: a query from the
+# fleet arrives with the customer's public source address, and the answer goes
+# out via Docker's bridge gateway, which has no path across networks. The reply
+# is dropped and the query times out — which celilo correctly records as
+# *undetermined* rather than a pass, so the check went quiet instead of firing.
+# Every host on internet-external exits by the ISP edge; this is that route.
+# See config/routing/public-sim-entrypoint.sh, and the recurrence gate in
+# src/public-sim-routes.test.ts that catches a simulator shipped without it.
+ip route del default 2>/dev/null || true
+ip route add default via 100.64.0.1 2>/dev/null || true
+
+/usr/sbin/unbound -d &
+
+echo "public-resolver ready on 100.64.0.64 - the rig's 1.1.1.1 (SIMULATOR_IPS.PUBLIC_RESOLVER)"
+sleep infinity

package/config/routing/resolver-internal-routes.sh

@@ -57,6 +57,11 @@ $(printf "${SPLIT_HORIZON_BLOCK}")
     # MinIO S3 simulator — backup/restore target for the migration e2e.
     # See SIMULATOR_IPS.MINIO.
     local-data: "minio.lab. IN A 100.64.0.61"
+    # IP echo simulator — the rig's api.ipify.org. celilo's public_dns check
+    # asks it the one thing the fleet cannot ask itself: what address it is
+    # currently reachable at. See SIMULATOR_IPS.IP_ECHO.
+    # (Backtick-free on purpose — see the note above; this heredoc is unquoted.)
+    local-data: "ip-echo.lab. IN A 100.64.0.65"
     # signal-cli daemon (real, unlinked) and its simulator counterpart. Both
     # sit on the `internal` network — celilo-mgr reaches a notification
     # transport locally; a transport needing public ingress could not tell

package/docker/Dockerfile.ip-echo

@@ -0,0 +1,20 @@
+FROM oven/bun:latest
+
+# iproute2 so the entrypoint can see its own addressing when debugging; the
+# service itself needs no routing changes (it sits on internet-external with
+# the other public simulators and answers requests that arrive there).
+RUN apt-get update && apt-get install -y iproute2 curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY simulators/ip-echo/server.ts /simulator/server.ts
+
+# Route the customer's public prefix via the ISP edge, like every other host
+# on internet-external. Without it a reply to the customer's WAN address goes
+# to Docker's bridge gateway, which has no path across networks, and the
+# connection simply hangs. See config/routing/public-sim-entrypoint.sh.
+COPY config/routing/public-sim-entrypoint.sh /usr/local/bin/public-sim-entrypoint.sh
+
+WORKDIR /simulator
+EXPOSE 80
+ENTRYPOINT ["/bin/sh", "/usr/local/bin/public-sim-entrypoint.sh"]
+CMD ["bun", "run", "server.ts"]

package/docker/Dockerfile.resolver

@@ -14,7 +14,11 @@ RUN chown -R root:unbound /etc/unbound
 COPY config/resolver/unbound.conf /etc/unbound/unbound.conf
 COPY config/resolver/root.hints /etc/unbound/root.hints
 COPY config/routing/resolver-routes.sh /startup.sh
-RUN chmod +x /startup.sh
+# The off-fleet public resolver runs from the SAME image with a different
+# startup (it sits on internet-external, so it needs no route through fw-ext).
+# One image, two `command:` values — see docker-compose-generator.ts.
+COPY config/routing/public-resolver-routes.sh /public-resolver-startup.sh
+RUN chmod +x /startup.sh /public-resolver-startup.sh
 
 EXPOSE 53/tcp 53/udp
 CMD ["/startup.sh"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@celilo/cli-display": "^0.1.10",
-    "@celilo/event-bus": "^0.2.0",
+    "@celilo/event-bus": "^0.4.0",
     "yaml": "^2.8.0",
     "zod": "^3.24.1"
   },

package/simulators/greenwave/state.ts

@@ -9,6 +9,25 @@ const PUBLIC_IP = process.env.ROUTER_PUBLIC_IP || '100.100.0.100';
 // The network interface facing the ISP (isp-external network)
 const EXTERNAL_INTERFACE = process.env.EXTERNAL_INTERFACE || 'eth1';
 
+/**
+ * Vendor extension prefix for TR-181 parameters.
+ *
+ * This simulator stands in for two real devices that speak an identical CGI
+ * protocol and differ only here: the GreenWave C4000XG (`X_GWS_Via`, plus the
+ * Lantiq-chipset `X_LANTIQ_COM_INTERFACE`) and the Axon Networks Q1000K, which
+ * folds both into `X_AXON_`. Defaults to the GreenWave spelling so the existing
+ * greenwave suite is byte-for-byte unaffected.
+ */
+const VENDOR_PREFIX = process.env.ROUTER_VENDOR_PREFIX || 'X_GWS_';
+const PARAM_VIA = `${VENDOR_PREFIX}Via`;
+
+/**
+ * The C4000XG named the interface selector after the chipset vendor rather than
+ * the device vendor, so it does NOT follow the prefix. Axon firmware does.
+ */
+const PARAM_INTERFACE =
+  VENDOR_PREFIX === 'X_GWS_' ? 'X_LANTIQ_COM_INTERFACE' : `${VENDOR_PREFIX}INTERFACE`;
+
 const sessions = new Map<string, Session>();
 const portMappings: PortMapping[] = [];
 let nextIndex = 1;
@@ -72,7 +91,7 @@ export function addPortMapping(params: Record<string, string>): CgiObject[] {
     enable: params.Enable || '1',
     description: params.Description || '',
     protocol: params.Protocol || 'TCP',
-    via: params.X_GWS_Via || 'UI',
+    via: params[PARAM_VIA] || 'UI',
     remoteHost: params.RemoteHost || '',
   };
 
@@ -200,9 +219,9 @@ function mappingToParams(pm: PortMapping): CgiParam[] {
     { ParamName: 'Enable', ParamValue: pm.enable },
     { ParamName: 'Description', ParamValue: pm.description },
     { ParamName: 'Protocol', ParamValue: pm.protocol },
-    { ParamName: 'X_GWS_Via', ParamValue: pm.via },
+    { ParamName: PARAM_VIA, ParamValue: pm.via },
     { ParamName: 'RemoteHost', ParamValue: pm.remoteHost },
-    { ParamName: 'X_LANTIQ_COM_INTERFACE', ParamValue: 'wan' },
+    { ParamName: PARAM_INTERFACE, ParamValue: 'wan' },
     { ParamName: 'AllInterfaces', ParamValue: '0' },
   ];
 }

package/simulators/ip-echo/server.ts

@@ -0,0 +1,76 @@
+/**
+ * IP echo simulator — the rig's stand-in for api.ipify.org.
+ *
+ * Reports the source address a request appears to come from, and nothing else.
+ * celilo's `public_dns` check uses it for the ONE thing it cannot ask itself:
+ * what address the fleet is currently reachable at.
+ *
+ * Why an independent service rather than the registrar's own answer: the DDNS
+ * response carries the address the registrar says it published, so comparing
+ * public DNS against it is self-agreement. Worse than useless, in fact —
+ * Namecheap returns `ErrCount 0` with the requested address echoed back for
+ * `www` updates it does not apply, so the response can be actively false. The
+ * expectation has to come from somewhere that has no stake in the answer.
+ *
+ * This sits on `internet-external`, so a request from the customer fleet
+ * arrives having been SNAT'd by the customer firewall — the source address it
+ * sees IS the firewall's external address, which is the address the internet
+ * must dial. That is the same mechanism DDNS source-IP detection relies on,
+ * observed independently.
+ *
+ * Plain HTTP on purpose. Real ipify is HTTPS, but TLS here would mean an ACME
+ * dance (Caddy + Pebble, as the isitup sim does) to test nothing this check
+ * cares about: the property under test is "an off-fleet observer reports our
+ * egress", not certificate validation. The path from the fleet to here still
+ * crosses both firewalls exactly as a real request would.
+ */
+
+// Marks this file a module rather than a script. Without it the simulators
+// share one global scope at typecheck time, and `PORT`/`server` collide with
+// the identically-named top-level bindings in simulators/isitup/server.ts.
+export {};
+
+interface RequestIpProvider {
+  requestIP(req: Request): { address: string } | null;
+}
+
+const PORT = Number(process.env.ECHO_PORT) || 80;
+
+/**
+ * Strip the IPv6-mapped-IPv4 prefix Bun returns for IPv4 connections over a
+ * dual-stack socket (`::ffff:100.100.0.100` → `100.100.0.100`). Same fix as
+ * the namecheap-ddns simulator: the mapped form is not a valid A-record value,
+ * and here it would simply never compare equal to what DNS serves.
+ */
+function normalizeIp(address: string): string {
+  const mapped = address.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/i);
+  return mapped ? mapped[1] : address;
+}
+
+function sourceIp(req: Request, server: RequestIpProvider): string | null {
+  const forwarded = req.headers.get('x-forwarded-for');
+  if (forwarded) return normalizeIp(forwarded.split(',')[0].trim());
+  const remote = server.requestIP(req);
+  return remote?.address ? normalizeIp(remote.address) : null;
+}
+
+const server = Bun.serve({
+  port: PORT,
+  fetch(req, server) {
+    const url = new URL(req.url);
+
+    if (url.pathname === '/health') return Response.json({ ok: true });
+
+    const ip = sourceIp(req, server);
+    if (!ip) {
+      // NEVER a sentinel. A prior version of the DDNS simulator returned
+      // `0.0.0.0` when it could not determine the source, and that value went
+      // straight into a public zone file. An unanswerable probe must read as
+      // an error, which celilo counts as *undetermined* — never as a pass.
+      return new Response('source address could not be determined', { status: 500 });
+    }
+    return new Response(ip, { headers: { 'Content-Type': 'text/plain' } });
+  },
+});
+
+console.log(`[ip-echo] listening on port ${server.port}`);

package/src/docker-compose-generator.ts

@@ -146,7 +146,7 @@ const PROTECTED_SUBNETS = (['dmz', 'app', 'secure', 'secure-mgmt'] as const)
   .map((zone) => ZONE_SUBNETS[zone])
   .join(' ');
 
-function defaultTopologyServices(): Record<string, unknown> {
+function defaultTopologyServices(config: NetworkConfig): Record<string, unknown> {
   return {
     'fw-main': baseService({
       build: { context: '.', dockerfile: 'docker/Dockerfile.firewall' },
@@ -178,6 +178,9 @@ function defaultTopologyServices(): Record<string, unknown> {
       devices: ['/dev/net/tun'],
       sysctls: ROUTER_SYSCTLS,
       volumes: ['ssh-keys:/ssh-keys:ro'],
+      // Which real ISP router this sim stands in for. The protocol is identical
+      // across the two; only the TR-181 vendor extension prefix differs.
+      environment: { ROUTER_VENDOR_PREFIX: config.routerVendorPrefix ?? 'X_GWS_' },
     }),
   };
 }
@@ -211,7 +214,10 @@ function directInternetTopologyServices(): Record<string, unknown> {
   };
 }
 
-const TOPOLOGY_SERVICES: Record<TopologyPreset, () => Record<string, unknown>> = {
+const TOPOLOGY_SERVICES: Record<
+  TopologyPreset,
+  (config: NetworkConfig) => Record<string, unknown>
+> = {
   default: defaultTopologyServices,
   'direct-internet': directInternetTopologyServices,
 };
@@ -315,6 +321,39 @@ export function generateSharedInfraYaml(): string {
     volumes: getRegistryVolumes(),
   });
 
+  // OFF-FLEET recursive resolver — the rig's 1.1.1.1, and the vantage point
+  // celilo's `public_dns` check resolves from.
+  //
+  // A peer of comcast-resolver, not a replacement: the fleet is configured to
+  // use THAT one (`system init` writes dns.primary=100.100.0.1), and celilo
+  // refuses to accept a resolver it already asks as evidence about the public
+  // internet — a check that used the fleet's own resolver would pass whatever
+  // the internet sees, which is the celilo#626 blindness one layer up. So the
+  // simulation needs two independent public resolvers, exactly as the real
+  // world has your ISP's and Cloudflare's.
+  //
+  // Same image as comcast-resolver (identical unbound.conf → identical
+  // authoritative answers) with a different startup: this one sits ON
+  // internet-external with the authoritative servers, so it needs no route
+  // through fw-ext to reach them.
+  services['public-resolver'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PUBLIC_RESOLVER } },
+    cap_add: ['NET_ADMIN'],
+    command: '/public-resolver-startup.sh',
+  });
+
+  // IP echo service — mimics api.ipify.org. Reports the source address a
+  // request appears to come from, which for the customer fleet is the
+  // firewall's external address after SNAT. This is where the `public_dns`
+  // check's EXPECTATION comes from; comparing public DNS against the
+  // registrar's own response would be self-agreement.
+  services['ip-echo'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.ip-echo' },
+    networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.IP_ECHO } },
+    cap_add: ['NET_ADMIN'],
+  });
+
   // External website prober simulator — mimics isitup.org's /api.json endpoint.
   // Sits on internet-external so probe requests exercise the real public-facing
   // path (target domain → DNS → ISP → firewall → Caddy). Uses comcast-resolver
@@ -532,7 +571,7 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
 
   // --- Topology-specific infrastructure (south of fw-ext) ---
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
-  Object.assign(services, topologyFn());
+  Object.assign(services, topologyFn(config));
 
   // Give fw-main a leg on the control-plane network when celilo-mgr lives there,
   // so the management box is routed like any other segmented network rather than
@@ -715,7 +754,7 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
   });
 
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
-  Object.assign(services, topologyFn());
+  Object.assign(services, topologyFn(config));
 
   const legacyCaddyIp =
     config.dmzMachines.find((m) => m.name === 'caddy')?.ip ??

package/src/network-builder.ts

@@ -59,6 +59,17 @@ export class NetworkBuilder {
     return this;
   }
 
+  /**
+   * Make the `fw-isp` simulator stand in for an Axon Networks Q1000K rather
+   * than the default GreenWave C4000XG. The two speak an identical CGI
+   * protocol; only the TR-181 vendor extension prefix differs, so this swaps
+   * `X_GWS_Via`/`X_LANTIQ_COM_INTERFACE` for `X_AXON_Via`/`X_AXON_INTERFACE`.
+   */
+  axonRouter(): this {
+    this.config.routerVendorPrefix = 'X_AXON_';
+    return this;
+  }
+
   /**
    * Place the celilo management container in a given zone. Defaults to
    * `internal`; `secure-mgmt` gives celilo-mgr its own control-plane network

package/src/simulator-ips.ts

@@ -44,6 +44,27 @@ export const SIMULATOR_IPS = {
   CPANEL_HOST: '100.64.0.63',
   /** signal-cli release host — serves the tarball the signal module downloads at deploy time. */
   SIGNAL_RELEASE: '100.64.0.62',
+  /**
+   * OFF-FLEET recursive resolver — the rig's stand-in for 1.1.1.1, and a peer
+   * of comcast-resolver rather than a replacement for it.
+   *
+   * The distinction is the whole point of celilo's `public_dns` check: the
+   * fleet's own resolver (its ISP's, or its internal split-horizon one)
+   * answers with whatever is correct for a client INSIDE, which is not
+   * evidence about what the internet sees. celilo REFUSES to use a resolver it
+   * is itself configured to use, so verifying public reachability requires a
+   * second, independent public resolver — and until this existed the topology
+   * had exactly one.
+   */
+  PUBLIC_RESOLVER: '100.64.0.64',
+  /**
+   * IP echo service — the rig's stand-in for api.ipify.org. Reports the source
+   * address a request appears to come from, which for the customer fleet is
+   * the firewall's external address after SNAT. The `public_dns` check's
+   * expectation comes from here rather than from the registrar's own response,
+   * which is self-agreement (and, for a Namecheap `www` update, false).
+   */
+  IP_ECHO: '100.64.0.65',
   /** Pebble ACME server (replaces production Let's Encrypt). */
   PEBBLE: '100.64.0.100',
 } as const;

package/src/types.ts

@@ -55,6 +55,15 @@ export interface NetworkConfig {
   domain: string;
   ddnsPassword: string;
   verifyRouting: boolean;
+  /**
+   * Which real ISP router the `fw-isp` simulator stands in for, expressed as
+   * its TR-181 vendor extension prefix. `X_GWS_` (the default) is the GreenWave
+   * C4000XG that `modules/greenwave` drives; `X_AXON_` is the Axon Networks
+   * Q1000K that `modules/axon` drives. The CGI protocol is identical across the
+   * two — this prefix IS the entire difference, which is why one simulator
+   * serves both rather than a near-duplicate second one.
+   */
+  routerVendorPrefix?: string;
   /** Extra volume mounts for the management container (host:container format) */
   managementVolumes: string[];
   /** Include a DHCP client container on the internal network */