@celilo/e2e 0.11.3 → 0.13.0

package/simulators/ip-echo/server.ts

@@ -0,0 +1,76 @@
+/**
+ * IP echo simulator — the rig's stand-in for api.ipify.org.
+ *
+ * Reports the source address a request appears to come from, and nothing else.
+ * celilo's `public_dns` check uses it for the ONE thing it cannot ask itself:
+ * what address the fleet is currently reachable at.
+ *
+ * Why an independent service rather than the registrar's own answer: the DDNS
+ * response carries the address the registrar says it published, so comparing
+ * public DNS against it is self-agreement. Worse than useless, in fact —
+ * Namecheap returns `ErrCount 0` with the requested address echoed back for
+ * `www` updates it does not apply, so the response can be actively false. The
+ * expectation has to come from somewhere that has no stake in the answer.
+ *
+ * This sits on `internet-external`, so a request from the customer fleet
+ * arrives having been SNAT'd by the customer firewall — the source address it
+ * sees IS the firewall's external address, which is the address the internet
+ * must dial. That is the same mechanism DDNS source-IP detection relies on,
+ * observed independently.
+ *
+ * Plain HTTP on purpose. Real ipify is HTTPS, but TLS here would mean an ACME
+ * dance (Caddy + Pebble, as the isitup sim does) to test nothing this check
+ * cares about: the property under test is "an off-fleet observer reports our
+ * egress", not certificate validation. The path from the fleet to here still
+ * crosses both firewalls exactly as a real request would.
+ */
+
+// Marks this file a module rather than a script. Without it the simulators
+// share one global scope at typecheck time, and `PORT`/`server` collide with
+// the identically-named top-level bindings in simulators/isitup/server.ts.
+export {};
+
+interface RequestIpProvider {
+  requestIP(req: Request): { address: string } | null;
+}
+
+const PORT = Number(process.env.ECHO_PORT) || 80;
+
+/**
+ * Strip the IPv6-mapped-IPv4 prefix Bun returns for IPv4 connections over a
+ * dual-stack socket (`::ffff:203.0.113.100` → `203.0.113.100`). Same fix as
+ * the namecheap-ddns simulator: the mapped form is not a valid A-record value,
+ * and here it would simply never compare equal to what DNS serves.
+ */
+function normalizeIp(address: string): string {
+  const mapped = address.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/i);
+  return mapped ? mapped[1] : address;
+}
+
+function sourceIp(req: Request, server: RequestIpProvider): string | null {
+  const forwarded = req.headers.get('x-forwarded-for');
+  if (forwarded) return normalizeIp(forwarded.split(',')[0].trim());
+  const remote = server.requestIP(req);
+  return remote?.address ? normalizeIp(remote.address) : null;
+}
+
+const server = Bun.serve({
+  port: PORT,
+  fetch(req, server) {
+    const url = new URL(req.url);
+
+    if (url.pathname === '/health') return Response.json({ ok: true });
+
+    const ip = sourceIp(req, server);
+    if (!ip) {
+      // NEVER a sentinel. A prior version of the DDNS simulator returned
+      // `0.0.0.0` when it could not determine the source, and that value went
+      // straight into a public zone file. An unanswerable probe must read as
+      // an error, which celilo counts as *undetermined* — never as a pass.
+      return new Response('source address could not be determined', { status: 500 });
+    }
+    return new Response(ip, { headers: { 'Content-Type': 'text/plain' } });
+  },
+});
+
+console.log(`[ip-echo] listening on port ${server.port}`);

package/src/address-plan.test.ts

@@ -17,8 +17,8 @@
 
 import { describe, expect, test } from 'bun:test';
 import { execSync } from 'node:child_process';
+import { subnetContains } from '@celilo/capabilities';
 import { SIM_PRIVATE_SUPERNET, ZONE_GATEWAYS, ZONE_SUBNETS, type Zone } from './types';
-import { inCidr } from './zone-classifier';
 
 const REPO_ROOT = execSync('git rev-parse --show-toplevel', {
   cwd: import.meta.dir,
@@ -73,9 +73,9 @@ describe('simulated address plan (#539)', () => {
   test('every zone lives inside the sim supernet', () => {
     for (const zone of Object.keys(ZONE_SUBNETS) as Zone[]) {
       const base = ZONE_SUBNETS[zone].split('/')[0];
-      expect(inCidr(base, SIM_PRIVATE_SUPERNET)).toBe(true);
-      expect(inCidr(ZONE_GATEWAYS[zone], SIM_PRIVATE_SUPERNET)).toBe(true);
-      expect(inCidr(ZONE_GATEWAYS[zone], ZONE_SUBNETS[zone])).toBe(true);
+      expect(subnetContains(SIM_PRIVATE_SUPERNET, base)).toBe(true);
+      expect(subnetContains(SIM_PRIVATE_SUPERNET, ZONE_GATEWAYS[zone])).toBe(true);
+      expect(subnetContains(ZONE_SUBNETS[zone], ZONE_GATEWAYS[zone])).toBe(true);
     }
   });
 

package/src/cli/build.ts

@@ -24,8 +24,16 @@ import { basename, join, resolve } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { stageAptRepo } from '../../scripts/stage-apt-repo';
 import { stageLibsignal } from '../../scripts/stage-libsignal';
+import { explainBuildFailure } from '../doctor';
 import { ensureRegistryServerBundle } from '../registry-bundle';
 
+/**
+ * Ceiling on a single `docker build`. The slowest image here is a few minutes
+ * cold; 15 leaves generous headroom while bounding a wedged build, which
+ * otherwise hangs the whole invocation with no signal at all.
+ */
+const PER_IMAGE_BUILD_TIMEOUT_MS = 15 * 60_000;
+
 /**
  * Walk up from pkgDir to find the celilo monorepo root.
  *
@@ -533,12 +541,34 @@ function buildDockerImages(pkgDir: string): void {
     const result = spawnSync('docker', ['build', '-t', tag, '-f', join(dockerDir, file), context], {
       cwd: pkgDir,
       stdio: 'pipe',
+      timeout: PER_IMAGE_BUILD_TIMEOUT_MS,
     });
     const elapsed = Math.round((Date.now() - start) / 1000);
 
+    // A wedged `docker build` used to hang here forever: spawnSync blocks the
+    // event loop, so even the run-lock heartbeat stops — the holder looks alive
+    // (its PID is) while beating nothing, and the only way anyone noticed was
+    // comparing beatAt to wall-clock by hand. Twice in one evening, ~20 and ~32
+    // minutes. A bounded wait turns that into an immediate, nameable failure.
+    if (result.error && (result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
+      console.log(`${red}✗ TIMED OUT${reset}  ${dim}${elapsed}s${reset}`);
+      console.error(
+        `\n${red}Building ${name} exceeded ${Math.round(PER_IMAGE_BUILD_TIMEOUT_MS / 60_000)} minutes and was killed.${reset}`,
+      );
+      console.error(
+        `${dim}Docker itself is usually still responsive. Reproduce it alone with:${reset}`,
+      );
+      console.error(`${dim}  docker build -f docker/${file} -t ${tag} .${reset}`);
+      console.error(
+        `${dim}If that succeeds, the stall was transient — re-run \`cele2e build-infra\`.${reset}\n`,
+      );
+      process.exit(1);
+    }
+
     if (result.status !== 0) {
       console.log(`${red}✗${reset}  ${dim}${elapsed}s${reset}`);
       console.error(result.stderr?.toString());
+      console.error(explainBuildFailure(result.stderr?.toString() ?? ''));
       process.exit(1);
     }
 

package/src/cli/command-registry.ts

@@ -99,6 +99,22 @@ export const COMMANDS: CommandDef[] = [
     description: 'Show run-lock holder + network status (exit 3 if busy)',
     flags: [{ name: '--json', description: 'Emit a machine-readable lock snapshot for polling' }],
   },
+  {
+    name: 'doctor',
+    description:
+      "Check everything a run needs (baked images, base images, lock, disk) — run's implicit preflight",
+    flags: [{ name: '--json', description: 'Emit the full report as JSON' }],
+  },
+  {
+    name: 'last',
+    description: "Show the most recent run's results dir and counts",
+    flags: [
+      {
+        name: '--json',
+        description: 'Emit {runId, resultsDir, total, passed, failed} for scripting',
+      },
+    ],
+  },
   {
     name: 'release',
     description: 'Free a run-lock left by --keep/up (does not tear down)',
@@ -141,4 +157,7 @@ export const RUN_FLAGS = [
 
 export const DOWN_FLAGS = ['--keep', '--all'];
 
+/** Read-only reporters whose only flag is `--json`. */
+export const JSON_FLAG_COMMANDS = ['status', 'doctor', 'last'];
+
 export const BUILD_INFRA_FLAGS = ['--save', '--skip-modules', '--published'];

package/src/cli/completion.ts

@@ -3,7 +3,14 @@
  * All completions are static (no DB lookups needed).
  */
 
-import { BUILD_INFRA_FLAGS, COMMANDS, DOWN_FLAGS, RUN_FLAGS, UP_PRESETS } from './command-registry';
+import {
+  BUILD_INFRA_FLAGS,
+  COMMANDS,
+  DOWN_FLAGS,
+  JSON_FLAG_COMMANDS,
+  RUN_FLAGS,
+  UP_PRESETS,
+} from './command-registry';
 import { findAllTestNames } from './module-discovery';
 
 export function getCompletions(words: string[], current: number): string[] {
@@ -62,6 +69,14 @@ export function getCompletions(words: string[], current: number): string[] {
     return filterSuggestions(['zsh', 'bash'], args[1] || '');
   }
 
+  if (JSON_FLAG_COMMANDS.includes(command) && currentIndex >= 1) {
+    const used = new Set(args.slice(1));
+    return filterSuggestions(
+      ['--json'].filter((f) => !used.has(f)),
+      args[currentIndex] || '',
+    );
+  }
+
   return [];
 }
 

package/src/cli/index.ts

@@ -11,6 +11,8 @@
  *   cele2e version
  *   cele2e shell [container]
  *   cele2e status
+ *   cele2e doctor
+ *   cele2e last [--json]
  *   cele2e load
  *   cele2e scaffold <test-name>
  */
@@ -19,11 +21,13 @@ import { spawnSync } from 'node:child_process';
 import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';
 import { basename, dirname, join, resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
+import { diagnose, formatHolderLine, formatReport } from '../doctor';
+import { readLastRun } from '../last-run';
 import {
   E2eBusyError,
+  type LockHolder,
   acquireRunLock,
   clearLock,
-  formatBusy,
   lockStatus,
   markKept,
 } from '../run-lock';
@@ -129,6 +133,19 @@ function discoverSuiteTests(suiteRoot: string): { name: string; file: string; gr
   return out;
 }
 
+/**
+ * Say out loud when we cleared this session's own kept stack. Silence would be
+ * worse than the old refusal: the operator asked for a kept stack, and it is
+ * gone now for a reason they should be able to read.
+ */
+function noteAutoRelease(holder: LockHolder | null): void {
+  if (!holder) return;
+  console.log(
+    `Auto-released your own kept stack (${holder.test}, held since ${holder.startedAt}) and continuing.`,
+  );
+  console.log('Use `cele2e run --reuse` instead if you meant to run against it.');
+}
+
 function runScript(script: string, args: string[], env?: Record<string, string>): never {
   const result = spawnSync(join(BIN_DIR, script), args, {
     stdio: 'inherit',
@@ -155,6 +172,8 @@ Commands:
   version           Show cele2e version
   shell [container] Shell into a running container (default: management)
   status            Show run-lock holder + network status (exit 3 if busy)
+  doctor            Check the environment a run needs (run's implicit preflight)
+  last              Show the most recent run's results dir + counts
   release           Free a run-lock left by --keep/up (does not tear down)
   load              Load cached Docker images from tarball
   scaffold <name>   Generate a new test file from template
@@ -180,6 +199,9 @@ Options for \`down\`:
   --keep            Stop containers but preserve volumes
   --all             Also stop competing module containers
 
+Options for \`doctor\` / \`last\` / \`status\`:
+  --json            Machine-readable output
+
 Options for \`build-infra\`:
   --save            Save Docker images to tarball after build
   --skip-modules    Skip .netapp packaging (only rebuild Docker images)
@@ -341,7 +363,11 @@ switch (command) {
     // kept state (cleared by `cele2e down`/`release`) to stop a concurrent run
     // from wiping it.
     try {
-      acquireRunLock({ test: 'up', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID() });
+      const outcome = acquireRunLock({
+        test: 'up',
+        runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID(),
+      });
+      noteAutoRelease(outcome.autoReleasedOwnKept);
       markKept();
     } catch (err) {
       if (err instanceof E2eBusyError) {
@@ -385,7 +411,19 @@ switch (command) {
     if (lock.free) {
       console.log('  free');
     } else if (lock.holder) {
-      console.log(`  BUSY — ${formatBusy(lock.holder)}`);
+      // Heartbeat age is always printed: a wedged holder keeps a live PID, so
+      // the only thing separating "slow" from "hung for 32 minutes" is how long
+      // ago it last beat — previously visible only by reading beatAt by hand.
+      console.log(`  BUSY — ${formatHolderLine(lock.holder)}`);
+      if (lock.suspect) {
+        console.log(
+          '  ⚠ SUSPECT: the process is alive but has stopped beating. It is probably wedged;',
+        );
+        console.log(`    inspect or kill pid ${lock.holder.pid}, then \`cele2e down\`.`);
+      }
+      if (lock.ownKept) {
+        console.log('  This is YOUR OWN kept stack — the next `cele2e run` auto-releases it.');
+      }
     }
     console.log('');
 
@@ -410,6 +448,48 @@ switch (command) {
     break;
   }
 
+  case 'doctor': {
+    // The single "is my environment sane?" answer. `run` calls the same
+    // diagnose() as its preflight, so what doctor reports is exactly what a run
+    // will refuse on — no second, drifting copy of the rules.
+    const report = diagnose({ pkgDir: PKG_DIR });
+    if (args.includes('--json')) {
+      console.log(JSON.stringify(report, null, 2));
+      process.exit(report.ok ? 0 : 1);
+    }
+    console.log('=== cele2e doctor ===');
+    for (const line of formatReport(report)) console.log(line);
+    console.log('');
+    console.log(
+      report.ok ? 'Environment looks runnable.' : 'Environment is NOT runnable — see fixes above.',
+    );
+    process.exit(report.ok ? 0 : 1);
+    break;
+  }
+
+  case 'last': {
+    // Which results directory was MINE? Inferring it from `ls -t results` returns
+    // the newest run, which after a refused start is somebody else's — a suite
+    // that never executed then reads as a clean pass.
+    const last = readLastRun();
+    if (args.includes('--json')) {
+      console.log(JSON.stringify(last));
+      process.exit(last ? 0 : 1);
+    }
+    if (!last) {
+      console.error('No cele2e run recorded yet.');
+      process.exit(1);
+    }
+    console.log(`runId:      ${last.runId}`);
+    console.log(`resultsDir: ${last.resultsDir}`);
+    console.log(`status:     ${last.status}`);
+    console.log(
+      `results:    ${last.passed} passed, ${last.failed} failed${last.skipped > 0 ? `, ${last.skipped} skipped (blocked by an earlier stage)` : ''} of ${last.total}`,
+    );
+    process.exit(0);
+    break;
+  }
+
   case 'load':
     runScript('e2e-load', args);
     break;
@@ -422,10 +502,11 @@ switch (command) {
     // build-infra mutates the shared Docker images — serialize against runs.
     // Released via the lock's exit handler (runBuild may process.exit on failure).
     try {
-      acquireRunLock({
+      const outcome = acquireRunLock({
         test: 'build-infra',
         runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID(),
       });
+      noteAutoRelease(outcome.autoReleasedOwnKept);
     } catch (err) {
       if (err instanceof E2eBusyError) {
         console.error(err.message);

package/src/container-manager.ts

@@ -1,14 +1,17 @@
 import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
-import { existsSync, writeFileSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
 import { startBrowser } from './browser';
 import { MIN_CLI_VERSION, checkCliVersion } from './cli-version-contract';
 import {
   SHARED_PROJECT_NAME,
+  firewallZoneLegs,
   generateTestComposeYaml,
   getAllMachines,
 } from './docker-compose-generator';
+import { explainBuildFailure } from './doctor';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
 import { startSocksProxy } from './socks-proxy';
@@ -20,8 +23,15 @@ import type {
   NetworkHandle,
   ProxyOptions,
   SocksProxyHandle,
+  TopologyPreset,
+} from './types';
+import {
+  CeliloCommandError,
+  ZONE_GATEWAYS,
+  ZONE_SUBNETS,
+  greenwaveRouterIp,
+  internalNatIp,
 } from './types';
-import { CeliloCommandError, ZONE_GATEWAYS, ZONE_SUBNETS, internalNatIp } from './types';
 
 /** Package root — where docker/, config/, simulators/ live */
 const PACKAGE_ROOT = join(__dirname, '..');
@@ -499,7 +509,11 @@ async function streamingBuild(
         console.log('[progress:done] images built');
         resolve();
       } else {
-        reject(new Error(`docker compose build failed (exit ${code}):\n${stderr.slice(-500)}`));
+        reject(
+          new Error(
+            `docker compose build failed (exit ${code}):\n${stderr.slice(-500)}${explainBuildFailure(stderr)}`,
+          ),
+        );
       }
     });
   });
@@ -515,6 +529,7 @@ function buildNetworkHandle(
   projectName: string,
   composeDir: string,
   celiloRoot: string | undefined,
+  topology: TopologyPreset,
 ): NetworkHandle {
   // Track host-side resources spawned via the handle (SOCKS proxies,
   // Playwright browsers). stop() tears these down before the docker
@@ -598,10 +613,21 @@ function buildNetworkHandle(
     },
 
     async deployFirewall(opts = {}): Promise<void> {
-      const zones = opts.zones ?? ['dmz', 'app', 'secure'];
+      // `opts.zones` says which zones get `provided_networks` written to system
+      // config. It does NOT say which NICs the box has — the compose topology
+      // decides that, and it wires fw-main to every segmented zone regardless of
+      // what a test passed. Roughly twenty call sites passed `['dmz']` against a
+      // four-leg firewall, so every one of them under-declared the hardware.
+      //
+      // Deriving the declared list from the topology makes under-declaring
+      // structurally impossible rather than something each future test author has
+      // to remember, and removes one more hand-maintained copy of a list the
+      // compose generator already owns.
+      const providedZones = opts.zones ?? ['dmz', 'app', 'secure'];
+      const declaredZones = firewallZoneLegs(topology);
       const firewallIp = ZONE_GATEWAYS.internal; // fw-main on internal
       const natIp = opts.natIp ?? internalNatIp();
-      const providedNetworks = zones.map((zone) => ({
+      const providedNetworks = providedZones.map((zone) => ({
         zone,
         subnet: ZONE_SUBNETS[zone],
         gateway: ZONE_GATEWAYS[zone],
@@ -637,15 +663,23 @@ function buildNetworkHandle(
       await handle.celilo('module import iptables');
       await handle.celilo(`module config set iptables firewall_ip ${firewallIp}`);
       await handle.celilo(`module config set iptables nat_ip ${natIp}`);
-      await handle.celilo(
-        `module config set iptables zones '${JSON.stringify(['internal', ...zones])}'`,
-      );
+      await handle.celilo(`module config set iptables zones '${JSON.stringify(declaredZones)}'`);
       await handle.celilo(
         `module config set iptables provided_networks '${JSON.stringify(providedNetworks)}'`,
       );
       await handle.celilo('module deploy iptables', 180_000);
     },
 
+    async deployGreenwave(): Promise<void> {
+      const routerIp = greenwaveRouterIp();
+      await handle.celilo(`machine add ${routerIp} --ssh-user root --earmark greenwave`);
+      await handle.celilo('module import greenwave');
+      await handle.celilo(`module config set greenwave router_ip ${routerIp}`);
+      await handle.celilo('module secret set greenwave router_username admin');
+      await handle.celilo('module secret set greenwave router_password admin');
+      await handle.celilo('module deploy greenwave', 180_000);
+    },
+
     async socksProxy(options: ProxyOptions = {}): Promise<SocksProxyHandle> {
       const proxy = await startSocksProxy(projectName, options);
       activeProxies.push(proxy);
@@ -889,7 +923,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   // means the test is replacing the infra resolver with a module under test
   // (e.g., knot-unbound-internal). The bare target machine has no DNS server
   // until its module deploys, so this wait would always time out. Management
-  // can still resolve names via the fallback `nameserver 100.100.0.1` in
+  // can still resolve names via the fallback `nameserver 203.0.113.1` in
   // its resolv.conf during deploy.
   //
   // The resolver-under-test may sit in ANY zone, not just `internal`: ISS-0156
@@ -1063,7 +1097,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
           config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
             ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
-        }dns.primary=100.100.0.1 \
+        }dns.primary=203.0.113.1 \
         dns.fallback=1.0.0.1,8.8.8.8`,
     );
     if (initResult.exitCode !== 0) {
@@ -1095,7 +1129,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   // Emit project name so the runner can persist it if --keep is set
   console.log(`[e2e:project] ${projectName}`);
 
-  return buildNetworkHandle(projectName, composeDir, celiloRoot);
+  return buildNetworkHandle(projectName, composeDir, celiloRoot, config.topology);
 }
 
 /**
@@ -1106,6 +1140,26 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
  *
  * If the project doesn't exist, throws with an actionable error.
  */
+/**
+ * Which topology a already-running stack was built with, read back from its
+ * generated compose file. `direct-internet` is the one where fw-main holds a leg
+ * on `isp-external`; everything else is `default`.
+ */
+function topologyFromComposeFile(composePath: string): TopologyPreset {
+  try {
+    const compose = parseYaml(readFileSync(composePath, 'utf-8')) as {
+      services?: Record<string, { networks?: Record<string, unknown> }>;
+    };
+    return compose.services?.['fw-main']?.networks?.['isp-external']
+      ? 'direct-internet'
+      : 'default';
+  } catch {
+    // An unreadable compose file means the stack was not built by this
+    // generator; `default` is the topology the fleet runs and the safer guess.
+    return 'default';
+  }
+}
+
 export function reconnectNetwork(projectName: string): NetworkHandle {
   const composeDir = PACKAGE_ROOT;
 
@@ -1128,5 +1182,14 @@ export function reconnectNetwork(projectName: string): NetworkHandle {
   console.log(`[progress:done] reconnected to existing network (${projectName})`);
   console.log(`[e2e:project] ${projectName}`);
 
-  return buildNetworkHandle(projectName, composeDir, findCeliloRoot());
+  // `--reuse` has no NetworkConfig to read, so the topology is recovered from the
+  // compose file that built the running stack — the same artifact the generator
+  // wrote. Guessing a default here would silently mis-declare the firewall's legs
+  // on exactly the runs used for debugging.
+  return buildNetworkHandle(
+    projectName,
+    composeDir,
+    findCeliloRoot(),
+    topologyFromComposeFile(join(composeDir, COMPOSE_FILE)),
+  );
 }