@celilo/e2e 0.3.0 → 0.5.0

package/bin/e2e-build

@@ -1,12 +1,12 @@
 #!/bin/bash
 #
-# Pre-build all E2E Docker images.
+# Pre-build all E2E Docker images and package standard modules as .netapps.
 #
 # Run once before the test suite. Subsequent test runs use the cached
 # images and skip the build phase entirely (~5s vs ~3min per test).
 #
 # Usage:
-#   ./e2e/bin/e2e-build          # build all images
+#   ./e2e/bin/e2e-build          # build netapps + all images
 #   ./e2e/bin/e2e-build --save   # build + save to tarball for colima restarts
 #
 
@@ -14,6 +14,11 @@ set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 E2E_DIR="$(dirname "$SCRIPT_DIR")"
+PKG_DIR="$E2E_DIR"
+REPO_ROOT="$(cd "$E2E_DIR/../.." && pwd)"
+MODULES_DIR="$REPO_ROOT/modules"
+NETAPPS_DIR="$E2E_DIR/netapps"
+
 cd "$E2E_DIR"
 
 GREEN='\033[0;32m'
@@ -21,6 +26,43 @@ DIM='\033[2m'
 BOLD='\033[1m'
 NC='\033[0m'
 
+# ── Step 1: Package standard modules as .netapp files ─────────────────────────
+
+STANDARD_MODULES=(namecheap greenwave iptables caddy authentik)
+
+echo -e "${BOLD}Packaging standard modules...${NC}"
+echo ""
+
+mkdir -p "$NETAPPS_DIR"
+
+for MODULE in "${STANDARD_MODULES[@]}"; do
+  SRC="$MODULES_DIR/$MODULE"
+  OUT="$NETAPPS_DIR/$MODULE.netapp"
+
+  if [ ! -d "$SRC" ]; then
+    echo -e "  ${DIM}skip${NC}  $MODULE  ${DIM}(not found at $SRC)${NC}"
+    continue
+  fi
+
+  printf "  %-20s " "$MODULE"
+  BUILD_START=$(date +%s)
+
+  # Use the wrapper script if available, else bun directly
+  if [ -x "$REPO_ROOT/celilo" ]; then
+    "$REPO_ROOT/celilo" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  else
+    bun run "$REPO_ROOT/apps/celilo/src/cli/index.ts" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  fi
+
+  BUILD_END=$(date +%s)
+  SIZE=$(du -h "$OUT" 2>/dev/null | cut -f1)
+  echo -e "${GREEN}✔${NC} ${DIM}$((BUILD_END - BUILD_START))s  ${SIZE}${NC}"
+done
+
+echo ""
+
+# ── Step 2: Build Docker images ───────────────────────────────────────────────
+
 echo -e "${BOLD}Building E2E Docker images...${NC}"
 echo ""
 
@@ -48,7 +90,8 @@ TOTAL_TIME=$((END - START))
 echo ""
 echo -e "${GREEN}All $TOTAL images built in ${TOTAL_TIME}s${NC}"
 
-# Optionally save to tarball
+# ── Step 3: Optionally save images to tarball ─────────────────────────────────
+
 if [ "$1" = "--save" ]; then
   echo ""
   echo -e "${BOLD}Saving images to tarball...${NC}"

package/bin/e2e-down

@@ -1,10 +1,13 @@
 #!/bin/bash
-# Tear down ALL E2E test artifacts — interactive sessions AND orphaned test runs.
+# Tear down E2E test artifacts:
+# - Persistent test networks (from `e2e-run --keep`)
+# - Interactive sessions (from `e2e-up`)
+# - Orphaned test runs
 #
 # Usage:
-#   ./bin/e2e-down           # Stop and remove everything
-#   ./bin/e2e-down --keep    # Stop containers but keep volumes (faster restart)
-#   ./bin/e2e-down --all     # Also stop module containers (authentik, etc.) that compete for resources
+#   ./bin/e2e-down           # tear down everything e2e-related
+#   ./bin/e2e-down --keep    # stop containers but keep volumes
+#   ./bin/e2e-down --all     # also stop module containers that compete for resources
 
 set -e
 SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
@@ -19,20 +22,42 @@ for arg in "$@"; do
   esac
 done
 
-# 1. Stop the interactive session (if running)
+# 1. Tear down any persistent test networks (look for marker files in common test dirs)
+for dir in "$(pwd)" "$(pwd)/../../e2e" "$(pwd)/../../../e2e"; do
+  MARKER="$dir/.e2e-persistent.json"
+  if [ -f "$MARKER" ]; then
+    PROJECT=$(bun -e "console.log(JSON.parse(require('fs').readFileSync('$MARKER', 'utf-8')).projectName)" 2>/dev/null || echo "")
+    if [ -n "$PROJECT" ]; then
+      echo "Tearing down persistent network: $PROJECT"
+      docker compose -f docker-compose.test.yml -p "$PROJECT" down --volumes --remove-orphans 2>/dev/null || true
+    fi
+    rm -f "$MARKER"
+  fi
+done
+
+# Also check E2E_TEST_DIR if set
+if [ -n "$E2E_TEST_DIR" ] && [ -f "$E2E_TEST_DIR/.e2e-persistent.json" ]; then
+  PROJECT=$(bun -e "console.log(JSON.parse(require('fs').readFileSync('$E2E_TEST_DIR/.e2e-persistent.json', 'utf-8')).projectName)" 2>/dev/null || echo "")
+  if [ -n "$PROJECT" ]; then
+    echo "Tearing down persistent network: $PROJECT"
+    docker compose -f docker-compose.test.yml -p "$PROJECT" down --volumes --remove-orphans 2>/dev/null || true
+  fi
+  rm -f "$E2E_TEST_DIR/.e2e-persistent.json"
+fi
+
+# 2. Stop the interactive session (if running)
 echo "Stopping interactive session..."
-docker compose -p "conductor-e2e-interactive" down --remove-orphans 2>/dev/null || true
+docker compose -p "celilo-e2e-interactive" down --remove-orphans 2>/dev/null || true
 
-# 2. Kill ALL containers matching conductor-e2e-* (orphaned test runs)
-CONTAINERS=$(docker ps -a --filter "name=conductor-e2e" --format "{{.ID}}" 2>/dev/null)
+# 3. Kill ALL containers matching celilo-e2e-* (orphaned test runs)
+CONTAINERS=$(docker ps -a --filter "name=celilo-e2e" --format "{{.ID}}" 2>/dev/null)
 if [ -n "$CONTAINERS" ]; then
   COUNT=$(echo "$CONTAINERS" | wc -l | tr -d ' ')
   echo "Removing $COUNT orphaned e2e containers..."
   echo "$CONTAINERS" | xargs docker rm -f 2>/dev/null || true
 fi
 
-# 3. Stop module containers that compete for resources (--all flag)
-# These come from local docker-compose runs, not from e2e, but starve the VM.
+# 4. Stop module containers that compete for resources (--all flag)
 if [ "$STOP_ALL" = true ]; then
   MODULE_PATTERNS="authentik- caddy- build-your-own-internet-"
   for pattern in $MODULE_PATTERNS; do
@@ -45,23 +70,23 @@ if [ "$STOP_ALL" = true ]; then
   done
 fi
 
-# 3. Remove all conductor-e2e networks
-NETWORKS=$(docker network ls --filter "name=conductor-e2e" --format "{{.ID}}" 2>/dev/null)
+# 5. Remove all celilo-e2e networks
+NETWORKS=$(docker network ls --filter "name=celilo-e2e" --format "{{.ID}}" 2>/dev/null)
 if [ -n "$NETWORKS" ]; then
   echo "Removing e2e networks..."
   echo "$NETWORKS" | xargs docker network rm 2>/dev/null || true
 fi
 
-# 4. Remove volumes (unless --keep)
+# 6. Remove volumes (unless --keep)
 if [ "$KEEP_VOLUMES" = false ]; then
-  VOLUMES=$(docker volume ls --filter "name=conductor-e2e" --format "{{.Name}}" 2>/dev/null)
+  VOLUMES=$(docker volume ls --filter "name=celilo-e2e" --format "{{.Name}}" 2>/dev/null)
   if [ -n "$VOLUMES" ]; then
     echo "Removing e2e volumes..."
     echo "$VOLUMES" | xargs docker volume rm 2>/dev/null || true
   fi
 fi
 
-# 5. Clean up generated compose file
+# 7. Clean up generated compose files
 rm -f docker-compose.test.yml
 
 echo "E2E environment cleaned."

package/config/dns/iamtheinternet.org.zone

@@ -2,7 +2,7 @@ $ORIGIN iamtheinternet.org.
 $TTL 300
 
 @ IN SOA ns1.iamtheinternet.org. admin.iamtheinternet.org. (
-    2024010769 300 60 604800 300
+    2024010778 300 60 604800 300
 )
 
 @       IN NS   ns1.iamtheinternet.org.

package/docker/Dockerfile.management

@@ -22,9 +22,19 @@ RUN update-ca-certificates
 # Point apt at the simulated apt cache proxy (for runtime apt usage)
 RUN echo 'Acquire::http::Proxy "http://100.64.0.2:3142";' > /etc/apt/apt.conf.d/00proxy
 
-# Install celilo CLI from npm — provides the `celilo` command
+# Install celilo CLI from npm — provides the fallback `celilo` command
 RUN bun add -g @celilo/cli@0.1.4
 
+# Shim: prefer /celilo source when mounted (dev loop), fall back to npm install.
+# This means `celilo ...` runs the monorepo's source code during e2e tests
+# instead of the pinned npm version — critical for iterating on fixes.
+RUN printf '#!/bin/bash\nif [ -d /celilo/apps/celilo ]; then\n  exec bun run /celilo/apps/celilo/src/cli/index.ts "$@"\nfi\nexec /root/.bun/bin/celilo "$@"\n' > /usr/local/bin/celilo \
+    && chmod +x /usr/local/bin/celilo
+
+# Standard modules pre-packaged as .netapp files — built by e2e-build.
+# Tests import from /netapps/<module>.netapp instead of the monorepo mount.
+COPY netapps/ /netapps/
+
 COPY config/ssh/generate-keys.sh /config/ssh/generate-keys.sh
 COPY config/routing/management-routes.sh /startup.sh
 RUN chmod +x /startup.sh /config/ssh/generate-keys.sh

package/docker/Dockerfile.target-machine

@@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/docker/Dockerfile.target-machine-docker

@@ -16,6 +16,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -21,6 +21,7 @@
     "config/",
     "simulators/",
     "bin/",
+    "netapps/",
     "README.md"
   ],
   "dependencies": {

package/src/container-manager.ts

@@ -158,6 +158,66 @@ async function waitFor(
   throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms`);
 }
 
+/**
+ * Run `docker compose build` with per-service progress signals.
+ * Uses --progress=plain so output is parseable on non-TTY.
+ * Emits one [progress:start] per service being built so the runner
+ * shows "building <service>" as each image is compiled.
+ */
+async function streamingBuild(
+  projectName: string,
+  composeDir: string,
+  composeFile: string,
+): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const proc = spawn(
+      'docker',
+      ['compose', '-f', composeFile, '-p', projectName, 'build', '--progress=plain'],
+      { cwd: composeDir },
+    );
+
+    const seenServices = new Set<string>();
+    let stderr = '';
+
+    function handleLine(line: string) {
+      // --progress=plain lines look like: #5 [management 2/4] RUN apt-get ...
+      const svcMatch = line.match(/\[([a-z][a-z0-9-]*)\s+\d+\/\d+\]/);
+      if (svcMatch) {
+        const svc = svcMatch[1];
+        if (!seenServices.has(svc)) {
+          seenServices.add(svc);
+          // Each new service auto-finalizes the previous [progress:start] in the runner.
+          console.log(`[progress:start] building image: ${svc} | ${svc} image ready`);
+        }
+      }
+    }
+
+    let buf = '';
+    function onData(chunk: Buffer) {
+      buf += chunk.toString();
+      const lines = buf.split('\n');
+      buf = lines.pop() ?? '';
+      for (const line of lines) handleLine(line.trim());
+    }
+
+    proc.stdout.on('data', onData);
+    proc.stderr.on('data', (chunk: Buffer) => {
+      stderr += chunk.toString();
+      onData(chunk);
+    });
+
+    proc.on('close', (code) => {
+      if (buf.trim()) handleLine(buf.trim());
+      if (code === 0) {
+        console.log('[progress:done] images built');
+        resolve();
+      } else {
+        reject(new Error(`docker compose build failed (exit ${code}):\n${stderr.slice(-500)}`));
+      }
+    });
+  });
+}
+
 export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
   // Pre-flight: fail if live containers are running that will compete for resources.
   try {
@@ -218,9 +278,9 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   const yaml = generateTestComposeYaml(config, celiloRoot);
   writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
-  // Build and start per-test containers only
-  console.log('[progress:start] building Docker images | images built');
-  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} build`, { cwd: composeDir, timeout: 600_000 });
+  // Build and start per-test containers only — stream output so the runner
+  // can show which service is currently being built (can take minutes on cold cache).
+  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
 
   console.log('[progress:start] starting containers | containers running');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, { cwd: composeDir, timeout: 120_000 });
@@ -310,6 +370,8 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   }
 
   console.log('[progress:done] network ready');
+  // Emit project name so the runner can persist it if --keep is set
+  console.log(`[e2e:project] ${projectName}`);
 
   const handle: NetworkHandle = {
     projectName,
@@ -362,6 +424,12 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     waitFor,
 
     async stop(): Promise<void> {
+      // Respect --keep / --reuse: don't tear down the network
+      if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') {
+        console.log(`[progress:done] network kept alive (project: ${projectName})`);
+        activeProjects.delete(projectName);  // don't re-teardown on exit
+        return;
+      }
       console.log('[progress:start] stopping network | network stopped');
       try {
         run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`,
@@ -375,3 +443,90 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
 
   return handle;
 }
+
+/**
+ * Reconnect to an existing test network (previously kept alive with --keep).
+ * Returns a NetworkHandle pointing to the existing Docker project. Skips
+ * all the setup (image build, container start, celilo init) — assumes the
+ * network is already running and healthy.
+ *
+ * If the project doesn't exist, throws with an actionable error.
+ */
+export function reconnectNetwork(projectName: string): NetworkHandle {
+  const composeDir = PACKAGE_ROOT;
+
+  // Verify the project actually exists
+  try {
+    const result = run(
+      `docker compose -f ${COMPOSE_FILE} -p ${projectName} ps -q`,
+      { cwd: composeDir, timeout: 10_000 },
+    );
+    if (!result.trim()) {
+      throw new Error(`No containers found for project ${projectName}`);
+    }
+  } catch (err) {
+    throw new Error(
+      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
+      `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
+    );
+  }
+
+  activeProjects.add(projectName);
+  console.log(`[progress:done] reconnected to existing network (${projectName})`);
+  console.log(`[e2e:project] ${projectName}`);
+
+  return {
+    projectName,
+
+    async celilo(cmd: string, optsOrTimeout?: number | { check?: boolean; timeoutMs?: number }): Promise<ExecResult> {
+      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
+      const timeoutMs = typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);
+
+      const result = await dockerExecAsync(projectName, composeDir, 'management',
+        `celilo ${cmd}`, timeoutMs);
+
+      if (check && result.exitCode !== 0) {
+        throw new CeliloCommandError(cmd, result);
+      }
+      return result;
+    },
+
+    exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
+      return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
+    },
+
+    async dig(name: string): Promise<string> {
+      const result = dockerExec(projectName, composeDir, 'management',
+        `dig +short ${name}`);
+      return result.stdout.trim();
+    },
+
+    async debug(container = 'management'): Promise<void> {
+      console.log(`[debug:pause] ${projectName} ${container}`);
+      const signalFile = join(composeDir, `.debug-resume-${process.pid}`);
+      const start = Date.now();
+      const timeout = 86_400_000;
+      while (!existsSync(signalFile) && Date.now() - start < timeout) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+      try { require('node:fs').unlinkSync(signalFile); } catch {}
+      console.log('[debug:resumed]');
+    },
+
+    waitFor,
+
+    async stop(): Promise<void> {
+      // Respect --keep / --reuse: don't tear down
+      if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') {
+        console.log(`[progress:done] network kept alive (project: ${projectName})`);
+        activeProjects.delete(projectName);
+        return;
+      }
+      try {
+        run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`,
+          { cwd: composeDir, timeout: 60_000 });
+      } catch {}
+      activeProjects.delete(projectName);
+    },
+  };
+}

package/src/index.ts

@@ -24,6 +24,9 @@
 // Builder API
 export { network, NetworkBuilder } from './network-builder';
 
+// Reconnect to an existing network (for --reuse mode)
+export { reconnectNetwork } from './container-manager';
+
 // Progress reporting (emit signals the test runner displays)
 export { progress } from './progress';
 

package/src/runner.ts

@@ -27,7 +27,13 @@ const PKG_DIR = resolve(import.meta.dir, '..');
 const testDir = process.env.E2E_TEST_DIR || process.cwd();
 const testsPath = join(testDir, 'tests');
 const timingFile = join(testDir, '.e2e-timing.json');
-const patterns = process.argv.slice(2);
+const persistentFile = join(testDir, '.e2e-persistent.json');
+
+// Parse flags out of argv
+const rawArgs = process.argv.slice(2);
+const flagKeep = rawArgs.includes('--keep');
+const flagReuse = rawArgs.includes('--reuse');
+const patterns = rawArgs.filter((a) => !a.startsWith('--'));
 
 // ─── Colors ──────────────────────────────────────────────────────────
 
@@ -309,6 +315,7 @@ interface TestResult {
   duration: number;
   error?: string;
   hadDebugPause?: boolean;
+  projectName?: string;
 }
 
 async function runTest(
@@ -323,13 +330,29 @@ async function runTest(
 
   display.reset(start, expectedDuration);
   let hadDebugPause = false;
+  let capturedProjectName: string | undefined;
 
   return new Promise<TestResult>((resolveTest) => {
+    // Env vars let the test detect --keep / --reuse modes
+    const testEnv: NodeJS.ProcessEnv = { ...process.env };
+    if (flagKeep) testEnv.CELILO_E2E_KEEP = '1';
+    if (flagReuse) {
+      // Read the persistent project and pass it to the test
+      try {
+        const data = JSON.parse(readFileSync(persistentFile, 'utf-8'));
+        testEnv.CELILO_E2E_REUSE = '1';
+        testEnv.CELILO_E2E_PROJECT = data.projectName;
+      } catch {
+        console.error(`${red}✗ --reuse: no persistent network found. Run with --keep first.${reset}`);
+        process.exit(1);
+      }
+    }
+
     // 1 hour timeout per test (debug sessions can be long)
     const proc = spawn('bun', ['test', '--timeout', '3600000', file], {
       cwd: PKG_DIR,
       stdio: ['pipe', 'pipe', 'pipe'],
-      env: { ...process.env },
+      env: testEnv,
     });
 
     let stdout = '';
@@ -393,6 +416,13 @@ async function runTest(
           continue;
         }
 
+        // Capture project name for persistent network tracking
+        const projectMatch = trimmed.match(/\[e2e:project\] (\S+)/);
+        if (projectMatch) {
+          capturedProjectName = projectMatch[1];
+          continue;
+        }
+
         parseLine(trimmed, display);
       }
     }
@@ -423,10 +453,11 @@ async function runTest(
             status: 'suspicious',
             duration,
             hadDebugPause,
+            projectName: capturedProjectName,
             error: '0 assertions — test likely exited before reaching assertions',
           });
         } else {
-          resolveTest({ name, status: 'pass', duration, hadDebugPause });
+          resolveTest({ name, status: 'pass', duration, hadDebugPause, projectName: capturedProjectName });
         }
       } else {
         resolveTest({
@@ -434,6 +465,7 @@ async function runTest(
           status: 'fail',
           duration,
           hadDebugPause,
+          projectName: capturedProjectName,
           error: extractFailureMessage(lines, code ?? 1),
         });
       }
@@ -556,6 +588,21 @@ async function main() {
       history[name] = result.duration;
     }
 
+    // Write persistent network info if --keep was set and we captured a project
+    if (flagKeep && result.projectName) {
+      writeFileSync(
+        persistentFile,
+        JSON.stringify({
+          projectName: result.projectName,
+          testFile: file,
+          createdAt: new Date().toISOString(),
+        }, null, 2) + '\n',
+      );
+      console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
+      console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
+      console.log(`${dim}  Tear down: e2e-down${reset}`);
+    }
+
     if (result.status === 'pass') {
       console.log(`  ${green}✔ passed (${result.duration}s)${reset}`);
       passed++;