@celilo/e2e 0.1.0 → 0.2.1

package/bin/e2e-infra

@@ -39,7 +39,7 @@ case "${1:-status}" in
 
     echo -e "${DIM}Waiting for DNS convergence...${NC}"
     for i in $(seq 1 30); do
-      DNS=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" exec -T comcast-resolver dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2 2>/dev/null || true)
+      DNS=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2 2>/dev/null || true)
       if [ -n "$DNS" ]; then
         echo -e "${GREEN}Shared infrastructure ready${NC}"
         echo ""
@@ -72,7 +72,7 @@ case "${1:-status}" in
       docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" ps --format "  {{.Name}}\t{{.Status}}"
 
       # Health check
-      DNS=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" exec -T comcast-resolver dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2 2>/dev/null || true)
+      DNS=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" exec -T namecheap-dns dig @127.0.0.1 iamtheinternet.org SOA +short +timeout=2 2>/dev/null || true)
       echo ""
       if [ -n "$DNS" ]; then
         echo -e "  DNS: ${GREEN}healthy${NC}"

package/bin/e2e-run

@@ -1,354 +1,5 @@
 #!/bin/bash
-#
-# E2E Test Runner with Live Progress Reporting
-#
-# Runs each test file sequentially with real-time progress display.
-# Shows sub-steps as they happen (building, deploying, configuring).
-# Captures full output per test in e2e/results/<timestamp>/.
-#
-# Shared infrastructure (DNS, Pebble, apt-cache) is started once
-# before the first test and torn down after the last test. Per-test
-# containers connect to the shared networks.
-#
-# Usage:
-#   ./e2e/bin/e2e-run              # run all tests
-#   ./e2e/bin/e2e-run smoke caddy  # run tests matching these patterns
-#
-
-set -e
-
+# E2E test runner — delegates to the TypeScript runner
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# Package root (where docker/, config/ live)
 PKG_DIR="$(dirname "$SCRIPT_DIR")"
-# Test directory — can be overridden via E2E_TEST_DIR env var.
-# Default: look for tests/ relative to CWD, or fall back to package dir.
-E2E_TEST_DIR="${E2E_TEST_DIR:-$(pwd)}"
-cd "$PKG_DIR"
-
-SHARED_PROJECT="celilo-e2e-shared"
-SHARED_COMPOSE="docker-compose.shared.yml"
-
-# Handle Ctrl+C: kill running test, clean up, and exit immediately.
-INTERRUPTED=false
-cleanup_and_exit() {
-  INTERRUPTED=true
-  echo ""
-  echo -e "${RED}Interrupted. Cleaning up...${NC}"
-  # Kill any running test process
-  [ -n "$TEST_PID" ] && kill $TEST_PID 2>/dev/null
-  # Remove named pipe
-  [ -n "$PIPE" ] && rm -f "$PIPE"
-  # Tear down shared infra
-  echo -e "${DIM}  Stopping shared infrastructure...${NC}"
-  docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" down --volumes --remove-orphans > /dev/null 2>&1 || true
-  # Clean up Docker
-  docker network prune -f > /dev/null 2>&1 || true
-  echo -e "${RED}Aborted.${NC}"
-  exit 130
-}
-trap cleanup_and_exit INT TERM
-
-# Colors
-GREEN='\033[0;32m'
-RED='\033[0;31m'
-DIM='\033[2m'
-BOLD='\033[1m'
-CYAN='\033[0;36m'
-NC='\033[0m'
-
-# Create results directory (in the test project, not the package)
-TIMESTAMP=$(date +%Y%m%d-%H%M%S)
-RESULTS_DIR="$E2E_TEST_DIR/results/$TIMESTAMP"
-mkdir -p "$RESULTS_DIR"
-
-# Discover test files from E2E_TEST_DIR
-TESTS_PATH="$E2E_TEST_DIR/tests"
-if [ ! -d "$TESTS_PATH" ]; then
-  echo -e "${RED}No tests/ directory found at $TESTS_PATH${NC}"
-  echo "Set E2E_TEST_DIR to the directory containing your tests/ folder."
-  exit 1
-fi
-
-if [ $# -gt 0 ]; then
-  TEST_FILES=()
-  for pattern in "$@"; do
-    for f in "$TESTS_PATH"/*${pattern}*.test.ts; do
-      [ -f "$f" ] && TEST_FILES+=("$f")
-    done
-  done
-else
-  TEST_FILES=("$TESTS_PATH"/*.test.ts)
-fi
-
-TOTAL=${#TEST_FILES[@]}
-PASSED=0
-FAILED=0
-RESULTS=()
-
-echo ""
-echo -e "${BOLD}╔══════════════════════════════════════════════════════════════╗${NC}"
-echo -e "${BOLD}║  Celilo E2E Tests                                          ║${NC}"
-printf  "${BOLD}║  %d test(s) queued — %s                              ║${NC}\n" "$TOTAL" "$(date +%H:%M:%S)"
-echo -e "${BOLD}╚══════════════════════════════════════════════════════════════╝${NC}"
-echo ""
-
-SUITE_START=$(date +%s)
-
-# ─── Shared Infrastructure Startup ────────────────────────────────────
-# The first test will call ensureSharedInfra() which starts the shared
-# project. But we also pre-check here so the runner can show progress.
-echo -e "${BOLD}Shared Infrastructure${NC}"
-
-# Check if shared infra is already running
-SHARED_RUNNING=false
-SHARED_CONTAINERS=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" ps -q 2>/dev/null || true)
-if [ -n "$SHARED_CONTAINERS" ]; then
-  # Verify DNS is healthy
-  DNS_CHECK=$(docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" exec -T comcast-resolver dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2 2>/dev/null || true)
-  if [ -n "$DNS_CHECK" ]; then
-    SHARED_RUNNING=true
-    echo -e "  ${GREEN}✔ already running (DNS healthy)${NC}"
-  else
-    echo -e "  ${DIM}DNS unhealthy, restarting...${NC}"
-    docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" down --volumes --remove-orphans > /dev/null 2>&1 || true
-  fi
-fi
-
-if [ "$SHARED_RUNNING" = false ]; then
-  echo -e "  ${DIM}Starting DNS hierarchy, Pebble ACME, apt-cache...${NC}"
-  # The actual startup happens inside the first test's ensureSharedInfra()
-  # call. We just report it here for visibility.
-fi
-
-SHARED_END=$(date +%s)
-echo ""
-
-# Track step count per test for percentage
-STEP_COUNT=0
-ESTIMATED_STEPS=15
-
-# Parse a line of test output and display as a progress step.
-show_progress() {
-  local line="$1"
-  local test_start="$2"
-  local step=""
-
-  # Shared infra progress (from ensureSharedInfra)
-  if [[ "$line" == *"[e2e] Shared infrastructure ready"* ]]; then
-    step="✔ shared infrastructure ready"
-  elif [[ "$line" == *"[e2e] Building shared"* ]]; then
-    step="building shared infrastructure images"
-  elif [[ "$line" == *"[e2e] Starting shared"* ]]; then
-    step="starting shared infrastructure"
-  elif [[ "$line" == *"[e2e] Cleaning up stale shared"* ]]; then
-    step="cleaning up stale shared infra"
-
-  # Per-test container manager progress
-  elif [[ "$line" == *"[e2e] Building images"* ]]; then
-    step="building test images"
-  elif [[ "$line" == *"[e2e] Starting containers"* ]]; then
-    step="starting test containers"
-  elif [[ "$line" == *"[e2e] Cleaning up"* ]]; then
-    step="cleaning up stale test resources"
-  elif [[ "$line" == *"[e2e] Waiting for DNS"* ]]; then
-    step="waiting for DNS convergence"
-  elif [[ "$line" == *"[e2e] Verifying routing"* ]]; then
-    step="verifying routing"
-  elif [[ "$line" == *"[e2e] Waiting for target"* ]]; then
-    step="waiting for target machines"
-  elif [[ "$line" == *"[e2e] Waiting for DHCP"* ]]; then
-    step="waiting for DHCP"
-  elif [[ "$line" == *"[e2e] Initializing celilo"* ]]; then
-    step="initializing celilo"
-  elif [[ "$line" == *"[e2e] Network ready"* ]]; then
-    step="network ready"
-  elif [[ "$line" == *"[e2e] Stopping network"* ]]; then
-    step="stopping network"
-
-  # Test-level progress (console.log from tests)
-  elif [[ "$line" == *"Adding machines"* ]]; then
-    step="adding machines"
-  elif [[ "$line" == *"Importing modules"* ]]; then
-    step="importing modules"
-  elif [[ "$line" == *"Configuring modules"* ]]; then
-    step="configuring modules"
-  elif [[ "$line" == *"Prerequisites ready"* ]]; then
-    step="prerequisites ready"
-
-  # Celilo CLI: "Deploying module: namecheap"
-  elif [[ "$line" == *"Deploying module:"* ]]; then
-    local module=$(echo "$line" | sed -n 's/.*Deploying module: \([^ ]*\).*/\1/p')
-    if [ -n "$module" ]; then
-      step="deploying $module"
-    else
-      step="deploying module"
-    fi
-  # Celilo CLI: "Deploying software..."
-  elif [[ "$line" == *"Deploying software"* ]]; then
-    step="running Ansible"
-
-  # Test console.log: "[45s] Deploying namecheap..."
-  elif [[ "$line" =~ Deploying\ ([a-z_-]+) ]]; then
-    step="deploying ${BASH_REMATCH[1]}"
-
-  # Celilo CLI: "✓ Module 'namecheap' deployed successfully"
-  elif [[ "$line" == *"deployed successfully"* ]]; then
-    local module=$(echo "$line" | sed -n "s/.*Module '\([^']*\)' deployed.*/\1/p")
-    step="✔ ${module:-module} deployed"
-
-  # Celilo CLI: "Successfully imported module: namecheap"
-  elif [[ "$line" == *"Successfully imported module:"* ]]; then
-    local module=$(echo "$line" | sed -n 's/.*imported module: \([^ ]*\).*/\1/p')
-    step="✔ ${module:-module} imported"
-
-  # Celilo CLI: "Infrastructure selected: machine"
-  elif [[ "$line" == *"Infrastructure selected"* ]]; then
-    step="infrastructure selected"
-  # Celilo CLI: "Terraform"
-  elif [[ "$line" == *"Terraform skipped"* ]]; then
-    step="terraform skipped (existing machine)"
-  elif [[ "$line" == *"terraform init"* ]] || [[ "$line" == *"Terraform init"* ]]; then
-    step="terraform init"
-  elif [[ "$line" == *"terraform apply"* ]] || [[ "$line" == *"Terraform apply"* ]]; then
-    step="terraform apply"
-
-  # Ansible health retries
-  elif [[ "$line" == *"RETRYING"* ]] && [[ "$line" == *"healthy"* ]]; then
-    local retries=$(echo "$line" | sed -n 's/.*(\([0-9]*\) retries left).*/\1/p')
-    step="waiting for health check ($retries retries left)"
-
-  # Skip noise
-  elif [[ "$line" == *"Set config for"* ]]; then return
-  elif [[ "$line" == *"Set secret for"* ]]; then return
-  elif [[ "$line" == *"Auto-generated"* ]]; then return
-  elif [[ "$line" == *"Warning: Permanently added"* ]]; then return
-
-  # Error detection
-  elif [[ "$line" == *"Error:"* ]] && [[ "$line" != *"vitest"* ]] && [[ "$line" != *"bun test"* ]] && [[ "$line" != *"ipv4: Address already"* ]]; then
-    local err=$(echo "$line" | sed 's/.*Error: //' | cut -c1-70)
-    step="✗ $err"
-  fi
-
-  if [ -n "$step" ]; then
-    STEP_COUNT=$((STEP_COUNT + 1))
-    # Update estimate upward if we're exceeding it
-    if [ $STEP_COUNT -gt $ESTIMATED_STEPS ]; then
-      ESTIMATED_STEPS=$((STEP_COUNT + 5))
-    fi
-    local pct=$(( (STEP_COUNT * 100) / ESTIMATED_STEPS ))
-    [ $pct -gt 99 ] && pct=99  # Don't show 100% until actually done
-    local now=$(date +%s)
-    local elapsed=$((now - test_start))
-    printf "  ${DIM}%3ds${NC} ${CYAN}%2d%%${NC} %s\n" "$elapsed" "$pct" "$step"
-  fi
-}
-
-for ((i=0; i<TOTAL; i++)); do
-  FILE="${TEST_FILES[$i]}"
-  NAME=$(basename "$FILE" .test.ts)
-  TEST_DIR="$RESULTS_DIR/$NAME"
-  mkdir -p "$TEST_DIR"
-
-  # Percentage complete (based on test count, not time)
-  PCT=$(( (i * 100) / TOTAL ))
-  ELAPSED_TOTAL=$(($(date +%s) - SUITE_START))
-  if [ $i -gt 0 ]; then
-    AVG_PER_TEST=$((ELAPSED_TOTAL / i))
-    REMAINING=$(( (TOTAL - i) * AVG_PER_TEST ))
-    ETA_MIN=$((REMAINING / 60))
-    ETA_SEC=$((REMAINING % 60))
-    ETA_STR=" — ~${ETA_MIN}m ${ETA_SEC}s remaining"
-  else
-    ETA_STR=""
-  fi
-
-  echo -e "${BOLD}[${i}/${TOTAL}] ${CYAN}${PCT}%${NC}${BOLD} ${NAME}${NC}${DIM}${ETA_STR}${NC}"
-
-  STEP_COUNT=0
-  TEST_START=$(date +%s)
-  PIPE=$(mktemp -u)
-  mkfifo "$PIPE"
-
-  # Background: run bun test, merge stdout+stderr, tee to log + pipe
-  (bun test --timeout 300000 "$FILE" 2>&1 | tee "$TEST_DIR/output.log" > "$PIPE") &
-  TEST_PID=$!
-
-  # Foreground: read pipe line-by-line and show progress
-  while IFS= read -r line; do
-    show_progress "$line" "$TEST_START"
-  done < "$PIPE"
-
-  # Wait for test to finish and get exit code
-  wait $TEST_PID 2>/dev/null
-  EXIT_CODE=$?
-  rm -f "$PIPE"
-
-  TEST_END=$(date +%s)
-  DURATION=$((TEST_END - TEST_START))
-
-  if [ $EXIT_CODE -eq 0 ]; then
-    echo -e "  ${GREEN}✔ passed (${DURATION}s)${NC}"
-    PASSED=$((PASSED + 1))
-    RESULTS+=("✓|$NAME|${DURATION}s")
-    echo '{"status":"pass","duration":'$DURATION'}' > "$TEST_DIR/result.json"
-  else
-    echo -e "  ${RED}✗ failed (${DURATION}s)${NC}"
-    FAILED=$((FAILED + 1))
-    ERROR=$(grep -E "Error:|expected.*to be|Timeout" "$TEST_DIR/output.log" 2>/dev/null | grep -v "vitest\|bun test" | head -1 | sed 's/.*Error: //' | cut -c1-80)
-    RESULTS+=("✗|$NAME|${DURATION}s|$ERROR")
-    echo '{"status":"fail","duration":'$DURATION',"error":"'"${ERROR//\"/\\\"}"'"}' > "$TEST_DIR/result.json"
-    echo -e "  ${RED}  └─ ${ERROR}${NC}"
-  fi
-  echo ""
-
-  # Clean up per-test Docker resources between tests (not shared infra)
-  docker network prune -f > /dev/null 2>&1 || true
-done
-
-# ─── Shared Infrastructure Teardown ───────────────────────────────────
-echo -e "${DIM}Stopping shared infrastructure...${NC}"
-docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" down --volumes --remove-orphans > /dev/null 2>&1 || true
-docker network prune -f > /dev/null 2>&1 || true
-
-SUITE_END=$(date +%s)
-SUITE_DURATION=$((SUITE_END - SUITE_START))
-SUITE_MINUTES=$((SUITE_DURATION / 60))
-SUITE_SECONDS=$((SUITE_DURATION % 60))
-
-echo ""
-echo -e "${BOLD}╔══════════════════════════════════════════════════════════════╗${NC}"
-printf  "${BOLD}║  Results: ${GREEN}%d passed${NC}${BOLD}, ${RED}%d failed${NC}${BOLD} — %dm %ds total             ║${NC}\n" "$PASSED" "$FAILED" "$SUITE_MINUTES" "$SUITE_SECONDS"
-echo -e "${BOLD}╠══════════════════════════════════════════════════════════════╣${NC}"
-
-for result in "${RESULTS[@]}"; do
-  IFS='|' read -r icon name duration error <<< "$result"
-  if [ "$icon" = "✓" ]; then
-    printf "${BOLD}║${NC}  ${GREEN}%s${NC} %-28s ${DIM}%6s${NC}\n" "$icon" "$name" "$duration"
-  else
-    printf "${BOLD}║${NC}  ${RED}%s${NC} %-28s ${DIM}%6s${NC}\n" "$icon" "$name" "$duration"
-    if [ -n "$error" ]; then
-      printf "${BOLD}║${NC}    ${RED}└─ %s${NC}\n" "$error"
-    fi
-  fi
-done
-
-echo -e "${BOLD}╚══════════════════════════════════════════════════════════════╝${NC}"
-echo ""
-echo -e "Full logs: ${DIM}$RESULTS_DIR${NC}"
-
-# Write summary
-cat > "$RESULTS_DIR/summary.json" <<EOF
-{
-  "timestamp": "$TIMESTAMP",
-  "total": $TOTAL,
-  "passed": $PASSED,
-  "failed": $FAILED,
-  "duration": $SUITE_DURATION
-}
-EOF
-
-# Add results dir to gitignore if not already there
-grep -q "^results/" "$E2E_TEST_DIR/.gitignore" 2>/dev/null || echo "results/" >> "$E2E_TEST_DIR/.gitignore"
-
-# Exit with failure if any tests failed
-[ $FAILED -eq 0 ]
+exec bun "$PKG_DIR/src/runner.ts" "$@"

package/config/dns/iamtheinternet.org.zone

@@ -2,14 +2,14 @@ $ORIGIN iamtheinternet.org.
 $TTL 300
 
 @ IN SOA ns1.iamtheinternet.org. admin.iamtheinternet.org. (
-    2024010686 300 60 604800 300
+    2024010764 300 60 604800 300
 )
 
 @       IN NS   ns1.iamtheinternet.org.
 ns1     IN A    100.64.0.55
 @    IN A    0.0.0.0
 caddy    IN A    100.100.0.100
-www    IN A    100.100.0.100
+www    IN A    10.0.10.10
 deathstar    IN A    100.100.0.100
 test    IN A    0.0.0.0
 auth    IN A    100.100.0.100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",

package/src/container-manager.ts

@@ -1,8 +1,9 @@
-import { execSync, spawn, type ExecSyncOptions } from 'node:child_process';
+import { execSync, spawn, spawnSync, type ExecSyncOptions } from 'node:child_process';
 import { existsSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import type { ExecResult, NetworkConfig, NetworkHandle } from './types';
-import { generateTestComposeYaml } from './docker-compose-generator';
+import { CeliloCommandError } from './types';
+import { generateTestComposeYaml, SHARED_PROJECT_NAME } from './docker-compose-generator';
 import { ensureSharedInfra } from './shared-infra';
 
 /** Package root — where docker/, config/, simulators/ live */
@@ -68,11 +69,22 @@ function run(cmd: string, opts?: ExecSyncOptions): string {
 }
 
 const COMPOSE_FILE = 'docker-compose.test.yml';
+const SHARED_COMPOSE_FILE = 'docker-compose.shared.yml';
+
+/** Containers that live in the shared infra project, not the per-test project */
+const SHARED_CONTAINERS = new Set([
+  'root-dns', 'tld-dns', 'namecheap-dns', 'letsencrypt', 'apt-cache',
+]);
 
 function dockerExec(projectName: string, composeDir: string, container: string, cmd: string, timeoutMs = 60_000): ExecResult {
+  // Use the shared infra project for shared containers
+  const isShared = SHARED_CONTAINERS.has(container);
+  const project = isShared ? SHARED_PROJECT_NAME : projectName;
+  const composeFile = isShared ? SHARED_COMPOSE_FILE : COMPOSE_FILE;
+
   try {
     const stdout = run(
-      `docker compose -f ${COMPOSE_FILE} -p ${projectName} exec -T ${container} bash -c ${JSON.stringify(cmd)}`,
+      `docker compose -f ${composeFile} -p ${project} exec -T ${container} bash -c ${JSON.stringify(cmd)}`,
       { cwd: composeDir, timeout: timeoutMs },
     );
     return { stdout, stderr: '', exitCode: 0 };
@@ -167,7 +179,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   await ensureSharedInfra();
 
   // Clean up stale per-test containers (self-healing from prior crashes)
-  console.log('[e2e] Cleaning up stale test resources...');
+  console.log('[progress:start] cleaning up stale test resources | cleanup complete');
   try {
     // Force-kill per-test containers (not shared infra)
     try {
@@ -205,18 +217,27 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
   // Build and start per-test containers only
-  console.log('[e2e] Building images...');
+  console.log('[progress:start] building Docker images | images built');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} build`, { cwd: composeDir, timeout: 300_000 });
 
-  console.log('[e2e] Starting containers...');
+  console.log('[progress:start] starting containers | containers running');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, { cwd: composeDir, timeout: 120_000 });
 
-  // DNS convergence is handled by shared infra — skip it here.
-  // Just verify routing from per-test containers to shared infra.
+  // Wait for DNS convergence (comcast-resolver is per-test, needs to reach shared DNS)
+  console.log('[progress:start] waiting for DNS convergence | DNS converged');
+  await waitFor(
+    async () => {
+      const result = dockerExec(projectName, composeDir, 'comcast-resolver',
+        'dig @127.0.0.1 iamtheinternet.org NS +short +timeout=2');
+      return result.exitCode === 0 && result.stdout.trim().length > 0;
+    },
+    60_000,
+    'DNS convergence',
+  );
 
   // Optional routing verification
   if (config.verifyRouting) {
-    console.log('[e2e] Verifying routing...');
+    console.log('[progress:start] verifying routing | routing verified');
     await waitFor(
       async () => {
         const result = dockerExec(projectName, composeDir, 'management',
@@ -231,7 +252,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   // Wait for all target machines to complete setup (systemd boot + routing)
   const allMachines = [...config.dmzMachines, ...config.appMachines, ...config.secureMachines, ...config.internalMachines];
   if (allMachines.length > 0) {
-    console.log('[e2e] Waiting for target machines...');
+    console.log('[progress:start] waiting for target machines | target machines ready');
   }
   for (const machine of allMachines) {
     await waitFor(
@@ -247,7 +268,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
 
   // Wait for DHCP client lease
   if (config.dhcpClient) {
-    console.log('[e2e] Waiting for DHCP client lease...');
+    console.log('[progress:start] waiting for DHCP client lease | DHCP lease acquired');
     await waitFor(
       async () => {
         const result = dockerExec(projectName, composeDir, 'dhcp-client',
@@ -273,7 +294,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   }
 
   // Initialize celilo
-  console.log('[e2e] Initializing celilo...');
+  console.log('[progress:start] initializing celilo | celilo initialized');
   const initResult = dockerExec(projectName, composeDir, 'management',
     `cd /celilo/apps/celilo && bun run src/cli/index.ts system init --accept-defaults \
       network.dmz.subnet=10.0.10.0/24 \
@@ -286,14 +307,22 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     throw new Error(`Celilo init failed: ${initResult.stderr}`);
   }
 
-  console.log(`[e2e] Network ready (project: ${projectName})`);
+  console.log('[progress:done] network ready');
 
   const handle: NetworkHandle = {
     projectName,
 
-    celilo(cmd: string, timeoutMs = 120_000): Promise<ExecResult> {
-      return dockerExecAsync(projectName, composeDir, 'management',
+    async celilo(cmd: string, optsOrTimeout?: number | { check?: boolean; timeoutMs?: number }): Promise<ExecResult> {
+      const check = typeof optsOrTimeout === 'object' ? (optsOrTimeout.check ?? true) : true;
+      const timeoutMs = typeof optsOrTimeout === 'number' ? optsOrTimeout : (optsOrTimeout?.timeoutMs ?? 120_000);
+
+      const result = await dockerExecAsync(projectName, composeDir, 'management',
         `cd /celilo/apps/celilo && bun run src/cli/index.ts ${cmd}`, timeoutMs);
+
+      if (check && result.exitCode !== 0) {
+        throw new CeliloCommandError(cmd, result);
+      }
+      return result;
     },
 
     exec(container: string, cmd: string, timeoutMs = 60_000): Promise<ExecResult> {
@@ -306,10 +335,32 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       return result.stdout.trim();
     },
 
+    async debug(container = 'management'): Promise<void> {
+      // Signal the runner to hand us the terminal
+      console.log(`[debug:pause] ${projectName} ${container}`);
+
+      // The runner intercepts [debug:pause] and spawns the interactive
+      // shell itself (since it owns the terminal). It writes a signal
+      // file when the user exits the shell.
+      const signalFile = join(composeDir, `.debug-resume-${process.pid}`);
+
+      // Wait for the runner to create the signal file
+      const start = Date.now();
+      const timeout = 86_400_000; // 24 hour max debug session
+      while (!existsSync(signalFile) && Date.now() - start < timeout) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+
+      // Clean up
+      try { require('node:fs').unlinkSync(signalFile); } catch {}
+
+      console.log('[debug:resumed]');
+    },
+
     waitFor,
 
     async stop(): Promise<void> {
-      console.log(`[e2e] Stopping network (project: ${projectName})...`);
+      console.log('[progress:start] stopping network | network stopped');
       try {
         run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} down --volumes --remove-orphans`,
           { cwd: composeDir, timeout: 60_000 });

package/src/docker-compose-generator.ts

@@ -65,7 +65,6 @@ export const SHARED_PROJECT_NAME = 'celilo-e2e-shared';
 
 /** Docker network names as created by the shared infrastructure project */
 export const SHARED_NETWORKS = {
-  'isp-external': `${SHARED_PROJECT_NAME}_isp-external`,
   'internet-external': `${SHARED_PROJECT_NAME}_internet-external`,
   'real-internet': `${SHARED_PROJECT_NAME}_real-internet`,
 } as const;
@@ -140,12 +139,14 @@ const TOPOLOGY_SERVICES: Record<TopologyPreset, () => Record<string, unknown>> =
 
 /**
  * Generate the compose YAML for shared infrastructure that runs once
- * per test suite. Contains: fw-ext, DNS hierarchy, Pebble ACME, apt-cache.
- * Owns the isp-external, internet-external, and real-internet networks.
+ * per test suite. Contains: DNS hierarchy, Pebble ACME, apt-cache.
+ * Owns the internet-external and real-internet networks.
+ *
+ * fw-ext and comcast-resolver are per-test because they route to
+ * per-test containers (fw-isp/fw-main) and need those to exist first.
  */
 export function generateSharedInfraYaml(): string {
   const networks: Record<string, unknown> = {
-    'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
     'internet-external': networkDef('100.64.0.0/24', '100.64.0.250'),
     'real-internet': {
       driver: 'bridge',
@@ -159,26 +160,6 @@ export function generateSharedInfraYaml(): string {
 
   const services: Record<string, unknown> = {};
 
-  services['fw-ext'] = baseService({
-    build: { context: '.', dockerfile: 'docker/Dockerfile.router' },
-    networks: {
-      'isp-external': { ipv4_address: '100.100.0.101' },
-      'internet-external': { ipv4_address: '100.64.0.1' },
-      'real-internet': { ipv4_address: '172.30.0.5' },
-    },
-    cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
-    sysctls: ROUTER_SYSCTLS,
-  });
-
-  services['comcast-resolver'] = baseService({
-    build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
-    networks: {
-      'isp-external': { ipv4_address: '100.100.0.1' },
-      'real-internet': { ipv4_address: '172.30.0.4' },
-    },
-    cap_add: ['NET_ADMIN'],
-  });
-
   services['root-dns'] = baseService({
     build: { context: '.', dockerfile: 'docker/Dockerfile.authoritative-dns' },
     networks: { 'internet-external': { ipv4_address: '100.64.0.53' } },
@@ -252,10 +233,15 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot = '..'
     dmz: networkDef('10.0.10.0/24', '10.0.10.250'),
     app: networkDef('10.0.20.0/24', '10.0.20.250'),
     secure: networkDef('10.0.30.0/24', '10.0.30.250'),
-    // isp-external is owned by the shared infra project — reference it
-    'isp-external': {
+    'isp-external': networkDef('100.100.0.0/24', '100.100.0.250'),
+    // internet-external and real-internet are owned by shared infra
+    'internet-external': {
       external: true,
-      name: SHARED_NETWORKS['isp-external'],
+      name: SHARED_NETWORKS['internet-external'],
+    },
+    'real-internet': {
+      external: true,
+      name: SHARED_NETWORKS['real-internet'],
     },
   };
 
@@ -287,6 +273,27 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot = '..'
   const topologyFn = TOPOLOGY_SERVICES[config.topology] || TOPOLOGY_SERVICES.default;
   Object.assign(services, topologyFn());
 
+  // --- Routing infrastructure (bridges per-test to shared infra) ---
+  services['fw-ext'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.router' },
+    networks: {
+      'isp-external': { ipv4_address: '100.100.0.101' },
+      'internet-external': { ipv4_address: '100.64.0.1' },
+      'real-internet': { ipv4_address: '172.30.0.5' },
+    },
+    cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
+    sysctls: ROUTER_SYSCTLS,
+  });
+
+  services['comcast-resolver'] = baseService({
+    build: { context: '.', dockerfile: 'docker/Dockerfile.resolver' },
+    networks: {
+      'isp-external': { ipv4_address: '100.100.0.1' },
+      'real-internet': { ipv4_address: '172.30.0.4' },
+    },
+    cap_add: ['NET_ADMIN'],
+  });
+
   // --- Dynamic test machines ---
   const allMachines = [
     ...config.dmzMachines,