@celilo/e2e 0.4.0 → 0.5.0

package/bin/e2e-build

@@ -1,12 +1,12 @@
 #!/bin/bash
 #
-# Pre-build all E2E Docker images.
+# Pre-build all E2E Docker images and package standard modules as .netapps.
 #
 # Run once before the test suite. Subsequent test runs use the cached
 # images and skip the build phase entirely (~5s vs ~3min per test).
 #
 # Usage:
-#   ./e2e/bin/e2e-build          # build all images
+#   ./e2e/bin/e2e-build          # build netapps + all images
 #   ./e2e/bin/e2e-build --save   # build + save to tarball for colima restarts
 #
 
@@ -14,6 +14,11 @@ set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 E2E_DIR="$(dirname "$SCRIPT_DIR")"
+PKG_DIR="$E2E_DIR"
+REPO_ROOT="$(cd "$E2E_DIR/../.." && pwd)"
+MODULES_DIR="$REPO_ROOT/modules"
+NETAPPS_DIR="$E2E_DIR/netapps"
+
 cd "$E2E_DIR"
 
 GREEN='\033[0;32m'
@@ -21,6 +26,43 @@ DIM='\033[2m'
 BOLD='\033[1m'
 NC='\033[0m'
 
+# ── Step 1: Package standard modules as .netapp files ─────────────────────────
+
+STANDARD_MODULES=(namecheap greenwave iptables caddy authentik)
+
+echo -e "${BOLD}Packaging standard modules...${NC}"
+echo ""
+
+mkdir -p "$NETAPPS_DIR"
+
+for MODULE in "${STANDARD_MODULES[@]}"; do
+  SRC="$MODULES_DIR/$MODULE"
+  OUT="$NETAPPS_DIR/$MODULE.netapp"
+
+  if [ ! -d "$SRC" ]; then
+    echo -e "  ${DIM}skip${NC}  $MODULE  ${DIM}(not found at $SRC)${NC}"
+    continue
+  fi
+
+  printf "  %-20s " "$MODULE"
+  BUILD_START=$(date +%s)
+
+  # Use the wrapper script if available, else bun directly
+  if [ -x "$REPO_ROOT/celilo" ]; then
+    "$REPO_ROOT/celilo" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  else
+    bun run "$REPO_ROOT/apps/celilo/src/cli/index.ts" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  fi
+
+  BUILD_END=$(date +%s)
+  SIZE=$(du -h "$OUT" 2>/dev/null | cut -f1)
+  echo -e "${GREEN}✔${NC} ${DIM}$((BUILD_END - BUILD_START))s  ${SIZE}${NC}"
+done
+
+echo ""
+
+# ── Step 2: Build Docker images ───────────────────────────────────────────────
+
 echo -e "${BOLD}Building E2E Docker images...${NC}"
 echo ""
 
@@ -48,7 +90,8 @@ TOTAL_TIME=$((END - START))
 echo ""
 echo -e "${GREEN}All $TOTAL images built in ${TOTAL_TIME}s${NC}"
 
-# Optionally save to tarball
+# ── Step 3: Optionally save images to tarball ─────────────────────────────────
+
 if [ "$1" = "--save" ]; then
   echo ""
   echo -e "${BOLD}Saving images to tarball...${NC}"

package/config/dns/iamtheinternet.org.zone

@@ -2,7 +2,7 @@ $ORIGIN iamtheinternet.org.
 $TTL 300
 
 @ IN SOA ns1.iamtheinternet.org. admin.iamtheinternet.org. (
-    2024010772 300 60 604800 300
+    2024010778 300 60 604800 300
 )
 
 @       IN NS   ns1.iamtheinternet.org.

package/docker/Dockerfile.management

@@ -22,9 +22,19 @@ RUN update-ca-certificates
 # Point apt at the simulated apt cache proxy (for runtime apt usage)
 RUN echo 'Acquire::http::Proxy "http://100.64.0.2:3142";' > /etc/apt/apt.conf.d/00proxy
 
-# Install celilo CLI from npm — provides the `celilo` command
+# Install celilo CLI from npm — provides the fallback `celilo` command
 RUN bun add -g @celilo/cli@0.1.4
 
+# Shim: prefer /celilo source when mounted (dev loop), fall back to npm install.
+# This means `celilo ...` runs the monorepo's source code during e2e tests
+# instead of the pinned npm version — critical for iterating on fixes.
+RUN printf '#!/bin/bash\nif [ -d /celilo/apps/celilo ]; then\n  exec bun run /celilo/apps/celilo/src/cli/index.ts "$@"\nfi\nexec /root/.bun/bin/celilo "$@"\n' > /usr/local/bin/celilo \
+    && chmod +x /usr/local/bin/celilo
+
+# Standard modules pre-packaged as .netapp files — built by e2e-build.
+# Tests import from /netapps/<module>.netapp instead of the monorepo mount.
+COPY netapps/ /netapps/
+
 COPY config/ssh/generate-keys.sh /config/ssh/generate-keys.sh
 COPY config/routing/management-routes.sh /startup.sh
 RUN chmod +x /startup.sh /config/ssh/generate-keys.sh

package/docker/Dockerfile.target-machine

@@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/docker/Dockerfile.target-machine-docker

@@ -16,6 +16,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -21,6 +21,7 @@
     "config/",
     "simulators/",
     "bin/",
+    "netapps/",
     "README.md"
   ],
   "dependencies": {

package/src/container-manager.ts

@@ -158,6 +158,66 @@ async function waitFor(
   throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms`);
 }
 
+/**
+ * Run `docker compose build` with per-service progress signals.
+ * Uses --progress=plain so output is parseable on non-TTY.
+ * Emits one [progress:start] per service being built so the runner
+ * shows "building <service>" as each image is compiled.
+ */
+async function streamingBuild(
+  projectName: string,
+  composeDir: string,
+  composeFile: string,
+): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const proc = spawn(
+      'docker',
+      ['compose', '-f', composeFile, '-p', projectName, 'build', '--progress=plain'],
+      { cwd: composeDir },
+    );
+
+    const seenServices = new Set<string>();
+    let stderr = '';
+
+    function handleLine(line: string) {
+      // --progress=plain lines look like: #5 [management 2/4] RUN apt-get ...
+      const svcMatch = line.match(/\[([a-z][a-z0-9-]*)\s+\d+\/\d+\]/);
+      if (svcMatch) {
+        const svc = svcMatch[1];
+        if (!seenServices.has(svc)) {
+          seenServices.add(svc);
+          // Each new service auto-finalizes the previous [progress:start] in the runner.
+          console.log(`[progress:start] building image: ${svc} | ${svc} image ready`);
+        }
+      }
+    }
+
+    let buf = '';
+    function onData(chunk: Buffer) {
+      buf += chunk.toString();
+      const lines = buf.split('\n');
+      buf = lines.pop() ?? '';
+      for (const line of lines) handleLine(line.trim());
+    }
+
+    proc.stdout.on('data', onData);
+    proc.stderr.on('data', (chunk: Buffer) => {
+      stderr += chunk.toString();
+      onData(chunk);
+    });
+
+    proc.on('close', (code) => {
+      if (buf.trim()) handleLine(buf.trim());
+      if (code === 0) {
+        console.log('[progress:done] images built');
+        resolve();
+      } else {
+        reject(new Error(`docker compose build failed (exit ${code}):\n${stderr.slice(-500)}`));
+      }
+    });
+  });
+}
+
 export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
   // Pre-flight: fail if live containers are running that will compete for resources.
   try {
@@ -218,9 +278,9 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   const yaml = generateTestComposeYaml(config, celiloRoot);
   writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
-  // Build and start per-test containers only
-  console.log('[progress:start] building Docker images | images built');
-  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} build`, { cwd: composeDir, timeout: 600_000 });
+  // Build and start per-test containers only — stream output so the runner
+  // can show which service is currently being built (can take minutes on cold cache).
+  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
 
   console.log('[progress:start] starting containers | containers running');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, { cwd: composeDir, timeout: 120_000 });