@celilo/e2e 0.4.0 → 0.5.1

package/bin/e2e-build

@@ -1,12 +1,12 @@
 #!/bin/bash
 #
-# Pre-build all E2E Docker images.
+# Pre-build all E2E Docker images and package standard modules as .netapps.
 #
 # Run once before the test suite. Subsequent test runs use the cached
 # images and skip the build phase entirely (~5s vs ~3min per test).
 #
 # Usage:
-#   ./e2e/bin/e2e-build          # build all images
+#   ./e2e/bin/e2e-build          # build netapps + all images
 #   ./e2e/bin/e2e-build --save   # build + save to tarball for colima restarts
 #
 
@@ -14,6 +14,11 @@ set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 E2E_DIR="$(dirname "$SCRIPT_DIR")"
+PKG_DIR="$E2E_DIR"
+REPO_ROOT="$(cd "$E2E_DIR/../.." && pwd)"
+MODULES_DIR="$REPO_ROOT/modules"
+NETAPPS_DIR="$E2E_DIR/netapps"
+
 cd "$E2E_DIR"
 
 GREEN='\033[0;32m'
@@ -21,6 +26,43 @@ DIM='\033[2m'
 BOLD='\033[1m'
 NC='\033[0m'
 
+# ── Step 1: Package standard modules as .netapp files ─────────────────────────
+
+STANDARD_MODULES=(namecheap greenwave iptables caddy authentik)
+
+echo -e "${BOLD}Packaging standard modules...${NC}"
+echo ""
+
+mkdir -p "$NETAPPS_DIR"
+
+for MODULE in "${STANDARD_MODULES[@]}"; do
+  SRC="$MODULES_DIR/$MODULE"
+  OUT="$NETAPPS_DIR/$MODULE.netapp"
+
+  if [ ! -d "$SRC" ]; then
+    echo -e "  ${DIM}skip${NC}  $MODULE  ${DIM}(not found at $SRC)${NC}"
+    continue
+  fi
+
+  printf "  %-20s " "$MODULE"
+  BUILD_START=$(date +%s)
+
+  # Use the wrapper script if available, else bun directly
+  if [ -x "$REPO_ROOT/celilo" ]; then
+    "$REPO_ROOT/celilo" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  else
+    bun run "$REPO_ROOT/apps/celilo/src/cli/index.ts" package "$SRC" --output "$OUT" > /dev/null 2>&1
+  fi
+
+  BUILD_END=$(date +%s)
+  SIZE=$(du -h "$OUT" 2>/dev/null | cut -f1)
+  echo -e "${GREEN}✔${NC} ${DIM}$((BUILD_END - BUILD_START))s  ${SIZE}${NC}"
+done
+
+echo ""
+
+# ── Step 2: Build Docker images ───────────────────────────────────────────────
+
 echo -e "${BOLD}Building E2E Docker images...${NC}"
 echo ""
 
@@ -48,7 +90,8 @@ TOTAL_TIME=$((END - START))
 echo ""
 echo -e "${GREEN}All $TOTAL images built in ${TOTAL_TIME}s${NC}"
 
-# Optionally save to tarball
+# ── Step 3: Optionally save images to tarball ─────────────────────────────────
+
 if [ "$1" = "--save" ]; then
   echo ""
   echo -e "${BOLD}Saving images to tarball...${NC}"

package/config/dns/iamtheinternet.org.zone

@@ -2,7 +2,7 @@ $ORIGIN iamtheinternet.org.
 $TTL 300
 
 @ IN SOA ns1.iamtheinternet.org. admin.iamtheinternet.org. (
-    2024010772 300 60 604800 300
+    2024010778 300 60 604800 300
 )
 
 @       IN NS   ns1.iamtheinternet.org.

package/config/routing/management-routes.sh

@@ -34,20 +34,34 @@ echo "nameserver 100.100.0.1" > /etc/resolv.conf
 # Create celilo data directory (Linux default: /var/lib/celilo/)
 mkdir -p /var/lib/celilo/modules
 
-# Dependencies come from the host bind mount (node_modules/)
-# If missing, run 'cd apps/celilo && bun install' on the host first.
-cd /celilo/apps/celilo
-if [ ! -d node_modules/@clack ]; then
-  echo "WARNING: node_modules not found. Run 'cd apps/celilo && bun install' on the host."
+# If the monorepo is mounted at /celilo, use it for shell completions (dev mode).
+# Otherwise use the globally installed CLI from npm.
+if [ -d /celilo/apps/celilo ]; then
+  CELILO_SRC="/celilo/apps/celilo/src/cli/index.ts"
+  CELILO_CMD="bun run $CELILO_SRC"
+
+  # Warn if node_modules are missing (developer forgot to bun install)
+  if [ ! -d /celilo/apps/celilo/node_modules/@clack ]; then
+    echo "WARNING: /celilo/apps/celilo/node_modules not found. Run 'cd apps/celilo && bun install' on the host."
+  fi
+
+  # Generate zsh completions from source
+  mkdir -p /root/.zsh/completions
+  bun run "$CELILO_SRC" completion zsh > /root/.zsh/completions/_celilo 2>/dev/null || true
+
+  START_DIR="/celilo/apps/celilo"
+else
+  CELILO_CMD="celilo"
+  START_DIR="/root"
+
+  # Generate zsh completions from npm-installed CLI
+  mkdir -p /root/.zsh/completions
+  celilo completion zsh > /root/.zsh/completions/_celilo 2>/dev/null || true
 fi
 
-# Generate zsh completion script from celilo
-mkdir -p /root/.zsh/completions
-bun run /celilo/apps/celilo/src/cli/index.ts completion zsh > /root/.zsh/completions/_celilo 2>/dev/null || true
-
-# Shell setup: aliases and completion for zsh
-cat > /root/.zshrc << 'ZSHRC'
-export PATH="/root/.bun/bin:$PATH"
+# Shell setup
+cat > /root/.zshrc << ZSHRC
+export PATH="/root/.bun/bin:/usr/local/bin:\$PATH"
 
 # Simulated network overrides
 export DDNS_ENDPOINT="http://100.64.0.55:8080/update"
@@ -69,27 +83,22 @@ bindkey '^R' history-incremental-search-backward
 # Prompt
 PROMPT='%F{cyan}celilo%f:%F{yellow}%~%f# '
 
-# Commands
-celilo() { bun run /celilo/apps/celilo/src/cli/index.ts "$@"; }
-c() { celilo "$@"; }
+# Commands — celilo shim in /usr/local/bin prefers source when /celilo is mounted
+alias c='celilo'
 alias ll='ls -la'
 
-# Tab completion — use celilo's built-in zsh completion generator
-fpath=(~/.zsh/completions $fpath)
+# Tab completion
+fpath=(~/.zsh/completions \$fpath)
 autoload -Uz compinit && compinit -u
 
-# Also bind `c` to the same completion
-compdef _celilo c
-
-cd /celilo/apps/celilo
+cd $START_DIR
 ZSHRC
 
-# Also set up bash aliases (for non-zsh shells)
-cat > /root/.bashrc << 'BASHRC'
-export PATH="/root/.bun/bin:$PATH"
-alias celilo='bun run /celilo/apps/celilo/src/cli/index.ts'
+# Bash fallback (for non-zsh shells)
+cat > /root/.bashrc << BASHRC
+export PATH="/root/.bun/bin:/usr/local/bin:\$PATH"
 alias c='celilo'
-cd /celilo/apps/celilo
+cd $START_DIR
 BASHRC
 
 echo "Management machine ready"

package/docker/Dockerfile.management

@@ -22,8 +22,18 @@ RUN update-ca-certificates
 # Point apt at the simulated apt cache proxy (for runtime apt usage)
 RUN echo 'Acquire::http::Proxy "http://100.64.0.2:3142";' > /etc/apt/apt.conf.d/00proxy
 
-# Install celilo CLI from npm — provides the `celilo` command
-RUN bun add -g @celilo/cli@0.1.4
+# Install celilo CLI from npm. bun installs the bin to /usr/local/bin/celilo
+# on the oven/bun base image. We rename it so our shim can fall back to it.
+RUN bun add -g @celilo/cli@0.1.5 \
+    && mv /usr/local/bin/celilo /usr/local/bin/celilo-npm
+
+# Shim: prefer /celilo source when mounted (dev loop), fall back to npm install.
+RUN printf '#!/bin/bash\nif [ -d /celilo/apps/celilo ]; then\n  exec bun run /celilo/apps/celilo/src/cli/index.ts "$@"\nfi\nexec /usr/local/bin/celilo-npm "$@"\n' > /usr/local/bin/celilo \
+    && chmod +x /usr/local/bin/celilo
+
+# Standard modules pre-packaged as .netapp files — built by e2e-build.
+# Tests import from /netapps/<module>.netapp instead of the monorepo mount.
+COPY netapps/ /netapps/
 
 COPY config/ssh/generate-keys.sh /config/ssh/generate-keys.sh
 COPY config/routing/management-routes.sh /startup.sh

package/docker/Dockerfile.target-machine

@@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/docker/Dockerfile.target-machine-docker

@@ -16,6 +16,9 @@ RUN apt-get update && apt-get install -y \
     dnsutils \
     procps \
     python3 \
+    python3-apt \
+    sqlite3 \
+    jq \
     sudo \
     curl \
     ca-certificates \

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -21,6 +21,7 @@
     "config/",
     "simulators/",
     "bin/",
+    "netapps/",
     "README.md"
   ],
   "dependencies": {

package/src/container-manager.ts

@@ -108,9 +108,11 @@ function dockerExecAsync(
   timeoutMs = 60_000,
 ): Promise<ExecResult> {
   return new Promise((resolve) => {
+    // stdbuf -oL forces line-buffered stdout so output streams in real time
+    // rather than appearing in one burst when the process's buffer flushes.
     const args = [
       'compose', '-f', COMPOSE_FILE, '-p', projectName,
-      'exec', '-T', container, 'bash', '-c', cmd,
+      'exec', '-T', container, 'stdbuf', '-oL', 'bash', '-c', cmd,
     ];
     const child = spawn('docker', args, { cwd: composeDir });
 
@@ -158,6 +160,66 @@ async function waitFor(
   throw new Error(`Timeout waiting for ${label} after ${timeoutMs}ms`);
 }
 
+/**
+ * Run `docker compose build` with per-service progress signals.
+ * Uses --progress=plain so output is parseable on non-TTY.
+ * Emits one [progress:start] per service being built so the runner
+ * shows "building <service>" as each image is compiled.
+ */
+async function streamingBuild(
+  projectName: string,
+  composeDir: string,
+  composeFile: string,
+): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const proc = spawn(
+      'docker',
+      ['compose', '-f', composeFile, '-p', projectName, 'build', '--progress=plain'],
+      { cwd: composeDir },
+    );
+
+    const seenServices = new Set<string>();
+    let stderr = '';
+
+    function handleLine(line: string) {
+      // --progress=plain lines look like: #5 [management 2/4] RUN apt-get ...
+      const svcMatch = line.match(/\[([a-z][a-z0-9-]*)\s+\d+\/\d+\]/);
+      if (svcMatch) {
+        const svc = svcMatch[1];
+        if (!seenServices.has(svc)) {
+          seenServices.add(svc);
+          // Each new service auto-finalizes the previous [progress:start] in the runner.
+          console.log(`[progress:start] building image: ${svc} | ${svc} image ready`);
+        }
+      }
+    }
+
+    let buf = '';
+    function onData(chunk: Buffer) {
+      buf += chunk.toString();
+      const lines = buf.split('\n');
+      buf = lines.pop() ?? '';
+      for (const line of lines) handleLine(line.trim());
+    }
+
+    proc.stdout.on('data', onData);
+    proc.stderr.on('data', (chunk: Buffer) => {
+      stderr += chunk.toString();
+      onData(chunk);
+    });
+
+    proc.on('close', (code) => {
+      if (buf.trim()) handleLine(buf.trim());
+      if (code === 0) {
+        console.log('[progress:done] images built');
+        resolve();
+      } else {
+        reject(new Error(`docker compose build failed (exit ${code}):\n${stderr.slice(-500)}`));
+      }
+    });
+  });
+}
+
 export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle> {
   // Pre-flight: fail if live containers are running that will compete for resources.
   try {
@@ -218,9 +280,9 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
   const yaml = generateTestComposeYaml(config, celiloRoot);
   writeFileSync(join(composeDir, COMPOSE_FILE), yaml);
 
-  // Build and start per-test containers only
-  console.log('[progress:start] building Docker images | images built');
-  run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} build`, { cwd: composeDir, timeout: 600_000 });
+  // Build and start per-test containers only — stream output so the runner
+  // can show which service is currently being built (can take minutes on cold cache).
+  await streamingBuild(projectName, composeDir, COMPOSE_FILE);
 
   console.log('[progress:start] starting containers | containers running');
   run(`docker compose -f ${COMPOSE_FILE} -p ${projectName} up -d`, { cwd: composeDir, timeout: 120_000 });