@celilo/e2e 0.19.2 → 0.20.0

package/README.md

@@ -281,10 +281,27 @@ describe('my deployment test', () => {
 | `exec(container, cmd, timeout?)` | Execute a command in any container |
 | `dig(name)` | Resolve a DNS name from management |
 | `waitFor(check, timeout, label)` | Poll until a condition is true |
+| `debug(container?)` | Pause the test and shell into a container (default `management`); exit resumes ([details](#mid-test-pause)) |
 | `socksProxy(opts?)` | Spawn a SOCKS5 proxy attached to the test network ([details](#browser-driven-tests)) |
 | `browser(opts)` | Launch a host-side browser routed into the network ([details](#browser-driven-tests)) |
 | `stop()` | Tear down the entire network |
 
+### Mid-test pause
+
+`net.debug()` freezes a running suite at the line that calls it and hands you a
+bash shell inside the named container (default `management`). Type `exit` and
+the test resumes. In a non-interactive run it instead prints
+`Resume: touch <signalFile>` and waits, so you can probe the stack from another
+terminal and resume by touching the file.
+
+This shows live state at the exact failing moment, which `--keep` cannot: a
+kept stack is only available after the suite finished, once later stages have
+already mutated it.
+
+The pause holds the machine-global run lock for up to 24 hours, so a forgotten
+debug session blocks every other session's run. Resume promptly. See
+`docs/RUNNING_CELE2E_TESTS.md` for the full write-up.
+
 ### Browser-driven tests
 
 `net.browser()` returns a Playwright `Browser` whose traffic enters the simulated network through a containerized SOCKS5 proxy, so a host-side browser sees the deployed app exactly as a real user would (DNS, TLS, ingress firewall, all in the loop).
@@ -347,20 +364,18 @@ const net = await network()
 
 ### Running Tests
 
-```bash
-cd e2e
-
-# Run all E2E tests
-bun run vitest run
-
-# Run a specific test
-bun run vitest run tests/caddy-deploy.test.ts
+The repo's own tests live in `e2e/tests/` at the repo root. They resolve
+`@celilo/e2e` through the root workspace install, so there is nothing to
+install in `e2e/` itself.
 
-# Run with verbose output
-bun run vitest run --reporter=verbose
+```bash
+# From the repo root, after `bun run setup`:
+cele2e run <test-name>        # Docker-backed suite, e.g. `cele2e run machine-add`
+cele2e run --all              # every module suite + the top-level e2e/tests/
 ```
 
-Tests run sequentially (fixed CIDRs prevent parallelism). The container manager automatically cleans up leftover networks from previous failed runs.
+See `docs/RUNNING_CELE2E_TESTS.md` for the full operator guide (run lock,
+build-infra triggers, `--keep` debugging).
 
 ## Key Design Decisions
 
@@ -368,9 +383,11 @@ Tests run sequentially (fixed CIDRs prevent parallelism). The container manager
 
 **No test parallelism:** Fixed network CIDRs mean only one test network can run at a time. Tests run sequentially via Vitest's `singleFork` config.
 
-**Bind-mounted source:** The celilo source tree is bind-mounted at `/celilo` on the management container. Code changes take effect immediately without rebuilding images.
+**Bind-mounted source:** The celilo source tree is bind-mounted at `/celilo` on the management container, which is where module sources are read from (`celilo module import /celilo/modules/<id>`).
+
+The CLI itself is NOT read from there. Tests run the `celilo` baked into the management image by `cele2e build-infra` — the artifact install.sh produces — so **a change under `apps/celilo/` or `packages/` needs a `build-infra` before it reaches a test.** `cele2e doctor` compares a fingerprint stamped on the image against the working tree and warns when they differ, and `cele2e run --source-cli` runs the mounted source instead for iteration. Running the source roughly doubles each celilo command's start-up (measured `celilo --version`: 0.14s installed against 0.31s from the mount on virtiofs, 0.16s against 0.56s on sshfs).
 
-**Docker image caching:** Images are built once and cached. Only config/simulator changes require rebuilds. The management container uses the bind mount, so celilo code changes are free.
+**Docker image caching:** Images are built once and cached. Only config/simulator changes require rebuilds. A run skips `docker compose build` entirely when every image it needs already exists — that check costs one `docker images` listing, against about 0.7s per service for a build that rebuilds nothing.
 
 **ACME via DNS interception:** Instead of overriding Caddy's ACME URL, `acme-v02.api.letsencrypt.org` resolves to Pebble (100.64.0.100) in the simulated DNS. The caddy module's `acme_ca` variable allows pointing to Pebble's `/dir` endpoint (vs Let's Encrypt's `/directory`).
 

package/bin/e2e-bake-management

@@ -26,6 +26,17 @@ import { execSync } from 'node:child_process';
 import { join } from 'node:path';
 import { ensureSharedInfra } from '../src/shared-infra';
 import { network } from '../src/network-builder';
+import { imageTag } from '../src/docker-compose-generator';
+import { findMonorepoRoot } from '../src/repo-root';
+import {
+  packNpmRegistryTarballs,
+  stageWebsiteDist,
+} from '../src/stage-simulator-inputs';
+import {
+  PUBLISHED_FINGERPRINT_PREFIX,
+  SOURCE_LABEL,
+} from '../src/source-fingerprint';
+import { readFileSync } from 'node:fs';
 
 const PACKAGE_ROOT = join(import.meta.dir, '..');
 const COMPOSE_FILE = join(PACKAGE_ROOT, 'docker-compose.test.yml');
@@ -42,13 +53,40 @@ interface ExecResult {
 type Exec = (cmd: string, timeoutMs?: number) => Promise<ExecResult>;
 
 /**
- * Write the source-mount dev-loop shim to /usr/local/bin/celilo and verify
- * the CLI works through it. Shared by both bake modes.
+ * Write the CLI shim to /usr/local/bin/celilo and verify the CLI works through
+ * it. Shared by both bake modes.
  *
  * The shim (at /usr/local/bin/celilo so plain `docker exec bash -c celilo`
- * finds it) prefers a mounted /celilo workspace, else falls through to the
- * real installed CLI at /root/.bun/bin/celilo. Two things must hold, both
- * bitten in ISS-0124:
+ * finds it) runs the INSTALLED CLI at /root/.bun/bin/celilo — the artifact
+ * install.sh just produced, which is the thing the rig exists to test. It runs
+ * the mounted /celilo workspace instead only when `CELILO_E2E_SOURCE_CLI` is
+ * set, which `cele2e run --source-cli` does.
+ *
+ * That default used to be the other way round, and both halves of the swap are
+ * worth stating.
+ *
+ * It cost real time per invocation. `bun run <source>` resolves 1522 modules
+ * across the bind mount rather than starting an installed binary. Measured
+ * inside this image on 2026-09-05, same command and same container, on a VM
+ * sized per the host policy: `celilo --version` took 0.14s installed and 0.31s
+ * from the mounted source over virtiofs, or 0.16s against 0.56s over sshfs. A
+ * test issues 25 to 40 celilo commands.
+ *
+ * (An earlier measurement put that gap at 2.3s. Most of that was a separate
+ * problem — the VM had been given 24 of the host's 32 GiB and the host was
+ * swapping 21.9 GiB. Fixing the memory removed about 1.9s of it. Worth stating,
+ * because a number measured with two variables moving blamed the wrong one.)
+ *
+ * And it made the verify below dishonest. This function exists because of
+ * ISS-0124 — a broken shim fallback shipping green — and its comment says the
+ * verify runs "with NO /celilo mount, i.e. exactly the shipped path". That was
+ * true of the published bake, which runs in a standalone container, and false
+ * of this one: the sim bake's container HAS the workspace mounted, so the old
+ * shim sent the verify down the source branch and the installed CLI was never
+ * executed at all. The check could not reach its subject. Now both modes verify
+ * the installed binary.
+ *
+ * Two things must hold, both bitten in ISS-0124:
  *
  *  1. `bun add -g` links the celilo binary at /usr/local/bin/celilo as a
  *     SYMLINK into the package (bun's global bin dir in this image is
@@ -65,12 +103,14 @@ type Exec = (cmd: string, timeoutMs?: number) => Promise<ExecResult>;
  * shipped path — so a dangling/looping fallback fails the bake instead of
  * shipping green.
  */
-async function installShimAndVerify(exec: Exec, label: string): Promise<void> {
+async function installShimAndVerify(exec: Exec, label: string): Promise<string> {
   process.stdout.write('  source-mount shim ...... ');
   const t0 = Date.now();
   const shim = [
     '#!/bin/bash',
-    'if [ -d /celilo/apps/celilo ]; then',
+    '# Opt-in source loop: `cele2e run --source-cli` sets this in the container',
+    '# environment. Unset (the default) runs the CLI install.sh actually installed.',
+    'if [ -n "$CELILO_E2E_SOURCE_CLI" ] && [ -d /celilo/apps/celilo ]; then',
     '  exec bun run /celilo/apps/celilo/src/cli/index.ts "$@"',
     'fi',
     'exec /root/.bun/bin/celilo "$@"',
@@ -101,6 +141,113 @@ async function installShimAndVerify(exec: Exec, label: string): Promise<void> {
     );
   }
   console.log(`✔ ${Math.round((Date.now() - t1) / 1000)}s  (${verify.stdout.trim()})`);
+  return verify.stdout.trim();
+}
+
+/**
+ * The image id currently on a tag, or '' when the tag is unused.
+ *
+ * Captured before a commit so the superseded image can be removed after it.
+ * Every bake used to orphan a 1.5 GB untagged image and nothing ever collected
+ * them: measured 2026-09-05, 3178 dangling images holding 25.8 GB, 1478 of them
+ * from a single 35-hour window. That pile is what drives an operator to reach
+ * for `docker image prune -a`, which deletes the TAGGED base images the next
+ * build needs and costs a full 27-image rebuild.
+ */
+function imageIdOnTag(tag: string): string {
+  try {
+    return run(`docker image inspect --format '{{.Id}}' ${tag}`).trim();
+  } catch {
+    return '';
+  }
+}
+
+/** Remove an image the bake just superseded. Best-effort: a tag or container still using it wins. */
+function removeSupersededImage(previousId: string, tag: string): void {
+  if (!previousId || previousId === imageIdOnTag(tag)) return;
+  try {
+    run(`docker image rm ${previousId}`);
+    console.log(`  superseded image ....... ✔ removed ${previousId.slice(7, 19)}`);
+  } catch {
+    // Still referenced (a running container, another tag). Leaving it is correct.
+  }
+}
+
+/**
+ * The sim tarballs the default bake installs, staged fresh by
+ * restageSimulatorInputs, carry a manifest recording the tree fingerprint they
+ * were packed from. The stamp IS that value: the label then describes the
+ * tarballs install.sh actually installs, not the tree the bake happened to run
+ * in. Stamping `computeSourceFingerprint(tree)` here is what produced a green
+ * doctor over a CLI without the tree's changes (celilo#1299): the tree moved
+ * between the last `cele2e build` and the bake, and the fingerprint followed
+ * the tree, not the stale cache the bake reinstalled.
+ */
+function simStampFromManifest(): string {
+  const manifestPath = join(PACKAGE_ROOT, '.npm-registry-cache', 'pack-manifest.json');
+  let manifest: { sourceFingerprint?: string | null };
+  try {
+    manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')) as {
+      sourceFingerprint?: string | null;
+    };
+  } catch {
+    throw new Error(
+      `No readable pack-manifest.json at ${manifestPath} — the sim tarballs were never staged. ` +
+        'Run the bake from a monorepo checkout; restageSimulatorInputs writes the manifest.',
+    );
+  }
+  if (!manifest.sourceFingerprint) {
+    throw new Error(
+      'pack-manifest.json carries no sourceFingerprint — the tarballs this bake installs cannot be attributed to a tree.',
+    );
+  }
+  return manifest.sourceFingerprint;
+}
+
+/**
+ * Stamp the source this image was baked from, so `cele2e doctor` can say
+ * whether the CLI under test is the code in the working tree. See
+ * `src/source-fingerprint.ts` for why that has to be measured, not remembered.
+ */
+function sourceStamp(published: boolean, version: string): string {
+  if (published) return `${PUBLISHED_FINGERPRINT_PREFIX}${version || 'unknown'}`;
+  return simStampFromManifest();
+}
+
+/**
+ * Re-stage the simulator inputs the default bake's install path depends on,
+ * and rebuild the two sim images that COPY them at docker-build time.
+ *
+ * Without this, the bake reinstalls whatever tarballs the last `cele2e build`
+ * left in `.npm-registry-cache/` — measured 2026-09-07: a bake at 00:17 UTC
+ * shipped @celilo/cli 2.1.0 from a cache packed at 15:18 the day before,
+ * silently dropping a CLI fix that had landed at 19:48 (celilo#1299). The
+ * rebuilds are not optional either: the per-test stack only builds MISSING
+ * images, so fresh tarballs on disk never reach the running sim otherwise.
+ * Both docker builds are cache-warm no-ops when nothing staged has changed.
+ */
+function restageSimulatorInputs(): void {
+  const repoRoot = findMonorepoRoot(PACKAGE_ROOT);
+  if (!repoRoot) {
+    throw new Error(
+      'The default bake installs the monorepo DEV cli via the sim registry — it needs a celilo checkout. For a published-CLI bake use --published.',
+    );
+  }
+  stageWebsiteDist(repoRoot, PACKAGE_ROOT);
+  packNpmRegistryTarballs(repoRoot, PACKAGE_ROOT);
+  for (const dockerfile of [
+    'docker/Dockerfile.celilo-website-sim',
+    'docker/Dockerfile.npm-registry-sim',
+  ]) {
+    const tag = imageTag(dockerfile);
+    process.stdout.write(`  rebuild ${tag.padEnd(22)} `);
+    const t0 = Date.now();
+    execSync(
+      `docker build -q -f ${join(PACKAGE_ROOT, dockerfile)} -t ${tag} ${PACKAGE_ROOT}`,
+      { stdio: 'pipe' },
+    );
+    console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);
+  }
 }
 
 /**
@@ -116,6 +263,11 @@ async function bakeViaSim(): Promise<void> {
   await ensureSharedInfra();
   console.log(`✔ ${Math.round((Date.now() - t0) / 1000)}s`);
 
+  // Fresh tarballs and site dist, and sim images that actually carry them,
+  // BEFORE install.sh runs — otherwise this bake reinstalls the last build's
+  // CLI (celilo#1299).
+  restageSimulatorInputs();
+
   process.stdout.write('  per-test stack ......... ');
   const t1 = Date.now();
   const net = await network().topology('default').managementVariant('vanilla').start();
@@ -166,8 +318,9 @@ async function bakeViaSim(): Promise<void> {
     }
     console.log(`✔ ${Math.round((Date.now() - t2) / 1000)}s`);
 
-    // Shim + verify (verify is post-shim, no mount — the shipped path).
-    await installShimAndVerify(exec, 'sim');
+    // Shim + verify. Post-shim and, since the shim now prefers the installed
+    // CLI, genuinely against the installed binary rather than the mount.
+    const version = await installShimAndVerify(exec, 'sim');
 
     process.stdout.write('  docker commit .......... ');
     const t5 = Date.now();
@@ -177,8 +330,12 @@ async function bakeViaSim(): Promise<void> {
     if (!containerId) {
       throw new Error('Could not resolve management container id');
     }
-    run(`docker commit ${containerId} celilo-e2e/management:latest`);
+    const superseded = imageIdOnTag('celilo-e2e/management:latest');
+    run(
+      `docker commit --change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(false, version)}`)} ${containerId} celilo-e2e/management:latest`,
+    );
     console.log(`✔ ${Math.round((Date.now() - t5) / 1000)}s`);
+    removeSupersededImage(superseded, 'celilo-e2e/management:latest');
   } finally {
     process.stdout.write('  tearing down ........... ');
     const tDown = Date.now();
@@ -265,7 +422,7 @@ async function bakeViaPublished(): Promise<void> {
     console.log(`✔ ${Math.round((Date.now() - t2) / 1000)}s`);
 
     // Shim + verify (verify is post-shim, no mount — the shipped path).
-    await installShimAndVerify(dexec, 'published');
+    const version = await installShimAndVerify(dexec, 'published');
 
     // Restore the sim @celilo scope override so the committed image is
     // drop-in compatible with the simulated topology it runs tests in.
@@ -280,10 +437,12 @@ async function bakeViaPublished(): Promise<void> {
     // real entrypoint (Dockerfile.management: CMD ["/startup.sh"], no
     // ENTRYPOINT). Without this the baked image never runs /startup.sh ->
     // ssh-keys volume stays empty -> fw-main crashloops (ce-um6).
+    const superseded = imageIdOnTag('celilo-e2e/management:latest');
     run(
-      `docker commit --change 'CMD ["/startup.sh"]' ${name} celilo-e2e/management:latest`,
+      `docker commit --change 'CMD ["/startup.sh"]' --change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(true, version)}`)} ${name} celilo-e2e/management:latest`,
     );
     console.log(`✔ ${Math.round((Date.now() - t3) / 1000)}s`);
+    removeSupersededImage(superseded, 'celilo-e2e/management:latest');
   } finally {
     process.stdout.write('  removing container ..... ');
     const t4 = Date.now();

package/bin/e2e-infra

@@ -34,7 +34,6 @@ case "${1:-status}" in
       console.log(generateSharedInfraYaml());
     " > "$SHARED_COMPOSE"
 
-    docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" build
     docker compose -f "$SHARED_COMPOSE" -p "$SHARED_PROJECT" up -d
 
     echo -e "${DIM}Waiting for DNS convergence...${NC}"

package/bin/e2e-up

@@ -96,9 +96,20 @@ if (customJson) {
 console.log(generateComposeYaml(config));
 " > docker-compose.yml
 
-# Build and start
-echo "Building images (this may take a while on first run)..."
-docker compose -p "$PROJECT_NAME" build
+# Verify baked images. The compose carries `image:` only, so nothing here
+# builds — a missing tag would surface as a confusing "pull access denied"
+# from `up`. Name the remedy instead.
+echo "Verifying baked images (suite time never builds)..."
+MISSING=()
+while read -r tag; do
+  [ -n "$tag" ] || continue
+  docker image inspect "$tag" >/dev/null 2>&1 || MISSING+=("$tag")
+done < <(awk '/image: celilo-e2e\//{print $3}' docker-compose.yml | sort -u)
+if [ ${#MISSING[@]} -gt 0 ]; then
+  echo "ERROR: ${#MISSING[@]} baked image(s) missing: ${MISSING[*]}"
+  echo "Remedy: cele2e build-infra"
+  exit 1
+fi
 
 echo "Starting containers..."
 docker compose -p "$PROJECT_NAME" up -d

package/docker/Dockerfile.observer

@@ -10,13 +10,24 @@ FROM ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y \
+# `--no-install-recommends` is load-bearing: `wireguard-tools` Recommends
+# `wireguard-modules | wireguard-dkms`, which apt satisfies with a realtime
+# KERNEL plus `linux-firmware` (1.1 GB). Measured 2026-09-05: 1.83 GB of this
+# image's 1.9 GB was that, for a probe toolbox that runs no kernel at all.
+# See the same note in Dockerfile.target-machine.
+#
+# `iptables` is named because it used to arrive via `wireguard-tools`' other
+# Recommends. This image dials a real tunnel (`wg-quick up wg0`), and wg-quick
+# reaches for `iptables-save` behind a `type -p` guard — so losing it would not
+# fail, it would silently skip part of teardown.
+RUN apt-get update && apt-get install -y --no-install-recommends \
     dnsutils \
     curl \
     openssl \
     netcat-openbsd \
     iputils-ping \
     iproute2 \
+    iptables \
     ca-certificates \
     wireguard-tools \
     wireguard-go \

package/docker/Dockerfile.target-machine

@@ -69,10 +69,31 @@ RUN systemctl enable ssh target-setup
 # --- Caddy pre-installation ---
 # Install Caddy (same steps as the Ansible role, but at image build time so
 # the role finds it already installed and skips the slow cloudsmith.io downloads)
-RUN apt-get update && apt-get install -y \
+#
+# `--no-install-recommends` is load-bearing here, not tidiness. `wireguard-tools`
+# Recommends `wireguard-modules | wireguard-dkms`, and apt satisfies that with
+# a KERNEL: `linux-modules-extra-*-realtime`, which drags in the realtime kernel
+# image, its modules and `linux-firmware`. Measured 2026-09-05: that one line
+# added 1.76 GB to this image (2.08 GB total), of which linux-firmware alone was
+# 1.1 GB. A container cannot load a kernel module, and wg-quick already falls
+# back to the userspace `wireguard-go` below, so none of it was ever used.
+# The price of `--no-install-recommends` is that anything this image USED to get
+# for free has to be named. Two things did, and both are load-bearing:
+#
+#   gnupg     — the very next step pipes the caddy signing key through `gpg
+#               --dearmor`. Without it the build dies with `gpg: not found`,
+#               which is how this was found: by building it.
+#   iptables  — arrived via `wireguard-tools`' other Recommends
+#               (`nftables | iptables`). The wireguard module's health check
+#               runs `iptables-save` on the host it lands on, and wg-quick
+#               consults `iptables-save` on teardown behind a `type -p` guard,
+#               so its absence would not error — it would quietly do less.
+RUN apt-get update && apt-get install -y --no-install-recommends \
     debian-keyring \
     debian-archive-keyring \
     apt-transport-https \
+    gnupg \
+    iptables \
     wireguard-tools \
     wireguard-go \
     && rm -rf /var/lib/apt/lists/*

package/npm-registry-server/package.json

@@ -9,7 +9,7 @@
     "test": "bun test"
   },
   "devDependencies": {
-    "bun-types": "latest",
+    "bun-types": "1.3.13",
     "typescript": "^5"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.19.2",
+  "version": "0.20.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -37,7 +37,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@celilo/capabilities": "^4.0.0",
+    "@celilo/capabilities": "^4.2.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/event-bus": "^0.6.0",
     "@celilo/terraform-fake": "^0.3.1",
@@ -45,7 +45,7 @@
     "zod": "^3.24.1"
   },
   "devDependencies": {
-    "@types/bun": "latest"
+    "@types/bun": "1.4.0"
   },
   "keywords": [
     "celilo",

package/registry-server/package.json

@@ -11,7 +11,7 @@
     "test": "bun test --timeout 30000"
   },
   "devDependencies": {
-    "bun-types": "latest",
+    "bun-types": "1.4.0",
     "typescript": "^5"
   }
 }

package/scripts/pack-celilo-packages.ts

@@ -38,6 +38,7 @@ import {
   workspaceVersionMap,
 } from '../../../scripts/workspace-graph';
 import { ensureRegistryServerBundle } from '../src/registry-bundle';
+import { computeSourceFingerprint } from '../src/source-fingerprint';
 
 /**
  * Resolve the celilo monorepo root by walking up from `startDir` (the caller's
@@ -242,6 +243,20 @@ function main(): void {
   }
 
   console.log(`✓ Packed ${results.length} package(s) to ${destination}`);
+
+  // Record the tree fingerprint these tarballs were packed FROM, next to the
+  // tarballs. The management-image bake stamps THIS value, not a fingerprint
+  // recomputed from the tree at bake time: the tarballs are what install.sh
+  // actually installs, so the stamp must describe them (celilo#1299 — a bake
+  // that reinstalls a stale cache used to stamp a fingerprint of the newer
+  // tree, claiming CLI content the image did not carry). The npm-registry
+  // server only serves *.tgz, so a JSON file in the same directory is inert.
+  const manifest = {
+    packedAt: new Date().toISOString(),
+    sourceFingerprint: computeSourceFingerprint(repoRoot()),
+    packages: results,
+  };
+  writeFileSync(join(destination, 'pack-manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`);
 }
 
 if (import.meta.main) {