@celilo/e2e 0.10.3 → 0.10.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.10.3",
+  "version": "0.10.4",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@celilo/cli-display": "^0.1.9",
-    "@celilo/event-bus": "^0.1.9",
+    "@celilo/event-bus": "^0.1.10",
     "yaml": "^2.8.0",
     "zod": "^3.24.1"
   },

package/src/container-manager.ts

@@ -93,14 +93,23 @@ const activeProjects: Set<string> = new Set();
 export function projectTeardownCommands(project: string): {
   listContainers: string;
   listNetworks: string;
+  listVolumes: string;
 } {
   return {
     listContainers: `docker ps -aq --filter name=${project}`,
     listNetworks: `docker network ls --format {{.Name}} --filter name=${project}`,
+    // Volumes were missing here, and the compose path that would have removed
+    // them (`down --volumes`) is the same best-effort call that silently does
+    // nothing without a resolvable compose file. So every run leaked its
+    // volumes: 58 `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder,
+    // the oldest months old. Nothing breaks from a leaked volume the way a
+    // leaked network breaks routing — it just accumulates unbounded, and the
+    // pile is the evidence that teardown has not been running.
+    listVolumes: `docker volume ls -q --filter name=${project}`,
   };
 }
 
-/** Force-remove a project's containers and networks by name. Never throws. */
+/** Force-remove a project's containers, networks and volumes by name. Never throws. */
 function forceRemoveProject(project: string): void {
   const cmds = projectTeardownCommands(project);
   try {
@@ -123,6 +132,19 @@ function forceRemoveProject(project: string): void {
       } catch {}
     }
   } catch {}
+  // Volumes last: a volume still attached to a container cannot be removed, so
+  // this must follow the container sweep above.
+  try {
+    const vols = execSync(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
+      .toString()
+      .split('\n')
+      .filter(Boolean);
+    for (const vol of vols) {
+      try {
+        execSync(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch {}
+    }
+  } catch {}
 }
 
 function cleanupOnExit() {
@@ -271,9 +293,7 @@ export async function scrubDnsZones(): Promise<void> {
   }
   if (served !== null && !served.split(/\s+/).includes(expected)) {
     throw new Error(
-      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} ` +
-        `(website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". ` +
-        `The DNS reset did not take effect — shared DNS would bleed across tests.`,
+      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} (website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". The DNS reset did not take effect — shared DNS would bleed across tests.`,
     );
   }
 }
@@ -330,9 +350,7 @@ function assertCliVersion(projectName: string, composeDir: string): void {
   const result = dockerExec(projectName, composeDir, 'management', 'celilo --version', 8_000);
   if (result.exitCode !== 0) {
     throw new Error(
-      `Could not read celilo version from the management image ` +
-        `(harness requires >=${MIN_CLI_VERSION}). ` +
-        `\`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
+      `Could not read celilo version from the management image (harness requires >=${MIN_CLI_VERSION}). \`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
     );
   }
   const problem = checkCliVersion(result.stdout);
@@ -564,11 +582,7 @@ function buildNetworkHandle(
       );
       // Spawn detached. nohup + & + disown cleanly survives the
       // dockerExec session ending.
-      const spawnCmd =
-        `nohup celilo events respond --values ${valuesPath} ` +
-        '--idle-timeout 1h --max-duration 2h ' +
-        '--emittedBy cele2e-responder ' +
-        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
+      const spawnCmd = `nohup celilo events respond --values ${valuesPath} --idle-timeout 1h --max-duration 2h --emittedBy cele2e-responder > /tmp/cele2e-responder.log 2>&1 < /dev/null & disown`;
       const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
       if (spawnResult.exitCode !== 0) {
         throw new Error(
@@ -772,9 +786,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     if (heavy.length > 0) {
       const names = heavy.map((c) => `   - ${c}`).join('\n');
       throw new Error(
-        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\n` +
-          'Live and e2e environments are mutually exclusive.\n' +
-          'Stop live containers first: cele2e down --all\n',
+        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\nLive and e2e environments are mutually exclusive.\nStop live containers first: cele2e down --all\n`,
       );
     }
   } catch (e) {
@@ -803,25 +815,33 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       }
     } catch {}
 
-    // Remove stale per-test networks
+    // Remove stale per-test networks AND their volumes, via the same
+    // by-name teardown the exit handler uses. This used to lean on
+    // `docker compose -p <project> down --volumes` for the volume half, which
+    // has no `-f` and no reliable cwd here either — compose exits "no
+    // configuration file provided" and the catch swallows it, so volumes were
+    // never removed on this path either. Reusing forceRemoveProject keeps the
+    // crash-recovery sweep and the exit handler from drifting apart.
     try {
       const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
         .split('\n')
         .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
-      if (networks.length > 0) {
-        const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
-        for (const project of projects) {
-          try {
-            run(`docker compose -p ${project} down --volumes --remove-orphans`, {
-              timeout: 30_000,
-            });
-          } catch {}
-        }
-        for (const net of networks) {
-          try {
-            run(`docker network rm ${net}`, { timeout: 5_000 });
-          } catch {}
-        }
+      const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
+      for (const project of projects) {
+        forceRemoveProject(project);
+      }
+    } catch {}
+    // Volumes can outlive every container and network of their project — that
+    // is exactly the 58-volume pile — so sweep by prefix too, not only for
+    // projects that still have a network to be discovered by.
+    try {
+      const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
+        .split('\n')
+        .filter(Boolean);
+      for (const vol of vols) {
+        try {
+          run(`docker volume rm ${vol}`, { timeout: 5_000 });
+        } catch {}
       }
     } catch {}
   } catch {}
@@ -1023,8 +1043,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
           // The control plane needs its subnet declared if celilo-mgr lives
           // there OR if a machine does — infrastructure selection for a module
           // declaring `zone: secure-mgmt` reads it either way (#436).
-          config.managementZone === 'secure-mgmt' ||
-          (config.secureMgmtMachines ?? []).length > 0
+          config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
             ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
         }dns.primary=100.100.0.1 \
@@ -1084,8 +1103,7 @@ export function reconnectNetwork(projectName: string): NetworkHandle {
     }
   } catch (err) {
     throw new Error(
-      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
-        `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
+      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\nFix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
     );
   }
 

package/src/exit-cleanup.test.ts

@@ -11,10 +11,10 @@
  * The leak's mechanism was a wiring bug, so the gate is on the wiring.
  */
 
+import { expect, test } from 'bun:test';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { expect, test } from 'bun:test';
 import { projectTeardownCommands } from './container-manager';
 
 const SRC = readFileSync(
@@ -34,7 +34,7 @@ test('cleanup honors --keep / --reuse so a failed run stays debuggable', () => {
   // an operator asked to keep, since --keep matters most after a failure.
   const body = SRC.slice(
     SRC.indexOf('function cleanupOnExit'),
-    SRC.indexOf('process.on(\'SIGTERM\''),
+    SRC.indexOf("process.on('SIGTERM'"),
   );
   expect(body).toContain('CELILO_E2E_KEEP');
   expect(body).toContain('CELILO_E2E_REUSE');
@@ -54,6 +54,20 @@ test('teardown does not depend on a compose file being findable', () => {
   }
 });
 
+// Containers and networks were removed; volumes were not. The only thing that
+// would have removed them was `docker compose down --volumes`, which is the
+// same best-effort call that silently does nothing without a resolvable compose
+// file — so in practice they were never removed at all. 58 leaked
+// `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder, the oldest
+// months old. A leaked volume breaks nothing on its own, which is why it went
+// unnoticed for months; the pile is the evidence teardown was not running.
+test('teardown removes volumes, not just containers and networks', () => {
+  const cmds = projectTeardownCommands('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).toContain('docker volume ls');
+  expect(cmds.listVolumes).toContain('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).not.toContain('docker compose');
+});
+
 test('teardown cannot match the shared project', () => {
   // Per-test projects carry a timestamp; `celilo-e2e-shared` does not, so a
   // name filter for one can never match the other. Shared infra must survive —