@celilo/e2e 0.10.2 → 0.10.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.10.2",
+  "version": "0.10.4",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@celilo/cli-display": "^0.1.9",
-    "@celilo/event-bus": "^0.1.9",
+    "@celilo/event-bus": "^0.1.10",
     "yaml": "^2.8.0",
     "zod": "^3.24.1"
   },

package/simulators/signal-cli/server.ts

@@ -113,6 +113,14 @@ function handleSend(id: unknown, params: Record<string, unknown>): Response {
   if (recipients.length === 0) return rpcError(id, -32602, 'No recipient given');
 
   for (const recipient of recipients) {
+    // The linked account is ALWAYS a valid recipient: messaging yourself is
+    // note-to-self, a first-class Signal feature and the default
+    // single-operator setup (design R2) — the route points at the very number
+    // the transport is a secondary device of. Rejecting it as "unregistered"
+    // is something no real daemon does, and it made the note-to-self ack path
+    // untestable end to end: the page could not be sent, so no token ever
+    // reached the operator to reply with.
+    if (recipient === ACCOUNT) continue;
     // Push back like Signal: an unregistered number is a hard failure, not a
     // silent no-op.
     if (KNOWN_RECIPIENTS.size > 0 && !KNOWN_RECIPIENTS.has(recipient)) {

package/src/container-manager.ts

@@ -75,6 +75,78 @@ const activeProjects: Set<string> = new Set();
  * MUST stay fully synchronous: it runs from an `exit` handler, where the
  * event loop is already closed and any async work is silently dropped.
  */
+/**
+ * The commands that remove a per-test project WITHOUT needing its compose file.
+ * Pure (Rule 10.1) so the no-compose-dependency property is testable.
+ *
+ * `docker compose -p <project> down` — what this used to run — resolves the
+ * project from a compose file in the CURRENT DIRECTORY. An exit handler has no
+ * reliable cwd, so compose exited "no configuration file provided", the error
+ * was swallowed by the surrounding catch, and nothing was removed. The handler
+ * fired correctly and tore down nothing.
+ *
+ * Removing by name needs only the project name, which we already have.
+ * Per-test projects are `celilo-e2e-<timestamp>`; the shared project is
+ * `celilo-e2e-shared`, which contains no timestamp and so is never matched —
+ * shared infra stays up, as it must.
+ */
+export function projectTeardownCommands(project: string): {
+  listContainers: string;
+  listNetworks: string;
+  listVolumes: string;
+} {
+  return {
+    listContainers: `docker ps -aq --filter name=${project}`,
+    listNetworks: `docker network ls --format {{.Name}} --filter name=${project}`,
+    // Volumes were missing here, and the compose path that would have removed
+    // them (`down --volumes`) is the same best-effort call that silently does
+    // nothing without a resolvable compose file. So every run leaked its
+    // volumes: 58 `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder,
+    // the oldest months old. Nothing breaks from a leaked volume the way a
+    // leaked network breaks routing — it just accumulates unbounded, and the
+    // pile is the evidence that teardown has not been running.
+    listVolumes: `docker volume ls -q --filter name=${project}`,
+  };
+}
+
+/** Force-remove a project's containers, networks and volumes by name. Never throws. */
+function forceRemoveProject(project: string): void {
+  const cmds = projectTeardownCommands(project);
+  try {
+    const ids = execSync(cmds.listContainers, { timeout: 15_000, stdio: 'pipe' })
+      .toString()
+      .split('\n')
+      .filter(Boolean);
+    if (ids.length > 0) {
+      execSync(`docker rm -f ${ids.join(' ')}`, { timeout: 30_000, stdio: 'pipe' });
+    }
+  } catch {}
+  try {
+    const nets = execSync(cmds.listNetworks, { timeout: 15_000, stdio: 'pipe' })
+      .toString()
+      .split('\n')
+      .filter(Boolean);
+    for (const net of nets) {
+      try {
+        execSync(`docker network rm ${net}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch {}
+    }
+  } catch {}
+  // Volumes last: a volume still attached to a container cannot be removed, so
+  // this must follow the container sweep above.
+  try {
+    const vols = execSync(cmds.listVolumes, { timeout: 15_000, stdio: 'pipe' })
+      .toString()
+      .split('\n')
+      .filter(Boolean);
+    for (const vol of vols) {
+      try {
+        execSync(`docker volume rm ${vol}`, { timeout: 10_000, stdio: 'pipe' });
+      } catch {}
+    }
+  } catch {}
+}
+
 function cleanupOnExit() {
   // `--keep` / `--reuse` mean the operator wants the stack to survive for
   // debugging — and they want it MOST after a failure, which is exactly the
@@ -82,12 +154,17 @@ function cleanupOnExit() {
   if (process.env.CELILO_E2E_KEEP === '1' || process.env.CELILO_E2E_REUSE === '1') return;
 
   for (const project of activeProjects) {
+    // Compose first when it can work — it also drops volumes and orphans — but
+    // it is best-effort, so never rely on it having done anything.
     try {
-      execSync(`docker compose -p ${project} down --volumes --remove-orphans`, {
+      execSync(`docker compose -f ${COMPOSE_FILE} -p ${project} down --volumes --remove-orphans`, {
+        cwd: PACKAGE_ROOT,
         timeout: 30_000,
         stdio: 'pipe',
       });
     } catch {}
+    // Authoritative: needs no compose file, no cwd, no working directory state.
+    forceRemoveProject(project);
   }
   activeProjects.clear();
   try {
@@ -216,9 +293,7 @@ export async function scrubDnsZones(): Promise<void> {
   }
   if (served !== null && !served.split(/\s+/).includes(expected)) {
     throw new Error(
-      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} ` +
-        `(website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". ` +
-        `The DNS reset did not take effect — shared DNS would bleed across tests.`,
+      `[dns-scrub] post-reset verification FAILED: celilo.computer apex should serve ${expected} (website-sim) after scrub, but namecheap-dns returns "${served || '(empty)'}". The DNS reset did not take effect — shared DNS would bleed across tests.`,
     );
   }
 }
@@ -275,9 +350,7 @@ function assertCliVersion(projectName: string, composeDir: string): void {
   const result = dockerExec(projectName, composeDir, 'management', 'celilo --version', 8_000);
   if (result.exitCode !== 0) {
     throw new Error(
-      `Could not read celilo version from the management image ` +
-        `(harness requires >=${MIN_CLI_VERSION}). ` +
-        `\`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
+      `Could not read celilo version from the management image (harness requires >=${MIN_CLI_VERSION}). \`celilo --version\` exited ${result.exitCode}: ${result.stderr || result.stdout || '(no output)'}`,
     );
   }
   const problem = checkCliVersion(result.stdout);
@@ -509,11 +582,7 @@ function buildNetworkHandle(
       );
       // Spawn detached. nohup + & + disown cleanly survives the
       // dockerExec session ending.
-      const spawnCmd =
-        `nohup celilo events respond --values ${valuesPath} ` +
-        '--idle-timeout 1h --max-duration 2h ' +
-        '--emittedBy cele2e-responder ' +
-        '> /tmp/cele2e-responder.log 2>&1 < /dev/null & disown';
+      const spawnCmd = `nohup celilo events respond --values ${valuesPath} --idle-timeout 1h --max-duration 2h --emittedBy cele2e-responder > /tmp/cele2e-responder.log 2>&1 < /dev/null & disown`;
       const spawnResult = dockerExec(projectName, composeDir, 'management', spawnCmd, 5_000);
       if (spawnResult.exitCode !== 0) {
         throw new Error(
@@ -717,9 +786,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     if (heavy.length > 0) {
       const names = heavy.map((c) => `   - ${c}`).join('\n');
       throw new Error(
-        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\n` +
-          'Live and e2e environments are mutually exclusive.\n' +
-          'Stop live containers first: cele2e down --all\n',
+        `\nCannot start e2e tests: ${heavy.length} live container(s) are running:\n${names}\n\nLive and e2e environments are mutually exclusive.\nStop live containers first: cele2e down --all\n`,
       );
     }
   } catch (e) {
@@ -748,25 +815,33 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
       }
     } catch {}
 
-    // Remove stale per-test networks
+    // Remove stale per-test networks AND their volumes, via the same
+    // by-name teardown the exit handler uses. This used to lean on
+    // `docker compose -p <project> down --volumes` for the volume half, which
+    // has no `-f` and no reliable cwd here either — compose exits "no
+    // configuration file provided" and the catch swallows it, so volumes were
+    // never removed on this path either. Reusing forceRemoveProject keeps the
+    // crash-recovery sweep and the exit handler from drifting apart.
     try {
       const networks = run('docker network ls --format "{{.Name}}" 2>/dev/null')
         .split('\n')
         .filter((n) => n.startsWith('celilo-e2e-1')); // per-test projects start with timestamp
-      if (networks.length > 0) {
-        const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
-        for (const project of projects) {
-          try {
-            run(`docker compose -p ${project} down --volumes --remove-orphans`, {
-              timeout: 30_000,
-            });
-          } catch {}
-        }
-        for (const net of networks) {
-          try {
-            run(`docker network rm ${net}`, { timeout: 5_000 });
-          } catch {}
-        }
+      const projects = [...new Set(networks.map((n) => n.replace(/_[^_]+$/, '')))];
+      for (const project of projects) {
+        forceRemoveProject(project);
+      }
+    } catch {}
+    // Volumes can outlive every container and network of their project — that
+    // is exactly the 58-volume pile — so sweep by prefix too, not only for
+    // projects that still have a network to be discovered by.
+    try {
+      const vols = run('docker volume ls -q --filter name=celilo-e2e-1 2>/dev/null')
+        .split('\n')
+        .filter(Boolean);
+      for (const vol of vols) {
+        try {
+          run(`docker volume rm ${vol}`, { timeout: 5_000 });
+        } catch {}
       }
     } catch {}
   } catch {}
@@ -968,8 +1043,7 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
           // The control plane needs its subnet declared if celilo-mgr lives
           // there OR if a machine does — infrastructure selection for a module
           // declaring `zone: secure-mgmt` reads it either way (#436).
-          config.managementZone === 'secure-mgmt' ||
-          (config.secureMgmtMachines ?? []).length > 0
+          config.managementZone === 'secure-mgmt' || (config.secureMgmtMachines ?? []).length > 0
             ? `network.secure-mgmt.subnet=${ZONE_SUBNETS['secure-mgmt']} network.secure-mgmt.gateway=${ZONE_GATEWAYS['secure-mgmt']} `
             : ''
         }dns.primary=100.100.0.1 \
@@ -1029,8 +1103,7 @@ export function reconnectNetwork(projectName: string): NetworkHandle {
     }
   } catch (err) {
     throw new Error(
-      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\n` +
-        `Fix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
+      `Cannot reconnect to network '${projectName}': ${err instanceof Error ? err.message : String(err)}\nFix: Run without --reuse to create a fresh network, or check if containers were torn down.`,
     );
   }
 

package/src/exit-cleanup.test.ts

@@ -11,10 +11,11 @@
  * The leak's mechanism was a wiring bug, so the gate is on the wiring.
  */
 
+import { expect, test } from 'bun:test';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { expect, test } from 'bun:test';
+import { projectTeardownCommands } from './container-manager';
 
 const SRC = readFileSync(
   join(dirname(fileURLToPath(import.meta.url)), 'container-manager.ts'),
@@ -33,12 +34,51 @@ test('cleanup honors --keep / --reuse so a failed run stays debuggable', () => {
   // an operator asked to keep, since --keep matters most after a failure.
   const body = SRC.slice(
     SRC.indexOf('function cleanupOnExit'),
-    SRC.indexOf('process.on(\'SIGTERM\''),
+    SRC.indexOf("process.on('SIGTERM'"),
   );
   expect(body).toContain('CELILO_E2E_KEEP');
   expect(body).toContain('CELILO_E2E_REUSE');
 });
 
+// The first version of this fix wired the handler correctly and still leaked,
+// because the teardown it ran was `docker compose -p <project> down` with no
+// `-f` and no cwd. Compose resolves the project from a compose file in the
+// CURRENT DIRECTORY; an exit handler has no reliable cwd, so it exited "no
+// configuration file provided", the surrounding catch swallowed it, and nothing
+// was removed. Asserting the hook exists was not enough — assert it can work.
+test('teardown does not depend on a compose file being findable', () => {
+  const cmds = projectTeardownCommands('celilo-e2e-1785546029645');
+  for (const cmd of Object.values(cmds)) {
+    expect(cmd).not.toContain('docker compose');
+    expect(cmd).toContain('celilo-e2e-1785546029645');
+  }
+});
+
+// Containers and networks were removed; volumes were not. The only thing that
+// would have removed them was `docker compose down --volumes`, which is the
+// same best-effort call that silently does nothing without a resolvable compose
+// file — so in practice they were never removed at all. 58 leaked
+// `celilo-e2e-<ts>_ssh-keys` volumes were found on the builder, the oldest
+// months old. A leaked volume breaks nothing on its own, which is why it went
+// unnoticed for months; the pile is the evidence teardown was not running.
+test('teardown removes volumes, not just containers and networks', () => {
+  const cmds = projectTeardownCommands('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).toContain('docker volume ls');
+  expect(cmds.listVolumes).toContain('celilo-e2e-1785546029645');
+  expect(cmds.listVolumes).not.toContain('docker compose');
+});
+
+test('teardown cannot match the shared project', () => {
+  // Per-test projects carry a timestamp; `celilo-e2e-shared` does not, so a
+  // name filter for one can never match the other. Shared infra must survive —
+  // it is torn down separately, and removing it mid-suite breaks every
+  // subsequent test.
+  const cmds = projectTeardownCommands('celilo-e2e-1785546029645');
+  for (const cmd of Object.values(cmds)) {
+    expect(cmd).not.toContain('celilo-e2e-shared');
+  }
+});
+
 test("the premise holds: process.exit() fires 'exit' but NOT 'beforeExit'", async () => {
   // The whole fix rests on this runtime behavior. If it ever changes, the
   // reasoning above is void and this should fail loudly rather than silently