@celilo/e2e 0.15.0 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/e2e",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
   "type": "module",
   "main": "./src/index.ts",
@@ -37,7 +37,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@celilo/capabilities": "^1.3.0",
+    "@celilo/capabilities": "^1.5.0",
     "@celilo/cli-display": "^0.2.0",
     "@celilo/event-bus": "^0.6.0",
     "@celilo/terraform-fake": "^0.3.0",

package/src/container-manager.ts

@@ -12,6 +12,7 @@ import {
   getAllMachines,
 } from './docker-compose-generator';
 import { explainBuildFailure } from './doctor';
+import { type ModuleHost, parseModuleHost } from './module-host';
 import { ensureSharedInfra } from './shared-infra';
 import { SIMULATOR_IPS } from './simulator-ips';
 import { startSocksProxy } from './socks-proxy';
@@ -369,6 +370,29 @@ function assertCliVersion(projectName: string, composeDir: string): void {
   if (problem) throw new Error(problem);
 }
 
+/**
+ * Exec into a container that is NOT part of the compose project.
+ *
+ * A guest the Proxmox simulator provisioned is a real container, but compose
+ * knows nothing about it, so `docker compose exec` cannot see it. Same wrapping
+ * as `dockerExec` so both behave identically from a test's point of view.
+ */
+function plainDockerExec(container: string, cmd: string, timeoutMs = 60_000): ExecResult {
+  try {
+    const stdout = run(`docker exec ${container} bash -c ${JSON.stringify(cmd)}`, {
+      timeout: timeoutMs,
+    });
+    return { stdout, stderr: '', exitCode: 0 };
+  } catch (err: unknown) {
+    const e = err as { stdout?: string; stderr?: string; status?: number };
+    return {
+      stdout: e.stdout ?? '',
+      stderr: e.stderr ?? String(err),
+      exitCode: e.status ?? 1,
+    };
+  }
+}
+
 function dockerExecAsync(
   projectName: string,
   composeDir: string,
@@ -569,6 +593,97 @@ function buildNetworkHandle(
       return Promise.resolve(dockerExec(projectName, composeDir, container, cmd, timeoutMs));
     },
 
+    async registerControlPlane(options = {}): Promise<void> {
+      const moduleDir =
+        options.moduleDir ?? join(celiloRoot ?? PACKAGE_ROOT, 'modules', 'celilo-mgmt');
+
+      await handle.publishModule(moduleDir);
+      // Tolerant: this box may already be in the pool depending on how the
+      // network came up, and "already there" satisfies the point of the step.
+      await handle.celilo('machine add 127.0.0.1 --zone secure-mgmt --earmark celilo-mgmt', {
+        check: false,
+      });
+      await handle.celilo('module import celilo-mgmt');
+      // Docker and terraform are already in the management image; installing
+      // them again costs minutes and proves nothing here.
+      await handle.celilo('module config set celilo-mgmt install_docker false');
+      await handle.celilo('module config set celilo-mgmt install_terraform false');
+      await handle.celilo(
+        'module config set celilo-mgmt db_path /root/.local/share/celilo/celilo.db',
+      );
+      // Give the box its fleet key BEFORE deploying, using the harness keypair.
+      //
+      // `ensureFleetKey` (modules/celilo-mgmt/scripts/discovery.ts) mints a new
+      // `celilo-fleet` key under the state dir when none exists, and the deploy
+      // then sets `ssh.public_key` to it. In a real fleet that is right. In the
+      // rig it is not survivable: every target container trusts the HARNESS key,
+      // baked into authorized_keys at boot, and a freshly minted key is trusted
+      // by nothing. celilo would hold a key no machine accepts, and the next
+      // `machine add` fails with "no matching private key was found in ~/.ssh/"
+      // — which reads as a missing fixture and is actually a rotated key.
+      //
+      // `ensureFleetKey` is idempotent and REUSES an existing key, so seeding it
+      // with the harness pair means the deploy runs its real code path and
+      // arrives at the key the fleet already trusts. This models an operator
+      // whose management box already has its fleet key, which is the ordinary
+      // case after the first deploy.
+      const stateSshDir = '/root/.local/share/celilo/.ssh';
+      dockerExec(
+        projectName,
+        composeDir,
+        'management',
+        `mkdir -p ${stateSshDir} && chmod 700 ${stateSshDir} && ` +
+          `cp /ssh-keys/id_ed25519 ${stateSshDir}/id_ed25519 && ` +
+          `cp /ssh-keys/id_ed25519.pub ${stateSshDir}/id_ed25519.pub && ` +
+          `chmod 600 ${stateSshDir}/id_ed25519`,
+      );
+
+      await handle.celilo('module deploy celilo-mgmt', 300_000);
+
+      // Deploying celilo-mgmt onto the management box REWRITES that box's own
+      // ~/.ssh. A `machine add` landing in that window fails with "Cannot find
+      // SSH private key", which reads as a missing fixture rather than as a
+      // race against the deploy that just ran. Wait for the key to be back and
+      // readable before handing control to the caller, so no suite has to know
+      // this happens.
+      await waitFor(
+        async () => {
+          const probe = dockerExec(
+            projectName,
+            composeDir,
+            'management',
+            'test -r /root/.ssh/id_ed25519 && echo ok',
+          );
+          return probe.stdout.trim() === 'ok';
+        },
+        60_000,
+        "the management box's SSH private key to be readable after celilo-mgmt deploy",
+      );
+    },
+
+    async moduleHost(moduleId: string): Promise<ModuleHost> {
+      const status = dockerExec(
+        projectName,
+        composeDir,
+        'management',
+        `celilo module status ${moduleId}`,
+      );
+      const host = parseModuleHost(status.stdout);
+      if (!host) {
+        throw new Error(
+          `Could not find where '${moduleId}' is deployed. Is it deployed yet? \`module status\` reported no placement:\n${status.stdout.slice(0, 400)}`,
+        );
+      }
+      return host;
+    },
+
+    async execOnModuleHost(moduleId, cmd, timeoutMs = 60_000): Promise<ExecResult> {
+      const host = await this.moduleHost(moduleId);
+      return host.reach === 'plain'
+        ? plainDockerExec(host.container, cmd, timeoutMs)
+        : dockerExec(projectName, composeDir, host.container, cmd, timeoutMs);
+    },
+
     async respondWith(values): Promise<void> {
       // Write the values JSON inside the management container at a
       // stable path, then start a detached `celilo events respond

package/src/module-host.test.ts

@@ -0,0 +1,71 @@
+/**
+ * Resolving a module's host from `module status`.
+ *
+ * The parsing is the risky half and needs no rig, so it is tested here rather
+ * than discovered during a 3-minute e2e run. The input shapes come from
+ * `formatPlacementLine` in `apps/celilo/src/services/placement-reconcile.ts`;
+ * if that function grows a case, one of these should fail.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { parseModuleHost } from './module-host';
+
+/** How the CLI really prints it — clack gutter and ANSI included. */
+const withGutter = (line: string) => `\x1b[1mModule: caddy\x1b[0m\n│  Placement:\n│  ${line}\n`;
+
+describe('parseModuleHost', () => {
+  test('a provisioned guest resolves to the container the simulator created', () => {
+    const host = parseModuleHost(withGutter('iot (vmid 2100) → pve1 (zone app)'));
+
+    expect(host).toEqual({
+      hostname: 'iot',
+      vmid: 2100,
+      container: 'celilo-e2e-lxc-2100',
+      reach: 'plain',
+    });
+  });
+
+  test('a pool machine resolves to the compose service of the same name', () => {
+    const host = parseModuleHost(withGutter('caddy — machine (zone dmz)'));
+
+    expect(host).toEqual({ hostname: 'caddy', container: 'caddy', reach: 'compose' });
+  });
+
+  test('the two are reached DIFFERENTLY, which is the whole point', () => {
+    // A guest the simulator created is a real container but not a compose
+    // service, so `docker compose exec` cannot see it. Getting this backwards
+    // fails in a way that reads as the module breaking.
+    const guest = parseModuleHost(withGutter('iot (vmid 2100) → pve1 (zone app)'));
+    const machine = parseModuleHost(withGutter('caddy — machine (zone dmz)'));
+
+    expect(guest?.reach).toBe('plain');
+    expect(machine?.reach).toBe('compose');
+  });
+
+  test('a module that is not deployed yet resolves to nothing, not a guess', () => {
+    // A real state — imported but never deployed — and distinguishable from a
+    // parse failure, because a test should react differently to each.
+    expect(parseModuleHost('Module: caddy\nState: CONFIGURED\n')).toBeNull();
+  });
+
+  test('reads placement even when Proxmox could not be reached', () => {
+    // `unreachable` still names the host and vmid, and the container exists
+    // regardless of whether celilo could confirm which node holds it.
+    const host = parseModuleHost(
+      withGutter('iot (vmid 2100) → node unknown — Proxmox unreachable (zone app)'),
+    );
+
+    expect(host?.container).toBe('celilo-e2e-lxc-2100');
+  });
+
+  test('is not fooled by the module id appearing elsewhere in the output', () => {
+    const output = [
+      'Module: caddy',
+      'Source: /root/.local/share/celilo/modules/caddy',
+      'Placement (live from Proxmox):',
+      '  web (vmid 305) → pve1 (zone dmz)',
+    ].join('\n');
+
+    expect(parseModuleHost(output)?.hostname).toBe('web');
+  });
+});

package/src/module-host.ts

@@ -0,0 +1,73 @@
+/**
+ * Reach a module's deployed host without knowing how it came to exist.
+ *
+ * A suite that says `net.exec('caddy', …)` is not naming a host — it is naming a
+ * COMPOSE SERVICE, which only exists because the test declared a machine in the
+ * topology. The moment the same module is placed on a container service, celilo
+ * provisions the host itself, there is no compose service by that name, and the
+ * call fails in a way that looks like the module broke.
+ *
+ * That coupling is the single thing standing between the suite and the
+ * conversions in celilo#815: every candidate suite execs on its module's host by
+ * compose-service name. `execOnModuleHost` asks celilo where the module actually
+ * landed and reaches it accordingly, so a suite stops caring which path placed
+ * it — which is what lets the SAME suite cover both.
+ */
+
+/** Where a module's host lives, as celilo reports it. */
+export interface ModuleHost {
+  hostname: string;
+  /** Present only for a guest celilo provisioned; absent for a pool machine. */
+  vmid?: number;
+  /**
+   * The docker container to exec into, and how to reach it.
+   *
+   * `compose` for a machine the topology declared — it is a service in the
+   * project. `plain` for a guest the simulator created, which is a real
+   * container but NOT part of the compose project, so `docker compose exec`
+   * cannot see it.
+   */
+  container: string;
+  reach: 'compose' | 'plain';
+}
+
+/**
+ * Parse `celilo module status <id>`'s placement section.
+ *
+ * The two shapes come from `formatPlacementLine`:
+ *   `<hostname> — machine (zone X)`                  ← machine pool
+ *   `<hostname> (vmid N) → <node> (zone X)`          ← container service
+ *
+ * Returns null when no placement line is present, which is a real state — the
+ * module is imported but not deployed — and is worth distinguishing from a parse
+ * failure, because "not deployed yet" and "I could not read the output" call for
+ * different reactions from a test.
+ */
+export function parseModuleHost(statusOutput: string): ModuleHost | null {
+  // Strip clack's `│  ` gutter and any ANSI before matching; CLI output carries
+  // both, and a raw match silently finds nothing.
+  const lines = statusOutput
+    .split('\n')
+    // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI is the point
+    .map((line) => line.replace(/\x1b\[[0-9;]*m/g, '').replace(/^[\s│|]+/, ''));
+
+  for (const line of lines) {
+    const container = /^(\S+)\s+\(vmid\s+(\d+)\)\s+→/.exec(line);
+    if (container?.[1] && container[2]) {
+      const vmid = Number(container[2]);
+      return {
+        hostname: container[1],
+        vmid,
+        container: `celilo-e2e-lxc-${vmid}`,
+        reach: 'plain',
+      };
+    }
+
+    const machine = /^(\S+)\s+—\s+machine\s+\(/.exec(line);
+    if (machine?.[1]) {
+      return { hostname: machine[1], container: machine[1], reach: 'compose' };
+    }
+  }
+
+  return null;
+}

package/src/types.ts

@@ -7,6 +7,8 @@
  * selection had nowhere to put it — so a module that named the zone correctly
  * could not be deployed by any test.
  */
+
+import type { ModuleHost } from './module-host';
 export type Zone = 'dmz' | 'app' | 'secure' | 'internal' | 'secure-mgmt';
 
 /**
@@ -252,6 +254,54 @@ export interface NetworkHandle {
   /** Execute an arbitrary command in any container */
   exec(container: string, cmd: string, timeoutMs?: number): Promise<ExecResult>;
 
+  /**
+   * Where celilo actually put a module, read from `module status`.
+   *
+   * Throws if the module is not deployed — "not deployed" and "I could not read
+   * the placement" are different problems and a test should not have to tell
+   * them apart from a silent empty result.
+   */
+  moduleHost(moduleId: string): Promise<ModuleHost>;
+
+  /**
+   * Tell celilo where its own control plane lives, and wait until the box is
+   * usable again.
+   *
+   * Required by any suite that runs celilo-mgr on `secure-mgmt` AND deploys a
+   * firewall — which is every container-service conversion, because the Proxmox
+   * simulator lives on `secure-mgmt` and `service add proxmox` has to reach it.
+   *
+   * The firewall's trusted sources are DERIVED from where celilo-mgmt is
+   * deployed, not hardcoded, and `secure-mgmt` is deliberately not a tier in the
+   * rule matrix — it reaches every tier by trust. So this must run BEFORE
+   * `deployFirewall()`, which is when those rules are computed. Skip it and
+   * fw-main's default-DROP FORWARD chain has no rule for the control-plane
+   * subnet: every SSH from celilo-mgr into a segmented zone times out, and it
+   * surfaces as the deployed module's hook failing rather than as a firewall
+   * policy gap.
+   *
+   * Also absorbs a race that is not the caller's business: the deploy rewrites
+   * the management box's own `~/.ssh`, so this does not return until the private
+   * key is readable again.
+   */
+  registerControlPlane(options?: { moduleDir?: string }): Promise<void>;
+
+  /**
+   * Execute a command on the host a module is deployed on, WITHOUT knowing how
+   * that host came to exist.
+   *
+   * `exec('caddy', …)` names a compose service, which exists only because the
+   * topology declared a machine. Place the same module on a container service
+   * and celilo provisions the host itself: no compose service by that name, and
+   * the call fails in a way that reads as the module breaking.
+   *
+   * This resolves the host from celilo and reaches it the right way — compose
+   * exec for a declared machine, plain docker exec for a provisioned guest,
+   * which is a real container the compose project knows nothing about. Use it
+   * in any suite that could run either way.
+   */
+  execOnModuleHost(moduleId: string, cmd: string, timeoutMs?: number): Promise<ExecResult>;
+
   /**
    * Pause the test and drop into an interactive shell for debugging.
    * The test resumes when you exit the shell.