@celilo/e2e 0.18.0 → 0.19.1

package/src/container-manager.ts

@@ -1,5 +1,5 @@
 import { type ExecSyncOptions, execSync, spawn } from 'node:child_process';
-import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { basename, join, resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
@@ -10,6 +10,7 @@ import {
   firewallZoneLegs,
   generateTestComposeYaml,
   getAllMachines,
+  registryUploadsHostDir,
 } from './docker-compose-generator';
 import { explainBuildFailure } from './doctor';
 import { type ModuleHost, parseModuleHost } from './module-host';
@@ -1005,12 +1006,19 @@ function buildNetworkHandle(
       }
 
       try {
-        // Copy directly into the registry container's /uploads dir.
-        // The registry rebuilds its index on every request, so no HTTP handshake needed.
-        run(
-          `docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} cp ${JSON.stringify(netappPath)} registry:/uploads/${moduleId}.netapp`,
-          { cwd: composeDir, timeout: 120_000 },
-        );
+        // Write HOST-side into the dir bound at the registry's /uploads. The
+        // registry rescans that dir on every request, so it needs no HTTP
+        // handshake and no restart.
+        //
+        // Deliberately not a `docker compose cp` into the container: /uploads
+        // is a READ-ONLY bind in a consumer install, so the container write
+        // failed for every npm-installed consumer while passing forever in the
+        // monorepo, where the same path is not a mount at all (celilo#1142).
+        // registryUploadsHostDir() is the same function the compose generator
+        // builds the mount from, so the two cannot drift apart again.
+        const uploadsDir = registryUploadsHostDir();
+        mkdirSync(uploadsDir, { recursive: true });
+        copyFileSync(netappPath, join(uploadsDir, `${moduleId}.netapp`));
       } finally {
         if (cleanup)
           try {
@@ -1264,8 +1272,14 @@ export async function startNetwork(config: NetworkConfig): Promise<NetworkHandle
     );
   }
 
-  // Wait for DHCP client lease
-  if (config.dhcpClient) {
+  // Wait for DHCP client lease.
+  //
+  // Skipped when the router's DHCP is off, because then NOTHING is serving yet:
+  // celilo's own DHCP server is a module, and modules deploy long after the
+  // network is up. Waiting here would time out at 60s on every such suite and
+  // report it as a network-start failure. Those suites drive the client
+  // themselves once their server is deployed.
+  if (config.dhcpClient && config.routerDhcp !== false) {
     console.log('[progress:start] waiting for DHCP client lease | DHCP lease acquired');
     // Self-diagnosing via waitFor's onTimeout (e2e-confidence #255): a DHCP-lease
     // timeout is an intermittent in-suite flake (passes solo). On timeout, dump

package/src/docker-compose-generator.ts

@@ -1,5 +1,5 @@
-import { existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
+import { basename, join } from 'node:path';
 import { stringify } from 'yaml';
 import { normalizeObservers, observerEnv, observerPlacement } from './observer';
 import { PROXMOX_SIM_IP, SIMULATOR_IPS } from './simulator-ips';
@@ -13,32 +13,76 @@ import {
   zoneIp,
 } from './types';
 
+const PACKAGE_DIR = join(import.meta.dir, '..');
+
+/**
+ * Is this @celilo/e2e running from the monorepo, or from an npm install?
+ *
+ * `<pkgDir>/../../modules` is `<repo>/modules` in the monorepo and
+ * `<consumer>/node_modules/modules` (nonexistent) in a consumer install.
+ */
+function isMonorepoInstall(): boolean {
+  return existsSync(join(PACKAGE_DIR, '..', '..', 'modules'));
+}
+
 /**
- * Volumes for the registry container. Picks between source-bootstrap and
- * uploads-bootstrap based on where the @celilo/e2e package lives:
+ * The host dir bound read-only at the registry's `/uploads` — and therefore
+ * the dir `publishModule` writes a `.netapp` into. The registry rescans it on
+ * every request, so a host-side write is picked up with no container write and
+ * no HTTP handshake.
+ *
+ * Both install shapes bind one. They differ only in WHICH, because they differ
+ * in where standard modules come from:
+ *
+ * - **Monorepo dev** — modules are served live from `/modules`
+ *   (`../../modules`, BOOTSTRAP_MODULES_DIR mode), so edits to a module's
+ *   source flow through without a manual repack. `/uploads` is therefore a
+ *   dedicated drop-zone, emptied on every shared-infra start (see
+ *   `prepareRegistryDropZone`) — a leftover `.netapp` would shadow live source.
  *
- * - **Monorepo dev** — `../../modules` from the compose file (which lives
- *   in `packages/e2e/`) resolves to `<repo>/modules`. The registry runs
- *   in BOOTSTRAP_MODULES_DIR mode and packs source modules on the fly,
- *   so edits to a module's source flow through without manual repack.
+ * - **npm-installed consumer** (lunacycle and friends) — `../../modules` does
+ *   not exist, so there is no source to serve. `/uploads` IS the netapp cache
+ *   `cele2e build-infra` fills from the public registry (see
+ *   stageNetappsFromRegistry in cli/build.ts), and a published module lands
+ *   beside the standard ones. Consumers vendor nothing.
+ *
+ * Read this through `registryUploadsHostDir()` rather than re-deriving it.
+ * publishModule used to `docker cp` into the container's `/uploads` instead,
+ * which works only where that path is NOT a mount — so it passed forever in
+ * the monorepo and failed for every consumer against the read-only bind
+ * (celilo#1142). One function now decides the mount and the write target
+ * together, so they cannot disagree again.
+ */
+export function registryUploadsHostDir(): string {
+  return join(PACKAGE_DIR, isMonorepoInstall() ? 'uploads' : 'netapps');
+}
+
+/**
+ * Make the drop-zone ready for a shared-infra bring-up.
  *
- * - **npm-installed consumer** (lunacycle and friends) — `../../modules`
- *   would resolve to `<consumer>/node_modules/modules`, which doesn't
- *   exist. Docker fails the mount with "permission denied" trying to
- *   create that path. Instead we bind the local `./netapps/` dir to
- *   `/uploads`, and the registry's scanUploadsDir picks up the .netapp
- *   files. That dir is populated at `cele2e build-infra` time by fetching
- *   the standard-module netapps from the public registry (see
- *   stageNetappsFromRegistry in cli/build.ts) — consumers vendor nothing.
+ * Creates it: Docker silently creates a MISSING bind source as an empty
+ * root-owned directory, which the host then cannot write into.
  *
- * Detection: check whether the would-be monorepo modules dir is real.
- * `import.meta.dir` is `<pkgDir>/src/`, so `../../../modules` from there
- * is `<repo>/modules` in the monorepo and `<consumer>/node_modules/modules`
- * (nonexistent) in a consumer install.
+ * Empties it, but only in the monorepo, where `/uploads` is dedicated — a
+ * `.netapp` left by a previous run would shadow the live module source it was
+ * packed from. In a consumer install the same dir IS the build-infra netapp
+ * cache, so clearing it would delete every standard module. (Before
+ * celilo#1142 published netapps lived in the registry's container layer and
+ * died with the container, so starting empty is the behaviour being kept, not
+ * a new one.)
  */
+export function prepareRegistryDropZone(): void {
+  const dir = registryUploadsHostDir();
+  mkdirSync(dir, { recursive: true });
+  if (!isMonorepoInstall()) return;
+  for (const file of readdirSync(dir)) {
+    if (file.endsWith('.netapp')) rmSync(join(dir, file), { force: true });
+  }
+}
+
 function getRegistryVolumes(): string[] {
-  const monorepoModules = join(import.meta.dir, '..', '..', '..', 'modules');
-  return existsSync(monorepoModules) ? ['../../modules:/modules:ro'] : ['./netapps:/uploads:ro'];
+  const uploads = `./${basename(registryUploadsHostDir())}:/uploads:ro`;
+  return isMonorepoInstall() ? ['../../modules:/modules:ro', uploads] : [uploads];
 }
 
 const NETWORK_DRIVER_OPTS = {
@@ -94,6 +138,12 @@ function baseService(opts: {
   environment?: Record<string, string>;
   tmpfs?: string[];
   privileged?: boolean;
+  /**
+   * Docker `security_opt` entries, e.g. `seccomp=unconfined`. Needed by the
+   * management box so the hook jail's bubblewrap can build a namespace inside
+   * a container — see the measured ladder at its call site.
+   */
+  security_opt?: string[];
   dns?: string[];
   /** Host devices to expose, e.g. `/dev/net/tun` for a userspace WireGuard tunnel. */
   devices?: string[];
@@ -215,7 +265,13 @@ function defaultTopologyServices(config: NetworkConfig): Record<string, unknown>
       volumes: ['ssh-keys:/ssh-keys:ro'],
       // Which real ISP router this sim stands in for. The protocol is identical
       // across the two; only the TR-181 vendor extension prefix differs.
-      environment: { ROUTER_VENDOR_PREFIX: config.routerVendorPrefix ?? 'X_GWS_' },
+      environment: {
+        ROUTER_VENDOR_PREFIX: config.routerVendorPrefix ?? 'X_GWS_',
+        // The simulated router serves DHCP on `internal` by default, which is
+        // what an operator's router does. A suite testing celilo's OWN DHCP
+        // turns it off, because two servers on one broadcast domain race.
+        ROUTER_DHCP: config.routerDhcp === false ? 'off' : 'on',
+      },
     }),
   };
 }
@@ -568,7 +624,34 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
       image:
         variant === 'vanilla' ? 'celilo-e2e/management:vanilla' : 'celilo-e2e/management:latest',
       networks: { [zone]: { ipv4_address: ip } },
-      cap_add: ['NET_ADMIN'],
+      // SYS_ADMIN is for the hook jail, NET_ADMIN for the routing the box does
+      // anyway. See the security_opt block below for why both are here.
+      cap_add: ['NET_ADMIN', 'SYS_ADMIN'],
+      // The hook jail runs each hook under bubblewrap, which has to build a
+      // user namespace and then mount inside it. Docker denies that four
+      // separate ways, and every one of these is load-bearing — measured on a
+      // colima Ubuntu 24.04.1 VM (kernel 6.8.0-50,
+      // kernel.apparmor_restrict_unprivileged_userns=1), by removing one at a
+      // time from the working set:
+      //
+      //   drop seccomp=unconfined      -> bwrap: pivot_root: Operation not permitted
+      //   drop apparmor=unconfined     -> bwrap: Failed to make / slave: Permission denied
+      //   drop systempaths=unconfined  -> bwrap: Can't mount proc on /newroot/proc
+      //   drop SYS_ADMIN               -> bwrap: setting up uid map: Permission denied
+      //
+      // Deliberately NOT `privileged: true`, which design D8 measured as
+      // working and task 4.1 asked to improve on. It would also grant every
+      // capability and every host device; this set grants four named things.
+      //
+      // `systempaths=unconfined` is the one D8's ladder was missing: it unmasks
+      // the /proc paths Docker hides, which is why "all three" of the earlier
+      // options still failed on the proc mount.
+      //
+      // NOTE this makes the container a WEAK place to test the AppArmor
+      // profile: apparmor=unconfined means the restriction the profile exists
+      // to satisfy is not in force here. That gate is the celilo-builder probe
+      // on a real kernel, never this container.
+      security_opt: ['seccomp=unconfined', 'apparmor=unconfined', 'systempaths=unconfined'],
       // ISS-0157: `module import`'s `bun install` hard-links packages from the
       // bun cache into node_modules, which on overlayfs forces a copy-up + fsync
       // per file. On a slow/contended builder disk that storm blows past the

package/src/network-builder.ts

@@ -140,6 +140,23 @@ export class NetworkBuilder {
     return this;
   }
 
+  /**
+   * Stop the simulated ISP router serving DHCP on `internal`.
+   *
+   * For suites where CELILO serves DHCP (`modules/dnsmasq-dhcp`). Two servers
+   * on one broadcast domain race, so without this a lease assertion is a coin
+   * toss. It is also what an operator does when moving DHCP to celilo, so the
+   * suite models the migration rather than dodging a conflict.
+   *
+   * NOTE: `withDhcpClient()`'s start-up lease wait expects SOMETHING to answer.
+   * Pair this with a suite that deploys celilo's DHCP server, or the client
+   * gets no lease and the network never comes up.
+   */
+  withoutRouterDhcp(): this {
+    this.config.routerDhcp = false;
+    return this;
+  }
+
   /**
    * Add the Proxmox API simulator on `secure-mgmt`.
    *

package/src/shared-infra.ts

@@ -17,6 +17,7 @@ import {
   SHARED_NETWORKS,
   SHARED_PROJECT_NAME,
   generateSharedInfraYaml,
+  prepareRegistryDropZone,
 } from './docker-compose-generator';
 import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
 
@@ -161,6 +162,11 @@ export async function ensureSharedInfra(): Promise<void> {
   const yaml = generateSharedInfraYaml();
   writeFileSync(join(e2eDir, SHARED_COMPOSE_FILE), yaml);
 
+  // The registry's /uploads bind source must exist before `up` (a missing
+  // bind source becomes an empty root-owned dir), and in the monorepo it
+  // starts empty so no stale .netapp shadows live module source.
+  prepareRegistryDropZone();
+
   // Seed the live DNS zone files from their templates BEFORE the compose
   // mounts them. config/dns/{iamtheinternet.org,example.net}.zone are
   // gitignored runtime state (scrubDnsZones rewrites them per-test); on a

package/src/types.ts

@@ -66,6 +66,19 @@ export interface NetworkConfig {
    * serves both rather than a near-duplicate second one.
    */
   routerVendorPrefix?: string;
+  /**
+   * Whether the simulated ISP router serves DHCP on `internal`. Defaults to
+   * true, which is what an operator's router does and what every existing suite
+   * relies on.
+   *
+   * Set false when celilo serves DHCP instead (`modules/dnsmasq-dhcp`). Two
+   * DHCP servers on one broadcast domain race — whichever answers a DISCOVER
+   * first wins — so a suite asserting WHICH server issued a lease would be
+   * asserting a coin toss. Turning the router's off is also what a real
+   * operator does when moving DHCP to celilo, so this models the migration
+   * rather than papering over a conflict.
+   */
+  routerDhcp?: boolean;
   /** Extra volume mounts for the management container (host:container format) */
   managementVolumes: string[];
   /** Include a DHCP client container on the internal network */