@celilo/e2e 0.19.3 → 0.20.0

package/src/cli/build.ts

@@ -19,13 +19,25 @@
  */
 
 import { execSync, spawnSync } from 'node:child_process';
-import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { basename, join, resolve } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { stageAptRepo } from '../../scripts/stage-apt-repo';
 import { stageLibsignal } from '../../scripts/stage-libsignal';
 import { explainBuildFailure } from '../doctor';
 import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../registry-bundle';
+import { findMonorepoRoot } from '../repo-root';
+import { packNpmRegistryTarballs, stageWebsiteDist } from '../stage-simulator-inputs';
 
 /**
  * Ceiling on a single `docker build`. The slowest image here is a few minutes
@@ -35,27 +47,15 @@ import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../regist
 const PER_IMAGE_BUILD_TIMEOUT_MS = 15 * 60_000;
 
 /**
- * Walk up from pkgDir to find the celilo monorepo root.
- *
- * The marker is `apps/celilo/package.json` — a file that only a real celilo
- * checkout has — NOT the presence of a `modules/` directory. A bare `modules/`
- * is too weak a signal: a stray empty `node_modules/modules` (left by a botched
- * `bun add -g`) made this return the global node_modules as "the monorepo root",
- * which skipped `stageFromPublic()` and crashed the npm-consumer path in the
- * workspace packer (ce-dbc). Matches the marker `container-manager.ts`'s
- * findCeliloRoot already uses. Returns null in any consumer install, so callers
- * correctly fall through to the published-source path.
+ * Ceiling on a single `celilo module package` run. The slowest module takes
+ * a couple of minutes; 10 leaves generous headroom. Without it a wedged
+ * packaging step sat for two hours (2026-09-04, technitium: all samples in
+ * kevent64, artifact truncated at 59 percent, no error, no exit).
  */
-export function findMonorepoRoot(pkgDir: string): string | null {
-  let dir = pkgDir;
-  for (let i = 0; i < 10; i++) {
-    if (existsSync(join(dir, 'apps', 'celilo', 'package.json'))) return dir;
-    const parent = resolve(dir, '..');
-    if (parent === dir) break;
-    dir = parent;
-  }
-  return null;
-}
+const PER_MODULE_PACKAGE_TIMEOUT_MS = 10 * 60_000;
+
+/** Re-exported so this module's existing callers and its recurrence test keep their import path. */
+export { findMonorepoRoot };
 
 /** Return paths to all modules that have a manifest.yml. */
 function discoverModuleDirs(repoRoot: string): string[] {
@@ -72,6 +72,84 @@ const red = '\x1b[31m';
 const yellow = '\x1b[33m';
 const reset = '\x1b[0m';
 
+/**
+ * Validate that a file is a complete gzip stream (full gunzip pass). This is
+ * the check a .netapp or .tgz needs: a stream truncated mid-write fails here
+ * with 'unexpected end of file' instead of surviving to the consumer, where
+ * the identical zlib error reads as a truncated DOWNLOAD and sends the
+ * debugging at the network while the artifact was born broken.
+ *
+ * Throws with an actionable message naming the file.
+ */
+export function assertGzipValid(filePath: string): void {
+  try {
+    gunzipSync(readFileSync(filePath));
+    return;
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `${filePath} is not a valid gzip stream (${reason}). The artifact was written truncated or corrupted at rest; serving it would surface as 'zlib: unexpected end of file' on the consumer side, indistinguishable from a truncated download.`,
+    );
+  }
+}
+
+/**
+ * Validate that a file is a real .deb (ar archive). A full dpkg parse is out
+ * of scope; the magic bytes catch the truncated-at-write case, which is the
+ * one that silently ships.
+ */
+function assertDebValid(filePath: string): void {
+  const fd = openSync(filePath, 'r');
+  try {
+    const magic = Buffer.alloc(8);
+    const read = readSync(fd, magic, 0, 8, 0);
+    if (read < 8 || magic.toString('latin1') !== '!<arch>\n') {
+      throw new Error(
+        `${filePath} is not a valid ar archive (missing !<arch> magic). The .deb was written truncated or corrupted at rest.`,
+      );
+    }
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * Verify a produced or downloaded .netapp is a complete gzip stream. Throws
+ * with an actionable message; callers fail the build rather than stage the
+ * artifact for the sim registry to serve.
+ */
+export function verifyNetapp(filePath: string): void {
+  if (!existsSync(filePath)) {
+    throw new Error(`${filePath} was not produced — packaging step exited 0 but left no artifact`);
+  }
+  assertGzipValid(filePath);
+}
+
+/**
+ * Verify every .netapp in the staging dir before declaring the stage done.
+ * The sim registry binds this directory and serves whatever is in it, so a
+ * corrupt file here becomes a 'zlib: unexpected end of file' on some consumer
+ * that looks like a network fault. Catches stale artifacts from a previous
+ * run too — the dir is not cleaned between builds.
+ */
+export function verifyStagedNetapps(netappsDir: string): void {
+  if (!existsSync(netappsDir)) return;
+  const netapps = readdirSync(netappsDir).filter((f) => f.endsWith('.netapp'));
+  const failures: string[] = [];
+  for (const netapp of netapps) {
+    try {
+      verifyNetapp(join(netappsDir, netapp));
+    } catch (err) {
+      failures.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+  if (failures.length > 0) {
+    throw new Error(
+      `${failures.length} of ${netapps.length} staged .netapp file(s) failed gzip integrity check:\n  ${failures.join('\n  ')}`,
+    );
+  }
+}
+
 interface BuildOptions {
   pkgDir: string;
   moduleDirs: string[];
@@ -87,19 +165,16 @@ interface BuildOptions {
 }
 
 /**
- * Stage two build-time inputs that the celilo-website-sim and
- * npm-registry-sim images COPY at docker-build time:
- *
- *   .celilo-website-cache/   ← modules/celilo-website/site/dist/
- *   .npm-registry-cache/     ← bun pm pack of each @celilo/* workspace pkg
+ * Stage the build-time inputs that the simulator images COPY at docker-build
+ * time: the website dist and the @celilo/* tarballs (via
+ * src/stage-simulator-inputs.ts), then the apt-repo and libsignal debs.
  *
- * Both are .gitignored — they're build outputs, not source. Without
- * this step, Dockerfile.celilo-website-sim and Dockerfile.npm-registry-sim
- * fail at COPY (or worse, ship stale tarballs that don't match the
- * current workspace versions, which is what the version-mismatch
- * regression on 2026-05-06 caught).
+ * Both cache dirs are .gitignored — they're build outputs, not source. Without
+ * this step, the simulator Dockerfiles fail at COPY (or worse, ship stale
+ * tarballs that don't match the current workspace versions, which is what the
+ * version-mismatch regression on 2026-05-06 caught).
  *
- * Pure no-op when the workspace doesn't have either cache target —
+ * Pure no-op when the workspace doesn't have the cache targets —
  * keeps build-infra working in npm-installed @celilo/e2e, where these
  * staging dirs aren't relevant.
  */
@@ -121,8 +196,6 @@ async function stageSimulatorInputs(pkgDir: string): Promise<void> {
   }
 
   const websiteSrc = join(repoRoot, 'modules', 'celilo-website', 'site');
-  const websiteCache = join(pkgDir, '.celilo-website-cache');
-  const npmCache = join(pkgDir, '.npm-registry-cache');
   const packScript = join(pkgDir, 'scripts', 'pack-celilo-packages.ts');
 
   // Skip silently if neither dir exists — we're in an npm-installed
@@ -131,61 +204,16 @@ async function stageSimulatorInputs(pkgDir: string): Promise<void> {
 
   console.log(`${bold}Staging simulator inputs...${reset}\n`);
 
-  // (a) Build the celilo-website static site and stage its dist/.
-  if (existsSync(websiteSrc)) {
-    process.stdout.write(`  ${'celilo-website (build)'.padEnd(28)} `);
-    const t0 = Date.now();
-    let result = spawnSync('bun', ['install'], { cwd: websiteSrc, stdio: 'pipe' });
-    if (result.status !== 0) {
-      console.log(`${red}✗${reset}`);
-      console.error(result.stderr?.toString());
-      process.exit(1);
-    }
-    result = spawnSync('bun', ['run', 'build'], { cwd: websiteSrc, stdio: 'pipe' });
-    if (result.status !== 0) {
-      console.log(`${red}✗${reset}`);
-      console.error(result.stderr?.toString());
-      process.exit(1);
-    }
-    console.log(`${green}✔${reset}  ${dim}${Math.round((Date.now() - t0) / 1000)}s${reset}`);
-
-    process.stdout.write(`  ${'celilo-website (stage)'.padEnd(28)} `);
-    rmSync(websiteCache, { recursive: true, force: true });
-    mkdirSync(websiteCache, { recursive: true });
-    cpSync(join(websiteSrc, 'dist'), websiteCache, { recursive: true });
-    console.log(`${green}✔${reset}`);
-  }
+  stageWebsiteDist(repoRoot, pkgDir);
+  packNpmRegistryTarballs(repoRoot, pkgDir);
 
-  // (b) Pack the @celilo/* workspace packages so they ship in
-  // npm-registry-sim. Tarball versions match current workspace
-  // package.json — re-runs every build-infra to stay fresh.
-  if (existsSync(packScript)) {
-    process.stdout.write(`  ${'npm-registry (pack)'.padEnd(28)} `);
-    const t0 = Date.now();
-    const result = spawnSync('bun', ['run', packScript], {
-      cwd: repoRoot,
-      stdio: 'pipe',
-    });
-    if (result.status !== 0) {
-      console.log(`${red}✗${reset}`);
-      console.error(result.stderr?.toString());
-      process.exit(1);
-    }
-    const tarballCount = existsSync(npmCache)
-      ? readdirSync(npmCache).filter((f) => f.endsWith('.tgz')).length
-      : 0;
-    console.log(
-      `${green}✔${reset}  ${dim}${Math.round((Date.now() - t0) / 1000)}s, ${tarballCount} tarball(s)${reset}`,
-    );
-  }
-
-  // (c) Build + stage the celilo/celilo-bootstrap .debs for apt-repo-sim.
+  // Build + stage the celilo/celilo-bootstrap .debs for apt-repo-sim.
   // Graceful no-op when packaging/ or nfpm is absent (npm-installed e2e).
   // Dockerfile.apt-repo-sim COPYs the staged .apt-repo-cache/pool at build
   // time, so this must run before buildDockerImages().
   stageAptRepo(repoRoot, pkgDir);
 
-  // (d) Build + stage the libsignal aarch64 native as a .deb, into the SAME
+  // Build + stage the libsignal aarch64 native as a .deb, into the SAME
   // pool stageAptRepo just populated (so it must run after it — that function
   // recreates the pool empty). Compiled here on the host's own network rather
   // than inside the sealed e2e network; see stage-libsignal.ts.
@@ -323,7 +351,9 @@ export async function fetchAptPool(base: string, pool: string): Promise<number>
     const url = `${base}/${filename}`;
     const res = await fetch(url);
     if (!res.ok) throw new Error(`fetching ${url}: HTTP ${res.status}`);
-    writeFileSync(join(pool, basename(filename)), Buffer.from(await res.arrayBuffer()));
+    const debPath = join(pool, basename(filename));
+    writeFileSync(debPath, Buffer.from(await res.arrayBuffer()));
+    assertDebValid(debPath);
   }
   return filenames.size;
 }
@@ -370,7 +400,12 @@ export async function fetchPublishedTarball(
   }
   const filename =
     tarballUrl.split('/').pop() ?? `${pkg.replace('@', '').replace('/', '-')}-${latest}.tgz`;
-  writeFileSync(join(destDir, filename), Buffer.from(await tgzRes.arrayBuffer()));
+  const tgzPath = join(destDir, filename);
+  writeFileSync(tgzPath, Buffer.from(await tgzRes.arrayBuffer()));
+  // A truncated tarball in .npm-registry-cache fails the consumer's bun add
+  // with a zlib error that reads like a network fault. Check here where the
+  // cause (a bad fetch) is still visible.
+  assertGzipValid(tgzPath);
 }
 
 /**
@@ -418,10 +453,14 @@ function packageNetapp(moduleDir: string, netappsDir: string): void {
 
   let result: ReturnType<typeof spawnSync>;
   if (celiloWrapper && existsSync(celiloWrapper)) {
-    result = spawnSync(celiloWrapper, ['package', absDir, '--output', out], { stdio: 'pipe' });
+    result = spawnSync(celiloWrapper, ['package', absDir, '--output', out], {
+      stdio: 'pipe',
+      timeout: PER_MODULE_PACKAGE_TIMEOUT_MS,
+    });
   } else if (celiloTs && existsSync(celiloTs)) {
     result = spawnSync('bun', ['run', celiloTs, 'package', absDir, '--output', out], {
       stdio: 'pipe',
+      timeout: PER_MODULE_PACKAGE_TIMEOUT_MS,
     });
   } else {
     console.error(`  ${red}skip${reset}  ${name}  ${dim}(celilo CLI not found)${reset}`);
@@ -429,9 +468,32 @@ function packageNetapp(moduleDir: string, netappsDir: string): void {
   }
 
   const elapsed = Math.round((Date.now() - start) / 1000);
+  if (result.error && (result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') {
+    console.log(`${red}✗ TIMED OUT${reset}  ${dim}${elapsed}s${reset}`);
+    console.error(
+      `\n${red}Packaging ${name} exceeded ${Math.round(PER_MODULE_PACKAGE_TIMEOUT_MS / 60_000)} minutes and was killed.${reset}`,
+    );
+    console.error(
+      `${dim}The stall is usually transient (2026-09-04: technitium wedged two hours, artifact truncated mid-write). Delete ${out} and re-run \`cele2e build-infra\`.${reset}`,
+    );
+    process.exit(1);
+  }
   if (result.status !== 0) {
     console.log(`${red}✗${reset}  ${dim}${result.stderr?.toString().trim()}${reset}`);
-    return;
+    // A failed package must fail the build, not silently shrink the module
+    // set: the sim registry binds netapps/ as-is, so the omission surfaces
+    // later as a confusing deploy failure instead of a nameable one.
+    process.exit(1);
+  }
+
+  // Verify what we just wrote before declaring the step done. A wedge in the
+  // packaging stream leaves a truncated file that still exits 0 (celilo#1257).
+  try {
+    verifyNetapp(out);
+  } catch (err) {
+    console.log(`${red}✗ CORRUPT${reset}  ${dim}${elapsed}s${reset}`);
+    console.error(`\n${red}${err instanceof Error ? err.message : String(err)}${reset}`);
+    process.exit(1);
   }
 
   const size = (() => {
@@ -489,7 +551,15 @@ export async function stageNetappsFromRegistry(netappsDir: string): Promise<void
       process.exit(1);
     }
     const bytes = new Uint8Array(await resp.arrayBuffer());
-    writeFileSync(join(netappsDir, `${name}.netapp`), bytes);
+    const netappPath = join(netappsDir, `${name}.netapp`);
+    writeFileSync(netappPath, bytes);
+    try {
+      verifyNetapp(netappPath);
+    } catch (err) {
+      console.log(`${red}✗ CORRUPT${reset}`);
+      console.error(`${red}  ${err instanceof Error ? err.message : String(err)}${reset}`);
+      process.exit(1);
+    }
     console.log(
       `${green}✔${reset}  ${dim}${max_version}  ${Math.round((Date.now() - t0) / 1000)}s  ${(bytes.length / 1024).toFixed(0)}KB${reset}`,
     );
@@ -705,6 +775,34 @@ function saveImages(pkgDir: string): void {
   console.log('\nTo restore after colima restart:\n  cele2e load');
 }
 
+/**
+ * Collect the untagged images this build (and every build before it) left
+ * behind.
+ *
+ * `docker image prune -f` — no `-a`, no `system prune` — removes only images
+ * that carry no tag AND no container. The tagged base images every Dockerfile
+ * starts `FROM` are therefore untouched, which is the whole distinction: the
+ * warning in this repo's operating notes is about `-a` and `system prune`,
+ * which DO delete `ubuntu:22.04` and cost a full 27-image rebuild that then
+ * gets misread as a network failure (`failed to solve: … TLS handshake
+ * timeout`). The bare form is the opposite — it is what stops the pile that
+ * drives people to reach for the destructive one.
+ *
+ * Measured 2026-09-05 before this existed: 3178 dangling images, 25.8 GB, 97%
+ * of the local image store, mostly management images superseded by a bake.
+ */
+function pruneSupersededImages(): void {
+  process.stdout.write(`${bold}Removing superseded (untagged) images...${reset} `);
+  try {
+    const out = execSync('docker image prune -f', { encoding: 'utf-8', timeout: 300_000 });
+    const reclaimed = out.match(/Total reclaimed space:\s*(.+)/)?.[1]?.trim() ?? '0B';
+    console.log(`${green}✔${reset} ${dim}${reclaimed} reclaimed${reset}`);
+  } catch {
+    // Never fail a build over housekeeping.
+    console.log(`${yellow}⚠ skipped${reset}`);
+  }
+}
+
 export async function runBuild(options: BuildOptions): Promise<void> {
   const { pkgDir, moduleDirs, save, skipModules, published } = options;
   const netappsDir = join(pkgDir, 'netapps');
@@ -745,6 +843,21 @@ export async function runBuild(options: BuildOptions): Promise<void> {
     }
   }
 
+  // Final gate before anything serves these files: the sim registry binds
+  // netapps/ verbatim, so every artifact in it (fresh or stale from a prior
+  // run) must be a complete gzip stream. A truncated .netapp served to a
+  // consumer reports 'zlib: unexpected end of file', byte-identical to a
+  // truncated download, which sends the debugging at the network (celilo#1257).
+  try {
+    verifyStagedNetapps(netappsDir);
+  } catch (err) {
+    console.error(`\n${red}${err instanceof Error ? err.message : String(err)}${reset}`);
+    console.error(
+      `${dim}Delete the corrupt file(s) and re-run \`cele2e build-infra\`. If packaging reproduced the corruption, the stall was likely transient — re-run before deeper debugging.${reset}`,
+    );
+    process.exit(1);
+  }
+
   console.log(`${bold}Building E2E Docker images...${reset}\n`);
   buildDockerImages(pkgDir);
 
@@ -768,4 +881,7 @@ export async function runBuild(options: BuildOptions): Promise<void> {
   if (save) {
     saveImages(pkgDir);
   }
+
+  // Last, so it also collects the image the bake just superseded.
+  pruneSupersededImages();
 }

package/src/cli/command-registry.ts

@@ -46,6 +46,11 @@ export const COMMANDS: CommandDef[] = [
       { name: '--live', description: 'Use live (non-simulated) internet' },
       { name: '--published', description: 'Use published .netapp packages' },
       { name: '--notify', description: 'Desktop notification when the run finishes' },
+      {
+        name: '--source-cli',
+        description:
+          'Run celilo from the mounted workspace instead of the baked image (roughly doubles each command start-up)',
+      },
       {
         name: '--shuffle',
         description: 'Randomize test order to surface order-dependent bugs (seed is logged)',
@@ -105,6 +110,19 @@ export const COMMANDS: CommandDef[] = [
       "Check everything a run needs (baked images, base images, lock, disk) — run's implicit preflight",
     flags: [{ name: '--json', description: 'Emit the full report as JSON' }],
   },
+  {
+    name: 'host',
+    description: 'Bring the Docker host VM up/down in the shape the rig needs',
+    subcommands: [
+      { name: 'status', description: 'Report the VM against the policy (exit 1 if out of policy)' },
+      { name: 'up', description: 'Start it, or say why a running one is out of policy' },
+      { name: 'down', description: 'Stop it (refuses while the e2e run-lock is held)' },
+      {
+        name: 'reset',
+        description: 'Recreate it — the only way to change mount type. DESTROYS every image',
+      },
+    ],
+  },
   {
     name: 'last',
     description: "Show the most recent run's results dir and counts",
@@ -153,8 +171,11 @@ export const RUN_FLAGS = [
   '--live',
   '--published',
   '--notify',
+  '--source-cli',
 ];
 
+export const HOST_VERBS = ['status', 'up', 'down', 'reset'];
+
 export const DOWN_FLAGS = ['--keep', '--all'];
 
 /** Read-only reporters whose only flag is `--json`. */

package/src/cli/command-tree-parser.ts

@@ -16,7 +16,7 @@ function runHelp(args = ''): string {
   }
 }
 
-function parseCommandsFromHelp(text: string): string[] {
+export function parseCommandsFromHelp(text: string): string[] {
   const commands: string[] = [];
   let inCommandsSection = false;
 
@@ -25,7 +25,16 @@ function parseCommandsFromHelp(text: string): string[] {
       inCommandsSection = true;
       continue;
     }
-    if (inCommandsSection && /^\s*(Options|Examples|Usage):/i.test(line)) {
+    // Any UNINDENTED heading ends the list. This used to name three headings
+    // literally (`Options:`, `Examples:`, `Usage:`), none of which the help
+    // actually prints — it writes `Options for \`run\`:` — so the section never
+    // closed and every 2-space-indented word after it was read as a command.
+    // Nothing showed that, because option lines start with `-` and the only
+    // other candidate block happened to sit after `Examples:`. It surfaced the
+    // day a command grew a verb list: `reset` was reported as a top-level
+    // command that the registry was missing. Keying on the help's real
+    // structure fixes the class rather than that instance.
+    if (inCommandsSection && /^\S.*:\s*$/.test(line)) {
       inCommandsSection = false;
       continue;
     }

package/src/cli/completion.ts

@@ -7,6 +7,7 @@ import {
   BUILD_INFRA_FLAGS,
   COMMANDS,
   DOWN_FLAGS,
+  HOST_VERBS,
   JSON_FLAG_COMMANDS,
   RUN_FLAGS,
   UP_PRESETS,
@@ -65,6 +66,14 @@ export function getCompletions(words: string[], current: number): string[] {
     );
   }
 
+  if (command === 'host' && currentIndex === 1) {
+    return filterSuggestions(HOST_VERBS, args[1] || '');
+  }
+
+  if (command === 'host' && args[1] === 'reset' && currentIndex === 2) {
+    return filterSuggestions(['--yes'], args[2] || '');
+  }
+
   if (command === 'completion' && currentIndex === 1) {
     return filterSuggestions(['zsh', 'bash'], args[1] || '');
   }