@celilo/e2e 0.5.1 → 0.6.0

package/src/cli/index.ts

@@ -0,0 +1,203 @@
+#!/usr/bin/env bun
+/**
+ * cele2e — E2E test CLI for Celilo-deployed applications.
+ *
+ * Usage:
+ *   cele2e run [pattern] [--keep] [--reuse] [--live] [--published]
+ *   cele2e up [--caddy|--full-stack|--infrastructure|--custom '<json>']
+ *   cele2e down [--keep] [--all]
+ *   cele2e build-infra [module-dir...] [--save]
+ *   cele2e clear-timing
+ *   cele2e version
+ *   cele2e shell [container]
+ *   cele2e status
+ *   cele2e load
+ *   cele2e scaffold <test-name>
+ */
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { runBuild } from './build';
+import { runScaffold } from './scaffold';
+import { generateBashCompletion, generateZshCompletion, getCompletions } from './completion';
+
+const PKG_DIR = resolve(import.meta.dir, '../..');
+const BIN_DIR = join(PKG_DIR, 'bin');
+
+function detectTestDir(): string {
+  if (process.env.E2E_TEST_DIR) return process.env.E2E_TEST_DIR;
+  const cwd = process.cwd();
+  // Check e2e/tests/ first — more specific than a generic tests/ directory
+  if (existsSync(join(cwd, 'e2e', 'tests'))) return join(cwd, 'e2e');
+  if (existsSync(join(cwd, 'tests'))) return cwd;
+  return cwd;
+}
+
+function runScript(script: string, args: string[], env?: Record<string, string>): never {
+  const result = spawnSync(join(BIN_DIR, script), args, {
+    stdio: 'inherit',
+    env: { ...process.env, ...env },
+  });
+  process.exit(result.status ?? 1);
+}
+
+function printHelp(): void {
+  console.log(`
+cele2e — E2E test CLI for Celilo-deployed applications
+
+Usage:
+  cele2e <command> [options]
+
+Commands:
+  run [pattern]     Run e2e tests (pattern filters by test file name)
+  up [preset]       Start interactive test network
+  down              Tear down running network
+  build-infra [dirs...] Rebuild Docker images and package all modules as .netapps
+  clear-timing      Clear saved timing history used for test ETAs
+  version           Show cele2e version
+  shell [container] Shell into a running container (default: management)
+  status            Show network status and DNS health
+  load              Load cached Docker images from tarball
+  scaffold <name>   Generate a new test file from template
+  completion <shell> Generate shell completion script (zsh|bash)
+
+Options for \`run\`:
+  --keep            Keep network running after tests
+  --reuse           Reuse existing network if running
+  --live            Use live (non-simulated) internet
+  --published       Use published .netapp packages
+
+Options for \`up\`:
+  --caddy           Start with caddy machine in DMZ
+  --full-stack      Start with caddy, IDP, and DB machines
+  --custom <json>   Custom machine spec JSON
+
+Options for \`down\`:
+  --keep            Stop containers but preserve volumes
+  --all             Also stop competing module containers
+
+Options for \`build-infra\`:
+  --save            Save Docker images to tarball after build
+  --skip-modules    Skip .netapp packaging (only rebuild Docker images)
+
+Examples:
+  cele2e run                        # run all tests
+  cele2e run deploy                 # run tests matching "deploy"
+  cele2e run --reuse deploy         # reuse existing network
+  cele2e up --full-stack            # interactive full-stack environment
+  cele2e shell caddy                # debug running caddy container
+  cele2e build-infra                # rebuild Docker images + repackage all modules
+  cele2e build-infra --skip-modules # rebuild Docker images only (faster)
+  cele2e scaffold lunacycle-auth    # create tests/lunacycle-auth.test.ts
+  cele2e completion zsh             # print zsh completion script
+`);
+}
+
+const [, , command, ...args] = process.argv;
+const testDir = detectTestDir();
+
+// Handle --get-completions before routing (used by shell completion scripts)
+if (command === '--get-completions') {
+  const words = args.slice(0, -1);
+  const current = Number.parseInt(args[args.length - 1] || '0', 10);
+  const suggestions = getCompletions(words, current);
+  process.stdout.write(`${suggestions.join('\n')}\n`);
+  process.exit(0);
+}
+
+switch (command) {
+  case 'run':
+    runScript('e2e-run', args, { E2E_TEST_DIR: testDir });
+
+  case 'up':
+    runScript('e2e-up', args);
+
+  case 'down':
+    runScript('e2e-down', args, { E2E_TEST_DIR: testDir });
+
+  case 'shell':
+    runScript('e2e-shell', args);
+
+  case 'status': {
+    const persistentFile = join(testDir, '.e2e-persistent.json');
+    let projectName: string | undefined;
+    if (existsSync(persistentFile)) {
+      try {
+        const data = JSON.parse(readFileSync(persistentFile, 'utf-8'));
+        projectName = data.projectName;
+        console.log('=== Persistent Test Network ===');
+        console.log(`  Project:  ${data.projectName}`);
+        console.log(`  Test:     ${data.testFile}`);
+        console.log(`  Created:  ${data.createdAt}`);
+        console.log(`  Reuse:    cele2e run --reuse`);
+        console.log(`  Tear down: cele2e down`);
+        console.log('');
+      } catch { /* ignore parse errors */ }
+    }
+    runScript('e2e-status', args, projectName ? { CELILO_E2E_PROJECT: projectName } : undefined);
+  }
+
+  case 'load':
+    runScript('e2e-load', args);
+
+  case 'build-infra': {
+    const save = args.includes('--save');
+    const skipModules = args.includes('--skip-modules');
+    const moduleDirs = args.filter((a) => !a.startsWith('--'));
+    runBuild({ pkgDir: PKG_DIR, moduleDirs, save, skipModules });
+    break;
+  }
+
+  case 'clear-timing': {
+    const timingFile = join(testDir, '.e2e-timing.json');
+    if (existsSync(timingFile)) {
+      rmSync(timingFile);
+      console.log(`Cleared timing data: ${timingFile}`);
+    } else {
+      console.log(`No timing data found at ${timingFile}`);
+    }
+    break;
+  }
+
+  case 'scaffold': {
+    const name = args[0];
+    if (!name) {
+      console.error('Error: scaffold requires a test name\n  cele2e scaffold <name>');
+      process.exit(1);
+    }
+    runScaffold({ testDir, name });
+    break;
+  }
+
+  case 'version': {
+    const pkg = await import('../../package.json', { with: { type: 'json' } });
+    console.log(`cele2e v${pkg.default.version}`);
+    break;
+  }
+
+  case 'completion': {
+    const shell = args[0];
+    if (shell === 'zsh') {
+      process.stdout.write(generateZshCompletion());
+    } else if (shell === 'bash') {
+      process.stdout.write(generateBashCompletion());
+    } else {
+      console.error('Usage: cele2e completion <zsh|bash>');
+      process.exit(1);
+    }
+    break;
+  }
+
+  case undefined:
+  case '--help':
+  case '-h':
+  case 'help':
+    printHelp();
+    break;
+
+  default:
+    console.error(`Unknown command: ${command}\n`);
+    printHelp();
+    process.exit(1);
+}

package/src/cli/scaffold.ts

@@ -0,0 +1,109 @@
+/**
+ * cele2e scaffold — generate a new e2e test file from template.
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const green = '\x1b[32m';
+const dim = '\x1b[2m';
+const reset = '\x1b[0m';
+
+interface ScaffoldOptions {
+  testDir: string;
+  name: string;
+}
+
+function toTitleCase(kebab: string): string {
+  return kebab
+    .split('-')
+    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+    .join(' ');
+}
+
+function generateTestTemplate(name: string): string {
+  const title = toTitleCase(name);
+  return `/**
+ * ${title} E2E Test
+ *
+ * Tests that ${title} deploys and operates correctly through Celilo's infrastructure.
+ *
+ * Run: cele2e run ${name}
+ * Run (reuse): cele2e run --reuse ${name}
+ */
+
+import { afterAll, describe, expect, test } from 'bun:test';
+import { network, progress, reconnectNetwork } from '@celilo/e2e';
+import type { NetworkHandle } from '@celilo/e2e/types';
+
+const REUSE = process.argv.includes('--reuse');
+const PROJECT_NAME = 'celilo-e2e-${name}';
+
+describe('${name}', () => {
+  let net: NetworkHandle;
+  let stageError: string | null = null;
+
+  function requireStage(stage: string) {
+    if (stageError) {
+      throw new Error(\`Skipped: \${stage} — \${stageError}\`);
+    }
+  }
+
+  afterAll(async () => {
+    await net?.stop();
+  });
+
+  test('start network', async () => {
+    if (REUSE) {
+      net = await reconnectNetwork(PROJECT_NAME);
+      return;
+    }
+
+    net = await network({ projectName: PROJECT_NAME })
+      .dmz({ 'machine-1': '10.0.10.10' })
+      // .app({ 'app-1': '10.0.20.10' })
+      .start();
+  }, 120_000);
+
+  test('deploy', async () => {
+    requireStage('deploy');
+    try {
+      progress('importing module', 'module imported');
+      await net.celilo('module import /path/to/module.netapp');
+      progress.done();
+
+      progress('deploying', 'deployed');
+      await net.celilo('module deploy my-module');
+      progress.done();
+    } catch (error) {
+      stageError = String(error);
+      throw error;
+    }
+  }, 300_000);
+
+  test('verify', async () => {
+    requireStage('verify');
+    // TODO: add verification steps
+    // const result = await net.exec('machine-1', 'curl -s http://localhost');
+    // expect(result.exitCode).toBe(0);
+  }, 60_000);
+});
+`;
+}
+
+export function runScaffold(options: ScaffoldOptions): void {
+  const { testDir, name } = options;
+
+  const testsDir = join(testDir, 'tests');
+  mkdirSync(testsDir, { recursive: true });
+
+  const outPath = join(testsDir, `${name}.test.ts`);
+  if (existsSync(outPath)) {
+    console.error(`Error: ${outPath} already exists`);
+    process.exit(1);
+  }
+
+  writeFileSync(outPath, generateTestTemplate(name));
+  console.log(`${green}✔${reset} Created ${outPath}`);
+  console.log(`\n${dim}Run it with:\n  cele2e run ${name}${reset}`);
+}

package/src/container-manager.ts

@@ -14,11 +14,21 @@ const PACKAGE_ROOT = join(__dirname, '..');
  * Looks for apps/celilo/ as the marker.
  */
 /**
- * Try to find the Celilo project root by walking up from the package directory.
- * Returns undefined if not found — the CLI is pre-installed in Docker so this
- * is only needed for development (mounting local source).
+ * Try to find the Celilo project root.
+ *
+ * Resolution order:
+ * 1. CELILO_E2E_INFRA_ROOT env var — explicit path (dev) or empty string
+ *    (force published mode). Empty string means "do not mount source".
+ * 2. Walk up from the package directory looking for apps/celilo/. Works
+ *    automatically when @celilo/e2e is used directly inside the infra repo.
+ * 3. undefined — falls back to the baked-in npm package inside Docker.
  */
 function findCeliloRoot(): string | undefined {
+  const envRoot = process.env.CELILO_E2E_INFRA_ROOT;
+  if (envRoot !== undefined) {
+    return envRoot === '' ? undefined : envRoot;
+  }
+
   let dir = PACKAGE_ROOT;
   for (let i = 0; i < 10; i++) {
     if (existsSync(join(dir, 'apps', 'celilo'))) {
@@ -108,11 +118,11 @@ function dockerExecAsync(
   timeoutMs = 60_000,
 ): Promise<ExecResult> {
   return new Promise((resolve) => {
-    // stdbuf -oL forces line-buffered stdout so output streams in real time
-    // rather than appearing in one burst when the process's buffer flushes.
+    // stdbuf -oL -eL forces line-buffered stdout and stderr so Ansible play/task
+    // markers stream in real time rather than arriving in one burst at the end.
     const args = [
       'compose', '-f', COMPOSE_FILE, '-p', projectName,
-      'exec', '-T', container, 'stdbuf', '-oL', 'bash', '-c', cmd,
+      'exec', '-T', container, 'stdbuf', '-oL', '-eL', 'bash', '-c', cmd,
     ];
     const child = spawn('docker', args, { cwd: composeDir });
 

package/src/docker-compose-generator.ts

@@ -306,13 +306,15 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
     const zoneNetwork = machine.zone;
     const gateway = ZONE_GATEWAYS[machine.zone];
 
-    const isAppZone = machine.zone === 'app';
-    const dockerfile = isAppZone
-      ? 'docker/Dockerfile.target-machine-docker'
-      : 'docker/Dockerfile.target-machine';
+    const needsDocker = machine.zone === 'app' || machine.docker === true;
+    const dockerfile = machine.dockerfile
+      || (needsDocker ? 'docker/Dockerfile.target-machine-docker' : 'docker/Dockerfile.target-machine');
 
     const machineVolumes = ['ssh-keys:/ssh-keys:ro'];
-    if (isAppZone) {
+
+    // Mount the host image cache into Docker-capable machines so the
+    // docker-image-preload service can load them at startup without pulling.
+    if (needsDocker) {
       machineVolumes.push('./docker-image-cache:/docker-image-cache:ro');
     }
 
@@ -469,13 +471,13 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
     const zoneNetwork = machine.zone;
     const gateway = ZONE_GATEWAYS[machine.zone];
 
-    const isAppZone = machine.zone === 'app';
-    const dockerfile = isAppZone
-      ? 'docker/Dockerfile.target-machine-docker'
-      : 'docker/Dockerfile.target-machine';
+    const needsDocker = machine.zone === 'app' || machine.docker === true;
+    const dockerfile = machine.dockerfile
+      || (needsDocker ? 'docker/Dockerfile.target-machine-docker' : 'docker/Dockerfile.target-machine');
 
     const machineVolumes = ['ssh-keys:/ssh-keys:ro'];
-    if (isAppZone) {
+
+    if (needsDocker) {
       machineVolumes.push('./docker-image-cache:/docker-image-cache:ro');
     }
 

package/src/network-builder.ts

@@ -78,6 +78,27 @@ export class NetworkBuilder {
     return this;
   }
 
+  /** Add internal-zone machines that need the Docker-capable image (e.g., Technitium) */
+  internalDocker(machines: Record<string, string>): this {
+    for (const [name, ip] of Object.entries(machines)) {
+      this.config.internalMachines.push({ name, ip, zone: 'internal', docker: true });
+    }
+    return this;
+  }
+
+  /** Add a Technitium DNS machine using the dedicated pre-installed image */
+  technitium(machines: Record<string, string>): this {
+    for (const [name, ip] of Object.entries(machines)) {
+      this.config.internalMachines.push({
+        name,
+        ip,
+        zone: 'internal',
+        dockerfile: 'docker/Dockerfile.target-technitium',
+      });
+    }
+    return this;
+  }
+
   withDhcpClient(): this {
     this.config.dhcpClient = true;
     return this;

package/src/parse-line.ts

@@ -0,0 +1,136 @@
+/**
+ * Line parser for the e2e test runner.
+ *
+ * Translates celilo CLI output lines into progress display calls.
+ * Extracted as a separate module so it can be unit-tested without
+ * spinning up Docker or a full ProgressDisplay.
+ */
+
+const dim = '\x1b[2m';
+const green = '\x1b[32m';
+const red = '\x1b[31m';
+const reset = '\x1b[0m';
+
+export interface DisplaySink {
+  startStep(doing: string, done: string): void;
+  doneStep(override?: string): void;
+  failStep(message: string): void;
+  instantEvent(message: string): void;
+  subEvent(message: string): void;
+}
+
+export function parseLine(line: string, display: DisplaySink): void {
+  // Protocol: [progress:start] doing | done
+  const startMatch = line.match(/\[progress:start\] (.*?) \| (.*)/);
+  if (startMatch) {
+    display.startStep(startMatch[1], startMatch[2]);
+    return;
+  }
+
+  // Protocol: [progress:done] optional message
+  const doneMatch = line.match(/\[progress:done\](.*)/);
+  if (doneMatch) {
+    const msg = doneMatch[1].trim();
+    display.doneStep(msg || undefined);
+    return;
+  }
+
+  // Protocol: [progress:fail] message
+  const failMatch = line.match(/\[progress:fail\] (.*)/);
+  if (failMatch) {
+    display.failStep(failMatch[1]);
+    return;
+  }
+
+  // Legacy: [progress] message (treat as instant event)
+  const legacyMatch = line.match(/\[progress\] (.*)/);
+  if (legacyMatch) {
+    display.instantEvent(legacyMatch[1]);
+    return;
+  }
+
+  const deployMatch = line.match(/Module '([^']+)' deployed successfully/);
+  if (deployMatch) {
+    display.subEvent(`${green}✔${reset} ${deployMatch[1]} deployed`);
+    return;
+  }
+
+  const importMatch = line.match(/Successfully imported module: (\S+)/);
+  if (importMatch) {
+    display.subEvent(`${green}✔${reset} ${importMatch[1]} imported`);
+    return;
+  }
+
+  if (line.includes('Deploying software')) {
+    display.subEvent('running Ansible');
+    return;
+  }
+  if (line.includes('Running on_install hook') || line.includes('on_install hook')) {
+    display.subEvent('running on_install hook');
+    return;
+  }
+  if (line.includes('Software deployed') || line.includes('Software deployed successfully')) {
+    display.subEvent(`${green}✔${reset} Ansible done`);
+    return;
+  }
+  if (line.includes('Infrastructure selected')) {
+    display.subEvent('infrastructure selected');
+    return;
+  }
+  if (line.includes('Terraform skipped')) {
+    display.subEvent('terraform skipped (existing machine)');
+    return;
+  }
+  if (line.includes('terraform init') || line.includes('Terraform init')) {
+    display.subEvent('running terraform init');
+    return;
+  }
+  if (line.includes('terraform apply') || line.includes('Terraform apply')) {
+    display.subEvent('running terraform apply');
+    return;
+  }
+
+  const retryMatch = line.match(/RETRYING.*?(\d+) retries left/);
+  if (retryMatch) {
+    display.subEvent(`health check (${retryMatch[1]} retries left)`);
+    return;
+  }
+
+  // Ansible progress markers emitted via process.stderr in deploy-ansible.ts
+  if (line.startsWith('[ansible:play] ')) {
+    display.subEvent(`${dim}▶${reset} ${line.slice(15)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:task] ')) {
+    display.subEvent(`  • ${line.slice(15)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:handler] ')) {
+    display.subEvent(`  ↺ ${line.slice(18)}`);
+    return;
+  }
+  if (line.startsWith('[ansible:recap] ')) {
+    const recap = line.slice(16);
+    const failed = recap.match(/failed=(\d+)/)?.[1];
+    const icon = failed && failed !== '0' ? `${red}✗${reset}` : `${green}✔${reset}`;
+    display.subEvent(`${icon} ${recap}`);
+    return;
+  }
+
+  // Hook log output: "[moduleId:hookName] message"
+  const hookLogMatch = line.match(/^\[[\w-]+:[\w_-]+\] (.*)/);
+  if (hookLogMatch) {
+    display.subEvent(`${dim}hook:${reset} ${hookLogMatch[1]}`);
+    return;
+  }
+
+  // Error lines
+  if (
+    line.includes('Error:') &&
+    !line.includes('bun test') &&
+    !line.includes('ipv4: Address already')
+  ) {
+    const err = line.replace(/.*Error: /, '').slice(0, 70);
+    display.subEvent(`${red}✗${reset} ${err}`);
+  }
+}