@matware/e2e-runner 1.0.0

package/src/runner.js

@@ -0,0 +1,309 @@
+/**
+ * E2E Test Execution Engine
+ *
+ * Runs tests in parallel using a pool of Chrome instances.
+ * Supports retries, test-level timeouts, and before/after hooks.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { connectToPool, waitForPool } from './pool.js';
+import { executeAction } from './actions.js';
+import { log, colors as C } from './logger.js';
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function timeDiff(start, end) {
+  const ms = new Date(end) - new Date(start);
+  return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`;
+}
+
+/** Merges suite-level hooks with global config hooks. Non-empty suite hooks win per key. */
+function mergeHooks(configHooks, suiteHooks) {
+  const base = configHooks || { beforeAll: [], afterAll: [], beforeEach: [], afterEach: [] };
+  if (!suiteHooks) return { ...base };
+  return {
+    beforeAll: suiteHooks.beforeAll?.length ? suiteHooks.beforeAll : base.beforeAll || [],
+    afterAll: suiteHooks.afterAll?.length ? suiteHooks.afterAll : base.afterAll || [],
+    beforeEach: suiteHooks.beforeEach?.length ? suiteHooks.beforeEach : base.beforeEach || [],
+    afterEach: suiteHooks.afterEach?.length ? suiteHooks.afterEach : base.afterEach || [],
+  };
+}
+
+/** Executes an array of hook actions on a page */
+async function executeHookActions(page, actions, config) {
+  for (const action of actions) {
+    await executeAction(page, action, config);
+  }
+}
+
+/** Normalizes raw JSON (array or object with hooks) into { tests, hooks } */
+function normalizeTestData(data) {
+  if (Array.isArray(data)) {
+    return { tests: data, hooks: {} };
+  }
+  return { tests: data.tests || [], hooks: data.hooks || {} };
+}
+
+/** Runs a single test end-to-end */
+export async function runTest(test, config, hooks = {}) {
+  let browser = null;
+  let page = null;
+
+  const result = {
+    name: test.name,
+    startTime: new Date().toISOString(),
+    actions: [],
+    success: true,
+    error: null,
+    consoleLogs: [],
+    networkErrors: [],
+  };
+
+  try {
+    browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
+    page = await browser.newPage();
+    await page.setViewport(config.viewport);
+
+    page.on('console', (msg) => {
+      result.consoleLogs.push({ type: msg.type(), text: msg.text() });
+    });
+    page.on('requestfailed', (req) => {
+      result.networkErrors.push({ url: req.url(), error: req.failure()?.errorText });
+    });
+
+    // Run beforeEach hook
+    if (hooks.beforeEach?.length) {
+      await executeHookActions(page, hooks.beforeEach, config);
+    }
+
+    for (const action of test.actions) {
+      const actionStart = Date.now();
+      try {
+        const actionResult = await executeAction(page, action, config);
+        result.actions.push({
+          ...action,
+          success: true,
+          duration: Date.now() - actionStart,
+          result: actionResult,
+        });
+      } catch (error) {
+        result.actions.push({
+          ...action,
+          success: false,
+          duration: Date.now() - actionStart,
+          error: error.message,
+        });
+        throw error;
+      }
+    }
+
+    // Run afterEach hook (success path)
+    if (hooks.afterEach?.length) {
+      await executeHookActions(page, hooks.afterEach, config);
+    }
+  } catch (error) {
+    result.success = false;
+    result.error = error.message;
+
+    // Run afterEach hook (failure path)
+    if (page && hooks.afterEach?.length) {
+      try { await executeHookActions(page, hooks.afterEach, config); } catch { /* */ }
+    }
+
+    if (page) {
+      try {
+        const errorScreenshot = path.join(config.screenshotsDir, `error-${test.name}-${Date.now()}.png`);
+        await page.screenshot({ path: errorScreenshot, fullPage: true });
+        result.errorScreenshot = errorScreenshot;
+      } catch { /* page may be dead */ }
+    }
+  } finally {
+    result.endTime = new Date().toISOString();
+    if (page) {
+      try { result.finalUrl = page.url(); } catch { /* */ }
+    }
+    if (browser) {
+      try { browser.disconnect(); } catch { /* */ }
+    }
+  }
+
+  return result;
+}
+
+/** Runs tests in parallel with limited concurrency, retries, timeouts, and hooks */
+export async function runTestsParallel(tests, config, suiteHooks = {}) {
+  const hooks = mergeHooks(config.hooks, suiteHooks);
+
+  // Run beforeAll hook
+  if (hooks.beforeAll?.length) {
+    log('🪝', `${C.dim}Running beforeAll hook...${C.reset}`);
+    let browser = null;
+    try {
+      browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
+      const page = await browser.newPage();
+      await page.setViewport(config.viewport);
+      await executeHookActions(page, hooks.beforeAll, config);
+      await page.close();
+    } catch (error) {
+      log('❌', `${C.red}beforeAll hook failed: ${error.message}${C.reset}`);
+      throw error;
+    } finally {
+      if (browser) try { browser.disconnect(); } catch { /* */ }
+    }
+  }
+
+  const results = [];
+  const queue = [...tests];
+  let activeCount = 0;
+
+  const worker = async () => {
+    while (queue.length > 0) {
+      const test = queue.shift();
+      activeCount++;
+      log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
+
+      const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
+      const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
+      let result;
+
+      for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        const timeoutPromise = new Promise((_, reject) => {
+          const timer = setTimeout(() => reject(new Error(`Test timed out after ${testTimeout}ms`)), testTimeout);
+          timer.unref();
+        });
+
+        try {
+          result = await Promise.race([runTest(test, config, hooks), timeoutPromise]);
+        } catch (error) {
+          result = {
+            name: test.name,
+            startTime: new Date().toISOString(),
+            endTime: new Date().toISOString(),
+            actions: [],
+            success: false,
+            error: error.message,
+            consoleLogs: [],
+            networkErrors: [],
+          };
+        }
+
+        result.attempt = attempt;
+        result.maxAttempts = maxAttempts;
+
+        if (result.success || attempt === maxAttempts) break;
+        log('🔄', `${C.yellow}${test.name}${C.reset} failed, retrying (${attempt}/${maxAttempts})...`);
+        await sleep(config.retryDelay || 1000);
+      }
+
+      results.push(result);
+      activeCount--;
+
+      if (result.success) {
+        const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
+        log('✅', `${C.green}${test.name}${C.reset} ${C.dim}(${timeDiff(result.startTime, result.endTime)})${C.reset}${flaky}`);
+      } else {
+        const attempts = result.maxAttempts > 1 ? ` (${result.maxAttempts} attempts)` : '';
+        log('❌', `${C.red}${test.name}${C.reset}: ${result.error}${attempts}`);
+      }
+    }
+  };
+
+  const concurrency = config.concurrency || 3;
+  const workers = [];
+  for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
+    workers.push(worker());
+  }
+  await Promise.all(workers);
+
+  // Run afterAll hook
+  if (hooks.afterAll?.length) {
+    log('🪝', `${C.dim}Running afterAll hook...${C.reset}`);
+    let browser = null;
+    try {
+      browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
+      const page = await browser.newPage();
+      await page.setViewport(config.viewport);
+      await executeHookActions(page, hooks.afterAll, config);
+      await page.close();
+    } catch (error) {
+      log('⚠️', `${C.yellow}afterAll hook failed: ${error.message}${C.reset}`);
+    } finally {
+      if (browser) try { browser.disconnect(); } catch { /* */ }
+    }
+  }
+
+  return results;
+}
+
+/** Loads tests from a JSON file — returns { tests, hooks } */
+export function loadTestFile(filePath) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`File not found: ${filePath}`);
+  }
+  const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  return normalizeTestData(data);
+}
+
+/** Loads a test suite by name — returns { tests, hooks } */
+export function loadTestSuite(suiteName, testsDir) {
+  // Match with or without numeric prefix (e.g. "agents" matches "03-agents.json")
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json'));
+  const exact = files.find(f => f === `${suiteName}.json`);
+  const prefixed = files.find(f => f.replace(/^\d+-/, '') === `${suiteName}.json`);
+  const match = exact || prefixed;
+
+  if (!match) {
+    throw new Error(`Suite not found: ${suiteName} in ${testsDir}`);
+  }
+
+  const data = JSON.parse(fs.readFileSync(path.join(testsDir, match), 'utf-8'));
+  return normalizeTestData(data);
+}
+
+/** Loads all test suites from the tests directory — returns { tests, hooks } */
+export function loadAllSuites(testsDir) {
+  if (!fs.existsSync(testsDir)) {
+    throw new Error(`Tests directory not found: ${testsDir}`);
+  }
+
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  let allTests = [];
+  let mergedHooks = {};
+
+  for (const file of files) {
+    const data = JSON.parse(fs.readFileSync(path.join(testsDir, file), 'utf-8'));
+    const { tests, hooks } = normalizeTestData(data);
+    allTests = allTests.concat(tests);
+    // Last suite's hooks win for each non-empty key
+    mergedHooks = mergeHooks(mergedHooks, hooks);
+    log('📋', `${C.cyan}${file}${C.reset} (${tests.length} tests)`);
+  }
+
+  return { tests: allTests, hooks: mergedHooks };
+}
+
+/** Lists all available test suites */
+export function listSuites(testsDir) {
+  if (!fs.existsSync(testsDir)) {
+    throw new Error(`Tests directory not found: ${testsDir}`);
+  }
+
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  const suites = [];
+
+  for (const file of files) {
+    const data = JSON.parse(fs.readFileSync(path.join(testsDir, file), 'utf-8'));
+    const { tests } = normalizeTestData(data);
+    suites.push({
+      name: file.replace('.json', ''),
+      file,
+      testCount: tests.length,
+      tests: tests.map(t => t.name),
+    });
+  }
+
+  return suites;
+}

package/templates/docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  chrome-pool:
+    image: browserless/chrome:1-puppeteer-21.3.6
+    container_name: e2e-chrome-pool
+    restart: unless-stopped
+    ports:
+      - "${PORT}:3000"
+    environment:
+      - MAX_CONCURRENT_SESSIONS=${MAX_SESSIONS}
+      - MAX_QUEUE_LENGTH=10
+      - CONNECTION_TIMEOUT=300000
+      - PREBOOT_CHROME=false
+      - ENABLE_CORS=true

package/templates/e2e.config.js

@@ -0,0 +1,55 @@
+export default {
+  // App URL (from inside Docker, use host.docker.internal to reach the host)
+  baseUrl: 'http://host.docker.internal:3000',
+
+  // Chrome Pool WebSocket URL
+  poolUrl: 'ws://localhost:3333',
+
+  // Directory containing JSON test files
+  testsDir: 'e2e/tests',
+
+  // Directory for screenshots and reports
+  screenshotsDir: 'e2e/screenshots',
+
+  // Parallel test workers
+  concurrency: 3,
+
+  // Browser viewport
+  viewport: { width: 1280, height: 720 },
+
+  // Timeout per action (ms)
+  defaultTimeout: 10000,
+
+  // Chrome Pool port (for pool start/stop)
+  poolPort: 3333,
+
+  // Max concurrent pool sessions
+  maxSessions: 5,
+
+  // Retry failed tests N times (0 = no retries)
+  retries: 0,
+
+  // Delay between retries (ms)
+  retryDelay: 1000,
+
+  // Per-test timeout (ms) — kills the test if it exceeds this
+  testTimeout: 60000,
+
+  // Report output format: 'json', 'junit', or 'both'
+  outputFormat: 'json',
+
+  // Global hooks — run actions before/after all tests or each test
+  // hooks: {
+  //   beforeAll: [{ type: 'goto', value: '/login' }, { type: 'type', selector: '#email', value: 'admin@example.com' }],
+  //   afterAll: [],
+  //   beforeEach: [{ type: 'goto', value: '/' }],
+  //   afterEach: [{ type: 'screenshot', value: 'after-test.png' }],
+  // },
+
+  // Environment profiles — override any config key per environment
+  // Use with --env <name> or E2E_ENV=<name>
+  // environments: {
+  //   staging: { baseUrl: 'https://staging.example.com' },
+  //   production: { baseUrl: 'https://example.com', concurrency: 5 },
+  // },
+};

package/templates/sample-test.json

@@ -0,0 +1,19 @@
+[
+  {
+    "name": "homepage-loads",
+    "actions": [
+      { "type": "goto", "value": "/" },
+      { "type": "wait", "value": "1000" },
+      { "type": "screenshot", "value": "homepage.png" },
+      { "type": "assert_url", "value": "/" }
+    ]
+  },
+  {
+    "name": "homepage-has-title",
+    "actions": [
+      { "type": "goto", "value": "/" },
+      { "type": "wait", "value": "1000" },
+      { "type": "assert_text", "text": "Welcome" }
+    ]
+  }
+]