@matware/e2e-runner 1.1.0 → 1.2.1

package/src/runner.js

@@ -9,12 +9,24 @@ import fs from 'fs';
 import path from 'path';
 import { connectToPool, waitForPool, getPoolStatus } from './pool.js';
 import { executeAction } from './actions.js';
+import { narrateAction } from './narrate.js';
 import { log, colors as C } from './logger.js';
+import { resolveTestData } from './module-resolver.js';
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+/** Simple glob matching with * wildcards for exclude patterns. */
+function matchesExclude(filename, excludePatterns) {
+  if (!excludePatterns?.length) return false;
+  const name = filename.replace('.json', '');
+  return excludePatterns.some(pattern => {
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
+    return regex.test(name) || regex.test(filename);
+  });
+}
+
 function timeDiff(start, end) {
   const ms = new Date(end) - new Date(start);
   return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`;
@@ -44,7 +56,13 @@ function normalizeTestData(data) {
   if (Array.isArray(data)) {
     return { tests: data, hooks: {} };
   }
-  return { tests: data.tests || [], hooks: data.hooks || {} };
+  // Support hooks nested under "hooks" key or directly at root level
+  const hooks = data.hooks || {};
+  if (!hooks.beforeAll && data.beforeAll) hooks.beforeAll = data.beforeAll;
+  if (!hooks.afterAll && data.afterAll) hooks.afterAll = data.afterAll;
+  if (!hooks.beforeEach && data.beforeEach) hooks.beforeEach = data.beforeEach;
+  if (!hooks.afterEach && data.afterEach) hooks.afterEach = data.afterEach;
+  return { tests: data.tests || [], hooks };
 }
 
 /** Waits until the pool has capacity before connecting */
@@ -70,6 +88,7 @@ async function waitForSlot(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
 /** Runs a single test end-to-end */
 export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   let browser = null;
+  let context = null;
   let page = null;
 
   const result = {
@@ -80,12 +99,16 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
     error: null,
     consoleLogs: [],
     networkErrors: [],
+    networkLogs: [],
   };
+  const pendingBodies = [];
 
   try {
     await waitForSlot(config.poolUrl);
     browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
-    page = await browser.newPage();
+    // Use incognito context for cookie isolation between concurrent tests
+    context = await browser.createBrowserContext();
+    page = await context.newPage();
     await page.setViewport(config.viewport);
 
     page.on('console', (msg) => {
@@ -95,6 +118,37 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       result.networkErrors.push({ url: req.url(), error: req.failure()?.errorText });
     });
 
+    const requestTimings = new Map();
+    page.on('request', (req) => {
+      const rt = req.resourceType();
+      if (rt === 'xhr' || rt === 'fetch') requestTimings.set(req, Date.now());
+    });
+    page.on('response', (resp) => {
+      const req = resp.request();
+      const startMs = requestTimings.get(req);
+      if (startMs !== undefined) {
+        requestTimings.delete(req);
+        const entry = {
+          url: req.url(),
+          method: req.method(),
+          status: resp.status(),
+          statusText: resp.statusText(),
+          duration: Date.now() - startMs,
+          requestHeaders: req.headers(),
+          requestBody: null,
+          responseHeaders: resp.headers(),
+          responseBody: null,
+        };
+        try { entry.requestBody = req.postData() || null; } catch { /* */ }
+        result.networkLogs.push(entry);
+        // Read response body async — collect promise for later flush
+        const bodyPromise = resp.text().then(body => {
+          entry.responseBody = body && body.length > 51200 ? body.slice(0, 51200) + '\n...[truncated]' : body;
+        }).catch(() => { /* response may be unavailable */ });
+        pendingBodies.push(bodyPromise);
+      }
+    });
+
     // Run beforeEach hook
     if (hooks.beforeEach?.length) {
       await executeHookActions(page, hooks.beforeEach, config);
@@ -102,30 +156,77 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
 
     for (let i = 0; i < test.actions.length; i++) {
       const action = test.actions[i];
-      const actionStart = Date.now();
-      try {
-        const actionResult = await executeAction(page, action, config);
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: true,
-          duration: actionDuration,
-          result: actionResult,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, screenshotPath: actionResult?.screenshot || null });
-      } catch (error) {
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: false,
-          duration: actionDuration,
-          error: error.message,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, error: error.message });
-        throw error;
+      const maxActionRetries = action.retries ?? config.actionRetries ?? 0;
+      const actionRetryDelay = config.actionRetryDelay ?? 500;
+      let lastError = null;
+
+      for (let attempt = 0; attempt <= maxActionRetries; attempt++) {
+        const actionStart = Date.now();
+        try {
+          let actionResult;
+          if (action.type === 'assert_no_network_errors') {
+            // Handled inline — needs access to result.networkErrors
+            if (result.networkErrors.length > 0) {
+              const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+              throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+            }
+            actionResult = null;
+          } else {
+            actionResult = await executeAction(page, action, config);
+          }
+          const actionDuration = Date.now() - actionStart;
+          const actionEntry = {
+            ...action,
+            success: true,
+            duration: actionDuration,
+            result: actionResult,
+          };
+          if (attempt > 0) actionEntry.actionRetries = attempt;
+          actionEntry.narrative = narrateAction(action, actionEntry);
+          result.actions.push(actionEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, narrative: actionEntry.narrative, screenshotPath: actionResult?.screenshot || null });
+          lastError = null;
+          break;
+        } catch (error) {
+          lastError = error;
+          if (attempt < maxActionRetries) {
+            log('🔄', `${C.dim}Action ${action.type} retry ${attempt + 1}/${maxActionRetries} (${error.message})${C.reset}`);
+            await sleep(actionRetryDelay);
+            continue;
+          }
+          const actionDuration = Date.now() - actionStart;
+          const failedEntry = {
+            ...action,
+            success: false,
+            duration: actionDuration,
+            error: error.message,
+          };
+          if (maxActionRetries > 0) failedEntry.actionRetries = attempt;
+          failedEntry.narrative = narrateAction(action, failedEntry);
+          result.actions.push(failedEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, narrative: failedEntry.narrative, error: error.message });
+          throw error;
+        }
       }
     }
 
+    // Fail the test if failOnNetworkError is enabled and network errors occurred
+    if (config.failOnNetworkError && result.networkErrors.length > 0) {
+      const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+      throw new Error(`Network errors detected (failOnNetworkError=true): ${result.networkErrors.length} error(s): ${summary}`);
+    }
+
+    // Auto-capture verification screenshot if test has "expect"
+    if (test.expect && page) {
+      result.expect = test.expect;
+      try {
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const verifyPath = path.join(config.screenshotsDir, `verify-${safeName}-${Date.now()}.png`);
+        await page.screenshot({ path: verifyPath, fullPage: true });
+        result.verificationScreenshot = verifyPath;
+      } catch { /* page may be dead */ }
+    }
+
     // Run afterEach hook (success path)
     if (hooks.afterEach?.length) {
       await executeHookActions(page, hooks.afterEach, config);
@@ -148,10 +249,17 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       } catch { /* page may be dead */ }
     }
   } finally {
+    // Flush pending response body reads before disconnecting
+    if (pendingBodies.length > 0) {
+      try { await Promise.allSettled(pendingBodies); } catch { /* */ }
+    }
     result.endTime = new Date().toISOString();
     if (page) {
       try { result.finalUrl = page.url(); } catch { /* */ }
     }
+    if (context) {
+      try { await context.close(); } catch { /* */ }
+    }
     if (browser) {
       try { browser.disconnect(); } catch { /* */ }
     }
@@ -166,6 +274,12 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 
   // Run beforeAll hook
   if (hooks.beforeAll?.length) {
+    const stateActions = hooks.beforeAll.filter(a =>
+      ['evaluate', 'goto', 'navigate', 'clear_cookies', 'type', 'click', 'select'].includes(a.type)
+    );
+    if (stateActions.length > 0) {
+      log('⚠️', `${C.yellow}beforeAll runs on a separate browser — state from ${stateActions.map(a => a.type).join(', ')} will NOT carry over to tests. Use beforeEach instead.${C.reset}`);
+    }
     log('🪝', `${C.dim}Running beforeAll hook...${C.reset}`);
     let browser = null;
     try {
@@ -186,11 +300,20 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   const _runId = Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
   const _proj = config.projectName || null;
   const _cwd = config._cwd || null;
-  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd });
+  const _triggeredBy = config.triggeredBy || 'unknown';
+  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd, triggeredBy: _triggeredBy });
+
+  // Split serial and parallel tests
+  const parallelTests = tests.filter(t => !t.serial);
+  const serialTests = tests.filter(t => t.serial);
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}${serialTests.length} serial test(s) will run after parallel batch${C.reset}`);
+  }
+
   _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
 
   const results = [];
-  const queue = [...tests];
+  const queue = [...parallelTests];
   let activeCount = 0;
 
   const worker = async () => {
@@ -198,7 +321,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       const test = queue.shift();
       activeCount++;
       log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
-      _progress({ event: 'test:start', name: test.name, activeCount, queueRemaining: queue.length });
+      _progress({ event: 'test:start', name: test.name, serial: test.serial || false, activeCount, queueRemaining: queue.length });
 
       const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
       const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
@@ -211,7 +334,8 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         });
 
         try {
-          result = await Promise.race([runTest(test, config, hooks, _progress), timeoutPromise]);
+          const testHooks = test._suiteHooks ? mergeHooks(config.hooks, test._suiteHooks) : hooks;
+          result = await Promise.race([runTest(test, config, testHooks, _progress), timeoutPromise]);
         } catch (error) {
           result = {
             name: test.name,
@@ -222,6 +346,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
             error: error.message,
             consoleLogs: [],
             networkErrors: [],
+            networkLogs: [],
           };
         }
 
@@ -238,7 +363,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       activeCount--;
 
       const screenshots = result.actions.filter(a => a.result?.screenshot).map(a => a.result.screenshot);
-      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, errorScreenshot: result.errorScreenshot, screenshots });
+      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots });
 
       if (result.success) {
         const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
@@ -259,11 +384,18 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   };
 
   const workers = [];
-  for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
+  for (let i = 0; i < Math.min(concurrency, parallelTests.length); i++) {
     workers.push(worker());
   }
   await Promise.all(workers);
 
+  // Run serial tests one at a time
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}Running ${serialTests.length} serial test(s)...${C.reset}`);
+    queue.push(...serialTests);
+    await worker();
+  }
+
   // Run afterAll hook
   if (hooks.afterAll?.length) {
     log('🪝', `${C.dim}Running afterAll hook...${C.reset}`);
@@ -291,16 +423,17 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 }
 
 /** Loads tests from a JSON file — returns { tests, hooks } */
-export function loadTestFile(filePath) {
+export function loadTestFile(filePath, modulesDir) {
   if (!fs.existsSync(filePath)) {
     throw new Error(`File not found: ${filePath}`);
   }
   const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  return modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
 }
 
 /** Loads a test suite by name — returns { tests, hooks } */
-export function loadTestSuite(suiteName, testsDir) {
+export function loadTestSuite(suiteName, testsDir, modulesDir) {
   // Match with or without numeric prefix (e.g. "agents" matches "03-agents.json")
   const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json'));
   const exact = files.find(f => f === `${suiteName}.json`);
@@ -312,29 +445,36 @@ export function loadTestSuite(suiteName, testsDir) {
   }
 
   const data = JSON.parse(fs.readFileSync(path.join(testsDir, match), 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  return modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
 }
 
 /** Loads all test suites from the tests directory — returns { tests, hooks } */
-export function loadAllSuites(testsDir) {
+export function loadAllSuites(testsDir, modulesDir, exclude = []) {
   if (!fs.existsSync(testsDir)) {
     throw new Error(`Tests directory not found: ${testsDir}`);
   }
 
-  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort()
+    .filter(f => !matchesExclude(f, exclude));
   let allTests = [];
-  let mergedHooks = {};
 
   for (const file of files) {
     const data = JSON.parse(fs.readFileSync(path.join(testsDir, file), 'utf-8'));
-    const { tests, hooks } = normalizeTestData(data);
+    let { tests, hooks } = normalizeTestData(data);
+    // Resolve modules per-suite before concatenating
+    if (modulesDir) {
+      ({ tests, hooks } = resolveTestData({ tests, hooks }, modulesDir));
+    }
+    // Tag each test with its own suite's hooks so they're preserved
+    for (const t of tests) {
+      t._suiteHooks = hooks;
+    }
     allTests = allTests.concat(tests);
-    // Last suite's hooks win for each non-empty key
-    mergedHooks = mergeHooks(mergedHooks, hooks);
     log('📋', `${C.cyan}${file}${C.reset} (${tests.length} tests)`);
   }
 
-  return { tests: allTests, hooks: mergedHooks };
+  return { tests: allTests, hooks: {} };
 }
 
 /** Lists all available test suites */

package/src/verify.js

@@ -25,7 +25,7 @@ export async function verifyIssue(url, config) {
   const issue = fetchIssue(url);
 
   // 2. Generate tests via Claude API
-  const { tests, suiteName } = await generateTests(issue, config);
+  const { tests, suiteName } = await generateTests(issue, config, config.testType || 'e2e');
 
   // 3. Save tests to a temp file (underscore prefix for cleanup identification)
   const testFile = path.join(config.testsDir, `_verify-${suiteName}.json`);
@@ -35,19 +35,33 @@ export async function verifyIssue(url, config) {
   fs.writeFileSync(testFile, JSON.stringify(tests, null, 2));
 
   try {
-    // 4. Wait for pool and run
+    // 4. Build hooks (inject auth if provided)
+    const hooks = {};
+    if (config.authToken) {
+      const esc = s => s.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+      const storageKey = esc(config.authStorageKey || 'accessToken');
+      const token = esc(config.authToken);
+      hooks.beforeEach = [
+        { type: 'goto', value: '/' },
+        { type: 'evaluate', value: `localStorage.setItem('${storageKey}', '${token}')` },
+        { type: 'goto', value: '/' },
+        { type: 'wait', value: '1000' },
+      ];
+    }
+
+    // 5. Wait for pool and run
     await waitForPool(config.poolUrl);
-    const results = await runTestsParallel(tests, config, {});
+    const results = await runTestsParallel(tests, config, hooks);
     const report = generateReport(results);
     saveReport(report, config.screenshotsDir, config);
     persistRun(report, config, suiteName);
 
-    // 5. Interpret results
+    // 6. Interpret results
     const bugConfirmed = report.summary.failed > 0;
 
     return { issue, report, bugConfirmed, tests, suiteName };
   } finally {
-    // 6. Clean up temp file
+    // 7. Clean up temp file
     try { fs.unlinkSync(testFile); } catch { /* already gone */ }
   }
 }

package/templates/build-dashboard.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+/**
+ * Build script — concatenates dashboard/template.html + dashboard/styles.css + dashboard/app.js
+ * into a single dashboard.html file.
+ *
+ * Usage: node templates/build-dashboard.js
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const dashDir = path.join(__dirname, 'dashboard');
+
+const template = fs.readFileSync(path.join(dashDir, 'template.html'), 'utf-8');
+const styles = fs.readFileSync(path.join(dashDir, 'styles.css'), 'utf-8');
+const script = fs.readFileSync(path.join(dashDir, 'app.js'), 'utf-8');
+
+const output = template
+  .replace('/* __STYLES__ */', () => styles)
+  .replace('/* __SCRIPT__ */', () => script);
+
+const outPath = path.join(__dirname, 'dashboard.html');
+fs.writeFileSync(outPath, output);
+
+const lines = output.split('\n').length;
+console.log(`Built ${outPath} (${lines} lines)`);