@matware/e2e-runner 1.1.1 → 1.2.1

package/src/runner.js

@@ -9,12 +9,24 @@ import fs from 'fs';
 import path from 'path';
 import { connectToPool, waitForPool, getPoolStatus } from './pool.js';
 import { executeAction } from './actions.js';
+import { narrateAction } from './narrate.js';
 import { log, colors as C } from './logger.js';
+import { resolveTestData } from './module-resolver.js';
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+/** Simple glob matching with * wildcards for exclude patterns. */
+function matchesExclude(filename, excludePatterns) {
+  if (!excludePatterns?.length) return false;
+  const name = filename.replace('.json', '');
+  return excludePatterns.some(pattern => {
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
+    return regex.test(name) || regex.test(filename);
+  });
+}
+
 function timeDiff(start, end) {
   const ms = new Date(end) - new Date(start);
   return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`;
@@ -44,7 +56,13 @@ function normalizeTestData(data) {
   if (Array.isArray(data)) {
     return { tests: data, hooks: {} };
   }
-  return { tests: data.tests || [], hooks: data.hooks || {} };
+  // Support hooks nested under "hooks" key or directly at root level
+  const hooks = data.hooks || {};
+  if (!hooks.beforeAll && data.beforeAll) hooks.beforeAll = data.beforeAll;
+  if (!hooks.afterAll && data.afterAll) hooks.afterAll = data.afterAll;
+  if (!hooks.beforeEach && data.beforeEach) hooks.beforeEach = data.beforeEach;
+  if (!hooks.afterEach && data.afterEach) hooks.afterEach = data.afterEach;
+  return { tests: data.tests || [], hooks };
 }
 
 /** Waits until the pool has capacity before connecting */
@@ -70,6 +88,7 @@ async function waitForSlot(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
 /** Runs a single test end-to-end */
 export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   let browser = null;
+  let context = null;
   let page = null;
 
   const result = {
@@ -87,7 +106,9 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   try {
     await waitForSlot(config.poolUrl);
     browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
-    page = await browser.newPage();
+    // Use incognito context for cookie isolation between concurrent tests
+    context = await browser.createBrowserContext();
+    page = await context.newPage();
     await page.setViewport(config.viewport);
 
     page.on('console', (msg) => {
@@ -135,37 +156,57 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
 
     for (let i = 0; i < test.actions.length; i++) {
       const action = test.actions[i];
-      const actionStart = Date.now();
-      try {
-        let actionResult;
-        if (action.type === 'assert_no_network_errors') {
-          // Handled inline — needs access to result.networkErrors
-          if (result.networkErrors.length > 0) {
-            const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
-            throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+      const maxActionRetries = action.retries ?? config.actionRetries ?? 0;
+      const actionRetryDelay = config.actionRetryDelay ?? 500;
+      let lastError = null;
+
+      for (let attempt = 0; attempt <= maxActionRetries; attempt++) {
+        const actionStart = Date.now();
+        try {
+          let actionResult;
+          if (action.type === 'assert_no_network_errors') {
+            // Handled inline — needs access to result.networkErrors
+            if (result.networkErrors.length > 0) {
+              const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+              throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+            }
+            actionResult = null;
+          } else {
+            actionResult = await executeAction(page, action, config);
+          }
+          const actionDuration = Date.now() - actionStart;
+          const actionEntry = {
+            ...action,
+            success: true,
+            duration: actionDuration,
+            result: actionResult,
+          };
+          if (attempt > 0) actionEntry.actionRetries = attempt;
+          actionEntry.narrative = narrateAction(action, actionEntry);
+          result.actions.push(actionEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, narrative: actionEntry.narrative, screenshotPath: actionResult?.screenshot || null });
+          lastError = null;
+          break;
+        } catch (error) {
+          lastError = error;
+          if (attempt < maxActionRetries) {
+            log('🔄', `${C.dim}Action ${action.type} retry ${attempt + 1}/${maxActionRetries} (${error.message})${C.reset}`);
+            await sleep(actionRetryDelay);
+            continue;
           }
-          actionResult = null;
-        } else {
-          actionResult = await executeAction(page, action, config);
+          const actionDuration = Date.now() - actionStart;
+          const failedEntry = {
+            ...action,
+            success: false,
+            duration: actionDuration,
+            error: error.message,
+          };
+          if (maxActionRetries > 0) failedEntry.actionRetries = attempt;
+          failedEntry.narrative = narrateAction(action, failedEntry);
+          result.actions.push(failedEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, narrative: failedEntry.narrative, error: error.message });
+          throw error;
         }
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: true,
-          duration: actionDuration,
-          result: actionResult,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, screenshotPath: actionResult?.screenshot || null });
-      } catch (error) {
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: false,
-          duration: actionDuration,
-          error: error.message,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, error: error.message });
-        throw error;
       }
     }
 
@@ -216,6 +257,9 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
     if (page) {
       try { result.finalUrl = page.url(); } catch { /* */ }
     }
+    if (context) {
+      try { await context.close(); } catch { /* */ }
+    }
     if (browser) {
       try { browser.disconnect(); } catch { /* */ }
     }
@@ -230,6 +274,12 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 
   // Run beforeAll hook
   if (hooks.beforeAll?.length) {
+    const stateActions = hooks.beforeAll.filter(a =>
+      ['evaluate', 'goto', 'navigate', 'clear_cookies', 'type', 'click', 'select'].includes(a.type)
+    );
+    if (stateActions.length > 0) {
+      log('⚠️', `${C.yellow}beforeAll runs on a separate browser — state from ${stateActions.map(a => a.type).join(', ')} will NOT carry over to tests. Use beforeEach instead.${C.reset}`);
+    }
     log('🪝', `${C.dim}Running beforeAll hook...${C.reset}`);
     let browser = null;
     try {
@@ -252,10 +302,18 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   const _cwd = config._cwd || null;
   const _triggeredBy = config.triggeredBy || 'unknown';
   const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd, triggeredBy: _triggeredBy });
+
+  // Split serial and parallel tests
+  const parallelTests = tests.filter(t => !t.serial);
+  const serialTests = tests.filter(t => t.serial);
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}${serialTests.length} serial test(s) will run after parallel batch${C.reset}`);
+  }
+
   _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
 
   const results = [];
-  const queue = [...tests];
+  const queue = [...parallelTests];
   let activeCount = 0;
 
   const worker = async () => {
@@ -263,7 +321,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       const test = queue.shift();
       activeCount++;
       log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
-      _progress({ event: 'test:start', name: test.name, activeCount, queueRemaining: queue.length });
+      _progress({ event: 'test:start', name: test.name, serial: test.serial || false, activeCount, queueRemaining: queue.length });
 
       const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
       const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
@@ -276,7 +334,8 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         });
 
         try {
-          result = await Promise.race([runTest(test, config, hooks, _progress), timeoutPromise]);
+          const testHooks = test._suiteHooks ? mergeHooks(config.hooks, test._suiteHooks) : hooks;
+          result = await Promise.race([runTest(test, config, testHooks, _progress), timeoutPromise]);
         } catch (error) {
           result = {
             name: test.name,
@@ -325,11 +384,18 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   };
 
   const workers = [];
-  for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
+  for (let i = 0; i < Math.min(concurrency, parallelTests.length); i++) {
     workers.push(worker());
   }
   await Promise.all(workers);
 
+  // Run serial tests one at a time
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}Running ${serialTests.length} serial test(s)...${C.reset}`);
+    queue.push(...serialTests);
+    await worker();
+  }
+
   // Run afterAll hook
   if (hooks.afterAll?.length) {
     log('🪝', `${C.dim}Running afterAll hook...${C.reset}`);
@@ -357,16 +423,17 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 }
 
 /** Loads tests from a JSON file — returns { tests, hooks } */
-export function loadTestFile(filePath) {
+export function loadTestFile(filePath, modulesDir) {
   if (!fs.existsSync(filePath)) {
     throw new Error(`File not found: ${filePath}`);
   }
   const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  return modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
 }
 
 /** Loads a test suite by name — returns { tests, hooks } */
-export function loadTestSuite(suiteName, testsDir) {
+export function loadTestSuite(suiteName, testsDir, modulesDir) {
   // Match with or without numeric prefix (e.g. "agents" matches "03-agents.json")
   const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json'));
   const exact = files.find(f => f === `${suiteName}.json`);
@@ -378,29 +445,36 @@ export function loadTestSuite(suiteName, testsDir) {
   }
 
   const data = JSON.parse(fs.readFileSync(path.join(testsDir, match), 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  return modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
 }
 
 /** Loads all test suites from the tests directory — returns { tests, hooks } */
-export function loadAllSuites(testsDir) {
+export function loadAllSuites(testsDir, modulesDir, exclude = []) {
   if (!fs.existsSync(testsDir)) {
     throw new Error(`Tests directory not found: ${testsDir}`);
   }
 
-  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort()
+    .filter(f => !matchesExclude(f, exclude));
   let allTests = [];
-  let mergedHooks = {};
 
   for (const file of files) {
     const data = JSON.parse(fs.readFileSync(path.join(testsDir, file), 'utf-8'));
-    const { tests, hooks } = normalizeTestData(data);
+    let { tests, hooks } = normalizeTestData(data);
+    // Resolve modules per-suite before concatenating
+    if (modulesDir) {
+      ({ tests, hooks } = resolveTestData({ tests, hooks }, modulesDir));
+    }
+    // Tag each test with its own suite's hooks so they're preserved
+    for (const t of tests) {
+      t._suiteHooks = hooks;
+    }
     allTests = allTests.concat(tests);
-    // Last suite's hooks win for each non-empty key
-    mergedHooks = mergeHooks(mergedHooks, hooks);
     log('📋', `${C.cyan}${file}${C.reset} (${tests.length} tests)`);
   }
 
-  return { tests: allTests, hooks: mergedHooks };
+  return { tests: allTests, hooks: {} };
 }
 
 /** Lists all available test suites */

package/src/verify.js

@@ -25,7 +25,7 @@ export async function verifyIssue(url, config) {
   const issue = fetchIssue(url);
 
   // 2. Generate tests via Claude API
-  const { tests, suiteName } = await generateTests(issue, config);
+  const { tests, suiteName } = await generateTests(issue, config, config.testType || 'e2e');
 
   // 3. Save tests to a temp file (underscore prefix for cleanup identification)
   const testFile = path.join(config.testsDir, `_verify-${suiteName}.json`);
@@ -38,10 +38,12 @@ export async function verifyIssue(url, config) {
     // 4. Build hooks (inject auth if provided)
     const hooks = {};
     if (config.authToken) {
-      const storageKey = config.authStorageKey || 'accessToken';
+      const esc = s => s.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+      const storageKey = esc(config.authStorageKey || 'accessToken');
+      const token = esc(config.authToken);
       hooks.beforeEach = [
         { type: 'goto', value: '/' },
-        { type: 'evaluate', value: `localStorage.setItem('${storageKey}', '${config.authToken}')` },
+        { type: 'evaluate', value: `localStorage.setItem('${storageKey}', '${token}')` },
         { type: 'goto', value: '/' },
         { type: 'wait', value: '1000' },
       ];

package/templates/build-dashboard.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+/**
+ * Build script — concatenates dashboard/template.html + dashboard/styles.css + dashboard/app.js
+ * into a single dashboard.html file.
+ *
+ * Usage: node templates/build-dashboard.js
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const dashDir = path.join(__dirname, 'dashboard');
+
+const template = fs.readFileSync(path.join(dashDir, 'template.html'), 'utf-8');
+const styles = fs.readFileSync(path.join(dashDir, 'styles.css'), 'utf-8');
+const script = fs.readFileSync(path.join(dashDir, 'app.js'), 'utf-8');
+
+const output = template
+  .replace('/* __STYLES__ */', () => styles)
+  .replace('/* __SCRIPT__ */', () => script);
+
+const outPath = path.join(__dirname, 'dashboard.html');
+fs.writeFileSync(outPath, output);
+
+const lines = output.split('\n').length;
+console.log(`Built ${outPath} (${lines} lines)`);