@matware/e2e-runner 1.1.1 → 1.3.0

package/src/runner.js

@@ -7,14 +7,30 @@
 
 import fs from 'fs';
 import path from 'path';
-import { connectToPool, waitForPool, getPoolStatus } from './pool.js';
+import http from 'http';
+import https from 'https';
+import { connectToPool } from './pool.js';
+import { getPoolUrls, selectPool, releasePending } from './pool-manager.js';
 import { executeAction } from './actions.js';
+import { narrateAction } from './narrate.js';
 import { log, colors as C } from './logger.js';
+import { resolveTestData, validateActionTypes } from './module-resolver.js';
+import { ensureProject, getVariables } from './db.js';
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+/** Simple glob matching with * wildcards for exclude patterns. */
+function matchesExclude(filename, excludePatterns) {
+  if (!excludePatterns?.length) return false;
+  const name = filename.replace('.json', '');
+  return excludePatterns.some(pattern => {
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
+    return regex.test(name) || regex.test(filename);
+  });
+}
+
 function timeDiff(start, end) {
   const ms = new Date(end) - new Date(start);
   return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`;
@@ -32,6 +48,46 @@ function mergeHooks(configHooks, suiteHooks) {
   };
 }
 
+/** Replaces {{var.KEY}} and {{env.KEY}} in all string fields of an action object.
+ *  Skips {{param}} patterns (no dot) — those are module params handled by module-resolver. */
+function resolveVarsInAction(action, vars) {
+  const resolved = { ...action };
+  for (const key of Object.keys(resolved)) {
+    if (typeof resolved[key] !== 'string') continue;
+    resolved[key] = resolved[key].replace(/\{\{(var|env)\.([^}]+)\}\}/g, (match, ns, name) => {
+      if (ns === 'env') {
+        if (process.env[name] !== undefined) return process.env[name];
+        throw new Error(`Unresolved variable: {{env.${name}}} — environment variable not set`);
+      }
+      // ns === 'var'
+      if (vars[name] !== undefined) return vars[name];
+      throw new Error(`Unresolved variable: {{var.${name}}} — not found in project or suite variables`);
+    });
+  }
+  return resolved;
+}
+
+/** Loads merged variables for a test (project scope + suite scope overlay). */
+function loadVarsForTest(config, suiteName) {
+  try {
+    const cwd = config._cwd || process.cwd();
+    const projectName = config.projectName || cwd.split('/').pop() || 'default';
+    const projectId = ensureProject(cwd, projectName, config.screenshotsDir, config.testsDir);
+    const projectVars = getVariables(projectId, 'project');
+    if (!suiteName) return projectVars;
+    const suiteVars = getVariables(projectId, suiteName);
+    return { ...projectVars, ...suiteVars };
+  } catch {
+    return {};
+  }
+}
+
+/** Resolves variables in an array of actions. */
+function resolveVarsInActions(actions, vars) {
+  if (!actions || !actions.length) return actions;
+  return actions.map(a => resolveVarsInAction(a, vars));
+}
+
 /** Executes an array of hook actions on a page */
 async function executeHookActions(page, actions, config) {
   for (const action of actions) {
@@ -44,32 +100,60 @@ function normalizeTestData(data) {
   if (Array.isArray(data)) {
     return { tests: data, hooks: {} };
   }
-  return { tests: data.tests || [], hooks: data.hooks || {} };
+  // Support hooks nested under "hooks" key or directly at root level
+  const hooks = data.hooks || {};
+  if (!hooks.beforeAll && data.beforeAll) hooks.beforeAll = data.beforeAll;
+  if (!hooks.afterAll && data.afterAll) hooks.afterAll = data.afterAll;
+  if (!hooks.beforeEach && data.beforeEach) hooks.beforeEach = data.beforeEach;
+  if (!hooks.afterEach && data.afterEach) hooks.afterEach = data.afterEach;
+  return { tests: data.tests || [], hooks };
 }
 
-/** Waits until the pool has capacity before connecting */
-async function waitForSlot(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
-  const start = Date.now();
-  while (Date.now() - start < maxWaitMs) {
-    try {
-      const status = await getPoolStatus(poolUrl);
-      if (status.available && status.running < status.maxConcurrent) {
-        return;
-      }
-      log('⏳', `${C.dim}Pool at capacity (${status.running}/${status.maxConcurrent}, ${status.queued} queued), waiting for slot...${C.reset}`);
-    } catch {
-      // Pool unreachable, let connectToPool handle the error
-      return;
-    }
-    await sleep(pollIntervalMs);
-  }
-  // Timeout — proceed anyway and let connectToPool deal with it
-  log('⚠️', `${C.yellow}Waited ${maxWaitMs / 1000}s for pool slot, proceeding anyway${C.reset}`);
+/** Extracts a value from an object using a dot-path (e.g. "data.token"). */
+function getByPath(obj, dotPath) {
+  return dotPath.split('.').reduce((o, key) => o?.[key], obj);
+}
+
+/** Fetches an auth token by POSTing credentials to a login endpoint. */
+function fetchAuthToken(endpoint, credentials, tokenPath) {
+  return new Promise((resolve, reject) => {
+    const url = new URL(endpoint);
+    const transport = url.protocol === 'https:' ? https : http;
+    const body = JSON.stringify(credentials);
+
+    const req = transport.request(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
+      timeout: 15000,
+    }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        if (res.statusCode < 200 || res.statusCode >= 300) {
+          return reject(new Error(`Auth login failed: HTTP ${res.statusCode} from ${endpoint}`));
+        }
+        try {
+          const json = JSON.parse(data);
+          const token = getByPath(json, tokenPath);
+          if (!token) {
+            return reject(new Error(`Auth login: token not found at path "${tokenPath}" in response`));
+          }
+          resolve(token);
+        } catch (e) {
+          reject(new Error(`Auth login: failed to parse response from ${endpoint}: ${e.message}`));
+        }
+      });
+    });
+    req.on('error', (e) => reject(new Error(`Auth login request failed: ${e.message}`)));
+    req.on('timeout', () => { req.destroy(); reject(new Error(`Auth login request timed out: ${endpoint}`)); });
+    req.end(body);
+  });
 }
 
 /** Runs a single test end-to-end */
 export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   let browser = null;
+  let context = null;
   let page = null;
 
   const result = {
@@ -85,16 +169,28 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   const pendingBodies = [];
 
   try {
-    await waitForSlot(config.poolUrl);
-    browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
-    page = await browser.newPage();
+    const chosenPool = await selectPool(getPoolUrls(config));
+    result.poolUrl = chosenPool;
+    const poolLabel = chosenPool.replace('ws://', '').replace('wss://', '');
+    const isMultiPool = getPoolUrls(config).length > 1;
+    if (isMultiPool) {
+      log('🔗', `${C.cyan}${test.name}${C.reset} ${C.dim}→ ${poolLabel}${C.reset}`);
+    }
+    progressFn({ event: 'test:pool', name: test.name, poolUrl: chosenPool });
+    browser = await connectToPool(chosenPool, config.connectRetries, config.connectRetryDelay);
+    // Use incognito context for cookie isolation between concurrent tests
+    context = await browser.createBrowserContext();
+    page = await context.newPage();
     await page.setViewport(config.viewport);
 
     page.on('console', (msg) => {
       result.consoleLogs.push({ type: msg.type(), text: msg.text() });
     });
     page.on('requestfailed', (req) => {
-      result.networkErrors.push({ url: req.url(), error: req.failure()?.errorText });
+      const url = req.url();
+      const ignoreDomains = config.networkIgnoreDomains || [];
+      if (ignoreDomains.length > 0 && ignoreDomains.some(d => url.includes(d))) return;
+      result.networkErrors.push({ url, error: req.failure()?.errorText });
     });
 
     const requestTimings = new Map();
@@ -128,44 +224,91 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       }
     });
 
+    // Auto-inject auth token into localStorage (runs BEFORE beforeEach hooks)
+    if (config.authToken) {
+      const storageKey = config.authStorageKey || 'accessToken';
+      await page.goto(config.baseUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
+      await page.evaluate((key, token) => {
+        localStorage.setItem(key, token);
+      }, storageKey, config.authToken);
+    }
+
+    // Resolve {{var.X}} and {{env.X}} in test actions and hooks
+    const vars = loadVarsForTest(config, config._suiteName);
+    if (Object.keys(vars).length > 0 || /\{\{(var|env)\./.test(JSON.stringify(test.actions))) {
+      test = { ...test, actions: resolveVarsInActions(test.actions, vars) };
+      if (hooks.beforeEach?.length) hooks = { ...hooks, beforeEach: resolveVarsInActions(hooks.beforeEach, vars) };
+      if (hooks.afterEach?.length) hooks = { ...hooks, afterEach: resolveVarsInActions(hooks.afterEach, vars) };
+    }
+
     // Run beforeEach hook
     if (hooks.beforeEach?.length) {
       await executeHookActions(page, hooks.beforeEach, config);
     }
 
+    // Auto-capture baseline screenshot if test has "expect" (BEFORE actions)
+    if (test.expect && page) {
+      try {
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const baselinePath = path.join(config.screenshotsDir, `baseline-${safeName}-${Date.now()}.png`);
+        await page.screenshot({ path: baselinePath, fullPage: true });
+        result.baselineScreenshot = baselinePath;
+      } catch { /* page may not be ready */ }
+    }
+
     for (let i = 0; i < test.actions.length; i++) {
       const action = test.actions[i];
-      const actionStart = Date.now();
-      try {
-        let actionResult;
-        if (action.type === 'assert_no_network_errors') {
-          // Handled inline — needs access to result.networkErrors
-          if (result.networkErrors.length > 0) {
-            const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
-            throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+      const maxActionRetries = action.retries ?? config.actionRetries ?? 0;
+      const actionRetryDelay = config.actionRetryDelay ?? 500;
+      let lastError = null;
+
+      for (let attempt = 0; attempt <= maxActionRetries; attempt++) {
+        const actionStart = Date.now();
+        try {
+          let actionResult;
+          if (action.type === 'assert_no_network_errors') {
+            // Handled inline — needs access to result.networkErrors
+            if (result.networkErrors.length > 0) {
+              const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+              throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+            }
+            actionResult = null;
+          } else {
+            actionResult = await executeAction(page, action, config);
+          }
+          const actionDuration = Date.now() - actionStart;
+          const actionEntry = {
+            ...action,
+            success: true,
+            duration: actionDuration,
+            result: actionResult,
+          };
+          if (attempt > 0) actionEntry.actionRetries = attempt;
+          actionEntry.narrative = narrateAction(action, actionEntry);
+          result.actions.push(actionEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, narrative: actionEntry.narrative, screenshotPath: actionResult?.screenshot || null });
+          lastError = null;
+          break;
+        } catch (error) {
+          lastError = error;
+          if (attempt < maxActionRetries) {
+            log('🔄', `${C.dim}Action ${action.type} retry ${attempt + 1}/${maxActionRetries} (${error.message})${C.reset}`);
+            await sleep(actionRetryDelay);
+            continue;
           }
-          actionResult = null;
-        } else {
-          actionResult = await executeAction(page, action, config);
+          const actionDuration = Date.now() - actionStart;
+          const failedEntry = {
+            ...action,
+            success: false,
+            duration: actionDuration,
+            error: error.message,
+          };
+          if (maxActionRetries > 0) failedEntry.actionRetries = attempt;
+          failedEntry.narrative = narrateAction(action, failedEntry);
+          result.actions.push(failedEntry);
+          progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, narrative: failedEntry.narrative, error: error.message });
+          throw error;
         }
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: true,
-          duration: actionDuration,
-          result: actionResult,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, screenshotPath: actionResult?.screenshot || null });
-      } catch (error) {
-        const actionDuration = Date.now() - actionStart;
-        result.actions.push({
-          ...action,
-          success: false,
-          duration: actionDuration,
-          error: error.message,
-        });
-        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, error: error.message });
-        throw error;
       }
     }
 
@@ -216,9 +359,16 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
     if (page) {
       try { result.finalUrl = page.url(); } catch { /* */ }
     }
+    if (context) {
+      try { await context.close(); } catch { /* */ }
+    }
     if (browser) {
       try { browser.disconnect(); } catch { /* */ }
     }
+    // Release local pending counter so selectPool() knows this slot is free
+    if (result.poolUrl) {
+      releasePending(result.poolUrl);
+    }
   }
 
   return result;
@@ -230,10 +380,17 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 
   // Run beforeAll hook
   if (hooks.beforeAll?.length) {
+    const stateActions = hooks.beforeAll.filter(a =>
+      ['evaluate', 'goto', 'navigate', 'clear_cookies', 'type', 'click', 'select'].includes(a.type)
+    );
+    if (stateActions.length > 0) {
+      log('⚠️', `${C.yellow}beforeAll runs on a separate browser — state from ${stateActions.map(a => a.type).join(', ')} will NOT carry over to tests. Use beforeEach instead.${C.reset}`);
+    }
     log('🪝', `${C.dim}Running beforeAll hook...${C.reset}`);
     let browser = null;
     try {
-      browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
+      const hookPool = await selectPool(getPoolUrls(config));
+      browser = await connectToPool(hookPool, config.connectRetries, config.connectRetryDelay);
       const page = await browser.newPage();
       await page.setViewport(config.viewport);
       await executeHookActions(page, hooks.beforeAll, config);
@@ -246,16 +403,40 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   }
 
+  // Auto-login: fetch auth token via API if configured and not already provided
+  if (config.authLoginEndpoint && !config.authToken && config.authCredentials) {
+    log('🔑', `${C.dim}Fetching auth token from ${config.authLoginEndpoint}...${C.reset}`);
+    try {
+      config.authToken = await fetchAuthToken(
+        config.authLoginEndpoint,
+        config.authCredentials,
+        config.authTokenPath || 'token'
+      );
+      log('✅', `${C.dim}Auth token acquired (${config.authToken.length} chars)${C.reset}`);
+    } catch (error) {
+      log('❌', `${C.red}Auth auto-login failed: ${error.message}${C.reset}`);
+      throw error;
+    }
+  }
+
   const concurrency = config.concurrency || 3;
   const _runId = Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
   const _proj = config.projectName || null;
   const _cwd = config._cwd || null;
   const _triggeredBy = config.triggeredBy || 'unknown';
   const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd, triggeredBy: _triggeredBy });
+
+  // Split serial and parallel tests
+  const parallelTests = tests.filter(t => !t.serial);
+  const serialTests = tests.filter(t => t.serial);
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}${serialTests.length} serial test(s) will run after parallel batch${C.reset}`);
+  }
+
   _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
 
   const results = [];
-  const queue = [...tests];
+  const queue = [...parallelTests];
   let activeCount = 0;
 
   const worker = async () => {
@@ -263,7 +444,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       const test = queue.shift();
       activeCount++;
       log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
-      _progress({ event: 'test:start', name: test.name, activeCount, queueRemaining: queue.length });
+      _progress({ event: 'test:start', name: test.name, serial: test.serial || false, activeCount, queueRemaining: queue.length });
 
       const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
       const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
@@ -276,7 +457,8 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         });
 
         try {
-          result = await Promise.race([runTest(test, config, hooks, _progress), timeoutPromise]);
+          const testHooks = test._suiteHooks ? mergeHooks(config.hooks, test._suiteHooks) : hooks;
+          result = await Promise.race([runTest(test, config, testHooks, _progress), timeoutPromise]);
         } catch (error) {
           result = {
             name: test.name,
@@ -304,7 +486,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       activeCount--;
 
       const screenshots = result.actions.filter(a => a.result?.screenshot).map(a => a.result.screenshot);
-      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots });
+      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots, poolUrl: result.poolUrl || null });
 
       if (result.success) {
         const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
@@ -325,17 +507,25 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   };
 
   const workers = [];
-  for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
+  for (let i = 0; i < Math.min(concurrency, parallelTests.length); i++) {
     workers.push(worker());
   }
   await Promise.all(workers);
 
+  // Run serial tests one at a time
+  if (serialTests.length > 0) {
+    log('🔒', `${C.dim}Running ${serialTests.length} serial test(s)...${C.reset}`);
+    queue.push(...serialTests);
+    await worker();
+  }
+
   // Run afterAll hook
   if (hooks.afterAll?.length) {
     log('🪝', `${C.dim}Running afterAll hook...${C.reset}`);
     let browser = null;
     try {
-      browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
+      const hookPool = await selectPool(getPoolUrls(config));
+      browser = await connectToPool(hookPool, config.connectRetries, config.connectRetryDelay);
       const page = await browser.newPage();
       await page.setViewport(config.viewport);
       await executeHookActions(page, hooks.afterAll, config);
@@ -357,16 +547,19 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 }
 
 /** Loads tests from a JSON file — returns { tests, hooks } */
-export function loadTestFile(filePath) {
+export function loadTestFile(filePath, modulesDir) {
   if (!fs.existsSync(filePath)) {
     throw new Error(`File not found: ${filePath}`);
   }
   const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  const resolved = modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
+  validateActionTypes(resolved, path.basename(filePath));
+  return resolved;
 }
 
 /** Loads a test suite by name — returns { tests, hooks } */
-export function loadTestSuite(suiteName, testsDir) {
+export function loadTestSuite(suiteName, testsDir, modulesDir) {
   // Match with or without numeric prefix (e.g. "agents" matches "03-agents.json")
   const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json'));
   const exact = files.find(f => f === `${suiteName}.json`);
@@ -378,29 +571,39 @@ export function loadTestSuite(suiteName, testsDir) {
   }
 
   const data = JSON.parse(fs.readFileSync(path.join(testsDir, match), 'utf-8'));
-  return normalizeTestData(data);
+  const normalized = normalizeTestData(data);
+  const resolved = modulesDir ? resolveTestData(normalized, modulesDir) : normalized;
+  validateActionTypes(resolved, match);
+  return resolved;
 }
 
 /** Loads all test suites from the tests directory — returns { tests, hooks } */
-export function loadAllSuites(testsDir) {
+export function loadAllSuites(testsDir, modulesDir, exclude = []) {
   if (!fs.existsSync(testsDir)) {
     throw new Error(`Tests directory not found: ${testsDir}`);
   }
 
-  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort()
+    .filter(f => !matchesExclude(f, exclude));
   let allTests = [];
-  let mergedHooks = {};
 
   for (const file of files) {
     const data = JSON.parse(fs.readFileSync(path.join(testsDir, file), 'utf-8'));
-    const { tests, hooks } = normalizeTestData(data);
+    let { tests, hooks } = normalizeTestData(data);
+    // Resolve modules per-suite before concatenating
+    if (modulesDir) {
+      ({ tests, hooks } = resolveTestData({ tests, hooks }, modulesDir));
+    }
+    validateActionTypes({ tests, hooks }, file);
+    // Tag each test with its own suite's hooks so they're preserved
+    for (const t of tests) {
+      t._suiteHooks = hooks;
+    }
     allTests = allTests.concat(tests);
-    // Last suite's hooks win for each non-empty key
-    mergedHooks = mergeHooks(mergedHooks, hooks);
     log('📋', `${C.cyan}${file}${C.reset} (${tests.length} tests)`);
   }
 
-  return { tests: allTests, hooks: mergedHooks };
+  return { tests: allTests, hooks: {} };
 }
 
 /** Lists all available test suites */