@matware/e2e-runner 1.1.0 → 1.1.1

package/src/mcp-tools.js

@@ -13,11 +13,11 @@ import path from 'path';
 import http from 'http';
 
 import { loadConfig } from './config.js';
-import { waitForPool, getPoolStatus } from './pool.js';
+import { waitForPool, getPoolStatus, connectToPool } from './pool.js';
 import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from './runner.js';
 import { generateReport, saveReport, persistRun } from './reporter.js';
 import { startDashboard, stopDashboard } from './dashboard.js';
-import { lookupScreenshotHash } from './db.js';
+import { lookupScreenshotHash, ensureProject, computeScreenshotHash, registerScreenshotHash } from './db.js';
 import { fetchIssue, checkCliAuth, detectProvider } from './issues.js';
 import { buildPrompt, hasApiKey } from './ai-generate.js';
 import { verifyIssue } from './verify.js';
@@ -56,6 +56,10 @@ export const TOOLS = [
           type: 'number',
           description: 'Number of retries for failed tests',
         },
+        failOnNetworkError: {
+          type: 'boolean',
+          description: 'Fail tests when network requests fail (e.g. ERR_CONNECTION_REFUSED). Default: false.',
+        },
         cwd: {
           type: 'string',
           description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
@@ -95,6 +99,7 @@ export const TOOLS = [
             type: 'object',
             properties: {
               name: { type: 'string', description: 'Test name' },
+              expect: { type: 'string', description: 'Human-readable description of the expected visual outcome. After the test runs, a verification screenshot is captured and Claude Code judges pass/fail against this description.' },
               actions: {
                 type: 'array',
                 description: 'Sequential browser actions',
@@ -206,6 +211,49 @@ export const TOOLS = [
           description:
             'prompt = return issue + prompt for Claude Code to create tests (default). verify = auto-generate tests via Claude API and run them.',
         },
+        authToken: {
+          type: 'string',
+          description: 'JWT or auth token to inject into localStorage before running tests (for authenticated apps)',
+        },
+        authStorageKey: {
+          type: 'string',
+          description: 'localStorage key name for the auth token (default: "accessToken")',
+        },
+        cwd: {
+          type: 'string',
+          description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
+        },
+      },
+      required: ['url'],
+    },
+  },
+  {
+    name: 'e2e_capture',
+    description:
+      'Capture a screenshot of any URL on demand. Connects to the Chrome pool, navigates to the URL, takes a screenshot, and returns the image with its ss:HASH.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: {
+          type: 'string',
+          description: 'Full URL to capture (e.g. "https://example.com" or "http://host.docker.internal:3000/dashboard")',
+        },
+        filename: {
+          type: 'string',
+          description: 'Output filename (default: capture-<timestamp>.png)',
+        },
+        fullPage: {
+          type: 'boolean',
+          description: 'Capture full scrollable page (default: false)',
+        },
+        selector: {
+          type: 'string',
+          description: 'Wait for this CSS selector before capturing',
+        },
+        delay: {
+          type: 'number',
+          description: 'Wait N milliseconds after page load before capturing (default: 0)',
+        },
         cwd: {
           type: 'string',
           description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
@@ -262,8 +310,10 @@ async function handleRun(args) {
   if (args.concurrency) configOverrides.concurrency = args.concurrency;
   if (args.baseUrl) configOverrides.baseUrl = args.baseUrl;
   if (args.retries !== undefined) configOverrides.retries = args.retries;
+  if (args.failOnNetworkError !== undefined) configOverrides.failOnNetworkError = args.failOnNetworkError;
 
   const config = await loadConfig(configOverrides, args.cwd);
+  config.triggeredBy = 'mcp';
 
   await waitForPool(config.poolUrl);
 
@@ -324,10 +374,28 @@ async function handleRun(args) {
     .filter(r => r.networkErrors?.length > 0)
     .map(r => ({ name: r.name, errors: r.networkErrors }));
 
+  const networkLogs = report.results
+    .filter(r => r.networkLogs?.length > 0)
+    .map(r => ({ name: r.name, requests: r.networkLogs }));
+
+  const verifications = report.results
+    .filter(r => r.expect && r.verificationScreenshot)
+    .map(r => ({
+      name: r.name,
+      expect: r.expect,
+      success: r.success,
+      screenshotHash: 'ss:' + computeScreenshotHash(r.verificationScreenshot),
+    }));
+
   if (flaky.length > 0) summary.flaky = flaky;
   if (failures.length > 0) summary.failures = failures;
   if (consoleErrors.length > 0) summary.consoleErrors = consoleErrors;
   if (networkErrors.length > 0) summary.networkErrors = networkErrors;
+  if (networkLogs.length > 0) summary.networkLogs = networkLogs;
+  if (verifications.length > 0) {
+    summary.verifications = verifications;
+    summary.verificationInstructions = 'For each verification, call e2e_screenshot with the screenshotHash to view the screenshot. Then compare what you see against the "expect" description. Report any mismatches as FAIL.';
+  }
 
   return textResult(JSON.stringify(summary, null, 2));
 }
@@ -441,6 +509,9 @@ async function handleIssue(args) {
       return errorResult('ANTHROPIC_API_KEY is required for verify mode. Set it as an environment variable.');
     }
 
+    if (args.authToken) config.authToken = args.authToken;
+    if (args.authStorageKey) config.authStorageKey = args.authStorageKey;
+
     const result = await verifyIssue(args.url, config);
     const status = result.bugConfirmed ? 'BUG CONFIRMED' : 'NOT REPRODUCIBLE';
     const summary = {
@@ -467,6 +538,62 @@ async function handleIssue(args) {
   return textResult(promptData.prompt);
 }
 
+async function handleCapture(args) {
+  if (!args.url) return errorResult('Missing required parameter: url');
+
+  const config = await loadConfig({}, args.cwd);
+
+  await waitForPool(config.poolUrl);
+
+  let browser;
+  try {
+    browser = await connectToPool(config.poolUrl);
+    const page = await browser.newPage();
+    await page.setViewport(config.viewport);
+    await page.goto(args.url, { waitUntil: 'networkidle2', timeout: 30000 });
+
+    if (args.selector) {
+      await page.waitForSelector(args.selector, { timeout: 10000 });
+    }
+
+    if (args.delay && args.delay > 0) {
+      await new Promise(r => setTimeout(r, args.delay));
+    }
+
+    // Build filename: sanitize and ensure .png
+    let filename = args.filename || `capture-${Date.now()}.png`;
+    filename = path.basename(filename);
+    if (!filename.endsWith('.png')) filename += '.png';
+
+    if (!fs.existsSync(config.screenshotsDir)) {
+      fs.mkdirSync(config.screenshotsDir, { recursive: true });
+    }
+
+    const screenshotPath = path.join(config.screenshotsDir, filename);
+    await page.screenshot({ path: screenshotPath, fullPage: !!args.fullPage });
+
+    // Register hash in SQLite
+    const cwd = args.cwd || process.cwd();
+    const projectName = config.projectName || path.basename(cwd);
+    const projectId = ensureProject(cwd, projectName, config.screenshotsDir, config.testsDir);
+    const hash = computeScreenshotHash(screenshotPath);
+    registerScreenshotHash(hash, screenshotPath, projectId, null);
+
+    // Read image for response
+    const data = fs.readFileSync(screenshotPath);
+    const base64 = data.toString('base64');
+
+    return {
+      content: [
+        { type: 'text', text: `Screenshot saved: ${screenshotPath}\nHash: ss:${hash}` },
+        { type: 'image', data: base64, mimeType: 'image/png' },
+      ],
+    };
+  } finally {
+    if (browser) browser.disconnect();
+  }
+}
+
 // Module-level state for stdio path only
 let dashboardHandle = null;
 
@@ -521,6 +648,8 @@ export async function dispatchTool(name, args = {}) {
       return await handleDashboardStop();
     case 'e2e_issue':
       return await handleIssue(args);
+    case 'e2e_capture':
+      return await handleCapture(args);
     default:
       return errorResult(`Unknown tool: ${name}`);
   }

package/src/reporter.js

@@ -152,7 +152,7 @@ export function persistRun(report, config, suiteName) {
 
   try {
     const projectId = ensureProject(config._cwd, config.projectName, config.screenshotsDir, config.testsDir);
-    saveRunToDb(projectId, report, runId, suiteName || null);
+    saveRunToDb(projectId, report, runId, suiteName || null, config.triggeredBy || null);
   } catch (err) {
     process.stderr.write(`[e2e-runner] SQLite write failed: ${err.message}\n`);
   }
@@ -210,6 +210,18 @@ export function printReport(report, screenshotsDir) {
     });
   }
 
+  const networkRequests = report.results.filter(r => r.networkLogs?.length > 0);
+  if (networkRequests.length > 0) {
+    console.log(`\n${C.cyan}${C.bold}NETWORK REQUESTS:${C.reset}`);
+    networkRequests.forEach(r => {
+      console.log(`  ${C.cyan}▸${C.reset} ${r.name}:`);
+      r.networkLogs.forEach(n => {
+        const statusColor = n.status < 300 ? C.green : n.status < 400 ? C.yellow : C.red;
+        console.log(`    ${C.dim}${n.method}${C.reset} ${statusColor}${n.status}${C.reset} ${n.url} ${C.dim}(${n.duration}ms)${C.reset}`);
+      });
+    });
+  }
+
   if (screenshotsDir) {
     console.log(`\n${C.dim}Report: ${path.join(screenshotsDir, 'report.json')}${C.reset}`);
     console.log(`${C.dim}Screenshots: ${screenshotsDir}${C.reset}\n`);

package/src/runner.js

@@ -80,7 +80,9 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
     error: null,
     consoleLogs: [],
     networkErrors: [],
+    networkLogs: [],
   };
+  const pendingBodies = [];
 
   try {
     await waitForSlot(config.poolUrl);
@@ -95,6 +97,37 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       result.networkErrors.push({ url: req.url(), error: req.failure()?.errorText });
     });
 
+    const requestTimings = new Map();
+    page.on('request', (req) => {
+      const rt = req.resourceType();
+      if (rt === 'xhr' || rt === 'fetch') requestTimings.set(req, Date.now());
+    });
+    page.on('response', (resp) => {
+      const req = resp.request();
+      const startMs = requestTimings.get(req);
+      if (startMs !== undefined) {
+        requestTimings.delete(req);
+        const entry = {
+          url: req.url(),
+          method: req.method(),
+          status: resp.status(),
+          statusText: resp.statusText(),
+          duration: Date.now() - startMs,
+          requestHeaders: req.headers(),
+          requestBody: null,
+          responseHeaders: resp.headers(),
+          responseBody: null,
+        };
+        try { entry.requestBody = req.postData() || null; } catch { /* */ }
+        result.networkLogs.push(entry);
+        // Read response body async — collect promise for later flush
+        const bodyPromise = resp.text().then(body => {
+          entry.responseBody = body && body.length > 51200 ? body.slice(0, 51200) + '\n...[truncated]' : body;
+        }).catch(() => { /* response may be unavailable */ });
+        pendingBodies.push(bodyPromise);
+      }
+    });
+
     // Run beforeEach hook
     if (hooks.beforeEach?.length) {
       await executeHookActions(page, hooks.beforeEach, config);
@@ -104,7 +137,17 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       const action = test.actions[i];
       const actionStart = Date.now();
       try {
-        const actionResult = await executeAction(page, action, config);
+        let actionResult;
+        if (action.type === 'assert_no_network_errors') {
+          // Handled inline — needs access to result.networkErrors
+          if (result.networkErrors.length > 0) {
+            const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+            throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+          }
+          actionResult = null;
+        } else {
+          actionResult = await executeAction(page, action, config);
+        }
         const actionDuration = Date.now() - actionStart;
         result.actions.push({
           ...action,
@@ -126,6 +169,23 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       }
     }
 
+    // Fail the test if failOnNetworkError is enabled and network errors occurred
+    if (config.failOnNetworkError && result.networkErrors.length > 0) {
+      const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+      throw new Error(`Network errors detected (failOnNetworkError=true): ${result.networkErrors.length} error(s): ${summary}`);
+    }
+
+    // Auto-capture verification screenshot if test has "expect"
+    if (test.expect && page) {
+      result.expect = test.expect;
+      try {
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const verifyPath = path.join(config.screenshotsDir, `verify-${safeName}-${Date.now()}.png`);
+        await page.screenshot({ path: verifyPath, fullPage: true });
+        result.verificationScreenshot = verifyPath;
+      } catch { /* page may be dead */ }
+    }
+
     // Run afterEach hook (success path)
     if (hooks.afterEach?.length) {
       await executeHookActions(page, hooks.afterEach, config);
@@ -148,6 +208,10 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
       } catch { /* page may be dead */ }
     }
   } finally {
+    // Flush pending response body reads before disconnecting
+    if (pendingBodies.length > 0) {
+      try { await Promise.allSettled(pendingBodies); } catch { /* */ }
+    }
     result.endTime = new Date().toISOString();
     if (page) {
       try { result.finalUrl = page.url(); } catch { /* */ }
@@ -186,7 +250,8 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
   const _runId = Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
   const _proj = config.projectName || null;
   const _cwd = config._cwd || null;
-  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd });
+  const _triggeredBy = config.triggeredBy || 'unknown';
+  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd, triggeredBy: _triggeredBy });
   _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
 
   const results = [];
@@ -222,6 +287,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
             error: error.message,
             consoleLogs: [],
             networkErrors: [],
+            networkLogs: [],
           };
         }
 
@@ -238,7 +304,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       activeCount--;
 
       const screenshots = result.actions.filter(a => a.result?.screenshot).map(a => a.result.screenshot);
-      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, errorScreenshot: result.errorScreenshot, screenshots });
+      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots });
 
       if (result.success) {
         const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';

package/src/verify.js

@@ -35,19 +35,31 @@ export async function verifyIssue(url, config) {
   fs.writeFileSync(testFile, JSON.stringify(tests, null, 2));
 
   try {
-    // 4. Wait for pool and run
+    // 4. Build hooks (inject auth if provided)
+    const hooks = {};
+    if (config.authToken) {
+      const storageKey = config.authStorageKey || 'accessToken';
+      hooks.beforeEach = [
+        { type: 'goto', value: '/' },
+        { type: 'evaluate', value: `localStorage.setItem('${storageKey}', '${config.authToken}')` },
+        { type: 'goto', value: '/' },
+        { type: 'wait', value: '1000' },
+      ];
+    }
+
+    // 5. Wait for pool and run
     await waitForPool(config.poolUrl);
-    const results = await runTestsParallel(tests, config, {});
+    const results = await runTestsParallel(tests, config, hooks);
     const report = generateReport(results);
     saveReport(report, config.screenshotsDir, config);
     persistRun(report, config, suiteName);
 
-    // 5. Interpret results
+    // 6. Interpret results
     const bugConfirmed = report.summary.failed > 0;
 
     return { issue, report, bugConfirmed, tests, suiteName };
   } finally {
-    // 6. Clean up temp file
+    // 7. Clean up temp file
     try { fs.unlinkSync(testFile); } catch { /* already gone */ }
   }
 }