@matware/e2e-runner 1.3.0 → 1.3.1

package/bin/cli.js

@@ -264,7 +264,7 @@ async function cmdRun() {
 
   // Verify pool connectivity
   log('🔌', `Checking Chrome Pool${poolUrls.length > 1 ? 's' : ''}...`);
-  const pressure = await waitForAnyPool(poolUrls);
+  const pressure = await waitForAnyPool(poolUrls, 30000, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
   log('✅', `Pool ready (${pressure.running}/${pressure.maxConcurrent} sessions, queued: ${pressure.queued})`);
 
   // Wire up live progress to dashboard if running
@@ -351,7 +351,7 @@ async function cmdPool() {
 
     case 'status': {
       const statusPoolUrls = getPoolUrls(config);
-      const aggregated = await getAggregatedPoolStatus(statusPoolUrls);
+      const aggregated = await getAggregatedPoolStatus(statusPoolUrls, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
       console.log(`\n${C.bold}Chrome Pool Status:${C.reset}\n`);
 
       if (statusPoolUrls.length > 1) {
@@ -494,11 +494,12 @@ async function cmdCapture() {
 
   const capturePoolUrls = getPoolUrls(config);
   log('🔌', 'Checking Chrome Pool...');
-  await waitForAnyPool(capturePoolUrls);
+  const captureDriverOpts = { poolDriver: config.poolDriver, maxSessions: config.maxSessions };
+  await waitForAnyPool(capturePoolUrls, 30000, captureDriverOpts);
 
   let browser;
   try {
-    const capturePool = await selectPool(capturePoolUrls);
+    const capturePool = await selectPool(capturePoolUrls, 2000, 60000, captureDriverOpts);
     browser = await connectToPool(capturePool);
     const page = await browser.newPage();
     await page.setViewport(config.viewport);

package/commands/capture.md

@@ -0,0 +1,45 @@
+---
+description: Capture a screenshot of any URL with automatic authentication
+user_invocable: true
+allowed_tools:
+  - mcp__e2e-runner__e2e_pool_status
+  - mcp__e2e-runner__e2e_capture
+  - mcp__e2e-runner__e2e_analyze
+  - mcp__e2e-runner__e2e_screenshot
+---
+
+# Quick Capture
+
+Take a screenshot of any URL in one step. Handles pool checks and authentication automatically.
+
+## Workflow
+
+1. **Check pool** — Call `e2e_pool_status` to confirm the Chrome pool is running. If not available, tell the user to run `npx e2e-runner pool start` via CLI and stop.
+
+2. **Capture** — Call `e2e_capture` with:
+   - `url`: The URL from the user's request (REQUIRED)
+   - `cwd`: The current working directory (REQUIRED — always pass this)
+   - `fullPage`: true if user says "full page", "full", "complete", or "toda la página"
+   - `selector`: CSS selector if user wants to wait for a specific element
+   - `delay`: milliseconds if user says "wait", "delay", or "espera"
+   - `waitUntil`: "domcontentloaded" if user mentions WebSocket, SSE, or real-time apps
+   - `filename`: if user specifies a name
+
+   **Authentication is automatic**: the tool reads `authToken`, `authLoginEndpoint`, and `authCredentials` from the project's `e2e.config.js`. You do NOT need to pass `authToken` unless the user explicitly provides one.
+
+3. **Show result** — The tool returns the screenshot as an inline image. Show it to the user with the file path.
+
+## Arguments
+
+The user passes the URL after the command:
+- `/e2e-runner:capture http://localhost:3000/dashboard` → capture that URL
+- `/e2e-runner:capture http://localhost/concept-maps --full-page` → full page capture
+- `/e2e-runner:capture http://localhost/admin --delay 3000` → wait 3s before capture
+
+If no URL is provided, ask the user for one.
+
+## Important
+
+- Do NOT try to manually authenticate, fetch tokens, write test files, or use curl. The tool handles auth automatically from project config.
+- Do NOT use the `e2e_run` tool — this is a screenshot capture, not a test run.
+- Keep it simple: one `e2e_pool_status` call + one `e2e_capture` call. That's it.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matware/e2e-runner",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "mcpName": "io.github.fastslack/e2e-runner",
   "description": "E2E test runner using Chrome Pool (browserless/chrome) with parallel execution",
   "type": "module",

package/src/actions.js

@@ -8,6 +8,8 @@
  */
 
 import path from 'path';
+import fs from 'fs';
+import { assertVisualMatch } from './visual-diff.js';
 
 /** All recognized action types — single source of truth for validation. */
 export const KNOWN_ACTION_TYPES = new Set([
@@ -22,6 +24,8 @@ export const KNOWN_ACTION_TYPES = new Set([
   'set_storage', 'click_icon', 'click_menu_item', 'click_in_context',
   'assert_text_in', 'assert_no_text',
   'gql', 'wait_network_idle',
+  'open_tab', 'switch_tab', 'close_tab', 'assert_tab_count', 'wait_for_tab',
+  'assert_visual',
 ]);
 
 function sleep(ms) {
@@ -752,6 +756,153 @@ export async function executeAction(page, action, config) {
       break;
     }
 
+    // ── Visual regression ───────────────────────────────────────────────────
+
+    case 'assert_visual': {
+      // Compares a live screenshot against a golden reference image.
+      //
+      // value: golden image filename (relative to screenshotsDir or goldenDir) — required
+      // selector: optional CSS selector — screenshot only that element instead of full page
+      // text: optional max diff percentage as string, e.g. "0.02" (default: config.verificationThreshold or 0.02)
+      //
+      // Additional fields via action object:
+      //   fullPage: boolean (default: true)
+      //   maskRegions: [{ x, y, width, height }] — regions to ignore (timestamps, avatars, etc.)
+      //   threshold: number — pixel color sensitivity 0-1 (default: 0.1)
+      //
+      // Returns: { diffPercentage, differentPixels, totalPixels, diffImagePath, baselinePath, currentPath }
+
+      if (!value) throw new Error('assert_visual requires "value" (golden image filename)');
+
+      // Resolve golden image path
+      const goldenDir = config.goldenDir || path.join(config.screenshotsDir, 'golden');
+      const goldenPath = path.isAbsolute(value) ? value : path.join(goldenDir, value);
+
+      if (!fs.existsSync(goldenPath)) {
+        // First run: save current screenshot as the golden reference
+        if (!fs.existsSync(goldenDir)) fs.mkdirSync(goldenDir, { recursive: true });
+        const screenshotOpts = { path: goldenPath, fullPage: action.fullPage !== false };
+        if (selector) {
+          const el = await page.$(selector);
+          if (!el) throw new Error(`assert_visual: selector "${selector}" not found`);
+          await el.screenshot(screenshotOpts);
+        } else {
+          await page.screenshot(screenshotOpts);
+        }
+        return {
+          goldenCreated: true,
+          goldenPath,
+          message: `Golden image saved: ${path.basename(goldenPath)}. Re-run to compare.`,
+        };
+      }
+
+      // Capture current screenshot
+      const safeName = path.basename(value, path.extname(value));
+      const currentPath = path.join(screenshotsDir, `current-${safeName}-${Date.now()}.png`);
+      const screenshotOpts = { path: currentPath, fullPage: action.fullPage !== false };
+      if (selector) {
+        const el = await page.$(selector);
+        if (!el) throw new Error(`assert_visual: selector "${selector}" not found`);
+        await el.screenshot(screenshotOpts);
+      } else {
+        await page.screenshot(screenshotOpts);
+      }
+
+      // Compare
+      const maxDiff = text ? parseFloat(text) : (config.verificationThreshold || 0.02);
+      const diffPath = path.join(screenshotsDir, `diff-${safeName}-${Date.now()}.png`);
+      const compareResult = assertVisualMatch(goldenPath, currentPath, maxDiff, {
+        threshold: action.threshold || 0.1,
+        maskRegions: action.maskRegions || [],
+        diffOutputPath: diffPath,
+        includeAntiAlias: action.includeAntiAlias || false,
+      });
+
+      if (!compareResult.passed) {
+        const pct = (compareResult.diffPercentage * 100).toFixed(2);
+        const maxPct = (maxDiff * 100).toFixed(2);
+        throw new Error(
+          `assert_visual failed: ${pct}% pixels differ (threshold: ${maxPct}%). ` +
+          `${compareResult.differentPixels}/${compareResult.totalPixels} pixels changed. ` +
+          `Diff: ${path.basename(diffPath)}`
+        );
+      }
+
+      return {
+        diffPercentage: compareResult.diffPercentage,
+        differentPixels: compareResult.differentPixels,
+        totalPixels: compareResult.totalPixels,
+        diffImagePath: compareResult.diffImagePath,
+        baselinePath: goldenPath,
+        currentPath,
+        screenshot: diffPath,
+      };
+    }
+
+    // ── Multi-tab actions ─────────────────────────────────────────────────────
+    // These actions are intercepted by the runner (runTest) which manages the
+    // tab registry and swaps the active page. The actual tab lifecycle happens
+    // in runner.js — these cases handle the in-page parts only.
+
+    case 'open_tab': {
+      // Opens a new tab and navigates to the given URL.
+      // value: URL (absolute or relative to baseUrl) — required
+      // text: optional label for the tab (used by switch_tab)
+      // The runner intercepts this to create the page and register it.
+      // If we reach here, it means the runner already created the page and
+      // we just need to navigate.
+      const tabUrl = value.startsWith('http') ? value : `${baseUrl}${value}`;
+      await page.goto(tabUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
+      break;
+    }
+
+    case 'switch_tab': {
+      // Switches to another open tab. The runner handles the actual page swap.
+      // This case is a no-op — the runner already switched the page reference.
+      break;
+    }
+
+    case 'close_tab': {
+      // Closes the current tab. The runner handles page cleanup and switching.
+      // This case is a no-op — the runner closes the page and swaps back.
+      break;
+    }
+
+    case 'assert_tab_count': {
+      // Asserts the number of open tabs.
+      // value: expected count (number or operator expression like ">=2")
+      // The runner injects __tabCount into the action result before we get here.
+      // If we reach here directly, we use browser context pages.
+      const tabCount = action.__tabCount;
+      if (tabCount === undefined) {
+        throw new Error('assert_tab_count: tab count not available (action must be run via runner)');
+      }
+      const opMatch = value.match(/^(>=|<=|>|<)\s*(\d+)$/);
+      if (opMatch) {
+        const [, op, numStr] = opMatch;
+        const expected = parseInt(numStr);
+        const passed = op === '>' ? tabCount > expected
+          : op === '>=' ? tabCount >= expected
+          : op === '<' ? tabCount < expected
+          : tabCount <= expected;
+        if (!passed) {
+          throw new Error(`assert_tab_count failed: ${tabCount} tabs open, expected ${op}${expected}`);
+        }
+      } else {
+        const expected = parseInt(value);
+        if (tabCount !== expected) {
+          throw new Error(`assert_tab_count failed: ${tabCount} tabs open, expected ${expected}`);
+        }
+      }
+      break;
+    }
+
+    case 'wait_for_tab': {
+      // Waits for a new tab/popup to appear. The runner handles this.
+      // This case is a no-op — the runner already waited and registered the new tab.
+      break;
+    }
+
     default:
       throw new Error(`Unknown action type: "${type}"`);
   }

package/src/ai-generate.js

@@ -87,6 +87,16 @@ Smart interaction actions:
 - click_menu_item: click a menu item by text. Searches [role="menuitem"], .dropdown-item, .menu-item, [class*="MenuItem"]. Optional "selector" scopes the search
 - click_in_context: click a child element within a container identified by text. "text" finds the container, "selector" is the child to click. Picks the smallest matching container
 
+Visual regression:
+- assert_visual: compare current page against a golden reference screenshot. "value" is the golden filename (e.g. "login-page.png"). First run auto-saves the golden. "text" is optional max diff percentage (default "0.02" = 2%). "selector" captures only that element. "maskRegions" ignores dynamic areas: [{ "x": 10, "y": 5, "width": 200, "height": 30 }]. Example: { "type": "assert_visual", "value": "dashboard.png", "text": "0.05" }
+
+Multi-tab actions (for OAuth, popups, admin+user flows):
+- open_tab: open a new tab with URL in "value". Optional "text" assigns a label for switch_tab. Example: { "type": "open_tab", "value": "/admin", "text": "admin" }
+- switch_tab: switch to a tab by label, title regex, URL substring, or index. Example: { "type": "switch_tab", "value": "admin" }
+- close_tab: close current tab or a named tab ("value" = label). Automatically switches to previous tab. Example: { "type": "close_tab", "value": "admin" }
+- wait_for_tab: wait for a popup/new tab opened by the page (window.open, target=_blank). Optional "text" labels it. Example: { "type": "wait_for_tab", "text": "oauth" }
+- assert_tab_count: verify number of open tabs. "value" is count or operator. Example: { "type": "assert_tab_count", "value": "2" }
+
 Assertion action reference:
 - assert_text: checks if text appears anywhere in the page body
 - assert_element_text: checks textContent of a specific element (use "value": "exact" for strict match)
@@ -239,6 +249,77 @@ Existing suites: ${existingSuites.join(', ') || 'none'}`;
   };
 }
 
+/**
+ * Generates a hindsight hint for a failed test result.
+ * Sends the error + action context to Claude API and returns a concrete fix suggestion.
+ * Returns null if API key is unavailable or on any error.
+ */
+export async function generateHindsightHint(failedResult, config = {}) {
+  const apiKey = config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) return null;
+
+  const model = config.hintsModel || config.anthropicModel || 'claude-sonnet-4-5-20250929';
+  const lastActions = (failedResult.actions || []).slice(-8);
+  const failedAction = lastActions.find(a => a.success === false);
+
+  const consoleErrors = (failedResult.consoleLogs || [])
+    .filter(l => l.type === 'error')
+    .slice(-5)
+    .map(l => l.text);
+
+  const networkErrors = (failedResult.networkErrors || [])
+    .slice(-5)
+    .map(e => `${e.url} (${e.error})`);
+
+  const prompt = `Analyze this failed E2E test and suggest a concrete fix.
+
+TEST: "${failedResult.name}"
+ERROR: ${failedResult.error}
+
+LAST ACTIONS:
+${lastActions.map((a, i) => `  ${i + 1}. ${a.type}${a.selector ? ' selector=' + a.selector : ''}${a.text ? ' text=' + a.text : ''}${a.value ? ' value=' + (a.value.length > 80 ? a.value.slice(0, 80) + '...' : a.value) : ''} → ${a.success === false ? 'FAILED: ' + a.error : 'OK'} (${a.duration}ms)`).join('\n')}
+
+${failedAction ? `FAILED ACTION: ${JSON.stringify({ type: failedAction.type, selector: failedAction.selector, text: failedAction.text, value: failedAction.value?.slice?.(0, 200) })}` : ''}
+${consoleErrors.length ? `CONSOLE ERRORS:\n${consoleErrors.join('\n')}` : ''}
+${networkErrors.length ? `NETWORK ERRORS:\n${networkErrors.join('\n')}` : ''}
+
+Respond with ONLY a JSON object: { "suggestion": "concrete fix description", "confidence": "high"|"medium"|"low", "fixType": "selector"|"wait"|"timeout"|"logic"|"infra"|"data" }`;
+
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 15000);
+
+    const response = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 1024,
+        system: 'You are an E2E test debugging expert. Given a failed test, suggest the most likely fix. Be specific: name exact selectors, wait times, or code changes. Keep suggestions under 100 words.',
+        messages: [{ role: 'user', content: prompt }],
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) return null;
+    const result = await response.json();
+    const text = result.content?.[0]?.text;
+    if (!text) return null;
+
+    const cleaned = text.replace(/^```(?:json)?\s*\n?/m, '').replace(/\n?```\s*$/m, '').trim();
+    const hint = JSON.parse(cleaned);
+    return { test: failedResult.name, ...hint };
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Checks if the Anthropic API key is available.
  * @returns {boolean}