@matware/e2e-runner 1.3.0 → 1.5.0

package/src/actions.js

@@ -8,6 +8,50 @@
  */
 
 import path from 'path';
+import fs from 'fs';
+import { assertVisualMatch } from './visual-diff.js';
+
+/**
+ * Returns false when the page has nothing useful to capture — used to
+ * skip screenshots that would otherwise be saved as pure-color PNGs
+ * (about:blank, fresh tab before navigation, DOM-only drivers that
+ * never paint, etc). Fails open: on any evaluation error we assume
+ * there *is* content so we don't lose legitimate captures.
+ */
+export async function pageHasRenderableContent(page) {
+  try {
+    const url = page.url();
+    if (!url || url === 'about:blank' || url === 'about:srcdoc') return false;
+    return await page
+      .evaluate(() => {
+        if (!document.body) return false;
+        if (document.body.children.length > 0) return true;
+        return (document.body.innerText || '').trim().length > 0;
+      })
+      .catch(() => true);
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * Heuristic post-capture guard: PNGs compressed under this size at
+ * typical viewport resolutions are almost certainly near-uniform color
+ * (about:blank, default Chrome BG, broken render). Catches cases the
+ * pre-capture DOM check can't — e.g. browserless rendering example.com
+ * to a 99%-gray frame even though navigation succeeded.
+ *
+ * 20 KB sits cleanly between the observed blank cluster (5 KB – 18 KB)
+ * and the smallest real captures in this project (~23 KB+).
+ */
+export const BLANK_PNG_BYTE_THRESHOLD = 20000;
+export const BLANK_JPEG_BYTE_THRESHOLD = 8000;
+
+export function looksLikeBlankCapture(buf, format = 'png') {
+  if (!Buffer.isBuffer(buf)) return false;
+  const threshold = format === 'jpeg' ? BLANK_JPEG_BYTE_THRESHOLD : BLANK_PNG_BYTE_THRESHOLD;
+  return buf.length < threshold;
+}
 
 /** All recognized action types — single source of truth for validation. */
 export const KNOWN_ACTION_TYPES = new Set([
@@ -18,10 +62,12 @@ export const KNOWN_ACTION_TYPES = new Set([
   'assert_no_network_errors', 'assert_storage',
   'get_text', 'select', 'clear', 'clear_cookies', 'press', 'scroll', 'hover',
   'navigate', 'evaluate',
-  'type_react', 'click_regex', 'click_option', 'focus_autocomplete', 'click_chip',
+  'type_react', 'click_regex', 'click_option', 'select_combobox', 'focus_autocomplete', 'click_chip',
   'set_storage', 'click_icon', 'click_menu_item', 'click_in_context',
   'assert_text_in', 'assert_no_text',
   'gql', 'wait_network_idle',
+  'open_tab', 'switch_tab', 'close_tab', 'assert_tab_count', 'wait_for_tab',
+  'assert_visual',
 ]);
 
 function sleep(ms) {
@@ -46,16 +92,35 @@ export async function executeAction(page, action, config) {
         await page.click(selector);
       } else if (text) {
         const clickTextSelector = 'button, a, [role="button"], [role="tab"], [role="menuitem"], [role="option"], [role="listitem"], div[class*="cursor"], span, li, td, th, label, p, h1, h2, h3, h4, h5, h6, dd, dt';
+        // Optional refinements (backward-compatible — defaults match old behavior):
+        //   scope: "dialog" → only match inside an open [role=dialog]/MuiDialog
+        //   visible: true   → skip hidden/zero-size matches (implied by scope:dialog)
+        //   last: true      → click the LAST match instead of the first
+        const scopeSel = action.scope === 'dialog' ? '[role="dialog"], .MuiDialog-root' : null;
+        const wantVisible = action.visible === true || action.scope === 'dialog';
+        const wantLast = action.last === true;
         await page.waitForFunction(
-          (t, sel) => [...document.querySelectorAll(sel)]
-            .find(el => el.textContent.includes(t)),
+          (t, sel, scope, vis) => {
+            const roots = scope ? [...document.querySelectorAll(scope)] : [document];
+            const isVis = el => { if (!vis) return true; const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'; };
+            for (const root of roots) {
+              if ([...root.querySelectorAll(sel)].some(el => el.textContent.includes(t) && isVis(el))) return true;
+            }
+            return false;
+          },
           { timeout },
-          text, clickTextSelector
+          text, clickTextSelector, scopeSel, wantVisible
         );
-        await page.$$eval(clickTextSelector, (els, t) => {
-          const el = els.find(e => e.textContent.includes(t));
-          if (el) el.click();
-        }, text);
+        const clicked = await page.evaluate((t, sel, scope, vis, last) => {
+          const roots = scope ? [...document.querySelectorAll(scope)] : [document];
+          const isVis = el => { if (!vis) return true; const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.display !== 'none' && s.visibility !== 'hidden'; };
+          const matches = [];
+          for (const root of roots) matches.push(...[...root.querySelectorAll(sel)].filter(el => el.textContent.includes(t) && isVis(el)));
+          const el = last ? matches[matches.length - 1] : matches[0];
+          if (el) { el.click(); return true; }
+          return false;
+        }, text, clickTextSelector, scopeSel, wantVisible, wantLast);
+        if (!clicked) throw new Error(`click failed: no element containing "${text}"${scopeSel ? ' in an open dialog' : ''} found`);
       }
       break;
 
@@ -67,8 +132,34 @@ export async function executeAction(page, action, config) {
       await page.type(selector, value, { delay: 20 });
       break;
 
-    case 'wait':
-      if (selector) {
+    case 'wait': {
+      // Condition waits (preferred over fixed sleeps):
+      //   { selector }            → wait until it appears
+      //   { text }                → wait until text appears in the page
+      //   { gone: "<css>" }       → wait until that selector disappears/hides (e.g. spinner)
+      //   { gone: true, selector }→ same, selector form
+      //   { gone: true, text }    → wait until text disappears
+      //   { value: "<ms>" }       → fixed sleep (last resort)
+      const goneSel = typeof action.gone === 'string' ? action.gone : (action.gone === true ? selector : null);
+      const goneTxt = action.gone === true && !selector ? text : null;
+      if (goneSel) {
+        try {
+          await page.waitForFunction((sel) => {
+            const el = document.querySelector(sel);
+            if (!el) return true;
+            const r = el.getBoundingClientRect(); const s = getComputedStyle(el);
+            return (r.width === 0 && r.height === 0) || s.display === 'none' || s.visibility === 'hidden' || s.opacity === '0';
+          }, { timeout }, goneSel);
+        } catch (e) {
+          throw new Error(`wait failed: "${goneSel}" still present/visible after ${timeout}ms`);
+        }
+      } else if (goneTxt) {
+        try {
+          await page.waitForFunction((t) => !document.body.innerText.includes(t), { timeout }, goneTxt);
+        } catch (e) {
+          throw new Error(`wait failed: text "${goneTxt}" still present after ${timeout}ms`);
+        }
+      } else if (selector) {
         try {
           await page.waitForSelector(selector, { timeout });
         } catch (e) {
@@ -88,6 +179,7 @@ export async function executeAction(page, action, config) {
         await sleep(parseInt(value));
       }
       break;
+    }
 
     case 'screenshot': {
       let filename = value || `screenshot-${Date.now()}.png`;
@@ -104,7 +196,20 @@ export async function executeAction(page, action, config) {
         filename = `${base}-${Date.now()}${ext}`;
       }
       const filepath = path.join(screenshotsDir, filename);
-      await page.screenshot({ path: filepath, fullPage: action.fullPage || false });
+      // Skip capture when page is at about:blank or DOM is empty — these
+      // produce uniform-color PNGs that pollute screenshotsDir with no
+      // diagnostic value.
+      if (!(await pageHasRenderableContent(page))) {
+        return { screenshot: null, skipped: 'blank-page' };
+      }
+      // Capture to buffer first so we can post-filter near-uniform frames
+      // (e.g. browserless returning a 99%-gray render). Only persist if
+      // the encoded PNG carries enough entropy to be informative.
+      const ssBuf = await page.screenshot({ fullPage: action.fullPage || false });
+      if (looksLikeBlankCapture(ssBuf, 'png')) {
+        return { screenshot: null, skipped: 'blank-render', bytes: ssBuf.length };
+      }
+      fs.writeFileSync(filepath, ssBuf);
       return { screenshot: filepath };
     }
 
@@ -352,8 +457,11 @@ export async function executeAction(page, action, config) {
     case 'type_react': {
       // Types into React controlled inputs using the native value setter.
       // This bypasses React's synthetic event system which ignores programmatic .value changes.
+      // Optional: blur (commit on blur for fields that validate then),
+      //           waitAfter (ms to wait after — e.g. for debounced autocomplete dropdowns).
       await page.waitForSelector(selector, { timeout });
-      await page.evaluate((sel, val) => {
+      const trBlur = action.blur === true;
+      await page.evaluate((sel, val, doBlur) => {
         const input = document.querySelector(sel);
         if (!input) throw new Error(`type_react: element "${sel}" not found`);
         const proto = input instanceof HTMLTextAreaElement
@@ -363,11 +471,13 @@ export async function executeAction(page, action, config) {
         if (!descriptor || !descriptor.set) {
           throw new Error(`type_react: element "${sel}" has no writable value property`);
         }
+        input.focus();
         descriptor.set.call(input, val);
         input.dispatchEvent(new Event('input', { bubbles: true }));
         input.dispatchEvent(new Event('change', { bubbles: true }));
-        input.focus();
-      }, selector, value);
+        if (doBlur) input.blur();
+      }, selector, value, trBlur);
+      if (action.waitAfter) await sleep(parseInt(action.waitAfter));
       break;
     }
 
@@ -414,6 +524,56 @@ export async function executeAction(page, action, config) {
       break;
     }
 
+    case 'select_combobox': {
+      // Open a MUI Autocomplete / Select, optionally type to filter, then click the
+      // option matching `text` (case-insensitive substring). Falls back across
+      // [role=option], MuiAutocomplete-option and MuiMenuItem so it works for both
+      // Autocomplete listboxes and Select dropdowns.
+      //   selector: combobox input (default input[role='combobox'])
+      //   text:     option to pick (required)
+      //   filter:   text typed into the input before picking (optional)
+      //   openWait/filterWait: ms tuning for async/debounced option loaders
+      const cbInput = selector || "input[role='combobox']";
+      const cbOption = text || action.option;
+      if (!cbOption) throw new Error("select_combobox requires 'text' (option to pick)");
+      const cbFilter = action.filter || '';
+      const cbOpenWait = action.openWait ? parseInt(action.openWait) : 400;
+      const cbFilterWait = action.filterWait ? parseInt(action.filterWait) : 600;
+      await page.waitForSelector(cbInput, { timeout });
+      await page.evaluate((sel, flt) => {
+        const input = document.querySelector(sel);
+        if (!input) throw new Error(`select_combobox: input "${sel}" not found`);
+        input.focus();
+        if (typeof input.click === 'function') input.click();
+        if (flt) {
+          const proto = input instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
+          const setter = Object.getOwnPropertyDescriptor(proto, 'value').set;
+          setter.call(input, flt);
+          input.dispatchEvent(new Event('input', { bubbles: true }));
+          input.dispatchEvent(new Event('change', { bubbles: true }));
+        }
+      }, cbInput, cbFilter);
+      await sleep(cbFilter ? cbFilterWait : cbOpenWait);
+      const cbOptionSel = '[role="option"], .MuiAutocomplete-option, li.MuiMenuItem-root, .MuiList-root li';
+      try {
+        await page.waitForFunction(
+          (sels, t) => [...document.querySelectorAll(sels)].some(o => (o.textContent || '').toLowerCase().includes(t.toLowerCase())),
+          { timeout }, cbOptionSel, cbOption
+        );
+      } catch (e) {
+        throw new Error(`select_combobox: no option matching "${cbOption}" appeared (filter="${cbFilter}")`);
+      }
+      const cbPicked = await page.evaluate((sels, t) => {
+        const c = [...document.querySelectorAll(sels)];
+        const m = c.find(o => (o.textContent || '').toLowerCase().includes(t.toLowerCase()));
+        if (m) { m.click(); return (m.textContent || '').trim().slice(0, 80); }
+        return null;
+      }, cbOptionSel, cbOption);
+      if (cbPicked === null) throw new Error(`select_combobox: option "${cbOption}" vanished before click`);
+      if (action.waitAfter) await sleep(parseInt(action.waitAfter));
+      break;
+    }
+
     case 'focus_autocomplete': {
       // Focus an autocomplete/combobox input by its label text.
       // Supports MUI Autocomplete (.MuiAutocomplete-root) and generic [role="combobox"].
@@ -752,6 +912,153 @@ export async function executeAction(page, action, config) {
       break;
     }
 
+    // ── Visual regression ───────────────────────────────────────────────────
+
+    case 'assert_visual': {
+      // Compares a live screenshot against a golden reference image.
+      //
+      // value: golden image filename (relative to screenshotsDir or goldenDir) — required
+      // selector: optional CSS selector — screenshot only that element instead of full page
+      // text: optional max diff percentage as string, e.g. "0.02" (default: config.verificationThreshold or 0.02)
+      //
+      // Additional fields via action object:
+      //   fullPage: boolean (default: true)
+      //   maskRegions: [{ x, y, width, height }] — regions to ignore (timestamps, avatars, etc.)
+      //   threshold: number — pixel color sensitivity 0-1 (default: 0.1)
+      //
+      // Returns: { diffPercentage, differentPixels, totalPixels, diffImagePath, baselinePath, currentPath }
+
+      if (!value) throw new Error('assert_visual requires "value" (golden image filename)');
+
+      // Resolve golden image path
+      const goldenDir = config.goldenDir || path.join(config.screenshotsDir, 'golden');
+      const goldenPath = path.isAbsolute(value) ? value : path.join(goldenDir, value);
+
+      if (!fs.existsSync(goldenPath)) {
+        // First run: save current screenshot as the golden reference
+        if (!fs.existsSync(goldenDir)) fs.mkdirSync(goldenDir, { recursive: true });
+        const screenshotOpts = { path: goldenPath, fullPage: action.fullPage !== false };
+        if (selector) {
+          const el = await page.$(selector);
+          if (!el) throw new Error(`assert_visual: selector "${selector}" not found`);
+          await el.screenshot(screenshotOpts);
+        } else {
+          await page.screenshot(screenshotOpts);
+        }
+        return {
+          goldenCreated: true,
+          goldenPath,
+          message: `Golden image saved: ${path.basename(goldenPath)}. Re-run to compare.`,
+        };
+      }
+
+      // Capture current screenshot
+      const safeName = path.basename(value, path.extname(value));
+      const currentPath = path.join(screenshotsDir, `current-${safeName}-${Date.now()}.png`);
+      const screenshotOpts = { path: currentPath, fullPage: action.fullPage !== false };
+      if (selector) {
+        const el = await page.$(selector);
+        if (!el) throw new Error(`assert_visual: selector "${selector}" not found`);
+        await el.screenshot(screenshotOpts);
+      } else {
+        await page.screenshot(screenshotOpts);
+      }
+
+      // Compare
+      const maxDiff = text ? parseFloat(text) : (config.verificationThreshold || 0.02);
+      const diffPath = path.join(screenshotsDir, `diff-${safeName}-${Date.now()}.png`);
+      const compareResult = assertVisualMatch(goldenPath, currentPath, maxDiff, {
+        threshold: action.threshold || 0.1,
+        maskRegions: action.maskRegions || [],
+        diffOutputPath: diffPath,
+        includeAntiAlias: action.includeAntiAlias || false,
+      });
+
+      if (!compareResult.passed) {
+        const pct = (compareResult.diffPercentage * 100).toFixed(2);
+        const maxPct = (maxDiff * 100).toFixed(2);
+        throw new Error(
+          `assert_visual failed: ${pct}% pixels differ (threshold: ${maxPct}%). ` +
+          `${compareResult.differentPixels}/${compareResult.totalPixels} pixels changed. ` +
+          `Diff: ${path.basename(diffPath)}`
+        );
+      }
+
+      return {
+        diffPercentage: compareResult.diffPercentage,
+        differentPixels: compareResult.differentPixels,
+        totalPixels: compareResult.totalPixels,
+        diffImagePath: compareResult.diffImagePath,
+        baselinePath: goldenPath,
+        currentPath,
+        screenshot: diffPath,
+      };
+    }
+
+    // ── Multi-tab actions ─────────────────────────────────────────────────────
+    // These actions are intercepted by the runner (runTest) which manages the
+    // tab registry and swaps the active page. The actual tab lifecycle happens
+    // in runner.js — these cases handle the in-page parts only.
+
+    case 'open_tab': {
+      // Opens a new tab and navigates to the given URL.
+      // value: URL (absolute or relative to baseUrl) — required
+      // text: optional label for the tab (used by switch_tab)
+      // The runner intercepts this to create the page and register it.
+      // If we reach here, it means the runner already created the page and
+      // we just need to navigate.
+      const tabUrl = value.startsWith('http') ? value : `${baseUrl}${value}`;
+      await page.goto(tabUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
+      break;
+    }
+
+    case 'switch_tab': {
+      // Switches to another open tab. The runner handles the actual page swap.
+      // This case is a no-op — the runner already switched the page reference.
+      break;
+    }
+
+    case 'close_tab': {
+      // Closes the current tab. The runner handles page cleanup and switching.
+      // This case is a no-op — the runner closes the page and swaps back.
+      break;
+    }
+
+    case 'assert_tab_count': {
+      // Asserts the number of open tabs.
+      // value: expected count (number or operator expression like ">=2")
+      // The runner injects __tabCount into the action result before we get here.
+      // If we reach here directly, we use browser context pages.
+      const tabCount = action.__tabCount;
+      if (tabCount === undefined) {
+        throw new Error('assert_tab_count: tab count not available (action must be run via runner)');
+      }
+      const opMatch = value.match(/^(>=|<=|>|<)\s*(\d+)$/);
+      if (opMatch) {
+        const [, op, numStr] = opMatch;
+        const expected = parseInt(numStr);
+        const passed = op === '>' ? tabCount > expected
+          : op === '>=' ? tabCount >= expected
+          : op === '<' ? tabCount < expected
+          : tabCount <= expected;
+        if (!passed) {
+          throw new Error(`assert_tab_count failed: ${tabCount} tabs open, expected ${op}${expected}`);
+        }
+      } else {
+        const expected = parseInt(value);
+        if (tabCount !== expected) {
+          throw new Error(`assert_tab_count failed: ${tabCount} tabs open, expected ${expected}`);
+        }
+      }
+      break;
+    }
+
+    case 'wait_for_tab': {
+      // Waits for a new tab/popup to appear. The runner handles this.
+      // This case is a no-op — the runner already waited and registered the new tab.
+      break;
+    }
+
     default:
       throw new Error(`Unknown action type: "${type}"`);
   }

package/src/ai-generate.js

@@ -87,6 +87,16 @@ Smart interaction actions:
 - click_menu_item: click a menu item by text. Searches [role="menuitem"], .dropdown-item, .menu-item, [class*="MenuItem"]. Optional "selector" scopes the search
 - click_in_context: click a child element within a container identified by text. "text" finds the container, "selector" is the child to click. Picks the smallest matching container
 
+Visual regression:
+- assert_visual: compare current page against a golden reference screenshot. "value" is the golden filename (e.g. "login-page.png"). First run auto-saves the golden. "text" is optional max diff percentage (default "0.02" = 2%). "selector" captures only that element. "maskRegions" ignores dynamic areas: [{ "x": 10, "y": 5, "width": 200, "height": 30 }]. Example: { "type": "assert_visual", "value": "dashboard.png", "text": "0.05" }
+
+Multi-tab actions (for OAuth, popups, admin+user flows):
+- open_tab: open a new tab with URL in "value". Optional "text" assigns a label for switch_tab. Example: { "type": "open_tab", "value": "/admin", "text": "admin" }
+- switch_tab: switch to a tab by label, title regex, URL substring, or index. Example: { "type": "switch_tab", "value": "admin" }
+- close_tab: close current tab or a named tab ("value" = label). Automatically switches to previous tab. Example: { "type": "close_tab", "value": "admin" }
+- wait_for_tab: wait for a popup/new tab opened by the page (window.open, target=_blank). Optional "text" labels it. Example: { "type": "wait_for_tab", "text": "oauth" }
+- assert_tab_count: verify number of open tabs. "value" is count or operator. Example: { "type": "assert_tab_count", "value": "2" }
+
 Assertion action reference:
 - assert_text: checks if text appears anywhere in the page body
 - assert_element_text: checks textContent of a specific element (use "value": "exact" for strict match)
@@ -239,6 +249,77 @@ Existing suites: ${existingSuites.join(', ') || 'none'}`;
   };
 }
 
+/**
+ * Generates a hindsight hint for a failed test result.
+ * Sends the error + action context to Claude API and returns a concrete fix suggestion.
+ * Returns null if API key is unavailable or on any error.
+ */
+export async function generateHindsightHint(failedResult, config = {}) {
+  const apiKey = config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) return null;
+
+  const model = config.hintsModel || config.anthropicModel || 'claude-sonnet-4-5-20250929';
+  const lastActions = (failedResult.actions || []).slice(-8);
+  const failedAction = lastActions.find(a => a.success === false);
+
+  const consoleErrors = (failedResult.consoleLogs || [])
+    .filter(l => l.type === 'error')
+    .slice(-5)
+    .map(l => l.text);
+
+  const networkErrors = (failedResult.networkErrors || [])
+    .slice(-5)
+    .map(e => `${e.url} (${e.error})`);
+
+  const prompt = `Analyze this failed E2E test and suggest a concrete fix.
+
+TEST: "${failedResult.name}"
+ERROR: ${failedResult.error}
+
+LAST ACTIONS:
+${lastActions.map((a, i) => `  ${i + 1}. ${a.type}${a.selector ? ' selector=' + a.selector : ''}${a.text ? ' text=' + a.text : ''}${a.value ? ' value=' + (a.value.length > 80 ? a.value.slice(0, 80) + '...' : a.value) : ''} → ${a.success === false ? 'FAILED: ' + a.error : 'OK'} (${a.duration}ms)`).join('\n')}
+
+${failedAction ? `FAILED ACTION: ${JSON.stringify({ type: failedAction.type, selector: failedAction.selector, text: failedAction.text, value: failedAction.value?.slice?.(0, 200) })}` : ''}
+${consoleErrors.length ? `CONSOLE ERRORS:\n${consoleErrors.join('\n')}` : ''}
+${networkErrors.length ? `NETWORK ERRORS:\n${networkErrors.join('\n')}` : ''}
+
+Respond with ONLY a JSON object: { "suggestion": "concrete fix description", "confidence": "high"|"medium"|"low", "fixType": "selector"|"wait"|"timeout"|"logic"|"infra"|"data" }`;
+
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 15000);
+
+    const response = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 1024,
+        system: 'You are an E2E test debugging expert. Given a failed test, suggest the most likely fix. Be specific: name exact selectors, wait times, or code changes. Keep suggestions under 100 words.',
+        messages: [{ role: 'user', content: prompt }],
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) return null;
+    const result = await response.json();
+    const text = result.content?.[0]?.text;
+    if (!text) return null;
+
+    const cleaned = text.replace(/^```(?:json)?\s*\n?/m, '').replace(/\n?```\s*$/m, '').trim();
+    const hint = JSON.parse(cleaned);
+    return { test: failedResult.name, ...hint };
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Checks if the Anthropic API key is available.
  * @returns {boolean}