@matware/e2e-runner 1.5.0 → 1.5.1

package/agents/test-improver.md

@@ -24,7 +24,7 @@ You are a specialist in refactoring and optimizing existing E2E tests without ch
 
 - **Evaluate replacement**: Replace verbose `evaluate` actions with equivalent built-in actions (`type_react`, `click_option`, `assert_element_text`, etc.)
 - **Duplication extraction**: Identify repeated action sequences across tests and extract them into reusable modules (`$use`)
-- **Selector hardening**: Replace brittle selectors (nth-child, deep nesting, generated classes) with stable alternatives (`data-testid`, `id`, text-based)
+- **Selector hardening**: Replace brittle selectors (nth-child, deep nesting, generated classes) with stable alternatives (`data-testid`, `id`, text-based). Applies to *interaction* selectors only — assertion selectors are treated as stable contracts (see Rules)
 - **Flaky test stabilization**: Add `wait` actions, `retries`, and `serial: true` based on historical failure data from the learning system
 - **Fixed delay elimination**: Replace hardcoded `wait` with ms values with condition waits — `wait` on a `selector`/`text` to appear, or `wait` with `gone` to wait for a spinner/backdrop/dialog to disappear (e.g. `{ "type": "wait", "gone": ".MuiBackdrop-root" }`)
 - **Visual verification**: Add `expect` fields to tests that lack visual verification
@@ -167,6 +167,7 @@ When extracting to a module, use `{{param}}` placeholders for values that vary b
 4. **Preserve test ordering** — don't reorder tests within a suite. Numeric prefix ordering is intentional.
 5. **Keep evaluates when no built-in exists** — if the evaluate does something that no built-in action covers (e.g., complex DOM manipulation, localStorage checks), leave it as-is.
 6. **Prefer selector waits over fixed delays** — replace `{ "type": "wait", "value": "3000" }` with `{ "type": "wait", "selector": ".expected-element" }` when possible. Only keep fixed delays when there's genuinely no element to wait for.
+7. **Assertion selectors are the contract** — interaction selectors may heal, but never retarget the `selector` of an `assert_*` action while hardening: that silently changes *what* the test verifies. If an assertion selector is genuinely broken, pin it to a stable `data-testid` and call it out in the summary instead of swapping it for whatever makes the test green.
 
 ## Output
 

package/bin/cli.js

@@ -35,6 +35,7 @@ import { loadConfig } from '../src/config.js';
 import { startPool, stopPool, restartPool, connectToPool } from '../src/pool.js';
 import { getPoolUrls, getAggregatedPoolStatus, waitForAnyPool, selectPool } from '../src/pool-manager.js';
 import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from '../src/runner.js';
+import { looksLikeBlankCapture } from '../src/actions.js';
 import { generateReport, saveReport, printReport, persistRun, printInsights } from '../src/reporter.js';
 import { startDashboard } from '../src/dashboard.js';
 import { startWatch } from '../src/watch.js';
@@ -152,6 +153,7 @@ ${C.bold}Usage:${C.reset}
   e2e-runner capture <url> --selector <sel>  Wait for selector before capture
   e2e-runner capture <url> --delay <ms> Wait before capturing
   e2e-runner capture <url> --filename <name> Custom filename
+  e2e-runner capture <url> --force      Save even if the frame is blank
 
   e2e-runner issue <url>                Fetch issue and show details
   e2e-runner issue <url> --generate     Generate test file via Claude API
@@ -546,7 +548,7 @@ async function cmdDashboard() {
 async function cmdCapture() {
   const url = args[1];
   if (!url || url.startsWith('--')) {
-    console.error(`${C.red}Usage: e2e-runner capture <url> [--filename <name>] [--full-page] [--selector <sel>] [--delay <ms>]${C.reset}`);
+    console.error(`${C.red}Usage: e2e-runner capture <url> [--filename <name>] [--full-page] [--selector <sel>] [--delay <ms>] [--force]${C.reset}`);
     process.exit(1);
   }
 
@@ -592,7 +594,16 @@ async function cmdCapture() {
 
     const screenshotPath = path.join(config.screenshotsDir, filename);
     const fullPage = hasFlag('--full-page');
-    await page.screenshot({ path: screenshotPath, fullPage });
+    const captureBuf = await page.screenshot({ fullPage });
+
+    // Blank frame (uniform color — page never rendered): skip the save
+    // unless the user explicitly forces it.
+    if (!hasFlag('--force') && looksLikeBlankCapture(captureBuf, 'png')) {
+      log('⚠️', `${C.yellow}Capture skipped:${C.reset} page rendered a blank (uniform-color) frame — nothing saved. Use ${C.dim}--force${C.reset} to save anyway.`);
+      console.log('');
+      return;
+    }
+    fs.writeFileSync(screenshotPath, captureBuf);
 
     // Register hash in SQLite
     const cwd = process.cwd();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matware/e2e-runner",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "mcpName": "io.github.fastslack/e2e-runner",
   "description": "E2E test runner using Chrome Pool (browserless/chrome) with parallel execution",
   "type": "module",
@@ -39,7 +39,7 @@
     "github-issues",
     "ai-testing"
   ],
-  "author": "Matware",
+  "author": "Matias Aguirre (Matware)",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",

package/skills/e2e-testing/SKILL.md

@@ -9,7 +9,7 @@ description: Create, run, and debug JSON-driven E2E browser tests with Chrome po
 
 `@matware/e2e-runner` is a JSON-driven E2E test runner. Tests are defined as JSON files with sequential browser actions — no JavaScript test code. Tests run in parallel against a Chrome pool (browserless/chrome via Docker) using Puppeteer.
 
-**Key capabilities:** 13 MCP tools for running tests, creating test files, capturing screenshots, analyzing network traffic, verifying GitHub/GitLab issues, and querying a learning system for stability insights.
+**Key capabilities:** 17 MCP tools for running tests, creating test files, capturing screenshots, analyzing network traffic, verifying GitHub/GitLab issues, and querying a learning system for stability insights.
 
 ## Prerequisites
 
@@ -159,6 +159,7 @@ Start/stop the web dashboard with `e2e_dashboard_start` / `e2e_dashboard_stop` f
 4. **`evaluate` is strict** — Returns starting with `FAIL:`/`ERROR:` or returning `false` will fail the test. Prefer granular assertion actions over `evaluate` with inline JS.
 5. **Serial tests** — Mark tests with `"serial": true` if they share mutable state. They run after all parallel tests.
 6. **Action retries** — Use `"retries": N` on individual actions for flaky selectors, or globally via config.
+7. **Assertion selectors are the contract** — when fixing flaky tests, heal *interaction* selectors freely, but never retarget an `assert_*` selector to make a test pass: pin assertion selectors to stable `data-testid`s.
 
 ## References
 

package/skills/e2e-testing/references/action-types.md

@@ -47,11 +47,25 @@ Complete catalog of all action types supported by @matware/e2e-runner.
 | `click_menu_item` | `text` (menu item text), `selector` (scope, optional) | Click a menu item by text. Searches `[role="menuitem"]`, `[role="menuitemradio"]`, `[role="menuitemcheckbox"]`, `.dropdown-item`, `.menu-item`, `[class*="MenuItem"]`, `[role="menu"] > li`. Waits for element to appear. |
 | `click_in_context` | `text` (container text), `selector` (child to click) | Find the smallest container whose text includes `text`, then click the `selector` child within it. Containers: `section`, `article`, `[class*="card"]`, `li`, `tr`, `div[class]`, etc. Both fields required. |
 
+## Multi-Tab
+
+All subsequent actions run in the active tab. The runner manages a tab registry keyed by label.
+
+| Action | Fields | Description |
+|--------|--------|-------------|
+| `open_tab` | `value` (URL), `text` (label, optional) | Open a new tab and navigate (relative to `baseUrl` or absolute). Label defaults to `tab-<n>`. The new tab becomes active. |
+| `switch_tab` | `value` | Switch active tab by label (exact), numeric index, or title/URL match (regex or substring). `"default"` returns to the original tab. |
+| `wait_for_tab` | `text` (label, optional), `timeout` | Wait for a tab/popup opened by the app (`window.open`, `target="_blank"`) and make it active. Use right after the action that triggers the popup. |
+| `assert_tab_count` | `value` | Assert number of open tabs: exact (`"2"`) or operators (`">=2"`). |
+| `close_tab` | `value` (label, optional) | Close the current (or named) tab and switch back to the last remaining one. Cannot close `default` while other tabs are open. |
+
 ## Assertions
 
 | Action | Fields | Description |
 |--------|--------|-------------|
 | `assert_text` | `text` | Check entire page body contains text (substring match). |
+| `assert_no_text` | `text` | Check text does NOT appear anywhere in the page body. Opposite of `assert_text`. |
+| `assert_text_in` | `selector`, `text`, `value` (`"exact"` optional) | Check text inside a scoped container. Joins `textContent` from all matching elements. Default: case-insensitive regex; with `value: "exact"`: case-sensitive substring. |
 | `assert_element_text` | `selector`, `text`, `value` (`"exact"` optional) | Check specific element's `textContent`. Default: substring match. With `value: "exact"`: strict `trim() ===` comparison. |
 | `assert_url` | `value` | Check current URL. Path-only (`/dashboard`) compares pathname. Full URL does substring match. |
 | `assert_visible` | `selector` | Element exists and is visible (`display`, `visibility`, `opacity` checks). |
@@ -62,11 +76,14 @@ Complete catalog of all action types supported by @matware/e2e-runner.
 | `assert_input_value` | `selector`, `value` | Checks `element.value.includes(value)` on input/select/textarea. |
 | `assert_matches` | `selector`, `value` (regex) | Tests element's `textContent` against `new RegExp(value)`. |
 | `assert_no_network_errors` | — | Checks accumulated `requestfailed` events during the test. Fails with error details if any exist. |
+| `assert_visual` | `value` (golden image filename), `selector` (optional element scope), `text` (max diff fraction, e.g. `"0.02"`), plus `fullPage`, `maskRegions`, `threshold` | Visual regression against a golden reference image (`goldenDir`, default `{screenshotsDir}/golden`). First run saves the golden and passes; later runs fail if more pixels differ than the max diff (default 2%) and write a diff image. `maskRegions: [{x,y,width,height}]` ignores dynamic areas (timestamps, avatars). |
 
 ### Assertion Disambiguation
 
 - **`assert_text`** → searches the **entire page body** (substring)
+- **`assert_no_text`** → asserts text is **absent** from the page body (do NOT use `assert_not_visible` with `text` — it requires `selector`)
 - **`assert_element_text`** → checks a **specific element** (substring, or exact with `value: "exact"`)
+- **`assert_text_in`** → checks text inside a **scoped container** (regex by default)
 - **`assert_matches`** → checks a specific element against a **regex** pattern
 - **`assert_input_value`** → reads the `.value` property (for form fields)
 
@@ -134,24 +151,6 @@ Delay between retries: `actionRetryDelay` config (default 500ms).
 { "type": "click_regex", "text": "add to cart", "selector": "button", "value": "last" }
 ```
 
-### Form validation assertions
-```json
-{ "type": "assert_attribute", "selector": "input#email", "value": "type=email" },
-{ "type": "assert_attribute", "selector": "button.submit", "value": "disabled" },
-{ "type": "assert_class", "selector": ".nav-item:first-child", "value": "active" },
-{ "type": "assert_input_value", "selector": "#email", "value": "user@example.com" },
-{ "type": "assert_matches", "selector": ".phone", "value": "\\d{3}-\\d{3}-\\d{4}" },
-{ "type": "assert_count", "selector": ".table-row", "value": ">3" }
-```
-
-### Storage operations
-```json
-{ "type": "set_storage", "value": "authToken=eyJhbGciOiJIUzI1NiJ9..." },
-{ "type": "assert_storage", "value": "authToken" },
-{ "type": "set_storage", "value": "theme=dark", "selector": "session" },
-{ "type": "assert_storage", "value": "theme=dark", "selector": "session" }
-```
-
 ### Icon, menu, and contextual clicks
 ```json
 { "type": "click_icon", "value": "edit" },

package/skills/e2e-testing/references/troubleshooting.md

@@ -193,32 +193,8 @@ The `KNOWN_ACTION_TYPES` Set in `src/actions.js` is the single source of truth.
 
 ## Screenshot Hashes
 
-Every screenshot captured during a run is assigned a short hash (`ss:a3f2b1c9`) — the first 8 hex chars of the SHA-256 of its file path. Hashes are deterministic and computed identically on the server (Node `crypto`) and in the browser (Web Crypto API).
-
-**Flow**: screenshot saved on disk → `saveRun()` registers hash in SQLite `screenshot_hashes` table → dashboard shows `[ss:XXXXXXXX]` badge (click to copy) → user pastes hash in Claude Code → `e2e_screenshot` MCP tool looks up hash, reads file, returns the image.
-
-- Hashes are registered inside the `saveRun()` transaction (covers action, error, verification, and baseline screenshots)
-- The `ss:` prefix is optional when calling `e2e_screenshot` — stripped during lookup
-- Dashboard computes hashes client-side (Web Crypto) for the Live view (before `persistRun()` writes to DB)
-- Run detail API (`/api/db/runs/:id`) includes `screenshotHashes` map per test result
-- Dashboard endpoint `/api/screenshot-hash/:hash` serves the image by hash
-- Dashboard Screenshots view has a **search bar** — type a hash to find and display the screenshot
+Every screenshot gets a short hash like `ss:a3f2b1c9` (deterministic, from its file path). The dashboard shows it as a click-to-copy badge; paste it (the `ss:` prefix is optional) and `e2e_screenshot` returns the image. The dashboard Screenshots view also has a hash search bar.
 
 ## Web Dashboard
 
-**`src/dashboard.js`** — HTTP server, REST API, WebSocket broadcast, pool polling.
-**`templates/dashboard.html`** — SPA, dark theme, vanilla JS, safe DOM (textContent + createEl helper).
-
-**Features:**
-- Live test execution with WebSocket updates
-- Run history with inline detail expansion
-- Screenshots gallery with hash badges and hash search
-- Network request logs with clickable expandable rows (full request/response detail)
-- Pool status monitoring
-- Multi-project support via project selector
-- Variables tab with masked values, inline edit, add, and delete
-
-**CLI:** `e2e-runner dashboard [--port 8484]`
-**MCP tools:** `e2e_dashboard_start`, `e2e_dashboard_stop`
-
-Config defaults: `dashboardPort: 8484`, `maxHistoryRuns: 100`
+Start with `e2e_dashboard_start` (or `e2e-runner dashboard [--port 8484]`), stop with `e2e_dashboard_stop`. Provides live execution (WebSocket), run history, screenshot gallery + hash search, expandable network logs, pool status, multi-project selector, and a variables tab.

package/src/actions.js

@@ -9,7 +9,7 @@
 
 import path from 'path';
 import fs from 'fs';
-import { assertVisualMatch } from './visual-diff.js';
+import { assertVisualMatch, isBlankImage } from './visual-diff.js';
 
 /**
  * Returns false when the page has nothing useful to capture — used to
@@ -50,7 +50,17 @@ export const BLANK_JPEG_BYTE_THRESHOLD = 8000;
 export function looksLikeBlankCapture(buf, format = 'png') {
   if (!Buffer.isBuffer(buf)) return false;
   const threshold = format === 'jpeg' ? BLANK_JPEG_BYTE_THRESHOLD : BLANK_PNG_BYTE_THRESHOLD;
-  return buf.length < threshold;
+  if (buf.length < threshold) return true;
+  // Byte size alone misses larger near-uniform frames (e.g. a white page
+  // whose PNG still compresses above 20 KB). For PNGs we can decode and
+  // check pixel uniformity directly — ≥98% of sampled pixels within
+  // tolerance of the mean color means there is nothing worth keeping.
+  // JPEGs (step captures) can't be decoded without deps, so they keep
+  // the byte heuristic only. Fails open on decode errors.
+  if (format === 'png') {
+    return isBlankImage(buf, { tolerance: 12, maxOutlierFraction: 0.02 }).blank;
+  }
+  return false;
 }
 
 /** All recognized action types — single source of truth for validation. */

package/src/dashboard.js

@@ -18,7 +18,7 @@ import { getPoolUrls, getAggregatedPoolStatus, waitForAnyPool } from './pool-man
 import { runTestsParallel, loadAllSuites, loadTestSuite, listSuites } from './runner.js';
 import { runModuleAnalysis } from './module-analysis.js';
 import { generateReport, generateJUnitXML, saveReport, persistRun, loadHistory, loadHistoryRun } from './reporter.js';
-import { listProjects as dbListProjects, listProjectsWithSparklines as dbListProjectsWithSparklines, getProjectRuns as dbGetProjectRuns, getRunDetail as dbGetRunDetail, getAllRuns as dbGetAllRuns, getRunCount as dbGetRunCount, getProjectScreenshotsDir as dbGetProjectScreenshotsDir, getProjectTestsDir as dbGetProjectTestsDir, getProjectCwd as dbGetProjectCwd, lookupScreenshotHash as dbLookupScreenshotHash, ensureProject as dbEnsureProject, getNetworkLogs as dbGetNetworkLogs, listVariables as dbListVariables, setVariable as dbSetVariable, deleteVariable as dbDeleteVariable, closeDb } from './db.js';
+import { listProjects as dbListProjects, listProjectsWithSparklines as dbListProjectsWithSparklines, getProjectRuns as dbGetProjectRuns, getRunDetail as dbGetRunDetail, getAllRuns as dbGetAllRuns, getRunCount as dbGetRunCount, getProjectScreenshotsDir as dbGetProjectScreenshotsDir, getProjectTestsDir as dbGetProjectTestsDir, getProjectCwd as dbGetProjectCwd, lookupScreenshotHash as dbLookupScreenshotHash, getScreenshotMetaByPaths as dbGetScreenshotMetaByPaths, ensureProject as dbEnsureProject, getNetworkLogs as dbGetNetworkLogs, listVariables as dbListVariables, setVariable as dbSetVariable, deleteVariable as dbDeleteVariable, closeDb } from './db.js';
 import { loadConfig } from './config.js';
 import { log, colors as C } from './logger.js';
 import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getHealthSnapshot, getActionHealthScores } from './learner-sqlite.js';
@@ -37,6 +37,25 @@ const { version: VERSION } = _require('../package.json');
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
+// Blank-PNG verdicts cached per path+size+mtime so the gallery listing
+// doesn't re-decode every PNG on each request. Non-PNG files are never
+// flagged (isBlankImage fails open on undecodable input).
+const blankVerdictCache = new Map();
+function isBlankScreenshotCached(filePath) {
+  if (!/\.png$/i.test(filePath)) return false;
+  try {
+    const st = fs.statSync(filePath);
+    const key = `${filePath}:${st.size}:${st.mtimeMs}`;
+    if (blankVerdictCache.has(key)) return blankVerdictCache.get(key);
+    const blank = isBlankImage(filePath).blank;
+    if (blankVerdictCache.size > 5000) blankVerdictCache.clear();
+    blankVerdictCache.set(key, blank);
+    return blank;
+  } catch {
+    return false;
+  }
+}
+
 /** Starts the dashboard server */
 export async function startDashboard(config) {
   const port = config.dashboardPort || 8484;
@@ -344,7 +363,9 @@ export async function startDashboard(config) {
         return;
       }
 
-      // API: DB — project screenshots list
+      // API: DB — project screenshots list (blank PNGs are hidden — they
+      // have no debug value and only waste gallery space; the blank-scan
+      // endpoint still finds them on disk for bulk deletion)
       const projectScreenshotsMatch = pathname.match(/^\/api\/db\/projects\/(\d+)\/screenshots$/);
       if (projectScreenshotsMatch) {
         try {
@@ -355,7 +376,30 @@ export async function startDashboard(config) {
             return;
           }
           const files = fs.readdirSync(dir).filter(f => /\.(png|jpg|jpeg|gif|webp)$/i.test(f)).sort();
-          jsonResponse(res, files.map(f => ({ name: f, path: path.join(dir, f) })));
+          const visible = files.filter(f => !isBlankScreenshotCached(path.join(dir, f)));
+          const fullPaths = visible.map(f => path.join(dir, f));
+          const meta = dbGetScreenshotMetaByPaths(fullPaths);
+          // Filenames embed the SANITIZED test name (runner's safeName); build a map
+          // back to the real name from DB-known entries so legacy files (no DB row)
+          // land in the same group as their DB-registered siblings
+          const sanitize = (s) => String(s).replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+          const sanitizedToReal = {};
+          for (const m of Object.values(meta)) {
+            if (m.testName) sanitizedToReal[sanitize(m.testName)] = m.testName;
+          }
+          jsonResponse(res, visible.map(f => {
+            const fp = path.join(dir, f);
+            const m = meta[fp];
+            // Fallback for files predating DB metadata: parse the test name out of
+            // runner-generated filenames (step-/error-/baseline-/verify-/current-/diff-<test>-<ts>.<ext>)
+            let testName = m?.testName || null;
+            let type = m?.type || null;
+            if (!testName) {
+              const fm = f.match(/^(step|error|baseline|verify|current|diff)-(.+?)(?:-\d{3})?-\d{10,}\.(?:png|jpe?g|gif|webp)$/i);
+              if (fm) { type = type || fm[1].toLowerCase(); testName = sanitizedToReal[fm[2]] || fm[2]; }
+            }
+            return { name: f, path: fp, testName, type };
+          }));
         } catch (error) {
           jsonResponse(res, { error: error.message }, 500);
         }
@@ -758,9 +802,10 @@ export async function startDashboard(config) {
           return;
         }
         const ext = path.extname(realPath).toLowerCase();
-        const mimeTypes = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.webp': 'image/webp' };
+        // .json covers step data captures (raw API responses saved instead of screenshots)
+        const mimeTypes = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.webp': 'image/webp', '.json': 'application/json' };
         if (!mimeTypes[ext]) {
-          jsonResponse(res, { error: 'Not an image' }, 400);
+          jsonResponse(res, { error: 'Unsupported file type' }, 400);
           return;
         }
         res.writeHead(200, { 'Content-Type': mimeTypes[ext], 'Cache-Control': 'no-store' });

package/src/db.js

@@ -375,6 +375,20 @@ export function getScreenshotHashes(filePaths) {
   return result;
 }
 
+/** Batch lookup with metadata: given an array of file paths, returns
+ *  { [path]: { hash, testName, type } } for paths registered in screenshot_hashes. */
+export function getScreenshotMetaByPaths(filePaths) {
+  if (!filePaths || filePaths.length === 0) return {};
+  const d = getDb();
+  const stmt = d.prepare('SELECT hash, file_path, test_name, screenshot_type FROM screenshot_hashes WHERE file_path = ?');
+  const result = {};
+  for (const fp of filePaths) {
+    const row = stmt.get(fp);
+    if (row) result[fp] = { hash: row.hash, testName: row.test_name || null, type: row.screenshot_type || null };
+  }
+  return result;
+}
+
 /** Save a run + its test results in a single transaction. Returns the run's DB id. */
 export function saveRun(projectId, report, runId, suiteName, triggeredBy, poolDriver) {
   const d = getDb();
@@ -430,6 +444,7 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy, poolDr
         error: a.error || undefined,
         actionRetries: a.actionRetries || undefined,
         autoScreenshot: a.autoScreenshot || undefined,
+        dataCapture: a.dataCapture || undefined,
         screenshot: a.result?.screenshot || undefined,
       }));