@matware/e2e-runner 1.2.1 → 1.3.1

package/src/ai-generate.js

@@ -52,7 +52,18 @@ The test format is:
       { "type": "click_regex", "text": "submit order", "selector": "button", "value": "last" },
       { "type": "click_option", "text": "Option Label" },
       { "type": "focus_autocomplete", "text": "Search by label" },
-      { "type": "click_chip", "text": "Tag Name" }
+      { "type": "click_chip", "text": "Tag Name" },
+      { "type": "set_storage", "value": "token=abc123" },
+      { "type": "set_storage", "value": "theme=dark", "selector": "session" },
+      { "type": "assert_storage", "value": "token" },
+      { "type": "assert_storage", "value": "theme=dark", "selector": "session" },
+      { "type": "click_icon", "value": "edit" },
+      { "type": "click_icon", "value": "delete", "selector": ".user-card" },
+      { "type": "click_menu_item", "text": "Delete" },
+      { "type": "click_menu_item", "text": "Export", "selector": ".actions-menu" },
+      { "type": "click_in_context", "text": "John Doe", "selector": "button.edit" },
+      { "type": "gql", "value": "{ users { id name } }" },
+      { "type": "gql", "value": "query($id: ID) { user(id: $id) { name } }", "text": "{\"id\": \"123\"}" }
     ]
   }
 ]
@@ -64,6 +75,28 @@ Framework-aware action reference (prefer these over evaluate for React/MUI apps)
 - focus_autocomplete: focus an autocomplete input by its label text (supports MUI .MuiAutocomplete-root and [role="combobox"])
 - click_chip: click a chip/tag element by text (searches [class*="Chip"], [data-chip])
 
+Storage actions:
+- set_storage: set a localStorage key. "value": "key=val". Use "selector": "session" for sessionStorage
+- assert_storage: assert a storage key exists ("value": "key") or has a value ("value": "key=expected"). Use "selector": "session" for sessionStorage
+
+GraphQL action:
+- gql: execute a GraphQL query/mutation via browser fetch. Auth token is read from localStorage automatically (configurable via gqlAuthHeader, gqlAuthKey, gqlAuthPrefix). "value" is the query string. "text" is variables as JSON string. "selector" is an optional JS assertion expression (receives response as "r"). Throws on GraphQL errors automatically. Also installs window.__e2eGql(query, vars) for use in subsequent evaluate actions
+
+Smart interaction actions:
+- click_icon: click an icon by identifier (data-testid fragment, class fragment, aria-label, SVG title). Walks up to nearest clickable parent (button, a, etc.). Optional "selector" scopes the search
+- click_menu_item: click a menu item by text. Searches [role="menuitem"], .dropdown-item, .menu-item, [class*="MenuItem"]. Optional "selector" scopes the search
+- click_in_context: click a child element within a container identified by text. "text" finds the container, "selector" is the child to click. Picks the smallest matching container
+
+Visual regression:
+- assert_visual: compare current page against a golden reference screenshot. "value" is the golden filename (e.g. "login-page.png"). First run auto-saves the golden. "text" is optional max diff percentage (default "0.02" = 2%). "selector" captures only that element. "maskRegions" ignores dynamic areas: [{ "x": 10, "y": 5, "width": 200, "height": 30 }]. Example: { "type": "assert_visual", "value": "dashboard.png", "text": "0.05" }
+
+Multi-tab actions (for OAuth, popups, admin+user flows):
+- open_tab: open a new tab with URL in "value". Optional "text" assigns a label for switch_tab. Example: { "type": "open_tab", "value": "/admin", "text": "admin" }
+- switch_tab: switch to a tab by label, title regex, URL substring, or index. Example: { "type": "switch_tab", "value": "admin" }
+- close_tab: close current tab or a named tab ("value" = label). Automatically switches to previous tab. Example: { "type": "close_tab", "value": "admin" }
+- wait_for_tab: wait for a popup/new tab opened by the page (window.open, target=_blank). Optional "text" labels it. Example: { "type": "wait_for_tab", "text": "oauth" }
+- assert_tab_count: verify number of open tabs. "value" is count or operator. Example: { "type": "assert_tab_count", "value": "2" }
+
 Assertion action reference:
 - assert_text: checks if text appears anywhere in the page body
 - assert_element_text: checks textContent of a specific element (use "value": "exact" for strict match)
@@ -80,8 +113,15 @@ Reusable modules:
 - Tests can reference shared action sequences: { "$use": "module-name", "params": { "key": "value" } }
 - Use modules for repeated flows like login, navigation, or setup
 
+Hooks and DRY patterns:
+- When multiple tests share the same setup (e.g. authentication), use beforeEach instead of repeating it per test
+- Object format with hooks: { "beforeEach": [...], "tests": [{ "name": "...", "actions": [...] }] }
+- Array format (no hooks): [{ "name": "...", "actions": [...] }]
+- If 3+ tests repeat the same action sequence (e.g. goto + wait + screenshot), extract it into a module
+- NEVER repeat the same $use call with identical params across all tests — move it to beforeEach
+
 Rules:
-- Output a JSON array of test objects
+- Output valid JSON: either a plain array of test objects, or an object with "beforeEach"/"tests" keys when hooks are needed
 - NEVER use evaluate with inline JS for assertions that can be done with native action types:
   * Use assert_element_text instead of evaluate to check element textContent
   * Use assert_attribute instead of evaluate to check HTML attributes
@@ -94,6 +134,12 @@ Rules:
   * Use click_option instead of evaluate with querySelectorAll('[role="option"]') patterns
   * Use focus_autocomplete instead of evaluate with MuiAutocomplete-root label search patterns
   * Use click_chip instead of evaluate with querySelectorAll('[class*="Chip"]') patterns
+  * Use set_storage instead of evaluate with localStorage.setItem or sessionStorage.setItem
+  * Use assert_storage instead of evaluate with localStorage.getItem or sessionStorage.getItem checks
+  * Use click_icon instead of evaluate with querySelector('svg[data-testid]').closest('button').click() patterns
+  * Use click_menu_item instead of evaluate with querySelectorAll('[role="menuitem"]') patterns
+  * Use click_in_context instead of evaluate that finds a container by text then clicks a child element
+  * Use gql instead of evaluate with fetch + JSON.stringify + GraphQL queries/mutations
   * Reserve evaluate ONLY for complex logic that cannot be expressed with existing action types
 - "click" with "text" (no selector) finds buttons/links by visible text
 - "goto" values starting with "/" are relative to the app's base URL
@@ -117,9 +163,11 @@ CRITICAL — UI-first testing rules:
 
 const API_RULES = `
 API testing rules:
-- Tests verify backend API behavior directly via evaluate actions
+- Tests verify backend API behavior directly via gql actions (preferred) or evaluate actions
 - Each test should: set up context → call API → assert response shape and values
-- Use evaluate for GraphQL mutations, queries, and REST calls
+- PREFER the gql action for GraphQL queries/mutations — it handles auth and error checking automatically
+- Use gql with "selector" field for inline assertions on the response (JS expression where "r" is the response)
+- Use evaluate with window.__e2eGql() for complex multi-step GraphQL operations (the helper is installed by any gql action)
 - Name tests clearly describing the API operation (e.g. "createUser-returns-new-user")
 - Include error case tests (invalid input, missing fields, auth failures)
 - No need for goto/click/type — this is not UI testing
@@ -201,6 +249,77 @@ Existing suites: ${existingSuites.join(', ') || 'none'}`;
   };
 }
 
+/**
+ * Generates a hindsight hint for a failed test result.
+ * Sends the error + action context to Claude API and returns a concrete fix suggestion.
+ * Returns null if API key is unavailable or on any error.
+ */
+export async function generateHindsightHint(failedResult, config = {}) {
+  const apiKey = config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) return null;
+
+  const model = config.hintsModel || config.anthropicModel || 'claude-sonnet-4-5-20250929';
+  const lastActions = (failedResult.actions || []).slice(-8);
+  const failedAction = lastActions.find(a => a.success === false);
+
+  const consoleErrors = (failedResult.consoleLogs || [])
+    .filter(l => l.type === 'error')
+    .slice(-5)
+    .map(l => l.text);
+
+  const networkErrors = (failedResult.networkErrors || [])
+    .slice(-5)
+    .map(e => `${e.url} (${e.error})`);
+
+  const prompt = `Analyze this failed E2E test and suggest a concrete fix.
+
+TEST: "${failedResult.name}"
+ERROR: ${failedResult.error}
+
+LAST ACTIONS:
+${lastActions.map((a, i) => `  ${i + 1}. ${a.type}${a.selector ? ' selector=' + a.selector : ''}${a.text ? ' text=' + a.text : ''}${a.value ? ' value=' + (a.value.length > 80 ? a.value.slice(0, 80) + '...' : a.value) : ''} → ${a.success === false ? 'FAILED: ' + a.error : 'OK'} (${a.duration}ms)`).join('\n')}
+
+${failedAction ? `FAILED ACTION: ${JSON.stringify({ type: failedAction.type, selector: failedAction.selector, text: failedAction.text, value: failedAction.value?.slice?.(0, 200) })}` : ''}
+${consoleErrors.length ? `CONSOLE ERRORS:\n${consoleErrors.join('\n')}` : ''}
+${networkErrors.length ? `NETWORK ERRORS:\n${networkErrors.join('\n')}` : ''}
+
+Respond with ONLY a JSON object: { "suggestion": "concrete fix description", "confidence": "high"|"medium"|"low", "fixType": "selector"|"wait"|"timeout"|"logic"|"infra"|"data" }`;
+
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 15000);
+
+    const response = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 1024,
+        system: 'You are an E2E test debugging expert. Given a failed test, suggest the most likely fix. Be specific: name exact selectors, wait times, or code changes. Keep suggestions under 100 words.',
+        messages: [{ role: 'user', content: prompt }],
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) return null;
+    const result = await response.json();
+    const text = result.content?.[0]?.text;
+    if (!text) return null;
+
+    const cleaned = text.replace(/^```(?:json)?\s*\n?/m, '').replace(/\n?```\s*$/m, '').trim();
+    const hint = JSON.parse(cleaned);
+    return { test: failedResult.name, ...hint };
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Checks if the Anthropic API key is available.
  * @returns {boolean}
@@ -247,7 +366,7 @@ Test Category: ${testType}
 ${categoryRules}
 Base URL: ${config.baseUrl}
 
-Output a JSON array of test objects. Nothing else.`;
+Output ONLY valid JSON. Either a plain array of test objects, or an object with "beforeEach" and "tests" keys if hooks are needed. Nothing else.`;
 
   const response = await fetch('https://api.anthropic.com/v1/messages', {
     method: 'POST',
@@ -288,9 +407,21 @@ Output a JSON array of test objects. Nothing else.`;
     throw new Error(`Failed to parse generated tests as JSON: ${err.message}\n\nRaw output:\n${text}`);
   }
 
-  if (!Array.isArray(tests)) {
-    throw new Error('Generated tests must be a JSON array');
+  // Accept both array format and object format with hooks
+  let hooks;
+  if (Array.isArray(tests)) {
+    // Plain array: [{ name, actions }]
+  } else if (tests && Array.isArray(tests.tests)) {
+    // Object with hooks: { beforeEach: [...], tests: [...] }
+    hooks = {};
+    for (const key of ['beforeAll', 'afterAll', 'beforeEach', 'afterEach']) {
+      if (Array.isArray(tests[key])) hooks[key] = tests[key];
+    }
+    if (Object.keys(hooks).length === 0) hooks = undefined;
+    tests = tests.tests;
+  } else {
+    throw new Error('Generated tests must be a JSON array or an object with a "tests" array');
   }
 
-  return { tests, suiteName };
+  return { tests, hooks, suiteName };
 }

package/src/app-pool.js

@@ -0,0 +1,339 @@
+/**
+ * App Pool — isolated application environments for test isolation.
+ *
+ * Provides each test with its own application instance via fast VM/container
+ * forking. Supports multiple drivers:
+ *
+ *   - "docker"    — Docker-based: runs a fresh container per test (slower, ~2-5s)
+ *   - "zeroboot"  — Firecracker microVM fork via Zeroboot SDK (~0.8ms)
+ *
+ * Lifecycle:
+ *   1. Template creation (one-time): boot app, wait for ready, snapshot state
+ *   2. Fork (per-test): clone template into isolated instance with unique port
+ *   3. Test runs against fork's baseUrl
+ *   4. Fork destroyed after test completes
+ *
+ * The app pool is independent of the Chrome pool — both are selected in
+ * parallel by pool-manager.js for maximum throughput.
+ */
+
+import { log, colors as C } from './logger.js';
+
+// ── Port allocator ────────────────────────────────────────────────────────────
+
+/** Tracks allocated ports to avoid collisions across concurrent forks. */
+const allocatedPorts = new Set();
+
+function allocatePort(basePort, maxForks) {
+  for (let offset = 0; offset < maxForks; offset++) {
+    const port = basePort + offset;
+    if (!allocatedPorts.has(port)) {
+      allocatedPorts.add(port);
+      return port;
+    }
+  }
+  throw new Error(`App pool: no free ports in range ${basePort}-${basePort + maxForks - 1}`);
+}
+
+function releasePort(port) {
+  allocatedPorts.delete(port);
+}
+
+// ── Fork registry ─────────────────────────────────────────────────────────────
+
+/**
+ * Active fork tracking.
+ * Maps forkId → { port, driver, testName, startTime, metadata }
+ */
+const activeForks = new Map();
+let forkCounter = 0;
+
+function generateForkId() {
+  return `fork-${Date.now().toString(36)}-${(++forkCounter).toString(36)}`;
+}
+
+// ── Health check ──────────────────────────────────────────────────────────────
+
+/**
+ * Polls a URL until it returns 2xx or timeout is reached.
+ * Used to verify a forked app instance is ready to receive traffic.
+ */
+async function waitForReady(url, timeoutMs = 10000, intervalMs = 200) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url, { signal: AbortSignal.timeout(2000) });
+      if (res.ok) return true;
+    } catch { /* not ready yet */ }
+    await new Promise(r => setTimeout(r, intervalMs));
+  }
+  throw new Error(`App pool: fork not ready after ${timeoutMs}ms (checked ${url})`);
+}
+
+// ── Driver: Docker ────────────────────────────────────────────────────────────
+
+/**
+ * Docker driver — runs a fresh container per fork.
+ * Slower (~2-5s) but works everywhere Docker is available.
+ *
+ * Expects appPool config:
+ *   image:       Docker image to run (required)
+ *   envVars:     { KEY: 'value' } environment variables for the container
+ *   readyCheck:  path to poll for readiness (e.g. '/health')
+ *   readyTimeout: ms to wait for ready (default 15000)
+ */
+async function dockerFork(config, port) {
+  const { execFile } = await import('child_process');
+  const { promisify } = await import('util');
+  const execFileAsync = promisify(execFile);
+
+  const appConfig = config.appPool;
+  const containerName = `e2e-app-${port}`;
+
+  const args = [
+    'run', '-d',
+    '--name', containerName,
+    '-p', `${port}:${appConfig.containerPort || 3000}`,
+  ];
+
+  // Add host.docker.internal access
+  args.push('--add-host', 'host.docker.internal:host-gateway');
+
+  // Environment variables
+  if (appConfig.envVars) {
+    for (const [key, value] of Object.entries(appConfig.envVars)) {
+      args.push('-e', `${key}=${value}`);
+    }
+  }
+
+  args.push(appConfig.image);
+
+  // Optional command override
+  if (appConfig.cmd) {
+    args.push(...(Array.isArray(appConfig.cmd) ? appConfig.cmd : appConfig.cmd.split(' ')));
+  }
+
+  await execFileAsync('docker', args);
+
+  return {
+    containerId: containerName,
+    cleanup: async () => {
+      try {
+        await execFileAsync('docker', ['rm', '-f', containerName]);
+      } catch { /* best effort */ }
+    },
+  };
+}
+
+async function dockerDestroy(metadata) {
+  if (metadata?.cleanup) {
+    await metadata.cleanup();
+  }
+}
+
+// ── Driver: Zeroboot ──────────────────────────────────────────────────────────
+
+/**
+ * Zeroboot driver — sub-millisecond VM forks via Firecracker snapshots.
+ *
+ * NOTE: Zeroboot currently has NO networking within VMs (serial I/O only).
+ * This driver is a forward-looking implementation for when networking is added.
+ * The interface is ready — only the SDK calls need updating.
+ *
+ * Expects appPool config:
+ *   zeroboot.apiUrl:     Zeroboot API endpoint (default: http://localhost:8484)
+ *   zeroboot.templateId: pre-created template ID (required)
+ *   readyCheck:          path to poll for readiness
+ *   readyTimeout:        ms to wait for ready (default 5000)
+ */
+
+/** Placeholder for Zeroboot SDK — replace with actual import when available. */
+function getZerobootClient(apiUrl) {
+  // When Zeroboot publishes their Node SDK:
+  //   import { ZerobootClient } from '@anthropic-ai/zeroboot';
+  //   return new ZerobootClient({ apiUrl });
+  return {
+    async fork(templateId, _options) {
+      // SDK call: creates a KVM fork from snapshot in ~0.8ms
+      // Returns: { forkId, port, host }
+      throw new Error(
+        'Zeroboot SDK not installed. Install with: npm install @anthropic-ai/zeroboot\n' +
+        'Zeroboot currently requires networking support (not yet available).\n' +
+        'See: https://github.com/zerobootdev/zeroboot'
+      );
+    },
+    async destroy(_forkId) {
+      // SDK call: destroys the forked VM
+    },
+    async status() {
+      // SDK call: returns template and fork status
+      return { templates: [], activeForks: 0, memoryUsed: 0 };
+    },
+  };
+}
+
+async function zerobootFork(config, port) {
+  const appConfig = config.appPool;
+  const apiUrl = appConfig.zeroboot?.apiUrl || 'http://localhost:8484';
+  const templateId = appConfig.zeroboot?.templateId;
+
+  if (!templateId) {
+    throw new Error('App pool (zeroboot): zeroboot.templateId is required in appPool config');
+  }
+
+  const client = getZerobootClient(apiUrl);
+  const fork = await client.fork(templateId, { port });
+
+  return {
+    zerobootForkId: fork.forkId,
+    client,
+    cleanup: async () => {
+      try {
+        await client.destroy(fork.forkId);
+      } catch { /* best effort */ }
+    },
+  };
+}
+
+async function zerobootDestroy(metadata) {
+  if (metadata?.cleanup) {
+    await metadata.cleanup();
+  }
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Forks a new isolated app instance.
+ *
+ * @param {object} config - Full e2e-runner config with appPool section
+ * @param {string} [testName] - Test name for logging/tracking
+ * @returns {{ forkId: string, baseUrl: string, port: number }}
+ */
+export async function forkAppInstance(config, testName = '') {
+  const appConfig = config.appPool;
+  if (!appConfig?.enabled) {
+    throw new Error('App pool is not enabled in config');
+  }
+
+  const driver = appConfig.driver || 'docker';
+  const basePort = appConfig.forkBasePort || 4000;
+  const maxForks = appConfig.maxForks || 10;
+  const port = allocatePort(basePort, maxForks);
+  const forkId = generateForkId();
+
+  log('🔱', `${C.cyan}Forking app${C.reset} ${C.dim}(${driver}, port ${port}${testName ? `, ${testName}` : ''})${C.reset}`);
+
+  const startMs = Date.now();
+  let metadata;
+
+  try {
+    if (driver === 'zeroboot') {
+      metadata = await zerobootFork(config, port);
+    } else if (driver === 'docker') {
+      metadata = await dockerFork(config, port);
+    } else {
+      throw new Error(`App pool: unknown driver "${driver}". Use "docker" or "zeroboot".`);
+    }
+
+    // Determine the baseUrl for the forked instance
+    const host = appConfig.forkHost || 'localhost';
+    const protocol = appConfig.forkProtocol || 'http';
+    const baseUrl = `${protocol}://${host}:${port}`;
+
+    // For Docker-based apps accessed from Chrome inside Docker:
+    const dockerBaseUrl = `http://host.docker.internal:${port}`;
+
+    // Wait for the app to be ready
+    if (appConfig.readyCheck) {
+      const checkUrl = `${baseUrl}${appConfig.readyCheck}`;
+      const readyTimeout = appConfig.readyTimeout || (driver === 'zeroboot' ? 5000 : 15000);
+      await waitForReady(checkUrl, readyTimeout);
+    }
+
+    const forkTimeMs = Date.now() - startMs;
+    log('🔱', `${C.green}App fork ready${C.reset} ${C.dim}(${forkTimeMs}ms, ${baseUrl})${C.reset}`);
+
+    const forkInfo = {
+      forkId,
+      port,
+      baseUrl,
+      dockerBaseUrl,
+      driver,
+      testName,
+      startTime: new Date().toISOString(),
+      forkTimeMs,
+      metadata,
+    };
+
+    activeForks.set(forkId, forkInfo);
+    return forkInfo;
+  } catch (error) {
+    releasePort(port);
+    throw error;
+  }
+}
+
+/**
+ * Destroys a forked app instance and releases its port.
+ *
+ * @param {string} forkId - Fork ID returned by forkAppInstance
+ */
+export async function destroyFork(forkId) {
+  const fork = activeForks.get(forkId);
+  if (!fork) return;
+
+  log('🔱', `${C.dim}Destroying app fork${C.reset} ${C.dim}(port ${fork.port}${fork.testName ? `, ${fork.testName}` : ''})${C.reset}`);
+
+  try {
+    if (fork.driver === 'zeroboot') {
+      await zerobootDestroy(fork.metadata);
+    } else if (fork.driver === 'docker') {
+      await dockerDestroy(fork.metadata);
+    }
+  } finally {
+    releasePort(fork.port);
+    activeForks.delete(forkId);
+  }
+}
+
+/**
+ * Returns the status of the app pool: active forks, port usage, per-fork details.
+ */
+export function getAppPoolStatus() {
+  const forks = [];
+  for (const [id, fork] of activeForks) {
+    forks.push({
+      forkId: id,
+      port: fork.port,
+      driver: fork.driver,
+      baseUrl: fork.baseUrl,
+      testName: fork.testName,
+      startTime: fork.startTime,
+      forkTimeMs: fork.forkTimeMs,
+    });
+  }
+
+  return {
+    activeForks: activeForks.size,
+    allocatedPorts: [...allocatedPorts].sort((a, b) => a - b),
+    forks,
+  };
+}
+
+/**
+ * Destroys all active forks. Called during cleanup/shutdown.
+ */
+export async function destroyAllForks() {
+  const ids = [...activeForks.keys()];
+  if (ids.length === 0) return;
+  log('🔱', `${C.dim}Destroying ${ids.length} app fork(s)...${C.reset}`);
+  await Promise.allSettled(ids.map(id => destroyFork(id)));
+}
+
+/**
+ * Checks if app pool is configured and enabled.
+ */
+export function isAppPoolEnabled(config) {
+  return config?.appPool?.enabled === true;
+}