@matware/e2e-runner 1.3.0 → 1.5.0

package/src/module-analysis.js

@@ -0,0 +1,247 @@
+/**
+ * Module Analysis — deterministic duplication detector.
+ *
+ * Scans every test JSON in testsDir, normalizes action sequences to
+ * signatures (literal values become placeholders, but selectors/text
+ * stay so two unrelated clicks don't collide), and reports sequences of
+ * length 3-8 that appear in 2+ different tests. These are the canonical
+ * candidates the test-improver agent would extract into a `$use` module.
+ *
+ * Also enumerates current modules and counts how often each is referenced
+ * via `$use` across the project so users can see adoption.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+const MIN_SEQ_LEN = 3;
+const MAX_SEQ_LEN = 8;
+
+/** Stable signature for one action — literals → '*', identifiers kept. */
+function signatureOf(action) {
+  if (!action || typeof action !== 'object') return '?';
+  if (action.$use) return `$use:${action.$use}`;
+  const type = action.type || '?';
+  // Keep selector + text (semantic identifiers); replace `value` with `*`
+  // since values are usually parameterizable.
+  const parts = [type];
+  if (action.selector) parts.push(`@${action.selector}`);
+  if (action.text != null) parts.push(`"${String(action.text).slice(0, 40)}"`);
+  if (action.value != null) parts.push('*');
+  return parts.join('|');
+}
+
+function walkTests(testsDir) {
+  const out = [];
+  let files = [];
+  try {
+    files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
+  } catch { return out; }
+  for (const file of files) {
+    const fp = path.join(testsDir, file);
+    let suite;
+    try { suite = JSON.parse(fs.readFileSync(fp, 'utf-8')); } catch { continue; }
+    const tests = Array.isArray(suite) ? suite : suite.tests || [];
+    for (const t of tests) {
+      if (!t || !Array.isArray(t.actions)) continue;
+      out.push({
+        file,
+        suite: file.replace(/\.json$/, ''),
+        test: t.name || '(unnamed)',
+        actions: t.actions,
+        signatures: t.actions.map(signatureOf),
+      });
+    }
+  }
+  return out;
+}
+
+function findCandidates(tests) {
+  // Map signature-sequence-string → [{testIdx, start}]
+  const seen = new Map();
+  for (let ti = 0; ti < tests.length; ti++) {
+    const sig = tests[ti].signatures;
+    for (let len = MIN_SEQ_LEN; len <= MAX_SEQ_LEN; len++) {
+      for (let i = 0; i + len <= sig.length; i++) {
+        const key = sig.slice(i, i + len).join(' >> ');
+        if (!seen.has(key)) seen.set(key, []);
+        seen.get(key).push({ testIdx: ti, start: i, len });
+      }
+    }
+  }
+
+  const candidates = [];
+  for (const [key, hits] of seen) {
+    // Distinct tests, not just same test repeated
+    const distinct = new Map();
+    for (const h of hits) {
+      const t = tests[h.testIdx];
+      const id = t.suite + '::' + t.test;
+      if (!distinct.has(id)) distinct.set(id, { test: t, hits: [] });
+      distinct.get(id).hits.push(h);
+    }
+    if (distinct.size < 2) continue;
+    candidates.push({
+      signature: key,
+      length: hits[0].len,
+      occurrenceCount: hits.length,
+      testCount: distinct.size,
+      // Best representative — first hit's actions, lifted from the test
+      sample: tests[hits[0].testIdx].actions.slice(hits[0].start, hits[0].start + hits[0].len),
+      usedBy: [...distinct.values()].map(d => ({
+        suite: d.test.suite,
+        test: d.test.test,
+        occurrences: d.hits.length,
+      })),
+    });
+  }
+
+  // Rank: maximize (savings ≈ length * (testCount - 1))
+  // Then prefer longer sequences over shorter ones.
+  candidates.sort((a, b) => {
+    const savingsA = a.length * (a.testCount - 1);
+    const savingsB = b.length * (b.testCount - 1);
+    if (savingsA !== savingsB) return savingsB - savingsA;
+    return b.length - a.length;
+  });
+
+  // Prune: drop sequences that are strict substrings of a higher-scored one
+  // covering the same set of tests (the shorter one is redundant once the
+  // longer one is extracted).
+  const kept = [];
+  for (const c of candidates) {
+    const covered = kept.find(k =>
+      k.signature.includes(c.signature) &&
+      JSON.stringify(k.usedBy.map(u => u.suite+'::'+u.test).sort()) ===
+      JSON.stringify(c.usedBy.map(u => u.suite+'::'+u.test).sort())
+    );
+    if (!covered) kept.push(c);
+  }
+
+  // Suggest a name from the dominant action types
+  for (const c of kept) {
+    c.suggestedName = suggestModuleName(c.sample);
+  }
+
+  return kept.slice(0, 30);
+}
+
+function suggestModuleName(actions) {
+  if (!actions || !actions.length) return 'module';
+  const types = actions.map(a => a?.type).filter(Boolean);
+  // Heuristics for common patterns
+  const hasGoto = types.includes('goto');
+  const hasType = types.includes('type') || types.includes('fill') || types.includes('type_react');
+  const hasClick = types.some(t => t && t.startsWith('click'));
+  const hasAssert = types.some(t => t && t.startsWith('assert'));
+  // Pull a noun-y hint from selector or text of first non-goto action
+  const hint = (function () {
+    for (const a of actions) {
+      const s = a?.text || a?.selector || a?.value;
+      if (typeof s === 'string' && s.length > 0 && s.length < 30) {
+        return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 24);
+      }
+    }
+    return '';
+  })();
+  if (hasGoto && hasType && hasClick) return 'navigate-and-submit' + (hint ? '-' + hint : '');
+  if (hasType && hasClick) return 'fill-form' + (hint ? '-' + hint : '');
+  if (hasGoto && hasAssert) return 'open-and-verify' + (hint ? '-' + hint : '');
+  if (hasGoto) return 'navigate' + (hint ? '-' + hint : '');
+  if (hint) return hint;
+  return 'extracted-module';
+}
+
+function loadModules(modulesDir) {
+  if (!modulesDir || !fs.existsSync(modulesDir)) return [];
+  const files = fs.readdirSync(modulesDir).filter(f => f.endsWith('.json')).sort();
+  return files.map(f => {
+    const fp = path.join(modulesDir, f);
+    let data = {};
+    try { data = JSON.parse(fs.readFileSync(fp, 'utf-8')); } catch { /* */ }
+    return {
+      name: f.replace(/\.json$/, ''),
+      file: f,
+      description: data.description || null,
+      params: data.params || [],
+      actionCount: Array.isArray(data.actions) ? data.actions.length : 0,
+    };
+  });
+}
+
+function countModuleUsage(tests, modules) {
+  const usage = new Map(modules.map(m => [m.name, { count: 0, usedBy: new Set() }]));
+  function walk(actions, testInfo) {
+    if (!Array.isArray(actions)) return;
+    for (const a of actions) {
+      if (a && a.$use) {
+        const u = usage.get(a.$use);
+        if (u) { u.count++; u.usedBy.add(testInfo); }
+      }
+    }
+  }
+  for (const t of tests) {
+    walk(t.actions, t.suite + '::' + t.test);
+  }
+  return modules.map(m => {
+    const u = usage.get(m.name) || { count: 0, usedBy: new Set() };
+    return { ...m, usageCount: u.count, usedBy: [...u.usedBy] };
+  });
+}
+
+export function runModuleAnalysis(testsDir, modulesDir) {
+  const tests = walkTests(testsDir);
+  const modules = loadModules(modulesDir);
+  const candidates = findCandidates(tests);
+  const modulesWithUsage = countModuleUsage(tests, modules);
+
+  // Build a Claude Code prompt the user can copy verbatim.
+  const prompt = buildAgentPrompt(testsDir, modulesDir, candidates, modulesWithUsage);
+
+  return {
+    testsDir,
+    modulesDir,
+    summary: {
+      testCount: tests.length,
+      moduleCount: modulesWithUsage.length,
+      candidateCount: candidates.length,
+      unusedModules: modulesWithUsage.filter(m => m.usageCount === 0).length,
+    },
+    modules: modulesWithUsage,
+    candidates,
+    agentPrompt: prompt,
+  };
+}
+
+function buildAgentPrompt(testsDir, modulesDir, candidates, modules) {
+  const topCandidates = candidates.slice(0, 10);
+  return [
+    'Analyze E2E test modules and recommend changes.',
+    '',
+    `Tests directory: ${testsDir}`,
+    `Modules directory: ${modulesDir}`,
+    '',
+    'Use the test-improver capabilities to:',
+    '1. Review the candidate sequences below and decide which should be extracted into reusable modules via `e2e_create_module`.',
+    '2. For each extracted module, suggest the parameters needed (selectors/text that vary between usages).',
+    '3. Check the current modules list for any that are unused or could be consolidated.',
+    '4. After creating modules, Edit the affected test files to replace inline action sequences with `{ "$use": "<module-name>", "params": {...} }`.',
+    '',
+    `## Top ${topCandidates.length} extraction candidates`,
+    '',
+    ...topCandidates.map((c, i) =>
+      `${i + 1}. **${c.suggestedName}** (${c.length} actions, used in ${c.testCount} tests, ${c.occurrenceCount} total occurrences)\n` +
+      `   Signature: \`${c.signature}\`\n` +
+      `   Used by: ${c.usedBy.map(u => `${u.suite}::${u.test}`).join(', ')}`
+    ),
+    '',
+    '## Current modules',
+    '',
+    ...modules.map(m =>
+      `- **${m.name}** — ${m.actionCount} actions, ${m.params.length} params, used ${m.usageCount}x` +
+      (m.description ? `\n  > ${m.description}` : '')
+    ),
+    '',
+    'After making changes, run the affected tests to confirm nothing broke.',
+  ].join('\n');
+}

package/src/module-resolver.js

@@ -14,6 +14,7 @@
 import fs from 'fs';
 import path from 'path';
 import { KNOWN_ACTION_TYPES } from './actions.js';
+import { KNOWN_DRIVERS } from './pool.js';
 
 /**
  * Loads all module definitions from a directory.
@@ -170,8 +171,17 @@ function resolveActions(actions, registry, contextParams = {}, visited = new Set
       const moduleActions = moduleDef.actions || [];
       const substituted = moduleActions.map(a => {
         if (a.$use) {
-          // Nested $use — pass through for recursive resolution
-          return { ...a, params: { ...mergedParams, ...a.params } };
+          // Nested $use — resolve {{param}} placeholders in the nested call's
+          // params against THIS module's scope (its merged params + defaults)
+          // so a module can forward its own params/defaults to a module it
+          // $uses. Then merge over the inherited context params as fallback.
+          const nestedParams = {};
+          for (const [k, v] of Object.entries(a.params || {})) {
+            nestedParams[k] = typeof v === 'string'
+              ? substituteParams(v, mergedParams, moduleDef.params)
+              : v;
+          }
+          return { ...a, params: { ...mergedParams, ...nestedParams } };
         }
         return substituteActionParams(a, mergedParams, moduleDef.params, moduleName);
       });
@@ -307,4 +317,27 @@ export function validateActionTypes(data, context) {
     const details = unknown.map(u => `"${u.type}" in ${u.location}`).join(', ');
     throw new Error(`Unknown action type(s) in ${context}: ${details}`);
   }
+
+  // Validate per-test driver / fallbackDriver fields
+  const knownExceptAuto = [...KNOWN_DRIVERS].filter(d => d !== 'auto');
+  for (const test of data.tests || []) {
+    if (test.driver !== undefined && !knownExceptAuto.includes(test.driver)) {
+      throw new Error(
+        `Invalid driver "${test.driver}" in test "${test.name}" (${context}). ` +
+        `Allowed: ${knownExceptAuto.join(', ')}.`
+      );
+    }
+    if (test.fallbackDriver !== undefined && !knownExceptAuto.includes(test.fallbackDriver)) {
+      throw new Error(
+        `Invalid fallbackDriver "${test.fallbackDriver}" in test "${test.name}" (${context}). ` +
+        `Allowed: ${knownExceptAuto.join(', ')}.`
+      );
+    }
+    if (test.fallbackDriver !== undefined && test.driver === undefined) {
+      throw new Error(
+        `Test "${test.name}" (${context}) declares fallbackDriver without driver. ` +
+        `fallbackDriver only applies when driver is set.`
+      );
+    }
+  }
 }

package/src/narrate.js

@@ -36,15 +36,24 @@ export function narrateAction(action, result) {
       return `Typed "${masked}" into "${selector}"${time}`;
     }
 
-    case 'wait':
+    case 'wait': {
+      const goneTarget = typeof action.gone === 'string' ? action.gone : (action.gone === true ? (selector || (text ? `text "${text}"` : null)) : null);
+      if (goneTarget) return `Waited for ${goneTarget.startsWith('text ') ? goneTarget : `"${goneTarget}"`} to disappear${time}`;
       if (selector) return `Waited for "${selector}" to appear${time}`;
       if (text) return `Waited for text "${text}" to appear${time}`;
       return `Waited ${value}ms`;
+    }
 
     case 'screenshot':
       if (result.result?.screenshot) {
         return `Captured screenshot: ${result.result.screenshot}`;
       }
+      if (result.result?.skipped) {
+        const reason = result.result.skipped === 'blank-page'
+          ? 'page was blank'
+          : 'render looked blank';
+        return `Skipped screenshot (${reason})`;
+      }
       return `Captured screenshot${value ? `: ${value}` : ''}`;
 
     case 'assert_text':
@@ -123,6 +132,9 @@ export function narrateAction(action, result) {
     case 'click_option':
       return `Clicked dropdown option "${text}"${time}`;
 
+    case 'select_combobox':
+      return `Selected "${text || action.option}" from combobox${action.filter ? ` (filtered "${action.filter}")` : ''}${time}`;
+
     case 'focus_autocomplete':
       return `Focused autocomplete labeled "${text}"${time}`;
 
@@ -167,6 +179,28 @@ export function narrateAction(action, result) {
       return `Executed JS: ${snippet}${time}`;
     }
 
+    case 'assert_visual': {
+      const vr = result.result;
+      if (vr?.goldenCreated) return `Saved golden reference: ${value}${time}`;
+      const pct = vr?.diffPercentage != null ? (vr.diffPercentage * 100).toFixed(2) + '% diff' : '';
+      return `Visual comparison against "${value}": ${pct}${time}`;
+    }
+
+    case 'open_tab':
+      return `Opened new tab${text ? ` "${text}"` : ''} → ${value}${time}`;
+
+    case 'switch_tab':
+      return `Switched to tab "${value}"${time}`;
+
+    case 'close_tab':
+      return `Closed tab${value ? ` "${value}"` : ''}${time}`;
+
+    case 'assert_tab_count':
+      return `Verified ${value} tab(s) open${time}`;
+
+    case 'wait_for_tab':
+      return `Waited for new tab to open${text ? ` (labeled "${text}")` : ''}${time}`;
+
     default:
       return `Unknown action "${type}"${time}`;
   }
@@ -210,6 +244,7 @@ function describeIntent(action) {
     case 'type_react':            return `Type into React input "${selector}"`;
     case 'click_regex':           return `Click element matching /${text}/i`;
     case 'click_option':          return `Click option "${text}"`;
+    case 'select_combobox':       return `Select "${text || action.option}" from combobox`;
     case 'focus_autocomplete':    return `Focus autocomplete "${text}"`;
     case 'click_chip':            return `Click chip "${text}"`;
     case 'set_storage':            return `Set ${selector === 'session' ? 'sessionStorage' : 'localStorage'} key "${value?.split('=')[0] || value}"`;
@@ -223,6 +258,12 @@ function describeIntent(action) {
     case 'click_menu_item':        return `Click menu item "${text}"`;
     case 'click_in_context':       return `Click "${selector}" in context of "${text}"`;
     case 'evaluate':   return 'Execute JS';
+    case 'assert_visual':           return `Visual compare against "${value}"`;
+    case 'open_tab':               return `Open new tab → ${value}`;
+    case 'switch_tab':             return `Switch to tab "${value}"`;
+    case 'close_tab':              return `Close tab${value ? ` "${value}"` : ''}`;
+    case 'assert_tab_count':       return `Assert ${value} tab(s) open`;
+    case 'wait_for_tab':           return 'Wait for new tab';
     default:           return `Action "${type}"`;
   }
 }

package/src/pool-manager.js

@@ -11,7 +11,7 @@
  * subsequent calls factor in connections that are in-flight.
  */
 
-import { getPoolStatus, connectToPool } from './pool.js';
+import { getPoolStatus, connectToPool, getCachedDriver } from './pool.js';
 import { log, colors as C } from './logger.js';
 
 function sleep(ms) {
@@ -42,16 +42,22 @@ function getPending(poolUrl) {
   return pendingConnections.get(poolUrl) || 0;
 }
 
+/** Extracts pool driver options from config for passing to getPoolStatus. */
+function driverOpts(config) {
+  if (!config) return {};
+  return { poolDriver: config.poolDriver || 'auto', maxSessions: config.maxSessions || 10 };
+}
+
 /** Returns the normalized pool URL array from config. Always an array, even for single pool. */
 export function getPoolUrls(config) {
   return config._poolUrls || [config.poolUrl];
 }
 
-/** Fetches /pressure from all pools in parallel. Returns [{ url, status, error }]. */
-export async function getAllPoolStatuses(poolUrls) {
+/** Fetches status from all pools in parallel. Returns [{ url, status, error }]. */
+export async function getAllPoolStatuses(poolUrls, options = {}) {
   return Promise.all(poolUrls.map(async (url) => {
     try {
-      const status = await getPoolStatus(url);
+      const status = await getPoolStatus(url, options);
       return { url, status, error: null };
     } catch (error) {
       return { url, status: null, error: error.message };
@@ -60,8 +66,8 @@ export async function getAllPoolStatuses(poolUrls) {
 }
 
 /** Combined view across all pools: totalRunning, totalMaxConcurrent, per-pool details. */
-export async function getAggregatedPoolStatus(poolUrls) {
-  const results = await getAllPoolStatuses(poolUrls);
+export async function getAggregatedPoolStatus(poolUrls, options = {}) {
+  const results = await getAllPoolStatuses(poolUrls, options);
 
   let totalRunning = 0;
   let totalMaxConcurrent = 0;
@@ -90,11 +96,11 @@ export async function getAggregatedPoolStatus(poolUrls) {
 }
 
 /** Blocks until at least one pool is reachable and available. */
-export async function waitForAnyPool(poolUrls, maxWaitMs = 30000) {
+export async function waitForAnyPool(poolUrls, maxWaitMs = 30000, options = {}) {
   const start = Date.now();
 
   while (Date.now() - start < maxWaitMs) {
-    const results = await getAllPoolStatuses(poolUrls);
+    const results = await getAllPoolStatuses(poolUrls, options);
     const available = results.find(r => r.status?.available);
     if (available) return available.status;
 
@@ -115,17 +121,17 @@ export async function waitForAnyPool(poolUrls, maxWaitMs = 30000) {
  * Picks the pool with the lowest pressure ratio.
  *
  * Algorithm:
- * 1. Query all pools' /pressure in parallel
+ * 1. Query all pools' status in parallel (driver-aware)
  * 2. Add local pending count to each pool's running total
  * 3. Filter to reachable pools with (running + pending) < maxConcurrent
  * 4. Sort by: lowest effective pressure → fewest queued → most free slots
  * 5. Track selection in pending counter, return best candidate URL
  * 6. If all full, poll every 2s up to 60s, then pick least-pressured anyway
  */
-export async function selectPool(poolUrls, pollIntervalMs = 2000, maxWaitMs = 60000) {
+export async function selectPool(poolUrls, pollIntervalMs = 2000, maxWaitMs = 60000, options = {}) {
   // Fast path: single pool
   if (poolUrls.length === 1) {
-    await waitForSlotOnPool(poolUrls[0], pollIntervalMs, maxWaitMs);
+    await waitForSlotOnPool(poolUrls[0], pollIntervalMs, maxWaitMs, options);
     trackPending(poolUrls[0]);
     return poolUrls[0];
   }
@@ -133,7 +139,7 @@ export async function selectPool(poolUrls, pollIntervalMs = 2000, maxWaitMs = 60
   const start = Date.now();
 
   while (Date.now() - start < maxWaitMs) {
-    const results = await getAllPoolStatuses(poolUrls);
+    const results = await getAllPoolStatuses(poolUrls, options);
     const candidates = results
       .filter(r => r.status && !r.error && r.status.available)
       .map(r => {
@@ -173,7 +179,7 @@ export async function selectPool(poolUrls, pollIntervalMs = 2000, maxWaitMs = 60
   }
 
   // Timeout — pick the least-pressured pool anyway (let connectToPool deal with it)
-  const results = await getAllPoolStatuses(poolUrls);
+  const results = await getAllPoolStatuses(poolUrls, options);
   const reachable = results
     .filter(r => r.status && !r.error)
     .sort((a, b) => {
@@ -195,19 +201,64 @@ export async function selectPool(poolUrls, pollIntervalMs = 2000, maxWaitMs = 60
   return poolUrls[0];
 }
 
+/**
+ * Filters pool URLs to those whose detected driver matches `driver`,
+ * with explicit opt-in fallback to `fallbackDriver`.
+ *
+ * Probes all pools once via getAllPoolStatuses() to warm the per-URL driver cache,
+ * then filters by getCachedDriver(url). Pools that are unreachable have a null
+ * detected driver and are excluded.
+ *
+ * Throws if no pool matches the requested driver and no usable fallback exists.
+ * Pool busyness is NOT a fallback trigger — selectPool() handles capacity waits
+ * inside the filtered set.
+ *
+ * @param {string[]} poolUrls
+ * @param {string} driver - Required driver name (e.g. 'obscura')
+ * @param {string|null} fallbackDriver - Explicit fallback driver, or null/undefined for hard error
+ * @param {object} options - { poolDriver, maxSessions } passed to getPoolStatus
+ * @returns {Promise<{urls: string[], driver: string, usedFallback: boolean}>}
+ */
+export async function resolvePoolsForTest(poolUrls, driver, fallbackDriver, options = {}) {
+  // Warm driver cache for all reachable pools
+  await getAllPoolStatuses(poolUrls, options);
+
+  const matching = poolUrls.filter(url => getCachedDriver(url) === driver);
+  if (matching.length > 0) {
+    return { urls: matching, driver, usedFallback: false };
+  }
+
+  if (fallbackDriver) {
+    const fallbackMatching = poolUrls.filter(url => getCachedDriver(url) === fallbackDriver);
+    if (fallbackMatching.length > 0) {
+      log('⚠️', `${C.yellow}No pool with driver=${driver}, falling back to ${fallbackDriver}${C.reset}`);
+      return { urls: fallbackMatching, driver: fallbackDriver, usedFallback: true };
+    }
+    throw new Error(
+      `No pool available for driver "${driver}" and fallback driver "${fallbackDriver}" also unavailable. ` +
+      `Reachable pools: ${poolUrls.map(u => `${u}=${getCachedDriver(u) || 'unreachable'}`).join(', ')}`
+    );
+  }
+
+  throw new Error(
+    `No pool available for driver "${driver}" and no fallbackDriver specified. ` +
+    `Reachable pools: ${poolUrls.map(u => `${u}=${getCachedDriver(u) || 'unreachable'}`).join(', ')}`
+  );
+}
+
 /** Convenience: selectPool + connectToPool in one call. */
 export async function selectAndConnect(config) {
   const poolUrls = getPoolUrls(config);
-  const chosenUrl = await selectPool(poolUrls);
+  const chosenUrl = await selectPool(poolUrls, 2000, 60000, driverOpts(config));
   return connectToPool(chosenUrl, config.connectRetries, config.connectRetryDelay);
 }
 
-/** Waits until a single pool has capacity (replaces the old waitForSlot from runner.js). */
-async function waitForSlotOnPool(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
+/** Waits until a single pool has capacity. */
+async function waitForSlotOnPool(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000, options = {}) {
   const start = Date.now();
   while (Date.now() - start < maxWaitMs) {
     try {
-      const status = await getPoolStatus(poolUrl);
+      const status = await getPoolStatus(poolUrl, options);
       if (status.available && status.running < status.maxConcurrent) {
         return;
       }