@matware/e2e-runner 1.2.1 → 1.3.1

package/src/index.js

@@ -8,23 +8,26 @@
  */
 
 export { loadConfig } from './config.js';
-export { waitForPool, connectToPool, startPool, stopPool, restartPool, getPoolStatus } from './pool.js';
+export { waitForPool, connectToPool, disconnectFromPool, startPool, stopPool, restartPool, getPoolStatus, clearDriverCache, getCachedDriver, trackCdpSession, releaseCdpSession, releaseSteelSession } from './pool.js';
+export { getPoolUrls, getAllPoolStatuses, getAggregatedPoolStatus, waitForAnyPool, selectPool, selectAndConnect } from './pool-manager.js';
 export { executeAction } from './actions.js';
-export { runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from './runner.js';
+export { runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites, fetchAuthToken } from './runner.js';
 export { generateReport, generateJUnitXML, saveReport, printReport, saveHistory, loadHistory, loadHistoryRun } from './reporter.js';
 export { startDashboard, stopDashboard } from './dashboard.js';
 export { fetchIssue, parseIssueUrl, detectProvider, checkCliAuth } from './issues.js';
 export { buildPrompt, generateTests, hasApiKey } from './ai-generate.js';
 export { verifyIssue } from './verify.js';
 export { resolveTestData, loadModuleRegistry, listModules } from './module-resolver.js';
-export { learnFromRun, categorizeError } from './learner.js';
-export { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights } from './learner-sqlite.js';
+export { learnFromRun, categorizeError, isInfraError, INFRA_CATEGORIES } from './learner.js';
+export { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestCreationContext, generateImprovements, getActionHealthScores } from './learner-sqlite.js';
 export { generateLearningsMarkdown } from './learner-markdown.js';
 export { writeToGraph, queryGraph, closeNeo4j } from './learner-neo4j.js';
 export { startNeo4j, stopNeo4j, getNeo4jStatus } from './neo4j-pool.js';
+export { forkAppInstance, destroyFork, destroyAllForks, getAppPoolStatus, isAppPoolEnabled } from './app-pool.js';
+export { compareImages, assertVisualMatch } from './visual-diff.js';
 
 import { loadConfig } from './config.js';
-import { waitForPool } from './pool.js';
+import { waitForAnyPool, getPoolUrls } from './pool-manager.js';
 import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites } from './runner.js';
 import { generateReport, saveReport, printReport } from './reporter.js';
 
@@ -41,7 +44,7 @@ export async function createRunner(userConfig = {}) {
 
     /** Runs all test suites from the tests directory */
     async runAll() {
-      await waitForPool(config.poolUrl);
+      await waitForAnyPool(getPoolUrls(config));
       const { tests, hooks } = loadAllSuites(config.testsDir, config.modulesDir, config.exclude);
       const results = await runTestsParallel(tests, config, hooks);
       const report = generateReport(results);
@@ -52,7 +55,7 @@ export async function createRunner(userConfig = {}) {
 
     /** Runs a single suite by name */
     async runSuite(name) {
-      await waitForPool(config.poolUrl);
+      await waitForAnyPool(getPoolUrls(config));
       const { tests, hooks } = loadTestSuite(name, config.testsDir, config.modulesDir);
       const results = await runTestsParallel(tests, config, hooks);
       const report = generateReport(results);
@@ -63,7 +66,7 @@ export async function createRunner(userConfig = {}) {
 
     /** Runs an array of test objects */
     async runTests(tests) {
-      await waitForPool(config.poolUrl);
+      await waitForAnyPool(getPoolUrls(config));
       const results = await runTestsParallel(tests, config);
       const report = generateReport(results);
       saveReport(report, config.screenshotsDir, config);
@@ -73,7 +76,7 @@ export async function createRunner(userConfig = {}) {
 
     /** Runs tests from a JSON file path */
     async runFile(filePath) {
-      await waitForPool(config.poolUrl);
+      await waitForAnyPool(getPoolUrls(config));
       const { tests, hooks } = loadTestFile(filePath, config.modulesDir);
       const results = await runTestsParallel(tests, config, hooks);
       const report = generateReport(results);

package/src/learner-sqlite.js

@@ -293,9 +293,160 @@ export function getRunInsights(projectId, report) {
     }
   }
 
+  // ── At-Least-One Guarantee: generate positive insights if none exist ──
+  if (insights.length === 0 && report.results.length > 0) {
+    const allPassed = report.results.every(r => r.success);
+
+    // Green streak detection
+    if (allPassed) {
+      const recentRuns = d.prepare(`
+        SELECT run_id, MIN(success) AS all_passed
+        FROM test_learnings
+        WHERE project_id = ?
+        GROUP BY run_id
+        ORDER BY created_at DESC
+        LIMIT 10
+      `).all(projectId);
+      const streak = recentRuns.findIndex(r => r.all_passed === 0);
+      const streakCount = streak === -1 ? recentRuns.length : streak;
+      if (streakCount >= 3) {
+        insights.push({
+          type: 'green-streak',
+          streak: streakCount,
+          message: `${streakCount}-run green streak — suite is stable.`,
+        });
+      }
+    }
+
+    // New tests (no historical data)
+    const newTests = report.results.filter(r => {
+      const h = d.prepare('SELECT COUNT(*) AS c FROM test_learnings WHERE project_id = ? AND test_name = ?').get(projectId, r.name);
+      return !h || h.c <= 1; // <= 1 because current run may already be written
+    });
+    if (newTests.length > 0) {
+      insights.push({
+        type: 'new-tests',
+        tests: newTests.map(t => t.name),
+        message: `${newTests.length} new test(s): ${newTests.map(t => t.name).slice(0, 3).join(', ')}${newTests.length > 3 ? '...' : ''}`,
+      });
+    }
+
+    // Pass rate improvement vs 7-day average
+    const avg7d = d.prepare(`
+      SELECT ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS pass_rate
+      FROM test_learnings
+      WHERE project_id = ? AND created_at >= datetime('now', '-7 days')
+    `).get(projectId);
+    const thisRunPassRate = Math.round((report.results.filter(r => r.success).length / report.results.length) * 1000) / 10;
+    if (avg7d?.pass_rate && thisRunPassRate > avg7d.pass_rate + 5) {
+      insights.push({
+        type: 'improved-pass-rate',
+        message: `Pass rate improved: ${thisRunPassRate}% this run vs ${avg7d.pass_rate}% 7-day average.`,
+      });
+    }
+
+    // Performance comparison
+    const avgDuration = d.prepare(`
+      SELECT ROUND(AVG(duration_ms)) AS avg_ms
+      FROM test_learnings
+      WHERE project_id = ? AND duration_ms IS NOT NULL AND created_at >= datetime('now', '-30 days')
+    `).get(projectId);
+    if (avgDuration?.avg_ms && report.results.length > 0) {
+      const thisAvg = report.results.reduce((s, r) => {
+        const ms = (r.endTime && r.startTime) ? new Date(r.endTime) - new Date(r.startTime) : 0;
+        return s + ms;
+      }, 0) / report.results.length;
+      const delta = Math.round(((thisAvg - avgDuration.avg_ms) / avgDuration.avg_ms) * 100);
+      if (Math.abs(delta) > 15) {
+        insights.push({
+          type: 'performance',
+          message: delta < 0
+            ? `This run was ${Math.abs(delta)}% faster than the 30-day average.`
+            : `This run was ${delta}% slower than the 30-day average — check for new slow pages.`,
+        });
+      }
+    }
+
+    // Stable selectors confirmed
+    if (allPassed) {
+      const usedSelectors = new Set();
+      for (const r of report.results) {
+        if (!r.actions) continue;
+        for (const a of r.actions) {
+          if (a.selector) usedSelectors.add(a.selector);
+        }
+      }
+      if (usedSelectors.size > 0) {
+        const stableCount = d.prepare(`
+          SELECT COUNT(DISTINCT selector) AS c
+          FROM selector_learnings
+          WHERE project_id = ? AND selector IN (${[...usedSelectors].map(() => '?').join(',')})
+          GROUP BY selector
+          HAVING SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) = 0 AND COUNT(*) > 3
+        `).all(projectId, ...usedSelectors).length;
+        if (stableCount > 0) {
+          insights.push({
+            type: 'stable-selectors',
+            count: stableCount,
+            message: `${stableCount} selector(s) confirmed stable across multiple runs.`,
+          });
+        }
+      }
+    }
+
+    // Fallback: if still no insights, report basic run stats
+    if (insights.length === 0) {
+      const passed = report.results.filter(r => r.success).length;
+      insights.push({
+        type: 'run-summary',
+        message: `${passed}/${report.results.length} tests passed (${thisRunPassRate}%).`,
+      });
+    }
+  }
+
   return insights;
 }
 
+/**
+ * Compact health snapshot for a project — used by CLI, MCP, and Dashboard.
+ * Returns null if no historical data exists.
+ */
+export function getHealthSnapshot(projectId) {
+  const summary = getLearningsSummary(projectId);
+  if (!summary || summary.totalRuns === 0) return null;
+
+  const flakyCount = summary.flakyTests ? summary.flakyTests.length : 0;
+  const unstableSelectorCount = summary.unstableSelectors ? summary.unstableSelectors.length : 0;
+  const topError = summary.topErrors && summary.topErrors.length > 0
+    ? { pattern: summary.topErrors[0].pattern, count: summary.topErrors[0].occurrence_count, category: summary.topErrors[0].category }
+    : null;
+
+  // Compute trend from recent daily data
+  let passRateTrend = 'stable'; // 'improving', 'declining', 'stable'
+  let trendDelta = 0;
+
+  const trends = getTestTrends(projectId, 7);
+  const trendData = trends?.data || trends || [];
+  if (Array.isArray(trendData) && trendData.length >= 2) {
+    const recent = trendData[trendData.length - 1].pass_rate;
+    const prior = trendData.slice(0, -1).reduce((s, t) => s + t.pass_rate, 0) / (trendData.length - 1);
+    trendDelta = Math.round((recent - prior) * 10) / 10;
+    if (trendDelta > 2) passRateTrend = 'improving';
+    else if (trendDelta < -2) passRateTrend = 'declining';
+  }
+
+  return {
+    passRate: summary.overallPassRate,
+    passRateTrend,
+    trendDelta,
+    flakyCount,
+    unstableSelectorCount,
+    topErrorPattern: topError,
+    totalRuns: summary.totalRuns,
+    totalTests: summary.totalTests,
+  };
+}
+
 /** Drill-down: history for a specific test. */
 export function getTestHistory(projectId, testName, days = 30) {
   const d = getDb();
@@ -352,3 +503,310 @@ export function getSelectorHistory(projectId, selector, days = 30) {
     ORDER BY created_at DESC
   `).all(projectId, selector, days);
 }
+
+/**
+ * Aggregated context for test authoring — curates the most actionable learnings
+ * into a compact object that AI agents can use to write better tests.
+ */
+/**
+ * Action health scores — composite per-action metrics aggregated by (action_type, selector).
+ * Score = (success_rate * 0.5) + (speed_score * 0.3) + (collateral_score * 0.2)
+ */
+export function getActionHealthScores(projectId, days = 30) {
+  const d = getDb();
+  const rows = d.prepare(`
+    SELECT
+      action_type,
+      selector,
+      page_url,
+      COUNT(*) AS total_uses,
+      ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS success_rate,
+      ROUND(AVG(duration_ms)) AS avg_duration_ms,
+      MAX(duration_ms) AS max_duration_ms,
+      ROUND(AVG(console_errors_after + network_errors_after), 1) AS avg_collateral_errors,
+      COUNT(DISTINCT test_name) AS used_by_tests
+    FROM action_health
+    WHERE project_id = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY action_type, selector
+    HAVING total_uses >= 2
+    ORDER BY success_rate ASC, total_uses DESC
+  `).all(projectId, days);
+
+  return rows.map(r => {
+    const speedScore = 100 - Math.min(100, ((r.avg_duration_ms || 0) / 5000) * 100);
+    const collateralScore = 100 - Math.min(100, (r.avg_collateral_errors || 0) * 20);
+    const healthScore = Math.round(r.success_rate * 0.5 + speedScore * 0.3 + collateralScore * 0.2);
+    return {
+      actionType: r.action_type,
+      selector: r.selector,
+      pageUrl: r.page_url,
+      totalUses: r.total_uses,
+      successRate: r.success_rate,
+      avgDurationMs: r.avg_duration_ms,
+      maxDurationMs: r.max_duration_ms,
+      avgCollateralErrors: r.avg_collateral_errors,
+      usedByTests: r.used_by_tests,
+      healthScore,
+    };
+  });
+}
+
+export function getTestCreationContext(projectId) {
+  const d = getDb();
+  const ctx = {};
+
+  // Top 5 unstable selectors (>20% fail rate)
+  const unstable = d.prepare(`
+    SELECT
+      selector,
+      ROUND(AVG(CASE WHEN success = 0 THEN 100.0 ELSE 0.0 END), 1) AS fail_rate,
+      MAX(CASE WHEN success = 0 THEN error END) AS last_error,
+      COUNT(*) AS total_uses
+    FROM selector_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY selector
+    HAVING fail_rate > 20
+    ORDER BY fail_rate DESC
+    LIMIT 5
+  `).all(projectId);
+
+  if (unstable.length > 0) {
+    ctx.unstableSelectors = unstable.map(s => ({
+      selector: s.selector,
+      failRate: s.fail_rate,
+      lastError: s.last_error,
+      suggestion: suggestSelectorFix(s.selector),
+    }));
+  }
+
+  // Top 10 stable selectors (0% fail rate, >5 uses)
+  const stable = d.prepare(`
+    SELECT
+      selector,
+      COUNT(*) AS total_uses,
+      COUNT(DISTINCT test_name) AS used_by_tests
+    FROM selector_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY selector
+    HAVING total_uses > 5 AND SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) = 0
+    ORDER BY total_uses DESC
+    LIMIT 10
+  `).all(projectId);
+
+  if (stable.length > 0) {
+    ctx.stableSelectors = stable.map(s => ({
+      selector: s.selector,
+      uses: s.total_uses,
+      tests: s.used_by_tests,
+    }));
+  }
+
+  // Top 5 error patterns
+  const errors = d.prepare(`
+    SELECT pattern, category, occurrence_count
+    FROM error_patterns
+    WHERE project_id = ?
+    ORDER BY occurrence_count DESC
+    LIMIT 5
+  `).all(projectId);
+
+  if (errors.length > 0) {
+    ctx.errorPatterns = errors.map(e => ({
+      pattern: e.pattern,
+      category: e.category,
+      count: e.occurrence_count,
+    }));
+  }
+
+  // Slow pages (avg load > 3s)
+  const slowPages = d.prepare(`
+    SELECT
+      url_path,
+      ROUND(AVG(load_time_ms)) AS avg_load_ms
+    FROM page_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY url_path
+    HAVING avg_load_ms > 3000
+    ORDER BY avg_load_ms DESC
+    LIMIT 5
+  `).all(projectId);
+
+  if (slowPages.length > 0) {
+    ctx.slowPages = slowPages.map(p => ({
+      page: p.url_path,
+      avgLoadMs: p.avg_load_ms,
+    }));
+  }
+
+  // Flaky tests
+  const flaky = d.prepare(`
+    SELECT test_name, SUM(flaky) AS flaky_count, COUNT(*) AS total_runs
+    FROM test_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY test_name
+    HAVING flaky_count > 0
+    ORDER BY flaky_count DESC
+    LIMIT 5
+  `).all(projectId);
+
+  if (flaky.length > 0) {
+    ctx.flakyTests = flaky.map(f => ({
+      name: f.test_name,
+      flakyCount: f.flaky_count,
+      totalRuns: f.total_runs,
+    }));
+  }
+
+  // API endpoints with >10% error rate
+  const apiIssues = d.prepare(`
+    SELECT
+      endpoint,
+      ROUND(AVG(CASE WHEN is_error = 1 THEN 100.0 ELSE 0.0 END), 1) AS error_rate,
+      COUNT(*) AS total_calls
+    FROM api_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY endpoint
+    HAVING error_rate > 10
+    ORDER BY error_rate DESC
+    LIMIT 5
+  `).all(projectId);
+
+  if (apiIssues.length > 0) {
+    ctx.apiIssues = apiIssues.map(a => ({
+      endpoint: a.endpoint,
+      errorRate: a.error_rate,
+      totalCalls: a.total_calls,
+    }));
+  }
+
+  // Overall pass rate
+  const stats = d.prepare(`
+    SELECT
+      COUNT(*) AS total_tests,
+      ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS pass_rate
+    FROM test_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+  `).get(projectId);
+
+  if (stats && stats.total_tests > 0) {
+    ctx.passRate = stats.pass_rate;
+  }
+
+  return Object.keys(ctx).length > 0 ? ctx : null;
+}
+
+/** Suggest a fix for an unstable selector based on its pattern. */
+function suggestSelectorFix(selector) {
+  if (/^\.Mui|^\.css-|^\.sc-/.test(selector)) return 'Prefer [data-testid] or click by text — generated class names are brittle';
+  if (/\s>\s/.test(selector) && selector.split('>').length > 3) return 'Deeply nested selector — simplify or use [data-testid]';
+  if (/nth-child|nth-of-type/.test(selector)) return 'Positional selector — prefer [data-testid] or text-based selection';
+  return 'Consider using [data-testid] or a more stable selector';
+}
+
+/**
+ * Cross-reference a run report with historical learnings to produce actionable
+ * improvement suggestions for the AI agent.
+ */
+export function generateImprovements(projectId, report) {
+  const d = getDb();
+  const improvements = [];
+
+  if (!report?.results) return improvements;
+
+  // Build a map of stable alternatives for unstable selectors
+  const stableAlts = d.prepare(`
+    SELECT selector, COUNT(*) AS uses
+    FROM selector_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY selector
+    HAVING uses > 3 AND SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) = 0
+    ORDER BY uses DESC
+  `).all(projectId);
+
+  const stableSet = new Set(stableAlts.map(s => s.selector));
+
+  // Unstable selectors with their fail rates
+  const unstableMap = new Map();
+  const unstableRows = d.prepare(`
+    SELECT
+      selector,
+      ROUND(AVG(CASE WHEN success = 0 THEN 100.0 ELSE 0.0 END), 1) AS fail_rate
+    FROM selector_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY selector
+    HAVING fail_rate > 20
+  `).all(projectId);
+  for (const row of unstableRows) unstableMap.set(row.selector, row.fail_rate);
+
+  // Flaky test counts
+  const flakyMap = new Map();
+  const flakyRows = d.prepare(`
+    SELECT test_name, SUM(flaky) AS flaky_count
+    FROM test_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-30 days')
+    GROUP BY test_name
+    HAVING flaky_count > 0
+  `).all(projectId);
+  for (const row of flakyRows) flakyMap.set(row.test_name, row.flaky_count);
+
+  for (const result of report.results) {
+    // Failed selector suggestions — find stable alternatives on the same page
+    if (!result.success && result.error) {
+      const selectorMatch = result.error.match(/selector ["']([^"']+)["']/i)
+        || result.error.match(/waiting for selector (.+)/i);
+      if (selectorMatch) {
+        const failedSelector = selectorMatch[1];
+        const failRate = unstableMap.get(failedSelector);
+        if (failRate) {
+          improvements.push({
+            type: 'unstable-selector',
+            test: result.name,
+            message: `Selector \`${failedSelector}\` failed (${failRate}% historical fail rate) → ${suggestSelectorFix(failedSelector)}`,
+          });
+        }
+      }
+
+      // Timeout suggestions
+      if (/timeout|timed?\s*out/i.test(result.error)) {
+        improvements.push({
+          type: 'timeout',
+          test: result.name,
+          message: `Test "${result.name}" timed out → add explicit { type: "wait", text: "..." } or increase timeout`,
+        });
+      }
+    }
+
+    // Check for tests using known unstable selectors (even if they passed this time)
+    if (result.actions) {
+      for (const action of result.actions) {
+        if (action.selector && unstableMap.has(action.selector)) {
+          const failRate = unstableMap.get(action.selector);
+          improvements.push({
+            type: 'at-risk-selector',
+            test: result.name,
+            message: `Selector \`${action.selector}\` has ${failRate}% fail rate → ${suggestSelectorFix(action.selector)}`,
+          });
+        }
+      }
+    }
+
+    // Flaky test suggestions
+    const flakyCount = flakyMap.get(result.name);
+    if (flakyCount && flakyCount >= 2) {
+      improvements.push({
+        type: 'flaky',
+        test: result.name,
+        message: `Test "${result.name}" is flaky (${flakyCount} flaky runs) → add { retries: 2 } to the test config`,
+      });
+    }
+  }
+
+  // Deduplicate by type+test (keep first occurrence)
+  const seen = new Set();
+  return improvements.filter(imp => {
+    const key = `${imp.type}:${imp.test}:${imp.message.slice(0, 60)}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}